From c90c84f700bf35cd7f57a43304d95cdf2dac8a77 Mon Sep 17 00:00:00 2001
From: Bukhtawar Khan <bukhtawa@amazon.com>
Date: Sun, 26 May 2019 18:03:49 +0530
Subject: [PATCH 1/9] Auto-release of read-only-allow-delete block when disk
 utilization falls below the low watermark. Relates to #39334

---
 .../modules/cluster/disk_allocator.asciidoc   |   6 +-
 .../allocation/DiskThresholdMonitor.java      |  69 +++++++-
 .../allocation/DiskThresholdSettings.java     |  18 +++
 .../allocation/DiskThresholdMonitorTests.java | 150 ++++++++++++++----
 .../DiskThresholdSettingsTests.java           |   3 +
 .../allocation/decider/MockDiskUsagesIT.java  |  95 +++++++++++
 6 files changed, 300 insertions(+), 41 deletions(-)

diff --git a/docs/reference/modules/cluster/disk_allocator.asciidoc b/docs/reference/modules/cluster/disk_allocator.asciidoc
index 5daa50d9c25b0..c0cc934734d6b 100644
--- a/docs/reference/modules/cluster/disk_allocator.asciidoc
+++ b/docs/reference/modules/cluster/disk_allocator.asciidoc
@@ -40,8 +40,10 @@ Elasticsearch enforces a read-only index block
 (`index.blocks.read_only_allow_delete`) on every index that has one or more
 shards allocated on the node that has at least one disk exceeding the flood
 stage. This is a last resort to prevent nodes from running out of disk space.
-The index block must be released manually once there is enough disk space
-available to allow indexing operations to continue.
+The index block is automatically released once the disk utilization falls below
+the high watermark.
+The automatic release can however be disabled in 7.x through a system property
+`es.disk.auto_release_flood_stage_block`
 
 NOTE: You can not mix the usage of percentage values and byte values within
 these settings. Either all are set to percentage values, or all are set to byte
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
index 4badab5a0cafa..a1a30361ec063 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
@@ -19,9 +19,12 @@
 
 package org.elasticsearch.cluster.routing.allocation;
 
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 import com.carrotsearch.hppc.ObjectLookupContainer;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
@@ -34,9 +37,11 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
@@ -55,6 +60,7 @@ public class DiskThresholdMonitor {
     private final Set<String> nodeHasPassedWatermark = Sets.newConcurrentHashSet();
     private final Supplier<ClusterState> clusterStateSupplier;
     private long lastRunNS;
+    private final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
 
     public DiskThresholdMonitor(Settings settings, Supplier<ClusterState> clusterStateSupplier, ClusterSettings clusterSettings,
                                 Client client) {
@@ -109,13 +115,24 @@ public void onNewInfo(ClusterInfo info) {
             }
             ClusterState state = clusterStateSupplier.get();
             Set<String> indicesToMarkReadOnly = new HashSet<>();
+            RoutingNodes routingNodes = state.getRoutingNodes();
+            Map<String, Boolean> indexAutoReleaseEligibility = new HashMap<>();
+            // Ensure we release indices on nodes that have a usage response from node stats
+            markNodesMissingUsageIneligibleForRelease(routingNodes, usages, indexAutoReleaseEligibility);
             for (ObjectObjectCursor<String, DiskUsage> entry : usages) {
                 String node = entry.key;
                 DiskUsage usage = entry.value;
                 warnAboutDiskIfNeeded(usage);
+                RoutingNode routingNode = state.getRoutingNodes().node(node);
+                // Only unblock index if all nodes that contain shards of it are below the high disk watermark
+                if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes()
+                    || usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) {
+                    markEligiblityForAutoRelease(routingNode, indexAutoReleaseEligibility, false);
+                } else {
+                    markEligiblityForAutoRelease(routingNode, indexAutoReleaseEligibility, true);
+                }
                 if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdFloodStage().getBytes() ||
                     usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdFloodStage()) {
-                    RoutingNode routingNode = state.getRoutingNodes().node(node);
                     if (routingNode != null) { // this might happen if we haven't got the full cluster-state yet?!
                         for (ShardRouting routing : routingNode) {
                             indicesToMarkReadOnly.add(routing.index().getName());
@@ -159,17 +176,59 @@ public void onNewInfo(ClusterInfo info) {
                 logger.info("rerouting shards: [{}]", explanation);
                 reroute();
             }
+
+            // Get set of indices that are eligible to be automatically unblocked
+            // Only collect indices that are currently blocked
+            Set<String> indicesToAutoRelease = indexAutoReleaseEligibility.entrySet().stream()
+                .filter(Map.Entry::getValue)
+                .map(Map.Entry::getKey)
+                .filter(index -> state.getBlocks().hasIndexBlock(index, IndexMetaData.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK))
+                .collect(Collectors.toCollection(HashSet::new));
+
+            if (indicesToAutoRelease.isEmpty() == false) {
+                if (diskThresholdSettings.isAutoReleaseIndexEnabled()) {
+                    logger.info("Releasing read-only allow delete block on indices: [{}]", indicesToAutoRelease);
+                    updateIndicesReadOnly(indicesToAutoRelease, false);
+                } else {
+                    deprecationLogger.deprecated("es.disk.auto_release_flood_stage_block will be removed in 8.0.0");
+                }
+            }
             indicesToMarkReadOnly.removeIf(index -> state.getBlocks().indexBlocked(ClusterBlockLevel.WRITE, index));
             if (indicesToMarkReadOnly.isEmpty() == false) {
-                markIndicesReadOnly(indicesToMarkReadOnly);
+                updateIndicesReadOnly(indicesToMarkReadOnly, true);
+            }
+        }
+    }
+
+
+    private void markNodesMissingUsageIneligibleForRelease(RoutingNodes routingNodes, ImmutableOpenMap<String, DiskUsage> usages,
+                                                           Map<String, Boolean> indexAutoReleaseEligibility) {
+        if (routingNodes.size() != usages.size()) {
+            for (RoutingNode routingNode : routingNodes) {
+                if (!usages.keys().contains(routingNode.nodeId())) {
+                    markEligiblityForAutoRelease(routingNode, indexAutoReleaseEligibility, false);
+                }
+            }
+        }
+
+    }
+
+    private void markEligiblityForAutoRelease(RoutingNode routingNode, Map<String, Boolean> indexAutoReleaseEligibility,
+                                              boolean eligible) {
+        if (routingNode != null) {
+            for (ShardRouting routing : routingNode) {
+                String indexName = routing.index().getName();
+                boolean value = indexAutoReleaseEligibility.getOrDefault(indexName, true);
+                indexAutoReleaseEligibility.put(indexName, value && eligible);
             }
         }
     }
 
-    protected void markIndicesReadOnly(Set<String> indicesToMarkReadOnly) {
+    protected void updateIndicesReadOnly(Set<String> indicesToUpdate, boolean readOnly) {
         // set read-only block but don't block on the response
-        client.admin().indices().prepareUpdateSettings(indicesToMarkReadOnly.toArray(Strings.EMPTY_ARRAY)).
-            setSettings(Settings.builder().put(IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE, true).build()).execute();
+        String value = readOnly ? Boolean.TRUE.toString() : null;
+        client.admin().indices().prepareUpdateSettings(indicesToUpdate.toArray(Strings.EMPTY_ARRAY)).
+            setSettings(Settings.builder().put(IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE, value).build()).execute();
     }
 
     protected void reroute() {
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java
index b8d234e9f1086..d71e764f6c022 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java
@@ -72,6 +72,20 @@ public class DiskThresholdSettings {
     private volatile TimeValue rerouteInterval;
     private volatile Double freeDiskThresholdFloodStage;
     private volatile ByteSizeValue freeBytesThresholdFloodStage;
+    private static boolean autoReleaseIndexEnabled;
+
+    static {
+        final String AUTO_RELEASE_INDEX_ENABLED_KEY = "es.disk.auto_release_flood_stage_block";
+        final String property = System.getProperty(AUTO_RELEASE_INDEX_ENABLED_KEY);
+        if (property == null) {
+            autoReleaseIndexEnabled = true;
+        } else if (Boolean.FALSE.toString().equals(property)){
+            autoReleaseIndexEnabled = false;
+        } else {
+            throw new IllegalArgumentException(AUTO_RELEASE_INDEX_ENABLED_KEY + " may only be unset or set to [false] but was [" +
+                property + "]");
+        }
+    }
 
     public DiskThresholdSettings(Settings settings, ClusterSettings clusterSettings) {
         final String lowWatermark = CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.get(settings);
@@ -286,6 +300,10 @@ public ByteSizeValue getFreeBytesThresholdFloodStage() {
         return freeBytesThresholdFloodStage;
     }
 
+    public boolean isAutoReleaseIndexEnabled() {
+        return autoReleaseIndexEnabled;
+    }
+
     public boolean includeRelocations() {
         return includeRelocations;
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
index b245b0d35d6c6..ebb6414184f2c 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
@@ -46,34 +46,12 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
 
 
     public void testMarkFloodStageIndicesReadOnly() {
-        AllocationService allocation = createAllocationService(Settings.builder()
-            .put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
         Settings settings = Settings.EMPTY;
-        MetaData metaData = MetaData.builder()
-            .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)
-                .put("index.routing.allocation.require._id", "node2")).numberOfShards(1).numberOfReplicas(0))
-            .put(IndexMetaData.builder("test_1").settings(settings(Version.CURRENT)
-                .put("index.routing.allocation.require._id", "node1")).numberOfShards(1).numberOfReplicas(0))
-            .put(IndexMetaData.builder("test_2").settings(settings(Version.CURRENT)
-                .put("index.routing.allocation.require._id", "node1")).numberOfShards(1).numberOfReplicas(0))
-            .build();
-        RoutingTable routingTable = RoutingTable.builder()
-            .addAsNew(metaData.index("test"))
-            .addAsNew(metaData.index("test_1"))
-            .addAsNew(metaData.index("test_2"))
-
-            .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
-            .metaData(metaData).routingTable(routingTable).build();
-        logger.info("adding two nodes and performing rerouting");
-        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))
-            .add(newNode("node2"))).build();
-        clusterState = allocation.reroute(clusterState, "reroute");
-        logger.info("start primary shard");
-        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+        ClusterState clusterState = bootstrapCluster();
         ClusterState finalState = clusterState;
         AtomicBoolean reroute = new AtomicBoolean(false);
-        AtomicReference<Set<String>> indices = new AtomicReference<>();
+        AtomicReference<Set<String>> indicesToMarkReadOnly = new AtomicReference<>();
+        AtomicReference<Set<String>> indicesToRelease = new AtomicReference<>();
         DiskThresholdMonitor monitor = new DiskThresholdMonitor(settings, () -> finalState,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null) {
             @Override
@@ -82,8 +60,12 @@ protected void reroute() {
             }
 
             @Override
-            protected void markIndicesReadOnly(Set<String> indicesToMarkReadOnly) {
-                assertTrue(indices.compareAndSet(null, indicesToMarkReadOnly));
+            protected void updateIndicesReadOnly(Set<String> indicesToUpdate, boolean readOnly) {
+                if (readOnly) {
+                    assertTrue(indicesToMarkReadOnly.compareAndSet(null, indicesToUpdate));
+                } else {
+                    assertTrue(indicesToRelease.compareAndSet(null, indicesToUpdate));
+                }
             }
         };
         ImmutableOpenMap.Builder<String, DiskUsage> builder = ImmutableOpenMap.builder();
@@ -91,15 +73,18 @@ protected void markIndicesReadOnly(Set<String> indicesToMarkReadOnly) {
         builder.put("node2", new DiskUsage("node2","node2", "/foo/bar", 100, 30));
         monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
         assertFalse(reroute.get());
-        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indices.get());
+        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
+        assertNull(indicesToRelease.get());
 
-        indices.set(null);
+        indicesToMarkReadOnly.set(null);
         builder = ImmutableOpenMap.builder();
         builder.put("node1", new DiskUsage("node1","node1", "/foo/bar", 100, 4));
         builder.put("node2", new DiskUsage("node2","node2", "/foo/bar", 100, 5));
         monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
         assertTrue(reroute.get());
-        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indices.get());
+        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
+        assertNull(indicesToRelease.get());
+
         IndexMetaData indexMetaData = IndexMetaData.builder(clusterState.metaData().index("test_2")).settings(Settings.builder()
             .put(clusterState.metaData()
             .index("test_2").getSettings())
@@ -121,18 +106,115 @@ protected void reroute() {
             }
 
             @Override
-            protected void markIndicesReadOnly(Set<String> indicesToMarkReadOnly) {
-                assertTrue(indices.compareAndSet(null, indicesToMarkReadOnly));
+            protected void updateIndicesReadOnly(Set<String> indicesToUpdate, boolean readOnly) {
+                if (readOnly) {
+                    assertTrue(indicesToMarkReadOnly.compareAndSet(null, indicesToUpdate));
+                } else {
+                    assertTrue(indicesToRelease.compareAndSet(null, indicesToUpdate));
+                }
             }
         };
 
-        indices.set(null);
+        indicesToMarkReadOnly.set(null);
         reroute.set(false);
         builder = ImmutableOpenMap.builder();
         builder.put("node1", new DiskUsage("node1","node1", "/foo/bar", 100, 4));
         builder.put("node2", new DiskUsage("node2","node2", "/foo/bar", 100, 5));
         monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
         assertTrue(reroute.get());
-        assertEquals(new HashSet<>(Arrays.asList("test_1")), indices.get());
+        assertEquals(new HashSet<>(Arrays.asList("test_1")), indicesToMarkReadOnly.get());
+        assertNull(indicesToRelease.get());
+    }
+
+    public void testAutoReleaseIndices() {
+        Settings settings = Settings.EMPTY;
+        ClusterState clusterState = bootstrapCluster();
+
+        AtomicReference<Set<String>> indicesToMarkReadOnly = new AtomicReference<>();
+        AtomicReference<Set<String>> indicesToRelease = new AtomicReference<>();
+
+        // Change cluster state so that "test" index is blocked (read only)
+        IndexMetaData indexMetaData = IndexMetaData.builder(clusterState.metaData().index("test")).settings(Settings.builder()
+            .put(clusterState.metaData()
+                .index("test").getSettings())
+            .put(IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), true)).build();
+
+        final ClusterState anotherFinalClusterState = ClusterState.builder(clusterState).metaData(MetaData.builder(clusterState.metaData())
+            .put(clusterState.metaData().index("test_1"), false)
+            .put(clusterState.metaData().index("test_2"), false)
+            .put(indexMetaData, true).build())
+            .blocks(ClusterBlocks.builder().addBlocks(indexMetaData).build()).build();
+
+        assertTrue(anotherFinalClusterState.blocks().indexBlocked(ClusterBlockLevel.WRITE, "test"));
+
+        DiskThresholdMonitor monitor = new DiskThresholdMonitor(settings, () -> anotherFinalClusterState,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null) {
+            @Override
+            protected void updateIndicesReadOnly(Set<String> indicesToUpdate, boolean readOnly) {
+                if (readOnly) {
+                    assertTrue(indicesToMarkReadOnly.compareAndSet(null, indicesToUpdate));
+                } else {
+                    assertTrue(indicesToRelease.compareAndSet(null, indicesToUpdate));
+                }
+            }
+        };
+
+        // When free disk on node2 goes above threshold (10% high watermark in this test case), index on node2 marked for
+        // auto release
+        ImmutableOpenMap.Builder<String, DiskUsage> builder = ImmutableOpenMap.builder();
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 4));
+        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 11));
+        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
+        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
+        assertEquals(new HashSet<>(Arrays.asList("test")), indicesToRelease.get());
+
+        // When free disk on node1 and node2 goes below threshold (10% high watermark in this test case).
+        // index with read only block isn't marked again to be read only and no indices to be auto-released
+        indicesToMarkReadOnly.set(null);
+        indicesToRelease.set(null);
+        builder = ImmutableOpenMap.builder();
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 4));
+        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 4));
+        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
+        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
+        assertNull(indicesToRelease.get());
+
+        // When free disk on node1 and node2 goes above threshold (10% high watermark in this test case).
+        // index with read only block to be auto-released
+        indicesToMarkReadOnly.set(null);
+        indicesToRelease.set(null);
+        builder = ImmutableOpenMap.builder();
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 14));
+        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 14));
+        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
+        assertNull(indicesToMarkReadOnly.get());
+        assertEquals(new HashSet<>(Arrays.asList("test")), indicesToRelease.get());
+    }
+
+    private ClusterState bootstrapCluster() {
+        AllocationService allocation = createAllocationService(Settings.builder()
+            .put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
+        Settings settings = Settings.EMPTY;
+        MetaData metaData = MetaData.builder()
+            .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)
+                .put("index.routing.allocation.require._id", "node2")).numberOfShards(1).numberOfReplicas(0))
+            .put(IndexMetaData.builder("test_1").settings(settings(Version.CURRENT)
+                .put("index.routing.allocation.require._id", "node1")).numberOfShards(1).numberOfReplicas(0))
+            .put(IndexMetaData.builder("test_2").settings(settings(Version.CURRENT)
+                .put("index.routing.allocation.require._id", "node1")).numberOfShards(1).numberOfReplicas(0))
+            .build();
+        RoutingTable routingTable = RoutingTable.builder()
+            .addAsNew(metaData.index("test"))
+            .addAsNew(metaData.index("test_1"))
+            .addAsNew(metaData.index("test_2"))
+            .build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .metaData(metaData).routingTable(routingTable).build();
+        logger.info("adding two nodes and performing rerouting");
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))
+            .add(newNode("node2"))).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
+        logger.info("start primary shard");
+        return allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettingsTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettingsTests.java
index d9e157187d581..0ce9a89127901 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettingsTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettingsTests.java
@@ -45,6 +45,9 @@ public void testDefaults() {
         assertEquals(60L, diskThresholdSettings.getRerouteInterval().seconds());
         assertTrue(diskThresholdSettings.isEnabled());
         assertTrue(diskThresholdSettings.includeRelocations());
+        assertEquals(zeroBytes, diskThresholdSettings.getFreeBytesThresholdFloodStage());
+        assertEquals(5.0D, diskThresholdSettings.getFreeDiskThresholdFloodStage(), 0.0D);
+        assertTrue(diskThresholdSettings.isAutoReleaseIndexEnabled());
     }
 
     public void testUpdate() {
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
index 8565beb1b89d7..efbd1232013f9 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
@@ -24,6 +24,8 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.DiskUsage;
 import org.elasticsearch.cluster.MockInternalClusterInfoService;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -41,6 +43,8 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class MockDiskUsagesIT extends ESIntegTestCase {
@@ -131,4 +135,95 @@ public void testRerouteOccursOnDiskPassingHighWatermark() throws Exception {
             assertThat("node3 has at least 3 shards", nodesToShardCount.get(realNodeNames.get(2)), greaterThanOrEqualTo(3));
         });
     }
+
+    public void testAutomaticReleaseOfIndexBlock() throws Exception {
+        List<String> nodes = internalCluster().startNodes(3);
+
+        // Wait for all 3 nodes to be up
+        assertBusy(() -> {
+            NodesStatsResponse resp = client().admin().cluster().prepareNodesStats().get();
+            assertThat(resp.getNodes().size(), equalTo(3));
+        });
+
+        // Start with all nodes at 50% usage
+        final MockInternalClusterInfoService cis = (MockInternalClusterInfoService)
+            internalCluster().getInstance(ClusterInfoService.class, internalCluster().getMasterName());
+        cis.setUpdateFrequency(TimeValue.timeValueMillis(100));
+        cis.onMaster();
+        cis.setN1Usage(nodes.get(0), new DiskUsage(nodes.get(0), "n1", "/dev/null", 100, 50));
+        cis.setN2Usage(nodes.get(1), new DiskUsage(nodes.get(1), "n2", "/dev/null", 100, 50));
+        cis.setN3Usage(nodes.get(2), new DiskUsage(nodes.get(2), "n3", "/dev/null", 100, 50));
+
+        final boolean watermarkBytes = randomBoolean(); // we have to consistently use bytes or percentage for the disk watermark settings
+        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
+            .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), watermarkBytes ? "15b" : "85%")
+            .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), watermarkBytes ? "10b" : "90%")
+            .put(
+                DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(),
+                watermarkBytes ? "5b" : "95%")
+            .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "150ms")).get();
+        // Create an index with 10 shards so we can check allocation for it
+        prepareCreate("test").setSettings(Settings.builder()
+            .put("number_of_shards", 6)
+            .put("number_of_replicas", 0)
+            .put("index.routing.allocation.exclude._name", "")).get();
+        ensureGreen("test");
+
+        // Block until the "fake" cluster info is retrieved at least once
+        assertBusy(() -> {
+            ClusterInfo info = cis.getClusterInfo();
+            logger.info("--> got: {} nodes", info.getNodeLeastAvailableDiskUsages().size());
+            assertThat(info.getNodeLeastAvailableDiskUsages().size(), greaterThan(0));
+        });
+
+        final List<String> realNodeNames = new ArrayList<>();
+        ClusterStateResponse resp = client().admin().cluster().prepareState().get();
+        Iterator<RoutingNode> iter = resp.getState().getRoutingNodes().iterator();
+        while (iter.hasNext()) {
+            RoutingNode node = iter.next();
+            realNodeNames.add(node.nodeId());
+            logger.info("--> node {} has {} shards",
+                node.nodeId(), resp.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
+        }
+
+        client().prepareIndex().setIndex("test").setType("doc").setId("1").setSource("foo", "bar").get();
+        refresh();
+        assertSearchHits(client().prepareSearch().get(), "1");
+
+        // Block all nodes so that re-balancing does not occur (BalancedShardsAllocator)
+        cis.setN1Usage(realNodeNames.get(0), new DiskUsage(nodes.get(0), "n1", "_na_", 100, 3));
+        cis.setN2Usage(realNodeNames.get(1), new DiskUsage(nodes.get(1), "n2", "_na_", 100, 3));
+        cis.setN3Usage(realNodeNames.get(2), new DiskUsage(nodes.get(2), "n3", "_na_", 100, 3));
+
+        refresh();
+        // Wait until index "test" is blocked
+        assertBusy(() -> {
+            assertBlocked(client().prepareIndex().setIndex("test").setType("doc").setId("2").setSource("foo", "bar"),
+                IndexMetaData.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK);
+        });
+
+        // Cannot add further documents
+        assertBlocked(client().prepareIndex().setIndex("test").setType("doc").setId("3").setSource("foo", "bar"),
+            IndexMetaData.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK);
+        refresh();
+        assertSearchHits(client().prepareSearch().get(), "1", "2");
+
+        // Update the disk usages so all nodes are back under the high and flood watermarks
+        cis.setN1Usage(realNodeNames.get(0), new DiskUsage(nodes.get(0), "n1", "_na_", 100, 11));
+        cis.setN2Usage(realNodeNames.get(1), new DiskUsage(nodes.get(1), "n2", "_na_", 100, 11));
+        cis.setN3Usage(realNodeNames.get(2), new DiskUsage(nodes.get(2), "n3", "_na_", 100, 11));
+
+        refresh();
+        // Attempt to create a new document until DiskUsageMonitor unblocks the index
+        assertBusy(() -> {
+            try {
+                client().prepareIndex().setIndex("test").setType("doc").setId("3").setSource("foo", "bar").get();
+            } catch (ClusterBlockException e) {
+                fail();
+            }
+        });
+        refresh();
+        assertSearchHits(client().prepareSearch().get(), "1", "2", "3");
+    }
+
 }

From 92e7d7b0edfb2afd264abfc683b0582911f191c5 Mon Sep 17 00:00:00 2001
From: Bukhtawar Khan <bukhtawa@amazon.com>
Date: Fri, 31 May 2019 21:49:34 +0530
Subject: [PATCH 2/9] Tracking indicesToMarkIneligibleForAutoRelease instead of
 a Map and addressing other minor comments

---
 .../allocation/DiskThresholdMonitor.java      | 39 ++++++++-----------
 .../allocation/DiskThresholdSettings.java     |  2 +-
 2 files changed, 17 insertions(+), 24 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
index a1a30361ec063..a1b1623ad83c3 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
@@ -19,9 +19,8 @@
 
 package org.elasticsearch.cluster.routing.allocation;
 
-import java.util.HashMap;
+import java.util.Arrays;
 import java.util.HashSet;
-import java.util.Map;
 import java.util.Set;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -116,20 +115,18 @@ public void onNewInfo(ClusterInfo info) {
             ClusterState state = clusterStateSupplier.get();
             Set<String> indicesToMarkReadOnly = new HashSet<>();
             RoutingNodes routingNodes = state.getRoutingNodes();
-            Map<String, Boolean> indexAutoReleaseEligibility = new HashMap<>();
-            // Ensure we release indices on nodes that have a usage response from node stats
-            markNodesMissingUsageIneligibleForRelease(routingNodes, usages, indexAutoReleaseEligibility);
+            Set<String> indicesToMarkIneligibleForAutoRelease = new HashSet<>();
+            //Ensure we release indices on nodes that have a usage response from node stats
+            markNodesMissingUsageIneligibleForRelease(routingNodes, usages, indicesToMarkIneligibleForAutoRelease);
             for (ObjectObjectCursor<String, DiskUsage> entry : usages) {
                 String node = entry.key;
                 DiskUsage usage = entry.value;
                 warnAboutDiskIfNeeded(usage);
-                RoutingNode routingNode = state.getRoutingNodes().node(node);
+                RoutingNode routingNode = routingNodes.node(node);
                 // Only unblock index if all nodes that contain shards of it are below the high disk watermark
                 if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes()
                     || usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) {
-                    markEligiblityForAutoRelease(routingNode, indexAutoReleaseEligibility, false);
-                } else {
-                    markEligiblityForAutoRelease(routingNode, indexAutoReleaseEligibility, true);
+                    markIneligiblityForAutoRelease(routingNode, indicesToMarkIneligibleForAutoRelease);
                 }
                 if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdFloodStage().getBytes() ||
                     usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdFloodStage()) {
@@ -179,9 +176,9 @@ public void onNewInfo(ClusterInfo info) {
 
             // Get set of indices that are eligible to be automatically unblocked
             // Only collect indices that are currently blocked
-            Set<String> indicesToAutoRelease = indexAutoReleaseEligibility.entrySet().stream()
-                .filter(Map.Entry::getValue)
-                .map(Map.Entry::getKey)
+            final String[] indices = state.routingTable().indicesRouting().keys().toArray(String.class);
+            Set<String> indicesToAutoRelease = Arrays.stream(indices)
+                .filter(index -> indicesToMarkIneligibleForAutoRelease.contains(index) == false)
                 .filter(index -> state.getBlocks().hasIndexBlock(index, IndexMetaData.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK))
                 .collect(Collectors.toCollection(HashSet::new));
 
@@ -190,7 +187,7 @@ public void onNewInfo(ClusterInfo info) {
                     logger.info("Releasing read-only allow delete block on indices: [{}]", indicesToAutoRelease);
                     updateIndicesReadOnly(indicesToAutoRelease, false);
                 } else {
-                    deprecationLogger.deprecated("es.disk.auto_release_flood_stage_block will be removed in 8.0.0");
+                    deprecationLogger.deprecated("[{}] will be removed in 8.0.0", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY);
                 }
             }
             indicesToMarkReadOnly.removeIf(index -> state.getBlocks().indexBlocked(ClusterBlockLevel.WRITE, index));
@@ -202,24 +199,20 @@ public void onNewInfo(ClusterInfo info) {
 
 
     private void markNodesMissingUsageIneligibleForRelease(RoutingNodes routingNodes, ImmutableOpenMap<String, DiskUsage> usages,
-                                                           Map<String, Boolean> indexAutoReleaseEligibility) {
-        if (routingNodes.size() != usages.size()) {
-            for (RoutingNode routingNode : routingNodes) {
-                if (!usages.keys().contains(routingNode.nodeId())) {
-                    markEligiblityForAutoRelease(routingNode, indexAutoReleaseEligibility, false);
-                }
+                                                           Set<String> indicesToMarkIneligibleForAutoRelease) {
+        for (RoutingNode routingNode : routingNodes) {
+            if (usages.containsKey(routingNode.nodeId()) == false) {
+                markIneligiblityForAutoRelease(routingNode, indicesToMarkIneligibleForAutoRelease);
             }
         }
 
     }
 
-    private void markEligiblityForAutoRelease(RoutingNode routingNode, Map<String, Boolean> indexAutoReleaseEligibility,
-                                              boolean eligible) {
+    private void markIneligiblityForAutoRelease(RoutingNode routingNode, Set<String> indicesToMarkIneligibleForAutoRelease) {
         if (routingNode != null) {
             for (ShardRouting routing : routingNode) {
                 String indexName = routing.index().getName();
-                boolean value = indexAutoReleaseEligibility.getOrDefault(indexName, true);
-                indexAutoReleaseEligibility.put(indexName, value && eligible);
+                indicesToMarkIneligibleForAutoRelease.add(indexName);
             }
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java
index d71e764f6c022..ebd4b90996d8e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java
@@ -73,9 +73,9 @@ public class DiskThresholdSettings {
     private volatile Double freeDiskThresholdFloodStage;
     private volatile ByteSizeValue freeBytesThresholdFloodStage;
     private static boolean autoReleaseIndexEnabled;
+    public static final String AUTO_RELEASE_INDEX_ENABLED_KEY = "es.disk.auto_release_flood_stage_block";
 
     static {
-        final String AUTO_RELEASE_INDEX_ENABLED_KEY = "es.disk.auto_release_flood_stage_block";
         final String property = System.getProperty(AUTO_RELEASE_INDEX_ENABLED_KEY);
         if (property == null) {
             autoReleaseIndexEnabled = true;

From ce28f9e36677486bd58b82b3fd287916645f9e09 Mon Sep 17 00:00:00 2001
From: Bukhtawar Khan <bukhtawa@amazon.com>
Date: Sun, 26 May 2019 18:03:49 +0530
Subject: [PATCH 3/9] Auto-release of read-only-allow-delete block when disk
 utilization falls below the low watermark. Relates to #39334
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Tracking indicesToMarkIneligibleForAutoRelease instead of a Map and addressing other minor comments

Unmute FullClusterRestartIT#testClosedIndices

Fixed in #39566
Closes #39576

Add debug log for retention leases (#42557)

We need more information to understand why CcrRetentionLeaseIT is
failing. This commit adds some debug log to retention leases and enables
them in CcrRetentionLeaseIT.

Improve how internal representation of pipelines are updated (#42257)

If a single pipeline is updated then the internal representation of
all pipelines was updated. With this change, only the internal representation
of the pipelines that have been modified will be updated.

Prior to this change the IngestMetadata of the previous and current cluster
was used to determine whether the internal representation of pipelines
should be updated. If applying the previous cluster state change failed then
subsequent cluster state changes that have no changes to IngestMetadata
will not attempt to update the internal representation of the pipelines.

This commit, changes how the IngestService updates the internal representation
by keeping track of the underlying configuration and use that to detect
against the new IngestMetadata whether a pipeline configuration has been
changed and if so, then the internal pipeline representation will be updated.

Fix RareClusterStateIT (#42430)

* It looks like we might be cancelling a previous publication instead of
the one triggered by the given request with a very low likelihood.
   * Fixed by adding a wait for no in-progress publications
   * Also added debug logging that would've identified this problem
* Closes #36813

Update script-fields.asciidoc (#42490)

Fixed typo in docker.asciidoc (#42455)

Remove unused mapStringsOrdered method (#42513)

Remove unused mapStringsOrdered method

Dry up BlobStoreRepository#basePath Implementations (#42578)

* This method is just a getter in every implementation => moved the field and concrete getter to the base class to simplify implementations

Add Infrastructure to Run 3rd Party Repository Tests (#42586)

* Add Infrastructure to Run 3rd Party Repository Tests

* Add infrastructure to run third party repository tests using our standard JUnit infrastructure
* This is a prerequisite of #42189

Add test ensure we can execute update requests in mixed cluster

Relates #42596

Allocate to data-only nodes in ReopenWhileClosingIT (#42560)

If all primary shards are allocated on the master node, then the
verifying before close step will never interact with mock transport
service. This change prefers to allocate shards on data-only nodes.

Closes #39757

Reset mock transport service in CcrRetentionLeaseIT (#42600)

testRetentionLeaseIsAddedIfItDisappearsWhileFollowing does not reset the
mock transport service after test. Surviving transport interceptors from
that test can sneaky remove retention leases and make other tests fail.

Closes #39331
Closes #39509
Closes #41428
Closes #41679
Closes #41737
Closes #41756

Fixed ignoring name parameter for percolator queries (#42598)

Closes #40405

[Ml Data Frame] Return bad_request on preview when config is invalid (#42447)

Mute AsyncTwoPhaseIndexerTests#testStateMachine() (#42609)

Relates #42084

[ML DataFrame] Use date histogram fixed_interval syntax
and remove test skip

Mute NodeTests (#42614)

Relates #42577

Fix Incorrect Time Math in MockTransport (#42595)

* Fix Incorrect Time Math in MockTransport

* The timeunit here must be nanos for the current time (we even convert it accordingly in the logging)
* Also, changed the log message when dumping stack traces a little to make it easier to grep for (otherwise it's the same as the message on unregister)

Remove PRE_60_NODE_CHECKPOINT (#42527)

This commit removes the obsolete `PRE_60_NODE_CHECKPOINT` constant for dealing
with 5.x nodes' lack of sequence number support.

Backported as #42531

Reset state recovery after successful recovery (#42576)

The problem this commit addresses is that state recovery is not reset on a node that then becomes
master with a cluster state that has a state not recovered flag in it. The situation that was observed
in a failed test run of MinimumMasterNodesIT.testThreeNodesNoMasterBlock (see below) is that we
have 3 master nodes (node_t0, node_t1, node_t2), two of them are shut down (node_t2 remains),
when the first one comes back (renamed to node_t4) it becomes leader in term 2 and sends state
(with state_not_recovered_block) to node_t2, which accepts. node_t2 becomes leader in term 3, and
as it was previously leader in term1 and successfully completed state recovery, does never retry
state recovery in term 3.

Closes #39172

[DOCS] Escape cross-ref link comma for Asciidoctor (#42402)

[DOCS] Fix API Quick Reference rollup attribute for Asciidoctor (#42403)

[ML] adding delayed_data_check_config to datafeed update docs (#42095)

* [ML] adding delayed_data_check_config to datafeed update docs

* [DOCS] Edits delayed data configuration details

Avoid loading retention leases while writing them (#42620)

Resolves #41430.

Validate routing commands using updated routing state (#42066)

When multiple commands are called in sequence, fetch shards
from mutable, up-to-date routing nodes to ensure each command's
changes are visible to subsequent commands.

This addresses an issue uncovered during work on #41050.

remove 6.4.x version constants (#42127)

relates refactoring initiative #41164.

[ML Data Frame] Set DF task state when stopping  (#42516)

Set the state to stopped prior to persisting

[DOCS] Reorg monitoring configuration for re-use (#42547)

Remove suppresions for "unchecked" for hamcrest varargs methods (#41528)

In hamcrest 2.1 warnings for unchecked varargs were fixed by hamcrest using @SafeVarargs for those matchers where this warning occurred.
This PR is aimed to remove these annotations when Matchers.contains ,Matchers.containsInAnyOrder or Matchers.hasItems was used

Remove support for chained multi-fields. (#42333)

Follow-up to #41926, where we deprecated support for multi-fields within
multi-fields.

Addresses #41267.

Lazily compute Java 8 home in reindex configuration (#42630)

In the reindex from old tests we require Java 8. Today when configuring
the reindex from old tests, we eagerly evalulate Java 8 home, which
means that we require JAVA8_HOME to be set even if the reindex from old
test tasks are not in the task graph. This is an onerous requirement if,
for example, all that you want to do is build a distribution. This
commit addresses this by making evaluation of Java 8 home lazy, so that
it is only done and required if the reindex from old test tasks would be
executed.

Remove "nodes/0" folder prefix from data path (#42489)

With the removal of node.max_local_storage_nodes, there is no need anymore to keep the data in
subfolders indexed by a node ordinal. This commit makes it so that ES 8.0 will store data directly in
$DATA_DIR instead of $DATA_DIR/nodes/$nodeOrdinal.

Upon startup, Elasticsearch will check to see if there is data in the old location, and automatically
move it to the new location. This automatic migration only works if $nodeOrdinal is 0, i.e., multiple
node instances have not previously run on the same data path, which required for
node.max_local_storage_nodes to explicitly be configured.

[DOCS] Set explicit anchors for Asciidoctor (#42521)

unmute 'Test url escaping with url mustache function' and bump logging (#42400)

check position before and after latch (#42623)

check position before and after latch

[DOCS] Fix X-Pack tag for Asciidoctor (#42443)

fix javadoc of SearchRequestBuilder#setTrackTotalHits (#42219)

[ML Data Frame] Mute stop start test

Relates to https://github.com/elastic/elasticsearch/issues/42650

Add 7.1.2 version constant. (#42643)

Relates to #42635

Adjust use of Deprecated Netty API (#42613)

* With the recent upgrade to Netty 4.1.36 this method became deprecated and I made the advised change to fix the deprecation

Fix a callout in the field alias docs.

Add explicit build flag for experimenting with test execution cacheability (#42649)

* Add build flag for ignoring random test seed as task input

* Fix checkstyle violations

Use correct global checkpoint sync interval (#42642)

A disruption test case need to use a lower checkpoint sync interval
since they verify sequence numbers after the test waiting max 10 seconds
for it to stabilize.

Closes #42637

Removes types from SearchRequest and QueryShardContext (#42112)

[ML-DataFrame] rewrite start and stop to answer with acknowledged (#42589)

rewrite start and stop to answer with acknowledged

fixes #42450

Added param ignore_throttled=false when indicesOptions.ignoreThrottled() is false (#42393)

and fixed test RequestConvertersTests and added ignore_throttled on all request

[DOCS] Set explicit anchors for TLS/SSL settings (#42524)

Testclusters: convert ccr tests (#42313)

un-mute ActivateWatchTests, bump up logging, and remove explicit sleeps (#42396)

un-mute Watcher rolling upgrade tests and bump up logging (#42377)

Fixes watcher test to remove typed api call

Muting WatcherRestIT webhook url escaping test

See #41172

[DOCS] Adds more monitoring tagged regions

Add warning scores are floats (#42667)

Allow aggregations using expressions to use _score (#42652)

_score was removed from use in aggregations using expressions
unintentionally when script contexts were added. This allows _score to once
again be used.

Refactor HLRC RequestConverters parameters to be more explicit (#42128)

The existing `RequestConverters.Params` is confusing, because it wraps
an underlying request object and mutations of the `Params` object
actually mutate the `Request` that was used in the construction of the
`Params`.

This leads to a situation where we create a `RequestConverter.Params`
object, mutate it, and then it appears nothing happens to it - it
appears to be unused. What happens behind the scenes is that the Request
object is mutated when methods on `Params` are invoked. This results in
unclear, confusing code where mutating one object changes another with
no obvious connection.

This commit refactors `RequestConverters.Params` to be a simple helper
class to produce a `Map` which must be passed explicitly to a Request
object. This makes it apparent that the `Params` are actually used, and
that they have an effect on the `request` object explicit and easier to
understand.

Co-authored-by: Ojas Gulati <ojasgulati100@gmail.com>

Propogate version in reindex from remote search (#42412)

This is related to #31908. In order to use the external version in a
reindex from remote request, the search request must be configured to
request the version (as it is not returned by default). This commit
modifies the search request to request the version. Additionally, it
modifies our current reindex from remote tests to randomly use the
external version_type.

Fix inverted condition so we never cache rest integ tests

Remove unused import

Geo: Refactor libs/geo parsers (#42549)

Refactors the WKT and GeoJSON parsers from an utility class into an
instantiatable objects. This is a preliminary step in
preparation for moving out coordinate validators from Geometry
constructors. This should allow us to make validators plugable.

Detect when security index is closed (#42191)

If the security index is closed, it should be treated as unavailable
for security purposes.

Prior to 8.0 (or in a mixed cluster) a closed security index has
no routing data, which would cause a NPE in the cluster change
handler, and the index state would not be updated correctly.
This commit fixese that problem

Fix testTokenExpiry flaky test (#42585)

Test was using ClockMock#rewind passing the amount of nanoseconds
in order to "strip" nanos from the time value. This was intentional
as the expiration time of the UserToken doesn't have nanosecond
precision.
However, ClockMock#rewind doesn't support nanos either, so when it's
called with a TimeValue, it rewinds the clock by the TimeValue's
millis instead. This was causing the clock to go enough millis
before token expiration time and the test was passing. Once every
few hundred times though, the TimeValue by which we attempted to
rewind the clock only had nanos and no millis, so rewind moved the
clock back just a few millis, but still after expiration time.

This change moves the clock explicitly to the same instant as expiration,
using clock.setTime and disregarding nanos.

Revert "un-mute Watcher rolling upgrade tests and bump up logging (#42377)"

This reverts commit 697c793dcbabf1df0351d75a3705047ac4435dca.

Log leader and handshake failures by default (#42342)

Today the `LeaderChecker` and `HandshakingTransportAddressConnector` do not log
anything above `DEBUG` level. However there are some situations where it is
appropriate for them to log at a higher level:

- if the low-level handshake succeeds but the high-level one fails then this
  indicates a config error that the user should resolve, and the exception
  will help them to do so.

- if leader checks fail repeatedly then we restart discovery, and the exception
  will help to determine what went wrong.

Resolves #42153

Deprecate CommonTermsQuery and cutoff_frequency (#42619)

* Deprecate CommonTermsQuery and cutoff_frequency

Since the max_score optimization landed in Elasticsearch 7,
the CommonTermsQuery is redundant and slower. Moreover the
cutoff_frequency parameter for MatchQuery and MultiMatchQuery
is redundant.

Relates to #27096

Fix Class Load Order in Netty4Plugin (#42591)

* Don't force the logger in the Netty4Plugin class already, at this point log4j might not be fully initialized.
   * The call was redundant anyway since we do the same thing in the Netty4Transport and Netty4HttpServerTransport classes already and there we do it properly after setting up log4j by initilizing the loggers
* Relates #42532

[DOCS] Rewrite 'wildcard' query (#42670)

[DOCS] path_hierarchy tokenizer examples (#39630)

Closes #17138

Fix error with mapping in docs

Fix refresh remote JWKS logic (#42662)

This change ensures that:

- We only attempt to refresh the remote JWKS when there is a
signature related error only ( BadJWSException instead of the
geric BadJOSEException )
- We do call OpenIDConnectAuthenticator#getUserClaims upon
successful refresh.
- We test this in OpenIdConnectAuthenticatorTests.

Without this fix, when using the OpenID Connect realm with a remote
JWKSet configured in `op.jwks_path`, the refresh would be triggered
for most configuration errors ( i.e. wrong value for `op.issuer` )
and the kibana wouldn't get a response and timeout since
`getUserClaims` wouldn't be called because
`ReloadableJWKSource#reloadAsync` wouldn't call `onResponse` on the
future.

[ML] [Data Frame] add support for weighted_avg agg (#42646)

Remove unused Gradle plugin (#42684)

Remove usage of deprecated compare gradle builds plugin (#42687)

* Remove usage of deprecated compare gradle builds plugin

* Remove system property only used by build comparison

Prevent merging nodes' data paths (#42665)

Today Elasticsearch does not prevent you from reconfiguring a node's
`path.data` to point to data paths that previously belonged to more than one
node. There's no good reason to be able to do this, and the consequences can be
quietly disastrous. Furthermore, #42489 might result in a user trying to split
up a previously-shared collection of data paths by hand and there's definitely
scope for mixing the paths up across nodes when doing this.

This change adds a check during startup to ensure that each data path belongs
to the same node.

Clarify the settings around limiting nested mappings. (#42686)

* Previously, we mentioned multiple times that each nested object was indexed as its own document. This is repetitive, and is also a bit confusing in the context of `index.mapping.nested_fields.limit`, as that applies to the number of distinct `nested` types in the mappings, not the number of nested objects. We now just describe the issue once at the beginning of the section, to illustrate why `nested` types can be expensive.
* Reference the ongoing example to clarify the meaning of the two settings.

Addresses #28363.

Make hashed token ids url safe (#42651)

This commit changes the way token ids are hashed so that the output is
url safe without requiring encoding. This follows the pattern that we
use for document ids that are autogenerated, see UUIDs and the
associated classes for additional details.

[DOCS] Disable Metricbeat system module (#42601)

Remove SecurityClient from x-pack (#42471)

This commit removes the SecurityClient class from x-pack. This client
class is a relic of the transport client, which is in the process of
being removed. Some tests were changed to use the high level rest
client and others use a client directly without the security client
wrapping it.

Remove Log4j 1.2 API as a dependency (#42702)

We had this as a dependency for legacy dependencies that still needed
the Log4j 1.2 API. This appears to no longer be necessary, so this
commit removes this artifact as a dependency.

To remove this dependency, we had to fix a few places where we were
accidentally relying on Log4j 1.2 instead of Log4j 2 (easy to do, since
both APIs were on the compile-time classpath).

Finally, we can remove our custom Netty logger factory. This was needed
when we were on Log4j 1.2 and handled logging in our own unique
way. When we migrated to Log4j 2 we could have dropped this
dependency. However, even then Netty would still pick up Log4j 1.2 since
it was on the classpath, thus the advantage to removing this as a
dependency now.

Remove client jar support from build (#42640)

The client jars were a way for modules and plugins to produce an
additional jar that contained classes for use by the transport client.
This commit removes that configuration as the transport client is being
removed.

relates #42638

mute failing search template test (#42730)

tracking issue #42664.

Remove groovy client docs (#42731)

The groovy client api was a wrapper around the transport client.
However, it has not been published since 2.4, as it had many issues with
the java security manager. This commit removes the docs from master for
the groovy client.

relates #42638

Fix docs typo in the certutil CSR mode (#42593)

Changes the mention of `cert` to `csr`.

Co-Authored-By: Alex Pang <pangyikhei+github@gmail.com>

Remove transport client docs (#42483)

This commit removes the transport client documentation.

remove v6.5.x and v6.6.x version constants (#42130)

related to refactoring initiative #41164.

Log the status of security on license change (#42488)

Whether security is enabled/disabled is dependent on the combination
of the node settings and the cluster license.

This commit adds a license state listener that logs when the license
change causes security to switch state (or to be initialised).

This is primarily useful for diagnosing cluster formation issues.

Remove leftover transport module docs (#42734)

This commit removes docs for alternate transport implementations which
were removed years ago. These were missed because they have redirects
masking their existsence.

Add option to ObjectParser to consume unknown fields (#42491)

ObjectParser has two ways of dealing with unknown fields: ignore them entirely,
or throw an error. Sometimes it can be useful instead to gather up these unknown
fields and record them separately, for example as arbitrary entries in a map.

This commit adds the ability to specify an unknown field consumer on an ObjectParser,
called with the field name and parsed value of each unknown field encountered during
parsing. The public API of ObjectParser is largely unchanged, with a single new
constructor method and interface definition.

Return NO_INTERVALS rather than null from empty TokenStream (#42750)

IntervalBuilder#analyzeText will currently return null if it is passed an
empty TokenStream, which can lead to a confusing NullPointerException
later on during querying. This commit changes the code to return
NO_INTERVALS instead.

Fixes #42587

[ML] [Data Frame] nesting group_by fields like other aggs (#42718)

[ML Data Frame] Refactor stop logic (#42644)

* Revert "invalid test"

This reverts commit 9dd8b52c13c716918ff97e6527aaf43aefc4695d.

* Testing

* mend

* Revert "[ML Data Frame] Mute Data Frame tests"

This reverts commit 5d837fa312b0e41a77a65462667a2d92d1114567.

* Call onStop and onAbort outside atomic update

* Don’t update CS

* Tidying up

* Remove invalid test that asserted logic that has been removed

* Add stopped event

* Revert "Add stopped event"

This reverts commit 02ba992f4818bebd838e1c7678bd2e1cc090bfab.

* Adding check for STOPPED in saveState

Re-enable token bwc tests (#42726)

This commit re-enables token bwc tests that run as part of the rolling
upgrade tests. These tests were muted while #42651 was being
backported.

[ML] Add Kibana application privilege to data frame admin/user roles (#42757)

Data frame transforms are restricted by different roles to ML, but
share the ML UI.  To prevent the ML UI being hidden for users who
only have the data frame admin or user role, it is necessary to add
the ML Kibana application privilege to the backend data frame roles.

[DOCS] Remove unneeded `ifdef::asciidoctor[]` conditionals (#42758)

Several `ifdef::asciidoctor` conditionals were added so that AsciiDoc
and Asciidoctor doc builds rendered consistently.

With https://github.com/elastic/docs/pull/827, Elasticsearch Reference
documentation migrated completely to Asciidoctor. We no longer need to
support AsciiDoc so we can remove these conditionals.

Resolves #41722

 Remove CommonTermsQuery and cutoff_frequency param  (#42654)

Remove `common` query and `cutoff_frequency` parameter of
`match` and `multi_match` queries. Both have already been
deprecated for the next 7.x version.

Closes: #37096

Clarify that inner_hits must be used to access nested fields. (#42724)

This PR updates the docs for `docvalue_fields` and `stored_fields` to clarify
that nested fields must be accessed through `inner_hits`. It also tweaks the
nested fields documentation to make this point more visible.

Addresses #23766.

Remove locale-dependent string checking

We were checking if an exception was caused by a specific reason "Not a
directory". Alas, this reason is locale-dependent and can fail on
systems that are not set to en_US.UTF-8. This commit addresses this by
deriving what the locale-dependent error message would be and using that
for comparison with the actual exception thrown.

Closes #41689

[DOCS] Remove unneeded options from `[source,sql]` code blocks (#42759)

In AsciiDoc, `subs="attributes,callouts,macros"` options were required
to render `include-tagged::` in a code block.

With elastic/docs#827, Elasticsearch Reference documentation migrated
from AsciiDoc to Asciidoctor.

In Asciidoctor, the `subs="attributes,callouts,macros"` options are no
longer needed to render `include-tagged::` in a code block. This commit
removes those unneeded options.

Resolves #41589

address SmokeTestWatcherWithSecurityIT#testSearchInputWithInsufficientPrivileges (#42764)

This commit adds busy wait and increases the interval for
SmokeTestWatcherWithSecurityIT#testSearchInputWithInsufficientPrivileges.

Watcher will not allow the same watch to be executed concurrently. If it
finds that case, it will update the watch history with a "not_executed_already_queued"
status. Given a slow machine, and 1 second interval this is possible.

To address this, this commit increases the interval so the watch can fire at most 2
times with a greater interval between the executions and adds a busy wait for the
expected state.

While this does not gaurntee a fix, it should greatly reduce the chances of this
test erroring.

Remove XPackClient from x-pack (#42729)

This commit removes the XPackClient class from x-pack. This class is a
relic of the TransportClient and simply a wrapper around it. Calls are
replaced with direct usage of a client. Additionally, the
XPackRestHandler class has been removed as it only served to provide
the XPackClient to implementing rest handlers.

Remove MonitoringClient from x-pack (#42770)

This commit removes the monitoring client from x-pack. This class is a
relic of the TransportClient and was only used in a test.

Use an anonymous inner class instead of lambda for UP-TO-DATE support

remove v6.8.x version constant and the backcompat code that uses it (#42146)

Remove Support for VERSION_CHECKPOINTS Translogs (#42782)

* Closes #42699

Remove some leftover refs to minimum_master_nodes (#42700)

Today `InternalTestCluster` has a few vestigial mentions of the
`minimum_master_nodes` setting. This commit removes them and simplifies some of
the surrounding logic.

Create client-only AnalyzeRequest/AnalyzeResponse classes (#42197)

This commit clones the existing AnalyzeRequest/AnalyzeResponse classes
to the high-level rest client, and adjusts request converters to use these new
classes.

This is a prerequisite to removing the Streamable interface from the internal
server version of these classes.

[ML] Better detection of binary input in find_file_structure (#42707)

This change helps to prevent the situation where a binary
file uploaded to the find_file_structure endpoint is
detected as being text in the UTF-16 character set, and
then causes a large amount of CPU to be spent analysing
the bogus text structure.

The approach is to check the distribution of zero bytes
between odd and even file positions, on the grounds that
UTF-16BE or UTF16-LE would have a very skewed distribution.

[Docs] Add example to reimplement stempel analyzer (#42676)

Adding an example of how to re-implement the polish stempel analyzer
in case a user want to modify or extend it. In order for the analyzer to be
able to use polish stopwords, also registering a polish_stop filter for the
stempel plugin.

Closes #13150

Clarify heap setting in Docker docs (#42754)

Add note in the Docker docs that even when container memory is limited,
we still require specifying -Xms/-Xmx using one of the supported
methods.

[ML] Add a limit on line merging in find_file_structure (#42501)

When analysing a semi-structured text file the
find_file_structure endpoint merges lines to form
multi-line messages using the assumption that the
first line in each message contains the timestamp.
However, if the timestamp is misdetected then this
can lead to excessive numbers of lines being merged
to form massive messages.

This commit adds a line_merge_size_limit setting
(default 10000 characters) that halts the analysis
if a message bigger than this is created.  This
prevents significant CPU time being spent subsequently
trying to determine the internal structure of the
huge bogus messages.

[DOCS] Adds redirect for deprecated `common` terms query (#42767)

Make Connection Future Err. Handling more Resilient (#42781)

* There were a number of possible (runtime-) exceptions that could be raised in the adjusted code and prevent resolving the listener
* Relates #42350

Read the default pipeline for bulk upsert through an alias (#41963)

This commit allows bulk upserts to correctly read the default pipeline
for the concrete index that belongs to an alias.

Bulk upserts are modeled differently from normal index requests such that
the index request is a request inside of the update request. The update
request (outer) contains the index or alias name is not part of the (inner)
index request. This commit adds a secondary check against the update request
(outer) if the index request (inner) does not find an alias.

RollupStart endpoint should return OK if job already started (#41502)

If a job is started or indexing, RollupStart should always return
a success (200 OK) response since the job is, in fact, started

SQL: [Docs] Fix links syntax (#42806)

Fix a couple of wrong links because of the order of the anchor
and the usage of backquotes.

More improvements to cluster coordination docs (#42799)

This commit addresses a few more frequently-asked questions:

* clarifies that bootstrapping doesn't happen even after a full cluster
  restart.

* removes the example that uses IP addresses, to try and further encourage the
  use of node names for bootstrapping.

* clarifies that auto-bootstrapping might form different clusters on different
  hosts, and gives a process for starting again if this wasn't what you wanted.

* adds the "do not stop half-or-more of the master-eligible nodes" slogan that
  was notably absent.

* reformats one of the console examples to a narrower width

Remove "template" field in IndexTemplateMetaData (#42099)

Remove "template" field from XContent parsing in IndexTemplateMetaData

Fix error with test conventions on tasks that require Docker (#42719)

[ML] [Data Frame] adding and modifying auditor messages (#42722)

* [ML] [Data Frame] adding and modifying auditor messages

* Update DataFrameTransformTask.java

Make high level rest client a fat jar (#42771)

The original intention of the high level rest client was to provide a
single jar. We tried this long ago, but had issues with intellij not
correctly resolving internal tests that relied on the HLRC. This
commit tweaks our use of the shadow plugin so we now produce a correct
fat jar (minus the LLRC and server jars, which we can address later),
with the module "client" dependencies included, as well as the
correct pom file omitting those dependencies.

relates #42638

Add Basic Date Docs to Painless (#42544)

[Docs] Add note for date patterns used for index search. (#42810)

Add an explanatory NOTE section to draw attention to the difference
between small and capital letters used for the index date patterns.
e.g.: HH vs hh, MM vs mm.

Closes: #22322

[Docs] Fix reference to `boost` and `slop` params (#42803)

For `multi_match` query: link `boost` param to the generic reference
for query usage and `slop` to the `match_phrase` query where its usage
is documented.

Fixes: #40091

Remove unnecessary usage of Gradle dependency substitution rules (#42773)

Don't require TLS for single node clusters (#42826)

This commit removes the TLS cluster join validator.

This validator existed to prevent v6.x nodes (which mandated
TLS) from joining an existing cluster of v5.x nodes (which did
not mandate TLS) unless the 6.x node (and by implication the
5.x nodes) was configured to use TLS.

Since 7.x nodes cannot talk to 5.x nodes, this validator is no longer
needed.

Removing the validator solves a problem where single node clusters
that were bound to local interfaces were incorrectly requiring TLS
when they recovered cluster state and joined their own cluster.

OIDC Guide additions (#42555)

- Call out the fact that the SSL Configuration is important and
offer a minimal example of configuring a custom CA for trust.
- Add information about the `op.issuer` that was missing and add
information about the `rp.post_logout_redirect` in the example
since `op.endsession_endpoint` was already mentioned there and
these two should be together
- Explain that `op.jwkset_path` can be a URL.

[ML] [Data Frame] Adding supported aggs in docs (#42728)

* [ML] [Data Frame] Adding supported aggs in docs

* [DOCS] Moves pivot to definitions list

[ML][Data Frame] forcing that no ptask => STOPPED state (#42800)

* [ML][Data Frame] forcing that no ptask => STOPPED state

* Addressing side-effect, early exit for stop when stopped

[Docs] Add to preference parameter docs (#42797)

Adding notes to the existing docs about how using `preference` might increase
request cache utilization but also add warning about the downsides.

Closes #24278

[DOCS] Fix broken bucket script agg link

Refactor control flow in TransportAnalyzeAction (#42801)

The control flow in TransportAnalyzeAction is currently spread across two large
methods, and is quite difficult to follow. This commit tidies things up a bit, to make
it clearer when we use pre-defined analyzers and when we use custom built ones.

[DOCS] Fix typo in bucket script aggregation link

Fix testNoMasterActionsWriteMasterBlock (#42798)

This commit performs the proper restore of network disruption.
Previously disruptionScheme.stopDisrupting() was called that does not
ensure that connectivity between cluster nodes is restored. The test
was checking that the cluster has green status, but it was not checking
that connectivity between nodes is restored.
Here we switch to internalCluster().clearDisruptionScheme(true) which
performs both checks before returning.

Closes #39688

Change shard allocation filter property and api (#42602)

The current example is not working and a bit confused. This change tries
to match it with the sample of the watcher blog.

NullPointerException when creating a watch with Jira action (#41922) (#42081)

NullPointerException when secured_url does not use proper scheme in jira action.
This commit will handle Expection and display proper message.

Eclipse libs projects setup fix (#42852)

Fallout from #42773 for eclipse users.

Replicate aliases in cross-cluster replication (#41815)

This commit adds functionality so that aliases that are manipulated on
leader indices are replicated by the shard follow tasks to the follower
indices. Note that we ignore write indices. This is due to the fact that
follower indices do not receive direct writes so the concept is not
useful.

Fix version parsing in various tests (#42871)

This commit fixes the version parsing in various tests. The issue here is that
the parsing was relying on java.version. However, java.version can contain
additional characters such as -ea for early access builds. See JEP 233:

Name                            Syntax
------------------------------  --------------
java.version                    $VNUM(\-$PRE)?
java.runtime.version            $VSTR
java.vm.version                 $VSTR
java.specification.version      $VNUM
java.vm.specification.version   $VNUM

Instead, we want java.specification.version.

Adjust BWC version on aliases replication

This commit adjusts the BWC version on aliases replication after the
change has been backported to 7.x (currently versioned as 7.3.0).

Enable testing against JDK 13 EA builds (#40829)

This commit adds JDK 13 to the CI rotation for testing. For now, we will
be testing against JDK 13 EA builds.

Avoid clobbering shared testcluster JAR files when installing modules (#42879)

Permit API Keys on Basic License (#42787)

Kibana alerting is going to be built using API Keys, and should be
permitted on a basic license.

This commit moves API Keys (but not Tokens) to the Basic license

Relates: kibana#36836

Deduplicate alias and concrete fields in query field expansion (#42328)

The full-text query parsers accept field pattern that are expanded using the mapping.
Alias field are also detected during the expansion but they are not deduplicated with the
concrete fields that are found from other patterns (or the same). This change ensures
that we deduplicate the target fields of the full-text query parsers in order to avoid
adding the same clause multiple times. Boolean queries are already able to deduplicate
clauses during rewrite but since we also use DisjunctionMaxQuery it is preferable to detect
 these duplicates early on.

Enable Parallel Deletes in Azure Repository (#42783)

* Parallel deletes via private thread pool

More logging in testRerouteOccursOnDiskPassingHighWatermark (#42864)

This test is failing because recoveries of these empty shards are not
completing in a reasonable time, but the reason for this is still obscure. This
commit adds yet more logging.

Relates #40174, #42424

Removes type from TermVectors APIs (#42198)

Use reader attributes to control term dict memory useage (#42838)

This change makes use of the reader attributes added in LUCENE-8671
to ensure that `_id` fields are always on-heap for best update performance
and term dicts are generally off-heap on Read-Only engines.

Closes #38390

Fix Stuck IO Thread Logging Time Precision (#42882)

* The precision of the timestamps we get from the cached time thread is only 200ms by default resulting in a number of needless ~200ms slow network thread execution logs
  * Fixed by making the warn threshold a function of the precision of the cached time thread found in the settings

Enable console audit logs for docker (#42671)

Enable audit logs in docker by creating console appenders for audit loggers.
also rename field @timestamp to timestamp and add field `type` with value audit

The docker build contains now two log4j configuration for oss or default versions. The build now allows override the default configuration.

Also changed the format of a timestamp from ISO8601 to include time zone as per this discussion https://github.com/elastic/elasticsearch/pull/36833#discussion_r244225243

closes #42666

[ML] Change dots in CSV column names to underscores (#42839)

Dots in the column names cause an error in the ingest
pipeline, as dots are special characters in ingest pipeline.
This PR changes dots into underscores in CSV field names
suggested by the ML find_file_structure endpoint _unless_
the field names are specifically overridden.  The reason for
allowing them in overrides is that fields that are not
mentioned in the ingest pipeline can contain dots.  But it's
more consistent that the default behaviour is to replace
them all.

Fixes elastic/kibana#26800

Disable building on JDK 13 in CI

This commit disables building on JDK 13 in CI. The reason for this is
because Gradle is not yet ready to run on JDK 13. We could re-introduce
infrastructure to enable Gralde to run on a different JDK than the build
JDK, but rather than introducing such complexity we will instead wait
for Gradle to be ready to run on JDK 13.

Add Ability to List Child Containers to BlobContainer (#42653)

* Add Ability to List Child Containers to BlobContainer
* This is a prerequisite of #42189

Fix Azure Plugin Compilation Issue

Fix Infinite Loops in ExceptionsHelper#unwrap (#42716)

* Fix Infinite Loops in ExceptionsHelper#unwrap

* Keep track of all seen exceptions and break out on loops
* Closes #42340

Add custom metadata to snapshots (#41281)

Adds a metadata field to snapshots which can be used to store arbitrary
key-value information. This may be useful for attaching a description of
why a snapshot was taken, tagging snapshots to make categorization
easier, or identifying the source of automatically-created snapshots.

Omit JDK sources archive from bundled JDK (#42821)

Clean Up Painless Datetime Docs (#42869)

This change abstracts the specific types away from the different
representations of datetime as a datetime representation in code can be all
kinds of different things. This defines the three most common types of
datetimes as numeric, string, and complex while outlining the type most
typically used for these as long, String, and ZonedDateTime, respectively.
Documentation uses the definitions while examples use the types. This makes
the documentation easier to consume especially for people from a non-Java
background.

Optimize Snapshot Finalization (#42723)

* Optimize Snapshot Finalization

* Delete index-N blobs and segement blobs in one single bulk delete instead of in separate ones to save RPC calls on implementations that have bulk deletes implemented
* Don't fail snapshot because deleting old index-N failed, this results in needlessly logging finalization failures and makes analysis of failures harder going forward as well as incorrect index.latest blobs

Make sibling pipeline agg ctor's protected (#42808)

SiblingPipelineAggregator is a public interfaces,
but the ctor was package-private.  These should be protected so that
plugin authors can extend and implement their own sibling pipeline agg.

[DOCS] Adds discovery.type (#42823)

Co-Authored-By: David Turner <david.turner@elastic.co>

[Docs] Clarify caveats for phonetic filters replace option (#42807)

The `replace` option in the phonetic token filter can have suprising side
effects, e.g. such as described in #26921. This PR adds a note to be mindful
about such scenarios and offers alternatives to using the `replace` option.

Closes #26921

Skip installation of pre-bundled integ-test modules (#42900)

Mute failing test

Remove alpha/beta/rc from version constants (#42778)

Prerelease qualifiers were moved outside of Version logic within
Elasticsearch for 7.0.0, where they are now just an external modifier on
the filename. However, they still existed inside code to support 6.x
constants. Now that those constants have been removed in master, the
prerelease logic can be removed.

Skip shadow jar logic for javadoc and sources jars (#42904)

For shadow jars, we place the original jar in a build/libs
directory. This is to avoid clobbering the original jar when building
the shadow jar. However, we need to skip this logic for javadoc and
sources jars otherwise they would never be copied to the
build/distributions directory during assembly.

Use jar task name constant in BuildPlugin

Rather than comparing to a raw string, this commit uses a built-in
constant to refer to the jar task name.

Relates #42904

Remove the transport client (#42538)

This commit removes the transport client and all remaining uses in the code.

Correct versions limits for snapshot metadata field (#42911)

Now that the snapshot metadata field has been backported, the version
restrictions used in tests and for serialization need to corrected.

[ML-DataFrame] increase the scheduler interval to 10s (#42845)

increases the scheduler interval to fire less frequently, namely changing it from 1s to 10s. The scheduler interval is used for retrying after an error condition.

[ML-DataFrame] reduce log spam: do not trigger indexer if state is indexing or stopping (#42849)

reduce log spam: do not trigger indexer if state is indexing or stopping

[ML] Add earliest and latest timestamps to field stats (#42890)

This change adds the earliest and latest timestamps into
the field stats for fields of type "date" in the output of
the ML find_file_structure endpoint.  This will enable the
cards for date fields in the file data visualizer in the UI
to be made to look more similar to the cards for date
fields in the index data visualizer in the UI.

[ML] Close sample stream in find_file_structure endpoint (#42896)

A static code analysis revealed that we are not closing
the input stream in the find_file_structure endpoint.
This actually makes no difference in practice, as the
particular InputStream implementation in this case is
org.elasticsearch.common.bytes.BytesReferenceStreamInput
and its close() method is a no-op.  However, it is good
practice to close the stream anyway.

Mute testEnableDisableBehaviour (#42929)

[ML] [Data Frame] Adding pending task wait to the hlrc cleanup (#42907)

Add a merge policy that prunes ID postings for soft-deleted but retained documents (#40741)

* Add a merge policy that prunes soft-deleted postings

This change adds a merge policy that drops all postings for documents that
are marked as deleted. This is usually unnecessary unless soft-deletes are used
with a rentention policy since otherwise a merge would remove deleted documents anyway.
Yet, this merge policy prevents extreme cases where a very large number of soft-deleted
documents are retained and are impacting search and update perfromance.
Note, using this merge policy will remove all search capabilities for soft-deleted documents.

* fix checkstyle

* fix assertion

* fix imports

* fix compilation

* add predicate to select fields to prune

* only purne ID field

* beef up test

* roll back retention query

* foo

* remove redundant modifier

* fix assumption about empty Terms

* remove null check

* Add test for the engine to check if we prune the IDs of retained docs away

Mute failing testPerformActionAttrsRequestFails (#42933)

[ML][Data Frame] pull state and states for indexer from index (#42856)

* [ML][Data Frame] pull state and states for indexer from index

* Update DataFrameTransformTask.java

Revert "Add a merge policy that prunes ID postings for soft-deleted but retained documents (#40741)"

This reverts commit 186b52c5738688b72543d9353539468e719fafce
github messed up the commit message due to a retry.
A followup commit will add this change again with a corrected
commit message.

Add a merge policy that prunes ID postings for soft-deleted but retained documents (#40741)

This change adds a merge policy that drops all _id postings for documents that
are marked as soft-deleted but retained across merges. This is usually unnecessary
unless soft-deletes are used with a retention policy since otherwise a merge would
remove deleted documents anyway.

Yet, this merge policy prevents extreme cases where a very large number of soft-deleted
documents are retained and are impacting update performance.
Note, using this merge policy will remove all lookup by ID capabilities for soft-deleted documents.

configure auto expand for dataframe indexes (#42924)

creates the dataframe destination index with auto expand for replicas (0-1)

Fix NPE when rejecting bulk updates (#42923)

Single updates use a different internal code path than updates that are wrapped in a bulk request.
While working on a refactoring to bring both closer together I've noticed that bulk updates were
failing some of the tests that single updates passed. In particular, bulk updates cause
NullPointerExceptions to be thrown and listeners not being properly notified when being rejected
from the thread pool.

Fix testPendingTasks (#42922)

Fixes a race in the test which can be reliably reproduced by adding Thread.sleep(100) to the end of
IndicesService.processPendingDeletes

Closes #18747

Fix `InternalEngineTests#testPruneAwayDeletedButRetainedIds`

The test failed because we had only a single document in the index
that got deleted such that some assertions that expected at least
one live doc failed.

Relates to: #40741

[TEST] Remove unnecessary log line

[DOCS] Rewrite terms query (#42889)

Reindex max_docs parameter name (#41894)

Previously, a reindex request had two different size specifications in the body:
* Outer level, determining the maximum documents to process
* Inside the source element, determining the scroll/batch size.

The outer level size has now been renamed to max_docs to
avoid confusion and clarify its semantics, with backwards compatibility and
deprecation warnings for using size.
Similarly, the size parameter has been renamed to max_docs for
update/delete-by-query to keep the 3 interfaces consistent.

Finally, all 3 endpoints now support max_docs in both body and URL.

Relates #24344

[DOCS] Move 'Scripting' section to top-level navigation. (#42939)

shrink may full copy when using multi data paths (#42913)

Additional scenario for full segment copy if hard link
cannot work across disks.

Fix concurrent search and index delete (#42621)

Changed order of listener invocation so that we notify before
registering search context and notify after unregistering same.

This ensures that count up/down like what we do in ShardSearchStats
works. Otherwise, we risk notifying onFreeScrollContext before notifying
onNewScrollContext (same for onFreeContext/onNewContext, but we
currently have no assertions failing in those).

Closes #28053

Wire query cache into sorting nested-filter computation (#42906)

Don't use Lucene's default query cache when filtering in sort.

Closes #42813

Make PR template reference supported architectures (#42919)

This commit changes the GitHub PR template to refer to supported "OS
and architecture" (rather than use OS) since we only accept PRs for
x86_64 (and not Linux ARM, s390, etc)

Relax timeout in NodeConnectionsServiceTests (#42934)

Today we assert that the connection thread is blocked by the time the test gets
to the barrier, but in fact this is not a valid assertion. The following
`Thread.sleep()` will cause the test to fail reasonably often.

```diff
diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
index 193cde3180d..0e57211cec4 100644
--- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
@@ -364,6 +364,7 @@ public class NodeConnectionsServiceTests extends ESTestCase {
             final CheckedRunnable<Exception> connectionBlock = nodeConnectionBlocks.get(node);
             if (connectionBlock != null) {
                 try {
+                    Thread.sleep(50);
                     connectionBlock.run();
                 } catch (Exception e) {
                     throw new AssertionError(e);
```

This change relaxes the test to allow some time for the connection thread to
hit the barrier.

Fixes #40170

Improve translog corruption detection (#42744)

Today we test for translog corruption by incrementing a byte by 1 somewhere in
a file, and verify that this leads to a `TranslogCorruptionException`.
However, we rely on _all_ corruptions leading to this exception in the
`RemoveCorruptedShardDataCommand`: this command fails if a translog file
corruption leads to a different kind of exception, and `EOFException` and
`NegativeArraySizeException` are both possible. This commit strengthens the
translog corruption detection tests by simulating the following:

- a bit is flipped
- all bits are cleared or set
- a random value is written
- the file is truncated

It also makes sure that we return a `TranslogCorruptionException` in all such
cases.

Fixes #42661

Fix FsRepositoryTests.testSnapshotAndRestore (#42925)

* The commit generation can be 3 or 2 here -> fixed by checking the actual generation on the second commit instead of hard coding 2
* Closes #42905

Only ignore IOException when fsyncing on dirs (#42972)

Today in the method IOUtils#fsync we ignore IOExceptions when fsyncing a
directory. However, the catch block here is too broad, for example it
would be ignoring IOExceptions when we try to open a non-existant
file. This commit addresses that by scoping the ignored exceptions only
to the invocation of FileChannel#force.

Remove Comma in Example (#41873)

The comma is there in error as there are no other parameter after 'value'

[ML][Data frame] make sure that fields exist when creating progress (#42943)

[TEST] Adding a BWC test for ML categorization config (#42981)

This test coverage was previously missing.

Remove WatcherClient from x-pack (#42815)

This commit removes the WatcherClient and WatcherRestHandler from the
codebase. The WatcherClient was a convenience wrapper around the
transport client, which is being removed so the client no longer serves
a purpose. The WatcherRestHandler is no longer needed as its primary
purpose was to provide a WatcherClient to the implementing handlers.

Remove the CcrClient (#42816)

This commit removes the CcrClient class, which is a wrapper around the
transport client. The transport client is being removed so the client
is no longer needed.

Remove the ILMClient (#42817)

This commit removes the ILMClient class, which is a wrapper around the
transport client. This class is not used in the codebase and the
transport client is being removed.

[DOCS] Add explicit `articles_case` parameter to Elision Token Filter example (#42987)

Update default shard count per index in readme (#42388)

The default shard count has been reduced from 5 to 1. This commit
updates the readme to reflect that changed default.

[ML][Data Frame] allow null values for aggs with sparse data (#42966)

* [ML][Data Frame] allow null values for aggs with sparse data

* Making classes static, memory allocation optimization

Drop dead code for socket permissions for transport (#42990)

This code has not been needed since the removal of tribe nodes, it was
left behind when those were dropped (note that regular transport
permissions are handled through transport profiles, even if they are not
explicitly in use).

Fix possible NPE in put mapping validators (#43000)

When applying put mapping validators, we apply all the validators in the
collection. If a failure occurs, we collect that as a top-level
exception, and suppress any additional failures into the top-level
exception. However, if a request passes the validator after a top-level
exception has been collected, we would try to suppress a null exception
into the top-level exception. This is a violation of the
Throwable#addSuppressed API. This commit addresses this, and adds test
to cover the logic of collecting the failures when validating a put
mapping request.

Fix put mapping request validators random test

This commit fixes a test bug in the request validators random test. In
particular, an assertion was not properly nested in a guard that would
ensure that was at least one failure.

Relates #43000

Fix IOUtils#fsync on Windows fsyncing directories (#43008)

Fsyncing directories on Windows is not possible. We always suppressed
this by allowing that an AccessDeniedException is thrown when attemping
to open the directory for reading. Yet, this suppression also allowed
other IOExceptions to be suppressed, and that was a bug (e.g., the
directory not existing, or a filesystem error and reasons that we might
get an access denied there, like genuine permissions issues). This
leniency was previously removed yet it exposed that we were suppressing
this case on Windows. Rather than relying on exceptions for flow control
and continuing to suppress there, we simply return early if attempting
to fsync a directory on Windows (we will not put this burden on the
caller).

Mute testLookupSeqNoByIdInLucene

Tracked at #42979

Mute AutodetectMemoryLimitIT#testTooManyPartitions

Relates #43013

Fix assertion in ReadOnlyEngine (#43010)

We should execute the assertion before throwing an exception;
otherwise, it's a noop.

Unmuted testRecoverBrokenIndexMetadata

These tests should be okay as we flush at the end of peer recovery.

Closes #40867

Refactor put mapping request validation for reuse (#43005)

This commit refactors put mapping request validation for reuse. The
concrete case that we are after here is the ability to apply effectively
the same framework to indices aliases requests. This commit refactors
the put mapping request validation framework to allow for that.

Do not allow modify aliases on followers (#43017)

Now that aliases are replicated by a follower from its leader, this
commit prevents directly modifying aliases on follower indices.

Adjust IndicesAliasesRequest origin BWC version

The work to add the origin field to the IndicesAliasesRequest has been
backported to 7.x. Since this version is currently 7.3.0, this commit
adjusts the version in master accordingly.

Add note to CCR docs regarding alias replication

This commit adds a note to the docs regarding the automatic replication
of aliases by a follower index from its leader index.

Add note to CCR docs about mapping/alias updates

This commit adds a note to the docs clarifying that it is not possible
to manually update the mapping nor the aliases of a follower index.

Unmute PermissionsIT test and enable debug logging for it (#42876)

This unmutes `testWhenUserLimitedByOnlyAliasOfIndexCanWriteToIndexWhichWasRolledoverByILMPolicy`
and enables DEBUG logging. The failure from this test case from a query
running rather than ILM itself, so more information is needed.

Relates to #41440

Since SQL is GA, remove the sql language plugin from this list (#41533)

SQL: cover the Integer type when extracting values from _source (#42859)

* Take into consideration a wider range of Numbers when extracting the
values from source, more specifically - BigInteger and BigDecimal.

Allow routing commands with ?retry_failed=true (#42658)

We respect allocation deciders, including the `MaxRetryAllocationDecider`, when
executing reroute commands. If you specify `?retry_failed=true` then the retry
counter is reset, but today this does not happen until after trying to execute
the reroute commands. This means that if an allocation has repeatedly failed,
but you want to take control and assign a shard to a particular node to work
around the repeated failures, you cannot execute the routing command in the
same call to `POST /_cluster/reroute` as the one that resets the failure
counter.

This commit fixes this by resetting the failure counter first, meaning that you
can now explicitly allocate a repeatedly-failed shard like this:

```
POST /_cluster/reroute?retry_failed=true
{
  "commands": [
    {
      "allocate_replica": {
        "index": "blahblah",
        "shard": 2,
        "node": "node-4"
      }
    }
  ]
}
```

Fixes #39546

Fix auto fuzziness in query_string query (#42897)

Setting `auto` after the fuzzy operator (e.g. `"query": "foo~auto"`) in the `query_string`
does not take the length of the term into account when computing the distance and always use
a max distance of 1. This change fixes this disrepancy by ensuring that the term is passed when
the fuzziness is computed.

Don't run build-tools integ tests on FIPS (#42986)

These run Gradle and FIPS isn't supported

Closes #41721

Fix typo in create-index.asciidoc (#41806)

Update regexp-syntax.asciidoc (#43021)

Corrects a typo.

Update search-settings.asciidoc (#43016)

Grammar and spelling fixes

[ML] Re-enable integration test (#41712)

Move construction of custom analyzers into AnalysisRegistry (#42940)

Both TransportAnalyzeAction and CategorizationAnalyzer have logic to build
custom analyzers for index-independent analysis. A lot of this code is duplicated,
and it requires the AnalysisRegistry to expose a number of internal provider
classes, as well as making some assumptions about when analysis components are
constructed.

This commit moves the build logic directly into AnalysisRegistry, reducing the
registry's API surface considerably.

Improve documentation for smart_cn analyzer (#42822)

Correct the description of generate_word_parts (#43026)

Clean up configuration when docker isn't available (#42745)

We initially added `requireDocker` for a way for tasks to say that they
absolutely must have it, like the  build docker image tasks.
Projects using the test fixtures plugin are not in this both, as the
intent with these is that they will be skipped if docker and docker-compose
is not available.

Before this change we were lenient, the docker image build would succeed
but produce nothing. The implementation was also confusing as it was not
immediately obvious this was the case due to all the indirection in the
code.

The reason we have this leniency is that when we added the docker image
build, docker was a fairly new requirement for us, and we didn't have
it deployed in CI widely enough nor had CI configured to prefer workers
with docker when possible. We are in a much better position now.
The other reason was other stack teams running `./gradlew assemble`
in their respective CI and the possibility of breaking them if docker is
not installed. We have been advocating for building specific distros for
some time now and I will also send out an additional notice

The PR also removes the use of `requireDocker` from tests that actually
use test fixtures and are ok without it, and fixes a bug in test
fixtures that would cause incorrect configuration and allow some tasks
to run when docker was not available and they shouldn't have.

Closes  #42680 and #42829  see also #42719

Better Exception in NetworkUtilsTests (#42109)

* We are still running into an exception here every so often
   * Adjusted exception to contain interface name
* Relates to #41549

Fix GCS Blob Repository 3rd Party Tests (#43030)

* We have to strip the trailing slash from child names here like we do for AWS
* closes #43029

[DOCS] Change `// TESTRESPONSE[_cat]` to `// TESTRESPONSE[non_json]` (#43006)

[ML] Get resources action should be lenient when sort field is unmapped (#42991)

Get resources action sorts on the resource id. When there are no resources at
all, then it is possible the index does not contain a mapping for the resource
id field. In that case, the search api fails by default.

This commit adjusts the search request to ignore unmapped fields.

Closes elastic/kibana#37870

Mute AzureDiscoveryClusterFormationTests (#43049)

Relates #43048

Fix IpFilteringIntegrationTests (#43019)

* Increase timeout to 5s since we saw 500ms+ GC pauses on CI
* closes #40689

Increase waiting time when check retention locks (#42994)

WriteActionsTests#testBulk and WriteActionsTests#testIndex sometimes
fail with a pending retention lock. We might leak retention locks when
switching to async recovery. However, it's more likely that ongoing
recoveries prevent the retention lock from releasing.

This change increases the waiting time when we check for no pending
retention lock and also ensures no ongoing recovery in
WriteActionsTests.

Closes #41054

[ML][Data Frame] Removes slice specification from DBQ. See #42996 (#43036)

Rename processor test fix (#43035)

If the source field name is a prefix of the target field name, the
source field still exists after rename processor has run. Adjusted test
case to handle that case.

Default distro run creates elastic-admin user (#43004)

When using gradle run by itself, this uses the default distro with a
basic license and enables security. There is a setup command to create
a elastic-admin user but only when the license is a trial license. Now
that security is available with the basic license, we should always run
this command when using the default distribution.

Fixing handling of auto slices in bulk scroll requests (#43050)

* Fixing handling of auto slices in bulk scroll requests

* adjusting assertions for tests

Unmute IndexFollowingIT#testFollowIndex

Fixed in #41987

Fix NPE in CcrRetentionLeaseIT (#43059)

The retention leases stats is null if the processing shard copy is being
closed. In this the case, we should check against null then retry to
avoid failing a test.

Closes #41237

[ML] Changes slice specification to auto. See #42996 (#43039)

[ML] Adding support for geo_shape, geo_centroid, geo_point in datafeeds (#42969)

* [ML] Adding support for geo_shape, geo_centroid, geo_point in datafeeds

* only supporting doc_values for geo_point fields

* moving validation into GeoPointField ctor

Upgrade AWS SDK to Latest Version (#42708)

* Just staying up to data on the SDK version
* Use `AbstractAmazonEC2` to shorten code

Better test diag output on OOM (#42989)

If linearizability checking fails with OOM (or other exception), we did
not get the serialized history written into the log, making it difficult
to debug in cases where the problem is hard to reproduce. Fixed to
always attempt dumping the serialized history.

Related to #42244

Refresh remote JWKs on all errors (#42850)

It turns out that key rotation on the OP, can manifest as both
a BadJWSException and a BadJOSEException in nimbus-jose-jwt. As
such we cannot depend on matching only BadJWSExceptions to
determine if we should poll the remote JWKs for an update.

This has the side-effect that a remote JWKs source will be polled
exactly one additional time too for errors that have to do with
configuration, or for errors that might be caused by not synched
clocks, forged JWTs, etc. ( These will throw a BadJWTException
which extends BadJOSEException also )

Split search in two when made against throttled and non throttled searches (#42510)

When a search on some indices takes a long time, it may cause problems to other indices that are being searched as part of the same search request and being written to as well, because their search context needs to stay open for a long time. This is especially a problem when searching against throttled and non-throttled indices as part of the same request. The problem can be generalized though: this may happen whenever read-only indices are searched together with indices that are being written to. Search contexts staying open for a long time is only an issue for indices that are being written to, in practice.

This commit splits the search in two sub-searches: one for read-only indices, and one for ordinary indices. This way the two don't interfere with each other. The split is done only when size is greater than 0, no scroll is provided and query_then_fetch is used as search type. Otherwise, the search executes like before. Note that the returned num_reduce_phases reflect the number of reduction phases that were run. If the search is split in two, there are three reductions: one non-final for each search, and a final one that merges the results of the previous two.

Closes #40900

[DOCS] Clarify phrase suggester docs smoothing parameter (#42947)

Closes #28512

remove path from rest-api-spec (#41452)

SQL: Clarify that the connections the jdbc driver creates are not pooled (#42992)

Restructure the SQL Language section to have proper sub-sections (#43007)

Rest docs page update
- have the section be on separate pages
- add an Overview page
- add other formats examples

Increase test logging for testSyncedFlushSkipOutOfSyncReplicas

Relates to #43086

Rename TESTRESPONSE[_cat] to TESTRESPONSE[non_json] (#43087)

Documents the new deprecations options on the rest-api-spec (#41444)

* Documents the new deprecations options on the rest-api-spec

Relates #41439 #38613 #35262

* remove reference to path now that #41452 is merged, also fixed missing a comma rendering the example json invalid

* removed one more instance of path

* make sure json examples are self contained and not excerpts

Handle empty terms index in TermsSliceQuery (#43078)

field on soft deleted documents. The TermsSliceQuery assumes that every document
has has an entry in the postings for that field so it doesn't check if the terms
index exists or not. This change fixes this bug by checking if the terms index for
the `_id` field is null and ignore the segment entirely if it's the case. This should
be harmless since segments without an `_id` terms index should only contain soft deleted
documents.

Closes #42996

Enable TRACE for testRecoverBrokenIndexMetadata (#43081)

Relates to #43034

CCR should not replicate private/internal settings (#43067)

With this change, CCR will not replicate internal or private settings to
follower indices.

Closes #41268

Defer reroute when nodes join (#42855)

Today the master eagerly reroutes the cluster as part of processing node joins.
However, it is not necessary to do this reroute straight away, and it is
sometimes preferable to defer it until later. For instance, when the master
wins its election it processes joins and performs a reroute, but it would be
better to defer the reroute until after the master has become properly
established.

This change defers this reroute into a separate task, and batches multiple such
tasks together.

Add SAS Token Authentication Support to Azure Repo Plugin (#42982)

* Added setting for SAS token
* Added support for the token in tests
* Relates #42117

Do not use runtime Java when starting BWC nodes (#43071)

When starting BWC nodes, it could be that runtime Java home is set. Yet,
runtime Java home can advance beyond what a BWC node might be compatible
with. For example, if runtime Java home is set to JDK 13 and we are
starting a 7.1.2 node, we do not have any guarantees that 7.1.2 is
compatible with JDK 13 (since we never did any work to make it so). This
will continue to be the case as JDK releases advance, but we still need
to test against BWC nodes. This commit stops applying runtime Java home
when starting a BWC node. Instead, we would use the bundled JDK.

Docs enhancement for quote_field_suffix. (#43093)

* Docs enhancement for quote_field_suffix.
Mentions the use of a fall-back field when specified field is missing.
Closes #40778

[ML] Fix race condition when closing time checker (#43098)

The tests for the ML TimeoutChecker rely on threads
not being interrupted after the TimeoutChecker is
closed.  This change ensures this by making the
close() and setTimeoutExceeded() methods synchronized
so that the code inside them cannot execute
simultaneously.

Fixes #43097

Fix unresponsive network simulation (#42579)

Unresponsive network simulation would throw away requests. However, then
we no longer have any guarantees that a transport action either succeeds
or fails, which could lead to hangs (example: unclosed IndexShard
permits).

Closes #42244

[ML][Data Frame] adding dest.index and id validations (#43053)

* [ML][Data Frame] adding dest.index and id validations

* adjusting message format

* Adjusting id validity pattern

* Update DataFrameStrings.java

Remove description from xpack feature sets (#43065)

The description field of xpack featuresets is optionally part of the
xpack info api, when using the verbose flag. However, this information
is unnecessary, as it is better left for documentation (and the existing
descriptions describe anything meaningful). This commit removes the
description field from feature sets.

[ML-DataFrame] stop task at completion of data frame function (#42955)

* stop data frame task after it finishes

* test auto stop

* adapt tests

* persist the state correctly and move stop into listener

* Calling `onStop` even if persistence fails, changing `stop` to rely on doSaveState

scheduleAtFixedRate would hang (#42993)

Though not in use in elasticsearch currently, it seems surprising that
ThreadPool.scheduler().scheduleAtFixedRate would hang. A recurring
scheduled task is never completed (except on failure) and we test for
exceptions using RunnableFuture.get(), which hangs for periodic tasks.
Fixed by checking that task is done before calling .get().

Remove the MachineLearningClient (#43108)

The transport client has been removed and this follow up change removes
the MachineLearningClient as we no longer need this wrapper class since
we have a user facing MachineLearningClient within the rest high level
client.

Adds a minimum interval to `auto_date_histogram`. (#42814)

Adds a minimum interval to `auto_date_histogram`. We do this by
restricting the roundings passed into to the aggregator.

Exclude internal SQL methods from Painless API Docs (#43116)

Use transport actions instead of guice for xpack usage (#43011)

The Xpack usage API internally gathers all xpack feature sets by using
guice to bind many implementations. The usage api then iterates over
these feature sets to gather usage information for each. However, since
each of these implementations are in different plugins, there is no
direct, non-guice way for them to register the implementations. This
commit removes the dependence on guice for the usage api by having the
core xpack jar contain an action instance for each plugin, and the
implementation of those actions are then registered within each xpack
feature plugin. The usage api then iterates over each of these actions
and calls them with the NodeClient.

Fix timing issue in CcrRetentionLeaseIT (#43054)

In these tests, we sleep for a small multiple of the renew interval,
then check that the retention leases are not changed. If a renewal
request takes longer than that interval because of GC or slow CI, then
the retention leases are not the same as before sleep. With this change,
we relax to assert that we eventually stop the renewable process.

Closes #39509

Metricbeat monitoring Elasticsearch: Reorder/remove steps (#42917)

[ML] Lock down extraction method when possible (#43104)

Docs change for exists query. (#43092)

Now emphasises the test is for indexed values.
Previous documentation only mentioned the state of the input JSON doc (null values) but this is only one of several reasons why an indexed value may not exist.

Closes #24256

Ensure Test Cluster in Azure Discovery Tests Closes (#43057)

* Use `internalCluster().close()` to force all nodes (and not just the datanodes) to shut down even if one fails to shut down in time
* Force closing httpServer to get cleaner logs if nodes still hang on shut down
* Relates #43048

Upgrade HPPC to version 0.8.1 (#43025)

Mute testDataFrameTransformCrud

Relates to #43139

SearchRequest#allowPartialSearchResults does not handle successful retries (#43095)

When set to false, allowPartialSearchResults option does not check if the
shard failures have been reseted to null. The atomic array, that is used to record
shard failures, is filled with a null value if a successful request on a shard happens
after a failure on a shard of another replica. In this case the atomic array is not empty
but contains only null values so this shouldn't be considered as a failure since all
shards are successful (some replicas have failed but the retries on another replica succeeded).
This change fixes this bug by checking the content of the atomic array and fails the request only
if allowPartialSearchResults is set to false and at least one shard failure is not null.

Closes #40743

Fix suggestions for empty indices (#42927)

Currently suggesters return null values on empty shards. Usually this gets replaced
by results from other non-epmty shards, but if the index is completely epmty (e.g. after
creation) the search responses "suggest" is also "null" and we don't render a corresponding
output in the REST response. This is an irritating edge case that requires special handling on
the user side (see #42473) and should be fixed.

This change makes sure every suggester type (completion, terms, phrase) returns at least an
empty skeleton suggestion output, even for empty shards. This way, even if we don't find
any suggestions anywhere, we still return and output the empty suggestion.

Closes #42473

[ML][Data Frame] change failure count reset logic (#43064)

[ML][Data Frame] add the src priv check for view_index_metadata (#43118)

[ML][Data Frame] cleaning up tests since tasks are cancelled onfinish (#43136)

* [ML][Data Frame] cleaning up usage test since tasks are cancelled onfinish

* Update DataFrameUsageIT.java

* Fixing additional test, waiting for task to complete

* removing unused import

* unmuting test

Move painless context api spec to test local (#43122)

The painless context api is internal and currently meant only for use in
generating docs. This commit moves the spec file for the api so that it
is only used by the test for this api, and not externally by any clients
building from the public rest spec.

Fix mustache search template test (#43120)

This commit fixes a race in the test for the new response format with
search templates. The test indexes a document and then executes a
search with the expectation of 0 results. In some instances, the index
will refresh prior to the search execution and 1 hit will be found
causing the test fail.

Closes #42664

Remove usage of FileSwitchDirectory  (#42937)

We are still using `FileSwitchDirectory` in the case a user configures file based pre-load of mmaps. This is trappy for multiple reasons if the both directories used by `FileSwitchDirectory` point to the same filesystem directory. One issue is LUCENE-8835 that cause issues like #37111 - unless LUCENE-8835 isn't fixed we should not use it in elasticsearch. Instead we use a similar trick as we use for HybridFS and subclass mmap directory directly.

IndexAnalyzers doesn't need to extend AbstractIndexComponent (#43149)

AIC doesn't add anything here, and it removes the need to pass index settings
to the constructor.

Stop SeedHostsResolver on shutdown (#42844)

Fixes an issue where tests would sometimes hang for 5 seconds when restarting a node. The reason
is that the SeedHostsResolver is blockingly waiting on a result for the full 5 seconds when the
corresponding threadpool is shut down.

[ML DataFrame] Reject Data Frame Ids containing upper case characters (#43145)

[DOCS] Adds dataframe authorization details (#43009)

Fix Operation Timestamps in Tests (#43155)

* For the issue in #43086 we were running into inactive shards because the random timestamps previously used would randomly make `org.elasticsearch.index.shard.IndexShard#checkIdle` see an incorrect+huge  inactive time
* Also fixed one other spot in tests that passed `ms` instead of `ns` for the same timestamp on an index op to correctly use relative `ns`
* Closes #43086

[ML][Data Frame] write a warning audit on bulk index failures (#43106)

* [ML][Data Frame] write a warning audit on bulk index failures

* adding failure message and moving to use volalitile

Also mmap terms index (`.tip`) files for hybridfs (#43150)

This change adds the terms index (`.tip`) to the list of extensions
that are memory-mapped by hybridfs. These files used to be accessed
only once to load the terms index on-heap but since #42838 they can
now be used to read the binary FST directly so it is benefical to
memory-map them instead of accessing them via NIO.

Introduce stability description to the REST API specification (#38413)

* introduce state to the REST API specification

* change state over to stability

* CCR is no GA updated to stable

* SQL is now GA so marked as stable

* Introduce `internal` as state for API's, marks stable in terms of lifetime but unstable in terms of guarantees on its output format since it exposes internal representations

* make setting a wrong stability value, or not setting it at all an error that causes the YAML test suite to fail

* update spec files to be explicit about their stability state

* Document the fact that stability needs to be defined

Otherwise the YAML test runner will fail (with a nice exception message)

* address check style violations

* update rest spec unit tests to include stability

* found one more test spec file not declaring stability, made sure stability appears after documentation everywhere

* cluster.state is stable, mark response in some way to denote its a key value format that can be changed during minors

* mark data frame API's as beta

* remove internal and private as states for an API

* removed the wrong enum values in the Stability Enum in the previous commit

Introduce unit tests for ValuesSourceType (#43174)

As the ValuesSourceType evolves, it is important to be
confident that new enum constants do not break
backwards-compatibility on the stream. Having dedicated
unit tests for this class will help be sure of that.

Enable krb5kdc-fixture, kerberos tests  mount urandom for kdc container (#41710)

Infra has fixed #10462 by installing `haveged` on CI workers.
This commit enables the disabled fixture and tests, and mounts
`/dev/urandom` for the container so there is enough
entropy required for kdc.
Note: hdfs-repository tests have been disabled, will raise a separate issue for it.

Closes #40624 Closes #40678

Convert PluginPropertiesExtension Groovy to Java (#39605)

Only load FST off heap if we are actually using mmaps for the term dictionary (#43158)

Given the significant performance impact that NIOFS has when term dicts are
loaded off-heap this change enforces FstLoadMode#AUTO that loads term dicts
off heap only if the underlying index input indicates a memory map.

Relates to #43150

Testclusters: graph (#43033)

Convert x-pack  graph to use testClusters

Document wildcard for network interfaces (#28839)

With this commit we mention how Elasticsearch behaves when
either `0` or `0.0.0.0` is used for `network.host`.

Report timing stats as part of the Job stats response (#42709)

Remote reindex failure parse fix (#42928)

A search request that partially fails with failures without an index
(index: null) in the failure would cause a parse error in reindex from
remote. This would hide the original exception, making it hard to debug
the root cause. This commit fixes this so that we can tolerate null
index entries in a search failure.

Fix roundUp parsing with composite patterns (#43080)

roundUp parsers were losing the composite pattern information when new
JavaDateFormatter was created from methods withLocale or withZone.

The roundUp parser should be preserved when calling these methods. This is the same approach in withLocale/Zone methods as in https://github.com/elastic/elasticsearch/blob/daa2ec8a605d385a65b9ab3e89d016b3fd0dffe2/server/src/main/java/org/elasticsearch/common/time/JavaDateFormatter.java

closes #42835

Converting DependencyLicensesTask and UpdateShasTask to java (#41921)

convert EmptyDirTask.groovy to .java (#34672)

Add 6.8 to the remote clusters compatibility table (#42389)

The table does not include 6.8 as it was written before we knew we were releasing it. This commit adds it.

Convert ILM tests to use testclusters (#43076)

Also improove the error message when bin scripts are not found

[ML] Restrict detection of epoch timestamps in find_file_structure (#43188)

Previously 10 digit numbers were considered candidates to be
timestamps recorded as seconds since the epoch and 13 digit
numbers as timestamps recorded as milliseconds since the epoch.

However, this meant that we could detect these formats for
numbers that would represent times far in the future.  As an
example ISBN numbers starting with 9 were detected as milliseconds
since the epoch since they had 13 digits.

This change tweaks the logic for detecting such timestamps to
require that they begin with 1 or 2.  This means that numbers
that would represent times beyond about 2065 are no longer
detected as epoch timestamps.  (We can add 3 to the definition
as we get closer to the cutoff date.)

Refactor Version class to make version bumps easier (#42668)

With this change we only have to add one line to add a new version.
The intent is to make it less error prone and easier to write a script
to automate the process.

[DOCS] Update node descriptions for default distribution (#42812)

Compile subset of build-tools classes with Java 8 target compatibility (#43177)

[DOCS] Fix link to ML node description

Disable x-pack:qa:kerberos-tests due to failures (#43210)

relates #40678

Add native code info to ML info api (#43172)

The machine learning feature of xpack has native binaries with a
different commit id than the rest of code. It is currently exposed in
the xpack info api. This commit adds that commit information to the ML
info api, so that it may be removed from the info api.

Enable node roles to be pluggable (#43175)

This commit introduces the possibility for a plugin to introduce
additional node roles.

Allow big integers and decimals to be mapped dynamically. (#42827)

This PR proposes to model big integers as longs (and big decimals as doubles)
in the context of dynamic mappings.

Previously, the dynamic mapping logic did not recognize big integers or
decimals, and would an error of the form "No matching token for number_type
[BIG_INTEGER]" when a dynamic big integer was encountered. It now accepts these
numeric types and interprets them as 'long' and 'double' respectively. This
allows `dynamic_templates` to accept and and remap them as another type such as
`keyword` or `scaled_float`.

Addresses #37846.

Adjust BWC version for discovery node roles

Now that pluggable node roles are backported to 7.x (versioned at
7.3.0), this commit adjusts the BWC version in master accordingly.

SQL: Fix wrong results when sorting on aggregate (#43154)

- Previously, when shorting on an aggregate function the bucket
processing ended early when the explicit (LIMIT XXX) or the impliciti
limit of 512 was reached. As a consequence, only a set of grouping
buckets was processed and the results returned didn't reflect the global
ordering.

- Previously, the priority queue shorting method had an inverse
comparison check and the final response from the priority queue was also
returned in the inversed order because of the calls to the `pop()`
method.

Fixes: #42851

Fix JDK-version dependent exception message parsing

This commit fixes some JDK-version dependent exception message checking
in the discovery node role tests.

[ML][Data Frame] have sum map to a double to prevent overflows (#43213)

Use SecureString for password length validation (#42884)

This replaces the use of char[] in the password length validation
code, with the use of SecureString

Although the use of char[] is not in itself problematic, using a
SecureString encourages callers to think about the lifetime of the
password object and to clear it after use.

Fix versions for reindex max_docs tests (#42951)

Versions were assuming this went into 7.2, but this ended up going into
7.3, updated test versions to reflect that.

Related to #41894

[Docs] Name remote cluster "two" (#42976)

All the text and the search example refer to the remote cluster as
"two" rather than "cluster_two". This commit changes the definition
that is stored through the settings API to match the rest of the
examples.

Remove explicily enabled build fixture task

Correctly configure testclsuters for fips (#43186)

Configuration was not being triggered for all projects with the way it
was previusly set up.
With this change we do it for each project directly.

Convert dataframes to use testclusters (#43032)

SimpleQ.S.B and QueryStringQ.S.B tests should avoid `now` in query (#43199)

Currently the randomization of the q.b. in these tests can create query strings
that can cause caching to be disabled for this query if we query all fields and
there is a date field present. This is pretty much an anomaly that we shouldn't
generally test for in the "testToQuery" tests where cache policies are checked.

This change makes sure we don't create offending query strings so the cache
checks never hit these cases and adds a special test method to check this edge
case.

Closes #43112

SQL: Increase hard limit for sorting on aggregates (#43220)

To be consistent with the `search.max_buckets` default setting,
set the hard limit of the PriorityQueue used for in memory sorting,
when sorting on an aggregate function, to 10000.

Fixes: #43168

Upgrade GCS Repository Dependencies (#43142)

* Upgrade to latest GCS SDK and transitive dependencies (I chose the later version here on conflict)
* Remove now unnecessary hack for custom endpoints (the linked bugs were both resolved in the SDK)

[DOCS] Adds size and from parameters to data frame APIs (#43212)

Move reindex request parsing into request (#43107)

Currently the fromXContent logic for reindex requests is implemented in
the rest action. This is inconsistent with other requests where the
logic is implemented in the request. Additionally, it requires access to
the rest action in order to parse the request. This commit moves the
logic and tests into the ReindexRequest.

Account soft deletes in committed segments (#43126)

This change fixes the delete count issue in segment stats where we don't
account soft-deleted documents from committed segments.

Relates #43103

BWC tests - move vector distance functions to 7.3

[ML] Close sample stream in post_data endpoint (#43235)

A static code analysis revealed that we are not closing
the input stream in the post_data endpoint.  This
actually makes no difference in practice, as the
particular InputStream implementation in this case is
org.elasticsearch.common.bytes.BytesReferenceStreamInput
and its close() method is a no-op. However, it is good
practice to close the stream anyway.

Ensure threads running before closing node (#43240)

There are a few tests within NodeTests that submit items to the
threadpool and then close the node. The tests are designed to check
how running tasks are affected during node close. These tests can cause
CI failures since the submitted tasks may not be running when the node
is closed and then execute after the thread context is closed, which
triggers an unexpected exception. This change ensures the threads are
running so we avoid the unexpected exception and can test these cases.

The test of task submittal while a node is closing is also important so
an additional but muted test has been added that tests the case where a
task may be getting submitted while the node is closing and ensuring we
do not trigger anything unexpected in these cases.

Relates #42774
Relates #42577

Report exponential_avg_bucket_processing_time which gives more weight to recent buckets (#43189)

TestClusters: convert kerberos-tests (#43232)

Looks like cluster formation tasks no longer plays nice wit
test.fixtures so we just convert this to use testclusters.

[DOCS] Simplifies wording. (#43226)

This PR simplifies the wording of the TOC and eventually makes it shorter.

[ML] Fix problem with lost shards in distributed failure test (#43153)

We were stopping a node in the cluster at a time when
the replica shards of the .ml-state index might not
have been created.  This change moves the wait for
green status to a point where the .ml-state index
exists.

Fixes #40546
Fixes #41742

Forward port of #43111

Make DiscoveryNodeRole into a value object (#43257)

Adds `equals()` and `hashcode()` methods to `DiscoveryNodeRole` to compare
these objects' values for equality, and adds a field to allow us to distinguish
unknown roles from known ones with the same name and abbreviation, for clearer
test failures.

Relates #43175

Allow cluster access during node restart (#42946)

This commit modifies InternalTestCluster to allow using client() and
other operations inside a RestartCallback (onStoppedNode typically).
Restarting nodes are now removed from the map and thus all
methods now return the state as if the restarting node does not exist.

This avoids various exceptions stemming from accessing the stopped
node(s).

[docs] Add missing comma (#43073)

Adds a missing comma to a code example

Make ILM force merging best effort (#43246)

It's possible for force merges kicked off by ILM to silently stop (due
to a node relocating for example). In which case, the segment count may
not reach what the user configured. In the subsequent `SegmentCountStep`
waiting for the expected segment count may wait indefinitely. Because of
this, this commit makes force merges "best effort" and then changes the
`SegmentCountStep` to simply report (at INFO level) if the merge was not
successful.

Relates to #42824
Resolves #43245

Remove unneeded backticks (#43256)

Relocation targets are assigned shards too (#43276)

Adds relocation targets to the output of
`IndexShardRoutingTable#assignedShards`.

Add Painless Docs for Datetime Inputs (#43128)

This changes add documentation for accessing datetimes in Painless scripts
from the three most common inputs of params, _source, and doc.

[ML] BWC tests for job_stats.timing_stats field (#43267)

[ML][Data Frame] only complete task after state persistence (#43230)

* [ML][Data Frame] only complete task after state persistence

There is a race condition where the task could be completed, but there
is still a pending document write. This change moves
the task cancellation into the actionlistener of the state persistence.

intermediate commit

intermediate commit

* removing unused import

* removing unused const

* refreshing internal index after waiting for task to complete

* adjusting test data generation

Rebuild version map when opening internal engine (#43202)

With this change, we will rebuild the live version map and local
checkpoint using documents (including soft-deleted) from the safe commit
when opening an internal engine. This allows us to safely prune away _id
of all soft-deleted documents as the version map is always in-sync with
the Lucene index.

Relates #40741
Supersedes #42979

[DOCS] Add introduction to Elasticsearch. (#43075)

* [DOCS] Add introduction to Elasticsearch.

* [DOCS] Incorporated review comments.

* [DOCS] Minor edits to add an abbreviated title and cross refs.

* [DOCS] Added sizing tips & link to quantatative sizing video.

Add elasticsearch distribution plugin (#43247)

Several types of distributions are built and tested in elasticsearch,
ranging from the current version, to building or downloading snapshot or
released versions. Currently tests relying on these have to contain
logic deciding where and how to pull down these distributions.

This commit adds an distributiond download plugin for each project to
manage which versions and variants the project needs. It abstracts away
all need for knowing where a particular version comes from, like a local
distribution or bwc project, or pulling from the elastic download
service. This will be used in a followup PR by the testclusters and
vagrant tests.

[DOCS] Fix typo: extraneous {es}

[DOCS] Sewing SME says it should be "size 70" needle.

Testclusters: convert ml qa tests (#43229)

* Testclusters: convert ml qa tests

This PR converts the ML tests to use testclusters.

Remove Azure Container Exists Check (#43288)

* This check is redundant, if the container doesn't exist subsequent operations will fail anyway. Since we are not running this exists check during verification I don't think there's much point to having it in snapshot initialization.
* This PR is mainly motivated by the fact that this forces more permissions to be available in shared environments

[ML] Speed up persistent task rechecks in ML failover tests (#43291)

The ML failover tests sometimes need to wait for jobs to be
assigned to new nodes following a node failure.  They wait
10 seconds for this to happen.  However, if the node that
failed was the master node and a new master was elected then
this 10 seconds might not be long enough as a refresh of the
memory stats will delay job assignment.  Once the memory
refresh completes the persistent task will be assigned when
the next cluster state update occurs or after the periodic
recheck interval, which defaults to 30 seconds.  Rather than
increase the length of the wait for assignment to 31 seconds,
this change decreases the periodic recheck interval to 1
second.

Fixes #43289

TestClusters: Convert the security plugin (#43242)

* TestClusters: Convert the security plugin

This PR moves security tests to use TestClusters.
The TLS test required support in testclusters itself, so the correct
wait condition is configgured based on the cluster settings.

* PR review

Allow election of nodes outside voting config (#43243)

Today we suppress election attempts on master-eligible nodes that are not in
the voting configuration. In fact this restriction is not necessary: any
master-eligible node can safely become master as long as it has a fresh enough
cluster state and can gather a quorum of votes. Moreover, this restriction is
sometimes undesirable: there may be a reason why we do not want any of the
nodes in the voting configuration to become master.

The reason for this restriction is as follows. If you want to shut the master
down then you might first exclude it from the voting configuration. When this
exclusion succeeds you might reasonably expect that a new master has been
elected, since the voting config exclusion is almost always a step towards
shutting the node down. If we allow nodes outside the voting configuration to
be the master then the excluded node will continue to be master, which is
confusing.

This commit adjusts the logic to allow master-eligible nodes to attempt an
election even if they are not in the voting configuration. If such a master is
successfully elected then it adds itself to the voting configuration. This
commit also adjusts the logic that causes master nodes to abdicate when they
are excluded from the voting configuration, to avoid the confusion described
above.

Relates #37712, #37802.

Local node is discovered when cluster fails (#43316)

Today the `ClusterFormationFailureHelper` does not include the local node in
the list of nodes it claims to have discovered. This means that it sometimes
reports that it has not discovered a quorum when in fact it has. This commit
adds the local node to the set of discovered nodes.

Recursive Delete on BlobContainer (#43281)

This is a prerequisite of #42189:

* Add directory delete method to blob container specific to each implementation:
  * Some notes on the implementations:
       * AWS + GCS: We can simply exploit the fact that both AWS and GCS return blobs lexicographically ordered which allows us to simply delete in the same order that we receive the blobs from the listing request. For AWS this simply required listing without the delimiter setting (so we get a deep listing) and for GCS the same behavior is achieved by not using the directory mode on the listing invocation. The nice thing about this is, that even for very large numbers of blobs the memory requirements are now capped nicely since we go page by page when deleting.
       * For Azure I extended the parallelization to the listing calls as well and made it work recursively. I verified that this works with thread count `1` since we only block once in the initial thread and then fan out to a "graph" of child listeners that never block.
       * HDFS and FS are trivial since we have directory delete methods available for them
* Enhances third party tests to ensure the new functionality works (I manually ran them for all cloud providers)

Create a separate job solely for the purpose of verifying its timing stats in mixed and upgraded cluster. (#43315)

Move dense_vector and sparse_vector to module (#43280)

Fixes formatting of CCS compatibility table (#43231)

Geo: Add coerce support to libs/geo WKT parser (#43273)

Adds support for coercing not closed polygons and ignoring Z value
to libs/geo WKT parser.

Closes #43173

Deprecate native code info in xpack info api (#43297)

The xpack info api currently returns native code info within each
feature. This commit deprecates retrieving that info, which is now
available directly in the ML info api.

specifies which index to search in docs for various queries (#43307)

the geo-bounding-box and phrase-suggest docs were susceptible to
failing due to other indices in the cluster. This change restricts
the queries to the index that is set up for the test.

relates to #43271.

Remove indices exists action (#43164)

The indices exists action was kept around only to not break transport
client, as RestGetIndicesAction has its own logic that ties into the
unified HEAD requests handling.

Now that tranport client is removed, we can remove also
`TransportIndicesExistsAction` as well as its corresponding request,
request builder, response, and action.

Properly format reproduction lines for test methods that contain periods (#43255)

Add AwaitsFix for #43342

Some Cleanup in BlobStoreRepository (#43323)

* Some Cleanup in BlobStoreRepository

* Extracted from #42833:
  * Dry up index and shard path handling
  * Shorten XContent handling

Fix sporadic failures in QueryStringQueryTests#testToQueryFuzzyQueryAutoFuziness (#43322)

This commit ensures that the test does not use reserved keyword (OR, AND, NOT)
when generating the random query strings.

Closes #43318

Fix randomization in testPerformActionAttrsRequestFails (#43304)

The randomization in this test would occasionally generate duplicate
node attribute keys, causing spurious test failures. This commit adjusts
the randomization to not generate duplicate keys and cleans up the data
structure used to hold the generated keys.

Add kerberos grant_type to get token in exchange for Kerberos ticket (#42847)

Kibana wants to create access_token/refresh_token pair using Token
management APIs in exchange for kerberos tickets. `client_credentials`
grant_type requires every user to have `cluster:admin/xpack/security/token/create`
cluster privilege.

This commit introduces `_kerberos` grant_type for generating `access_token`
and `refresh_token` in exchange for a valid base64 encoded kerberos ticket.
In addition, `kibana_user` role now has cluster privilege to create tokens.
This allows Kibana to create access_token/refresh_token pair in exchange for
kerberos tickets.

Note:
The lifetime from the kerberos ticket is not used in ES and so even after it expires
the access_token/refresh_token pair will be valid. Care must be taken to invalidate
such tokens using token management APIs if required.

Closes #41943

Remove trace logging from ML datafeeds in tests

This was added to investigate a test failure over two years ago, yet
left behind. Since the test failure has been addressed since then, this
commit removes the trace logging.

Remove trace logging for ML native multi-node tests

This trace logging looks like it was copy/pasted from another test,
where the logging in that test was only added to investigate a test
failure. This commit removes the trace logging.

Remove token service trace logging in tests

This commit removes some trace logging for the token service in the
rolling upgrade tests. If there is an active investigation here, it
would be best to annotate this line with a comment in the source
indicating such. From my digging, it does not appear there is an active
investigation that relies on this logging, so we remove it.

Enable trace log for testOpenCloseApiWildcards

Relates #39578

Refactor aggregation base classes to remove doEquals() and doHashCode() (#43214)

A number of the aggregation base classes have an abstract doEquals() and doHashCode() (e.g. InternalAggregation.java, AbstractPipelineAggregationBuilder.java).

Theoretically this is so the sub-classes can add to the equals/hashCode and don't need to worry about calling super.equals(). In practice, it's mostly just confusing/inconsistent. And if there are more than two levels, we end up with situations like InternalMappedSignificantTerms which has to call super.doEquals() which defeats the point of having these overridable methods.

This PR removes the do versions and just use equals/hashCode ensuring the super when necessary.

This PR is part of #41713 refactoring meta

* Refactored all subclasses of InternalAggregation to remove doEquals() and doHashCode()
* Refactored all subclasses of AbstractPipelineAggregationBuilder to remove doEquals() and doHashCode()
* Refactored all subclasses of AbstractAggregationBuilder and CompositeValuesSourceBuilder to remove doEquals() and doHashCode()

SearchPhaseContext to not extend ActionListener (#43269)

The fact that SearchPhaseContext extends ActionListener makes it hard
to reason about when the original listener is notified and to trace
those calls. Also, the corresponding onFailure and onResponse were
only needed in two places, one each, where they can be replaced by a
more intuitive call, like sendSearchResponse for onResponse.

Unmute timing_stats BWC test (#43358)

Replace Streamable w/ Writeable in SingleShardRequest and subclasses (#43222)

This commit replaces usages of Streamable with Writeable for the
SingleShardRequest / TransportSingleShardAction classes and subclasses of
these classes.

Note that where possible response fields were made final and default
constructors were removed.

Relates to #34389

change version to 7.3, since backport work has been merged to 7.x

Rename AutoDetectResultProcessor* to AutodetectResultProcessor* for consistency with other classes where the spelling is "Autodetect" (#43359)

Fix NetworkUtilsTests (#43295)

* Follow up to #42109:
   * Adjust test to only check that interface lookup by name works not actually lookup IPs which is brittle since virtual interfaces can be destroyed/created by Docker while the tests are running

Co-authored-by:  Jason Tedor <jason@tedor.me>

Get snapshots support for multiple repositories (#42090)

This commit adds multiple repositories support to get snapshots
request.
If some repository throws an exception this method does not fail fast
instead, it returns results for all repositories.
This PR is opened in favour of #41799, because we decided to change
the response format in a non-BwC manner. It makes sense to read a
discussion of the aforementioned PR.
This is the continuation of work done here #15151.

Do not use soft-deletes to resolve indexing strategy (#43336)

This PR reverts #35230.

Previously, we reply on soft-deletes to fill the mismatch between the
version map and the Lucene index. This is no longer needed after #43202
where we rebuild the version map when opening an engine. Moreover,
PrunePostingsMergePolicy can prune _id of soft-deleted documents out of
order; thus the lookup result including soft-deletes sometimes does not
return the latest version (although it's okay as we only use a valid
result in an engine).

With this change, we use only live documents in Lucene to resolve the
indexing strategy. This is perfectly safe since we keep all deleted
documents after the local checkpoint in the version map.

Closes #42979

Remove native code info from xpack info api (#43125)

The native code info is a portion of the xpack info api that emits
arbitrary info about feature plugins that are backed by native code.
This is currently only used by machine learning. Additionally, it is the
only non enabled/available information reported by the info api.

This commit moves the native code info reporting to the usage api for
machine learning. The commit info reported is only used for debugging
purposes, and not captured by the current uses of the info api
(monitoring and telemetry) since it requires passing the verbose flag.

Long term, this information would be better suited to node info. Until
node info is extendable by plugins, this new location will allow
cleaning up the xpack info api to be implemented in a generic way for
all xpack features.

Return 0 for negative "free" and "total" memory reported by the OS (#42725)

* Return 0 for negative "free" and "total" memory reported by the OS

We've had a situation where the MX bean reported negative values for the
free memory of the OS, in those rare cases we want to return a value of
0 rather than blowing up later down the pipeline.

In the event that there is a serialization or creation error with regard
to memory use, this adds asserts so the failure will occur as soon as
possible and give us a better location for investigation.

Resolves #42157

* Fix test passing in invalid memory value

* Fix another test passing in invalid memory value

* Also change mem check in MachineLearning.machineMemoryFromStats

* Add background documentation for why we prevent negative return values

* Clarify comment a bit more

Fix GeoIP custom database directory in docs (#43383)

These docs were misleading for package installations of
Elasticsearch. Instead, we should refer to $ES_CONFIG/ingest-geoip as
the path to place the custom database files. For non-package
installations, this is the same as $ES_HOME/config, but for package
installations this is not the case as the config directory for package
installations is /etc/elasticsearch, and is not relative to
$ES_HOME. This commit corrects the docs.

[ML][Data Frame] adds new pipeline field to dest config (#43124)

* [ML][Data Frame] adds new pipeline field to dest config

* Adding pipeline support to _preview

* removing unused import

* moving towards extracting _source from pipeline simulation

* fixing permission requirement, adding _index entry to doc

Muting DataFrameTransformIT.testGetStats

See #43324

[ML][Data Frame] make response.count be total count of hits (#43241)

* [ML][Data Frame] make response.count be total count of hits

* addressing line length check

* changing response count for filters

* adjusting serialization, variable name, and total count logic

* making count mandatory for creation

Allocate memory lazily in BestBucketsDeferringCollector (#43339)

While investigating memory consumption of deeply nested aggregations for #43091
the memory used to keep track of the doc ids and buckets in the BestBucketsDeferringCollector
showed up as one of the main contributor. In my tests half of the memory held in the
 BestBucketsDeferringCollector is associated to segments that don't have matching docs
 in the selected buckets. This is expected on fields that have a big cardinality since each
 bucket can appear in very few segments. By allocating the builders lazily this change
 reduces the memory consumption by a factor 2 (from 1GB to 512MB), hence reducing the
impact on gcs for these volatile allocations. This commit also switches the PackedLongValues.Builder
with a RoaringDocIdSet in order to handle very sparse buckets more efficiently.

I ran all my tests on the `geoname` rally track with the following query:

````
{
    "size": 0,
    "aggs": {
        "country_population": {
            "terms": {
                "size": 100,
                "field": "country_code.raw"
            },
            "aggs": {
                "admin1_code": {
                    "terms": {
                        "size": 100,
                        "field": "admin1_code.raw"
                    },
                    "aggs": {
                        "admin2_code": {
                            "terms": {
                                "size": 100,
                                "field": "admin2_code.raw"
                            },
                            "aggs": {
                                "sum_population": {
                                    "sum": {
                                        "field": "population"
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}
````

Narrow period of Shrink action in which ILM prevents stopping (#43254)

* Narrow period of Shrink action in which ILM prevents stopping

Prior to this change, we would prevent stopping of ILM if the index was
anywhere in the shrink action. This commit changes
`IndexLifecycleService` to allow stopping when in any of the innocuous
steps during shrink. This changes ILM only to prevent stopping if
absolutely necessary.

Resolves #43253

* Rename variable for ignore actions -> ignore steps

* Fix comment

* Factor test out to test *all* stoppable steps

Prevent NullPointerException in TransportRolloverAction (#43353)

It's possible for the passed in `IndexMetaData` to be null (for
instance, cluster state passed in does not have the index in its
metadata) which in turn can cause a `NullPointerException` when
evaluating the conditions for an index. This commit adds null protection
and unit tests for this case.

Resolves #43296

Reduce the number of docvalues iterator created in the global ordinals fielddata (#43091)

Today the fielddata for global ordinals re-creates docvalues readers of each segment
when building the iterator of a single segment. This is required because the lookup of
global ordinals needs to access the docvalues's TermsEnum of each segment to retrieve
the original terms. This also means that we need to create NxN (where N is the number of segment in the index) docvalues iterators
each time we want to collect global ordinal values. This wasn't an issue in previous versions since docvalues readers are stateless
before 6.0 so they are reused on each segment but now that docvalues are iterators we need to create a new instance each time
we want to access the values. In order to avoid creating too many iterators this change splits
the global ordinals fielddata in two classes, one that is used to cache a single instance per directory reader and one
that is created from the cached instance that can be used by a single consumer. The latter creates the TermsEnum of each segment
once and reuse them to create the segment's iterator. This prevents the creation of all TermsEnums each time we want to access
the value of a single segment, hence reducing the number of docvalues iterator to create to Nx2 (one iterator and one lookup per segment).

Remove stale test logging annotations (#43403)

This commit removes some very old test logging annotations that appeared
to be added to investigate test failures that are long since closed. If
these are needed, they can be added back on a case-by-case basis with a
comment associating them to a test failure.

Testclusters: conver remaining x-pack  (#43335)

Convert x-pack tests

Advance checkpoints only after persisting ops (#43205)

Local and global checkpoints currently do not correctly reflect what's persisted to disk. The issue is
that the local checkpoint is adapted as soon as an operation is processed (but not fsynced yet). This
leaves room for the history below the global checkpoint to still change in case of a crash. As we rely
on global checkpoints for CCR as well as operation-based recoveries, this has the risk of shard
copies / follower clusters going out of sync.

This commit required changing some core classes in the system:

- The LocalCheckpointTracker keeps track now not only of the information whether an operation has
been processed, but also whether that operation has been persisted to disk.
- TranslogWriter now keeps track of the sequence numbers that have not been fsynced yet. Once
they are fsynced, TranslogWriter notifies LocalCheckpointTracker of this.
- ReplicationTracker now keeps track of the persisted local and persisted global checkpoints of all
shard copies when in primary mode. The computed global checkpoint (which represents the
minimum of all persisted local checkpoints of all in-sync shard copies), which was previously stored
in the checkpoint entry for the local shard copy, has been moved to an extra field.
- The periodic global checkpoint sync now also takes async durability into account, where the local
checkpoints on shards only advance when the translog is asynchronously fsynced. This means that
the previous condition to detect inactivity (max sequence number is equal to global checkpoint) is
not sufficient anymore.
- The new index closing API does not work when combined with async durability. The shard
verification step is now requires an additional pre-flight step to fsync the translog, so that the main
verify shard step has the most up-to-date global checkpoint at disposition.

Fix NPE in case of subsequent scrolled requests for a CSV/TSV formatted response (#43365)

Remove mentions of "fields with the same name in the same index" (#43077)

Together with types removal, any mention of "fields with the same name in the same index" doesn't make sense anymore.

Reconnect remote cluster when seeds are changed (#43379)

The RemoteClusterService should close the current
RemoteClusterConnection and should build it again if
the seeds are changed, similarly to what is done when
the ping interval or the compression settings are changed.

Closes #37799

[Docs] Remove boost parameter from intervals-query example (#43331)

The boost factor doesn't seem to be needed and can be removed.

Adapt BWC conditions after backport of #43205

[ML] Unmute NetworkDisruptionIT.testJobRelocation (#43268)

This reverts commit 27346a076b5f3ca9a81dabbf1796182d3ffc7e00.

Reduce shard started failure logging (#43330)

If the master is stepping or shutting down, the error-level logging can cause quite a bit of noise.

Fix testGlobalCheckpointSync

The test needed adaption after #43205, as the ReplicationTracker now distinguishes between the
knowledge of the persisted global checkpoint and the computed global checkpoint on the primary

Follow-up to #43205

Docs: Add description of the coerce parameter in geo_shape mapper (#43340)

Explains the effect of the coerce parameter on the geo_shape field.

Relates #35059

Fail connection attempts earlier in tests (#43320)

Today the `DisruptibleMockTransport` always allows a connection to a node to be
established, and then fails requests sent to that node such as the subsequent
handshake. Since #42342, we log handshake failures on an open connection as a
warning, and this makes the test logs rather noisy. This change fails the
connection attempt first, avoiding these unrealistic warnings.

Mute test NetworkDisruptionIT.testJobRelocation

Remove Confusing Comment (#43400)

[DOCS] Describe setup for monitoring logs (#42655)

Mute put-transform docs test

Relates to #43271

[DOCS] Add brew install instructions. Closes #42914 (#42915)

Better support for unmapped fields in AggregatorTestCase (#43405)

AggregatorTestCase will NPE if only a single, null MappedFieldType
is provided (which is required to simulate an unmapped field).  While
it's possible to test unmapped fields by supplying other, non-related
field types... that's clunky and unnecessary.  AggregatorTestCase
just needs to filter out null field types when setting up.

Adapt local checkpoint assertion

With async durability, it does not hold true anymore after #43205. This is fine.

Remove old Painless doc generator (#43404)

This removes the previous Painless API Doc Generator prior to contexts
existing. It has been replaced with the new doc generator that uses the
documentation rest API.

Increase timeout for assertSeqNos

Helps with tests that do async translog syncing

Added parsing of erroneous field value (#42321)

[DOCS] Fixed path to install directory. (#43443)

Log Blocked IO Thread State (#43424)

* Let's log the state of the thread to find out if it's dead-locked or just stuck after being suspended
* Relates #43392

Add painless method getByPath, get value from nested collections with dotted path (#43170)

Given a nested structure composed of Lists and Maps, getByPath will return the value
keyed by path.  getByPath is a method on Lists and Maps.

The path is string Map keys and integer List indices separated by dot. An optional third
argument returns a default value if the path lookup fails due to a missing value.

Eg.
['key0': ['a', 'b'], 'key1': ['c', 'd']].getByPath('key1') = ['c', 'd']
['key0': ['a', 'b'], 'key1': ['c', 'd']].getByPath('key1.0') = 'c'
['key0': ['a', 'b'], 'key1': ['c', 'd']].getByPath('key2', 'x') = 'x'
[['key0': 'value0'], ['key1': 'value1']].getByPath('1.key1') = 'value1'

Throws IllegalArgumentException if an item cannot be found and a default is not given.
Throws NumberFormatException if a path element operating on a List is not an integer.

Fixes #42769

[DOCS] Added blurb & link to Elasticsearch Service. (#43452)

Fix round up of date range without rounding (#43303)

Today when searching for an exclusive range the java date math parser rounds up the value
with the granularity of the operation. So when searching for values that are greater than
"now-2M" the parser rounds up the operation to "now-1M". This behavior was introduced when
we migrated to java date but it looks like a bug since the joda math parser rounds up values
but only when a rounding is used. So "now/M" is rounded to "now-1ms" (minus 1ms to get the largest inclusive value)
in the joda parser if the result should be exclusive but no rounding is applied if the input
is a simple operation like "now-1M". This change restores the joda behavior in order to have
a consistent parsing in all versions.

Closes #43277

[DOCS] Updated the URL for starting in the cloud.

Clarify unsupported secure settings behavior (#43454)

This commit tweaks the docs for secure settings to ensure the user is
aware adding non secure settings to the keystore will result in
elasticsearch not starting.

fixes #43328

Co-Authored-By: James Rodewig <james.rodewig@elastic.co>

Reduce TestLogging usage in DisruptionIT tests (#43411)

Removes `@TestLogging` annotations in `*DisruptionIT` tests, so that the only
tests with annotations are those with open issues. Also adds links to the open
issues in the remaining cases.

Relates #43403

Make Recovery API support `detailed` params (#29076)

Properly forwards the `detailed` parameter to show the recovery stats details.

Closes #28910

Change BWC condition after backporting #29076

SecurityIndexSearcherWrapper doesn't always carry over caches and similarity (#43436)

If DocumentLevelSecurity is enabled SecurityIndexSearcherWrapper doesn't carry
over the cache, cache policy and similarity from the incoming searcher.

Remove types exist action (#43344)

The types exists action was kept around only to not break transport
client, as RestGetMappingAction has its own logic that ties into the
unified HEAD requests handling.

Now that tranport client is removed, we can remove also
TransportTypesExistsAction as well as its corresponding request,
request builder, response, and action.

Resolve NetworkDisruptionIT (#43441)

[ML][Data Frame] Add version and create_time to transform config (#43384)

* [ML][Data Frame] Add version and create_time to transform config

* s/transform_version/version s/Date/Instant

* fixing getter/setter for version

Fix DefaultShardOperationFailedException subclass xcontent serialization (#43435)

The current toXContent implementation can fail when the superclasses toXContent
is called (see #43423). This change makes sure that
DefaultShardOperationFailedException#toXContent is final and implementations
need to add special fields in #innerToXContent. All implementations should write
to self-contained xContent objects. Also adding a test for xContent deserialization
to CloseIndexResponseTests.

Closes #43423

Example of how to set slow logs dynamically per-index (#42384)

* Example of how to set slow logs dynamically per-index

* Make _settings API example more explicit

Co-Authored-By: James Rodewig <james.rodewig@elastic.co>

* Add TEST directive to fix CI

Co-Authored-By: James Rodewig <james.rodewig@elastic.co>

Recursively Delete Unreferenced Index Directories (#42189)

* Use ability to list child "folders" in the blob store to implement recursive delete on all stale index folders when cleaning up instead of using the diff between two `RepositoryData` instances to cover aborted deletes
* Runs after ever delete operation
* Relates  #13159 (fixing most of this issues caused by unreferenced indices, leaving some meta files to be cleaned up only)

Simplify BlobStoreRepository (Flatten Nested Classes) (#42833)

* In the current codebase it is hardly obvious what code operates on a shard and is run by a datanode what code operates on the global metadata and is run on master
   * Fixed by adjusting the method names accordingly
* The nested context classes don't add much if any value, they simply spread out the parameters that go into a shard snapshot create or delete all over the place since their
constructors can be inlined in all spots
   * Fixed by flattening the nested classes into BlobStoreRepository
* Also:
  * Inlined the other single use inner classes

Assert ServerSocketChannel is not Blocking (#43479)

* Assert ServerSocketChannel is not Blocking

* Relates #43387 which appears to run into blocking accept calls

[DOCS] Rewrite term-level queries overview (#43337)

[DOCS] Rewrite `constant_score` query (#43374)

[ML][Data Frame] fixing some data frame hlrc tests (#43446)

* [ML][Data Frame] fixing some data frame hlrc tests

* adding task|indexer state checks back

Remove aliases exist action (#43430)

The aliases exists action was kept around only to not break transport
client, as RestGetAliasesAction has its own logic that ties into the
unified HEAD requests handling.

Now that transport client is removed, we can remove also
TransportAliasesExistAction as well as its corresponding request,
request builder, response, and action.

Mute failing test

Mute LicenseDocumentationIT#testGetLicense (#43505)

Account for node versions during allocation in ILM Shrink (#43300)

This commit ensures that ILM's Shrink action will take node versions into
account when choosing which node to allocate to when shrinking an
index. Prior to this change, ILM could pick a node with a lower version
than some shards are already allocated to, which causes the new
allocation to fail as shards can't be relocated onto a node with a lower
version than they are already on.

As part of this, when making the decision about which node to allocate
to prior to Shrink, all shards in the index are considered, rather than
choosing a random shard to consider.

Further, the unit tests for the logic that chooses a node to allocate
shards to pre-shrink has been improved to validate the behavior in more
realistic and varied initial conditions.

Don't Consistency Check Broken Repository in Test (#43499)

* Missed this one in #42189 and it randomly runs into a situation where the broken mock repo is broken such that we can't get to a consistent end state via a delete
* Closes #43498

UserRoleMapper.UserData non-null groups and metadata (#41774)

The `UserRoleMapper.UserData` is constructed by each realm and is used to
"match" role mapping expressions that eventually supply the role names of the principal.

`null` values as group names or metadata values were not properly supported.
Most of the time they were filtered out, but under some circumstances they might
cause NPEs (depending of how the OIDC and SAML's underlying libraries parse
purposefully crafted assertions - these libraries don't have internal null checks the
same way that the unboundid library does).

This PR enforces non-null collection values (lists and maps) earliest in the process,
on first building the collection, and utilizes java 9 unmodifiable collections API, with
the goal of eliminating redundant null filtering and repeated wrapping in
`Colllections.unmodifiable*`.

Cleanup legacy logic in CombinedDeletionPolicy (#43484)

This change removes the support for pre-v6 index commits which
do not have sequence numbers.

Replace Streamable w/ Writable in AcknowledgedResponse and subclasses (#43414)

This commit replaces usages of Streamable with Writeable for the
AcknowledgedResponse and its subclasses, plus associated actions.

Note that where possible response fields were made final and default
constructors were removed.

This is a large PR, but the change is mostly mechanical.

Relates to #34389

Fix the bundled jdk flag to be passed through windows startup (#43502)

This commit fixes a typo in elasticsearch.bat that prevented the windows
distribution from knowing whether it is using the bundled jdk.

Use transport actions instead of guice for xpack info (#43449)

The Xpack info API internally gathers all xpack feature sets by using
guice to bind many implementations. The info api then iterates over
these feature sets to gather usage information for each. However, since
each of these implementations are in different plugins, there is no
direct, non-guice way for them to register the implementations. This
commit removes the dependence on guice for the info api by having the
core xpack jar contain an action instance for each plugin, and the
implementation of those actions are then registered within each xpack
feature plugin. The info api then iterates over each of these actions
and calls them with the NodeClient.

Clarify storage location of ML Snapshots (#43437)

The existing language was misleading about the model snapshots and where they are located. Saying "to disk" sounds like files external to Elasticsearch IMO. It raises the obvious question, where on disk? which node? Is it in the Elasticsearch snapshot repo? The model snapshots are held in an internal index.

Improove error on waiting for cluster to come up (#43416)

Always include all messages from exceptions to make the reason for
failure more visible.

Test clusters: convert x-pack qa tests (#43283)

Assert that NOOPs must succeed (#43483)

We currently assert that adding deletion tombstones to Lucene must always succeed if it's not a
tragic exception, and the same should also hold true for NOOP tombstones. We rely on this
assumption, as without this, we have the risk of creating gaps in the history, which will break
operation-based recoveries and CCR.

Cleanup IndicesService#CacheCleaner Scheduling (#42060)

* Follow up to #42016

Add additional logging for #43034

It's unclear why sometimes the shard is not flushed on closing

Do not hang on unsupported HTTP methods (#43362)

Unsupported HTTP methods are detected during requests dispatching
which generates an appropriate error response. Sadly, this error is
never sent back to the client because the method of the original
request is checked again in DefaultRestChannel which throws again
an IllegalArgumentException that is never handled.

This pull request changes the DefaultRestChannel so that the latest
exception is swallowed, allowing the error message to be sent back
to the client. It also eagerly adds the objects to close to the toClose
list so that resources are more likely to be released if something
 goes wrong during the response creation and sending.

Move nio ip filter rule to be a channel handler (#43507)

Currently nio implements ip filtering at the channel context level. This
is kind of a hack as the application logic should be implemented at the
handler level. This commit moves the ip filtering into a channel
handler. This requires adding an indicator to the channel handler to
show when a channel should be closed.

Fix testNoMasterActions (#43471)

This commit performs the proper restore of network disruption.
Previously disruptionScheme.stopDisrupting() was called that does not
ensure that connectivity between cluster nodes is restored. The test
was checking that the cluster has green status, but it was not checking
that connectivity between nodes is restored.
Here we switch to internalCluster().clearDisruptionScheme(true) which
performs both checks before returning.
Similar to #42798
Closes #42051

[DOCS] Rename "job" to "transform" in data frame transform docs (#43534)

Add annotations to Painless whitelist (#43239)

* Start to parse annotations in the whitelist

* Finish parsing annotations

* Finish annotation parsing

* Add docs

* Parser changes.

* Add annotation parser.

* Add annotation tests for Painless core.

* Add example annotation to Painless example extension.

* Response to PR comments.

Add TimeValue.toHumanReadableString() to allow specifying frac… (#43346)

* Enhance TimeValue.toString() to allow specifying fractional values.

This enhances the `TimeValue` class to allow specifying the number of
truncated fractional decimals when calling `toString()`. The default
remains 1, however, more (or less, such as 0) can be specified to change
the output.

This commit also re-organizes some things in `TimeValue` such as putting
all the class variables near the top of the class, and moving the
constructors to the first methods in the class, in order to follow the
structure of our other code.

* Rename `toString(...)` to `toHumanReadableString(...)`

Set document on script when using Bytes.WithScript (#43390)

Long and Double ValuesSource set the current document on the script
before executing, but Bytes was missing this method call.  That meant
it was possible to generate an OutOfBoundsException when using
a "value" script (field + script) on keyword or other bytes
fields.

This adds in the method call, and a few yaml tests to verify correct
behavior.

Distribution download plugin cleanup (#43461)

This commit addresses some post-review comments.

relates #43247

Sync global checkpoint on pending in-sync shards (#43526)

At the end of a peer recovery the primary wants to mark the replica as in-sync. For that the
persisted local checkpoint of the replica needs to have caught up with the global checkpoint on the
primary. If translog durability is set to ASYNC, this means that information about the persisted local
checkpoint can lag on the primary and might need to be explicitly fetched through a global
checkpoint sync action. Unfortunately, that action will only be triggered after 30 seconds, and, even
worse, will only run based on what the in-sync shard copies say (see
IndexShard.maybeSyncGlobalCheckpoint). As the replica has not been marked as in-sync yet, it is
not taken into consideration, and the primary might have its global checkpoint equal to the max seq
no, so it thinks nothing needs to be done.

Closes #43486

[DOCS] Adds administering section (#43493)

Fix score mode of the MinimumScoreCollector (#43527)

This change fixes the score mode of the minimum score collector to
be set based on the score mode of the child collector (top docs).

Closes #43497

Fix CreateRepository Requeset in HLRC (#43522)

* verify = false is the non-default case for this request -> adjusted the code accordingly and expanded the test to cover this case
* Closes #43521

Properly serialize remote query in ReindexRequest (#43457)

This commit modifies the RemoteInfo to clarify that a search query
must always be serialized as JSON. Additionally, it adds an assertion
to ensure that this is the case. This fixes #43406.

Additionally, this PR implements AbstractXContentTestCase for the
reindex request. This is related to #43456.

Remove debug log in testOpenCloseApiWildcards

Relates #39578

[DOCS] Edited title/subtitle. (#43552)

Enable Kerberos tests (#43519)

Now that the fix krb5-kdc fixture (entropy problem in docker container)
is in and the converting `kerberos-tests` to testclusters is done,
enabling the kerberos-tests

Closes #40678

Fix for PemTrustConfigTests.testTrustConfigReloadsFileContents failure (#43539)

The test `PemTrustConfigTests.testTrustConfigReloadsFileContents` failed
intermittently with `ArrayIndexOutOfBoundsException` while parsing
the randomly generated bytes array representing DER encoded stream.
This seems to be a bug in JDK (once confirmed we can raise the bug
in JDK bugs system).

The problem arises when the `X509Factory#parseX509orPKCS7()` tries to
[create `PKCS7` block](https://github.com/AdoptOpenJDK/openjdk-jdk11/blob/19fb8f93c59dfd791f62d41f332db9e306bc1422/src/java.base/share/classes/sun/security/provider/X509Factory.java#L460) from der encoded stream. While constructing PKCS7
block it tries to create `ContentInfo` type but fails to do so for the
stream where the length after the DER SEQUENCE is 0.
`DerInputStream#getSequence` [may return empty array of `DerValue`](https://github.com/AdoptOpenJDK/openjdk-jdk11/blob/19fb8f93c59dfd791f62d41f332db9e306bc1422/src/java.base/share/classes/sun/security/util/DerInputStream.java#L409..L412) but
[the code in `ContentInfo`](https://github.com/AdoptOpenJDK/openjdk-jdk11/blob/19fb8f93c59dfd791f62d41f332db9e306bc1422/src/java.base/share/classes/sun/security/pkcs/ContentInfo.java#L135) does not check for the empty thereby throwing
`ArrayIndexOutOfBoundsException`.

Closes #42509

Extract TimingStats-related functionality into TimingStatsReporter (#43371)

Get snapshots response example for the breaking change (#43475)

for requesting snapshots from multiple repositories. And it has changed
the response format in a non-BwC way.
There is a mentioning of a response format change in the breaking
changes docs, however, there is no example of how new format looks
like. Pointed out by @dakrone.
This commit adds the missing example.

[ML] Machine learning data frame analytics (#43544)

This merges the initial work that adds a framework for performing
machine learning analytics on data frames. The feature is currently experimental
and requires a platinum license. Note that the original commits can be
found in the `feature-ml-data-frame-analytics` branch.

A new set of APIs is added which allows the creation of data frame analytics
jobs. Configuration allows specifying different types of analysis to be performed
on a data frame. At first there is support for outlier detection.

The APIs are:

- PUT _ml/data_frame/analysis/{id}
- GET _ml/data_frame/analysis/{id}
- GET _ml/data_frame/analysis/{id}/_stats
- POST _ml/data_frame/analysis/{id}/_start
- POST _ml/data_frame/analysis/{id}/_stop
- DELETE _ml/data_frame/analysis/{id}

When a data frame analytics job is started a persistent task is created and started.
The main steps of the task are:

1. reindex the source index into the dest index
2. analyze the data through the data_frame_analyzer c++ process
3. merge the results of the process back into the destination index

In addition, an evaluation API is added which packages commonly used metrics
that provide evaluation of various analysis:

- POST _ml/data_frame/_evaluate

Make role descriptors optional when creating API keys (#43481)

This commit changes the `role_descriptors` field from required
to optional when creating API key. The default behavior in .NET ES
client is to omit properties with `null` value requiring additional
workarounds. The behavior for the API does not change.
Field names (`id`, `name`) in the invalidate api keys API documentation have been
corrected where they were wrong.

Closes #42053

Fix DieWithDignity logs assertion (#43543)

This test is likely to kill the server in the middle of writing logs. This means that we can end up with logs with partially written json log lines and standard json parsers would fail on this.
This fix is to use regular expressions on json logs.(just like the previous approach on plain text logs)
closes #43413

Fix testPostOperationGlobalCheckpointSync

The conditions in this test do not hold true anymore after #43205.

Relates to #43205

[ML] Improve message when native controller cannot connect (#43565)

The error message if the native controller failed to run
(for example due to running Elasticsearch on an unsupported
platform) was not easy to understand.  This change removes
pointless detail from the message and adds some hints about
likely causes.

Fixes #42341

Fix DedicatedClusterSnapshotRestoreIT testSnapshotWithStuckNode (#43537)

* Fix DedicatedClusterSnapshotRestoreIT testSnapshotWithStuckNode

* See comment in the test: The problem is that when the snapshot delete works out partially on master failover and the retry fails on `SnapshotMissingException` no repository cleanup is run => we still failed even with repo cleanup logic in the delete path now
   * Fixed the test by rerunning a create snapshot and delete loop to clean up the repo before verifying file counts
* Closes #39852

[ML][Data Frame] Adjusting error message (#43455)

* Adjusting error message

* Update TransportPutDataFrameTransformAction.java

* Update TransportPutDataFrameTransformAction.java

[ML][TEST] Refresh doc index in setup of test running df analytics

Fix GET /_snapshot/_all/_all if there are no repos (#43558)

When there are no repositories, a request to GET /_snapshot/_all/_all
returns a 504 timeout error.
This happens because try to create GroupedActionListener with the
size of zero, which leads to an exception.
This commit short-circuits if there are no repos and adds a test to
verify the fix.

Closes #43547

Default number of shards is now 1 instead of 5 (#43573)

As specified in the [Breaking changes for 7.X](https://www.elastic.co/guide/en/elasticsearch/reference/7.1/breaking-changes-7.0.html#breaking_70_indices_changes), the default number of shards for an index is now `1` instead of `5`.

[ML] Construct ML native controller in standard place (#43570)

The ML native controller used to be used within the
ML feature set which necessitated constructing it
outside of the usual createComponents() method.
Now that the ML native code info is reported by the
ML info action rather than by the feature set it is
possible to move construction to the standard place,
which avoids surprises for maintainers.

Fix search template request (#43509)

A seed was hit in (#43157) that caused mutateInstance to generate an identical
instance. This change prevents that.

[ML][TEST] Also refresh doc index in another df analytics test

Fixing backquote in fail_on_unsupported_field (#43572)

Fix indices shown in _cat/indices (#43286)

After two recent changes (#38824 and #33888), the _cat/indices API
no longer report information for active recovering indices and
non-replicated closed indices. It also misreport replicated closed
indices that are potentially not authorized for the user.

This commit changes how the cat action works by first using the
Get Settings API in order to resolve authorized indices. It then uses
the Cluster State, Cluster Health and Indices Stats APIs to retrieve
 information about the indices.

Closes #39933

Add voting-only master node (#43410)

A voting-only master-eligible node is a node that can participate in master elections but will not act
as a master in the cluster. In particular, a voting-only node can help elect another master-eligible
node as master, and can serve as a tiebreaker in elections. High availability (HA) clusters require at
least three master-eligible nodes, so that if one of the three nodes is down, then the remaining two
can still elect a master amongst them-selves. This only requires one of the two remaining nodes to
have the capability to act as master, but both need to have voting powers. This means that one of
the three master-eligible nodes can be made as voting-only. If this voting-only node is a dedicated
master, a less powerful machine or a smaller heap-size can be chosen for this node. Alternatively, a
voting-only non-dedicated master node can play the role of the third master-eligible node, which
allows running an HA cluster with only two dedicated master nodes.

Closes #14340

Co-authored-by: David Turner <david.turner@elastic.co>

Fix failing LicensingDocumentationIT test (#43533)

This PR brings corrections for cluster name after migrating to testclusters.
Not sure how this slipped trough the cracks when converting.

Closes #43504

[ML] Introduce a setting for the process connect timeout (#43234)

This change introduces a new setting,
xpack.ml.process_connect_timeout, to enable
the timeout for one of the external ML processes
to connect to the ES JVM to be increased.

The timeout may need to be increased if many
processes are being started simultaneously on
the same machine. This is unlikely in clusters
with many ML nodes, as we balance the processes
across the ML nodes, but can happen in clusters
with a single ML node and a high value for
xpack.ml.node_concurrent_job_allocations.

Disable testing conventions on Windows (#43532)

Tests are disabled on Windows. Conventions also need to be disabled.

Testclusters: convert left-overs from checkPart1 (#43370)

* Testclusters: convert left-overs from checkPart1

Add scripting support to AggregatorTestCase (#43494)

This refactors AggregatorTestCase to allow testing mock scripts.
The main change is to QueryShardContext.  This was previously mocked,
but to get the ScriptService you have to invoke a final method
which can't be mocked.

Instead, we just create a mostly-empty QueryShardContext and populate
the fields that are needed for testing.  It also introduces a few
new helper methods that can be overridden to change the default
behavior a bit.

Most tests should be able to override getMockScriptService() to supply
a ScriptService to the context, which is later used by the aggs.
More complicated tests can override queryShardContextMock() as before.

Adds a test to MaxAggregatorTests to test out the new functionality.

[DOCS] Add redirect for painless examples anchor

Fix merge issue between #43410 and #43449

PR builds on #43410 had not picked up the change merged in #43449

include 7.2.1 as a version (#43584)

Allow extra time for a warning to be logged (#43597)

Today we assert that a warning is logged after no more than
`discovery.cluster_formation_warning_timeout`, but the deterministic scheduler
adds a small amount of extra randomness to the timing of future events, causing
the following build to fail:

    ./gradlew :server:test --tests "org.elasticsearch.cluster.coordination.CoordinatorTests.testLogsWarningPeriodicallyIfClusterNotFormed" -Dtests.seed=DF35C28D4FA9EE2D

This commit adds an allowance for this extra time.

[DOCS] Rewrite `range` query (#43282)

Ensure relocation target still tracked when start handoff (#42201)

If the master removes the relocating shard, but recovery isn't aware of
it, then we can enter an invalid state where ReplicationTracker does not
include the local shard.

Adjust bwc assertion after backporting #42201

Remove guice modules from plugins (#43555)

Guice modules provide a way to supply injected parameters to other
injected constructors. The last remiaing use in plugins is to provide
constructor arguments to transport actions. This commit removes the
ability for plugins to provide guice modules. Any transport action
parameters should be concretely typed and returned from
createComponents, which are still injected. While this does not remove
guice completely, it removes it from all remaining plugin apis.

Remove unused method not caught by merge

relates #43555

Reindex remote version lookup test refactor (#43576)

Refactor test to reuse code and ease maintenance

Reindex remove outer level size (#43373)

This commit finalizes the work done to rename size to max_docs in
reindex and update/delete by query. size is no longer supported in URL
or outer level body for the 3 APIs (though size in update/delete-by-query
will and has always been interpreted as scroll_size, it is not to be relied
upon).

Continuation of #41894

Closes #24344

Upgrade jcodings dependency to 1.0.44 (#43334)

Mute DiskDisruptionIT#testGlobalCheckpointIsSafe

Relates to #43626

Mute VotingOnlyNodeCoordinatorTests#testDoesNotElectVotingOnlyMasterNode

Relates to #43631

[TEST] Wait for replicas before stopping nodes in ML distributed test (#43622)

If we stop a node before replicas exist then
the test can fail because we lose a whole
index if we stop the node with the primary on.

[ML][Data Frame] improve pivot nested field validations (#43548)

* [ML][Data Frame] improve pivot nested field validations

* addressing pr comments

[ML][Data Frame] Add support for allow_no_match for endpoints (#43490)

* [ML][Data Frame] Add support for allow_no_match parameter in endpoints

Adds support for:
* Get Transforms
* Get Transforms stats
* stop transforms

Remove unreleased 7.1.2 version constant (#43629)

This was breaking BWC tests as the presence of the constant implied 7.1.2 was released

Add prefix intervals source (#43635)

This commit adds a prefix intervals source, allowing you to search
for intervals that contain terms starting with a given prefix. The source
can make use of the index_prefixes mapping option.

Relates to #43198

[ML][Data Frame] fixing tag end for df doc tests (#43640)

[ML] Tag destination index with data frame metadata (#43567)

Optimize Selector Wakeups (#43515)

* Use atomic boolean to guard wakeups
* Don't trigger wakeups from the select loops thread itself for registering and closing channels
* Don't needlessly queue writes

Co-authored-by:  Tim Brooks <tim@uncontended.net>

[Ml Data Frame] Size the GET stats search by number of Ids requested (#43206)

Set the size of the search request to the number of ids limited by 10,000

Avoid AssertionError when closing engine (#43638)

Lucene throwing an AlreadyClosedException when closing the engine is fine, and should not trigger
an AssertionError.

Closes #43626

Remove blank file

Relates to #43410

Remove non task-aware version of master and node operations (#43563)

TransportNodeAction and TransportMasterNodeAction contains two
variations of their abstract operation methods: one taking a Task and
one that does not. This commit removes the variant that does not take a
Task, eliminating the need to override that method and throw an
unsupported operation exception in those cases when the task is
required.

[ML][Data Frame] adds tests for continuous DF (#43601)

[DOCS] Change 'X-Pack APIs' section to 'REST APIs' (#43451)

[DOCS] Updates data frame APIs to use API template (#43610)

Remove compile-time dependency on test fixtures (#43651)

Mute failing test

Tracked in #43670

Make the ignore_above docs tests more robust. (#43349)

It is possible for internal ML indices like `.data-frame-notifications-1` to leak,
causing other docs tests to fail when they accidentally search over these
indices. This PR updates the ignore_above tests to only search a specific index.

Require [articles] setting in elision filter (#43083)

We should throw an exception at construction time if a list of
articles is not provided, otherwise we can get random NPEs during
indexing.

Relates to #43002

Use preconfigured filters correctly in Analyze API (#43568)

When a named token filter or char filter is passed as part of an Analyze API
request with no index, we currently try and build the relevant filter using no
index settings. However, this can miss cases where there is a pre-configured
filter defined in the analysis registry. One example here is the elision filter, which
has a pre-configured version built with the french elision set; when used as part
of normal analysis, this preconfigured set is used, but when used as part of the
Analyze API we end up with NPEs because it tries to instantiate the filter with
no index settings.

This commit changes the Analyze API to check for pre-configured filters in the case
that the request has no index defined, and is using a name rather than a custom
definition for a filter.

It also changes the pre-configured `word_delimiter_graph` filter and `edge_ngram`
tokenizer to make their settings consistent with the defaults used when creating
them with no settings

Closes #43002
Closes #43621
Closes #43582

Remove Unused AWS KMS Dependency (#43671)

* We don't make use of KMS at the moment, no need to have this dependency here

Move query builder caching check to dedicated tests (#43238)

Currently `AbstractQueryTestCase#testToQuery` checks the search context cachable
flag. This is a bit fragile due to the high randomization of query builders
performed by this general test. Also we might only rarely check the
"interesting" cases because they rarely get generated when fully randomizing the
query builder.

This change moved the general checks out ot #testToQuery and instead adds
dedicated cache tests for those query builders that exhibit something other than
the default behaviour.

Closes #43200

[ML][DataFrame] Consider data frame templates internal in REST tests (#43692)

The data frame index template pattern was not in the list
considered as internal and therefore not needing cleanup
after every test.

Remove preconfigured `delimited_payload_filter` (#43686)

cleanup of pre-version 7 restrictions, but missed removing the
preconfigured version due to #43684.

Deduplicate two similar TimeUtils classes.

* Deduplicate org.elasticsearch.xpack.core.dataframe.utils.TimeUtils and org.elasticsearch.xpack.core.ml.utils.time.TimeUtils into a common class: org.elasticsearch.xpack.core.common.time.TimeUtils.
* Add unit tests for parseTimeField and parseTimeFieldToInstant methods

Fix UOE on search requests that match a sparse role query (#43668)

Search requests executed through the SecurityIndexSearcherWrapper throw
an UnsupportedOperationException if they match a sparse role query.
When low level cancellation is activated (which is the default since #42857),
the context index searcher creates a weight that doesn't handle #scorer.
This change fixes this bug and adds a test to ensure that we check this case.

Fix propagation of enablePositionIncrements in QueryStringQueryBuilder (#43578)

This change fixes the propagation of the enablePositionIncrements option
to the underlying QueryBuilder.

Closes #43574

Remove deprecated sort options: nested_path and nested_filter (#42809)

This commit removes the nested_path and nested_filter options deprecated in 6x.
This change also checks that the sort field has a [nested] option if it is under a nested
object and throws an exception if it's not the case.

Closes #27098

[ML] Don't write timing stats on no-op (#43680)

Similar to elastic/ml-cpp#512, if a job opens and closes and
does nothing in between we shouldn't write timing stats to
the results index.

Handle situation where only voting-only nodes are bootstrapped (#43628)

Adds support for the situation where only voting-only nodes are bootstrapped. In that case, they will
still try to become elected and bring full master nodes into the cluster.

Allow reloading of search time analyzers (#43313)

Currently changing resources (like dictionaries, synonym files etc...) of search
time analyzers is only possible by closing an index, changing the underlying
resource (e.g. synonym files) and then re-opening the index for the change to
take effect.

This PR adds a new API endpoint that allows triggering reloading of certain
analysis resources (currently token filters) that will then pick up changes in
underlying file resources. To achieve this we introduce a new type of custom
analyzer (ReloadableCustomAnalyzer) that uses a ReuseStrategy that allows
swapping out analysis components. Custom analyzers that contain filters that are
markes as "updateable" will automatically choose this implementation. This PR
also adds this capability to `synonym` token filters for use in search time
analyzers.

Relates to #29051

[DOCS] Adds anchors and attributes to ML APIs

Add version and create_time to data frame analytics config (#43683)

Remove nodeId from BaseNodeRequest (#43658)

TransportNodesAction provides a mechanism to easily broadcast a request
to many nodes, and collect the respones into a high level response. Each
node has its own request type, with a base class of BaseNodeRequest.
This base request requires passing the nodeId to which the request will
be sent. However, that nodeId is not used anywhere. It is private to the
base class, yet serialized to each node, where the node could just as
easily find the nodeId of the node it is on locally.

This commit removes passing the nodeId through to the node request
creation, and guards its serialization so that we can remove the base
request class altogether in the future.

Geo: Makes coordinate validator in libs/geo plugable (#43657)

Moves coordinate validation from Geometry constructors into
parser.

Relates #43644

[DOCS] Updates ML APIs to use new API template (#43711)

[DOCS] Adds data frame API response codes for allow_no_match (#43666)

Do not use MockInternalEngine in GatewayIndexStateIT (#43716)

GatewayIndexStateIT#testRecoverBrokenIndexMetadata replies on the
flushing on shutdown. This behaviour, however, can be randomly disabled
in MockInternalEngine.

Closes #43034

[ML] Allowing stopped status in HLRC testStartStop (#43710)

[FEATURE][ML] Support multiple source indices for df-analytics (#43702)

This commit adds support for multiple source indices.
In order to deal with multiple indices having different mappings,
it attempts a best-effort approach to merge the mappings assuming
there are no conflicts. In case conflicts exists an error will be
returned.

To allow users creating custom mappings for special use cases,
the destination index is now allowed to exist before the analytics
job runs. In addition, settings are no longer copied except for
the `index.number_of_shards` and `index.number_of_replicas`.

[ML] Rename outlier score setting to feature_influence_threshold (#43705)

Renames outlier score setting `minimum_score_to_write_feature_influence`
to `feature_influence_threshold`.

[TEST][ML] Fix yml test failure for data_frame_analytics_crud

Oddly, this passed CI on the PR that introduced it...

Remove rests of StandardHtmlStripAnalyzer (#43485)

StandardHtmlStripAnalyzer has been deprecated in 6.x and cannot be used for new
indices from 7.0 on. This change removes it entirely and also removes the from
tests and deprecation logging that has still been around during the 7.x
versions.

Enabled cannot be updated (#43701)

Removed the invalid tip that enabled can be updated for existing fields
and clarified instead that it cannot.

Related to #33566 and #33933

Add support for 'flattened object' fields. (#42541)

This commit merges the `object-fields` feature branch. The new 'flattened
object' field type allows an entire JSON object to be indexed into a field, and
provides limited search functionality over the field's contents.

[DOCS] Rewrite boosting query (#43647)

Wildcard intervals (#43691)

This commit adds a wildcard intervals source, similar to the prefix. It
also changes the term parameter in prefix to read prefix, to bring it
in to line with the pattern parameter in wildcard.

Closes #43198

[ML][Data Frame] removing format support (#43659)

Add missing dependencies so we can build in parallel  (#43672)

Testclusters: Convert additional projects (#43625)

* Testclusters: Convert additional projects

Found some more that were not using testclusters from elasticsearch-ci/1

* Allow IOException too

* Make the client more resilient

Refactor IndexSearcherWrapper to disallow the wrapping of IndexSearcher (#43645)

This change removes the ability to wrap an IndexSearcher in plugins. The IndexSearcherWrapper is replaced by an IndexReaderWrapper and allows to wrap the DirectoryReader only. This simplifies the creation of the context IndexSearcher that is used on a per request basis. This change also moves the optimization that was implemented in the security index searcher wrapper to the ContextIndexSearcher that now checks the live docs to determine how the search should be executed. If the underlying live docs is a sparse bit set the searcher will compute the intersection
betweeen the query and the live docs instead of checking the live docs on every document that match the query.

Fix threshold spelling errors (#43326)

Substitutes treshold by threshold

Trim translog for closed indices (#43156)

Today when an index is closed all its shards are forced flushed
but the translog files are left around. As explained in #42445
we'd like to trim the translog for closed indices in order to
consume less disk space. This commit reuses the existing
AsyncTrimTranslogTask task and reenables it for closed indices.

At the time the task is executed, we should have the guarantee
that nothing holds the translog files that are going to be removed.
It also leaves a short period of time (10 min) during which translog
files of a recently closed index are still present on disk. This could
 also help in some cases where the closed index is reopened
shortly after being closed (in order to update an index setting
for example).

Relates to #42445

Convert some action.execute cases to using NodeClient (#43715)

There are several cases where execute is called on actions directly,
mostly within other action implementations. Now that transport client is
gone, these internal actions can be registered just like normal actions,
and called with the NodeClient to execute locally on the same node. This
commit converts a few of these cases for shard level actions to use this
approach.

[DOCS] Rewrite `terms_set` query (#43060)

Re-enable bwc tests (#43723)

relates #43658

[ML] Mark ml-cpp dependency as regularly changing (#43760)

Since #41817 was merged the ml-cpp zip file for any
given version has been cached indefinitely by Gradle.
This is problematic, particularly in the case of the
master branch where the version 8.0.0-SNAPSHOT will
be in use for more than a year.

This change tells Gradle that the ml-cpp zip file is
a "changing" dependency, and to check whether it has
changed every two hours. Two hours is a compromise
between checking on every build and annoying developers
with slow internet connections and checking rarely
causing bug fixes in the ml-cpp code to take a long
time to propagate through to elasticsearch PRs that
rely on them.

Migrate watcher hlrc response tests to use AbstractResponseTestCase (#43478)

Relates to #43472

Add StreamableResponseAction to aid in deprecation of Streamable (#43770)

The Action base class currently works for both Streamable and Writeable
response types. This commit intorduces StreamableResponseAction, for
which only the legacy Action implementions which provide newResponse()
will extend. This eliminates the need for overriding newResponse() with
an UnsupportedOperationException.

relates #34389

[ML] Rename df-analytics `_id_copy` to `ml__id_copy` (#43754)

Renames `_id_copy` to `ml__id_copy` as field names starting with
underscore are deprecated. The new field name `ml__id_copy` was
chosen as an obscure enough field that users won't have in their data.
Otherwise, this field is only intented to be used by df-analytics.

[ML] Assert that a no-op job creates no results nor state (#43681)

If a job is opened and then closed and does nothing in
between then it should not persist any results or state
documents. This change adapts the no-op job test to
assert no results in addition to no state, and to log
any documents that cause this assertion to fail.

Relates elastic/ml-cpp#512
Relates #43680

Consistent Secure Settings (#40416)

Introduces a new `ConsistentSecureSettingsValidatorService` service that exposes
a single public method, namely `allSecureSettingsConsistent`. The method returns
`true` if the local node's secure settings (inside the keystore) are equal to the
master's, and `false` otherwise. Technically, the local node has to have exactly
the same secure settings - setting names should not be missing or in surplus
- for all `SecureSetting` instances that are flagged with the newly introduced
`Property.Consistent`. It is worth highlighting that the `allSecureSettingsConsistent`
is not a consensus view across the cluster, but rather the local node's perspective
in relation to the master.

Rename Action to ActionType (#43778)

Action is a class that encapsulates meta information about an action
that allows it to be called remotely, specifically the action name and
response type. With recent refactoring, the action class can now be
constructed as a static constant, instead of needing to create a
subclass. This makes the old pattern of creating a singleton INSTANCE
both misnamed and lacking a common placement.

This commit renames Action to ActionType, thus allowing the old INSTANCE
naming pattern to be TYPE on the transport action itself. ActionType
also conveys that this class is also not the action itself, although
this change does not rename any concrete classes as those will be
removed organically as they are converted to TYPE constants.

relates #34389

Make peer recovery clean files step async (#43787)

Relates #36195

Reenable bwc tests after #40416 (#43794)

Nothing to see here.

AsyncIOProcessor preserve thread context (#43729)

AsyncIOProcessor now preserves thread context, ensuring that deprecation
warnings are not duplicated to other concurrent operations on the same
shard.

Avoid parallel reroutes in DiskThresholdMonitor (#43381)

Today the `DiskThresholdMonitor` limits the frequency with which it submits
reroute tasks, but it might still submit these tasks faster than the master can
process them if, for instance, each reroute takes over 60 seconds. This causes
a problem since the reroute task runs with priority `IMMEDIATE` and is always
scheduled when there is a node over the high watermark, so this can starve any
other pending tasks on the master.

This change avoids further updates from the monitor while its last task(s) are
still in progress, and it measures the time of each update from the completion
time of the reroute task rather than its start time, to allow a larger window
for other tasks to run.

It also now makes use of the `RoutingService` to submit the reroute task, in
order to batch this task with any other pending reroutes. It enhances the
`RoutingService` to notify its listeners on completion.

Fixes #40174
Relates #42559

[ML] Correct df-analytics version introduced to 7.3.0 (#43784)

Make peer recovery send file info step async (#43792)

Relates #36195

show a full ingest example in the index page, to let user fast understand ingest node. (#43476)

Async IO Processor release before notify (#43682)

This commit changes async IO processor to release the promiseSemaphore
before notifying consumers. This ensures that a bad consumer that
sometimes does blocking (or otherwise slow) operations does not halt the
processor. This should slightly increase the concurrency for shard
fsync, but primarily improves safety so that one bad piece of code has
less effect on overall system performance.

relax trigger count for transform stats test (#43753)

relax trigger count test as we can not guarantee it due to async behaviour

Testclusters: improove timeout handling (#43440)

Switch WriteActionsTests.testBulk to use hamcrest (#43725)

If an item in the bulk request fails, that could be for a variety of
reasons - it may be that the underlying behaviour of security has
changed, or it may just be a transient failure during testing.

Simply asserting a `true`/`false` value produces failure messages that
are difficult to diagnose and debug. Using hamcert (`assertThat`) will
make it easier to understand the causes of failures in this test.

Add "manage_api_key" cluster privilege (#43728)

This adds a new cluster privilege for manage_api_key. Users with this
privilege are able to create new API keys (as a child of their own
user identity) and may also get and invalidate any/all API keys
(including those owned by other users).

Update TLS configuration in Docker docs (#43748)

Following the removal of the `unzip` package from the Elasticsearch
Docker image in #39040, update setup instructions for TLS in Docker.

Also avoid cross-platform ownership+permission issues by not relying
on local bind mounts for storing generated certs and don't require
`curl` locally installed.

HLRC changes for kerberos grant type (#43642)

The TODO from last PR for kerbero grant type was missed.
This commit adds the changes for kerberos grant type in HLRC.

Avoid IP addresses for bootstrapping in setup docs (#43802)

Removes the suggestion to use IP addresses for `cluster.initial_master_nodes`
in the "important settings" discovery docs, leaving only the suggestion to use
node names.

Relates #41179, #41569

Expose translog stats in ReadOnlyEngine (#43752)

[ML][Data Frame] reduce audit frequency, change log msg, and level (#43771)

[ML][Data Frame] account for delay in writing stats docs (#43703)

Optimize Azure Directory Delete (#43341)

* Follow up to #43281:
* Optimizing the Azure directory delete operation:
   * Same as with GCS and S3 we can simply flat list a prefix and then delete as we iterate instead of listing the directories recursively. This should require fewer actual list RPC calls and the logic becomes simpler

Update docs for Open/Close API (#43809)

Relates #43530

Add RareTerms aggregation (#35718)

This adds a `rare_terms` aggregation.  It is an aggregation designed
to identify the long-tail of keywords, e.g. terms that are "rare" or
have low doc counts.

This aggregation is designed to be more memory efficient than the
alternative, which is setting a terms aggregation to size: LONG_MAX
(or worse, ordering a terms agg by count ascending, which has
unbounded error).

This aggregation works by maintaining a map of terms that have
been seen. A counter associated with each value is incremented
when we see the term again.  If the counter surpasses a predefined
threshold, the term is removed from the map and inserted into a cuckoo
filter.  If a future term is found in the cuckoo filter we assume it
was previously removed from the map and is "common".

The map keys are the "rare" terms after collection is done.

Yet another `the the` cleanup (#43815)

Revert "[TEST] Mute RemoteClusterServiceTests.testCollectNodes"

This reverts commit d184056bf5cf9968db95c16cb27f6f2ec505bc3e.

Update rare_term version skips, fix SetBackedScalingCuckooFilter javadoc

Add more logging to investigate #41067

Upgrade to Gradle 5.5 (#43788)

Fix logger usage check

SmokeTestWatcherWithSecurityIT: Retry if failures searching (#43781)

Retry if failures searching .watcher-history

Catch exceptions and fail (which allows the assertBusy to retry)
if an error happens from the search against .watcher-history.

Related: #42409
Related: #39169

[ML][Data Frame] using transform creation version for node assignment (#43764)

* [ML][Data Frame] using transform creation version for node assignment

* removing unused imports

* Addressing PR comment

[ML][Data Frame] Add deduced mappings to _preview response payload (#43742)

* [ML][Data Frame] Add deduced mappings to _preview response payload

* updating preview docs

Add timezone documentation for Painless datetimes (#43768)

Document API-Key APIs require manage_api_key priv (#43811)

Add the "Authorization" section to the API key API docs.
These APIs require The new manage_api_key cluster privilege.

Relates: #43728

Rename and refactor RoutingService (#43827)

The `RoutingService` has a confusing name, since it doesn't really have
anything to do with routing. Its responsibility is submitting reroute commands
to the master.

This commit renames this class to `BatchedRerouteService`, and extracts the
`RerouteService` interface to avoid passing `BiConsumer`s everywhere. It also
removes that `BatchedRerouteService extends AbstractLifecycleComponent` since
this service has no meaningful lifecycle. Finally, it introduces a small
wrapper class to allow for lazy initialization to deal with the dependency loop
when constructing a `Node`.

Update docs to refer to 6.8 instead of 6.7 (#43685)

A few places in the documentation had mentioned 6.7 as the version to
upgrade from, when doing an upgrade to 7.0. While this is technically
possible, this commit will replace all those mentions to 6.8, as this is
the latest version with the latest bugfixes, deprecation checks and
ugprade assistant features - which should be the one used for upgrades.

Co-Authored-By: James Rodewig <james.rodewig@elastic.co>

Use explicit string keys in data_frame test (#43824)

Disable BWC tests (#43856)

Until #43823 is backported to 7.x

Fix credentials encoding for OIDC token request (#43808)

As defined in https://tools.ietf.org/html/rfc6749#section-2.3.1
both client id and client secret need to be encoded with the
application/x-www-form-urlencoded encoding algorithm when used as
credentials for HTTP Basic Authentication in requests to the OP.

Resolves #43709

[ML] Get df-analytics action should require monitor privilege (#43831)

Fix DieWithDignity test when waiting on jps (#43861)

the test often hangs on executing jps command
we don't need to wait for this command to finish.

closes #43413

Remove sort by primary term when reading soft-deletes (#43845)

With Lucene rollback (#33473), we should never have more than one
primary term for each sequence number. Therefore we don't have to sort
by the primary term when reading soft-deletes.

Return reloaded analyzers in _reload_search_ananlyzer response (#43813)

Currently the repsonse of the "_reload_search_analyzer" endpoint contains the
index names and nodeIds of indices were analyzers reloading was triggered. This
change add the names of the search-time analyzers that were reloaded.

Closes #43804

[ML][Data Frame] fix progress measurement for continuous transforms (#43838)

* [ML][Data Frame] fix progress measurement for continuous transforms

* Update DataFrameIndexer.java

Clarify voting-only master node docs (#43857)

Clarifies the roles of a dedicated voting-only master-eligible node.

Co-Authored-By: James Rodewig <james.rodewig@elastic.co>
Co-Authored-By: David Turner <david.turner@elastic.co>

[ML-Data Frame] Add data frame transform cluster privileges to HLRC (#43879)

Adds the monitor_data_frame_transforms and
manage_data_frame_transforms cluster privileges to
the high level rest client.

The ALL_ARRAY variable is only used in randomized
tests at the within the Elasticsearch code, so it's
not a major problem that these cluster privileges
weren't added from the start.  But since ALL_ARRAY
is public HLRC users may be using it to find out
which cluster privileges exist, so it's best that
it contains them all.

Reenable bwc tests (#43872)

This reenables the BWC tests disabled in #43856

Link rare_terms docs from index page (#43882)

Docs for rare_terms were added in #35718, but neglected to
link it from the bucket index page

Dry up inputstream to bytesreference (#43675)

* Dry up Reading InputStream to BytesReference

* Dry up spots where we use the same pattern to get from an InputStream to a BytesReferences

Convert replication calls from action.execute to nodeclient (#43834)

Replication relies on some internal actions to do local replication
operations. This commit converts these actions to registered transport
actions so NodeClient can be used.

[ML][Data Frame] add node attr to GET _stats (#43842)

* [ML][Data Frame] add node attr to GET _stats

* addressing testing issues with node.attributes

Add dims parameter to dense_vector mapping (#43444)

Typically, dense vectors of both documents and queries must have the same
number of dimensions. Different number of dimensions among documents
or query vector indicate an error. This PR enforces that all vectors
for the same field have the same number of dimensions. It also enforces
that query vectors have the same number of dimensions.

[DOCS] Revise GS intro & remove redundant conceptual info. Closes #43846 (#43847)

* [DOCS] Revise GS intro and remove redundant conceptual content. Closes #43846.

* [DOCS] Incorporated feedback.

* [DOCS] Abbreviated titles for parts don't work in  AsciiDoctor.

Add Datetime Now to Painless Documentation (#43852)

This change explains why Painless doesn't natively support datetime now, and
gives examples of how to create a version of now through user-defined
parameters.

Enable caching of rest tests which use integ-test distribution (#43782)

Extend timeout for TimeSeriesLifecycleActionsIT> testFullPolicy (#43891)

ilm test - allow more time for policy completion (#43844)

Always attach system user to internal actions (#43468)

All valid licenses permit security, and the only license state where
we don't support security is when there is a missing license.
However, for safety we should attach the system (or xpack/security)
user to internally originated actions even if the license is missing
(or, more strictly, doesn't support security).

This allows all nodes to communicate and send internal actions (shard
state, handshake/pings, etc) even if a license is transitioning
between a broken state and a valid state.

Relates: #42215

Use separate BitSet cache in Doc Level Security (#43669)

Document level security was depending on the shared
"BitsetFilterCache" which (by design) never expires its entries.

However, when using DLS queries - particularly templated ones - the
number (and memory usage) of generated bitsets can be significant.

This change introduces a new cache specifically for BitSets used in
DLS queries, that has memory usage constraints and access time expiry.

The whole cache is automatically cleared if the role cache is cleared.
Individual bitsets are cleared when the corresponding lucene index
reader is closed.

The cache defaults to 50MB, and entries expire if unused for 7 days.

Support builtin privileges in get privileges API (#42134)

Adds a new "/_security/privilege/_builtin" endpoint so that builtin
index and cluster privileges can be retrieved via the Rest API

Resolves: #29771

Refresh translog stats after translog trimming in NoOpEngine (#43825)

This commit changes NoOpEngine so that it refreshes its translog
stats once translog is trimmed.

Relates #43156

Fix index_prefix sub field name on nested text fields (#43862)

This change fixes the name of the index_prefix sub field when the `index_prefix`
option is set on a text field that is nested under an object or a multi-field.
We don't use the full path of the parent field to set the index_prefix field name
so the field is registered under the wrong name. This doesn't break queries since
we always retrieve the prefix field through its parent field but this breaks other
APIs like _field_caps which tries to find the parent of the `index_prefix` field
in the mapping but fails.

Closes #43741

Adapt version check after backport

Relates #43862

[ML] Set df-analytics task state to failed when appropriate (#43880)

This introduces a `failed` state to which the data frame analytics
persistent task is set to when something unexpected fails. It could
be the process crashing, the results processor hitting some error,
etc. The failure message is then captured and set on the task state.
From there, it becomes available via the _stats API as `failure_reason`.

The df-analytics stop API now has a `force` boolean parameter. This allows
the user to call it for a failed task in order to reset it to `stopped` after
we have ensured the failure has been communicated to the user.

This commit also adds the analytics version in the persistent task
params as this allows us to prevent tasks to run on unsuitable nodes in
the future.

Watcher: Allow to execute actions for each element in array (#41997)

This adds the ability to execute an action for each element that occurs
in an array, for example you could sent a dedicated slack action for
each search hit returned from a search.

There is also a limit for the number of actions executed, which is
hardcoded to 100 right now, to prevent having watches run forever.

The watch history logs each action result and the total number of actions
the were executed.

Relates #34546

Add _reload_search_analyzers endpoint to HLRC (#43733)

This change adds the new endpoint that allows reloading of search analyzers to
the high-level java rest client.

Relates to #43313

Remove the client transport profile filter (#43236)

Now that the transport client has been removed, the client transport
profile filter can be removed from security. This filter prevented node
actions from being executed using a transport client.

Ensure to access RecoveryState#fileDetails under lock

Closes #43840

[DOCS] Rewrite dis max query (#43586)

Move CORS Config into :server package (#43779)

This commit moves the config that stores Cors options into the server
package. Currently both nio and netty modules must have a copy of this
config. Moving it into server allows one copy and the tests to be in a
common location.

Shortcut simple patterns ending in `*` (#43904)

When profiling a call to `AllocationService#reroute()` in a large cluster
containing allocation filters of the form `node-name-*` I observed a nontrivial
amount of time spent in `Regex#simpleMatch` due to these allocation filters.
Patterns ending in a wildcard are not uncommon, and this change treats them as
a special case in `Regex#simpleMatch` in order to shave a bit of time off this
calculation. It also uses `String#regionMatches()` to avoid an allocation in
the case that the pattern's only wildcard is at the start.

Microbenchmark results before this change:

    Result "org.elasticsearch.common.regex.RegexStartsWithBenchmark.performSimpleMatch":
      1113.839 ±(99.9%) 6.338 ns/op [Average]
      (min, avg, max) = (1102.388, 1113.839, 1135.783), stdev = 9.486
      CI (99.9%): [1107.502, 1120.177] (assumes normal distribution)

Microbenchmark results with this change applied:

    Result "org.elasticsearch.common.regex.RegexStartsWithBenchmark.performSimpleMatch":
      433.190 ±(99.9%) 0.644 ns/op [Average]
      (min, avg, max) = (431.518, 433.190, 435.456), stdev = 0.964
      CI (99.9%): [432.546, 433.833] (assumes normal distribution)

The microbenchmark in question was:

    @Fork(3)
    @Warmup(iterations = 10)
    @Measurement(iterations = 10)
    @BenchmarkMode(Mode.AverageTime)
    @OutputTimeUnit(TimeUnit.NANOSECONDS)
    @State(Scope.Benchmark)
    @SuppressWarnings("unused") //invoked by benchmarking framework
    public class RegexStartsWithBenchmark {

        private static final String testString = "abcdefghijklmnopqrstuvwxyz";
        private static final String[] patterns;

        static {
            patterns = new String[testString.length() + 1];
            for (int i = 0; i <= testString.length(); i++) {
                patterns[i] = testString.substring(0, i) + "*";
            }
        }

        @Benchmark
        public void performSimpleMatch() {
            for (int i = 0; i < patterns.length; i++) {
                Regex.simpleMatch(patterns[i], testString);
            }
        }
    }

Mute failing test

Tracked in #43924

Actually close IndexAnalyzers contents (#43914)

IndexAnalyzers has a close() method that should iterate through all its wrapped
analyzers and close each one in turn. However, instead of delegating to the
analyzers' close() methods, it instead wraps them in a Closeable interface,
which just returns a list of the analyzers. In addition, whitespace normalizers are
ignored entirely.

Geo: Refactors libs/geo parser to provide serialization logic as well (#43717)

Enables libs/geo parser to return a geometry format object that can
perform both serialization and deserialization functions. This can
be useful for ingest nodes that are trying to modify an existing
geometry in the source.

Relates to #43554

[ML][Data Frame] Adding bwc tests for pivot transform (#43506)

* [ML][Data Frame] Adding bwc tests for pivot transform

* adding continuous transforms

* adding continuous dataframes to bwc

* adding continuous data frame tests

* Adding rolling upgrade tests for continuous df

* Fixing test

* Adjusting indices used in BWC, and handling NPE for seq_no_stats

* updating and muting specific bwc test

Enable validating user-supplied missing values on unmapped fields (#43718)

Provides a hook for aggregations to introspect the `ValuesSourceType` for a user supplied Missing value on an unmapped field, when the type would otherwise be `ANY`.  Mapped field behavior is unchanged, and still applies the `ValuesSourceType` of the field.  This PR just provides the hook for doing this, no existing aggregations have their behavior changed.

Add version 7.4. (#43930)

Relates #43927.

Adjust version in vectors tests after backport

After backport to 7.3 adjust yml tests

Relates to #43444

Remove generic on AggregatorFactory (#43664)

AggregatorFactory was generic over itself, but it doesn't appear we
use this functionality anywhere (e.g. to allow the super class
to declare arguments/return types generically for subclasses to
override).  Most places use a wildcard constraint, and even when a
concrete type is specified it wasn't used.

But since AggFactories are widely used, this led to
the generic touching many pieces of code and making type signatures
fairly complex

 Ensure test cluster classpath inputs have predictable ordering (#43938)
---
 .ci/java-versions.properties                  |    1 +
 .ci/matrix-runtime-javas.yml                  |    1 +
 .github/PULL_REQUEST_TEMPLATE.md              |    2 +-
 CONTRIBUTING.md                               |    3 +-
 README.textile                                |    6 +-
 TESTING.asciidoc                              |    6 -
 benchmarks/build.gradle                       |    2 +-
 .../routing/allocation/Allocators.java        |    5 +-
 build.gradle                                  |   99 +-
 buildSrc/build.gradle                         |   44 +-
 .../elasticsearch/gradle/BuildPlugin.groovy   |  158 +-
 .../gradle/doc/SnippetsTask.groovy            |    6 +-
 .../gradle/plugin/PluginBuildPlugin.groovy    |   36 +-
 .../plugin/PluginPropertiesExtension.groovy   |   94 -
 .../precommit/DependencyLicensesTask.groovy   |  268 --
 .../gradle/precommit/PrecommitTasks.groovy    |   38 +-
 .../gradle/precommit/UpdateShasTask.groovy    |   66 -
 .../gradle/test/ClusterFormationTasks.groovy  |    8 +-
 .../gradle/test/MessyTestPlugin.groovy        |   63 -
 .../elasticsearch/gradle/test/NodeInfo.groovy |    5 +
 .../gradle/test/RestIntegTestTask.groovy      |   11 +-
 .../test/StandaloneRestTestPlugin.groovy      |    5 +-
 .../AbstractLazyPropertyCollection.java       |   27 +
 .../org/elasticsearch/gradle/BwcVersions.java |    7 +-
 .../gradle/DistributionDownloadPlugin.java    |  275 ++
 .../gradle/ElasticsearchDistribution.java     |  230 ++
 .../elasticsearch/gradle/EmptyDirTask.java    |   83 +
 .../gradle/JdkDownloadPlugin.java             |    8 +-
 .../gradle/LazyPropertyList.java              |  205 ++
 .../elasticsearch/gradle/LazyPropertyMap.java |  167 +
 .../gradle/PropertyNormalization.java         |   13 +
 ...emPropertyCommandLineArgumentProvider.java |   30 +
 .../gradle/http/WaitForHttpResource.java      |   20 +-
 .../gradle/info/GlobalBuildInfoPlugin.java    |    2 +-
 .../plugin/PluginPropertiesExtension.java     |  141 +
 .../precommit/DependencyLicensesTask.java     |  328 ++
 .../gradle/precommit/UpdateShasTask.java      |   86 +
 .../gradle/test/RestTestRunnerTask.java       |   37 +
 .../testclusters/ElasticsearchCluster.java    |   93 +-
 .../testclusters/ElasticsearchNode.java       |  500 ++-
 .../TestClusterCleanupOnShutdown.java         |   59 +
 .../TestClusterConfiguration.java             |   37 +-
 .../TestClustersCleanupExtension.java         |   74 +
 .../testclusters/TestClustersPlugin.java      |  149 +-
 .../testfixtures/TestFixturesPlugin.java      |  121 +-
 .../gradle/tool/ClasspathUtils.java           |   23 +
 ...icsearch.distribution-download.properties} |    2 +-
 buildSrc/src/main/resources/buildSrc.marker   |    0
 .../src/main/resources/minimumGradleVersion   |    2 +-
 .../elasticsearch/gradle/JdkJarHellCheck.java |    0
 .../gradle/LazyFileOutputStream.java          |    0
 .../org/elasticsearch/gradle/LoggedExec.java  |    9 +-
 .../org/elasticsearch/gradle/Version.java     |    0
 .../gradle/VersionProperties.java             |    0
 .../gradle/DistributionDownloadPluginIT.java  |  136 +
 .../DistributionDownloadPluginTests.java      |  260 ++
 .../gradle/EmptyDirTaskTests.java             |   78 +
 .../PluginPropertiesExtensionTests.java       |   58 +
 .../DependencyLicensesTaskTests.java          |  268 ++
 .../gradle/precommit/UpdateShasTaskTests.java |  140 +
 .../distribution-download/build.gradle        |   44 +
 .../distribution/build.gradle                 |   35 +
 .../files/fake_elasticsearch.tar.gz           |  Bin 0 -> 188 bytes
 .../distribution/files/fake_elasticsearch.zip |  Bin 0 -> 568 bytes
 .../distribution-download/settings.gradle     |    8 +
 .../subproj/build.gradle                      |   58 +
 client/benchmark/README.md                    |    4 +-
 client/benchmark/build.gradle                 |   10 +-
 .../client/benchmark/BenchmarkMain.java       |    4 -
 .../transport/TransportClientBenchmark.java   |  127 -
 .../noop/action/bulk/NoopBulkAction.java      |    4 +-
 .../noop/action/search/NoopSearchAction.java  |    9 +-
 client/rest-high-level/build.gradle           |   60 +-
 .../client/CcrRequestConverters.java          |    3 +-
 .../client/ClusterRequestConverters.java      |   12 +-
 .../client/DataFrameRequestConverters.java    |   42 +-
 .../IndexLifecycleRequestConverters.java      |   27 +-
 .../elasticsearch/client/IndicesClient.java   |   30 +-
 .../client/IndicesRequestConverters.java      |  137 +-
 .../client/IngestRequestConverters.java       |   14 +-
 .../client/LicenseRequestConverters.java      |   15 +-
 .../client/MLRequestConverters.java           |  156 +-
 .../client/MachineLearningClient.java         |  294 ++
 .../client/RequestConverters.java             |  116 +-
 .../client/RollupRequestConverters.java       |   13 +-
 .../elasticsearch/client/SecurityClient.java  |   30 +
 .../client/SecurityRequestConverters.java     |   38 +-
 .../client/SnapshotRequestConverters.java     |   35 +-
 .../client/TasksRequestConverters.java        |    9 +-
 .../client/WatcherRequestConverters.java      |   10 +-
 .../org/elasticsearch/client/XPackClient.java |    4 +-
 .../client/XPackRequestConverters.java        |    3 +-
 .../client/ccr/IndicesFollowStats.java        |   80 +-
 .../client/core/BroadcastResponse.java        |    8 +-
 .../client/core/TermVectorsResponse.java      |   30 +-
 .../DataFrameNamedXContentProvider.java       |   29 +-
 .../GetDataFrameTransformRequest.java         |   16 +-
 .../GetDataFrameTransformStatsRequest.java    |   15 +-
 .../PreviewDataFrameTransformResponse.java    |   19 +-
 .../StartDataFrameTransformResponse.java      |   12 +-
 .../StopDataFrameTransformRequest.java        |   14 +-
 .../StopDataFrameTransformResponse.java       |   13 +-
 .../transforms/DataFrameTransformConfig.java  |   94 +-
 .../transforms/DataFrameTransformState.java   |   18 +-
 .../dataframe/transforms/DestConfig.java      |   54 +-
 .../dataframe/transforms/NodeAttributes.java  |  156 +
 .../dataframe/transforms/SyncConfig.java      |   30 +
 .../dataframe/transforms/TimeSyncConfig.java  |  108 +
 .../pivot/DateHistogramGroupSource.java       |   35 +-
 .../dataframe/transforms/util/TimeUtil.java   |   59 +
 .../client/indices/AnalyzeRequest.java        |  343 ++
 .../client/indices/AnalyzeResponse.java       |  183 ++
 .../client/indices/DetailAnalyzeResponse.java |  214 ++
 .../indices/ReloadAnalyzersRequest.java       |   68 +
 .../indices/ReloadAnalyzersResponse.java      |  108 +
 .../ml/DeleteDataFrameAnalyticsRequest.java   |   64 +
 .../client/ml/EvaluateDataFrameRequest.java   |  136 +
 .../client/ml/EvaluateDataFrameResponse.java  |  117 +
 .../client/ml/FindFileStructureRequest.java   |   18 +-
 .../ml/GetDataFrameAnalyticsRequest.java      |  104 +
 .../ml/GetDataFrameAnalyticsResponse.java     |   74 +
 .../ml/GetDataFrameAnalyticsStatsRequest.java |   99 +
 .../GetDataFrameAnalyticsStatsResponse.java   |  102 +
 .../client/ml/NodeAttributes.java             |    6 +
 .../ml/PutDataFrameAnalyticsRequest.java      |   76 +
 .../ml/PutDataFrameAnalyticsResponse.java     |   57 +
 .../ml/StartDataFrameAnalyticsRequest.java    |   74 +
 .../ml/StopDataFrameAnalyticsRequest.java     |  100 +
 .../ml/StopDataFrameAnalyticsResponse.java    |   87 +
 .../ml/dataframe/DataFrameAnalysis.java       |   27 +
 .../dataframe/DataFrameAnalyticsConfig.java   |  259 ++
 .../ml/dataframe/DataFrameAnalyticsDest.java  |  123 +
 .../dataframe/DataFrameAnalyticsSource.java   |  128 +
 .../ml/dataframe/DataFrameAnalyticsState.java |   34 +
 .../ml/dataframe/DataFrameAnalyticsStats.java |  145 +
 ...ataFrameAnalysisNamedXContentProvider.java |   37 +
 .../client/ml/dataframe/OutlierDetection.java |  175 +
 .../client/ml/dataframe/QueryConfig.java      |   82 +
 .../ml/dataframe/evaluation/Evaluation.java   |   32 +
 .../evaluation/EvaluationMetric.java          |   32 +-
 .../MlEvaluationNamedXContentProvider.java    |   57 +
 .../AbstractConfusionMatrixMetric.java        |   47 +
 .../softclassification/AucRocMetric.java      |  241 ++
 .../BinarySoftClassification.java             |  129 +
 .../ConfusionMatrixMetric.java                |  206 ++
 .../softclassification/PrecisionMetric.java   |  123 +
 .../softclassification/RecallMetric.java      |  123 +
 .../ml/filestructurefinder/FieldStats.java    |   30 +-
 .../client/ml/job/process/TimingStats.java    |  159 +
 .../client/ml/job/stats/JobStats.java         |   23 +-
 .../client/security/CreateTokenRequest.java   |   34 +-
 .../client/security/CreateTokenResponse.java  |   19 +-
 .../security/GetBuiltinPrivilegesRequest.java |   40 +
 .../GetBuiltinPrivilegesResponse.java         |   82 +
 .../client/security/user/privileges/Role.java |    7 +-
 .../client/xpack/XPackInfoResponse.java       |   10 +-
 ...icsearch.plugins.spi.NamedXContentProvider |    5 +-
 ...AbstractHlrcWriteableXContentTestCase.java |   70 +
 .../client/AbstractResponseTestCase.java      |    6 +-
 .../elasticsearch/client/BulkProcessorIT.java |    2 -
 .../BulkRequestWithGlobalParametersIT.java    |    5 -
 .../DataFrameRequestConvertersTests.java      |   26 +-
 .../client/DataFrameTransformIT.java          |   77 +-
 .../client/ESRestHighLevelClientTestCase.java |    2 +-
 .../elasticsearch/client/IndicesClientIT.java |   20 +-
 .../client/IndicesRequestConvertersTests.java |   30 +-
 .../client/MLRequestConvertersTests.java      |  137 +-
 .../client/MachineLearningIT.java             |  517 ++-
 .../client/MlTestStateCleaner.java            |   13 +
 .../elasticsearch/client/PingAndInfoIT.java   |   13 +-
 .../client/RequestConvertersTests.java        |   65 +-
 .../client/RestHighLevelClientTests.java      |   28 +-
 .../org/elasticsearch/client/RollupIT.java    |   22 +-
 .../SecurityRequestConvertersTests.java       |    4 +-
 .../org/elasticsearch/client/SnapshotIT.java  |  109 +-
 .../SnapshotRequestConvertersTests.java       |   18 +-
 .../client/XPackInfoResponseTests.java        |   19 +-
 .../client/ccr/CcrStatsResponseTests.java     |    3 +
 .../client/ccr/FollowStatsResponseTests.java  |    2 +
 .../client/core/TermVectorsResponseTests.java |    8 +-
 .../GetDataFrameTransformResponseTests.java   |    7 +-
 ...PreviewDataFrameTransformRequestTests.java |    8 +-
 ...reviewDataFrameTransformResponseTests.java |    8 +-
 .../PutDataFrameTransformRequestTests.java    |    8 +-
 .../DataFrameTransformConfigTests.java        |   25 +-
 .../DataFrameTransformStateTests.java         |   11 +-
 .../dataframe/transforms/DestConfigTests.java |    3 +-
 .../transforms/NodeAttributesTests.java       |   64 +
 .../transforms/QueryConfigTests.java          |    2 +-
 .../transforms/SourceConfigTests.java         |    2 +-
 .../transforms/TimeSyncConfigTests.java       |   41 +-
 .../DataFrameTransformStateAndStatsTests.java |    2 +-
 .../hlrc/DataFrameTransformStateTests.java    |   36 +-
 .../transforms/hlrc/TimeSyncConfigTests.java  |   59 +
 .../pivot/AggregationConfigTests.java         |    2 +-
 .../pivot/DateHistogramGroupSourceTests.java  |    1 -
 .../transforms/pivot/PivotConfigTests.java    |    2 +-
 .../hlrc/DateHistogramGroupSourceTests.java   |    4 -
 .../documentation/CRUDDocumentationIT.java    |   25 +-
 .../DataFrameTransformDocumentationIT.java    |   41 +-
 .../IndicesClientDocumentationIT.java         |  136 +-
 .../LicensingDocumentationIT.java             |    4 +-
 .../MlClientDocumentationIT.java              |  570 +++-
 .../QueryDSLDocumentationTests.java           |    8 -
 .../SecurityDocumentationIT.java              |   61 +-
 .../SnapshotClientDocumentationIT.java        |    6 +-
 .../indices/AnalyzeGlobalRequestTests.java    |   58 +
 .../indices/AnalyzeIndexRequestTests.java     |   62 +
 .../client/indices/AnalyzeRequestTests.java   |   54 +
 .../client/indices/AnalyzeResponseTests.java  |  174 +
 .../indices/ReloadAnalyzersResponseTests.java |  111 +
 .../license/PutLicenseResponseTests.java      |   11 +-
 .../license/StartBasicResponseTests.java      |   11 +-
 .../ml/AucRocMetricAucRocPointTests.java      |   39 +-
 .../client/ml/AucRocMetricResultTests.java    |   63 +
 ...usionMatrixMetricConfusionMatrixTests.java |   47 +
 .../ml/ConfusionMatrixMetricResultTests.java  |   62 +
 .../DeleteDataFrameAnalyticsRequestTests.java |   39 +
 .../ml/EvaluateDataFrameResponseTests.java    |   76 +
 .../ml/FindFileStructureRequestTests.java     |    4 +
 .../ml/GetDataFrameAnalyticsRequestTests.java |   39 +
 ...etDataFrameAnalyticsStatsRequestTests.java |   39 +
 .../client/ml/PrecisionMetricResultTests.java |   60 +
 .../ml/PutDataFrameAnalyticsRequestTests.java |   74 +
 .../client/ml/RecallMetricResultTests.java    |   60 +
 .../StartDataFrameAnalyticsRequestTests.java  |   43 +
 .../StopDataFrameAnalyticsRequestTests.java   |   43 +
 .../StopDataFrameAnalyticsResponseTests.java  |   34 +-
 .../DataFrameAnalyticsConfigTests.java        |   96 +
 .../DataFrameAnalyticsDestTests.java          |   50 +
 .../DataFrameAnalyticsSourceTests.java        |   70 +
 .../DataFrameAnalyticsStatsTests.java         |   70 +
 .../ml/dataframe/OutlierDetectionTests.java   |   73 +
 .../client/ml/dataframe/QueryConfigTests.java |   62 +
 .../filestructurefinder/FieldStatsTests.java  |   10 +-
 .../ml/job/config/AnalysisConfigTests.java    |    2 +-
 .../client/ml/job/config/JobTests.java        |    2 +-
 .../ml/job/process/TimingStatsTests.java      |   97 +
 .../client/ml/job/stats/JobStatsTests.java    |    6 +-
 .../security/CreateTokenRequestTests.java     |   51 +-
 .../security/CreateTokenResponseTests.java    |    5 +
 .../watcher/ExecuteWatchResponseTests.java    |  115 -
 .../client/watcher/GetWatchResponseTests.java |  144 +-
 .../hlrc/DeleteWatchResponseTests.java        |   28 +-
 .../hlrc/ExecuteWatchResponseTests.java       |   28 +-
 .../watcher/hlrc/PutWatchResponseTests.java   |   32 +-
 client/rest/build.gradle                      |    4 +-
 .../org/elasticsearch/client/Request.java     |    4 +
 .../elasticsearch/client/RequestOptions.java  |    9 +-
 .../client/RestClientBuilderIntegTests.java   |    2 +-
 client/sniffer/build.gradle                   |    6 +-
 client/test/build.gradle                      |    2 +-
 client/transport/build.gradle                 |   57 -
 .../client/PreBuiltTransportClient.java       |  144 -
 .../client/PreBuiltTransportClientTests.java  |   66 -
 distribution/archives/build.gradle            |    8 +-
 distribution/build.gradle                     |   12 +-
 distribution/docker/build.gradle              |   11 +-
 .../src/docker/config/log4j2.properties       |  121 +
 .../src/docker/config/log4j2.properties       |    0
 distribution/src/bin/elasticsearch.bat        |    2 +-
 distribution/tools/keystore-cli/build.gradle  |    6 +-
 .../common/settings/KeyStoreWrapperTests.java |   23 +
 distribution/tools/plugin-cli/build.gradle    |    6 +-
 docs/README.asciidoc                          |    6 +-
 docs/build.gradle                             |    9 +
 docs/groovy-api/anatomy.asciidoc              |  102 -
 docs/groovy-api/client.asciidoc               |   59 -
 docs/groovy-api/delete.asciidoc               |   16 -
 docs/groovy-api/get.asciidoc                  |   19 -
 docs/groovy-api/index.asciidoc                |   48 -
 docs/groovy-api/index_.asciidoc               |   32 -
 docs/groovy-api/search.asciidoc               |  116 -
 docs/java-api/admin/cluster/health.asciidoc   |   76 -
 docs/java-api/admin/cluster/index.asciidoc    |   16 -
 .../admin/cluster/stored-scripts.asciidoc     |   29 -
 docs/java-api/admin/index.asciidoc            |   18 -
 .../admin/indices/create-index.asciidoc       |   28 -
 .../admin/indices/get-settings.asciidoc       |   22 -
 docs/java-api/admin/indices/index.asciidoc    |   21 -
 .../admin/indices/put-mapping.asciidoc        |   30 -
 docs/java-api/admin/indices/refresh.asciidoc  |   19 -
 .../admin/indices/update-settings.asciidoc    |   16 -
 docs/java-api/aggregations/bucket.asciidoc    |   33 -
 .../bucket/children-aggregation.asciidoc      |   35 -
 .../bucket/datehistogram-aggregation.asciidoc |   73 -
 .../bucket/daterange-aggregation.asciidoc     |   59 -
 .../bucket/filter-aggregation.asciidoc        |   34 -
 .../bucket/filters-aggregation.asciidoc       |   51 -
 .../bucket/geodistance-aggregation.asciidoc   |   58 -
 .../bucket/geohashgrid-aggregation.asciidoc   |   57 -
 .../bucket/global-aggregation.asciidoc        |   35 -
 .../bucket/histogram-aggregation.asciidoc     |   48 -
 .../bucket/iprange-aggregation.asciidoc       |   79 -
 .../bucket/missing-aggregation.asciidoc       |   34 -
 .../bucket/nested-aggregation.asciidoc        |   34 -
 .../bucket/range-aggregation.asciidoc         |   58 -
 .../reverse-nested-aggregation.asciidoc       |   50 -
 .../significantterms-aggregation.asciidoc     |   47 -
 .../bucket/terms-aggregation.asciidoc         |   97 -
 docs/java-api/aggregations/metrics.asciidoc   |   27 -
 .../metrics/avg-aggregation.asciidoc          |   37 -
 .../metrics/cardinality-aggregation.asciidoc  |   38 -
 .../extendedstats-aggregation.asciidoc        |   44 -
 .../metrics/geobounds-aggregation.asciidoc    |   46 -
 .../metrics/max-aggregation.asciidoc          |   37 -
 .../metrics/min-aggregation.asciidoc          |   37 -
 .../metrics/percentile-aggregation.asciidoc   |   68 -
 .../percentile-rank-aggregation.asciidoc      |   55 -
 .../scripted-metric-aggregation.asciidoc      |  100 -
 .../metrics/stats-aggregation.asciidoc        |   41 -
 .../metrics/sum-aggregation.asciidoc          |   37 -
 .../metrics/tophits-aggregation.asciidoc      |   79 -
 .../metrics/valuecount-aggregation.asciidoc   |   37 -
 docs/java-api/aggs.asciidoc                   |   63 -
 docs/java-api/client.asciidoc                 |  110 -
 docs/java-api/docs.asciidoc                   |   36 -
 docs/java-api/docs/bulk.asciidoc              |  190 --
 docs/java-api/docs/delete.asciidoc            |   42 -
 docs/java-api/docs/get.asciidoc               |   14 -
 docs/java-api/docs/index_.asciidoc            |  167 -
 docs/java-api/docs/multi-get.asciidoc         |   30 -
 docs/java-api/docs/reindex.asciidoc           |   11 -
 docs/java-api/docs/update-by-query.asciidoc   |  166 -
 docs/java-api/docs/update.asciidoc            |  118 -
 docs/java-api/index.asciidoc                  |  149 -
 docs/java-api/query-dsl.asciidoc              |   40 -
 docs/java-api/query-dsl/bool-query.asciidoc   |   13 -
 .../query-dsl/boosting-query.asciidoc         |   12 -
 .../query-dsl/common-terms-query.asciidoc     |   11 -
 .../query-dsl/compound-queries.asciidoc       |   45 -
 .../query-dsl/constant-score-query.asciidoc   |   11 -
 .../java-api/query-dsl/dis-max-query.asciidoc |   13 -
 docs/java-api/query-dsl/exists-query.asciidoc |   10 -
 .../query-dsl/full-text-queries.asciidoc      |   44 -
 .../query-dsl/function-score-query.asciidoc   |   19 -
 docs/java-api/query-dsl/fuzzy-query.asciidoc  |   11 -
 .../query-dsl/geo-bounding-box-query.asciidoc |   12 -
 .../query-dsl/geo-distance-query.asciidoc     |   12 -
 .../query-dsl/geo-polygon-query.asciidoc      |   11 -
 docs/java-api/query-dsl/geo-queries.asciidoc  |   34 -
 .../query-dsl/geo-shape-query.asciidoc        |   56 -
 .../query-dsl/has-child-query.asciidoc        |   23 -
 .../query-dsl/has-parent-query.asciidoc       |   23 -
 docs/java-api/query-dsl/ids-query.asciidoc    |   10 -
 .../query-dsl/joining-queries.asciidoc        |   28 -
 .../query-dsl/match-all-query.asciidoc        |    9 -
 docs/java-api/query-dsl/match-query.asciidoc  |   11 -
 docs/java-api/query-dsl/mlt-query.asciidoc    |   13 -
 .../query-dsl/multi-match-query.asciidoc      |   11 -
 docs/java-api/query-dsl/nested-query.asciidoc |   12 -
 .../query-dsl/percolate-query.asciidoc        |   61 -
 docs/java-api/query-dsl/prefix-query.asciidoc |   11 -
 .../query-dsl/query-string-query.asciidoc     |    9 -
 docs/java-api/query-dsl/range-query.asciidoc  |   22 -
 docs/java-api/query-dsl/regexp-query.asciidoc |   11 -
 docs/java-api/query-dsl/script-query.asciidoc |   29 -
 .../simple-query-string-query.asciidoc        |    9 -
 .../query-dsl/span-containing-query.asciidoc  |   11 -
 .../query-dsl/span-first-query.asciidoc       |   11 -
 .../query-dsl/span-multi-term-query.asciidoc  |   11 -
 .../query-dsl/span-near-query.asciidoc        |   12 -
 .../query-dsl/span-not-query.asciidoc         |   11 -
 .../java-api/query-dsl/span-or-query.asciidoc |   10 -
 docs/java-api/query-dsl/span-queries.asciidoc |   65 -
 .../query-dsl/span-term-query.asciidoc        |   11 -
 .../query-dsl/span-within-query.asciidoc      |   11 -
 .../query-dsl/special-queries.asciidoc        |   31 -
 .../query-dsl/term-level-queries.asciidoc     |   77 -
 docs/java-api/query-dsl/term-query.asciidoc   |   11 -
 docs/java-api/query-dsl/terms-query.asciidoc  |   11 -
 .../query-dsl/wildcard-query.asciidoc         |   11 -
 .../java-api/query-dsl/wrapper-query.asciidoc |   11 -
 docs/java-api/search.asciidoc                 |  250 --
 .../dataframe/get_data_frame.asciidoc         |    1 +
 .../dataframe/get_data_frame_stats.asciidoc   |   16 +-
 .../dataframe/put_data_frame.asciidoc         |   12 +-
 .../dataframe/stop_data_frame.asciidoc        |    1 +
 .../document/delete-by-query.asciidoc         |    4 +-
 .../high-level/document/reindex.asciidoc      |    6 +-
 .../high-level/document/term-vectors.asciidoc |    7 +-
 .../document/update-by-query.asciidoc         |    4 +-
 .../high-level/indices/analyze.asciidoc       |   10 +-
 .../indices/reload_analyzers.asciidoc         |   50 +
 .../ml/delete-data-frame-analytics.asciidoc   |   28 +
 .../ml/evaluate-data-frame.asciidoc           |   45 +
 .../get-data-frame-analytics-stats.asciidoc   |   34 +
 .../ml/get-data-frame-analytics.asciidoc      |   34 +
 .../ml/put-data-frame-analytics.asciidoc      |  115 +
 .../ml/start-data-frame-analytics.asciidoc    |   28 +
 .../ml/stop-data-frame-analytics.asciidoc     |   29 +
 .../high-level/query-builders.asciidoc        |    3 +-
 .../security/get-builtin-privileges.asciidoc  |   27 +
 .../high-level/supported-apis.asciidoc        |   19 +-
 .../packages.asciidoc                         |    6 +
 .../packages.asciidoc                         |   48 +
 docs/painless/painless-guide/index.asciidoc   |    4 +
 .../painless-guide/painless-datetime.asciidoc |  720 ++++
 docs/painless/redirects.asciidoc              |    9 +
 docs/plugins/analysis-phonetic.asciidoc       |    8 +
 docs/plugins/analysis-smartcn.asciidoc        |  412 ++-
 docs/plugins/analysis-stempel.asciidoc        |  104 +-
 docs/plugins/api.asciidoc                     |    3 -
 docs/plugins/repository-azure.asciidoc        |   10 +-
 docs/reference/administering.asciidoc         |   25 +
 docs/reference/aggregations/bucket.asciidoc   |    3 +
 .../autodatehistogram-aggregation.asciidoc    |   33 +
 .../bucket/rare-terms-aggregation.asciidoc    |  357 ++
 .../tokenfilters/elision-tokenfilter.asciidoc |    9 +-
 .../tokenfilters/synonym-tokenfilter.asciidoc |    2 +
 .../word-delimiter-tokenfilter.asciidoc       |    2 +-
 docs/reference/analysis/tokenizers.asciidoc   |    4 +
 .../pathhierarchy-tokenizer-examples.asciidoc |  191 ++
 .../pathhierarchy-tokenizer.asciidoc          |    4 +
 docs/reference/api-conventions.asciidoc       |    4 +
 docs/reference/cat.asciidoc                   |    6 +-
 docs/reference/cat/alias.asciidoc             |    2 +-
 docs/reference/cat/allocation.asciidoc        |    2 +-
 docs/reference/cat/count.asciidoc             |    4 +-
 docs/reference/cat/fielddata.asciidoc         |    6 +-
 docs/reference/cat/health.asciidoc            |    4 +-
 docs/reference/cat/indices.asciidoc           |   10 +-
 docs/reference/cat/master.asciidoc            |    2 +-
 docs/reference/cat/nodeattrs.asciidoc         |    4 +-
 docs/reference/cat/nodes.asciidoc             |    6 +-
 docs/reference/cat/pending_tasks.asciidoc     |    2 +-
 docs/reference/cat/plugins.asciidoc           |    2 +-
 docs/reference/cat/recovery.asciidoc          |    6 +-
 docs/reference/cat/repositories.asciidoc      |    2 +-
 docs/reference/cat/segments.asciidoc          |    2 +-
 docs/reference/cat/shards.asciidoc            |   10 +-
 docs/reference/cat/snapshots.asciidoc         |   25 +-
 docs/reference/cat/templates.asciidoc         |    2 +-
 docs/reference/cat/thread_pool.asciidoc       |    4 +-
 .../ccr/apis/follow-request-body.asciidoc     |    1 -
 .../ccr/apis/follow/get-follow-stats.asciidoc |    5 +
 .../reference/ccr/apis/get-ccr-stats.asciidoc |    2 +
 docs/reference/ccr/overview.asciidoc          |    7 +-
 docs/reference/cluster.asciidoc               |   18 +-
 docs/reference/cluster/stats.asciidoc         |    9 +-
 docs/reference/commands/certutil.asciidoc     |    2 +-
 docs/reference/commands/shard-tool.asciidoc   |   10 +-
 .../apis/delete-transform.asciidoc            |   35 +-
 .../apis/get-transform-stats.asciidoc         |   92 +-
 .../data-frames/apis/get-transform.asciidoc   |  101 +-
 .../reference/data-frames/apis/index.asciidoc |    8 +-
 .../data-frames/apis/pivotresource.asciidoc   |   26 +
 .../apis/preview-transform.asciidoc           |   50 +-
 .../data-frames/apis/put-transform.asciidoc   |   68 +-
 .../data-frames/apis/start-transform.asciidoc |   34 +-
 .../data-frames/apis/stop-transform.asciidoc  |   96 +-
 docs/reference/docs/data-replication.asciidoc |   24 +-
 docs/reference/docs/delete-by-query.asciidoc  |   10 +-
 docs/reference/docs/reindex.asciidoc          |   20 +-
 docs/reference/docs/termvectors.asciidoc      |    3 -
 docs/reference/docs/update-by-query.asciidoc  |   10 +-
 docs/reference/frozen-indices.asciidoc        |    2 +-
 docs/reference/getting-started.asciidoc       |  136 +-
 .../how-to/recipes/stemming.asciidoc          |    3 +
 .../reference/ilm/policy-definitions.asciidoc |    6 +-
 .../images/rare_terms/accuracy_0001.png       |  Bin 0 -> 25315 bytes
 .../images/rare_terms/accuracy_001.png        |  Bin 0 -> 27086 bytes
 .../images/rare_terms/accuracy_01.png         |  Bin 0 -> 25075 bytes
 docs/reference/images/rare_terms/memory.png   |  Bin 0 -> 20465 bytes
 docs/reference/index-modules.asciidoc         |    3 +-
 docs/reference/index-modules/slowlog.asciidoc |   39 +-
 docs/reference/index.asciidoc                 |    8 +-
 .../indices/apis/reload-analyzers.asciidoc    |  105 +
 docs/reference/indices/create-index.asciidoc  |    2 +-
 docs/reference/indices/open-close.asciidoc    |   64 +-
 docs/reference/indices/recovery.asciidoc      |    2 +-
 docs/reference/indices/shrink-index.asciidoc  |    5 +-
 docs/reference/ingest.asciidoc                |   47 +-
 .../processors/date-index-name.asciidoc       |    2 +-
 .../ingest/processors/geoip.asciidoc          |    2 +-
 docs/reference/ingest/processors/set.asciidoc |    2 +-
 docs/reference/intro.asciidoc                 |  270 ++
 docs/reference/mapping.asciidoc               |   10 +-
 docs/reference/mapping/params/coerce.asciidoc |    3 +-
 .../reference/mapping/params/enabled.asciidoc |    4 +-
 .../mapping/params/ignore-above.asciidoc      |    2 +-
 .../mapping/params/ignore-malformed.asciidoc  |    3 +-
 .../mapping/params/multi-fields.asciidoc      |    3 +-
 .../mapping/params/normalizer.asciidoc        |    2 +-
 docs/reference/mapping/params/norms.asciidoc  |    5 +-
 .../mapping/removal_of_types.asciidoc         |   14 +-
 docs/reference/mapping/types.asciidoc         |    9 +-
 docs/reference/mapping/types/alias.asciidoc   |    2 +-
 .../mapping/types/dense-vector.asciidoc       |   16 +-
 .../mapping/types/flattened.asciidoc          |  188 ++
 .../mapping/types/geo-shape.asciidoc          |    2 +
 docs/reference/mapping/types/nested.asciidoc  |   59 +-
 .../mapping/types/sparse-vector.asciidoc      |    2 +
 docs/reference/migration/migrate_8_0.asciidoc |    4 +
 .../migration/migrate_8_0/mappings.asciidoc   |    9 +
 .../migration/migrate_8_0/node.asciidoc       |   22 +
 .../migration/migrate_8_0/reindex.asciidoc    |   16 +-
 .../migration/migrate_8_0/rollup.asciidoc     |   20 +
 .../migration/migrate_8_0/search.asciidoc     |   19 +-
 .../migration/migrate_8_0/security.asciidoc   |    8 +
 .../migration/migrate_8_0/snapshots.asciidoc  |   53 +
 docs/reference/ml/apis/close-job.asciidoc     |   37 +-
 .../ml/apis/datafeedresource.asciidoc         |   30 +-
 .../ml/apis/delete-calendar-event.asciidoc    |   29 +-
 .../ml/apis/delete-calendar-job.asciidoc      |   30 +-
 .../ml/apis/delete-calendar.asciidoc          |   29 +-
 .../ml/apis/delete-datafeed.asciidoc          |   43 +-
 .../ml/apis/delete-expired-data.asciidoc      |   24 +-
 docs/reference/ml/apis/delete-filter.asciidoc |   29 +-
 .../ml/apis/delete-forecast.asciidoc          |   45 +-
 docs/reference/ml/apis/delete-job.asciidoc    |   37 +-
 .../ml/apis/delete-snapshot.asciidoc          |   34 +-
 .../ml/apis/find-file-structure.asciidoc      |   70 +-
 docs/reference/ml/apis/flush-job.asciidoc     |   45 +-
 docs/reference/ml/apis/forecast.asciidoc      |   36 +-
 docs/reference/ml/apis/get-bucket.asciidoc    |   65 +-
 .../ml/apis/get-calendar-event.asciidoc       |   43 +-
 docs/reference/ml/apis/get-calendar.asciidoc  |   42 +-
 docs/reference/ml/apis/get-category.asciidoc  |   52 +-
 .../ml/apis/get-datafeed-stats.asciidoc       |   35 +-
 docs/reference/ml/apis/get-datafeed.asciidoc  |   35 +-
 docs/reference/ml/apis/get-filter.asciidoc    |   42 +-
 .../reference/ml/apis/get-influencer.asciidoc |   55 +-
 docs/reference/ml/apis/get-job-stats.asciidoc |   47 +-
 docs/reference/ml/apis/get-job.asciidoc       |   33 +-
 docs/reference/ml/apis/get-ml-info.asciidoc   |   34 +-
 .../ml/apis/get-overall-buckets.asciidoc      |   59 +-
 docs/reference/ml/apis/get-record.asciidoc    |   56 +-
 docs/reference/ml/apis/get-snapshot.asciidoc  |   49 +-
 docs/reference/ml/apis/jobcounts.asciidoc     |   33 +-
 docs/reference/ml/apis/open-job.asciidoc      |   43 +-
 .../ml/apis/post-calendar-event.asciidoc      |   42 +-
 docs/reference/ml/apis/post-data.asciidoc     |   56 +-
 .../ml/apis/preview-datafeed.asciidoc         |   48 +-
 .../ml/apis/put-calendar-job.asciidoc         |   30 +-
 docs/reference/ml/apis/put-calendar.asciidoc  |   37 +-
 docs/reference/ml/apis/put-datafeed.asciidoc  |   89 +-
 docs/reference/ml/apis/put-filter.asciidoc    |   37 +-
 docs/reference/ml/apis/put-job.asciidoc       |   57 +-
 .../ml/apis/revert-snapshot.asciidoc          |   40 +-
 .../ml/apis/set-upgrade-mode.asciidoc         |   31 +-
 .../ml/apis/snapshotresource.asciidoc         |    2 +-
 .../reference/ml/apis/start-datafeed.asciidoc |   50 +-
 docs/reference/ml/apis/stop-datafeed.asciidoc |   43 +-
 .../ml/apis/update-datafeed.asciidoc          |   70 +-
 docs/reference/ml/apis/update-filter.asciidoc |   39 +-
 docs/reference/ml/apis/update-job.asciidoc    |   30 +-
 .../ml/apis/update-snapshot.asciidoc          |   42 +-
 .../ml/apis/validate-detector.asciidoc        |   28 +-
 docs/reference/ml/apis/validate-job.asciidoc  |   28 +-
 docs/reference/ml/configuring.asciidoc        |    2 +-
 docs/reference/ml/functions/geo.asciidoc      |   38 +-
 docs/reference/modules.asciidoc               |   12 -
 .../cluster/allocation_awareness.asciidoc     |    2 +-
 .../discovery/adding-removing-nodes.asciidoc  |   32 +-
 .../modules/discovery/bootstrapping.asciidoc  |   44 +-
 .../discovery/discovery-settings.asciidoc     |   19 +-
 .../modules/indices/search-settings.asciidoc  |    6 +-
 docs/reference/modules/memcached.asciidoc     |   69 -
 docs/reference/modules/ml-node.asciidoc       |  116 -
 docs/reference/modules/network.asciidoc       |   11 +-
 docs/reference/modules/node.asciidoc          |  232 +-
 .../modules/remote-clusters.asciidoc          |   35 +-
 docs/reference/modules/snapshots.asciidoc     |   59 +-
 docs/reference/modules/thrift.asciidoc        |   25 -
 .../monitoring/configuring-filebeat.asciidoc  |  187 ++
 .../configuring-metricbeat.asciidoc           |  198 +-
 .../configuring-monitoring.asciidoc           |    3 +
 .../query-dsl/boosting-query.asciidoc         |   52 +-
 .../query-dsl/common-terms-query.asciidoc     |  299 --
 .../query-dsl/constant-score-query.asciidoc   |   28 +-
 .../query-dsl/dis-max-query.asciidoc          |   87 +-
 .../reference/query-dsl/exists-query.asciidoc |   21 +-
 .../query-dsl/full-text-queries.asciidoc      |    6 -
 .../query-dsl/geo-bounding-box-query.asciidoc |   18 +-
 .../query-dsl/intervals-query.asciidoc        |   50 +-
 docs/reference/query-dsl/match-query.asciidoc |   47 +-
 docs/reference/query-dsl/mlt-query.asciidoc   |    5 +-
 .../query-dsl/multi-match-query.asciidoc      |   20 +-
 .../query-dsl/query_filter_context.asciidoc   |   19 +-
 docs/reference/query-dsl/range-query.asciidoc |  268 +-
 .../query-dsl/regexp-syntax.asciidoc          |    2 +-
 .../query-dsl/script-score-query.asciidoc     |    5 +-
 .../query-dsl/term-level-queries.asciidoc     |   73 +-
 docs/reference/query-dsl/terms-query.asciidoc |  291 +-
 .../query-dsl/terms-set-query.asciidoc        |  248 +-
 .../query-dsl/wildcard-query.asciidoc         |   86 +-
 docs/reference/redirects.asciidoc             |   20 +-
 docs/reference/rest-api/defs.asciidoc         |    2 +
 docs/reference/rest-api/index.asciidoc        |   15 +-
 docs/reference/rest-api/info.asciidoc         |   29 +-
 docs/reference/rollup/api-quickref.asciidoc   |    2 +-
 docs/reference/rollup/rollup-api.asciidoc     |    2 +-
 .../{modules => }/scripting.asciidoc          |   16 +-
 .../{modules => }/scripting/engine.asciidoc   |    2 +-
 .../scripting/expression.asciidoc             |    2 +-
 .../{modules => }/scripting/fields.asciidoc   |    8 +-
 .../{modules => }/scripting/painless.asciidoc |    2 +-
 .../{modules => }/scripting/security.asciidoc |    2 +-
 .../{modules => }/scripting/using.asciidoc    |   13 +-
 .../search/request/docvalue-fields.asciidoc   |    4 +
 .../search/request/preference.asciidoc        |   12 +-
 .../search/request/script-fields.asciidoc     |    2 +-
 docs/reference/search/request/sort.asciidoc   |   16 +-
 .../search/request/stored-fields.asciidoc     |    5 +
 .../search/suggesters/phrase-suggest.asciidoc |   31 +-
 .../configuring-tls-docker.asciidoc           |   56 +-
 .../separating-node-client-traffic.asciidoc   |   68 -
 docs/reference/settings/ml-settings.asciidoc  |    8 +
 .../settings/monitoring-settings.asciidoc     |    1 +
 .../settings/notification-settings.asciidoc   |    7 +-
 .../settings/security-settings.asciidoc       |    2 +
 docs/reference/settings/ssl-settings.asciidoc |    4 +-
 .../discovery-settings.asciidoc               |    6 +-
 docs/reference/setup/install.asciidoc         |    8 +
 docs/reference/setup/install/brew.asciidoc    |   69 +
 docs/reference/setup/install/docker.asciidoc  |    8 +-
 docs/reference/setup/secure-settings.asciidoc |    8 +-
 docs/reference/sql/endpoints/jdbc.asciidoc    |    5 +
 docs/reference/sql/endpoints/rest.asciidoc    |  156 +-
 .../sql/functions/conditional.asciidoc        |   42 +-
 .../sql/functions/date-time.asciidoc          |   80 +-
 docs/reference/sql/functions/geo.asciidoc     |    4 +-
 .../reference/sql/functions/grouping.asciidoc |   10 +-
 .../sql/functions/like-rlike.asciidoc         |    4 +-
 .../sql/functions/operators.asciidoc          |   36 +-
 docs/reference/sql/functions/search.asciidoc  |   25 +-
 docs/reference/sql/functions/string.asciidoc  |   42 +-
 docs/reference/sql/functions/system.asciidoc  |    4 +-
 .../sql/functions/type-conversion.asciidoc    |   10 +-
 docs/reference/sql/getting-started.asciidoc   |    2 +-
 .../sql/language/data-types.asciidoc          |    4 +-
 docs/reference/sql/language/indices.asciidoc  |   20 +-
 .../syntax/commands/describe-table.asciidoc   |    2 +-
 .../language/syntax/commands/index.asciidoc   |    2 +-
 .../language/syntax/commands/select.asciidoc  |   64 +-
 .../syntax/commands/show-columns.asciidoc     |    2 +-
 .../syntax/commands/show-functions.asciidoc   |   10 +-
 .../syntax/commands/show-tables.asciidoc      |   12 +-
 .../sql/language/syntax/lexic/index.asciidoc  |   29 +-
 docs/reference/sql/limitations.asciidoc       |    4 +-
 docs/reference/sql/security.asciidoc          |    2 +-
 docs/reference/upgrade.asciidoc               |    4 +-
 .../upgrade/cluster_restart.asciidoc          |    2 +-
 .../upgrade/reindex_upgrade.asciidoc          |    6 +-
 .../upgrade/rolling_upgrade.asciidoc          |    4 +-
 gradle.properties                             |    2 +-
 gradle/wrapper/gradle-wrapper.properties      |    4 +-
 gradlew                                       |    2 +-
 gradlew.bat                                   |    2 +-
 libs/build.gradle                             |   10 +-
 libs/cli/build.gradle                         |    2 +-
 libs/core/build.gradle                        |   18 +-
 .../elasticsearch/common/unit/TimeValue.java  |  128 +-
 .../core/internal/io/IOUtils.java             |   32 +-
 libs/core/src/test/eclipse-build.gradle       |    2 +-
 .../common/unit/TimeValueTests.java           |   20 +
 .../core/internal/io/IOUtilsTests.java        |   44 +
 libs/dissect/build.gradle                     |   14 +-
 libs/dissect/src/test/eclipse-build.gradle    |    2 +-
 libs/geo/build.gradle                         |    6 +-
 .../elasticsearch/geo/geometry/Circle.java    |    2 -
 .../geo/geometry/GeometryUtils.java           |   70 -
 .../org/elasticsearch/geo/geometry/Line.java  |    4 -
 .../org/elasticsearch/geo/geometry/Point.java |    2 -
 .../elasticsearch/geo/geometry/Rectangle.java |   30 -
 .../geo/utils/GeographyValidator.java         |  178 +
 .../geo/utils/GeometryValidator.java          |   34 +
 .../geo/utils/WellKnownText.java              |   92 +-
 .../geo/geometry/BaseGeometryTestCase.java    |    6 +-
 .../geo/geometry/CircleTests.java             |   22 +-
 .../geo/geometry/GeometryCollectionTests.java |   10 +-
 .../geo/geometry/GeometryValidatorTests.java  |  127 +
 .../elasticsearch/geo/geometry/LineTests.java |   31 +-
 .../geo/geometry/LinearRingTests.java         |   19 +-
 .../geo/geometry/MultiLineTests.java          |   10 +-
 .../geo/geometry/MultiPointTests.java         |   18 +-
 .../geo/geometry/MultiPolygonTests.java       |   10 +-
 .../geo/geometry/PointTests.java              |   26 +-
 .../geo/geometry/PolygonTests.java            |   40 +-
 .../geo/geometry/RectangleTests.java          |   24 +-
 libs/grok/build.gradle                        |   12 +-
 libs/grok/licenses/jcodings-1.0.12.jar.sha1   |    1 -
 libs/grok/licenses/jcodings-1.0.44.jar.sha1   |    1 +
 libs/grok/src/test/eclipse-build.gradle       |    2 +-
 libs/nio/build.gradle                         |   18 +-
 .../nio/BytesChannelContext.java              |   11 +-
 .../elasticsearch/nio/BytesWriteHandler.java  |    7 +-
 .../org/elasticsearch/nio/ChannelFactory.java |    2 +
 .../elasticsearch/nio/DelegatingHandler.java  |   68 +
 ...iteHandler.java => NioChannelHandler.java} |   11 +-
 .../org/elasticsearch/nio/NioSelector.java    |   99 +-
 .../nio/SocketChannelContext.java             |   25 +-
 .../org/elasticsearch/nio/WriteOperation.java |    2 +-
 libs/nio/src/test/eclipse-build.gradle        |    2 +-
 .../elasticsearch/nio/EventHandlerTests.java  |    6 +-
 .../elasticsearch/nio/NioSelectorTests.java   |   75 +-
 .../nio/SocketChannelContextTests.java        |   33 +-
 libs/secure-sm/build.gradle                   |   18 +-
 libs/secure-sm/src/test/eclipse-build.gradle  |    2 +-
 libs/ssl-config/build.gradle                  |    8 +-
 .../common/ssl/PemTrustConfigTests.java       |   27 +-
 libs/x-content/build.gradle                   |   18 +-
 .../common/xcontent/ObjectParser.java         |  103 +-
 .../common/xcontent/XContentParser.java       |    6 +-
 .../common/xcontent/XContentSubParser.java    |    5 -
 .../xcontent/json/JsonXContentParser.java     |    4 +
 .../support/AbstractXContentParser.java       |   19 +-
 libs/x-content/src/test/eclipse-build.gradle  |    2 +-
 .../common/xcontent/ObjectParserTests.java    |   39 +
 .../common/xcontent/XContentParserTests.java  |    2 +-
 modules/aggs-matrix-stats/build.gradle        |    1 -
 .../matrix/stats/InternalMatrixStats.java     |   10 +-
 .../stats/MatrixStatsAggregationBuilder.java  |   12 +-
 .../stats/MatrixStatsAggregatorFactory.java   |    5 +-
 .../ArrayValuesSourceAggregationBuilder.java  |   41 +-
 .../ArrayValuesSourceAggregatorFactory.java   |    6 +-
 .../analysis/common/CommonAnalysisPlugin.java |   35 +-
 .../common/ElisionTokenFilterFactory.java     |    3 +
 .../common/StandardHtmlStripAnalyzer.java     |   58 -
 .../common/SynonymTokenFilterFactory.java     |   13 +
 .../common/EdgeNGramTokenizerTests.java       |   98 +
 .../common/ElisionFilterFactoryTests.java     |   43 +
 .../common/HighlighterWithAnalyzersTests.java |    2 +-
 ...DelimiterGraphTokenFilterFactoryTests.java |   57 +
 .../test/analysis-common/20_analyzers.yml     |    9 -
 .../test/analysis-common/40_token_filters.yml |   14 +
 .../search.query/50_queries_with_synonyms.yml |  116 +-
 modules/ingest-common/build.gradle            |    4 +-
 .../ingest/common/GrokProcessorGetAction.java |    4 +-
 .../ingest/common/RenameProcessorTests.java   |    6 +-
 .../test/ingest/200_default_pipeline.yml      |   23 +
 .../ExpressionAggregationScript.java          |   27 +-
 .../expression/ExpressionScriptEngine.java    |    9 +-
 .../ExpressionFieldScriptTests.java           |    2 +-
 .../ExpressionNumberSortScriptTests.java      |    9 +-
 .../ExpressionTermsSetQueryTests.java         |    9 +-
 .../expression/MoreExpressionTests.java       |   13 +-
 .../expression/StoredExpressionTests.java     |    2 +-
 .../test/lang_expression/20_search.yml        |   20 +-
 modules/lang-mustache/build.gradle            |    1 -
 .../mustache/MultiSearchTemplateAction.java   |    9 +-
 .../script/mustache/SearchTemplateAction.java |    9 +-
 .../MultiSearchTemplateRequestTests.java      |    3 -
 .../script/mustache/SearchTemplateIT.java     |   14 +-
 .../mustache/SearchTemplateRequestTests.java  |    4 +-
 .../mustache/simple-msearch-template.json     |    4 +-
 .../test/lang_mustache/30_search_template.yml |    3 +-
 modules/lang-painless/build.gradle            |    2 +-
 modules/lang-painless/spi/build.gradle        |    2 +-
 .../elasticsearch/painless/spi/Whitelist.java |    5 +-
 .../painless/spi/WhitelistClass.java          |   26 +-
 .../painless/spi/WhitelistClassBinding.java   |   18 +-
 .../painless/spi/WhitelistConstructor.java    |   16 +-
 .../painless/spi/WhitelistField.java          |   18 +-
 .../spi/WhitelistInstanceBinding.java         |   18 +-
 .../painless/spi/WhitelistLoader.java         |  239 +-
 .../painless/spi/WhitelistMethod.java         |   17 +-
 .../spi/annotation/DeprecatedAnnotation.java  |   19 +-
 .../DeprecatedAnnotationParser.java           |   44 +
 .../spi/annotation/NoImportAnnotation.java}   |   13 +-
 .../annotation/NoImportAnnotationParser.java  |   21 +-
 .../annotation/WhitelistAnnotationParser.java |   42 +
 .../painless/ContextDocGenerator.java         |    1 +
 .../painless/PainlessPlugin.java              |   19 +-
 .../action/PainlessContextAction.java         |    9 +-
 .../action/PainlessExecuteAction.java         |   48 +-
 .../painless/api/Augmentation.java            |  113 +
 .../lookup/PainlessLookupBuilder.java         |    4 +-
 .../elasticsearch/painless/spi/java.util.txt  |    4 +
 .../painless/spi/org.elasticsearch.score.txt  |   11 +-
 .../painless/spi/org.elasticsearch.txt        |   18 +-
 .../painless/AnnotationTestObject.java        |  102 +
 .../painless/AugmentationTests.java           |    1 -
 .../elasticsearch/painless/BindingsTests.java |    4 +-
 .../elasticsearch/painless/DateTimeTests.java |  194 ++
 .../painless/GetByPathAugmentationTests.java  |  259 ++
 .../painless/NeedsScoreTests.java             |    2 +-
 .../painless/PainlessDocGenerator.java        |  451 ---
 .../painless/WhitelistLoaderTests.java        |   85 +
 .../action/PainlessExecuteRequestTests.java   |   13 +-
 .../spi/org.elasticsearch.painless.annotation |    7 +
 .../spi/org.elasticsearch.painless.test       |    2 +-
 .../api/scripts_painless_context.json         |    2 +-
 .../test/painless/100_terms_agg.yml           |  104 +
 .../test/painless/71_context_api.yml          |    4 +-
 .../index/mapper/MapperExtrasPlugin.java      |    2 -
 ...asticsearch.painless.spi.PainlessExtension |    1 -
 .../index/query/docvalues_whitelist.txt       |   32 -
 .../mapper/DenseVectorFieldMapperTests.java   |  105 -
 .../TokenCountFieldMapperIntegrationIT.java   |    3 +-
 modules/parent-join/build.gradle              |    1 -
 .../ChildrenAggregationBuilder.java           |   13 +-
 .../ChildrenAggregatorFactory.java            |    4 +-
 .../ParentAggregationBuilder.java             |   13 +-
 .../aggregations/ParentAggregatorFactory.java |    4 +-
 .../join/query/HasChildQueryBuilderTests.java |    5 +-
 .../query/HasParentQueryBuilderTests.java     |    4 -
 modules/percolator/build.gradle               |    1 -
 .../percolator/PercolateQueryBuilder.java     |   21 +-
 .../percolator/QueryAnalyzer.java             |    9 -
 .../percolator/CandidateQueryTests.java       |    9 +-
 .../PercolateQueryBuilderTests.java           |   38 +-
 .../percolator/PercolatorQuerySearchIT.java   |   14 +-
 .../percolator/QueryAnalyzerTests.java        |   24 +-
 .../percolator/QueryBuilderStoreTests.java    |    4 +-
 modules/rank-eval/build.gradle                |    1 -
 .../index/rankeval/RankEvalAction.java        |    6 +-
 .../rankeval/RankEvalRequestBuilder.java      |    4 +-
 .../index/rankeval/RatedRequestsTests.java    |    2 +-
 modules/reindex/build.gradle                  |    9 +-
 .../AbstractAsyncBulkByScrollAction.java      |   14 +-
 .../AbstractBaseReindexRestHandler.java       |   18 +-
 .../AbstractBulkByQueryRestHandler.java       |   10 +-
 .../BulkByScrollParallelizationHelper.java    |   10 +-
 .../reindex/RestDeleteByQueryAction.java      |    1 +
 .../index/reindex/RestReindexAction.java      |  184 +-
 .../reindex/RestUpdateByQueryAction.java      |    1 +
 .../index/reindex/RethrottleAction.java       |    9 +-
 .../reindex/RethrottleRequestBuilder.java     |    4 +-
 .../reindex/remote/RemoteRequestBuilders.java |   24 +-
 .../reindex/remote/RemoteResponseParsers.java |    2 +-
 .../documentation/ReindexDocumentationIT.java |    4 +-
 .../reindex/AsyncBulkByScrollActionTests.java |    4 +-
 .../index/reindex/CancelTests.java            |    6 +-
 .../reindex/DeleteByQueryBasicTests.java      |   27 +-
 .../index/reindex/ManyDocumentsIT.java        |   39 +-
 .../index/reindex/ReindexBasicTests.java      |   18 +-
 .../index/reindex/ReindexFailureTests.java    |    4 +-
 ...ReindexFromRemoteBuildRestClientTests.java |    9 +-
 .../ReindexFromRemoteWhitelistTests.java      |    8 +-
 .../reindex/ReindexRestClientSslTests.java    |    5 +-
 .../ReindexSourceTargetValidationTests.java   |    7 +-
 .../index/reindex/RestReindexActionTests.java |  158 +-
 .../index/reindex/RethrottleTests.java        |    2 -
 .../index/reindex/RoundTripTests.java         |    4 +-
 .../reindex/UpdateByQueryBasicTests.java      |   15 +-
 .../remote/ReindexFromOldRemoteIT.java        |   43 +-
 .../index/reindex/remote/RemoteInfoTests.java |   16 +-
 .../remote/RemoteRequestBuildersTests.java    |   66 +-
 .../remote/RemoteResponseParsersTests.java    |   52 +
 .../RemoteScrollableHitSourceTests.java       |   49 +-
 .../test/delete_by_query/10_basic.yml         |   93 +-
 .../test/delete_by_query/20_validation.yml    |   10 +-
 .../test/reindex/20_validation.yml            |   75 +-
 .../rest-api-spec/test/reindex/30_search.yml  |   84 +-
 .../rest-api-spec/test/reindex/90_remote.yml  |    8 +-
 .../test/update_by_query/10_basic.yml         |   70 +-
 .../test/update_by_query/20_validation.yml    |   49 +-
 .../blobstore/url/URLBlobContainer.java       |   11 +
 .../repositories/url/URLRepository.java       |   10 +-
 .../url/URLSnapshotRestoreTests.java          |   10 +-
 .../test/repository_url/10_basic.yml          |   25 +-
 modules/transport-netty4/build.gradle         |    5 +-
 .../netty4/Netty4HttpServerTransport.java     |   65 +-
 .../http/netty4/cors/Netty4CorsConfig.java    |  221 --
 .../netty4/cors/Netty4CorsConfigBuilder.java  |  265 --
 .../http/netty4/cors/Netty4CorsHandler.java   |   18 +-
 .../elasticsearch/transport/Netty4Plugin.java |    5 -
 .../netty4/Netty4InternalESLogger.java        |  187 --
 .../transport/netty4/Netty4Transport.java     |    8 +-
 .../transport/netty4/Netty4Utils.java         |   17 -
 .../http/netty4/Netty4CorsTests.java          |    3 +-
 .../netty4/Netty4HttpRequestSizeLimitIT.java  |    2 +-
 .../Netty4HttpServerTransportTests.java       |   59 +-
 .../rest/discovery/Zen2RestApiIT.java         |    2 +-
 ...Netty4TransportMultiPortIntegrationIT.java |   24 -
 plugins/analysis-icu/build.gradle             |    1 -
 .../ICUCollationKeywordFieldMapperIT.java     |   24 +-
 .../SmartChineseStopTokenFilterFactory.java   |   61 +
 .../smartcn/AnalysisSmartChinesePlugin.java   |   10 +-
 .../pl/PolishStopTokenFilterFactory.java      |   73 +
 .../stempel/AnalysisStempelPlugin.java        |    4 +-
 plugins/discovery-azure-classic/build.gradle  |    1 +
 .../licenses/log4j-1.2-api-2.11.1.jar.sha1    |    0
 .../licenses/log4j-LICENSE.txt                |    0
 .../licenses/log4j-NOTICE.txt                 |    0
 .../AzureDiscoveryClusterFormationTests.java  |   14 +-
 plugins/discovery-ec2/build.gradle            |    3 +-
 .../aws-java-sdk-core-1.11.505.jar.sha1       |    1 -
 .../aws-java-sdk-core-1.11.562.jar.sha1       |    1 +
 .../aws-java-sdk-ec2-1.11.505.jar.sha1        |    1 -
 .../aws-java-sdk-ec2-1.11.562.jar.sha1        |    1 +
 .../licenses/log4j-1.2-api-2.11.1.jar.sha1    |    1 +
 .../discovery-ec2/licenses/log4j-LICENSE.txt  |  202 ++
 .../discovery-ec2/licenses/log4j-NOTICE.txt   |    5 +
 .../discovery-ec2/qa/amazon-ec2/build.gradle  |    8 +-
 .../discovery/ec2/AmazonEC2Mock.java          | 2904 +----------------
 plugins/discovery-gce/build.gradle            |    1 +
 .../licenses/log4j-1.2-api-2.11.1.jar.sha1    |    1 +
 .../discovery-gce/licenses/log4j-LICENSE.txt  |  202 ++
 .../discovery-gce/licenses/log4j-NOTICE.txt   |    5 +
 plugins/discovery-gce/qa/gce/build.gradle     |    9 +-
 plugins/examples/build.gradle                 |   19 +
 .../customsuggester/CustomSuggester.java      |   22 +-
 .../ExamplePainlessAnnotation.java            |   23 +-
 .../ExampleWhitelistAnnotationParser.java     |   62 +
 .../ExampleWhitelistExtension.java            |   16 +-
 .../ExampleWhitelistedClass.java              |    5 +
 .../painlesswhitelist/example_whitelist.txt   |    3 +
 .../rest-api-spec/api/cat.example.json        |    2 +-
 .../AnnotatedTextFieldMapperTests.java        |    2 +-
 plugins/repository-azure/build.gradle         |   23 +
 .../qa/microsoft-azure-storage/build.gradle   |   21 +-
 .../test/repository_azure/10_repository.yml   |   16 +-
 .../azure/AzureBlobContainer.java             |   60 +-
 .../repositories/azure/AzureBlobStore.java    |   24 +-
 .../repositories/azure/AzureRepository.java   |   57 +-
 .../azure/AzureRepositoryPlugin.java          |   15 +
 .../azure/AzureStorageService.java            |   97 +-
 .../azure/AzureStorageSettings.java           |   57 +-
 .../azure/AzureBlobStoreContainerTests.java   |   20 +-
 .../azure/AzureBlobStoreTests.java            |   20 +-
 .../AzureStorageCleanupThirdPartyTests.java   |   75 +
 .../azure/AzureStorageServiceMock.java        |    5 -
 .../azure/AzureStorageServiceTests.java       |   12 +-
 plugins/repository-gcs/build.gradle           |   71 +-
 .../licenses/api-common-1.7.0.jar.sha1        |    1 -
 .../licenses/api-common-1.8.1.jar.sha1        |    1 +
 .../licenses/gax-1.30.0.jar.sha1              |    1 -
 .../licenses/gax-1.45.0.jar.sha1              |    1 +
 .../licenses/gax-httpjson-0.47.0.jar.sha1     |    1 -
 .../licenses/gax-httpjson-0.62.0.jar.sha1     |    1 +
 .../google-api-client-1.24.1.jar.sha1         |    1 -
 .../google-api-client-1.28.0.jar.sha1         |    1 +
 ...services-storage-v1-rev135-1.24.1.jar.sha1 |    1 -
 ...ces-storage-v1-rev20190426-1.28.0.jar.sha1 |    1 +
 ...e-auth-library-credentials-0.10.0.jar.sha1 |    1 -
 ...e-auth-library-credentials-0.16.1.jar.sha1 |    1 +
 ...e-auth-library-oauth2-http-0.10.0.jar.sha1 |    1 -
 ...e-auth-library-oauth2-http-0.16.1.jar.sha1 |    1 +
 .../google-cloud-core-1.59.0.jar.sha1         |    1 -
 .../google-cloud-core-1.77.0.jar.sha1         |    1 +
 .../google-cloud-core-http-1.59.0.jar.sha1    |    1 -
 .../google-cloud-core-http-1.77.0.jar.sha1    |    1 +
 .../google-cloud-storage-1.59.0.jar.sha1      |    1 -
 .../google-cloud-storage-1.77.0.jar.sha1      |    1 +
 .../google-http-client-1.24.1.jar.sha1        |    1 -
 .../google-http-client-1.30.1.jar.sha1        |    1 +
 ...ogle-http-client-appengine-1.24.1.jar.sha1 |    1 -
 ...ogle-http-client-appengine-1.29.2.jar.sha1 |    1 +
 ...google-http-client-jackson-1.24.1.jar.sha1 |    1 -
 ...oogle-http-client-jackson2-1.24.1.jar.sha1 |    1 -
 ...oogle-http-client-jackson2-1.29.2.jar.sha1 |    1 +
 .../google-oauth-client-1.24.1.jar.sha1       |    1 -
 .../google-oauth-client-1.28.0.jar.sha1       |    1 +
 .../licenses/guava-20.0.jar.sha1              |    1 -
 .../licenses/guava-26.0-jre.jar.sha1          |    1 +
 .../repository-gcs/licenses/jackson-LICENSE   |    8 -
 .../repository-gcs/licenses/jackson-NOTICE    |   20 -
 .../licenses/jackson-core-asl-1.9.11.jar.sha1 |    1 -
 .../licenses/log4j-1.2-api-2.11.1.jar.sha1    |    1 +
 .../repository-gcs/licenses/log4j-LICENSE.txt |  202 ++
 .../repository-gcs/licenses/log4j-NOTICE.txt  |    5 +
 .../licenses/opencensus-api-0.15.0.jar.sha1   |    1 -
 .../licenses/opencensus-api-0.18.0.jar.sha1   |    1 +
 ...encensus-contrib-http-util-0.15.0.jar.sha1 |    1 -
 ...encensus-contrib-http-util-0.18.0.jar.sha1 |    1 +
 ...proto-google-common-protos-1.12.0.jar.sha1 |    1 -
 ...proto-google-common-protos-1.16.0.jar.sha1 |    1 +
 .../licenses/protobuf-java-3.6.0.jar.sha1     |    1 -
 .../licenses/protobuf-java-3.7.1.jar.sha1     |    1 +
 .../protobuf-java-util-3.6.0.jar.sha1         |    1 -
 .../protobuf-java-util-3.7.1.jar.sha1         |    1 +
 .../qa/google-cloud-storage/build.gradle      |    9 +-
 .../test/repository_gcs/10_repository.yml     |   16 +-
 .../gcs/GoogleCloudStorageBlobContainer.java  |   11 +
 .../gcs/GoogleCloudStorageBlobStore.java      |   61 +-
 .../gcs/GoogleCloudStorageRepository.java     |   25 +-
 .../gcs/GoogleCloudStorageService.java        |   62 +-
 .../GoogleCloudStorageThirdPartyTests.java    |   64 +
 .../repositories/gcs/MockStorage.java         |   10 +
 plugins/repository-hdfs/build.gradle          |   41 +-
 .../licenses/log4j-1.2-api-2.11.1.jar.sha1    |    1 +
 .../licenses/log4j-LICENSE.txt                |  202 ++
 .../repository-hdfs/licenses/log4j-NOTICE.txt |    5 +
 .../repositories/hdfs/HdfsBlobContainer.java  |   29 +-
 .../repositories/hdfs/HdfsRepository.java     |    8 +-
 .../hdfs/HdfsRepositoryTests.java             |   61 +
 .../repositories/hdfs/HdfsTests.java          |    9 +-
 .../test/hdfs_repository/30_snapshot_get.yml  |    8 +-
 .../hdfs_repository/30_snapshot_readonly.yml  |    2 +-
 plugins/repository-s3/build.gradle            |   62 +-
 .../aws-java-sdk-core-1.11.505.jar.sha1       |    1 -
 .../aws-java-sdk-core-1.11.562.jar.sha1       |    1 +
 .../aws-java-sdk-kms-1.11.505.jar.sha1        |    1 -
 .../aws-java-sdk-s3-1.11.505.jar.sha1         |    1 -
 .../aws-java-sdk-s3-1.11.562.jar.sha1         |    1 +
 .../licenses/jmespath-java-1.11.505.jar.sha1  |    1 -
 .../licenses/jmespath-java-1.11.562.jar.sha1  |    1 +
 .../licenses/log4j-1.2-api-2.11.1.jar.sha1    |    1 +
 .../repository-s3/licenses/log4j-LICENSE.txt  |  202 ++
 .../repository-s3/licenses/log4j-NOTICE.txt   |    5 +
 .../repositories/s3/S3BlobContainer.java      |  101 +-
 .../repositories/s3/S3Repository.java         |   31 +-
 .../s3/S3RepositoryThirdPartyTests.java       |  114 +
 .../20_repository_permanent_credentials.yml   |   17 +-
 .../30_repository_temporary_credentials.yml   |   16 +-
 .../40_repository_ec2_credentials.yml         |   16 +-
 .../50_repository_ecs_credentials.yml         |   16 +-
 .../smbmmapfs/SmbMmapFsDirectoryFactory.java  |    5 +-
 plugins/transport-nio/build.gradle            |    7 +-
 .../http/nio/HttpReadWriteHandler.java        |   13 +-
 .../http/nio/NioHttpServerTransport.java      |   56 +-
 .../http/nio/cors/NioCorsConfig.java          |  221 --
 .../http/nio/cors/NioCorsConfigBuilder.java   |  265 --
 .../http/nio/cors/NioCorsHandler.java         |   17 +-
 .../http/nio/HttpReadWriteHandlerTests.java   |    9 +-
 .../elasticsearch/http/nio/NioHttpClient.java |    9 +-
 .../http/nio/NioHttpServerTransportTests.java |   58 +-
 qa/build.gradle                               |   29 +-
 qa/ccs-unavailable-clusters/build.gradle      |    3 +-
 qa/die-with-dignity/build.gradle              |   10 +-
 .../elasticsearch/DieWithDignityPlugin.java   |    4 +
 .../qa/die_with_dignity/DieWithDignityIT.java |  123 +-
 qa/evil-tests/build.gradle                    |    1 +
 .../env/NodeEnvironmentEvilTests.java         |    9 +-
 .../upgrades/FullClusterRestartIT.java        |   20 +-
 .../upgrades/QueryBuilderBWCIT.java           |    2 +-
 qa/logging-config/build.gradle                |   10 +-
 .../common/logging/JsonLoggerTests.java       |    4 +-
 .../custom_logging/CustomLoggingConfigIT.java |    2 +-
 qa/multi-cluster-search/build.gradle          |   41 +-
 .../elasticsearch/upgrades/RecoveryIT.java    |   20 +-
 qa/smoke-test-client/build.gradle             |   52 -
 .../smoketest/ESSmokeClientTestCase.java      |  170 -
 .../smoketest/SmokeTestClientIT.java          |   73 -
 qa/smoke-test-http/build.gradle               |    3 +-
 qa/smoke-test-ingest-disabled/build.gradle    |    3 +-
 .../build.gradle                              |    1 +
 qa/smoke-test-multinode/build.gradle          |   15 +-
 qa/smoke-test-plugins/build.gradle            |   13 +-
 qa/unconfigured-node-name/build.gradle        |   13 +-
 qa/vagrant/build.gradle                       |    2 +-
 qa/wildfly/build.gradle                       |   11 +-
 .../org/elasticsearch/wildfly/WildflyIT.java  |    7 +-
 rest-api-spec/README.markdown                 |   94 +-
 .../resources/rest-api-spec/api/bulk.json     |    2 +-
 .../rest-api-spec/api/cat.aliases.json        |    2 +-
 .../rest-api-spec/api/cat.allocation.json     |    2 +-
 .../rest-api-spec/api/cat.count.json          |    2 +-
 .../rest-api-spec/api/cat.fielddata.json      |    2 +-
 .../rest-api-spec/api/cat.health.json         |    2 +-
 .../resources/rest-api-spec/api/cat.help.json |    2 +-
 .../rest-api-spec/api/cat.indices.json        |    2 +-
 .../rest-api-spec/api/cat.master.json         |    2 +-
 .../rest-api-spec/api/cat.nodeattrs.json      |    2 +-
 .../rest-api-spec/api/cat.nodes.json          |    2 +-
 .../rest-api-spec/api/cat.pending_tasks.json  |    2 +-
 .../rest-api-spec/api/cat.plugins.json        |    2 +-
 .../rest-api-spec/api/cat.recovery.json       |    2 +-
 .../rest-api-spec/api/cat.repositories.json   |    2 +-
 .../rest-api-spec/api/cat.segments.json       |    2 +-
 .../rest-api-spec/api/cat.shards.json         |    2 +-
 .../rest-api-spec/api/cat.snapshots.json      |    2 +-
 .../rest-api-spec/api/cat.tasks.json          |    2 +-
 .../rest-api-spec/api/cat.templates.json      |    2 +-
 .../rest-api-spec/api/cat.thread_pool.json    |    2 +-
 .../rest-api-spec/api/clear_scroll.json       |    2 +-
 .../api/cluster.allocation_explain.json       |    2 +-
 .../api/cluster.get_settings.json             |    2 +-
 .../rest-api-spec/api/cluster.health.json     |    2 +-
 .../api/cluster.pending_tasks.json            |    2 +-
 .../api/cluster.put_settings.json             |    2 +-
 .../api/cluster.remote_info.json              |    2 +-
 .../rest-api-spec/api/cluster.reroute.json    |    2 +-
 .../rest-api-spec/api/cluster.state.json      |    7 +-
 .../rest-api-spec/api/cluster.stats.json      |    2 +-
 .../resources/rest-api-spec/api/count.json    |    9 +-
 .../resources/rest-api-spec/api/create.json   |    2 +-
 .../resources/rest-api-spec/api/delete.json   |    2 +-
 .../rest-api-spec/api/delete_by_query.json    |    6 +-
 .../api/delete_by_query_rethrottle.json       |    2 +-
 .../rest-api-spec/api/delete_script.json      |    2 +-
 .../resources/rest-api-spec/api/exists.json   |    2 +-
 .../rest-api-spec/api/exists_source.json      |    2 +-
 .../resources/rest-api-spec/api/explain.json  |    2 +-
 .../rest-api-spec/api/field_caps.json         |    2 +-
 .../main/resources/rest-api-spec/api/get.json |    2 +-
 .../rest-api-spec/api/get_script.json         |    2 +-
 .../rest-api-spec/api/get_source.json         |    2 +-
 .../resources/rest-api-spec/api/index.json    |    2 +-
 .../rest-api-spec/api/indices.analyze.json    |    2 +-
 .../api/indices.clear_cache.json              |    2 +-
 .../rest-api-spec/api/indices.close.json      |    2 +-
 .../rest-api-spec/api/indices.create.json     |    2 +-
 .../rest-api-spec/api/indices.delete.json     |    2 +-
 .../api/indices.delete_alias.json             |    2 +-
 .../api/indices.delete_template.json          |    2 +-
 .../rest-api-spec/api/indices.exists.json     |    2 +-
 .../api/indices.exists_alias.json             |    2 +-
 .../api/indices.exists_template.json          |    2 +-
 .../api/indices.exists_type.json              |    2 +-
 .../rest-api-spec/api/indices.flush.json      |    2 +-
 .../api/indices.flush_synced.json             |    2 +-
 .../rest-api-spec/api/indices.forcemerge.json |    2 +-
 .../rest-api-spec/api/indices.get.json        |    2 +-
 .../rest-api-spec/api/indices.get_alias.json  |    2 +-
 .../api/indices.get_field_mapping.json        |    2 +-
 .../api/indices.get_mapping.json              |    2 +-
 .../api/indices.get_settings.json             |    2 +-
 .../api/indices.get_template.json             |    2 +-
 .../api/indices.get_upgrade.json              |    2 +-
 .../rest-api-spec/api/indices.open.json       |    2 +-
 .../rest-api-spec/api/indices.put_alias.json  |    2 +-
 .../api/indices.put_mapping.json              |    2 +-
 .../api/indices.put_settings.json             |    2 +-
 .../api/indices.put_template.json             |    2 +-
 .../rest-api-spec/api/indices.recovery.json   |    2 +-
 .../rest-api-spec/api/indices.refresh.json    |    2 +-
 .../rest-api-spec/api/indices.rollover.json   |    2 +-
 .../rest-api-spec/api/indices.segments.json   |    2 +-
 .../api/indices.shard_stores.json             |    2 +-
 .../rest-api-spec/api/indices.shrink.json     |    2 +-
 .../rest-api-spec/api/indices.split.json      |    2 +-
 .../rest-api-spec/api/indices.stats.json      |    2 +-
 .../api/indices.update_aliases.json           |    2 +-
 .../rest-api-spec/api/indices.upgrade.json    |    2 +-
 .../api/indices.validate_query.json           |    2 +-
 .../resources/rest-api-spec/api/info.json     |    2 +-
 .../api/ingest.delete_pipeline.json           |    2 +-
 .../api/ingest.get_pipeline.json              |    2 +-
 .../api/ingest.processor_grok.json            |    2 +-
 .../api/ingest.put_pipeline.json              |    2 +-
 .../rest-api-spec/api/ingest.simulate.json    |    2 +-
 .../resources/rest-api-spec/api/mget.json     |    2 +-
 .../resources/rest-api-spec/api/msearch.json  |    2 +-
 .../rest-api-spec/api/msearch_template.json   |    2 +-
 .../rest-api-spec/api/mtermvectors.json       |    2 +-
 .../rest-api-spec/api/nodes.hot_threads.json  |    2 +-
 .../rest-api-spec/api/nodes.info.json         |    2 +-
 .../api/nodes.reload_secure_settings.json     |    2 +-
 .../rest-api-spec/api/nodes.stats.json        |    2 +-
 .../rest-api-spec/api/nodes.usage.json        |    2 +-
 .../resources/rest-api-spec/api/ping.json     |    2 +-
 .../rest-api-spec/api/put_script.json         |    2 +-
 .../rest-api-spec/api/rank_eval.json          |    2 +-
 .../resources/rest-api-spec/api/reindex.json  |    6 +-
 .../rest-api-spec/api/reindex_rethrottle.json |    2 +-
 .../api/render_search_template.json           |    2 +-
 .../api/scripts_painless_execute.json         |    2 +-
 .../resources/rest-api-spec/api/scroll.json   |    2 +-
 .../resources/rest-api-spec/api/search.json   |    2 +-
 .../rest-api-spec/api/search_shards.json      |    2 +-
 .../rest-api-spec/api/search_template.json    |    2 +-
 .../rest-api-spec/api/snapshot.create.json    |    2 +-
 .../api/snapshot.create_repository.json       |    2 +-
 .../rest-api-spec/api/snapshot.delete.json    |    2 +-
 .../api/snapshot.delete_repository.json       |    2 +-
 .../rest-api-spec/api/snapshot.get.json       |    2 +-
 .../api/snapshot.get_repository.json          |    2 +-
 .../rest-api-spec/api/snapshot.restore.json   |    2 +-
 .../rest-api-spec/api/snapshot.status.json    |    2 +-
 .../api/snapshot.verify_repository.json       |    2 +-
 .../rest-api-spec/api/tasks.cancel.json       |    2 +-
 .../rest-api-spec/api/tasks.get.json          |    2 +-
 .../rest-api-spec/api/tasks.list.json         |    2 +-
 .../rest-api-spec/api/termvectors.json        |    2 +-
 .../resources/rest-api-spec/api/update.json   |    2 +-
 .../rest-api-spec/api/update_by_query.json    |    6 +-
 .../api/update_by_query_rethrottle.json       |    2 +-
 .../test/cat.snapshots/10_basic.yml           |   12 +-
 .../test/count/11_basic_with_types.yml        |   66 -
 .../test/indices.analyze/10_analyze.yml       |    1 -
 .../test/indices.recovery/10_basic.yml        |   25 +
 .../test/indices.stats/20_translog.yml        |   44 +
 .../rest-api-spec/test/mlt/20_docs.yml        |    2 -
 .../rest-api-spec/test/mlt/30_unlike.yml      |    2 -
 .../test/msearch/12_basic_with_types.yml      |   97 -
 .../test/mtermvectors/20_deprecated.yml       |    1 -
 .../200_top_hits_metric.yml                   |    9 +-
 .../search.aggregation/280_rare_terms.yml     |  316 ++
 .../test/search/230_interval_query.yml        |   40 +
 .../test/search/310_match_bool_prefix.yml     |   16 -
 .../test/snapshot.get/10_basic.yml            |   93 +-
 .../test/termvectors/20_issue7121.yml         |    1 -
 server/build.gradle                           |   22 +-
 server/licenses/hppc-0.7.1.jar.sha1           |    1 -
 server/licenses/hppc-0.8.1.jar.sha1           |    1 +
 .../lucene/queries/BlendedTermQuery.java      |   44 +-
 .../queries/ExtendedCommonTermsQuery.java     |   76 -
 .../apache/lucene/util/CombinedBitSet.java    |  117 +
 .../elasticsearch/ElasticsearchException.java |    2 +-
 .../org/elasticsearch/ExceptionsHelper.java   |   70 +-
 .../main/java/org/elasticsearch/Version.java  |  207 +-
 .../elasticsearch/action/ActionModule.java    |  106 +-
 .../action/ActionRequestBuilder.java          |    4 +-
 .../action/{Action.java => ActionType.java}   |   32 +-
 .../action/RequestValidators.java             |   67 +
 .../action/RoutingMissingException.java       |    5 +
 .../action/StreamableResponseActionType.java  |   51 +
 .../action/TransportActionNodeProxy.java      |    4 +-
 .../ClusterAllocationExplainAction.java       |    6 +-
 ...ansportClusterAllocationExplainAction.java |    3 +-
 .../AddVotingConfigExclusionsAction.java      |    9 +-
 .../ClearVotingConfigExclusionsAction.java    |    9 +-
 ...nsportAddVotingConfigExclusionsAction.java |    3 +-
 ...portClearVotingConfigExclusionsAction.java |    3 +-
 .../cluster/health/ClusterHealthAction.java   |    4 +-
 .../health/TransportClusterHealthAction.java  |    7 -
 .../hotthreads/NodesHotThreadsAction.java     |    4 +-
 .../TransportNodesHotThreadsAction.java       |   10 +-
 .../cluster/node/info/NodesInfoAction.java    |    4 +-
 .../cluster/node/info/NodesInfoResponse.java  |    6 +-
 .../node/info/TransportNodesInfoAction.java   |   10 +-
 .../NodesReloadSecureSettingsAction.java      |    4 +-
 ...nsportNodesReloadSecureSettingsAction.java |   10 +-
 .../admin/cluster/node/stats/NodeStats.java   |    5 +-
 .../cluster/node/stats/NodesStatsAction.java  |    4 +-
 .../node/stats/TransportNodesStatsAction.java |   10 +-
 .../node/tasks/cancel/CancelTasksAction.java  |   11 +-
 .../cluster/node/tasks/get/GetTaskAction.java |    6 +-
 .../tasks/get/TransportGetTaskAction.java     |    2 +-
 .../node/tasks/list/ListTasksAction.java      |   11 +-
 .../node/tasks/list/ListTasksResponse.java    |    5 +-
 .../cluster/node/usage/NodesUsageAction.java  |    4 +-
 .../node/usage/NodesUsageRequestBuilder.java  |    4 +-
 .../node/usage/TransportNodesUsageAction.java |   10 +-
 .../cluster/remote/RemoteInfoAction.java      |    4 +-
 .../delete/DeleteRepositoryAction.java        |    9 +-
 .../TransportDeleteRepositoryAction.java      |   13 +-
 .../get/GetRepositoriesAction.java            |    9 +-
 .../get/GetRepositoriesResponse.java          |   13 +-
 .../get/TransportGetRepositoriesAction.java   |   12 +-
 .../repositories/put/PutRepositoryAction.java |    9 +-
 .../put/TransportPutRepositoryAction.java     |   13 +-
 .../TransportVerifyRepositoryAction.java      |    3 +-
 .../verify/VerifyRepositoryAction.java        |    4 +-
 .../cluster/reroute/ClusterRerouteAction.java |    9 +-
 .../reroute/ClusterRerouteResponse.java       |   17 +-
 .../TransportClusterRerouteAction.java        |   12 +-
 .../settings/ClusterUpdateSettingsAction.java |    9 +-
 .../ClusterUpdateSettingsResponse.java        |   32 +-
 .../TransportClusterUpdateSettingsAction.java |   13 +-
 .../shards/ClusterSearchShardsAction.java     |    9 +-
 .../TransportClusterSearchShardsAction.java   |    3 +-
 .../create/CreateSnapshotAction.java          |    4 +-
 .../create/CreateSnapshotRequest.java         |   53 +-
 .../create/TransportCreateSnapshotAction.java |    5 +-
 .../delete/DeleteSnapshotAction.java          |    9 +-
 .../delete/TransportDeleteSnapshotAction.java |   13 +-
 .../snapshots/get/GetSnapshotsAction.java     |    9 +-
 .../snapshots/get/GetSnapshotsRequest.java    |   58 +-
 .../get/GetSnapshotsRequestBuilder.java       |   21 +-
 .../snapshots/get/GetSnapshotsResponse.java   |  248 +-
 .../get/TransportGetSnapshotsAction.java      |  172 +-
 .../restore/RestoreSnapshotAction.java        |    4 +-
 .../TransportRestoreSnapshotAction.java       |    3 +-
 .../status/SnapshotsStatusAction.java         |    4 +-
 .../status/TransportNodesSnapshotsStatus.java |   12 +-
 .../TransportSnapshotsStatusAction.java       |    3 +-
 .../cluster/state/ClusterStateAction.java     |    4 +-
 .../state/TransportClusterStateAction.java    |    3 +-
 .../cluster/stats/ClusterStatsAction.java     |    4 +-
 .../cluster/stats/ClusterStatsNodes.java      |   27 +-
 .../stats/TransportClusterStatsAction.java    |   10 +-
 .../DeleteStoredScriptAction.java             |    9 +-
 .../storedscripts/GetStoredScriptAction.java  |    4 +-
 .../storedscripts/PutStoredScriptAction.java  |   11 +-
 .../TransportDeleteStoredScriptAction.java    |   13 +-
 .../TransportGetStoredScriptAction.java       |    3 +-
 .../TransportPutStoredScriptAction.java       |   13 +-
 .../tasks/PendingClusterTasksAction.java      |    4 +-
 .../TransportPendingClusterTasksAction.java   |    3 +-
 .../indices/alias/IndicesAliasesAction.java   |    9 +-
 .../indices/alias/IndicesAliasesRequest.java  |   22 +
 .../alias/TransportIndicesAliasesAction.java  |   48 +-
 .../exists/TransportAliasesExistAction.java   |   68 -
 .../alias/get/BaseAliasesRequestBuilder.java  |    4 +-
 .../indices/alias/get/GetAliasesAction.java   |    4 +-
 .../alias/get/TransportGetAliasesAction.java  |    3 +-
 .../admin/indices/analyze/AnalyzeAction.java  |  779 ++++-
 .../admin/indices/analyze/AnalyzeRequest.java |  302 --
 .../analyze/AnalyzeRequestBuilder.java        |    7 +-
 .../indices/analyze/AnalyzeResponse.java      |  320 --
 .../analyze/DetailAnalyzeResponse.java        |  400 ---
 .../analyze/TransportAnalyzeAction.java       |  516 +--
 .../cache/clear/ClearIndicesCacheAction.java  |    4 +-
 .../admin/indices/close/CloseIndexAction.java |    9 +-
 .../indices/close/CloseIndexResponse.java     |   36 +-
 .../close/TransportCloseIndexAction.java      |   16 +-
 ...TransportVerifyShardBeforeCloseAction.java |   38 +-
 .../indices/create/CreateIndexAction.java     |    9 +-
 .../indices/create/CreateIndexResponse.java   |   16 +-
 .../create/TransportCreateIndexAction.java    |   13 +-
 .../indices/delete/DeleteIndexAction.java     |    9 +-
 .../delete/TransportDeleteIndexAction.java    |   11 +-
 .../exists/indices/IndicesExistsRequest.java  |  102 -
 .../indices/IndicesExistsRequestBuilder.java  |   52 -
 .../indices/TransportIndicesExistsAction.java |   82 -
 .../types/TransportTypesExistsAction.java     |   95 -
 .../exists/types/TypesExistsRequest.java      |  108 -
 .../types/TypesExistsRequestBuilder.java      |   67 -
 .../admin/indices/flush/FlushAction.java      |    4 +-
 .../indices/flush/SyncedFlushAction.java      |    4 +-
 .../indices/forcemerge/ForceMergeAction.java  |    4 +-
 .../admin/indices/get/GetIndexAction.java     |    4 +-
 .../mapping/get/GetFieldMappingsAction.java   |    9 +-
 .../get/GetFieldMappingsIndexRequest.java     |   27 +-
 .../mapping/get/GetMappingsAction.java        |    4 +-
 .../get/TransportGetFieldMappingsAction.java  |   12 +-
 .../TransportGetFieldMappingsIndexAction.java |    2 +
 .../mapping/put/MappingRequestValidator.java  |   40 -
 .../indices/mapping/put/PutMappingAction.java |    9 +-
 .../mapping/put/PutMappingRequest.java        |    1 -
 .../put/TransportPutMappingAction.java        |   59 +-
 .../admin/indices/open/OpenIndexAction.java   |    9 +-
 .../admin/indices/open/OpenIndexResponse.java |    9 +-
 .../open/TransportOpenIndexAction.java        |   12 +-
 .../indices/recovery/RecoveryAction.java      |    4 +-
 .../admin/indices/refresh/RefreshAction.java  |    4 +-
 .../indices/rollover/RolloverAction.java      |    9 +-
 .../indices/rollover/RolloverResponse.java    |   45 +-
 .../rollover/TransportRolloverAction.java     |   41 +-
 .../segments/IndicesSegmentsAction.java       |    4 +-
 .../settings/get/GetSettingsAction.java       |    4 +-
 .../get/TransportGetSettingsAction.java       |    4 +-
 .../put/TransportUpdateSettingsAction.java    |   13 +-
 .../settings/put/UpdateSettingsAction.java    |    9 +-
 .../IndicesShardStoreRequestBuilder.java      |    4 +-
 .../shards/IndicesShardStoresAction.java      |    6 +-
 .../shards/IndicesShardStoresResponse.java    |    7 +-
 .../TransportIndicesShardStoresAction.java    |    3 +-
 .../admin/indices/shrink/ResizeAction.java    |    9 +-
 .../indices/shrink/ResizeRequestBuilder.java  |    4 +-
 .../admin/indices/shrink/ResizeResponse.java  |    6 +-
 .../admin/indices/shrink/ShrinkAction.java    |    9 +-
 .../indices/shrink/TransportResizeAction.java |   20 +-
 .../indices/stats/IndicesStatsAction.java     |    4 +-
 .../delete/DeleteIndexTemplateAction.java     |    9 +-
 .../TransportDeleteIndexTemplateAction.java   |   13 +-
 .../template/get/GetIndexTemplatesAction.java |    4 +-
 .../get/TransportGetIndexTemplatesAction.java |    3 +-
 .../template/put/PutIndexTemplateAction.java  |    9 +-
 .../put/TransportPutIndexTemplateAction.java  |   13 +-
 .../upgrade/get/UpgradeStatusAction.java      |    4 +-
 .../post/TransportUpgradeSettingsAction.java  |   13 +-
 .../indices/upgrade/post/UpgradeAction.java   |    4 +-
 .../upgrade/post/UpgradeSettingsAction.java   |    9 +-
 .../query/TransportValidateQueryAction.java   |    2 +-
 .../validate/query/ValidateQueryAction.java   |    4 +-
 .../elasticsearch/action/bulk/BulkAction.java |    4 +-
 .../action/bulk/TransportBulkAction.java      |   24 +-
 .../action/bulk/TransportShardBulkAction.java |   11 +-
 .../action/delete/DeleteAction.java           |    4 +-
 .../action/explain/ExplainAction.java         |    9 +-
 .../action/explain/ExplainRequest.java        |   27 +-
 .../explain/TransportExplainAction.java       |   12 +-
 .../fieldcaps/FieldCapabilitiesAction.java    |    4 +-
 .../FieldCapabilitiesIndexRequest.java        |   17 +-
 .../TransportFieldCapabilitiesAction.java     |   15 +-
 ...TransportFieldCapabilitiesIndexAction.java |    3 +
 .../elasticsearch/action/get/GetAction.java   |    9 +-
 .../elasticsearch/action/get/GetRequest.java  |   31 +-
 .../action/get/MultiGetAction.java            |    4 +-
 .../action/get/MultiGetShardRequest.java      |   35 +-
 .../action/get/TransportMultiGetAction.java   |    9 +-
 .../get/TransportShardMultiGetAction.java     |    2 +
 .../action/index/IndexAction.java             |    4 +-
 .../action/ingest/DeletePipelineAction.java   |    9 +-
 .../ingest/DeletePipelineTransportAction.java |   15 +-
 .../action/ingest/GetPipelineAction.java      |    4 +-
 .../ingest/GetPipelineTransportAction.java    |    3 +-
 .../action/ingest/IngestActionForwarder.java  |    4 +-
 .../action/ingest/PutPipelineAction.java      |    9 +-
 .../ingest/PutPipelineTransportAction.java    |   12 +-
 .../action/ingest/SimulatePipelineAction.java |    4 +-
 .../elasticsearch/action/main/MainAction.java |    4 +-
 .../search/AbstractSearchAsyncAction.java     |   50 +-
 .../action/search/ClearScrollAction.java      |    4 +-
 .../action/search/ExpandSearchPhase.java      |    3 +-
 .../action/search/FetchSearchPhase.java       |   12 +-
 .../action/search/InitialSearchPhase.java     |    4 +-
 .../action/search/MultiSearchAction.java      |    9 +-
 .../action/search/MultiSearchRequest.java     |    9 -
 .../action/search/SearchAction.java           |    9 +-
 .../action/search/SearchPhaseContext.java     |   13 +-
 .../action/search/SearchPhaseController.java  |    1 +
 .../action/search/SearchRequest.java          |   54 +-
 .../action/search/SearchRequestBuilder.java   |   17 +-
 .../action/search/SearchScrollAction.java     |    9 +-
 .../action/search/TransportSearchAction.java  |  178 +-
 .../DefaultShardOperationFailedException.java |    7 +-
 .../support/ThreadedActionListener.java       |   39 -
 .../BroadcastOperationRequestBuilder.java     |    4 +-
 .../master/AcknowledgedRequestBuilder.java    |    4 +-
 .../support/master/AcknowledgedResponse.java  |    9 +-
 .../MasterNodeOperationRequestBuilder.java    |    4 +-
 ...MasterNodeReadOperationRequestBuilder.java |    4 +-
 .../master/ShardsAcknowledgedResponse.java    |   21 +-
 .../master/TransportMasterNodeAction.java     |   10 +-
 .../info/ClusterInfoRequestBuilder.java       |    4 +-
 .../info/TransportClusterInfoAction.java      |    4 +-
 .../action/support/nodes/BaseNodeRequest.java |   20 +-
 .../support/nodes/BaseNodesRequest.java       |    3 +-
 .../nodes/NodesOperationRequestBuilder.java   |    4 +-
 .../support/nodes/TransportNodesAction.java   |   10 +-
 .../replication/ReplicationOperation.java     |   24 +-
 .../ReplicationRequestBuilder.java            |    4 +-
 .../TransportReplicationAction.java           |   13 +-
 .../InstanceShardOperationRequestBuilder.java |    4 +-
 .../SingleShardOperationRequestBuilder.java   |    4 +-
 .../single/shard/SingleShardRequest.java      |   19 +-
 .../shard/TransportSingleShardAction.java     |    7 +-
 .../support/tasks/TasksRequestBuilder.java    |    4 +-
 .../termvectors/MultiTermVectorsAction.java   |    4 +-
 .../MultiTermVectorsItemResponse.java         |   10 -
 .../termvectors/MultiTermVectorsRequest.java  |   10 +-
 .../MultiTermVectorsRequestBuilder.java       |    9 +-
 .../termvectors/MultiTermVectorsResponse.java |   29 +-
 .../MultiTermVectorsShardRequest.java         |   25 +-
 .../action/termvectors/TermVectorsAction.java |    9 +-
 .../termvectors/TermVectorsRequest.java       |  142 +-
 .../TermVectorsRequestBuilder.java            |   12 +-
 .../termvectors/TermVectorsResponse.java      |   22 +-
 .../TransportMultiTermVectorsAction.java      |   18 +-
 .../TransportShardMultiTermsVectorAction.java |   14 +-
 .../TransportTermVectorsAction.java           |    2 +-
 .../action/update/UpdateAction.java           |    4 +-
 .../org/elasticsearch/bootstrap/Security.java |   18 +-
 .../java/org/elasticsearch/client/Client.java |    7 +-
 .../client/ClusterAdminClient.java            |    8 +-
 .../client/ElasticsearchClient.java           |   10 +-
 .../elasticsearch/client/FilterClient.java    |    4 +-
 .../client/IndicesAdminClient.java            |   88 +-
 .../client/OriginSettingClient.java           |    4 +-
 .../client/ParentTaskAssigningClient.java     |    4 +-
 .../org/elasticsearch/client/Requests.java    |   20 +-
 .../elasticsearch/client/node/NodeClient.java |   30 +-
 .../client/support/AbstractClient.java        |   93 +-
 .../client/transport/TransportClient.java     |  404 ---
 .../TransportClientNodesService.java          |  582 ----
 .../transport/TransportProxyClient.java       |   59 -
 .../elasticsearch/cluster/ClusterModule.java  |   34 -
 .../elasticsearch/cluster/ClusterState.java   |   19 +-
 .../cluster/InternalClusterInfoService.java   |   10 +-
 .../cluster/SnapshotsInProgress.java          |   30 +-
 .../action/shard/ShardStateAction.java        |   22 +-
 .../ClusterFormationFailureHelper.java        |   15 +-
 .../coordination/CoordinationState.java       |   55 +-
 .../cluster/coordination/Coordinator.java     |  106 +-
 .../coordination/ElectionStrategy.java        |   74 +
 .../cluster/coordination/JoinHelper.java      |    5 +-
 .../coordination/JoinTaskExecutor.java        |   13 +-
 .../cluster/coordination/LeaderChecker.java   |   40 +-
 .../coordination/PreVoteCollector.java        |   27 +-
 .../cluster/coordination/Reconfigurator.java  |   10 +-
 .../cluster/metadata/AliasMetaData.java       |    8 +-
 .../cluster/metadata/DiffableStringMap.java   |    4 +
 .../cluster/metadata/IndexMetaData.java       |    4 +-
 .../metadata/IndexTemplateMetaData.java       |   14 +-
 .../cluster/metadata/MetaData.java            |   83 +-
 .../metadata/MetaDataIndexStateService.java   |   19 +-
 .../metadata/MetaDataIndexUpgradeService.java |    2 +-
 .../cluster/node/DiscoveryNode.java           |  155 +-
 .../cluster/node/DiscoveryNodeRole.java       |  167 +
 .../cluster/node/DiscoveryNodes.java          |   18 +-
 .../routing/BatchedRerouteService.java        |  135 +
 .../routing/IndexShardRoutingTable.java       |   11 +-
 .../LazilyInitializedRerouteService.java      |   42 +
 .../cluster/routing/RerouteService.java       |   29 +
 .../cluster/routing/RoutingService.java       |  122 -
 .../routing/allocation/AllocationService.java |   12 +-
 .../allocation/IndexMetaDataUpdater.java      |    6 +-
 ...AllocateEmptyPrimaryAllocationCommand.java |   13 +-
 .../AllocateReplicaAllocationCommand.java     |   26 +-
 ...AllocateStalePrimaryAllocationCommand.java |   13 +-
 .../decider/DiskThresholdDecider.java         |    6 +-
 .../cluster/service/ClusterService.java       |    2 +-
 .../org/elasticsearch/common/TriConsumer.java |   43 +-
 .../common/blobstore/BlobContainer.java       |   16 +
 .../common/blobstore/BlobPath.java            |   16 +
 .../common/blobstore/fs/FsBlobContainer.java  |   21 +-
 .../common/compress/CompressorFactory.java    |   10 +-
 .../org/elasticsearch/common/geo/GeoJson.java |   43 +-
 .../common/geo/GeometryFormat.java}           |   35 +-
 .../common/geo/GeometryParser.java            |   73 +-
 .../common/hash/MessageDigests.java           |   19 +-
 .../common/hash/MurmurHash3.java              |   38 +
 .../elasticsearch/common/inject/Binder.java   |    2 +-
 .../assistedinject/FactoryProvider.java       |    2 +-
 .../org/elasticsearch/common/io/Streams.java  |   12 +
 .../common/io/stream/StreamInput.java         |   28 +
 .../common/io/stream/StreamOutput.java        |   26 +
 .../elasticsearch/common/lucene/Lucene.java   |   38 -
 .../common/lucene/MinimumScoreCollector.java  |    2 +-
 .../common/lucene/search/Queries.java         |    3 -
 .../common/lucene/search/XMoreLikeThis.java   |    2 +-
 .../uid/PerThreadIDVersionAndSeqNoLookup.java |   79 +-
 .../lucene/uid/VersionsAndSeqNoResolver.java  |   21 +-
 .../common/network/NetworkModule.java         |   23 +-
 .../common/network/NetworkUtils.java          |    8 +-
 .../org/elasticsearch/common/regex/Regex.java |   15 +-
 .../common/settings/ClusterSettings.java      |   11 +-
 .../settings/ConsistentSettingsService.java   |  256 ++
 .../common/settings/KeyStoreWrapper.java      |   61 +-
 .../common/settings/SecureSetting.java        |   19 +-
 .../common/settings/SecureSettings.java       |    2 +
 .../common/settings/Setting.java              |   20 +
 .../common/settings/Settings.java             |    9 +-
 .../common/settings/SettingsModule.java       |   22 +-
 .../common/time/DateMathParser.java           |   18 +-
 .../common/time/JavaDateFormatter.java        |   17 +-
 .../common/time/JavaDateMathParser.java       |   53 +-
 .../common/util/CuckooFilter.java             |  521 +++
 .../util/SetBackedScalingCuckooFilter.java    |  408 +++
 .../util/concurrent/AsyncIOProcessor.java     |   56 +-
 .../common/util/concurrent/EsExecutors.java   |    1 +
 .../discovery/DiscoveryModule.java            |   24 +-
 .../HandshakingTransportAddressConnector.java |    8 +
 .../discovery/SeedHostsResolver.java          |   15 +-
 .../elasticsearch/env/NodeEnvironment.java    |  186 +-
 .../gateway/GatewayAllocator.java             |   16 +-
 .../elasticsearch/gateway/GatewayService.java |   11 +-
 .../TransportNodesListGatewayMetaState.java   |   15 +-
 ...ransportNodesListGatewayStartedShards.java |   10 +-
 .../http/AbstractHttpServerTransport.java     |    2 +
 .../org/elasticsearch/http/CorsHandler.java   |  258 ++
 .../http/DefaultRestChannel.java              |   57 +-
 .../org/elasticsearch/http/HttpRequest.java   |    6 +
 .../org/elasticsearch/index/IndexModule.java  |   49 +-
 .../org/elasticsearch/index/IndexService.java |   53 +-
 .../elasticsearch/index/IndexSettings.java    |    2 +-
 .../elasticsearch/index/SearchSlowLog.java    |    9 +-
 .../index/analysis/AnalysisRegistry.java      |  164 +-
 .../index/analysis/AnalyzerComponents.java    |  111 +
 .../analysis/AnalyzerComponentsProvider.java  |   13 +-
 .../index/analysis/CustomAnalyzer.java        |   51 +-
 .../analysis/CustomAnalyzerProvider.java      |   72 +-
 .../index/analysis/IndexAnalyzers.java        |   22 +-
 .../index/analysis/NameOrDefinition.java      |  115 +
 .../index/analysis/NamedAnalyzer.java         |    4 +-
 .../analysis/ReloadableCustomAnalyzer.java    |  162 +
 .../index/engine/CombinedDeletionPolicy.java  |   24 +-
 .../index/engine/CombinedDocValues.java       |   90 +
 .../elasticsearch/index/engine/Engine.java    |   12 +-
 .../index/engine/InternalEngine.java          |  273 +-
 .../index/engine/LuceneChangesSnapshot.java   |   69 +-
 .../index/engine/NoOpEngine.java              |   62 +-
 .../engine/PrunePostingsMergePolicy.java      |  224 ++
 .../index/engine/ReadOnlyEngine.java          |   43 +-
 .../index/fielddata/IndexFieldData.java       |    8 +-
 .../fielddata/IndexOrdinalsFieldData.java     |    7 +
 .../ordinals/GlobalOrdinalMapping.java        |   12 +-
 .../GlobalOrdinalsIndexFieldData.java         |  162 +-
 .../plain/AbstractIndexOrdinalsFieldData.java |   17 +
 .../SortedSetDVOrdinalsIndexFieldData.java    |   17 +
 .../fieldvisitor/IdOnlyFieldVisitor.java      |   60 +
 .../index/mapper/BaseGeoShapeFieldMapper.java |    2 +-
 .../index/mapper/ContentPath.java             |    4 +
 .../index/mapper/DocumentParser.java          |    8 +-
 .../index/mapper/DynamicKeyFieldMapper.java   |   54 +
 .../index/mapper/FieldMapper.java             |   28 +-
 .../index/mapper/FieldTypeLookup.java         |  127 +-
 .../mapper/LegacyGeoShapeFieldMapper.java     |    5 +-
 .../index/mapper/MappedFieldType.java         |    3 +-
 .../index/mapper/MapperService.java           |   34 +-
 .../index/mapper/ParseContext.java            |   17 +-
 .../index/mapper/TextFieldMapper.java         |   61 +-
 .../index/mapper/TypeParsers.java             |   18 +-
 .../index/query/CommonTermsQueryBuilder.java  |  406 ---
 .../index/query/IdsQueryBuilder.java          |   75 +-
 .../index/query/IntervalBuilder.java          |    4 +-
 .../index/query/IntervalsSourceProvider.java  |  232 +-
 .../index/query/MatchQueryBuilder.java        |   44 +-
 .../index/query/MoreLikeThisQueryBuilder.java |   97 +-
 .../index/query/MultiMatchQueryBuilder.java   |   54 +-
 .../index/query/QueryBuilders.java            |   10 -
 .../index/query/QueryShardContext.java        |   57 +-
 .../ScriptScoreQueryBuilder.java              |    8 +-
 .../reindex/AbstractBulkByScrollRequest.java  |   35 +-
 .../AbstractBulkByScrollRequestBuilder.java   |   20 +-
 ...stractBulkIndexByScrollRequestBuilder.java |    6 +-
 .../reindex/ClientScrollableHitSource.java    |    5 +-
 .../index/reindex/DeleteByQueryAction.java    |    4 +-
 .../index/reindex/DeleteByQueryRequest.java   |   51 -
 .../reindex/DeleteByQueryRequestBuilder.java  |    6 +-
 .../index/reindex/ReindexAction.java          |    4 +-
 .../index/reindex/ReindexRequest.java         |  199 +-
 .../index/reindex/ReindexRequestBuilder.java  |    6 +-
 .../index/reindex/RemoteInfo.java             |   57 +
 .../index/reindex/UpdateByQueryAction.java    |    4 +-
 .../index/reindex/UpdateByQueryRequest.java   |   28 -
 .../reindex/UpdateByQueryRequestBuilder.java  |    6 +-
 .../index/search/MatchQuery.java              |   46 +-
 .../index/search/MultiMatchQuery.java         |   27 +-
 .../index/search/NestedHelper.java            |    2 +-
 .../index/search/QueryParserHelper.java       |   27 +-
 .../index/search/QueryStringQueryParser.java  |    9 +-
 .../seqno/GlobalCheckpointSyncAction.java     |   30 +-
 .../index/seqno/LocalCheckpointTracker.java   |  151 +-
 .../index/seqno/ReplicationTracker.java       |  189 +-
 .../index/seqno/RetentionLeaseActions.java    |   52 +-
 .../RetentionLeaseBackgroundSyncAction.java   |   46 +-
 .../index/seqno/RetentionLeaseSyncAction.java |   44 +-
 .../elasticsearch/index/seqno/SeqNoStats.java |    2 +-
 .../index/seqno/SequenceNumbers.java          |    4 -
 .../index/shard/IndexSearcherWrapper.java     |  138 -
 .../elasticsearch/index/shard/IndexShard.java |  154 +-
 .../index/shard/PrimaryReplicaSyncer.java     |    2 +-
 .../RemoveCorruptedShardDataCommand.java      |    5 +-
 .../index/similarity/SimilarityService.java   |   28 +-
 .../index/store/FsDirectoryFactory.java       |  133 +-
 .../org/elasticsearch/index/store/Store.java  |  147 +-
 .../index/termvectors/TermVectorsService.java |   15 +-
 .../index/translog/BaseTranslogReader.java    |    2 +-
 .../index/translog/Translog.java              |   22 +-
 .../index/translog/TranslogHeader.java        |   92 +-
 .../index/translog/TranslogSnapshot.java      |   22 +-
 .../index/translog/TranslogWriter.java        |   30 +-
 .../translog/TruncateTranslogAction.java      |   14 +-
 .../elasticsearch/indices/IndicesModule.java  |    2 -
 .../elasticsearch/indices/IndicesService.java |    9 +-
 .../cluster/IndicesClusterStateService.java   |  110 +-
 .../recovery/PeerRecoveryTargetService.java   |   18 +-
 .../recovery/RecoverySourceHandler.java       |  219 +-
 .../indices/recovery/RecoveryState.java       |    4 +-
 .../indices/recovery/RecoveryTarget.java      |  117 +-
 .../recovery/RecoveryTargetHandler.java       |    6 +-
 .../recovery/RemoteRecoveryTargetHandler.java |   18 +-
 .../TransportNodesListShardStoreMetaData.java |   10 +-
 .../elasticsearch/ingest/IngestService.java   |  219 +-
 .../org/elasticsearch/monitor/os/OsProbe.java |   44 +-
 .../org/elasticsearch/monitor/os/OsStats.java |    4 +
 .../node/InternalSettingsPreparer.java        |    5 +-
 .../java/org/elasticsearch/node/Node.java     |   51 +-
 .../CompletionPersistentTaskAction.java       |    9 +-
 .../PersistentTasksNodeService.java           |    2 +-
 .../persistent/PersistentTasksService.java    |    4 +-
 .../RemovePersistentTaskAction.java           |    7 +-
 .../persistent/StartPersistentTaskAction.java |    7 +-
 .../UpdatePersistentTaskStatusAction.java     |    7 +-
 .../elasticsearch/plugins/ActionPlugin.java   |   53 +-
 .../plugins/DiscoveryPlugin.java              |    8 +
 .../org/elasticsearch/plugins/Plugin.java     |   22 +-
 .../plugins/PluginLoaderIndirection.java      |    1 +
 .../elasticsearch/plugins/PluginsService.java |   21 +-
 .../repositories/FilterRepository.java        |    6 +-
 .../repositories/Repository.java              |    4 +-
 .../repositories/RepositoryData.java          |    4 +-
 .../blobstore/BlobStoreRepository.java        |  908 +++---
 .../blobstore/ChecksumBlobStoreFormat.java    |   33 +-
 .../repositories/blobstore/package-info.java  |   16 +-
 .../repositories/fs/FsRepository.java         |   10 +-
 .../elasticsearch/rest/RestController.java    |   90 +-
 .../org/elasticsearch/rest/RestRequest.java   |    6 +
 .../admin/cluster/RestGetSnapshotsAction.java |    4 +-
 .../admin/indices/RestAnalyzeAction.java      |  104 +-
 .../rest/action/cat/RestIndicesAction.java    |  286 +-
 .../rest/action/cat/RestNodesAction.java      |    3 +-
 .../rest/action/cat/RestSnapshotAction.java   |   74 +-
 .../document/RestMultiTermVectorsAction.java  |    4 -
 .../document/RestTermVectorsAction.java       |    3 +-
 .../rest/action/search/RestCountAction.java   |   15 -
 .../action/search/RestMultiSearchAction.java  |    2 +-
 .../search/DefaultSearchContext.java          |   22 +-
 .../org/elasticsearch/search/SearchHits.java  |    2 +-
 .../elasticsearch/search/SearchModule.java    |   49 +-
 .../elasticsearch/search/SearchService.java   |   41 +-
 .../AbstractAggregationBuilder.java           |   31 +-
 .../aggregations/AggregationBuilder.java      |    2 +-
 .../aggregations/AggregatorFactories.java     |   12 +-
 .../aggregations/AggregatorFactory.java       |   14 +-
 .../aggregations/InternalAggregation.java     |   28 +-
 .../PipelineAggregationBuilder.java           |    2 +-
 .../bucket/BestBucketsDeferringCollector.java |   68 +-
 .../bucket/BucketsAggregator.java             |    4 +-
 .../InternalSingleBucketAggregation.java      |   10 +-
 .../MergingBucketsDeferringCollector.java     |  254 +-
 .../AdjacencyMatrixAggregationBuilder.java    |   11 +-
 .../AdjacencyMatrixAggregatorFactory.java     |    4 +-
 .../adjacency/InternalAdjacencyMatrix.java    |   10 +-
 .../CompositeAggregationBuilder.java          |   15 +-
 .../CompositeAggregationFactory.java          |    4 +-
 .../CompositeValuesSourceBuilder.java         |   11 +-
 .../DateHistogramValuesSourceBuilder.java     |   10 +-
 .../HistogramValuesSourceBuilder.java         |   10 +-
 .../bucket/composite/InternalComposite.java   |   10 +-
 .../composite/TermsValuesSourceBuilder.java   |   10 -
 .../filter/FilterAggregationBuilder.java      |   11 +-
 .../filter/FilterAggregatorFactory.java       |    4 +-
 .../filter/FiltersAggregationBuilder.java     |   11 +-
 .../filter/FiltersAggregatorFactory.java      |    4 +-
 .../bucket/filter/InternalFilters.java        |   10 +-
 .../geogrid/GeoGridAggregationBuilder.java    |   30 +-
 .../GeoHashGridAggregationBuilder.java        |    4 +-
 .../geogrid/GeoHashGridAggregatorFactory.java |    4 +-
 .../GeoTileGridAggregationBuilder.java        |    4 +-
 .../geogrid/GeoTileGridAggregatorFactory.java |    4 +-
 .../bucket/geogrid/InternalGeoGrid.java       |   14 +-
 .../global/GlobalAggregationBuilder.java      |   12 +-
 .../global/GlobalAggregatorFactory.java       |    4 +-
 .../AutoDateHistogramAggregationBuilder.java  |  119 +-
 .../AutoDateHistogramAggregator.java          |    2 +-
 .../AutoDateHistogramAggregatorFactory.java   |   13 +-
 .../DateHistogramAggregationBuilder.java      |   13 +-
 .../DateHistogramAggregatorFactory.java       |    4 +-
 .../HistogramAggregationBuilder.java          |   25 +-
 .../histogram/HistogramAggregatorFactory.java |    4 +-
 .../histogram/InternalAutoDateHistogram.java  |   10 +-
 .../histogram/InternalDateHistogram.java      |   10 +-
 .../bucket/histogram/InternalHistogram.java   |   10 +-
 .../missing/MissingAggregationBuilder.java    |   16 +-
 .../missing/MissingAggregatorFactory.java     |    4 +-
 .../nested/NestedAggregationBuilder.java      |   12 +-
 .../nested/NestedAggregatorFactory.java       |    4 +-
 .../ReverseNestedAggregationBuilder.java      |   14 +-
 .../ReverseNestedAggregatorFactory.java       |    4 +-
 .../range/AbstractRangeAggregatorFactory.java |    5 +-
 .../bucket/range/AbstractRangeBuilder.java    |    9 +-
 .../range/BinaryRangeAggregatorFactory.java   |    4 +-
 .../range/DateRangeAggregationBuilder.java    |    2 +-
 .../range/DateRangeAggregatorFactory.java     |    4 +-
 .../range/GeoDistanceAggregationBuilder.java  |   13 +-
 .../GeoDistanceRangeAggregatorFactory.java    |    4 +-
 .../bucket/range/InternalBinaryRange.java     |   12 +-
 .../bucket/range/InternalRange.java           |   10 +-
 .../range/IpRangeAggregationBuilder.java      |   15 +-
 .../bucket/range/RangeAggregationBuilder.java |    2 +-
 .../bucket/range/RangeAggregatorFactory.java  |    4 +-
 .../DiversifiedAggregationBuilder.java        |   17 +-
 .../sampler/DiversifiedAggregatorFactory.java |    4 +-
 .../sampler/SamplerAggregationBuilder.java    |   11 +-
 .../sampler/SamplerAggregatorFactory.java     |    4 +-
 .../InternalMappedSignificantTerms.java       |   13 +-
 .../significant/InternalSignificantTerms.java |   10 +-
 .../significant/SignificantStringTerms.java   |    4 +
 .../SignificantTermsAggregationBuilder.java   |   13 +-
 .../SignificantTermsAggregatorFactory.java    |    4 +-
 .../SignificantTextAggregationBuilder.java    |   14 +-
 .../SignificantTextAggregatorFactory.java     |    4 +-
 .../terms/AbstractRareTermsAggregator.java    |  134 +
 .../bucket/terms/IncludeExclude.java          |    8 +-
 .../bucket/terms/InternalMappedRareTerms.java |  182 ++
 .../bucket/terms/InternalMappedTerms.java     |   13 +-
 .../bucket/terms/InternalRareTerms.java       |  205 ++
 .../bucket/terms/InternalTerms.java           |   10 +-
 .../bucket/terms/LongRareTerms.java           |  156 +
 .../bucket/terms/LongRareTermsAggregator.java |  169 +
 .../aggregations/bucket/terms/RareTerms.java  |   48 +
 .../terms/RareTermsAggregationBuilder.java    |  203 ++
 .../terms/RareTermsAggregatorFactory.java     |  164 +
 .../bucket/terms/StringRareTerms.java         |  159 +
 .../terms/StringRareTermsAggregator.java      |  175 +
 .../bucket/terms/StringTermsAggregator.java   |   14 +-
 .../bucket/terms/TermsAggregationBuilder.java |   24 +-
 .../bucket/terms/TermsAggregatorFactory.java  |   15 +-
 .../bucket/terms/UnmappedRareTerms.java       |  119 +
 .../AbstractInternalHDRPercentiles.java       |   14 +-
 .../AbstractInternalTDigestPercentiles.java   |   10 +-
 .../metrics/AvgAggregationBuilder.java        |   12 +-
 .../metrics/AvgAggregatorFactory.java         |    4 +-
 .../CardinalityAggregationBuilder.java        |   11 +-
 .../metrics/CardinalityAggregatorFactory.java |    4 +-
 .../ExtendedStatsAggregationBuilder.java      |   11 +-
 .../ExtendedStatsAggregatorFactory.java       |    4 +-
 .../metrics/GeoBoundsAggregationBuilder.java  |   11 +-
 .../metrics/GeoBoundsAggregatorFactory.java   |    4 +-
 .../GeoCentroidAggregationBuilder.java        |   12 +-
 .../metrics/GeoCentroidAggregatorFactory.java |    4 +-
 .../HDRPercentileRanksAggregatorFactory.java  |    4 +-
 .../HDRPercentilesAggregatorFactory.java      |    4 +-
 .../aggregations/metrics/InternalAvg.java     |    9 +-
 .../metrics/InternalCardinality.java          |   11 +-
 .../metrics/InternalExtendedStats.java        |   13 +-
 .../metrics/InternalGeoBounds.java            |   10 +-
 .../metrics/InternalGeoCentroid.java          |   11 +-
 .../aggregations/metrics/InternalMax.java     |    9 +-
 .../InternalMedianAbsoluteDeviation.java      |    9 +-
 .../aggregations/metrics/InternalMin.java     |    9 +-
 .../InternalNumericMetricsAggregation.java    |   15 +-
 .../metrics/InternalScriptedMetric.java       |   10 +-
 .../aggregations/metrics/InternalStats.java   |   10 +-
 .../aggregations/metrics/InternalSum.java     |   10 +-
 .../aggregations/metrics/InternalTopHits.java |   13 +-
 .../metrics/InternalValueCount.java           |   10 +-
 .../metrics/InternalWeightedAvg.java          |    9 +-
 .../metrics/MaxAggregationBuilder.java        |   12 +-
 .../metrics/MaxAggregatorFactory.java         |    4 +-
 ...anAbsoluteDeviationAggregationBuilder.java |   13 +-
 ...ianAbsoluteDeviationAggregatorFactory.java |    5 +-
 .../metrics/MinAggregationBuilder.java        |   12 +-
 .../metrics/MinAggregatorFactory.java         |    4 +-
 .../PercentileRanksAggregationBuilder.java    |   47 +-
 .../PercentilesAggregationBuilder.java        |   40 +-
 .../ScriptedMetricAggregationBuilder.java     |   19 +-
 .../ScriptedMetricAggregatorFactory.java      |    4 +-
 .../metrics/StatsAggregationBuilder.java      |   12 +-
 .../metrics/StatsAggregatorFactory.java       |    4 +-
 .../metrics/SumAggregationBuilder.java        |   12 +-
 .../metrics/SumAggregatorFactory.java         |    4 +-
 ...igestPercentileRanksAggregatorFactory.java |    4 +-
 .../TDigestPercentilesAggregatorFactory.java  |    4 +-
 .../metrics/TopHitsAggregationBuilder.java    |   37 +-
 .../metrics/TopHitsAggregatorFactory.java     |    4 +-
 .../metrics/ValueCountAggregationBuilder.java |   12 +-
 .../metrics/ValueCountAggregatorFactory.java  |    4 +-
 .../WeightedAvgAggregationBuilder.java        |   14 +-
 .../metrics/WeightedAvgAggregatorFactory.java |    4 +-
 .../AbstractPipelineAggregationBuilder.java   |   32 +-
 .../AvgBucketPipelineAggregationBuilder.java  |   10 -
 ...cketMetricsPipelineAggregationBuilder.java |   18 +-
 ...ucketScriptPipelineAggregationBuilder.java |   15 +-
 ...ketSelectorPipelineAggregationBuilder.java |   15 +-
 .../BucketSortPipelineAggregationBuilder.java |   11 +-
 ...mulativeSumPipelineAggregationBuilder.java |   11 +-
 .../DerivativePipelineAggregationBuilder.java |   24 +-
 ...StatsBucketPipelineAggregationBuilder.java |   11 +-
 .../pipeline/InternalBucketMetricValue.java   |    9 +-
 .../pipeline/InternalDerivative.java          |    9 +-
 .../pipeline/InternalPercentilesBucket.java   |   10 +-
 .../pipeline/InternalSimpleValue.java         |    9 +-
 .../MaxBucketPipelineAggregationBuilder.java  |   10 -
 .../MinBucketPipelineAggregationBuilder.java  |   10 -
 .../MovFnPipelineAggregationBuilder.java      |   11 +-
 ...tilesBucketPipelineAggregationBuilder.java |   14 +-
 .../SerialDiffPipelineAggregationBuilder.java |   12 +-
 .../pipeline/SiblingPipelineAggregator.java   |    2 +-
 ...StatsBucketPipelineAggregationBuilder.java |   10 -
 .../SumBucketPipelineAggregationBuilder.java  |   10 -
 .../MultiValuesSourceAggregationBuilder.java  |   36 +-
 .../MultiValuesSourceAggregatorFactory.java   |    6 +-
 .../aggregations/support/ValuesSource.java    |   15 +
 .../ValuesSourceAggregationBuilder.java       |   48 +-
 .../ValuesSourceAggregatorFactory.java        |   22 +-
 .../support/ValuesSourceConfig.java           |   12 +-
 .../search/fetch/FetchPhase.java              |   38 +-
 .../highlight/FragmentBuilderHelper.java      |    7 +-
 .../search/internal/ContextIndexSearcher.java |  127 +-
 .../internal/ShardSearchLocalRequest.java     |   31 +-
 .../search/internal/ShardSearchRequest.java   |    2 -
 .../internal/ShardSearchTransportRequest.java |    5 -
 .../search/lookup/DocLookup.java              |   13 +-
 .../search/lookup/FieldsLookup.java           |    8 +-
 .../search/lookup/LeafDocLookup.java          |   11 +-
 .../search/lookup/LeafFieldsLookup.java       |   10 +-
 .../search/lookup/SearchLookup.java           |    8 +-
 .../search/slice/SliceBuilder.java            |    8 +-
 .../search/slice/TermsSliceQuery.java         |    3 +
 .../search/sort/FieldSortBuilder.java         |  235 +-
 .../search/sort/GeoDistanceSortBuilder.java   |  132 +-
 .../search/sort/ScriptSortBuilder.java        |  121 +-
 .../search/sort/SortBuilder.java              |   15 +-
 .../search/suggest/Suggester.java             |    7 +-
 .../completion/CompletionSuggester.java       |   19 +-
 .../suggest/phrase/PhraseSuggester.java       |    9 +
 .../phrase/PhraseSuggestionBuilder.java       |    7 +-
 .../search/suggest/term/TermSuggester.java    |   12 +
 .../elasticsearch/snapshots/SnapshotInfo.java |   71 +-
 .../snapshots/SnapshotShardsService.java      |    3 +-
 .../snapshots/SnapshotsService.java           |   11 +-
 .../elasticsearch/snapshots/package-info.java |    6 +-
 .../elasticsearch/threadpool/Scheduler.java   |    7 +-
 .../transport/RemoteClusterAware.java         |    4 +-
 .../transport/RemoteClusterAwareClient.java   |    4 +-
 .../transport/RemoteClusterConnection.java    |    1 +
 .../transport/RemoteClusterService.java       |   13 +-
 .../elasticsearch/transport/TcpTransport.java |   56 +-
 .../transport/TransportService.java           |    9 +-
 .../elasticsearch/bootstrap/security.policy   |    4 +-
 .../CustomUnifiedHighlighterTests.java        |   16 -
 .../lucene/util/CombinedBitSetTests.java      |  111 +
 .../java/org/elasticsearch/BuildTests.java    |   10 -
 .../elasticsearch/ExceptionsHelperTests.java  |   39 +-
 .../java/org/elasticsearch/VersionTests.java  |   72 +-
 .../action/ActionModuleTests.java             |    8 +-
 .../org/elasticsearch/action/ActionTests.java |    2 +-
 .../action/IndicesRequestIT.java              |    7 +-
 .../action/ListenerActionIT.java              |    8 +-
 .../action/RequestValidatorsTests.java        |   94 +
 .../ShardValidateQueryRequestTests.java       |    2 +-
 ...AddVotingConfigExclusionsRequestTests.java |   58 +-
 ...tAddVotingConfigExclusionsActionTests.java |   13 +-
 .../node/tasks/CancellableTasksTests.java     |   28 +-
 .../node/tasks/TaskManagerTestCase.java       |    3 +-
 .../admin/cluster/node/tasks/TasksIT.java     |    7 +-
 .../cluster/node/tasks/TestTaskPlugin.java    |   35 +-
 .../node/tasks/TransportTasksActionTests.java |   11 +-
 .../ClusterUpdateSettingsResponseTests.java   |   10 +-
 .../ClusterSearchShardsResponseTests.java     |    2 +-
 .../cluster/snapshots/SnapshotBlocksIT.java   |   11 +-
 .../create/CreateSnapshotRequestTests.java    |   63 +
 .../create/CreateSnapshotResponseTests.java   |   15 +-
 .../get/GetSnapshotsResponseTests.java        |  101 +-
 .../state/ClusterStateRequestTests.java       |    6 +-
 .../admin/cluster/stats/ClusterStatsIT.java   |   14 +-
 .../indices/TransportAnalyzeActionTests.java  |  240 +-
 .../ValidateIndicesAliasesRequestIT.java      |  133 +
 .../indices/analyze/AnalyzeRequestTests.java  |   41 +-
 .../indices/analyze/AnalyzeResponseTests.java |  109 +-
 .../clear/ClearIndicesCacheResponseTests.java |    2 +-
 .../indices/close/CloseIndexRequestTests.java |    8 +-
 .../close/CloseIndexResponseTests.java        |   76 +-
 ...portVerifyShardBeforeCloseActionTests.java |   18 +-
 .../create/CreateIndexResponseTests.java      |    9 +-
 .../admin/indices/create/ShrinkIndexIT.java   |    2 -
 .../admin/indices/exists/IndicesExistsIT.java |   47 -
 .../indices/flush/FlushResponseTests.java     |    2 +-
 .../forcemerge/ForceMergeResponseTests.java   |    2 +-
 .../put/ValidateMappingRequestPluginIT.java   |    9 +-
 .../indices/open/OpenIndexResponseTests.java  |    9 +-
 .../indices/refresh/RefreshResponseTests.java |    2 +-
 .../rollover/RolloverResponseTests.java       |   10 +-
 .../TransportRolloverActionTests.java         |  126 +-
 .../settings/get/GetSettingsActionTests.java  |    6 +-
 .../shards/IndicesShardStoreRequestIT.java    |    3 -
 .../indices/shrink/ResizeResponseTests.java   |    9 +-
 .../shrink/TransportResizeActionTests.java    |    3 +-
 .../query/ValidateQueryResponseTests.java     |    2 +-
 .../action/bulk/BulkProcessorIT.java          |   44 -
 .../action/bulk/BulkProcessorRetryIT.java     |    1 -
 .../action/bulk/BulkRejectionIT.java          |   17 +
 ...ActionIndicesThatCannotBeCreatedTests.java |    2 +-
 .../bulk/TransportBulkActionIngestTests.java  |   28 +-
 .../action/bulk/TransportBulkActionTests.java |    2 +-
 .../bulk/TransportBulkActionTookTests.java    |   10 +-
 .../action/explain/ExplainRequestTests.java   |    5 +-
 .../action/get/MultiGetShardRequestTests.java |    4 +-
 .../get/TransportMultiGetActionTests.java     |    4 +-
 .../action/search/ExpandSearchPhaseTests.java |   48 +-
 .../action/search/FetchSearchPhaseTests.java  |  109 +-
 .../action/search/MockSearchPhaseContext.java |   12 +-
 .../search/MultiSearchRequestTests.java       |   23 +-
 .../action/search/SearchAsyncActionTests.java |   97 +
 .../search/SearchRequestBuilderTests.java     |   36 +-
 .../action/search/SearchRequestTests.java     |    7 -
 .../action/search/SearchResponseTests.java    |    2 +-
 .../TransportMultiSearchActionTests.java      |    7 +-
 .../TransportSearchActionSingleNodeTests.java |   61 +
 .../search/TransportSearchActionTests.java    |   75 +
 .../support/ActiveShardsObserverIT.java       |    2 +-
 .../master/IndexingMasterFailoverIT.java      |    2 +-
 .../ShardsAcknowledgedResponseTests.java      |   64 +
 .../TransportMasterNodeActionTests.java       |   12 +-
 .../TransportMasterNodeActionUtils.java       |    8 +-
 .../nodes/TransportNodesActionTests.java      |   11 +-
 .../ClusterStateCreationUtils.java            |    4 +-
 .../ReplicationOperationTests.java            |    6 +
 ...ReplicationAllPermitsAcquisitionTests.java |    4 +-
 .../AbstractTermVectorsTestCase.java          |    2 +-
 .../action/termvectors/GetTermVectorsIT.java  |   55 +-
 .../termvectors/GetTermVectorsTests.java      |    2 +-
 .../termvectors/MultiTermVectorsIT.java       |   28 +-
 .../termvectors/TermVectorsUnitTests.java     |   68 +-
 .../TransportMultiTermVectorsActionTests.java |   12 +-
 .../elasticsearch/aliases/IndexAliasesIT.java |   70 +-
 .../elasticsearch/blocks/SimpleBlocksIT.java  |   24 +-
 .../RecoveryWithUnsupportedIndicesIT.java     |   17 +-
 .../client/AbstractClientHeadersTestCase.java |    6 +-
 .../client/OriginSettingClientTests.java      |    4 +-
 .../ParentTaskAssigningClientTests.java       |    4 +-
 .../client/node/NodeClientHeadersTests.java   |   10 +-
 .../cluster/ClusterChangedEventTests.java     |   15 +-
 .../cluster/ClusterModuleTests.java           |   27 -
 .../cluster/ClusterStateDiffIT.java           |   14 +-
 .../elasticsearch/cluster/DiskUsageTests.java |    2 +-
 .../cluster/FeatureAwareTests.java            |   21 +-
 .../cluster/MinimumMasterNodesIT.java         |    4 +-
 .../elasticsearch/cluster/NoMasterNodeIT.java |    9 +-
 .../cluster/NodeConnectionsServiceTests.java  |    6 +-
 .../cluster/SimpleClusterStateIT.java         |    2 -
 .../cluster/SimpleDataNodesIT.java            |   54 +-
 .../cluster/SnapshotsInProgressTests.java     |    3 +-
 .../cluster/SpecificMasterNodesIT.java        |    4 +-
 .../action/shard/ShardStateActionTests.java   |    6 +-
 .../ClusterBootstrapServiceTests.java         |   33 +-
 .../ClusterFormationFailureHelperTests.java   |  114 +-
 .../coordination/CoordinationStateTests.java  |  202 +-
 .../coordination/CoordinatorTests.java        | 1332 +-------
 .../coordination/FollowersCheckerTests.java   |    6 +-
 .../cluster/coordination/JoinHelperTests.java |    4 +-
 .../coordination/JoinTaskExecutorTests.java   |   29 +-
 .../coordination/LeaderCheckerTests.java      |   15 +-
 .../cluster/coordination/NodeJoinTests.java   |   14 +-
 .../coordination/PreVoteCollectorTests.java   |    6 +-
 .../coordination/PublicationTests.java        |    8 +-
 .../coordination/RareClusterStateIT.java      |    6 +-
 .../coordination/ReconfiguratorTests.java     |    2 +-
 .../UnsafeBootstrapAndDetachCommandIT.java    |    6 +-
 .../coordination/VotingConfigurationIT.java   |   75 +-
 .../cluster/coordination/ZenDiscoveryIT.java  |    2 -
 .../metadata/AutoExpandReplicasTests.java     |   16 +-
 .../MetaDataCreateIndexServiceTests.java      |    9 +-
 .../MetaDataDeleteIndexServiceTests.java      |    4 +-
 .../MetaDataIndexStateServiceTests.java       |    9 +-
 .../metadata/TemplateUpgradeServiceTests.java |    6 +-
 .../cluster/node/DiscoveryNodeRoleIT.java     |   99 +
 .../cluster/node/DiscoveryNodeRoleTests.java  |  100 +
 .../cluster/node/DiscoveryNodesTests.java     |   38 +-
 .../routing/BatchedRerouteServiceTests.java   |  146 +
 .../DelayedAllocationServiceTests.java        |    4 +-
 .../cluster/routing/PrimaryAllocationIT.java  |    3 -
 .../allocation/AllocationCommandsTests.java   |   80 +-
 .../allocation/BalanceConfigurationTests.java |   22 +-
 .../allocation/FailedNodeRoutingTests.java    |   17 +-
 .../RetryFailedAllocationTests.java           |   97 +
 .../decider/DiskThresholdDeciderTests.java    |    7 +-
 .../DiskThresholdDeciderUnitTests.java        |   14 +-
 .../allocation/decider/MockDiskUsagesIT.java  |    6 +-
 ...storeInProgressAllocationDeciderTests.java |    3 +-
 .../cluster/service/ClusterServiceIT.java     |    2 -
 .../cluster/shards/ClusterShardLimitIT.java   |    2 +-
 .../common/geo/BaseGeoParsingTestCase.java    |    3 +-
 .../common/geo/GeoDistanceTests.java          |   17 -
 .../common/geo/GeoJsonParserTests.java        |   47 +-
 .../common/geo/GeoJsonSerializationTests.java |    4 +-
 .../common/geo/GeometryParserTests.java       |   42 +-
 .../common/io/stream/StreamTests.java         |   29 +
 .../common/joda/JodaDateMathParserTests.java  |    5 +
 .../common/lucene/uid/VersionLookupTests.java |    5 +
 .../common/lucene/uid/VersionsTests.java      |   13 +
 .../common/network/NetworkModuleTests.java    |   26 +-
 .../common/network/NetworkUtilsTests.java     |   30 +-
 .../common/regex/RegexTests.java              |   20 +
 .../common/settings/ConsistentSettingsIT.java |  188 ++
 .../ConsistentSettingsServiceTests.java       |  159 +
 .../common/settings/SettingsModuleTests.java  |   36 +
 .../common/time/JavaDateMathParserTests.java  |   18 +
 .../common/util/CuckooFilterTests.java        |  135 +
 .../SetBackedScalingCuckooFilterTests.java    |  231 ++
 .../concurrent/AsyncIOProcessorTests.java     |  134 +-
 ...ueueResizingEsThreadPoolExecutorTests.java |    7 +-
 .../discovery/ClusterDisruptionIT.java        |    4 +-
 .../discovery/DiscoveryDisruptionIT.java      |    2 -
 .../discovery/DiscoveryModuleTests.java       |    4 +-
 .../discovery/DiskDisruptionIT.java           |  178 +
 .../discovery/MasterDisruptionIT.java         |   17 -
 .../discovery/SeedHostsResolverTests.java     |   64 +-
 .../discovery/SnapshotDisruptionIT.java       |    4 +-
 .../discovery/StableMasterDisruptionIT.java   |    3 +-
 .../single/SingleNodeDiscoveryIT.java         |    5 +-
 .../elasticsearch/env/NodeEnvironmentIT.java  |  117 +-
 .../env/NodeEnvironmentTests.java             |   12 +-
 .../gateway/AsyncShardFetchTests.java         |    5 +-
 .../gateway/ClusterStateUpdatersTests.java    |    5 +-
 .../gateway/GatewayIndexStateIT.java          |   57 +-
 .../GatewayMetaStatePersistedStateTests.java  |    3 +-
 .../gateway/GatewayMetaStateTests.java        |    4 +-
 .../gateway/RecoverAfterNodesIT.java          |    2 +-
 .../elasticsearch/http/CorsHandlerTests.java  |   90 +
 .../http/DefaultRestChannelTests.java         |   80 +-
 .../elasticsearch/index/IndexModuleTests.java |   20 +-
 .../index/IndexServiceTests.java              |   47 +-
 .../index/IndexSettingsTests.java             |   15 +-
 .../index/SearchSlowLogTests.java             |    5 -
 .../index/SettingsListenerIT.java             |   17 +-
 .../index/analysis/IndexAnalyzersTests.java   |   53 +-
 .../ReloadableCustomAnalyzerTests.java        |  168 +
 .../engine/CombinedDeletionPolicyTests.java   |   48 -
 .../index/engine/InternalEngineMergeIT.java   |    2 -
 .../index/engine/InternalEngineTests.java     |  527 ++-
 .../engine/LuceneChangesSnapshotTests.java    |   17 +-
 .../index/engine/NoOpEngineTests.java         |   64 +-
 .../engine/PrunePostingsMergePolicyTests.java |  165 +
 .../index/engine/ReadOnlyEngineTests.java     |   66 +-
 .../fielddata/AbstractFieldDataTestCase.java  |    2 +-
 .../AbstractStringFieldDataTestCase.java      |    7 +-
 .../fielddata/IndexFieldDataServiceTests.java |    2 +-
 .../index/mapper/BooleanFieldMapperTests.java |   38 +-
 .../index/mapper/DateFieldTypeTests.java      |    4 +-
 .../index/mapper/DocumentParserTests.java     |   60 +
 .../mapper/ExternalFieldMapperTests.java      |   89 -
 .../ExternalValuesMapperIntegrationIT.java    |   10 +-
 .../mapper/FieldNamesFieldTypeTests.java      |    4 +-
 .../index/mapper/IdFieldTypeTests.java        |    7 -
 .../index/mapper/KeywordFieldMapperTests.java |   79 +
 .../index/mapper/MapperServiceTests.java      |  109 +-
 .../index/mapper/NestedObjectMapperTests.java |   42 +
 .../index/mapper/RangeFieldTypeTests.java     |    2 +-
 .../index/mapper/TextFieldMapperTests.java    |   89 +-
 .../index/mapper/TypeParsersTests.java        |   50 +-
 .../query/AbstractQueryBuilderTests.java      |    2 +-
 .../query/CommonTermsQueryBuilderTests.java   |  208 --
 .../query/CommonTermsQueryParserTests.java    |   50 -
 .../index/query/FullTextQueryTestCase.java    |   60 -
 .../query/GeoShapeQueryBuilderTests.java      |    2 +-
 .../index/query/IdsQueryBuilderTests.java     |   75 +-
 .../index/query/InnerHitBuilderTests.java     |    2 +-
 .../index/query/IntervalBuilderTests.java     |    6 +
 .../query/IntervalQueryBuilderTests.java      |  110 +-
 .../index/query/MatchQueryBuilderTests.java   |   49 +-
 .../query/MoreLikeThisQueryBuilderTests.java  |   51 +-
 .../query/MultiMatchQueryBuilderTests.java    |   58 +-
 .../index/query/QueryShardContextTests.java   |    2 +-
 .../query/QueryStringQueryBuilderTests.java   |  155 +-
 .../index/query/RandomQueryBuilder.java       |    3 +-
 .../index/query/RangeQueryBuilderTests.java   |   21 +
 .../index/query/RangeQueryRewriteTests.java   |   12 +-
 .../index/query/ScriptQueryBuilderTests.java  |   11 +-
 .../query/ScriptScoreQueryBuilderTests.java   |   11 +-
 .../query/SimpleQueryStringBuilderTests.java  |   50 +-
 .../query/SpanMultiTermQueryBuilderTests.java |   24 +-
 .../index/query/TermsQueryBuilderTests.java   |    7 -
 .../query/TermsSetQueryBuilderTests.java      |   36 +-
 .../index/query/TypeQueryBuilderTests.java    |    6 +
 .../FunctionScoreQueryBuilderTests.java       |   36 +-
 .../AbstractBulkByScrollRequestTestCase.java  |   15 +-
 .../reindex/DeleteByQueryRequestTests.java    |   51 +-
 .../index/reindex/ReindexRequestTests.java    |  248 +-
 .../reindex/UpdateByQueryRequestTests.java    |   27 +
 .../IndexLevelReplicationTests.java           |   17 +-
 .../RecoveryDuringReplicationTests.java       |   36 +-
 .../index/search/MatchPhraseQueryIT.java      |    5 +-
 .../index/search/MultiMatchQueryTests.java    |   14 +-
 .../search/nested/NestedSortingTests.java     |    2 +
 .../GlobalCheckpointSyncActionTests.java      |    2 +-
 .../index/seqno/GlobalCheckpointSyncIT.java   |   37 +-
 .../seqno/LocalCheckpointTrackerTests.java    |  148 +-
 .../index/seqno/ReplicationTrackerTests.java  |    4 +-
 ...tentionLeaseBackgroundSyncActionTests.java |   79 -
 .../seqno/RetentionLeaseSyncActionTests.java  |   74 -
 ...ests.java => IndexReaderWrapperTests.java} |   91 +-
 .../index/shard/IndexShardIT.java             |   14 +-
 .../index/shard/IndexShardTests.java          |  152 +-
 .../index/shard/NewPathForShardTests.java     |    2 +-
 .../shard/PrimaryReplicaSyncerTests.java      |    4 +-
 .../RemoveCorruptedShardDataCommandTests.java |   16 +-
 .../index/store/CorruptedFileIT.java          |    8 +-
 .../index/store/FsDirectoryFactoryTests.java  |   92 +-
 .../elasticsearch/index/store/StoreTests.java |   44 -
 .../termvectors/TermVectorsServiceTests.java  |   22 +-
 .../index/translog/TestTranslog.java          |   69 +-
 .../translog/TranslogDeletionPolicyTests.java |    2 +-
 .../index/translog/TranslogHeaderTests.java   |   50 +-
 .../index/translog/TranslogTests.java         |   86 +-
 .../elasticsearch/indexing/IndexActionIT.java |    4 +-
 .../IndexingMemoryControllerTests.java        |    5 +-
 .../indices/IndicesOptionsIntegrationIT.java  |   79 +-
 .../indices/IndicesRequestCacheIT.java        |    2 -
 .../indices/IndicesServiceTests.java          |    6 +-
 .../indices/analyze/AnalyzeActionIT.java      |   42 +-
 .../indices/cluster/ClusterStateChanges.java  |    2 +-
 .../FakeThreadPoolMasterServiceTests.java     |    4 +-
 ...ClusterStateServiceRandomUpdatesTests.java |   31 +-
 .../IndicesClusterStateServiceTests.java      |  221 ++
 .../exists/indices/IndicesExistsIT.java       |   91 -
 .../indices/exists/types/TypesExistsIT.java   |  107 -
 .../elasticsearch/indices/flush/FlushIT.java  |   26 +-
 .../RandomExceptionCircuitBreakerIT.java      |    8 +-
 .../recovery/IndexPrimaryRelocationIT.java    |    4 -
 .../indices/recovery/IndexRecoveryIT.java     |    3 +-
 .../PeerRecoveryTargetServiceTests.java       |   10 +-
 .../recovery/RecoverySourceHandlerTests.java  |   11 +-
 .../indices/recovery/RecoveryTests.java       |   12 +-
 .../InternalOrPrivateSettingsPlugin.java      |   11 +-
 .../indices/state/CloseIndexIT.java           |    5 +-
 .../state/CloseWhileRelocatingShardsIT.java   |    3 -
 .../indices/state/OpenCloseIndexIT.java       |   40 +-
 .../indices/state/ReopenWhileClosingIT.java   |   15 +-
 .../indices/stats/IndexStatsIT.java           |    2 +-
 .../ingest/IngestServiceTests.java            |   70 +-
 .../org/elasticsearch/node/NodeTests.java     |   55 +-
 .../DestructiveOperationsIT.java              |    2 +-
 .../PersistentTasksClusterServiceTests.java   |    3 +-
 .../PersistentTasksCustomMetaDataTests.java   |   37 +-
 .../PersistentTasksExecutorFullRestartIT.java |    2 -
 .../persistent/TestPersistentTasksPlugin.java |    9 +-
 .../plugins/PluginsServiceTests.java          |   12 +-
 .../recovery/RecoveryWhileUnderLoadIT.java    |    3 -
 .../elasticsearch/recovery/RelocationIT.java  |    9 -
 .../recovery/TruncatedRecoveryIT.java         |    3 +-
 .../RepositoriesServiceTests.java             |    3 +-
 .../repositories/fs/FsRepositoryTests.java    |    2 +-
 .../rest/RestControllerTests.java             |   87 +
 .../rest/action/RestActionsTests.java         |    2 +-
 .../admin/indices/RestAnalyzeActionTests.java |   62 +-
 .../indices/RestValidateQueryActionTests.java |    4 +-
 .../action/cat/RestIndicesActionTests.java    |  266 +-
 .../action/search/RestCountActionTests.java   |   62 -
 .../routing/SimpleRoutingIT.java              |   17 +-
 .../search/AbstractSearchTestCase.java        |    2 +-
 .../search/DefaultSearchContextTests.java     |    1 -
 .../search/SearchModuleTests.java             |   53 +-
 .../search/SearchServiceTests.java            |   19 +-
 .../AggregatorFactoriesBuilderTests.java      |    2 +-
 .../AggregatorFactoriesTests.java             |    2 +-
 .../BasePipelineAggregationTestCase.java      |    2 +-
 .../InternalAggregationsTests.java            |    2 +-
 .../bucket/AutoDateHistogramTests.java        |    1 +
 .../aggregations/bucket/BooleanTermsIT.java   |    6 +-
 .../aggregations/bucket/DateHistogramIT.java  |    1 -
 .../bucket/DiversifiedSamplerIT.java          |    4 +-
 .../aggregations/bucket/DoubleTermsIT.java    |    3 +-
 .../aggregations/bucket/HistogramIT.java      |    4 +-
 .../aggregations/bucket/LongTermsIT.java      |    4 +-
 .../aggregations/bucket/MinDocCountIT.java    |   13 +-
 .../search/aggregations/bucket/NestedIT.java  |    4 +-
 .../aggregations/bucket/RareTermsTests.java   |  101 +
 .../search/aggregations/bucket/SamplerIT.java |    6 +-
 .../aggregations/bucket/ShardSizeTermsIT.java |   32 +-
 .../bucket/ShardSizeTestCase.java             |    4 +-
 .../SignificantTermsSignificanceScoreIT.java  |   20 +-
 .../bucket/TermsDocCountErrorIT.java          |   94 +-
 ...djacencyMatrixAggregationBuilderTests.java |    2 +-
 ...oDateHistogramAggregationBuilderTests.java |   67 +
 .../bucket/histogram/ExtendedBoundsTests.java |    2 +-
 .../histogram/HistogramAggregatorTests.java   |   82 +
 .../InternalAutoDateHistogramTests.java       |   16 +-
 .../bucket/range/IpRangeAggregatorTests.java  |   41 +
 .../SignificanceHeuristicTests.java           |    4 +-
 .../terms/RareTermsAggregatorTests.java       |  600 ++++
 .../bucket/terms/StringTermsIT.java           |   57 +-
 .../bucket/terms/TermsAggregatorTests.java    |  113 +-
 .../metrics/CardinalityAggregatorTests.java   |   61 +-
 .../aggregations/metrics/CardinalityIT.java   |   34 +-
 .../metrics/MaxAggregatorTests.java           |   79 +-
 .../ScriptedMetricAggregatorTests.java        |    7 +-
 .../metrics/ValueCountAggregatorTests.java    |   63 +-
 .../WeightedAvgAggregationBuilderTests.java   |    2 +-
 .../AbstractBucketMetricsTestCase.java        |    1 -
 .../PipelineAggregationHelperTests.java       |    3 +-
 .../aggregations/pipeline/SerialDiffIT.java   |    4 +-
 .../support/ScriptValuesTests.java            |    6 +-
 .../support/ValuesSourceTypeTests.java        |   69 +
 .../basic/SearchWhileCreatingIndexIT.java     |    2 -
 .../basic/SearchWithRandomExceptionsIT.java   |    8 +-
 .../basic/SearchWithRandomIOExceptionsIT.java |    7 +-
 .../search/collapse/CollapseBuilderTests.java |    2 +-
 .../search/fetch/FetchSubPhasePluginIT.java   |    2 +-
 .../subphase/FetchSourceSubPhaseTests.java    |    4 +-
 .../highlight/CustomHighlighterSearchIT.java  |    9 +-
 .../highlight/HighlightBuilderTests.java      |    6 +-
 .../highlight/HighlighterSearchIT.java        |   59 +-
 .../search/fields/SearchFieldsIT.java         |    6 +-
 .../search/geo/GeoDistanceIT.java             |    1 -
 .../search/geo/GeoShapeQueryTests.java        |   10 +-
 .../internal/ContextIndexSearcherTests.java   |  425 +++
 .../ShardSearchTransportRequestTests.java     |    1 -
 .../search/lookup/LeafDocLookupTests.java     |   26 +-
 .../search/lookup/LeafFieldsLookupTests.java  |    4 +-
 .../search/morelikethis/MoreLikeThisIT.java   |   64 +-
 .../search/nested/SimpleNestedIT.java         |   72 +-
 .../search/profile/query/QueryProfilerIT.java |    1 -
 .../profile/query/QueryProfilerTests.java     |    9 +-
 .../profile/query/RandomQueryGenerator.java   |   33 +-
 .../search/query/MultiMatchQueryIT.java       |   73 -
 .../search/query/QueryPhaseTests.java         |   33 +
 .../search/query/QuerySearchResultTests.java  |    2 +-
 .../search/query/QueryStringIT.java           |    1 +
 .../search/query/SearchQueryIT.java           |  116 +-
 .../search/query/SimpleQueryStringIT.java     |    5 +-
 .../rescore/QueryRescorerBuilderTests.java    |   10 +-
 .../search/scroll/DuelScrollIT.java           |    9 +-
 .../search/scroll/SearchScrollIT.java         |    4 +-
 .../search/slice/SliceBuilderTests.java       |    5 -
 .../search/slice/TermsSliceQueryTests.java    |   19 +-
 .../search/sort/AbstractSortTestCase.java     |    7 +-
 .../search/sort/FieldSortBuilderTests.java    |   73 +-
 .../search/sort/FieldSortIT.java              |   15 +-
 .../search/sort/GeoDistanceIT.java            |   26 +-
 .../sort/GeoDistanceSortBuilderTests.java     |   91 +-
 .../search/sort/NestedSortBuilderTests.java   |    2 +-
 .../search/sort/ScriptSortBuilderTests.java   |   36 +-
 .../search/sort/SortBuilderTests.java         |   12 +-
 .../AbstractSuggestionBuilderTestCase.java    |    6 +-
 .../suggest/CompletionSuggestSearchIT.java    |   23 +-
 .../search/suggest/SuggestBuilderTests.java   |    2 +-
 .../search/suggest/SuggestSearchIT.java       |   66 +-
 .../search/suggest/SuggestTests.java          |    2 +-
 .../AbstractSnapshotIntegTestCase.java        |   30 +-
 .../DedicatedClusterSnapshotRestoreIT.java    |   52 +-
 ...etadataLoadingDuringSnapshotRestoreIT.java |    6 +-
 .../MinThreadsSnapshotRestoreIT.java          |    6 +-
 .../snapshots/RepositoriesIT.java             |    2 +
 .../SharedClusterSnapshotRestoreIT.java       |  280 +-
 .../snapshots/SnapshotInfoTests.java          |  149 +
 .../snapshots/SnapshotResiliencyTests.java    |  200 +-
 .../snapshots/SnapshotShardsServiceIT.java    |    4 +-
 ...SnapshotsInProgressSerializationTests.java |    3 +-
 .../mockstore/BlobContainerWrapper.java       |   10 +
 .../snapshots/mockstore/MockRepository.java   |   22 +
 .../SharedSignificantTermsTestMethods.java    |    2 +-
 .../threadpool/SchedulerTests.java            |   12 +
 .../RemoteClusterConnectionTests.java         |    2 -
 .../transport/RemoteClusterServiceTests.java  |  108 +-
 .../TransportServiceHandshakeTests.java       |   39 -
 .../org/elasticsearch/update/UpdateIT.java    |   14 +-
 .../validate/SimpleValidateQueryIT.java       |   24 +-
 .../ConcurrentSeqNoVersioningIT.java          |   24 +-
 .../action/search/simple-msearch1.json        |    2 +-
 .../action/search/simple-msearch2.json        |    2 +-
 .../action/search/simple-msearch3.json        |    4 +-
 .../action/search/simple-msearch4.json        |    4 +-
 .../action/termvectors/multiRequest1.json     |    3 +-
 .../action/termvectors/multiRequest2.json     |    6 +-
 .../action/termvectors/multiRequest3.json     |    1 -
 .../index/query/commonTerms-query1.json       |   11 -
 .../index/query/commonTerms-query2.json       |   11 -
 .../index/query/commonTerms-query3.json       |    9 -
 .../index/translog/translog-v2.binary         |  Bin 0 -> 43 bytes
 settings.gradle                               |   72 +-
 .../src/main/java/hdfs/MiniHDFS.java          |   25 +-
 test/fixtures/krb5kdc-fixture/build.gradle    |    3 -
 .../krb5kdc-fixture/docker-compose.yml        |    4 +
 test/framework/build.gradle                   |   16 +-
 .../cluster/ESAllocationTestCase.java         |    8 +-
 .../AbstractCoordinatorTestCase.java          | 1362 ++++++++
 .../CoordinationStateTestCluster.java         |  264 ++
 .../coordination/LinearizabilityChecker.java  |    0
 .../common/settings/MockSecureSettings.java   |   13 +
 .../gateway/MockGatewayMetaState.java         |    2 -
 .../index/engine/EngineTestCase.java          |   22 +-
 .../index/engine/TranslogHandler.java         |    3 +-
 .../ESIndexLevelReplicationTestCase.java      |   23 +-
 .../index/shard/IndexShardTestCase.java       |   47 +-
 .../index/shard/RestoreOnlyRepository.java    |    2 +-
 .../cluster/FakeThreadPoolMasterService.java  |    0
 .../indices/recovery/AsyncRecoveryTarget.java |   11 +-
 .../AbstractThirdPartyRepositoryTestCase.java |  300 ++
 .../blobstore/BlobStoreTestUtil.java          |  126 +
 .../ESBlobStoreRepositoryIntegTestCase.java   |    2 +-
 .../search/RandomSearchRequestGenerator.java  |    3 -
 .../aggregations/AggregatorTestCase.java      |   82 +-
 .../AbstractBroadcastResponseTestCase.java    |    3 +-
 .../test/AbstractBuilderTestCase.java         |    8 +-
 .../test/AbstractQueryTestCase.java           |   28 +-
 .../elasticsearch/test/BackgroundIndexer.java |   65 +-
 .../test/ClusterServiceUtils.java             |    6 +-
 .../elasticsearch/test/ESIntegTestCase.java   |   39 +-
 .../org/elasticsearch/test/ESTestCase.java    |   17 +
 .../test/InternalAggregationTestCase.java     |    2 +-
 .../test/InternalTestCluster.java             |  145 +-
 .../test/MockIndexEventListener.java          |   17 +-
 .../elasticsearch/test/MockLogAppender.java   |    2 +-
 .../test/NodeConfigurationSource.java         |   15 -
 .../elasticsearch/test/client/NoOpClient.java |    4 +-
 .../disruption/DisruptableMockTransport.java  |   14 +-
 .../test/engine/MockEngineSupport.java        |   19 +-
 .../hamcrest/ElasticsearchAssertions.java     |   17 +-
 .../test/hamcrest/OptionalMatchers.java       |   84 +
 .../junit/listeners/ReproduceInfoPrinter.java |   10 +-
 .../test/rest/ESRestTestCase.java             |   14 +-
 .../yaml/restspec/ClientYamlSuiteRestApi.java |   13 +
 .../ClientYamlSuiteRestApiParser.java         |   17 +
 .../test/transport/MockTransportService.java  |   23 +-
 .../threadpool/TestThreadPool.java            |    8 +-
 .../transport/MockTransportClient.java        |   69 -
 .../transport/nio/MockNioTransport.java       |   21 +-
 .../search/MockSearchServiceTests.java        |    4 +-
 .../elasticsearch/test/VersionUtilsTests.java |   49 +-
 .../DisruptableMockTransportTests.java        |   25 +
 .../test/disruption/NetworkDisruptionIT.java  |   90 +-
 ...entYamlSuiteRestApiParserFailingTests.java |    4 +
 .../ClientYamlSuiteRestApiParserTests.java    |    4 +
 .../test/test/InternalTestClusterIT.java      |   25 +
 .../test/test/InternalTestClusterTests.java   |   77 +-
 .../transport/nio/TestEventHandlerTests.java  |    5 +-
 test/logger-usage/build.gradle                |    2 +-
 x-pack/build.gradle                           |   14 -
 x-pack/docs/build.gradle                      |    1 -
 x-pack/docs/en/rest-api/security.asciidoc     |    2 +
 .../security/create-api-keys.asciidoc         |   12 +-
 .../rest-api/security/get-api-keys.asciidoc   |    4 +
 .../security/get-builtin-privileges.asciidoc  |  107 +
 .../en/rest-api/security/get-tokens.asciidoc  |   43 +-
 .../security/invalidate-api-keys.asciidoc     |   11 +-
 .../authentication/oidc-guide.asciidoc        |   52 +-
 .../cross-cluster.asciidoc                    |    6 +-
 .../docs/en/security/configuring-es.asciidoc  |    3 -
 .../security/securing-communications.asciidoc |    5 -
 x-pack/docs/en/watcher/actions.asciidoc       |   43 +
 .../watcher/condition/array-compare.asciidoc  |    2 +-
 .../en/watcher/how-watcher-works.asciidoc     |    6 +-
 x-pack/license-tools/build.gradle             |    6 +-
 x-pack/plugin/build.gradle                    |  106 +-
 x-pack/plugin/ccr/build.gradle                |    2 +-
 x-pack/plugin/ccr/qa/build.gradle             |    1 +
 .../downgrade-to-basic-license/build.gradle   |   58 +-
 .../xpack/ccr/FollowIndexIT.java              |    2 +-
 .../plugin/ccr/qa/multi-cluster/build.gradle  |   75 +-
 .../ccr/qa/non-compliant-license/build.gradle |   42 +-
 x-pack/plugin/ccr/qa/rest/build.gradle        |   20 +-
 x-pack/plugin/ccr/qa/restart/build.gradle     |   69 +-
 x-pack/plugin/ccr/qa/security/build.gradle    |   76 +-
 .../xpack/ccr/ESCCRRestTestCase.java          |    5 +
 .../java/org/elasticsearch/xpack/ccr/Ccr.java |   27 +-
 .../xpack/ccr/CcrLicenseChecker.java          |    6 +-
 .../xpack/ccr/action/CcrRequests.java         |   59 +-
 .../xpack/ccr/action/ShardChangesAction.java  |   79 +-
 .../xpack/ccr/action/ShardFollowNodeTask.java |   67 +-
 .../xpack/ccr/action/ShardFollowTask.java     |    2 +-
 .../ccr/action/ShardFollowTasksExecutor.java  |  135 +-
 .../ccr/action/TransportCcrStatsAction.java   |    7 +-
 ...ransportDeleteAutoFollowPatternAction.java |   12 +-
 .../ccr/action/TransportFollowInfoAction.java |    3 +-
 .../TransportGetAutoFollowPatternAction.java  |    3 +-
 .../action/TransportPauseFollowAction.java    |   12 +-
 .../TransportPutAutoFollowPatternAction.java  |   12 +-
 .../ccr/action/TransportPutFollowAction.java  |    7 +-
 .../action/TransportResumeFollowAction.java   |   11 +-
 .../ccr/action/TransportUnfollowAction.java   |   16 +-
 .../bulk/BulkShardOperationsAction.java       |    4 +-
 .../TransportBulkShardOperationsAction.java   |    7 +-
 .../ClearCcrRestoreSessionAction.java         |    9 +-
 .../DeleteInternalCcrRepositoryAction.java    |    9 +-
 .../GetCcrRestoreFileChunkAction.java         |    9 +-
 .../PutCcrRestoreSessionAction.java           |    9 +-
 .../PutCcrRestoreSessionRequest.java          |   16 +-
 .../PutInternalCcrRepositoryAction.java       |    9 +-
 .../ccr/index/engine/FollowingEngine.java     |    2 +-
 .../xpack/ccr/repository/CcrRepository.java   |   17 +-
 .../elasticsearch/xpack/CcrIntegTestCase.java |   10 -
 .../elasticsearch/xpack/ccr/AutoFollowIT.java |   32 +-
 .../xpack/ccr/CCRFeatureSetUsageTests.java    |   11 +-
 ....java => CCRInfoTransportActionTests.java} |   39 +-
 .../elasticsearch/xpack/ccr/CcrAliasesIT.java |  412 +++
 .../xpack/ccr/CcrRetentionLeaseIT.java        |  224 +-
 .../xpack/ccr/FollowerFailOverIT.java         |   13 +-
 .../xpack/ccr/IndexFollowingIT.java           |  112 +-
 .../xpack/ccr/LocalIndexFollowingIT.java      |    3 +-
 .../action/AutoFollowCoordinatorTests.java    |   11 +-
 .../ccr/action/ShardChangesActionTests.java   |   20 +-
 .../ccr/action/ShardChangesRequestTests.java  |    9 +-
 .../ccr/action/ShardChangesResponseTests.java |    2 +
 .../ShardFollowNodeTaskRandomTests.java       |   48 +-
 .../ShardFollowNodeTaskStatusTests.java       |    2 +
 .../ccr/action/ShardFollowNodeTaskTests.java  |  154 +-
 .../ShardFollowTaskReplicationTests.java      |   53 +-
 .../xpack/ccr/action/StatsResponsesTests.java |    1 +
 .../TransportResumeFollowActionTests.java     |   11 -
 .../action/bulk/BulkShardOperationsTests.java |    2 +-
 .../index/engine/FollowingEngineTests.java    |   36 +-
 .../ccr/FollowStatsMonitoringDocTests.java    |    4 +
 .../collector/ccr/StatsCollectorTests.java    |    8 +-
 x-pack/plugin/core/build.gradle               |    9 +-
 .../licenses/log4j-1.2-api-2.11.1.jar.sha1    |    1 +
 x-pack/plugin/core/licenses/log4j-LICENSE.txt |  202 ++
 x-pack/plugin/core/licenses/log4j-NOTICE.txt  |    5 +
 .../core/src/main/config/log4j2.properties    |    3 +-
 .../index/engine/FrozenEngine.java            |   12 +-
 .../license/DeleteLicenseAction.java          |    9 +-
 .../license/GetBasicStatusAction.java         |    4 +-
 .../license/GetBasicStatusRequestBuilder.java |    4 +-
 .../license/GetLicenseAction.java             |    4 +-
 .../license/GetTrialStatusAction.java         |    4 +-
 .../license/GetTrialStatusRequestBuilder.java |    4 +-
 .../elasticsearch/license/LicenseUtils.java   |   12 +-
 .../license/LicensingClient.java              |    8 +-
 .../license/PostStartBasicAction.java         |    9 +-
 .../license/PostStartBasicRequestBuilder.java |    4 +-
 .../license/PostStartBasicResponse.java       |   42 +-
 .../license/PostStartTrialAction.java         |    4 +-
 .../license/PostStartTrialRequestBuilder.java |    4 +-
 .../license/PutLicenseAction.java             |    9 +-
 .../license/RestDeleteLicenseAction.java      |   12 +-
 .../license/RestGetBasicStatus.java           |   12 +-
 .../license/RestGetLicenseAction.java         |   14 +-
 .../license/RestGetTrialStatus.java           |   12 +-
 .../license/RestPostStartBasicLicense.java    |   12 +-
 .../license/RestPostStartTrialLicense.java    |   14 +-
 .../license/RestPutLicenseAction.java         |   15 +-
 .../license/TransportDeleteLicenseAction.java |   15 +-
 .../TransportGetBasicStatusAction.java        |    3 +-
 .../license/TransportGetLicenseAction.java    |    3 +-
 .../TransportGetTrialStatusAction.java        |    3 +-
 .../TransportPostStartBasicAction.java        |   13 +-
 .../TransportPostStartTrialAction.java        |    3 +-
 .../license/TransportPutLicenseAction.java    |   15 +-
 .../license/XPackLicenseState.java            |   31 +-
 .../protocol/xpack/XPackInfoResponse.java     |   52 +-
 .../xpack/license/PutLicenseResponse.java     |   42 +-
 .../snapshots/SourceOnlySnapshot.java         |    4 +-
 .../SourceOnlySnapshotRepository.java         |    5 +-
 .../CCRInfoTransportAction.java}              |   63 +-
 .../xpack/ccr/CCRUsageTransportAction.java    |   75 +
 .../xpack/core/ClientHelper.java              |    6 +-
 .../xpack/core/EmptyXPackFeatureSet.java      |   43 -
 .../elasticsearch/xpack/core/XPackClient.java |  128 -
 .../xpack/core/XPackClientPlugin.java         |   68 +-
 .../xpack/core/XPackFeatureSet.java           |    8 -
 .../elasticsearch/xpack/core/XPackField.java  |    6 +
 .../elasticsearch/xpack/core/XPackPlugin.java |   45 +-
 .../xpack/core/XPackSettings.java             |    9 +
 .../AbstractTransportGetResourcesAction.java  |   11 +-
 .../core/action/ReloadAnalyzerAction.java     |   23 +
 .../core/action/ReloadAnalyzersRequest.java   |   43 +
 .../core/action/ReloadAnalyzersResponse.java  |  132 +
 .../action/TransportFreezeIndexAction.java    |   31 +-
 .../TransportReloadAnalyzersAction.java       |  160 +
 .../core/action/TransportXPackInfoAction.java |   34 +-
 .../action/TransportXPackUsageAction.java     |   34 +-
 .../xpack/core/action/XPackInfoAction.java    |    4 +-
 .../core/action/XPackInfoFeatureAction.java   |   58 +
 .../core/action/XPackInfoFeatureResponse.java |   42 +
 .../XPackInfoFeatureTransportAction.java      |   38 +
 .../xpack/core/action/XPackUsageAction.java   |    4 +-
 .../core/action/XPackUsageFeatureAction.java  |   58 +
 .../action/XPackUsageFeatureResponse.java     |   42 +
 .../XPackUsageFeatureTransportAction.java     |   41 +
 .../xpack/core/ccr/AutoFollowMetadata.java    |    2 +-
 .../xpack/core/ccr/AutoFollowStats.java       |   15 +-
 .../core/ccr/ShardFollowNodeTaskStatus.java   |   31 +-
 .../xpack/core/ccr/action/CcrStatsAction.java |    9 +-
 .../action/DeleteAutoFollowPatternAction.java |    9 +-
 .../core/ccr/action/FollowInfoAction.java     |    9 +-
 .../core/ccr/action/FollowStatsAction.java    |    9 +-
 .../core/ccr/action/ForgetFollowerAction.java |    4 +-
 .../action/GetAutoFollowPatternAction.java    |    9 +-
 .../core/ccr/action/PauseFollowAction.java    |    9 +-
 .../action/PutAutoFollowPatternAction.java    |    9 +-
 .../core/ccr/action/PutFollowAction.java      |    9 +-
 .../core/ccr/action/ResumeFollowAction.java   |    9 +-
 .../xpack/core/ccr/action/UnfollowAction.java |    9 +-
 .../xpack/core/ccr/client/CcrClient.java      |  147 -
 .../{ml/utils => common}/time/TimeUtils.java  |   17 +-
 .../xpack/core/dataframe/DataFrameField.java  |    5 +
 .../core/dataframe/DataFrameMessages.java     |   12 +-
 .../DataFrameNamedXContentProvider.java       |   26 +
 .../DeleteDataFrameTransformAction.java       |   15 +-
 .../action/GetDataFrameTransformsAction.java  |   13 +-
 .../GetDataFrameTransformsStatsAction.java    |   76 +-
 .../PreviewDataFrameTransformAction.java      |   89 +-
 .../action/PutDataFrameTransformAction.java   |   34 +-
 .../action/StartDataFrameTransformAction.java |   29 +-
 .../StartDataFrameTransformTaskAction.java    |    9 +-
 .../action/StopDataFrameTransformAction.java  |   55 +-
 .../notifications/DataFrameAuditMessage.java  |    2 +-
 .../transforms/DataFrameTransform.java        |   30 +-
 .../transforms/DataFrameTransformConfig.java  |  155 +-
 .../transforms/DataFrameTransformState.java   |   46 +-
 .../core/dataframe/transforms/DestConfig.java |   30 +-
 .../dataframe/transforms/NodeAttributes.java  |  171 +
 .../core/dataframe/transforms/SyncConfig.java |   25 +
 .../dataframe/transforms/TimeSyncConfig.java  |  148 +
 .../pivot/DateHistogramGroupSource.java       |   43 +-
 .../pivot/HistogramGroupSource.java           |   13 +
 .../transforms/pivot/PivotConfig.java         |    6 +-
 .../transforms/pivot/SingleGroupSource.java   |    6 +
 .../transforms/pivot/TermsGroupSource.java    |   13 +
 .../dataframe/utils/DataFrameStrings.java     |   46 +
 .../xpack/core/dataframe/utils/TimeUtils.java |  129 -
 .../deprecation/DeprecationInfoAction.java    |    4 +-
 .../NodesDeprecationCheckAction.java          |   10 +-
 .../flattened/FlattenedFeatureSetUsage.java   |   24 +
 .../core/graph/action/GraphExploreAction.java |    4 +-
 .../core/indexing/AsyncTwoPhaseIndexer.java   |   36 +-
 .../xpack/core/indexing/IndexerJobStats.java  |   29 +-
 .../IndexLifecycleMetadata.java               |    2 +-
 .../core/indexlifecycle/SegmentCountStep.java |   36 +-
 .../SetSingleNodeAllocateStep.java            |   47 +-
 .../action/DeleteLifecycleAction.java         |   12 +-
 .../action/ExplainLifecycleAction.java        |    4 +-
 .../action/GetLifecycleAction.java            |    4 +-
 .../action/GetStatusAction.java               |    4 +-
 .../action/MoveToStepAction.java              |   12 +-
 .../action/PutLifecycleAction.java            |   12 +-
 .../RemoveIndexLifecyclePolicyAction.java     |    4 +-
 .../indexlifecycle/action/RetryAction.java    |   12 +-
 .../indexlifecycle/action/StartILMAction.java |    9 +-
 .../indexlifecycle/action/StopILMAction.java  |    9 +-
 .../core/indexlifecycle/client/ILMClient.java |  165 -
 .../xpack/core/ml/MachineLearningField.java   |    2 +-
 .../elasticsearch/xpack/core/ml/MlTasks.java  |   31 +
 .../xpack/core/ml/action/CloseJobAction.java  |    9 +-
 .../core/ml/action/DeleteCalendarAction.java  |    9 +-
 .../ml/action/DeleteCalendarEventAction.java  |    9 +-
 .../DeleteDataFrameAnalyticsAction.java       |   95 +
 .../core/ml/action/DeleteDatafeedAction.java  |    9 +-
 .../ml/action/DeleteExpiredDataAction.java    |    4 +-
 .../core/ml/action/DeleteFilterAction.java    |    9 +-
 .../core/ml/action/DeleteForecastAction.java  |    9 +-
 .../xpack/core/ml/action/DeleteJobAction.java |    9 +-
 .../ml/action/DeleteModelSnapshotAction.java  |    9 +-
 .../ml/action/EvaluateDataFrameAction.java    |  215 ++
 .../ml/action/FinalizeJobExecutionAction.java |    9 +-
 .../ml/action/FindFileStructureAction.java    |   30 +-
 .../xpack/core/ml/action/FlushJobAction.java  |    9 +-
 .../core/ml/action/ForecastJobAction.java     |    9 +-
 .../core/ml/action/GetBucketsAction.java      |    4 +-
 .../ml/action/GetCalendarEventsAction.java    |    4 +-
 .../core/ml/action/GetCalendarsAction.java    |    4 +-
 .../core/ml/action/GetCategoriesAction.java   |    4 +-
 .../action/GetDataFrameAnalyticsAction.java   |   80 +
 .../GetDataFrameAnalyticsStatsAction.java     |  325 ++
 .../core/ml/action/GetDatafeedsAction.java    |    4 +-
 .../ml/action/GetDatafeedsStatsAction.java    |    4 +-
 .../core/ml/action/GetFiltersAction.java      |    5 +-
 .../core/ml/action/GetInfluencersAction.java  |    5 +-
 .../xpack/core/ml/action/GetJobsAction.java   |    4 +-
 .../core/ml/action/GetJobsStatsAction.java    |   76 +-
 .../ml/action/GetModelSnapshotsAction.java    |    4 +-
 .../ml/action/GetOverallBucketsAction.java    |    4 +-
 .../core/ml/action/GetRecordsAction.java      |    4 +-
 .../core/ml/action/IsolateDatafeedAction.java |    9 +-
 .../core/ml/action/KillProcessAction.java     |    9 +-
 .../xpack/core/ml/action/MlInfoAction.java    |    4 +-
 .../xpack/core/ml/action/OpenJobAction.java   |   17 +-
 .../core/ml/action/PersistJobAction.java      |    9 +-
 .../ml/action/PostCalendarEventsAction.java   |    4 +-
 .../xpack/core/ml/action/PostDataAction.java  |    9 +-
 .../core/ml/action/PreviewDatafeedAction.java |    4 +-
 .../core/ml/action/PutCalendarAction.java     |    4 +-
 .../action/PutDataFrameAnalyticsAction.java   |  153 +
 .../core/ml/action/PutDatafeedAction.java     |    4 +-
 .../xpack/core/ml/action/PutFilterAction.java |    4 +-
 .../xpack/core/ml/action/PutJobAction.java    |    4 +-
 .../ml/action/RevertModelSnapshotAction.java  |    4 +-
 .../core/ml/action/SetUpgradeModeAction.java  |    9 +-
 .../action/StartDataFrameAnalyticsAction.java |  245 ++
 .../core/ml/action/StartDatafeedAction.java   |   21 +-
 .../action/StopDataFrameAnalyticsAction.java  |  234 ++
 .../core/ml/action/StopDatafeedAction.java    |    9 +-
 .../ml/action/UpdateCalendarJobAction.java    |    4 +-
 .../core/ml/action/UpdateDatafeedAction.java  |    4 +-
 .../core/ml/action/UpdateFilterAction.java    |    4 +-
 .../xpack/core/ml/action/UpdateJobAction.java |    4 +-
 .../ml/action/UpdateModelSnapshotAction.java  |    4 +-
 .../core/ml/action/UpdateProcessAction.java   |    9 +-
 .../ml/action/ValidateDetectorAction.java     |    9 +-
 .../ml/action/ValidateJobConfigAction.java    |    9 +-
 .../xpack/core/ml/annotations/Annotation.java |    2 +-
 .../core/ml/calendars/ScheduledEvent.java     |    2 +-
 .../core/ml/client/MachineLearningClient.java |  385 ---
 .../core/ml/datafeed/DatafeedConfig.java      |   28 +-
 .../core/ml/datafeed/DatafeedUpdate.java      |   15 +-
 .../ml/datafeed/DelayedDataCheckConfig.java   |    2 +-
 .../dataframe/DataFrameAnalyticsConfig.java   |  385 +++
 .../ml/dataframe/DataFrameAnalyticsDest.java  |  106 +
 .../dataframe/DataFrameAnalyticsSource.java   |  149 +
 .../ml/dataframe/DataFrameAnalyticsState.java |   44 +
 .../DataFrameAnalyticsTaskState.java          |  121 +
 .../dataframe/analyses/DataFrameAnalysis.java |   16 +
 ...ataFrameAnalysisNamedXContentProvider.java |   37 +
 .../dataframe/analyses/OutlierDetection.java  |  167 +
 .../ml/dataframe/evaluation/Evaluation.java   |   37 +
 .../evaluation/EvaluationMetricResult.java    |   20 +
 .../MlEvaluationNamedXContentProvider.java    |   69 +
 .../AbstractConfusionMatrixMetric.java        |  102 +
 .../evaluation/softclassification/AucRoc.java |  350 ++
 .../BinarySoftClassification.java             |  212 ++
 .../softclassification/ConfusionMatrix.java   |  163 +
 .../softclassification/Precision.java         |   91 +
 .../evaluation/softclassification/Recall.java |   91 +
 .../ScoreByThresholdResult.java               |   63 +
 .../SoftClassificationMetric.java             |   60 +
 .../ml/filestructurefinder/FieldStats.java    |   57 +-
 .../core/ml/job/config/AnalysisConfig.java    |    2 +-
 .../config/CategorizationAnalyzerConfig.java  |   92 +-
 .../core/ml/job/config/DetectionRule.java     |    3 -
 .../xpack/core/ml/job/config/Detector.java    |    6 +-
 .../xpack/core/ml/job/config/Job.java         |    2 +-
 .../xpack/core/ml/job/config/JobUpdate.java   |   10 +-
 .../xpack/core/ml/job/config/MlFilter.java    |   13 +-
 .../xpack/core/ml/job/messages/Messages.java  |    3 +
 .../persistence/ElasticsearchMappings.java    |   86 +-
 .../output/FlushAcknowledgement.java          |    2 +-
 .../process/autodetect/state/DataCounts.java  |    2 +-
 .../autodetect/state/ModelSizeStats.java      |    2 +-
 .../autodetect/state/ModelSnapshot.java       |    2 +-
 .../process/autodetect/state/TimingStats.java |  222 ++
 .../core/ml/job/results/AnomalyRecord.java    |   11 +-
 .../xpack/core/ml/job/results/Bucket.java     |   11 +-
 .../core/ml/job/results/BucketInfluencer.java |    2 +-
 .../ml/job/results/CategoryDefinition.java    |    9 +-
 .../xpack/core/ml/job/results/Forecast.java   |    2 +-
 .../xpack/core/ml/job/results/Influencer.java |    2 +-
 .../xpack/core/ml/job/results/ModelPlot.java  |    2 +-
 .../ml/job/results/ReservedFieldNames.java    |   27 +
 .../core/ml/notifications/AuditMessage.java   |    2 +-
 .../core/ml/process/writer/RecordWriter.java  |    2 +-
 .../xpack/core/ml/utils/ExceptionsHelper.java |   13 +
 .../ml/{datafeed => utils}/QueryProvider.java |   23 +-
 .../action/MonitoringBulkAction.java          |    4 +-
 .../monitoring/client/MonitoringClient.java   |   60 -
 .../xpack/core/rest/XPackRestHandler.java     |   30 -
 .../rest/action/RestFreezeIndexAction.java    |   11 +-
 .../action/RestReloadAnalyzersAction.java     |   40 +
 .../core/rest/action/RestXPackInfoAction.java |   15 +-
 .../rest/action/RestXPackUsageAction.java     |   14 +-
 .../rollup/action/DeleteRollupJobAction.java  |    9 +-
 .../rollup/action/GetRollupCapsAction.java    |    5 +-
 .../action/GetRollupIndexCapsAction.java      |    5 +-
 .../rollup/action/GetRollupJobsAction.java    |    9 +-
 .../rollup/action/PutRollupJobAction.java     |    9 +-
 .../rollup/action/RollupSearchAction.java     |    9 +-
 .../rollup/action/StartRollupJobAction.java   |    9 +-
 .../rollup/action/StopRollupJobAction.java    |   22 +-
 .../security/action/CreateApiKeyAction.java   |   11 +-
 .../security/action/CreateApiKeyRequest.java  |   12 +-
 .../action/CreateApiKeyRequestBuilder.java    |    2 +-
 .../core/security/action/GetApiKeyAction.java |   11 +-
 .../action/InvalidateApiKeyAction.java        |   11 +-
 .../oidc/OpenIdConnectAuthenticateAction.java |   11 +-
 .../oidc/OpenIdConnectLogoutAction.java       |    9 +-
 ...nIdConnectPrepareAuthenticationAction.java |    9 +-
 .../privilege/DeletePrivilegesAction.java     |    6 +-
 .../DeletePrivilegesRequestBuilder.java       |    4 +-
 .../privilege/GetBuiltinPrivilegesAction.java |   26 +
 .../GetBuiltinPrivilegesRequest.java          |   23 +
 .../GetBuiltinPrivilegesResponse.java         |   61 +
 .../action/privilege/GetPrivilegesAction.java |    6 +-
 .../privilege/GetPrivilegesRequest.java       |    1 -
 .../GetPrivilegesRequestBuilder.java          |    4 +-
 .../privilege/GetPrivilegesResponse.java      |   12 +-
 .../action/privilege/PutPrivilegesAction.java |    6 +-
 .../PutPrivilegesRequestBuilder.java          |    4 +-
 .../action/realm/ClearRealmCacheAction.java   |    4 +-
 .../action/realm/ClearRealmCacheRequest.java  |    3 +-
 .../action/role/ClearRolesCacheAction.java    |    4 +-
 .../action/role/ClearRolesCacheRequest.java   |    3 +-
 .../action/role/DeleteRoleAction.java         |    6 +-
 .../security/action/role/GetRolesAction.java  |    6 +-
 .../security/action/role/PutRoleAction.java   |    6 +-
 .../rolemapping/DeleteRoleMappingAction.java  |    6 +-
 .../DeleteRoleMappingRequestBuilder.java      |    5 +-
 .../rolemapping/GetRoleMappingsAction.java    |    6 +-
 .../GetRoleMappingsRequestBuilder.java        |    4 +-
 .../rolemapping/PutRoleMappingAction.java     |    6 +-
 .../PutRoleMappingRequestBuilder.java         |    4 +-
 .../action/saml/SamlAuthenticateAction.java   |    6 +-
 .../saml/SamlInvalidateSessionAction.java     |    6 +-
 .../action/saml/SamlLogoutAction.java         |    6 +-
 .../saml/SamlPrepareAuthenticationAction.java |    6 +-
 .../action/token/CreateTokenAction.java       |    6 +-
 .../action/token/CreateTokenRequest.java      |  132 +-
 .../token/CreateTokenRequestBuilder.java      |    4 +-
 .../action/token/CreateTokenResponse.java     |   28 +-
 .../action/token/InvalidateTokenAction.java   |    6 +-
 .../action/token/RefreshTokenAction.java      |    6 +-
 .../action/user/AuthenticateAction.java       |    4 +-
 .../action/user/ChangePasswordAction.java     |    4 +-
 .../user/ChangePasswordRequestBuilder.java    |    2 +-
 .../action/user/DeleteUserAction.java         |    6 +-
 .../action/user/GetUserPrivilegesAction.java  |    6 +-
 .../security/action/user/GetUsersAction.java  |    6 +-
 .../action/user/HasPrivilegesAction.java      |    4 +-
 .../security/action/user/PutUserAction.java   |    6 +-
 .../action/user/PutUserRequestBuilder.java    |   17 +-
 .../action/user/SetEnabledAction.java         |    4 +-
 .../core/security/authc/support/Hasher.java   |    4 +-
 .../DocumentSubsetBitsetCache.java            |  206 ++
 .../accesscontrol/DocumentSubsetReader.java   |   44 +-
 .../SecurityIndexReaderWrapper.java           |  110 +
 .../SecurityIndexSearcherWrapper.java         |  217 --
 .../authz/privilege/ApplicationPrivilege.java |    2 +-
 .../authz/privilege/ClusterPrivilege.java     |    7 +
 .../authz/privilege/IndexPrivilege.java       |   15 +-
 .../authz/store/ReservedRolesStore.java       |   16 +-
 .../core/security/client/SecurityClient.java  |  392 ---
 .../security/support/CacheIteratorHelper.java |   59 +
 .../core/security/support/MetadataUtils.java  |    3 +-
 .../core/security/support/Validation.java     |    9 +-
 .../core/security/user/APMSystemUser.java     |    2 -
 .../security/user/RemoteMonitoringUser.java   |    3 -
 .../xpack/core/security/user/User.java        |    7 +-
 .../xpack/core/ssl/CertParsingUtils.java      |   22 +-
 .../xpack/core/ssl/TrustConfig.java           |   14 +-
 .../ssl/action/GetCertificateInfoAction.java  |    4 +-
 .../core/template/IndexTemplateRegistry.java  |    3 +-
 .../xpack/core/template/TemplateUtils.java    |   11 +-
 .../upgrade/actions/IndexUpgradeAction.java   |    4 +-
 .../actions/IndexUpgradeInfoAction.java       |    4 +-
 .../core/vectors/VectorsFeatureSetUsage.java  |   24 +
 .../VotingOnlyNodeFeatureSetUsage.java        |   22 +
 .../core/watcher/actions/ActionWrapper.java   |  113 +-
 .../watcher/client/WatchSourceBuilder.java    |   17 +-
 .../core/watcher/client/WatcherClient.java    |  326 --
 .../WatcherIndexTemplateRegistryField.java    |    3 +-
 .../xcontent/WatcherXContentParser.java       |    5 -
 .../transport/actions/ack/AckWatchAction.java |    4 +-
 .../actions/activate/ActivateWatchAction.java |    4 +-
 .../actions/delete/DeleteWatchAction.java     |    4 +-
 .../actions/execute/ExecuteWatchAction.java   |    4 +-
 .../transport/actions/get/GetWatchAction.java |    4 +-
 .../transport/actions/put/PutWatchAction.java |    4 +-
 .../actions/service/WatcherServiceAction.java |    9 +-
 .../actions/stats/WatcherStatsAction.java     |    6 +-
 .../actions/stats/WatcherStatsRequest.java    |    3 +-
 .../xpack/core/watcher/watch/WatchField.java  |    1 +
 .../src/main/resources/monitoring-es.json     |    3 +
 .../src/main/resources/watch-history.json     |    7 +
 .../index/engine/FrozenEngineTests.java       |   12 +-
 .../index/engine/FrozenIndexTests.java        |  133 +-
 .../license/XPackLicenseStateTests.java       |   16 +
 .../snapshots/SourceOnlySnapshotIT.java       |    7 +-
 .../SourceOnlySnapshotShardTests.java         |    4 +-
 .../core/LocalStateCompositeXPackPlugin.java  |   51 +-
 .../action/ReloadAnalyzersResponseTests.java  |   56 +
 .../action/ReloadSynonymAnalyzerTests.java    |  107 +
 .../action/TransportXPackInfoActionTests.java |   52 +-
 .../AbstractAuditMessageTests.java            |    2 +-
 .../core/common}/time/TimeUtilsTests.java     |   49 +-
 .../AbstractSerializingDataFrameTestCase.java |   10 +-
 ...tractWireSerializingDataFrameTestCase.java |   10 +-
 ...ataFrameTransformsActionResponseTests.java |    6 +-
 ...ameTransformsStatsActionResponseTests.java |    2 +-
 ...wDataFrameTransformActionRequestTests.java |   12 +-
 ...ataFrameTransformsActionResponseTests.java |   27 +-
 ...pDataFrameTransformActionRequestTests.java |   11 +-
 .../AbstractSerializingDataFrameTestCase.java |    6 +-
 .../DataFrameTransformConfigTests.java        |   86 +-
 .../DataFrameTransformStateTests.java         |    4 +-
 .../transforms/DataFrameTransformTests.java   |   52 +
 .../dataframe/transforms/DestConfigTests.java |    3 +-
 .../MockDeprecatedAggregationBuilder.java     |   14 +-
 .../transforms/NodeAttributeTests.java        |   52 +
 .../transforms/TimeSyncConfigTests.java       |   38 +
 .../pivot/DateHistogramGroupSourceTests.java  |   19 +-
 .../utils/DataFrameStringsTests.java          |   30 +
 .../indexing/AsyncTwoPhaseIndexerTests.java   |   24 +-
 .../indexlifecycle/SegmentCountStepTests.java |    6 +-
 .../SetSingleNodeAllocateStepTests.java       |  319 +-
 .../WaitForFollowShardTasksStepTests.java     |    1 +
 .../action/DeleteLifecycleResponseTests.java  |    9 +-
 .../action/MoveToStepResponseTests.java       |    9 +-
 .../action/PutLifecycleResponseTests.java     |    9 +-
 .../action/RetryResponseTests.java            |    9 +-
 .../xpack/core/ml/MlTasksTests.java           |    7 +
 .../EvaluateDataFrameActionRequestTests.java  |   58 +
 .../FindFileStructureActionRequestTests.java  |   16 +
 ...DataFrameAnalyticsActionResponseTests.java |   55 +
 .../GetDataFrameAnalyticsRequestTests.java    |   27 +
 ...rameAnalyticsStatsActionResponseTests.java |   38 +
 ...etDataFrameAnalyticsStatsRequestTests.java |   26 +
 .../GetDatafeedStatsActionResponseTests.java  |    6 +-
 .../GetDatafeedsActionResponseTests.java      |    2 +-
 .../GetJobStatsActionResponseTests.java       |   40 +-
 ...tDataFrameAnalyticsActionRequestTests.java |   67 +
 ...DataFrameAnalyticsActionResponseTests.java |   48 +
 .../action/PutDatafeedActionRequestTests.java |    4 +-
 .../PutDatafeedActionResponseTests.java       |    2 +-
 ...taFrameAnalyticsActionTaskParamsTests.java |   37 +
 .../StartDataFrameAnalyticsRequestTests.java  |   28 +
 ...DataFrameAnalyticsActionResponseTests.java |   23 +
 .../StopDataFrameAnalyticsRequestTests.java   |   43 +
 .../UpdateDatafeedActionRequestTests.java     |    4 +-
 .../core/ml/datafeed/AggProviderTests.java    |    6 +-
 .../core/ml/datafeed/DatafeedConfigTests.java |   11 +-
 .../core/ml/datafeed/DatafeedUpdateTests.java |    9 +-
 .../DataFrameAnalyticsConfigTests.java        |  312 ++
 .../DataFrameAnalyticsDestTests.java          |   55 +
 .../DataFrameAnalyticsSourceTests.java        |   64 +
 .../DataFrameAnalyticsStateTests.java         |   41 +
 .../DataFrameAnalyticsTaskStateTests.java     |   35 +
 .../analyses/OutlierDetectionTests.java       |   59 +
 .../softclassification/AucRocTests.java       |  127 +
 .../BinarySoftClassificationTests.java        |   79 +
 .../ConfusionMatrixTests.java                 |   79 +
 .../softclassification/PrecisionTests.java    |   93 +
 .../softclassification/RecallTests.java       |   93 +
 .../filestructurefinder/FieldStatsTests.java  |   10 +-
 .../integration/MlRestTestStateCleaner.java   |   17 +
 .../ElasticsearchMappingsTests.java           |    2 +
 .../autodetect/state/TimingStatsTests.java    |  149 +
 .../QueryProviderTests.java                   |   12 +-
 .../utils/XContentObjectTransformerTests.java |    2 +-
 .../rest/action/ReloadSynonymAnalyzerIT.java  |  125 +
 .../CreateApiKeyRequestBuilderTests.java      |   18 +
 .../action/CreateApiKeyRequestTests.java      |   24 +-
 .../GetBuiltinPrivilegesResponseTests.java    |   32 +
 .../privilege/GetPrivilegesRequestTests.java  |    3 +-
 .../privilege/GetPrivilegesResponseTests.java |   22 +-
 .../action/token/CreateTokenRequestTests.java |   66 +-
 .../token/CreateTokenResponseTests.java       |    4 +-
 .../DocumentSubsetBitsetCacheTests.java       |  247 ++
 .../DocumentSubsetReaderTests.java            |   50 +-
 ...tyIndexReaderWrapperIntegrationTests.java} |   58 +-
 .../SecurityIndexReaderWrapperUnitTests.java  |  225 ++
 ...SecurityIndexSearcherWrapperUnitTests.java |  539 ---
 .../authz/store/ReservedRolesStoreTests.java  |   27 +
 .../SecurityQueryTemplateEvaluatorTests.java  |    2 +-
 .../security/support/ValidationTests.java     |    7 +-
 .../xpack/core/security/user/UserTests.java   |    4 +-
 .../xpack/core/ssl/CertParsingUtilsTests.java |    6 +-
 .../ssl/SSLConfigurationReloaderTests.java    |   19 +-
 .../xpack/core/ssl/SSLConfigurationTests.java |   19 +-
 x-pack/plugin/data-frame/build.gradle         |    2 +-
 .../qa/multi-node-tests/build.gradle          |   26 +-
 .../integration/DataFrameIntegTestCase.java   |   73 +-
 .../integration/DataFrameTransformIT.java     |  114 +-
 .../qa/single-node-tests/build.gradle         |   21 +-
 .../integration/DataFrameAuditorIT.java       |    8 +-
 .../DataFrameConfigurationIndexIT.java        |    2 -
 .../DataFrameGetAndGetStatsIT.java            |  123 +-
 .../integration/DataFrameMetaDataIT.java      |    2 -
 .../integration/DataFramePivotRestIT.java     |  353 +-
 .../integration/DataFrameRestTestCase.java    |  130 +-
 .../DataFrameTaskFailedStateIT.java           |    2 -
 .../DataFrameTransformProgressIT.java         |   32 +-
 .../integration/DataFrameUsageIT.java         |   82 +-
 .../xpack/dataframe/DataFrame.java            |   29 +-
 ...java => DataFrameInfoTransportAction.java} |  111 +-
 .../DataFrameUsageTransportAction.java        |  130 +
 ...ansportDeleteDataFrameTransformAction.java |   25 +-
 ...TransportGetDataFrameTransformsAction.java |    2 +-
 ...portGetDataFrameTransformsStatsAction.java |   91 +-
 ...nsportPreviewDataFrameTransformAction.java |   96 +-
 .../TransportPutDataFrameTransformAction.java |   67 +-
 ...ransportStartDataFrameTransformAction.java |   33 +-
 ...TransportStopDataFrameTransformAction.java |   47 +-
 .../DataFrameTransformsCheckpointService.java |   27 +-
 .../DataFrameTransformsConfigManager.java     |   16 +-
 .../dataframe/persistence/DataframeIndex.java |   84 +-
 .../RestGetDataFrameTransformsAction.java     |    3 +
 ...RestGetDataFrameTransformsStatsAction.java |    3 +
 .../RestStopDataFrameTransformAction.java     |    8 +-
 .../transforms/DataFrameIndexer.java          |  150 +-
 ...FrameTransformPersistentTasksExecutor.java |   71 +-
 .../transforms/DataFrameTransformTask.java    |  291 +-
 .../transforms/TransformProgressGatherer.java |   26 +-
 .../pivot/AggregationResultUtils.java         |   97 +-
 .../transforms/pivot/Aggregations.java        |    3 +-
 .../dataframe/transforms/pivot/Pivot.java     |   87 +-
 .../transforms/pivot/SchemaUtil.java          |    3 +-
 ...=> DataFrameInfoTransportActionTests.java} |   45 +-
 .../dataframe/action/DataFrameNodesTests.java |    4 +-
 ...meTransformCheckpointServiceNodeTests.java |    6 +-
 ...DataFrameTransformsConfigManagerTests.java |   38 +-
 .../persistence/DataframeIndexTests.java      |   75 +
 .../transforms/DataFrameIndexerTests.java     |   14 +-
 ...TransformPersistentTasksExecutorTests.java |   83 +
 .../pivot/AggregationResultUtilsTests.java    |   38 +-
 .../transforms/pivot/AggregationsTests.java   |    8 +-
 .../transforms/pivot/PivotTests.java          |   18 +-
 x-pack/plugin/deprecation/build.gradle        |    2 +-
 .../TransportDeprecationInfoAction.java       |    3 +-
 .../TransportNodeDeprecationCheckAction.java  |    7 +-
 .../deprecation/MlDeprecationChecksTests.java |    2 +-
 x-pack/plugin/graph/build.gradle              |    1 -
 x-pack/plugin/graph/qa/build.gradle           |   28 +-
 .../graph/qa/with-security/build.gradle       |   33 +-
 .../org/elasticsearch/xpack/graph/Graph.java  |   24 +-
 ...Set.java => GraphInfoTransportAction.java} |   33 +-
 .../graph/GraphUsageTransportAction.java      |   48 +
 .../action/TransportGraphExploreAction.java   |    4 +-
 .../graph/rest/action/RestGraphAction.java    |   18 +-
 ...ava => GraphInfoTransportActionTests.java} |   29 +-
 x-pack/plugin/ilm/build.gradle                |    1 -
 .../plugin/ilm/qa/multi-cluster/build.gradle  |   55 +-
 .../indexlifecycle/CCRIndexLifecycleIT.java   |   88 +-
 x-pack/plugin/ilm/qa/multi-node/build.gradle  |   16 +-
 .../ChangePolicyforIndexIT.java               |   15 +-
 .../TimeSeriesLifecycleActionsIT.java         |   30 +-
 x-pack/plugin/ilm/qa/rest/build.gradle        |   32 +-
 .../plugin/ilm/qa/with-security/build.gradle  |   35 +-
 .../xpack/security/PermissionsIT.java         |    3 +-
 .../xpack/indexlifecycle/IndexLifecycle.java  |   23 +-
 .../IndexLifecycleFeatureSet.java             |   97 -
 .../IndexLifecycleInfoTransportAction.java    |   46 +
 .../indexlifecycle/IndexLifecycleService.java |   28 +-
 .../IndexLifecycleUsageTransportAction.java   |   81 +
 .../LifecyclePolicySecurityClient.java        |    6 +-
 .../TransportDeleteLifecycleAction.java       |   12 +-
 .../action/TransportGetLifecycleAction.java   |    3 +-
 .../action/TransportGetStatusAction.java      |    3 +-
 .../action/TransportMoveToStepAction.java     |   13 +-
 .../action/TransportPutLifecycleAction.java   |   12 +-
 ...sportRemoveIndexLifecyclePolicyAction.java |    3 +-
 .../action/TransportRetryAction.java          |   13 +-
 .../action/TransportStartILMAction.java       |   13 +-
 .../action/TransportStopILMAction.java        |   13 +-
 ...dexLifecycleInfoTransportActionTests.java} |   38 +-
 .../IndexLifecycleInitialisationTests.java    |    6 +-
 .../IndexLifecycleMetadataTests.java          |    4 +-
 .../IndexLifecycleServiceTests.java           |   67 +-
 x-pack/plugin/logstash/build.gradle           |    1 -
 .../xpack/logstash/Logstash.java              |   26 +-
 ....java => LogstashInfoTransportAction.java} |   34 +-
 .../LogstashUsageTransportAction.java         |   49 +
 ... => LogstashInfoTransportActionTests.java} |   37 +-
 x-pack/plugin/mapper-flattened/build.gradle   |   27 +
 .../FlattenedInfoTransportAction.java         |   47 +
 .../flattened/FlattenedMapperPlugin.java      |   50 +
 .../FlattenedUsageTransportAction.java        |   49 +
 .../mapper/FlatObjectFieldMapper.java         |  616 ++++
 .../mapper/FlatObjectFieldParser.java         |  167 +
 .../KeyedFlatObjectAtomicFieldData.java       |  251 ++
 .../mapper/FlatObjectFieldLookupTests.java    |  188 ++
 .../mapper/FlatObjectFieldMapperTests.java    |  457 +++
 .../mapper/FlatObjectFieldParserTests.java    |  318 ++
 .../mapper/FlatObjectIndexFieldDataTests.java |  106 +
 .../mapper/FlatObjectSearchTests.java         |  512 +++
 .../KeyedFlatObjectAtomicFieldDataTests.java  |  204 ++
 .../mapper/KeyedFlatObjectFieldTypeTests.java |  155 +
 .../mapper/RootFlatObjectFieldTypeTests.java  |  116 +
 x-pack/plugin/ml/build.gradle                 |   12 +-
 .../ml/qa/basic-multi-node/build.gradle       |    8 +-
 x-pack/plugin/ml/qa/disabled/build.gradle     |    7 +-
 .../ml/qa/ml-with-security/build.gradle       |   90 +-
 .../plugin/ml/qa/ml-with-security/roles.yml   |    4 +-
 .../smoketest/MlWithSecurityUserRoleIT.java   |   33 +-
 .../qa/native-multi-node-tests/build.gradle   |   46 +-
 .../integration/AutodetectMemoryLimitIT.java  |    1 +
 .../ml/integration/DatafeedJobsRestIT.java    |   65 +
 .../ml/integration/DeleteExpiredDataIT.java   |    1 -
 ...NativeDataFrameAnalyticsIntegTestCase.java |  118 +
 .../ml/integration/MlNativeIntegTestCase.java |   15 +-
 .../xpack/ml/integration/PersistJobIT.java    |   12 +-
 .../integration/RunDataFrameAnalyticsIT.java  |  386 +++
 .../ml/qa/no-bootstrap-tests/build.gradle     |    4 +-
 .../ml/qa/single-node-tests/build.gradle      |    6 +-
 .../xpack/ml/MachineLearning.java             |  140 +-
 .../xpack/ml/MachineLearningFeatureSet.java   |  303 --
 .../MachineLearningInfoTransportAction.java   |   46 +
 .../MachineLearningUsageTransportAction.java  |  209 ++
 .../ml/MlConfigMigrationEligibilityCheck.java |   10 -
 .../xpack/ml/MlLifeCycleService.java          |   48 +-
 .../action/TransportDeleteCalendarAction.java |    3 +-
 ...ansportDeleteDataFrameAnalyticsAction.java |  114 +
 .../action/TransportDeleteDatafeedAction.java |   13 +-
 .../action/TransportDeleteForecastAction.java |    5 +-
 .../ml/action/TransportDeleteJobAction.java   |   19 +-
 .../TransportEvaluateDataFrameAction.java     |   61 +
 .../TransportFinalizeJobExecutionAction.java  |   12 +-
 .../TransportFindFileStructureAction.java     |   10 +-
 .../TransportGetDataFrameAnalyticsAction.java |   82 +
 ...sportGetDataFrameAnalyticsStatsAction.java |  196 ++
 .../action/TransportGetDatafeedsAction.java   |    3 +-
 .../TransportGetDatafeedsStatsAction.java     |    3 +-
 .../ml/action/TransportGetJobsAction.java     |    3 +-
 .../action/TransportGetJobsStatsAction.java   |   24 +-
 .../ml/action/TransportMlInfoAction.java      |   14 +-
 .../ml/action/TransportOpenJobAction.java     |  296 +-
 .../ml/action/TransportPostDataAction.java    |    5 +-
 .../TransportPutDataFrameAnalyticsAction.java |  166 +
 .../ml/action/TransportPutDatafeedAction.java |    2 +-
 .../ml/action/TransportPutJobAction.java      |    2 +-
 .../TransportRevertModelSnapshotAction.java   |    3 +-
 .../action/TransportSetUpgradeModeAction.java |   14 +-
 ...ransportStartDataFrameAnalyticsAction.java |  490 +++
 .../action/TransportStartDatafeedAction.java  |   12 +-
 ...TransportStopDataFrameAnalyticsAction.java |  270 ++
 .../action/TransportUpdateDatafeedAction.java |    3 +-
 .../ml/action/TransportUpdateJobAction.java   |    4 +-
 .../xpack/ml/datafeed/DatafeedJob.java        |    8 +-
 .../AggregationToJsonProcessor.java           |   13 +-
 .../extractor/fields/ExtractedField.java      |  144 +-
 .../extractor/fields/ExtractedFields.java     |   20 +
 .../ml/dataframe/DataFrameAnalyticsIndex.java |  197 ++
 .../dataframe/DataFrameAnalyticsManager.java  |  219 ++
 .../xpack/ml/dataframe/MappingsMerger.java    |  100 +
 .../ml/dataframe/SourceDestValidator.java     |   68 +
 .../extractor/DataFrameDataExtractor.java     |  276 ++
 .../DataFrameDataExtractorContext.java        |   35 +
 .../DataFrameDataExtractorFactory.java        |  170 +
 .../extractor/ExtractedFieldsDetector.java    |  169 +
 .../DataFrameAnalyticsConfigProvider.java     |  122 +
 .../dataframe/process/AnalyticsBuilder.java   |   74 +
 .../AnalyticsControlMessageWriter.java        |   38 +
 .../dataframe/process/AnalyticsProcess.java   |   34 +
 .../process/AnalyticsProcessConfig.java       |   76 +
 .../process/AnalyticsProcessFactory.java      |   24 +
 .../process/AnalyticsProcessManager.java      |  301 ++
 .../ml/dataframe/process/AnalyticsResult.java |   77 +
 .../process/AnalyticsResultProcessor.java     |   87 +
 .../process/DataFrameRowsJoiner.java          |  186 ++
 .../process/NativeAnalyticsProcess.java       |   52 +
 .../NativeAnalyticsProcessFactory.java        |   94 +
 .../dataframe/process/results/RowResults.java |   73 +
 .../DelimitedFileStructureFinder.java         |    7 +-
 .../DelimitedFileStructureFinderFactory.java  |    3 +-
 .../FieldStatsCalculator.java                 |   88 +-
 .../FileStructureFinderFactory.java           |    4 +-
 .../FileStructureFinderManager.java           |   55 +-
 .../FileStructureUtils.java                   |    9 +-
 .../GrokPatternCreator.java                   |   97 +-
 .../NdJsonFileStructureFinder.java            |    3 +-
 .../NdJsonFileStructureFinderFactory.java     |    3 +-
 .../TextLogFileStructureFinder.java           |   26 +-
 .../TextLogFileStructureFinderFactory.java    |    4 +-
 .../filestructurefinder/TimeoutChecker.java   |   14 +-
 .../TimestampFormatFinder.java                |    6 +-
 .../XmlFileStructureFinder.java               |    3 +-
 .../XmlFileStructureFinderFactory.java        |    2 +-
 .../xpack/ml/job/JobManager.java              |    2 +-
 .../xpack/ml/job/JobNodeSelector.java         |  329 ++
 .../CategorizationAnalyzer.java               |  188 +-
 .../persistence/JobDataCountsPersister.java   |    2 +-
 .../ml/job/persistence/JobDataDeleter.java    |    3 +-
 .../job/persistence/JobResultsPersister.java  |   12 +
 .../job/persistence/JobResultsProvider.java   |   28 +
 .../job/persistence/TimingStatsReporter.java  |   87 +
 .../autodetect/AutodetectCommunicator.java    |   39 +-
 .../autodetect/AutodetectProcessManager.java  |   34 +-
 .../autodetect/NativeAutodetectProcess.java   |   27 +-
 .../NativeAutodetectProcessFactory.java       |   27 +-
 ...or.java => AutodetectResultProcessor.java} |  128 +-
 .../autodetect/params/AutodetectParams.java   |   22 +-
 .../autodetect/params/FlushJobParams.java     |    2 +-
 .../process/autodetect/params/TimeRange.java  |    2 +-
 .../writer/AbstractDataToProcessWriter.java   |    4 +-
 .../normalizer/NativeNormalizerProcess.java   |    6 +-
 .../NativeNormalizerProcessFactory.java       |   18 +-
 .../process/normalizer/NormalizerResult.java  |    9 +-
 .../retention/ExpiredForecastsRemover.java    |    3 +-
 .../job/retention/ExpiredResultsRemover.java  |    3 +-
 .../ml/process/AbstractNativeProcess.java     |   23 +-
 .../xpack/ml/process/DummyController.java     |   26 +
 .../xpack/ml/process/MlController.java        |   36 +
 .../xpack/ml/process/MlControllerHolder.java  |   28 +
 .../xpack/ml/process/MlMemoryTracker.java     |  191 +-
 .../xpack/ml/process/NativeController.java    |   13 +-
 .../ml/process/NativeControllerHolder.java    |   60 -
 .../ProcessResultsParser.java}                |   34 +-
 .../ml/rest/RestFindFileStructureAction.java  |    2 +
 .../RestDeleteDataFrameAnalyticsAction.java   |   39 +
 .../RestEvaluateDataFrameAction.java          |   36 +
 .../RestGetDataFrameAnalyticsAction.java      |   51 +
 .../RestGetDataFrameAnalyticsStatsAction.java |   52 +
 .../RestPutDataFrameAnalyticsAction.java      |   43 +
 .../RestStartDataFrameAnalyticsAction.java    |   50 +
 .../RestStopDataFrameAnalyticsAction.java     |   49 +
 .../plugin-security-test.policy               |    5 +
 .../MachineLearningLicensingTests.java        |   76 +-
 ...hineLearningInfoTransportActionTests.java} |  108 +-
 .../xpack/ml/MachineLearningTests.java        |    4 +-
 ...lConfigMigrationEligibilityCheckTests.java |   83 +-
 .../xpack/ml/MlConfigMigratorTests.java       |    2 +-
 .../xpack/ml/MlMetadataTests.java             |    4 +-
 .../xpack/ml/MlSingleNodeTestCase.java        |    2 +-
 ...nsportFinalizeJobExecutionActionTests.java |    2 +-
 .../TransportGetJobsStatsActionTests.java     |   25 +-
 .../action/TransportOpenJobActionTests.java   |  399 +--
 .../TransportStartDatafeedActionTests.java    |    2 +-
 ...portStopDataFrameAnalyticsActionTests.java |   85 +
 .../datafeed/DatafeedJobValidatorTests.java   |    2 +-
 .../DelayedDataDetectorFactoryTests.java      |    2 +-
 .../extractor/DataExtractorFactoryTests.java  |    2 +-
 .../AggregationDataExtractorFactoryTests.java |    2 +-
 .../aggregation/AggregationTestUtils.java     |   11 +
 .../AggregationToJsonProcessorTests.java      |   15 +
 .../ChunkedDataExtractorFactoryTests.java     |    2 +-
 .../extractor/fields/ExtractedFieldTests.java |   28 +
 .../DataFrameAnalyticsIndexTests.java         |  175 +
 .../ml/dataframe/MappingsMergerTests.java     |  137 +
 .../dataframe/SourceDestValidatorTests.java   |  180 +
 .../DataFrameDataExtractorTests.java          |  392 +++
 .../ExtractedFieldsDetectorTests.java         |  319 ++
 .../AnalyticsControlMessageWriterTests.java   |   50 +
 .../AnalyticsResultProcessorTests.java        |   86 +
 .../process/AnalyticsResultTests.java         |   39 +
 .../process/DataFrameRowsJoinerTests.java     |  280 ++
 .../process/results/RowResultsTests.java      |   42 +
 .../DelimitedFileStructureFinderTests.java    |   59 +-
 .../FieldStatsCalculatorTests.java            |  102 +-
 .../FileStructureFinderManagerTests.java      |   55 +-
 .../FileStructureUtilsTests.java              |    6 +-
 .../GrokPatternCreatorTests.java              |   20 +-
 .../NdJsonFileStructureFinderTests.java       |    2 +-
 .../TextLogFileStructureFinderTests.java      |   44 +-
 .../TimestampFormatFinderTests.java           |   13 +-
 .../XmlFileStructureFinderTests.java          |    2 +-
 .../ml/integration/AnnotationIndexIT.java     |    3 +-
 .../AutodetectResultProcessorIT.java          |  139 +-
 .../integration/BasicDistributedJobsIT.java   |   17 +-
 .../ml/integration/MlConfigMigratorIT.java    |    3 +-
 .../integration/MlDistributedFailureIT.java   |    9 +-
 .../ml/integration/NetworkDisruptionIT.java   |    6 -
 .../xpack/ml/job/JobManagerTests.java         |    2 +-
 .../xpack/ml/job/JobNodeSelectorTests.java    |  576 ++++
 .../CategorizationAnalyzerTests.java          |   55 +-
 .../BatchedDocumentsIteratorTests.java        |    1 -
 .../persistence/JobResultsPersisterTests.java |   32 +
 .../persistence/JobResultsProviderTests.java  |   55 +
 .../ml/job/persistence/MockClientBuilder.java |   24 +-
 .../job/persistence/StateStreamerTests.java   |    1 -
 .../persistence/TimingStatsReporterTests.java |  124 +
 .../AutodetectCommunicatorTests.java          |   34 +-
 .../AutodetectProcessManagerTests.java        |    4 -
 .../NativeAutodetectProcessFactoryTests.java  |   61 +
 .../NativeAutodetectProcessTests.java         |   24 +-
 ...va => AutodetectResultProcessorTests.java} |  348 +-
 .../output/AutodetectResultsParserTests.java  |  422 ---
 .../params/AutodetectParamsTests.java         |   31 +
 .../AbstractDataToProcessWriterTests.java     |    2 +-
 .../writer/CsvDataToProcessWriterTests.java   |    2 +-
 .../writer/JsonDataToProcessWriterTests.java  |    2 +-
 .../xpack/ml/job/results/BucketTests.java     |    2 +-
 .../ml/process/MlMemoryTrackerTests.java      |   63 +-
 .../ml/process/ProcessResultsParserTests.java |  113 +
 .../xpack/ml/support/BaseMlIntegTestCase.java |   43 +-
 x-pack/plugin/monitoring/build.gradle         |   11 +-
 .../xpack/monitoring/Monitoring.java          |   36 +-
 .../monitoring/MonitoringFeatureSet.java      |   88 -
 .../MonitoringInfoTransportAction.java        |   45 +
 .../monitoring/MonitoringUsageServices.java   |   23 +
 .../MonitoringUsageTransportAction.java       |   72 +
 .../collector/ccr/StatsCollector.java         |   12 +-
 .../collector/ml/JobStatsCollector.java       |   10 +-
 .../exporter/ClusterAlertsUtil.java           |   13 +-
 .../exporter/local/LocalExporter.java         |   31 +-
 .../rest/action/RestMonitoringBulkAction.java |   53 +-
 .../monitoring/LocalStateMonitoring.java      |   32 +
 ...> MonitoringInfoTransportActionTests.java} |   27 +-
 .../ClusterStatsMonitoringDocTests.java       |   13 +-
 .../IndexRecoveryMonitoringDocTests.java      |    5 +-
 .../collector/ml/JobStatsCollectorTests.java  |    9 +-
 .../ml/JobStatsMonitoringDocTests.java        |   21 +-
 .../node/NodeStatsMonitoringDocTests.java     |    2 +-
 .../exporter/http/HttpExporterSslIT.java      |    3 +-
 .../local/LocalExporterIntegTests.java        |   10 +-
 .../action/RestMonitoringBulkActionTests.java |   51 +-
 .../test/MonitoringIntegTestCase.java         |   15 +-
 x-pack/plugin/rollup/build.gradle             |    5 +-
 .../elasticsearch/xpack/rollup/Rollup.java    |   26 +-
 ...et.java => RollupInfoTransportAction.java} |   33 +-
 .../rollup/RollupUsageTransportAction.java    |   50 +
 .../action/TransportPutRollupJobAction.java   |   13 +-
 .../action/TransportRollupSearchAction.java   |    4 +-
 .../xpack/rollup/job/IndexerUtils.java        |    5 +-
 .../xpack/rollup/job/RollupJobTask.java       |   24 +-
 ...va => RollupInfoTransportActionTests.java} |   30 +-
 .../rollup/RollupRequestTranslationTests.java |    2 +-
 .../action/PutJobStateMachineTests.java       |    5 +-
 .../rollup/action/SearchActionTests.java      |    2 +-
 .../job/RollupIndexerIndexingTests.java       |    2 +-
 .../xpack/rollup/job/RollupJobTaskTests.java  |    9 +-
 x-pack/plugin/security/build.gradle           |    3 +-
 x-pack/plugin/security/cli/build.gradle       |    5 +-
 .../qa/basic-enable-security/build.gradle     |   94 +-
 .../security/qa/security-basic/build.gradle   |   22 +-
 .../security/SecurityWithBasicLicenseIT.java  |   14 +-
 .../plugin/security/qa/tls-basic/build.gradle |   27 +-
 .../xpack/security/Security.java              |  110 +-
 .../security/SecurityInfoTransportAction.java |   48 +
 .../xpack/security/SecurityUsageServices.java |   30 +
 ...java => SecurityUsageTransportAction.java} |   85 +-
 .../security/action/SecurityActionMapper.java |    5 +-
 .../TransportGetBuiltinPrivilegesAction.java  |   39 +
 .../TransportGetPrivilegesAction.java         |    6 +-
 .../realm/TransportClearRealmCacheAction.java |    7 +-
 .../role/TransportClearRolesCacheAction.java  |    7 +-
 .../token/TransportCreateTokenAction.java     |   76 +-
 .../token/TransportRefreshTokenAction.java    |    2 +-
 .../audit/logfile/LoggingAuditTrail.java      |    2 +
 .../xpack/security/authc/ApiKeyService.java   |   46 +-
 .../xpack/security/authc/TokenService.java    |    2 +-
 .../authc/esnative/NativeUsersStore.java      |   12 +-
 .../esnative/tool/SetupPasswordTool.java      |    2 +-
 .../security/authc/file/tool/UsersTool.java   |   30 +-
 .../authc/kerberos/KerberosRealm.java         |   13 +-
 .../ldap/ActiveDirectoryGroupsResolver.java   |    7 +-
 .../xpack/security/authc/ldap/LdapRealm.java  |    3 +-
 .../authc/ldap/SearchGroupsResolver.java      |    6 +-
 .../ldap/UserAttributeGroupsResolver.java     |   11 +-
 .../ldap/support/LdapMetaDataResolver.java    |   16 +-
 .../oidc/OpenIdConnectAuthenticator.java      |   10 +-
 .../authc/oidc/OpenIdConnectRealm.java        |   41 +-
 .../xpack/security/authc/pki/PkiRealm.java    |    7 +-
 .../security/authc/saml/SamlAttributes.java   |   25 +-
 .../xpack/security/authc/saml/SamlRealm.java  |   18 +-
 .../authc/support/UserRoleMapper.java         |   12 +-
 .../mapper/NativeRoleMappingStore.java        |   12 +-
 .../authz/IndicesAndAliasesResolver.java      |    7 -
 .../authz/store/CompositeRolesStore.java      |   56 +-
 .../authz/store/NativePrivilegeStore.java     |   10 +-
 .../authz/store/NativeRolesStore.java         |   10 +-
 .../action/apikey/RestCreateApiKeyAction.java |   17 +-
 .../action/oauth2/RestGetTokenAction.java     |   42 +-
 .../privilege/RestDeletePrivilegesAction.java |   32 +-
 .../RestGetBuiltinPrivilegesAction.java       |   58 +
 .../privilege/RestGetPrivilegesAction.java    |   54 +-
 .../privilege/RestPutPrivilegesAction.java    |    7 +-
 .../realm/RestClearRealmCacheAction.java      |    8 +-
 .../role/RestClearRolesCacheAction.java       |    8 +-
 .../action/role/RestDeleteRoleAction.java     |   23 +-
 .../rest/action/role/RestGetRolesAction.java  |    6 +-
 .../rest/action/role/RestPutRoleAction.java   |    9 +-
 .../RestDeleteRoleMappingAction.java          |   21 +-
 .../RestGetRoleMappingsAction.java            |   35 +-
 .../rolemapping/RestPutRoleMappingAction.java |    9 +-
 .../saml/RestSamlAuthenticateAction.java      |    6 +-
 .../action/user/RestChangePasswordAction.java |   23 +-
 .../action/user/RestDeleteUserAction.java     |   25 +-
 .../user/RestGetUserPrivilegesAction.java     |    3 +-
 .../rest/action/user/RestGetUsersAction.java  |    4 +-
 .../action/user/RestHasPrivilegesAction.java  |    5 +-
 .../rest/action/user/RestPutUserAction.java   |    9 +-
 .../action/user/RestSetEnabledAction.java     |   18 +-
 .../support/SecurityIndexManager.java         |   77 +-
 .../support/SecurityStatusChangeListener.java |   45 +
 .../SecurityServerTransportInterceptor.java   |  106 +-
 .../transport/ServerTransportFilter.java      |  185 +-
 .../security/transport/nio/NioIPFilter.java   |   39 +-
 .../transport/nio/SSLChannelContext.java      |   21 +-
 .../nio/SecurityNioHttpServerTransport.java   |   16 +-
 .../transport/nio/SecurityNioTransport.java   |   25 +-
 .../integration/ClearRealmsCacheTests.java    |    6 +-
 .../integration/ClearRolesCacheTests.java     |   79 +-
 .../DocumentLevelSecurityTests.java           |   39 +-
 .../integration/FieldLevelSecurityTests.java  |   64 +-
 .../MultipleIndicesPermissionsTests.java      |   16 +-
 .../SecurityCachePermissionTests.java         |    5 +-
 .../integration/SecurityClearScrollTests.java |    6 +-
 .../elasticsearch/license/LicensingTests.java |    4 +-
 .../test/NativeRealmIntegTestCase.java        |   11 +-
 .../test/SecurityIntegTestCase.java           |   37 +-
 .../test/SecuritySettingsSource.java          |   96 +-
 .../test/SecuritySingleNodeTestCase.java      |    6 -
 .../PutPrivilegesRequestBuilderTests.java     |    8 +-
 .../xpack/security/LocalStateSecurity.java    |   54 +
 .../security/PkiRealmBootstrapCheckTests.java |    2 +-
 ... => SecurityInfoTransportActionTests.java} |   43 +-
 .../xpack/security/SecurityTests.java         |   79 -
 .../action/SecurityActionMapperTests.java     |    9 +-
 ...ansportOpenIdConnectLogoutActionTests.java |    2 +-
 .../PutRoleMappingRequestTests.java           |    9 +-
 ...sportSamlInvalidateSessionActionTests.java |    4 +-
 .../TransportCreateTokenActionTests.java      |   68 +-
 .../audit/logfile/LoggingAuditTrailTests.java |    6 +-
 .../security/authc/ApiKeyIntegTests.java      |   82 +-
 .../authc/AuthenticationServiceTests.java     |   16 +-
 .../xpack/security/authc/RunAsIntegTests.java |   10 +-
 .../security/authc/TokenAuthIntegTests.java   |  525 ++-
 .../security/authc/TokenServiceTests.java     |   21 +-
 .../authc/esnative/NativeRealmIntegTests.java |  287 +-
 .../authc/esnative/NativeRealmTests.java      |    6 +-
 .../authc/esnative/NativeUsersStoreTests.java |    4 +-
 .../esnative/ReservedRealmIntegTests.java     |   42 +-
 .../esnative/tool/SetupPasswordToolTests.java |   14 +-
 .../SessionFactoryLoadBalancingTests.java     |    2 -
 .../oidc/OpenIdConnectAuthenticatorTests.java |   35 +
 .../authc/pki/PkiAuthenticationTests.java     |    3 +-
 .../authc/saml/SamlMetadataCommandTests.java  |    2 +-
 .../security/authc/saml/SamlRealmTests.java   |    3 +-
 .../mapper/ExpressionRoleMappingTests.java    |   19 +-
 .../mapper/NativeRoleMappingStoreTests.java   |   23 +-
 .../authz/AuthorizationServiceTests.java      |   14 +-
 .../authz/IndicesAndAliasesResolverTests.java |   24 -
 .../security/authz/ReadActionsTests.java      |   34 +-
 .../security/authz/SecurityScrollTests.java   |    9 +-
 .../authz/SnapshotUserRoleIntegTests.java     |   49 +-
 .../security/authz/WriteActionsTests.java     |  100 +-
 .../authz/store/CompositeRolesStoreTests.java |   53 +-
 .../store/NativePrivilegeStoreTests.java      |    6 +-
 .../ingest/SetSecurityUserProcessorTests.java |    9 +-
 .../apikey/RestCreateApiKeyActionTests.java   |    7 +-
 .../apikey/RestGetApiKeyActionTests.java      |    7 +-
 .../RestInvalidateApiKeyActionTests.java      |    7 +-
 .../oauth2/RestGetTokenActionTests.java       |   42 +-
 .../SecurityIndexManagerIntegTests.java       |   22 +-
 .../support/SecurityIndexManagerTests.java    |   62 +-
 .../SecurityStatusChangeListenerTests.java    |  115 +
 ...stractSimpleSecurityTransportTestCase.java |    7 +-
 ...curityServerTransportInterceptorTests.java |   43 +-
 ...ServerTransportFilterIntegrationTests.java |  208 --
 .../transport/ServerTransportFilterTests.java |   43 +-
 .../filter/IpFilteringIntegrationTests.java   |    4 +-
 .../transport/nio/NioIPFilterTests.java       |   36 +-
 .../transport/nio/SSLChannelContextTests.java |    6 +-
 .../transport/ssl/EllipticCurveSSLTests.java  |    2 +-
 .../transport/ssl/SslIntegrationTests.java    |    2 +-
 .../xpack/ssl/SSLClientAuthTests.java         |    3 +-
 .../xpack/ssl/SSLTrustRestrictionsTests.java  |    3 -
 x-pack/plugin/sql/build.gradle                |    5 +-
 x-pack/plugin/sql/jdbc/build.gradle           |    6 +-
 .../xpack/sql/jdbc/TypeConverter.java         |    5 +-
 x-pack/plugin/sql/qa/build.gradle             |   20 +-
 x-pack/plugin/sql/qa/multi-node/build.gradle  |    4 +-
 x-pack/plugin/sql/qa/security/build.gradle    |   33 +-
 .../sql/qa/security/with-ssl/build.gradle     |   74 +-
 .../sql/qa/security/without-ssl/build.gradle  |   14 +-
 x-pack/plugin/sql/qa/single-node/build.gradle |    7 +-
 .../sql/qa/single_node/JdbcCsvSpecIT.java     |    2 +-
 .../xpack/sql/qa/ErrorsTestCase.java          |    1 +
 .../sql/qa/cli/CliIntegrationTestCase.java    |    8 +-
 .../xpack/sql/qa/cli/ErrorsTestCase.java      |    9 +-
 .../xpack/sql/qa/cli/FetchSizeTestCase.java   |   28 +
 .../xpack/sql/qa/jdbc/DebugCsvSpec.java       |   64 -
 .../xpack/sql/qa/jdbc/DebugSqlSpec.java       |    5 +-
 .../xpack/sql/qa/jdbc/ErrorsTestCase.java     |   10 +
 .../xpack/sql/qa/jdbc/JdbcAssert.java         |    5 +-
 .../xpack/sql/qa/jdbc/SqlSpecTestCase.java    |    7 +
 .../xpack/sql/qa/rest/RestSqlTestCase.java    |  104 +-
 .../src/main/resources/agg-ordering.sql-spec  |    6 +-
 .../qa/src/main/resources/docs/docs.csv-spec  |    9 +-
 .../qa/src/main/resources/fulltext.csv-spec   |    8 +-
 .../single-node-only/command-sys-geo.csv-spec |   12 +-
 .../single-node-only/command-sys.csv-spec     |  162 +-
 x-pack/plugin/sql/sql-action/build.gradle     |    6 +-
 .../sql/action/SqlClearCursorAction.java      |    4 +-
 .../xpack/sql/action/SqlQueryAction.java      |    4 +-
 .../xpack/sql/action/SqlTranslateAction.java  |    4 +-
 .../sql/action/SqlQueryRequestTests.java      |    4 +-
 .../sql/action/SqlTranslateRequestTests.java  |    4 +-
 x-pack/plugin/sql/sql-cli/build.gradle        |    8 +-
 x-pack/plugin/sql/sql-client/build.gradle     |    2 +-
 x-pack/plugin/sql/sql-proto/build.gradle      |    6 +-
 .../xpack/sql/SqlFeatureSet.java              |   87 -
 .../xpack/sql/SqlInfoTransportAction.java     |   46 +
 .../xpack/sql/SqlUsageTransportAction.java    |   75 +
 .../search/CompositeAggregationCursor.java    |   33 +-
 .../execution/search/CompositeAggsRowSet.java |   20 +-
 .../xpack/sql/execution/search/Querier.java   |  155 +-
 .../search/SchemaCompositeAggsRowSet.java     |    2 +-
 .../sql/execution/search/SourceGenerator.java |   10 +-
 .../search/extractor/FieldHitExtractor.java   |    6 +-
 .../function/scalar/geo/GeoShape.java         |   16 +-
 .../elasticsearch/xpack/sql/package-info.java |   12 +-
 .../xpack/sql/plugin/SqlPlugin.java           |   22 +-
 .../xpack/sql/plugin/SqlStatsAction.java      |    4 +-
 .../xpack/sql/plugin/SqlStatsRequest.java     |    3 +-
 .../xpack/sql/plugin/TextFormat.java          |    2 +-
 .../sql/plugin/TransportSqlStatsAction.java   |    7 +-
 .../xpack/sql/querydsl/query/MatchQuery.java  |    1 -
 .../sql/querydsl/query/MultiMatchQuery.java   |    1 -
 ....java => SqlInfoTransportActionTests.java} |   25 +-
 .../CompositeAggregationCursorTests.java      |    5 +-
 .../sql/execution/search/QuerierTests.java    |  109 +
 .../extractor/FieldHitExtractorTests.java     |   26 +-
 .../extractor/TestMultiValueAggregation.java  |   10 -
 .../extractor/TestSingleValueAggregation.java |    9 -
 .../scalar/geo/StWkttosqlProcessorTests.java  |    9 +
 .../xpack/sql/plugin/SqlPluginTests.java      |    3 +-
 .../api/ccr.delete_auto_follow_pattern.json   |    2 +-
 .../rest-api-spec/api/ccr.follow.json         |    2 +-
 .../rest-api-spec/api/ccr.follow_info.json    |    2 +-
 .../rest-api-spec/api/ccr.follow_stats.json   |    2 +-
 .../api/ccr.forget_follower.json              |    2 +-
 .../api/ccr.get_auto_follow_pattern.json      |    2 +-
 .../rest-api-spec/api/ccr.pause_follow.json   |    2 +-
 .../api/ccr.put_auto_follow_pattern.json      |    2 +-
 .../rest-api-spec/api/ccr.resume_follow.json  |    2 +-
 .../rest-api-spec/api/ccr.stats.json          |    2 +-
 .../rest-api-spec/api/ccr.unfollow.json       |    2 +-
 ...ata_frame.delete_data_frame_transform.json |    2 +-
 .../data_frame.get_data_frame_transform.json  |    7 +-
 ..._frame.get_data_frame_transform_stats.json |    7 +-
 ...ta_frame.preview_data_frame_transform.json |    2 +-
 .../data_frame.put_data_frame_transform.json  |    2 +-
 ...data_frame.start_data_frame_transform.json |    2 +-
 .../data_frame.stop_data_frame_transform.json |    7 +-
 .../rest-api-spec/api/graph.explore.json      |    2 +-
 .../api/ilm.delete_lifecycle.json             |    2 +-
 .../api/ilm.explain_lifecycle.json            |    2 +-
 .../rest-api-spec/api/ilm.get_lifecycle.json  |    2 +-
 .../rest-api-spec/api/ilm.get_status.json     |    2 +-
 .../rest-api-spec/api/ilm.move_to_step.json   |    2 +-
 .../rest-api-spec/api/ilm.put_lifecycle.json  |    2 +-
 .../rest-api-spec/api/ilm.remove_policy.json  |    2 +-
 .../rest-api-spec/api/ilm.retry.json          |    2 +-
 .../rest-api-spec/api/ilm.start.json          |    2 +-
 .../resources/rest-api-spec/api/ilm.stop.json |    2 +-
 .../rest-api-spec/api/indices.freeze.json     |    2 +-
 .../api/indices.reload_search_analyzers.json  |   33 +
 .../rest-api-spec/api/indices.unfreeze.json   |    2 +-
 .../rest-api-spec/api/license.delete.json     |    2 +-
 .../rest-api-spec/api/license.get.json        |    2 +-
 .../api/license.get_basic_status.json         |    2 +-
 .../api/license.get_trial_status.json         |    2 +-
 .../rest-api-spec/api/license.post.json       |    2 +-
 .../api/license.post_start_basic.json         |    2 +-
 .../api/license.post_start_trial.json         |    2 +-
 .../api/migration.deprecations.json           |    2 +-
 .../rest-api-spec/api/ml.close_job.json       |    2 +-
 .../rest-api-spec/api/ml.delete_calendar.json |    2 +-
 .../api/ml.delete_calendar_event.json         |    2 +-
 .../api/ml.delete_calendar_job.json           |    2 +-
 .../api/ml.delete_data_frame_analytics.json   |   18 +
 .../rest-api-spec/api/ml.delete_datafeed.json |    2 +-
 .../api/ml.delete_expired_data.json           |    2 +-
 .../rest-api-spec/api/ml.delete_filter.json   |    2 +-
 .../rest-api-spec/api/ml.delete_forecast.json |    2 +-
 .../rest-api-spec/api/ml.delete_job.json      |    2 +-
 .../api/ml.delete_model_snapshot.json         |    2 +-
 .../api/ml.evaluate_data_frame.json           |   15 +
 .../api/ml.find_file_structure.json           |    7 +-
 .../rest-api-spec/api/ml.flush_job.json       |    6 +-
 .../rest-api-spec/api/ml.forecast.json        |    2 +-
 .../rest-api-spec/api/ml.get_buckets.json     |    2 +-
 .../api/ml.get_calendar_events.json           |    2 +-
 .../rest-api-spec/api/ml.get_calendars.json   |    2 +-
 .../rest-api-spec/api/ml.get_categories.json  |    2 +-
 .../api/ml.get_data_frame_analytics.json      |   38 +
 .../ml.get_data_frame_analytics_stats.json    |   38 +
 .../api/ml.get_datafeed_stats.json            |    2 +-
 .../rest-api-spec/api/ml.get_datafeeds.json   |    2 +-
 .../rest-api-spec/api/ml.get_filters.json     |    2 +-
 .../rest-api-spec/api/ml.get_influencers.json |    2 +-
 .../rest-api-spec/api/ml.get_job_stats.json   |    2 +-
 .../rest-api-spec/api/ml.get_jobs.json        |    2 +-
 .../api/ml.get_model_snapshots.json           |    2 +-
 .../api/ml.get_overall_buckets.json           |    2 +-
 .../rest-api-spec/api/ml.get_records.json     |    2 +-
 .../resources/rest-api-spec/api/ml.info.json  |    2 +-
 .../rest-api-spec/api/ml.open_job.json        |    2 +-
 .../api/ml.post_calendar_events.json          |    2 +-
 .../rest-api-spec/api/ml.post_data.json       |    2 +-
 .../api/ml.preview_datafeed.json              |    2 +-
 .../rest-api-spec/api/ml.put_calendar.json    |    2 +-
 .../api/ml.put_calendar_job.json              |    2 +-
 .../api/ml.put_data_frame_analytics.json      |   21 +
 .../rest-api-spec/api/ml.put_datafeed.json    |    2 +-
 .../rest-api-spec/api/ml.put_filter.json      |    2 +-
 .../rest-api-spec/api/ml.put_job.json         |    2 +-
 .../api/ml.revert_model_snapshot.json         |    2 +-
 .../api/ml.set_upgrade_mode.json              |    2 +-
 .../api/ml.start_data_frame_analytics.json    |   27 +
 .../rest-api-spec/api/ml.start_datafeed.json  |    2 +-
 .../api/ml.stop_data_frame_analytics.json     |   37 +
 .../rest-api-spec/api/ml.stop_datafeed.json   |    6 +-
 .../rest-api-spec/api/ml.update_datafeed.json |    2 +-
 .../rest-api-spec/api/ml.update_filter.json   |    2 +-
 .../rest-api-spec/api/ml.update_job.json      |    2 +-
 .../api/ml.update_model_snapshot.json         |    2 +-
 .../rest-api-spec/api/ml.validate.json        |    2 +-
 .../api/ml.validate_detector.json             |    2 +-
 .../rest-api-spec/api/monitoring.bulk.json    |    2 +-
 .../rest-api-spec/api/rollup.delete_job.json  |    2 +-
 .../rest-api-spec/api/rollup.get_jobs.json    |    2 +-
 .../api/rollup.get_rollup_caps.json           |    2 +-
 .../api/rollup.get_rollup_index_caps.json     |    2 +-
 .../rest-api-spec/api/rollup.put_job.json     |    2 +-
 .../api/rollup.rollup_search.json             |    2 +-
 .../rest-api-spec/api/rollup.start_job.json   |    2 +-
 .../rest-api-spec/api/rollup.stop_job.json    |    2 +-
 .../api/security.authenticate.json            |    2 +-
 .../api/security.change_password.json         |    2 +-
 .../api/security.clear_cached_realms.json     |    2 +-
 .../api/security.clear_cached_roles.json      |    2 +-
 .../api/security.create_api_key.json          |    2 +-
 .../api/security.delete_privileges.json       |    2 +-
 .../api/security.delete_role.json             |    2 +-
 .../api/security.delete_role_mapping.json     |    2 +-
 .../api/security.delete_user.json             |    2 +-
 .../api/security.disable_user.json            |    2 +-
 .../api/security.enable_user.json             |    2 +-
 .../api/security.get_api_key.json             |    2 +-
 .../api/security.get_builtin_privileges.json  |   15 +
 .../api/security.get_privileges.json          |    4 +-
 .../rest-api-spec/api/security.get_role.json  |    2 +-
 .../api/security.get_role_mapping.json        |    2 +-
 .../rest-api-spec/api/security.get_token.json |    2 +-
 .../rest-api-spec/api/security.get_user.json  |    2 +-
 .../api/security.get_user_privileges.json     |    2 +-
 .../api/security.has_privileges.json          |    2 +-
 .../api/security.invalidate_api_key.json      |    2 +-
 .../api/security.invalidate_token.json        |    2 +-
 .../api/security.put_privileges.json          |    2 +-
 .../rest-api-spec/api/security.put_role.json  |    2 +-
 .../api/security.put_role_mapping.json        |    2 +-
 .../rest-api-spec/api/security.put_user.json  |    2 +-
 .../rest-api-spec/api/sql.clear_cursor.json   |    2 +-
 .../rest-api-spec/api/sql.query.json          |    2 +-
 .../rest-api-spec/api/sql.translate.json      |    2 +-
 .../rest-api-spec/api/ssl.certificates.json   |    2 +-
 .../rest-api-spec/api/watcher.ack_watch.json  |    2 +-
 .../api/watcher.activate_watch.json           |    2 +-
 .../api/watcher.deactivate_watch.json         |    2 +-
 .../api/watcher.delete_watch.json             |    3 +-
 .../api/watcher.execute_watch.json            |    2 +-
 .../rest-api-spec/api/watcher.get_watch.json  |    2 +-
 .../rest-api-spec/api/watcher.put_watch.json  |    2 +-
 .../rest-api-spec/api/watcher.start.json      |    2 +-
 .../rest-api-spec/api/watcher.stats.json      |    2 +-
 .../rest-api-spec/api/watcher.stop.json       |    2 +-
 .../rest-api-spec/api/xpack.info.json         |    2 +-
 .../rest-api-spec/api/xpack.usage.json        |    2 +-
 .../rest-api-spec/test/api_key/10_basic.yml   |    6 +-
 .../test/data_frame/preview_transforms.yml    |  135 +-
 .../test/data_frame/transforms_crud.yml       |  165 +-
 .../test/data_frame/transforms_start_stop.yml |  149 +-
 .../test/data_frame/transforms_stats.yml      |   67 +-
 .../rest-api-spec/test/flattened/10_basic.yml |  111 +
 .../test/indices.freeze/10_basic.yml          |   19 +
 .../test/indices.freeze/20_stats.yml          |   62 +
 .../test/ml/data_frame_analytics_crud.yml     |  926 ++++++
 .../test/ml/evaluate_data_frame.yml           |  520 +++
 .../rest-api-spec/test/ml/filter_crud.yml     |   23 +-
 .../test/ml/find_file_structure.yml           |    7 +-
 .../rest-api-spec/test/ml/jobs_get_stats.yml  |  100 +-
 .../test/ml/start_data_frame_analytics.yml    |   74 +
 .../test/ml/stop_data_frame_analytics.yml     |   70 +
 .../test/privileges/11_builtin.yml            |   19 +
 .../rest-api-spec/test/rollup/start_job.yml   |    3 +-
 .../test/security/authz/14_cat_indices.yml    |  125 +
 .../test/vectors/10_dense_vector_basic.yml    |    5 +-
 .../vectors/20_dense_vector_special_cases.yml |   82 +-
 .../test/vectors/30_sparse_vector_basic.yml   |    4 +-
 .../40_sparse_vector_special_cases.yml        |    4 +-
 .../test/voting_only_node/10_basic.yml        |   16 +
 .../test/watcher/execute_watch/80_foreach.yml |  111 +
 .../rest-api-spec/test/xpack/15_basic.yml     |    8 -
 x-pack/plugin/vectors/build.gradle            |   22 +
 .../elasticsearch/xpack/vectors/Vectors.java  |   55 +
 .../vectors/VectorsInfoTransportAction.java   |   46 +
 .../vectors/VectorsUsageTransportAction.java  |   48 +
 .../mapper/DenseVectorFieldMapper.java        |   99 +-
 .../mapper/SparseVectorFieldMapper.java       |   29 +-
 .../vectors}/mapper/VectorEncoderDecoder.java |   31 +-
 .../query/DocValuesWhitelistExtension.java    |   30 +
 .../vectors}/query/ScoreScriptUtils.java      |   39 +-
 .../query/VectorDVAtomicFieldData.java        |   22 +-
 .../query/VectorDVIndexFieldData.java         |   22 +-
 .../vectors}/query/VectorScriptDocValues.java |   22 +-
 ...asticsearch.painless.spi.PainlessExtension |    1 +
 .../xpack/vectors/query/whitelist.txt         |   18 +
 .../VectorsInfoTransportActionTests.java      |   80 +
 .../mapper/DenseVectorFieldMapperTests.java   |  125 +
 .../mapper/DenseVectorFieldTypeTests.java     |   19 +
 .../mapper/SparseVectorFieldMapperTests.java  |   31 +-
 .../mapper/SparseVectorFieldTypeTests.java    |   19 +
 .../mapper/VectorEncoderDecoderTests.java     |   41 +-
 .../vectors}/query/ScoreScriptUtilsTests.java |   45 +-
 x-pack/plugin/voting-only-node/build.gradle   |   22 +
 .../VotingOnlyNodeFeatureSet.java             |  102 +
 .../coordination/VotingOnlyNodePlugin.java    |  230 ++
 .../LocalStateVotingOnlyNodePlugin.java       |   20 +
 .../VotingOnlyNodeCoordinationStateTests.java |   30 +
 .../VotingOnlyNodeCoordinatorTests.java       |   72 +
 .../VotingOnlyNodePluginTests.java            |  105 +
 x-pack/plugin/watcher/build.gradle            |    6 +-
 .../xpack/watcher/ClockHolder.java            |   23 +
 .../elasticsearch/xpack/watcher/Watcher.java  |   29 +-
 .../xpack/watcher/WatcherFeatureSet.java      |   92 -
 .../watcher/WatcherInfoTransportAction.java   |   46 +
 .../watcher/WatcherUsageTransportAction.java  |   82 +
 .../xpack/watcher/common/http/Scheme.java     |    2 +
 .../notification/NotificationService.java     |   17 +-
 .../notification/jira/JiraAccount.java        |    5 +-
 .../watcher/rest/WatcherRestHandler.java      |   31 -
 .../rest/action/RestAckWatchAction.java       |   19 +-
 .../rest/action/RestActivateWatchAction.java  |   55 +-
 .../rest/action/RestDeleteWatchAction.java    |   17 +-
 .../rest/action/RestExecuteWatchAction.java   |   25 +-
 .../rest/action/RestGetWatchAction.java       |   13 +-
 .../rest/action/RestPutWatchAction.java       |   15 +-
 .../rest/action/RestWatchServiceAction.java   |   23 +-
 .../rest/action/RestWatcherStatsAction.java   |   19 +-
 .../search/WatcherSearchTemplateService.java  |    3 -
 .../actions/ack/TransportAckWatchAction.java  |    5 +-
 .../TransportActivateWatchAction.java         |    5 +-
 .../execute/TransportExecuteWatchAction.java  |    5 +-
 .../actions/get/TransportGetWatchAction.java  |    5 +-
 .../actions/put/TransportPutWatchAction.java  |    5 +-
 .../TransportWatcherServiceAction.java        |   13 +-
 .../stats/TransportWatcherStatsAction.java    |    7 +-
 .../watcher/WatcherConcreteIndexTests.java    |    3 +-
 .../watcher/WatcherIndexingListenerTests.java |    9 +-
 ...a => WatcherInfoTransportActionTests.java} |   29 +-
 .../watcher/WatcherLifeCycleServiceTests.java |   12 +-
 .../xpack/watcher/WatcherPluginTests.java     |    3 +-
 .../xpack/watcher/WatcherServiceTests.java    |    5 +-
 .../watcher/WatcherXpackUsageStatsTests.java  |   52 -
 .../actions/ActionErrorIntegrationTests.java  |    6 +-
 .../watcher/actions/ActionWrapperTests.java   |  147 +-
 .../actions/TimeThrottleIntegrationTests.java |   12 +-
 .../actions/email/EmailAttachmentTests.java   |    7 +-
 .../throttler/ActionThrottleTests.java        |   51 +-
 .../webhook/WebhookHttpsIntegrationTests.java |    8 +-
 .../webhook/WebhookIntegrationTests.java      |   10 +-
 .../watcher/common/http/HttpClientTests.java  |    3 +-
 .../ExecuteWatchQueuedStatsTests.java         |   16 +-
 .../execution/ExecutionServiceTests.java      |   27 +-
 .../history/HistoryActionConditionTests.java  |    5 +-
 .../HistoryTemplateEmailMappingsTests.java    |    6 +-
 .../HistoryTemplateHttpMappingsTests.java     |   10 +-
 ...storyTemplateIndexActionMappingsTests.java |    3 +-
 ...storyTemplateSearchInputMappingsTests.java |    3 +-
 .../HistoryTemplateTimeMappingsTests.java     |    3 +-
 ...HistoryTemplateTransformMappingsTests.java |   10 +-
 .../input/chain/ChainIntegrationTests.java    |    3 +-
 .../NotificationServiceTests.java             |    7 +
 .../email/EmailSecretsIntegrationTests.java   |   11 +-
 .../notification/jira/JiraAccountTests.java   |   10 +
 .../action/RestExecuteWatchActionTests.java   |   27 +-
 .../WatcherIndexTemplateRegistryTests.java    |    4 -
 .../AbstractWatcherIntegrationTestCase.java   |   26 +-
 .../xpack/watcher/test/WatcherTestUtils.java  |    4 +-
 .../WatcherExecutorServiceBenchmark.java      |   10 +-
 .../bench/WatcherScheduleEngineBenchmark.java |   10 +-
 .../test/integration/BasicWatcherTests.java   |   49 +-
 .../test/integration/BootStrapTests.java      |   26 +-
 .../ExecutionVarsIntegrationTests.java        |   17 +-
 .../integration/HistoryIntegrationTests.java  |   21 +-
 .../HttpSecretsIntegrationTests.java          |   18 +-
 .../integration/RejectedExecutionTests.java   |    5 +-
 .../test/integration/SearchInputTests.java    |    2 +-
 .../test/integration/SingleNodeTests.java     |    3 +-
 .../test/integration/WatchAckTests.java       |   31 +-
 .../test/integration/WatchMetadataTests.java  |    8 +-
 .../transform/TransformIntegrationTests.java  |   18 +-
 .../action/activate/ActivateWatchTests.java   |   47 +-
 .../action/delete/DeleteWatchTests.java       |   16 +-
 .../action/execute/ExecuteWatchTests.java     |   17 +-
 .../transport/action/get/GetWatchTests.java   |   12 +-
 .../ack/TransportAckWatchActionTests.java     |    4 +-
 .../put/TransportPutWatchActionTests.java     |    3 +-
 .../TransportWatcherStatsActionTests.java     |    4 +-
 .../watch/WatchStatusIntegrationTests.java    |    8 +-
 .../xpack/watcher/watch/WatchTests.java       |    8 +-
 .../build.gradle                              |   41 +-
 .../xpack/restart/FullClusterRestartIT.java   |    1 -
 .../MlMigrationFullClusterRestartIT.java      |   53 -
 x-pack/qa/kerberos-tests/build.gradle         |   52 +-
 .../kerberos/KerberosAuthenticationIT.java    |   45 +-
 ...SpnegoHttpClientConfigCallbackHandler.java |   76 +
 .../build.gradle                              |   67 +-
 x-pack/qa/multi-node/build.gradle             |   23 +-
 x-pack/qa/oidc-op-tests/build.gradle          |   42 +-
 .../authc/oidc/OpenIdConnectAuthIT.java       |   29 +-
 x-pack/qa/openldap-tests/build.gradle         |    1 -
 ...OpenLdapUserSearchSessionFactoryTests.java |    2 -
 .../authc/ldap/SearchGroupsResolverTests.java |    1 -
 .../reindex-tests-with-security/build.gradle  |   42 +-
 x-pack/qa/rolling-upgrade/build.gradle        |    8 +-
 .../upgrades/DataFrameSurvivesUpgradeIT.java  |  279 ++
 .../test/mixed_cluster/30_ml_jobs_crud.yml    |   51 +
 .../mixed_cluster/80_data_frame_jobs_crud.yml |  292 ++
 .../test/old_cluster/30_ml_jobs_crud.yml      |  117 +
 .../old_cluster/80_data_frame_jobs_crud.yml   |  218 ++
 .../test/upgraded_cluster/30_ml_jobs_crud.yml |   56 +
 .../80_data_frame_jobs_crud.yml               |  259 ++
 x-pack/qa/saml-idp-tests/build.gradle         |   82 +-
 .../build.gradle                              |   36 +-
 .../build.gradle                              |   23 +-
 .../build.gradle                              |   14 +-
 .../MonitoringWithWatcherRestIT.java          |    2 -
 x-pack/qa/smoke-test-plugins-ssl/build.gradle |   35 +-
 .../SmokeTestMonitoringWithSecurityIT.java    |    2 +-
 x-pack/qa/smoke-test-plugins/build.gradle     |   28 +-
 .../build.gradle                              |   17 +-
 .../build.gradle                              |   32 +-
 .../SmokeTestWatcherWithSecurityIT.java       |   13 +-
 x-pack/qa/smoke-test-watcher/build.gradle     |    8 +-
 .../SmokeTestWatcherTestSuiteIT.java          |    4 +-
 .../test/mustache/50_webhook_url_escaping.yml |    5 +-
 .../test/rest/XPackRestTestConstants.java     |    8 +-
 .../ldap/AbstractAdLdapRealmTestCase.java     |    6 +-
 .../ActiveDirectoryGroupsResolverTests.java   |    1 -
 .../ActiveDirectorySessionFactoryTests.java   |    9 -
 .../UserAttributeGroupsResolverTests.java     |    3 -
 x-pack/qa/third-party/jira/build.gradle       |   28 +-
 x-pack/qa/third-party/pagerduty/build.gradle  |   21 +-
 x-pack/qa/third-party/slack/build.gradle      |   21 +-
 x-pack/test/feature-aware/build.gradle        |    6 +-
 3726 files changed, 94944 insertions(+), 49742 deletions(-)
 delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy
 delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy
 delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/UpdateShasTask.groovy
 delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/test/MessyTestPlugin.groovy
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/AbstractLazyPropertyCollection.java
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/EmptyDirTask.java
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/LazyPropertyMap.java
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/PropertyNormalization.java
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/SystemPropertyCommandLineArgumentProvider.java
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.java
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/precommit/UpdateShasTask.java
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/RestTestRunnerTask.java
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterCleanupOnShutdown.java
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersCleanupExtension.java
 create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/tool/ClasspathUtils.java
 rename buildSrc/src/main/resources/META-INF/gradle-plugins/{elasticsearch.messy-test.properties => elasticsearch.distribution-download.properties} (91%)
 create mode 100644 buildSrc/src/main/resources/buildSrc.marker
 rename buildSrc/src/{main => minimumRuntime}/java/org/elasticsearch/gradle/JdkJarHellCheck.java (100%)
 rename buildSrc/src/{main => minimumRuntime}/java/org/elasticsearch/gradle/LazyFileOutputStream.java (100%)
 rename buildSrc/src/{main => minimumRuntime}/java/org/elasticsearch/gradle/LoggedExec.java (94%)
 rename buildSrc/src/{main => minimumRuntime}/java/org/elasticsearch/gradle/Version.java (100%)
 rename buildSrc/src/{main => minimumRuntime}/java/org/elasticsearch/gradle/VersionProperties.java (100%)
 create mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java
 create mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginTests.java
 create mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/EmptyDirTaskTests.java
 create mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtensionTests.java
 create mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/precommit/DependencyLicensesTaskTests.java
 create mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/precommit/UpdateShasTaskTests.java
 create mode 100644 buildSrc/src/testKit/distribution-download/build.gradle
 create mode 100644 buildSrc/src/testKit/distribution-download/distribution/build.gradle
 create mode 100644 buildSrc/src/testKit/distribution-download/distribution/files/fake_elasticsearch.tar.gz
 create mode 100644 buildSrc/src/testKit/distribution-download/distribution/files/fake_elasticsearch.zip
 create mode 100644 buildSrc/src/testKit/distribution-download/settings.gradle
 create mode 100644 buildSrc/src/testKit/distribution-download/subproj/build.gradle
 delete mode 100644 client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java
 rename modules/mapper-extras/src/main/java/org/elasticsearch/index/query/DocValuesWhitelistExtension.java => client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/DataFrameNamedXContentProvider.java (52%)
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/NodeAttributes.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/SyncConfig.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/TimeSyncConfig.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/AnalyzeRequest.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/AnalyzeResponse.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DetailAnalyzeResponse.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/ReloadAnalyzersRequest.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/ReloadAnalyzersResponse.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteDataFrameAnalyticsRequest.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EvaluateDataFrameRequest.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EvaluateDataFrameResponse.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsResponse.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsStatsRequest.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsStatsResponse.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutDataFrameAnalyticsRequest.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutDataFrameAnalyticsResponse.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/StartDataFrameAnalyticsRequest.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/StopDataFrameAnalyticsRequest.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/StopDataFrameAnalyticsResponse.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalysis.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsDest.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsSource.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsState.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/MlDataFrameAnalysisNamedXContentProvider.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/OutlierDetection.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/QueryConfig.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/Evaluation.java
 rename server/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/AliasesExistAction.java => client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/EvaluationMetric.java (57%)
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/AbstractConfusionMatrixMetric.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/AucRocMetric.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/BinarySoftClassification.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/ConfusionMatrixMetric.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/PrecisionMetric.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/RecallMetric.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/process/TimingStats.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetBuiltinPrivilegesRequest.java
 create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetBuiltinPrivilegesResponse.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/AbstractHlrcWriteableXContentTestCase.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/NodeAttributesTests.java
 rename server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TypesExistsResponse.java => client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/TimeSyncConfigTests.java (52%)
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/TimeSyncConfigTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/AnalyzeGlobalRequestTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/AnalyzeIndexRequestTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/AnalyzeRequestTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/AnalyzeResponseTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/ReloadAnalyzersResponseTests.java
 rename server/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/AliasesExistResponse.java => client/rest-high-level/src/test/java/org/elasticsearch/client/ml/AucRocMetricAucRocPointTests.java (51%)
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/AucRocMetricResultTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/ConfusionMatrixMetricConfusionMatrixTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/ConfusionMatrixMetricResultTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/DeleteDataFrameAnalyticsRequestTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/EvaluateDataFrameResponseTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequestTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsStatsRequestTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PrecisionMetricResultTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PutDataFrameAnalyticsRequestTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/RecallMetricResultTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/StartDataFrameAnalyticsRequestTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/StopDataFrameAnalyticsRequestTests.java
 rename server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/IndicesExistsResponse.java => client/rest-high-level/src/test/java/org/elasticsearch/client/ml/StopDataFrameAnalyticsResponseTests.java (53%)
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsDestTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsSourceTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStatsTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/OutlierDetectionTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/QueryConfigTests.java
 create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/process/TimingStatsTests.java
 delete mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/ExecuteWatchResponseTests.java
 delete mode 100644 client/transport/build.gradle
 delete mode 100644 client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
 delete mode 100644 client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java
 create mode 100644 distribution/docker/docker-build-context/src/docker/config/log4j2.properties
 rename distribution/docker/{ => oss-docker-build-context}/src/docker/config/log4j2.properties (100%)
 delete mode 100644 docs/groovy-api/anatomy.asciidoc
 delete mode 100644 docs/groovy-api/client.asciidoc
 delete mode 100644 docs/groovy-api/delete.asciidoc
 delete mode 100644 docs/groovy-api/get.asciidoc
 delete mode 100644 docs/groovy-api/index.asciidoc
 delete mode 100644 docs/groovy-api/index_.asciidoc
 delete mode 100644 docs/groovy-api/search.asciidoc
 delete mode 100644 docs/java-api/admin/cluster/health.asciidoc
 delete mode 100644 docs/java-api/admin/cluster/index.asciidoc
 delete mode 100644 docs/java-api/admin/cluster/stored-scripts.asciidoc
 delete mode 100644 docs/java-api/admin/index.asciidoc
 delete mode 100644 docs/java-api/admin/indices/create-index.asciidoc
 delete mode 100644 docs/java-api/admin/indices/get-settings.asciidoc
 delete mode 100644 docs/java-api/admin/indices/index.asciidoc
 delete mode 100644 docs/java-api/admin/indices/put-mapping.asciidoc
 delete mode 100644 docs/java-api/admin/indices/refresh.asciidoc
 delete mode 100644 docs/java-api/admin/indices/update-settings.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket/children-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket/datehistogram-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket/daterange-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket/filter-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket/filters-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket/geodistance-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket/geohashgrid-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket/global-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket/histogram-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket/iprange-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket/missing-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket/nested-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket/range-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket/reverse-nested-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket/significantterms-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/bucket/terms-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/metrics.asciidoc
 delete mode 100644 docs/java-api/aggregations/metrics/avg-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/metrics/cardinality-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/metrics/extendedstats-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/metrics/geobounds-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/metrics/max-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/metrics/min-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/metrics/percentile-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/metrics/percentile-rank-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/metrics/scripted-metric-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/metrics/stats-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/metrics/sum-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/metrics/tophits-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggregations/metrics/valuecount-aggregation.asciidoc
 delete mode 100644 docs/java-api/aggs.asciidoc
 delete mode 100644 docs/java-api/client.asciidoc
 delete mode 100644 docs/java-api/docs.asciidoc
 delete mode 100644 docs/java-api/docs/bulk.asciidoc
 delete mode 100644 docs/java-api/docs/delete.asciidoc
 delete mode 100644 docs/java-api/docs/get.asciidoc
 delete mode 100644 docs/java-api/docs/index_.asciidoc
 delete mode 100644 docs/java-api/docs/multi-get.asciidoc
 delete mode 100644 docs/java-api/docs/reindex.asciidoc
 delete mode 100644 docs/java-api/docs/update-by-query.asciidoc
 delete mode 100644 docs/java-api/docs/update.asciidoc
 delete mode 100644 docs/java-api/index.asciidoc
 delete mode 100644 docs/java-api/query-dsl.asciidoc
 delete mode 100644 docs/java-api/query-dsl/bool-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/boosting-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/common-terms-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/compound-queries.asciidoc
 delete mode 100644 docs/java-api/query-dsl/constant-score-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/dis-max-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/exists-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/full-text-queries.asciidoc
 delete mode 100644 docs/java-api/query-dsl/function-score-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/fuzzy-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/geo-bounding-box-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/geo-distance-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/geo-polygon-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/geo-queries.asciidoc
 delete mode 100644 docs/java-api/query-dsl/geo-shape-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/has-child-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/has-parent-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/ids-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/joining-queries.asciidoc
 delete mode 100644 docs/java-api/query-dsl/match-all-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/match-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/mlt-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/multi-match-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/nested-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/percolate-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/prefix-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/query-string-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/range-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/regexp-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/script-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/simple-query-string-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/span-containing-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/span-first-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/span-multi-term-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/span-near-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/span-not-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/span-or-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/span-queries.asciidoc
 delete mode 100644 docs/java-api/query-dsl/span-term-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/span-within-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/special-queries.asciidoc
 delete mode 100644 docs/java-api/query-dsl/term-level-queries.asciidoc
 delete mode 100644 docs/java-api/query-dsl/term-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/terms-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/wildcard-query.asciidoc
 delete mode 100644 docs/java-api/query-dsl/wrapper-query.asciidoc
 delete mode 100644 docs/java-api/search.asciidoc
 create mode 100644 docs/java-rest/high-level/indices/reload_analyzers.asciidoc
 create mode 100644 docs/java-rest/high-level/ml/delete-data-frame-analytics.asciidoc
 create mode 100644 docs/java-rest/high-level/ml/evaluate-data-frame.asciidoc
 create mode 100644 docs/java-rest/high-level/ml/get-data-frame-analytics-stats.asciidoc
 create mode 100644 docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc
 create mode 100644 docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc
 create mode 100644 docs/java-rest/high-level/ml/start-data-frame-analytics.asciidoc
 create mode 100644 docs/java-rest/high-level/ml/stop-data-frame-analytics.asciidoc
 create mode 100644 docs/java-rest/high-level/security/get-builtin-privileges.asciidoc
 create mode 100644 docs/painless/painless-guide/painless-datetime.asciidoc
 create mode 100644 docs/painless/redirects.asciidoc
 create mode 100644 docs/reference/administering.asciidoc
 create mode 100644 docs/reference/aggregations/bucket/rare-terms-aggregation.asciidoc
 create mode 100644 docs/reference/analysis/tokenizers/pathhierarchy-tokenizer-examples.asciidoc
 create mode 100644 docs/reference/data-frames/apis/pivotresource.asciidoc
 create mode 100644 docs/reference/images/rare_terms/accuracy_0001.png
 create mode 100644 docs/reference/images/rare_terms/accuracy_001.png
 create mode 100644 docs/reference/images/rare_terms/accuracy_01.png
 create mode 100644 docs/reference/images/rare_terms/memory.png
 create mode 100644 docs/reference/indices/apis/reload-analyzers.asciidoc
 create mode 100644 docs/reference/intro.asciidoc
 create mode 100644 docs/reference/mapping/types/flattened.asciidoc
 create mode 100644 docs/reference/migration/migrate_8_0/rollup.asciidoc
 delete mode 100644 docs/reference/modules/memcached.asciidoc
 delete mode 100644 docs/reference/modules/ml-node.asciidoc
 delete mode 100644 docs/reference/modules/thrift.asciidoc
 create mode 100644 docs/reference/monitoring/configuring-filebeat.asciidoc
 delete mode 100644 docs/reference/query-dsl/common-terms-query.asciidoc
 rename docs/reference/{modules => }/scripting.asciidoc (86%)
 rename docs/reference/{modules => }/scripting/engine.asciidoc (97%)
 rename docs/reference/{modules => }/scripting/expression.asciidoc (99%)
 rename docs/reference/{modules => }/scripting/fields.asciidoc (98%)
 rename docs/reference/{modules => }/scripting/painless.asciidoc (97%)
 rename docs/reference/{modules => }/scripting/security.asciidoc (99%)
 rename docs/reference/{modules => }/scripting/using.asciidoc (97%)
 delete mode 100644 docs/reference/security/securing-communications/separating-node-client-traffic.asciidoc
 create mode 100644 docs/reference/setup/install/brew.asciidoc
 delete mode 100644 libs/geo/src/main/java/org/elasticsearch/geo/geometry/GeometryUtils.java
 create mode 100644 libs/geo/src/main/java/org/elasticsearch/geo/utils/GeographyValidator.java
 create mode 100644 libs/geo/src/main/java/org/elasticsearch/geo/utils/GeometryValidator.java
 create mode 100644 libs/geo/src/test/java/org/elasticsearch/geo/geometry/GeometryValidatorTests.java
 delete mode 100644 libs/grok/licenses/jcodings-1.0.12.jar.sha1
 create mode 100644 libs/grok/licenses/jcodings-1.0.44.jar.sha1
 create mode 100644 libs/nio/src/main/java/org/elasticsearch/nio/DelegatingHandler.java
 rename libs/nio/src/main/java/org/elasticsearch/nio/{ReadWriteHandler.java => NioChannelHandler.java} (90%)
 delete mode 100644 modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StandardHtmlStripAnalyzer.java
 create mode 100644 modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java
 create mode 100644 modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ElisionFilterFactoryTests.java
 rename server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TypesExistsAction.java => modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DeprecatedAnnotation.java (63%)
 create mode 100644 modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DeprecatedAnnotationParser.java
 rename modules/{mapper-extras/src/test/java/org/elasticsearch/index/mapper/SparseVectorFieldTypeTests.java => lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NoImportAnnotation.java} (75%)
 rename server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/IndicesExistsAction.java => modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NoImportAnnotationParser.java (59%)
 create mode 100644 modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/AnnotationTestObject.java
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/DateTimeTests.java
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/GetByPathAugmentationTests.java
 delete mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/WhitelistLoaderTests.java
 create mode 100644 modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.annotation
 rename {rest-api-spec/src/main => modules/lang-painless/src/test}/resources/rest-api-spec/api/scripts_painless_context.json (88%)
 create mode 100644 modules/lang-painless/src/test/resources/rest-api-spec/test/painless/100_terms_agg.yml
 delete mode 100644 modules/mapper-extras/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension
 delete mode 100644 modules/mapper-extras/src/main/resources/org/elasticsearch/index/query/docvalues_whitelist.txt
 delete mode 100644 modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/DenseVectorFieldMapperTests.java
 create mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsersTests.java
 delete mode 100644 modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfig.java
 delete mode 100644 modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfigBuilder.java
 delete mode 100644 modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java
 create mode 100644 plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseStopTokenFilterFactory.java
 create mode 100644 plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStopTokenFilterFactory.java
 rename {server => plugins/discovery-azure-classic}/licenses/log4j-1.2-api-2.11.1.jar.sha1 (100%)
 rename {server => plugins/discovery-azure-classic}/licenses/log4j-LICENSE.txt (100%)
 rename {server => plugins/discovery-azure-classic}/licenses/log4j-NOTICE.txt (100%)
 delete mode 100644 plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.505.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.562.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.505.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.562.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/log4j-1.2-api-2.11.1.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/log4j-LICENSE.txt
 create mode 100644 plugins/discovery-ec2/licenses/log4j-NOTICE.txt
 create mode 100644 plugins/discovery-gce/licenses/log4j-1.2-api-2.11.1.jar.sha1
 create mode 100644 plugins/discovery-gce/licenses/log4j-LICENSE.txt
 create mode 100644 plugins/discovery-gce/licenses/log4j-NOTICE.txt
 rename server/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/AliasesExistRequestBuilder.java => plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExamplePainlessAnnotation.java (62%)
 create mode 100644 plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistAnnotationParser.java
 create mode 100644 plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
 delete mode 100644 plugins/repository-gcs/licenses/api-common-1.7.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/api-common-1.8.1.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/gax-1.30.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/gax-1.45.0.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/gax-httpjson-0.47.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/gax-httpjson-0.62.0.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-api-client-1.24.1.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-api-client-1.28.0.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-api-services-storage-v1-rev135-1.24.1.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20190426-1.28.0.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-auth-library-credentials-0.10.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-auth-library-credentials-0.16.1.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-auth-library-oauth2-http-0.10.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-auth-library-oauth2-http-0.16.1.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-cloud-core-1.59.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-cloud-core-1.77.0.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-cloud-core-http-1.59.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-cloud-core-http-1.77.0.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-cloud-storage-1.59.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-cloud-storage-1.77.0.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-http-client-1.24.1.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-http-client-1.30.1.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-http-client-appengine-1.24.1.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-http-client-appengine-1.29.2.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-http-client-jackson-1.24.1.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-http-client-jackson2-1.24.1.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-http-client-jackson2-1.29.2.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/google-oauth-client-1.24.1.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-oauth-client-1.28.0.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/guava-20.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/guava-26.0-jre.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/jackson-LICENSE
 delete mode 100644 plugins/repository-gcs/licenses/jackson-NOTICE
 delete mode 100644 plugins/repository-gcs/licenses/jackson-core-asl-1.9.11.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/log4j-1.2-api-2.11.1.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/log4j-LICENSE.txt
 create mode 100644 plugins/repository-gcs/licenses/log4j-NOTICE.txt
 delete mode 100644 plugins/repository-gcs/licenses/opencensus-api-0.15.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/opencensus-api-0.18.0.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.15.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.18.0.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/proto-google-common-protos-1.12.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/proto-google-common-protos-1.16.0.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/protobuf-java-3.6.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/protobuf-java-3.7.1.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/protobuf-java-util-3.6.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/protobuf-java-util-3.7.1.jar.sha1
 create mode 100644 plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java
 create mode 100644 plugins/repository-hdfs/licenses/log4j-1.2-api-2.11.1.jar.sha1
 create mode 100644 plugins/repository-hdfs/licenses/log4j-LICENSE.txt
 create mode 100644 plugins/repository-hdfs/licenses/log4j-NOTICE.txt
 create mode 100644 plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsRepositoryTests.java
 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.505.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.562.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.505.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.505.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.562.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.505.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.562.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/log4j-1.2-api-2.11.1.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/log4j-LICENSE.txt
 create mode 100644 plugins/repository-s3/licenses/log4j-NOTICE.txt
 create mode 100644 plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java
 delete mode 100644 plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsConfig.java
 delete mode 100644 plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsConfigBuilder.java
 delete mode 100644 qa/smoke-test-client/build.gradle
 delete mode 100644 qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java
 delete mode 100644 qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/SmokeTestClientIT.java
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/count/11_basic_with_types.yml
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/msearch/12_basic_with_types.yml
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/280_rare_terms.yml
 delete mode 100644 server/licenses/hppc-0.7.1.jar.sha1
 create mode 100644 server/licenses/hppc-0.8.1.jar.sha1
 delete mode 100644 server/src/main/java/org/apache/lucene/queries/ExtendedCommonTermsQuery.java
 create mode 100644 server/src/main/java/org/apache/lucene/util/CombinedBitSet.java
 rename server/src/main/java/org/elasticsearch/action/{Action.java => ActionType.java} (75%)
 create mode 100644 server/src/main/java/org/elasticsearch/action/RequestValidators.java
 create mode 100644 server/src/main/java/org/elasticsearch/action/StreamableResponseActionType.java
 delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/TransportAliasesExistAction.java
 delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java
 delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java
 delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java
 delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/IndicesExistsRequest.java
 delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/IndicesExistsRequestBuilder.java
 delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java
 delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TransportTypesExistsAction.java
 delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TypesExistsRequest.java
 delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TypesExistsRequestBuilder.java
 delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/MappingRequestValidator.java
 delete mode 100644 server/src/main/java/org/elasticsearch/client/transport/TransportClient.java
 delete mode 100644 server/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
 delete mode 100644 server/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/coordination/ElectionStrategy.java
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/routing/BatchedRerouteService.java
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/routing/LazilyInitializedRerouteService.java
 create mode 100644 server/src/main/java/org/elasticsearch/cluster/routing/RerouteService.java
 delete mode 100644 server/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
 rename buildSrc/src/main/groovy/org/elasticsearch/gradle/EmptyDirTask.groovy => server/src/main/java/org/elasticsearch/common/TriConsumer.java (59%)
 rename server/src/main/java/org/{apache/lucene/search/XIndexSearcher.java => elasticsearch/common/geo/GeometryFormat.java} (52%)
 create mode 100644 server/src/main/java/org/elasticsearch/common/settings/ConsistentSettingsService.java
 create mode 100644 server/src/main/java/org/elasticsearch/common/util/CuckooFilter.java
 create mode 100644 server/src/main/java/org/elasticsearch/common/util/SetBackedScalingCuckooFilter.java
 create mode 100644 server/src/main/java/org/elasticsearch/http/CorsHandler.java
 create mode 100644 server/src/main/java/org/elasticsearch/index/analysis/AnalyzerComponents.java
 rename modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/DenseVectorFieldTypeTests.java => server/src/main/java/org/elasticsearch/index/analysis/AnalyzerComponentsProvider.java (76%)
 create mode 100644 server/src/main/java/org/elasticsearch/index/analysis/NameOrDefinition.java
 create mode 100644 server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java
 create mode 100644 server/src/main/java/org/elasticsearch/index/engine/CombinedDocValues.java
 create mode 100644 server/src/main/java/org/elasticsearch/index/engine/PrunePostingsMergePolicy.java
 create mode 100644 server/src/main/java/org/elasticsearch/index/fieldvisitor/IdOnlyFieldVisitor.java
 create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java
 delete mode 100644 server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
 delete mode 100644 server/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractRareTermsAggregator.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedRareTerms.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalRareTerms.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongRareTerms.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongRareTermsAggregator.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTerms.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorFactory.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringRareTerms.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringRareTermsAggregator.java
 create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/UnmappedRareTerms.java
 create mode 100644 server/src/test/java/org/apache/lucene/util/CombinedBitSetTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/action/RequestValidatorsTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/alias/ValidateIndicesAliasesRequestIT.java
 delete mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java
 create mode 100644 server/src/test/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponseTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleIT.java
 create mode 100644 server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/cluster/routing/BatchedRerouteServiceTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/cluster/routing/allocation/RetryFailedAllocationTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/common/settings/ConsistentSettingsIT.java
 create mode 100644 server/src/test/java/org/elasticsearch/common/settings/ConsistentSettingsServiceTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/common/util/CuckooFilterTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/common/util/SetBackedScalingCuckooFilterTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/discovery/DiskDisruptionIT.java
 create mode 100644 server/src/test/java/org/elasticsearch/http/CorsHandlerTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzerTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/index/engine/PrunePostingsMergePolicyTests.java
 delete mode 100644 server/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java
 delete mode 100644 server/src/test/java/org/elasticsearch/index/query/CommonTermsQueryParserTests.java
 delete mode 100644 server/src/test/java/org/elasticsearch/index/query/FullTextQueryTestCase.java
 rename server/src/test/java/org/elasticsearch/index/shard/{IndexSearcherWrapperTests.java => IndexReaderWrapperTests.java} (65%)
 create mode 100644 server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceTests.java
 delete mode 100644 server/src/test/java/org/elasticsearch/indices/exists/indices/IndicesExistsIT.java
 delete mode 100644 server/src/test/java/org/elasticsearch/indices/exists/types/TypesExistsIT.java
 delete mode 100644 server/src/test/java/org/elasticsearch/rest/action/search/RestCountActionTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/RareTermsTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilderTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceTypeTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
 create mode 100644 server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java
 delete mode 100644 server/src/test/resources/org/elasticsearch/index/query/commonTerms-query1.json
 delete mode 100644 server/src/test/resources/org/elasticsearch/index/query/commonTerms-query2.json
 delete mode 100644 server/src/test/resources/org/elasticsearch/index/query/commonTerms-query3.json
 create mode 100644 server/src/test/resources/org/elasticsearch/index/translog/translog-v2.binary
 create mode 100644 test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java
 create mode 100644 test/framework/src/main/java/org/elasticsearch/cluster/coordination/CoordinationStateTestCluster.java
 rename {server/src/test => test/framework/src/main}/java/org/elasticsearch/cluster/coordination/LinearizabilityChecker.java (100%)
 rename {server/src/test => test/framework/src/main}/java/org/elasticsearch/gateway/MockGatewayMetaState.java (95%)
 rename {server/src/test => test/framework/src/main}/java/org/elasticsearch/indices/cluster/FakeThreadPoolMasterService.java (100%)
 create mode 100644 test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java
 create mode 100644 test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java
 rename {server/src/test/java/org/elasticsearch/action/support/broadcast => test/framework/src/main/java/org/elasticsearch/test}/AbstractBroadcastResponseTestCase.java (98%)
 create mode 100644 test/framework/src/main/java/org/elasticsearch/test/hamcrest/OptionalMatchers.java
 delete mode 100644 test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java
 create mode 100644 x-pack/docs/en/rest-api/security/get-builtin-privileges.asciidoc
 rename x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/{CCRFeatureSetTests.java => CCRInfoTransportActionTests.java} (75%)
 create mode 100644 x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrAliasesIT.java
 create mode 100644 x-pack/plugin/core/licenses/log4j-1.2-api-2.11.1.jar.sha1
 create mode 100644 x-pack/plugin/core/licenses/log4j-LICENSE.txt
 create mode 100644 x-pack/plugin/core/licenses/log4j-NOTICE.txt
 rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/{core/ccr/CCRFeatureSet.java => ccr/CCRInfoTransportAction.java} (65%)
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/ccr/CCRUsageTransportAction.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/EmptyXPackFeatureSet.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClient.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/ReloadAnalyzerAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/ReloadAnalyzersRequest.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/ReloadAnalyzersResponse.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportReloadAnalyzersAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureResponse.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureTransportAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureResponse.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureTransportAction.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/client/CcrClient.java
 rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/{ml/utils => common}/time/TimeUtils.java (87%)
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameNamedXContentProvider.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/NodeAttributes.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/SyncConfig.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/TimeSyncConfig.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/utils/DataFrameStrings.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/utils/TimeUtils.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/client/ILMClient.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteDataFrameAnalyticsAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/EvaluateDataFrameAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsAction.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/client/MachineLearningClient.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsDest.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsSource.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsState.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/MlDataFrameAnalysisNamedXContentProvider.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/Evaluation.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationMetricResult.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AbstractConfusionMatrixMetric.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AucRoc.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/BinarySoftClassification.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/ConfusionMatrix.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/Precision.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/Recall.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/ScoreByThresholdResult.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/SoftClassificationMetric.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStats.java
 rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/{datafeed => utils}/QueryProvider.java (81%)
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/client/MonitoringClient.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/XPackRestHandler.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestReloadAnalyzersAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesAction.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesRequest.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponse.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCache.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/CacheIteratorHelper.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/vectors/VectorsFeatureSetUsage.java
 create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/votingonly/VotingOnlyNodeFeatureSetUsage.java
 delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatcherClient.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadAnalyzersResponseTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadSynonymAnalyzerTests.java
 rename x-pack/plugin/{ml/src/test/java/org/elasticsearch/xpack/ml/utils => core/src/test/java/org/elasticsearch/xpack/core/common}/time/TimeUtilsTests.java (64%)
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/NodeAttributeTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/TimeSyncConfigTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/utils/DataFrameStringsTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/EvaluateDataFrameActionRequestTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsActionResponseTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsRequestTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsRequestTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionRequestTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionResponseTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsActionTaskParamsTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsRequestTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsActionResponseTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsRequestTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsDestTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsSourceTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsStateTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskStateTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetectionTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AucRocTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/BinarySoftClassificationTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/ConfusionMatrixTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/PrecisionTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/RecallTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStatsTests.java
 rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/{datafeed => utils}/QueryProviderTests.java (92%)
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rest/action/ReloadSynonymAnalyzerIT.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponseTests.java
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java
 rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/{SecurityIndexSearcherWrapperIntegrationTests.java => SecurityIndexReaderWrapperIntegrationTests.java} (84%)
 create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperUnitTests.java
 delete mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java
 rename x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/{DataFrameFeatureSet.java => DataFrameInfoTransportAction.java} (54%)
 create mode 100644 x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameUsageTransportAction.java
 rename x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/{DataFrameFeatureSetTests.java => DataFrameInfoTransportActionTests.java} (74%)
 create mode 100644 x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndexTests.java
 rename x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/{GraphFeatureSet.java => GraphInfoTransportAction.java} (57%)
 create mode 100644 x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphUsageTransportAction.java
 rename x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/{GraphFeatureSetTests.java => GraphInfoTransportActionTests.java} (62%)
 delete mode 100644 x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSet.java
 create mode 100644 x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInfoTransportAction.java
 create mode 100644 x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleUsageTransportAction.java
 rename x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/{IndexLifecycleFeatureSetTests.java => IndexLifecycleInfoTransportActionTests.java} (79%)
 rename x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/{LogstashFeatureSet.java => LogstashInfoTransportAction.java} (53%)
 create mode 100644 x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashUsageTransportAction.java
 rename x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/{LogstashFeatureSetTests.java => LogstashInfoTransportActionTests.java} (55%)
 create mode 100644 x-pack/plugin/mapper-flattened/build.gradle
 create mode 100644 x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/FlattenedInfoTransportAction.java
 create mode 100644 x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/FlattenedMapperPlugin.java
 create mode 100644 x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/FlattenedUsageTransportAction.java
 create mode 100644 x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java
 create mode 100644 x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldParser.java
 create mode 100644 x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectAtomicFieldData.java
 create mode 100644 x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java
 create mode 100644 x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java
 create mode 100644 x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldParserTests.java
 create mode 100644 x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectIndexFieldDataTests.java
 create mode 100644 x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectSearchTests.java
 create mode 100644 x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectAtomicFieldDataTests.java
 create mode 100644 x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectFieldTypeTests.java
 create mode 100644 x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/RootFlatObjectFieldTypeTests.java
 create mode 100644 x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java
 create mode 100644 x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java
 delete mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportAction.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEvaluateDataFrameAction.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndex.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/SourceDestValidator.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/persistence/DataFrameAnalyticsConfigProvider.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsBuilder.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsControlMessageWriter.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcess.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessFactory.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResult.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoiner.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcess.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/results/RowResults.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobNodeSelector.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/TimingStatsReporter.java
 rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/{AutoDetectResultProcessor.java => AutodetectResultProcessor.java} (79%)
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/DummyController.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlController.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlControllerHolder.java
 delete mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeControllerHolder.java
 rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/{job/process/autodetect/output/AutodetectResultsParser.java => process/ProcessResultsParser.java} (72%)
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestDeleteDataFrameAnalyticsAction.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestEvaluateDataFrameAction.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsStatsAction.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestPutDataFrameAnalyticsAction.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestStartDataFrameAnalyticsAction.java
 create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestStopDataFrameAnalyticsAction.java
 create mode 100644 x-pack/plugin/ml/src/main/plugin-metadata/plugin-security-test.policy
 rename x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/{MachineLearningFeatureSetTests.java => MachineLearningInfoTransportActionTests.java} (82%)
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsActionTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndexTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/MappingsMergerTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/SourceDestValidatorTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsControlMessageWriterTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoinerTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/results/RowResultsTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobNodeSelectorTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/TimingStatsReporterTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactoryTests.java
 rename x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/{AutoDetectResultProcessorTests.java => AutodetectResultProcessorTests.java} (52%)
 delete mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParserTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/AutodetectParamsTests.java
 create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/ProcessResultsParserTests.java
 delete mode 100644 x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringFeatureSet.java
 create mode 100644 x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportAction.java
 create mode 100644 x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringUsageServices.java
 create mode 100644 x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringUsageTransportAction.java
 rename x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/{MonitoringFeatureSetTests.java => MonitoringInfoTransportActionTests.java} (82%)
 rename x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/{RollupFeatureSet.java => RollupInfoTransportAction.java} (50%)
 create mode 100644 x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupUsageTransportAction.java
 rename x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/{RollupFeatureSetTests.java => RollupInfoTransportActionTests.java} (58%)
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityInfoTransportAction.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageServices.java
 rename x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/{SecurityFeatureSet.java => SecurityUsageTransportAction.java} (73%)
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetBuiltinPrivilegesAction.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java
 create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityStatusChangeListener.java
 rename x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/{SecurityFeatureSetTests.java => SecurityInfoTransportActionTests.java} (88%)
 create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityStatusChangeListenerTests.java
 delete mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java
 delete mode 100644 x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java
 delete mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlFeatureSet.java
 create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlInfoTransportAction.java
 create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlUsageTransportAction.java
 rename x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/{SqlFeatureSetTests.java => SqlInfoTransportActionTests.java} (77%)
 create mode 100644 x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/QuerierTests.java
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/indices.reload_search_analyzers.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_data_frame_analytics.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/ml.evaluate_data_frame.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics_stats.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_data_frame_analytics.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/ml.start_data_frame_analytics.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/ml.stop_data_frame_analytics.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_builtin_privileges.json
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/10_basic.yml
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/20_stats.yml
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/ml/evaluate_data_frame.yml
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/ml/stop_data_frame_analytics.yml
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/11_builtin.yml
 rename modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/10_basic.yml => x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/10_dense_vector_basic.yml (95%)
 rename modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/20_special_cases.yml => x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/20_dense_vector_special_cases.yml (60%)
 rename modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/10_basic.yml => x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/30_sparse_vector_basic.yml (96%)
 rename modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/20_special_cases.yml => x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/40_sparse_vector_special_cases.yml (98%)
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/voting_only_node/10_basic.yml
 create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/execute_watch/80_foreach.yml
 create mode 100644 x-pack/plugin/vectors/build.gradle
 create mode 100644 x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/Vectors.java
 create mode 100644 x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/VectorsInfoTransportAction.java
 create mode 100644 x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/VectorsUsageTransportAction.java
 rename {modules/mapper-extras/src/main/java/org/elasticsearch/index => x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors}/mapper/DenseVectorFieldMapper.java (67%)
 rename {modules/mapper-extras/src/main/java/org/elasticsearch/index => x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors}/mapper/SparseVectorFieldMapper.java (89%)
 rename {modules/mapper-extras/src/main/java/org/elasticsearch/index => x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors}/mapper/VectorEncoderDecoder.java (84%)
 create mode 100644 x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DocValuesWhitelistExtension.java
 rename {modules/mapper-extras/src/main/java/org/elasticsearch/index => x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors}/query/ScoreScriptUtils.java (87%)
 rename {modules/mapper-extras/src/main/java/org/elasticsearch/index => x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors}/query/VectorDVAtomicFieldData.java (69%)
 rename {modules/mapper-extras/src/main/java/org/elasticsearch/index => x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors}/query/VectorDVIndexFieldData.java (72%)
 rename {modules/mapper-extras/src/main/java/org/elasticsearch/index => x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors}/query/VectorScriptDocValues.java (67%)
 create mode 100644 x-pack/plugin/vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension
 create mode 100644 x-pack/plugin/vectors/src/main/resources/org/elasticsearch/xpack/vectors/query/whitelist.txt
 create mode 100644 x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/VectorsInfoTransportActionTests.java
 create mode 100644 x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java
 create mode 100644 x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldTypeTests.java
 rename {modules/mapper-extras/src/test/java/org/elasticsearch/index => x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors}/mapper/SparseVectorFieldMapperTests.java (90%)
 create mode 100644 x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java
 rename {modules/mapper-extras/src/test/java/org/elasticsearch/index => x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors}/mapper/VectorEncoderDecoderTests.java (69%)
 rename {modules/mapper-extras/src/test/java/org/elasticsearch/index => x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors}/query/ScoreScriptUtilsTests.java (61%)
 create mode 100644 x-pack/plugin/voting-only-node/build.gradle
 create mode 100644 x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/VotingOnlyNodeFeatureSet.java
 create mode 100644 x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/VotingOnlyNodePlugin.java
 create mode 100644 x-pack/plugin/voting-only-node/src/test/java/org/elasticsearch/cluster/coordination/LocalStateVotingOnlyNodePlugin.java
 create mode 100644 x-pack/plugin/voting-only-node/src/test/java/org/elasticsearch/cluster/coordination/VotingOnlyNodeCoordinationStateTests.java
 create mode 100644 x-pack/plugin/voting-only-node/src/test/java/org/elasticsearch/cluster/coordination/VotingOnlyNodeCoordinatorTests.java
 create mode 100644 x-pack/plugin/voting-only-node/src/test/java/org/elasticsearch/cluster/coordination/VotingOnlyNodePluginTests.java
 create mode 100644 x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/ClockHolder.java
 delete mode 100644 x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherFeatureSet.java
 create mode 100644 x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportAction.java
 create mode 100644 x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherUsageTransportAction.java
 delete mode 100644 x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/WatcherRestHandler.java
 rename x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/{WatcherFeatureSetTests.java => WatcherInfoTransportActionTests.java} (81%)
 delete mode 100644 x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherXpackUsageStatsTests.java
 create mode 100644 x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataFrameSurvivesUpgradeIT.java
 create mode 100644 x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_data_frame_jobs_crud.yml
 create mode 100644 x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/80_data_frame_jobs_crud.yml
 create mode 100644 x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_data_frame_jobs_crud.yml

diff --git a/.ci/java-versions.properties b/.ci/java-versions.properties
index e40470bf51b89..16b760507324d 100644
--- a/.ci/java-versions.properties
+++ b/.ci/java-versions.properties
@@ -8,3 +8,4 @@ ES_BUILD_JAVA=openjdk12
 ES_RUNTIME_JAVA=java11
 GRADLE_TASK=build
 
+
diff --git a/.ci/matrix-runtime-javas.yml b/.ci/matrix-runtime-javas.yml
index 61746ea59e1d0..583acce078055 100644
--- a/.ci/matrix-runtime-javas.yml
+++ b/.ci/matrix-runtime-javas.yml
@@ -9,6 +9,7 @@ ES_RUNTIME_JAVA:
   - java11
   - java12
   - openjdk12
+  - openjdk13
   - zulu11
   - zulu12
   - corretto11
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 6a4531f1bdefa..57f0992b9172d 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -10,5 +10,5 @@ attention.
 - Have you followed the [contributor guidelines](https://github.com/elastic/elasticsearch/blob/master/CONTRIBUTING.md)?
 - If submitting code, have you built your formula locally prior to submission with `gradle check`?
 - If submitting code, is your pull request against master? Unless there is a good reason otherwise, we prefer pull requests against master and will backport as needed.
-- If submitting code, have you checked that your submission is for an [OS that we support](https://www.elastic.co/support/matrix#show_os)?
+- If submitting code, have you checked that your submission is for an [OS and architecture that we support](https://www.elastic.co/support/matrix#show_os)?
 - If you are submitting this code for a class then read our [policy](https://github.com/elastic/elasticsearch/blob/master/CONTRIBUTING.md#contributing-as-part-of-a-class) for that.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 9bf11f2b49fd5..c2bae4e206256 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -325,8 +325,7 @@ Commercially licensed code that integrates with the rest of Elasticsearch. The
 `docs` subdirectory functions just like the top level `docs` subdirectory and
 the `qa` subdirectory functions just like the top level `qa` subdirectory. The
 `plugin` subdirectory contains the x-pack module which runs inside the
-Elasticsearch process. The `transport-client` subdirectory contains extensions
-to Elasticsearch's standard transport client to work properly with x-pack.
+Elasticsearch process.
 
 ### Gradle Build
 
diff --git a/README.textile b/README.textile
index 5912c0371fa2e..6f907f370e7a5 100644
--- a/README.textile
+++ b/README.textile
@@ -150,13 +150,13 @@ curl -XPUT 'http://localhost:9200/kimchy/_doc/2?pretty' -H 'Content-Type: applic
 
 The above will index information into the @kimchy@ index. Each user will get their own special index.
 
-Complete control on the index level is allowed. As an example, in the above case, we would want to change from the default 5 shards with 1 replica per index, to only 1 shard with 1 replica per index (== per twitter user). Here is how this can be done (the configuration can be in yaml as well):
+Complete control on the index level is allowed. As an example, in the above case, we might want to change from the default 1 shard with 1 replica per index, to 2 shards with 1 replica per index (because this user tweets a lot). Here is how this can be done (the configuration can be in yaml as well):
 
 <pre>
 curl -XPUT http://localhost:9200/another_user?pretty -H 'Content-Type: application/json' -d '
 {
     "index" : {
-        "number_of_shards" : 1,
+        "number_of_shards" : 2,
         "number_of_replicas" : 1
     }
 }'
@@ -191,7 +191,7 @@ h3. Distributed, Highly Available
 
 Let's face it, things will fail....
 
-Elasticsearch is a highly available and distributed search engine. Each index is broken down into shards, and each shard can have one or more replicas. By default, an index is created with 5 shards and 1 replica per shard (5/1). There are many topologies that can be used, including 1/10 (improve search performance), or 20/1 (improve indexing performance, with search executed in a map reduce fashion across shards).
+Elasticsearch is a highly available and distributed search engine. Each index is broken down into shards, and each shard can have one or more replicas. By default, an index is created with 1 shards and 1 replica per shard (1/1). There are many topologies that can be used, including 1/10 (improve search performance), or 20/1 (improve indexing performance, with search executed in a map reduce fashion across shards).
 
 In order to play with the distributed nature of Elasticsearch, simply bring more nodes up and shut down nodes. The system will continue to serve requests (make sure you use the correct http port) with the latest data indexed.
 
diff --git a/TESTING.asciidoc b/TESTING.asciidoc
index 5a443127745e6..022ebc47f7ec0 100644
--- a/TESTING.asciidoc
+++ b/TESTING.asciidoc
@@ -305,12 +305,6 @@ e.g. -Dtests.rest.suite=index,get,create/10_with_id
 blacklisted and need to be skipped
 e.g. -Dtests.rest.blacklist=index/*/Index document,get/10_basic/*
 
-Note that the REST tests, like all the integration tests, can be run against an external
-cluster by specifying the `tests.cluster` property, which if present needs to contain a
-comma separated list of nodes to connect to (e.g. localhost:9300). A transport client will
-be created based on that and used for all the before|after test operations, and to extract
-the http addresses of the nodes so that REST requests can be sent to them.
-
 == Testing packaging
 
 The packaging tests use Vagrant virtual machines to verify that installing
diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
index e85f9a5608644..376ad4d4e6716 100644
--- a/benchmarks/build.gradle
+++ b/benchmarks/build.gradle
@@ -27,7 +27,7 @@ archivesBaseName = 'elasticsearch-benchmarks'
 test.enabled = false
 
 dependencies {
-    compile("org.elasticsearch:elasticsearch:${version}") {
+    compile(project(":server")) {
         // JMH ships with the conflicting version 4.6. This prevents us from using jopt-simple in benchmarks (which should be ok) but allows
         // us to invoke the JMH uberjar as usual.
         exclude group: 'net.sf.jopt-simple', module: 'jopt-simple'
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
index 39a2bdfca0953..90f03c26dcb27 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedShard;
@@ -92,7 +93,7 @@ public static AllocationDeciders defaultAllocationDeciders(Settings settings, Cl
 
     public static DiscoveryNode newNode(String nodeId, Map<String, String> attributes) {
         return new DiscoveryNode("", nodeId, new TransportAddress(TransportAddress.META_ADDRESS,
-            portGenerator.incrementAndGet()), attributes, Sets.newHashSet(DiscoveryNode.Role.MASTER,
-            DiscoveryNode.Role.DATA), Version.CURRENT);
+            portGenerator.incrementAndGet()), attributes, Sets.newHashSet(DiscoveryNodeRole.MASTER_ROLE,
+            DiscoveryNodeRole.DATA_ROLE), Version.CURRENT);
     }
 }
diff --git a/build.gradle b/build.gradle
index 1de3f919d9c49..5c1fe80668283 100644
--- a/build.gradle
+++ b/build.gradle
@@ -104,10 +104,7 @@ subprojects {
 
 /* Introspect all versions of ES that may be tested against for backwards
  * compatibility. It is *super* important that this logic is the same as the
- * logic in VersionUtils.java, throwing out alphas because they don't have any
- * backwards compatibility guarantees and only keeping the latest beta or rc
- * in a branch if there are only betas and rcs in the branch so we have
- * *something* to test against. */
+ * logic in VersionUtils.java. */
 BwcVersions versions = new BwcVersions(file('server/src/main/java/org/elasticsearch/Version.java').readLines('UTF-8'))
 
 // build metadata from previous build, contains eg hashes for bwc builds
@@ -209,68 +206,7 @@ allprojects {
     javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet')
   }
 
-  /* Sets up the dependencies that we build as part of this project but
-    register as though they were external to resolve internally. We register
-    them as external dependencies so the build plugin that we use can be used
-    to build elasticsearch plugins outside of the elasticsearch source tree. */
-  ext.projectSubstitutions = [
-    "org.elasticsearch.gradle:build-tools:${version}": ':build-tools',
-    "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
-    "org.elasticsearch:elasticsearch:${version}": ':server',
-    "org.elasticsearch:elasticsearch-cli:${version}": ':libs:elasticsearch-cli',
-    "org.elasticsearch:elasticsearch-core:${version}": ':libs:core',
-    "org.elasticsearch:elasticsearch-nio:${version}": ':libs:nio',
-    "org.elasticsearch:elasticsearch-x-content:${version}": ':libs:x-content',
-    "org.elasticsearch:elasticsearch-geo:${version}": ':libs:elasticsearch-geo',
-    "org.elasticsearch:elasticsearch-secure-sm:${version}": ':libs:secure-sm',
-    "org.elasticsearch:elasticsearch-ssl-config:${version}": ':libs:elasticsearch-ssl-config',
-    "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
-    "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer',
-    "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',
-    "org.elasticsearch.client:test:${version}": ':client:test',
-    "org.elasticsearch.client:transport:${version}": ':client:transport',
-    "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
-    "org.elasticsearch.test:framework:${version}": ':test:framework',
-    "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
-    "org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware',
-    // for transport client
-    "org.elasticsearch.plugin:transport-netty4-client:${version}": ':modules:transport-netty4',
-    "org.elasticsearch.plugin:reindex-client:${version}": ':modules:reindex',
-    "org.elasticsearch.plugin:lang-mustache-client:${version}": ':modules:lang-mustache',
-    "org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
-    "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
-    "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
-    "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
-    // for security example plugins
-    "org.elasticsearch.plugin:x-pack-core:${version}": ':x-pack:plugin:core'
-  ]
-
-  /*
-   * Gradle only resolve project substitutions during dependency resolution but
-   * we sometimes want to do the resolution at other times. This creates a
-   * convenient method we can call to do it.
-   */
-  ext.dependencyToProject = { Dependency dep ->
-    if (dep instanceof ProjectDependency) {
-      return dep.dependencyProject
-    } else {
-      String substitution = projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
-      if (substitution != null) {
-        return findProject(substitution)
-      }
-      return null
-    }
-  }
-
   project.afterEvaluate {
-    configurations.matching { it.canBeResolved }.all {
-      resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->
-        projectSubstitutions.each { k,v ->
-          subs.substitute(subs.module(k)).with(subs.project(v))
-        }
-      }
-    }
-
     // Handle javadoc dependencies across projects. Order matters: the linksOffline for
     // org.elasticsearch:elasticsearch must be the last one or all the links for the
     // other packages (e.g org.elasticsearch.client) will point to server rather than
@@ -279,10 +215,10 @@ allprojects {
       String artifactsHost = VersionProperties.elasticsearch.endsWith("-SNAPSHOT") ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
       Closure sortClosure = { a, b -> b.group <=> a.group }
       Closure depJavadocClosure = { shadowed, dep ->
-        if (dep.group == null || false == dep.group.startsWith('org.elasticsearch')) {
+        if ((dep instanceof ProjectDependency) == false) {
           return
         }
-        Project upstreamProject = project.ext.dependencyToProject(dep)
+        Project upstreamProject = dep.dependencyProject
         if (upstreamProject == null) {
           return
         }
@@ -338,8 +274,8 @@ gradle.projectsEvaluated {
       integTest.mustRunAfter test
     }
     configurations.matching { it.canBeResolved }.all { Configuration configuration ->
-      dependencies.all { Dependency dep ->
-        Project upstreamProject = dependencyToProject(dep)
+      dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
+        Project upstreamProject = dep.dependencyProject
         if (upstreamProject != null) {
           if (project.path == upstreamProject.path) {
             // TODO: distribution integ tests depend on themselves (!), fix that
@@ -552,31 +488,6 @@ gradle.projectsEvaluated {
   }
 }
 
-if (System.properties.get("build.compare") != null) {
-  apply plugin: 'compare-gradle-builds'
-  compareGradleBuilds {
-    ext.referenceProject = System.properties.get("build.compare")
-    doFirst {
-      if (file(referenceProject).exists() == false) {
-        throw new GradleException(
-                "Use git worktree to check out a version to compare against to ../elasticsearch_build_reference"
-        )
-      }
-    }
-    sourceBuild {
-      gradleVersion = gradle.getGradleVersion()
-      projectDir = referenceProject
-      tasks = ["clean", "assemble"]
-      arguments = ["-Dbuild.compare_friendly=true"]
-    }
-    targetBuild {
-      tasks = ["clean", "assemble"]
-      // use -Dorg.gradle.java.home= to alter jdk versions
-      arguments = ["-Dbuild.compare_friendly=true"]
-    }
-  }
-}
-
 allprojects {
   task resolveAllDependencies {
       dependsOn tasks.matching { it.name == "pullFixture"}
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index d3a16f55277d7..a3ca1c7ffa348 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -67,8 +67,26 @@ if (JavaVersion.current() < JavaVersion.VERSION_11) {
 }
 
 // Keep compatibility with Java 8 for external users of build-tools that haven't migrated to Java 11
-targetCompatibility = '8'
-sourceCompatibility = '8'
+targetCompatibility = '11'
+sourceCompatibility = '11'
+
+sourceSets {
+    // We have a few classes that need to be compiled for older java versions
+    minimumRuntime { }
+}
+
+compileMinimumRuntimeJava {
+    targetCompatibility = 8
+    sourceCompatibility = 8
+}
+
+jar {
+    from sourceSets.minimumRuntime.output
+}
+
+javadoc {
+    source sourceSets.minimumRuntime.allSource
+}
 
 /*****************************************************************************
  *                    Dependencies used by the entire build                  *
@@ -79,8 +97,15 @@ repositories {
 }
 
 dependencies {
+  if (project.ext.has("isEclipse") == false || project.ext.isEclipse == false) {
+    // eclipse is confused if this is set explicitly
+    compile sourceSets.minimumRuntime.output
+  }
+
   compile localGroovy()
   
+  compile 'commons-codec:commons-codec:1.12'
+
   compile 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
   compile 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'
   compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
@@ -94,6 +119,9 @@ dependencies {
   testCompile "junit:junit:${props.getProperty('junit')}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
   testCompile 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
+  minimumRuntimeCompile "junit:junit:${props.getProperty('junit')}"
+  minimumRuntimeCompile localGroovy()
+  minimumRuntimeCompile gradleApi()
 }
 
 /*****************************************************************************
@@ -121,10 +149,6 @@ if (project != rootProject) {
   apply plugin: 'nebula.maven-base-publish'
   apply plugin: 'nebula.maven-scm'
 
-  // we need to apply these again to override the build plugin
-  targetCompatibility = "10"
-  sourceCompatibility = "10"
-
   // groovydoc succeeds, but has some weird internal exception...
   groovydoc.enabled = false
 
@@ -132,6 +156,7 @@ if (project != rootProject) {
   dependencyLicenses.enabled = false
   dependenciesInfo.enabled = false
   forbiddenApisMain.enabled = false
+  forbiddenApisMinimumRuntime.enabled = false
   forbiddenApisTest.enabled = false
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
@@ -148,6 +173,11 @@ if (project != rootProject) {
     distribution project(':distribution:archives:linux-tar')
     distribution project(':distribution:archives:oss-linux-tar')
   }
+    
+  // for external projects we want to remove the marker file indicating we are running the Elasticsearch project
+  processResources {
+      exclude 'buildSrc.marker'
+  }
 
   String localDownloads = "${rootProject.buildDir}/local-downloads"
   task setupLocalDownloads(type:Copy) {
@@ -178,6 +208,8 @@ if (project != rootProject) {
       systemProperty 'test.lucene-snapshot-revision', isLuceneSnapshot[0][1]
     }
     maxParallelForks System.getProperty('tests.jvms', project.rootProject.ext.defaultParallel.toString()) as Integer
+    // These tests run Gradle which doesn't have FIPS support
+    onlyIf { project.inFipsJvm == false }
   }
   check.dependsOn(integTest)
 
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index b5c69a418cceb..72af6b8c330f8 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.gradle
 
+import com.github.jengelman.gradle.plugins.shadow.ShadowExtension
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
 import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 import groovy.transform.CompileDynamic
@@ -32,6 +33,7 @@ import org.elasticsearch.gradle.precommit.DependencyLicensesTask
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.elasticsearch.gradle.test.ErrorReportingTestListener
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
+import org.elasticsearch.gradle.testclusters.TestClustersPlugin
 import org.gradle.api.Action
 import org.gradle.api.GradleException
 import org.gradle.api.InvalidUserDataException
@@ -77,7 +79,6 @@ import org.gradle.authentication.http.HttpHeaderAuthentication
 import org.gradle.external.javadoc.CoreJavadocOptions
 import org.gradle.internal.jvm.Jvm
 import org.gradle.language.base.plugins.LifecycleBasePlugin
-import org.gradle.process.CommandLineArgumentProvider
 import org.gradle.process.ExecResult
 import org.gradle.process.ExecSpec
 import org.gradle.util.GradleVersion
@@ -99,13 +100,17 @@ class BuildPlugin implements Plugin<Project> {
         project.rootProject.pluginManager.apply(GlobalBuildInfoPlugin)
 
         if (project.pluginManager.hasPlugin('elasticsearch.standalone-rest-test')) {
-              throw new InvalidUserDataException('elasticsearch.standalone-test, '
-                + 'elasticsearch.standalone-rest-test, and elasticsearch.build '
-                + 'are mutually exclusive')
+            throw new InvalidUserDataException('elasticsearch.standalone-test, '
+                    + 'elasticsearch.standalone-rest-test, and elasticsearch.build '
+                    + 'are mutually exclusive')
         }
         String minimumGradleVersion = null
         InputStream is = getClass().getResourceAsStream("/minimumGradleVersion")
-        try { minimumGradleVersion = IOUtils.toString(is, StandardCharsets.UTF_8.toString()) } finally { is.close() }
+        try {
+            minimumGradleVersion = IOUtils.toString(is, StandardCharsets.UTF_8.toString())
+        } finally {
+            is.close()
+        }
         if (GradleVersion.current() < GradleVersion.version(minimumGradleVersion.trim())) {
             throw new GradleException(
                     "Gradle ${minimumGradleVersion}+ is required to use elasticsearch.build plugin"
@@ -139,23 +144,25 @@ class BuildPlugin implements Plugin<Project> {
         configurePrecommit(project)
         configureDependenciesInfo(project)
 
-        // Common config when running with a FIPS-140 runtime JVM
+
+        configureFips140(project)
+    }
+
+    public static void configureFips140(Project project) {
         // Need to do it here to support external plugins
-        if (project == project.rootProject) {
-            GlobalInfoExtension globalInfo = project.extensions.getByType(GlobalInfoExtension)
-
-            // wait until global info is populated because we don't know if we are running in a fips jvm until execution time
-            globalInfo.ready {
-                project.subprojects { Project subproject ->
-                    ExtraPropertiesExtension ext = subproject.extensions.getByType(ExtraPropertiesExtension)
-                    // Common config when running with a FIPS-140 runtime JVM
-                    if (ext.has('inFipsJvm') && ext.get('inFipsJvm')) {
-                        subproject.tasks.withType(Test) { Test task ->
-                            task.systemProperty 'javax.net.ssl.trustStorePassword', 'password'
-                            task.systemProperty 'javax.net.ssl.keyStorePassword', 'password'
-                        }
-                        project.pluginManager.withPlugin("elasticsearch.testclusters") {
-                            NamedDomainObjectContainer<ElasticsearchCluster> testClusters = subproject.extensions.getByName('testClusters') as NamedDomainObjectContainer<ElasticsearchCluster>
+        GlobalInfoExtension globalInfo = project.rootProject.extensions.getByType(GlobalInfoExtension)
+        // wait until global info is populated because we don't know if we are running in a fips jvm until execution time
+        globalInfo.ready {
+                ExtraPropertiesExtension ext = project.extensions.getByType(ExtraPropertiesExtension)
+                // Common config when running with a FIPS-140 runtime JVM
+                if (ext.has('inFipsJvm') && ext.get('inFipsJvm')) {
+                    project.tasks.withType(Test) { Test task ->
+                        task.systemProperty 'javax.net.ssl.trustStorePassword', 'password'
+                        task.systemProperty 'javax.net.ssl.keyStorePassword', 'password'
+                    }
+                    project.pluginManager.withPlugin("elasticsearch.testclusters") {
+                        NamedDomainObjectContainer<ElasticsearchCluster> testClusters = project.extensions.findByName(TestClustersPlugin.EXTENSION_NAME) as NamedDomainObjectContainer<ElasticsearchCluster>
+                        if (testClusters != null) {
                             testClusters.all { ElasticsearchCluster cluster ->
                                 cluster.systemProperty 'javax.net.ssl.trustStorePassword', 'password'
                                 cluster.systemProperty 'javax.net.ssl.keyStorePassword', 'password'
@@ -163,7 +170,6 @@ class BuildPlugin implements Plugin<Project> {
                         }
                     }
                 }
-            }
         }
     }
 
@@ -256,15 +262,11 @@ class BuildPlugin implements Plugin<Project> {
             }
         }
 
-        if (ext.get('buildDocker')) {
-            (ext.get('requiresDocker') as List<Task>).add(task)
-        } else {
-            task.enabled = false
-        }
+        (ext.get('requiresDocker') as List<Task>).add(task)
     }
 
     protected static void checkDockerVersionRecent(String dockerVersion) {
-        final Matcher matcher = dockerVersion =~ /Docker version (\d+\.\d+)\.\d+(?:-ce)?, build [0-9a-f]{7,40}/
+        final Matcher matcher = dockerVersion =~ /Docker version (\d+\.\d+)\.\d+(?:-[a-zA-Z0-9]+)?, build [0-9a-f]{7,40}/
         assert matcher.matches(): dockerVersion
         final dockerMajorMinorVersion = matcher.group(1)
         final String[] majorMinor = dockerMajorMinorVersion.split("\\.")
@@ -530,39 +532,43 @@ class BuildPlugin implements Plugin<Project> {
     static void configurePomGeneration(Project project) {
         // Only works with  `enableFeaturePreview('STABLE_PUBLISHING')`
         // https://github.com/gradle/gradle/issues/5696#issuecomment-396965185
-        project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom generatePOMTask ->
-            // The GenerateMavenPom task is aggressive about setting the destination, instead of fighting it,
-            // just make a copy.
-            ExtraPropertiesExtension ext = generatePOMTask.extensions.getByType(ExtraPropertiesExtension)
-            ext.set('pomFileName', null)
-            generatePOMTask.doLast {
-                project.copy { CopySpec spec ->
-                    spec.from generatePOMTask.destination
-                    spec.into "${project.buildDir}/distributions"
-                    spec.rename {
-                        ext.has('pomFileName') && ext.get('pomFileName') == null ?
-                            "${project.convention.getPlugin(BasePluginConvention).archivesBaseName}-${project.version}.pom" :
-                            ext.get('pomFileName')
+        // dummy task to depend on the real pom generation
+        project.plugins.withType(MavenPublishPlugin).whenPluginAdded {
+            Task generatePomTask = project.tasks.create("generatePom")
+            Task assemble = project.tasks.findByName('assemble')
+            if (assemble) {
+                assemble.dependsOn(generatePomTask)
+            }
+            project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom pomTask ->
+                // The GenerateMavenPom task is aggressive about setting the destination, instead of fighting it,
+                // just make a copy.
+                ExtraPropertiesExtension ext = pomTask.extensions.getByType(ExtraPropertiesExtension)
+                ext.set('pomFileName', null)
+                pomTask.doLast {
+                    project.copy { CopySpec spec ->
+                        spec.from pomTask.destination
+                        spec.into "${project.buildDir}/distributions"
+                        spec.rename {
+                            ext.has('pomFileName') && ext.get('pomFileName') == null ?
+                                    "${project.convention.getPlugin(BasePluginConvention).archivesBaseName}-${project.version}.pom" :
+                                    ext.get('pomFileName')
+                        }
                     }
                 }
             }
-            // build poms with assemble (if the assemble task exists)
-            Task assemble = project.tasks.findByName('assemble')
-            if (assemble && assemble.enabled) {
-                assemble.dependsOn(generatePOMTask)
-            }
-        }
-        project.plugins.withType(MavenPublishPlugin).whenPluginAdded {
+            generatePomTask.dependsOn = ['generatePomFileForNebulaPublication']
             PublishingExtension publishing = project.extensions.getByType(PublishingExtension)
             publishing.publications.all { MavenPublication publication -> // we only deal with maven
                 // add exclusions to the pom directly, for each of the transitive deps of this project's deps
                 publication.pom.withXml(fixupDependencies(project))
             }
             project.plugins.withType(ShadowPlugin).whenPluginAdded {
-                MavenPublication publication = publishing.publications.maybeCreate('nebula', MavenPublication)
+                MavenPublication publication = publishing.publications.maybeCreate('shadow', MavenPublication)
                 publication.with {
-                    artifacts = [ project.tasks.getByName('shadowJar') ]
+                    ShadowExtension shadow = project.extensions.getByType(ShadowExtension)
+                    shadow.component(publication)
                 }
+                generatePomTask.dependsOn = ['generatePomFileForShadowPublication']
             }
         }
     }
@@ -690,6 +696,15 @@ class BuildPlugin implements Plugin<Project> {
         project.tasks.withType(Jar) { Jar jarTask ->
             // we put all our distributable files under distributions
             jarTask.destinationDir = new File(project.buildDir, 'distributions')
+            project.plugins.withType(ShadowPlugin).whenPluginAdded {
+                /*
+                 * Ensure the original jar task places its output in 'libs' so that we don't overwrite it with the shadow jar. We only do
+                 * this for tasks named jar to exclude javadoc and sources jars.
+                 */
+                if (jarTask instanceof ShadowJar == false && jarTask.name == JavaPlugin.JAR_TASK_NAME) {
+                    jarTask.destinationDir = new File(project.buildDir, 'libs')
+                }
+            }
             // fixup the jar manifest
             jarTask.doFirst {
                 // this doFirst is added before the info plugin, therefore it will run
@@ -716,10 +731,6 @@ class BuildPlugin implements Plugin<Project> {
                         jarTask.manifest.attributes('Change': shortHash)
                     }
                 }
-                // Force manifest entries that change by nature to a constant to be able to compare builds more effectively
-                if (System.properties.getProperty("build.compare_friendly", "false") == "true") {
-                    jarTask.manifest.getAttributes().clear()
-                }
             }
 
             // add license/notice files
@@ -744,12 +755,6 @@ class BuildPlugin implements Plugin<Project> {
             }
         }
         project.plugins.withType(ShadowPlugin).whenPluginAdded {
-            /*
-             * When we use the shadow plugin we entirely replace the
-             * normal jar with the shadow jar so we no longer want to run
-             * the jar task.
-             */
-            project.tasks.getByName(JavaPlugin.JAR_TASK_NAME).enabled = false
             project.tasks.getByName('shadowJar').configure { ShadowJar shadowJar ->
                 /*
                  * Replace the default "shadow" classifier with null
@@ -768,7 +773,6 @@ class BuildPlugin implements Plugin<Project> {
             }
             // Make sure we assemble the shadow jar
             project.tasks.getByName(BasePlugin.ASSEMBLE_TASK_NAME).dependsOn project.tasks.getByName('shadowJar')
-            project.artifacts.add('apiElements', project.tasks.getByName('shadowJar'))
         }
     }
 
@@ -817,7 +821,7 @@ class BuildPlugin implements Plugin<Project> {
                 }
 
                 test.jvmArgumentProviders.add(nonInputProperties)
-                test.extensions.getByType(ExtraPropertiesExtension).set('nonInputProperties', nonInputProperties)
+                test.extensions.add('nonInputProperties', nonInputProperties)
 
                 test.executable = "${ext.get('runtimeJavaHome')}/bin/java"
                 test.workingDir = project.file("${project.buildDir}/testrun/${test.name}")
@@ -842,17 +846,25 @@ class BuildPlugin implements Plugin<Project> {
 
                 // we use './temp' since this is per JVM and tests are forbidden from writing to CWD
                 test.systemProperties 'gradle.dist.lib': new File(project.class.location.toURI()).parent,
-                        'gradle.worker.jar': "${project.gradle.getGradleUserHomeDir()}/caches/${project.gradle.gradleVersion}/workerMain/gradle-worker.jar",
-                        'gradle.user.home': project.gradle.getGradleUserHomeDir(),
                         'java.io.tmpdir': './temp',
                         'java.awt.headless': 'true',
                         'tests.gradle': 'true',
                         'tests.artifact': project.name,
                         'tests.task': test.path,
                         'tests.security.manager': 'true',
-                        'tests.seed': project.property('testSeed'),
                         'jna.nosys': 'true'
 
+                // ignore changing test seed when build is passed -Dignore.tests.seed for cacheability experimentation
+                if (System.getProperty('ignore.tests.seed') != null) {
+                    nonInputProperties.systemProperty('tests.seed', project.property('testSeed'))
+                } else {
+                    test.systemProperty('tests.seed', project.property('testSeed'))
+                }
+
+                // don't track these as inputs since they contain absolute paths and break cache relocatability
+                nonInputProperties.systemProperty('gradle.worker.jar', "${project.gradle.getGradleUserHomeDir()}/caches/${project.gradle.gradleVersion}/workerMain/gradle-worker.jar")
+                nonInputProperties.systemProperty('gradle.user.home', project.gradle.getGradleUserHomeDir())
+
                 nonInputProperties.systemProperty('compiler.java', "${-> (ext.get('compilerJavaVersion') as JavaVersion).getMajorVersion()}")
 
                 // TODO: remove setting logging level via system property
@@ -874,6 +886,7 @@ class BuildPlugin implements Plugin<Project> {
 
                 project.plugins.withType(ShadowPlugin).whenPluginAdded {
                     // Test against a shadow jar if we made one
+                    test.classpath -= project.configurations.getByName('bundle')
                     test.classpath -= project.tasks.getByName('compileJava').outputs.files
                     test.classpath += project.tasks.getByName('shadowJar').outputs.files
 
@@ -965,19 +978,4 @@ class BuildPlugin implements Plugin<Project> {
             })
         }
     }
-
-    private static class SystemPropertyCommandLineArgumentProvider implements CommandLineArgumentProvider {
-        private final Map<String, Object> systemProperties = [:]
-
-        void systemProperty(String key, Object value) {
-            systemProperties.put(key, value)
-        }
-
-        @Override
-        Iterable<String> asArguments() {
-            return systemProperties.collect { key, value ->
-                "-D${key}=${value.toString()}".toString()
-            }
-        }
-    }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
index 83a6a05ec5df7..c1dbddd9e9d52 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
@@ -43,7 +43,7 @@ public class SnippetsTask extends DefaultTask {
     private static final String SKIP = /skip:([^\]]+)/
     private static final String SETUP = /setup:([^ \]]+)/
     private static final String WARNING = /warning:(.+)/
-    private static final String CAT = /(_cat)/
+    private static final String NON_JSON = /(non_json)/
     private static final String TEST_SYNTAX =
         /(?:$CATCH|$SUBSTITUTION|$SKIP|(continued)|$SETUP|$WARNING|(skip_shard_failures)) ?/
 
@@ -255,12 +255,12 @@ public class SnippetsTask extends DefaultTask {
                             substitutions = []
                         }
                         String loc = "$file:$lineNumber"
-                        parse(loc, matcher.group(2), /(?:$SUBSTITUTION|$CAT|$SKIP) ?/) {
+                        parse(loc, matcher.group(2), /(?:$SUBSTITUTION|$NON_JSON|$SKIP) ?/) {
                             if (it.group(1) != null) {
                                 // TESTRESPONSE[s/adsf/jkl/]
                                 substitutions.add([it.group(1), it.group(2)])
                             } else if (it.group(3) != null) {
-                                // TESTRESPONSE[_cat]
+                                // TESTRESPONSE[non_json]
                                 substitutions.add(['^', '/'])
                                 substitutions.add(['\n$', '\\\\s*/'])
                                 substitutions.add(['( +)', '$1\\\\s+'])
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
index e04d0966c412d..a25c0415f6483 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
@@ -19,7 +19,6 @@
 package org.elasticsearch.gradle.plugin
 
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
-import nebula.plugin.publishing.maven.MavenScmPlugin
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.NoticeTask
 import org.elasticsearch.gradle.Version
@@ -27,17 +26,16 @@ import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.test.RunTask
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
+import org.elasticsearch.gradle.tool.ClasspathUtils
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.publish.maven.MavenPublication
 import org.gradle.api.publish.maven.plugins.MavenPublishPlugin
-import org.gradle.api.publish.maven.tasks.GenerateMavenPom
 import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.SourceSet
 import org.gradle.api.tasks.bundling.Zip
-import org.gradle.jvm.tasks.Jar
 
 import java.util.regex.Matcher
 import java.util.regex.Pattern
@@ -131,31 +129,21 @@ class PluginBuildPlugin implements Plugin<Project> {
     }
 
     private void configurePublishing(Project project, PluginPropertiesExtension extension) {
-        // Only configure publishing if applied externally
-        if (extension.hasClientJar) {
-            project.plugins.apply(MavenScmPlugin.class)
-            // Only change Jar tasks, we don't want a -client zip so we can't change archivesBaseName
-            project.tasks.withType(Jar) {
-                baseName = baseName + "-client"
-            }
-            // always configure publishing for client jars
-            project.plugins.apply(MavenScmPlugin.class)
-            project.publishing.publications.nebula(MavenPublication).artifactId(extension.name + "-client")
-            project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom generatePOMTask ->
-                generatePOMTask.ext.pomFileName = "${project.archivesBaseName}-client-${project.versions.elasticsearch}.pom"
-            }
-        } else {
-            if (project.plugins.hasPlugin(MavenPublishPlugin)) {
-                project.publishing.publications.nebula(MavenPublication).artifactId(extension.name)
-            }
-
+        if (project.plugins.hasPlugin(MavenPublishPlugin)) {
+            project.publishing.publications.nebula(MavenPublication).artifactId(extension.name)
         }
+
     }
 
     private static void configureDependencies(Project project) {
         project.dependencies {
-            compileOnly "org.elasticsearch:elasticsearch:${project.versions.elasticsearch}"
-            testCompile "org.elasticsearch.test:framework:${project.versions.elasticsearch}"
+            if (ClasspathUtils.isElasticsearchProject()) {
+                compileOnly project.project(':server')
+                testCompile project.project(':test:framework')
+            } else {
+                compileOnly "org.elasticsearch:elasticsearch:${project.versions.elasticsearch}"
+                testCompile "org.elasticsearch.test:framework:${project.versions.elasticsearch}"
+            }
             // we "upgrade" these optional deps to provided for plugins, since they will run
             // with a full elasticsearch server that includes optional deps
             compileOnly "org.locationtech.spatial4j:spatial4j:${project.versions.spatial4j}"
@@ -260,8 +248,6 @@ class PluginBuildPlugin implements Plugin<Project> {
         project.artifacts.add('zip', bundle)
     }
 
-    /** Adds a task to move jar and associated files to a "-client" name. */
-
     static final Pattern GIT_PATTERN = Pattern.compile(/git@([^:]+):([^\.]+)\.git/)
 
     /** Find the reponame. */
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy
deleted file mode 100644
index c250d7695a832..0000000000000
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.gradle.plugin
-
-import org.gradle.api.Project
-import org.gradle.api.tasks.Input
-import org.gradle.api.tasks.InputFile
-
-/**
- * A container for plugin properties that will be written to the plugin descriptor, for easy
- * manipulation in the gradle DSL.
- */
-class PluginPropertiesExtension {
-
-    @Input
-    String name
-
-    @Input
-    String version
-
-    @Input
-    String description
-
-    @Input
-    String classname
-
-    /** Other plugins this plugin extends through SPI */
-    @Input
-    List<String> extendedPlugins = []
-
-    @Input
-    boolean hasNativeController = false
-
-    /** Indicates whether the plugin jar should be made available for the transport client. */
-    @Input
-    boolean hasClientJar = false
-
-    /** True if the plugin requires the elasticsearch keystore to exist, false otherwise. */
-    @Input
-    boolean requiresKeystore = false
-
-    /** A license file that should be included in the built plugin zip. */
-    private File licenseFile = null
-
-    /**
-     * A notice file that should be included in the built plugin zip. This will be
-     * extended with notices from the {@code licenses/} directory.
-     */
-    private File noticeFile = null
-
-    Project project = null
-
-    PluginPropertiesExtension(Project project) {
-        name = project.name
-        version = project.version
-        this.project = project
-    }
-
-    @InputFile
-    File getLicenseFile() {
-        return licenseFile
-    }
-
-    void setLicenseFile(File licenseFile) {
-        project.ext.licenseFile = licenseFile
-        this.licenseFile = licenseFile
-    }
-
-    @InputFile
-    File getNoticeFile() {
-        return noticeFile
-    }
-
-    void setNoticeFile(File noticeFile) {
-        project.ext.noticeFile = noticeFile
-        this.noticeFile = noticeFile
-    }
-}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy
deleted file mode 100644
index 04fb023e2051a..0000000000000
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.gradle.precommit
-
-import org.gradle.api.DefaultTask
-import org.gradle.api.GradleException
-import org.gradle.api.InvalidUserDataException
-import org.gradle.api.file.FileCollection
-import org.gradle.api.tasks.Input
-import org.gradle.api.tasks.InputDirectory
-import org.gradle.api.tasks.InputFiles
-import org.gradle.api.tasks.TaskAction
-
-import java.nio.file.Files
-import java.security.MessageDigest
-import java.util.regex.Matcher
-import java.util.regex.Pattern
-
-/**
- * A task to check licenses for dependencies.
- *
- * There are two parts to the check:
- * <ul>
- *   <li>LICENSE and NOTICE files</li>
- *   <li>SHA checksums for each dependency jar</li>
- * </ul>
- *
- * The directory to find the license and sha files in defaults to the dir @{code licenses}
- * in the project directory for this task. You can override this directory:
- * <pre>
- *   dependencyLicenses {
- *     licensesDir = project.file('mybetterlicensedir')
- *   }
- * </pre>
- *
- * The jar files to check default to the dependencies from the default configuration. You
- * can override this, for example, to only check compile dependencies:
- * <pre>
- *   dependencyLicenses {
- *     dependencies = project.configurations.compile
- *   }
- * </pre>
- *
- * Every jar must have a {@code .sha1} file in the licenses dir. These can be managed
- * automatically using the {@code updateShas} helper task that is created along
- * with this task. It will add {@code .sha1} files for new jars that are in dependencies
- * and remove old {@code .sha1} files that are no longer needed.
- *
- * Every jar must also have a LICENSE and NOTICE file. However, multiple jars can share
- * LICENSE and NOTICE files by mapping a pattern to the same name.
- * <pre>
- *   dependencyLicenses {
- *     mapping from: &#47;lucene-.*&#47;, to: 'lucene'
- *   }
- * </pre>
- */
-public class DependencyLicensesTask extends DefaultTask {
-    static final String SHA_EXTENSION = '.sha1'
-
-    // TODO: we should be able to default this to eg compile deps, but we need to move the licenses
-    // check from distribution to core (ie this should only be run on java projects)
-    /** A collection of jar files that should be checked. */
-    @InputFiles
-    public FileCollection dependencies
-
-    /** The directory to find the license and sha files in. */
-    @InputDirectory
-    public File licensesDir = new File(project.projectDir, 'licenses')
-
-    /** A map of patterns to prefix, used to find the LICENSE and NOTICE file. */
-    private LinkedHashMap<String, String> mappings = new LinkedHashMap<>()
-
-    /** Names of dependencies whose shas should not exist. */
-    private Set<String> ignoreShas = new HashSet<>()
-
-    /**
-     * Add a mapping from a regex pattern for the jar name, to a prefix to find
-     * the LICENSE and NOTICE file for that jar.
-     */
-    @Input
-    public void mapping(Map<String, String> props) {
-        String from = props.remove('from')
-        if (from == null) {
-            throw new InvalidUserDataException('Missing "from" setting for license name mapping')
-        }
-        String to = props.remove('to')
-        if (to == null) {
-            throw new InvalidUserDataException('Missing "to" setting for license name mapping')
-        }
-        if (props.isEmpty() == false) {
-            throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: ${props.keySet()}")
-        }
-        mappings.put(from, to)
-    }
-
-    public LinkedHashMap<String, String> getMappings() {
-        return new LinkedHashMap<>(mappings)
-    }
-
-    /**
-     * Add a rule which will skip SHA checking for the given dependency name. This should be used for
-     * locally build dependencies, which cause the sha to change constantly.
-     */
-    @Input
-    public void ignoreSha(String dep) {
-        ignoreShas.add(dep)
-    }
-
-    @TaskAction
-    public void checkDependencies() {
-        if (dependencies.isEmpty()) {
-            if (licensesDir.exists()) {
-                throw new GradleException("Licenses dir ${licensesDir} exists, but there are no dependencies")
-            }
-            return // no dependencies to check
-        } else if (licensesDir.exists() == false) {
-            throw new GradleException("Licences dir ${licensesDir} does not exist, but there are dependencies")
-        }
-
-        Map<String, Integer> licenses = new HashMap<>()
-        Map<String, Integer> notices = new HashMap<>()
-        Set<File> shaFiles = new HashSet<File>()
-
-        licensesDir.eachFile {
-            String name = it.getName()
-            if (name.endsWith(SHA_EXTENSION)) {
-                shaFiles.add(it)
-            } else if (name.endsWith('-LICENSE') || name.endsWith('-LICENSE.txt')) {
-                // TODO: why do we support suffix of LICENSE *and* LICENSE.txt??
-                licenses.put(name, 0)
-            } else if (name.contains('-NOTICE') || name.contains('-NOTICE.txt')) {
-                notices.put(name, 0)
-            }
-        }
-
-        for (File dependency : dependencies) {
-            String jarName = dependency.getName()
-            String depName = jarName - ~/\-v?\d+.*/
-            if (ignoreShas.contains(depName)) {
-                // local deps should not have sha files!
-                if (getShaFile(jarName).exists()) {
-                    throw new GradleException("SHA file ${getShaFile(jarName)} exists for ignored dependency ${depName}")
-                }
-            } else {
-                logger.info("Checking sha for " + jarName)
-                checkSha(dependency, jarName, shaFiles)
-            }
-
-            final String dependencyName = getDependencyName(mappings, depName)
-            logger.info("mapped dependency name ${depName} to ${dependencyName} for license/notice check")
-            checkFile(dependencyName, jarName, licenses, 'LICENSE')
-            checkFile(dependencyName, jarName, notices, 'NOTICE')
-        }
-
-        licenses.each { license, count ->
-            if (count == 0) {
-                throw new GradleException("Unused license ${license}")
-            }
-        }
-        notices.each { notice, count ->
-            if (count == 0) {
-                throw new GradleException("Unused notice ${notice}")
-            }
-        }
-        if (shaFiles.isEmpty() == false) {
-            throw new GradleException("Unused sha files found: \n${shaFiles.join('\n')}")
-        }
-    }
-
-    public static String getDependencyName(final LinkedHashMap<String, String> mappings, final String dependencyName) {
-        // order is the same for keys and values iteration since we use a linked hashmap
-        List<String> mapped = new ArrayList<>(mappings.values())
-        Pattern mappingsPattern = Pattern.compile('(' + mappings.keySet().join(')|(') + ')')
-        Matcher match = mappingsPattern.matcher(dependencyName)
-        if (match.matches()) {
-            int i = 0
-            while (i < match.groupCount() && match.group(i + 1) == null) ++i;
-            return mapped.get(i)
-        }
-        return dependencyName
-    }
-
-    private File getShaFile(String jarName) {
-        return new File(licensesDir, jarName + SHA_EXTENSION)
-    }
-
-    private void checkSha(File jar, String jarName, Set<File> shaFiles) {
-        File shaFile = getShaFile(jarName)
-        if (shaFile.exists() == false) {
-            throw new GradleException("Missing SHA for ${jarName}. Run 'gradle updateSHAs' to create")
-        }
-        // TODO: shouldn't have to trim, sha files should not have trailing newline
-        String expectedSha = shaFile.getText('UTF-8').trim()
-        String sha = MessageDigest.getInstance("SHA-1").digest(jar.getBytes()).encodeHex().toString()
-        if (expectedSha.equals(sha) == false) {
-            throw new GradleException("SHA has changed! Expected ${expectedSha} for ${jarName} but got ${sha}. " +
-                                      "\nThis usually indicates a corrupt dependency cache or artifacts changed upstream." +
-                                      "\nEither wipe your cache, fix the upstream artifact, or delete ${shaFile} and run updateShas")
-        }
-        shaFiles.remove(shaFile)
-    }
-
-    private void checkFile(String name, String jarName, Map<String, Integer> counters, String type) {
-        String fileName = "${name}-${type}"
-        Integer count = counters.get(fileName)
-        if (count == null) {
-            // try the other suffix...TODO: get rid of this, just support ending in .txt
-            fileName = "${fileName}.txt"
-            counters.get(fileName)
-        }
-        count = counters.get(fileName)
-        if (count == null) {
-            throw new GradleException("Missing ${type} for ${jarName}, expected in ${fileName}")
-        }
-        counters.put(fileName, count + 1)
-    }
-
-    /** A helper task to update the sha files in the license dir. */
-    public static class UpdateShasTask extends DefaultTask {
-        private DependencyLicensesTask parentTask
-
-        @TaskAction
-        public void updateShas() {
-            Set<File> shaFiles = new HashSet<File>()
-            parentTask.licensesDir.eachFile {
-                String name = it.getName()
-                if (name.endsWith(SHA_EXTENSION)) {
-                    shaFiles.add(it)
-                }
-            }
-            for (File dependency : parentTask.dependencies) {
-                String jarName = dependency.getName()
-                String depName = jarName - ~/\-\d+.*/
-                if (parentTask.ignoreShas.contains(depName)) {
-                    continue
-                }
-                File shaFile = new File(parentTask.licensesDir, jarName + SHA_EXTENSION)
-                if (shaFile.exists() == false) {
-                    logger.lifecycle("Adding sha for ${jarName}")
-                    String sha = MessageDigest.getInstance("SHA-1").digest(dependency.getBytes()).encodeHex().toString()
-                    shaFile.setText(sha, 'UTF-8')
-                } else {
-                    shaFiles.remove(shaFile)
-                }
-            }
-            shaFiles.each { shaFile ->
-                logger.lifecycle("Removing unused sha ${shaFile.getName()}")
-                Files.delete(shaFile.toPath())
-            }
-        }
-    }
-}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
index f656f177ce67b..a5d4f3fcd9495 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
@@ -23,11 +23,13 @@ import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
 import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
 import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.tool.ClasspathUtils
 import org.gradle.api.JavaVersion
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.plugins.quality.Checkstyle
+
 /**
  * Validation tasks which should be run before committing. These run before tests.
  */
@@ -40,18 +42,18 @@ class PrecommitTasks {
     public static Task create(Project project, boolean includeDependencyLicenses) {
         project.configurations.create("forbiddenApisCliJar")
         project.dependencies {
-            forbiddenApisCliJar ('de.thetaphi:forbiddenapis:2.6')
+            forbiddenApisCliJar('de.thetaphi:forbiddenapis:2.6')
         }
 
         List<Task> precommitTasks = [
-            configureCheckstyle(project),
-            configureForbiddenApisCli(project),
-            project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class),
-            project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
-            project.tasks.create('filepermissions', FilePermissionsTask.class),
-            configureJarHell(project),
-            configureThirdPartyAudit(project),
-            configureTestingConventions(project)
+                configureCheckstyle(project),
+                configureForbiddenApisCli(project),
+                project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class),
+                project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
+                project.tasks.create('filepermissions', FilePermissionsTask.class),
+                configureJarHell(project),
+                configureThirdPartyAudit(project),
+                configureTestingConventions(project)
         ]
 
         // tasks with just tests don't need dependency licenses, so this flag makes adding
@@ -85,10 +87,10 @@ class PrecommitTasks {
         }
 
         return project.tasks.create([
-            name: 'precommit',
-            group: JavaBasePlugin.VERIFICATION_GROUP,
-            description: 'Runs all non-test checks.',
-            dependsOn: precommitTasks
+                name       : 'precommit',
+                group      : JavaBasePlugin.VERIFICATION_GROUP,
+                description: 'Runs all non-test checks.',
+                dependsOn  : precommitTasks
         ])
     }
 
@@ -168,7 +170,7 @@ class PrecommitTasks {
                 )
             }
         }
-        Task forbiddenApis =  project.tasks.getByName("forbiddenApis")
+        Task forbiddenApis = project.tasks.getByName("forbiddenApis")
         forbiddenApis.group = ""
         return forbiddenApis
     }
@@ -211,7 +213,7 @@ class PrecommitTasks {
         project.checkstyle {
             config = project.resources.text.fromFile(checkstyleConf, 'UTF-8')
             configProperties = [
-                suppressions: checkstyleSuppressions
+                    suppressions: checkstyleSuppressions
             ]
             toolVersion = CHECKSTYLE_VERSION
         }
@@ -229,9 +231,11 @@ class PrecommitTasks {
     }
 
     private static Task configureLoggerUsage(Project project) {
+        Object dependency = ClasspathUtils.isElasticsearchProject() ? project.project(':test:logger-usage') :
+                "org.elasticsearch.test:logger-usage:${VersionProperties.elasticsearch}"
+
         project.configurations.create('loggerUsagePlugin')
-        project.dependencies.add('loggerUsagePlugin',
-                "org.elasticsearch.test:logger-usage:${VersionProperties.elasticsearch}")
+        project.dependencies.add('loggerUsagePlugin', dependency)
         return project.tasks.create('loggerUsageCheck', LoggerUsageTask.class) {
             classpath = project.configurations.loggerUsagePlugin
         }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/UpdateShasTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/UpdateShasTask.groovy
deleted file mode 100644
index 4a174688aa1e5..0000000000000
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/UpdateShasTask.groovy
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.gradle.precommit
-
-import org.gradle.api.DefaultTask
-import org.gradle.api.tasks.TaskAction
-
-import java.nio.file.Files
-import java.security.MessageDigest
-
-/**
- * A task to update shas used by {@code DependencyLicensesCheck}
- */
-public class UpdateShasTask extends DefaultTask {
-
-    /** The parent dependency licenses task to use configuration from */
-    public DependencyLicensesTask parentTask
-
-    public UpdateShasTask() {
-        description = 'Updates the sha files for the dependencyLicenses check'
-        onlyIf { parentTask.licensesDir.exists() }
-    }
-
-    @TaskAction
-    public void updateShas() {
-        Set<File> shaFiles = new HashSet<File>()
-        parentTask.licensesDir.eachFile {
-            String name = it.getName()
-            if (name.endsWith(DependencyLicensesTask.SHA_EXTENSION)) {
-                shaFiles.add(it)
-            }
-        }
-        for (File dependency : parentTask.dependencies) {
-            String jarName = dependency.getName()
-            File shaFile = new File(parentTask.licensesDir, jarName + DependencyLicensesTask.SHA_EXTENSION)
-            if (shaFile.exists() == false) {
-                logger.lifecycle("Adding sha for ${jarName}")
-                String sha = MessageDigest.getInstance("SHA-1").digest(dependency.getBytes()).encodeHex().toString()
-                shaFile.setText(sha, 'UTF-8')
-            } else {
-                shaFiles.remove(shaFile)
-            }
-        }
-        shaFiles.each { shaFile ->
-            logger.lifecycle("Removing unused sha ${shaFile.getName()}")
-            Files.delete(shaFile.toPath())
-        }
-    }
-}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index bc5c7ff0871bb..f70f258b80a7d 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -657,8 +657,8 @@ class ClusterFormationTasks {
     static Task configureExecTask(String name, Project project, Task setup, NodeInfo node, Object[] execArgs) {
         return project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) { Exec exec ->
             exec.workingDir node.cwd
-            if (project.isRuntimeJavaHomeSet || node.nodeVersion.before(Version.fromString("7.0.0")) ||
-                node.config.distribution == 'integ-test-zip') {
+            if ((project.isRuntimeJavaHomeSet && node.isBwcNode == false) // runtime Java might not be compatible with old nodes
+                    || node.config.distribution == 'integ-test-zip') {
                 exec.environment.put('JAVA_HOME', project.runtimeJavaHome)
             } else {
                 // force JAVA_HOME to *not* be set
@@ -683,8 +683,8 @@ class ClusterFormationTasks {
             ant.exec(executable: node.executable, spawn: node.config.daemonize, newenvironment: true,
                      dir: node.cwd, taskname: 'elasticsearch') {
                 node.env.each { key, value -> env(key: key, value: value) }
-                if (project.isRuntimeJavaHomeSet || node.nodeVersion.before(Version.fromString("7.0.0")) ||
-                    node.config.distribution == 'integ-test-zip') {
+                if ((project.isRuntimeJavaHomeSet && node.isBwcNode == false) // runtime Java might not be compatible with old nodes
+                        || node.config.distribution == 'integ-test-zip') {
                     env(key: 'JAVA_HOME', value: project.runtimeJavaHome)
                 }
                 node.args.each { arg(value: it) }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/MessyTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/MessyTestPlugin.groovy
deleted file mode 100644
index 1c0aec1bc00f3..0000000000000
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/MessyTestPlugin.groovy
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.gradle.test
-
-import org.elasticsearch.gradle.plugin.PluginBuildPlugin
-import org.gradle.api.Project
-import org.gradle.api.artifacts.Dependency
-import org.gradle.api.artifacts.ProjectDependency
-import org.gradle.api.tasks.Copy
-
-/**
- * A plugin to run messy tests, which are generally tests that depend on plugins.
- *
- * This plugin will add the same test configuration as standalone tests, except
- * also add the plugin-metadata and properties files for each plugin project
- * dependency.
- */
-class MessyTestPlugin extends StandaloneTestPlugin {
-    @Override
-    public void apply(Project project) {
-        super.apply(project)
-
-        project.configurations.testCompile.dependencies.all { Dependency dep ->
-            // this closure is run every time a compile dependency is added
-            if (dep instanceof ProjectDependency && dep.dependencyProject.plugins.hasPlugin(PluginBuildPlugin)) {
-                project.gradle.projectsEvaluated {
-                    addPluginResources(project, dep.dependencyProject)
-                }
-            }
-        }
-    }
-
-    private static addPluginResources(Project project, Project pluginProject) {
-        String outputDir = "${project.buildDir}/generated-resources/${pluginProject.name}"
-        String taskName = ClusterFormationTasks.pluginTaskName("copy", pluginProject.name, "Metadata")
-        Copy copyPluginMetadata = project.tasks.create(taskName, Copy.class)
-        copyPluginMetadata.into(outputDir)
-        copyPluginMetadata.from(pluginProject.tasks.pluginProperties)
-        copyPluginMetadata.from(pluginProject.file('src/main/plugin-metadata'))
-        project.sourceSets.test.output.dir(outputDir, builtBy: taskName)
-
-        // add each generated dir to the test classpath in IDEs
-        project.idea.module.singleEntryLibraries= ['TEST': [project.file(outputDir)]]
-        // Eclipse doesn't need this because it gets the entire module as a dependency
-    }
-}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
index ae365038ccf3f..cb7a8397ed011 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
@@ -23,6 +23,7 @@ import com.sun.jna.Native
 import com.sun.jna.WString
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionProperties
 import org.gradle.api.Project
 
 import java.nio.file.Files
@@ -107,6 +108,9 @@ class NodeInfo {
     /** the version of elasticsearch that this node runs */
     Version nodeVersion
 
+    /** true if the node is not the current version */
+    boolean isBwcNode
+
     /** Holds node configuration for part of a test cluster. */
     NodeInfo(ClusterConfiguration config, int nodeNum, Project project, String prefix, String nodeVersion, File sharedDir) {
         this.config = config
@@ -121,6 +125,7 @@ class NodeInfo {
         baseDir = new File(project.buildDir, "cluster/${prefix} node${nodeNum}")
         pidFile = new File(baseDir, 'es.pid')
         this.nodeVersion = Version.fromString(nodeVersion)
+        this.isBwcNode = this.nodeVersion.before(VersionProperties.elasticsearch)
         homeDir = new File(baseDir, "elasticsearch-${nodeVersion}")
         pathConf = new File(homeDir, 'config')
         if (config.dataDir != null) {
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
index 40cefdcc25fb9..9857a1bc29ed7 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
@@ -26,13 +26,13 @@ import org.gradle.api.Task
 import org.gradle.api.execution.TaskExecutionAdapter
 import org.gradle.api.logging.Logger
 import org.gradle.api.logging.Logging
+import org.gradle.api.specs.Specs
 import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.TaskState
 import org.gradle.api.tasks.options.Option
 import org.gradle.api.tasks.testing.Test
 import org.gradle.plugins.ide.idea.IdeaPlugin
-import org.gradle.process.CommandLineArgumentProvider
 
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
@@ -59,7 +59,7 @@ class RestIntegTestTask extends DefaultTask {
     Boolean includePackaged = false
 
     RestIntegTestTask() {
-        runner = project.tasks.create("${name}Runner", Test.class)
+        runner = project.tasks.create("${name}Runner", RestTestRunnerTask.class)
         super.dependsOn(runner)
         clusterInit = project.tasks.create(name: "${name}Cluster#init", dependsOn: project.testClasses)
         runner.dependsOn(clusterInit)
@@ -77,10 +77,6 @@ class RestIntegTestTask extends DefaultTask {
             runner.useCluster project.testClusters."$name"
         }
 
-        // disable the build cache for rest test tasks
-        // there are a number of inputs we aren't properly tracking here so we'll just not cache these for now
-        runner.outputs.doNotCacheIf('Caching is disabled for REST integration tests') { true }
-
         // override/add more for rest tests
         runner.maxParallelForks = 1
         runner.include('**/*IT.class')
@@ -252,7 +248,7 @@ class RestIntegTestTask extends DefaultTask {
             restSpec
         }
         project.dependencies {
-            restSpec "org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}"
+            restSpec project.project(':rest-api-spec')
         }
         Task copyRestSpec = project.tasks.findByName('copyRestSpec')
         if (copyRestSpec != null) {
@@ -285,4 +281,5 @@ class RestIntegTestTask extends DefaultTask {
         }
         return copyRestSpec
     }
+
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
index c9a26eb74b54d..cfe610fa568c3 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
@@ -25,6 +25,7 @@ import groovy.transform.CompileStatic
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.JavaVersion
@@ -57,12 +58,14 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
                 + 'elasticsearch.standalone-rest-test, and elasticsearch.build '
                 + 'are mutually exclusive')
         }
+        project.rootProject.pluginManager.apply(GlobalBuildInfoPlugin)
         project.pluginManager.apply(JavaBasePlugin)
 
         project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)
         BuildPlugin.configureRepositories(project)
         BuildPlugin.configureTestTasks(project)
         BuildPlugin.configureInputNormalization(project)
+        BuildPlugin.configureFips140(project)
 
         ExtraPropertiesExtension ext = project.extensions.getByType(ExtraPropertiesExtension)
         project.extensions.getByType(JavaPluginExtension).sourceCompatibility = ext.get('minimumRuntimeVersion') as JavaVersion
@@ -79,7 +82,7 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
 
         // create a compileOnly configuration as others might expect it
         project.configurations.create("compileOnly")
-        project.dependencies.add('testCompile', "org.elasticsearch.test:framework:${VersionProperties.elasticsearch}")
+        project.dependencies.add('testCompile', project.project(':test:framework'))
 
         EclipseModel eclipse = project.extensions.getByType(EclipseModel)
         eclipse.classpath.sourceSets = [testSourceSet]
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/AbstractLazyPropertyCollection.java b/buildSrc/src/main/java/org/elasticsearch/gradle/AbstractLazyPropertyCollection.java
new file mode 100644
index 0000000000000..94cdb091eac1b
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/AbstractLazyPropertyCollection.java
@@ -0,0 +1,27 @@
+package org.elasticsearch.gradle;
+
+import java.util.List;
+
+public abstract class AbstractLazyPropertyCollection {
+
+    final String name;
+    final Object owner;
+
+    public AbstractLazyPropertyCollection(String name) {
+        this(name, null);
+    }
+
+    public AbstractLazyPropertyCollection(String name, Object owner) {
+        this.name = name;
+        this.owner = owner;
+    }
+
+    abstract List<? extends Object> getNormalizedCollection();
+
+    void assertNotNull(Object value, String description) {
+        if (value == null) {
+            throw new NullPointerException(name + " " + description + " was null" + (owner != null ? " when configuring " + owner : ""));
+        }
+    }
+
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/BwcVersions.java b/buildSrc/src/main/java/org/elasticsearch/gradle/BwcVersions.java
index daa8c4130dd3c..b5f1aa5f551dc 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/BwcVersions.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/BwcVersions.java
@@ -109,7 +109,7 @@ public BwcVersions(List<String> versionLines) {
     }
 
     protected BwcVersions(List<String> versionLines, Version currentVersionProperty) {
-        SortedSet<Version> allVersions = versionLines.stream()
+        this(versionLines.stream()
             .map(LINE_PATTERN::matcher)
             .filter(Matcher::matches)
             .map(match -> new Version(
@@ -117,8 +117,11 @@ protected BwcVersions(List<String> versionLines, Version currentVersionProperty)
                 Integer.parseInt(match.group(2)),
                 Integer.parseInt(match.group(3))
             ))
-            .collect(Collectors.toCollection(TreeSet::new));
+            .collect(Collectors.toCollection(TreeSet::new)), currentVersionProperty);
+    }
 
+    // for testkit tests, until BwcVersions is extracted into an extension
+    public BwcVersions(SortedSet<Version> allVersions, Version currentVersionProperty) {
         if (allVersions.isEmpty()) {
             throw new IllegalArgumentException("Could not parse any versions");
         }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
new file mode 100644
index 0000000000000..28748c00f46e1
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor;
+import org.elasticsearch.gradle.ElasticsearchDistribution.Platform;
+import org.elasticsearch.gradle.ElasticsearchDistribution.Type;
+import org.gradle.api.GradleException;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.UnknownTaskException;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.ConfigurationContainer;
+import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.artifacts.dsl.DependencyHandler;
+import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
+import org.gradle.api.credentials.HttpHeaderCredentials;
+import org.gradle.api.file.FileTree;
+import org.gradle.api.plugins.ExtraPropertiesExtension;
+import org.gradle.api.tasks.Sync;
+import org.gradle.api.tasks.TaskProvider;
+import org.gradle.authentication.http.HttpHeaderAuthentication;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.function.Supplier;
+
+/**
+ * A plugin to manage getting and extracting distributions of Elasticsearch.
+ *
+ * The source of the distribution could be from a local snapshot, a locally built
+ * bwc snapshot, or the Elastic downloads service.
+ */
+public class DistributionDownloadPlugin implements Plugin<Project> {
+
+    private static final String FAKE_IVY_GROUP = "elasticsearch-distribution";
+    private static final String DOWNLOAD_REPO_NAME = "elasticsearch-downloads";
+
+    private BwcVersions bwcVersions;
+    private NamedDomainObjectContainer<ElasticsearchDistribution> distributionsContainer;
+
+    @Override
+    public void apply(Project project) {
+        distributionsContainer = project.container(ElasticsearchDistribution.class, name -> {
+            Configuration fileConfiguration = project.getConfigurations().create("es_distro_file_" + name);
+            Configuration extractedConfiguration = project.getConfigurations().create("es_distro_extracted_" + name);
+            return new ElasticsearchDistribution(name, project.getObjects(), fileConfiguration, extractedConfiguration);
+        });
+        project.getExtensions().add("elasticsearch_distributions", distributionsContainer);
+
+        setupDownloadServiceRepo(project);
+
+        ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties();
+        this.bwcVersions = (BwcVersions) extraProperties.get("bwcVersions");
+        // TODO: setup snapshot dependency instead of pointing to bwc distribution projects for external projects
+
+        project.afterEvaluate(this::setupDistributions);
+    }
+
+    // pkg private for tests
+    void setupDistributions(Project project) {
+        for (ElasticsearchDistribution distribution : distributionsContainer) {
+            distribution.finalizeValues();
+
+            DependencyHandler dependencies = project.getDependencies();
+            // for the distribution as a file, just depend on the artifact directly
+            dependencies.add(distribution.configuration.getName(), dependencyNotation(project, distribution));
+
+            // no extraction allowed for rpm or deb
+            if (distribution.getType() != Type.RPM && distribution.getType() != Type.DEB) {
+                // for the distribution extracted, add a root level task that does the extraction, and depend on that
+                // extracted configuration as an artifact consisting of the extracted distribution directory
+                dependencies.add(distribution.getExtracted().configuration.getName(),
+                    projectDependency(project, ":", configName("extracted_elasticsearch", distribution)));
+                // ensure a root level download task exists
+                setupRootDownload(project.getRootProject(), distribution);
+            }
+        }
+    }
+
+    private void setupRootDownload(Project rootProject, ElasticsearchDistribution distribution) {
+        String extractTaskName = extractTaskName(distribution);
+        // NOTE: this is *horrendous*, but seems to be the only way to check for the existence of a registered task
+        try {
+            rootProject.getTasks().named(extractTaskName);
+            // already setup this version
+            return;
+        } catch (UnknownTaskException e) {
+            // fall through: register the task
+        }
+        setupDownloadServiceRepo(rootProject);
+
+        final ConfigurationContainer configurations = rootProject.getConfigurations();
+        String downloadConfigName = configName("elasticsearch", distribution);
+        String extractedConfigName = "extracted_" + downloadConfigName;
+        final Configuration downloadConfig = configurations.create(downloadConfigName);
+        configurations.create(extractedConfigName);
+        rootProject.getDependencies().add(downloadConfigName, dependencyNotation(rootProject, distribution));
+
+        // add task for extraction, delaying resolving config until runtime
+        if (distribution.getType() == Type.ARCHIVE || distribution.getType() == Type.INTEG_TEST_ZIP) {
+            Supplier<File> archiveGetter = downloadConfig::getSingleFile;
+            String extractDir = rootProject.getBuildDir().toPath().resolve("elasticsearch-distros").resolve(extractedConfigName).toString();
+            TaskProvider<Sync> extractTask = rootProject.getTasks().register(extractTaskName, Sync.class, syncTask -> {
+                syncTask.dependsOn(downloadConfig);
+                syncTask.into(extractDir);
+                syncTask.from((Callable<FileTree>)() -> {
+                    File archiveFile = archiveGetter.get();
+                    String archivePath = archiveFile.toString();
+                    if (archivePath.endsWith(".zip")) {
+                        return rootProject.zipTree(archiveFile);
+                    } else if (archivePath.endsWith(".tar.gz")) {
+                        return rootProject.tarTree(rootProject.getResources().gzip(archiveFile));
+                    }
+                    throw new IllegalStateException("unexpected file extension on [" + archivePath + "]");
+                });
+            });
+            rootProject.getArtifacts().add(extractedConfigName,
+                rootProject.getLayout().getProjectDirectory().dir(extractDir),
+                artifact -> artifact.builtBy(extractTask));
+        }
+    }
+
+    private static void setupDownloadServiceRepo(Project project) {
+        if (project.getRepositories().findByName(DOWNLOAD_REPO_NAME) != null) {
+            return;
+        }
+        project.getRepositories().ivy(ivyRepo -> {
+            ivyRepo.setName(DOWNLOAD_REPO_NAME);
+            ivyRepo.setUrl("https://artifacts.elastic.co");
+            ivyRepo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
+            // this header is not a credential but we hack the capability to send this header to avoid polluting our download stats
+            ivyRepo.credentials(HttpHeaderCredentials.class, creds -> {
+                creds.setName("X-Elastic-No-KPI");
+                creds.setValue("1");
+            });
+            ivyRepo.getAuthentication().create("header", HttpHeaderAuthentication.class);
+            ivyRepo.patternLayout(layout -> layout.artifact("/downloads/elasticsearch/[module]-[revision](-[classifier]).[ext]"));
+            ivyRepo.content(content -> content.includeGroup(FAKE_IVY_GROUP));
+        });
+        project.getRepositories().all(repo -> {
+            if (repo.getName().equals(DOWNLOAD_REPO_NAME) == false) {
+                // all other repos should ignore the special group name
+                repo.content(content -> content.excludeGroup(FAKE_IVY_GROUP));
+            }
+        });
+        // TODO: need maven repo just for integ-test-zip, but only in external cases
+    }
+
+    /**
+     * Returns a dependency object representing the given distribution.
+     *
+     * The returned object is suitable to be passed to {@link DependencyHandler}.
+     * The concrete type of the object will either be a project {@link Dependency} or
+     * a set of maven coordinates as a {@link String}. Project dependencies point to
+     * a project in the Elasticsearch repo either under `:distribution:bwc`,
+     * `:distribution:archives` or :distribution:packages`. Maven coordinates point to
+     * either the integ-test-zip coordinates on maven central, or a set of artificial
+     * coordinates that resolve to the Elastic download service through an ivy repository.
+     */
+    private Object dependencyNotation(Project project, ElasticsearchDistribution distribution) {
+
+        if (Version.fromString(VersionProperties.getElasticsearch()).equals(distribution.getVersion())) {
+            return projectDependency(project, distributionProjectPath(distribution), "default");
+            // TODO: snapshot dep when not in ES repo
+        }
+        BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.unreleasedInfo(distribution.getVersion());
+        if (unreleasedInfo != null) {
+            assert distribution.getBundledJdk();
+            return projectDependency(project, unreleasedInfo.gradleProjectPath, distributionProjectName(distribution));
+        }
+
+        if (distribution.getType() == Type.INTEG_TEST_ZIP) {
+            return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion();
+        }
+
+        String extension = distribution.getType().toString();
+        String classifier = "x86_64";
+        if (distribution.getType() == Type.ARCHIVE) {
+            extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz";
+            classifier = distribution.getPlatform() + "-" + classifier;
+        }
+        return FAKE_IVY_GROUP + ":elasticsearch" + (distribution.getFlavor() == Flavor.OSS ? "-oss:" : ":")
+            + distribution.getVersion() + ":" + classifier + "@" + extension;
+    }
+
+    private static Dependency projectDependency(Project project, String projectPath, String projectConfig) {
+
+        if (project.findProject(projectPath) == null) {
+            throw new GradleException("no project [" + projectPath + "], project names: " + project.getRootProject().getAllprojects());
+        }
+        Map<String, Object> depConfig = new HashMap<>();
+        depConfig.put("path", projectPath);
+        depConfig.put("configuration", projectConfig);
+        return project.getDependencies().project(depConfig);
+    }
+
+    private static String distributionProjectPath(ElasticsearchDistribution distribution) {
+        String projectPath = ":distribution";
+        if (distribution.getType() == Type.INTEG_TEST_ZIP) {
+            projectPath += ":archives:integ-test-zip";
+        } else {
+            projectPath += distribution.getType() == Type.ARCHIVE ? ":archives:" : ":packages:";
+            projectPath += distributionProjectName(distribution);
+        }
+        return projectPath;
+    }
+
+    private static String distributionProjectName(ElasticsearchDistribution distribution) {
+        String projectName = "";
+        if (distribution.getFlavor() == Flavor.OSS) {
+            projectName += "oss-";
+        }
+        if (distribution.getBundledJdk() == false) {
+            projectName += "no-jdk-";
+        }
+        if (distribution.getType() == Type.ARCHIVE) {
+            Platform platform = distribution.getPlatform();
+            projectName += platform.toString() + (platform == Platform.WINDOWS ? "-zip" : "-tar");
+        } else {
+            projectName += distribution.getType();
+        }
+        return projectName;
+    }
+
+    private static String configName(String prefix, ElasticsearchDistribution distribution) {
+        return prefix + "_" + distribution.getVersion() + "_" + distribution.getType() + "_" +
+            (distribution.getPlatform() == null ? "" : distribution.getPlatform() + "_")
+            + distribution.getFlavor() + (distribution.getBundledJdk() ? "" : "_nojdk");
+    }
+
+    private static String capitalize(String s) {
+        return s.substring(0, 1).toUpperCase(Locale.ROOT) + s.substring(1);
+    }
+
+    private static String extractTaskName(ElasticsearchDistribution distribution) {
+        String taskName = "extractElasticsearch";
+        if (distribution.getType() != Type.INTEG_TEST_ZIP) {
+            if (distribution.getFlavor() == Flavor.OSS) {
+                taskName += "Oss";
+            }
+            if (distribution.getBundledJdk() == false) {
+                taskName += "NoJdk";
+            }
+        }
+        if (distribution.getType() == Type.ARCHIVE) {
+            taskName += capitalize(distribution.getPlatform().toString());
+        } else if (distribution.getType() != Type.INTEG_TEST_ZIP) {
+            taskName += capitalize(distribution.getType().toString());
+        }
+        taskName += distribution.getVersion();
+        return taskName;
+    }
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java
new file mode 100644
index 0000000000000..ac18209a43373
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.gradle.api.Buildable;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.Property;
+import org.gradle.api.tasks.TaskDependency;
+
+import java.io.File;
+import java.util.Iterator;
+import java.util.Locale;
+
+public class ElasticsearchDistribution implements Buildable {
+
+    public enum Platform {
+        LINUX,
+        WINDOWS,
+        DARWIN;
+
+        @Override
+        public String toString() {
+            return super.toString().toLowerCase(Locale.ROOT);
+        }
+    }
+
+    public enum Type {
+        INTEG_TEST_ZIP,
+        ARCHIVE,
+        RPM,
+        DEB;
+
+        @Override
+        public String toString() {
+            return super.toString().toLowerCase(Locale.ROOT);
+        }
+    }
+
+    public enum Flavor {
+        DEFAULT,
+        OSS;
+
+        @Override
+        public String toString() {
+            return super.toString().toLowerCase(Locale.ROOT);
+        }
+    }
+
+    // package private to tests can use
+    static final Platform CURRENT_PLATFORM = OS.<Platform>conditional()
+        .onLinux(() -> Platform.LINUX)
+        .onWindows(() -> Platform.WINDOWS)
+        .onMac(() -> Platform.DARWIN)
+        .supply();
+
+    public static final class Extracted implements Buildable, Iterable<File> {
+
+        // pkg private so plugin can configure
+        final Configuration configuration;
+
+        private Extracted(Configuration configuration) {
+            this.configuration = configuration;
+        }
+
+        @Override
+        public Iterator<File> iterator() {
+            return configuration.iterator();
+        }
+
+        @Override
+        public TaskDependency getBuildDependencies() {
+            return configuration.getBuildDependencies();
+        }
+
+        @Override
+        public String toString() {
+            return configuration.getSingleFile().toString();
+        }
+    }
+
+    private final String name;
+    // pkg private so plugin can configure
+    final Configuration configuration;
+    private final Extracted extracted;
+
+    private final Property<Version> version;
+    private final Property<Type> type;
+    private final Property<Platform> platform;
+    private final Property<Flavor> flavor;
+    private final Property<Boolean> bundledJdk;
+
+    ElasticsearchDistribution(String name, ObjectFactory objectFactory, Configuration fileConfiguration,
+                              Configuration extractedConfiguration) {
+        this.name = name;
+        this.configuration = fileConfiguration;
+        this.version = objectFactory.property(Version.class);
+        this.version.convention(Version.fromString(VersionProperties.getElasticsearch()));
+        this.type = objectFactory.property(Type.class);
+        this.type.convention(Type.ARCHIVE);
+        this.platform = objectFactory.property(Platform.class);
+        this.flavor = objectFactory.property(Flavor.class);
+        this.bundledJdk = objectFactory.property(Boolean.class);
+        this.extracted = new Extracted(extractedConfiguration);
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public Version getVersion() {
+        return version.get();
+    }
+
+    public void setVersion(String version) {
+        this.version.set(Version.fromString(version));
+    }
+
+    public Platform getPlatform() {
+        return platform.getOrNull();
+    }
+
+    public void setPlatform(Platform platform) {
+        this.platform.set(platform);
+    }
+
+    public Type getType() {
+        return type.get();
+    }
+
+    public void setType(Type type) {
+        this.type.set(type);
+    }
+
+    public Flavor getFlavor() {
+        return flavor.getOrNull();
+    }
+
+    public void setFlavor(Flavor flavor) {
+        this.flavor.set(flavor);
+    }
+
+    public boolean getBundledJdk() {
+        return bundledJdk.getOrElse(true);
+    }
+
+    public void setBundledJdk(boolean bundledJdk) {
+        this.bundledJdk.set(bundledJdk);
+    }
+
+    @Override
+    public String toString() {
+        return configuration.getSingleFile().toString();
+    }
+
+    public Extracted getExtracted() {
+        if (getType() == Type.RPM || getType() == Type.DEB) {
+            throw new UnsupportedOperationException("distribution type [" + getType() + "] for " +
+                "elasticsearch distribution [" + name + "] cannot be extracted");
+        }
+        return extracted;
+    }
+
+    @Override
+    public TaskDependency getBuildDependencies() {
+        return configuration.getBuildDependencies();
+    }
+
+    // internal, make this distribution's configuration unmodifiable
+    void finalizeValues() {
+
+        if (getType() == Type.INTEG_TEST_ZIP) {
+            if (platform.isPresent()) {
+                throw new IllegalArgumentException(
+                    "platform not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]");
+            }
+            if (flavor.isPresent()) {
+                throw new IllegalArgumentException(
+                    "flavor not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]");
+            }
+            if (bundledJdk.isPresent()) {
+                throw new IllegalArgumentException(
+                    "bundledJdk not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]");
+            }
+            return;
+        }
+
+        if (getType() == Type.ARCHIVE) {
+            // defaults for archive, set here instead of via convention so integ-test-zip can verify they are not set
+            if (platform.isPresent() == false) {
+                platform.set(CURRENT_PLATFORM);
+            }
+        } else { // rpm or deb
+            if (platform.isPresent()) {
+                throw new IllegalArgumentException("platform not allowed for elasticsearch distribution ["
+                    + name + "] of type [" + getType() + "]");
+            }
+        }
+
+        if (flavor.isPresent() == false) {
+            flavor.set(Flavor.DEFAULT);
+        }
+        if (bundledJdk.isPresent() == false) {
+            bundledJdk.set(true);
+        }
+
+        version.finalizeValue();
+        platform.finalizeValue();
+        type.finalizeValue();
+        flavor.finalizeValue();
+        bundledJdk.finalizeValue();
+    }
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/EmptyDirTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/EmptyDirTask.java
new file mode 100644
index 0000000000000..ec9b774767cd1
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/EmptyDirTask.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle;
+
+import java.io.File;
+
+import javax.inject.Inject;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.TaskAction;
+import org.gradle.internal.nativeintegration.filesystem.Chmod;
+
+/**
+ * Creates an empty directory.
+ */
+public class EmptyDirTask extends DefaultTask {
+
+    private File dir;
+    private int dirMode = 0755;
+
+    /**
+     * Creates an empty directory with the configured permissions.
+     */
+    @TaskAction
+    public void create() {
+        dir.mkdirs();
+        getChmod().chmod(dir, dirMode);
+    }
+
+    @Inject
+    public Chmod getChmod() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Input
+    public File getDir() {
+        return dir;
+    }
+
+    /**
+     * @param dir The directory to create
+     */
+    public void setDir(File dir) {
+        this.dir = dir;
+    }
+
+    /**
+     * @param dir The path of the directory to create. Takes a String and coerces it to a file.
+     */
+    public void setDir(String dir) {
+        this.dir = getProject().file(dir);
+    }
+
+    @Input
+    public int getDirMode() {
+        return dirMode;
+    }
+
+    /**
+     * @param dirMode The permissions to apply to the new directory
+     */
+    public void setDirMode(int dirMode) {
+        this.dirMode = dirMode;
+    }
+
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
index a408b66ec817d..d4f0d9941dacf 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
@@ -23,6 +23,7 @@
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.Task;
 import org.gradle.api.UnknownTaskException;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.ConfigurationContainer;
@@ -165,7 +166,12 @@ private static void setupRootJdkDownload(Project rootProject, String platform, S
         }
         String extractDir = rootProject.getBuildDir().toPath().resolve("jdks/openjdk-" + jdkVersion + "_" + platform).toString();
         TaskProvider<Copy> extractTask = rootProject.getTasks().register(extractTaskName, Copy.class, copyTask -> {
-            copyTask.doFirst(t -> rootProject.delete(extractDir));
+            copyTask.doFirst(new Action<Task>() {
+                @Override
+                public void execute(Task t) {
+                    rootProject.delete(extractDir);
+                }
+            });
             copyTask.into(extractDir);
             copyTask.from(fileGetter, removeRootDir);
         });
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java b/buildSrc/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java
new file mode 100644
index 0000000000000..d23c9b9e748d5
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java
@@ -0,0 +1,205 @@
+package org.elasticsearch.gradle;
+
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.Nested;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+public class LazyPropertyList<T> extends AbstractLazyPropertyCollection implements List<T> {
+
+    private final List<PropertyListEntry<T>> delegate = new ArrayList<>();
+
+    public LazyPropertyList(String name) {
+        super(name);
+    }
+
+    public LazyPropertyList(String name, Object owner) {
+        super(name, owner);
+    }
+
+    @Override
+    public int size() {
+        return delegate.size();
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return delegate.isEmpty();
+    }
+
+    @Override
+    public boolean contains(Object o) {
+        return delegate.stream().anyMatch(entry -> entry.getValue().equals(o));
+    }
+
+    @Override
+    public Iterator<T> iterator() {
+        return delegate.stream().peek(this::validate).map(PropertyListEntry::getValue).iterator();
+    }
+
+    @Override
+    public Object[] toArray() {
+        return delegate.stream().peek(this::validate).map(PropertyListEntry::getValue).toArray();
+    }
+
+    @Override
+    public <T1> T1[] toArray(T1[] a) {
+        return delegate.stream().peek(this::validate).map(PropertyListEntry::getValue).collect(Collectors.toList()).toArray(a);
+    }
+
+    @Override
+    public boolean add(T t) {
+        return delegate.add(new PropertyListEntry<>(() -> t, PropertyNormalization.DEFAULT));
+    }
+
+    public boolean add(Supplier<T> supplier) {
+        return delegate.add(new PropertyListEntry<>(supplier, PropertyNormalization.DEFAULT));
+    }
+
+    public boolean add(Supplier<T> supplier, PropertyNormalization normalization) {
+        return delegate.add(new PropertyListEntry<>(supplier, normalization));
+    }
+
+    @Override
+    public boolean remove(Object o) {
+        throw new UnsupportedOperationException(this.getClass().getName() + " does not support remove()");
+    }
+
+    @Override
+    public boolean containsAll(Collection<?> c) {
+        return delegate.stream().map(PropertyListEntry::getValue).collect(Collectors.toList()).containsAll(c);
+    }
+
+    @Override
+    public boolean addAll(Collection<? extends T> c) {
+        c.forEach(this::add);
+        return true;
+    }
+
+    @Override
+    public boolean addAll(int index, Collection<? extends T> c) {
+        int i = index;
+        for (T item : c) {
+            this.add(i++, item);
+        }
+        return true;
+    }
+
+    @Override
+    public boolean removeAll(Collection<?> c) {
+        throw new UnsupportedOperationException(this.getClass().getName() + " does not support removeAll()");
+    }
+
+    @Override
+    public boolean retainAll(Collection<?> c) {
+        throw new UnsupportedOperationException(this.getClass().getName() + " does not support retainAll()");
+    }
+
+    @Override
+    public void clear() {
+        delegate.clear();
+    }
+
+    @Override
+    public T get(int index) {
+        PropertyListEntry<T> entry = delegate.get(index);
+        validate(entry);
+        return entry.getValue();
+    }
+
+    @Override
+    public T set(int index, T element) {
+        return delegate.set(index, new PropertyListEntry<>(() -> element, PropertyNormalization.DEFAULT)).getValue();
+    }
+
+    @Override
+    public void add(int index, T element) {
+        delegate.add(index, new PropertyListEntry<>(() -> element, PropertyNormalization.DEFAULT));
+    }
+
+    @Override
+    public T remove(int index) {
+        return delegate.remove(index).getValue();
+    }
+
+    @Override
+    public int indexOf(Object o) {
+        for (int i = 0; i < delegate.size(); i++) {
+            if (delegate.get(i).getValue().equals(o)) {
+                return i;
+            }
+        }
+
+        return -1;
+    }
+
+    @Override
+    public int lastIndexOf(Object o) {
+        int lastIndex = -1;
+        for (int i = 0; i < delegate.size(); i++) {
+            if (delegate.get(i).getValue().equals(o)) {
+                lastIndex = i;
+            }
+        }
+
+        return lastIndex;
+    }
+
+    @Override
+    public ListIterator<T> listIterator() {
+        return delegate.stream().map(PropertyListEntry::getValue).collect(Collectors.toList()).listIterator();
+    }
+
+    @Override
+    public ListIterator<T> listIterator(int index) {
+        return delegate.stream().peek(this::validate).map(PropertyListEntry::getValue).collect(Collectors.toList()).listIterator(index);
+    }
+
+    @Override
+    public List<T> subList(int fromIndex, int toIndex) {
+        return delegate.stream()
+            .peek(this::validate)
+            .map(PropertyListEntry::getValue)
+            .collect(Collectors.toList())
+            .subList(fromIndex, toIndex);
+    }
+
+    @Override
+    @Nested
+    List<? extends Object> getNormalizedCollection() {
+        return delegate.stream()
+            .peek(this::validate)
+            .filter(entry -> entry.getNormalization() != PropertyNormalization.IGNORE_VALUE)
+            .collect(Collectors.toList());
+    }
+
+    private void validate(PropertyListEntry<T> entry) {
+        assertNotNull(entry.getValue(), "entry");
+    }
+
+    private class PropertyListEntry<T> {
+        private final Supplier<T> supplier;
+        private final PropertyNormalization normalization;
+
+        PropertyListEntry(Supplier<T> supplier, PropertyNormalization normalization) {
+            this.supplier = supplier;
+            this.normalization = normalization;
+        }
+
+        public PropertyNormalization getNormalization() {
+            return normalization;
+        }
+
+        @Input
+        public T getValue() {
+            assertNotNull(supplier, "supplier");
+            return supplier.get();
+        }
+    }
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/LazyPropertyMap.java b/buildSrc/src/main/java/org/elasticsearch/gradle/LazyPropertyMap.java
new file mode 100644
index 0000000000000..781bb095bb5cb
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/LazyPropertyMap.java
@@ -0,0 +1,167 @@
+package org.elasticsearch.gradle;
+
+import org.gradle.api.Named;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.Nested;
+
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiFunction;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+public class LazyPropertyMap<K, V> extends AbstractLazyPropertyCollection implements Map<K, V> {
+
+    private final Map<K, PropertyMapEntry<K, V>> delegate = new LinkedHashMap<>();
+    private final BiFunction<K, V, ?> normalizationMapper;
+
+    public LazyPropertyMap(String name) {
+        this(name, null);
+    }
+
+    public LazyPropertyMap(String name, Object owner) {
+        this(name, owner, null);
+    }
+
+    public LazyPropertyMap(String name, Object owner, BiFunction<K, V, ?> normalizationMapper) {
+        super(name, owner);
+        this.normalizationMapper = normalizationMapper;
+    }
+
+    @Override
+    public int size() {
+        return delegate.size();
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return delegate.isEmpty();
+    }
+
+    @Override
+    public boolean containsKey(Object key) {
+        return delegate.containsKey(key);
+    }
+
+    @Override
+    public boolean containsValue(Object value) {
+        return delegate.values().stream().map(PropertyMapEntry::getValue).anyMatch(v -> v.equals(value));
+    }
+
+    @Override
+    public V get(Object key) {
+        PropertyMapEntry<K, V> entry = delegate.get(key);
+        if (entry != null) {
+            V value = entry.getValue();
+            assertNotNull(value, "value for key '" + key + "'");
+            return value;
+        } else {
+            return null;
+        }
+    }
+
+    @Override
+    public V put(K key, V value) {
+        return put(key, value, PropertyNormalization.DEFAULT);
+    }
+
+    public V put(K key, V value, PropertyNormalization normalization) {
+        assertNotNull(value, "value for key '" + key + "'");
+        return put(key, () -> value, normalization);
+    }
+
+    public V put(K key, Supplier<V> supplier) {
+        return put(key, supplier, PropertyNormalization.DEFAULT);
+    }
+
+    public V put(K key, Supplier<V> supplier, PropertyNormalization normalization) {
+        assertNotNull(supplier, "supplier for key '" + key + "'");
+        PropertyMapEntry<K, V> previous = delegate.put(key, new PropertyMapEntry<>(key, supplier, normalization));
+        return previous == null ? null : previous.getValue();
+    }
+
+    @Override
+    public V remove(Object key) {
+        PropertyMapEntry<K, V> previous = delegate.remove(key);
+        return previous == null ? null : previous.getValue();
+    }
+
+    @Override
+    public void putAll(Map<? extends K, ? extends V> m) {
+        throw new UnsupportedOperationException(this.getClass().getName() + " does not support putAll()");
+    }
+
+    @Override
+    public void clear() {
+        delegate.clear();
+    }
+
+    @Override
+    public Set<K> keySet() {
+        return delegate.keySet();
+    }
+
+    @Override
+    public Collection<V> values() {
+        return delegate.values().stream().peek(this::validate).map(PropertyMapEntry::getValue).collect(Collectors.toList());
+    }
+
+    @Override
+    public Set<Entry<K, V>> entrySet() {
+        return delegate.entrySet().stream()
+            .peek(this::validate)
+            .collect(Collectors.toMap(Entry::getKey, entry -> entry.getValue().getValue())).entrySet();
+    }
+
+    @Override
+    @Nested
+    List<? extends Object> getNormalizedCollection() {
+        return delegate.values().stream()
+            .peek(this::validate)
+            .filter(entry -> entry.getNormalization() != PropertyNormalization.IGNORE_VALUE)
+            .map(entry -> normalizationMapper == null ? entry : normalizationMapper.apply(entry.getKey(), entry.getValue()))
+            .collect(Collectors.toList());
+    }
+
+    private void validate(Map.Entry<K, PropertyMapEntry<K, V>> entry) {
+        validate(entry.getValue());
+    }
+
+    private void validate(PropertyMapEntry<K, V> supplier) {
+        assertNotNull(supplier, "key '" + supplier.getKey() + "' supplier value");
+    }
+
+    private static class PropertyMapEntry<K, V> implements Named {
+        private final K key;
+        private final Supplier<V> value;
+        private final PropertyNormalization normalization;
+
+        PropertyMapEntry(K key, Supplier<V> value, PropertyNormalization normalization) {
+            this.key = key;
+            this.value = value;
+            this.normalization = normalization;
+        }
+
+        public PropertyNormalization getNormalization() {
+            return normalization;
+        }
+
+        @Override
+        public String getName() {
+            return getKey().toString();
+        }
+
+        @Input
+        public K getKey() {
+            return key;
+        }
+
+        @Input
+        public V getValue() {
+            return value.get();
+        }
+    }
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/PropertyNormalization.java b/buildSrc/src/main/java/org/elasticsearch/gradle/PropertyNormalization.java
new file mode 100644
index 0000000000000..85fdede56c392
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/PropertyNormalization.java
@@ -0,0 +1,13 @@
+package org.elasticsearch.gradle;
+
+public enum PropertyNormalization {
+    /**
+     * Uses default strategy based on runtime property type.
+     */
+    DEFAULT,
+
+    /**
+     * Ignores property value completely for the purposes of input snapshotting.
+     */
+    IGNORE_VALUE
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/SystemPropertyCommandLineArgumentProvider.java b/buildSrc/src/main/java/org/elasticsearch/gradle/SystemPropertyCommandLineArgumentProvider.java
new file mode 100644
index 0000000000000..7e808724035df
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/SystemPropertyCommandLineArgumentProvider.java
@@ -0,0 +1,30 @@
+package org.elasticsearch.gradle;
+
+import org.gradle.api.tasks.Input;
+import org.gradle.process.CommandLineArgumentProvider;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class SystemPropertyCommandLineArgumentProvider implements CommandLineArgumentProvider {
+    private final Map<String, Object> systemProperties = new LinkedHashMap<>();
+
+    public void systemProperty(String key, Object value) {
+        systemProperties.put(key, value);
+    }
+
+    @Override
+    public Iterable<String> asArguments() {
+        return systemProperties.entrySet()
+            .stream()
+            .map(entry -> "-D" + entry.getKey() + "=" + entry.getValue())
+            .collect(Collectors.toList());
+    }
+
+    // Track system property keys as an input so our build cache key will change if we add properties but values are still ignored
+    @Input
+    public Iterable<String> getPropertyNames() {
+        return systemProperties.keySet();
+    }
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java b/buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java
index 3ac4a53910c26..52447b6bc20a1 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java
@@ -129,18 +129,14 @@ public boolean wait(int durationInMs) throws GeneralSecurityException, Interrupt
     }
 
     protected void checkResource(SSLContext ssl) throws IOException {
-        try {
-            final HttpURLConnection connection = buildConnection(ssl);
-            connection.connect();
-            final Integer response = connection.getResponseCode();
-            if (validResponseCodes.contains(response)) {
-                logger.info("Got successful response [{}] from URL [{}]", response, url);
-                return;
-            } else {
-                throw new IOException(response + " " + connection.getResponseMessage());
-            }
-        } catch (IOException e) {
-            throw e;
+        final HttpURLConnection connection = buildConnection(ssl);
+        connection.connect();
+        final Integer response = connection.getResponseCode();
+        if (validResponseCodes.contains(response)) {
+            logger.info("Got successful response [{}] from URL [{}]", response, url);
+            return;
+        } else {
+            throw new IOException(response + " " + connection.getResponseMessage());
         }
     }
 
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java
index f0f34e84261c6..32f9e8e15f928 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java
@@ -116,7 +116,7 @@ private static String findJavaHome(String version) {
         String versionedJavaHome = System.getenv(getJavaHomeEnvVarName(version));
         if (versionedJavaHome == null) {
             throw new GradleException(
-                "$versionedVarName must be set to build Elasticsearch. " +
+                "$" + getJavaHomeEnvVarName(version) + " must be set to build Elasticsearch. " +
                     "Note that if the variable was just set you might have to run `./gradlew --stop` for " +
                     "it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details."
             );
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.java
new file mode 100644
index 0000000000000..28f9a2f3d426e
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.plugin;
+
+import org.gradle.api.Project;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A container for plugin properties that will be written to the plugin descriptor, for easy
+ * manipulation in the gradle DSL.
+ */
+public class PluginPropertiesExtension {
+    private String name;
+
+    private String version;
+
+    private String description;
+
+    private String classname;
+
+    /** Other plugins this plugin extends through SPI */
+    private List<String> extendedPlugins = new ArrayList<>();
+
+    private boolean hasNativeController;
+
+    /** True if the plugin requires the elasticsearch keystore to exist, false otherwise. */
+    private boolean requiresKeystore;
+
+    /** A license file that should be included in the built plugin zip. */
+    private File licenseFile;
+
+    /**
+     * A notice file that should be included in the built plugin zip. This will be
+     * extended with notices from the {@code licenses/} directory.
+     */
+    private File noticeFile;
+
+    private final Project project;
+
+    public PluginPropertiesExtension(Project project) {
+        this.project = project;
+    }
+
+    public String getName() {
+        return name == null ? project.getName() : name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getVersion() {
+        return version == null ? project.getVersion().toString() : version;
+    }
+
+    public void setVersion(String version) {
+        this.version = version;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public void setDescription(String description) {
+        this.description = description;
+    }
+
+    public String getClassname() {
+        return classname;
+    }
+
+    public void setClassname(String classname) {
+        this.classname = classname;
+    }
+
+    public List<String> getExtendedPlugins() {
+        return this.extendedPlugins;
+    }
+
+    public boolean isHasNativeController() {
+        return hasNativeController;
+    }
+
+    public void setHasNativeController(boolean hasNativeController) {
+        this.hasNativeController = hasNativeController;
+    }
+
+    public boolean isRequiresKeystore() {
+        return requiresKeystore;
+    }
+
+    public void setRequiresKeystore(boolean requiresKeystore) {
+        this.requiresKeystore = requiresKeystore;
+    }
+
+    public File getLicenseFile() {
+        return licenseFile;
+    }
+
+    public void setLicenseFile(File licenseFile) {
+        this.project.getExtensions().getExtraProperties().set("licenseFile", licenseFile);
+        this.licenseFile = licenseFile;
+    }
+
+    public File getNoticeFile() {
+        return noticeFile;
+    }
+
+    public void setNoticeFile(File noticeFile) {
+        this.project.getExtensions().getExtraProperties().set("noticeFile", noticeFile);
+        this.noticeFile = noticeFile;
+    }
+
+    public Project getProject() {
+        return project;
+    }
+
+    public void setExtendedPlugins(List<String> extendedPlugins) {
+        this.extendedPlugins = extendedPlugins;
+    }
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java
new file mode 100644
index 0000000000000..d884207d590d3
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java
@@ -0,0 +1,328 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.precommit;
+
+import org.apache.commons.codec.binary.Hex;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.GradleException;
+import org.gradle.api.InvalidUserDataException;
+import org.gradle.api.file.FileCollection;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.InputDirectory;
+import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.Optional;
+import org.gradle.api.tasks.TaskAction;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * A task to check licenses for dependencies.
+ *
+ * There are two parts to the check:
+ * <ul>
+ *   <li>LICENSE and NOTICE files</li>
+ *   <li>SHA checksums for each dependency jar</li>
+ * </ul>
+ *
+ * The directory to find the license and sha files in defaults to the dir @{code licenses}
+ * in the project directory for this task. You can override this directory:
+ * <pre>
+ *   dependencyLicenses {
+ *     licensesDir = getProject().file("mybetterlicensedir")
+ *   }
+ * </pre>
+ *
+ * The jar files to check default to the dependencies from the default configuration. You
+ * can override this, for example, to only check compile dependencies:
+ * <pre>
+ *   dependencyLicenses {
+ *     dependencies = getProject().configurations.compile
+ *   }
+ * </pre>
+ *
+ * Every jar must have a {@code .sha1} file in the licenses dir. These can be managed
+ * automatically using the {@code updateShas} helper task that is created along
+ * with this task. It will add {@code .sha1} files for new jars that are in dependencies
+ * and remove old {@code .sha1} files that are no longer needed.
+ *
+ * Every jar must also have a LICENSE and NOTICE file. However, multiple jars can share
+ * LICENSE and NOTICE files by mapping a pattern to the same name.
+ * <pre>
+ *   dependencyLicenses {
+ *     mapping from: &#47;lucene-.*&#47;, to: "lucene"
+ *   }
+ * </pre>
+ */
+public class DependencyLicensesTask extends DefaultTask {
+
+    private final Pattern regex = Pattern.compile("-v?\\d+.*");
+
+    private final Logger logger = Logging.getLogger(getClass());
+
+    private static final String SHA_EXTENSION = ".sha1";
+
+    // TODO: we should be able to default this to eg compile deps, but we need to move the licenses
+    // check from distribution to core (ie this should only be run on java projects)
+    /** A collection of jar files that should be checked. */
+    private FileCollection dependencies;
+
+    /** The directory to find the license and sha files in. */
+    private File licensesDir = new File(getProject().getProjectDir(), "licenses");
+
+    /** A map of patterns to prefix, used to find the LICENSE and NOTICE file. */
+    private Map<String, String> mappings = new LinkedHashMap<>();
+
+    /** Names of dependencies whose shas should not exist. */
+    private Set<String> ignoreShas = new HashSet<>();
+
+    /**
+     * Add a mapping from a regex pattern for the jar name, to a prefix to find
+     * the LICENSE and NOTICE file for that jar.
+     */
+    public void mapping(Map<String, String> props) {
+        String from = props.remove("from");
+        if (from == null) {
+            throw new InvalidUserDataException("Missing \"from\" setting for license name mapping");
+        }
+        String to = props.remove("to");
+        if (to == null) {
+            throw new InvalidUserDataException("Missing \"to\" setting for license name mapping");
+        }
+        if (props.isEmpty() == false) {
+            throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: " + props.keySet());
+        }
+        mappings.put(from, to);
+    }
+
+    @InputFiles
+    public FileCollection getDependencies() {
+        return dependencies;
+    }
+
+    public void setDependencies(FileCollection dependencies) {
+        this.dependencies = dependencies;
+    }
+
+    @Optional
+    @InputDirectory
+    public File getLicensesDir() {
+        if (licensesDir.exists()) {
+            return licensesDir;
+        }
+
+        return null;
+    }
+
+    public void setLicensesDir(File licensesDir) {
+        this.licensesDir = licensesDir;
+    }
+
+    /**
+     * Add a rule which will skip SHA checking for the given dependency name. This should be used for
+     * locally build dependencies, which cause the sha to change constantly.
+     */
+    public void ignoreSha(String dep) {
+        ignoreShas.add(dep);
+    }
+
+    @TaskAction
+    public void checkDependencies() throws IOException, NoSuchAlgorithmException {
+        if (dependencies == null) {
+            throw new GradleException("No dependencies variable defined.");
+        }
+
+        if (dependencies.isEmpty()) {
+            if (licensesDir.exists()) {
+                throw new GradleException("Licenses dir " + licensesDir + " exists, but there are no dependencies");
+            }
+            return; // no dependencies to check
+        } else if (licensesDir.exists() == false) {
+            throw new GradleException("Licences dir " + licensesDir + " does not exist, but there are dependencies");
+        }
+
+        Map<String, Boolean> licenses = new HashMap<>();
+        Map<String, Boolean> notices = new HashMap<>();
+        Set<File> shaFiles = new HashSet<>();
+
+        for (File file : licensesDir.listFiles()) {
+            String name = file.getName();
+            if (name.endsWith(SHA_EXTENSION)) {
+                shaFiles.add(file);
+            } else if (name.endsWith("-LICENSE") || name.endsWith("-LICENSE.txt")) {
+                // TODO: why do we support suffix of LICENSE *and* LICENSE.txt??
+                licenses.put(name, false);
+            } else if (name.contains("-NOTICE") || name.contains("-NOTICE.txt")) {
+                notices.put(name, false);
+            }
+        }
+
+        checkDependencies(licenses, notices, shaFiles);
+
+        licenses.forEach((item, exists) -> failIfAnyMissing(item, exists, "license"));
+
+        notices.forEach((item, exists) -> failIfAnyMissing(item, exists, "notice"));
+
+        if (shaFiles.isEmpty() == false) {
+            throw new GradleException("Unused sha files found: \n" + joinFilenames(shaFiles));
+        }
+    }
+
+    private void failIfAnyMissing(String item, Boolean exists, String type) {
+        if (exists == false) {
+            throw new GradleException("Unused " + type + " " + item);
+        }
+    }
+
+    private void checkDependencies(Map<String, Boolean> licenses, Map<String, Boolean> notices, Set<File> shaFiles)
+        throws NoSuchAlgorithmException, IOException {
+        for (File dependency : dependencies) {
+            String jarName = dependency.getName();
+            String depName = regex.matcher(jarName).replaceFirst("");
+
+            validateSha(shaFiles, dependency, jarName, depName);
+
+            String dependencyName = getDependencyName(mappings, depName);
+            logger.info("mapped dependency name {} to {} for license/notice check", depName, dependencyName);
+            checkFile(dependencyName, jarName, licenses, "LICENSE");
+            checkFile(dependencyName, jarName, notices, "NOTICE");
+        }
+    }
+
+    private void validateSha(Set<File> shaFiles, File dependency, String jarName, String depName)
+        throws NoSuchAlgorithmException, IOException {
+        if (ignoreShas.contains(depName)) {
+            // local deps should not have sha files!
+            if (getShaFile(jarName).exists()) {
+                throw new GradleException("SHA file " + getShaFile(jarName) + " exists for ignored dependency " + depName);
+            }
+        } else {
+            logger.info("Checking sha for {}", jarName);
+            checkSha(dependency, jarName, shaFiles);
+        }
+    }
+
+    private String joinFilenames(Set<File> shaFiles) {
+        List<String> names = shaFiles.stream().map(File::getName).collect(Collectors.toList());
+        return String.join("\n", names);
+    }
+
+    public static String getDependencyName(Map<String, String> mappings, String dependencyName) {
+        // order is the same for keys and values iteration since we use a linked hashmap
+        List<String> mapped = new ArrayList<>(mappings.values());
+        Pattern mappingsPattern = Pattern.compile("(" + String.join(")|(", mappings.keySet()) + ")");
+        Matcher match = mappingsPattern.matcher(dependencyName);
+        if (match.matches()) {
+            int i = 0;
+            while (i < match.groupCount() && match.group(i + 1) == null) {
+                ++i;
+            }
+            return mapped.get(i);
+        }
+        return dependencyName;
+    }
+
+    private void checkSha(File jar, String jarName, Set<File> shaFiles) throws NoSuchAlgorithmException, IOException {
+        File shaFile = getShaFile(jarName);
+        if (shaFile.exists() == false) {
+            throw new GradleException("Missing SHA for " + jarName + ". Run \"gradle updateSHAs\" to create them");
+        }
+
+        // TODO: shouldn't have to trim, sha files should not have trailing newline
+        byte[] fileBytes = Files.readAllBytes(shaFile.toPath());
+        String expectedSha = new String(fileBytes, StandardCharsets.UTF_8).trim();
+
+        String sha = getSha1(jar);
+
+        if (expectedSha.equals(sha) == false) {
+            throw new GradleException(
+                "SHA has changed! Expected " + expectedSha + " for " + jarName + " but got " + sha + ". " +
+                    "\nThis usually indicates a corrupt dependency cache or artifacts changed upstream." +
+                    "\nEither wipe your cache, fix the upstream artifact, or delete " + shaFile + " and run updateShas");
+        }
+        shaFiles.remove(shaFile);
+    }
+
+    private void checkFile(String name, String jarName, Map<String, Boolean> counters, String type) {
+        String fileName = getFileName(name, counters, type);
+
+        if (counters.containsKey(fileName) == false) {
+            throw new GradleException("Missing " + type + " for " + jarName + ", expected in " + fileName);
+        }
+
+        counters.put(fileName, true);
+    }
+
+    private String getFileName(String name, Map<String, ?> counters, String type) {
+        String fileName = name + "-" + type;
+
+        if (counters.containsKey(fileName) == false) {
+            // try the other suffix...TODO: get rid of this, just support ending in .txt
+            return fileName + ".txt";
+        }
+
+        return fileName;
+    }
+
+    @Input
+    public LinkedHashMap<String, String> getMappings() {
+        return new LinkedHashMap<>(mappings);
+    }
+
+    File getShaFile(String jarName) {
+        return new File(licensesDir, jarName + SHA_EXTENSION);
+    }
+
+    Set<File> getShaFiles() {
+        File[] array = licensesDir.listFiles();
+        if (array == null) {
+            throw new GradleException("\"" + licensesDir.getPath() + "\" isn't a valid directory");
+        }
+
+        return Arrays.stream(array)
+            .filter(file -> file.getName().endsWith(SHA_EXTENSION))
+            .collect(Collectors.toSet());
+    }
+
+    String getSha1(File file) throws IOException, NoSuchAlgorithmException {
+        byte[] bytes =  Files.readAllBytes(file.toPath());
+
+        MessageDigest digest = MessageDigest.getInstance("SHA-1");
+        char[] encoded = Hex.encodeHex(digest.digest(bytes));
+        return String.copyValueOf(encoded);
+    }
+
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/UpdateShasTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/UpdateShasTask.java
new file mode 100644
index 0000000000000..db3148da696a3
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/UpdateShasTask.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.precommit;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+import org.gradle.api.tasks.TaskAction;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
+import java.security.NoSuchAlgorithmException;
+import java.util.Set;
+
+/**
+ * A task to update shas used by {@code DependencyLicensesCheck}
+ */
+public class UpdateShasTask extends DefaultTask {
+
+    private final Logger logger = Logging.getLogger(getClass());
+
+    /** The parent dependency licenses task to use configuration from */
+    private DependencyLicensesTask parentTask;
+
+    public UpdateShasTask() {
+        setDescription("Updates the sha files for the dependencyLicenses check");
+        setOnlyIf(element -> parentTask.getLicensesDir() != null);
+    }
+
+    @TaskAction
+    public void updateShas() throws NoSuchAlgorithmException, IOException {
+        Set<File> shaFiles = parentTask.getShaFiles();
+
+        for (File dependency : parentTask.getDependencies()) {
+            String jarName = dependency.getName();
+            File shaFile = parentTask.getShaFile(jarName);
+
+            if (shaFile.exists() == false) {
+                createSha(dependency, jarName, shaFile);
+            } else {
+                shaFiles.remove(shaFile);
+            }
+        }
+
+        for (File shaFile : shaFiles) {
+            logger.lifecycle("Removing unused sha " + shaFile.getName());
+            shaFile.delete();
+        }
+    }
+
+    private void createSha(File dependency, String jarName, File shaFile) throws IOException, NoSuchAlgorithmException {
+        logger.lifecycle("Adding sha for " + jarName);
+
+        String sha = parentTask.getSha1(dependency);
+
+        Files.write(shaFile.toPath(), sha.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
+    }
+
+    public DependencyLicensesTask getParentTask() {
+        return parentTask;
+    }
+
+    public void setParentTask(DependencyLicensesTask parentTask) {
+        this.parentTask = parentTask;
+    }
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestTestRunnerTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestTestRunnerTask.java
new file mode 100644
index 0000000000000..eff05f64f9c33
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestTestRunnerTask.java
@@ -0,0 +1,37 @@
+package org.elasticsearch.gradle.test;
+
+import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
+import org.gradle.api.tasks.CacheableTask;
+import org.gradle.api.tasks.Nested;
+import org.gradle.api.tasks.testing.Test;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+import static org.elasticsearch.gradle.Distribution.INTEG_TEST;
+
+/**
+ * Customized version of Gradle {@link Test} task which tracks a collection of {@link ElasticsearchCluster} as a task input. We must do this
+ * as a custom task type because the current {@link org.gradle.api.tasks.TaskInputs} runtime API does not have a way to register
+ * {@link Nested} inputs.
+ */
+@CacheableTask
+public class RestTestRunnerTask extends Test {
+
+    private Collection<ElasticsearchCluster> clusters = new ArrayList<>();
+
+    public RestTestRunnerTask() {
+        super();
+        this.getOutputs().doNotCacheIf("Build cache is only enabled for tests against clusters using the 'integ-test' distribution",
+            task -> clusters.stream().flatMap(c -> c.getNodes().stream()).anyMatch(n -> n.getDistribution() != INTEG_TEST));
+    }
+
+    @Nested
+    public Collection<ElasticsearchCluster> getClusters() {
+        return clusters;
+    }
+
+    public void testCluster(ElasticsearchCluster cluster) {
+        this.clusters.add(cluster);
+    }
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java
index e245fb0ead95a..c343f56525aea 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java
@@ -18,15 +18,17 @@
  */
 package org.elasticsearch.gradle.testclusters;
 
-import org.elasticsearch.GradleServicesAdapter;
 import org.elasticsearch.gradle.Distribution;
 import org.elasticsearch.gradle.FileSupplier;
+import org.elasticsearch.gradle.PropertyNormalization;
 import org.elasticsearch.gradle.Version;
 import org.elasticsearch.gradle.http.WaitForHttpResource;
+import org.gradle.api.Named;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Project;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
+import org.gradle.api.tasks.Nested;
 
 import java.io.File;
 import java.io.IOException;
@@ -47,7 +49,7 @@
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
-public class ElasticsearchCluster implements TestClusterConfiguration {
+public class ElasticsearchCluster implements TestClusterConfiguration, Named {
 
     private static final Logger LOGGER = Logging.getLogger(ElasticsearchNode.class);
     private static final int CLUSTER_UP_TIMEOUT = 40;
@@ -60,21 +62,23 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
     private final File workingDirBase;
     private final File artifactsExtractDir;
     private final LinkedHashMap<String, Predicate<TestClusterConfiguration>> waitConditions = new LinkedHashMap<>();
-    private final GradleServicesAdapter services;
+    private final Project project;
 
     public ElasticsearchCluster(String path, String clusterName, Project project, File artifactsExtractDir, File workingDirBase) {
         this.path = path;
         this.clusterName = clusterName;
+        this.project = project;
         this.workingDirBase = workingDirBase;
         this.artifactsExtractDir = artifactsExtractDir;
-        this.services = GradleServicesAdapter.getInstance(project);
         this.nodes = project.container(ElasticsearchNode.class);
         this.nodes.add(
             new ElasticsearchNode(
                 path, clusterName + "-0",
-                services, artifactsExtractDir, workingDirBase
+                project, artifactsExtractDir, workingDirBase
             )
         );
+        // configure the cluster name eagerly so nodes know about it
+        this.nodes.all((node) -> node.defaultConfig.put("cluster.name", safeName(clusterName)));
 
         addWaitForClusterHealth();
     }
@@ -94,7 +98,7 @@ public void setNumberOfNodes(int numberOfNodes) {
 
         for (int i = nodes.size() ; i < numberOfNodes; i++) {
             this.nodes.add(new ElasticsearchNode(
-                path, clusterName + "-" + i, services, artifactsExtractDir, workingDirBase
+                path, clusterName + "-" + i, project, artifactsExtractDir, workingDirBase
             ));
         }
     }
@@ -151,6 +155,11 @@ public void keystore(String key, File value) {
         nodes.all(each -> each.keystore(key, value));
     }
 
+    @Override
+    public void keystore(String key, File value, PropertyNormalization normalization) {
+        nodes.all(each -> each.keystore(key, value, normalization));
+    }
+
     @Override
     public void keystore(String key, FileSupplier valueSupplier) {
         nodes.all(each -> each.keystore(key, valueSupplier));
@@ -161,11 +170,21 @@ public void setting(String key, String value) {
         nodes.all(each -> each.setting(key, value));
     }
 
+    @Override
+    public void setting(String key, String value, PropertyNormalization normalization) {
+        nodes.all(each -> each.setting(key, value, normalization));
+    }
+
     @Override
     public void setting(String key, Supplier<CharSequence> valueSupplier) {
         nodes.all(each -> each.setting(key, valueSupplier));
     }
 
+    @Override
+    public void setting(String key, Supplier<CharSequence> valueSupplier, PropertyNormalization normalization) {
+        nodes.all(each -> each.setting(key, valueSupplier, normalization));
+    }
+
     @Override
     public void systemProperty(String key, String value) {
         nodes.all(each -> each.systemProperty(key, value));
@@ -176,6 +195,11 @@ public void systemProperty(String key, Supplier<CharSequence> valueSupplier) {
         nodes.all(each -> each.systemProperty(key, valueSupplier));
     }
 
+    @Override
+    public void systemProperty(String key, Supplier<CharSequence> valueSupplier, PropertyNormalization normalization) {
+        nodes.all(each -> each.systemProperty(key, valueSupplier, normalization));
+    }
+
     @Override
     public void environment(String key, String value) {
         nodes.all(each -> each.environment(key, value));
@@ -186,6 +210,16 @@ public void environment(String key, Supplier<CharSequence> valueSupplier) {
         nodes.all(each -> each.environment(key, valueSupplier));
     }
 
+    @Override
+    public void environment(String key, Supplier<CharSequence> valueSupplier, PropertyNormalization normalization) {
+        nodes.all(each -> each.environment(key, valueSupplier, normalization));
+    }
+
+    @Override
+    public void jvmArgs(String... values) {
+        nodes.all(each -> each.jvmArgs(values));
+    }
+
     @Override
     public void freeze() {
         nodes.forEach(ElasticsearchNode::freeze);
@@ -205,22 +239,40 @@ public void setJavaHome(File javaHome) {
 
     @Override
     public void start() {
-        String nodeNames = nodes.stream().map(ElasticsearchNode::getName).collect(Collectors.joining(","));
+        final String nodeNames;
+        if (nodes.stream().map(ElasticsearchNode::getName).anyMatch( name -> name == null)) {
+            nodeNames = null;
+        } else {
+            nodeNames = nodes.stream().map(ElasticsearchNode::getName).collect(Collectors.joining(","));
+        };
         for (ElasticsearchNode node : nodes) {
-            node.defaultConfig.put("cluster.name", safeName(clusterName));
-            if (Version.fromString(node.getVersion()).getMajor() >= 7) {
-                node.defaultConfig.put("cluster.initial_master_nodes", "[" + nodeNames + "]");
-                node.defaultConfig.put("discovery.seed_providers", "file");
+            if (nodeNames != null) {
+                // Can only configure master nodes if we have node names defined
+                if (Version.fromString(node.getVersion()).getMajor() >= 7) {
+                    node.defaultConfig.put("cluster.initial_master_nodes", "[" + nodeNames + "]");
+                    node.defaultConfig.put("discovery.seed_providers", "file");
+                    node.defaultConfig.put("discovery.seed_hosts", "[]");
+                }
             }
             node.start();
         }
     }
 
+    @Override
+    public void restart() {
+        nodes.forEach(ElasticsearchNode::restart);
+    }
+
     @Override
     public void extraConfigFile(String destination, File from) {
         nodes.all(node -> node.extraConfigFile(destination, from));
     }
 
+    @Override
+    public void extraConfigFile(String destination, File from, PropertyNormalization normalization) {
+        nodes.all(node -> node.extraConfigFile(destination, from, normalization));
+    }
+
     @Override
     public void user(Map<String, String> userSpec) {
         nodes.all(node -> node.user(userSpec));
@@ -262,14 +314,13 @@ public List<String> getAllTransportPortURI() {
     }
 
     public void waitForAllConditions() {
-        long startedAt = System.currentTimeMillis();
         LOGGER.info("Waiting for nodes");
         nodes.forEach(ElasticsearchNode::waitForAllConditions);
 
         writeUnicastHostsFiles();
 
         LOGGER.info("Starting to wait for cluster to form");
-        waitForConditions(waitConditions, startedAt, CLUSTER_UP_TIMEOUT, CLUSTER_UP_TIMEOUT_UNIT, this);
+        waitForConditions(waitConditions, System.currentTimeMillis(), CLUSTER_UP_TIMEOUT, CLUSTER_UP_TIMEOUT_UNIT, this);
     }
 
     @Override
@@ -305,9 +356,16 @@ public ElasticsearchNode singleNode() {
     private void addWaitForClusterHealth() {
         waitConditions.put("cluster health yellow", (node) -> {
             try {
+                boolean httpSslEnabled = getFirstNode().isHttpSslEnabled();
                 WaitForHttpResource wait = new WaitForHttpResource(
-                    "http", getFirstNode().getHttpSocketURI(), nodes.size()
+                    httpSslEnabled ? "https" : "http",
+                    getFirstNode().getHttpSocketURI(),
+                    nodes.size()
                 );
+                if (httpSslEnabled) {
+
+                    getFirstNode().configureHttpWait(wait);
+                }
                 List<Map<String, String>> credentials = getFirstNode().getCredentials();
                 if (getFirstNode().getCredentials().isEmpty() == false) {
                     wait.setUsername(credentials.get(0).get("useradd"));
@@ -315,7 +373,7 @@ private void addWaitForClusterHealth() {
                 }
                 return wait.wait(500);
             } catch (IOException e) {
-                throw new IllegalStateException("Connection attempt to " + this + " failed", e);
+                throw new UncheckedIOException("IO error while waiting cluster", e);
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
                 throw new TestClustersException("Interrupted while waiting for " + this, e);
@@ -325,6 +383,11 @@ private void addWaitForClusterHealth() {
         });
     }
 
+    @Nested
+    public NamedDomainObjectContainer<ElasticsearchNode> getNodes() {
+        return nodes;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
index bba94f6c7d173..025cee2ffafb1 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
@@ -18,13 +18,29 @@
  */
 package org.elasticsearch.gradle.testclusters;
 
-import org.elasticsearch.GradleServicesAdapter;
 import org.elasticsearch.gradle.Distribution;
 import org.elasticsearch.gradle.FileSupplier;
+import org.elasticsearch.gradle.LazyPropertyList;
+import org.elasticsearch.gradle.LazyPropertyMap;
+import org.elasticsearch.gradle.LoggedExec;
 import org.elasticsearch.gradle.OS;
+import org.elasticsearch.gradle.PropertyNormalization;
 import org.elasticsearch.gradle.Version;
+import org.elasticsearch.gradle.http.WaitForHttpResource;
+import org.gradle.api.Action;
+import org.gradle.api.Named;
+import org.gradle.api.Project;
+import org.gradle.api.file.FileCollection;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
+import org.gradle.api.tasks.Classpath;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.InputFile;
+import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.Nested;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
+import org.gradle.api.tasks.util.PatternFilterable;
 
 import java.io.ByteArrayInputStream;
 import java.io.File;
@@ -36,9 +52,12 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
@@ -63,8 +82,10 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private static final Logger LOGGER = Logging.getLogger(ElasticsearchNode.class);
     private static final int ES_DESTROY_TIMEOUT = 20;
     private static final TimeUnit ES_DESTROY_TIMEOUT_UNIT = TimeUnit.SECONDS;
-    private static final int NODE_UP_TIMEOUT = 60;
-    private static final TimeUnit NODE_UP_TIMEOUT_UNIT = TimeUnit.SECONDS;
+    private static final int NODE_UP_TIMEOUT = 2;
+    private static final TimeUnit NODE_UP_TIMEOUT_UNIT = TimeUnit.MINUTES;
+    private static final int ADDITIONAL_CONFIG_TIMEOUT = 15;
+    private static final TimeUnit ADDITIONAL_CONFIG_TIMEOUT_UNIT = TimeUnit.SECONDS;
     private static final List<String> OVERRIDABLE_SETTINGS = Arrays.asList(
         "path.repo",
         "discovery.seed_providers"
@@ -72,23 +93,23 @@ public class ElasticsearchNode implements TestClusterConfiguration {
 
     private final String path;
     private final String name;
-    private final GradleServicesAdapter services;
+    private final Project project;
     private final AtomicBoolean configurationFrozen = new AtomicBoolean(false);
     private final Path artifactsExtractDir;
     private final Path workingDir;
 
-
     private final LinkedHashMap<String, Predicate<TestClusterConfiguration>> waitConditions = new LinkedHashMap<>();
     private final List<URI> plugins = new ArrayList<>();
     private final List<File> modules = new ArrayList<>();
-    private final Map<String, Supplier<CharSequence>> settings = new LinkedHashMap<>();
-    private final Map<String, Supplier<CharSequence>> keystoreSettings = new LinkedHashMap<>();
-    private final Map<String, FileSupplier> keystoreFiles = new LinkedHashMap<>();
-    private final Map<String, Supplier<CharSequence>> systemProperties = new LinkedHashMap<>();
-    private final Map<String, Supplier<CharSequence>> environment = new LinkedHashMap<>();
-    private final Map<String, File> extraConfigFiles = new HashMap<>();
-    final LinkedHashMap<String, String> defaultConfig = new LinkedHashMap<>();
+    private final LazyPropertyMap<String, CharSequence> settings = new LazyPropertyMap<>("Settings", this);
+    private final LazyPropertyMap<String, CharSequence> keystoreSettings = new LazyPropertyMap<>("Keystore", this);
+    private final LazyPropertyMap<String, File> keystoreFiles = new LazyPropertyMap<>("Keystore files", this, FileEntry::new);
+    private final LazyPropertyMap<String, CharSequence> systemProperties = new LazyPropertyMap<>("System properties", this);
+    private final LazyPropertyMap<String, CharSequence> environment = new LazyPropertyMap<>("Environment", this);
+    private final LazyPropertyList<CharSequence> jvmArgs = new LazyPropertyList<>("JVM arguments", this);
+    private final LazyPropertyMap<String, File> extraConfigFiles = new LazyPropertyMap<>("Extra config files", this, FileEntry::new);
     private final List<Map<String, String>> credentials = new ArrayList<>();
+    final LinkedHashMap<String, String> defaultConfig = new LinkedHashMap<>();
 
     private final Path confPathRepo;
     private final Path configFile;
@@ -105,11 +126,12 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private File javaHome;
     private volatile Process esProcess;
     private Function<String, String> nameCustomization = Function.identity();
+    private boolean isWorkingDirConfigured = false;
 
-    ElasticsearchNode(String path, String name, GradleServicesAdapter services, File artifactsExtractDir, File workingDirBase) {
+    ElasticsearchNode(String path, String name, Project project, File artifactsExtractDir, File workingDirBase) {
         this.path = path;
         this.name = name;
-        this.services = services;
+        this.project = project;
         this.artifactsExtractDir = artifactsExtractDir.toPath();
         this.workingDir = workingDirBase.toPath().resolve(safeName(name)).toAbsolutePath();
         confPathRepo = workingDir.resolve("repo");
@@ -128,6 +150,7 @@ public String getName() {
         return nameCustomization.apply(name);
     }
 
+    @Input
     public String getVersion() {
         return version;
     }
@@ -139,6 +162,7 @@ public void setVersion(String version) {
         this.version = version;
     }
 
+    @Input
     public Distribution getDistribution() {
         return distribution;
     }
@@ -169,72 +193,81 @@ public void module(File module) {
 
     @Override
     public void keystore(String key, String value) {
-        addSupplier("Keystore", keystoreSettings, key, value);
+        keystoreSettings.put(key, value);
     }
 
     @Override
     public void keystore(String key, Supplier<CharSequence> valueSupplier) {
-        addSupplier("Keystore", keystoreSettings, key, valueSupplier);
+        keystoreSettings.put(key, valueSupplier);
     }
 
     @Override
     public void keystore(String key, File value) {
-        requireNonNull(value, "keystore value was null when configuring test cluster`" + this + "`");
-        keystore(key, () -> value);
+        keystoreFiles.put(key, value);
+    }
+
+    @Override
+    public void keystore(String key, File value, PropertyNormalization normalization) {
+        keystoreFiles.put(key, value, normalization);
     }
 
     @Override
     public void keystore(String key, FileSupplier valueSupplier) {
-        requireNonNull(key, "Keystore" + " key was null when configuring test cluster `" + this + "`");
-        requireNonNull(valueSupplier, "Keystore" + " value supplier was null when configuring test cluster `" + this + "`");
         keystoreFiles.put(key, valueSupplier);
     }
 
     @Override
     public void setting(String key, String value) {
-        addSupplier("Settings", settings, key, value);
+        settings.put(key, value);
+    }
+
+    @Override
+    public void setting(String key, String value, PropertyNormalization normalization) {
+        settings.put(key, value, normalization);
     }
 
     @Override
     public void setting(String key, Supplier<CharSequence> valueSupplier) {
-        addSupplier("Setting", settings, key, valueSupplier);
+        settings.put(key, valueSupplier);
+    }
+
+    @Override
+    public void setting(String key, Supplier<CharSequence> valueSupplier, PropertyNormalization normalization) {
+        settings.put(key, valueSupplier, normalization);
     }
 
     @Override
     public void systemProperty(String key, String value) {
-        addSupplier("Java System property", systemProperties, key, value);
+        systemProperties.put(key, value);
     }
 
     @Override
     public void systemProperty(String key, Supplier<CharSequence> valueSupplier) {
-        addSupplier("Java System property", systemProperties, key, valueSupplier);
+        systemProperties.put(key, valueSupplier);
     }
 
     @Override
-    public void environment(String key, String value) {
-        addSupplier("Environment variable", environment, key, value);
+    public void systemProperty(String key, Supplier<CharSequence> valueSupplier, PropertyNormalization normalization) {
+        systemProperties.put(key, valueSupplier, normalization);
     }
 
     @Override
-    public void environment(String key, Supplier<CharSequence> valueSupplier) {
-        addSupplier("Environment variable", environment, key, valueSupplier);
+    public void environment(String key, String value) {
+        environment.put(key, value);
     }
 
-    private void addSupplier(String name, Map<String, Supplier<CharSequence>> collector, String key, Supplier<CharSequence> valueSupplier) {
-        requireNonNull(key, name + " key was null when configuring test cluster `" + this + "`");
-        requireNonNull(valueSupplier, name + " value supplier was null when configuring test cluster `" + this + "`");
-        collector.put(key, valueSupplier);
+    @Override
+    public void environment(String key, Supplier<CharSequence> valueSupplier) {
+        environment.put(key, valueSupplier);
     }
 
-    private void addSupplier(String name, Map<String, Supplier<CharSequence>> collector, String key, String actualValue) {
-        requireNonNull(actualValue, name + " value was null when configuring test cluster `" + this + "`");
-        addSupplier(name, collector, key, () -> actualValue);
+    @Override
+    public void environment(String key, Supplier<CharSequence> valueSupplier, PropertyNormalization normalization) {
+        environment.put(key, valueSupplier, normalization);
     }
 
-    private void checkSuppliers(String name, Map<String, Supplier<CharSequence>> collector) {
-        collector.forEach((key, value) -> {
-            requireNonNull(value.get().toString(), name + " supplied value was null when configuring test cluster `" + this + "`");
-        });
+    public void jvmArgs(String... values) {
+        jvmArgs.addAll(Arrays.asList(values));
     }
 
     public Path getConfigDir() {
@@ -277,39 +310,43 @@ public Stream<String> logLines() throws IOException {
     public synchronized void start() {
         LOGGER.info("Starting `{}`", this);
 
-        Path distroArtifact = artifactsExtractDir
-            .resolve(distribution.getGroup())
-            .resolve("elasticsearch-" + getVersion());
-
-        if (Files.exists(distroArtifact) == false) {
-            throw new TestClustersException("Can not start " + this + ", missing: " + distroArtifact);
+        if (Files.exists(getExtractedDistributionDir()) == false) {
+            throw new TestClustersException("Can not start " + this + ", missing: " + getExtractedDistributionDir());
         }
-        if (Files.isDirectory(distroArtifact) == false) {
-            throw new TestClustersException("Can not start " + this + ", is not a directory: " + distroArtifact);
+        if (Files.isDirectory(getExtractedDistributionDir()) == false) {
+            throw new TestClustersException("Can not start " + this + ", is not a directory: " + getExtractedDistributionDir());
         }
 
         try {
-            createWorkingDir(distroArtifact);
+            if (isWorkingDirConfigured == false) {
+                logToProcessStdout("Configuring working directory: " + workingDir);
+                // Only configure working dir once so we don't loose data on restarts
+                isWorkingDirConfigured = true;
+                createWorkingDir(getExtractedDistributionDir());
+            }
         } catch (IOException e) {
             throw new UncheckedIOException("Failed to create working directory for " + this, e);
         }
         createConfiguration();
 
-        plugins.forEach(plugin -> runElaticsearchBinScript(
-            "elasticsearch-plugin",
-            "install", "--batch", plugin.toString())
-        );
+        if (plugins.isEmpty() == false) {
+            logToProcessStdout("Installing " + plugins.size() + " plugins");
+            plugins.forEach(plugin -> runElaticsearchBinScript(
+                "elasticsearch-plugin",
+                "install", "--batch", plugin.toString())
+            );
+        }
 
         if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) {
+            logToProcessStdout("Adding " + keystoreSettings.size() + " keystore settings and " + keystoreFiles.size() + " keystore files");
             runElaticsearchBinScript("elasticsearch-keystore", "create");
 
-            checkSuppliers("Keystore", keystoreSettings);
             keystoreSettings.forEach((key, value) ->
-                runElaticsearchBinScriptWithInput(value.get().toString(), "elasticsearch-keystore", "add", "-x", key)
+                runElaticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key)
             );
 
-            for (Map.Entry<String, FileSupplier> entry : keystoreFiles.entrySet()) {
-                File file = entry.getValue().get();
+            for (Map.Entry<String, File> entry : keystoreFiles.entrySet()) {
+                File file = entry.getValue();
                 requireNonNull(file, "supplied keystoreFile was null when configuring " + this);
                 if (file.exists() == false) {
                     throw new TestClustersException("supplied keystore file " + file + " does not exist, require for " + this);
@@ -323,62 +360,95 @@ public synchronized void start() {
         copyExtraConfigFiles();
 
         if (isSettingMissingOrTrue("xpack.security.enabled")) {
+            logToProcessStdout("Setting up " + credentials.size() + " users");
             if (credentials.isEmpty()) {
                 user(Collections.emptyMap());
             }
             credentials.forEach(paramMap -> runElaticsearchBinScript(
                 "elasticsearch-users",
-                    paramMap.entrySet().stream()
-                        .flatMap(entry -> Stream.of(entry.getKey(), entry.getValue()))
-                        .toArray(String[]::new)
+                paramMap.entrySet().stream()
+                    .flatMap(entry -> Stream.of(entry.getKey(), entry.getValue()))
+                    .toArray(String[]::new)
             ));
         }
 
+        logToProcessStdout("Starting Elasticsearch process");
         startElasticsearchProcess();
     }
 
+    private void logToProcessStdout(String message) {
+        try {
+            if (Files.exists(esStdoutFile.getParent()) == false) {
+                Files.createDirectories(esStdoutFile.getParent());
+            }
+            Files.write(
+                esStdoutFile,
+                ("[" + Instant.now().toString() + "] [BUILD] " + message + "\n").getBytes(StandardCharsets.UTF_8),
+                StandardOpenOption.CREATE, StandardOpenOption.APPEND
+            );
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    @Override
+    public void restart() {
+        LOGGER.info("Restarting {}", this);
+        stop(false);
+        try {
+            Files.delete(httpPortsFile);
+            Files.delete(transportPortFile);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+
+        start();
+    }
+
     private boolean isSettingMissingOrTrue(String name) {
-        return Boolean.valueOf(settings.getOrDefault(name, () -> "false").get().toString());
+        return Boolean.valueOf(settings.getOrDefault(name, "false").toString());
     }
 
     private void copyExtraConfigFiles() {
+        if (extraConfigFiles.isEmpty() == false) {
+            logToProcessStdout("Setting up " + extraConfigFiles.size() + " additional config files");
+        }
         extraConfigFiles.forEach((destination, from) -> {
-                if (Files.exists(from.toPath()) == false) {
-                    throw new TestClustersException("Can't create extra config file from " + from + " for " + this +
-                        " as it does not exist");
-                }
-                Path dst = configFile.getParent().resolve(destination);
-                try {
-                    Files.createDirectories(dst);
-                    Files.copy(from.toPath(), dst, StandardCopyOption.REPLACE_EXISTING);
-                    LOGGER.info("Added extra config file {} for {}", destination, this);
-                } catch (IOException e) {
-                    throw new UncheckedIOException("Can't create extra config file for", e);
-                }
-            });
+            if (Files.exists(from.toPath()) == false) {
+                throw new TestClustersException("Can't create extra config file from " + from + " for " + this +
+                    " as it does not exist");
+            }
+            Path dst = configFile.getParent().resolve(destination);
+            try {
+                Files.createDirectories(dst.getParent());
+                Files.copy(from.toPath(), dst, StandardCopyOption.REPLACE_EXISTING);
+                LOGGER.info("Added extra config file {} for {}", destination, this);
+            } catch (IOException e) {
+                throw new UncheckedIOException("Can't create extra config file for", e);
+            }
+        });
     }
 
     private void installModules() {
         if (distribution == Distribution.INTEG_TEST) {
-            modules.forEach(module -> services.copy(spec -> {
-                if (module.getName().toLowerCase().endsWith(".zip")) {
-                    spec.from(services.zipTree(module));
-                } else if (module.isDirectory()) {
-                    spec.from(module);
-                } else {
-                    throw new IllegalArgumentException("Not a valid module " + module + " for " + this);
+            logToProcessStdout("Installing " + modules.size() + "modules");
+            for (File module : modules) {
+                Path destination = workingDir.resolve("modules").resolve(module.getName().replace(".zip", "").replace("-" + version, ""));
+
+                // only install modules that are not already bundled with the integ-test distribution
+                if (Files.exists(destination) == false) {
+                    project.copy(spec -> {
+                        if (module.getName().toLowerCase().endsWith(".zip")) {
+                            spec.from(project.zipTree(module));
+                        } else if (module.isDirectory()) {
+                            spec.from(module);
+                        } else {
+                            throw new IllegalArgumentException("Not a valid module " + module + " for " + this);
+                        }
+                        spec.into(destination);
+                    });
                 }
-                spec.into(
-                    workingDir
-                        .resolve("modules")
-                        .resolve(
-                            module.getName()
-                                .replace(".zip", "")
-                                .replace("-" + version, "")
-                        )
-                        .toFile()
-                );
-            }));
+            }
         } else {
             LOGGER.info("Not installing " + modules.size() + "(s) since the " + distribution + " distribution already " +
                 "has them");
@@ -394,6 +464,15 @@ public void extraConfigFile(String destination, File from) {
         extraConfigFiles.put(destination, from);
     }
 
+    @Override
+    public void extraConfigFile(String destination, File from, PropertyNormalization normalization) {
+        if (destination.contains("..")) {
+            throw new IllegalArgumentException("extra config file destination can't be relative, was " + destination +
+                " for " + this);
+        }
+        extraConfigFiles.put(destination, from, normalization);
+    }
+
     @Override
     public void user(Map<String, String> userSpec) {
         Set<String> keys = new HashSet<>(userSpec.keySet());
@@ -403,16 +482,23 @@ public void user(Map<String, String> userSpec) {
         if (keys.isEmpty() == false) {
             throw new TestClustersException("Unknown keys in user definition " + keys + " for " + this);
         }
-        Map<String,String> cred = new LinkedHashMap<>();
-        cred.put("useradd", userSpec.getOrDefault("username","test_user"));
-        cred.put("-p", userSpec.getOrDefault("password","x-pack-test-password"));
+        Map<String, String> cred = new LinkedHashMap<>();
+        cred.put("useradd", userSpec.getOrDefault("username", "test_user"));
+        cred.put("-p", userSpec.getOrDefault("password", "x-pack-test-password"));
         cred.put("-r", userSpec.getOrDefault("role", "superuser"));
         credentials.add(cred);
     }
 
     private void runElaticsearchBinScriptWithInput(String input, String tool, String... args) {
+        if (
+            Files.exists(workingDir.resolve("bin").resolve(tool)) == false &&
+                Files.exists(workingDir.resolve("bin").resolve(tool + ".bat")) == false
+        ) {
+            throw new TestClustersException("Can't run bin script: `" + tool + "` does not exist. " +
+                "Is this the distribution you expect it to be ?");
+        }
         try (InputStream byteArrayInputStream = new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8))) {
-            services.loggedExec(spec -> {
+            LoggedExec.exec(project, spec -> {
                 spec.setEnvironment(getESEnvironment());
                 spec.workingDir(workingDir);
                 spec.executable(
@@ -453,12 +539,25 @@ private Map<String, String> getESEnvironment() {
         defaultEnv.put("ES_PATH_CONF", configFile.getParent().toString());
         String systemPropertiesString = "";
         if (systemProperties.isEmpty() == false) {
-            checkSuppliers("Java System property", systemProperties);
             systemPropertiesString = " " + systemProperties.entrySet().stream()
-                .map(entry -> "-D" + entry.getKey() + "=" + entry.getValue().get())
+                .map(entry -> "-D" + entry.getKey() + "=" + entry.getValue())
+                .collect(Collectors.joining(" "));
+        }
+        String jvmArgsString = "";
+        if (jvmArgs.isEmpty() == false) {
+            jvmArgsString = " " + jvmArgs.stream()
+                .peek(argument -> {
+                    if (argument.toString().startsWith("-D")) {
+                        throw new TestClustersException("Invalid jvm argument `" + argument +
+                            "` configure as systemProperty instead for " + this
+                        );
+                    }
+                })
                 .collect(Collectors.joining(" "));
         }
-        defaultEnv.put("ES_JAVA_OPTS", "-Xms512m -Xmx512m -ea -esa" + systemPropertiesString);
+        defaultEnv.put("ES_JAVA_OPTS", "-Xms512m -Xmx512m -ea -esa" +
+            systemPropertiesString + jvmArgsString
+        );
         defaultEnv.put("ES_TMPDIR", tmpDir.toString());
         // Windows requires this as it defaults to `c:\windows` despite ES_TMPDIR
         defaultEnv.put("TMP", tmpDir.toString());
@@ -471,8 +570,7 @@ private Map<String, String> getESEnvironment() {
             );
         }
 
-        checkSuppliers("Environment variable", environment);
-        environment.forEach((key, value) -> defaultEnv.put(key, value.get().toString()));
+        environment.forEach((key, value) -> defaultEnv.put(key, value.toString()));
         return defaultEnv;
     }
 
@@ -520,6 +618,14 @@ public List<String> getAllTransportPortURI() {
         return getTransportPortInternal();
     }
 
+    public File getServerLog() {
+        return confPathLogs.resolve(defaultConfig.get("cluster.name") + "_server.json").toFile();
+    }
+
+    public File getAuditLog() {
+        return confPathLogs.resolve(defaultConfig.get("cluster.name") + "_audit.json").toFile();
+    }
+
     @Override
     public synchronized void stop(boolean tailLogs) {
         if (esProcess == null && tailLogs) {
@@ -588,7 +694,7 @@ private void logProcessInfo(String prefix, ProcessHandle.Info info) {
 
     private void logFileContents(String description, Path from) {
         LOGGER.error("{} `{}`", description, this);
-        try(Stream<String> lines = Files.lines(from, StandardCharsets.UTF_8)) {
+        try (Stream<String> lines = Files.lines(from, StandardCharsets.UTF_8)) {
             lines
                 .map(line -> "  " + line)
                 .forEach(LOGGER::error);
@@ -624,12 +730,12 @@ private void createWorkingDir(Path distroExtractDir) throws IOException {
      * We remove write permissions to make sure files are note mistakenly edited ( e.x. the config file ) and changes
      * reflected across all copies. Permissions are retained to be able to replace the links.
      *
-     * @param sourceRoot where to copy from
+     * @param sourceRoot      where to copy from
      * @param destinationRoot destination to link to
      */
     private void syncWithLinks(Path sourceRoot, Path destinationRoot) {
         if (Files.exists(destinationRoot)) {
-            services.delete(destinationRoot);
+            project.delete(destinationRoot);
         }
 
         try (Stream<Path> stream = Files.walk(sourceRoot)) {
@@ -662,8 +768,11 @@ private void syncWithLinks(Path sourceRoot, Path destinationRoot) {
         }
     }
 
-    private void createConfiguration()  {
-        defaultConfig.put("node.name", nameCustomization.apply(safeName(name)));
+    private void createConfiguration() {
+        String nodeName = nameCustomization.apply(safeName(name));
+        if (nodeName != null) {
+            defaultConfig.put("node.name", nodeName);
+        }
         defaultConfig.put("path.repo", confPathRepo.toAbsolutePath().toString());
         defaultConfig.put("path.data", confPathData.toAbsolutePath().toString());
         defaultConfig.put("path.logs", confPathLogs.toAbsolutePath().toString());
@@ -688,24 +797,21 @@ private void createConfiguration()  {
         // over and the REST client will not retry on circuit breaking exceptions yet (see #31986 for details). Once the REST client
         // can retry on circuit breaking exceptions, we can revert again to the default configuration.
         if (Version.fromString(version).getMajor() >= 7) {
-            defaultConfig.put("indices.breaker.total.use_real_memory",  "false");
+            defaultConfig.put("indices.breaker.total.use_real_memory", "false");
         }
         // Don't wait for state, just start up quickly. This will also allow new and old nodes in the BWC case to become the master
-        defaultConfig.put("discovery.initial_state_timeout",  "0s");
+        defaultConfig.put("discovery.initial_state_timeout", "0s");
 
-        checkSuppliers("Settings", settings);
-        Map<String, String> userConfig = settings.entrySet().stream()
-            .collect(Collectors.toMap(entry -> entry.getKey(), entry -> entry.getValue().get().toString()));
         HashSet<String> overriden = new HashSet<>(defaultConfig.keySet());
-        overriden.retainAll(userConfig.keySet());
+        overriden.retainAll(settings.keySet());
         overriden.removeAll(OVERRIDABLE_SETTINGS);
-        if (overriden.isEmpty() ==false) {
+        if (overriden.isEmpty() == false) {
             throw new IllegalArgumentException(
                 "Testclusters does not allow the following settings to be changed:" + overriden + " for " + this
             );
         }
         // Make sure no duplicate config keys
-        userConfig.keySet().stream()
+        settings.keySet().stream()
             .filter(OVERRIDABLE_SETTINGS::contains)
             .forEach(defaultConfig::remove);
 
@@ -716,7 +822,7 @@ private void createConfiguration()  {
             Files.write(
                 configFile,
                 Stream.concat(
-                    userConfig.entrySet().stream(),
+                    settings.entrySet().stream(),
                     defaultConfig.entrySet().stream()
                 )
                     .map(entry -> entry.getKey() + ": " + entry.getValue())
@@ -731,7 +837,7 @@ private void createConfiguration()  {
 
     private void checkFrozen() {
         if (configurationFrozen.get()) {
-            throw new IllegalStateException("Configuration for " + this +  " can not be altered, already locked");
+            throw new IllegalStateException("Configuration for " + this + " can not be altered, already locked");
         }
     }
 
@@ -756,11 +862,97 @@ private List<String> getHttpPortInternal() {
     }
 
     private List<String> readPortsFile(Path file) throws IOException {
-        try(Stream<String> lines = Files.lines(file, StandardCharsets.UTF_8)) {
+        try (Stream<String> lines = Files.lines(file, StandardCharsets.UTF_8)) {
             return lines.map(String::trim).collect(Collectors.toList());
         }
     }
 
+    private Path getExtractedDistributionDir() {
+        return artifactsExtractDir.resolve(distribution.getGroup()).resolve("elasticsearch-" + getVersion());
+    }
+
+    private List<File> getInstalledFileSet(Action<? super PatternFilterable> filter) {
+        return Stream.concat(
+            plugins.stream().filter(uri -> uri.getScheme().equalsIgnoreCase("file")).map(File::new),
+            modules.stream()
+        )
+            .filter(File::exists)
+            // TODO: We may be able to simplify this with Gradle 5.6
+            // https://docs.gradle.org/nightly/release-notes.html#improved-handling-of-zip-archives-on-classpaths
+            .map(zipFile -> project.zipTree(zipFile).matching(filter))
+            .flatMap(tree -> tree.getFiles().stream())
+            .sorted(Comparator.comparing(File::getName))
+            .collect(Collectors.toList());
+    }
+
+    @Input
+    private Set<URI> getRemotePlugins() {
+        Set<URI> file = plugins.stream().filter(uri -> uri.getScheme().equalsIgnoreCase("file") == false).collect(Collectors.toSet());
+        return file;
+    }
+
+    @Classpath
+    private List<File> getInstalledClasspath() {
+        return getInstalledFileSet(filter -> filter.include("**/*.jar"));
+    }
+
+    @InputFiles
+    @PathSensitive(PathSensitivity.RELATIVE)
+    private List<File> getInstalledFiles() {
+        return getInstalledFileSet(filter -> filter.exclude("**/*.jar"));
+    }
+
+    @Classpath
+    private List<File> getDistributionClasspath() {
+        ArrayList<File> files = new ArrayList<>(project.fileTree(getExtractedDistributionDir())
+            .matching(filter -> filter.include("**/*.jar"))
+            .getFiles());
+        files.sort(Comparator.comparing(File::getName));
+
+        return files;
+    }
+
+    @InputFiles
+    @PathSensitive(PathSensitivity.RELATIVE)
+    private FileCollection getDistributionFiles() {
+        return project.fileTree(getExtractedDistributionDir()).minus(project.files(getDistributionClasspath()));
+    }
+
+    @Nested
+    private Map<String, CharSequence> getKeystoreSettings() {
+        return keystoreSettings;
+    }
+
+    @Nested
+    private Map<String, File> getKeystoreFiles() {
+        return keystoreFiles;
+    }
+
+    @Nested
+    private Map<String, CharSequence> getSettings() {
+        return settings;
+    }
+
+    @Nested
+    private Map<String, CharSequence> getSystemProperties() {
+        return systemProperties;
+    }
+
+    @Nested
+    private Map<String, CharSequence> getEnvironment() {
+        return environment;
+    }
+
+    @Nested
+    private List<CharSequence> getJvmArgs() {
+        return jvmArgs;
+    }
+
+    @Nested
+    private Map<String, File> getExtraConfigFiles() {
+        return extraConfigFiles;
+    }
+
     @Override
     public boolean isProcessAlive() {
         requireNonNull(
@@ -771,7 +963,23 @@ public boolean isProcessAlive() {
     }
 
     void waitForAllConditions() {
-        waitForConditions(waitConditions, System.currentTimeMillis(), NODE_UP_TIMEOUT, NODE_UP_TIMEOUT_UNIT, this);
+        waitForConditions(
+            waitConditions,
+            System.currentTimeMillis(),
+            NODE_UP_TIMEOUT_UNIT.toMillis(NODE_UP_TIMEOUT) +
+                // Installing plugins at config time and loading them when nods start requires additional time we need to
+                // account for
+                ADDITIONAL_CONFIG_TIMEOUT_UNIT.toMillis(ADDITIONAL_CONFIG_TIMEOUT *
+                    (
+                        plugins.size() +
+                            keystoreFiles.size() +
+                            keystoreSettings.size() +
+                            credentials.size()
+                    )
+                ),
+            TimeUnit.MILLISECONDS,
+            this
+        );
     }
 
     @Override
@@ -793,6 +1001,7 @@ public String toString() {
         return "node{" + path + ":" + name + "}";
     }
 
+    @Input
     List<Map<String, String>> getCredentials() {
         return credentials;
     }
@@ -809,4 +1018,61 @@ private boolean checkPortsFilesExistWithDelay(TestClusterConfiguration node) {
         }
         return Files.exists(httpPortsFile) && Files.exists(transportPortFile);
     }
+
+    public boolean isHttpSslEnabled() {
+        return Boolean.valueOf(
+            settings.getOrDefault("xpack.security.http.ssl.enabled", "false").toString()
+        );
+    }
+
+    void configureHttpWait(WaitForHttpResource wait) {
+        if (settings.containsKey("xpack.security.http.ssl.certificate_authorities")) {
+            wait.setCertificateAuthorities(
+                getConfigDir()
+                    .resolve(settings.get("xpack.security.http.ssl.certificate_authorities").toString())
+                    .toFile()
+            );
+        }
+        if (settings.containsKey("xpack.security.http.ssl.certificate")) {
+            wait.setCertificateAuthorities(
+                getConfigDir()
+                    .resolve(settings.get("xpack.security.http.ssl.certificate").toString())
+                    .toFile()
+            );
+        }
+        if (settings.containsKey("xpack.security.http.ssl.keystore.path")) {
+            wait.setTrustStoreFile(
+                getConfigDir()
+                    .resolve(settings.get("xpack.security.http.ssl.keystore.path").toString())
+                    .toFile()
+            );
+        }
+        if (keystoreSettings.containsKey("xpack.security.http.ssl.keystore.secure_password")) {
+            wait.setTrustStorePassword(
+                keystoreSettings.get("xpack.security.http.ssl.keystore.secure_password").toString()
+            );
+        }
+    }
+
+    private static class FileEntry implements Named {
+        private String name;
+        private File file;
+
+        FileEntry(String name, File file) {
+            this.name = name;
+            this.file = file;
+        }
+
+        @Input
+        @Override
+        public String getName() {
+            return name;
+        }
+
+        @InputFile
+        @PathSensitive(PathSensitivity.NONE)
+        public File getFile() {
+            return file;
+        }
+    }
 }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterCleanupOnShutdown.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterCleanupOnShutdown.java
new file mode 100644
index 0000000000000..0381cece108e2
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterCleanupOnShutdown.java
@@ -0,0 +1,59 @@
+package org.elasticsearch.gradle.testclusters;
+
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+
+/**
+ * Keep an inventory of all running Clusters and stop them when interrupted
+ *
+ * This takes advantage of the fact that Gradle interrupts all the threads in the daemon when the build completes.
+ */
+public class TestClusterCleanupOnShutdown implements Runnable {
+
+    private final Logger logger =  Logging.getLogger(TestClusterCleanupOnShutdown.class);
+
+    private Set<ElasticsearchCluster> clustersToWatch = new HashSet<>();
+
+    public void watch(Collection<ElasticsearchCluster> cluster) {
+        synchronized (clustersToWatch) {
+            clustersToWatch.addAll(clustersToWatch);
+        }
+    }
+
+    public void unWatch(Collection<ElasticsearchCluster> cluster) {
+        synchronized (clustersToWatch) {
+            clustersToWatch.removeAll(clustersToWatch);
+        }
+    }
+
+    @Override
+    public void run() {
+        try {
+            while (true) {
+                Thread.sleep(Long.MAX_VALUE);
+            }
+        } catch (InterruptedException interrupted) {
+            synchronized (clustersToWatch) {
+                if (clustersToWatch.isEmpty()) {
+                    return;
+                }
+                logger.info("Cleanup thread was interrupted, shutting down all clusters");
+                Iterator<ElasticsearchCluster> iterator = clustersToWatch.iterator();
+                while (iterator.hasNext()) {
+                    ElasticsearchCluster cluster = iterator.next();
+                    iterator.remove();
+                    try {
+                        cluster.stop(false);
+                    } catch (Exception e) {
+                        logger.warn("Could not shut down {}", cluster, e);
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java
index 1ccbeabd4b88a..f290b4aa91b8f 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java
@@ -20,6 +20,7 @@
 
 import org.elasticsearch.gradle.Distribution;
 import org.elasticsearch.gradle.FileSupplier;
+import org.elasticsearch.gradle.PropertyNormalization;
 import org.gradle.api.logging.Logging;
 import org.slf4j.Logger;
 
@@ -52,28 +53,44 @@ public interface TestClusterConfiguration {
 
     void keystore(String key, File value);
 
+    void keystore(String key, File value, PropertyNormalization normalization);
+
     void keystore(String key, FileSupplier valueSupplier);
 
     void setting(String key, String value);
 
+    void setting(String key, String value, PropertyNormalization normalization);
+
     void setting(String key, Supplier<CharSequence> valueSupplier);
 
+    void setting(String key, Supplier<CharSequence> valueSupplier, PropertyNormalization normalization);
+
     void systemProperty(String key, String value);
 
     void systemProperty(String key, Supplier<CharSequence> valueSupplier);
 
+    void systemProperty(String key, Supplier<CharSequence> valueSupplier, PropertyNormalization normalization);
+
     void environment(String key, String value);
 
     void environment(String key, Supplier<CharSequence> valueSupplier);
 
+    void environment(String key, Supplier<CharSequence> valueSupplier, PropertyNormalization normalization);
+
+    void jvmArgs(String... values);
+
     void freeze();
 
     void setJavaHome(File javaHome);
 
     void start();
 
+    void restart();
+
     void extraConfigFile(String destination, File from);
 
+    void extraConfigFile(String destination, File from, PropertyNormalization normalization);
+
     void user(Map<String, String> userSpec);
 
     String getHttpSocketURI();
@@ -116,11 +133,7 @@ default void waitForConditions(
                 } catch (TestClustersException e) {
                     throw e;
                 } catch (Exception e) {
-                    if (lastException == null) {
-                        lastException = e;
-                    } else {
-                        lastException = e;
-                    }
+                    lastException = e;
                 }
             }
             if (conditionMet == false) {
@@ -129,7 +142,17 @@ default void waitForConditions(
                 if (lastException == null) {
                     throw new TestClustersException(message);
                 } else {
-                    throw new TestClustersException(message, lastException);
+                    String extraCause = "";
+                    Throwable cause = lastException;
+                    int ident = 2;
+                    while (cause != null) {
+                        if (cause.getMessage() != null && cause.getMessage().isEmpty() == false) {
+                            extraCause += "\n" + " ".repeat(ident) + cause.getMessage();
+                            ident += 2;
+                        }
+                        cause = cause.getCause();
+                    }
+                    throw new TestClustersException(message + extraCause, lastException);
                 }
             }
             logger.info(
@@ -146,7 +169,5 @@ default String safeName(String name) {
             .replaceAll("[^a-zA-Z0-9]+", "-");
     }
 
-
-
     boolean isProcessAlive();
 }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersCleanupExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersCleanupExtension.java
new file mode 100644
index 0000000000000..14bdfa952db0f
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersCleanupExtension.java
@@ -0,0 +1,74 @@
+package org.elasticsearch.gradle.testclusters;
+
+import org.gradle.api.Project;
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This extensions was meant to be used internally by testclusters
+ *
+ * It holds synchronization primitives needed to implement the rate limiting.
+ * This is tricky because we can't use Gradle workers as there's no way to make sure that tests and their clusters are
+ * allocated atomically, so we could be in a situation where all workers are tests waiting for clusters to start up.
+ *
+ * Also auto configures cleanup of executors to make sure we don't leak threads in the daemon.
+ */
+public class TestClustersCleanupExtension {
+
+    private static final int EXECUTOR_SHUTDOWN_TIMEOUT = 1;
+    private static final TimeUnit EXECUTOR_SHUTDOWN_TIMEOUT_UNIT = TimeUnit.MINUTES;
+
+    private static final Logger logger =  Logging.getLogger(TestClustersCleanupExtension.class);
+
+    private final ExecutorService executorService;
+    private final TestClusterCleanupOnShutdown cleanupThread;
+
+    public TestClustersCleanupExtension() {
+        executorService = Executors.newSingleThreadExecutor();
+        cleanupThread = new TestClusterCleanupOnShutdown();
+        executorService.submit(cleanupThread);
+    }
+
+
+    public static void createExtension(Project project) {
+        if (project.getRootProject().getExtensions().findByType(TestClustersCleanupExtension.class) != null) {
+            return;
+        }
+        // Configure the extension on the root project so we have a single instance per run
+        TestClustersCleanupExtension ext = project.getRootProject().getExtensions().create(
+            "__testclusters_rate_limit",
+            TestClustersCleanupExtension.class
+        );
+        Thread shutdownHook = new Thread(ext.cleanupThread::run);
+        Runtime.getRuntime().addShutdownHook(shutdownHook);
+        project.getGradle().buildFinished(buildResult -> {
+            ext.executorService.shutdownNow();
+            try {
+                if (ext.executorService.awaitTermination(EXECUTOR_SHUTDOWN_TIMEOUT, EXECUTOR_SHUTDOWN_TIMEOUT_UNIT) == false) {
+                    throw new IllegalStateException(
+                        "Failed to shut down executor service after " +
+                            EXECUTOR_SHUTDOWN_TIMEOUT + " " + EXECUTOR_SHUTDOWN_TIMEOUT_UNIT
+                    );
+                }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+            try {
+                if (false == Runtime.getRuntime().removeShutdownHook(shutdownHook)) {
+                    logger.warn("Trying to deregister shutdown hook when it was not registered.");
+                }
+            } catch (IllegalStateException ese) {
+                // Thrown when shutdown is in progress
+                logger.warn("Can't remove shutdown hook", ese);
+            }
+        });
+    }
+
+    public TestClusterCleanupOnShutdown getCleanupThread() {
+        return cleanupThread;
+    }
+}
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
index daca1f5ebb191..3c50108d9a179 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
@@ -21,6 +21,7 @@
 import groovy.lang.Closure;
 import org.elasticsearch.gradle.BwcVersions;
 import org.elasticsearch.gradle.Version;
+import org.elasticsearch.gradle.test.RestTestRunnerTask;
 import org.elasticsearch.gradle.tool.Boilerplate;
 import org.gradle.api.Action;
 import org.gradle.api.NamedDomainObjectContainer;
@@ -43,31 +44,25 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 public class TestClustersPlugin implements Plugin<Project> {
 
     private static final String LIST_TASK_NAME = "listTestClusters";
-    private static final String NODE_EXTENSION_NAME = "testClusters";
+    public static final String EXTENSION_NAME = "testClusters";
     private static final String HELPER_CONFIGURATION_PREFIX = "testclusters";
     private static final String SYNC_ARTIFACTS_TASK_NAME = "syncTestClustersArtifacts";
-    private static final int EXECUTOR_SHUTDOWN_TIMEOUT = 1;
-    private static final TimeUnit EXECUTOR_SHUTDOWN_TIMEOUT_UNIT = TimeUnit.MINUTES;
 
     private static final Logger logger =  Logging.getLogger(TestClustersPlugin.class);
+    private static final String TESTCLUSTERS_INSPECT_FAILURE = "testclusters.inspect.failure";
 
     private final Map<Task, List<ElasticsearchCluster>> usedClusters = new HashMap<>();
     private final Map<ElasticsearchCluster, Integer> claimsInventory = new HashMap<>();
-    private final Set<ElasticsearchCluster> runningClusters =new HashSet<>();
-    private final Thread shutdownHook = new Thread(this::shutDownAllClusters);
-    private ExecutorService executorService = Executors.newSingleThreadExecutor();
+    private final Set<ElasticsearchCluster> runningClusters = new HashSet<>();
+    private final Boolean allowClusterToSurvive = Boolean.valueOf(System.getProperty(TESTCLUSTERS_INSPECT_FAILURE, "false"));
 
     public static String getHelperConfigurationName(String version) {
         return HELPER_CONFIGURATION_PREFIX + "-" + version;
@@ -80,6 +75,8 @@ public void apply(Project project) {
         // enable the DSL to describe clusters
         NamedDomainObjectContainer<ElasticsearchCluster> container = createTestClustersContainerExtension(project);
 
+        TestClustersCleanupExtension.createExtension(project);
+
         // provide a task to be able to list defined clusters.
         createListClustersTask(project, container);
 
@@ -98,9 +95,6 @@ public void apply(Project project) {
         // After each task we determine if there are clusters that are no longer needed.
         configureStopClustersHook(project);
 
-        // configure hooks to make sure no test cluster processes survive the build
-        configureCleanupHooks(project);
-
         // Since we have everything modeled in the DSL, add all the required dependencies e.x. the distribution to the
         // configuration so the user doesn't have to repeat this.
         autoConfigureClusterDependencies(project, rootProject, container);
@@ -122,7 +116,7 @@ private NamedDomainObjectContainer<ElasticsearchCluster> createTestClustersConta
                 new File(project.getBuildDir(), "testclusters")
             )
         );
-        project.getExtensions().add(NODE_EXTENSION_NAME, container);
+        project.getExtensions().add(EXTENSION_NAME, container);
         return container;
     }
 
@@ -162,6 +156,9 @@ public void doCall(ElasticsearchCluster cluster) {
                             ((Task) thisObject).dependsOn(
                                 project.getRootProject().getTasks().getByName(SYNC_ARTIFACTS_TASK_NAME)
                             );
+                            if (thisObject instanceof RestTestRunnerTask) {
+                                ((RestTestRunnerTask) thisObject).testCluster(cluster);
+                            }
                         }
                     })
         );
@@ -194,8 +191,19 @@ private void configureStartClustersHook(Project project) {
                 @Override
                 public void beforeActions(Task task) {
                     // we only start the cluster before the actions, so we'll not start it if the task is up-to-date
-                    usedClusters.getOrDefault(task, Collections.emptyList()).stream()
-                        .filter(each -> runningClusters.contains(each) == false)
+                    List<ElasticsearchCluster> neededButNotRunning = usedClusters.getOrDefault(
+                        task,
+                        Collections.emptyList()
+                    )
+                        .stream()
+                        .filter(cluster -> runningClusters.contains(cluster) == false)
+                        .collect(Collectors.toList());
+
+                    project.getRootProject().getExtensions()
+                        .getByType(TestClustersCleanupExtension.class)
+                        .getCleanupThread()
+                        .watch(neededButNotRunning);
+                    neededButNotRunning
                         .forEach(elasticsearchCluster -> {
                             elasticsearchCluster.start();
                             runningClusters.add(elasticsearchCluster);
@@ -218,22 +226,36 @@ public void afterExecute(Task task, TaskState state) {
                         task,
                         Collections.emptyList()
                     );
+                    if (clustersUsedByTask.isEmpty()) {
+                        return;
+                    }
+                    logger.info("Clusters were used, stopping and releasing permits");
+                    final int permitsToRelease;
                     if (state.getFailure() != null) {
                         // If the task fails, and other tasks use this cluster, the other task will likely never be
-                        // executed at all, so we will never get to un-claim and terminate it.
-                        clustersUsedByTask.forEach(each -> each.stop(true));
+                        // executed at all, so we will never be called again to un-claim and terminate it.
+                        clustersUsedByTask.forEach(cluster -> stopCluster(cluster, true));
+                        permitsToRelease = clustersUsedByTask.stream()
+                            .map(cluster -> cluster.getNumberOfNodes())
+                            .reduce(Integer::sum).get();
                     } else {
                         clustersUsedByTask.forEach(
-                            each -> claimsInventory.put(each, claimsInventory.getOrDefault(each, 0) - 1)
+                            cluster -> claimsInventory.put(cluster, claimsInventory.getOrDefault(cluster, 0) - 1)
                         );
-                        claimsInventory.entrySet().stream()
+                        List<ElasticsearchCluster> stoppingClusers = claimsInventory.entrySet().stream()
                             .filter(entry -> entry.getValue() == 0)
                             .filter(entry -> runningClusters.contains(entry.getKey()))
                             .map(Map.Entry::getKey)
-                            .forEach(each -> {
-                                each.stop(false);
-                                runningClusters.remove(each);
-                            });
+                            .collect(Collectors.toList());
+                        stoppingClusers.forEach(cluster -> {
+                            stopCluster(cluster, false);
+                            runningClusters.remove(cluster);
+                        });
+
+                        project.getRootProject().getExtensions()
+                            .getByType(TestClustersCleanupExtension.class)
+                            .getCleanupThread()
+                            .unWatch(stoppingClusers);
                     }
                 }
                 @Override
@@ -242,6 +264,28 @@ public void beforeExecute(Task task) {}
         );
     }
 
+    private void stopCluster(ElasticsearchCluster cluster, boolean taskFailed) {
+        if (allowClusterToSurvive) {
+            logger.info("Not stopping clusters, disabled by property");
+            if (taskFailed) {
+                // task failed or this is the last one to stop
+                for (int i=1 ; ; i += i) {
+                    logger.lifecycle(
+                        "No more test clusters left to run, going to sleep because {} was set," +
+                            " interrupt (^C) to stop clusters.", TESTCLUSTERS_INSPECT_FAILURE
+                    );
+                    try {
+                        Thread.sleep(1000 * i);
+                    } catch (InterruptedException e) {
+                        Thread.currentThread().interrupt();
+                        return;
+                    }
+                }
+            }
+        }
+        cluster.stop(taskFailed);
+    }
+
     /**
      * Boilerplate to get testClusters container extension
      *
@@ -250,7 +294,7 @@ public void beforeExecute(Task task) {}
     @SuppressWarnings("unchecked")
     public static NamedDomainObjectContainer<ElasticsearchCluster> getNodeExtension(Project project) {
         return (NamedDomainObjectContainer<ElasticsearchCluster>)
-            project.getExtensions().getByName(NODE_EXTENSION_NAME);
+            project.getExtensions().getByName(EXTENSION_NAME);
     }
 
     private static void autoConfigureClusterDependencies(
@@ -382,59 +426,4 @@ public void execute(Task task) {
             })));
     }
 
-    private void configureCleanupHooks(Project project) {
-        // When the Gradle daemon is used, it will interrupt all threads when the build concludes.
-        // This is our signal to clean up
-        executorService.submit(() -> {
-            while (true) {
-                try {
-                    Thread.sleep(Long.MAX_VALUE);
-                } catch (InterruptedException interrupted) {
-                    shutDownAllClusters();
-                    Thread.currentThread().interrupt();
-                    return;
-                }
-            }
-        });
-
-        // When the Daemon is not used, or runs into issues, rely on a shutdown hook
-        // When the daemon is used, but does not work correctly and eventually dies off (e.x. due to non interruptible
-        // thread in the build) process will be stopped eventually when the daemon dies.
-        Runtime.getRuntime().addShutdownHook(shutdownHook);
-
-        // When we don't run into anything out of the ordinary, and the build completes, makes sure to clean up
-        project.getGradle().buildFinished(buildResult -> {
-            shutdownExecutorService();
-            if (false == Runtime.getRuntime().removeShutdownHook(shutdownHook)) {
-                logger.info("Trying to deregister shutdown hook when it was not registered.");
-            }
-        });
-    }
-
-    private void shutdownExecutorService() {
-        executorService.shutdownNow();
-        try {
-            if (executorService.awaitTermination(EXECUTOR_SHUTDOWN_TIMEOUT, EXECUTOR_SHUTDOWN_TIMEOUT_UNIT) == false) {
-                throw new IllegalStateException(
-                    "Failed to shut down executor service after " +
-                    EXECUTOR_SHUTDOWN_TIMEOUT + " " + EXECUTOR_SHUTDOWN_TIMEOUT_UNIT
-                );
-            }
-        } catch (InterruptedException e) {
-            logger.info("Wait for testclusters shutdown interrupted", e);
-            Thread.currentThread().interrupt();
-        }
-    }
-
-    private void shutDownAllClusters() {
-        synchronized (runningClusters) {
-            Iterator<ElasticsearchCluster> iterator = runningClusters.iterator();
-            while (iterator.hasNext()) {
-                iterator.remove();
-                iterator.next().stop(true);
-            }
-        }
-    }
-
-
 }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java
index b930955236fb8..b9eaa2ab979c6 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java
@@ -22,9 +22,11 @@
 import com.avast.gradle.dockercompose.DockerComposePlugin;
 import com.avast.gradle.dockercompose.tasks.ComposeUp;
 import org.elasticsearch.gradle.OS;
+import org.elasticsearch.gradle.SystemPropertyCommandLineArgumentProvider;
 import org.elasticsearch.gradle.precommit.JarHellTask;
 import org.elasticsearch.gradle.precommit.TestingConventionsTasks;
 import org.elasticsearch.gradle.precommit.ThirdPartyAuditTask;
+import org.gradle.api.Action;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -34,6 +36,7 @@
 import org.gradle.api.tasks.TaskContainer;
 import org.gradle.api.tasks.testing.Test;
 
+import java.io.File;
 import java.util.Collections;
 import java.util.function.BiConsumer;
 
@@ -56,46 +59,47 @@ public void apply(Project project) {
             disableTaskByType(tasks, ThirdPartyAuditTask.class);
             disableTaskByType(tasks, JarHellTask.class);
 
+            // the project that defined a test fixture can also use it
+            extension.fixtures.add(project);
+
             Task buildFixture = project.getTasks().create("buildFixture");
             Task pullFixture = project.getTasks().create("pullFixture");
             Task preProcessFixture = project.getTasks().create("preProcessFixture");
             buildFixture.dependsOn(preProcessFixture);
             pullFixture.dependsOn(preProcessFixture);
             Task postProcessFixture = project.getTasks().create("postProcessFixture");
+            postProcessFixture.dependsOn(buildFixture);
+            preProcessFixture.onlyIf(spec -> buildFixture.getEnabled());
+            postProcessFixture.onlyIf(spec -> buildFixture.getEnabled());
 
-            if (dockerComposeSupported(project) == false) {
+            if (dockerComposeSupported() == false) {
                 preProcessFixture.setEnabled(false);
                 postProcessFixture.setEnabled(false);
                 buildFixture.setEnabled(false);
                 pullFixture.setEnabled(false);
-                return;
-            }
-            preProcessFixture.onlyIf(spec -> buildFixture.getEnabled());
-            postProcessFixture.onlyIf(spec -> buildFixture.getEnabled());
-
-            project.apply(spec -> spec.plugin(BasePlugin.class));
-            project.apply(spec -> spec.plugin(DockerComposePlugin.class));
-            ComposeExtension composeExtension = project.getExtensions().getByType(ComposeExtension.class);
-            composeExtension.setUseComposeFiles(Collections.singletonList(DOCKER_COMPOSE_YML));
-            composeExtension.setRemoveContainers(true);
-            composeExtension.setExecutable(
-                project.file("/usr/local/bin/docker-compose").exists() ?
-                    "/usr/local/bin/docker-compose" : "/usr/bin/docker-compose"
-            );
+            } else {
+                project.apply(spec -> spec.plugin(BasePlugin.class));
+                project.apply(spec -> spec.plugin(DockerComposePlugin.class));
+                ComposeExtension composeExtension = project.getExtensions().getByType(ComposeExtension.class);
+                composeExtension.setUseComposeFiles(Collections.singletonList(DOCKER_COMPOSE_YML));
+                composeExtension.setRemoveContainers(true);
+                composeExtension.setExecutable(
+                    project.file("/usr/local/bin/docker-compose").exists() ?
+                        "/usr/local/bin/docker-compose" : "/usr/bin/docker-compose"
+                );
 
-            buildFixture.dependsOn(tasks.getByName("composeUp"));
-            pullFixture.dependsOn(tasks.getByName("composePull"));
-            tasks.getByName("composeUp").mustRunAfter(preProcessFixture);
-            tasks.getByName("composePull").mustRunAfter(preProcessFixture);
-            postProcessFixture.dependsOn(buildFixture);
+                buildFixture.dependsOn(tasks.getByName("composeUp"));
+                pullFixture.dependsOn(tasks.getByName("composePull"));
+                tasks.getByName("composeUp").mustRunAfter(preProcessFixture);
+                tasks.getByName("composePull").mustRunAfter(preProcessFixture);
 
-            configureServiceInfoForTask(
-                postProcessFixture,
-                project,
-                (name, port) -> postProcessFixture.getExtensions()
-                    .getByType(ExtraPropertiesExtension.class).set(name, port)
-            );
-            extension.fixtures.add(project);
+                configureServiceInfoForTask(
+                    postProcessFixture,
+                    project,
+                    (name, port) -> postProcessFixture.getExtensions()
+                        .getByType(ExtraPropertiesExtension.class).set(name, port)
+                );
+            }
         }
 
         extension.fixtures
@@ -107,7 +111,7 @@ public void apply(Project project) {
         conditionTaskByType(tasks, extension, TestingConventionsTasks.class);
         conditionTaskByType(tasks, extension, ComposeUp.class);
 
-        if (dockerComposeSupported(project) == false) {
+        if (dockerComposeSupported() == false) {
             project.getLogger().warn(
                 "Tests for {} require docker-compose at /usr/local/bin/docker-compose or /usr/bin/docker-compose " +
                     "but none could be found so these will be skipped", project.getPath()
@@ -122,7 +126,8 @@ public void apply(Project project) {
                 configureServiceInfoForTask(
                     task,
                     fixtureProject,
-                    task::systemProperty
+                    (name, host) ->
+                        task.getExtensions().getByType(SystemPropertyCommandLineArgumentProvider.class).systemProperty(name, host)
                 );
                 task.dependsOn(fixtureProject.getTasks().getByName("postProcessFixture"));
             })
@@ -135,7 +140,9 @@ private void conditionTaskByType(TaskContainer tasks, TestFixtureExtension exten
             taskClass,
             task -> task.onlyIf(spec ->
                 extension.fixtures.stream()
-                    .anyMatch(fixtureProject -> fixtureProject.getTasks().getByName("buildFixture").getEnabled() == false) == false
+                    .anyMatch(fixtureProject ->
+                        fixtureProject.getTasks().getByName("buildFixture").getEnabled() == false
+                    ) == false
             )
         );
     }
@@ -143,37 +150,41 @@ private void conditionTaskByType(TaskContainer tasks, TestFixtureExtension exten
     private void configureServiceInfoForTask(Task task, Project fixtureProject, BiConsumer<String, Integer> consumer) {
         // Configure ports for the tests as system properties.
         // We only know these at execution time so we need to do it in doFirst
-        task.doFirst(theTask ->
-            fixtureProject.getExtensions().getByType(ComposeExtension.class).getServicesInfos()
-                .forEach((service, infos) -> {
-                    infos.getTcpPorts()
-                        .forEach((container, host) -> {
-                            String name = "test.fixtures." + service + ".tcp." + container;
-                            theTask.getLogger().info("port mapping property: {}={}", name, host);
-                            consumer.accept(
-                                name,
-                                host
-                            );
-                        });
-                    infos.getUdpPorts()
-                        .forEach((container, host) -> {
-                            String name = "test.fixtures." + service + ".udp." + container;
-                            theTask.getLogger().info("port mapping property: {}={}", name, host);
-                            consumer.accept(
-                                name,
-                                host
-                            );
-                        });
-                })
+        task.doFirst(new Action<Task>() {
+                         @Override
+                         public void execute(Task theTask) {
+                             fixtureProject.getExtensions().getByType(ComposeExtension.class).getServicesInfos()
+                                 .forEach((service, infos) -> {
+                                     infos.getTcpPorts()
+                                         .forEach((container, host) -> {
+                                             String name = "test.fixtures." + service + ".tcp." + container;
+                                             theTask.getLogger().info("port mapping property: {}={}", name, host);
+                                             consumer.accept(
+                                                 name,
+                                                 host
+                                             );
+                                         });
+                                     infos.getUdpPorts()
+                                         .forEach((container, host) -> {
+                                             String name = "test.fixtures." + service + ".udp." + container;
+                                             theTask.getLogger().info("port mapping property: {}={}", name, host);
+                                             consumer.accept(
+                                                 name,
+                                                 host
+                                             );
+                                         });
+                                 });
+                         }
+                     }
         );
     }
 
-    public boolean dockerComposeSupported(Project project) {
+    public static boolean dockerComposeSupported() {
         if (OS.current().equals(OS.WINDOWS)) {
             return false;
         }
-        final boolean hasDockerCompose = project.file("/usr/local/bin/docker-compose").exists() ||
-            project.file("/usr/bin/docker-compose").exists();
+        final boolean hasDockerCompose = (new File("/usr/local/bin/docker-compose")).exists() ||
+            (new File("/usr/bin/docker-compose").exists());
         return hasDockerCompose && Boolean.parseBoolean(System.getProperty("tests.fixture.enabled", "true"));
     }
 
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/tool/ClasspathUtils.java b/buildSrc/src/main/java/org/elasticsearch/gradle/tool/ClasspathUtils.java
new file mode 100644
index 0000000000000..40ec6bd71830a
--- /dev/null
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/tool/ClasspathUtils.java
@@ -0,0 +1,23 @@
+package org.elasticsearch.gradle.tool;
+
+public class ClasspathUtils {
+    private static boolean isElasticsearchProject;
+
+    static {
+        // look for buildSrc marker file, if it exists then we are running in the context of the elastic/elasticsearch build
+        isElasticsearchProject = ClasspathUtils.class.getResource("/buildSrc.marker") != null;
+    }
+
+    private ClasspathUtils() {
+    }
+
+    /**
+     * Determine if we are running in the context of the `elastic/elasticsearch` project. This method will return {@code false} when
+     * the build-tools project is pulled in as an external dependency.
+     *
+     * @return if we are currently running in the `elastic/elasticsearch` project
+     */
+    public static boolean isElasticsearchProject() {
+        return isElasticsearchProject;
+    }
+}
diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.messy-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distribution-download.properties
similarity index 91%
rename from buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.messy-test.properties
rename to buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distribution-download.properties
index 507a0f85a0468..f400ea1ad67f1 100644
--- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.messy-test.properties
+++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distribution-download.properties
@@ -17,4 +17,4 @@
 # under the License.
 #
 
-implementation-class=org.elasticsearch.gradle.test.MessyTestPlugin
+implementation-class=org.elasticsearch.gradle.DistributionDownloadPlugin
\ No newline at end of file
diff --git a/buildSrc/src/main/resources/buildSrc.marker b/buildSrc/src/main/resources/buildSrc.marker
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/buildSrc/src/main/resources/minimumGradleVersion b/buildSrc/src/main/resources/minimumGradleVersion
index 04edabda285a6..e5e7441d3e938 100644
--- a/buildSrc/src/main/resources/minimumGradleVersion
+++ b/buildSrc/src/main/resources/minimumGradleVersion
@@ -1 +1 @@
-5.4.1
\ No newline at end of file
+5.5
\ No newline at end of file
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/JdkJarHellCheck.java
similarity index 100%
rename from buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java
rename to buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/JdkJarHellCheck.java
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/LazyFileOutputStream.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LazyFileOutputStream.java
similarity index 100%
rename from buildSrc/src/main/java/org/elasticsearch/gradle/LazyFileOutputStream.java
rename to buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LazyFileOutputStream.java
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/LoggedExec.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java
similarity index 94%
rename from buildSrc/src/main/java/org/elasticsearch/gradle/LoggedExec.java
rename to buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java
index 0921c611895fa..89031916160b2 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/LoggedExec.java
+++ b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java
@@ -17,7 +17,6 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.UnsupportedEncodingException;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -79,7 +78,13 @@ public void setSpoolOutput(boolean spoolOutput) {
             };
         } else {
             out = new ByteArrayOutputStream();
-            outputLogger = logger -> logger.error(((ByteArrayOutputStream) out).toString(StandardCharsets.UTF_8));
+            outputLogger = logger -> {
+                try {
+                    logger.error(((ByteArrayOutputStream) out).toString("UTF-8"));
+                } catch (UnsupportedEncodingException e) {
+                    throw new RuntimeException(e);
+                }
+            };
         }
         setStandardOutput(out);
         setErrorOutput(out);
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Version.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/Version.java
similarity index 100%
rename from buildSrc/src/main/java/org/elasticsearch/gradle/Version.java
rename to buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/Version.java
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/VersionProperties.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/VersionProperties.java
similarity index 100%
rename from buildSrc/src/main/java/org/elasticsearch/gradle/VersionProperties.java
rename to buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/VersionProperties.java
diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java
new file mode 100644
index 0000000000000..d83de5f21739a
--- /dev/null
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import com.github.tomakehurst.wiremock.WireMockServer;
+import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
+import org.gradle.testkit.runner.BuildResult;
+import org.gradle.testkit.runner.GradleRunner;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.head;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+
+public class DistributionDownloadPluginIT extends GradleIntegrationTestCase {
+    // TODO: check reuse of root task across projects MOVE TO UNIT TEST
+    // TODO: future: check integ-test-zip to maven, snapshots to snapshot service for external project
+
+    public void testCurrent() throws Exception {
+        String projectName = ":distribution:archives:linux-tar";
+        assertExtractedDistro(VersionProperties.getElasticsearch(), "archive", "linux", null, null,
+            "tests.local_distro.config", "default",
+            "tests.local_distro.project", projectName);
+    }
+
+    public void testBwc() throws Exception {
+        assertExtractedDistro("1.1.0", "archive", "linux", null, null,
+            "tests.local_distro.config", "linux-tar",
+            "tests.local_distro.project", ":distribution:bwc:minor",
+            "tests.current_version", "2.0.0");
+    }
+
+    public void testReleased() throws Exception {
+        WireMockServer wireMock = new WireMockServer(0);
+        try {
+            final byte[] filebytes;
+            try (InputStream stream =
+                     Files.newInputStream(Paths.get("src/testKit/distribution-download/distribution/files/fake_elasticsearch.zip"))) {
+                filebytes = stream.readAllBytes();
+            }
+            String urlPath = "/downloads/elasticsearch/elasticsearch-1.0.0-windows-x86_64.zip";
+            wireMock.stubFor(head(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200)));
+            wireMock.stubFor(get(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200).withBody(filebytes)));
+            wireMock.start();
+
+            assertExtractedDistro("1.0.0", "archive", "windows", null, null,
+                "tests.download_service", wireMock.baseUrl());
+        } catch (Exception e) {
+            // for debugging
+            System.err.println("missed requests: " + wireMock.findUnmatchedRequests().getRequests());
+            throw e;
+        } finally {
+            wireMock.stop();
+        }
+    }
+
+    private void assertFileDistro(String version, String type, String platform, String flavor, Boolean bundledJdk,
+                                  String... sysProps) throws IOException {
+        List<String> finalSysProps = new ArrayList<>();
+        addDistroSysProps(finalSysProps, version, type, platform, flavor, bundledJdk);
+        finalSysProps.addAll(Arrays.asList(sysProps));
+        runBuild(":subproj:assertDistroFile", finalSysProps.toArray(new String[0]));
+    }
+
+    private void assertExtractedDistro(String version, String type, String platform, String flavor, Boolean bundledJdk,
+                                       String... sysProps) throws IOException {
+        List<String> finalSysProps = new ArrayList<>();
+        addDistroSysProps(finalSysProps, version, type, platform, flavor, bundledJdk);
+        finalSysProps.addAll(Arrays.asList(sysProps));
+        runBuild(":subproj:assertDistroExtracted", finalSysProps.toArray(new String[0]));
+    }
+
+    private BuildResult runBuild(String taskname, String... sysProps) throws IOException {
+        assert sysProps.length % 2 == 0;
+        List<String> args = new ArrayList<>();
+        args.add(taskname);
+        args.add("-Dlocal.repo.path=" + getLocalTestRepoPath());
+        for (int i = 0; i < sysProps.length; i += 2) {
+            args.add("-D" + sysProps[i] + "=" + sysProps[i + 1]);
+        }
+        args.add("-i");
+        GradleRunner runner = getGradleRunner("distribution-download").withArguments(args);
+
+        BuildResult result = runner.build();
+        System.out.println(result.getOutput());
+        return result;
+    }
+
+    private void addDistroSysProps(List<String> sysProps, String version, String type, String platform, String flavor, Boolean bundledJdk) {
+        if (version != null) {
+            sysProps.add("tests.distro.version");
+            sysProps.add(version);
+        }
+        if (type != null) {
+            sysProps.add("tests.distro.type");
+            sysProps.add(type);
+        }
+        if (platform != null) {
+            sysProps.add("tests.distro.platform");
+            sysProps.add(platform);
+        }
+        if (flavor != null) {
+            sysProps.add("tests.distro.flavor");
+            sysProps.add(flavor);
+        }
+        if (bundledJdk != null) {
+            sysProps.add("tests.distro.bundledJdk");
+            sysProps.add(bundledJdk.toString());
+        }
+    }
+}
diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginTests.java
new file mode 100644
index 0000000000000..69169d13d1977
--- /dev/null
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginTests.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor;
+import org.elasticsearch.gradle.ElasticsearchDistribution.Platform;
+import org.elasticsearch.gradle.ElasticsearchDistribution.Type;
+import org.elasticsearch.gradle.test.GradleUnitTestCase;
+import org.gradle.api.NamedDomainObjectContainer;
+import org.gradle.api.Project;
+import org.gradle.testfixtures.ProjectBuilder;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.TreeSet;
+
+import static org.hamcrest.core.StringContains.containsString;
+
+public class DistributionDownloadPluginTests extends GradleUnitTestCase {
+    private static Project rootProject;
+    private static Project archivesProject;
+    private static Project packagesProject;
+    private static Project bwcProject;
+
+    private static final Version BWC_MAJOR_VERSION = Version.fromString("2.0.0");
+    private static final Version BWC_MINOR_VERSION = Version.fromString("1.1.0");
+    private static final Version BWC_STAGED_VERSION = Version.fromString("1.0.0");
+    private static final Version BWC_BUGFIX_VERSION = Version.fromString("1.0.1");
+    private static final Version BWC_MAINTENANCE_VERSION = Version.fromString("0.90.1");
+    private static final BwcVersions BWC_MINOR =
+        new BwcVersions(new TreeSet<>(Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)), BWC_MAJOR_VERSION);
+    private static final BwcVersions BWC_STAGED =
+        new BwcVersions(new TreeSet<>(Arrays.asList(BWC_STAGED_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)), BWC_MAJOR_VERSION);
+    private static final BwcVersions BWC_BUGFIX =
+        new BwcVersions(new TreeSet<>(Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)), BWC_MAJOR_VERSION);
+    private static final BwcVersions BWC_MAINTENANCE =
+        new BwcVersions(new TreeSet<>(Arrays.asList(BWC_MAINTENANCE_VERSION, BWC_STAGED_VERSION, BWC_MINOR_VERSION)), BWC_MINOR_VERSION);
+
+    public void testVersionDefault() {
+        ElasticsearchDistribution distro = checkDistro(createProject(null),
+            "testdistro", null, Type.ARCHIVE, Platform.LINUX, Flavor.OSS, true);
+        assertEquals(distro.getVersion(), Version.fromString(VersionProperties.getElasticsearch()));
+    }
+
+    public void testBadVersionFormat() {
+        assertDistroError(createProject(null), "testdistro", "badversion", Type.ARCHIVE, Platform.LINUX, Flavor.OSS, true,
+            "Invalid version format: 'badversion'");
+    }
+
+    public void testTypeDefault() {
+        ElasticsearchDistribution distro = checkDistro(createProject(null),
+            "testdistro", "5.0.0", null, Platform.LINUX, Flavor.OSS, true);
+        assertEquals(distro.getType(), Type.ARCHIVE);
+    }
+
+    public void testPlatformDefault() {
+        ElasticsearchDistribution distro = checkDistro(createProject(null),
+            "testdistro", "5.0.0", Type.ARCHIVE, null, Flavor.OSS, true);
+        assertEquals(distro.getPlatform(), ElasticsearchDistribution.CURRENT_PLATFORM);
+    }
+
+    public void testPlatformForIntegTest() {
+        assertDistroError(createProject(null), "testdistro", "5.0.0", Type.INTEG_TEST_ZIP, Platform.LINUX, null, null,
+            "platform not allowed for elasticsearch distribution [testdistro]");
+    }
+
+    public void testFlavorDefault() {
+        ElasticsearchDistribution distro = checkDistro(createProject(null),
+            "testdistro", "5.0.0", Type.ARCHIVE, Platform.LINUX, null, true);
+        assertEquals(distro.getFlavor(), Flavor.DEFAULT);
+    }
+
+    public void testFlavorForIntegTest() {
+        assertDistroError(createProject(null),
+            "testdistro", "5.0.0", Type.INTEG_TEST_ZIP, null, Flavor.OSS, null,
+            "flavor not allowed for elasticsearch distribution [testdistro]");
+    }
+
+    public void testBundledJdkDefault() {
+        ElasticsearchDistribution distro = checkDistro(createProject(null),
+            "testdistro", "5.0.0", Type.ARCHIVE, Platform.LINUX, null, true);
+        assertTrue(distro.getBundledJdk());
+    }
+
+    public void testBundledJdkForIntegTest() {
+        assertDistroError(createProject(null), "testdistro", "5.0.0", Type.INTEG_TEST_ZIP, null, null, true,
+            "bundledJdk not allowed for elasticsearch distribution [testdistro]");
+    }
+
+    public void testCurrentVersionIntegTestZip() {
+        Project project = createProject(null);
+        Project archiveProject = ProjectBuilder.builder().withParent(archivesProject).withName("integ-test-zip").build();
+        archiveProject.getConfigurations().create("default");
+        archiveProject.getArtifacts().add("default", new File("doesnotmatter"));
+        createDistro(project, "distro",
+            VersionProperties.getElasticsearch(), Type.INTEG_TEST_ZIP, null, null, null);
+        checkPlugin(project);
+    }
+
+    public void testCurrentVersionArchives() {
+        for (Platform platform : Platform.values()) {
+            for (Flavor flavor : Flavor.values()) {
+                for (boolean bundledJdk : new boolean[] { true, false}) {
+                    // create a new project in each iteration, so that we know we are resolving the only additional project being created
+                    Project project = createProject(null);
+                    String projectName = projectName(platform.toString(), flavor, bundledJdk);
+                    projectName += (platform == Platform.WINDOWS ? "-zip" : "-tar");
+                    Project archiveProject = ProjectBuilder.builder().withParent(archivesProject).withName(projectName).build();
+                    archiveProject.getConfigurations().create("default");
+                    archiveProject.getArtifacts().add("default", new File("doesnotmatter"));
+                    createDistro(project, "distro",
+                        VersionProperties.getElasticsearch(), Type.ARCHIVE, platform, flavor, bundledJdk);
+                    checkPlugin(project);
+                }
+            }
+        }
+    }
+
+    public void testCurrentVersionPackages() {
+        for (Type packageType : new Type[] { Type.RPM, Type.DEB }) {
+            for (Flavor flavor : Flavor.values()) {
+                for (boolean bundledJdk : new boolean[] { true, false}) {
+                    Project project = createProject(null);
+                    String projectName = projectName(packageType.toString(), flavor, bundledJdk);
+                    Project packageProject = ProjectBuilder.builder().withParent(packagesProject).withName(projectName).build();
+                    packageProject.getConfigurations().create("default");
+                    packageProject.getArtifacts().add("default", new File("doesnotmatter"));
+                    createDistro(project, "distro",
+                        VersionProperties.getElasticsearch(), packageType, null, flavor, bundledJdk);
+                    checkPlugin(project);
+                }
+            }
+        }
+    }
+
+    public void testLocalBwcArchives() {
+        for (Platform platform : Platform.values()) {
+            for (Flavor flavor : Flavor.values()) {
+                // note: no non bundled jdk for bwc
+                String configName = projectName(platform.toString(), flavor, true);
+                configName += (platform == Platform.WINDOWS ? "-zip" : "-tar");
+
+                checkBwc("minor", configName, BWC_MINOR_VERSION, BWC_MINOR, Type.ARCHIVE, platform, flavor);
+                checkBwc("staged", configName, BWC_STAGED_VERSION, BWC_STAGED, Type.ARCHIVE, platform, flavor);
+                checkBwc("bugfix", configName, BWC_BUGFIX_VERSION, BWC_BUGFIX, Type.ARCHIVE, platform, flavor);
+                checkBwc("maintenance", configName, BWC_MAINTENANCE_VERSION, BWC_MAINTENANCE, Type.ARCHIVE, platform, flavor);
+            }
+        }
+    }
+
+    public void testLocalBwcPackages() {
+        for (Type packageType : new Type[] { Type.RPM, Type.DEB }) {
+            for (Flavor flavor : Flavor.values()) {
+                // note: no non bundled jdk for bwc
+                String configName = projectName(packageType.toString(), flavor, true);
+
+                checkBwc("minor", configName, BWC_MINOR_VERSION, BWC_MINOR, packageType, null, flavor);
+                checkBwc("staged", configName, BWC_STAGED_VERSION, BWC_STAGED, packageType, null, flavor);
+                checkBwc("bugfix", configName, BWC_BUGFIX_VERSION, BWC_BUGFIX, packageType, null, flavor);
+                checkBwc("maintenance", configName, BWC_MAINTENANCE_VERSION, BWC_MAINTENANCE, packageType, null, flavor);
+            }
+        }
+    }
+
+    private void assertDistroError(Project project, String name, String version, Type type, Platform platform,
+                                   Flavor flavor, Boolean bundledJdk, String message) {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> checkDistro(project, name, version, type, platform, flavor, bundledJdk));
+        assertThat(e.getMessage(), containsString(message));
+    }
+
+    private ElasticsearchDistribution createDistro(Project project, String name, String version, Type type,
+                              Platform platform, Flavor flavor, Boolean bundledJdk) {
+        @SuppressWarnings("unchecked")
+        NamedDomainObjectContainer<ElasticsearchDistribution> distros =
+            (NamedDomainObjectContainer<ElasticsearchDistribution>) project.getExtensions().getByName("elasticsearch_distributions");
+        return distros.create(name, distro -> {
+            if (version != null) {
+                distro.setVersion(version);
+            }
+            if (type != null) {
+                distro.setType(type);
+            }
+            if (platform != null) {
+                distro.setPlatform(platform);
+            }
+            if (flavor != null) {
+                distro.setFlavor(flavor);
+            }
+            if (bundledJdk != null) {
+                distro.setBundledJdk(bundledJdk);
+            }
+        });
+    }
+
+    // create a distro and finalize its configuration
+    private ElasticsearchDistribution checkDistro(Project project, String name, String version, Type type,
+                                                  Platform platform, Flavor flavor, Boolean bundledJdk) {
+        ElasticsearchDistribution distribution = createDistro(project, name, version, type, platform, flavor, bundledJdk);
+        distribution.finalizeValues();
+        return distribution;
+    }
+
+    // check the download plugin can be fully configured
+    private void checkPlugin(Project project) {
+        DistributionDownloadPlugin plugin = project.getPlugins().getPlugin(DistributionDownloadPlugin.class);
+        plugin.setupDistributions(project);
+    }
+
+    private void checkBwc(String projectName, String config, Version version, BwcVersions bwcVersions,
+                          Type type, Platform platform, Flavor flavor) {
+        Project project = createProject(bwcVersions);
+        Project archiveProject = ProjectBuilder.builder().withParent(bwcProject).withName(projectName).build();
+        archiveProject.getConfigurations().create(config);
+        archiveProject.getArtifacts().add(config, new File("doesnotmatter"));
+        createDistro(project, "distro", version.toString(), type, platform, flavor, true);
+        checkPlugin(project);
+    }
+
+    private Project createProject(BwcVersions bwcVersions) {
+        rootProject = ProjectBuilder.builder().build();
+        Project distributionProject = ProjectBuilder.builder().withParent(rootProject).withName("distribution").build();
+        archivesProject = ProjectBuilder.builder().withParent(distributionProject).withName("archives").build();
+        packagesProject = ProjectBuilder.builder().withParent(distributionProject).withName("packages").build();
+        bwcProject = ProjectBuilder.builder().withParent(distributionProject).withName("bwc").build();
+        Project project = ProjectBuilder.builder().withParent(rootProject).build();
+        project.getExtensions().getExtraProperties().set("bwcVersions", bwcVersions);
+        project.getPlugins().apply("elasticsearch.distribution-download");
+        return project;
+    }
+
+    private static String projectName(String base, Flavor flavor, boolean bundledJdk) {
+        String prefix = "";
+        if (flavor == Flavor.OSS) {
+            prefix += "oss-";
+        }
+        if (bundledJdk == false) {
+            prefix += "no-jdk-";
+        }
+
+        return prefix + base;
+    }
+}
diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/EmptyDirTaskTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/EmptyDirTaskTests.java
new file mode 100644
index 0000000000000..9cb49e8efd418
--- /dev/null
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/EmptyDirTaskTests.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.elasticsearch.gradle.test.GradleUnitTestCase;
+import org.gradle.api.Project;
+import org.gradle.testfixtures.ProjectBuilder;
+
+public class EmptyDirTaskTests extends GradleUnitTestCase {
+
+    public void testCreateEmptyDir() throws Exception {
+        Project project = ProjectBuilder.builder().build();
+        EmptyDirTask emptyDirTask = project.getTasks().create("emptyDirTask", EmptyDirTask.class);
+        assertEquals(0755, emptyDirTask.getDirMode());
+
+        // generate a new temporary folder and make sure it does not exists
+        File newEmptyFolder = getNewNonExistingTempFolderFile(project);
+
+        emptyDirTask.setDir(newEmptyFolder);
+        emptyDirTask.create();
+
+        assertTrue(newEmptyFolder.exists());
+        assertTrue(newEmptyFolder.isDirectory());
+        assertTrue(newEmptyFolder.canExecute());
+        assertTrue(newEmptyFolder.canRead());
+        assertTrue(newEmptyFolder.canWrite());
+
+        // cleanup
+        newEmptyFolder.delete();
+    }
+
+    public void testCreateEmptyDirNoPermissions() throws Exception {
+        Project project = ProjectBuilder.builder().build();
+        EmptyDirTask emptyDirTask = project.getTasks().create("emptyDirTask", EmptyDirTask.class);
+        emptyDirTask.setDirMode(0000);
+
+        // generate a new temporary folder and make sure it does not exists
+        File newEmptyFolder = getNewNonExistingTempFolderFile(project);
+
+        emptyDirTask.setDir(newEmptyFolder);
+        emptyDirTask.create();
+
+        assertTrue(newEmptyFolder.exists());
+        assertTrue(newEmptyFolder.isDirectory());
+        assertFalse(newEmptyFolder.canExecute());
+        assertFalse(newEmptyFolder.canRead());
+        assertFalse(newEmptyFolder.canWrite());
+
+        // cleanup
+        newEmptyFolder.delete();
+    }
+
+    private File getNewNonExistingTempFolderFile(Project project) throws IOException {
+        File newEmptyFolder = new File(project.getBuildDir(), "empty-dir");
+        assertFalse(newEmptyFolder.exists());
+        return newEmptyFolder;
+    }
+
+}
diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtensionTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtensionTests.java
new file mode 100644
index 0000000000000..b93d400f3e057
--- /dev/null
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtensionTests.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.plugin;
+
+import org.elasticsearch.gradle.test.GradleUnitTestCase;
+import org.gradle.api.Project;
+import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.testfixtures.ProjectBuilder;
+
+public class PluginPropertiesExtensionTests extends GradleUnitTestCase {
+
+    public void testCreatingPluginPropertiesExtensionWithNameAndVersion() {
+        String projectName = "Test";
+        String projectVersion = "5.0";
+
+        PluginPropertiesExtension pluginPropertiesExtension  =
+                new PluginPropertiesExtension(this.createProject(projectName, projectVersion));
+
+        assertEquals(projectName, pluginPropertiesExtension.getName());
+        assertEquals(projectVersion, pluginPropertiesExtension.getVersion());
+    }
+
+    public void testCreatingPluginPropertiesExtensionWithNameWithoutVersion() {
+        String projectName = "Test";
+
+        PluginPropertiesExtension pluginPropertiesExtension  =
+                new PluginPropertiesExtension(this.createProject(projectName, null));
+
+        assertEquals(projectName, pluginPropertiesExtension.getName());
+        assertEquals("unspecified", pluginPropertiesExtension.getVersion());
+    }
+
+    private Project createProject(String projectName, String version) {
+        Project project = ProjectBuilder.builder().withName(projectName).build();
+        project.setVersion(version);
+
+        project.getPlugins().apply(JavaPlugin.class);
+
+        return project;
+    }
+}
diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/DependencyLicensesTaskTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/DependencyLicensesTaskTests.java
new file mode 100644
index 0000000000000..397c5938fba23
--- /dev/null
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/DependencyLicensesTaskTests.java
@@ -0,0 +1,268 @@
+package org.elasticsearch.gradle.precommit;
+
+import org.elasticsearch.gradle.test.GradleUnitTestCase;
+import org.gradle.api.GradleException;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.file.FileCollection;
+import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.testfixtures.ProjectBuilder;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.security.NoSuchAlgorithmException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.containsString;
+
+public class DependencyLicensesTaskTests extends GradleUnitTestCase {
+
+    @Rule
+    public ExpectedException expectedException = ExpectedException.none();
+
+    private UpdateShasTask updateShas;
+
+    private DependencyLicensesTask task;
+
+    private Project project;
+
+    private Dependency dependency;
+
+    @Before
+    public void prepare() {
+        project = createProject();
+        task = createDependencyLicensesTask(project);
+        updateShas = createUpdateShasTask(project, task);
+        dependency = project.getDependencies().localGroovy();
+    }
+
+    @Test
+    public void givenProjectWithLicensesDirButNoDependenciesThenShouldThrowException() throws Exception {
+        expectedException.expect(GradleException.class);
+        expectedException.expectMessage(containsString("exists, but there are no dependencies"));
+
+        getLicensesDir(project).mkdir();
+        task.checkDependencies();
+    }
+
+    @Test
+    public void givenProjectWithoutLicensesDirButWithDependenciesThenShouldThrowException() throws Exception {
+        expectedException.expect(GradleException.class);
+        expectedException.expectMessage(containsString("does not exist, but there are dependencies"));
+
+        project.getDependencies().add("compile", dependency);
+        task.checkDependencies();
+    }
+
+    @Test
+    public void givenProjectWithoutLicensesDirNorDependenciesThenShouldReturnSilently() throws Exception {
+        task.checkDependencies();
+    }
+
+    @Test
+    public void givenProjectWithDependencyButNoShaFileThenShouldReturnException() throws Exception {
+        expectedException.expect(GradleException.class);
+        expectedException.expectMessage(containsString("Missing SHA for "));
+
+        File licensesDir = getLicensesDir(project);
+        createFileIn(licensesDir, "groovy-all-LICENSE.txt", "");
+        createFileIn(licensesDir, "groovy-all-NOTICE.txt", "");
+
+        project.getDependencies().add("compile", project.getDependencies().localGroovy());
+        task.checkDependencies();
+    }
+
+    @Test
+    public void givenProjectWithDependencyButNoLicenseFileThenShouldReturnException() throws Exception {
+        expectedException.expect(GradleException.class);
+        expectedException.expectMessage(containsString("Missing LICENSE for "));
+
+        project.getDependencies().add("compile", project.getDependencies().localGroovy());
+
+        getLicensesDir(project).mkdir();
+        updateShas.updateShas();
+        task.checkDependencies();
+    }
+
+    @Test
+    public void givenProjectWithDependencyButNoNoticeFileThenShouldReturnException() throws Exception {
+        expectedException.expect(GradleException.class);
+        expectedException.expectMessage(containsString("Missing NOTICE for "));
+
+        project.getDependencies().add("compile", dependency);
+
+        createFileIn(getLicensesDir(project), "groovy-all-LICENSE.txt", "");
+
+        updateShas.updateShas();
+        task.checkDependencies();
+    }
+
+    @Test
+    public void givenProjectWithDependencyAndEverythingInOrderThenShouldReturnSilently() throws Exception {
+        project.getDependencies().add("compile", dependency);
+
+        File licensesDir = getLicensesDir(project);
+
+        createAllDefaultDependencyFiles(licensesDir, "groovy-all");
+        task.checkDependencies();
+    }
+
+    @Test
+    public void givenProjectWithALicenseButWithoutTheDependencyThenShouldThrowException() throws Exception {
+        expectedException.expect(GradleException.class);
+        expectedException.expectMessage(containsString("Unused license "));
+
+        project.getDependencies().add("compile", dependency);
+
+        File licensesDir = getLicensesDir(project);
+        createAllDefaultDependencyFiles(licensesDir, "groovy-all");
+        createFileIn(licensesDir, "non-declared-LICENSE.txt", "");
+
+        task.checkDependencies();
+    }
+
+    @Test
+    public void givenProjectWithANoticeButWithoutTheDependencyThenShouldThrowException() throws Exception {
+        expectedException.expect(GradleException.class);
+        expectedException.expectMessage(containsString("Unused notice "));
+
+        project.getDependencies().add("compile", dependency);
+
+        File licensesDir = getLicensesDir(project);
+        createAllDefaultDependencyFiles(licensesDir, "groovy-all");
+        createFileIn(licensesDir, "non-declared-NOTICE.txt", "");
+
+        task.checkDependencies();
+    }
+
+    @Test
+    public void givenProjectWithAShaButWithoutTheDependencyThenShouldThrowException() throws Exception {
+        expectedException.expect(GradleException.class);
+        expectedException.expectMessage(containsString("Unused sha files found: \n"));
+
+        project.getDependencies().add("compile", dependency);
+
+        File licensesDir = getLicensesDir(project);
+        createAllDefaultDependencyFiles(licensesDir, "groovy-all");
+        createFileIn(licensesDir, "non-declared.sha1", "");
+
+        task.checkDependencies();
+    }
+
+    @Test
+    public void givenProjectWithADependencyWithWrongShaThenShouldThrowException() throws Exception {
+        expectedException.expect(GradleException.class);
+        expectedException.expectMessage(containsString("SHA has changed! Expected "));
+
+        project.getDependencies().add("compile", dependency);
+
+        File licensesDir = getLicensesDir(project);
+        createAllDefaultDependencyFiles(licensesDir, "groovy-all");
+
+        Path groovySha = Files
+            .list(licensesDir.toPath())
+            .filter(file -> file.toFile().getName().contains("sha"))
+            .findFirst().get();
+
+        Files.write(groovySha, new byte[] { 1 }, StandardOpenOption.CREATE);
+
+        task.checkDependencies();
+    }
+
+    @Test
+    public void givenProjectWithADependencyMappingThenShouldReturnSilently() throws Exception {
+        project.getDependencies().add("compile", dependency);
+
+        File licensesDir = getLicensesDir(project);
+        createAllDefaultDependencyFiles(licensesDir, "groovy");
+
+        Map<String, String> mappings = new HashMap<>();
+        mappings.put("from", "groovy-all");
+        mappings.put("to", "groovy");
+
+        task.mapping(mappings);
+        task.checkDependencies();
+    }
+
+    @Test
+    public void givenProjectWithAIgnoreShaConfigurationAndNoShaFileThenShouldReturnSilently() throws Exception {
+        project.getDependencies().add("compile", dependency);
+
+        File licensesDir = getLicensesDir(project);
+        createFileIn(licensesDir, "groovy-all-LICENSE.txt", "");
+        createFileIn(licensesDir, "groovy-all-NOTICE.txt", "");
+
+        task.ignoreSha("groovy-all");
+        task.checkDependencies();
+    }
+
+    @Test
+    public void givenProjectWithoutLicensesDirWhenAskingForShaFilesThenShouldThrowException() {
+        expectedException.expect(GradleException.class);
+        expectedException.expectMessage(containsString("isn't a valid directory"));
+
+        task.getShaFiles();
+    }
+
+    private Project createProject() {
+        Project project = ProjectBuilder.builder().build();
+        project.getPlugins().apply(JavaPlugin.class);
+
+        return project;
+    }
+
+    private void createAllDefaultDependencyFiles(File licensesDir, String dependencyName) throws IOException, NoSuchAlgorithmException {
+        createFileIn(licensesDir, dependencyName + "-LICENSE.txt", "");
+        createFileIn(licensesDir, dependencyName + "-NOTICE.txt", "");
+
+        updateShas.updateShas();
+    }
+
+    private File getLicensesDir(Project project) {
+        return getFile(project, "licenses");
+    }
+
+    private File getFile(Project project, String fileName) {
+        return project.getProjectDir().toPath().resolve(fileName).toFile();
+    }
+
+    private void createFileIn(File parent, String name, String content) throws IOException {
+        parent.mkdir();
+
+        Path file = parent.toPath().resolve(name);
+        file.toFile().createNewFile();
+
+        Files.write(file, content.getBytes(StandardCharsets.UTF_8));
+    }
+
+    private UpdateShasTask createUpdateShasTask(Project project, DependencyLicensesTask dependencyLicensesTask) {
+        UpdateShasTask task =  project.getTasks()
+            .register("updateShas", UpdateShasTask.class)
+            .get();
+
+        task.setParentTask(dependencyLicensesTask);
+        return task;
+    }
+
+    private DependencyLicensesTask createDependencyLicensesTask(Project project) {
+        DependencyLicensesTask task =  project.getTasks()
+            .register("dependencyLicenses", DependencyLicensesTask.class)
+            .get();
+
+        task.setDependencies(getDependencies(project));
+        return task;
+    }
+
+    private FileCollection getDependencies(Project project) {
+        return project.getConfigurations().getByName("compile");
+    }
+}
diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/UpdateShasTaskTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/UpdateShasTaskTests.java
new file mode 100644
index 0000000000000..62ac9600a8307
--- /dev/null
+++ b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/UpdateShasTaskTests.java
@@ -0,0 +1,140 @@
+package org.elasticsearch.gradle.precommit;
+
+import org.apache.commons.io.FileUtils;
+import org.elasticsearch.gradle.test.GradleUnitTestCase;
+import org.gradle.api.GradleException;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.file.FileCollection;
+import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.testfixtures.ProjectBuilder;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.security.NoSuchAlgorithmException;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class UpdateShasTaskTests extends GradleUnitTestCase {
+
+    @Rule
+    public ExpectedException expectedException = ExpectedException.none();
+
+    private UpdateShasTask task;
+
+    private Project project;
+
+    private Dependency dependency;
+
+    @Before
+    public void prepare() throws IOException {
+        project = createProject();
+        task = createUpdateShasTask(project);
+        dependency = project.getDependencies().localGroovy();
+
+    }
+
+    @Test
+    public void whenDependencyDoesntExistThenShouldDeleteDependencySha()
+        throws IOException, NoSuchAlgorithmException {
+
+        File unusedSha = createFileIn(getLicensesDir(project), "test.sha1", "");
+        task.updateShas();
+
+        assertFalse(unusedSha.exists());
+    }
+
+    @Test
+    public void whenDependencyExistsButShaNotThenShouldCreateNewShaFile()
+        throws IOException, NoSuchAlgorithmException {
+        project.getDependencies().add("compile", dependency);
+
+        getLicensesDir(project).mkdir();
+        task.updateShas();
+
+        Path groovySha = Files
+            .list(getLicensesDir(project).toPath())
+            .findFirst().get();
+
+        assertTrue(groovySha.toFile().getName().startsWith("groovy-all"));
+    }
+
+    @Test
+    public void whenDependencyAndWrongShaExistsThenShouldNotOverwriteShaFile()
+        throws IOException, NoSuchAlgorithmException {
+        project.getDependencies().add("compile", dependency);
+
+        File groovyJar = task.getParentTask().getDependencies().getFiles().iterator().next();
+        String groovyShaName = groovyJar.getName() + ".sha1";
+
+        File groovySha = createFileIn(getLicensesDir(project), groovyShaName, "content");
+        task.updateShas();
+
+        assertThat(FileUtils.readFileToString(groovySha), equalTo("content"));
+    }
+
+    @Test
+    public void whenLicensesDirDoesntExistThenShouldThrowException()
+        throws IOException, NoSuchAlgorithmException {
+        expectedException.expect(GradleException.class);
+        expectedException.expectMessage(containsString("isn't a valid directory"));
+
+        task.updateShas();
+    }
+
+    private Project createProject() {
+        Project project = ProjectBuilder.builder().build();
+        project.getPlugins().apply(JavaPlugin.class);
+
+        return project;
+    }
+
+    private File getLicensesDir(Project project) {
+        return getFile(project, "licenses");
+    }
+
+    private File getFile(Project project, String fileName) {
+        return project.getProjectDir().toPath().resolve(fileName).toFile();
+    }
+
+    private File createFileIn(File parent, String name, String content) throws IOException {
+        parent.mkdir();
+
+        Path path = parent.toPath().resolve(name);
+        File file = path.toFile();
+
+        Files.write(path, content.getBytes(), StandardOpenOption.CREATE);
+
+        return file;
+    }
+
+    private UpdateShasTask createUpdateShasTask(Project project) {
+        UpdateShasTask task =  project.getTasks()
+            .register("updateShas", UpdateShasTask.class)
+            .get();
+
+        task.setParentTask(createDependencyLicensesTask(project));
+        return task;
+    }
+
+    private DependencyLicensesTask createDependencyLicensesTask(Project project) {
+        DependencyLicensesTask task =  project.getTasks()
+            .register("dependencyLicenses", DependencyLicensesTask.class)
+            .get();
+
+        task.setDependencies(getDependencies(project));
+        return task;
+    }
+
+    private FileCollection getDependencies(Project project) {
+        return project.getConfigurations().getByName("compile");
+    }
+}
diff --git a/buildSrc/src/testKit/distribution-download/build.gradle b/buildSrc/src/testKit/distribution-download/build.gradle
new file mode 100644
index 0000000000000..82f76ff65acb7
--- /dev/null
+++ b/buildSrc/src/testKit/distribution-download/build.gradle
@@ -0,0 +1,44 @@
+import org.elasticsearch.gradle.BwcVersions
+import org.elasticsearch.gradle.Version
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// bring in build-tools onto the classpath
+plugins {
+    id 'elasticsearch.global-build-info' apply false
+}
+project.gradle.projectsEvaluated {
+    // wire the download service url to wiremock
+    String fakeDownloadService = System.getProperty('tests.download_service')
+    if (fakeDownloadService != null) {
+        println rootProject.repositories.asMap.keySet()
+        IvyArtifactRepository repository = (IvyArtifactRepository) rootProject.repositories.getByName("elasticsearch-downloads")
+        repository.setUrl(fakeDownloadService)
+        repository = (IvyArtifactRepository) project('subproj').repositories.getByName("elasticsearch-downloads")
+        repository.setUrl(fakeDownloadService)
+    }
+}
+
+Version currentVersion = Version.fromString("2.0.0")
+BwcVersions versions = new BwcVersions(new TreeSet<>(
+        Arrays.asList(Version.fromString("1.0.0"), Version.fromString("1.0.1"), Version.fromString("1.1.0"), currentVersion)),
+        currentVersion)
+allprojects {
+    ext.bwcVersions = versions
+}
\ No newline at end of file
diff --git a/buildSrc/src/testKit/distribution-download/distribution/build.gradle b/buildSrc/src/testKit/distribution-download/distribution/build.gradle
new file mode 100644
index 0000000000000..d0fdf038a602d
--- /dev/null
+++ b/buildSrc/src/testKit/distribution-download/distribution/build.gradle
@@ -0,0 +1,35 @@
+
+String distroConfig = System.getProperty('tests.local_distro.config')
+if (distroConfig != null) {
+  // setup the test distribution as an artifact of this project
+  String distroType = System.getProperty('tests.distro.type')
+
+  Task buildDistro
+  File buildFile
+  if (['rpm', 'deb'].contains(distroType)) {
+    buildDistro = project.tasks.create("build" + distroType.capitalize(), Copy) {
+      from 'files'
+      into 'build/files'
+      include 'fake_elasticsearch.tar.gz'
+      // this shouldn't be extracted so we just rename the file so it is a dummy package
+      rename { filename -> filename.replace('tar.gz', distroType) }
+    }
+    buildFile = project.file("build/files/fake_elasticsearch." + distroType)
+  } else {
+    String distroPlatform = System.getProperty('tests.distro.platform')
+    String extension = "tar.gz"
+    if (distroType == 'archive' && distroPlatform == 'windows' || distroType == 'integ-test-zip') {
+      extension = "zip"
+    }
+    // copy file as is
+    buildDistro = project.tasks.create("buildArchive", Copy) {
+      from 'files'
+      include "fake_elasticsearch.${extension}"
+      into 'build/files'
+    }
+    buildFile = project.file("build/files/fake_elasticsearch.${extension}")
+  }
+
+  configurations.create(distroConfig)
+  artifacts.add(distroConfig, [file: buildFile, builtBy: buildDistro])
+}
diff --git a/buildSrc/src/testKit/distribution-download/distribution/files/fake_elasticsearch.tar.gz b/buildSrc/src/testKit/distribution-download/distribution/files/fake_elasticsearch.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..a9c43bc71c5ce6ea28b33720cbbbfcf2ab49e474
GIT binary patch
literal 188
zcmV;t07L&DiwFSabnRRK1MSl73c?^5hT$EGC(v?!%;Q<Z(IWOk*ZC)H1i@B}BpBV-
zpI1cm)klAA;@G#*H=#Wrs#2BuSY#%pbk>STt*0uLnd(eb+BmJ8QqBlj*~*zh*xZdQ
zxy7*wsg-c?VTj}Ey0on8-t05q<5_pYFXVsfhpc~KOa3~IQ~qg8{=cx6|I7TBXQcP2
qjoFaDlE&IX$SgPf;Lr0100000000000002MzTE*MbZ*7~C;$N2s$rf0

literal 0
HcmV?d00001

diff --git a/buildSrc/src/testKit/distribution-download/distribution/files/fake_elasticsearch.zip b/buildSrc/src/testKit/distribution-download/distribution/files/fake_elasticsearch.zip
new file mode 100644
index 0000000000000000000000000000000000000000..42b572e37f3fbfbb90dee93ecbff745d37988879
GIT binary patch
literal 568
zcmWIWW@h1H00E1zJ$_&Yln`N%VMxtMEH24RE>2A>O3u(V)HBjE)(;KgWMJM?@;2u7
zG^W_n3T_5QmamKq3@jo*eF1ROtk6u8z+qZaW*)@E1EiX$ipNA0cY_@aGJYa92QxCs
zG2;pV320CVFuZjHF;R?#2L~%8I52_&VKQc5Ae#&db!;X>f(B?h$n7MUuF3#2oq=IV
e;{rUUqu3f6g3BsaHlP^{%s{9NjOhO$4g&zPL2_UK

literal 0
HcmV?d00001

diff --git a/buildSrc/src/testKit/distribution-download/settings.gradle b/buildSrc/src/testKit/distribution-download/settings.gradle
new file mode 100644
index 0000000000000..1a64261480d6f
--- /dev/null
+++ b/buildSrc/src/testKit/distribution-download/settings.gradle
@@ -0,0 +1,8 @@
+include 'subproj'
+
+String distroProject = System.getProperty('tests.local_distro.project')
+System.out.println("local distro project: " + distroProject);
+if (distroProject != null) {
+  include distroProject
+  project(distroProject).projectDir = new File(rootDir, 'distribution')
+}
diff --git a/buildSrc/src/testKit/distribution-download/subproj/build.gradle b/buildSrc/src/testKit/distribution-download/subproj/build.gradle
new file mode 100644
index 0000000000000..7584ce9a76d1c
--- /dev/null
+++ b/buildSrc/src/testKit/distribution-download/subproj/build.gradle
@@ -0,0 +1,58 @@
+
+plugins {
+    id 'elasticsearch.distribution-download'
+}
+
+String distroVersion = System.getProperty('tests.distro.version')
+String distroType = System.getProperty('tests.distro.type')
+String distroPlatform = System.getProperty('tests.distro.platform')
+String distroFlavor = System.getProperty('tests.distro.flavor')
+String distroBundledJdk = System.getProperty('tests.distro.bundledJdk')
+
+elasticsearch_distributions {
+  test_distro {
+    if (distroVersion != null) {
+      version = distroVersion
+    }
+    if (distroType != null) {
+      type = distroType
+    }
+    if (distroPlatform != null) {
+      platform = distroPlatform
+    }
+    if (distroFlavor != null) {
+      flavor = distroFlavor
+    }
+    if (distroBundledJdk != null) {
+      bundledJdk = Boolean.parseBoolean(distroBundledJdk)
+    }
+  }
+}
+
+task assertDistroFile {
+  dependsOn elasticsearch_distributions.test_distro
+  doLast {
+    File distroFile = new File(elasticsearch_distributions.test_distro.toString())
+    if (distroFile.exists() == false) {
+      throw new GradleException("distro file does not exist: ${distroFile}")
+    }
+    if (distroFile.isFile() == false) {
+      throw new GradleException("distro file is not a regular file: ${distroFile}")
+    }
+  }
+}
+
+if (['rpm', 'deb'].contains(distroType) == false) {
+  task assertDistroExtracted {
+    dependsOn elasticsearch_distributions.test_distro.extracted, assertDistroFile
+    doLast {
+      File distroExtracted = new File(elasticsearch_distributions.test_distro.extracted.toString())
+      if (distroExtracted.exists() == false) {
+        throw new GradleException("extracted does not exist: ${distroExtracted}")
+      }
+      if (distroExtracted.isDirectory() == false) {
+        throw new GradleException("extracted distro is not a directory: ${distroExtracted}")
+      }
+    }
+  }
+}
diff --git a/client/benchmark/README.md b/client/benchmark/README.md
index 2562a139bec0d..31ac1ea8ae548 100644
--- a/client/benchmark/README.md
+++ b/client/benchmark/README.md
@@ -54,9 +54,9 @@ Example invocation:
 
 The parameters are in order:
 
-* Client type: Use either "rest" or "transport"
+* Client type: Always "rest"
 * Benchmark type: Use either "bulk" or "search"
 * Benchmark target host IP (the host where Elasticsearch is running)
 * name of the index
-* a search request body (remember to escape double quotes). The `TransportClientBenchmark` uses `QueryBuilders.wrapperQuery()` internally which automatically adds a root key `query`, so it must not be present in the command line parameter.
+* a search request body (remember to escape double quotes).
 * A comma-separated list of target throughput rates
diff --git a/client/benchmark/build.gradle b/client/benchmark/build.gradle
index a53f102034009..41e166558d522 100644
--- a/client/benchmark/build.gradle
+++ b/client/benchmark/build.gradle
@@ -34,16 +34,10 @@ test.enabled = false
 dependencies {
   compile 'org.apache.commons:commons-math3:3.2'
 
-  compile("org.elasticsearch.client:elasticsearch-rest-client:${version}")
+  compile project(':server')
+  compile project(':client:rest')
   // bottleneck should be the client, not Elasticsearch
   compile project(path: ':client:client-benchmark-noop-api-plugin')
-  // for transport client
-  compile("org.elasticsearch:elasticsearch:${version}")
-  compile("org.elasticsearch.client:transport:${version}")
-  compile project(path: ':modules:transport-netty4', configuration: 'runtime')
-  compile project(path: ':modules:reindex', configuration: 'runtime')
-  compile project(path: ':modules:lang-mustache', configuration: 'runtime')
-  compile project(path: ':modules:percolator', configuration: 'runtime')
 }
 
 // No licenses for our benchmark deps (we don't ship benchmarks)
diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkMain.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkMain.java
index 317f0bf479779..d1363c0811190 100644
--- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkMain.java
+++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/BenchmarkMain.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.client.benchmark;
 
 import org.elasticsearch.client.benchmark.rest.RestClientBenchmark;
-import org.elasticsearch.client.benchmark.transport.TransportClientBenchmark;
 import org.elasticsearch.common.SuppressForbidden;
 
 import java.util.Arrays;
@@ -30,9 +29,6 @@ public static void main(String[] args) throws Exception {
         String type = args[0];
         AbstractBenchmark<?> benchmark = null;
         switch (type) {
-            case "transport":
-                benchmark = new TransportClientBenchmark();
-                break;
             case "rest":
                 benchmark = new RestClientBenchmark();
                 break;
diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java
deleted file mode 100644
index 66b3d7bf060ad..0000000000000
--- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.client.benchmark.transport;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.benchmark.AbstractBenchmark;
-import org.elasticsearch.client.benchmark.ops.bulk.BulkRequestExecutor;
-import org.elasticsearch.client.benchmark.ops.search.SearchRequestExecutor;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.plugin.noop.NoopPlugin;
-import org.elasticsearch.plugin.noop.action.bulk.NoopBulkAction;
-import org.elasticsearch.plugin.noop.action.search.NoopSearchAction;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.transport.client.PreBuiltTransportClient;
-
-import java.net.InetAddress;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-
-public final class TransportClientBenchmark extends AbstractBenchmark<TransportClient> {
-    public static void main(String[] args) throws Exception {
-        TransportClientBenchmark benchmark = new TransportClientBenchmark();
-        benchmark.run(args);
-    }
-
-    @Override
-    protected TransportClient client(String benchmarkTargetHost) throws Exception {
-        TransportClient client = new PreBuiltTransportClient(Settings.EMPTY, NoopPlugin.class);
-        client.addTransportAddress(new TransportAddress(InetAddress.getByName(benchmarkTargetHost), 9300));
-        return client;
-    }
-
-    @Override
-    protected BulkRequestExecutor bulkRequestExecutor(TransportClient client, String indexName, String typeName) {
-        return new TransportBulkRequestExecutor(client, indexName, typeName);
-    }
-
-    @Override
-    protected SearchRequestExecutor searchRequestExecutor(TransportClient client, String indexName) {
-        return new TransportSearchRequestExecutor(client, indexName);
-    }
-
-    private static final class TransportBulkRequestExecutor implements BulkRequestExecutor {
-        private final TransportClient client;
-        private final String indexName;
-        private final String typeName;
-
-        TransportBulkRequestExecutor(TransportClient client, String indexName, String typeName) {
-            this.client = client;
-            this.indexName = indexName;
-            this.typeName = typeName;
-        }
-
-        @Override
-        public boolean bulkIndex(List<String> bulkData) {
-            BulkRequest bulkRequest = new BulkRequest();
-            for (String bulkItem : bulkData) {
-                bulkRequest.add(new IndexRequest(indexName, typeName).source(bulkItem.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
-            }
-            BulkResponse bulkResponse;
-            try {
-                bulkResponse = client.execute(NoopBulkAction.INSTANCE, bulkRequest).get();
-            } catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-                return false;
-            } catch (ExecutionException e) {
-                throw new ElasticsearchException(e);
-            }
-            return !bulkResponse.hasFailures();
-        }
-    }
-
-    private static final class TransportSearchRequestExecutor implements SearchRequestExecutor {
-        private final TransportClient client;
-        private final String indexName;
-
-        private TransportSearchRequestExecutor(TransportClient client, String indexName) {
-            this.client = client;
-            this.indexName = indexName;
-        }
-
-        @Override
-        public boolean search(String source) {
-            final SearchResponse response;
-            try {
-                final SearchRequest searchRequest = new SearchRequest(indexName);
-                SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
-                searchRequest.source(searchSourceBuilder);
-                searchSourceBuilder.query(QueryBuilders.wrapperQuery(source));
-                response = client.execute(NoopSearchAction.INSTANCE, searchRequest).get();
-                return response.status() == RestStatus.OK;
-            } catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-                return false;
-            } catch (ExecutionException e) {
-                throw new ElasticsearchException(e);
-            }
-        }
-    }
-}
diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkAction.java
index 2bfd3b0cc8ed4..e6412099fee72 100644
--- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkAction.java
+++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkAction.java
@@ -18,10 +18,10 @@
  */
 package org.elasticsearch.plugin.noop.action.bulk;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.bulk.BulkResponse;
 
-public class NoopBulkAction extends Action<BulkResponse> {
+public class NoopBulkAction extends StreamableResponseActionType<BulkResponse> {
     public static final String NAME = "mock:data/write/bulk";
 
     public static final NoopBulkAction INSTANCE = new NoopBulkAction();
diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/NoopSearchAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/NoopSearchAction.java
index 9b390e1ffddbc..fb83bda148b11 100644
--- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/NoopSearchAction.java
+++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/NoopSearchAction.java
@@ -18,11 +18,11 @@
  */
 package org.elasticsearch.plugin.noop.action.search;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.io.stream.Writeable;
 
-public class NoopSearchAction extends Action<SearchResponse> {
+public class NoopSearchAction extends ActionType<SearchResponse> {
     public static final NoopSearchAction INSTANCE = new NoopSearchAction();
     public static final String NAME = "mock:data/read/search";
 
@@ -30,11 +30,6 @@ private NoopSearchAction() {
         super(NAME);
     }
 
-    @Override
-    public SearchResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<SearchResponse> getResponseReader() {
         return SearchResponse::new;
diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle
index c9a3fc486f1da..0e05eb567687a 100644
--- a/client/rest-high-level/build.gradle
+++ b/client/rest-high-level/build.gradle
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'nebula.maven-base-publish'
@@ -46,32 +47,34 @@ idea {
 }
 
 dependencies {
-  /*
-   * Everything in the "shadow" configuration is *not* copied into the
-   * shadowJar.
-   */
-  compile "org.elasticsearch:elasticsearch:${version}"
-  compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
-  compile "org.elasticsearch.plugin:parent-join-client:${version}"
-  compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
-  compile "org.elasticsearch.plugin:rank-eval-client:${version}"
-  compile "org.elasticsearch.plugin:lang-mustache-client:${version}"
+  compile project(':server')
+  compile project(':client:rest')
+  compile project(':modules:parent-join')
+  compile project(':modules:aggs-matrix-stats')
+  compile project(':modules:rank-eval')
+  compile project(':modules:lang-mustache')
+  shadow project(':server')
+  shadow project(':client:rest')
+  bundle project(':modules:parent-join')
+  bundle project(':modules:aggs-matrix-stats')
+  bundle project(':modules:rank-eval')
+  bundle project(':modules:lang-mustache')
 
-  testCompile "org.elasticsearch.client:test:${version}"
-  testCompile "org.elasticsearch.test:framework:${version}"
+  testCompile project(':client:test')
+  testCompile project(':test:framework')
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   //this is needed to make RestHighLevelClientTests#testApiNamingConventions work from IDEs
-  testCompile "org.elasticsearch:rest-api-spec:${version}"
+  testCompile project(":rest-api-spec")
   // Needed for serialization tests:
   // (In order to serialize a server side class to a client side class or the other way around)
   if (isEclipse == false || project.path == ":client:rest-high-level-tests") {
-    testCompile("org.elasticsearch.plugin:x-pack-core:${version}") {
+    testCompile(project(':x-pack:plugin:core')) {
         exclude group: 'org.elasticsearch', module: 'elasticsearch-rest-high-level-client'
     }
   }
 
-  restSpec "org.elasticsearch:rest-api-spec:${version}"
+  restSpec project(':rest-api-spec')
 }
 
 //we need to copy the yaml spec so we can check naming (see RestHighlevelClientTests#testApiNamingConventions)
@@ -113,14 +116,15 @@ if (isEclipse) {
 File nodeCert = file("./testnode.crt")
 File nodeTrustStore = file("./testnode.jks")
 
-integTestRunner {
+integTest.runner {
   systemProperty 'tests.rest.cluster.username', System.getProperty('tests.rest.cluster.username', 'test_user')
   systemProperty 'tests.rest.cluster.password', System.getProperty('tests.rest.cluster.password', 'test-password')
 }
 
-integTestCluster {
+testClusters.integTest {
+  distribution = "DEFAULT"
   systemProperty 'es.scripting.update.ctx_in_params', 'false'
-  setting 'reindex.remote.whitelist', ['"[::1]:*"', '"127.0.0.1:*"']
+  setting 'reindex.remote.whitelist', '[ "[::1]:*", "127.0.0.1:*" ]'
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.security.authc.token.enabled', 'true'
@@ -129,22 +133,10 @@ integTestCluster {
   setting 'xpack.security.http.ssl.certificate_authorities', 'testnode.crt'
   setting 'xpack.security.transport.ssl.truststore.path', 'testnode.jks'
   setting 'indices.lifecycle.poll_interval', '1000ms'
-  keystoreSetting 'xpack.security.transport.ssl.truststore.secure_password', 'testnode'
-  setupCommand 'setupDummyUser',
-          'bin/elasticsearch-users',
-          'useradd', System.getProperty('tests.rest.cluster.username', 'test_user'),
-          '-p', System.getProperty('tests.rest.cluster.password', 'test-password'),
-          '-r', 'superuser'
+  keystore 'xpack.security.transport.ssl.truststore.secure_password', 'testnode'
+  user username:  System.getProperty('tests.rest.cluster.username', 'test_user'),
+          password:  System.getProperty('tests.rest.cluster.password', 'test-password')
+
   extraConfigFile nodeCert.name, nodeCert
   extraConfigFile nodeTrustStore.name, nodeTrustStore
-  waitCondition = { node, ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-            dest: tmpFile.toString(),
-            username: System.getProperty('tests.rest.cluster.username', 'test_user'),
-            password: System.getProperty('tests.rest.cluster.password', 'test-password'),
-            ignoreerrors: true,
-            retries: 10)
-    return tmpFile.exists()
-  }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java
index a3f5d7e79fda7..8272e5d73bb4f 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java
@@ -48,9 +48,10 @@ static Request putFollow(PutFollowRequest putFollowRequest) throws IOException {
             .addPathPartAsIs("_ccr", "follow")
             .build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withWaitForActiveShards(putFollowRequest.waitForActiveShards());
         request.setEntity(createEntity(putFollowRequest, REQUEST_BODY_CONTENT_TYPE));
+        request.addParameters(parameters.asMap());
         return request;
     }
 
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterRequestConverters.java
index 4da8d128b98d1..a246402b505cc 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterRequestConverters.java
@@ -36,22 +36,21 @@ private ClusterRequestConverters() {}
     static Request clusterPutSettings(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest) throws IOException {
         Request request = new Request(HttpPut.METHOD_NAME, "/_cluster/settings");
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(clusterUpdateSettingsRequest.timeout());
         parameters.withMasterTimeout(clusterUpdateSettingsRequest.masterNodeTimeout());
-
+        request.addParameters(parameters.asMap());
         request.setEntity(RequestConverters.createEntity(clusterUpdateSettingsRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
 
     static Request clusterGetSettings(ClusterGetSettingsRequest clusterGetSettingsRequest) throws IOException {
         Request request = new Request(HttpGet.METHOD_NAME, "/_cluster/settings");
-
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withLocal(clusterGetSettingsRequest.local());
         parameters.withIncludeDefaults(clusterGetSettingsRequest.includeDefaults());
         parameters.withMasterTimeout(clusterGetSettingsRequest.masterNodeTimeout());
-
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -63,7 +62,7 @@ static Request clusterHealth(ClusterHealthRequest healthRequest) {
             .build();
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
 
-        new RequestConverters.Params(request)
+        RequestConverters.Params params = new RequestConverters.Params()
             .withWaitForStatus(healthRequest.waitForStatus())
             .withWaitForNoRelocatingShards(healthRequest.waitForNoRelocatingShards())
             .withWaitForNoInitializingShards(healthRequest.waitForNoInitializingShards())
@@ -74,6 +73,7 @@ static Request clusterHealth(ClusterHealthRequest healthRequest) {
             .withMasterTimeout(healthRequest.masterNodeTimeout())
             .withLocal(healthRequest.local())
             .withLevel(healthRequest.level());
+        request.addParameters(params.asMap());
         return request;
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/DataFrameRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/DataFrameRequestConverters.java
index 0e887994da7bf..18dfc2305575d 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/DataFrameRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/DataFrameRequestConverters.java
@@ -37,6 +37,7 @@
 
 import static org.elasticsearch.client.RequestConverters.REQUEST_BODY_CONTENT_TYPE;
 import static org.elasticsearch.client.RequestConverters.createEntity;
+import static org.elasticsearch.client.dataframe.GetDataFrameTransformRequest.ALLOW_NO_MATCH;
 
 final class DataFrameRequestConverters {
 
@@ -64,6 +65,9 @@ static Request getDataFrameTransform(GetDataFrameTransformRequest getRequest) {
         if (getRequest.getPageParams() != null && getRequest.getPageParams().getSize() != null) {
             request.addParameter(PageParams.SIZE.getPreferredName(), getRequest.getPageParams().getSize().toString());
         }
+        if (getRequest.getAllowNoMatch() != null) {
+            request.addParameter(ALLOW_NO_MATCH, getRequest.getAllowNoMatch().toString());
+        }
         return request;
     }
 
@@ -82,28 +86,33 @@ static Request startDataFrameTransform(StartDataFrameTransformRequest startReque
                 .addPathPartAsIs("_start")
                 .build();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         if (startRequest.getTimeout() != null) {
             params.withTimeout(startRequest.getTimeout());
         }
+        request.addParameters(params.asMap());
         return request;
     }
 
     static Request stopDataFrameTransform(StopDataFrameTransformRequest stopRequest) {
-            String endpoint = new RequestConverters.EndpointBuilder()
-                    .addPathPartAsIs("_data_frame", "transforms")
-                    .addPathPart(stopRequest.getId())
-                    .addPathPartAsIs("_stop")
-                    .build();
-            Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-            RequestConverters.Params params = new RequestConverters.Params(request);
-            if (stopRequest.getWaitForCompletion() != null) {
-                params.withWaitForCompletion(stopRequest.getWaitForCompletion());
-            }
-            if (stopRequest.getTimeout() != null) {
-                params.withTimeout(stopRequest.getTimeout());
-            }
-            return request;
+        String endpoint = new RequestConverters.EndpointBuilder()
+            .addPathPartAsIs("_data_frame", "transforms")
+            .addPathPart(stopRequest.getId())
+            .addPathPartAsIs("_stop")
+            .build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        RequestConverters.Params params = new RequestConverters.Params();
+        if (stopRequest.getWaitForCompletion() != null) {
+            params.withWaitForCompletion(stopRequest.getWaitForCompletion());
+        }
+        if (stopRequest.getTimeout() != null) {
+            params.withTimeout(stopRequest.getTimeout());
+        }
+        if (stopRequest.getAllowNoMatch() != null) {
+            request.addParameter(ALLOW_NO_MATCH, stopRequest.getAllowNoMatch().toString());
+        }
+        request.addParameters(params.asMap());
+        return request;
     }
 
     static Request previewDataFrameTransform(PreviewDataFrameTransformRequest previewRequest) throws IOException {
@@ -128,6 +137,9 @@ static Request getDataFrameTransformStats(GetDataFrameTransformStatsRequest stat
         if (statsRequest.getPageParams() != null && statsRequest.getPageParams().getSize() != null) {
             request.addParameter(PageParams.SIZE.getPreferredName(), statsRequest.getPageParams().getSize().toString());
         }
+        if (statsRequest.getAllowNoMatch() != null) {
+            request.addParameter(ALLOW_NO_MATCH, statsRequest.getAllowNoMatch().toString());
+        }
         return request;
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndexLifecycleRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndexLifecycleRequestConverters.java
index ff6d8a90cdb2d..f39f2b36cebc0 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndexLifecycleRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndexLifecycleRequestConverters.java
@@ -44,9 +44,10 @@ static Request getLifecyclePolicy(GetLifecyclePolicyRequest getLifecyclePolicyRe
         String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_ilm/policy")
                 .addCommaSeparatedPathParts(getLifecyclePolicyRequest.getPolicyNames()).build();
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withMasterTimeout(getLifecyclePolicyRequest.masterNodeTimeout());
         params.withTimeout(getLifecyclePolicyRequest.timeout());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -56,9 +57,10 @@ static Request putLifecyclePolicy(PutLifecyclePolicyRequest putLifecycleRequest)
             .addPathPartAsIs(putLifecycleRequest.getName())
             .build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withMasterTimeout(putLifecycleRequest.masterNodeTimeout());
         params.withTimeout(putLifecycleRequest.timeout());
+        request.addParameters(params.asMap());
         request.setEntity(RequestConverters.createEntity(putLifecycleRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -69,9 +71,10 @@ static Request deleteLifecyclePolicy(DeleteLifecyclePolicyRequest deleteLifecycl
                 .addPathPartAsIs("_ilm/policy")
                 .addPathPartAsIs(deleteLifecyclePolicyRequest.getLifecyclePolicy())
                 .build());
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withMasterTimeout(deleteLifecyclePolicyRequest.masterNodeTimeout());
         params.withTimeout(deleteLifecyclePolicyRequest.timeout());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -83,9 +86,10 @@ static Request removeIndexLifecyclePolicy(RemoveIndexLifecyclePolicyRequest remo
                         .addCommaSeparatedPathParts(indices)
                         .addPathPartAsIs("_ilm", "remove")
                         .build());
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withIndicesOptions(removePolicyRequest.indicesOptions());
         params.withMasterTimeout(removePolicyRequest.masterNodeTimeout());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -95,9 +99,10 @@ static Request startILM(StartILMRequest startILMRequest) {
                 .addPathPartAsIs("_ilm")
                 .addPathPartAsIs("start")
             .build());
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withMasterTimeout(startILMRequest.masterNodeTimeout());
         params.withTimeout(startILMRequest.timeout());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -107,9 +112,10 @@ static Request stopILM(StopILMRequest stopILMRequest) {
                 .addPathPartAsIs("_ilm")
                 .addPathPartAsIs("stop")
             .build());
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withMasterTimeout(stopILMRequest.masterNodeTimeout());
         params.withTimeout(stopILMRequest.timeout());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -119,9 +125,10 @@ static Request lifecycleManagementStatus(LifecycleManagementStatusRequest lifecy
                 .addPathPartAsIs("_ilm")
                 .addPathPartAsIs("status")
             .build());
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withMasterTimeout(lifecycleManagementStatusRequest.masterNodeTimeout());
         params.withTimeout(lifecycleManagementStatusRequest.timeout());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -132,9 +139,10 @@ static Request explainLifecycle(ExplainLifecycleRequest explainLifecycleRequest)
                 .addPathPartAsIs("_ilm")
                 .addPathPartAsIs("explain")
             .build());
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withIndicesOptions(explainLifecycleRequest.indicesOptions());
         params.withMasterTimeout(explainLifecycleRequest.masterNodeTimeout());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -145,9 +153,10 @@ static Request retryLifecycle(RetryLifecyclePolicyRequest retryLifecyclePolicyRe
                 .addPathPartAsIs("_ilm")
                 .addPathPartAsIs("retry")
                 .build());
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withMasterTimeout(retryLifecyclePolicyRequest.masterNodeTimeout());
         params.withTimeout(retryLifecyclePolicyRequest.timeout());
+        request.addParameters(params.asMap());
         return request;
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java
index cbb1d95feae1b..584bdad745026 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java
@@ -22,8 +22,6 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
@@ -47,6 +45,8 @@
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.core.ShardsAcknowledgedResponse;
+import org.elasticsearch.client.indices.AnalyzeRequest;
+import org.elasticsearch.client.indices.AnalyzeResponse;
 import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.client.indices.CreateIndexResponse;
 import org.elasticsearch.client.indices.FreezeIndexRequest;
@@ -61,6 +61,8 @@
 import org.elasticsearch.client.indices.IndexTemplatesExistRequest;
 import org.elasticsearch.client.indices.PutIndexTemplateRequest;
 import org.elasticsearch.client.indices.PutMappingRequest;
+import org.elasticsearch.client.indices.ReloadAnalyzersRequest;
+import org.elasticsearch.client.indices.ReloadAnalyzersResponse;
 import org.elasticsearch.client.indices.UnfreezeIndexRequest;
 import org.elasticsearch.client.indices.rollover.RolloverRequest;
 import org.elasticsearch.client.indices.rollover.RolloverResponse;
@@ -1328,4 +1330,28 @@ public void deleteTemplateAsync(DeleteIndexTemplateRequest request, RequestOptio
         restHighLevelClient.performRequestAsyncAndParseEntity(request, IndicesRequestConverters::deleteTemplate,
             options, AcknowledgedResponse::fromXContent, listener, emptySet());
     }
+
+    /**
+     * Synchronously calls the _reload_search_analyzers API
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     */
+    public ReloadAnalyzersResponse reloadAnalyzers(ReloadAnalyzersRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, IndicesRequestConverters::reloadAnalyzers, options,
+                ReloadAnalyzersResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously calls the _reload_search_analyzers API
+     *
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void reloadAnalyzersAsync(ReloadAnalyzersRequest request, RequestOptions options,
+            ActionListener<ReloadAnalyzersResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, IndicesRequestConverters::reloadAnalyzers, options,
+                ReloadAnalyzersResponse::fromXContent, listener, emptySet());
+    }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java
index cc5adffd33483..7de53510b5080 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java
@@ -26,7 +26,6 @@
 import org.apache.http.client.methods.HttpPut;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
@@ -41,6 +40,7 @@
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
 import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
+import org.elasticsearch.client.indices.AnalyzeRequest;
 import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.client.indices.FreezeIndexRequest;
 import org.elasticsearch.client.indices.GetFieldMappingsRequest;
@@ -50,6 +50,7 @@
 import org.elasticsearch.client.indices.IndexTemplatesExistRequest;
 import org.elasticsearch.client.indices.PutIndexTemplateRequest;
 import org.elasticsearch.client.indices.PutMappingRequest;
+import org.elasticsearch.client.indices.ReloadAnalyzersRequest;
 import org.elasticsearch.client.indices.UnfreezeIndexRequest;
 import org.elasticsearch.client.indices.rollover.RolloverRequest;
 import org.elasticsearch.common.Strings;
@@ -67,10 +68,11 @@ static Request deleteIndex(DeleteIndexRequest deleteIndexRequest) {
         String endpoint = RequestConverters.endpoint(deleteIndexRequest.indices());
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(deleteIndexRequest.timeout());
         parameters.withMasterTimeout(deleteIndexRequest.masterNodeTimeout());
         parameters.withIndicesOptions(deleteIndexRequest.indicesOptions());
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -78,11 +80,12 @@ static Request openIndex(OpenIndexRequest openIndexRequest) {
         String endpoint = RequestConverters.endpoint(openIndexRequest.indices(), "_open");
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(openIndexRequest.timeout());
         parameters.withMasterTimeout(openIndexRequest.masterNodeTimeout());
         parameters.withWaitForActiveShards(openIndexRequest.waitForActiveShards());
         parameters.withIndicesOptions(openIndexRequest.indicesOptions());
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -90,10 +93,11 @@ static Request closeIndex(CloseIndexRequest closeIndexRequest) {
         String endpoint = RequestConverters.endpoint(closeIndexRequest.indices(), "_close");
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(closeIndexRequest.timeout());
         parameters.withMasterTimeout(closeIndexRequest.masterNodeTimeout());
         parameters.withIndicesOptions(closeIndexRequest.indicesOptions());
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -102,11 +106,11 @@ static Request createIndex(CreateIndexRequest createIndexRequest) throws IOExcep
             .addPathPart(createIndexRequest.index()).build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(createIndexRequest.timeout());
         parameters.withMasterTimeout(createIndexRequest.masterNodeTimeout());
         parameters.withWaitForActiveShards(createIndexRequest.waitForActiveShards());
-
+        request.addParameters(parameters.asMap());
         request.setEntity(RequestConverters.createEntity(createIndexRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -116,12 +120,12 @@ static Request createIndex(org.elasticsearch.action.admin.indices.create.CreateI
         String endpoint = RequestConverters.endpoint(createIndexRequest.indices());
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(createIndexRequest.timeout());
         parameters.withMasterTimeout(createIndexRequest.masterNodeTimeout());
         parameters.withWaitForActiveShards(createIndexRequest.waitForActiveShards());
         parameters.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true");
-
+        request.addParameters(parameters.asMap());
         request.setEntity(RequestConverters.createEntity(createIndexRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -129,10 +133,10 @@ static Request createIndex(org.elasticsearch.action.admin.indices.create.CreateI
     static Request updateAliases(IndicesAliasesRequest indicesAliasesRequest) throws IOException {
         Request request = new Request(HttpPost.METHOD_NAME, "/_aliases");
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(indicesAliasesRequest.timeout());
         parameters.withMasterTimeout(indicesAliasesRequest.masterNodeTimeout());
-
+        request.addParameters(parameters.asMap());
         request.setEntity(RequestConverters.createEntity(indicesAliasesRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -141,9 +145,10 @@ static Request updateAliases(IndicesAliasesRequest indicesAliasesRequest) throws
     static Request putMapping(PutMappingRequest putMappingRequest) throws IOException {
         Request request = new Request(HttpPut.METHOD_NAME, RequestConverters.endpoint(putMappingRequest.indices(), "_mapping"));
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(putMappingRequest.timeout());
         parameters.withMasterTimeout(putMappingRequest.masterNodeTimeout());
+        request.addParameters(parameters.asMap());
         request.setEntity(RequestConverters.createEntity(putMappingRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -162,11 +167,11 @@ static Request putMapping(org.elasticsearch.action.admin.indices.mapping.put.Put
         Request request = new Request(HttpPut.METHOD_NAME, RequestConverters.endpoint(putMappingRequest.indices(),
             "_mapping", putMappingRequest.type()));
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(putMappingRequest.timeout());
         parameters.withMasterTimeout(putMappingRequest.masterNodeTimeout());
         parameters.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true");
-
+        request.addParameters(parameters.asMap());
         request.setEntity(RequestConverters.createEntity(putMappingRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -176,11 +181,11 @@ static Request getMappings(GetMappingsRequest getMappingsRequest) {
 
         Request request = new Request(HttpGet.METHOD_NAME, RequestConverters.endpoint(indices, "_mapping"));
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withMasterTimeout(getMappingsRequest.masterNodeTimeout());
         parameters.withIndicesOptions(getMappingsRequest.indicesOptions());
         parameters.withLocal(getMappingsRequest.local());
-
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -191,12 +196,12 @@ static Request getMappings(org.elasticsearch.action.admin.indices.mapping.get.Ge
 
         Request request = new Request(HttpGet.METHOD_NAME, RequestConverters.endpoint(indices, "_mapping", types));
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withMasterTimeout(getMappingsRequest.masterNodeTimeout());
         parameters.withIndicesOptions(getMappingsRequest.indicesOptions());
         parameters.withLocal(getMappingsRequest.local());
         parameters.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true");
-
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -213,11 +218,11 @@ static Request getFieldMapping(GetFieldMappingsRequest getFieldMappingsRequest)
 
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withIndicesOptions(getFieldMappingsRequest.indicesOptions());
         parameters.withIncludeDefaults(getFieldMappingsRequest.includeDefaults());
         parameters.withLocal(getFieldMappingsRequest.local());
-
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -234,12 +239,12 @@ static Request getFieldMapping(org.elasticsearch.action.admin.indices.mapping.ge
 
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withIndicesOptions(getFieldMappingsRequest.indicesOptions());
         parameters.withIncludeDefaults(getFieldMappingsRequest.includeDefaults());
         parameters.withLocal(getFieldMappingsRequest.local());
         parameters.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true");
-
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -247,8 +252,9 @@ static Request refresh(RefreshRequest refreshRequest) {
         String[] indices = refreshRequest.indices() == null ? Strings.EMPTY_ARRAY : refreshRequest.indices();
         Request request = new Request(HttpPost.METHOD_NAME, RequestConverters.endpoint(indices, "_refresh"));
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withIndicesOptions(refreshRequest.indicesOptions());
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -256,18 +262,20 @@ static Request flush(FlushRequest flushRequest) {
         String[] indices = flushRequest.indices() == null ? Strings.EMPTY_ARRAY : flushRequest.indices();
         Request request = new Request(HttpPost.METHOD_NAME, RequestConverters.endpoint(indices, "_flush"));
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withIndicesOptions(flushRequest.indicesOptions());
         parameters.putParam("wait_if_ongoing", Boolean.toString(flushRequest.waitIfOngoing()));
         parameters.putParam("force", Boolean.toString(flushRequest.force()));
+        request.addParameters(parameters.asMap());
         return request;
     }
 
     static Request flushSynced(SyncedFlushRequest syncedFlushRequest) {
         String[] indices = syncedFlushRequest.indices() == null ? Strings.EMPTY_ARRAY : syncedFlushRequest.indices();
         Request request = new Request(HttpPost.METHOD_NAME, RequestConverters.endpoint(indices, "_flush/synced"));
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withIndicesOptions(syncedFlushRequest.indicesOptions());
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -275,11 +283,12 @@ static Request forceMerge(ForceMergeRequest forceMergeRequest) {
         String[] indices = forceMergeRequest.indices() == null ? Strings.EMPTY_ARRAY : forceMergeRequest.indices();
         Request request = new Request(HttpPost.METHOD_NAME, RequestConverters.endpoint(indices, "_forcemerge"));
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withIndicesOptions(forceMergeRequest.indicesOptions());
         parameters.putParam("max_num_segments", Integer.toString(forceMergeRequest.maxNumSegments()));
         parameters.putParam("only_expunge_deletes", Boolean.toString(forceMergeRequest.onlyExpungeDeletes()));
         parameters.putParam("flush", Boolean.toString(forceMergeRequest.flush()));
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -287,12 +296,13 @@ static Request clearCache(ClearIndicesCacheRequest clearIndicesCacheRequest) {
         String[] indices = clearIndicesCacheRequest.indices() == null ? Strings.EMPTY_ARRAY :clearIndicesCacheRequest.indices();
         Request request = new Request(HttpPost.METHOD_NAME, RequestConverters.endpoint(indices, "_cache/clear"));
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withIndicesOptions(clearIndicesCacheRequest.indicesOptions());
         parameters.putParam("query", Boolean.toString(clearIndicesCacheRequest.queryCache()));
         parameters.putParam("fielddata", Boolean.toString(clearIndicesCacheRequest.fieldDataCache()));
         parameters.putParam("request", Boolean.toString(clearIndicesCacheRequest.requestCache()));
         parameters.putParam("fields", String.join(",", clearIndicesCacheRequest.fields()));
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -306,9 +316,10 @@ static Request existsAlias(GetAliasesRequest getAliasesRequest) {
 
         Request request = new Request(HttpHead.METHOD_NAME, RequestConverters.endpoint(indices, "_alias", aliases));
 
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withIndicesOptions(getAliasesRequest.indicesOptions());
         params.withLocal(getAliasesRequest.local());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -332,11 +343,11 @@ private static Request resize(ResizeRequest resizeRequest) throws IOException {
                 .addPathPart(resizeRequest.getTargetIndexRequest().index()).build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
 
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withTimeout(resizeRequest.timeout());
         params.withMasterTimeout(resizeRequest.masterNodeTimeout());
         params.withWaitForActiveShards(resizeRequest.getTargetIndexRequest().waitForActiveShards());
-
+        request.addParameters(params.asMap());
         request.setEntity(RequestConverters.createEntity(resizeRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -346,14 +357,14 @@ static Request rollover(RolloverRequest rolloverRequest) throws IOException {
             .addPathPart(rolloverRequest.getNewIndexName()).build();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
 
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withTimeout(rolloverRequest.timeout());
         params.withMasterTimeout(rolloverRequest.masterNodeTimeout());
         params.withWaitForActiveShards(rolloverRequest.getCreateIndexRequest().waitForActiveShards());
         if (rolloverRequest.isDryRun()) {
             params.putParam("dry_run", Boolean.TRUE.toString());
         }
-
+        request.addParameters(params.asMap());
         request.setEntity(RequestConverters.createEntity(rolloverRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -364,7 +375,7 @@ static Request rollover(org.elasticsearch.action.admin.indices.rollover.Rollover
             .addPathPart(rolloverRequest.getNewIndexName()).build();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
 
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withTimeout(rolloverRequest.timeout());
         params.withMasterTimeout(rolloverRequest.masterNodeTimeout());
         params.withWaitForActiveShards(rolloverRequest.getCreateIndexRequest().waitForActiveShards());
@@ -373,7 +384,7 @@ static Request rollover(org.elasticsearch.action.admin.indices.rollover.Rollover
         }
         params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true");
         request.setEntity(RequestConverters.createEntity(rolloverRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
-
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -384,12 +395,12 @@ static Request getSettings(GetSettingsRequest getSettingsRequest) {
         String endpoint = RequestConverters.endpoint(indices, "_settings", names);
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
 
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withIndicesOptions(getSettingsRequest.indicesOptions());
         params.withLocal(getSettingsRequest.local());
         params.withIncludeDefaults(getSettingsRequest.includeDefaults());
         params.withMasterTimeout(getSettingsRequest.masterNodeTimeout());
-
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -404,14 +415,14 @@ static Request getIndex(org.elasticsearch.action.admin.indices.get.GetIndexReque
         String endpoint = RequestConverters.endpoint(indices);
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
 
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withIndicesOptions(getIndexRequest.indicesOptions());
         params.withLocal(getIndexRequest.local());
         params.withIncludeDefaults(getIndexRequest.includeDefaults());
         params.withHuman(getIndexRequest.humanReadable());
         params.withMasterTimeout(getIndexRequest.masterNodeTimeout());
         params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true");
-
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -421,13 +432,13 @@ static Request getIndex(GetIndexRequest getIndexRequest) {
         String endpoint = RequestConverters.endpoint(indices);
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
 
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withIndicesOptions(getIndexRequest.indicesOptions());
         params.withLocal(getIndexRequest.local());
         params.withIncludeDefaults(getIndexRequest.includeDefaults());
         params.withHuman(getIndexRequest.humanReadable());
         params.withMasterTimeout(getIndexRequest.masterNodeTimeout());
-
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -437,35 +448,35 @@ static Request getIndex(GetIndexRequest getIndexRequest) {
      */
     @Deprecated
     static Request indicesExist(org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest) {
-        // this can be called with no indices as argument by transport client, not via REST though
         if (getIndexRequest.indices() == null || getIndexRequest.indices().length == 0) {
             throw new IllegalArgumentException("indices are mandatory");
         }
         String endpoint = RequestConverters.endpoint(getIndexRequest.indices(), "");
         Request request = new Request(HttpHead.METHOD_NAME, endpoint);
 
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withLocal(getIndexRequest.local());
         params.withHuman(getIndexRequest.humanReadable());
         params.withIndicesOptions(getIndexRequest.indicesOptions());
         params.withIncludeDefaults(getIndexRequest.includeDefaults());
         params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true");
+        request.addParameters(params.asMap());
         return request;
     }
 
     static Request indicesExist(GetIndexRequest getIndexRequest) {
-        // this can be called with no indices as argument by transport client, not via REST though
         if (getIndexRequest.indices() == null || getIndexRequest.indices().length == 0) {
             throw new IllegalArgumentException("indices are mandatory");
         }
         String endpoint = RequestConverters.endpoint(getIndexRequest.indices(), "");
         Request request = new Request(HttpHead.METHOD_NAME, endpoint);
 
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withLocal(getIndexRequest.local());
         params.withHuman(getIndexRequest.humanReadable());
         params.withIndicesOptions(getIndexRequest.indicesOptions());
         params.withIncludeDefaults(getIndexRequest.includeDefaults());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -473,12 +484,12 @@ static Request indexPutSettings(UpdateSettingsRequest updateSettingsRequest) thr
         String[] indices = updateSettingsRequest.indices() == null ? Strings.EMPTY_ARRAY : updateSettingsRequest.indices();
         Request request = new Request(HttpPut.METHOD_NAME, RequestConverters.endpoint(indices, "_settings"));
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(updateSettingsRequest.timeout());
         parameters.withMasterTimeout(updateSettingsRequest.masterNodeTimeout());
         parameters.withIndicesOptions(updateSettingsRequest.indicesOptions());
         parameters.withPreserveExisting(updateSettingsRequest.isPreserveExisting());
-
+        request.addParameters(parameters.asMap());
         request.setEntity(RequestConverters.createEntity(updateSettingsRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -493,7 +504,7 @@ static Request putTemplate(org.elasticsearch.action.admin.indices.template.put.P
         String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_template")
             .addPathPart(putIndexTemplateRequest.name()).build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withMasterTimeout(putIndexTemplateRequest.masterNodeTimeout());
         if (putIndexTemplateRequest.create()) {
             params.putParam("create", Boolean.TRUE.toString());
@@ -502,6 +513,7 @@ static Request putTemplate(org.elasticsearch.action.admin.indices.template.put.P
             params.putParam("cause", putIndexTemplateRequest.cause());
         }
         params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true");
+        request.addParameters(params.asMap());
         request.setEntity(RequestConverters.createEntity(putIndexTemplateRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -510,7 +522,7 @@ static Request putTemplate(PutIndexTemplateRequest putIndexTemplateRequest) thro
         String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_template")
             .addPathPart(putIndexTemplateRequest.name()).build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withMasterTimeout(putIndexTemplateRequest.masterNodeTimeout());
         if (putIndexTemplateRequest.create()) {
             params.putParam("create", Boolean.TRUE.toString());
@@ -518,6 +530,7 @@ static Request putTemplate(PutIndexTemplateRequest putIndexTemplateRequest) thro
         if (Strings.hasText(putIndexTemplateRequest.cause())) {
             params.putParam("cause", putIndexTemplateRequest.cause());
         }
+        request.addParameters(params.asMap());
         request.setEntity(RequestConverters.createEntity(putIndexTemplateRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -527,11 +540,12 @@ static Request validateQuery(ValidateQueryRequest validateQueryRequest) throws I
         String[] types = validateQueryRequest.types() == null || indices.length <= 0 ? Strings.EMPTY_ARRAY : validateQueryRequest.types();
         String endpoint = RequestConverters.endpoint(indices, types, "_validate/query");
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withIndicesOptions(validateQueryRequest.indicesOptions());
         params.putParam("explain", Boolean.toString(validateQueryRequest.explain()));
         params.putParam("all_shards", Boolean.toString(validateQueryRequest.allShards()));
         params.putParam("rewrite", Boolean.toString(validateQueryRequest.rewrite()));
+        request.addParameters(params.asMap());
         request.setEntity(RequestConverters.createEntity(validateQueryRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -541,9 +555,10 @@ static Request getAlias(GetAliasesRequest getAliasesRequest) {
         String[] aliases = getAliasesRequest.aliases() == null ? Strings.EMPTY_ARRAY : getAliasesRequest.aliases();
         String endpoint = RequestConverters.endpoint(indices, "_alias", aliases);
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withIndicesOptions(getAliasesRequest.indicesOptions());
         params.withLocal(getAliasesRequest.local());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -562,12 +577,13 @@ private static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRe
             .addCommaSeparatedPathParts(getIndexTemplatesRequest.names())
             .build();
         final Request request = new Request(HttpGet.METHOD_NAME, endpoint);
-        final RequestConverters.Params params = new RequestConverters.Params(request);
+        final RequestConverters.Params params = new RequestConverters.Params();
         params.withLocal(getIndexTemplatesRequest.isLocal());
         params.withMasterTimeout(getIndexTemplatesRequest.getMasterNodeTimeout());
         if (includeTypeName) {
             params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true");
         }
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -577,9 +593,10 @@ static Request templatesExist(IndexTemplatesExistRequest indexTemplatesExistRequ
             .addCommaSeparatedPathParts(indexTemplatesExistRequest.names())
             .build();
         final Request request = new Request(HttpHead.METHOD_NAME, endpoint);
-        final RequestConverters.Params params = new RequestConverters.Params(request);
+        final RequestConverters.Params params = new RequestConverters.Params();
         params.withLocal(indexTemplatesExistRequest.isLocal());
         params.withMasterTimeout(indexTemplatesExistRequest.getMasterNodeTimeout());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -598,22 +615,24 @@ static Request analyze(AnalyzeRequest request) throws IOException {
     static Request freezeIndex(FreezeIndexRequest freezeIndexRequest) {
         String endpoint = RequestConverters.endpoint(freezeIndexRequest.getIndices(), "_freeze");
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(freezeIndexRequest.timeout());
         parameters.withMasterTimeout(freezeIndexRequest.masterNodeTimeout());
         parameters.withIndicesOptions(freezeIndexRequest.indicesOptions());
         parameters.withWaitForActiveShards(freezeIndexRequest.getWaitForActiveShards());
+        request.addParameters(parameters.asMap());
         return request;
     }
 
     static Request unfreezeIndex(UnfreezeIndexRequest unfreezeIndexRequest) {
         String endpoint = RequestConverters.endpoint(unfreezeIndexRequest.getIndices(), "_unfreeze");
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(unfreezeIndexRequest.timeout());
         parameters.withMasterTimeout(unfreezeIndexRequest.masterNodeTimeout());
         parameters.withIndicesOptions(unfreezeIndexRequest.indicesOptions());
         parameters.withWaitForActiveShards(unfreezeIndexRequest.getWaitForActiveShards());
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -621,8 +640,18 @@ static Request deleteTemplate(DeleteIndexTemplateRequest deleteIndexTemplateRequ
         String name = deleteIndexTemplateRequest.name();
         String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_template").addPathPart(name).build();
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withMasterTimeout(deleteIndexTemplateRequest.masterNodeTimeout());
+        request.addParameters(params.asMap());
+        return request;
+    }
+
+    static Request reloadAnalyzers(ReloadAnalyzersRequest reloadAnalyzersRequest) {
+        String endpoint = RequestConverters.endpoint(reloadAnalyzersRequest.getIndices(), "_reload_search_analyzers");
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        RequestConverters.Params parameters = new RequestConverters.Params();
+        parameters.withIndicesOptions(reloadAnalyzersRequest.indicesOptions());
+        request.addParameters(parameters.asMap());
         return request;
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestRequestConverters.java
index 06b4c0fd62ac0..c2ffc4ee5da4d 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestRequestConverters.java
@@ -41,8 +41,9 @@ static Request getPipeline(GetPipelineRequest getPipelineRequest) {
             .build();
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withMasterTimeout(getPipelineRequest.masterNodeTimeout());
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -53,10 +54,10 @@ static Request putPipeline(PutPipelineRequest putPipelineRequest) throws IOExcep
             .build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(putPipelineRequest.timeout());
         parameters.withMasterTimeout(putPipelineRequest.masterNodeTimeout());
-
+        request.addParameters(parameters.asMap());
         request.setEntity(RequestConverters.createEntity(putPipelineRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -68,10 +69,10 @@ static Request deletePipeline(DeletePipelineRequest deletePipelineRequest) {
             .build();
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(deletePipelineRequest.timeout());
         parameters.withMasterTimeout(deletePipelineRequest.masterNodeTimeout());
-
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -83,8 +84,9 @@ static Request simulatePipeline(SimulatePipelineRequest simulatePipelineRequest)
         builder.addPathPartAsIs("_simulate");
         String endpoint = builder.build();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.putParam("verbose", Boolean.toString(simulatePipelineRequest.isVerbose()));
+        request.addParameters(params.asMap());
         request.setEntity(RequestConverters.createEntity(simulatePipelineRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/LicenseRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/LicenseRequestConverters.java
index 73ecce0f0467c..ec5ae8e3a6073 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/LicenseRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/LicenseRequestConverters.java
@@ -36,12 +36,13 @@ private LicenseRequestConverters() {}
     static Request putLicense(PutLicenseRequest putLicenseRequest) {
         String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_license").build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(putLicenseRequest.timeout());
         parameters.withMasterTimeout(putLicenseRequest.masterNodeTimeout());
         if (putLicenseRequest.isAcknowledge()) {
             parameters.putParam("acknowledge", "true");
         }
+        request.addParameters(parameters.asMap());
         request.setJsonEntity(putLicenseRequest.getLicenseDefinition());
         return request;
     }
@@ -49,17 +50,19 @@ static Request putLicense(PutLicenseRequest putLicenseRequest) {
     static Request getLicense(GetLicenseRequest getLicenseRequest) {
         String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_license").build();
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withLocal(getLicenseRequest.isLocal());
+        request.addParameters(parameters.asMap());
         return request;
     }
 
     static Request deleteLicense(DeleteLicenseRequest deleteLicenseRequest) {
         String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_license").build();
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(deleteLicenseRequest.timeout());
         parameters.withMasterTimeout(deleteLicenseRequest.masterNodeTimeout());
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -67,11 +70,12 @@ static Request startTrial(StartTrialRequest startTrialRequest) {
         final String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_license", "start_trial").build();
         final Request request = new Request(HttpPost.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.putParam("acknowledge", Boolean.toString(startTrialRequest.isAcknowledge()));
         if (startTrialRequest.getLicenseType() != null) {
             parameters.putParam("type", startTrialRequest.getLicenseType());
         }
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -80,12 +84,13 @@ static Request startBasic(StartBasicRequest startBasicRequest) {
             .addPathPartAsIs("_license", "start_basic")
             .build();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(startBasicRequest.timeout());
         parameters.withMasterTimeout(startBasicRequest.masterNodeTimeout());
         if (startBasicRequest.isAcknowledge()) {
             parameters.putParam("acknowledge", "true");
         }
+        request.addParameters(parameters.asMap());
         return request;
     }
 
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java
index 99884ee49c868..651851e345df9 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java
@@ -32,12 +32,14 @@
 import org.elasticsearch.client.ml.DeleteCalendarEventRequest;
 import org.elasticsearch.client.ml.DeleteCalendarJobRequest;
 import org.elasticsearch.client.ml.DeleteCalendarRequest;
+import org.elasticsearch.client.ml.DeleteDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.DeleteDatafeedRequest;
 import org.elasticsearch.client.ml.DeleteExpiredDataRequest;
 import org.elasticsearch.client.ml.DeleteFilterRequest;
 import org.elasticsearch.client.ml.DeleteForecastRequest;
 import org.elasticsearch.client.ml.DeleteJobRequest;
 import org.elasticsearch.client.ml.DeleteModelSnapshotRequest;
+import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
 import org.elasticsearch.client.ml.FindFileStructureRequest;
 import org.elasticsearch.client.ml.FlushJobRequest;
 import org.elasticsearch.client.ml.ForecastJobRequest;
@@ -45,6 +47,8 @@
 import org.elasticsearch.client.ml.GetCalendarEventsRequest;
 import org.elasticsearch.client.ml.GetCalendarsRequest;
 import org.elasticsearch.client.ml.GetCategoriesRequest;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsRequest;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsStatsRequest;
 import org.elasticsearch.client.ml.GetDatafeedRequest;
 import org.elasticsearch.client.ml.GetDatafeedStatsRequest;
 import org.elasticsearch.client.ml.GetFiltersRequest;
@@ -61,12 +65,15 @@
 import org.elasticsearch.client.ml.PreviewDatafeedRequest;
 import org.elasticsearch.client.ml.PutCalendarJobRequest;
 import org.elasticsearch.client.ml.PutCalendarRequest;
+import org.elasticsearch.client.ml.PutDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.PutDatafeedRequest;
 import org.elasticsearch.client.ml.PutFilterRequest;
 import org.elasticsearch.client.ml.PutJobRequest;
 import org.elasticsearch.client.ml.RevertModelSnapshotRequest;
 import org.elasticsearch.client.ml.SetUpgradeModeRequest;
+import org.elasticsearch.client.ml.StartDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.StartDatafeedRequest;
+import org.elasticsearch.client.ml.StopDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.StopDatafeedRequest;
 import org.elasticsearch.client.ml.UpdateDatafeedRequest;
 import org.elasticsearch.client.ml.UpdateFilterRequest;
@@ -105,11 +112,11 @@ static Request getJob(GetJobRequest getJobRequest) {
             .build();
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
 
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         if (getJobRequest.getAllowNoJobs() != null) {
             params.putParam("allow_no_jobs", Boolean.toString(getJobRequest.getAllowNoJobs()));
         }
-
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -122,10 +129,11 @@ static Request getJobStats(GetJobStatsRequest getJobStatsRequest) {
                 .build();
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
 
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         if (getJobStatsRequest.getAllowNoJobs() != null) {
             params.putParam("allow_no_jobs", Boolean.toString(getJobStatsRequest.getAllowNoJobs()));
         }
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -171,14 +179,14 @@ static Request deleteJob(DeleteJobRequest deleteJobRequest) {
                 .build();
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
 
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         if (deleteJobRequest.getForce() != null) {
             params.putParam("force", Boolean.toString(deleteJobRequest.getForce()));
         }
         if (deleteJobRequest.getWaitForCompletion() != null) {
             params.putParam("wait_for_completion", Boolean.toString(deleteJobRequest.getWaitForCompletion()));
         }
-
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -249,12 +257,12 @@ static Request getDatafeed(GetDatafeedRequest getDatafeedRequest) {
                 .build();
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
 
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         if (getDatafeedRequest.getAllowNoDatafeeds() != null) {
             params.putParam(GetDatafeedRequest.ALLOW_NO_DATAFEEDS.getPreferredName(),
                     Boolean.toString(getDatafeedRequest.getAllowNoDatafeeds()));
         }
-
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -265,10 +273,11 @@ static Request deleteDatafeed(DeleteDatafeedRequest deleteDatafeedRequest) {
                 .addPathPart(deleteDatafeedRequest.getDatafeedId())
                 .build();
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         if (deleteDatafeedRequest.getForce() != null) {
             params.putParam("force", Boolean.toString(deleteDatafeedRequest.getForce()));
         }
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -305,10 +314,11 @@ static Request getDatafeedStats(GetDatafeedStatsRequest getDatafeedStatsRequest)
             .build();
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
 
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         if (getDatafeedStatsRequest.getAllowNoDatafeeds() != null) {
             params.putParam("allow_no_datafeeds", Boolean.toString(getDatafeedStatsRequest.getAllowNoDatafeeds()));
         }
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -331,13 +341,14 @@ static Request deleteForecast(DeleteForecastRequest deleteForecastRequest) {
             .addPathPart(Strings.collectionToCommaDelimitedString(deleteForecastRequest.getForecastIds()))
             .build();
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         if (deleteForecastRequest.getAllowNoForecasts() != null) {
             params.putParam("allow_no_forecasts", Boolean.toString(deleteForecastRequest.getAllowNoForecasts()));
         }
         if (deleteForecastRequest.timeout() != null) {
             params.putParam("timeout", deleteForecastRequest.timeout().getStringRep());
         }
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -453,7 +464,7 @@ static Request postData(PostDataRequest postDataRequest) {
             .build();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
 
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         if (postDataRequest.getResetStart() != null) {
             params.putParam(PostDataRequest.RESET_START.getPreferredName(), postDataRequest.getResetStart());
         }
@@ -461,6 +472,7 @@ static Request postData(PostDataRequest postDataRequest) {
             params.putParam(PostDataRequest.RESET_END.getPreferredName(), postDataRequest.getResetEnd());
         }
         BytesReference content = postDataRequest.getContent();
+        request.addParameters(params.asMap());
         if (content != null) {
             BytesRef source = postDataRequest.getContent().toBytesRef();
             HttpEntity byteEntity = new NByteArrayEntity(source.bytes,
@@ -576,6 +588,118 @@ static Request deleteCalendarEvent(DeleteCalendarEventRequest deleteCalendarEven
         return new Request(HttpDelete.METHOD_NAME, endpoint);
     }
 
+    static Request putDataFrameAnalytics(PutDataFrameAnalyticsRequest putRequest) throws IOException {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_ml", "data_frame", "analytics")
+            .addPathPart(putRequest.getConfig().getId())
+            .build();
+        Request request = new Request(HttpPut.METHOD_NAME, endpoint);
+        request.setEntity(createEntity(putRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
+    static Request getDataFrameAnalytics(GetDataFrameAnalyticsRequest getRequest) {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_ml", "data_frame", "analytics")
+            .addPathPart(Strings.collectionToCommaDelimitedString(getRequest.getIds()))
+            .build();
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        RequestConverters.Params params = new RequestConverters.Params();
+        if (getRequest.getPageParams() != null) {
+            PageParams pageParams = getRequest.getPageParams();
+            if (pageParams.getFrom() != null) {
+                params.putParam(PageParams.FROM.getPreferredName(), pageParams.getFrom().toString());
+            }
+            if (pageParams.getSize() != null) {
+                params.putParam(PageParams.SIZE.getPreferredName(), pageParams.getSize().toString());
+            }
+        }
+        if (getRequest.getAllowNoMatch() != null) {
+            params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getRequest.getAllowNoMatch()));
+        }
+        request.addParameters(params.asMap());
+        return request;
+    }
+
+    static Request getDataFrameAnalyticsStats(GetDataFrameAnalyticsStatsRequest getStatsRequest) {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_ml", "data_frame", "analytics")
+            .addPathPart(Strings.collectionToCommaDelimitedString(getStatsRequest.getIds()))
+            .addPathPartAsIs("_stats")
+            .build();
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        RequestConverters.Params params = new RequestConverters.Params();
+        if (getStatsRequest.getPageParams() != null) {
+            PageParams pageParams = getStatsRequest.getPageParams();
+            if (pageParams.getFrom() != null) {
+                params.putParam(PageParams.FROM.getPreferredName(), pageParams.getFrom().toString());
+            }
+            if (pageParams.getSize() != null) {
+                params.putParam(PageParams.SIZE.getPreferredName(), pageParams.getSize().toString());
+            }
+        }
+        if (getStatsRequest.getAllowNoMatch() != null) {
+            params.putParam(GetDataFrameAnalyticsStatsRequest.ALLOW_NO_MATCH.getPreferredName(),
+                Boolean.toString(getStatsRequest.getAllowNoMatch()));
+        }
+        request.addParameters(params.asMap());
+        return request;
+    }
+
+    static Request startDataFrameAnalytics(StartDataFrameAnalyticsRequest startRequest) {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_ml", "data_frame", "analytics")
+            .addPathPart(startRequest.getId())
+            .addPathPartAsIs("_start")
+            .build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        RequestConverters.Params params = new RequestConverters.Params();
+        if (startRequest.getTimeout() != null) {
+            params.withTimeout(startRequest.getTimeout());
+        }
+        request.addParameters(params.asMap());
+        return request;
+    }
+
+    static Request stopDataFrameAnalytics(StopDataFrameAnalyticsRequest stopRequest) {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_ml", "data_frame", "analytics")
+            .addPathPart(stopRequest.getId())
+            .addPathPartAsIs("_stop")
+            .build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        RequestConverters.Params params = new RequestConverters.Params();
+        if (stopRequest.getTimeout() != null) {
+            params.withTimeout(stopRequest.getTimeout());
+        }
+        if (stopRequest.getAllowNoMatch() != null) {
+            params.putParam(
+                StopDataFrameAnalyticsRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(stopRequest.getAllowNoMatch()));
+        }
+        if (stopRequest.getForce() != null) {
+            params.putParam(StopDataFrameAnalyticsRequest.FORCE.getPreferredName(), Boolean.toString(stopRequest.getForce()));
+        }
+        request.addParameters(params.asMap());
+        return request;
+    }
+
+    static Request deleteDataFrameAnalytics(DeleteDataFrameAnalyticsRequest deleteRequest) {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_ml", "data_frame", "analytics")
+            .addPathPart(deleteRequest.getId())
+            .build();
+        return new Request(HttpDelete.METHOD_NAME, endpoint);
+    }
+
+    static Request evaluateDataFrame(EvaluateDataFrameRequest evaluateRequest) throws IOException {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_ml", "data_frame", "_evaluate")
+            .build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        request.setEntity(createEntity(evaluateRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
     static Request putFilter(PutFilterRequest putFilterRequest) throws IOException {
         String endpoint = new EndpointBuilder()
             .addPathPartAsIs("_ml")
@@ -594,13 +718,14 @@ static Request getFilter(GetFiltersRequest getFiltersRequest) {
             .addPathPart(getFiltersRequest.getFilterId())
             .build();
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         if (getFiltersRequest.getSize() != null) {
             params.putParam(PageParams.SIZE.getPreferredName(), getFiltersRequest.getSize().toString());
         }
         if (getFiltersRequest.getFrom() != null) {
             params.putParam(PageParams.FROM.getPreferredName(), getFiltersRequest.getFrom().toString());
         }
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -628,11 +753,12 @@ static Request deleteFilter(DeleteFilterRequest deleteFilterRequest) {
     static Request setUpgradeMode(SetUpgradeModeRequest setUpgradeModeRequest) {
         String endpoint = new EndpointBuilder().addPathPartAsIs("_ml", "set_upgrade_mode").build();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.putParam(SetUpgradeModeRequest.ENABLED.getPreferredName(), Boolean.toString(setUpgradeModeRequest.isEnabled()));
         if (setUpgradeModeRequest.getTimeout() != null) {
             params.putParam(SetUpgradeModeRequest.TIMEOUT.getPreferredName(), setUpgradeModeRequest.getTimeout().toString());
         }
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -650,7 +776,7 @@ static Request findFileStructure(FindFileStructureRequest findFileStructureReque
             .build();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
 
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         if (findFileStructureRequest.getLinesToSample() != null) {
             params.putParam(FindFileStructureRequest.LINES_TO_SAMPLE.getPreferredName(),
                 findFileStructureRequest.getLinesToSample().toString());
@@ -695,7 +821,7 @@ static Request findFileStructure(FindFileStructureRequest findFileStructureReque
         if (findFileStructureRequest.getExplain() != null) {
             params.putParam(FindFileStructureRequest.EXPLAIN.getPreferredName(), findFileStructureRequest.getExplain().toString());
         }
-
+        request.addParameters(params.asMap());
         BytesReference sample = findFileStructureRequest.getSample();
         BytesRef source = sample.toBytesRef();
         HttpEntity byteEntity = new NByteArrayEntity(source.bytes, source.offset, source.length, createContentType(XContentType.JSON));
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
index 2e359931c1025..ea72c355a02e7 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.client.ml.DeleteCalendarEventRequest;
 import org.elasticsearch.client.ml.DeleteCalendarJobRequest;
 import org.elasticsearch.client.ml.DeleteCalendarRequest;
+import org.elasticsearch.client.ml.DeleteDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.DeleteDatafeedRequest;
 import org.elasticsearch.client.ml.DeleteExpiredDataRequest;
 import org.elasticsearch.client.ml.DeleteExpiredDataResponse;
@@ -33,6 +34,8 @@
 import org.elasticsearch.client.ml.DeleteJobRequest;
 import org.elasticsearch.client.ml.DeleteJobResponse;
 import org.elasticsearch.client.ml.DeleteModelSnapshotRequest;
+import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
+import org.elasticsearch.client.ml.EvaluateDataFrameResponse;
 import org.elasticsearch.client.ml.FindFileStructureRequest;
 import org.elasticsearch.client.ml.FindFileStructureResponse;
 import org.elasticsearch.client.ml.FlushJobRequest;
@@ -47,6 +50,10 @@
 import org.elasticsearch.client.ml.GetCalendarsResponse;
 import org.elasticsearch.client.ml.GetCategoriesRequest;
 import org.elasticsearch.client.ml.GetCategoriesResponse;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsRequest;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsResponse;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsStatsRequest;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsStatsResponse;
 import org.elasticsearch.client.ml.GetDatafeedRequest;
 import org.elasticsearch.client.ml.GetDatafeedResponse;
 import org.elasticsearch.client.ml.GetDatafeedStatsRequest;
@@ -78,6 +85,8 @@
 import org.elasticsearch.client.ml.PutCalendarJobRequest;
 import org.elasticsearch.client.ml.PutCalendarRequest;
 import org.elasticsearch.client.ml.PutCalendarResponse;
+import org.elasticsearch.client.ml.PutDataFrameAnalyticsRequest;
+import org.elasticsearch.client.ml.PutDataFrameAnalyticsResponse;
 import org.elasticsearch.client.ml.PutDatafeedRequest;
 import org.elasticsearch.client.ml.PutDatafeedResponse;
 import org.elasticsearch.client.ml.PutFilterRequest;
@@ -87,8 +96,11 @@
 import org.elasticsearch.client.ml.RevertModelSnapshotRequest;
 import org.elasticsearch.client.ml.RevertModelSnapshotResponse;
 import org.elasticsearch.client.ml.SetUpgradeModeRequest;
+import org.elasticsearch.client.ml.StartDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.StartDatafeedRequest;
 import org.elasticsearch.client.ml.StartDatafeedResponse;
+import org.elasticsearch.client.ml.StopDataFrameAnalyticsRequest;
+import org.elasticsearch.client.ml.StopDataFrameAnalyticsResponse;
 import org.elasticsearch.client.ml.StopDatafeedRequest;
 import org.elasticsearch.client.ml.StopDatafeedResponse;
 import org.elasticsearch.client.ml.UpdateDatafeedRequest;
@@ -1877,4 +1889,286 @@ public void setUpgradeModeAsync(SetUpgradeModeRequest request, RequestOptions op
             listener,
             Collections.emptySet());
     }
+
+    /**
+     * Creates a new Data Frame Analytics config
+     * <p>
+     * For additional info
+     * see <a href="https://www.TODO.com">PUT Data Frame Analytics documentation</a>
+     *
+     * @param request The {@link PutDataFrameAnalyticsRequest} containing the
+     * {@link org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return The {@link PutDataFrameAnalyticsResponse} containing the created
+     * {@link org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig}
+     * @throws IOException when there is a serialization issue sending the request or receiving the response
+     */
+    public PutDataFrameAnalyticsResponse putDataFrameAnalytics(PutDataFrameAnalyticsRequest request,
+                                                               RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            MLRequestConverters::putDataFrameAnalytics,
+            options,
+            PutDataFrameAnalyticsResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Creates a new Data Frame Analytics config asynchronously and notifies listener upon completion
+     * <p>
+     * For additional info
+     * see <a href="https://www.TODO.com">PUT Data Frame Analytics documentation</a>
+     *
+     * @param request The {@link PutDataFrameAnalyticsRequest} containing the
+     * {@link org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void putDataFrameAnalyticsAsync(PutDataFrameAnalyticsRequest request, RequestOptions options,
+                                           ActionListener<PutDataFrameAnalyticsResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            MLRequestConverters::putDataFrameAnalytics,
+            options,
+            PutDataFrameAnalyticsResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+
+    /**
+     * Gets a single or multiple Data Frame Analytics configs
+     * <p>
+     * For additional info
+     * see <a href="https://www.TODO.com">GET Data Frame Analytics documentation</a>
+     *
+     * @param request The {@link GetDataFrameAnalyticsRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return {@link GetDataFrameAnalyticsResponse} response object containing the
+     * {@link org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig} objects
+     */
+    public GetDataFrameAnalyticsResponse getDataFrameAnalytics(GetDataFrameAnalyticsRequest request,
+                                                               RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            MLRequestConverters::getDataFrameAnalytics,
+            options,
+            GetDataFrameAnalyticsResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Gets a single or multiple Data Frame Analytics configs asynchronously and notifies listener upon completion
+     * <p>
+     * For additional info
+     * see <a href="https://www.TODO.com">GET Data Frame Analytics documentation</a>
+     *
+     * @param request The {@link GetDataFrameAnalyticsRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void getDataFrameAnalyticsAsync(GetDataFrameAnalyticsRequest request, RequestOptions options,
+                                           ActionListener<GetDataFrameAnalyticsResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            MLRequestConverters::getDataFrameAnalytics,
+            options,
+            GetDataFrameAnalyticsResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+
+    /**
+     * Gets the running statistics of a Data Frame Analytics
+     * <p>
+     * For additional info
+     * see <a href="https://www.TODO.com">GET Data Frame Analytics Stats documentation</a>
+     *
+     * @param request The {@link GetDataFrameAnalyticsStatsRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return {@link GetDataFrameAnalyticsStatsResponse} response object
+     */
+    public GetDataFrameAnalyticsStatsResponse getDataFrameAnalyticsStats(GetDataFrameAnalyticsStatsRequest request,
+                                                                         RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            MLRequestConverters::getDataFrameAnalyticsStats,
+            options,
+            GetDataFrameAnalyticsStatsResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Gets the running statistics of a Data Frame Analytics asynchronously and notifies listener upon completion
+     * <p>
+     * For additional info
+     * see <a href="https://www.TODO.com">GET Data Frame Analytics Stats documentation</a>
+     *
+     * @param request The {@link GetDataFrameAnalyticsStatsRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void getDataFrameAnalyticsStatsAsync(GetDataFrameAnalyticsStatsRequest request, RequestOptions options,
+                                                ActionListener<GetDataFrameAnalyticsStatsResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            MLRequestConverters::getDataFrameAnalyticsStats,
+            options,
+            GetDataFrameAnalyticsStatsResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+
+    /**
+     * Starts Data Frame Analytics
+     * <p>
+     * For additional info
+     * see <a href="https://www.TODO.com">Start Data Frame Analytics documentation</a>
+     *
+     * @param request The {@link StartDataFrameAnalyticsRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return action acknowledgement
+     * @throws IOException when there is a serialization issue sending the request or receiving the response
+     */
+    public AcknowledgedResponse startDataFrameAnalytics(StartDataFrameAnalyticsRequest request,
+                                                        RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            MLRequestConverters::startDataFrameAnalytics,
+            options,
+            AcknowledgedResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Starts Data Frame Analytics asynchronously and notifies listener upon completion
+     * <p>
+     * For additional info
+     * see <a href="https://www.TODO.com">Start Data Frame Analytics documentation</a>
+     *
+     * @param request The {@link StartDataFrameAnalyticsRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void startDataFrameAnalyticsAsync(StartDataFrameAnalyticsRequest request, RequestOptions options,
+                                             ActionListener<AcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            MLRequestConverters::startDataFrameAnalytics,
+            options,
+            AcknowledgedResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+
+    /**
+     * Stops Data Frame Analytics
+     * <p>
+     * For additional info
+     * see <a href="https://www.TODO.com">Stop Data Frame Analytics documentation</a>
+     *
+     * @param request The {@link StopDataFrameAnalyticsRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return {@link StopDataFrameAnalyticsResponse}
+     * @throws IOException when there is a serialization issue sending the request or receiving the response
+     */
+    public StopDataFrameAnalyticsResponse stopDataFrameAnalytics(StopDataFrameAnalyticsRequest request,
+                                                                 RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            MLRequestConverters::stopDataFrameAnalytics,
+            options,
+            StopDataFrameAnalyticsResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Stops Data Frame Analytics asynchronously and notifies listener upon completion
+     * <p>
+     * For additional info
+     * see <a href="https://www.TODO.com">Stop Data Frame Analytics documentation</a>
+     *
+     * @param request The {@link StopDataFrameAnalyticsRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void stopDataFrameAnalyticsAsync(StopDataFrameAnalyticsRequest request, RequestOptions options,
+                                            ActionListener<StopDataFrameAnalyticsResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            MLRequestConverters::stopDataFrameAnalytics,
+            options,
+            StopDataFrameAnalyticsResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+
+    /**
+     * Deletes the given Data Frame Analytics config
+     * <p>
+     * For additional info
+     * see <a href="https://www.TODO.com">DELETE Data Frame Analytics documentation</a>
+     *
+     * @param request The {@link DeleteDataFrameAnalyticsRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return action acknowledgement
+     * @throws IOException when there is a serialization issue sending the request or receiving the response
+     */
+    public AcknowledgedResponse deleteDataFrameAnalytics(DeleteDataFrameAnalyticsRequest request,
+                                                         RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            MLRequestConverters::deleteDataFrameAnalytics,
+            options,
+            AcknowledgedResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Deletes the given Data Frame Analytics config asynchronously and notifies listener upon completion
+     * <p>
+     * For additional info
+     * see <a href="https://www.TODO.com">DELETE Data Frame Analytics documentation</a>
+     *
+     * @param request The {@link DeleteDataFrameAnalyticsRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void deleteDataFrameAnalyticsAsync(DeleteDataFrameAnalyticsRequest request, RequestOptions options,
+                                              ActionListener<AcknowledgedResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            MLRequestConverters::deleteDataFrameAnalytics,
+            options,
+            AcknowledgedResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+
+    /**
+     * Evaluates the given Data Frame
+     * <p>
+     * For additional info
+     * see <a href="https://www.TODO.com">Evaluate Data Frame documentation</a>
+     *
+     * @param request The {@link EvaluateDataFrameRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return {@link EvaluateDataFrameResponse} response object
+     * @throws IOException when there is a serialization issue sending the request or receiving the response
+     */
+    public EvaluateDataFrameResponse evaluateDataFrame(EvaluateDataFrameRequest request,
+                                                       RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            MLRequestConverters::evaluateDataFrame,
+            options,
+            EvaluateDataFrameResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Evaluates the given Data Frame asynchronously and notifies listener upon completion
+     * <p>
+     * For additional info
+     * see <a href="https://www.TODO.com">Evaluate Data Frame documentation</a>
+     *
+     * @param request The {@link EvaluateDataFrameRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void evaluateDataFrameAsync(EvaluateDataFrameRequest request, RequestOptions options,
+                                       ActionListener<EvaluateDataFrameResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            MLRequestConverters::evaluateDataFrame,
+            options,
+            EvaluateDataFrameResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
index 023bd1fe63786..6adc287645c7c 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.explain.ExplainRequest;
@@ -52,6 +51,7 @@
 import org.elasticsearch.client.core.CountRequest;
 import org.elasticsearch.client.core.MultiTermVectorsRequest;
 import org.elasticsearch.client.core.TermVectorsRequest;
+import org.elasticsearch.client.indices.AnalyzeRequest;
 import org.elasticsearch.client.security.RefreshPolicy;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.Nullable;
@@ -88,8 +88,10 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.charset.Charset;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.StringJoiner;
 
 final class RequestConverters {
@@ -103,7 +105,7 @@ static Request delete(DeleteRequest deleteRequest) {
         String endpoint = endpoint(deleteRequest.index(), deleteRequest.type(), deleteRequest.id());
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
 
-        Params parameters = new Params(request);
+        Params parameters = new Params();
         parameters.withRouting(deleteRequest.routing());
         parameters.withTimeout(deleteRequest.timeout());
         parameters.withVersion(deleteRequest.version());
@@ -112,6 +114,7 @@ static Request delete(DeleteRequest deleteRequest) {
         parameters.withIfPrimaryTerm(deleteRequest.ifPrimaryTerm());
         parameters.withRefreshPolicy(deleteRequest.getRefreshPolicy());
         parameters.withWaitForActiveShards(deleteRequest.waitForActiveShards());
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -122,7 +125,7 @@ static Request info() {
     static Request bulk(BulkRequest bulkRequest) throws IOException {
         Request request = new Request(HttpPost.METHOD_NAME, "/_bulk");
 
-        Params parameters = new Params(request);
+        Params parameters = new Params();
         parameters.withTimeout(bulkRequest.timeout());
         parameters.withRefreshPolicy(bulkRequest.getRefreshPolicy());
         parameters.withPipeline(bulkRequest.pipeline());
@@ -249,6 +252,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
                 content.write(separator);
             }
         }
+        request.addParameters(parameters.asMap());
         request.setEntity(new NByteArrayEntity(content.toByteArray(), 0, content.size(), requestContentType));
         return request;
     }
@@ -264,7 +268,7 @@ static Request get(GetRequest getRequest) {
     private static Request getStyleRequest(String method, GetRequest getRequest) {
         Request request = new Request(method, endpoint(getRequest.index(), getRequest.type(), getRequest.id()));
 
-        Params parameters = new Params(request);
+        Params parameters = new Params();
         parameters.withPreference(getRequest.preference());
         parameters.withRouting(getRequest.routing());
         parameters.withRefresh(getRequest.refresh());
@@ -273,7 +277,7 @@ private static Request getStyleRequest(String method, GetRequest getRequest) {
         parameters.withVersion(getRequest.version());
         parameters.withVersionType(getRequest.versionType());
         parameters.withFetchSourceContext(getRequest.fetchSourceContext());
-
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -286,23 +290,24 @@ static Request sourceExists(GetRequest getRequest) {
             endpoint = endpoint(getRequest.index(), optionalType, getRequest.id(), "_source");
         }
         Request request = new Request(HttpHead.METHOD_NAME, endpoint);
-        Params parameters = new Params(request);
+        Params parameters = new Params();
         parameters.withPreference(getRequest.preference());
         parameters.withRouting(getRequest.routing());
         parameters.withRefresh(getRequest.refresh());
         parameters.withRealtime(getRequest.realtime());
         // Version params are not currently supported by the source exists API so are not passed
+        request.addParameters(parameters.asMap());
         return request;
     }
 
     static Request multiGet(MultiGetRequest multiGetRequest) throws IOException {
         Request request = new Request(HttpPost.METHOD_NAME, "/_mget");
 
-        Params parameters = new Params(request);
+        Params parameters = new Params();
         parameters.withPreference(multiGetRequest.preference());
         parameters.withRealtime(multiGetRequest.realtime());
         parameters.withRefresh(multiGetRequest.refresh());
-
+        request.addParameters(parameters.asMap());
         request.setEntity(createEntity(multiGetRequest, REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -321,7 +326,7 @@ static Request index(IndexRequest indexRequest) {
 
         Request request = new Request(method, endpoint);
 
-        Params parameters = new Params(request);
+        Params parameters = new Params();
         parameters.withRouting(indexRequest.routing());
         parameters.withTimeout(indexRequest.timeout());
         parameters.withVersion(indexRequest.version());
@@ -334,6 +339,7 @@ static Request index(IndexRequest indexRequest) {
 
         BytesRef source = indexRequest.source().toBytesRef();
         ContentType contentType = createContentType(indexRequest.getContentType());
+        request.addParameters(parameters.asMap());
         request.setEntity(new NByteArrayEntity(source.bytes, source.offset, source.length, contentType));
         return request;
     }
@@ -348,7 +354,7 @@ static Request update(UpdateRequest updateRequest) throws IOException {
             : endpoint(updateRequest.index(), updateRequest.type(), updateRequest.id(), "_update");
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
 
-        Params parameters = new Params(request);
+        Params parameters = new Params();
         parameters.withRouting(updateRequest.routing());
         parameters.withTimeout(updateRequest.timeout());
         parameters.withRefreshPolicy(updateRequest.getRefreshPolicy());
@@ -379,6 +385,7 @@ static Request update(UpdateRequest updateRequest) throws IOException {
         if (xContentType == null) {
             xContentType = Requests.INDEX_CONTENT_TYPE;
         }
+        request.addParameters(parameters.asMap());
         request.setEntity(createEntity(updateRequest, xContentType));
         return request;
     }
@@ -391,14 +398,15 @@ static Request update(UpdateRequest updateRequest) throws IOException {
      *    searches.
      */
     static Request search(SearchRequest searchRequest, String searchEndpoint) throws IOException {
-        Request request = new Request(HttpPost.METHOD_NAME, endpoint(searchRequest.indices(), searchRequest.types(), searchEndpoint));
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint(searchRequest.indices(), searchEndpoint));
 
-        Params params = new Params(request);
+        Params params = new Params();
         addSearchRequestParams(params, searchRequest);
 
         if (searchRequest.source() != null) {
             request.setEntity(createEntity(searchRequest.source(), REQUEST_BODY_CONTENT_TYPE));
         }
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -436,7 +444,7 @@ static Request clearScroll(ClearScrollRequest clearScrollRequest) throws IOExcep
     static Request multiSearch(MultiSearchRequest multiSearchRequest) throws IOException {
         Request request = new Request(HttpPost.METHOD_NAME, "/_msearch");
 
-        Params params = new Params(request);
+        Params params = new Params();
         params.putParam(RestSearchAction.TYPED_KEYS_PARAM, "true");
         if (multiSearchRequest.maxConcurrentSearchRequests() != MultiSearchRequest.MAX_CONCURRENT_SEARCH_REQUESTS_DEFAULT) {
             params.putParam("max_concurrent_searches", Integer.toString(multiSearchRequest.maxConcurrentSearchRequests()));
@@ -444,6 +452,7 @@ static Request multiSearch(MultiSearchRequest multiSearchRequest) throws IOExcep
 
         XContent xContent = REQUEST_BODY_CONTENT_TYPE.xContent();
         byte[] source = MultiSearchRequest.writeMultiLineFormat(multiSearchRequest, xContent);
+        request.addParameters(params.asMap());
         request.setEntity(new NByteArrayEntity(source, createContentType(xContent.type())));
         return request;
     }
@@ -455,11 +464,12 @@ static Request searchTemplate(SearchTemplateRequest searchTemplateRequest) throw
             request = new Request(HttpGet.METHOD_NAME, "_render/template");
         } else {
             SearchRequest searchRequest = searchTemplateRequest.getRequest();
-            String endpoint = endpoint(searchRequest.indices(), searchRequest.types(), "_search/template");
+            String endpoint = endpoint(searchRequest.indices(), "_search/template");
             request = new Request(HttpGet.METHOD_NAME, endpoint);
 
-            Params params = new Params(request);
+            Params params = new Params();
             addSearchRequestParams(params, searchRequest);
+            request.addParameters(params.asMap());
         }
 
         request.setEntity(createEntity(searchTemplateRequest, REQUEST_BODY_CONTENT_TYPE));
@@ -469,7 +479,7 @@ static Request searchTemplate(SearchTemplateRequest searchTemplateRequest) throw
     static Request multiSearchTemplate(MultiSearchTemplateRequest multiSearchTemplateRequest) throws IOException {
         Request request = new Request(HttpPost.METHOD_NAME, "/_msearch/template");
 
-        Params params = new Params(request);
+        Params params = new Params();
         params.putParam(RestSearchAction.TYPED_KEYS_PARAM, "true");
         if (multiSearchTemplateRequest.maxConcurrentSearchRequests() != MultiSearchRequest.MAX_CONCURRENT_SEARCH_REQUESTS_DEFAULT) {
             params.putParam("max_concurrent_searches", Integer.toString(multiSearchTemplateRequest.maxConcurrentSearchRequests()));
@@ -483,10 +493,11 @@ static Request multiSearchTemplate(MultiSearchTemplateRequest multiSearchTemplat
 
     static Request count(CountRequest countRequest) throws IOException {
         Request request = new Request(HttpPost.METHOD_NAME, endpoint(countRequest.indices(), countRequest.types(), "_count"));
-        Params params = new Params(request);
+        Params params = new Params();
         params.withRouting(countRequest.routing());
         params.withPreference(countRequest.preference());
         params.withIndicesOptions(countRequest.indicesOptions());
+        request.addParameters(params.asMap());
         request.setEntity(createEntity(countRequest.source(), REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -497,11 +508,12 @@ static Request explain(ExplainRequest explainRequest) throws IOException {
             : endpoint(explainRequest.index(), explainRequest.type(), explainRequest.id(), "_explain");
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
 
-        Params params = new Params(request);
+        Params params = new Params();
         params.withStoredFields(explainRequest.storedFields());
         params.withFetchSourceContext(explainRequest.fetchSourceContext());
         params.withRouting(explainRequest.routing());
         params.withPreference(explainRequest.preference());
+        request.addParameters(params.asMap());
         request.setEntity(createEntity(explainRequest, REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -509,18 +521,19 @@ static Request explain(ExplainRequest explainRequest) throws IOException {
     static Request fieldCaps(FieldCapabilitiesRequest fieldCapabilitiesRequest) {
         Request request = new Request(HttpGet.METHOD_NAME, endpoint(fieldCapabilitiesRequest.indices(), "_field_caps"));
 
-        Params params = new Params(request);
+        Params params = new Params();
         params.withFields(fieldCapabilitiesRequest.fields());
         params.withIndicesOptions(fieldCapabilitiesRequest.indicesOptions());
+        request.addParameters(params.asMap());
         return request;
     }
 
     static Request rankEval(RankEvalRequest rankEvalRequest) throws IOException {
         Request request = new Request(HttpGet.METHOD_NAME, endpoint(rankEvalRequest.indices(), Strings.EMPTY_ARRAY, "_rank_eval"));
 
-        Params params = new Params(request);
+        Params params = new Params();
         params.withIndicesOptions(rankEvalRequest.indicesOptions());
-
+        request.addParameters(params.asMap());
         request.setEntity(createEntity(rankEvalRequest.getRankEvalSpec(), REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -536,7 +549,7 @@ static Request submitReindex(ReindexRequest reindexRequest) throws IOException {
     private static Request prepareReindexRequest(ReindexRequest reindexRequest, boolean waitForCompletion) throws IOException {
         String endpoint = new EndpointBuilder().addPathPart("_reindex").build();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        Params params = new Params(request)
+        Params params = new Params()
             .withWaitForCompletion(waitForCompletion)
             .withRefresh(reindexRequest.isRefresh())
             .withTimeout(reindexRequest.getTimeout())
@@ -546,15 +559,15 @@ private static Request prepareReindexRequest(ReindexRequest reindexRequest, bool
         if (reindexRequest.getScrollTime() != null) {
             params.putParam("scroll", reindexRequest.getScrollTime());
         }
+        request.addParameters(params.asMap());
         request.setEntity(createEntity(reindexRequest, REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
 
     static Request updateByQuery(UpdateByQueryRequest updateByQueryRequest) throws IOException {
-        String endpoint =
-            endpoint(updateByQueryRequest.indices(), updateByQueryRequest.getDocTypes(), "_update_by_query");
+        String endpoint = endpoint(updateByQueryRequest.indices(), "_update_by_query");
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        Params params = new Params(request)
+        Params params = new Params()
             .withRouting(updateByQueryRequest.getRouting())
             .withPipeline(updateByQueryRequest.getPipeline())
             .withRefresh(updateByQueryRequest.isRefresh())
@@ -571,18 +584,18 @@ static Request updateByQuery(UpdateByQueryRequest updateByQueryRequest) throws I
         if (updateByQueryRequest.getScrollTime() != AbstractBulkByScrollRequest.DEFAULT_SCROLL_TIMEOUT) {
             params.putParam("scroll", updateByQueryRequest.getScrollTime());
         }
-        if (updateByQueryRequest.getSize() > 0) {
-            params.putParam("size", Integer.toString(updateByQueryRequest.getSize()));
+        if (updateByQueryRequest.getMaxDocs() > 0) {
+            params.putParam("max_docs", Integer.toString(updateByQueryRequest.getMaxDocs()));
         }
+        request.addParameters(params.asMap());
         request.setEntity(createEntity(updateByQueryRequest, REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
 
     static Request deleteByQuery(DeleteByQueryRequest deleteByQueryRequest) throws IOException {
-        String endpoint =
-            endpoint(deleteByQueryRequest.indices(), deleteByQueryRequest.getDocTypes(), "_delete_by_query");
+        String endpoint = endpoint(deleteByQueryRequest.indices(), "_delete_by_query");
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        Params params = new Params(request)
+        Params params = new Params()
             .withRouting(deleteByQueryRequest.getRouting())
             .withRefresh(deleteByQueryRequest.isRefresh())
             .withTimeout(deleteByQueryRequest.getTimeout())
@@ -598,9 +611,10 @@ static Request deleteByQuery(DeleteByQueryRequest deleteByQueryRequest) throws I
         if (deleteByQueryRequest.getScrollTime() != AbstractBulkByScrollRequest.DEFAULT_SCROLL_TIMEOUT) {
             params.putParam("scroll", deleteByQueryRequest.getScrollTime());
         }
-        if (deleteByQueryRequest.getSize() > 0) {
-            params.putParam("size", Integer.toString(deleteByQueryRequest.getSize()));
+        if (deleteByQueryRequest.getMaxDocs() > 0) {
+            params.putParam("max_docs", Integer.toString(deleteByQueryRequest.getMaxDocs()));
         }
+        request.addParameters(params.asMap());
         request.setEntity(createEntity(deleteByQueryRequest, REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -621,22 +635,24 @@ private static Request rethrottle(RethrottleRequest rethrottleRequest, String fi
         String endpoint = new EndpointBuilder().addPathPart(firstPathPart).addPathPart(rethrottleRequest.getTaskId().toString())
                 .addPathPart("_rethrottle").build();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        Params params = new Params(request)
+        Params params = new Params()
                 .withRequestsPerSecond(rethrottleRequest.getRequestsPerSecond());
         // we set "group_by" to "none" because this is the response format we can parse back
         params.putParam("group_by", "none");
+        request.addParameters(params.asMap());
         return request;
     }
 
     static Request putScript(PutStoredScriptRequest putStoredScriptRequest) throws IOException {
         String endpoint = new EndpointBuilder().addPathPartAsIs("_scripts").addPathPart(putStoredScriptRequest.id()).build();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        Params params = new Params(request);
+        Params params = new Params();
         params.withTimeout(putStoredScriptRequest.timeout());
         params.withMasterTimeout(putStoredScriptRequest.masterNodeTimeout());
         if (Strings.hasText(putStoredScriptRequest.context())) {
             params.putParam("context", putStoredScriptRequest.context());
         }
+        request.addParameters(params.asMap());
         request.setEntity(createEntity(putStoredScriptRequest, REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -667,11 +683,12 @@ static Request termVectors(TermVectorsRequest tvrequest) throws IOException {
         }
 
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
-        Params params = new Params(request);
+        Params params = new Params();
         params.withRouting(tvrequest.getRouting());
         params.withPreference(tvrequest.getPreference());
         params.withFields(tvrequest.getFields());
         params.withRealtime(tvrequest.getRealtime());
+        request.addParameters(params.asMap());
         request.setEntity(createEntity(tvrequest, REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -686,17 +703,19 @@ static Request mtermVectors(MultiTermVectorsRequest mtvrequest) throws IOExcepti
     static Request getScript(GetStoredScriptRequest getStoredScriptRequest) {
         String endpoint = new EndpointBuilder().addPathPartAsIs("_scripts").addPathPart(getStoredScriptRequest.id()).build();
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
-        Params params = new Params(request);
+        Params params = new Params();
         params.withMasterTimeout(getStoredScriptRequest.masterNodeTimeout());
+        request.addParameters(params.asMap());
         return request;
     }
 
     static Request deleteScript(DeleteStoredScriptRequest deleteStoredScriptRequest) {
         String endpoint = new EndpointBuilder().addPathPartAsIs("_scripts").addPathPart(deleteStoredScriptRequest.id()).build();
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
-        Params params = new Params(request);
+        Params params = new Params();
         params.withTimeout(deleteStoredScriptRequest.timeout());
         params.withMasterTimeout(deleteStoredScriptRequest.masterNodeTimeout());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -710,10 +729,12 @@ static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType,
         return new NByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));
     }
 
+    @Deprecated
     static String endpoint(String index, String type, String id) {
         return new EndpointBuilder().addPathPart(index, type, id).build();
     }
 
+    @Deprecated
     static String endpoint(String index, String type, String id, String endpoint) {
         return new EndpointBuilder().addPathPart(index, type, id).addPathPartAsIs(endpoint).build();
     }
@@ -726,6 +747,7 @@ static String endpoint(String[] indices, String endpoint) {
         return new EndpointBuilder().addCommaSeparatedPathParts(indices).addPathPartAsIs(endpoint).build();
     }
 
+    @Deprecated
     static String endpoint(String[] indices, String[] types, String endpoint) {
         return new EndpointBuilder().addCommaSeparatedPathParts(indices).addCommaSeparatedPathParts(types)
                 .addPathPartAsIs(endpoint).build();
@@ -736,6 +758,7 @@ static String endpoint(String[] indices, String endpoint, String[] suffixes) {
                 .addCommaSeparatedPathParts(suffixes).build();
     }
 
+    @Deprecated
     static String endpoint(String[] indices, String endpoint, String type) {
         return new EndpointBuilder().addCommaSeparatedPathParts(indices).addPathPartAsIs(endpoint).addPathPart(type).build();
     }
@@ -756,15 +779,14 @@ public static ContentType createContentType(final XContentType xContentType) {
      * a {@link Request} and adds the parameters to it directly.
      */
     static class Params {
-        private final Request request;
+        private final Map<String,String> parameters = new HashMap<>();
 
-        Params(Request request) {
-            this.request = request;
+        Params() {
         }
 
         Params putParam(String name, String value) {
             if (Strings.hasLength(value)) {
-                request.addParameter(name, value);
+                parameters.put(name,value);
             }
             return this;
         }
@@ -776,6 +798,10 @@ Params putParam(String key, TimeValue value) {
             return this;
         }
 
+        Map<String, String> asMap(){
+            return parameters;
+        }
+
         Params withDocAsUpsert(boolean docAsUpsert) {
             if (docAsUpsert) {
                 return putParam("doc_as_upsert", Boolean.TRUE.toString());
@@ -939,6 +965,7 @@ Params withIndicesOptions(IndicesOptions indicesOptions) {
                     expandWildcards = joiner.toString();
                 }
                 putParam("expand_wildcards", expandWildcards);
+                putParam("ignore_throttled", Boolean.toString(indicesOptions.ignoreThrottled()));
             }
             return this;
         }
@@ -1016,13 +1043,6 @@ Params withParentTaskId(TaskId parentTaskId) {
             return this;
         }
 
-        Params withVerify(boolean verify) {
-            if (verify) {
-                return putParam("verify", Boolean.TRUE.toString());
-            }
-            return this;
-        }
-
         Params withWaitForStatus(ClusterHealthStatus status) {
             if (status != null) {
                 return putParam("wait_for_status", status.name().toLowerCase(Locale.ROOT));
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupRequestConverters.java
index 4cd3be057a9f5..a9f6ff6d5f9fb 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RollupRequestConverters.java
@@ -68,11 +68,12 @@ static Request stopJob(final StopRollupJobRequest stopRollupJobRequest) throws I
             .build();
 
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withTimeout(stopRollupJobRequest.timeout());
         if (stopRollupJobRequest.waitForCompletion() != null) {
             parameters.withWaitForCompletion(stopRollupJobRequest.waitForCompletion());
         }
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -95,16 +96,6 @@ static Request deleteJob(final DeleteRollupJobRequest deleteRollupJobRequest) th
     }
 
     static Request search(final SearchRequest request) throws IOException {
-        if (request.types().length > 0) {
-            /*
-             * Ideally we'd check this with the standard validation framework
-             * but we don't have a special request for rollup search so that'd
-             * be difficult. 
-             */
-            ValidationException ve = new ValidationException();
-            ve.addValidationError("types are not allowed in rollup search");
-            throw ve;
-        }
         return RequestConverters.search(request, "_rollup_search");
     }
 
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java
index fefb5771dc801..8c29cfaae54e1 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java
@@ -43,6 +43,8 @@
 import org.elasticsearch.client.security.EnableUserRequest;
 import org.elasticsearch.client.security.GetApiKeyRequest;
 import org.elasticsearch.client.security.GetApiKeyResponse;
+import org.elasticsearch.client.security.GetBuiltinPrivilegesRequest;
+import org.elasticsearch.client.security.GetBuiltinPrivilegesResponse;
 import org.elasticsearch.client.security.GetPrivilegesRequest;
 import org.elasticsearch.client.security.GetPrivilegesResponse;
 import org.elasticsearch.client.security.GetRoleMappingsRequest;
@@ -751,6 +753,34 @@ public void invalidateTokenAsync(InvalidateTokenRequest request, RequestOptions
             InvalidateTokenResponse::fromXContent, listener, emptySet());
     }
 
+    /**
+     * Synchronously get builtin (cluster &amp; index) privilege(s).
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-builtin-privileges.html">
+     * the docs</a> for more.
+     *
+     * @param options the request options (e.g. headers), use
+     *                {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response from the get builtin privileges call
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public GetBuiltinPrivilegesResponse getBuiltinPrivileges(final RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(GetBuiltinPrivilegesRequest.INSTANCE,
+            GetBuiltinPrivilegesRequest::getRequest, options, GetBuiltinPrivilegesResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously get builtin (cluster &amp; index) privilege(s).
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-builtin-privileges.html">
+     * the docs</a> for more.
+     *
+     * @param options  the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void getBuiltinPrivilegesAsync(final RequestOptions options, final ActionListener<GetBuiltinPrivilegesResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(GetBuiltinPrivilegesRequest.INSTANCE,
+            GetBuiltinPrivilegesRequest::getRequest, options, GetBuiltinPrivilegesResponse::fromXContent, listener, emptySet());
+    }
+
     /**
      * Synchronously get application privilege(s).
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-privileges.html">
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java
index f686167e211bb..18ecc2cea281a 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java
@@ -66,8 +66,9 @@ static Request changePassword(ChangePasswordRequest changePasswordRequest) throw
             .build();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
         request.setEntity(createEntity(changePasswordRequest, REQUEST_BODY_CONTENT_TYPE));
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withRefreshPolicy(changePasswordRequest.getRefreshPolicy());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -87,8 +88,9 @@ static Request putUser(PutUserRequest putUserRequest) throws IOException {
             .build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
         request.setEntity(createEntity(putUserRequest, REQUEST_BODY_CONTENT_TYPE));
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withRefreshPolicy(putUserRequest.getRefreshPolicy());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -98,8 +100,9 @@ static Request deleteUser(DeleteUserRequest deleteUserRequest) {
             .addPathPart(deleteUserRequest.getName())
             .build();
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withRefreshPolicy(deleteUserRequest.getRefreshPolicy());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -110,8 +113,9 @@ static Request putRoleMapping(final PutRoleMappingRequest putRoleMappingRequest)
             .build();
         final Request request = new Request(HttpPut.METHOD_NAME, endpoint);
         request.setEntity(createEntity(putRoleMappingRequest, REQUEST_BODY_CONTENT_TYPE));
-        final RequestConverters.Params params = new RequestConverters.Params(request);
+        final RequestConverters.Params params = new RequestConverters.Params();
         params.withRefreshPolicy(putRoleMappingRequest.getRefreshPolicy());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -139,8 +143,9 @@ private static Request setUserEnabled(SetUserEnabledRequest setUserEnabledReques
             .addPathPart(setUserEnabledRequest.isEnabled() ? "_enable" : "_disable")
             .build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withRefreshPolicy(setUserEnabledRequest.getRefreshPolicy());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -161,8 +166,9 @@ static Request clearRealmCache(ClearRealmCacheRequest clearRealmCacheRequest) {
         final String endpoint = builder.addPathPartAsIs("_clear_cache").build();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
         if (clearRealmCacheRequest.getUsernames().isEmpty() == false) {
-            RequestConverters.Params params = new RequestConverters.Params(request);
+            RequestConverters.Params params = new RequestConverters.Params();
             params.putParam("usernames", Strings.collectionToCommaDelimitedString(clearRealmCacheRequest.getUsernames()));
+            request.addParameters(params.asMap());
         }
         return request;
     }
@@ -182,8 +188,9 @@ static Request deleteRoleMapping(DeleteRoleMappingRequest deleteRoleMappingReque
             .addPathPart(deleteRoleMappingRequest.getName())
             .build();
         final Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
-        final RequestConverters.Params params = new RequestConverters.Params(request);
+        final RequestConverters.Params params = new RequestConverters.Params();
         params.withRefreshPolicy(deleteRoleMappingRequest.getRefreshPolicy());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -193,8 +200,9 @@ static Request deleteRole(DeleteRoleRequest deleteRoleRequest) {
             .addPathPart(deleteRoleRequest.getName())
             .build();
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withRefreshPolicy(deleteRoleRequest.getRefreshPolicy());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -231,8 +239,9 @@ static Request getPrivileges(GetPrivilegesRequest getPrivilegesRequest) {
     static Request putPrivileges(final PutPrivilegesRequest putPrivilegesRequest) throws IOException {
         Request request = new Request(HttpPut.METHOD_NAME, "/_security/privilege");
         request.setEntity(createEntity(putPrivilegesRequest, REQUEST_BODY_CONTENT_TYPE));
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withRefreshPolicy(putPrivilegesRequest.getRefreshPolicy());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -243,8 +252,9 @@ static Request deletePrivileges(DeletePrivilegesRequest deletePrivilegeRequest)
             .addCommaSeparatedPathParts(deletePrivilegeRequest.getPrivileges())
             .build();
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withRefreshPolicy(deletePrivilegeRequest.getRefreshPolicy());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -255,16 +265,18 @@ static Request putRole(final PutRoleRequest putRoleRequest) throws IOException {
             .build();
         final Request request = new Request(HttpPut.METHOD_NAME, endpoint);
         request.setEntity(createEntity(putRoleRequest, REQUEST_BODY_CONTENT_TYPE));
-        final RequestConverters.Params params = new RequestConverters.Params(request);
+        final RequestConverters.Params params = new RequestConverters.Params();
         params.withRefreshPolicy(putRoleRequest.getRefreshPolicy());
+        request.addParameters(params.asMap());
         return request;
     }
 
     static Request createApiKey(final CreateApiKeyRequest createApiKeyRequest) throws IOException {
         final Request request = new Request(HttpPost.METHOD_NAME, "/_security/api_key");
         request.setEntity(createEntity(createApiKeyRequest, REQUEST_BODY_CONTENT_TYPE));
-        final RequestConverters.Params params = new RequestConverters.Params(request);
+        final RequestConverters.Params params = new RequestConverters.Params();
         params.withRefreshPolicy(createApiKeyRequest.getRefreshPolicy());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -282,13 +294,13 @@ static Request getApiKey(final GetApiKeyRequest getApiKeyRequest) throws IOExcep
         if (Strings.hasText(getApiKeyRequest.getRealmName())) {
             request.addParameter("realm_name", getApiKeyRequest.getRealmName());
         }
+
         return request;
     }
 
     static Request invalidateApiKey(final InvalidateApiKeyRequest invalidateApiKeyRequest) throws IOException {
         final Request request = new Request(HttpDelete.METHOD_NAME, "/_security/api_key");
         request.setEntity(createEntity(invalidateApiKeyRequest, REQUEST_BODY_CONTENT_TYPE));
-        final RequestConverters.Params params = new RequestConverters.Params(request);
         return request;
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java
index 93fb10bd56136..406470ea52cda 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java
@@ -46,9 +46,10 @@ static Request getRepositories(GetRepositoriesRequest getRepositoriesRequest) {
             .build();
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withMasterTimeout(getRepositoriesRequest.masterNodeTimeout());
         parameters.withLocal(getRepositoriesRequest.local());
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -56,11 +57,13 @@ static Request createRepository(PutRepositoryRequest putRepositoryRequest) throw
         String endpoint = new RequestConverters.EndpointBuilder().addPathPart("_snapshot").addPathPart(putRepositoryRequest.name()).build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withMasterTimeout(putRepositoryRequest.masterNodeTimeout());
         parameters.withTimeout(putRepositoryRequest.timeout());
-        parameters.withVerify(putRepositoryRequest.verify());
-
+        if (putRepositoryRequest.verify() == false) {
+            parameters.putParam("verify", "false");
+        }
+        request.addParameters(parameters.asMap());
         request.setEntity(RequestConverters.createEntity(putRepositoryRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -70,9 +73,10 @@ static Request deleteRepository(DeleteRepositoryRequest deleteRepositoryRequest)
             .build();
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withMasterTimeout(deleteRepositoryRequest.masterNodeTimeout());
         parameters.withTimeout(deleteRepositoryRequest.timeout());
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -83,9 +87,10 @@ static Request verifyRepository(VerifyRepositoryRequest verifyRepositoryRequest)
             .build();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withMasterTimeout(verifyRepositoryRequest.masterNodeTimeout());
         parameters.withTimeout(verifyRepositoryRequest.timeout());
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -95,16 +100,17 @@ static Request createSnapshot(CreateSnapshotRequest createSnapshotRequest) throw
             .addPathPart(createSnapshotRequest.snapshot())
             .build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withMasterTimeout(createSnapshotRequest.masterNodeTimeout());
         params.withWaitForCompletion(createSnapshotRequest.waitForCompletion());
+        request.addParameters(params.asMap());
         request.setEntity(RequestConverters.createEntity(createSnapshotRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
 
     static Request getSnapshots(GetSnapshotsRequest getSnapshotsRequest) {
         RequestConverters.EndpointBuilder endpointBuilder = new RequestConverters.EndpointBuilder().addPathPartAsIs("_snapshot")
-            .addPathPart(getSnapshotsRequest.repository());
+            .addCommaSeparatedPathParts(getSnapshotsRequest.repositories());
         String endpoint;
         if (getSnapshotsRequest.snapshots().length == 0) {
             endpoint = endpointBuilder.addPathPart("_all").build();
@@ -114,11 +120,11 @@ static Request getSnapshots(GetSnapshotsRequest getSnapshotsRequest) {
 
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withMasterTimeout(getSnapshotsRequest.masterNodeTimeout());
         parameters.putParam("ignore_unavailable", Boolean.toString(getSnapshotsRequest.ignoreUnavailable()));
         parameters.putParam("verbose", Boolean.toString(getSnapshotsRequest.verbose()));
-
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -130,9 +136,10 @@ static Request snapshotsStatus(SnapshotsStatusRequest snapshotsStatusRequest) {
             .build();
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withMasterTimeout(snapshotsStatusRequest.masterNodeTimeout());
         parameters.withIgnoreUnavailable(snapshotsStatusRequest.ignoreUnavailable());
+        request.addParameters(parameters.asMap());
         return request;
     }
 
@@ -143,9 +150,10 @@ static Request restoreSnapshot(RestoreSnapshotRequest restoreSnapshotRequest) th
             .addPathPartAsIs("_restore")
             .build();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withMasterTimeout(restoreSnapshotRequest.masterNodeTimeout());
         parameters.withWaitForCompletion(restoreSnapshotRequest.waitForCompletion());
+        request.addParameters(parameters.asMap());
         request.setEntity(RequestConverters.createEntity(restoreSnapshotRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE));
         return request;
     }
@@ -157,8 +165,9 @@ static Request deleteSnapshot(DeleteSnapshotRequest deleteSnapshotRequest) {
             .build();
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
 
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withMasterTimeout(deleteSnapshotRequest.masterNodeTimeout());
+        request.addParameters(parameters.asMap());
         return request;
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/TasksRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/TasksRequestConverters.java
index f0e9cf4e025f6..f30efabc823e3 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/TasksRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/TasksRequestConverters.java
@@ -32,12 +32,13 @@ private TasksRequestConverters() {}
 
     static Request cancelTasks(CancelTasksRequest cancelTasksRequest) {
         Request request = new Request(HttpPost.METHOD_NAME, "/_tasks/_cancel");
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withTimeout(cancelTasksRequest.getTimeout())
             .withTaskId(cancelTasksRequest.getTaskId())
             .withNodes(cancelTasksRequest.getNodes())
             .withParentTaskId(cancelTasksRequest.getParentTaskId())
             .withActions(cancelTasksRequest.getActions());
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -46,7 +47,7 @@ static Request listTasks(ListTasksRequest listTaskRequest) {
             throw new IllegalArgumentException("TaskId cannot be used for list tasks request");
         }
         Request request  = new Request(HttpGet.METHOD_NAME, "/_tasks");
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withTimeout(listTaskRequest.getTimeout())
             .withDetailed(listTaskRequest.getDetailed())
             .withWaitForCompletion(listTaskRequest.getWaitForCompletion())
@@ -54,6 +55,7 @@ static Request listTasks(ListTasksRequest listTaskRequest) {
             .withNodes(listTaskRequest.getNodes())
             .withActions(listTaskRequest.getActions())
             .putParam("group_by", "none");
+        request.addParameters(params.asMap());
         return request;
     }
 
@@ -62,9 +64,10 @@ static Request getTask(GetTaskRequest getTaskRequest) {
                 .addPathPartAsIs(getTaskRequest.getNodeId() + ":" + Long.toString(getTaskRequest.getTaskId()))
                 .build();
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         params.withTimeout(getTaskRequest.getTimeout())
             .withWaitForCompletion(getTaskRequest.getWaitForCompletion());
+        request.addParameters(params.asMap());
         return request;
     }
     
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherRequestConverters.java
index 9718607d8b80e..84559fb182306 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherRequestConverters.java
@@ -69,12 +69,13 @@ static Request putWatch(PutWatchRequest putWatchRequest) {
             .build();
 
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request)
+        RequestConverters.Params params = new RequestConverters.Params()
             .withIfSeqNo(putWatchRequest.ifSeqNo())
             .withIfPrimaryTerm(putWatchRequest.ifPrimaryTerm());
         if (putWatchRequest.isActive() == false) {
             params.putParam("active", "false");
         }
+        request.addParameters(params.asMap());
         ContentType contentType = RequestConverters.createContentType(putWatchRequest.xContentType());
         BytesReference source = putWatchRequest.getSource();
         request.setEntity(new NByteArrayEntity(source.toBytesRef().bytes, 0, source.length(), contentType));
@@ -118,7 +119,7 @@ static Request executeWatch(ExecuteWatchRequest executeWatchRequest) throws IOEx
             .addPathPartAsIs("_execute").build();
 
         Request request = new Request(HttpPost.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request);
+        RequestConverters.Params params = new RequestConverters.Params();
         if (executeWatchRequest.isDebug()) {
             params.putParam("debug", "true");
         }
@@ -128,7 +129,7 @@ static Request executeWatch(ExecuteWatchRequest executeWatchRequest) throws IOEx
         if (executeWatchRequest.recordExecution()) {
             params.putParam("record_execution", "true");
         }
-
+        request.addParameters(params.asMap());
         request.setEntity(RequestConverters.createEntity(executeWatchRequest, XContentType.JSON));
         return request;
     }
@@ -158,7 +159,7 @@ static Request watcherStats(WatcherStatsRequest watcherStatsRequest) {
         RequestConverters.EndpointBuilder builder = new RequestConverters.EndpointBuilder().addPathPartAsIs("_watcher", "stats");
         String endpoint = builder.build();
         Request request = new Request(HttpGet.METHOD_NAME, endpoint);
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         StringBuilder metric = new StringBuilder();
         if (watcherStatsRequest.includeCurrentWatches()) {
             metric.append("current_watches");
@@ -172,6 +173,7 @@ static Request watcherStats(WatcherStatsRequest watcherStatsRequest) {
         if (metric.length() > 0) {
             parameters.putParam("metric", metric.toString());
         }
+        request.addParameters(parameters.asMap());
         return request;
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
index a20dfd1ba328a..69cdd329e3984 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java
@@ -35,8 +35,8 @@
  * default distribution of Elasticsearch. All of these APIs will 404 if run
  * against the OSS distribution of Elasticsearch.
  * <p>
- * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/xpack-api.html">
- * X-Pack APIs on elastic.co</a> for more information.
+ * See the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/rest-apis.html">
+ * REST APIs on elastic.co</a> for more information.
  */
 public final class XPackClient {
 
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackRequestConverters.java
index 9e0c1527403d5..e4742e8c1cbe2 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackRequestConverters.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackRequestConverters.java
@@ -46,8 +46,9 @@ static Request info(XPackInfoRequest infoRequest) {
 
     static Request usage(XPackUsageRequest usageRequest) {
         Request request = new Request(HttpGet.METHOD_NAME, "/_xpack/usage");
-        RequestConverters.Params parameters = new RequestConverters.Params(request);
+        RequestConverters.Params parameters = new RequestConverters.Params();
         parameters.withMasterTimeout(usageRequest.masterNodeTimeout());
+        request.addParameters(parameters.asMap());
         return request;
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/IndicesFollowStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/IndicesFollowStats.java
index 7d3af08577b16..ae6217d167447 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/IndicesFollowStats.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/IndicesFollowStats.java
@@ -101,6 +101,7 @@ public static final class ShardFollowStats {
         static final ParseField WRITE_BUFFER_SIZE_IN_BYTES_FIELD = new ParseField("write_buffer_size_in_bytes");
         static final ParseField FOLLOWER_MAPPING_VERSION_FIELD = new ParseField("follower_mapping_version");
         static final ParseField FOLLOWER_SETTINGS_VERSION_FIELD = new ParseField("follower_settings_version");
+        static final ParseField FOLLOWER_ALIASES_VERSION_FIELD = new ParseField("follower_aliases_version");
         static final ParseField TOTAL_READ_TIME_MILLIS_FIELD = new ParseField("total_read_time_millis");
         static final ParseField TOTAL_READ_REMOTE_EXEC_TIME_MILLIS_FIELD = new ParseField("total_read_remote_exec_time_millis");
         static final ParseField SUCCESSFUL_READ_REQUESTS_FIELD = new ParseField("successful_read_requests");
@@ -117,41 +118,42 @@ public static final class ShardFollowStats {
 
         @SuppressWarnings("unchecked")
         static final ConstructingObjectParser<ShardFollowStats, Void> PARSER =
-            new ConstructingObjectParser<>(
-                "shard-follow-stats",
-                true,
-                args -> new ShardFollowStats(
-                    (String) args[0],
-                    (String) args[1],
-                    (String) args[2],
-                    (int) args[3],
-                    (long) args[4],
-                    (long) args[5],
-                    (long) args[6],
-                    (long) args[7],
-                    (long) args[8],
-                    (int) args[9],
-                    (int) args[10],
-                    (int) args[11],
-                    (long) args[12],
-                    (long) args[13],
-                    (long) args[14],
-                    (long) args[15],
-                    (long) args[16],
-                    (long) args[17],
-                    (long) args[18],
-                    (long) args[19],
-                    (long) args[20],
-                    (long) args[21],
-                    (long) args[22],
-                    (long) args[23],
-                    (long) args[24],
-                    (long) args[25],
-                    new TreeMap<>(
-                        ((List<Map.Entry<Long, Tuple<Integer, ElasticsearchException>>>) args[26])
-                            .stream()
-                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))),
-                    (ElasticsearchException) args[27]));
+                new ConstructingObjectParser<>(
+                        "shard-follow-stats",
+                        true,
+                        args -> new ShardFollowStats(
+                                (String) args[0],
+                                (String) args[1],
+                                (String) args[2],
+                                (int) args[3],
+                                (long) args[4],
+                                (long) args[5],
+                                (long) args[6],
+                                (long) args[7],
+                                (long) args[8],
+                                (int) args[9],
+                                (int) args[10],
+                                (int) args[11],
+                                (long) args[12],
+                                (long) args[13],
+                                (long) args[14],
+                                (long) args[15],
+                                (long) args[16],
+                                (long) args[17],
+                                (long) args[18],
+                                (long) args[19],
+                                (long) args[20],
+                                (long) args[21],
+                                (long) args[22],
+                                (long) args[23],
+                                (long) args[24],
+                                (long) args[25],
+                                (long) args[26],
+                                new TreeMap<>(
+                                        ((List<Map.Entry<Long, Tuple<Integer, ElasticsearchException>>>) args[27])
+                                                .stream()
+                                                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))),
+                                (ElasticsearchException) args[28]));
 
         static final ConstructingObjectParser<Map.Entry<Long, Tuple<Integer, ElasticsearchException>>, Void> READ_EXCEPTIONS_ENTRY_PARSER =
             new ConstructingObjectParser<>(
@@ -175,6 +177,7 @@ public static final class ShardFollowStats {
             PARSER.declareLong(ConstructingObjectParser.constructorArg(), WRITE_BUFFER_SIZE_IN_BYTES_FIELD);
             PARSER.declareLong(ConstructingObjectParser.constructorArg(), FOLLOWER_MAPPING_VERSION_FIELD);
             PARSER.declareLong(ConstructingObjectParser.constructorArg(), FOLLOWER_SETTINGS_VERSION_FIELD);
+            PARSER.declareLong(ConstructingObjectParser.constructorArg(), FOLLOWER_ALIASES_VERSION_FIELD);
             PARSER.declareLong(ConstructingObjectParser.constructorArg(), TOTAL_READ_TIME_MILLIS_FIELD);
             PARSER.declareLong(ConstructingObjectParser.constructorArg(), TOTAL_READ_REMOTE_EXEC_TIME_MILLIS_FIELD);
             PARSER.declareLong(ConstructingObjectParser.constructorArg(), SUCCESSFUL_READ_REQUESTS_FIELD);
@@ -220,6 +223,7 @@ public static final class ShardFollowStats {
         private final long writeBufferSizeInBytes;
         private final long followerMappingVersion;
         private final long followerSettingsVersion;
+        private final long followerAliasesVersion;
         private final long totalReadTimeMillis;
         private final long totalReadRemoteExecTimeMillis;
         private final long successfulReadRequests;
@@ -249,6 +253,7 @@ public static final class ShardFollowStats {
                          long writeBufferSizeInBytes,
                          long followerMappingVersion,
                          long followerSettingsVersion,
+                         long followerAliasesVersion,
                          long totalReadTimeMillis,
                          long totalReadRemoteExecTimeMillis,
                          long successfulReadRequests,
@@ -277,6 +282,7 @@ public static final class ShardFollowStats {
             this.writeBufferSizeInBytes = writeBufferSizeInBytes;
             this.followerMappingVersion = followerMappingVersion;
             this.followerSettingsVersion = followerSettingsVersion;
+            this.followerAliasesVersion = followerAliasesVersion;
             this.totalReadTimeMillis = totalReadTimeMillis;
             this.totalReadRemoteExecTimeMillis = totalReadRemoteExecTimeMillis;
             this.successfulReadRequests = successfulReadRequests;
@@ -352,6 +358,10 @@ public long getFollowerSettingsVersion() {
             return followerSettingsVersion;
         }
 
+        public long getFollowerAliasesVersion() {
+            return followerAliasesVersion;
+        }
+
         public long getTotalReadTimeMillis() {
             return totalReadTimeMillis;
         }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/BroadcastResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/BroadcastResponse.java
index 3665ba5bf5009..35ce0f55d717b 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/BroadcastResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/BroadcastResponse.java
@@ -45,7 +45,7 @@ public Shards shards() {
         return shards;
     }
 
-    BroadcastResponse(final Shards shards) {
+    protected BroadcastResponse(final Shards shards) {
         this.shards = Objects.requireNonNull(shards);
     }
 
@@ -56,7 +56,7 @@ public Shards shards() {
             a -> new BroadcastResponse((Shards) a[0]));
 
     static {
-        PARSER.declareObject(ConstructingObjectParser.constructorArg(), Shards.SHARDS_PARSER, SHARDS_FIELD);
+        declareShardsField(PARSER);
     }
 
     /**
@@ -70,6 +70,10 @@ public static BroadcastResponse fromXContent(final XContentParser parser) throws
         return PARSER.parse(parser, null);
     }
 
+    protected static <T extends BroadcastResponse> void declareShardsField(ConstructingObjectParser<T, Void> PARSER) {
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), Shards.SHARDS_PARSER, SHARDS_FIELD);
+    }
+
     /**
      * Represents the results of a collection of shards on which a request was executed against.
      */
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/TermVectorsResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/TermVectorsResponse.java
index eef76ed7cb043..8f4232cca6e2c 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/TermVectorsResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/TermVectorsResponse.java
@@ -33,17 +33,14 @@
 
 public class TermVectorsResponse {
     private final String index;
-    private final String type;
     private final String id;
     private final long docVersion;
     private final boolean found;
     private final long tookInMillis;
     private final List<TermVector> termVectorList;
 
-    public TermVectorsResponse(
-            String index, String type, String id, long version, boolean found, long tookInMillis, List<TermVector> termVectorList) {
+    public TermVectorsResponse(String index, String id, long version, boolean found, long tookInMillis, List<TermVector> termVectorList) {
         this.index = index;
-        this.type = type;
         this.id = id;
         this.docVersion = version;
         this.found = found;
@@ -53,18 +50,17 @@ public TermVectorsResponse(
 
     private static ConstructingObjectParser<TermVectorsResponse, Void> PARSER = new ConstructingObjectParser<>("term_vectors", true,
         args -> {
-            // as the response comes from server, we are sure that args[6] will be a list of TermVector
-            @SuppressWarnings("unchecked") List<TermVector> termVectorList = (List<TermVector>) args[6];
+            // as the response comes from server, we are sure that args[5] will be a list of TermVector
+            @SuppressWarnings("unchecked") List<TermVector> termVectorList = (List<TermVector>) args[5];
             if (termVectorList != null) {
                 Collections.sort(termVectorList, Comparator.comparing(TermVector::getFieldName));
             }
             return new TermVectorsResponse(
                 (String) args[0],
                 (String) args[1],
-                (String) args[2],
-                (long) args[3],
-                (boolean) args[4],
-                (long) args[5],
+                (long) args[2],
+                (boolean) args[3],
+                (long) args[4],
                 termVectorList
             );
         }
@@ -72,7 +68,6 @@ public TermVectorsResponse(
 
     static {
         PARSER.declareString(constructorArg(), new ParseField("_index"));
-        PARSER.declareString(constructorArg(), new ParseField("_type"));
         PARSER.declareString(optionalConstructorArg(), new ParseField("_id"));
         PARSER.declareLong(constructorArg(), new ParseField("_version"));
         PARSER.declareBoolean(constructorArg(), new ParseField("found"));
@@ -92,16 +87,6 @@ public String getIndex() {
         return index;
     }
 
-    /**
-     * Returns the type for the response
-     *
-     * @deprecated Types are in the process of being removed.
-     */
-    @Deprecated
-    public String getType() {
-        return type;
-    }
-
     /**
      * Returns the id of the request
      * can be NULL if there is no document ID
@@ -145,7 +130,6 @@ public boolean equals(Object obj) {
         if (!(obj instanceof TermVectorsResponse)) return false;
         TermVectorsResponse other = (TermVectorsResponse) obj;
         return index.equals(other.index)
-            && type.equals(other.type)
             && Objects.equals(id, other.id)
             && docVersion == other.docVersion
             && found == other.found
@@ -155,7 +139,7 @@ public boolean equals(Object obj) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(index, type, id, docVersion, found, tookInMillis, termVectorList);
+        return Objects.hash(index, id, docVersion, found, tookInMillis, termVectorList);
     }
 
 
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/DocValuesWhitelistExtension.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/DataFrameNamedXContentProvider.java
similarity index 52%
rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/query/DocValuesWhitelistExtension.java
rename to client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/DataFrameNamedXContentProvider.java
index f463135d69f71..940b136c93daa 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/DocValuesWhitelistExtension.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/DataFrameNamedXContentProvider.java
@@ -17,26 +17,25 @@
  * under the License.
  */
 
-package org.elasticsearch.index.query;
+package org.elasticsearch.client.dataframe;
 
+import org.elasticsearch.client.dataframe.transforms.SyncConfig;
+import org.elasticsearch.client.dataframe.transforms.TimeSyncConfig;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.plugins.spi.NamedXContentProvider;
 
-import org.elasticsearch.painless.spi.PainlessExtension;
-import org.elasticsearch.painless.spi.Whitelist;
-import org.elasticsearch.painless.spi.WhitelistLoader;
-import org.elasticsearch.script.ScoreScript;
-import org.elasticsearch.script.ScriptContext;
-
-import java.util.Collections;
+import java.util.Arrays;
 import java.util.List;
-import java.util.Map;
-
-public class DocValuesWhitelistExtension implements PainlessExtension {
 
-    private static final Whitelist WHITELIST =
-        WhitelistLoader.loadFromResourceFiles(DocValuesWhitelistExtension.class, "docvalues_whitelist.txt");
+public class DataFrameNamedXContentProvider implements NamedXContentProvider {
 
     @Override
-    public Map<ScriptContext<?>, List<Whitelist>> getContextWhitelists() {
-        return Collections.singletonMap(ScoreScript.CONTEXT, Collections.singletonList(WHITELIST));
+    public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
+        return Arrays.asList(
+                new NamedXContentRegistry.Entry(SyncConfig.class,
+                        new ParseField(TimeSyncConfig.NAME),
+                        TimeSyncConfig::fromXContent));
     }
+
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/GetDataFrameTransformRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/GetDataFrameTransformRequest.java
index c50f37a27c885..cc69e0bd4cd4e 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/GetDataFrameTransformRequest.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/GetDataFrameTransformRequest.java
@@ -30,6 +30,7 @@
 
 public class GetDataFrameTransformRequest implements Validatable {
 
+    public static final String ALLOW_NO_MATCH = "allow_no_match";
     /**
      * Helper method to create a request that will get ALL Data Frame Transforms
      * @return new {@link GetDataFrameTransformRequest} object for the id "_all"
@@ -40,6 +41,7 @@ public static GetDataFrameTransformRequest getAllDataFrameTransformsRequest() {
 
     private final List<String> ids;
     private PageParams pageParams;
+    private Boolean allowNoMatch;
 
     public GetDataFrameTransformRequest(String... ids) {
         this.ids = Arrays.asList(ids);
@@ -57,6 +59,14 @@ public void setPageParams(PageParams pageParams) {
         this.pageParams = pageParams;
     }
 
+    public Boolean getAllowNoMatch() {
+        return allowNoMatch;
+    }
+
+    public void setAllowNoMatch(Boolean allowNoMatch) {
+        this.allowNoMatch = allowNoMatch;
+    }
+
     @Override
     public Optional<ValidationException> validate() {
         if (ids == null || ids.isEmpty()) {
@@ -70,7 +80,7 @@ public Optional<ValidationException> validate() {
 
     @Override
     public int hashCode() {
-        return Objects.hash(ids, pageParams);
+        return Objects.hash(ids, pageParams, allowNoMatch);
     }
 
     @Override
@@ -83,6 +93,8 @@ public boolean equals(Object obj) {
             return false;
         }
         GetDataFrameTransformRequest other = (GetDataFrameTransformRequest) obj;
-        return Objects.equals(ids, other.ids) && Objects.equals(pageParams, other.pageParams);
+        return Objects.equals(ids, other.ids)
+            && Objects.equals(pageParams, other.pageParams)
+            && Objects.equals(allowNoMatch, other.allowNoMatch);
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsRequest.java
index 4a105f7b40c7e..7522ae0d67c26 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsRequest.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/GetDataFrameTransformStatsRequest.java
@@ -29,6 +29,7 @@
 public class GetDataFrameTransformStatsRequest implements Validatable {
     private final String id;
     private PageParams pageParams;
+    private Boolean allowNoMatch;
 
     public GetDataFrameTransformStatsRequest(String id) {
         this.id = id;
@@ -46,6 +47,14 @@ public void setPageParams(PageParams pageParams) {
         this.pageParams = pageParams;
     }
 
+    public Boolean getAllowNoMatch() {
+        return allowNoMatch;
+    }
+
+    public void setAllowNoMatch(Boolean allowNoMatch) {
+        this.allowNoMatch = allowNoMatch;
+    }
+
     @Override
     public Optional<ValidationException> validate() {
         if (id == null) {
@@ -59,7 +68,7 @@ public Optional<ValidationException> validate() {
 
     @Override
     public int hashCode() {
-        return Objects.hash(id, pageParams);
+        return Objects.hash(id, pageParams, allowNoMatch);
     }
 
     @Override
@@ -72,6 +81,8 @@ public boolean equals(Object obj) {
             return false;
         }
         GetDataFrameTransformStatsRequest other = (GetDataFrameTransformStatsRequest) obj;
-        return Objects.equals(id, other.id) && Objects.equals(pageParams, other.pageParams);
+        return Objects.equals(id, other.id)
+            && Objects.equals(pageParams, other.pageParams)
+            && Objects.equals(allowNoMatch, other.allowNoMatch);
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponse.java
index 5d5f67dd65ed2..40e87b5768b5d 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponse.java
@@ -29,23 +29,32 @@
 public class PreviewDataFrameTransformResponse {
 
     private static final String PREVIEW = "preview";
+    private static final String MAPPINGS = "mappings";
 
     @SuppressWarnings("unchecked")
     public static PreviewDataFrameTransformResponse fromXContent(final XContentParser parser) throws IOException {
-        Object previewDocs = parser.map().get(PREVIEW);
-        return new PreviewDataFrameTransformResponse((List<Map<String, Object>>) previewDocs);
+        Map<String, Object> previewMap = parser.mapOrdered();
+        Object previewDocs = previewMap.get(PREVIEW);
+        Object mappings = previewMap.get(MAPPINGS);
+        return new PreviewDataFrameTransformResponse((List<Map<String, Object>>) previewDocs, (Map<String, Object>) mappings);
     }
 
     private List<Map<String, Object>> docs;
+    private Map<String, Object> mappings;
 
-    public PreviewDataFrameTransformResponse(List<Map<String, Object>> docs) {
+    public PreviewDataFrameTransformResponse(List<Map<String, Object>> docs, Map<String, Object> mappings) {
         this.docs = docs;
+        this.mappings = mappings;
     }
 
     public List<Map<String, Object>> getDocs() {
         return docs;
     }
 
+    public Map<String, Object> getMappings() {
+        return mappings;
+    }
+
     @Override
     public boolean equals(Object obj) {
         if (obj == this) {
@@ -57,12 +66,12 @@ public boolean equals(Object obj) {
         }
 
         PreviewDataFrameTransformResponse other = (PreviewDataFrameTransformResponse) obj;
-        return Objects.equals(other.docs, docs);
+        return Objects.equals(other.docs, docs) && Objects.equals(other.mappings, mappings);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hashCode(docs);
+        return Objects.hash(docs, mappings);
     }
 
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/StartDataFrameTransformResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/StartDataFrameTransformResponse.java
index f11ecd096c16e..9b358ffdfa151 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/StartDataFrameTransformResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/StartDataFrameTransformResponse.java
@@ -30,21 +30,19 @@
 
 public class StartDataFrameTransformResponse extends AcknowledgedTasksResponse {
 
-    private static final String STARTED = "started";
+    private static final String ACKNOWLEDGED = "acknowledged";
 
     private static final ConstructingObjectParser<StartDataFrameTransformResponse, Void> PARSER =
-            AcknowledgedTasksResponse.generateParser("start_data_frame_transform_response", StartDataFrameTransformResponse::new, STARTED);
+            AcknowledgedTasksResponse.generateParser("start_data_frame_transform_response", StartDataFrameTransformResponse::new,
+                    ACKNOWLEDGED);
 
     public static StartDataFrameTransformResponse fromXContent(final XContentParser parser) throws IOException {
         return PARSER.parse(parser, null);
     }
 
-    public StartDataFrameTransformResponse(boolean started, @Nullable List<TaskOperationFailure> taskFailures,
+    public StartDataFrameTransformResponse(boolean acknowledged, @Nullable List<TaskOperationFailure> taskFailures,
                                           @Nullable List<? extends ElasticsearchException> nodeFailures) {
-        super(started, taskFailures, nodeFailures);
+        super(acknowledged, taskFailures, nodeFailures);
     }
 
-    public boolean isStarted() {
-        return isAcknowledged();
-    }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/StopDataFrameTransformRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/StopDataFrameTransformRequest.java
index 0bc690ad79076..4fb6164f2cca9 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/StopDataFrameTransformRequest.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/StopDataFrameTransformRequest.java
@@ -31,6 +31,7 @@ public class StopDataFrameTransformRequest implements Validatable {
     private final String id;
     private Boolean waitForCompletion;
     private TimeValue timeout;
+    private Boolean allowNoMatch;
 
     public StopDataFrameTransformRequest(String id) {
         this.id = id;
@@ -64,6 +65,14 @@ public TimeValue getTimeout() {
         return timeout;
     }
 
+    public Boolean getAllowNoMatch() {
+        return allowNoMatch;
+    }
+
+    public void setAllowNoMatch(Boolean allowNoMatch) {
+        this.allowNoMatch = allowNoMatch;
+    }
+
     @Override
     public Optional<ValidationException> validate() {
         if (id == null) {
@@ -77,7 +86,7 @@ public Optional<ValidationException> validate() {
 
     @Override
     public int hashCode() {
-        return Objects.hash(id, waitForCompletion, timeout);
+        return Objects.hash(id, waitForCompletion, timeout, allowNoMatch);
     }
 
     @Override
@@ -92,7 +101,8 @@ public boolean equals(Object obj) {
         StopDataFrameTransformRequest other = (StopDataFrameTransformRequest) obj;
         return Objects.equals(this.id, other.id)
                 && Objects.equals(this.waitForCompletion, other.waitForCompletion)
-                && Objects.equals(this.timeout, other.timeout);
+                && Objects.equals(this.timeout, other.timeout)
+                && Objects.equals(this.allowNoMatch, other.allowNoMatch);
     }
 
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/StopDataFrameTransformResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/StopDataFrameTransformResponse.java
index 3224dfb4703e5..6d32474f70c1a 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/StopDataFrameTransformResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/StopDataFrameTransformResponse.java
@@ -30,21 +30,18 @@
 
 public class StopDataFrameTransformResponse extends AcknowledgedTasksResponse {
 
-    private static final String STOPPED = "stopped";
+    private static final String ACKNOWLEDGED = "acknowledged";
 
-    private static final ConstructingObjectParser<StopDataFrameTransformResponse, Void> PARSER =
-            AcknowledgedTasksResponse.generateParser("stop_data_frame_transform_response", StopDataFrameTransformResponse::new, STOPPED);
+    private static final ConstructingObjectParser<StopDataFrameTransformResponse, Void> PARSER = AcknowledgedTasksResponse
+            .generateParser("stop_data_frame_transform_response", StopDataFrameTransformResponse::new, ACKNOWLEDGED);
 
     public static StopDataFrameTransformResponse fromXContent(final XContentParser parser) throws IOException {
         return PARSER.parse(parser, null);
     }
 
-    public StopDataFrameTransformResponse(boolean stopped, @Nullable List<TaskOperationFailure> taskFailures,
+    public StopDataFrameTransformResponse(boolean acknowledged, @Nullable List<TaskOperationFailure> taskFailures,
                                           @Nullable List<? extends ElasticsearchException> nodeFailures) {
-        super(stopped, taskFailures, nodeFailures);
+        super(acknowledged, taskFailures, nodeFailures);
     }
 
-    public boolean isStopped() {
-        return isAcknowledged();
-    }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java
index 8465ae8342827..355e3ad9bbc0f 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java
@@ -19,16 +19,21 @@
 
 package org.elasticsearch.client.dataframe.transforms;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.client.dataframe.transforms.pivot.PivotConfig;
+import org.elasticsearch.client.dataframe.transforms.util.TimeUtil;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.Objects;
 
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
@@ -40,14 +45,20 @@ public class DataFrameTransformConfig implements ToXContentObject {
     public static final ParseField SOURCE = new ParseField("source");
     public static final ParseField DEST = new ParseField("dest");
     public static final ParseField DESCRIPTION = new ParseField("description");
+    public static final ParseField SYNC = new ParseField("sync");
+    public static final ParseField VERSION = new ParseField("version");
+    public static final ParseField CREATE_TIME = new ParseField("create_time");
     // types of transforms
     public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot");
 
     private final String id;
     private final SourceConfig source;
     private final DestConfig dest;
+    private final SyncConfig syncConfig;
     private final PivotConfig pivotConfig;
     private final String description;
+    private final Version transformVersion;
+    private final Instant createTime;
 
     public static final ConstructingObjectParser<DataFrameTransformConfig, Void> PARSER =
             new ConstructingObjectParser<>("data_frame_transform", true,
@@ -55,19 +66,42 @@ public class DataFrameTransformConfig implements ToXContentObject {
                     String id = (String) args[0];
                     SourceConfig source = (SourceConfig) args[1];
                     DestConfig dest = (DestConfig) args[2];
-                    PivotConfig pivotConfig = (PivotConfig) args[3];
-                    String description = (String)args[4];
-                    return new DataFrameTransformConfig(id, source, dest, pivotConfig, description);
+                    SyncConfig syncConfig = (SyncConfig) args[3];
+                    PivotConfig pivotConfig = (PivotConfig) args[4];
+                    String description = (String)args[5];
+                    Instant createTime = (Instant)args[6];
+                    String transformVersion = (String)args[7];
+                    return new DataFrameTransformConfig(id,
+                        source,
+                        dest,
+                        syncConfig,
+                        pivotConfig,
+                        description,
+                        createTime,
+                        transformVersion);
                 });
 
     static {
         PARSER.declareString(constructorArg(), ID);
         PARSER.declareObject(constructorArg(), (p, c) -> SourceConfig.PARSER.apply(p, null), SOURCE);
         PARSER.declareObject(constructorArg(), (p, c) -> DestConfig.PARSER.apply(p, null), DEST);
+        PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseSyncConfig(p), SYNC);
         PARSER.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p), PIVOT_TRANSFORM);
         PARSER.declareString(optionalConstructorArg(), DESCRIPTION);
+        PARSER.declareField(optionalConstructorArg(),
+            p -> TimeUtil.parseTimeFieldToInstant(p, CREATE_TIME.getPreferredName()), CREATE_TIME, ObjectParser.ValueType.VALUE);
+        PARSER.declareString(optionalConstructorArg(), VERSION);
     }
 
+    private static SyncConfig parseSyncConfig(XContentParser parser) throws IOException {
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+        SyncConfig syncConfig = parser.namedObject(SyncConfig.class, parser.currentName(), true);
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        return syncConfig;
+    }
+
+
     public static DataFrameTransformConfig fromXContent(final XContentParser parser) {
         return PARSER.apply(parser, null);
     }
@@ -84,19 +118,25 @@ public static DataFrameTransformConfig fromXContent(final XContentParser parser)
      * @return A DataFrameTransformConfig to preview, NOTE it will have a {@code null} id, destination and index.
      */
     public static DataFrameTransformConfig forPreview(final SourceConfig source, final PivotConfig pivotConfig) {
-        return new DataFrameTransformConfig(null, source, null, pivotConfig, null);
+        return new DataFrameTransformConfig(null, source, null, null, pivotConfig, null, null, null);
     }
 
     DataFrameTransformConfig(final String id,
                              final SourceConfig source,
                              final DestConfig dest,
+                             final SyncConfig syncConfig,
                              final PivotConfig pivotConfig,
-                             final String description) {
+                             final String description,
+                             final Instant createTime,
+                             final String version) {
         this.id = id;
         this.source = source;
         this.dest = dest;
+        this.syncConfig = syncConfig;
         this.pivotConfig = pivotConfig;
         this.description = description;
+        this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli());
+        this.transformVersion = version == null ? null : Version.fromString(version);
     }
 
     public String getId() {
@@ -111,10 +151,22 @@ public DestConfig getDestination() {
         return dest;
     }
 
+    public SyncConfig getSyncConfig() {
+        return syncConfig;
+    }
+
     public PivotConfig getPivotConfig() {
         return pivotConfig;
     }
 
+    public Version getVersion() {
+        return transformVersion;
+    }
+
+    public Instant getCreateTime() {
+        return createTime;
+    }
+
     @Nullable
     public String getDescription() {
         return description;
@@ -132,12 +184,23 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
         if (dest != null) {
             builder.field(DEST.getPreferredName(), dest);
         }
+        if (syncConfig != null) {
+            builder.startObject(SYNC.getPreferredName());
+            builder.field(syncConfig.getName(), syncConfig);
+            builder.endObject();
+        }
         if (pivotConfig != null) {
             builder.field(PIVOT_TRANSFORM.getPreferredName(), pivotConfig);
         }
         if (description != null) {
             builder.field(DESCRIPTION.getPreferredName(), description);
         }
+        if (createTime != null) {
+            builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli());
+        }
+        if (transformVersion != null) {
+            builder.field(VERSION.getPreferredName(), transformVersion);
+        }
         builder.endObject();
         return builder;
     }
@@ -155,15 +218,18 @@ public boolean equals(Object other) {
         final DataFrameTransformConfig that = (DataFrameTransformConfig) other;
 
         return Objects.equals(this.id, that.id)
-                && Objects.equals(this.source, that.source)
-                && Objects.equals(this.dest, that.dest)
-                && Objects.equals(this.description, that.description)
-                && Objects.equals(this.pivotConfig, that.pivotConfig);
+            && Objects.equals(this.source, that.source)
+            && Objects.equals(this.dest, that.dest)
+            && Objects.equals(this.description, that.description)
+            && Objects.equals(this.syncConfig, that.syncConfig)
+            && Objects.equals(this.transformVersion, that.transformVersion)
+            && Objects.equals(this.createTime, that.createTime)
+            && Objects.equals(this.pivotConfig, that.pivotConfig);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(id, source, dest, pivotConfig, description);
+        return Objects.hash(id, source, dest, syncConfig, pivotConfig, description);
     }
 
     @Override
@@ -180,6 +246,7 @@ public static class Builder {
         private String id;
         private SourceConfig source;
         private DestConfig dest;
+        private SyncConfig syncConfig;
         private PivotConfig pivotConfig;
         private String description;
 
@@ -198,6 +265,11 @@ public Builder setDest(DestConfig dest) {
             return this;
         }
 
+        public Builder setSyncConfig(SyncConfig syncConfig) {
+            this.syncConfig = syncConfig;
+            return this;
+        }
+
         public Builder setPivotConfig(PivotConfig pivotConfig) {
             this.pivotConfig = pivotConfig;
             return this;
@@ -209,7 +281,7 @@ public Builder setDescription(String description) {
         }
 
         public DataFrameTransformConfig build() {
-            return new DataFrameTransformConfig(id, source, dest, pivotConfig, description);
+            return new DataFrameTransformConfig(id, source, dest, syncConfig, pivotConfig, description, null, null);
         }
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformState.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformState.java
index 352cbfb67fcde..186c67bf42ce2 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformState.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformState.java
@@ -43,6 +43,7 @@ public class DataFrameTransformState {
     private static final ParseField CHECKPOINT = new ParseField("checkpoint");
     private static final ParseField REASON = new ParseField("reason");
     private static final ParseField PROGRESS = new ParseField("progress");
+    private static final ParseField NODE = new ParseField("node");
 
     @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<DataFrameTransformState, Void> PARSER =
@@ -52,7 +53,8 @@ public class DataFrameTransformState {
                         (Map<String, Object>) args[2],
                         (long) args[3],
                         (String) args[4],
-                        (DataFrameTransformProgress) args[5]));
+                        (DataFrameTransformProgress) args[5],
+                        (NodeAttributes) args[6]));
 
     static {
         PARSER.declareField(constructorArg(), p -> DataFrameTransformTaskState.fromString(p.text()), TASK_STATE, ValueType.STRING);
@@ -61,6 +63,7 @@ public class DataFrameTransformState {
         PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), CHECKPOINT);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), REASON);
         PARSER.declareField(optionalConstructorArg(), DataFrameTransformProgress::fromXContent, PROGRESS, ValueType.OBJECT);
+        PARSER.declareField(optionalConstructorArg(), NodeAttributes.PARSER::apply, NODE, ValueType.OBJECT);
     }
 
     public static DataFrameTransformState fromXContent(XContentParser parser) throws IOException {
@@ -73,19 +76,22 @@ public static DataFrameTransformState fromXContent(XContentParser parser) throws
     private final Map<String, Object> currentPosition;
     private final String reason;
     private final DataFrameTransformProgress progress;
+    private final NodeAttributes node;
 
     public DataFrameTransformState(DataFrameTransformTaskState taskState,
                                    IndexerState indexerState,
                                    @Nullable Map<String, Object> position,
                                    long checkpoint,
                                    @Nullable String reason,
-                                   @Nullable DataFrameTransformProgress progress) {
+                                   @Nullable DataFrameTransformProgress progress,
+                                   @Nullable NodeAttributes node) {
         this.taskState = taskState;
         this.indexerState = indexerState;
         this.currentPosition = position == null ? null : Collections.unmodifiableMap(new LinkedHashMap<>(position));
         this.checkpoint = checkpoint;
         this.reason = reason;
         this.progress = progress;
+        this.node = node;
     }
 
     public IndexerState getIndexerState() {
@@ -115,6 +121,11 @@ public DataFrameTransformProgress getProgress() {
         return progress;
     }
 
+    @Nullable
+    public NodeAttributes getNode() {
+        return node;
+    }
+
     @Override
     public boolean equals(Object other) {
         if (this == other) {
@@ -132,12 +143,13 @@ public boolean equals(Object other) {
             Objects.equals(this.currentPosition, that.currentPosition) &&
             Objects.equals(this.progress, that.progress) &&
             this.checkpoint == that.checkpoint &&
+            Objects.equals(this.node, that.node) &&
             Objects.equals(this.reason, that.reason);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(taskState, indexerState, currentPosition, checkpoint, reason, progress);
+        return Objects.hash(taskState, indexerState, currentPosition, checkpoint, reason, progress, node);
     }
 
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DestConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DestConfig.java
index 5e81a368f66b4..f808fa867209a 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DestConfig.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DestConfig.java
@@ -28,6 +28,7 @@
 import java.util.Objects;
 
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 /**
  * Configuration containing the destination index for the {@link DataFrameTransformConfig}
@@ -35,29 +36,40 @@
 public class DestConfig implements ToXContentObject {
 
     public static final ParseField INDEX = new ParseField("index");
+    public static final ParseField PIPELINE = new ParseField("pipeline");
 
     public static final ConstructingObjectParser<DestConfig, Void> PARSER = new ConstructingObjectParser<>("data_frame_config_dest",
         true,
-        args -> new DestConfig((String)args[0]));
+        args -> new DestConfig((String)args[0], (String)args[1]));
 
     static {
         PARSER.declareString(constructorArg(), INDEX);
+        PARSER.declareString(optionalConstructorArg(), PIPELINE);
     }
 
     private final String index;
+    private final String pipeline;
 
-    public DestConfig(String index) {
+    DestConfig(String index, String pipeline) {
         this.index = Objects.requireNonNull(index, INDEX.getPreferredName());
+        this.pipeline = pipeline;
     }
 
     public String getIndex() {
         return index;
     }
 
+    public String getPipeline() {
+        return pipeline;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(INDEX.getPreferredName(), index);
+        if (pipeline != null) {
+            builder.field(PIPELINE.getPreferredName(), pipeline);
+        }
         builder.endObject();
         return builder;
     }
@@ -72,11 +84,45 @@ public boolean equals(Object other) {
         }
 
         DestConfig that = (DestConfig) other;
-        return Objects.equals(index, that.index);
+        return Objects.equals(index, that.index) &&
+            Objects.equals(pipeline, that.pipeline);
     }
 
     @Override
     public int hashCode(){
-        return Objects.hash(index);
+        return Objects.hash(index, pipeline);
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static class Builder {
+        private String index;
+        private String pipeline;
+
+        /**
+         * Sets which index to which to write the data
+         * @param index where to write the data
+         * @return The {@link Builder} with index set
+         */
+        public Builder setIndex(String index) {
+            this.index = Objects.requireNonNull(index, INDEX.getPreferredName());
+            return this;
+        }
+
+        /**
+         * Sets the pipeline through which the indexed documents should be processed
+         * @param pipeline The pipeline ID
+         * @return The {@link Builder} with pipeline set
+         */
+        public Builder setPipeline(String pipeline) {
+            this.pipeline = pipeline;
+            return this;
+        }
+
+        public DestConfig build() {
+            return new DestConfig(index, pipeline);
+        }
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/NodeAttributes.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/NodeAttributes.java
new file mode 100644
index 0000000000000..85c2b9644c2fd
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/NodeAttributes.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.dataframe.transforms;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * A Pojo class containing an Elastic Node's attributes
+ */
+public class NodeAttributes implements ToXContentObject {
+
+    public static final ParseField ID = new ParseField("id");
+    public static final ParseField NAME = new ParseField("name");
+    public static final ParseField EPHEMERAL_ID = new ParseField("ephemeral_id");
+    public static final ParseField TRANSPORT_ADDRESS = new ParseField("transport_address");
+    public static final ParseField ATTRIBUTES = new ParseField("attributes");
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<NodeAttributes, Void> PARSER =
+        new ConstructingObjectParser<>("node", true,
+            (a) -> {
+                int i = 0;
+                String id = (String) a[i++];
+                String name = (String) a[i++];
+                String ephemeralId = (String) a[i++];
+                String transportAddress = (String) a[i++];
+                Map<String, String> attributes = (Map<String, String>) a[i];
+                return new NodeAttributes(id, name, ephemeralId, transportAddress, attributes);
+            });
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), ID);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), EPHEMERAL_ID);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), TRANSPORT_ADDRESS);
+        PARSER.declareField(ConstructingObjectParser.constructorArg(),
+            (p, c) -> p.mapStrings(),
+            ATTRIBUTES,
+            ObjectParser.ValueType.OBJECT);
+    }
+
+    private final String id;
+    private final String name;
+    private final String ephemeralId;
+    private final String transportAddress;
+    private final Map<String, String> attributes;
+
+    public NodeAttributes(String id, String name, String ephemeralId, String transportAddress, Map<String, String> attributes) {
+        this.id = id;
+        this.name = name;
+        this.ephemeralId = ephemeralId;
+        this.transportAddress = transportAddress;
+        this.attributes = Collections.unmodifiableMap(attributes);
+    }
+
+    /**
+     * The unique identifier of the node.
+     */
+    public String getId() {
+        return id;
+    }
+
+    /**
+     * The node name.
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * The ephemeral id of the node.
+     */
+    public String getEphemeralId() {
+        return ephemeralId;
+    }
+
+    /**
+     * The host and port where transport HTTP connections are accepted.
+     */
+    public String getTransportAddress() {
+        return transportAddress;
+    }
+
+    /**
+     * Additional attributes related to this node
+     */
+    public Map<String, String> getAttributes() {
+        return attributes;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(ID.getPreferredName(), id);
+        builder.field(NAME.getPreferredName(), name);
+        builder.field(EPHEMERAL_ID.getPreferredName(), ephemeralId);
+        builder.field(TRANSPORT_ADDRESS.getPreferredName(), transportAddress);
+        builder.field(ATTRIBUTES.getPreferredName(), attributes);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, name, ephemeralId, transportAddress, attributes);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        NodeAttributes that = (NodeAttributes) other;
+        return Objects.equals(id, that.id) &&
+            Objects.equals(name, that.name) &&
+            Objects.equals(ephemeralId, that.ephemeralId) &&
+            Objects.equals(transportAddress, that.transportAddress) &&
+            Objects.equals(attributes, that.attributes);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/SyncConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/SyncConfig.java
new file mode 100644
index 0000000000000..3ead35d0a491a
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/SyncConfig.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.dataframe.transforms;
+
+import org.elasticsearch.common.xcontent.ToXContentObject;
+
+public interface SyncConfig extends ToXContentObject {
+
+    /**
+     * Returns the name of the writeable object
+     */
+    String getName();
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/TimeSyncConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/TimeSyncConfig.java
new file mode 100644
index 0000000000000..797ca3f896138
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/TimeSyncConfig.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.dataframe.transforms;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class TimeSyncConfig implements SyncConfig {
+
+    public static final String NAME = "time";
+
+    private static final ParseField FIELD = new ParseField("field");
+    private static final ParseField DELAY = new ParseField("delay");
+
+    private final String field;
+    private final TimeValue delay;
+
+    private static final ConstructingObjectParser<TimeSyncConfig, Void> PARSER = new ConstructingObjectParser<>("time_sync_config", true,
+            args -> new TimeSyncConfig((String) args[0], args[1] != null ? (TimeValue) args[1] : TimeValue.ZERO));
+
+    static {
+        PARSER.declareString(constructorArg(), FIELD);
+        PARSER.declareField(optionalConstructorArg(), (p, c) -> TimeValue.parseTimeValue(p.textOrNull(), DELAY.getPreferredName()), DELAY,
+                ObjectParser.ValueType.STRING_OR_NULL);
+    }
+
+    public static TimeSyncConfig fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public TimeSyncConfig(String field, TimeValue delay) {
+        this.field = field;
+        this.delay = delay;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(FIELD.getPreferredName(), field);
+        if (delay.duration() > 0) {
+            builder.field(DELAY.getPreferredName(), delay.getStringRep());
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    public String getField() {
+        return field;
+    }
+
+    public TimeValue getDelay() {
+        return delay;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        final TimeSyncConfig that = (TimeSyncConfig) other;
+
+        return Objects.equals(this.field, that.field)
+                && Objects.equals(this.delay, that.delay);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(field, delay);
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/pivot/DateHistogramGroupSource.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/pivot/DateHistogramGroupSource.java
index d880bfd82140b..c8fb885896d9c 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/pivot/DateHistogramGroupSource.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/pivot/DateHistogramGroupSource.java
@@ -45,7 +45,6 @@
 public class DateHistogramGroupSource extends SingleGroupSource implements ToXContentObject {
 
     private static final ParseField TIME_ZONE = new ParseField("time_zone");
-    private static final ParseField FORMAT = new ParseField("format");
 
     // From DateHistogramAggregationBuilder in core, transplanted and modified to a set
     // so we don't need to import a dependency on the class
@@ -195,8 +194,7 @@ public int hashCode() {
                    }
 
                    ZoneId zoneId = (ZoneId) args[3];
-                   String format = (String) args[4];
-                   return new DateHistogramGroupSource(field, interval, format, zoneId);
+                   return new DateHistogramGroupSource(field, interval, zoneId);
                 });
 
     static {
@@ -212,8 +210,6 @@ public int hashCode() {
                 return ZoneOffset.ofHours(p.intValue());
             }
         }, TIME_ZONE, ObjectParser.ValueType.LONG);
-
-        PARSER.declareString(optionalConstructorArg(), FORMAT);
     }
 
     public static DateHistogramGroupSource fromXContent(final XContentParser parser) {
@@ -221,13 +217,11 @@ public static DateHistogramGroupSource fromXContent(final XContentParser parser)
     }
 
     private final Interval interval;
-    private final String format;
     private final ZoneId timeZone;
 
-    DateHistogramGroupSource(String field, Interval interval, String format, ZoneId timeZone) {
+    DateHistogramGroupSource(String field, Interval interval, ZoneId timeZone) {
         super(field);
         this.interval = interval;
-        this.format = format;
         this.timeZone = timeZone;
     }
 
@@ -240,10 +234,6 @@ public Interval getInterval() {
         return interval;
     }
 
-    public String getFormat() {
-        return format;
-    }
-
     public ZoneId getTimeZone() {
         return timeZone;
     }
@@ -258,9 +248,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (timeZone != null) {
             builder.field(TIME_ZONE.getPreferredName(), timeZone.toString());
         }
-        if (format != null) {
-            builder.field(FORMAT.getPreferredName(), format);
-        }
         builder.endObject();
         return builder;
     }
@@ -279,13 +266,12 @@ public boolean equals(Object other) {
 
         return Objects.equals(this.field, that.field) &&
                 Objects.equals(this.interval, that.interval) &&
-                Objects.equals(this.timeZone, that.timeZone) &&
-                Objects.equals(this.format, that.format);
+                Objects.equals(this.timeZone, that.timeZone);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(field, interval, timeZone, format);
+        return Objects.hash(field, interval, timeZone);
     }
 
     public static Builder builder() {
@@ -296,7 +282,6 @@ public static class Builder {
 
         private String field;
         private Interval interval;
-        private String format;
         private ZoneId timeZone;
 
         /**
@@ -319,16 +304,6 @@ public Builder setInterval(Interval interval) {
             return this;
         }
 
-        /**
-         * Set the optional String formatting for the time interval.
-         * @param format The format of the output for the time interval key
-         * @return The {@link Builder} with the format set.
-         */
-        public Builder setFormat(String format) {
-            this.format = format;
-            return this;
-        }
-
         /**
          * Sets the time zone to use for this aggregation
          * @param timeZone The zoneId for the timeZone
@@ -340,7 +315,7 @@ public Builder setTimeZone(ZoneId timeZone) {
         }
 
         public DateHistogramGroupSource build() {
-            return new DateHistogramGroupSource(field, interval, format, timeZone);
+            return new DateHistogramGroupSource(field, interval, timeZone);
         }
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java
new file mode 100644
index 0000000000000..2470c3f7a4aa8
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/util/TimeUtil.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.dataframe.transforms.util;
+
+import org.elasticsearch.common.time.DateFormatters;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+import java.util.Date;
+
+public final class TimeUtil {
+
+    /**
+     * Parse out a Date object given the current parser and field name.
+     *
+     * @param parser current XContentParser
+     * @param fieldName the field's preferred name (utilized in exception)
+     * @return parsed Date object
+     * @throws IOException from XContentParser
+     */
+    public static Date parseTimeField(XContentParser parser, String fieldName) throws IOException {
+        if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
+            return new Date(parser.longValue());
+        } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
+            return new Date(DateFormatters.from(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant().toEpochMilli());
+        }
+        throw new IllegalArgumentException(
+            "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]");
+    }
+
+    public static Instant parseTimeFieldToInstant(XContentParser parser, String fieldName) throws IOException {
+        if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
+            return Instant.ofEpochMilli(parser.longValue());
+        } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
+            return DateFormatters.from(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant();
+        }
+        throw new IllegalArgumentException(
+            "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]");
+    }
+
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/AnalyzeRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/AnalyzeRequest.java
new file mode 100644
index 0000000000000..1aed59227e8bd
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/AnalyzeRequest.java
@@ -0,0 +1,343 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indices;
+
+import org.elasticsearch.client.Validatable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * A request to analyze text
+ */
+public class AnalyzeRequest implements Validatable, ToXContentObject {
+
+    private String index;
+
+    private String[] text;
+
+    private String analyzer;
+
+    private NameOrDefinition tokenizer;
+
+    private final List<NameOrDefinition> tokenFilters = new ArrayList<>();
+
+    private final List<NameOrDefinition> charFilters = new ArrayList<>();
+
+    private String field;
+
+    private boolean explain = false;
+
+    private String[] attributes = Strings.EMPTY_ARRAY;
+
+    private String normalizer;
+
+    /**
+     * Analyzes text using a global analyzer
+     */
+    public static AnalyzeRequest withGlobalAnalyzer(String analyzer, String... text) {
+        return new AnalyzeRequest(null, analyzer, null, null, text);
+    }
+
+    /**
+     * Analyzes text using a custom analyzer built from global components
+     */
+    public static CustomAnalyzerBuilder buildCustomAnalyzer(String tokenizer) {
+        return new CustomAnalyzerBuilder(null, new NameOrDefinition(tokenizer));
+    }
+
+    /**
+     * Analyzes text using a custom analyzer built from global components
+     */
+    public static CustomAnalyzerBuilder buildCustomAnalyzer(Map<String, Object> tokenizerSettings) {
+        return new CustomAnalyzerBuilder(null, new NameOrDefinition(tokenizerSettings));
+    }
+
+    /**
+     * Analyzes text using a custom analyzer built from components defined on an index
+     */
+    public static CustomAnalyzerBuilder buildCustomAnalyzer(String index, String tokenizer) {
+        return new CustomAnalyzerBuilder(index, new NameOrDefinition(tokenizer));
+    }
+
+    /**
+     * Analyzes text using a custom analyzer built from components defined on an index
+     */
+    public static CustomAnalyzerBuilder buildCustomAnalyzer(String index, Map<String, Object> tokenizerSettings) {
+        return new CustomAnalyzerBuilder(index, new NameOrDefinition(tokenizerSettings));
+    }
+
+    /**
+     * Analyzes text using a named analyzer on an index
+     */
+    public static AnalyzeRequest withIndexAnalyzer(String index, String analyzer, String... text) {
+        return new AnalyzeRequest(index, analyzer, null, null, text);
+    }
+
+    /**
+     * Analyzes text using the analyzer defined on a specific field within an index
+     */
+    public static AnalyzeRequest withField(String index, String field, String... text) {
+        return new AnalyzeRequest(index, null, null, field, text);
+    }
+
+    /**
+     * Analyzes text using a named normalizer on an index
+     */
+    public static AnalyzeRequest withNormalizer(String index, String normalizer, String... text) {
+        return new AnalyzeRequest(index, null, normalizer, null, text);
+    }
+
+    /**
+     * Analyzes text using a custom normalizer built from global components
+     */
+    public static CustomAnalyzerBuilder buildCustomNormalizer() {
+        return new CustomAnalyzerBuilder(null, null);
+    }
+
+    /**
+     * Analyzes text using a custom normalizer built from components defined on an index
+     */
+    public static CustomAnalyzerBuilder buildCustomNormalizer(String index) {
+        return new CustomAnalyzerBuilder(index, null);
+    }
+
+    /**
+     * Helper class to build custom analyzer definitions
+     */
+    public static class CustomAnalyzerBuilder {
+
+        final NameOrDefinition tokenizer;
+        final String index;
+        List<NameOrDefinition> charFilters = new ArrayList<>();
+        List<NameOrDefinition> tokenFilters = new ArrayList<>();
+
+        CustomAnalyzerBuilder(String index, NameOrDefinition tokenizer) {
+            this.tokenizer = tokenizer;
+            this.index = index;
+        }
+
+        public CustomAnalyzerBuilder addCharFilter(String name) {
+            charFilters.add(new NameOrDefinition(name));
+            return this;
+        }
+
+        public CustomAnalyzerBuilder addCharFilter(Map<String, Object> settings) {
+            charFilters.add(new NameOrDefinition(settings));
+            return this;
+        }
+
+        public CustomAnalyzerBuilder addTokenFilter(String name) {
+            tokenFilters.add(new NameOrDefinition(name));
+            return this;
+        }
+
+        public CustomAnalyzerBuilder addTokenFilter(Map<String, Object> settings) {
+            tokenFilters.add(new NameOrDefinition(settings));
+            return this;
+        }
+
+        public AnalyzeRequest build(String... text) {
+            return new AnalyzeRequest(index, tokenizer, charFilters, tokenFilters, text);
+        }
+    }
+
+    private AnalyzeRequest(String index, String analyzer, String normalizer, String field, String... text) {
+        this.index = index;
+        this.analyzer = analyzer;
+        this.normalizer = normalizer;
+        this.field = field;
+        this.text = text;
+    }
+
+    private AnalyzeRequest(String index, NameOrDefinition tokenizer, List<NameOrDefinition> charFilters,
+                           List<NameOrDefinition> tokenFilters, String... text) {
+        this.index = index;
+        this.analyzer = null;
+        this.normalizer = null;
+        this.field = null;
+        this.tokenizer = tokenizer;
+        this.charFilters.addAll(charFilters);
+        this.tokenFilters.addAll(tokenFilters);
+        this.text = text;
+    }
+
+    static class NameOrDefinition implements ToXContentFragment {
+        // exactly one of these two members is not null
+        public final String name;
+        public final Settings definition;
+
+        NameOrDefinition(String name) {
+            this.name = Objects.requireNonNull(name);
+            this.definition = null;
+        }
+
+        NameOrDefinition(Settings settings) {
+            this.name = null;
+            this.definition = Objects.requireNonNull(settings);
+        }
+
+        NameOrDefinition(Map<String, ?> definition) {
+            this.name = null;
+            Objects.requireNonNull(definition);
+            try {
+                XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+                builder.map(definition);
+                this.definition = Settings.builder().loadFromSource(Strings.toString(builder), builder.contentType()).build();
+            } catch (IOException e) {
+                throw new IllegalArgumentException("Failed to parse [" + definition + "]", e);
+            }
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            if (definition == null) {
+                return builder.value(name);
+            }
+            builder.startObject();
+            definition.toXContent(builder, params);
+            builder.endObject();
+            return builder;
+        }
+
+    }
+
+    /**
+     * Returns the index that the request should be executed against, or {@code null} if
+     * no index is specified
+     */
+    public String index() {
+        return this.index;
+    }
+
+    /**
+     * Returns the text to be analyzed
+     */
+    public String[] text() {
+        return this.text;
+    }
+
+    /**
+     * Returns the named analyzer used for analysis, if defined
+     */
+    public String analyzer() {
+        return this.analyzer;
+    }
+
+    /**
+     * Returns the named tokenizer used for analysis, if defined
+     */
+    public String normalizer() {
+        return this.normalizer;
+    }
+
+    /**
+     * Returns a custom Tokenizer used for analysis, if defined
+     */
+    public NameOrDefinition tokenizer() {
+        return this.tokenizer;
+    }
+
+    /**
+     * Returns the custom token filters used for analysis, if defined
+     */
+    public List<NameOrDefinition> tokenFilters() {
+        return this.tokenFilters;
+    }
+
+    /**
+     * Returns the custom character filters used for analysis, if defined
+     */
+    public List<NameOrDefinition> charFilters() {
+        return this.charFilters;
+    }
+
+    /**
+     * Returns the field to take an Analyzer from, if defined
+     */
+    public String field() {
+        return this.field;
+    }
+
+    /**
+     * Set whether or not detailed explanations of analysis should be returned
+     */
+    public AnalyzeRequest explain(boolean explain) {
+        this.explain = explain;
+        return this;
+    }
+
+    public boolean explain() {
+        return this.explain;
+    }
+
+    public AnalyzeRequest attributes(String... attributes) {
+        if (attributes == null) {
+            throw new IllegalArgumentException("attributes must not be null");
+        }
+        this.attributes = attributes;
+        return this;
+    }
+
+    public String[] attributes() {
+        return this.attributes;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field("text", text);
+        if (Strings.isNullOrEmpty(analyzer) == false) {
+            builder.field("analyzer", analyzer);
+        }
+        if (tokenizer != null) {
+            builder.field("tokenizer", tokenizer);
+        }
+        if (tokenFilters.size() > 0) {
+            builder.field("filter", tokenFilters);
+        }
+        if (charFilters.size() > 0) {
+            builder.field("char_filter", charFilters);
+        }
+        if (Strings.isNullOrEmpty(field) == false) {
+            builder.field("field", field);
+        }
+        if (explain) {
+            builder.field("explain", true);
+        }
+        if (attributes.length > 0) {
+            builder.field("attributes", attributes);
+        }
+        if (Strings.isNullOrEmpty(normalizer) == false) {
+            builder.field("normalizer", normalizer);
+        }
+        return builder.endObject();
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/AnalyzeResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/AnalyzeResponse.java
new file mode 100644
index 0000000000000..aaba8653dee84
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/AnalyzeResponse.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indices;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class AnalyzeResponse {
+
+    private static final String TOKENS = "tokens";
+    private static final String DETAIL = "detail";
+
+    public static class AnalyzeToken {
+        private String term;
+        private int startOffset;
+        private int endOffset;
+        private int position;
+        private int positionLength = 1;
+        private String type;
+        private final Map<String, Object> attributes = new HashMap<>();
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            AnalyzeResponse.AnalyzeToken that = (AnalyzeResponse.AnalyzeToken) o;
+            return startOffset == that.startOffset &&
+                endOffset == that.endOffset &&
+                position == that.position &&
+                positionLength == that.positionLength &&
+                Objects.equals(term, that.term) &&
+                Objects.equals(attributes, that.attributes) &&
+                Objects.equals(type, that.type);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(term, startOffset, endOffset, position, positionLength, attributes, type);
+        }
+
+        public String getTerm() {
+            return this.term;
+        }
+
+        private void setTerm(String term) {
+            this.term = term;
+        }
+
+        public int getStartOffset() {
+            return this.startOffset;
+        }
+
+        private void setStartOffset(int startOffset) {
+            this.startOffset = startOffset;
+        }
+
+        public int getEndOffset() {
+            return this.endOffset;
+        }
+
+        private void setEndOffset(int endOffset) {
+            this.endOffset = endOffset;
+        }
+
+        public int getPosition() {
+            return this.position;
+        }
+
+        private void setPosition(int position) {
+            this.position = position;
+        }
+
+        public int getPositionLength() {
+            return this.positionLength;
+        }
+
+        private void setPositionLength(int positionLength) {
+            this.positionLength = positionLength;
+        }
+
+        public String getType() {
+            return this.type;
+        }
+
+        private void setType(String type) {
+            this.type = type;
+        }
+
+        public Map<String, Object> getAttributes() {
+            return this.attributes;
+        }
+
+        private void setAttribute(String key, Object value) {
+            this.attributes.put(key, value);
+        }
+
+        private static final ObjectParser<AnalyzeToken, Void> PARSER
+            = new ObjectParser<>("analyze_token", AnalyzeToken::setAttribute, AnalyzeToken::new);
+        static {
+            PARSER.declareString(AnalyzeToken::setTerm, new ParseField("token"));
+            PARSER.declareString(AnalyzeToken::setType, new ParseField("type"));
+            PARSER.declareInt(AnalyzeToken::setPosition, new ParseField("position"));
+            PARSER.declareInt(AnalyzeToken::setStartOffset, new ParseField("start_offset"));
+            PARSER.declareInt(AnalyzeToken::setEndOffset, new ParseField("end_offset"));
+            PARSER.declareInt(AnalyzeToken::setPositionLength, new ParseField("positionLength"));
+        }
+
+        public static AnalyzeToken fromXContent(XContentParser parser) throws IOException {
+            return PARSER.parse(parser, null);
+        }
+    }
+
+    private final DetailAnalyzeResponse detail;
+    private final List<AnalyzeResponse.AnalyzeToken> tokens;
+
+    private AnalyzeResponse(List<AnalyzeResponse.AnalyzeToken> tokens, DetailAnalyzeResponse detail) {
+        this.tokens = tokens;
+        this.detail = detail;
+    }
+
+    public List<AnalyzeResponse.AnalyzeToken> getTokens() {
+        return this.tokens;
+    }
+
+    public DetailAnalyzeResponse detail() {
+        return this.detail;
+    }
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<AnalyzeResponse, Void> PARSER = new ConstructingObjectParser<>("analyze_response",
+        true, args -> new AnalyzeResponse((List<AnalyzeResponse.AnalyzeToken>) args[0], (DetailAnalyzeResponse) args[1]));
+
+    static {
+        PARSER.declareObjectArray(optionalConstructorArg(), AnalyzeToken.PARSER, new ParseField(TOKENS));
+        PARSER.declareObject(optionalConstructorArg(), DetailAnalyzeResponse.PARSER, new ParseField(DETAIL));
+    }
+
+    public static AnalyzeResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AnalyzeResponse that = (AnalyzeResponse) o;
+        return Objects.equals(detail, that.detail) &&
+            Objects.equals(tokens, that.tokens);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(detail, tokens);
+    }
+
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DetailAnalyzeResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DetailAnalyzeResponse.java
new file mode 100644
index 0000000000000..36cf8afad0d58
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DetailAnalyzeResponse.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indices;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class DetailAnalyzeResponse {
+
+    private final boolean customAnalyzer;
+    private final AnalyzeTokenList analyzer;
+    private final CharFilteredText[] charfilters;
+    private final AnalyzeTokenList tokenizer;
+    private final AnalyzeTokenList[] tokenfilters;
+
+    private DetailAnalyzeResponse(boolean customAnalyzer,
+                                 AnalyzeTokenList analyzer,
+                                 List<CharFilteredText> charfilters,
+                                 AnalyzeTokenList tokenizer,
+                                 List<AnalyzeTokenList> tokenfilters) {
+        this.customAnalyzer = customAnalyzer;
+        this.analyzer = analyzer;
+        this.charfilters = charfilters == null ? null : charfilters.toArray(new CharFilteredText[]{});
+        this.tokenizer = tokenizer;
+        this.tokenfilters = tokenfilters == null ? null : tokenfilters.toArray(new AnalyzeTokenList[]{});
+    }
+
+    public AnalyzeTokenList analyzer() {
+        return this.analyzer;
+    }
+
+    public CharFilteredText[] charfilters() {
+        return this.charfilters;
+    }
+
+    public AnalyzeTokenList tokenizer() {
+        return tokenizer;
+    }
+
+    public AnalyzeTokenList[] tokenfilters() {
+        return tokenfilters;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        DetailAnalyzeResponse that = (DetailAnalyzeResponse) o;
+        return customAnalyzer == that.customAnalyzer &&
+            Objects.equals(analyzer, that.analyzer) &&
+            Arrays.equals(charfilters, that.charfilters) &&
+            Objects.equals(tokenizer, that.tokenizer) &&
+            Arrays.equals(tokenfilters, that.tokenfilters);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = Objects.hash(customAnalyzer, analyzer, tokenizer);
+        result = 31 * result + Arrays.hashCode(charfilters);
+        result = 31 * result + Arrays.hashCode(tokenfilters);
+        return result;
+    }
+
+    @SuppressWarnings("unchecked")
+    static final ConstructingObjectParser<DetailAnalyzeResponse, Void> PARSER = new ConstructingObjectParser<>("detail",
+        true, args -> new DetailAnalyzeResponse(
+        (boolean) args[0],
+        (AnalyzeTokenList) args[1],
+        (List<CharFilteredText>)args[2],
+        (AnalyzeTokenList) args[3],
+        (List<AnalyzeTokenList>)args[4]));
+
+    static {
+        PARSER.declareBoolean(constructorArg(), new ParseField("custom_analyzer"));
+        PARSER.declareObject(optionalConstructorArg(), AnalyzeTokenList.PARSER, new ParseField("analyzer"));
+        PARSER.declareObjectArray(optionalConstructorArg(), CharFilteredText.PARSER, new ParseField("charfilters"));
+        PARSER.declareObject(optionalConstructorArg(), AnalyzeTokenList.PARSER, new ParseField("tokenizer"));
+        PARSER.declareObjectArray(optionalConstructorArg(), AnalyzeTokenList.PARSER, new ParseField("tokenfilters"));
+    }
+
+    public static DetailAnalyzeResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    public static class AnalyzeTokenList {
+        private final String name;
+        private final AnalyzeResponse.AnalyzeToken[] tokens;
+
+        private static final String TOKENS = "tokens";
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            AnalyzeTokenList that = (AnalyzeTokenList) o;
+            return Objects.equals(name, that.name) &&
+                Arrays.equals(tokens, that.tokens);
+        }
+
+        @Override
+        public int hashCode() {
+            int result = Objects.hash(name);
+            result = 31 * result + Arrays.hashCode(tokens);
+            return result;
+        }
+
+        public AnalyzeTokenList(String name, List<AnalyzeResponse.AnalyzeToken> tokens) {
+            this.name = name;
+            this.tokens = tokens.toArray(new AnalyzeResponse.AnalyzeToken[]{});
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public AnalyzeResponse.AnalyzeToken[] getTokens() {
+            return tokens;
+        }
+
+        @SuppressWarnings("unchecked")
+        private static final ConstructingObjectParser<AnalyzeTokenList, Void> PARSER = new ConstructingObjectParser<>("token_list",
+            true, args -> new AnalyzeTokenList((String) args[0],
+            (List<AnalyzeResponse.AnalyzeToken>)args[1]));
+
+        static {
+            PARSER.declareString(constructorArg(), new ParseField("name"));
+            PARSER.declareObjectArray(constructorArg(), (p, c) -> AnalyzeResponse.AnalyzeToken.fromXContent(p),
+                new ParseField("tokens"));
+        }
+
+        public static AnalyzeTokenList fromXContent(XContentParser parser) throws IOException {
+            return PARSER.parse(parser, null);
+        }
+
+    }
+
+    public static class CharFilteredText {
+        private final String name;
+        private final String[] texts;
+
+        CharFilteredText(String name, String[] texts) {
+            this.name = name;
+            if (texts != null) {
+                this.texts = texts;
+            } else {
+                this.texts = Strings.EMPTY_ARRAY;
+            }
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public String[] getTexts() {
+            return texts;
+        }
+
+        @SuppressWarnings("unchecked")
+        private static final ConstructingObjectParser<CharFilteredText, Void> PARSER = new ConstructingObjectParser<>("char_filtered_text",
+            true, args -> new CharFilteredText((String) args[0], ((List<String>) args[1]).toArray(new String[0])));
+
+        static {
+            PARSER.declareString(constructorArg(), new ParseField("name"));
+            PARSER.declareStringArray(constructorArg(), new ParseField("filtered_text"));
+        }
+
+        public static CharFilteredText fromXContent(XContentParser parser) throws IOException {
+            return PARSER.parse(parser, null);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            CharFilteredText that = (CharFilteredText) o;
+            return Objects.equals(name, that.name) &&
+                Arrays.equals(texts, that.texts);
+        }
+
+        @Override
+        public int hashCode() {
+            int result = Objects.hash(name);
+            result = 31 * result + Arrays.hashCode(texts);
+            return result;
+        }
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/ReloadAnalyzersRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/ReloadAnalyzersRequest.java
new file mode 100644
index 0000000000000..e815d91bbe8f5
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/ReloadAnalyzersRequest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indices;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.Validatable;
+
+import java.util.Objects;
+
+/**
+ * Request for the _reload_search_analyzers API
+ */
+public final class ReloadAnalyzersRequest implements Validatable {
+
+    private final String[] indices;
+    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
+
+    /**
+     * Creates a new reload analyzers request
+     * @param indices the index for which to reload analyzers
+     */
+    public ReloadAnalyzersRequest(String... indices) {
+        this.indices = Objects.requireNonNull(indices);
+    }
+
+    /**
+     * Returns the indices
+     */
+    public String[] getIndices() {
+        return indices;
+    }
+
+    /**
+     * Specifies what type of requested indices to ignore and how to deal with wildcard expressions.
+     * For example indices that don't exist.
+     *
+     * @return the current behaviour when it comes to index names and wildcard indices expressions
+     */
+    public IndicesOptions indicesOptions() {
+        return indicesOptions;
+    }
+
+    /**
+     * Specifies what type of requested indices to ignore and how to deal with wildcard expressions.
+     * For example indices that don't exist.
+     *
+     * @param indicesOptions the desired behaviour regarding indices to ignore and wildcard indices expressions
+     */
+    public void setIndicesOptions(IndicesOptions indicesOptions) {
+        this.indicesOptions = indicesOptions;
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/ReloadAnalyzersResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/ReloadAnalyzersResponse.java
new file mode 100644
index 0000000000000..e2c39d0a7aeba
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/ReloadAnalyzersResponse.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.indices;
+
+import org.elasticsearch.client.core.BroadcastResponse;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
+/**
+ * The response object that will be returned when reloading analyzers
+ */
+public class ReloadAnalyzersResponse extends BroadcastResponse {
+
+    private final Map<String, ReloadDetails> reloadDetails;
+
+    ReloadAnalyzersResponse(final Shards shards, Map<String, ReloadDetails> reloadDetails) {
+        super(shards);
+        this.reloadDetails = reloadDetails;
+    }
+
+    @SuppressWarnings({ "unchecked" })
+    private static final ConstructingObjectParser<ReloadAnalyzersResponse, Void> PARSER = new ConstructingObjectParser<>("reload_analyzer",
+            true, arg -> {
+                Shards shards = (Shards) arg[0];
+                List<Tuple<String, ReloadDetails>> results = (List<Tuple<String, ReloadDetails>>) arg[1];
+                Map<String, ReloadDetails> reloadDetails = new HashMap<>();
+                for (Tuple<String, ReloadDetails> result : results) {
+                    reloadDetails.put(result.v1(), result.v2());
+                }
+                return new ReloadAnalyzersResponse(shards, reloadDetails);
+            });
+
+    @SuppressWarnings({ "unchecked" })
+    private static final ConstructingObjectParser<Tuple<String, ReloadDetails>, Void> ENTRY_PARSER = new ConstructingObjectParser<>(
+            "reload_analyzer.entry", true, arg -> {
+                String index = (String) arg[0];
+                Set<String> nodeIds = new HashSet<>((List<String>) arg[1]);
+                Set<String> analyzers = new HashSet<>((List<String>) arg[2]);
+                return new Tuple<>(index, new ReloadDetails(index, nodeIds, analyzers));
+            });
+
+    static {
+        declareShardsField(PARSER);
+        PARSER.declareObjectArray(constructorArg(), ENTRY_PARSER, new ParseField("reload_details"));
+        ENTRY_PARSER.declareString(constructorArg(), new ParseField("index"));
+        ENTRY_PARSER.declareStringArray(constructorArg(), new ParseField("reloaded_node_ids"));
+        ENTRY_PARSER.declareStringArray(constructorArg(), new ParseField("reloaded_analyzers"));
+    }
+
+    public static ReloadAnalyzersResponse fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public Map<String, ReloadDetails> getReloadedDetails() {
+        return reloadDetails;
+    }
+
+    public static class ReloadDetails {
+
+        private final String indexName;
+        private final Set<String> reloadedIndicesNodes;
+        private final Set<String> reloadedAnalyzers;
+
+        public ReloadDetails(String name, Set<String> reloadedIndicesNodes, Set<String> reloadedAnalyzers) {
+            this.indexName = name;
+            this.reloadedIndicesNodes = reloadedIndicesNodes;
+            this.reloadedAnalyzers = reloadedAnalyzers;
+        }
+
+        public String getIndexName() {
+            return indexName;
+        }
+
+        public Set<String> getReloadedIndicesNodes() {
+            return reloadedIndicesNodes;
+        }
+
+        public Set<String> getReloadedAnalyzers() {
+            return reloadedAnalyzers;
+        }
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteDataFrameAnalyticsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteDataFrameAnalyticsRequest.java
new file mode 100644
index 0000000000000..f03466632304d
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteDataFrameAnalyticsRequest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.Validatable;
+import org.elasticsearch.client.ValidationException;
+
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Request to delete a data frame analytics config
+ */
+public class DeleteDataFrameAnalyticsRequest implements Validatable {
+
+    private final String id;
+
+    public DeleteDataFrameAnalyticsRequest(String id) {
+        this.id = id;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    @Override
+    public Optional<ValidationException> validate() {
+        if (id == null) {
+            return Optional.of(ValidationException.withError("data frame analytics id must not be null"));
+        }
+        return Optional.empty();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        DeleteDataFrameAnalyticsRequest other = (DeleteDataFrameAnalyticsRequest) o;
+        return Objects.equals(id, other.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EvaluateDataFrameRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EvaluateDataFrameRequest.java
new file mode 100644
index 0000000000000..2e3bbb170509c
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EvaluateDataFrameRequest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.Validatable;
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.client.ml.dataframe.evaluation.Evaluation;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
+public class EvaluateDataFrameRequest implements ToXContentObject, Validatable {
+
+    private static final ParseField INDEX = new ParseField("index");
+    private static final ParseField EVALUATION = new ParseField("evaluation");
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<EvaluateDataFrameRequest, Void> PARSER =
+        new ConstructingObjectParser<>(
+            "evaluate_data_frame_request", true, args -> new EvaluateDataFrameRequest((List<String>) args[0], (Evaluation) args[1]));
+
+    static {
+        PARSER.declareStringArray(constructorArg(), INDEX);
+        PARSER.declareObject(constructorArg(), (p, c) -> parseEvaluation(p), EVALUATION);
+    }
+
+    private static Evaluation parseEvaluation(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+        Evaluation evaluation = parser.namedObject(Evaluation.class, parser.currentName(), null);
+        ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        return evaluation;
+    }
+
+    public static EvaluateDataFrameRequest fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    private List<String> indices;
+    private Evaluation evaluation;
+
+    public EvaluateDataFrameRequest(String index, Evaluation evaluation) {
+        this(Arrays.asList(index), evaluation);
+    }
+
+    public EvaluateDataFrameRequest(List<String> indices, Evaluation evaluation) {
+        setIndices(indices);
+        setEvaluation(evaluation);
+    }
+
+    public List<String> getIndices() {
+        return Collections.unmodifiableList(indices);
+    }
+
+    public final void setIndices(List<String> indices) {
+        Objects.requireNonNull(indices);
+        this.indices = new ArrayList<>(indices);
+    }
+
+    public Evaluation getEvaluation() {
+        return evaluation;
+    }
+
+    public final void setEvaluation(Evaluation evaluation) {
+        this.evaluation = evaluation;
+    }
+
+    @Override
+    public Optional<ValidationException> validate() {
+        List<String> errors = new ArrayList<>();
+        if (indices.isEmpty()) {
+            errors.add("At least one index must be specified");
+        }
+        if (evaluation == null) {
+            errors.add("evaluation must not be null");
+        }
+        return errors.isEmpty()
+            ? Optional.empty()
+            : Optional.of(ValidationException.withErrors(errors));
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder
+            .startObject()
+                .array(INDEX.getPreferredName(), indices.toArray())
+                .startObject(EVALUATION.getPreferredName())
+                    .field(evaluation.getName(), evaluation)
+                .endObject()
+            .endObject();
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(indices, evaluation);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        EvaluateDataFrameRequest that = (EvaluateDataFrameRequest) o;
+        return Objects.equals(indices, that.indices)
+            && Objects.equals(evaluation, that.evaluation);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EvaluateDataFrameResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EvaluateDataFrameResponse.java
new file mode 100644
index 0000000000000..0709021ed4bd5
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/EvaluateDataFrameResponse.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.NamedObjectNotFoundException;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
+public class EvaluateDataFrameResponse implements ToXContentObject {
+
+    public static EvaluateDataFrameResponse fromXContent(XContentParser parser) throws IOException {
+        if (parser.currentToken() == null) {
+            parser.nextToken();
+        }
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+        String evaluationName = parser.currentName();
+        parser.nextToken();
+        Map<String, EvaluationMetric.Result> metrics = parser.map(LinkedHashMap::new, EvaluateDataFrameResponse::parseMetric);
+        List<EvaluationMetric.Result> knownMetrics =
+            metrics.values().stream()
+                .filter(Objects::nonNull)  // Filter out null values returned by {@link EvaluateDataFrameResponse::parseMetric}.
+                .collect(Collectors.toList());
+        ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        return new EvaluateDataFrameResponse(evaluationName, knownMetrics);
+    }
+
+    private static EvaluationMetric.Result parseMetric(XContentParser parser) throws IOException {
+        String metricName = parser.currentName();
+        try {
+            return parser.namedObject(EvaluationMetric.Result.class, metricName, null);
+        } catch (NamedObjectNotFoundException e) {
+            parser.skipChildren();
+            // Metric name not recognized. Return {@code null} value here and filter it out later.
+            return null;
+        }
+    }
+
+    private final String evaluationName;
+    private final Map<String, EvaluationMetric.Result> metrics;
+
+    public EvaluateDataFrameResponse(String evaluationName, List<EvaluationMetric.Result> metrics) {
+        this.evaluationName = Objects.requireNonNull(evaluationName);
+        this.metrics = Objects.requireNonNull(metrics).stream().collect(Collectors.toUnmodifiableMap(m -> m.getMetricName(), m -> m));
+    }
+
+    public String getEvaluationName() {
+        return evaluationName;
+    }
+
+    public List<EvaluationMetric.Result> getMetrics() {
+        return metrics.values().stream().collect(Collectors.toList());
+    }
+
+    @SuppressWarnings("unchecked")
+    public <T extends EvaluationMetric.Result> T getMetricByName(String metricName) {
+        Objects.requireNonNull(metricName);
+        return (T) metrics.get(metricName);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        return builder
+            .startObject()
+            .field(evaluationName, metrics)
+            .endObject();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        EvaluateDataFrameResponse that = (EvaluateDataFrameResponse) o;
+        return Objects.equals(evaluationName, that.evaluationName)
+            && Objects.equals(metrics, that.metrics);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(evaluationName, metrics);
+    }
+
+    @Override
+    public final String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureRequest.java
index adfee92bd6171..fed417e9582d8 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureRequest.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureRequest.java
@@ -37,6 +37,7 @@
 public class FindFileStructureRequest implements Validatable, ToXContentFragment {
 
     public static final ParseField LINES_TO_SAMPLE = new ParseField("lines_to_sample");
+    public static final ParseField LINE_MERGE_SIZE_LIMIT = new ParseField("line_merge_size_limit");
     public static final ParseField TIMEOUT = new ParseField("timeout");
     public static final ParseField CHARSET = FileStructure.CHARSET;
     public static final ParseField FORMAT = FileStructure.FORMAT;
@@ -52,6 +53,7 @@ public class FindFileStructureRequest implements Validatable, ToXContentFragment
     public static final ParseField EXPLAIN = new ParseField("explain");
 
     private Integer linesToSample;
+    private Integer lineMergeSizeLimit;
     private TimeValue timeout;
     private String charset;
     private FileStructure.Format format;
@@ -77,6 +79,14 @@ public void setLinesToSample(Integer linesToSample) {
         this.linesToSample = linesToSample;
     }
 
+    public Integer getLineMergeSizeLimit() {
+        return lineMergeSizeLimit;
+    }
+
+    public void setLineMergeSizeLimit(Integer lineMergeSizeLimit) {
+        this.lineMergeSizeLimit = lineMergeSizeLimit;
+    }
+
     public TimeValue getTimeout() {
         return timeout;
     }
@@ -228,6 +238,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (linesToSample != null) {
             builder.field(LINES_TO_SAMPLE.getPreferredName(), linesToSample);
         }
+        if (lineMergeSizeLimit != null) {
+            builder.field(LINE_MERGE_SIZE_LIMIT.getPreferredName(), lineMergeSizeLimit);
+        }
         if (timeout != null) {
             builder.field(TIMEOUT.getPreferredName(), timeout);
         }
@@ -270,8 +283,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public int hashCode() {
-        return Objects.hash(linesToSample, timeout, charset, format, columnNames, hasHeaderRow, delimiter, grokPattern, timestampFormat,
-            timestampField, explain, sample);
+        return Objects.hash(linesToSample, lineMergeSizeLimit, timeout, charset, format, columnNames, hasHeaderRow, delimiter, grokPattern,
+            timestampFormat, timestampField, explain, sample);
     }
 
     @Override
@@ -287,6 +300,7 @@ public boolean equals(Object other) {
 
         FindFileStructureRequest that = (FindFileStructureRequest) other;
         return Objects.equals(this.linesToSample, that.linesToSample) &&
+            Objects.equals(this.lineMergeSizeLimit, that.lineMergeSizeLimit) &&
             Objects.equals(this.timeout, that.timeout) &&
             Objects.equals(this.charset, that.charset) &&
             Objects.equals(this.format, that.format) &&
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java
new file mode 100644
index 0000000000000..40698c4b528fa
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.Validatable;
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.client.core.PageParams;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+public class GetDataFrameAnalyticsRequest implements Validatable {
+
+    public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
+
+    private final List<String> ids;
+    private Boolean allowNoMatch;
+    private PageParams pageParams;
+
+    /**
+     * Helper method to create a request that will get ALL Data Frame Analytics
+     * @return new {@link GetDataFrameAnalyticsRequest} object for the id "_all"
+     */
+    public static GetDataFrameAnalyticsRequest getAllDataFrameAnalyticsRequest() {
+        return new GetDataFrameAnalyticsRequest("_all");
+    }
+
+    public GetDataFrameAnalyticsRequest(String... ids) {
+        this.ids = Arrays.asList(ids);
+    }
+
+    public List<String> getIds() {
+        return ids;
+    }
+
+    public Boolean getAllowNoMatch() {
+        return allowNoMatch;
+    }
+
+    /**
+     * Whether to ignore if a wildcard expression matches no data frame analytics.
+     *
+     * @param allowNoMatch If this is {@code false}, then an error is returned when a wildcard (or {@code _all})
+     *                    does not match any data frame analytics
+     */
+    public GetDataFrameAnalyticsRequest setAllowNoMatch(boolean allowNoMatch) {
+        this.allowNoMatch = allowNoMatch;
+        return this;
+    }
+
+    public PageParams getPageParams() {
+        return pageParams;
+    }
+
+    public GetDataFrameAnalyticsRequest setPageParams(@Nullable PageParams pageParams) {
+        this.pageParams = pageParams;
+        return this;
+    }
+
+    @Override
+    public Optional<ValidationException> validate() {
+        if (ids == null || ids.isEmpty()) {
+            return Optional.of(ValidationException.withError("data frame analytics id must not be null"));
+        }
+        return Optional.empty();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        GetDataFrameAnalyticsRequest other = (GetDataFrameAnalyticsRequest) o;
+        return Objects.equals(ids, other.ids)
+            && Objects.equals(allowNoMatch, other.allowNoMatch)
+            && Objects.equals(pageParams, other.pageParams);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(ids, allowNoMatch, pageParams);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsResponse.java
new file mode 100644
index 0000000000000..76996e9d4d0b6
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsResponse.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
+public class GetDataFrameAnalyticsResponse {
+
+    public static final ParseField DATA_FRAME_ANALYTICS = new ParseField("data_frame_analytics");
+
+    @SuppressWarnings("unchecked")
+    static final ConstructingObjectParser<GetDataFrameAnalyticsResponse, Void> PARSER =
+        new ConstructingObjectParser<>(
+            "get_data_frame_analytics",
+            true,
+            args -> new GetDataFrameAnalyticsResponse((List<DataFrameAnalyticsConfig>) args[0]));
+
+    static {
+        PARSER.declareObjectArray(constructorArg(), (p, c) -> DataFrameAnalyticsConfig.fromXContent(p), DATA_FRAME_ANALYTICS);
+    }
+
+    public static GetDataFrameAnalyticsResponse fromXContent(final XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    private List<DataFrameAnalyticsConfig> analytics;
+
+    public GetDataFrameAnalyticsResponse(List<DataFrameAnalyticsConfig> analytics) {
+        this.analytics = analytics;
+    }
+
+    public List<DataFrameAnalyticsConfig> getAnalytics() {
+        return analytics;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        GetDataFrameAnalyticsResponse other = (GetDataFrameAnalyticsResponse) o;
+        return Objects.equals(this.analytics, other.analytics);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(analytics);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsStatsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsStatsRequest.java
new file mode 100644
index 0000000000000..f1e4a35fb661b
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsStatsRequest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.Validatable;
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.client.core.PageParams;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Request to get data frame analytics stats
+ */
+public class GetDataFrameAnalyticsStatsRequest implements Validatable {
+
+    public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
+
+    private final List<String> ids;
+    private Boolean allowNoMatch;
+    private PageParams pageParams;
+
+    public GetDataFrameAnalyticsStatsRequest(String... ids) {
+        this.ids = Arrays.asList(ids);
+    }
+
+    public List<String> getIds() {
+        return ids;
+    }
+
+    public Boolean getAllowNoMatch() {
+        return allowNoMatch;
+    }
+
+    /**
+     * Whether to ignore if a wildcard expression matches no data frame analytics.
+     *
+     * @param allowNoMatch If this is {@code false}, then an error is returned when a wildcard (or {@code _all})
+     *                        does not match any data frame analytics
+     */
+    public GetDataFrameAnalyticsStatsRequest setAllowNoMatch(boolean allowNoMatch) {
+        this.allowNoMatch = allowNoMatch;
+        return this;
+    }
+
+    public PageParams getPageParams() {
+        return pageParams;
+    }
+
+    public GetDataFrameAnalyticsStatsRequest setPageParams(@Nullable PageParams pageParams) {
+        this.pageParams = pageParams;
+        return this;
+    }
+
+    @Override
+    public Optional<ValidationException> validate() {
+        if (ids == null || ids.isEmpty()) {
+            return Optional.of(ValidationException.withError("data frame analytics id must not be null"));
+        }
+        return Optional.empty();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        GetDataFrameAnalyticsStatsRequest other = (GetDataFrameAnalyticsStatsRequest) o;
+        return Objects.equals(ids, other.ids)
+            && Objects.equals(allowNoMatch, other.allowNoMatch)
+            && Objects.equals(pageParams, other.pageParams);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(ids, allowNoMatch, pageParams);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsStatsResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsStatsResponse.java
new file mode 100644
index 0000000000000..5391a576e98b0
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsStatsResponse.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.TaskOperationFailure;
+import org.elasticsearch.client.dataframe.AcknowledgedTasksResponse;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsStats;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class GetDataFrameAnalyticsStatsResponse {
+
+    public static GetDataFrameAnalyticsStatsResponse fromXContent(XContentParser parser) {
+        return GetDataFrameAnalyticsStatsResponse.PARSER.apply(parser, null);
+    }
+
+    private static final ParseField DATA_FRAME_ANALYTICS = new ParseField("data_frame_analytics");
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<GetDataFrameAnalyticsStatsResponse, Void> PARSER =
+        new ConstructingObjectParser<>(
+            "get_data_frame_analytics_stats_response", true,
+            args -> new GetDataFrameAnalyticsStatsResponse(
+                (List<DataFrameAnalyticsStats>) args[0],
+                (List<TaskOperationFailure>) args[1],
+                (List<ElasticsearchException>) args[2]));
+
+    static {
+        PARSER.declareObjectArray(constructorArg(), (p, c) -> DataFrameAnalyticsStats.fromXContent(p), DATA_FRAME_ANALYTICS);
+        PARSER.declareObjectArray(
+            optionalConstructorArg(), (p, c) -> TaskOperationFailure.fromXContent(p), AcknowledgedTasksResponse.TASK_FAILURES);
+        PARSER.declareObjectArray(
+            optionalConstructorArg(), (p, c) -> ElasticsearchException.fromXContent(p), AcknowledgedTasksResponse.NODE_FAILURES);
+    }
+
+    private final List<DataFrameAnalyticsStats> analyticsStats;
+    private final List<TaskOperationFailure> taskFailures;
+    private final List<ElasticsearchException> nodeFailures;
+
+    public GetDataFrameAnalyticsStatsResponse(List<DataFrameAnalyticsStats> analyticsStats,
+                                              @Nullable List<TaskOperationFailure> taskFailures,
+                                              @Nullable List<? extends ElasticsearchException> nodeFailures) {
+        this.analyticsStats = analyticsStats;
+        this.taskFailures = taskFailures == null ? Collections.emptyList() : Collections.unmodifiableList(taskFailures);
+        this.nodeFailures = nodeFailures == null ? Collections.emptyList() : Collections.unmodifiableList(nodeFailures);
+    }
+
+    public List<DataFrameAnalyticsStats> getAnalyticsStats() {
+        return analyticsStats;
+    }
+
+    public List<ElasticsearchException> getNodeFailures() {
+        return nodeFailures;
+    }
+
+    public List<TaskOperationFailure> getTaskFailures() {
+        return taskFailures;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        GetDataFrameAnalyticsStatsResponse other = (GetDataFrameAnalyticsStatsResponse) o;
+        return Objects.equals(analyticsStats, other.analyticsStats)
+            && Objects.equals(nodeFailures, other.nodeFailures)
+            && Objects.equals(taskFailures, other.taskFailures);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(analyticsStats, nodeFailures, taskFailures);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/NodeAttributes.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/NodeAttributes.java
index 892df340abd6b..a0f0d25f2ca01 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/NodeAttributes.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/NodeAttributes.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.client.ml;
 
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
@@ -147,4 +148,9 @@ public boolean equals(Object other) {
             Objects.equals(transportAddress, that.transportAddress) &&
             Objects.equals(attributes, that.attributes);
     }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutDataFrameAnalyticsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutDataFrameAnalyticsRequest.java
new file mode 100644
index 0000000000000..2624b68a98318
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutDataFrameAnalyticsRequest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.Validatable;
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Optional;
+
+public class PutDataFrameAnalyticsRequest implements ToXContentObject, Validatable {
+
+    private final DataFrameAnalyticsConfig config;
+
+    public PutDataFrameAnalyticsRequest(DataFrameAnalyticsConfig config) {
+        this.config = config;
+    }
+
+    public DataFrameAnalyticsConfig getConfig() {
+        return config;
+    }
+
+    @Override
+    public Optional<ValidationException> validate() {
+        if (config == null) {
+            return Optional.of(ValidationException.withError("put requires a non-null data frame analytics config"));
+        }
+        return Optional.empty();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return config.toXContent(builder, params);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        PutDataFrameAnalyticsRequest other = (PutDataFrameAnalyticsRequest) o;
+        return Objects.equals(config, other.config);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(config);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutDataFrameAnalyticsResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutDataFrameAnalyticsResponse.java
new file mode 100644
index 0000000000000..e6c4be15987d4
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PutDataFrameAnalyticsResponse.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class PutDataFrameAnalyticsResponse {
+
+    public static PutDataFrameAnalyticsResponse fromXContent(XContentParser parser) throws IOException {
+        return new PutDataFrameAnalyticsResponse(DataFrameAnalyticsConfig.fromXContent(parser));
+    }
+
+    private final DataFrameAnalyticsConfig config;
+
+    public PutDataFrameAnalyticsResponse(DataFrameAnalyticsConfig config) {
+        this.config = config;
+    }
+
+    public DataFrameAnalyticsConfig getConfig() {
+        return config;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        PutDataFrameAnalyticsResponse other = (PutDataFrameAnalyticsResponse) o;
+        return Objects.equals(config, other.config);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(config);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/StartDataFrameAnalyticsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/StartDataFrameAnalyticsRequest.java
new file mode 100644
index 0000000000000..68a925d15019a
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/StartDataFrameAnalyticsRequest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.Validatable;
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.util.Objects;
+import java.util.Optional;
+
+public class StartDataFrameAnalyticsRequest implements Validatable {
+
+    private final String id;
+    private TimeValue timeout;
+
+    public StartDataFrameAnalyticsRequest(String id) {
+        this.id = id;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public TimeValue getTimeout() {
+        return timeout;
+    }
+
+    public StartDataFrameAnalyticsRequest setTimeout(@Nullable TimeValue timeout) {
+        this.timeout = timeout;
+        return this;
+    }
+
+    @Override
+    public Optional<ValidationException> validate() {
+        if (id == null) {
+            return Optional.of(ValidationException.withError("data frame analytics id must not be null"));
+        }
+        return Optional.empty();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        StartDataFrameAnalyticsRequest other = (StartDataFrameAnalyticsRequest) o;
+        return Objects.equals(id, other.id)
+            && Objects.equals(timeout, other.timeout);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, timeout);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/StopDataFrameAnalyticsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/StopDataFrameAnalyticsRequest.java
new file mode 100644
index 0000000000000..4ba6af852f61c
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/StopDataFrameAnalyticsRequest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.Validatable;
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.util.Objects;
+import java.util.Optional;
+
+public class StopDataFrameAnalyticsRequest implements Validatable {
+
+    public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
+    public static final ParseField FORCE = new ParseField("force");
+
+    private final String id;
+    private Boolean allowNoMatch;
+    private Boolean force;
+    private TimeValue timeout;
+
+    public StopDataFrameAnalyticsRequest(String id) {
+        this.id = id;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public TimeValue getTimeout() {
+        return timeout;
+    }
+
+    public StopDataFrameAnalyticsRequest setTimeout(@Nullable TimeValue timeout) {
+        this.timeout = timeout;
+        return this;
+    }
+
+    public Boolean getAllowNoMatch() {
+        return allowNoMatch;
+    }
+
+    public StopDataFrameAnalyticsRequest setAllowNoMatch(boolean allowNoMatch) {
+        this.allowNoMatch = allowNoMatch;
+        return this;
+    }
+
+    public Boolean getForce() {
+        return force;
+    }
+
+    public StopDataFrameAnalyticsRequest setForce(boolean force) {
+        this.force = force;
+        return this;
+    }
+
+    @Override
+    public Optional<ValidationException> validate() {
+        if (id == null) {
+            return Optional.of(ValidationException.withError("data frame analytics id must not be null"));
+        }
+        return Optional.empty();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        StopDataFrameAnalyticsRequest other = (StopDataFrameAnalyticsRequest) o;
+        return Objects.equals(id, other.id)
+            && Objects.equals(timeout, other.timeout)
+            && Objects.equals(allowNoMatch, other.allowNoMatch)
+            && Objects.equals(force, other.force);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, timeout, allowNoMatch, force);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/StopDataFrameAnalyticsResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/StopDataFrameAnalyticsResponse.java
new file mode 100644
index 0000000000000..5f45c6f9ea51f
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/StopDataFrameAnalyticsResponse.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Response indicating if the Machine Learning Data Frame Analytics is now stopped or not
+ */
+public class StopDataFrameAnalyticsResponse implements ToXContentObject {
+
+    private static final ParseField STOPPED = new ParseField("stopped");
+
+    public static final ConstructingObjectParser<StopDataFrameAnalyticsResponse, Void> PARSER =
+        new ConstructingObjectParser<>(
+            "stop_data_frame_analytics_response",
+            true,
+            args -> new StopDataFrameAnalyticsResponse((Boolean) args[0]));
+
+    static {
+        PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), STOPPED);
+    }
+
+    public static StopDataFrameAnalyticsResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    private final boolean stopped;
+
+    public StopDataFrameAnalyticsResponse(boolean stopped) {
+        this.stopped = stopped;
+    }
+
+    /**
+     * Has the Data Frame Analytics stopped or not
+     *
+     * @return boolean value indicating the Data Frame Analytics stopped status
+     */
+    public boolean isStopped() {
+        return stopped;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        StopDataFrameAnalyticsResponse other = (StopDataFrameAnalyticsResponse) o;
+        return stopped == other.stopped;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(stopped);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder
+            .startObject()
+            .field(STOPPED.getPreferredName(), stopped)
+            .endObject();
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalysis.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalysis.java
new file mode 100644
index 0000000000000..81b19eefce573
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalysis.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe;
+
+import org.elasticsearch.common.xcontent.ToXContentObject;
+
+public interface DataFrameAnalysis extends ToXContentObject {
+
+    String getName();
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java
new file mode 100644
index 0000000000000..62adb06294558
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.client.dataframe.transforms.util.TimeUtil;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Objects;
+
+public class DataFrameAnalyticsConfig implements ToXContentObject {
+
+    public static DataFrameAnalyticsConfig fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null).build();
+    }
+
+    public static Builder builder(String id) {
+        return new Builder().setId(id);
+    }
+
+    private static final ParseField ID = new ParseField("id");
+    private static final ParseField SOURCE = new ParseField("source");
+    private static final ParseField DEST = new ParseField("dest");
+    private static final ParseField ANALYSIS = new ParseField("analysis");
+    private static final ParseField ANALYZED_FIELDS = new ParseField("analyzed_fields");
+    private static final ParseField MODEL_MEMORY_LIMIT = new ParseField("model_memory_limit");
+    private static final ParseField CREATE_TIME = new ParseField("create_time");
+    private static final ParseField VERSION = new ParseField("version");
+
+    private static ObjectParser<Builder, Void> PARSER = new ObjectParser<>("data_frame_analytics_config", true, Builder::new);
+
+    static {
+        PARSER.declareString(Builder::setId, ID);
+        PARSER.declareObject(Builder::setSource, (p, c) -> DataFrameAnalyticsSource.fromXContent(p), SOURCE);
+        PARSER.declareObject(Builder::setDest, (p, c) -> DataFrameAnalyticsDest.fromXContent(p), DEST);
+        PARSER.declareObject(Builder::setAnalysis, (p, c) -> parseAnalysis(p), ANALYSIS);
+        PARSER.declareField(Builder::setAnalyzedFields,
+            (p, c) -> FetchSourceContext.fromXContent(p),
+            ANALYZED_FIELDS,
+            ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING);
+        PARSER.declareField(Builder::setModelMemoryLimit,
+            (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MODEL_MEMORY_LIMIT.getPreferredName()),
+            MODEL_MEMORY_LIMIT,
+            ValueType.VALUE);
+        PARSER.declareField(Builder::setCreateTime,
+            p -> TimeUtil.parseTimeFieldToInstant(p, CREATE_TIME.getPreferredName()),
+            CREATE_TIME,
+            ValueType.VALUE);
+        PARSER.declareField(Builder::setVersion,
+            p -> {
+                if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                    return Version.fromString(p.text());
+                }
+                throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+            },
+            VERSION,
+            ValueType.STRING);
+    }
+
+    private static DataFrameAnalysis parseAnalysis(XContentParser parser) throws IOException {
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+        DataFrameAnalysis analysis = parser.namedObject(DataFrameAnalysis.class, parser.currentName(), true);
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        return analysis;
+    }
+
+    private final String id;
+    private final DataFrameAnalyticsSource source;
+    private final DataFrameAnalyticsDest dest;
+    private final DataFrameAnalysis analysis;
+    private final FetchSourceContext analyzedFields;
+    private final ByteSizeValue modelMemoryLimit;
+    private final Instant createTime;
+    private final Version version;
+
+    private DataFrameAnalyticsConfig(String id, DataFrameAnalyticsSource source, DataFrameAnalyticsDest dest, DataFrameAnalysis analysis,
+                                     @Nullable FetchSourceContext analyzedFields, @Nullable ByteSizeValue modelMemoryLimit,
+                                     @Nullable Instant createTime, @Nullable Version version) {
+        this.id = Objects.requireNonNull(id);
+        this.source = Objects.requireNonNull(source);
+        this.dest = Objects.requireNonNull(dest);
+        this.analysis = Objects.requireNonNull(analysis);
+        this.analyzedFields = analyzedFields;
+        this.modelMemoryLimit = modelMemoryLimit;
+        this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli());;
+        this.version = version;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public DataFrameAnalyticsSource getSource() {
+        return source;
+    }
+
+    public DataFrameAnalyticsDest getDest() {
+        return dest;
+    }
+
+    public DataFrameAnalysis getAnalysis() {
+        return analysis;
+    }
+
+    public FetchSourceContext getAnalyzedFields() {
+        return analyzedFields;
+    }
+
+    public ByteSizeValue getModelMemoryLimit() {
+        return modelMemoryLimit;
+    }
+
+    public Instant getCreateTime() {
+        return createTime;
+    }
+
+    public Version getVersion() {
+        return version;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(ID.getPreferredName(), id);
+        builder.field(SOURCE.getPreferredName(), source);
+        builder.field(DEST.getPreferredName(), dest);
+        builder.startObject(ANALYSIS.getPreferredName());
+        builder.field(analysis.getName(), analysis);
+        builder.endObject();
+        if (analyzedFields != null) {
+            builder.field(ANALYZED_FIELDS.getPreferredName(), analyzedFields);
+        }
+        if (modelMemoryLimit != null) {
+            builder.field(MODEL_MEMORY_LIMIT.getPreferredName(), modelMemoryLimit.getStringRep());
+        }
+        if (createTime != null) {
+            builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli());
+        }
+        if (version != null) {
+            builder.field(VERSION.getPreferredName(), version);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        DataFrameAnalyticsConfig other = (DataFrameAnalyticsConfig) o;
+        return Objects.equals(id, other.id)
+            && Objects.equals(source, other.source)
+            && Objects.equals(dest, other.dest)
+            && Objects.equals(analysis, other.analysis)
+            && Objects.equals(analyzedFields, other.analyzedFields)
+            && Objects.equals(modelMemoryLimit, other.modelMemoryLimit)
+            && Objects.equals(createTime, other.createTime)
+            && Objects.equals(version, other.version);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    public static class Builder {
+
+        private String id;
+        private DataFrameAnalyticsSource source;
+        private DataFrameAnalyticsDest dest;
+        private DataFrameAnalysis analysis;
+        private FetchSourceContext analyzedFields;
+        private ByteSizeValue modelMemoryLimit;
+        private Instant createTime;
+        private Version version;
+
+        private Builder() {}
+
+        public Builder setId(String id) {
+            this.id = Objects.requireNonNull(id);
+            return this;
+        }
+
+        public Builder setSource(DataFrameAnalyticsSource source) {
+            this.source = Objects.requireNonNull(source);
+            return this;
+        }
+
+        public Builder setDest(DataFrameAnalyticsDest dest) {
+            this.dest = Objects.requireNonNull(dest);
+            return this;
+        }
+
+        public Builder setAnalysis(DataFrameAnalysis analysis) {
+            this.analysis = Objects.requireNonNull(analysis);
+            return this;
+        }
+
+        public Builder setAnalyzedFields(FetchSourceContext fields) {
+            this.analyzedFields = fields;
+            return this;
+        }
+
+        public Builder setModelMemoryLimit(ByteSizeValue modelMemoryLimit) {
+            this.modelMemoryLimit = modelMemoryLimit;
+            return this;
+        }
+
+        public Builder setCreateTime(Instant createTime) {
+            this.createTime = createTime;
+            return this;
+        }
+
+        public Builder setVersion(Version version) {
+            this.version = version;
+            return this;
+        }
+
+        public DataFrameAnalyticsConfig build() {
+            return new DataFrameAnalyticsConfig(id, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version);
+        }
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsDest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsDest.java
new file mode 100644
index 0000000000000..4123f85ee2f43
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsDest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static java.util.Objects.requireNonNull;
+
+public class DataFrameAnalyticsDest implements ToXContentObject {
+
+    public static DataFrameAnalyticsDest fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null).build();
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    private static final ParseField INDEX = new ParseField("index");
+    private static final ParseField RESULTS_FIELD = new ParseField("results_field");
+
+    private static ObjectParser<Builder, Void> PARSER = new ObjectParser<>("data_frame_analytics_dest", true, Builder::new);
+
+    static {
+        PARSER.declareString(Builder::setIndex, INDEX);
+        PARSER.declareString(Builder::setResultsField, RESULTS_FIELD);
+    }
+
+    private final String index;
+    private final String resultsField;
+
+    private DataFrameAnalyticsDest(String index, @Nullable String resultsField) {
+        this.index = requireNonNull(index);
+        this.resultsField = resultsField;
+    }
+
+    public String getIndex() {
+        return index;
+    }
+
+    public String getResultsField() {
+        return resultsField;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(INDEX.getPreferredName(), index);
+        if (resultsField != null) {
+            builder.field(RESULTS_FIELD.getPreferredName(), resultsField);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        DataFrameAnalyticsDest other = (DataFrameAnalyticsDest) o;
+        return Objects.equals(index, other.index)
+            && Objects.equals(resultsField, other.resultsField);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(index, resultsField);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    public static class Builder {
+
+        private String index;
+        private String resultsField;
+
+        private Builder() {}
+
+        public Builder setIndex(String index) {
+            this.index = index;
+            return this;
+        }
+
+        public Builder setResultsField(String resultsField) {
+            this.resultsField = resultsField;
+            return this;
+        }
+
+        public DataFrameAnalyticsDest build() {
+            return new DataFrameAnalyticsDest(index, resultsField);
+        }
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsSource.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsSource.java
new file mode 100644
index 0000000000000..9a6de159bea3e
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsSource.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class DataFrameAnalyticsSource implements ToXContentObject {
+
+    public static DataFrameAnalyticsSource fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null).build();
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    private static final ParseField INDEX = new ParseField("index");
+    private static final ParseField QUERY = new ParseField("query");
+
+    private static ObjectParser<Builder, Void> PARSER = new ObjectParser<>("data_frame_analytics_source", true, Builder::new);
+
+    static {
+        PARSER.declareStringArray(Builder::setIndex, INDEX);
+        PARSER.declareObject(Builder::setQueryConfig, (p, c) -> QueryConfig.fromXContent(p), QUERY);
+    }
+
+    private final String[] index;
+    private final QueryConfig queryConfig;
+
+    private DataFrameAnalyticsSource(String[] index, @Nullable QueryConfig queryConfig) {
+        this.index = Objects.requireNonNull(index);
+        this.queryConfig = queryConfig;
+    }
+
+    public String[] getIndex() {
+        return index;
+    }
+
+    public QueryConfig getQueryConfig() {
+        return queryConfig;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(INDEX.getPreferredName(), index);
+        if (queryConfig != null) {
+            builder.field(QUERY.getPreferredName(), queryConfig.getQuery());
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        DataFrameAnalyticsSource other = (DataFrameAnalyticsSource) o;
+        return Arrays.equals(index, other.index)
+            && Objects.equals(queryConfig, other.queryConfig);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(Arrays.asList(index), queryConfig);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    public static class Builder {
+
+        private String[] index;
+        private QueryConfig queryConfig;
+
+        private Builder() {}
+
+        public Builder setIndex(String... index) {
+            this.index = index;
+            return this;
+        }
+
+        public Builder setIndex(List<String> index) {
+            this.index = index.toArray(new String[0]);
+            return this;
+        }
+
+        public Builder setQueryConfig(QueryConfig queryConfig) {
+            this.queryConfig = queryConfig;
+            return this;
+        }
+
+        public DataFrameAnalyticsSource build() {
+            return new DataFrameAnalyticsSource(index, queryConfig);
+        }
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsState.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsState.java
new file mode 100644
index 0000000000000..6ee349b8e8d38
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsState.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe;
+
+import java.util.Locale;
+
+public enum DataFrameAnalyticsState {
+    STARTED, REINDEXING, ANALYZING, STOPPING, STOPPED;
+
+    public static DataFrameAnalyticsState fromString(String name) {
+        return valueOf(name.trim().toUpperCase(Locale.ROOT));
+    }
+
+    public String value() {
+        return name().toLowerCase(Locale.ROOT);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java
new file mode 100644
index 0000000000000..4e04204e65021
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStats.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe;
+
+import org.elasticsearch.client.ml.NodeAttributes;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.inject.internal.ToStringBuilder;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class DataFrameAnalyticsStats {
+
+    public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    static final ParseField ID = new ParseField("id");
+    static final ParseField STATE = new ParseField("state");
+    static final ParseField FAILURE_REASON = new ParseField("failure_reason");
+    static final ParseField PROGRESS_PERCENT = new ParseField("progress_percent");
+    static final ParseField NODE = new ParseField("node");
+    static final ParseField ASSIGNMENT_EXPLANATION = new ParseField("assignment_explanation");
+
+    private static final ConstructingObjectParser<DataFrameAnalyticsStats, Void> PARSER =
+        new ConstructingObjectParser<>("data_frame_analytics_stats", true,
+            args -> new DataFrameAnalyticsStats(
+                (String) args[0],
+                (DataFrameAnalyticsState) args[1],
+                (String) args[2],
+                (Integer) args[3],
+                (NodeAttributes) args[4],
+                (String) args[5]));
+
+    static {
+        PARSER.declareString(constructorArg(), ID);
+        PARSER.declareField(constructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return DataFrameAnalyticsState.fromString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, STATE, ObjectParser.ValueType.STRING);
+        PARSER.declareString(optionalConstructorArg(), FAILURE_REASON);
+        PARSER.declareInt(optionalConstructorArg(), PROGRESS_PERCENT);
+        PARSER.declareObject(optionalConstructorArg(), NodeAttributes.PARSER, NODE);
+        PARSER.declareString(optionalConstructorArg(), ASSIGNMENT_EXPLANATION);
+    }
+
+    private final String id;
+    private final DataFrameAnalyticsState state;
+    private final String failureReason;
+    private final Integer progressPercent;
+    private final NodeAttributes node;
+    private final String assignmentExplanation;
+
+    public DataFrameAnalyticsStats(String id, DataFrameAnalyticsState state, @Nullable String failureReason,
+                                   @Nullable Integer progressPercent, @Nullable NodeAttributes node,
+                                   @Nullable String assignmentExplanation) {
+        this.id = id;
+        this.state = state;
+        this.failureReason = failureReason;
+        this.progressPercent = progressPercent;
+        this.node = node;
+        this.assignmentExplanation = assignmentExplanation;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public DataFrameAnalyticsState getState() {
+        return state;
+    }
+
+    public String getFailureReason() {
+        return failureReason;
+    }
+
+    public Integer getProgressPercent() {
+        return progressPercent;
+    }
+
+    public NodeAttributes getNode() {
+        return node;
+    }
+
+    public String getAssignmentExplanation() {
+        return assignmentExplanation;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        DataFrameAnalyticsStats other = (DataFrameAnalyticsStats) o;
+        return Objects.equals(id, other.id)
+            && Objects.equals(state, other.state)
+            && Objects.equals(failureReason, other.failureReason)
+            && Objects.equals(progressPercent, other.progressPercent)
+            && Objects.equals(node, other.node)
+            && Objects.equals(assignmentExplanation, other.assignmentExplanation);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, state, failureReason, progressPercent, node, assignmentExplanation);
+    }
+
+    @Override
+    public String toString() {
+        return new ToStringBuilder(getClass())
+            .add("id", id)
+            .add("state", state)
+            .add("failureReason", failureReason)
+            .add("progressPercent", progressPercent)
+            .add("node", node)
+            .add("assignmentExplanation", assignmentExplanation)
+            .toString();
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/MlDataFrameAnalysisNamedXContentProvider.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/MlDataFrameAnalysisNamedXContentProvider.java
new file mode 100644
index 0000000000000..3b78c60be91fd
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/MlDataFrameAnalysisNamedXContentProvider.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe;
+
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.plugins.spi.NamedXContentProvider;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class MlDataFrameAnalysisNamedXContentProvider implements NamedXContentProvider {
+
+    @Override
+    public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
+        return Arrays.asList(
+            new NamedXContentRegistry.Entry(
+                DataFrameAnalysis.class,
+                OutlierDetection.NAME,
+                (p, c) -> OutlierDetection.fromXContent(p)));
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/OutlierDetection.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/OutlierDetection.java
new file mode 100644
index 0000000000000..fe5094fb7190a
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/OutlierDetection.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+
+public class OutlierDetection implements DataFrameAnalysis {
+
+    public static OutlierDetection fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null).build();
+    }
+
+    public static OutlierDetection createDefault() {
+        return builder().build();
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static final ParseField NAME = new ParseField("outlier_detection");
+    static final ParseField N_NEIGHBORS = new ParseField("n_neighbors");
+    static final ParseField METHOD = new ParseField("method");
+    public static final ParseField FEATURE_INFLUENCE_THRESHOLD = new ParseField("feature_influence_threshold");
+
+    private static ObjectParser<Builder, Void> PARSER = new ObjectParser<>(NAME.getPreferredName(), true, Builder::new);
+
+    static {
+        PARSER.declareInt(Builder::setNNeighbors, N_NEIGHBORS);
+        PARSER.declareField(Builder::setMethod, p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return Method.fromString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, METHOD, ObjectParser.ValueType.STRING);
+        PARSER.declareDouble(Builder::setFeatureInfluenceThreshold, FEATURE_INFLUENCE_THRESHOLD);
+    }
+
+    private final Integer nNeighbors;
+    private final Method method;
+    private final Double featureInfluenceThreshold;
+
+    /**
+     * Constructs the outlier detection configuration
+     * @param nNeighbors The number of neighbors. Leave unspecified for dynamic detection.
+     * @param method The method. Leave unspecified for a dynamic mixture of methods.
+     * @param featureInfluenceThreshold The min outlier score required to calculate feature influence. Defaults to 0.1.
+     */
+    private OutlierDetection(@Nullable Integer nNeighbors, @Nullable Method method, @Nullable Double featureInfluenceThreshold) {
+        this.nNeighbors = nNeighbors;
+        this.method = method;
+        this.featureInfluenceThreshold = featureInfluenceThreshold;
+    }
+
+    @Override
+    public String getName() {
+        return NAME.getPreferredName();
+    }
+
+    public Integer getNNeighbors() {
+        return nNeighbors;
+    }
+
+    public Method getMethod() {
+        return method;
+    }
+
+    public Double getFeatureInfluenceThreshold() {
+        return featureInfluenceThreshold;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (nNeighbors != null) {
+            builder.field(N_NEIGHBORS.getPreferredName(), nNeighbors);
+        }
+        if (method != null) {
+            builder.field(METHOD.getPreferredName(), method);
+        }
+        if (featureInfluenceThreshold != null) {
+            builder.field(FEATURE_INFLUENCE_THRESHOLD.getPreferredName(), featureInfluenceThreshold);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        OutlierDetection other = (OutlierDetection) o;
+        return Objects.equals(nNeighbors, other.nNeighbors)
+            && Objects.equals(method, other.method)
+            && Objects.equals(featureInfluenceThreshold, other.featureInfluenceThreshold);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(nNeighbors, method, featureInfluenceThreshold);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    public enum Method {
+        LOF, LDOF, DISTANCE_KTH_NN, DISTANCE_KNN;
+
+        public static Method fromString(String value) {
+            return Method.valueOf(value.toUpperCase(Locale.ROOT));
+        }
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
+    }
+
+    public static class Builder {
+
+        private Integer nNeighbors;
+        private Method method;
+        private Double featureInfluenceThreshold;
+
+        private Builder() {}
+
+        public Builder setNNeighbors(Integer nNeighbors) {
+            this.nNeighbors = nNeighbors;
+            return this;
+        }
+
+        public Builder setMethod(Method method) {
+            this.method = method;
+            return this;
+        }
+
+        public Builder setFeatureInfluenceThreshold(Double featureInfluenceThreshold) {
+            this.featureInfluenceThreshold = featureInfluenceThreshold;
+            return this;
+        }
+
+        public OutlierDetection build() {
+            return new OutlierDetection(nNeighbors, method, featureInfluenceThreshold);
+        }
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/QueryConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/QueryConfig.java
new file mode 100644
index 0000000000000..ae704db9f800e
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/QueryConfig.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Object for encapsulating the desired Query for a DataFrameAnalysis
+ */
+public class QueryConfig implements ToXContentObject {
+
+    public static QueryConfig fromXContent(XContentParser parser) throws IOException {
+        QueryBuilder query = AbstractQueryBuilder.parseInnerQueryBuilder(parser);
+        return new QueryConfig(query);
+    }
+
+    private final QueryBuilder query;
+
+    public QueryConfig(QueryBuilder query) {
+        this.query = requireNonNull(query);
+    }
+
+    public QueryConfig(QueryConfig queryConfig) {
+        this(requireNonNull(queryConfig).query);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        query.toXContent(builder, params);
+        return builder;
+    }
+
+    public QueryBuilder getQuery() {
+        return query;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        QueryConfig other = (QueryConfig) o;
+        return Objects.equals(query, other.query);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(query);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/Evaluation.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/Evaluation.java
new file mode 100644
index 0000000000000..78578597e195b
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/Evaluation.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.evaluation;
+
+import org.elasticsearch.common.xcontent.ToXContentObject;
+
+/**
+ * Defines an evaluation
+ */
+public interface Evaluation extends ToXContentObject {
+
+    /**
+     * Returns the evaluation name
+     */
+    String getName();
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/AliasesExistAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/EvaluationMetric.java
similarity index 57%
rename from server/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/AliasesExistAction.java
rename to client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/EvaluationMetric.java
index dfaebab076c66..a0f77838f1fd0 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/AliasesExistAction.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/EvaluationMetric.java
@@ -7,7 +7,7 @@
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
@@ -16,22 +16,28 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+package org.elasticsearch.client.ml.dataframe.evaluation;
 
-package org.elasticsearch.action.admin.indices.alias.exists;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 
-import org.elasticsearch.action.Action;
-
-public class AliasesExistAction extends Action<AliasesExistResponse> {
+/**
+ * Defines an evaluation metric
+ */
+public interface EvaluationMetric extends ToXContentObject {
 
-    public static final AliasesExistAction INSTANCE = new AliasesExistAction();
-    public static final String NAME = "indices:admin/aliases/exists";
+    /**
+     * Returns the name of the metric
+     */
+    String getName();
 
-    private AliasesExistAction() {
-        super(NAME);
-    }
+    /**
+     * The result of an evaluation metric
+     */
+    interface Result extends ToXContentObject {
 
-    @Override
-    public AliasesExistResponse newResponse() {
-        return new AliasesExistResponse();
+        /**
+         * Returns the name of the metric
+         */
+        String getMetricName();
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java
new file mode 100644
index 0000000000000..764ff41de86e0
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.evaluation;
+
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.BinarySoftClassification;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.plugins.spi.NamedXContentProvider;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.AucRocMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.PrecisionMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.RecallMetric;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class MlEvaluationNamedXContentProvider implements NamedXContentProvider {
+
+    @Override
+    public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
+        return Arrays.asList(
+            // Evaluations
+            new NamedXContentRegistry.Entry(
+                Evaluation.class, new ParseField(BinarySoftClassification.NAME), BinarySoftClassification::fromXContent),
+            // Evaluation metrics
+            new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(AucRocMetric.NAME), AucRocMetric::fromXContent),
+            new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(PrecisionMetric.NAME), PrecisionMetric::fromXContent),
+            new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(RecallMetric.NAME), RecallMetric::fromXContent),
+            new NamedXContentRegistry.Entry(
+                EvaluationMetric.class, new ParseField(ConfusionMatrixMetric.NAME), ConfusionMatrixMetric::fromXContent),
+            // Evaluation metrics results
+            new NamedXContentRegistry.Entry(
+                EvaluationMetric.Result.class, new ParseField(AucRocMetric.NAME), AucRocMetric.Result::fromXContent),
+            new NamedXContentRegistry.Entry(
+                EvaluationMetric.Result.class, new ParseField(PrecisionMetric.NAME), PrecisionMetric.Result::fromXContent),
+            new NamedXContentRegistry.Entry(
+                EvaluationMetric.Result.class, new ParseField(RecallMetric.NAME), RecallMetric.Result::fromXContent),
+            new NamedXContentRegistry.Entry(
+                EvaluationMetric.Result.class, new ParseField(ConfusionMatrixMetric.NAME), ConfusionMatrixMetric.Result::fromXContent));
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/AbstractConfusionMatrixMetric.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/AbstractConfusionMatrixMetric.java
new file mode 100644
index 0000000000000..f41c13f248ab9
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/AbstractConfusionMatrixMetric.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+abstract class AbstractConfusionMatrixMetric implements EvaluationMetric {
+
+    protected static final ParseField AT = new ParseField("at");
+
+    protected final double[] thresholds;
+
+    protected AbstractConfusionMatrixMetric(List<Double> at) {
+        this.thresholds = Objects.requireNonNull(at).stream().mapToDouble(Double::doubleValue).toArray();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        return builder
+            .startObject()
+            .field(AT.getPreferredName(), thresholds)
+            .endObject();
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/AucRocMetric.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/AucRocMetric.java
new file mode 100644
index 0000000000000..78c713c592581
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/AucRocMetric.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * Area under the curve (AUC) of the receiver operating characteristic (ROC).
+ * The ROC curve is a plot of the TPR (true positive rate) against
+ * the FPR (false positive rate) over a varying threshold.
+ */
+public class AucRocMetric implements EvaluationMetric {
+
+    public static final String NAME = "auc_roc";
+
+    public static final ParseField INCLUDE_CURVE = new ParseField("include_curve");
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<AucRocMetric, Void> PARSER =
+        new ConstructingObjectParser<>(NAME, args -> new AucRocMetric((Boolean) args[0]));
+
+    static {
+        PARSER.declareBoolean(optionalConstructorArg(), INCLUDE_CURVE);
+    }
+
+    public static AucRocMetric fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public static AucRocMetric withCurve() {
+        return new AucRocMetric(true);
+    }
+
+    private final boolean includeCurve;
+
+    public AucRocMetric(Boolean includeCurve) {
+        this.includeCurve = includeCurve == null ? false : includeCurve;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        return builder
+            .startObject()
+            .field(INCLUDE_CURVE.getPreferredName(), includeCurve)
+            .endObject();
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AucRocMetric that = (AucRocMetric) o;
+        return Objects.equals(includeCurve, that.includeCurve);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(includeCurve);
+    }
+
+    public static class Result implements EvaluationMetric.Result {
+
+        public static Result fromXContent(XContentParser parser) {
+            return PARSER.apply(parser, null);
+        }
+
+        private static final ParseField SCORE = new ParseField("score");
+        private static final ParseField CURVE = new ParseField("curve");
+
+        @SuppressWarnings("unchecked")
+        private static final ConstructingObjectParser<Result, Void> PARSER =
+            new ConstructingObjectParser<>("auc_roc_result", true, args -> new Result((double) args[0], (List<AucRocPoint>) args[1]));
+
+        static {
+            PARSER.declareDouble(constructorArg(), SCORE);
+            PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> AucRocPoint.fromXContent(p), CURVE);
+        }
+
+        private final double score;
+        private final List<AucRocPoint> curve;
+
+        public Result(double score, @Nullable List<AucRocPoint> curve) {
+            this.score = score;
+            this.curve = curve;
+        }
+
+        @Override
+        public String getMetricName() {
+            return NAME;
+        }
+
+        public double getScore() {
+            return score;
+        }
+
+        public List<AucRocPoint> getCurve() {
+            return curve == null ? null : Collections.unmodifiableList(curve);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+            builder.startObject();
+            builder.field(SCORE.getPreferredName(), score);
+            if (curve != null && curve.isEmpty() == false) {
+                builder.field(CURVE.getPreferredName(), curve);
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Result that = (Result) o;
+            return Objects.equals(score, that.score)
+                && Objects.equals(curve, that.curve);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(score, curve);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+
+    public static final class AucRocPoint implements ToXContentObject {
+
+        public static AucRocPoint fromXContent(XContentParser parser) {
+            return PARSER.apply(parser, null);
+        }
+
+        private static final ParseField TPR = new ParseField("tpr");
+        private static final ParseField FPR = new ParseField("fpr");
+        private static final ParseField THRESHOLD = new ParseField("threshold");
+
+        @SuppressWarnings("unchecked")
+        private static final ConstructingObjectParser<AucRocPoint, Void> PARSER =
+            new ConstructingObjectParser<>(
+                "auc_roc_point",
+                true,
+                args -> new AucRocPoint((double) args[0], (double) args[1], (double) args[2]));
+
+        static {
+            PARSER.declareDouble(constructorArg(), TPR);
+            PARSER.declareDouble(constructorArg(), FPR);
+            PARSER.declareDouble(constructorArg(), THRESHOLD);
+        }
+
+        private final double tpr;
+        private final double fpr;
+        private final double threshold;
+
+        public AucRocPoint(double tpr, double fpr, double threshold) {
+            this.tpr = tpr;
+            this.fpr = fpr;
+            this.threshold = threshold;
+        }
+
+        public double getTruePositiveRate() {
+            return tpr;
+        }
+
+        public double getFalsePositiveRate() {
+            return fpr;
+        }
+
+        public double getThreshold() {
+            return threshold;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder
+                .startObject()
+                .field(TPR.getPreferredName(), tpr)
+                .field(FPR.getPreferredName(), fpr)
+                .field(THRESHOLD.getPreferredName(), threshold)
+                .endObject();
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            AucRocPoint that = (AucRocPoint) o;
+            return tpr == that.tpr && fpr == that.fpr && threshold == that.threshold;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(tpr, fpr, threshold);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/BinarySoftClassification.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/BinarySoftClassification.java
new file mode 100644
index 0000000000000..6d5fa04da38e5
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/BinarySoftClassification.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.client.ml.dataframe.evaluation.Evaluation;
+import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * Evaluation of binary soft classification methods, e.g. outlier detection.
+ * This is useful to evaluate problems where a model outputs a probability of whether
+ * a data frame row belongs to one of two groups.
+ */
+public class BinarySoftClassification implements Evaluation {
+
+    public static final String NAME = "binary_soft_classification";
+
+    private static final ParseField ACTUAL_FIELD = new ParseField("actual_field");
+    private static final ParseField PREDICTED_PROBABILITY_FIELD = new ParseField("predicted_probability_field");
+    private static final ParseField METRICS = new ParseField("metrics");
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<BinarySoftClassification, Void> PARSER =
+        new ConstructingObjectParser<>(
+            NAME,
+            args -> new BinarySoftClassification((String) args[0], (String) args[1], (List<EvaluationMetric>) args[2]));
+
+    static {
+        PARSER.declareString(constructorArg(), ACTUAL_FIELD);
+        PARSER.declareString(constructorArg(), PREDICTED_PROBABILITY_FIELD);
+        PARSER.declareNamedObjects(optionalConstructorArg(), (p, c, n) -> p.namedObject(EvaluationMetric.class, n, null), METRICS);
+    }
+
+    public static BinarySoftClassification fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    /**
+     * The field where the actual class is marked up.
+     * The value of this field is assumed to either be 1 or 0, or true or false.
+     */
+    private final String actualField;
+
+    /**
+     * The field of the predicted probability in [0.0, 1.0].
+     */
+    private final String predictedProbabilityField;
+
+    /**
+     * The list of metrics to calculate
+     */
+    private final List<EvaluationMetric> metrics;
+
+    public BinarySoftClassification(String actualField, String predictedProbabilityField, EvaluationMetric... metric) {
+        this(actualField, predictedProbabilityField, Arrays.asList(metric));
+    }
+
+    public BinarySoftClassification(String actualField, String predictedProbabilityField,
+                                    @Nullable List<EvaluationMetric> metrics) {
+        this.actualField = Objects.requireNonNull(actualField);
+        this.predictedProbabilityField = Objects.requireNonNull(predictedProbabilityField);
+        this.metrics = Objects.requireNonNull(metrics);
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
+        builder.field(ACTUAL_FIELD.getPreferredName(), actualField);
+        builder.field(PREDICTED_PROBABILITY_FIELD.getPreferredName(), predictedProbabilityField);
+
+        builder.startObject(METRICS.getPreferredName());
+        for (EvaluationMetric metric : metrics) {
+            builder.field(metric.getName(), metric);
+        }
+        builder.endObject();
+
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        BinarySoftClassification that = (BinarySoftClassification) o;
+        return Objects.equals(actualField, that.actualField)
+            && Objects.equals(predictedProbabilityField, that.predictedProbabilityField)
+            && Objects.equals(metrics, that.metrics);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(actualField, predictedProbabilityField, metrics);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/ConfusionMatrixMetric.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/ConfusionMatrixMetric.java
new file mode 100644
index 0000000000000..d5e4307c9cc74
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/ConfusionMatrixMetric.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
+public class ConfusionMatrixMetric extends AbstractConfusionMatrixMetric {
+
+    public static final String NAME = "confusion_matrix";
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<ConfusionMatrixMetric, Void> PARSER =
+        new ConstructingObjectParser<>(NAME, args -> new ConfusionMatrixMetric((List<Double>) args[0]));
+
+    static {
+        PARSER.declareDoubleArray(constructorArg(), AT);
+    }
+
+    public static ConfusionMatrixMetric fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public static ConfusionMatrixMetric at(Double... at) {
+        return new ConfusionMatrixMetric(Arrays.asList(at));
+    }
+
+    public ConfusionMatrixMetric(List<Double> at) {
+        super(at);
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ConfusionMatrixMetric that = (ConfusionMatrixMetric) o;
+        return Arrays.equals(thresholds, that.thresholds);
+    }
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(thresholds);
+    }
+
+    public static class Result implements EvaluationMetric.Result {
+
+        public static Result fromXContent(XContentParser parser) throws IOException {
+            return new Result(parser.map(LinkedHashMap::new, ConfusionMatrix::fromXContent));
+        }
+
+        private final Map<String, ConfusionMatrix> results;
+
+        public Result(Map<String, ConfusionMatrix> results) {
+            this.results = Objects.requireNonNull(results);
+        }
+
+        @Override
+        public String getMetricName() {
+            return NAME;
+        }
+
+        public ConfusionMatrix getScoreByThreshold(String threshold) {
+            return results.get(threshold);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+            return builder.map(results);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Result that = (Result) o;
+            return Objects.equals(results, that.results);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(results);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+
+    public static final class ConfusionMatrix implements ToXContentObject {
+
+        public static ConfusionMatrix fromXContent(XContentParser parser) {
+            return PARSER.apply(parser, null);
+        }
+
+        private static final ParseField TP = new ParseField("tp");
+        private static final ParseField FP = new ParseField("fp");
+        private static final ParseField TN = new ParseField("tn");
+        private static final ParseField FN = new ParseField("fn");
+
+        @SuppressWarnings("unchecked")
+        private static final ConstructingObjectParser<ConfusionMatrix, Void> PARSER =
+            new ConstructingObjectParser<>(
+                "confusion_matrix", true, args -> new ConfusionMatrix((long) args[0], (long) args[1], (long) args[2], (long) args[3]));
+
+        static {
+            PARSER.declareLong(constructorArg(), TP);
+            PARSER.declareLong(constructorArg(), FP);
+            PARSER.declareLong(constructorArg(), TN);
+            PARSER.declareLong(constructorArg(), FN);
+        }
+
+        private final long tp;
+        private final long fp;
+        private final long tn;
+        private final long fn;
+
+        public ConfusionMatrix(long tp, long fp, long tn, long fn) {
+            this.tp = tp;
+            this.fp = fp;
+            this.tn = tn;
+            this.fn = fn;
+        }
+
+        public long getTruePositives() {
+            return tp;
+        }
+
+        public long getFalsePositives() {
+            return fp;
+        }
+
+        public long getTrueNegatives() {
+            return tn;
+        }
+
+        public long getFalseNegatives() {
+            return fn;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder
+                .startObject()
+                .field(TP.getPreferredName(), tp)
+                .field(FP.getPreferredName(), fp)
+                .field(TN.getPreferredName(), tn)
+                .field(FN.getPreferredName(), fn)
+                .endObject();
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            ConfusionMatrix that = (ConfusionMatrix) o;
+            return tp == that.tp && fp == that.fp && tn == that.tn && fn == that.fn;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(tp, fp, tn, fn);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/PrecisionMetric.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/PrecisionMetric.java
new file mode 100644
index 0000000000000..2a0f1499461d6
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/PrecisionMetric.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
+public class PrecisionMetric extends AbstractConfusionMatrixMetric {
+
+    public static final String NAME = "precision";
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<PrecisionMetric, Void> PARSER =
+        new ConstructingObjectParser<>(NAME, args -> new PrecisionMetric((List<Double>) args[0]));
+
+    static {
+        PARSER.declareDoubleArray(constructorArg(), AT);
+    }
+
+    public static PrecisionMetric fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public static PrecisionMetric at(Double... at) {
+        return new PrecisionMetric(Arrays.asList(at));
+    }
+
+    public PrecisionMetric(List<Double> at) {
+        super(at);
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        PrecisionMetric that = (PrecisionMetric) o;
+        return Arrays.equals(thresholds, that.thresholds);
+    }
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(thresholds);
+    }
+
+    public static class Result implements EvaluationMetric.Result {
+
+        public static Result fromXContent(XContentParser parser) throws IOException {
+            return new Result(parser.map(LinkedHashMap::new, p -> p.doubleValue()));
+        }
+
+        private final Map<String, Double> results;
+
+        public Result(Map<String, Double> results) {
+            this.results = Objects.requireNonNull(results);
+        }
+
+        @Override
+        public String getMetricName() {
+            return NAME;
+        }
+
+        public Double getScoreByThreshold(String threshold) {
+            return results.get(threshold);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+            return builder.map(results);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Result that = (Result) o;
+            return Objects.equals(results, that.results);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(results);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/RecallMetric.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/RecallMetric.java
new file mode 100644
index 0000000000000..505ff1b34d7c5
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/softclassification/RecallMetric.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
+public class RecallMetric extends AbstractConfusionMatrixMetric {
+
+    public static final String NAME = "recall";
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<RecallMetric, Void> PARSER =
+        new ConstructingObjectParser<>(NAME, args -> new RecallMetric((List<Double>) args[0]));
+
+    static {
+        PARSER.declareDoubleArray(constructorArg(), AT);
+    }
+
+    public static RecallMetric fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public static RecallMetric at(Double... at) {
+        return new RecallMetric(Arrays.asList(at));
+    }
+
+    public RecallMetric(List<Double> at) {
+        super(at);
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        RecallMetric that = (RecallMetric) o;
+        return Arrays.equals(thresholds, that.thresholds);
+    }
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(thresholds);
+    }
+
+    public static class Result implements EvaluationMetric.Result {
+
+        public static Result fromXContent(XContentParser parser) throws IOException {
+            return new Result(parser.map(LinkedHashMap::new, p -> p.doubleValue()));
+        }
+
+        private final Map<String, Double> results;
+
+        public Result(Map<String, Double> results) {
+            this.results = Objects.requireNonNull(results);
+        }
+
+        @Override
+        public String getMetricName() {
+            return NAME;
+        }
+
+        public Double getScoreByThreshold(String threshold) {
+            return results.get(threshold);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+            return builder.map(results);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Result that = (Result) o;
+            return Objects.equals(results, that.results);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(results);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/filestructurefinder/FieldStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/filestructurefinder/FieldStats.java
index 4391d03f6d940..adb8e68393e82 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/filestructurefinder/FieldStats.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/filestructurefinder/FieldStats.java
@@ -38,12 +38,14 @@ public class FieldStats implements ToXContentObject {
     public static final ParseField MAX_VALUE = new ParseField("max_value");
     public static final ParseField MEAN_VALUE = new ParseField("mean_value");
     public static final ParseField MEDIAN_VALUE = new ParseField("median_value");
+    public static final ParseField EARLIEST = new ParseField("earliest");
+    public static final ParseField LATEST = new ParseField("latest");
     public static final ParseField TOP_HITS = new ParseField("top_hits");
 
     @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<FieldStats, Void> PARSER = new ConstructingObjectParser<>("field_stats", true,
         a -> new FieldStats((long) a[0], (int) a[1], (Double) a[2], (Double) a[3], (Double) a[4], (Double) a[5],
-            (List<Map<String, Object>>) a[6]));
+            (String) a[6], (String) a[7], (List<Map<String, Object>>) a[8]));
 
     static {
         PARSER.declareLong(ConstructingObjectParser.constructorArg(), COUNT);
@@ -52,6 +54,8 @@ public class FieldStats implements ToXContentObject {
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), MAX_VALUE);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), MEAN_VALUE);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), MEDIAN_VALUE);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), EARLIEST);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), LATEST);
         PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.mapOrdered(), TOP_HITS);
     }
 
@@ -61,16 +65,20 @@ public class FieldStats implements ToXContentObject {
     private final Double maxValue;
     private final Double meanValue;
     private final Double medianValue;
+    private final String earliestTimestamp;
+    private final String latestTimestamp;
     private final List<Map<String, Object>> topHits;
 
     FieldStats(long count, int cardinality, Double minValue, Double maxValue, Double meanValue, Double medianValue,
-               List<Map<String, Object>> topHits) {
+               String earliestTimestamp, String latestTimestamp, List<Map<String, Object>> topHits) {
         this.count = count;
         this.cardinality = cardinality;
         this.minValue = minValue;
         this.maxValue = maxValue;
         this.meanValue = meanValue;
         this.medianValue = medianValue;
+        this.earliestTimestamp = earliestTimestamp;
+        this.latestTimestamp = latestTimestamp;
         this.topHits = (topHits == null) ? Collections.emptyList() : Collections.unmodifiableList(topHits);
     }
 
@@ -98,6 +106,14 @@ public Double getMedianValue() {
         return medianValue;
     }
 
+    public String getEarliestTimestamp() {
+        return earliestTimestamp;
+    }
+
+    public String getLatestTimestamp() {
+        return latestTimestamp;
+    }
+
     public List<Map<String, Object>> getTopHits() {
         return topHits;
     }
@@ -120,6 +136,12 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
         if (medianValue != null) {
             builder.field(MEDIAN_VALUE.getPreferredName(), toIntegerIfInteger(medianValue));
         }
+        if (earliestTimestamp != null) {
+            builder.field(EARLIEST.getPreferredName(), earliestTimestamp);
+        }
+        if (latestTimestamp != null) {
+            builder.field(LATEST.getPreferredName(), latestTimestamp);
+        }
         if (topHits.isEmpty() == false) {
             builder.field(TOP_HITS.getPreferredName(), topHits);
         }
@@ -140,7 +162,7 @@ static Number toIntegerIfInteger(double d) {
     @Override
     public int hashCode() {
 
-        return Objects.hash(count, cardinality, minValue, maxValue, meanValue, medianValue, topHits);
+        return Objects.hash(count, cardinality, minValue, maxValue, meanValue, medianValue, earliestTimestamp, latestTimestamp, topHits);
     }
 
     @Override
@@ -161,6 +183,8 @@ public boolean equals(Object other) {
             Objects.equals(this.maxValue, that.maxValue) &&
             Objects.equals(this.meanValue, that.meanValue) &&
             Objects.equals(this.medianValue, that.medianValue) &&
+            Objects.equals(this.earliestTimestamp, that.earliestTimestamp) &&
+            Objects.equals(this.latestTimestamp, that.latestTimestamp) &&
             Objects.equals(this.topHits, that.topHits);
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/process/TimingStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/process/TimingStats.java
new file mode 100644
index 0000000000000..73393140f30af
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/process/TimingStats.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.job.process;
+
+import org.elasticsearch.client.ml.job.config.Job;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * Stats that give more insight into timing of various operations performed as part of anomaly detection job.
+ */
+public class TimingStats implements ToXContentObject {
+
+    public static final ParseField BUCKET_COUNT = new ParseField("bucket_count");
+    public static final ParseField MIN_BUCKET_PROCESSING_TIME_MS = new ParseField("minimum_bucket_processing_time_ms");
+    public static final ParseField MAX_BUCKET_PROCESSING_TIME_MS = new ParseField("maximum_bucket_processing_time_ms");
+    public static final ParseField AVG_BUCKET_PROCESSING_TIME_MS = new ParseField("average_bucket_processing_time_ms");
+    public static final ParseField EXPONENTIAL_AVG_BUCKET_PROCESSING_TIME_MS =
+        new ParseField("exponential_average_bucket_processing_time_ms");
+
+    public static final ConstructingObjectParser<TimingStats, Void> PARSER =
+        new ConstructingObjectParser<>(
+            "timing_stats",
+            true,
+            args ->
+                new TimingStats((String) args[0], (long) args[1], (Double) args[2], (Double) args[3], (Double) args[4], (Double) args[5]));
+
+    static {
+        PARSER.declareString(constructorArg(), Job.ID);
+        PARSER.declareLong(constructorArg(), BUCKET_COUNT);
+        PARSER.declareDouble(optionalConstructorArg(), MIN_BUCKET_PROCESSING_TIME_MS);
+        PARSER.declareDouble(optionalConstructorArg(), MAX_BUCKET_PROCESSING_TIME_MS);
+        PARSER.declareDouble(optionalConstructorArg(), AVG_BUCKET_PROCESSING_TIME_MS);
+        PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_BUCKET_PROCESSING_TIME_MS);
+    }
+
+    private final String jobId;
+    private long bucketCount;
+    private Double minBucketProcessingTimeMs;
+    private Double maxBucketProcessingTimeMs;
+    private Double avgBucketProcessingTimeMs;
+    private Double exponentialAvgBucketProcessingTimeMs;
+
+    public TimingStats(
+            String jobId,
+            long bucketCount,
+            @Nullable Double minBucketProcessingTimeMs,
+            @Nullable Double maxBucketProcessingTimeMs,
+            @Nullable Double avgBucketProcessingTimeMs,
+            @Nullable Double exponentialAvgBucketProcessingTimeMs) {
+        this.jobId = jobId;
+        this.bucketCount = bucketCount;
+        this.minBucketProcessingTimeMs = minBucketProcessingTimeMs;
+        this.maxBucketProcessingTimeMs = maxBucketProcessingTimeMs;
+        this.avgBucketProcessingTimeMs = avgBucketProcessingTimeMs;
+        this.exponentialAvgBucketProcessingTimeMs = exponentialAvgBucketProcessingTimeMs;
+    }
+
+    public String getJobId() {
+        return jobId;
+    }
+
+    public long getBucketCount() {
+        return bucketCount;
+    }
+
+    public Double getMinBucketProcessingTimeMs() {
+        return minBucketProcessingTimeMs;
+    }
+
+    public Double getMaxBucketProcessingTimeMs() {
+        return maxBucketProcessingTimeMs;
+    }
+
+    public Double getAvgBucketProcessingTimeMs() {
+        return avgBucketProcessingTimeMs;
+    }
+
+    public Double getExponentialAvgBucketProcessingTimeMs() {
+        return exponentialAvgBucketProcessingTimeMs;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
+        builder.field(Job.ID.getPreferredName(), jobId);
+        builder.field(BUCKET_COUNT.getPreferredName(), bucketCount);
+        if (minBucketProcessingTimeMs != null) {
+            builder.field(MIN_BUCKET_PROCESSING_TIME_MS.getPreferredName(), minBucketProcessingTimeMs);
+        }
+        if (maxBucketProcessingTimeMs != null) {
+            builder.field(MAX_BUCKET_PROCESSING_TIME_MS.getPreferredName(), maxBucketProcessingTimeMs);
+        }
+        if (avgBucketProcessingTimeMs != null) {
+            builder.field(AVG_BUCKET_PROCESSING_TIME_MS.getPreferredName(), avgBucketProcessingTimeMs);
+        }
+        if (exponentialAvgBucketProcessingTimeMs != null) {
+            builder.field(EXPONENTIAL_AVG_BUCKET_PROCESSING_TIME_MS.getPreferredName(), exponentialAvgBucketProcessingTimeMs);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        TimingStats that = (TimingStats) o;
+        return Objects.equals(this.jobId, that.jobId)
+            && this.bucketCount == that.bucketCount
+            && Objects.equals(this.minBucketProcessingTimeMs, that.minBucketProcessingTimeMs)
+            && Objects.equals(this.maxBucketProcessingTimeMs, that.maxBucketProcessingTimeMs)
+            && Objects.equals(this.avgBucketProcessingTimeMs, that.avgBucketProcessingTimeMs)
+            && Objects.equals(this.exponentialAvgBucketProcessingTimeMs, that.exponentialAvgBucketProcessingTimeMs);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(
+            jobId,
+            bucketCount,
+            minBucketProcessingTimeMs,
+            maxBucketProcessingTimeMs,
+            avgBucketProcessingTimeMs,
+            exponentialAvgBucketProcessingTimeMs);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/stats/JobStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/stats/JobStats.java
index df5be4aa4c5cc..078f781d4b21d 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/stats/JobStats.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/stats/JobStats.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.client.ml.job.config.JobState;
 import org.elasticsearch.client.ml.job.process.DataCounts;
 import org.elasticsearch.client.ml.job.process.ModelSizeStats;
+import org.elasticsearch.client.ml.job.process.TimingStats;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.unit.TimeValue;
@@ -42,6 +43,7 @@ public class JobStats implements ToXContentObject {
 
     private static final ParseField DATA_COUNTS = new ParseField("data_counts");
     private static final ParseField MODEL_SIZE_STATS = new ParseField("model_size_stats");
+    private static final ParseField TIMING_STATS = new ParseField("timing_stats");
     private static final ParseField FORECASTS_STATS = new ParseField("forecasts_stats");
     private static final ParseField STATE = new ParseField("state");
     private static final ParseField NODE = new ParseField("node");
@@ -58,6 +60,7 @@ public class JobStats implements ToXContentObject {
                 JobState jobState = (JobState) a[i++];
                 ModelSizeStats.Builder modelSizeStatsBuilder = (ModelSizeStats.Builder) a[i++];
                 ModelSizeStats modelSizeStats = modelSizeStatsBuilder == null ? null : modelSizeStatsBuilder.build();
+                TimingStats timingStats = (TimingStats) a[i++];
                 ForecastStats forecastStats = (ForecastStats) a[i++];
                 NodeAttributes node = (NodeAttributes) a[i++];
                 String assignmentExplanation = (String) a[i++];
@@ -66,6 +69,7 @@ public class JobStats implements ToXContentObject {
                     dataCounts,
                     jobState,
                     modelSizeStats,
+                    timingStats,
                     forecastStats,
                     node,
                     assignmentExplanation,
@@ -80,6 +84,7 @@ public class JobStats implements ToXContentObject {
             STATE,
             ObjectParser.ValueType.VALUE);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ModelSizeStats.PARSER, MODEL_SIZE_STATS);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), TimingStats.PARSER, TIMING_STATS);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ForecastStats.PARSER, FORECASTS_STATS);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), NodeAttributes.PARSER, NODE);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), ASSIGNMENT_EXPLANATION);
@@ -94,22 +99,24 @@ public class JobStats implements ToXContentObject {
     private final DataCounts dataCounts;
     private final JobState state;
     private final ModelSizeStats modelSizeStats;
+    private final TimingStats timingStats;
     private final ForecastStats forecastStats;
     private final NodeAttributes node;
     private final String assignmentExplanation;
     private final TimeValue openTime;
 
     JobStats(String jobId, DataCounts dataCounts, JobState state, @Nullable ModelSizeStats modelSizeStats,
-                    @Nullable ForecastStats forecastStats, @Nullable NodeAttributes node,
-                    @Nullable String assignmentExplanation, @Nullable TimeValue opentime) {
+             @Nullable TimingStats timingStats, @Nullable ForecastStats forecastStats, @Nullable NodeAttributes node,
+             @Nullable String assignmentExplanation, @Nullable TimeValue openTime) {
         this.jobId = Objects.requireNonNull(jobId);
         this.dataCounts = Objects.requireNonNull(dataCounts);
         this.state = Objects.requireNonNull(state);
         this.modelSizeStats = modelSizeStats;
+        this.timingStats = timingStats;
         this.forecastStats = forecastStats;
         this.node = node;
         this.assignmentExplanation = assignmentExplanation;
-        this.openTime = opentime;
+        this.openTime = openTime;
     }
 
     /**
@@ -135,6 +142,10 @@ public ModelSizeStats getModelSizeStats() {
         return modelSizeStats;
     }
 
+    public TimingStats getTimingStats() {
+        return timingStats;
+    }
+
     /**
      * An object that provides statistical information about forecasts of this job.
      * See {@link ForecastStats}
@@ -182,6 +193,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (modelSizeStats != null) {
             builder.field(MODEL_SIZE_STATS.getPreferredName(), modelSizeStats);
         }
+        if (timingStats != null) {
+            builder.field(TIMING_STATS.getPreferredName(), timingStats);
+        }
         if (forecastStats != null) {
             builder.field(FORECASTS_STATS.getPreferredName(), forecastStats);
         }
@@ -199,7 +213,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public int hashCode() {
-        return Objects.hash(jobId, dataCounts, modelSizeStats, forecastStats, state, node, assignmentExplanation, openTime);
+        return Objects.hash(jobId, dataCounts, modelSizeStats, timingStats, forecastStats, state, node, assignmentExplanation, openTime);
     }
 
     @Override
@@ -216,6 +230,7 @@ public boolean equals(Object obj) {
         return Objects.equals(jobId, other.jobId) &&
             Objects.equals(this.dataCounts, other.dataCounts) &&
             Objects.equals(this.modelSizeStats, other.modelSizeStats) &&
+            Objects.equals(this.timingStats, other.timingStats) &&
             Objects.equals(this.forecastStats, other.forecastStats) &&
             Objects.equals(this.state, other.state) &&
             Objects.equals(this.node, other.node) &&
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateTokenRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateTokenRequest.java
index 6c1b394355e1e..4f1302533d9b5 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateTokenRequest.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateTokenRequest.java
@@ -40,6 +40,7 @@ public final class CreateTokenRequest implements Validatable, ToXContentObject {
     private final String username;
     private final char[] password;
     private final String refreshToken;
+    private final char[] kerberosTicket;
 
     /**
      * General purpose constructor. This constructor is typically not useful, and one of the following factory methods should be used
@@ -48,10 +49,11 @@ public final class CreateTokenRequest implements Validatable, ToXContentObject {
      * <li>{@link #passwordGrant(String, char[])}</li>
      * <li>{@link #refreshTokenGrant(String)}</li>
      * <li>{@link #clientCredentialsGrant()}</li>
+     * <li>{@link #kerberosGrant(char[])}</li>
      * </ul>
      */
     public CreateTokenRequest(String grantType, @Nullable String scope, @Nullable String username, @Nullable char[] password,
-                              @Nullable String refreshToken) {
+                              @Nullable String refreshToken, @Nullable char[] kerberosTicket) {
         if (Strings.isNullOrEmpty(grantType)) {
             throw new IllegalArgumentException("grant_type is required");
         }
@@ -60,6 +62,7 @@ public CreateTokenRequest(String grantType, @Nullable String scope, @Nullable St
         this.password = password;
         this.scope = scope;
         this.refreshToken = refreshToken;
+        this.kerberosTicket = kerberosTicket;
     }
 
     public static CreateTokenRequest passwordGrant(String username, char[] password) {
@@ -69,18 +72,25 @@ public static CreateTokenRequest passwordGrant(String username, char[] password)
         if (password == null || password.length == 0) {
             throw new IllegalArgumentException("password is required");
         }
-        return new CreateTokenRequest("password", null, username, password, null);
+        return new CreateTokenRequest("password", null, username, password, null, null);
     }
 
     public static CreateTokenRequest refreshTokenGrant(String refreshToken) {
         if (Strings.isNullOrEmpty(refreshToken)) {
             throw new IllegalArgumentException("refresh_token is required");
         }
-        return new CreateTokenRequest("refresh_token", null, null, null, refreshToken);
+        return new CreateTokenRequest("refresh_token", null, null, null, refreshToken, null);
     }
 
     public static CreateTokenRequest clientCredentialsGrant() {
-        return new CreateTokenRequest("client_credentials", null, null, null, null);
+        return new CreateTokenRequest("client_credentials", null, null, null, null, null);
+    }
+
+    public static CreateTokenRequest kerberosGrant(char[] kerberosTicket) {
+        if (kerberosTicket == null || kerberosTicket.length == 0) {
+            throw new IllegalArgumentException("kerberos ticket is required");
+        }
+        return new CreateTokenRequest("_kerberos", null, null, null, null, kerberosTicket);
     }
 
     public String getGrantType() {
@@ -103,6 +113,10 @@ public String getRefreshToken() {
         return refreshToken;
     }
 
+    public char[] getKerberosTicket() {
+        return kerberosTicket;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject()
@@ -124,6 +138,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (refreshToken != null) {
             builder.field("refresh_token", refreshToken);
         }
+        if (kerberosTicket != null) {
+            byte[] kerberosTicketBytes = CharArrays.toUtf8Bytes(kerberosTicket);
+            try {
+                builder.field("kerberos_ticket").utf8Value(kerberosTicketBytes, 0, kerberosTicketBytes.length);
+            } finally {
+                Arrays.fill(kerberosTicketBytes, (byte) 0);
+            }
+        }
         return builder.endObject();
     }
 
@@ -140,13 +162,15 @@ public boolean equals(Object o) {
             Objects.equals(scope, that.scope) &&
             Objects.equals(username, that.username) &&
             Arrays.equals(password, that.password) &&
-            Objects.equals(refreshToken, that.refreshToken);
+            Objects.equals(refreshToken, that.refreshToken) &&
+            Arrays.equals(kerberosTicket, that.kerberosTicket);
     }
 
     @Override
     public int hashCode() {
         int result = Objects.hash(grantType, scope, username, refreshToken);
         result = 31 * result + Arrays.hashCode(password);
+        result = 31 * result + Arrays.hashCode(kerberosTicket);
         return result;
     }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateTokenResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateTokenResponse.java
index 32d298d1a9bc0..dc71d49f4b770 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateTokenResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateTokenResponse.java
@@ -41,13 +41,16 @@ public final class CreateTokenResponse {
     private final TimeValue expiresIn;
     private final String scope;
     private final String refreshToken;
+    private final String kerberosAuthenticationResponseToken;
 
-    public CreateTokenResponse(String accessToken, String type, TimeValue expiresIn, String scope, String refreshToken) {
+    public CreateTokenResponse(String accessToken, String type, TimeValue expiresIn, String scope, String refreshToken,
+                               String kerberosAuthenticationResponseToken) {
         this.accessToken = accessToken;
         this.type = type;
         this.expiresIn = expiresIn;
         this.scope = scope;
         this.refreshToken = refreshToken;
+        this.kerberosAuthenticationResponseToken = kerberosAuthenticationResponseToken;
     }
 
     public String getAccessToken() {
@@ -70,6 +73,10 @@ public String getRefreshToken() {
         return refreshToken;
     }
 
+    public String getKerberosAuthenticationResponseToken() {
+        return kerberosAuthenticationResponseToken;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {
@@ -83,17 +90,18 @@ public boolean equals(Object o) {
             Objects.equals(type, that.type) &&
             Objects.equals(expiresIn, that.expiresIn) &&
             Objects.equals(scope, that.scope) &&
-            Objects.equals(refreshToken, that.refreshToken);
+            Objects.equals(refreshToken, that.refreshToken) &&
+            Objects.equals(kerberosAuthenticationResponseToken, that.kerberosAuthenticationResponseToken);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(accessToken, type, expiresIn, scope, refreshToken);
+        return Objects.hash(accessToken, type, expiresIn, scope, refreshToken, kerberosAuthenticationResponseToken);
     }
 
     private static final ConstructingObjectParser<CreateTokenResponse, Void> PARSER = new ConstructingObjectParser<>(
-        "create_token_response", true, args -> new CreateTokenResponse(
-        (String) args[0], (String) args[1], TimeValue.timeValueSeconds((Long) args[2]), (String) args[3], (String) args[4]));
+            "create_token_response", true, args -> new CreateTokenResponse((String) args[0], (String) args[1],
+                    TimeValue.timeValueSeconds((Long) args[2]), (String) args[3], (String) args[4], (String) args[5]));
 
     static {
         PARSER.declareString(constructorArg(), new ParseField("access_token"));
@@ -101,6 +109,7 @@ public int hashCode() {
         PARSER.declareLong(constructorArg(), new ParseField("expires_in"));
         PARSER.declareStringOrNull(optionalConstructorArg(), new ParseField("scope"));
         PARSER.declareStringOrNull(optionalConstructorArg(), new ParseField("refresh_token"));
+        PARSER.declareStringOrNull(optionalConstructorArg(), new ParseField("kerberos_authentication_response_token"));
     }
 
     public static CreateTokenResponse fromXContent(XContentParser parser) throws IOException {
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetBuiltinPrivilegesRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetBuiltinPrivilegesRequest.java
new file mode 100644
index 0000000000000..543665580756c
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetBuiltinPrivilegesRequest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.apache.http.client.methods.HttpGet;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Validatable;
+
+/**
+ * Request object to retrieve the privilege that are builtin to the Elasticsearch cluster.
+ */
+public final class GetBuiltinPrivilegesRequest implements Validatable {
+
+    public static final GetBuiltinPrivilegesRequest INSTANCE = new GetBuiltinPrivilegesRequest();
+
+    private GetBuiltinPrivilegesRequest() {
+    }
+
+    public Request getRequest() {
+        return new Request(HttpGet.METHOD_NAME, "/_security/privilege/_builtin");
+    }
+
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetBuiltinPrivilegesResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetBuiltinPrivilegesResponse.java
new file mode 100644
index 0000000000000..a900ee4f976f4
--- /dev/null
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetBuiltinPrivilegesResponse.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.security;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Objects;
+import java.util.Set;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
+/**
+ * Get builtin privileges response
+ */
+public final class GetBuiltinPrivilegesResponse {
+
+    private final Set<String> clusterPrivileges;
+    private final Set<String> indexPrivileges;
+
+    public GetBuiltinPrivilegesResponse(Collection<String> cluster, Collection<String> index) {
+        this.clusterPrivileges = Set.copyOf(cluster);
+        this.indexPrivileges = Set.copyOf(index);
+    }
+
+    public Set<String> getClusterPrivileges() {
+        return clusterPrivileges;
+    }
+
+    public Set<String> getIndexPrivileges() {
+        return indexPrivileges;
+    }
+
+    public static GetBuiltinPrivilegesResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        GetBuiltinPrivilegesResponse that = (GetBuiltinPrivilegesResponse) o;
+        return Objects.equals(this.clusterPrivileges, that.clusterPrivileges)
+        && Objects.equals(this.indexPrivileges, that.indexPrivileges);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(clusterPrivileges, indexPrivileges);
+    }
+
+
+    @SuppressWarnings("unchecked")
+    private static final ConstructingObjectParser<GetBuiltinPrivilegesResponse, Void> PARSER = new ConstructingObjectParser<>(
+        "get_builtin_privileges", true,
+        args -> new GetBuiltinPrivilegesResponse((Collection<String>) args[0], (Collection<String>) args[1]));
+
+    static {
+        PARSER.declareStringArray(constructorArg(), new ParseField("cluster"));
+        PARSER.declareStringArray(constructorArg(), new ParseField("index"));
+    }
+}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/Role.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/Role.java
index a3263e7f6e920..e8e1a104d8c2c 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/Role.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/Role.java
@@ -299,10 +299,12 @@ public static class ClusterPrivilegeName {
         public static final String NONE = "none";
         public static final String ALL = "all";
         public static final String MONITOR = "monitor";
+        public static final String MONITOR_DATA_FRAME_TRANSFORMS = "monitor_data_frame_transforms";
         public static final String MONITOR_ML = "monitor_ml";
         public static final String MONITOR_WATCHER = "monitor_watcher";
         public static final String MONITOR_ROLLUP = "monitor_rollup";
         public static final String MANAGE = "manage";
+        public static final String MANAGE_DATA_FRAME_TRANSFORMS = "manage_data_frame_transforms";
         public static final String MANAGE_ML = "manage_ml";
         public static final String MANAGE_WATCHER = "manage_watcher";
         public static final String MANAGE_ROLLUP = "manage_rollup";
@@ -318,8 +320,9 @@ public static class ClusterPrivilegeName {
         public static final String READ_CCR = "read_ccr";
         public static final String MANAGE_ILM = "manage_ilm";
         public static final String READ_ILM = "read_ilm";
-        public static final String[] ALL_ARRAY = new String[] { NONE, ALL, MONITOR, MONITOR_ML, MONITOR_WATCHER, MONITOR_ROLLUP, MANAGE,
-                MANAGE_ML, MANAGE_WATCHER, MANAGE_ROLLUP, MANAGE_INDEX_TEMPLATES, MANAGE_INGEST_PIPELINES, TRANSPORT_CLIENT,
+        public static final String[] ALL_ARRAY = new String[] { NONE, ALL, MONITOR, MONITOR_DATA_FRAME_TRANSFORMS, MONITOR_ML,
+            MONITOR_WATCHER, MONITOR_ROLLUP, MANAGE, MANAGE_DATA_FRAME_TRANSFORMS,
+            MANAGE_ML, MANAGE_WATCHER, MANAGE_ROLLUP, MANAGE_INDEX_TEMPLATES, MANAGE_INGEST_PIPELINES, TRANSPORT_CLIENT,
             MANAGE_SECURITY, MANAGE_SAML, MANAGE_OIDC, MANAGE_TOKEN, MANAGE_PIPELINE, MANAGE_CCR, READ_CCR, MANAGE_ILM, READ_ILM};
     }
 
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/xpack/XPackInfoResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/xpack/XPackInfoResponse.java
index e8fc2ce89b251..62f19570c6bc8 100644
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/xpack/XPackInfoResponse.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/xpack/XPackInfoResponse.java
@@ -318,11 +318,6 @@ public String name() {
                 return name;
             }
 
-            @Nullable
-            public String description() {
-                return description;
-            }
-
             public boolean available() {
                 return available;
             }
@@ -331,6 +326,11 @@ public boolean enabled() {
                 return enabled;
             }
 
+            /**
+             * Return native code info
+             * @deprecated Use ML info api to find native code info
+             */
+            @Deprecated
             @Nullable
             public Map<String, Object> nativeCodeInfo() {
                 return nativeCodeInfo;
diff --git a/client/rest-high-level/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider b/client/rest-high-level/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
index 4204a868246a5..dde81e43867d8 100644
--- a/client/rest-high-level/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
+++ b/client/rest-high-level/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider
@@ -1 +1,4 @@
-org.elasticsearch.client.indexlifecycle.IndexLifecycleNamedXContentProvider
\ No newline at end of file
+org.elasticsearch.client.dataframe.DataFrameNamedXContentProvider
+org.elasticsearch.client.indexlifecycle.IndexLifecycleNamedXContentProvider
+org.elasticsearch.client.ml.dataframe.MlDataFrameAnalysisNamedXContentProvider
+org.elasticsearch.client.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider
\ No newline at end of file
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/AbstractHlrcWriteableXContentTestCase.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/AbstractHlrcWriteableXContentTestCase.java
new file mode 100644
index 0000000000000..432c2db99643b
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/AbstractHlrcWriteableXContentTestCase.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
+
+/**
+ * @deprecated Use {@link AbstractResponseTestCase} instead of this class.
+ */
+// TODO: Remove and change subclasses to use AbstractResponseTestCase instead
+@Deprecated
+public abstract class AbstractHlrcWriteableXContentTestCase<T extends ToXContent & Writeable, H>
+        extends AbstractSerializingTestCase<T> {
+
+    /**
+     * Generic test that creates new instance of HLRC request/response from the test instance and checks
+     * both for equality and asserts equality on the two queries.
+     */
+    public final void testHlrcFromXContent() throws IOException {
+        xContentTester(this::createParser, this::createTestInstance, getToXContentParams(),
+            p -> convertHlrcToInternal(doHlrcParseInstance(p)))
+            .numberOfTestRuns(NUMBER_OF_TEST_RUNS)
+            .supportsUnknownFields(supportsUnknownFields())
+            .shuffleFieldsExceptions(getShuffleFieldsExceptions())
+            .randomFieldsExcludeFilter(getRandomFieldsExcludeFilter())
+            .assertEqualsConsumer(this::assertEqualInstances)
+            .assertToXContentEquivalence(true)
+            .test();
+    }
+
+    /**
+     * Parses to a new HLRC instance using the provided {@link XContentParser}
+     */
+    public abstract H doHlrcParseInstance(XContentParser parser) throws IOException;
+
+    /**
+     * Converts a HLRC instance to a XPack instance
+     */
+    public abstract T convertHlrcToInternal(H instance);
+
+    //TODO this would be final ideally: why do both responses need to parse from xcontent, only one (H) should? I think that T#fromXContent
+    //are only there for testing and could go away? Then the additional testHlrcFromXContent is also no longer needed.
+    @Override
+    protected T doParseInstance(XContentParser parser) throws IOException {
+        return convertHlrcToInternal(doHlrcParseInstance(parser));
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/AbstractResponseTestCase.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/AbstractResponseTestCase.java
index 8565ca14a908a..2a91a639a5ac3 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/AbstractResponseTestCase.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/AbstractResponseTestCase.java
@@ -45,7 +45,7 @@ public final void testFromXContent() throws IOException {
         final S serverTestInstance = createServerTestInstance();
 
         final XContentType xContentType = randomFrom(XContentType.values());
-        final BytesReference bytes = toShuffledXContent(serverTestInstance, xContentType, ToXContent.EMPTY_PARAMS, randomBoolean());
+        final BytesReference bytes = toShuffledXContent(serverTestInstance, xContentType, getParams(), randomBoolean());
 
         final XContent xContent = XContentFactory.xContent(xContentType);
         final XContentParser parser = xContent.createParser(
@@ -62,4 +62,8 @@ public final void testFromXContent() throws IOException {
 
     protected abstract void assertInstances(S serverTestInstance, C clientInstance);
 
+    protected ToXContent.Params getParams() {
+        return ToXContent.EMPTY_PARAMS;
+    }
+
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java
index 762e927551b8b..2aa9457bcd897 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java
@@ -291,7 +291,6 @@ public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception
         assertMultiGetResponse(highLevelClient().mget(multiGetRequest, RequestOptions.DEFAULT), testDocs);
     }
 
-    @SuppressWarnings("unchecked")
     public void testGlobalParametersAndSingleRequest() throws Exception {
         createIndexWithMultipleShards("test");
 
@@ -326,7 +325,6 @@ public void testGlobalParametersAndSingleRequest() throws Exception {
         assertThat(blogs, everyItem(hasProperty(fieldFromSource("fieldNameXYZ"), equalTo("valueXYZ"))));
     }
 
-    @SuppressWarnings("unchecked")
     public void testGlobalParametersAndBulkProcessor() throws Exception {
         createIndexWithMultipleShards("test");
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkRequestWithGlobalParametersIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkRequestWithGlobalParametersIT.java
index 3020eb0329b5c..dc49e6f88a6e4 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkRequestWithGlobalParametersIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkRequestWithGlobalParametersIT.java
@@ -44,7 +44,6 @@
 
 public class BulkRequestWithGlobalParametersIT extends ESRestHighLevelClientTestCase {
 
-    @SuppressWarnings("unchecked")
     public void testGlobalPipelineOnBulkRequest() throws IOException {
         createFieldAddingPipleine("xyz", "fieldNameXYZ", "valueXYZ");
 
@@ -83,7 +82,6 @@ public void testPipelineOnRequestOverridesGlobalPipeline() throws IOException {
         assertThat(hits, everyItem(hasProperty(fieldFromSource("fieldXYZ"), nullValue())));
     }
 
-    @SuppressWarnings("unchecked")
     public void testMixPipelineOnRequestAndGlobal() throws IOException {
         createFieldAddingPipleine("globalId", "fieldXYZ", "valueXYZ");
         createFieldAddingPipleine("perIndexId", "someNewField", "someValue");
@@ -153,7 +151,6 @@ public void testGlobalType() throws IOException {
         assertThat(hits, everyItem(hasType("global_type")));
     }
 
-    @SuppressWarnings("unchecked")
     public void testTypeGlobalAndPerRequest() throws IOException {
         BulkRequest request = new BulkRequest(null, "global_type");
         request.add(new IndexRequest("index1", "local_type", "1")
@@ -171,7 +168,6 @@ public void testTypeGlobalAndPerRequest() throws IOException {
                 .and(hasType("global_type"))));
     }
 
-    @SuppressWarnings("unchecked")
     public void testGlobalRouting() throws IOException {
         createIndexWithMultipleShards("index");
         BulkRequest request = new BulkRequest(null);
@@ -189,7 +185,6 @@ public void testGlobalRouting() throws IOException {
         assertThat(hits, containsInAnyOrder(hasId("1"), hasId("2")));
     }
 
-    @SuppressWarnings("unchecked")
     public void testMixLocalAndGlobalRouting() throws IOException {
         BulkRequest request = new BulkRequest(null);
         request.routing("globalRouting");
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameRequestConvertersTests.java
index 7a1e5e2389316..db111904f4704 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameRequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameRequestConvertersTests.java
@@ -23,6 +23,7 @@
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
+import org.elasticsearch.client.dataframe.DataFrameNamedXContentProvider;
 import org.elasticsearch.client.core.PageParams;
 import org.elasticsearch.client.dataframe.DeleteDataFrameTransformRequest;
 import org.elasticsearch.client.dataframe.GetDataFrameTransformRequest;
@@ -43,7 +44,9 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 
+import static org.elasticsearch.client.dataframe.GetDataFrameTransformRequest.ALLOW_NO_MATCH;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
@@ -52,8 +55,11 @@ public class DataFrameRequestConvertersTests extends ESTestCase {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
-        return new NamedXContentRegistry(searchModule.getNamedXContents());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
+        List<NamedXContentRegistry.Entry> namedXContents = searchModule.getNamedXContents();
+        namedXContents.addAll(new DataFrameNamedXContentProvider().getNamedXContentParsers());
+
+        return new NamedXContentRegistry(namedXContents);
     }
 
     public void testPutDataFrameTransform() throws IOException {
@@ -110,7 +116,6 @@ public void testStopDataFrameTransform() {
         }
         StopDataFrameTransformRequest stopRequest = new StopDataFrameTransformRequest(id, waitForCompletion, timeValue);
 
-
         Request request = DataFrameRequestConverters.stopDataFrameTransform(stopRequest);
         assertEquals(HttpPost.METHOD_NAME, request.getMethod());
         assertThat(request.getEndpoint(), equalTo("/_data_frame/transforms/" + stopRequest.getId() + "/_stop"));
@@ -128,6 +133,11 @@ public void testStopDataFrameTransform() {
         } else {
             assertFalse(request.getParameters().containsKey("timeout"));
         }
+
+        assertFalse(request.getParameters().containsKey(ALLOW_NO_MATCH));
+        stopRequest.setAllowNoMatch(randomBoolean());
+        request = DataFrameRequestConverters.stopDataFrameTransform(stopRequest);
+        assertEquals(stopRequest.getAllowNoMatch(), Boolean.parseBoolean(request.getParameters().get(ALLOW_NO_MATCH)));
     }
 
     public void testPreviewDataFrameTransform() throws IOException {
@@ -153,6 +163,7 @@ public void testGetDataFrameTransformStats() {
 
         assertFalse(request.getParameters().containsKey("from"));
         assertFalse(request.getParameters().containsKey("size"));
+        assertFalse(request.getParameters().containsKey(ALLOW_NO_MATCH));
 
         getStatsRequest.setPageParams(new PageParams(0, null));
         request = DataFrameRequestConverters.getDataFrameTransformStats(getStatsRequest);
@@ -167,6 +178,10 @@ public void testGetDataFrameTransformStats() {
         getStatsRequest.setPageParams(new PageParams(0, 10));
         request = DataFrameRequestConverters.getDataFrameTransformStats(getStatsRequest);
         assertThat(request.getParameters(), allOf(hasEntry("from", "0"), hasEntry("size", "10")));
+
+        getStatsRequest.setAllowNoMatch(false);
+        request = DataFrameRequestConverters.getDataFrameTransformStats(getStatsRequest);
+        assertThat(request.getParameters(), hasEntry("allow_no_match", "false"));
     }
 
     public void testGetDataFrameTransform() {
@@ -178,6 +193,7 @@ public void testGetDataFrameTransform() {
 
         assertFalse(request.getParameters().containsKey("from"));
         assertFalse(request.getParameters().containsKey("size"));
+        assertFalse(request.getParameters().containsKey(ALLOW_NO_MATCH));
 
         getRequest.setPageParams(new PageParams(0, null));
         request = DataFrameRequestConverters.getDataFrameTransform(getRequest);
@@ -192,6 +208,10 @@ public void testGetDataFrameTransform() {
         getRequest.setPageParams(new PageParams(0, 10));
         request = DataFrameRequestConverters.getDataFrameTransform(getRequest);
         assertThat(request.getParameters(), allOf(hasEntry("from", "0"), hasEntry("size", "10")));
+
+        getRequest.setAllowNoMatch(false);
+        request = DataFrameRequestConverters.getDataFrameTransform(getRequest);
+        assertThat(request.getParameters(), hasEntry("allow_no_match", "false"));
     }
 
     public void testGetDataFrameTransform_givenMulitpleIds() {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java
index 40cd6f454cdab..d7371468b1506 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java
@@ -66,10 +66,12 @@
 import java.util.Optional;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.CoreMatchers.nullValue;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.oneOf;
@@ -140,7 +142,7 @@ private void indexData(String indexName) throws IOException {
     }
 
     @After
-    public void cleanUpTransforms() throws IOException {
+    public void cleanUpTransforms() throws Exception {
         for (String transformId : transformsToClean) {
             highLevelClient().dataFrame().stopDataFrameTransform(
                     new StopDataFrameTransformRequest(transformId, Boolean.TRUE, null), RequestOptions.DEFAULT);
@@ -152,6 +154,7 @@ public void cleanUpTransforms() throws IOException {
         }
 
         transformsToClean = new ArrayList<>();
+        waitForPendingTasks(adminClient());
     }
 
     public void testCreateDelete() throws IOException {
@@ -185,16 +188,14 @@ public void testGetTransform() throws IOException {
         DataFrameTransformConfig transform = validDataFrameTransformConfig(id, sourceIndex, "pivot-dest");
 
         DataFrameClient client = highLevelClient().dataFrame();
-        AcknowledgedResponse ack = execute(new PutDataFrameTransformRequest(transform), client::putDataFrameTransform,
-                client::putDataFrameTransformAsync);
-        assertTrue(ack.isAcknowledged());
+        putTransform(transform);
 
         GetDataFrameTransformRequest getRequest = new GetDataFrameTransformRequest(id);
         GetDataFrameTransformResponse getResponse = execute(getRequest, client::getDataFrameTransform,
                 client::getDataFrameTransformAsync);
         assertNull(getResponse.getInvalidTransforms());
         assertThat(getResponse.getTransformConfigurations(), hasSize(1));
-        assertEquals(transform, getResponse.getTransformConfigurations().get(0));
+        assertEquals(transform.getId(), getResponse.getTransformConfigurations().get(0).getId());
     }
 
     public void testGetAllAndPageTransforms() throws IOException {
@@ -204,21 +205,17 @@ public void testGetAllAndPageTransforms() throws IOException {
         DataFrameClient client = highLevelClient().dataFrame();
 
         DataFrameTransformConfig transform = validDataFrameTransformConfig("test-get-all-1", sourceIndex, "pivot-dest-1");
-        AcknowledgedResponse ack = execute(new PutDataFrameTransformRequest(transform), client::putDataFrameTransform,
-                client::putDataFrameTransformAsync);
-        assertTrue(ack.isAcknowledged());
+        putTransform(transform);
 
         transform = validDataFrameTransformConfig("test-get-all-2", sourceIndex, "pivot-dest-2");
-        ack = execute(new PutDataFrameTransformRequest(transform), client::putDataFrameTransform,
-                client::putDataFrameTransformAsync);
-        assertTrue(ack.isAcknowledged());
+        putTransform(transform);
 
         GetDataFrameTransformRequest getRequest = new GetDataFrameTransformRequest("_all");
         GetDataFrameTransformResponse getResponse = execute(getRequest, client::getDataFrameTransform,
                 client::getDataFrameTransformAsync);
         assertNull(getResponse.getInvalidTransforms());
         assertThat(getResponse.getTransformConfigurations(), hasSize(2));
-        assertEquals(transform, getResponse.getTransformConfigurations().get(1));
+        assertEquals(transform.getId(), getResponse.getTransformConfigurations().get(1).getId());
 
         getRequest.setPageParams(new PageParams(0,1));
         getResponse = execute(getRequest, client::getDataFrameTransform,
@@ -250,32 +247,38 @@ public void testStartStop() throws IOException {
         DataFrameTransformConfig transform = validDataFrameTransformConfig(id, sourceIndex, "pivot-dest");
 
         DataFrameClient client = highLevelClient().dataFrame();
-        AcknowledgedResponse ack = execute(new PutDataFrameTransformRequest(transform), client::putDataFrameTransform,
-                client::putDataFrameTransformAsync);
-        assertTrue(ack.isAcknowledged());
-        transformsToClean.add(id);
+        putTransform(transform);
 
         StartDataFrameTransformRequest startRequest = new StartDataFrameTransformRequest(id);
         StartDataFrameTransformResponse startResponse =
                 execute(startRequest, client::startDataFrameTransform, client::startDataFrameTransformAsync);
-        assertTrue(startResponse.isStarted());
+        assertTrue(startResponse.isAcknowledged());
         assertThat(startResponse.getNodeFailures(), empty());
         assertThat(startResponse.getTaskFailures(), empty());
 
         GetDataFrameTransformStatsResponse statsResponse = execute(new GetDataFrameTransformStatsRequest(id),
                 client::getDataFrameTransformStats, client::getDataFrameTransformStatsAsync);
         assertThat(statsResponse.getTransformsStateAndStats(), hasSize(1));
-        IndexerState indexerState = statsResponse.getTransformsStateAndStats().get(0).getTransformState().getIndexerState();
-        assertThat(indexerState, is(oneOf(IndexerState.STARTED, IndexerState.INDEXING)));
+        DataFrameTransformTaskState taskState = statsResponse.getTransformsStateAndStats().get(0).getTransformState().getTaskState();
+
+        // Since we are non-continuous, the transform could auto-stop between being started earlier and us gathering the statistics
+        assertThat(taskState, is(oneOf(DataFrameTransformTaskState.STARTED, DataFrameTransformTaskState.STOPPED)));
 
         StopDataFrameTransformRequest stopRequest = new StopDataFrameTransformRequest(id, Boolean.TRUE, null);
         StopDataFrameTransformResponse stopResponse =
                 execute(stopRequest, client::stopDataFrameTransform, client::stopDataFrameTransformAsync);
-        assertTrue(stopResponse.isStopped());
+        assertTrue(stopResponse.isAcknowledged());
         assertThat(stopResponse.getNodeFailures(), empty());
         assertThat(stopResponse.getTaskFailures(), empty());
+
+        // Calling stop with wait_for_completion assures that we will be in the `STOPPED` state for the transform task
+        statsResponse = execute(new GetDataFrameTransformStatsRequest(id),
+            client::getDataFrameTransformStats, client::getDataFrameTransformStatsAsync);
+        taskState = statsResponse.getTransformsStateAndStats().get(0).getTransformState().getTaskState();
+        assertThat(taskState, is(DataFrameTransformTaskState.STOPPED));
     }
 
+    @SuppressWarnings("unchecked")
     public void testPreview() throws IOException {
         String sourceIndex = "transform-source";
         createIndex(sourceIndex);
@@ -297,6 +300,12 @@ public void testPreview() throws IOException {
         Optional<Map<String, Object>> michel = docs.stream().filter(doc -> "michel".equals(doc.get("reviewer"))).findFirst();
         assertTrue(michel.isPresent());
         assertEquals(3.6d, (double) michel.get().get("avg_rating"), 0.1d);
+
+        Map<String, Object> mappings = preview.getMappings();
+        assertThat(mappings, hasKey("properties"));
+        Map<String, Object> fields = (Map<String, Object>)mappings.get("properties");
+        assertThat(fields.get("reviewer"), equalTo(Map.of("type", "keyword")));
+        assertThat(fields.get("avg_rating"), equalTo(Map.of("type", "double")));
     }
 
     private DataFrameTransformConfig validDataFrameTransformConfig(String id, String source, String destination) {
@@ -306,7 +315,7 @@ private DataFrameTransformConfig validDataFrameTransformConfig(String id, String
         aggBuilder.addAggregator(AggregationBuilders.avg("avg_rating").field("stars"));
         PivotConfig pivotConfig = PivotConfig.builder().setGroups(groupConfig).setAggregations(aggBuilder).build();
 
-        DestConfig destConfig = (destination != null) ? new DestConfig(destination) : null;
+        DestConfig destConfig = (destination != null) ? DestConfig.builder().setIndex(destination).build() : null;
 
         return DataFrameTransformConfig.builder()
             .setId(id)
@@ -317,6 +326,7 @@ private DataFrameTransformConfig validDataFrameTransformConfig(String id, String
             .build();
     }
 
+    // TODO add tests to cover continuous situations
     public void testGetStats() throws Exception {
         String sourceIndex = "transform-source";
         createIndex(sourceIndex);
@@ -332,16 +342,13 @@ public void testGetStats() throws Exception {
         DataFrameTransformConfig transform = DataFrameTransformConfig.builder()
             .setId(id)
             .setSource(SourceConfig.builder().setIndex(sourceIndex).setQuery(new MatchAllQueryBuilder()).build())
-            .setDest(new DestConfig("pivot-dest"))
+            .setDest(DestConfig.builder().setIndex("pivot-dest").build())
             .setPivotConfig(pivotConfig)
             .setDescription("transform for testing stats")
             .build();
 
         DataFrameClient client = highLevelClient().dataFrame();
-        AcknowledgedResponse ack = execute(new PutDataFrameTransformRequest(transform), client::putDataFrameTransform,
-                client::putDataFrameTransformAsync);
-        assertTrue(ack.isAcknowledged());
-        transformsToClean.add(id);
+        putTransform(transform);
 
         GetDataFrameTransformStatsResponse statsResponse = execute(new GetDataFrameTransformStatsRequest(id),
                 client::getDataFrameTransformStats, client::getDataFrameTransformStatsAsync);
@@ -358,20 +365,30 @@ public void testGetStats() throws Exception {
         StartDataFrameTransformResponse startTransformResponse = execute(new StartDataFrameTransformRequest(id),
             client::startDataFrameTransform,
             client::startDataFrameTransformAsync);
-        assertThat(startTransformResponse.isStarted(), is(true));
+        assertThat(startTransformResponse.isAcknowledged(), is(true));
         assertBusy(() -> {
             GetDataFrameTransformStatsResponse response = execute(new GetDataFrameTransformStatsRequest(id),
                     client::getDataFrameTransformStats, client::getDataFrameTransformStatsAsync);
             DataFrameTransformStateAndStats stateAndStats = response.getTransformsStateAndStats().get(0);
-            assertEquals(IndexerState.STARTED, stateAndStats.getTransformState().getIndexerState());
-            assertEquals(DataFrameTransformTaskState.STARTED, stateAndStats.getTransformState().getTaskState());
-            assertEquals(null, stateAndStats.getTransformState().getReason());
             assertNotEquals(zeroIndexerStats, stateAndStats.getTransformStats());
             assertNotNull(stateAndStats.getTransformState().getProgress());
+            assertThat(stateAndStats.getTransformState().getTaskState(),
+                is(oneOf(DataFrameTransformTaskState.STARTED, DataFrameTransformTaskState.STOPPED)));
+            assertThat(stateAndStats.getTransformState().getIndexerState(),
+                is(oneOf(IndexerState.STARTED, IndexerState.STOPPED)));
             assertThat(stateAndStats.getTransformState().getProgress().getPercentComplete(), equalTo(100.0));
             assertThat(stateAndStats.getTransformState().getProgress().getTotalDocs(), greaterThan(0L));
             assertThat(stateAndStats.getTransformState().getProgress().getRemainingDocs(), equalTo(0L));
+            assertThat(stateAndStats.getTransformState().getReason(), is(nullValue()));
         });
     }
+
+    void putTransform(DataFrameTransformConfig config) throws IOException {
+        DataFrameClient client = highLevelClient().dataFrame();
+        AcknowledgedResponse ack = execute(new PutDataFrameTransformRequest(config), client::putDataFrameTransform,
+            client::putDataFrameTransformAsync);
+        assertTrue(ack.isAcknowledged());
+        transformsToClean.add(config.getId());
+    }
 }
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java
index f758156c222a8..afbdb9fe60058 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java
@@ -131,7 +131,7 @@ protected interface AsyncMethodNoRequest<Response> {
 
     private static class HighLevelClient extends RestHighLevelClient {
         private HighLevelClient(RestClient restClient) {
-            super(restClient, (client) -> {}, new SearchModule(Settings.EMPTY, false, Collections.emptyList()).getNamedXContents());
+            super(restClient, (client) -> {}, new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents());
         }
     }
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
index d9adf61782b3d..59d76142566e6 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java
@@ -28,8 +28,6 @@
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
@@ -58,6 +56,8 @@
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.core.ShardsAcknowledgedResponse;
+import org.elasticsearch.client.indices.AnalyzeRequest;
+import org.elasticsearch.client.indices.AnalyzeResponse;
 import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.client.indices.CreateIndexResponse;
 import org.elasticsearch.client.indices.FreezeIndexRequest;
@@ -73,6 +73,8 @@
 import org.elasticsearch.client.indices.IndexTemplatesExistRequest;
 import org.elasticsearch.client.indices.PutIndexTemplateRequest;
 import org.elasticsearch.client.indices.PutMappingRequest;
+import org.elasticsearch.client.indices.ReloadAnalyzersRequest;
+import org.elasticsearch.client.indices.ReloadAnalyzersResponse;
 import org.elasticsearch.client.indices.UnfreezeIndexRequest;
 import org.elasticsearch.client.indices.rollover.RolloverRequest;
 import org.elasticsearch.client.indices.rollover.RolloverResponse;
@@ -1852,12 +1854,12 @@ public void testAnalyze() throws Exception {
 
         RestHighLevelClient client = highLevelClient();
 
-        AnalyzeRequest noindexRequest = new AnalyzeRequest().text("One two three").analyzer("english");
+        AnalyzeRequest noindexRequest = AnalyzeRequest.withGlobalAnalyzer("english", "One two three");
         AnalyzeResponse noindexResponse = execute(noindexRequest, client.indices()::analyze, client.indices()::analyzeAsync);
 
         assertThat(noindexResponse.getTokens(), hasSize(3));
 
-        AnalyzeRequest detailsRequest = new AnalyzeRequest().text("One two three").analyzer("english").explain(true);
+        AnalyzeRequest detailsRequest = AnalyzeRequest.withGlobalAnalyzer("english", "One two three").explain(true);
         AnalyzeResponse detailsResponse = execute(detailsRequest, client.indices()::analyze, client.indices()::analyzeAsync);
 
         assertNotNull(detailsResponse.detail());
@@ -1877,4 +1879,14 @@ public void testFreezeAndUnfreeze() throws IOException {
         assertTrue(unfreeze.isShardsAcknowledged());
         assertTrue(unfreeze.isAcknowledged());
     }
+
+    public void testReloadAnalyzer() throws IOException {
+        createIndex("test", Settings.EMPTY);
+        RestHighLevelClient client = highLevelClient();
+
+        ReloadAnalyzersResponse reloadResponse = execute(new ReloadAnalyzersRequest("test"), client.indices()::reloadAnalyzers,
+            client.indices()::reloadAnalyzersAsync);
+        assertNotNull(reloadResponse.shards());
+        assertTrue(reloadResponse.getReloadedDetails().containsKey("test"));
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java
index f7d5ac51a73ac..d0f4177635797 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
@@ -45,6 +44,7 @@
 import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.client.indices.AnalyzeRequest;
 import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.client.indices.GetFieldMappingsRequest;
 import org.elasticsearch.client.indices.GetIndexRequest;
@@ -54,6 +54,7 @@
 import org.elasticsearch.client.indices.PutIndexTemplateRequest;
 import org.elasticsearch.client.indices.PutMappingRequest;
 import org.elasticsearch.client.indices.RandomCreateIndexGenerator;
+import org.elasticsearch.client.indices.ReloadAnalyzersRequest;
 import org.elasticsearch.client.indices.rollover.RolloverRequest;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Strings;
@@ -86,18 +87,14 @@
 public class IndicesRequestConvertersTests extends ESTestCase {
 
     public void testAnalyzeRequest() throws Exception {
-        AnalyzeRequest indexAnalyzeRequest = new AnalyzeRequest()
-            .text("Here is some text")
-            .index("test_index")
-            .analyzer("test_analyzer");
+        AnalyzeRequest indexAnalyzeRequest
+            = AnalyzeRequest.withIndexAnalyzer("test_index", "test_analyzer", "Here is some text");
 
         Request request = IndicesRequestConverters.analyze(indexAnalyzeRequest);
         assertThat(request.getEndpoint(), equalTo("/test_index/_analyze"));
         RequestConvertersTests.assertToXContentBody(indexAnalyzeRequest, request.getEntity());
 
-        AnalyzeRequest analyzeRequest = new AnalyzeRequest()
-            .text("more text")
-            .analyzer("test_analyzer");
+        AnalyzeRequest analyzeRequest = AnalyzeRequest.withGlobalAnalyzer("test_analyzer", "more text");
         assertThat(IndicesRequestConverters.analyze(analyzeRequest).getEndpoint(), equalTo("/_analyze"));
     }
 
@@ -1219,4 +1216,21 @@ public void testDeleteTemplateRequest() {
         Assert.assertThat(request.getParameters(), equalTo(expectedParams));
         Assert.assertThat(request.getEntity(), nullValue());
     }
+
+    public void testReloadAnalyzers() {
+        String[] indices = RequestConvertersTests.randomIndicesNames(1, 5);
+        StringJoiner endpoint = new StringJoiner("/", "/", "");
+        if (indices != null && indices.length > 0) {
+            endpoint.add(String.join(",", indices));
+        }
+        ReloadAnalyzersRequest reloadRequest = new ReloadAnalyzersRequest(indices);
+        Map<String, String> expectedParams = new HashMap<>();
+        RequestConvertersTests.setRandomIndicesOptions(reloadRequest::setIndicesOptions, reloadRequest::indicesOptions,
+                expectedParams);
+        Request request = IndicesRequestConverters.reloadAnalyzers(reloadRequest);
+        Assert.assertThat(request.getMethod(), equalTo(HttpPost.METHOD_NAME));
+        Assert.assertThat(request.getEndpoint(), equalTo(endpoint + "/_reload_search_analyzers"));
+        Assert.assertThat(request.getParameters(), equalTo(expectedParams));
+        Assert.assertThat(request.getEntity(), nullValue());
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java
index fd867a12204d0..aff25b4aa2d3b 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java
@@ -28,12 +28,14 @@
 import org.elasticsearch.client.ml.DeleteCalendarEventRequest;
 import org.elasticsearch.client.ml.DeleteCalendarJobRequest;
 import org.elasticsearch.client.ml.DeleteCalendarRequest;
+import org.elasticsearch.client.ml.DeleteDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.DeleteDatafeedRequest;
 import org.elasticsearch.client.ml.DeleteExpiredDataRequest;
 import org.elasticsearch.client.ml.DeleteFilterRequest;
 import org.elasticsearch.client.ml.DeleteForecastRequest;
 import org.elasticsearch.client.ml.DeleteJobRequest;
 import org.elasticsearch.client.ml.DeleteModelSnapshotRequest;
+import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
 import org.elasticsearch.client.ml.FindFileStructureRequest;
 import org.elasticsearch.client.ml.FindFileStructureRequestTests;
 import org.elasticsearch.client.ml.FlushJobRequest;
@@ -42,6 +44,8 @@
 import org.elasticsearch.client.ml.GetCalendarEventsRequest;
 import org.elasticsearch.client.ml.GetCalendarsRequest;
 import org.elasticsearch.client.ml.GetCategoriesRequest;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsRequest;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsStatsRequest;
 import org.elasticsearch.client.ml.GetDatafeedRequest;
 import org.elasticsearch.client.ml.GetDatafeedStatsRequest;
 import org.elasticsearch.client.ml.GetFiltersRequest;
@@ -58,13 +62,16 @@
 import org.elasticsearch.client.ml.PreviewDatafeedRequest;
 import org.elasticsearch.client.ml.PutCalendarJobRequest;
 import org.elasticsearch.client.ml.PutCalendarRequest;
+import org.elasticsearch.client.ml.PutDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.PutDatafeedRequest;
 import org.elasticsearch.client.ml.PutFilterRequest;
 import org.elasticsearch.client.ml.PutJobRequest;
 import org.elasticsearch.client.ml.RevertModelSnapshotRequest;
 import org.elasticsearch.client.ml.SetUpgradeModeRequest;
+import org.elasticsearch.client.ml.StartDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.StartDatafeedRequest;
 import org.elasticsearch.client.ml.StartDatafeedRequestTests;
+import org.elasticsearch.client.ml.StopDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.StopDatafeedRequest;
 import org.elasticsearch.client.ml.UpdateFilterRequest;
 import org.elasticsearch.client.ml.UpdateJobRequest;
@@ -75,6 +82,12 @@
 import org.elasticsearch.client.ml.calendars.ScheduledEventTests;
 import org.elasticsearch.client.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.client.ml.datafeed.DatafeedConfigTests;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.client.ml.dataframe.MlDataFrameAnalysisNamedXContentProvider;
+import org.elasticsearch.client.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.BinarySoftClassification;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.PrecisionMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.RecallMetric;
 import org.elasticsearch.client.ml.filestructurefinder.FileStructure;
 import org.elasticsearch.client.ml.job.config.AnalysisConfig;
 import org.elasticsearch.client.ml.job.config.Detector;
@@ -84,23 +97,30 @@
 import org.elasticsearch.client.ml.job.config.MlFilter;
 import org.elasticsearch.client.ml.job.config.MlFilterTests;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfigTests.randomDataFrameAnalyticsConfig;
+import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsNull.nullValue;
 
@@ -154,7 +174,6 @@ public void testGetJobStats() {
         assertEquals(Boolean.toString(true), request.getParameters().get("allow_no_jobs"));
     }
 
-
     public void testOpenJob() throws Exception {
         String jobId = "some-job-id";
         OpenJobRequest openJobRequest = new OpenJobRequest(jobId);
@@ -669,6 +688,113 @@ public void testDeleteCalendarEvent() {
         assertEquals("/_ml/calendars/" + calendarId + "/events/" + eventId, request.getEndpoint());
     }
 
+    public void testPutDataFrameAnalytics() throws IOException {
+        PutDataFrameAnalyticsRequest putRequest = new PutDataFrameAnalyticsRequest(randomDataFrameAnalyticsConfig());
+        Request request = MLRequestConverters.putDataFrameAnalytics(putRequest);
+        assertEquals(HttpPut.METHOD_NAME, request.getMethod());
+        assertEquals("/_ml/data_frame/analytics/" + putRequest.getConfig().getId(), request.getEndpoint());
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, request.getEntity().getContent())) {
+            DataFrameAnalyticsConfig parsedConfig = DataFrameAnalyticsConfig.fromXContent(parser);
+            assertThat(parsedConfig, equalTo(putRequest.getConfig()));
+        }
+    }
+
+    public void testGetDataFrameAnalytics() {
+        String configId1 = randomAlphaOfLength(10);
+        String configId2 = randomAlphaOfLength(10);
+        String configId3 = randomAlphaOfLength(10);
+        GetDataFrameAnalyticsRequest getRequest = new GetDataFrameAnalyticsRequest(configId1, configId2, configId3)
+            .setAllowNoMatch(false)
+            .setPageParams(new PageParams(100, 300));
+
+        Request request = MLRequestConverters.getDataFrameAnalytics(getRequest);
+        assertEquals(HttpGet.METHOD_NAME, request.getMethod());
+        assertEquals("/_ml/data_frame/analytics/" + configId1 + "," + configId2 + "," + configId3, request.getEndpoint());
+        assertThat(request.getParameters(), allOf(hasEntry("from", "100"), hasEntry("size", "300"), hasEntry("allow_no_match", "false")));
+        assertNull(request.getEntity());
+    }
+
+    public void testGetDataFrameAnalyticsStats() {
+        String configId1 = randomAlphaOfLength(10);
+        String configId2 = randomAlphaOfLength(10);
+        String configId3 = randomAlphaOfLength(10);
+        GetDataFrameAnalyticsStatsRequest getStatsRequest = new GetDataFrameAnalyticsStatsRequest(configId1, configId2, configId3)
+            .setAllowNoMatch(false)
+            .setPageParams(new PageParams(100, 300));
+
+        Request request = MLRequestConverters.getDataFrameAnalyticsStats(getStatsRequest);
+        assertEquals(HttpGet.METHOD_NAME, request.getMethod());
+        assertEquals("/_ml/data_frame/analytics/" + configId1 + "," + configId2 + "," + configId3 + "/_stats", request.getEndpoint());
+        assertThat(request.getParameters(), allOf(hasEntry("from", "100"), hasEntry("size", "300"), hasEntry("allow_no_match", "false")));
+        assertNull(request.getEntity());
+    }
+
+    public void testStartDataFrameAnalytics() {
+        StartDataFrameAnalyticsRequest startRequest = new StartDataFrameAnalyticsRequest(randomAlphaOfLength(10));
+        Request request = MLRequestConverters.startDataFrameAnalytics(startRequest);
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
+        assertEquals("/_ml/data_frame/analytics/" + startRequest.getId() + "/_start", request.getEndpoint());
+        assertNull(request.getEntity());
+    }
+
+    public void testStartDataFrameAnalytics_WithTimeout() {
+        StartDataFrameAnalyticsRequest startRequest = new StartDataFrameAnalyticsRequest(randomAlphaOfLength(10))
+            .setTimeout(TimeValue.timeValueMinutes(1));
+        Request request = MLRequestConverters.startDataFrameAnalytics(startRequest);
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
+        assertEquals("/_ml/data_frame/analytics/" + startRequest.getId() + "/_start", request.getEndpoint());
+        assertThat(request.getParameters(), hasEntry("timeout", "1m"));
+        assertNull(request.getEntity());
+    }
+
+    public void testStopDataFrameAnalytics() {
+        StopDataFrameAnalyticsRequest stopRequest = new StopDataFrameAnalyticsRequest(randomAlphaOfLength(10));
+        Request request = MLRequestConverters.stopDataFrameAnalytics(stopRequest);
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
+        assertEquals("/_ml/data_frame/analytics/" + stopRequest.getId() + "/_stop", request.getEndpoint());
+        assertNull(request.getEntity());
+    }
+
+    public void testStopDataFrameAnalytics_WithParams() {
+        StopDataFrameAnalyticsRequest stopRequest = new StopDataFrameAnalyticsRequest(randomAlphaOfLength(10))
+            .setTimeout(TimeValue.timeValueMinutes(1))
+            .setAllowNoMatch(false)
+            .setForce(true);
+        Request request = MLRequestConverters.stopDataFrameAnalytics(stopRequest);
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
+        assertEquals("/_ml/data_frame/analytics/" + stopRequest.getId() + "/_stop", request.getEndpoint());
+        assertThat(request.getParameters(), allOf(
+            hasEntry("timeout", "1m"),
+            hasEntry("allow_no_match", "false"),
+            hasEntry("force", "true")));
+        assertNull(request.getEntity());
+    }
+
+    public void testDeleteDataFrameAnalytics() {
+        DeleteDataFrameAnalyticsRequest deleteRequest = new DeleteDataFrameAnalyticsRequest(randomAlphaOfLength(10));
+        Request request = MLRequestConverters.deleteDataFrameAnalytics(deleteRequest);
+        assertEquals(HttpDelete.METHOD_NAME, request.getMethod());
+        assertEquals("/_ml/data_frame/analytics/" + deleteRequest.getId(), request.getEndpoint());
+        assertNull(request.getEntity());
+    }
+
+    public void testEvaluateDataFrame() throws IOException {
+        EvaluateDataFrameRequest evaluateRequest =
+            new EvaluateDataFrameRequest(
+                Arrays.asList(generateRandomStringArray(1, 10, false, false)),
+                new BinarySoftClassification(
+                    randomAlphaOfLengthBetween(1, 10),
+                    randomAlphaOfLengthBetween(1, 10),
+                    PrecisionMetric.at(0.5), RecallMetric.at(0.6, 0.7)));
+        Request request = MLRequestConverters.evaluateDataFrame(evaluateRequest);
+        assertEquals(HttpPost.METHOD_NAME, request.getMethod());
+        assertEquals("/_ml/data_frame/_evaluate", request.getEndpoint());
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, request.getEntity().getContent())) {
+            EvaluateDataFrameRequest parsedRequest = EvaluateDataFrameRequest.fromXContent(parser);
+            assertThat(parsedRequest, equalTo(evaluateRequest));
+        }
+    }
+
     public void testPutFilter() throws IOException {
         MlFilter filter = MlFilterTests.createRandomBuilder("foo").build();
         PutFilterRequest putFilterRequest = new PutFilterRequest(filter);
@@ -835,6 +961,15 @@ public void testSetUpgradeMode() {
         assertThat(request.getParameters().get(SetUpgradeModeRequest.TIMEOUT.getPreferredName()), is("1h"));
     }
 
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
+        namedXContent.addAll(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents());
+        namedXContent.addAll(new MlDataFrameAnalysisNamedXContentProvider().getNamedXContentParsers());
+        namedXContent.addAll(new MlEvaluationNamedXContentProvider().getNamedXContentParsers());
+        return new NamedXContentRegistry(namedXContent);
+    }
+
     private static Job createValidJob(String jobId) {
         AnalysisConfig.Builder analysisConfig = AnalysisConfig.builder(Collections.singletonList(
                 Detector.builder().setFunction("count").build()));
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
index 25e610d67814f..93417cab1b393 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
@@ -29,11 +29,13 @@
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.client.core.PageParams;
 import org.elasticsearch.client.indices.CreateIndexRequest;
+import org.elasticsearch.client.indices.GetIndexRequest;
 import org.elasticsearch.client.ml.CloseJobRequest;
 import org.elasticsearch.client.ml.CloseJobResponse;
 import org.elasticsearch.client.ml.DeleteCalendarEventRequest;
 import org.elasticsearch.client.ml.DeleteCalendarJobRequest;
 import org.elasticsearch.client.ml.DeleteCalendarRequest;
+import org.elasticsearch.client.ml.DeleteDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.DeleteDatafeedRequest;
 import org.elasticsearch.client.ml.DeleteExpiredDataRequest;
 import org.elasticsearch.client.ml.DeleteExpiredDataResponse;
@@ -42,6 +44,8 @@
 import org.elasticsearch.client.ml.DeleteJobRequest;
 import org.elasticsearch.client.ml.DeleteJobResponse;
 import org.elasticsearch.client.ml.DeleteModelSnapshotRequest;
+import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
+import org.elasticsearch.client.ml.EvaluateDataFrameResponse;
 import org.elasticsearch.client.ml.FindFileStructureRequest;
 import org.elasticsearch.client.ml.FindFileStructureResponse;
 import org.elasticsearch.client.ml.FlushJobRequest;
@@ -52,6 +56,10 @@
 import org.elasticsearch.client.ml.GetCalendarEventsResponse;
 import org.elasticsearch.client.ml.GetCalendarsRequest;
 import org.elasticsearch.client.ml.GetCalendarsResponse;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsRequest;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsResponse;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsStatsRequest;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsStatsResponse;
 import org.elasticsearch.client.ml.GetDatafeedRequest;
 import org.elasticsearch.client.ml.GetDatafeedResponse;
 import org.elasticsearch.client.ml.GetDatafeedStatsRequest;
@@ -77,6 +85,8 @@
 import org.elasticsearch.client.ml.PutCalendarJobRequest;
 import org.elasticsearch.client.ml.PutCalendarRequest;
 import org.elasticsearch.client.ml.PutCalendarResponse;
+import org.elasticsearch.client.ml.PutDataFrameAnalyticsRequest;
+import org.elasticsearch.client.ml.PutDataFrameAnalyticsResponse;
 import org.elasticsearch.client.ml.PutDatafeedRequest;
 import org.elasticsearch.client.ml.PutDatafeedResponse;
 import org.elasticsearch.client.ml.PutFilterRequest;
@@ -86,8 +96,11 @@
 import org.elasticsearch.client.ml.RevertModelSnapshotRequest;
 import org.elasticsearch.client.ml.RevertModelSnapshotResponse;
 import org.elasticsearch.client.ml.SetUpgradeModeRequest;
+import org.elasticsearch.client.ml.StartDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.StartDatafeedRequest;
 import org.elasticsearch.client.ml.StartDatafeedResponse;
+import org.elasticsearch.client.ml.StopDataFrameAnalyticsRequest;
+import org.elasticsearch.client.ml.StopDataFrameAnalyticsResponse;
 import org.elasticsearch.client.ml.StopDatafeedRequest;
 import org.elasticsearch.client.ml.StopDatafeedResponse;
 import org.elasticsearch.client.ml.UpdateDatafeedRequest;
@@ -103,6 +116,18 @@
 import org.elasticsearch.client.ml.datafeed.DatafeedState;
 import org.elasticsearch.client.ml.datafeed.DatafeedStats;
 import org.elasticsearch.client.ml.datafeed.DatafeedUpdate;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsDest;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsSource;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsStats;
+import org.elasticsearch.client.ml.dataframe.OutlierDetection;
+import org.elasticsearch.client.ml.dataframe.QueryConfig;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.AucRocMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.BinarySoftClassification;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.PrecisionMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.RecallMetric;
 import org.elasticsearch.client.ml.filestructurefinder.FileStructure;
 import org.elasticsearch.client.ml.job.config.AnalysisConfig;
 import org.elasticsearch.client.ml.job.config.DataDescription;
@@ -113,9 +138,12 @@
 import org.elasticsearch.client.ml.job.config.MlFilter;
 import org.elasticsearch.client.ml.job.process.ModelSnapshot;
 import org.elasticsearch.client.ml.job.stats.JobStats;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.junit.After;
@@ -136,6 +164,7 @@
 import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.CoreMatchers.hasItems;
 import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -528,18 +557,7 @@ public void testStartDatafeed() throws Exception {
         String indexName = "start_data_1";
 
         // Set up the index and docs
-        CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
-        createIndexRequest.mapping(XContentFactory.jsonBuilder().startObject()
-            .startObject("properties")
-                .startObject("timestamp")
-                    .field("type", "date")
-                .endObject()
-                .startObject("total")
-                    .field("type", "long")
-                .endObject()
-            .endObject()
-        .endObject());
-        highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT);
+        createIndex(indexName, defaultMappingForTest());
         BulkRequest bulk = new BulkRequest();
         bulk.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
         long now = (System.currentTimeMillis()/1000)*1000;
@@ -611,18 +629,7 @@ public void testStopDatafeed() throws Exception {
         String indexName = "stop_data_1";
 
         // Set up the index
-        CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
-        createIndexRequest.mapping(XContentFactory.jsonBuilder().startObject()
-            .startObject("properties")
-                .startObject("timestamp")
-                    .field("type", "date")
-                .endObject()
-                .startObject("total")
-                    .field("type", "long")
-                .endObject()
-            .endObject()
-        .endObject());
-        highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT);
+        createIndex(indexName, defaultMappingForTest());
 
         // create the job and the datafeed
         Job job1 = buildJob(jobId1);
@@ -684,18 +691,7 @@ public void testGetDatafeedStats() throws Exception {
         String indexName = "datafeed_stats_data_1";
 
         // Set up the index
-        CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
-        createIndexRequest.mapping(XContentFactory.jsonBuilder().startObject()
-            .startObject("properties")
-                .startObject("timestamp")
-                    .field("type", "date")
-                .endObject()
-                .startObject("total")
-                    .field("type", "long")
-                .endObject()
-            .endObject()
-        .endObject());
-        highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT);
+        createIndex(indexName, defaultMappingForTest());
 
         // create the job and the datafeed
         Job job1 = buildJob(jobId1);
@@ -762,18 +758,7 @@ public void testPreviewDatafeed() throws Exception {
         String indexName = "preview_data_1";
 
         // Set up the index and docs
-        CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
-        createIndexRequest.mapping(XContentFactory.jsonBuilder().startObject()
-            .startObject("properties")
-                .startObject("timestamp")
-                    .field("type", "date")
-                .endObject()
-                .startObject("total")
-                    .field("type", "long")
-                .endObject()
-            .endObject()
-        .endObject());
-        highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT);
+        createIndex(indexName, defaultMappingForTest());
         BulkRequest bulk = new BulkRequest();
         bulk.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
         long now = (System.currentTimeMillis()/1000)*1000;
@@ -826,21 +811,9 @@ public void testDeleteExpiredDataGivenNothingToDelete() throws Exception {
     }
 
     private  String createExpiredData(String jobId) throws Exception {
-        String indexId = jobId + "-data";
+        String indexName = jobId + "-data";
         // Set up the index and docs
-        CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexId);
-        createIndexRequest.mapping(XContentFactory.jsonBuilder().startObject()
-            .startObject("properties")
-                .startObject("timestamp")
-                    .field("type", "date")
-                    .field("format", "epoch_millis")
-                .endObject()
-                .startObject("total")
-                    .field("type", "long")
-                .endObject()
-            .endObject()
-        .endObject());
-        highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT);
+        createIndex(indexName, defaultMappingForTest());
         BulkRequest bulk = new BulkRequest();
         bulk.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
 
@@ -853,7 +826,7 @@ private  String createExpiredData(String jobId) throws Exception {
             long timestamp = nowMillis - TimeValue.timeValueHours(totalBuckets - bucket).getMillis();
             int bucketRate = bucket == anomalousBucket ? anomalousRate : normalRate;
             for (int point = 0; point < bucketRate; point++) {
-                IndexRequest indexRequest = new IndexRequest(indexId);
+                IndexRequest indexRequest = new IndexRequest(indexName);
                 indexRequest.source(XContentType.JSON, "timestamp", timestamp, "total", randomInt(1000));
                 bulk.add(indexRequest);
             }
@@ -872,7 +845,7 @@ private  String createExpiredData(String jobId) throws Exception {
         Job job = buildJobForExpiredDataTests(jobId);
         putJob(job);
         openJob(job);
-        String datafeedId = createAndPutDatafeed(jobId, indexId);
+        String datafeedId = createAndPutDatafeed(jobId, indexName);
 
         startDatafeed(datafeedId, String.valueOf(0), String.valueOf(nowMillis - TimeValue.timeValueHours(24).getMillis()));
 
@@ -1230,6 +1203,420 @@ public void testDeleteCalendarEvent() throws IOException {
         assertThat(remainingIds, not(hasItem(deletedEvent)));
     }
 
+    public void testPutDataFrameAnalyticsConfig() throws Exception {
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+        String configId = "put-test-config";
+        DataFrameAnalyticsConfig config = DataFrameAnalyticsConfig.builder(configId)
+            .setSource(DataFrameAnalyticsSource.builder()
+                .setIndex("put-test-source-index")
+                .build())
+            .setDest(DataFrameAnalyticsDest.builder()
+                .setIndex("put-test-dest-index")
+                .build())
+            .setAnalysis(OutlierDetection.createDefault())
+            .build();
+
+        createIndex("put-test-source-index", defaultMappingForTest());
+
+        PutDataFrameAnalyticsResponse putDataFrameAnalyticsResponse = execute(
+            new PutDataFrameAnalyticsRequest(config),
+            machineLearningClient::putDataFrameAnalytics, machineLearningClient::putDataFrameAnalyticsAsync);
+        DataFrameAnalyticsConfig createdConfig = putDataFrameAnalyticsResponse.getConfig();
+        assertThat(createdConfig.getId(), equalTo(config.getId()));
+        assertThat(createdConfig.getSource().getIndex(), equalTo(config.getSource().getIndex()));
+        assertThat(createdConfig.getSource().getQueryConfig(), equalTo(new QueryConfig(new MatchAllQueryBuilder())));  // default value
+        assertThat(createdConfig.getDest().getIndex(), equalTo(config.getDest().getIndex()));
+        assertThat(createdConfig.getDest().getResultsField(), equalTo("ml"));  // default value
+        assertThat(createdConfig.getAnalysis(), equalTo(config.getAnalysis()));
+        assertThat(createdConfig.getAnalyzedFields(), equalTo(config.getAnalyzedFields()));
+        assertThat(createdConfig.getModelMemoryLimit(), equalTo(ByteSizeValue.parseBytesSizeValue("1gb", "")));  // default value
+    }
+
+    public void testGetDataFrameAnalyticsConfig_SingleConfig() throws Exception {
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+        String configId = "get-test-config";
+        DataFrameAnalyticsConfig config = DataFrameAnalyticsConfig.builder(configId)
+            .setSource(DataFrameAnalyticsSource.builder()
+                .setIndex("get-test-source-index")
+                .build())
+            .setDest(DataFrameAnalyticsDest.builder()
+                .setIndex("get-test-dest-index")
+                .build())
+            .setAnalysis(OutlierDetection.createDefault())
+            .build();
+
+        createIndex("get-test-source-index", defaultMappingForTest());
+
+        PutDataFrameAnalyticsResponse putDataFrameAnalyticsResponse = execute(
+            new PutDataFrameAnalyticsRequest(config),
+            machineLearningClient::putDataFrameAnalytics, machineLearningClient::putDataFrameAnalyticsAsync);
+        DataFrameAnalyticsConfig createdConfig = putDataFrameAnalyticsResponse.getConfig();
+
+        GetDataFrameAnalyticsResponse getDataFrameAnalyticsResponse = execute(
+            new GetDataFrameAnalyticsRequest(configId),
+            machineLearningClient::getDataFrameAnalytics, machineLearningClient::getDataFrameAnalyticsAsync);
+        assertThat(getDataFrameAnalyticsResponse.getAnalytics(), hasSize(1));
+        assertThat(getDataFrameAnalyticsResponse.getAnalytics(), contains(createdConfig));
+    }
+
+    public void testGetDataFrameAnalyticsConfig_MultipleConfigs() throws Exception {
+        createIndex("get-test-source-index", defaultMappingForTest());
+
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+        String configIdPrefix = "get-test-config-";
+        int numberOfConfigs = 10;
+        List<DataFrameAnalyticsConfig> createdConfigs = new ArrayList<>();
+        for (int i = 0; i < numberOfConfigs; ++i) {
+            String configId = configIdPrefix + i;
+            DataFrameAnalyticsConfig config = DataFrameAnalyticsConfig.builder(configId)
+                .setSource(DataFrameAnalyticsSource.builder()
+                    .setIndex("get-test-source-index")
+                    .build())
+                .setDest(DataFrameAnalyticsDest.builder()
+                    .setIndex("get-test-dest-index")
+                    .build())
+                .setAnalysis(OutlierDetection.createDefault())
+                .build();
+
+            PutDataFrameAnalyticsResponse putDataFrameAnalyticsResponse = execute(
+                new PutDataFrameAnalyticsRequest(config),
+                machineLearningClient::putDataFrameAnalytics, machineLearningClient::putDataFrameAnalyticsAsync);
+            DataFrameAnalyticsConfig createdConfig = putDataFrameAnalyticsResponse.getConfig();
+            createdConfigs.add(createdConfig);
+        }
+
+        {
+            GetDataFrameAnalyticsResponse getDataFrameAnalyticsResponse = execute(
+                GetDataFrameAnalyticsRequest.getAllDataFrameAnalyticsRequest(),
+                machineLearningClient::getDataFrameAnalytics, machineLearningClient::getDataFrameAnalyticsAsync);
+            assertThat(getDataFrameAnalyticsResponse.getAnalytics(), hasSize(numberOfConfigs));
+            assertThat(getDataFrameAnalyticsResponse.getAnalytics(), containsInAnyOrder(createdConfigs.toArray()));
+        }
+        {
+            GetDataFrameAnalyticsResponse getDataFrameAnalyticsResponse = execute(
+                new GetDataFrameAnalyticsRequest(configIdPrefix + "*"),
+                machineLearningClient::getDataFrameAnalytics, machineLearningClient::getDataFrameAnalyticsAsync);
+            assertThat(getDataFrameAnalyticsResponse.getAnalytics(), hasSize(numberOfConfigs));
+            assertThat(getDataFrameAnalyticsResponse.getAnalytics(), containsInAnyOrder(createdConfigs.toArray()));
+        }
+        {
+            GetDataFrameAnalyticsResponse getDataFrameAnalyticsResponse = execute(
+                new GetDataFrameAnalyticsRequest(configIdPrefix + "9", configIdPrefix + "1", configIdPrefix + "4"),
+                machineLearningClient::getDataFrameAnalytics, machineLearningClient::getDataFrameAnalyticsAsync);
+            assertThat(getDataFrameAnalyticsResponse.getAnalytics(), hasSize(3));
+            assertThat(
+                getDataFrameAnalyticsResponse.getAnalytics(),
+                containsInAnyOrder(createdConfigs.get(1), createdConfigs.get(4), createdConfigs.get(9)));
+        }
+        {
+            GetDataFrameAnalyticsRequest getDataFrameAnalyticsRequest = new GetDataFrameAnalyticsRequest(configIdPrefix + "*");
+            getDataFrameAnalyticsRequest.setPageParams(new PageParams(3, 4));
+            GetDataFrameAnalyticsResponse getDataFrameAnalyticsResponse = execute(
+                getDataFrameAnalyticsRequest,
+                machineLearningClient::getDataFrameAnalytics, machineLearningClient::getDataFrameAnalyticsAsync);
+            assertThat(getDataFrameAnalyticsResponse.getAnalytics(), hasSize(4));
+            assertThat(
+                getDataFrameAnalyticsResponse.getAnalytics(),
+                containsInAnyOrder(createdConfigs.get(3), createdConfigs.get(4), createdConfigs.get(5), createdConfigs.get(6)));
+        }
+    }
+
+    public void testGetDataFrameAnalyticsConfig_ConfigNotFound() {
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+        GetDataFrameAnalyticsRequest request = new GetDataFrameAnalyticsRequest("config_that_does_not_exist");
+        ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class,
+            () -> execute(request, machineLearningClient::getDataFrameAnalytics, machineLearningClient::getDataFrameAnalyticsAsync));
+        assertThat(exception.status().getStatus(), equalTo(404));
+    }
+
+    public void testGetDataFrameAnalyticsStats() throws Exception {
+        String sourceIndex = "get-stats-test-source-index";
+        String destIndex = "get-stats-test-dest-index";
+        createIndex(sourceIndex, defaultMappingForTest());
+        highLevelClient().index(new IndexRequest(sourceIndex).source(XContentType.JSON, "total", 10000), RequestOptions.DEFAULT);
+
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+        String configId = "get-stats-test-config";
+        DataFrameAnalyticsConfig config = DataFrameAnalyticsConfig.builder(configId)
+            .setSource(DataFrameAnalyticsSource.builder()
+                .setIndex(sourceIndex)
+                .build())
+            .setDest(DataFrameAnalyticsDest.builder()
+                .setIndex(destIndex)
+                .build())
+            .setAnalysis(OutlierDetection.createDefault())
+            .build();
+
+        execute(
+            new PutDataFrameAnalyticsRequest(config),
+            machineLearningClient::putDataFrameAnalytics, machineLearningClient::putDataFrameAnalyticsAsync);
+
+        GetDataFrameAnalyticsStatsResponse statsResponse = execute(
+            new GetDataFrameAnalyticsStatsRequest(configId),
+            machineLearningClient::getDataFrameAnalyticsStats, machineLearningClient::getDataFrameAnalyticsStatsAsync);
+
+        assertThat(statsResponse.getAnalyticsStats(), hasSize(1));
+        DataFrameAnalyticsStats stats = statsResponse.getAnalyticsStats().get(0);
+        assertThat(stats.getId(), equalTo(configId));
+        assertThat(stats.getState(), equalTo(DataFrameAnalyticsState.STOPPED));
+        assertNull(stats.getFailureReason());
+        assertNull(stats.getProgressPercent());
+        assertNull(stats.getNode());
+        assertNull(stats.getAssignmentExplanation());
+        assertThat(statsResponse.getNodeFailures(), hasSize(0));
+        assertThat(statsResponse.getTaskFailures(), hasSize(0));
+    }
+
+    public void testStartDataFrameAnalyticsConfig() throws Exception {
+        String sourceIndex = "start-test-source-index";
+        String destIndex = "start-test-dest-index";
+        createIndex(sourceIndex, defaultMappingForTest());
+        highLevelClient().index(new IndexRequest(sourceIndex).source(XContentType.JSON, "total", 10000)
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT);
+
+        // Verify that the destination index does not exist. Otherwise, analytics' reindexing step would fail.
+        assertFalse(highLevelClient().indices().exists(new GetIndexRequest(destIndex), RequestOptions.DEFAULT));
+
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+        String configId = "start-test-config";
+        DataFrameAnalyticsConfig config = DataFrameAnalyticsConfig.builder(configId)
+            .setSource(DataFrameAnalyticsSource.builder()
+                .setIndex(sourceIndex)
+                .build())
+            .setDest(DataFrameAnalyticsDest.builder()
+                .setIndex(destIndex)
+                .build())
+            .setAnalysis(OutlierDetection.createDefault())
+            .build();
+
+        execute(
+            new PutDataFrameAnalyticsRequest(config),
+            machineLearningClient::putDataFrameAnalytics, machineLearningClient::putDataFrameAnalyticsAsync);
+        assertThat(getAnalyticsState(configId), equalTo(DataFrameAnalyticsState.STOPPED));
+
+        AcknowledgedResponse startDataFrameAnalyticsResponse = execute(
+            new StartDataFrameAnalyticsRequest(configId),
+            machineLearningClient::startDataFrameAnalytics, machineLearningClient::startDataFrameAnalyticsAsync);
+        assertTrue(startDataFrameAnalyticsResponse.isAcknowledged());
+
+        // Wait for the analytics to stop.
+        assertBusy(() -> assertThat(getAnalyticsState(configId), equalTo(DataFrameAnalyticsState.STOPPED)), 30, TimeUnit.SECONDS);
+
+        // Verify that the destination index got created.
+        assertTrue(highLevelClient().indices().exists(new GetIndexRequest(destIndex), RequestOptions.DEFAULT));
+    }
+
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/43924")
+    public void testStopDataFrameAnalyticsConfig() throws Exception {
+        String sourceIndex = "stop-test-source-index";
+        String destIndex = "stop-test-dest-index";
+        createIndex(sourceIndex, mappingForClassification());
+        highLevelClient().index(new IndexRequest(sourceIndex).source(XContentType.JSON, "total", 10000)
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT);
+
+        // Verify that the destination index does not exist. Otherwise, analytics' reindexing step would fail.
+        assertFalse(highLevelClient().indices().exists(new GetIndexRequest(destIndex), RequestOptions.DEFAULT));
+
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+        String configId = "stop-test-config";
+        DataFrameAnalyticsConfig config = DataFrameAnalyticsConfig.builder(configId)
+            .setSource(DataFrameAnalyticsSource.builder()
+                .setIndex(sourceIndex)
+                .build())
+            .setDest(DataFrameAnalyticsDest.builder()
+                .setIndex(destIndex)
+                .build())
+            .setAnalysis(OutlierDetection.createDefault())
+            .build();
+
+        execute(
+            new PutDataFrameAnalyticsRequest(config),
+            machineLearningClient::putDataFrameAnalytics, machineLearningClient::putDataFrameAnalyticsAsync);
+        assertThat(getAnalyticsState(configId), equalTo(DataFrameAnalyticsState.STOPPED));
+
+        AcknowledgedResponse startDataFrameAnalyticsResponse = execute(
+            new StartDataFrameAnalyticsRequest(configId),
+            machineLearningClient::startDataFrameAnalytics, machineLearningClient::startDataFrameAnalyticsAsync);
+        assertTrue(startDataFrameAnalyticsResponse.isAcknowledged());
+        assertThat(getAnalyticsState(configId), equalTo(DataFrameAnalyticsState.STARTED));
+
+        StopDataFrameAnalyticsResponse stopDataFrameAnalyticsResponse = execute(
+            new StopDataFrameAnalyticsRequest(configId),
+            machineLearningClient::stopDataFrameAnalytics, machineLearningClient::stopDataFrameAnalyticsAsync);
+        assertTrue(stopDataFrameAnalyticsResponse.isStopped());
+        assertThat(getAnalyticsState(configId), equalTo(DataFrameAnalyticsState.STOPPED));
+    }
+
+    private DataFrameAnalyticsState getAnalyticsState(String configId) throws IOException {
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+        GetDataFrameAnalyticsStatsResponse statsResponse =
+            machineLearningClient.getDataFrameAnalyticsStats(new GetDataFrameAnalyticsStatsRequest(configId), RequestOptions.DEFAULT);
+        assertThat(statsResponse.getAnalyticsStats(), hasSize(1));
+        DataFrameAnalyticsStats stats = statsResponse.getAnalyticsStats().get(0);
+        return stats.getState();
+    }
+
+    public void testDeleteDataFrameAnalyticsConfig() throws Exception {
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+        String configId = "delete-test-config";
+        DataFrameAnalyticsConfig config = DataFrameAnalyticsConfig.builder(configId)
+            .setSource(DataFrameAnalyticsSource.builder()
+                .setIndex("delete-test-source-index")
+                .build())
+            .setDest(DataFrameAnalyticsDest.builder()
+                .setIndex("delete-test-dest-index")
+                .build())
+            .setAnalysis(OutlierDetection.createDefault())
+            .build();
+
+        createIndex("delete-test-source-index", defaultMappingForTest());
+
+        GetDataFrameAnalyticsResponse getDataFrameAnalyticsResponse = execute(
+            new GetDataFrameAnalyticsRequest(configId + "*"),
+            machineLearningClient::getDataFrameAnalytics, machineLearningClient::getDataFrameAnalyticsAsync);
+        assertThat(getDataFrameAnalyticsResponse.getAnalytics(), hasSize(0));
+
+        execute(
+            new PutDataFrameAnalyticsRequest(config),
+            machineLearningClient::putDataFrameAnalytics, machineLearningClient::putDataFrameAnalyticsAsync);
+
+        getDataFrameAnalyticsResponse = execute(
+            new GetDataFrameAnalyticsRequest(configId + "*"),
+            machineLearningClient::getDataFrameAnalytics, machineLearningClient::getDataFrameAnalyticsAsync);
+        assertThat(getDataFrameAnalyticsResponse.getAnalytics(), hasSize(1));
+
+        AcknowledgedResponse deleteDataFrameAnalyticsResponse = execute(
+            new DeleteDataFrameAnalyticsRequest(configId),
+            machineLearningClient::deleteDataFrameAnalytics, machineLearningClient::deleteDataFrameAnalyticsAsync);
+        assertTrue(deleteDataFrameAnalyticsResponse.isAcknowledged());
+
+        getDataFrameAnalyticsResponse = execute(
+            new GetDataFrameAnalyticsRequest(configId + "*"),
+            machineLearningClient::getDataFrameAnalytics, machineLearningClient::getDataFrameAnalyticsAsync);
+        assertThat(getDataFrameAnalyticsResponse.getAnalytics(), hasSize(0));
+    }
+
+    public void testDeleteDataFrameAnalyticsConfig_ConfigNotFound() {
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+        DeleteDataFrameAnalyticsRequest request = new DeleteDataFrameAnalyticsRequest("config_that_does_not_exist");
+        ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class,
+            () -> execute(
+                request, machineLearningClient::deleteDataFrameAnalytics, machineLearningClient::deleteDataFrameAnalyticsAsync));
+        assertThat(exception.status().getStatus(), equalTo(404));
+    }
+
+    public void testEvaluateDataFrame() throws IOException {
+        String indexName = "evaluate-test-index";
+        createIndex(indexName, mappingForClassification());
+        BulkRequest bulk = new BulkRequest()
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .add(docForClassification(indexName, false, 0.1))  // #0
+            .add(docForClassification(indexName, false, 0.2))  // #1
+            .add(docForClassification(indexName, false, 0.3))  // #2
+            .add(docForClassification(indexName, false, 0.4))  // #3
+            .add(docForClassification(indexName, false, 0.7))  // #4
+            .add(docForClassification(indexName, true, 0.2))  // #5
+            .add(docForClassification(indexName, true, 0.3))  // #6
+            .add(docForClassification(indexName, true, 0.4))  // #7
+            .add(docForClassification(indexName, true, 0.8))  // #8
+            .add(docForClassification(indexName, true, 0.9));  // #9
+        highLevelClient().bulk(bulk, RequestOptions.DEFAULT);
+
+        MachineLearningClient machineLearningClient = highLevelClient().machineLearning();
+        EvaluateDataFrameRequest evaluateDataFrameRequest =
+            new EvaluateDataFrameRequest(
+                indexName,
+                new BinarySoftClassification(
+                    actualField,
+                    probabilityField,
+                    PrecisionMetric.at(0.4, 0.5, 0.6), RecallMetric.at(0.5, 0.7), ConfusionMatrixMetric.at(0.5), AucRocMetric.withCurve()));
+
+        EvaluateDataFrameResponse evaluateDataFrameResponse =
+            execute(evaluateDataFrameRequest, machineLearningClient::evaluateDataFrame, machineLearningClient::evaluateDataFrameAsync);
+        assertThat(evaluateDataFrameResponse.getEvaluationName(), equalTo(BinarySoftClassification.NAME));
+        assertThat(evaluateDataFrameResponse.getMetrics().size(), equalTo(4));
+
+        PrecisionMetric.Result precisionResult = evaluateDataFrameResponse.getMetricByName(PrecisionMetric.NAME);
+        assertThat(precisionResult.getMetricName(), equalTo(PrecisionMetric.NAME));
+        // Precision is 3/5=0.6 as there were 3 true examples (#7, #8, #9) among the 5 positive examples (#3, #4, #7, #8, #9)
+        assertThat(precisionResult.getScoreByThreshold("0.4"), closeTo(0.6, 1e-9));
+        // Precision is 2/3=0.(6) as there were 2 true examples (#8, #9) among the 3 positive examples (#4, #8, #9)
+        assertThat(precisionResult.getScoreByThreshold("0.5"), closeTo(0.666666666, 1e-9));
+        // Precision is 2/3=0.(6) as there were 2 true examples (#8, #9) among the 3 positive examples (#4, #8, #9)
+        assertThat(precisionResult.getScoreByThreshold("0.6"), closeTo(0.666666666, 1e-9));
+        assertNull(precisionResult.getScoreByThreshold("0.1"));
+
+        RecallMetric.Result recallResult = evaluateDataFrameResponse.getMetricByName(RecallMetric.NAME);
+        assertThat(recallResult.getMetricName(), equalTo(RecallMetric.NAME));
+        // Recall is 2/5=0.4 as there were 2 true positive examples (#8, #9) among the 5 true examples (#5, #6, #7, #8, #9)
+        assertThat(recallResult.getScoreByThreshold("0.5"), closeTo(0.4, 1e-9));
+        // Recall is 2/5=0.4 as there were 2 true positive examples (#8, #9) among the 5 true examples (#5, #6, #7, #8, #9)
+        assertThat(recallResult.getScoreByThreshold("0.7"), closeTo(0.4, 1e-9));
+        assertNull(recallResult.getScoreByThreshold("0.1"));
+
+        ConfusionMatrixMetric.Result confusionMatrixResult = evaluateDataFrameResponse.getMetricByName(ConfusionMatrixMetric.NAME);
+        assertThat(confusionMatrixResult.getMetricName(), equalTo(ConfusionMatrixMetric.NAME));
+        ConfusionMatrixMetric.ConfusionMatrix confusionMatrix = confusionMatrixResult.getScoreByThreshold("0.5");
+        assertThat(confusionMatrix.getTruePositives(), equalTo(2L));  // docs #8 and #9
+        assertThat(confusionMatrix.getFalsePositives(), equalTo(1L));  // doc #4
+        assertThat(confusionMatrix.getTrueNegatives(), equalTo(4L));  // docs #0, #1, #2 and #3
+        assertThat(confusionMatrix.getFalseNegatives(), equalTo(3L));  // docs #5, #6 and #7
+        assertNull(confusionMatrixResult.getScoreByThreshold("0.1"));
+
+        AucRocMetric.Result aucRocResult = evaluateDataFrameResponse.getMetricByName(AucRocMetric.NAME);
+        assertThat(aucRocResult.getMetricName(), equalTo(AucRocMetric.NAME));
+        assertThat(aucRocResult.getScore(), closeTo(0.70025, 1e-9));
+        assertNotNull(aucRocResult.getCurve());
+        List<AucRocMetric.AucRocPoint> curve = aucRocResult.getCurve();
+        AucRocMetric.AucRocPoint curvePointAtThreshold0 = curve.stream().filter(p -> p.getThreshold() == 0.0).findFirst().get();
+        assertThat(curvePointAtThreshold0.getTruePositiveRate(), equalTo(1.0));
+        assertThat(curvePointAtThreshold0.getFalsePositiveRate(), equalTo(1.0));
+        assertThat(curvePointAtThreshold0.getThreshold(), equalTo(0.0));
+        AucRocMetric.AucRocPoint curvePointAtThreshold1 = curve.stream().filter(p -> p.getThreshold() == 1.0).findFirst().get();
+        assertThat(curvePointAtThreshold1.getTruePositiveRate(), equalTo(0.0));
+        assertThat(curvePointAtThreshold1.getFalsePositiveRate(), equalTo(0.0));
+        assertThat(curvePointAtThreshold1.getThreshold(), equalTo(1.0));
+    }
+
+    private static XContentBuilder defaultMappingForTest() throws IOException {
+        return XContentFactory.jsonBuilder().startObject()
+            .startObject("properties")
+               .startObject("timestamp")
+                    .field("type", "date")
+                .endObject()
+                .startObject("total")
+                    .field("type", "long")
+                .endObject()
+            .endObject()
+        .endObject();
+    }
+
+    private static final String actualField = "label";
+    private static final String probabilityField = "p";
+
+    private static XContentBuilder mappingForClassification() throws IOException {
+        return XContentFactory.jsonBuilder().startObject()
+            .startObject("properties")
+                .startObject(actualField)
+                    .field("type", "keyword")
+                .endObject()
+                .startObject(probabilityField)
+                    .field("type", "double")
+                .endObject()
+            .endObject()
+        .endObject();
+    }
+
+    private static IndexRequest docForClassification(String indexName, boolean isTrue, double p) {
+        return new IndexRequest()
+            .index(indexName)
+            .source(XContentType.JSON, actualField, Boolean.toString(isTrue), probabilityField, p);
+    }
+
+    private void createIndex(String indexName, XContentBuilder mapping) throws IOException {
+        highLevelClient().indices().create(new CreateIndexRequest(indexName).mapping(mapping), RequestOptions.DEFAULT);
+    }
+
     public void testPutFilter() throws Exception {
         String filterId = "filter-job-test";
         MlFilter mlFilter = MlFilter.builder(filterId)
@@ -1286,7 +1673,7 @@ public void testGetFilters() throws Exception {
             GetFiltersResponse getFiltersResponse = execute(getFiltersRequest,
                 machineLearningClient::getFilter,
                 machineLearningClient::getFilterAsync);
-            assertThat(getFiltersResponse.count(), equalTo(2L));
+            assertThat(getFiltersResponse.count(), equalTo(3L));
             assertThat(getFiltersResponse.filters().size(), equalTo(2));
             assertThat(getFiltersResponse.filters().stream().map(MlFilter::getId).collect(Collectors.toList()),
                 containsInAnyOrder("get-filter-test-2", "get-filter-test-3"));
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MlTestStateCleaner.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MlTestStateCleaner.java
index c565af7c37202..f5776e99fd0eb 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MlTestStateCleaner.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MlTestStateCleaner.java
@@ -20,14 +20,18 @@
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.ml.CloseJobRequest;
+import org.elasticsearch.client.ml.DeleteDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.DeleteDatafeedRequest;
 import org.elasticsearch.client.ml.DeleteJobRequest;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsRequest;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsResponse;
 import org.elasticsearch.client.ml.GetDatafeedRequest;
 import org.elasticsearch.client.ml.GetDatafeedResponse;
 import org.elasticsearch.client.ml.GetJobRequest;
 import org.elasticsearch.client.ml.GetJobResponse;
 import org.elasticsearch.client.ml.StopDatafeedRequest;
 import org.elasticsearch.client.ml.datafeed.DatafeedConfig;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.client.ml.job.config.Job;
 
 import java.io.IOException;
@@ -48,6 +52,7 @@ public MlTestStateCleaner(Logger logger, MachineLearningClient mlClient) {
     public void clearMlMetadata() throws IOException {
         deleteAllDatafeeds();
         deleteAllJobs();
+        deleteAllDataFrameAnalytics();
     }
 
     private void deleteAllDatafeeds() throws IOException {
@@ -99,4 +104,12 @@ private void closeAllJobs() {
             throw new RuntimeException("Had to resort to force-closing jobs, something went wrong?", e1);
         }
     }
+
+    private void deleteAllDataFrameAnalytics() throws IOException {
+        GetDataFrameAnalyticsResponse getDataFrameAnalyticsResponse =
+            mlClient.getDataFrameAnalytics(GetDataFrameAnalyticsRequest.getAllDataFrameAnalyticsRequest(), RequestOptions.DEFAULT);
+        for (DataFrameAnalyticsConfig config : getDataFrameAnalyticsResponse.getAnalytics()) {
+            mlClient.deleteDataFrameAnalytics(new DeleteDataFrameAnalyticsRequest(config.getId()), RequestOptions.DEFAULT);
+        }
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java
index 1b4ca84c47144..eee2491c7d42f 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java
@@ -71,31 +71,20 @@ public void testXPackInfo() throws IOException {
         assertEquals(LicenseStatus.ACTIVE, info.getLicenseInfo().getStatus());
 
         FeatureSet graph = info.getFeatureSetsInfo().getFeatureSets().get("graph");
-        assertNotNull(graph.description());
         assertTrue(graph.available());
         assertTrue(graph.enabled());
-        assertNull(graph.nativeCodeInfo());
         FeatureSet monitoring = info.getFeatureSetsInfo().getFeatureSets().get("monitoring");
-        assertNotNull(monitoring.description());
         assertTrue(monitoring.available());
         assertTrue(monitoring.enabled());
-        assertNull(monitoring.nativeCodeInfo());
         FeatureSet ml = info.getFeatureSetsInfo().getFeatureSets().get("ml");
-        assertNotNull(ml.description());
         assertTrue(ml.available());
         assertTrue(ml.enabled());
-        assertEquals(mainResponse.getVersion().getNumber(), ml.nativeCodeInfo().get("version").toString());
     }
 
     public void testXPackInfoEmptyRequest() throws IOException {
         XPackInfoResponse info = highLevelClient().xpack().info(new XPackInfoRequest(), RequestOptions.DEFAULT);
 
-        /*
-         * The default in the transport client is non-verbose and returning
-         * no categories which is the opposite of the default when you use
-         * the API over REST. We don't want to break the transport client
-         * even though it doesn't feel like a good default.
-         */
+        // TODO: reconsider this leniency now that the transport client is gone
         assertNull(info.getBuildInfo());
         assertNull(info.getLicenseInfo());
         assertNull(info.getFeatureSetsInfo());
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
index 9c5137d54427a..a1946baa3e0e1 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkShardRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
@@ -56,6 +55,7 @@
 import org.elasticsearch.client.core.CountRequest;
 import org.elasticsearch.client.core.MultiTermVectorsRequest;
 import org.elasticsearch.client.core.TermVectorsRequest;
+import org.elasticsearch.client.indices.AnalyzeRequest;
 import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -404,9 +404,6 @@ public void testReindex() throws IOException {
             );
             reindexRequest.setRemoteInfo(remoteInfo);
         }
-        if (randomBoolean()) {
-            reindexRequest.setSourceDocTypes("doc", "tweet");
-        }
         if (randomBoolean()) {
             reindexRequest.setSourceBatchSize(randomInt(100));
         }
@@ -430,7 +427,7 @@ public void testReindex() throws IOException {
             reindexRequest.setDestRouting("=cat");
         }
         if (randomBoolean()) {
-            reindexRequest.setSize(randomIntBetween(100, 1000));
+            reindexRequest.setMaxDocs(randomIntBetween(100, 1000));
         }
         if (randomBoolean()) {
             reindexRequest.setAbortOnVersionConflict(false);
@@ -457,9 +454,6 @@ public void testUpdateByQuery() throws IOException {
         UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest();
         updateByQueryRequest.indices(randomIndicesNames(1, 5));
         Map<String, String> expectedParams = new HashMap<>();
-        if (randomBoolean()) {
-            updateByQueryRequest.setDocTypes(generateRandomStringArray(5, 5, false, false));
-        }
         if (randomBoolean()) {
             int batchSize = randomInt(100);
             updateByQueryRequest.setBatchSize(batchSize);
@@ -481,9 +475,9 @@ public void testUpdateByQuery() throws IOException {
             expectedParams.put("routing", "=cat");
         }
         if (randomBoolean()) {
-            int size = randomIntBetween(100, 1000);
-            updateByQueryRequest.setSize(size);
-            expectedParams.put("size", Integer.toString(size));
+            int maxDocs = randomIntBetween(100, 1000);
+            updateByQueryRequest.setMaxDocs(maxDocs);
+            expectedParams.put("max_docs", Integer.toString(maxDocs));
         }
         if (randomBoolean()) {
             updateByQueryRequest.setAbortOnVersionConflict(false);
@@ -505,8 +499,6 @@ public void testUpdateByQuery() throws IOException {
         Request request = RequestConverters.updateByQuery(updateByQueryRequest);
         StringJoiner joiner = new StringJoiner("/", "/", "");
         joiner.add(String.join(",", updateByQueryRequest.indices()));
-        if (updateByQueryRequest.getDocTypes().length > 0)
-            joiner.add(String.join(",", updateByQueryRequest.getDocTypes()));
         joiner.add("_update_by_query");
         assertEquals(joiner.toString(), request.getEndpoint());
         assertEquals(HttpPost.METHOD_NAME, request.getMethod());
@@ -518,9 +510,6 @@ public void testDeleteByQuery() throws IOException {
         DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest();
         deleteByQueryRequest.indices(randomIndicesNames(1, 5));
         Map<String, String> expectedParams = new HashMap<>();
-        if (randomBoolean()) {
-            deleteByQueryRequest.setDocTypes(generateRandomStringArray(5, 5, false, false));
-        }
         if (randomBoolean()) {
             int batchSize = randomInt(100);
             deleteByQueryRequest.setBatchSize(batchSize);
@@ -531,9 +520,9 @@ public void testDeleteByQuery() throws IOException {
             expectedParams.put("routing", "=cat");
         }
         if (randomBoolean()) {
-            int size = randomIntBetween(100, 1000);
-            deleteByQueryRequest.setSize(size);
-            expectedParams.put("size", Integer.toString(size));
+            int maxDocs = randomIntBetween(100, 1000);
+            deleteByQueryRequest.setMaxDocs(maxDocs);
+            expectedParams.put("max_docs", Integer.toString(maxDocs));
         }
         if (randomBoolean()) {
             deleteByQueryRequest.setAbortOnVersionConflict(false);
@@ -559,8 +548,6 @@ public void testDeleteByQuery() throws IOException {
         Request request = RequestConverters.deleteByQuery(deleteByQueryRequest);
         StringJoiner joiner = new StringJoiner("/", "/", "");
         joiner.add(String.join(",", deleteByQueryRequest.indices()));
-        if (deleteByQueryRequest.getDocTypes().length > 0)
-            joiner.add(String.join(",", deleteByQueryRequest.getDocTypes()));
         joiner.add("_delete_by_query");
         assertEquals(joiner.toString(), request.getEndpoint());
         assertEquals(HttpPost.METHOD_NAME, request.getMethod());
@@ -1065,13 +1052,6 @@ public void testSearch() throws Exception {
         String[] indices = randomIndicesNames(0, 5);
         SearchRequest searchRequest = new SearchRequest(indices);
 
-        int numTypes = randomIntBetween(0, 5);
-        String[] types = new String[numTypes];
-        for (int i = 0; i < numTypes; i++) {
-            types[i] = "type-" + randomAlphaOfLengthBetween(2, 5);
-        }
-        searchRequest.types(types);
-
         Map<String, String> expectedParams = new HashMap<>();
         setRandomSearchParams(searchRequest, expectedParams);
         setRandomIndicesOptions(searchRequest::indicesOptions, searchRequest::indicesOptions, expectedParams);
@@ -1128,10 +1108,6 @@ public void testSearch() throws Exception {
         if (Strings.hasLength(index)) {
             endpoint.add(index);
         }
-        String type = String.join(",", types);
-        if (Strings.hasLength(type)) {
-            endpoint.add(type);
-        }
         endpoint.add(searchEndpoint);
         assertEquals(HttpPost.METHOD_NAME, request.getMethod());
         assertEquals(endpoint.toString(), request.getEndpoint());
@@ -1142,7 +1118,6 @@ public void testSearch() throws Exception {
     public void testSearchNullIndicesAndTypes() {
         expectThrows(NullPointerException.class, () -> new SearchRequest((String[]) null));
         expectThrows(NullPointerException.class, () -> new SearchRequest().indices((String[]) null));
-        expectThrows(NullPointerException.class, () -> new SearchRequest().types((String[]) null));
     }
 
      public void testCountNotNullSource() throws IOException {
@@ -1257,7 +1232,7 @@ public void testMultiSearch() throws IOException {
             requests.add(searchRequest);
         };
         MultiSearchRequest.readMultiLineFormat(new BytesArray(EntityUtils.toByteArray(request.getEntity())),
-                REQUEST_BODY_CONTENT_TYPE.xContent(), consumer, null, multiSearchRequest.indicesOptions(), null, null, null, null,
+                REQUEST_BODY_CONTENT_TYPE.xContent(), consumer, null, multiSearchRequest.indicesOptions(), null, null, null,
                 xContentRegistry(), true);
         assertEquals(requests, multiSearchRequest.requests());
     }
@@ -1561,7 +1536,7 @@ public void testFieldCaps() {
         endpoint.add("_field_caps");
 
         assertEquals(endpoint.toString(), request.getEndpoint());
-        assertEquals(4, request.getParameters().size());
+        assertEquals(5, request.getParameters().size());
 
         // Note that we don't check the field param value explicitly, as field names are
         // passed through
@@ -1595,7 +1570,7 @@ public void testRankEval() throws Exception {
         }
         endpoint.add(RestRankEvalAction.ENDPOINT);
         assertEquals(endpoint.toString(), request.getEndpoint());
-        assertEquals(3, request.getParameters().size());
+        assertEquals(4, request.getParameters().size());
         assertEquals(expectedParams, request.getParameters());
         assertToXContentBody(spec, request.getEntity());
     }
@@ -1637,18 +1612,14 @@ public void testPutScript() throws Exception {
     }
 
     public void testAnalyzeRequest() throws Exception {
-        AnalyzeRequest indexAnalyzeRequest = new AnalyzeRequest()
-            .text("Here is some text")
-            .index("test_index")
-            .analyzer("test_analyzer");
+        AnalyzeRequest indexAnalyzeRequest
+            = AnalyzeRequest.withIndexAnalyzer("test_index", "test_analyzer", "Here is some text");
 
         Request request = RequestConverters.analyze(indexAnalyzeRequest);
         assertThat(request.getEndpoint(), equalTo("/test_index/_analyze"));
         assertToXContentBody(indexAnalyzeRequest, request.getEntity());
 
-        AnalyzeRequest analyzeRequest = new AnalyzeRequest()
-            .text("more text")
-            .analyzer("test_analyzer");
+        AnalyzeRequest analyzeRequest = AnalyzeRequest.withGlobalAnalyzer("test_analyzer", "more text");
         assertThat(RequestConverters.analyze(analyzeRequest).getEndpoint(), equalTo("/_analyze"));
     }
 
@@ -1922,7 +1893,8 @@ static void setRandomIndicesOptions(Consumer<IndicesOptions> setter, Supplier<In
                                         Map<String, String> expectedParams) {
 
         if (randomBoolean()) {
-            setter.accept(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
+            setter.accept(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
+                true, false, false, randomBoolean()));
         }
         expectedParams.put("ignore_unavailable", Boolean.toString(getter.get().ignoreUnavailable()));
         expectedParams.put("allow_no_indices", Boolean.toString(getter.get().allowNoIndices()));
@@ -1935,11 +1907,13 @@ static void setRandomIndicesOptions(Consumer<IndicesOptions> setter, Supplier<In
         } else {
             expectedParams.put("expand_wildcards", "none");
         }
+        expectedParams.put("ignore_throttled", Boolean.toString(getter.get().ignoreThrottled()));
     }
 
     static IndicesOptions setRandomIndicesOptions(IndicesOptions indicesOptions, Map<String, String> expectedParams) {
         if (randomBoolean()) {
-            indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
+            indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(),
+                true, false, false, randomBoolean());
         }
         expectedParams.put("ignore_unavailable", Boolean.toString(indicesOptions.ignoreUnavailable()));
         expectedParams.put("allow_no_indices", Boolean.toString(indicesOptions.allowNoIndices()));
@@ -1952,6 +1926,7 @@ static IndicesOptions setRandomIndicesOptions(IndicesOptions indicesOptions, Map
         } else {
             expectedParams.put("expand_wildcards", "none");
         }
+        expectedParams.put("ignore_throttled", Boolean.toString(indicesOptions.ignoreThrottled()));
         return indicesOptions;
     }
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
index ed5d7b66d80c1..ae1cd5eb45edf 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -46,6 +46,8 @@
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.client.core.MainRequest;
 import org.elasticsearch.client.core.MainResponse;
+import org.elasticsearch.client.dataframe.transforms.SyncConfig;
+import org.elasticsearch.client.dataframe.transforms.TimeSyncConfig;
 import org.elasticsearch.client.indexlifecycle.AllocateAction;
 import org.elasticsearch.client.indexlifecycle.DeleteAction;
 import org.elasticsearch.client.indexlifecycle.ForceMergeAction;
@@ -56,6 +58,13 @@
 import org.elasticsearch.client.indexlifecycle.SetPriorityAction;
 import org.elasticsearch.client.indexlifecycle.ShrinkAction;
 import org.elasticsearch.client.indexlifecycle.UnfollowAction;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalysis;
+import org.elasticsearch.client.ml.dataframe.OutlierDetection;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.AucRocMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.BinarySoftClassification;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.PrecisionMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.RecallMetric;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
@@ -109,6 +118,7 @@
 import static org.hamcrest.CoreMatchers.endsWith;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.Matchers.hasItems;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
@@ -131,7 +141,7 @@ public class RestHighLevelClientTests extends ESTestCase {
         // core
         "ping", "info",
         // security
-        "security.get_ssl_certificates", "security.authenticate", "security.get_user_privileges",
+        "security.get_ssl_certificates", "security.authenticate", "security.get_user_privileges", "security.get_builtin_privileges",
         // license
         "license.get_trial_status", "license.get_basic_status"
 
@@ -664,7 +674,7 @@ public void testDefaultNamedXContents() {
 
     public void testProvidedNamedXContents() {
         List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
-        assertEquals(20, namedXContents.size());
+        assertEquals(31, namedXContents.size());
         Map<Class<?>, Integer> categories = new HashMap<>();
         List<String> names = new ArrayList<>();
         for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@@ -674,7 +684,7 @@ public void testProvidedNamedXContents() {
                 categories.put(namedXContent.categoryClass, counter + 1);
             }
         }
-        assertEquals("Had: " + categories, 4, categories.size());
+        assertEquals("Had: " + categories, 9, categories.size());
         assertEquals(Integer.valueOf(3), categories.get(Aggregation.class));
         assertTrue(names.contains(ChildrenAggregationBuilder.NAME));
         assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME));
@@ -698,6 +708,16 @@ public void testProvidedNamedXContents() {
         assertTrue(names.contains(ShrinkAction.NAME));
         assertTrue(names.contains(FreezeAction.NAME));
         assertTrue(names.contains(SetPriorityAction.NAME));
+        assertEquals(Integer.valueOf(1), categories.get(DataFrameAnalysis.class));
+        assertTrue(names.contains(OutlierDetection.NAME.getPreferredName()));
+        assertEquals(Integer.valueOf(1), categories.get(SyncConfig.class));
+        assertTrue(names.contains(TimeSyncConfig.NAME));
+        assertEquals(Integer.valueOf(1), categories.get(org.elasticsearch.client.ml.dataframe.evaluation.Evaluation.class));
+        assertThat(names, hasItems(BinarySoftClassification.NAME));
+        assertEquals(Integer.valueOf(4), categories.get(org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric.class));
+        assertThat(names, hasItems(AucRocMetric.NAME, PrecisionMetric.NAME, RecallMetric.NAME, ConfusionMatrixMetric.NAME));
+        assertEquals(Integer.valueOf(4), categories.get(org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric.Result.class));
+        assertThat(names, hasItems(AucRocMetric.NAME, PrecisionMetric.NAME, RecallMetric.NAME, ConfusionMatrixMetric.NAME));
     }
 
     public void testApiNamingConventions() throws Exception {
@@ -731,7 +751,6 @@ public void testApiNamingConventions() throws Exception {
             "nodes.hot_threads",
             "nodes.usage",
             "nodes.reload_secure_settings",
-            "scripts_painless_context",
             "search_shards",
         };
         List<String> booleanReturnMethods = Arrays.asList(
@@ -805,6 +824,7 @@ public void testApiNamingConventions() throws Exception {
                                 apiName.startsWith("ccr.") == false &&
                                 apiName.startsWith("data_frame") == false &&
                                 apiName.endsWith("freeze") == false &&
+                                apiName.endsWith("reload_analyzers") == false &&
                                 // IndicesClientIT.getIndexTemplate should be renamed "getTemplate" in version 8.0 when we
                                 // can get rid of 7.0's deprecated "getTemplate"
                                 apiName.equals("indices.get_index_template") == false) {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java
index d92c6ad73a738..a7c8a85131355 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java
@@ -29,8 +29,8 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.client.core.IndexerState;
 import org.elasticsearch.client.core.AcknowledgedResponse;
+import org.elasticsearch.client.core.IndexerState;
 import org.elasticsearch.client.rollup.DeleteRollupJobRequest;
 import org.elasticsearch.client.rollup.GetRollupCapsRequest;
 import org.elasticsearch.client.rollup.GetRollupCapsResponse;
@@ -40,10 +40,10 @@
 import org.elasticsearch.client.rollup.GetRollupJobResponse;
 import org.elasticsearch.client.rollup.GetRollupJobResponse.JobWrapper;
 import org.elasticsearch.client.rollup.PutRollupJobRequest;
-import org.elasticsearch.client.rollup.StartRollupJobRequest;
-import org.elasticsearch.client.rollup.StartRollupJobResponse;
 import org.elasticsearch.client.rollup.RollableIndexCaps;
 import org.elasticsearch.client.rollup.RollupJobCaps;
+import org.elasticsearch.client.rollup.StartRollupJobRequest;
+import org.elasticsearch.client.rollup.StartRollupJobResponse;
 import org.elasticsearch.client.rollup.StopRollupJobRequest;
 import org.elasticsearch.client.rollup.StopRollupJobResponse;
 import org.elasticsearch.client.rollup.job.config.DateHistogramGroupConfig;
@@ -54,10 +54,10 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
-import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -259,20 +259,6 @@ public void testSearch() throws Exception {
         assertThat(avg.value(), closeTo(sum / numDocs, 0.00000001));
     }
 
-    public void testSearchWithType() throws Exception {
-        SearchRequest search = new SearchRequest(rollupIndex);
-        search.types("a", "b", "c");
-        search.source(new SearchSourceBuilder()
-                .size(0)
-                .aggregation(new AvgAggregationBuilder("avg").field("value")));
-        try {
-            highLevelClient().rollup().search(search, RequestOptions.DEFAULT);
-            fail("types are not allowed but didn't fail");
-        } catch (ValidationException e) {
-            assertEquals("Validation Failed: 1: types are not allowed in rollup search;", e.getMessage());
-        }
-    }
-
     public void testGetMissingRollupJob() throws Exception {
         GetRollupJobRequest getRollupJobRequest = new GetRollupJobRequest("missing");
         RollupClient rollupClient = highLevelClient().rollup();
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java
index 99350fc29db8a..4c99cb323969e 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java
@@ -315,7 +315,7 @@ public void testGetApplicationPrivilege() throws Exception {
         assertNull(request.getEntity());
     }
 
-    public void testGetAllApplicationPrivileges() throws Exception {
+    public void testGetAllPrivilegesForApplication() throws Exception {
         final String application = randomAlphaOfLength(6);
         GetPrivilegesRequest getPrivilegesRequest = GetPrivilegesRequest.getApplicationPrivileges(application);
         Request request = SecurityRequestConverters.getPrivileges(getPrivilegesRequest);
@@ -339,7 +339,7 @@ public void testGetMultipleApplicationPrivileges() throws Exception {
         assertNull(request.getEntity());
     }
 
-    public void testGetAllPrivileges() throws Exception {
+    public void testGetAllApplicationPrivileges() throws Exception {
         GetPrivilegesRequest getPrivilegesRequest = GetPrivilegesRequest.getAllPrivileges();
         Request request = SecurityRequestConverters.getPrivileges(getPrivilegesRequest);
         assertEquals(HttpGet.METHOD_NAME, request.getMethod());
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
index 616850c513af7..8e4001442b0cc 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
@@ -41,14 +41,16 @@
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.RestoreInfo;
+import org.mockito.internal.util.collections.Sets;
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.stream.Collectors;
+import java.util.HashMap;
+import java.util.Map;
 
-import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 
 public class SnapshotIT extends ESRestHighLevelClientTestCase {
@@ -58,14 +60,14 @@ private AcknowledgedResponse createTestRepository(String repository, String type
         request.settings(settings, XContentType.JSON);
         request.type(type);
         return execute(request, highLevelClient().snapshot()::createRepository,
-            highLevelClient().snapshot()::createRepositoryAsync);
+                highLevelClient().snapshot()::createRepositoryAsync);
     }
 
     private CreateSnapshotResponse createTestSnapshot(CreateSnapshotRequest createSnapshotRequest) throws IOException {
         // assumes the repository already exists
 
         return execute(createSnapshotRequest, highLevelClient().snapshot()::create,
-            highLevelClient().snapshot()::createAsync);
+                highLevelClient().snapshot()::createAsync);
     }
 
     public void testCreateRepository() throws IOException {
@@ -81,7 +83,7 @@ public void testSnapshotGetRepositoriesUsingParams() throws IOException {
         GetRepositoriesRequest request = new GetRepositoriesRequest();
         request.repositories(new String[]{testRepository});
         GetRepositoriesResponse response = execute(request, highLevelClient().snapshot()::getRepository,
-            highLevelClient().snapshot()::getRepositoryAsync);
+                highLevelClient().snapshot()::getRepositoryAsync);
         assertThat(1, equalTo(response.repositories().size()));
     }
 
@@ -90,7 +92,7 @@ public void testSnapshotGetDefaultRepositories() throws IOException {
         assertTrue(createTestRepository("test", FsRepository.TYPE, "{\"location\": \".\"}").isAcknowledged());
 
         GetRepositoriesResponse response = execute(new GetRepositoriesRequest(), highLevelClient().snapshot()::getRepository,
-            highLevelClient().snapshot()::getRepositoryAsync);
+                highLevelClient().snapshot()::getRepositoryAsync);
         assertThat(2, equalTo(response.repositories().size()));
     }
 
@@ -98,11 +100,11 @@ public void testSnapshotGetRepositoriesNonExistent() {
         String repository = "doesnotexist";
         GetRepositoriesRequest request = new GetRepositoriesRequest(new String[]{repository});
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(request,
-            highLevelClient().snapshot()::getRepository, highLevelClient().snapshot()::getRepositoryAsync));
+                highLevelClient().snapshot()::getRepository, highLevelClient().snapshot()::getRepositoryAsync));
 
         assertThat(exception.status(), equalTo(RestStatus.NOT_FOUND));
         assertThat(exception.getMessage(), equalTo(
-            "Elasticsearch exception [type=repository_missing_exception, reason=[" + repository + "] missing]"));
+                "Elasticsearch exception [type=repository_missing_exception, reason=[" + repository + "] missing]"));
     }
 
     public void testSnapshotDeleteRepository() throws IOException {
@@ -111,12 +113,12 @@ public void testSnapshotDeleteRepository() throws IOException {
 
         GetRepositoriesRequest request = new GetRepositoriesRequest();
         GetRepositoriesResponse response = execute(request, highLevelClient().snapshot()::getRepository,
-            highLevelClient().snapshot()::getRepositoryAsync);
+                highLevelClient().snapshot()::getRepositoryAsync);
         assertThat(1, equalTo(response.repositories().size()));
 
         DeleteRepositoryRequest deleteRequest = new DeleteRepositoryRequest(repository);
         AcknowledgedResponse deleteResponse = execute(deleteRequest, highLevelClient().snapshot()::deleteRepository,
-            highLevelClient().snapshot()::deleteRepositoryAsync);
+                highLevelClient().snapshot()::deleteRepositoryAsync);
 
         assertTrue(deleteResponse.isAcknowledged());
     }
@@ -127,7 +129,7 @@ public void testVerifyRepository() throws IOException {
 
         VerifyRepositoryRequest request = new VerifyRepositoryRequest("test");
         VerifyRepositoryResponse response = execute(request, highLevelClient().snapshot()::verifyRepository,
-            highLevelClient().snapshot()::verifyRepositoryAsync);
+                highLevelClient().snapshot()::verifyRepositoryAsync);
         assertThat(response.getNodes().size(), equalTo(1));
     }
 
@@ -139,6 +141,9 @@ public void testCreateSnapshot() throws IOException {
         CreateSnapshotRequest request = new CreateSnapshotRequest(repository, snapshot);
         boolean waitForCompletion = randomBoolean();
         request.waitForCompletion(waitForCompletion);
+        if (randomBoolean()) {
+            request.userMetadata(randomUserMetadata());
+        }
         request.partial(randomBoolean());
         request.includeGlobalState(randomBoolean());
 
@@ -147,47 +152,59 @@ public void testCreateSnapshot() throws IOException {
         if (waitForCompletion == false) {
             // If we don't wait for the snapshot to complete we have to cancel it to not leak the snapshot task
             AcknowledgedResponse deleteResponse = execute(
-                new DeleteSnapshotRequest(repository, snapshot),
-                highLevelClient().snapshot()::delete, highLevelClient().snapshot()::deleteAsync
+                    new DeleteSnapshotRequest(repository, snapshot),
+                    highLevelClient().snapshot()::delete, highLevelClient().snapshot()::deleteAsync
             );
             assertTrue(deleteResponse.isAcknowledged());
         }
     }
 
     public void testGetSnapshots() throws IOException {
-        String repository = "test_repository";
+        String repository1 = "test_repository1";
+        String repository2 = "test_repository2";
         String snapshot1 = "test_snapshot1";
         String snapshot2 = "test_snapshot2";
 
-        AcknowledgedResponse putRepositoryResponse = createTestRepository(repository, FsRepository.TYPE, "{\"location\": \".\"}");
+        AcknowledgedResponse putRepositoryResponse =
+                createTestRepository(repository1, FsRepository.TYPE, "{\"location\": \"loc1\"}");
         assertTrue(putRepositoryResponse.isAcknowledged());
 
-        CreateSnapshotRequest createSnapshotRequest1 = new CreateSnapshotRequest(repository, snapshot1);
+        AcknowledgedResponse putRepositoryResponse2 =
+                createTestRepository(repository2, FsRepository.TYPE, "{\"location\": \"loc2\"}");
+        assertTrue(putRepositoryResponse2.isAcknowledged());
+
+        CreateSnapshotRequest createSnapshotRequest1 = new CreateSnapshotRequest(repository1, snapshot1);
         createSnapshotRequest1.waitForCompletion(true);
         CreateSnapshotResponse putSnapshotResponse1 = createTestSnapshot(createSnapshotRequest1);
-        CreateSnapshotRequest createSnapshotRequest2 = new CreateSnapshotRequest(repository, snapshot2);
+        CreateSnapshotRequest createSnapshotRequest2 = new CreateSnapshotRequest(repository2, snapshot2);
         createSnapshotRequest2.waitForCompletion(true);
+        Map<String, Object> originalMetadata = randomUserMetadata();
+        createSnapshotRequest2.userMetadata(originalMetadata);
         CreateSnapshotResponse putSnapshotResponse2 = createTestSnapshot(createSnapshotRequest2);
         // check that the request went ok without parsing JSON here. When using the high level client, check acknowledgement instead.
         assertEquals(RestStatus.OK, putSnapshotResponse1.status());
         assertEquals(RestStatus.OK, putSnapshotResponse2.status());
 
-        GetSnapshotsRequest request;
-        if (randomBoolean()) {
-            request = new GetSnapshotsRequest(repository);
-        } else if (randomBoolean()) {
-            request = new GetSnapshotsRequest(repository, new String[] {"_all"});
+        GetSnapshotsRequest request = new GetSnapshotsRequest(
+                randomFrom(new String[]{"_all"}, new String[]{"*"}, new String[]{repository1, repository2}),
+                randomFrom(new String[]{"_all"}, new String[]{"*"}, new String[]{snapshot1, snapshot2})
+        );
+        request.ignoreUnavailable(true);
 
-        } else {
-            request = new GetSnapshotsRequest(repository, new String[] {snapshot1, snapshot2});
-        }
         GetSnapshotsResponse response = execute(request, highLevelClient().snapshot()::get, highLevelClient().snapshot()::getAsync);
 
-        assertEquals(2, response.getSnapshots().size());
-        assertThat(response.getSnapshots().stream().map((s) -> s.snapshotId().getName()).collect(Collectors.toList()),
-            contains("test_snapshot1", "test_snapshot2"));
+        assertThat(response.isFailed(), is(false));
+        assertThat(response.getRepositories(), equalTo(Sets.newSet(repository1, repository2)));
+
+        assertThat(response.getSnapshots(repository1), hasSize(1));
+        assertThat(response.getSnapshots(repository1).get(0).snapshotId().getName(), equalTo(snapshot1));
+
+        assertThat(response.getSnapshots(repository2), hasSize(1));
+        assertThat(response.getSnapshots(repository2).get(0).snapshotId().getName(), equalTo(snapshot2));
+        assertThat(response.getSnapshots(repository2).get(0).userMetadata(), equalTo(originalMetadata));
     }
 
+
     public void testSnapshotsStatus() throws IOException {
         String testRepository = "test";
         String testSnapshot = "snapshot";
@@ -209,7 +226,7 @@ public void testSnapshotsStatus() throws IOException {
         request.repository(testRepository);
         request.snapshots(new String[]{testSnapshot});
         SnapshotsStatusResponse response = execute(request, highLevelClient().snapshot()::status,
-            highLevelClient().snapshot()::statusAsync);
+                highLevelClient().snapshot()::statusAsync);
         assertThat(response.getSnapshots().size(), equalTo(1));
         assertThat(response.getSnapshots().get(0).getSnapshot().getRepository(), equalTo(testRepository));
         assertThat(response.getSnapshots().get(0).getSnapshot().getSnapshotId().getName(), equalTo(testSnapshot));
@@ -231,6 +248,9 @@ public void testRestoreSnapshot() throws IOException {
         CreateSnapshotRequest createSnapshotRequest = new CreateSnapshotRequest(testRepository, testSnapshot);
         createSnapshotRequest.indices(testIndex);
         createSnapshotRequest.waitForCompletion(true);
+        if (randomBoolean()) {
+            createSnapshotRequest.userMetadata(randomUserMetadata());
+        }
         CreateSnapshotResponse createSnapshotResponse = createTestSnapshot(createSnapshotRequest);
         assertEquals(RestStatus.OK, createSnapshotResponse.status());
 
@@ -243,7 +263,7 @@ public void testRestoreSnapshot() throws IOException {
         request.renameReplacement(restoredIndex);
 
         RestoreSnapshotResponse response = execute(request, highLevelClient().snapshot()::restore,
-            highLevelClient().snapshot()::restoreAsync);
+                highLevelClient().snapshot()::restoreAsync);
 
         RestoreInfo restoreInfo = response.getRestoreInfo();
         assertThat(restoreInfo.name(), equalTo(testSnapshot));
@@ -261,6 +281,9 @@ public void testDeleteSnapshot() throws IOException {
 
         CreateSnapshotRequest createSnapshotRequest = new CreateSnapshotRequest(repository, snapshot);
         createSnapshotRequest.waitForCompletion(true);
+        if (randomBoolean()) {
+            createSnapshotRequest.userMetadata(randomUserMetadata());
+        }
         CreateSnapshotResponse createSnapshotResponse = createTestSnapshot(createSnapshotRequest);
         // check that the request went ok without parsing JSON here. When using the high level client, check acknowledgement instead.
         assertEquals(RestStatus.OK, createSnapshotResponse.status());
@@ -270,4 +293,28 @@ public void testDeleteSnapshot() throws IOException {
 
         assertTrue(response.isAcknowledged());
     }
-}
+
+    private static Map<String, Object> randomUserMetadata() {
+        if (randomBoolean()) {
+            return null;
+        }
+
+        Map<String, Object> metadata = new HashMap<>();
+        long fields = randomLongBetween(0, 4);
+        for (int i = 0; i < fields; i++) {
+            if (randomBoolean()) {
+                metadata.put(randomValueOtherThanMany(metadata::containsKey, () -> randomAlphaOfLengthBetween(2,10)),
+                        randomAlphaOfLengthBetween(5, 5));
+            } else {
+                Map<String, Object> nested = new HashMap<>();
+                long nestedFields = randomLongBetween(0, 4);
+                for (int j = 0; j < nestedFields; j++) {
+                    nested.put(randomValueOtherThanMany(nested::containsKey, () -> randomAlphaOfLengthBetween(2,10)),
+                            randomAlphaOfLengthBetween(5, 5));
+                }
+                metadata.put(randomValueOtherThanMany(metadata::containsKey, () -> randomAlphaOfLengthBetween(2,10)), nested);
+            }
+        }
+        return metadata;
+    }
+}
\ No newline at end of file
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java
index 66720b70ee3a6..23789d390357b 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java
@@ -41,7 +41,6 @@
 
 import java.io.IOException;
 import java.nio.file.Path;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
@@ -78,7 +77,8 @@ public void testCreateRepository() throws IOException {
         Path repositoryLocation = PathUtils.get(".");
         PutRepositoryRequest putRepositoryRequest = new PutRepositoryRequest(repository);
         putRepositoryRequest.type(FsRepository.TYPE);
-        putRepositoryRequest.verify(randomBoolean());
+        final boolean verify = randomBoolean();
+        putRepositoryRequest.verify(verify);
 
         putRepositoryRequest.settings(
             Settings.builder()
@@ -90,6 +90,11 @@ public void testCreateRepository() throws IOException {
         Request request = SnapshotRequestConverters.createRepository(putRepositoryRequest);
         assertThat(request.getEndpoint(), equalTo(endpoint));
         assertThat(request.getMethod(), equalTo(HttpPut.METHOD_NAME));
+        if (verify) {
+            assertThat(request.getParameters().get("verify"), nullValue());
+        } else {
+            assertThat(request.getParameters().get("verify"), equalTo("false"));
+        }
         RequestConvertersTests.assertToXContentBody(putRepositoryRequest, request.getEntity());
     }
 
@@ -148,15 +153,16 @@ public void testCreateSnapshot() throws IOException {
 
     public void testGetSnapshots() {
         Map<String, String> expectedParams = new HashMap<>();
-        String repository = RequestConvertersTests.randomIndicesNames(1, 1)[0];
+        String repository1 = randomAlphaOfLength(10);
+        String repository2 = randomAlphaOfLength(10);
         String snapshot1 = "snapshot1-" + randomAlphaOfLengthBetween(2, 5).toLowerCase(Locale.ROOT);
         String snapshot2 = "snapshot2-" + randomAlphaOfLengthBetween(2, 5).toLowerCase(Locale.ROOT);
 
-        String endpoint = String.format(Locale.ROOT, "/_snapshot/%s/%s,%s", repository, snapshot1, snapshot2);
+        String endpoint = String.format(Locale.ROOT, "/_snapshot/%s,%s/%s,%s", repository1, repository2, snapshot1, snapshot2);
 
         GetSnapshotsRequest getSnapshotsRequest = new GetSnapshotsRequest();
-        getSnapshotsRequest.repository(repository);
-        getSnapshotsRequest.snapshots(Arrays.asList(snapshot1, snapshot2).toArray(new String[0]));
+        getSnapshotsRequest.repositories(repository1, repository2);
+        getSnapshotsRequest.snapshots(new String[]{snapshot1, snapshot2});
         RequestConvertersTests.setRandomMasterTimeout(getSnapshotsRequest, expectedParams);
 
         if (randomBoolean()) {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/XPackInfoResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/XPackInfoResponseTests.java
index 052e700a59f1a..59a7a830a6304 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/XPackInfoResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/XPackInfoResponseTests.java
@@ -70,8 +70,7 @@ private LicenseInfo convertHlrcToInternal(org.elasticsearch.client.xpack.XPackIn
     private FeatureSetsInfo convertHlrcToInternal(org.elasticsearch.client.xpack.XPackInfoResponse.FeatureSetsInfo featureSetsInfo) {
         return featureSetsInfo != null
             ? new FeatureSetsInfo(featureSetsInfo.getFeatureSets().values().stream()
-            .map(fs -> new FeatureSet(fs.name(), fs.description(), fs.available(), fs.enabled(),
-                fs.nativeCodeInfo()))
+            .map(fs -> new FeatureSet(fs.name(), fs.available(), fs.enabled()))
             .collect(Collectors.toSet()))
             : null;
     }
@@ -169,21 +168,7 @@ private FeatureSetsInfo mutateFeatureSetsInfo(FeatureSetsInfo featureSetsInfo) {
     private FeatureSet randomFeatureSet() {
         return new FeatureSet(
             randomAlphaOfLength(5),
-            randomBoolean() ? null : randomAlphaOfLength(20),
             randomBoolean(),
-            randomBoolean(),
-            randomNativeCodeInfo());
-    }
-
-    private Map<String, Object> randomNativeCodeInfo() {
-        if (randomBoolean()) {
-            return null;
-        }
-        int size = between(0, 10);
-        Map<String, Object> nativeCodeInfo = new HashMap<>(size);
-        while (nativeCodeInfo.size() < size) {
-            nativeCodeInfo.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
-        }
-        return nativeCodeInfo;
+            randomBoolean());
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/CcrStatsResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/CcrStatsResponseTests.java
index d56b762520c55..eaf6103a0ecfe 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/CcrStatsResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/CcrStatsResponseTests.java
@@ -106,6 +106,7 @@ static FollowStatsAction.StatsResponses createStatsResponse() {
                 randomNonNegativeLong(),
                 randomNonNegativeLong(),
                 randomNonNegativeLong(),
+                randomNonNegativeLong(),
                 Collections.emptyNavigableMap(),
                 randomLong(),
                 randomBoolean() ? new ElasticsearchException("fatal error") : null);
@@ -190,6 +191,8 @@ protected void assertInstances(CcrStatsAction.Response serverTestInstance, CcrSt
                         equalTo(expectedShardFollowStats.followerMappingVersion()));
                     assertThat(actualShardFollowStats.getFollowerSettingsVersion(),
                         equalTo(expectedShardFollowStats.followerSettingsVersion()));
+                    assertThat(actualShardFollowStats.getFollowerAliasesVersion(),
+                            equalTo(expectedShardFollowStats.followerAliasesVersion()));
                     assertThat(actualShardFollowStats.getTotalReadTimeMillis(),
                         equalTo(expectedShardFollowStats.totalReadTimeMillis()));
                     assertThat(actualShardFollowStats.getSuccessfulReadRequests(),
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/FollowStatsResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/FollowStatsResponseTests.java
index cd7257342c724..ff93c8df33eda 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/FollowStatsResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/FollowStatsResponseTests.java
@@ -93,6 +93,8 @@ protected void assertInstances(FollowStatsAction.StatsResponses serverTestInstan
                     equalTo(expectedShardFollowStats.followerMappingVersion()));
                 assertThat(actualShardFollowStats.getFollowerSettingsVersion(),
                     equalTo(expectedShardFollowStats.followerSettingsVersion()));
+                assertThat(actualShardFollowStats.getFollowerAliasesVersion(),
+                        equalTo(expectedShardFollowStats.followerAliasesVersion()));
                 assertThat(actualShardFollowStats.getTotalReadTimeMillis(),
                     equalTo(expectedShardFollowStats.totalReadTimeMillis()));
                 assertThat(actualShardFollowStats.getSuccessfulReadRequests(),
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/core/TermVectorsResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/core/TermVectorsResponseTests.java
index 714a7269a19d9..debe08879339d 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/core/TermVectorsResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/core/TermVectorsResponseTests.java
@@ -22,11 +22,11 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.ArrayList;
-import java.util.List;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.List;
 
 import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
 
@@ -47,7 +47,6 @@ public void testFromXContent() throws IOException {
     static void toXContent(TermVectorsResponse response, XContentBuilder builder) throws IOException {
         builder.startObject();
         builder.field("_index", response.getIndex());
-        builder.field("_type", response.getType());
         if (response.getId() != null) {
             builder.field("_id", response.getId());
         }
@@ -119,7 +118,6 @@ private static void toXContent(TermVectorsResponse.TermVector tv, XContentBuilde
 
     static TermVectorsResponse createTestInstance() {
         String index = randomAlphaOfLength(5);
-        String type = randomAlphaOfLength(5);
         String id = String.valueOf(randomIntBetween(1,100));
         long version = randomNonNegativeLong();
         long tookInMillis = randomNonNegativeLong();
@@ -142,7 +140,7 @@ static TermVectorsResponse createTestInstance() {
                     fieldName, hasFieldStatistics, hasTermStatistics, hasScores, hasOffsets, hasPositions, hasPayloads));
             }
         }
-        TermVectorsResponse tvresponse = new TermVectorsResponse(index, type, id, version, found, tookInMillis, tvList);
+        TermVectorsResponse tvresponse = new TermVectorsResponse(index, id, version, found, tookInMillis, tvList);
         return tvresponse;
     }
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/GetDataFrameTransformResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/GetDataFrameTransformResponseTests.java
index f7386e936301b..c37e8f8997185 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/GetDataFrameTransformResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/GetDataFrameTransformResponseTests.java
@@ -78,7 +78,10 @@ private static void toXContent(GetDataFrameTransformResponse response, XContentB
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
-        return new NamedXContentRegistry(searchModule.getNamedXContents());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
+        List<NamedXContentRegistry.Entry> namedXContents = searchModule.getNamedXContents();
+        namedXContents.addAll(new DataFrameNamedXContentProvider().getNamedXContentParsers());
+
+        return new NamedXContentRegistry(namedXContents);
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformRequestTests.java
index c91e1cbb1dd91..d335e6a497a8e 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformRequestTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformRequestTests.java
@@ -31,6 +31,7 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Optional;
 
 import static org.elasticsearch.client.dataframe.transforms.SourceConfigTests.randomSourceConfig;
@@ -54,8 +55,11 @@ protected boolean supportsUnknownFields() {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
-        return new NamedXContentRegistry(searchModule.getNamedXContents());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
+        List<NamedXContentRegistry.Entry> namedXContents = searchModule.getNamedXContents();
+        namedXContents.addAll(new DataFrameNamedXContentProvider().getNamedXContentParsers());
+
+        return new NamedXContentRegistry(namedXContents);
     }
 
     public void testValidate() {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponseTests.java
index 31f1a26d6f1d0..c0e4cfb7eeb31 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformResponseTests.java
@@ -53,8 +53,13 @@ private PreviewDataFrameTransformResponse createTestInstance() {
             }
             docs.add(doc);
         }
+        int numMappingEntries = randomIntBetween(5, 10);
+        Map<String, Object> mappings = new HashMap<>(numMappingEntries);
+        for (int i = 0; i < numMappingEntries; i++) {
+            mappings.put(randomAlphaOfLength(10), Map.of("type", randomAlphaOfLength(10)));
+        }
 
-        return new PreviewDataFrameTransformResponse(docs);
+        return new PreviewDataFrameTransformResponse(docs, mappings);
     }
 
     private void toXContent(PreviewDataFrameTransformResponse response, XContentBuilder builder) throws IOException {
@@ -64,6 +69,7 @@ private void toXContent(PreviewDataFrameTransformResponse response, XContentBuil
             builder.map(doc);
         }
         builder.endArray();
+        builder.field("mappings", response.getMappings());
         builder.endObject();
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PutDataFrameTransformRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PutDataFrameTransformRequestTests.java
index 28fd92dcf913f..01e1db2cb3823 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PutDataFrameTransformRequestTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PutDataFrameTransformRequestTests.java
@@ -31,6 +31,7 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Optional;
 
 import static org.hamcrest.Matchers.containsString;
@@ -70,7 +71,10 @@ protected boolean supportsUnknownFields() {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
-        return new NamedXContentRegistry(searchModule.getNamedXContents());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
+        List<NamedXContentRegistry.Entry> namedXContents = searchModule.getNamedXContents();
+        namedXContents.addAll(new DataFrameNamedXContentProvider().getNamedXContentParsers());
+
+        return new NamedXContentRegistry(namedXContents);
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java
index 1b5228d96229f..79b7e85098e04 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.client.dataframe.transforms;
 
+import org.elasticsearch.client.dataframe.DataFrameNamedXContentProvider;
+import org.elasticsearch.Version;
 import org.elasticsearch.client.dataframe.transforms.pivot.PivotConfigTests;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -27,7 +29,9 @@
 import org.elasticsearch.test.AbstractXContentTestCase;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.Collections;
+import java.util.List;
 import java.util.function.Predicate;
 
 import static org.elasticsearch.client.dataframe.transforms.DestConfigTests.randomDestConfig;
@@ -36,8 +40,18 @@
 public class DataFrameTransformConfigTests extends AbstractXContentTestCase<DataFrameTransformConfig> {
 
     public static DataFrameTransformConfig randomDataFrameTransformConfig() {
-        return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomSourceConfig(),
-                randomDestConfig(), PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100));
+        return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10),
+            randomSourceConfig(),
+            randomDestConfig(),
+            randomBoolean() ? null : randomSyncConfig(), 
+            PivotConfigTests.randomPivotConfig(),
+            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100),
+            randomBoolean() ? null : Instant.now(),
+            randomBoolean() ? null : Version.CURRENT.toString());
+    }
+
+    public static SyncConfig randomSyncConfig() {
+        return TimeSyncConfigTests.randomTimeSyncConfig();
     }
 
     @Override
@@ -63,7 +77,10 @@ protected Predicate<String> getRandomFieldsExcludeFilter() {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
-        return new NamedXContentRegistry(searchModule.getNamedXContents());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
+        List<NamedXContentRegistry.Entry> namedXContents = searchModule.getNamedXContents();
+        namedXContents.addAll(new DataFrameNamedXContentProvider().getNamedXContentParsers());
+
+        return new NamedXContentRegistry(namedXContents);
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateTests.java
index 4ada50c20d219..ebb62890c3cdd 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformStateTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.client.dataframe.transforms;
 
 import org.elasticsearch.client.core.IndexerState;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.test.ESTestCase;
 
@@ -37,7 +38,8 @@ public void testFromXContent() throws IOException {
                 DataFrameTransformStateTests::toXContent,
                 DataFrameTransformState::fromXContent)
                 .supportsUnknownFields(true)
-                .randomFieldsExcludeFilter(field -> field.equals("current_position"))
+                .randomFieldsExcludeFilter(field -> field.equals("current_position") ||
+                    field.equals("node.attributes"))
                 .test();
     }
 
@@ -47,7 +49,8 @@ public static DataFrameTransformState randomDataFrameTransformState() {
             randomPositionMap(),
             randomLongBetween(0,10),
             randomBoolean() ? null : randomAlphaOfLength(10),
-            randomBoolean() ? null : DataFrameTransformProgressTests.randomInstance());
+            randomBoolean() ? null : DataFrameTransformProgressTests.randomInstance(),
+            randomBoolean() ? null : NodeAttributesTests.createRandom());
     }
 
     public static void toXContent(DataFrameTransformState state, XContentBuilder builder) throws IOException {
@@ -65,6 +68,10 @@ public static void toXContent(DataFrameTransformState state, XContentBuilder bui
             builder.field("progress");
             DataFrameTransformProgressTests.toXContent(state.getProgress(), builder);
         }
+        if (state.getNode() != null) {
+            builder.field("node");
+            state.getNode().toXContent(builder, ToXContent.EMPTY_PARAMS);
+        }
         builder.endObject();
     }
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DestConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DestConfigTests.java
index f2950b64cf7c9..0dc8f99d7631b 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DestConfigTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DestConfigTests.java
@@ -27,7 +27,8 @@
 public class DestConfigTests extends AbstractXContentTestCase<DestConfig> {
 
     public static DestConfig randomDestConfig() {
-        return new DestConfig(randomAlphaOfLength(10));
+        return new DestConfig(randomAlphaOfLength(10),
+            randomBoolean() ? null : randomAlphaOfLength(10));
     }
 
     @Override
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/NodeAttributesTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/NodeAttributesTests.java
new file mode 100644
index 0000000000000..661aa9f7a30a4
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/NodeAttributesTests.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.dataframe.transforms;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Predicate;
+
+public class NodeAttributesTests extends AbstractXContentTestCase<NodeAttributes> {
+
+    public static NodeAttributes createRandom() {
+        int numberOfAttributes = randomIntBetween(1, 10);
+        Map<String, String> attributes = new HashMap<>(numberOfAttributes);
+        for(int i = 0; i < numberOfAttributes; i++) {
+            String val = randomAlphaOfLength(10);
+            attributes.put("key-"+i, val);
+        }
+        return new NodeAttributes(randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            attributes);
+    }
+
+    @Override
+    protected NodeAttributes createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected NodeAttributes doParseInstance(XContentParser parser) throws IOException {
+        return NodeAttributes.PARSER.parse(parser, null);
+    }
+
+    @Override
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        return field -> !field.isEmpty();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/QueryConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/QueryConfigTests.java
index 644858cab286f..bfa14d4a27a90 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/QueryConfigTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/QueryConfigTests.java
@@ -56,7 +56,7 @@ protected boolean supportsUnknownFields() {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/SourceConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/SourceConfigTests.java
index 722c265c9a05e..e9e5548e93938 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/SourceConfigTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/SourceConfigTests.java
@@ -62,7 +62,7 @@ protected SourceConfig createTestInstance() {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TypesExistsResponse.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/TimeSyncConfigTests.java
similarity index 52%
rename from server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TypesExistsResponse.java
rename to client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/TimeSyncConfigTests.java
index c492c392933e0..dd2a17eb0260d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TypesExistsResponse.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/TimeSyncConfigTests.java
@@ -16,41 +16,34 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.action.admin.indices.exists.types;
 
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
+package org.elasticsearch.client.dataframe.transforms;
 
-import java.io.IOException;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
 
-/**
- * Whether all of the existed types exist.
- */
-public class TypesExistsResponse extends ActionResponse {
+import java.io.IOException;
 
-    private boolean exists;
+public class TimeSyncConfigTests extends AbstractXContentTestCase<TimeSyncConfig> {
 
-    TypesExistsResponse() {
+    public static TimeSyncConfig randomTimeSyncConfig() {
+        return new TimeSyncConfig(randomAlphaOfLengthBetween(1, 10), new TimeValue(randomNonNegativeLong()));
     }
 
-    public TypesExistsResponse(boolean exists) {
-        this.exists = exists;
-    }
-
-    public boolean isExists() {
-        return this.exists;
+    @Override
+    protected TimeSyncConfig createTestInstance() {
+        return randomTimeSyncConfig();
     }
 
     @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        exists = in.readBoolean();
+    protected TimeSyncConfig doParseInstance(XContentParser parser) throws IOException {
+        return TimeSyncConfig.fromXContent(parser);
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeBoolean(exists);
+    protected boolean supportsUnknownFields() {
+        return true;
     }
-}
\ No newline at end of file
+
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsTests.java
index ad08881fb5641..dde44898bf90b 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsTests.java
@@ -64,7 +64,7 @@ protected boolean supportsUnknownFields() {
 
     @Override
     protected Predicate<String> getRandomFieldsExcludeFilter() {
-        return field -> field.equals("state.current_position");
+        return field -> field.equals("state.current_position") || field.equals("state.node") || field.equals("state.node.attributes");
     }
 }
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateTests.java
index 4c80365bc539a..b97e0a72c1fa2 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/DataFrameTransformStateTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformState;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformTaskState;
+import org.elasticsearch.xpack.core.dataframe.transforms.NodeAttributes;
 import org.elasticsearch.xpack.core.indexing.IndexerState;
 
 import java.io.IOException;
@@ -40,8 +41,20 @@ public class DataFrameTransformStateTests extends AbstractHlrcXContentTestCase<D
 
     public static DataFrameTransformState fromHlrc(org.elasticsearch.client.dataframe.transforms.DataFrameTransformState instance) {
         return new DataFrameTransformState(DataFrameTransformTaskState.fromString(instance.getTaskState().value()),
-                IndexerState.fromString(instance.getIndexerState().value()), instance.getPosition(), instance.getCheckpoint(),
-                instance.getReason(), DataFrameTransformProgressTests.fromHlrc(instance.getProgress()));
+            IndexerState.fromString(instance.getIndexerState().value()),
+            instance.getPosition(),
+            instance.getCheckpoint(),
+            instance.getReason(),
+            DataFrameTransformProgressTests.fromHlrc(instance.getProgress()),
+            fromHlrc(instance.getNode()));
+    }
+
+    public static NodeAttributes fromHlrc(org.elasticsearch.client.dataframe.transforms.NodeAttributes attributes) {
+        return attributes == null ? null : new NodeAttributes(attributes.getId(),
+            attributes.getName(),
+            attributes.getEphemeralId(),
+            attributes.getTransportAddress(),
+            attributes.getAttributes());
     }
 
     @Override
@@ -72,7 +85,7 @@ protected boolean supportsUnknownFields() {
 
     @Override
     protected Predicate<String> getRandomFieldsExcludeFilter() {
-        return field -> field.equals("current_position");
+        return field -> field.equals("current_position") || field.equals("node.attributes");
     }
 
     public static DataFrameTransformStateAndStats randomDataFrameTransformStateAndStats(String id) {
@@ -97,6 +110,20 @@ public static DataFrameTransformProgress randomDataFrameTransformProgress() {
         return new DataFrameTransformProgress(totalDocs, remainingDocs);
     }
 
+    public static NodeAttributes randomNodeAttributes() {
+        int numberOfAttributes = randomIntBetween(1, 10);
+        Map<String, String> attributes = new HashMap<>(numberOfAttributes);
+        for(int i = 0; i < numberOfAttributes; i++) {
+            String val = randomAlphaOfLength(10);
+            attributes.put("key-"+i, val);
+        }
+        return new NodeAttributes(randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            attributes);
+    }
+
     public static DataFrameIndexerTransformStats randomStats(String transformId) {
         return new DataFrameIndexerTransformStats(transformId, randomLongBetween(10L, 10000L),
             randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L),
@@ -110,7 +137,8 @@ public static DataFrameTransformState randomDataFrameTransformState() {
             randomPosition(),
             randomLongBetween(0,10),
             randomBoolean() ? null : randomAlphaOfLength(10),
-            randomBoolean() ? null : randomDataFrameTransformProgress());
+            randomBoolean() ? null : randomDataFrameTransformProgress(),
+            randomBoolean() ? null : randomNodeAttributes());
     }
 
     private static Map<String, Object> randomPosition() {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/TimeSyncConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/TimeSyncConfigTests.java
new file mode 100644
index 0000000000000..0c6a0350882a4
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/TimeSyncConfigTests.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.dataframe.transforms.hlrc;
+
+import org.elasticsearch.client.AbstractResponseTestCase;
+import org.elasticsearch.client.dataframe.transforms.TimeSyncConfig;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+
+public class TimeSyncConfigTests
+        extends AbstractResponseTestCase<org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig, TimeSyncConfig> {
+
+    public static org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig randomTimeSyncConfig() {
+        return new org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig(randomAlphaOfLengthBetween(1, 10),
+                new TimeValue(randomNonNegativeLong()));
+    }
+
+    public static void assertHlrcEquals(org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig serverTestInstance,
+            TimeSyncConfig clientInstance) {
+        assertEquals(serverTestInstance.getField(), clientInstance.getField());
+        assertEquals(serverTestInstance.getDelay(), clientInstance.getDelay());
+    }
+
+    @Override
+    protected org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig createServerTestInstance() {
+        return randomTimeSyncConfig();
+    }
+
+    @Override
+    protected TimeSyncConfig doParseToClientInstance(XContentParser parser) throws IOException {
+        return TimeSyncConfig.fromXContent(parser);
+    }
+
+    @Override
+    protected void assertInstances(org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig serverTestInstance,
+            TimeSyncConfig clientInstance) {
+        assertHlrcEquals(serverTestInstance, clientInstance);
+    }
+
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/pivot/AggregationConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/pivot/AggregationConfigTests.java
index 210dc59329c02..2408f766b8af8 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/pivot/AggregationConfigTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/pivot/AggregationConfigTests.java
@@ -66,7 +66,7 @@ protected boolean supportsUnknownFields() {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/pivot/DateHistogramGroupSourceTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/pivot/DateHistogramGroupSourceTests.java
index 32605f5c286ad..ece1c4fb743ee 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/pivot/DateHistogramGroupSourceTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/pivot/DateHistogramGroupSourceTests.java
@@ -39,7 +39,6 @@ public static DateHistogramGroupSource randomDateHistogramGroupSource() {
         String field = randomAlphaOfLengthBetween(1, 20);
         return new DateHistogramGroupSource(field,
                 randomDateHistogramInterval(),
-                randomBoolean() ? randomAlphaOfLength(10) : null,
                 randomBoolean() ? randomZone() : null);
     }
 
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/pivot/PivotConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/pivot/PivotConfigTests.java
index 5cafcb9f419b5..a433000f62d1f 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/pivot/PivotConfigTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/pivot/PivotConfigTests.java
@@ -60,7 +60,7 @@ protected Predicate<String> getRandomFieldsExcludeFilter() {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/pivot/hlrc/DateHistogramGroupSourceTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/pivot/hlrc/DateHistogramGroupSourceTests.java
index dc31004607dcd..fd98e52a1527e 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/pivot/hlrc/DateHistogramGroupSourceTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/pivot/hlrc/DateHistogramGroupSourceTests.java
@@ -44,9 +44,6 @@ public static DateHistogramGroupSource randomDateHistogramGroupSource() {
         if (randomBoolean()) {
             dateHistogramGroupSource.setTimeZone(randomZone());
         }
-        if (randomBoolean()) {
-            dateHistogramGroupSource.setFormat(randomAlphaOfLength(10));
-        }
         return dateHistogramGroupSource;
     }
 
@@ -64,7 +61,6 @@ protected org.elasticsearch.client.dataframe.transforms.pivot.DateHistogramGroup
     protected void assertInstances(DateHistogramGroupSource serverTestInstance,
             org.elasticsearch.client.dataframe.transforms.pivot.DateHistogramGroupSource clientInstance) {
         assertThat(serverTestInstance.getField(), equalTo(clientInstance.getField()));
-        assertThat(serverTestInstance.getFormat(), equalTo(clientInstance.getFormat()));
         assertSameInterval(serverTestInstance.getInterval(), clientInstance.getInterval());
         assertThat(serverTestInstance.getTimeZone(), equalTo(clientInstance.getTimeZone()));
         assertThat(serverTestInstance.getType().name(), equalTo(clientInstance.getType().name()));
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java
index fe003d691a830..062f90c0eb4db 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java
@@ -703,7 +703,7 @@ public void testBulk() throws Exception {
             request.add(new IndexRequest("posts").id("4")  // <3>
                     .source(XContentType.JSON,"field", "baz"));
             // end::bulk-request-with-mixed-operations
-            BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); 
+            BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
             assertSame(RestStatus.OK, bulkResponse.status());
             assertFalse(bulkResponse.hasFailures());
 
@@ -824,9 +824,9 @@ public void testReindex() throws Exception {
             // tag::reindex-request-conflicts
             request.setConflicts("proceed"); // <1>
             // end::reindex-request-conflicts
-            // tag::reindex-request-size
-            request.setSize(10); // <1>
-            // end::reindex-request-size
+            // tag::reindex-request-maxDocs
+            request.setMaxDocs(10); // <1>
+            // end::reindex-request-maxDocs
             // tag::reindex-request-sourceSize
             request.setSourceBatchSize(100); // <1>
             // end::reindex-request-sourceSize
@@ -1026,9 +1026,9 @@ public void testUpdateByQuery() throws Exception {
             // tag::update-by-query-request-query
             request.setQuery(new TermQueryBuilder("user", "kimchy")); // <1>
             // end::update-by-query-request-query
-            // tag::update-by-query-request-size
-            request.setSize(10); // <1>
-            // end::update-by-query-request-size
+            // tag::update-by-query-request-maxDocs
+            request.setMaxDocs(10); // <1>
+            // end::update-by-query-request-maxDocs
             // tag::update-by-query-request-scrollSize
             request.setBatchSize(100); // <1>
             // end::update-by-query-request-scrollSize
@@ -1148,9 +1148,9 @@ public void testDeleteByQuery() throws Exception {
             // tag::delete-by-query-request-query
             request.setQuery(new TermQueryBuilder("user", "kimchy")); // <1>
             // end::delete-by-query-request-query
-            // tag::delete-by-query-request-size
-            request.setSize(10); // <1>
-            // end::delete-by-query-request-size
+            // tag::delete-by-query-request-maxDocs
+            request.setMaxDocs(10); // <1>
+            // end::delete-by-query-request-maxDocs
             // tag::delete-by-query-request-scrollSize
             request.setBatchSize(100); // <1>
             // end::delete-by-query-request-scrollSize
@@ -1614,9 +1614,8 @@ public void testTermVectors() throws Exception {
 
         // tag::term-vectors-response
         String index = response.getIndex(); // <1>
-        String type = response.getType(); // <2>
-        String id = response.getId(); // <3>
-        boolean found = response.getFound(); // <4>
+        String id = response.getId(); // <2>
+        boolean found = response.getFound(); // <3>
         // end::term-vectors-response
 
         if (response.getTermVectorsList() != null) {
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java
index 6f7832cbf3cff..731d42f902c50 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java
@@ -45,6 +45,7 @@
 import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStats;
 import org.elasticsearch.client.dataframe.transforms.DataFrameTransformTaskState;
 import org.elasticsearch.client.dataframe.transforms.DestConfig;
+import org.elasticsearch.client.dataframe.transforms.NodeAttributes;
 import org.elasticsearch.client.dataframe.transforms.QueryConfig;
 import org.elasticsearch.client.dataframe.transforms.SourceConfig;
 import org.elasticsearch.client.dataframe.transforms.pivot.AggregationConfig;
@@ -74,7 +75,7 @@ public class DataFrameTransformDocumentationIT extends ESRestHighLevelClientTest
     private List<String> transformsToClean = new ArrayList<>();
 
     @After
-    public void cleanUpTransforms() throws IOException {
+    public void cleanUpTransforms() throws Exception {
         for (String transformId : transformsToClean) {
             highLevelClient().dataFrame().stopDataFrameTransform(
                     new StopDataFrameTransformRequest(transformId, Boolean.TRUE, TimeValue.timeValueSeconds(20)), RequestOptions.DEFAULT);
@@ -86,6 +87,7 @@ public void cleanUpTransforms() throws IOException {
         }
 
         transformsToClean = new ArrayList<>();
+        waitForPendingTasks(adminClient());
     }
 
     private void createIndex(String indexName) throws IOException {
@@ -124,6 +126,11 @@ public void testPutDataFrameTransform() throws IOException, InterruptedException
             .setIndex("source-index")
             .setQueryConfig(queryConfig).build();
         // end::put-data-frame-transform-source-config
+        // tag::put-data-frame-transform-dest-config
+        DestConfig destConfig = DestConfig.builder()
+            .setIndex("pivot-destination")
+            .setPipeline("my-pipeline").build();
+        // end::put-data-frame-transform-dest-config
         // tag::put-data-frame-transform-group-config
         GroupConfig groupConfig = GroupConfig.builder()
             .groupBy("reviewer", // <1>
@@ -148,7 +155,7 @@ public void testPutDataFrameTransform() throws IOException, InterruptedException
             .builder()
             .setId("reviewer-avg-rating") // <1>
             .setSource(sourceConfig) // <2>
-            .setDest(new DestConfig("pivot-destination")) // <3>
+            .setDest(destConfig) // <3>
             .setPivotConfig(pivotConfig) // <4>
             .setDescription("This is my test transform") // <5>
             .build();
@@ -165,6 +172,7 @@ public void testPutDataFrameTransform() throws IOException, InterruptedException
                     client.dataFrame().putDataFrameTransform(
                             request, RequestOptions.DEFAULT);
             // end::put-data-frame-transform-execute
+            transformsToClean.add(request.getConfig().getId());
 
             assertTrue(response.isAcknowledged());
         }
@@ -202,6 +210,7 @@ public void onFailure(Exception e) {
             // end::put-data-frame-transform-execute-async
 
             assertTrue(latch.await(30L, TimeUnit.SECONDS));
+            transformsToClean.add(request.getConfig().getId());
         }
     }
 
@@ -221,7 +230,7 @@ public void testStartStop() throws IOException, InterruptedException {
         DataFrameTransformConfig transformConfig = DataFrameTransformConfig.builder()
             .setId("mega-transform")
             .setSource(SourceConfig.builder().setIndex("source-data").setQueryConfig(queryConfig).build())
-            .setDest(new DestConfig("pivot-dest"))
+            .setDest(DestConfig.builder().setIndex("pivot-dest").build())
             .setPivotConfig(pivotConfig)
             .build();
 
@@ -244,7 +253,7 @@ public void testStartStop() throws IOException, InterruptedException {
                             request, RequestOptions.DEFAULT);
             // end::start-data-frame-transform-execute
 
-            assertTrue(response.isStarted());
+            assertTrue(response.isAcknowledged());
         }
         {
             // tag::stop-data-frame-transform-request
@@ -255,6 +264,7 @@ public void testStartStop() throws IOException, InterruptedException {
             // tag::stop-data-frame-transform-request-options
             request.setWaitForCompletion(Boolean.TRUE);  // <1>
             request.setTimeout(TimeValue.timeValueSeconds(30));  // <2>
+            request.setAllowNoMatch(true); // <3>
             // end::stop-data-frame-transform-request-options
 
             // tag::stop-data-frame-transform-execute
@@ -263,7 +273,7 @@ public void testStartStop() throws IOException, InterruptedException {
                             request, RequestOptions.DEFAULT);
             // end::stop-data-frame-transform-execute
 
-            assertTrue(response.isStopped());
+            assertTrue(response.isAcknowledged());
         }
         {
             // tag::start-data-frame-transform-execute-listener
@@ -343,7 +353,7 @@ public void testDeleteDataFrameTransform() throws IOException, InterruptedExcept
                 .setIndex("source-data")
                 .setQuery(new MatchAllQueryBuilder())
                 .build())
-            .setDest(new DestConfig("pivot-dest"))
+            .setDest(DestConfig.builder().setIndex("pivot-dest").build())
             .setPivotConfig(pivotConfig)
             .build();
         DataFrameTransformConfig transformConfig2 = DataFrameTransformConfig.builder()
@@ -352,7 +362,7 @@ public void testDeleteDataFrameTransform() throws IOException, InterruptedExcept
                 .setIndex("source-data")
                 .setQuery(new MatchAllQueryBuilder())
                 .build())
-            .setDest(new DestConfig("pivot-dest2"))
+            .setDest(DestConfig.builder().setIndex("pivot-dest2").build())
             .setPivotConfig(pivotConfig)
             .build();
 
@@ -425,6 +435,7 @@ public void testPreview() throws IOException, InterruptedException {
                     .setQueryConfig(queryConfig)
                     .build(), // <1>
                 pivotConfig); // <2>
+
         PreviewDataFrameTransformRequest request =
                 new PreviewDataFrameTransformRequest(transformConfig); // <3>
         // end::preview-data-frame-transform-request
@@ -437,6 +448,7 @@ public void testPreview() throws IOException, InterruptedException {
             // end::preview-data-frame-transform-execute
 
             assertNotNull(response.getDocs());
+            assertNotNull(response.getMappings());
         }
         {
             // tag::preview-data-frame-transform-execute-listener
@@ -472,7 +484,6 @@ public void testGetStats() throws IOException, InterruptedException {
 
         RestHighLevelClient client = highLevelClient();
 
-        QueryConfig queryConfig = new QueryConfig(new MatchAllQueryBuilder());
         GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer",
             TermsGroupSource.builder().setField("user_id").build()).build();
         AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder();
@@ -487,16 +498,22 @@ public void testGetStats() throws IOException, InterruptedException {
                 .setIndex("source-data")
                 .setQuery(new MatchAllQueryBuilder())
                 .build())
-            .setDest(new DestConfig("pivot-dest"))
+            .setDest(DestConfig.builder().setIndex("pivot-dest").build())
             .setPivotConfig(pivotConfig)
             .build();
         client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig), RequestOptions.DEFAULT);
+        transformsToClean.add(id);
 
         // tag::get-data-frame-transform-stats-request
         GetDataFrameTransformStatsRequest request =
                 new GetDataFrameTransformStatsRequest(id); // <1>
         // end::get-data-frame-transform-stats-request
 
+        // tag::get-data-frame-transform-stats-request-options
+        request.setPageParams(new PageParams(0, 100)); // <1>
+        request.setAllowNoMatch(true); // <2>
+        // end::get-data-frame-transform-stats-request-options
+
         {
             // tag::get-data-frame-transform-stats-execute
             GetDataFrameTransformStatsResponse response =
@@ -517,6 +534,8 @@ public void testGetStats() throws IOException, InterruptedException {
                 stateAndStats.getTransformStats();              // <4>
             DataFrameTransformProgress progress =
                 stateAndStats.getTransformState().getProgress(); // <5>
+            NodeAttributes node =
+                stateAndStats.getTransformState().getNode(); // <6>
             // end::get-data-frame-transform-stats-response
 
             assertEquals(IndexerState.STOPPED, indexerState);
@@ -558,7 +577,6 @@ public void onFailure(Exception e) {
     public void testGetDataFrameTransform() throws IOException, InterruptedException {
         createIndex("source-data");
 
-        QueryConfig queryConfig = new QueryConfig(new MatchAllQueryBuilder());
         GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer",
             TermsGroupSource.builder().setField("user_id").build()).build();
         AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder();
@@ -573,7 +591,7 @@ public void testGetDataFrameTransform() throws IOException, InterruptedException
                 .setIndex("source-data")
                 .setQuery(new MatchAllQueryBuilder())
                 .build())
-            .setDest(new DestConfig("pivot-dest"))
+            .setDest(DestConfig.builder().setIndex("pivot-dest").build())
             .setPivotConfig(pivotConfig)
             .build();
 
@@ -589,6 +607,7 @@ public void testGetDataFrameTransform() throws IOException, InterruptedException
 
             // tag::get-data-frame-transform-request-options
             request.setPageParams(new PageParams(0, 100)); // <1>
+            request.setAllowNoMatch(true); // <2>
             // end::get-data-frame-transform-request-options
 
             // tag::get-data-frame-transform-execute
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java
index 14def60b277e8..f878f0f6f7d88 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java
@@ -26,9 +26,6 @@
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
-import org.elasticsearch.action.admin.indices.analyze.DetailAnalyzeResponse;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
@@ -61,9 +58,13 @@
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.client.SyncedFlushResponse;
+import org.elasticsearch.client.core.BroadcastResponse.Shards;
 import org.elasticsearch.client.core.ShardsAcknowledgedResponse;
+import org.elasticsearch.client.indices.AnalyzeRequest;
+import org.elasticsearch.client.indices.AnalyzeResponse;
 import org.elasticsearch.client.indices.CreateIndexRequest;
 import org.elasticsearch.client.indices.CreateIndexResponse;
+import org.elasticsearch.client.indices.DetailAnalyzeResponse;
 import org.elasticsearch.client.indices.FreezeIndexRequest;
 import org.elasticsearch.client.indices.GetFieldMappingsRequest;
 import org.elasticsearch.client.indices.GetFieldMappingsResponse;
@@ -77,6 +78,9 @@
 import org.elasticsearch.client.indices.IndexTemplatesExistRequest;
 import org.elasticsearch.client.indices.PutIndexTemplateRequest;
 import org.elasticsearch.client.indices.PutMappingRequest;
+import org.elasticsearch.client.indices.ReloadAnalyzersRequest;
+import org.elasticsearch.client.indices.ReloadAnalyzersResponse;
+import org.elasticsearch.client.indices.ReloadAnalyzersResponse.ReloadDetails;
 import org.elasticsearch.client.indices.UnfreezeIndexRequest;
 import org.elasticsearch.client.indices.rollover.RolloverRequest;
 import org.elasticsearch.client.indices.rollover.RolloverResponse;
@@ -2418,32 +2422,29 @@ public void testAnalyze() throws IOException, InterruptedException {
 
         {
             // tag::analyze-builtin-request
-            AnalyzeRequest request = new AnalyzeRequest();
-            request.text("Some text to analyze", "Some more text to analyze");  // <1>
-            request.analyzer("english");    // <2>
+            AnalyzeRequest request = AnalyzeRequest.withGlobalAnalyzer("english", // <1>
+                "Some text to analyze", "Some more text to analyze");       // <2>
             // end::analyze-builtin-request
         }
 
         {
             // tag::analyze-custom-request
-            AnalyzeRequest request = new AnalyzeRequest();
-            request.text("<b>Some text to analyze</b>");
-            request.addCharFilter("html_strip");                // <1>
-            request.tokenizer("standard");                      // <2>
-            request.addTokenFilter("lowercase");                // <3>
-
             Map<String, Object> stopFilter = new HashMap<>();
             stopFilter.put("type", "stop");
-            stopFilter.put("stopwords", new String[]{ "to" });  // <4>
-            request.addTokenFilter(stopFilter);                 // <5>
+            stopFilter.put("stopwords", new String[]{ "to" });  // <1>
+            AnalyzeRequest request = AnalyzeRequest.buildCustomAnalyzer("standard")  // <2>
+                .addCharFilter("html_strip")    // <3>
+                .addTokenFilter("lowercase")    // <4>
+                .addTokenFilter(stopFilter)     // <5>
+                .build("<b>Some text to analyze</b>");
             // end::analyze-custom-request
         }
 
         {
             // tag::analyze-custom-normalizer-request
-            AnalyzeRequest request = new AnalyzeRequest();
-            request.text("<b>BaR</b>");
-            request.addTokenFilter("lowercase");
+            AnalyzeRequest request = AnalyzeRequest.buildCustomNormalizer()
+                .addTokenFilter("lowercase")
+                .build("<b>BaR</b>");
             // end::analyze-custom-normalizer-request
 
             // tag::analyze-request-explain
@@ -2484,10 +2485,11 @@ public void testAnalyze() throws IOException, InterruptedException {
 
         {
             // tag::analyze-index-request
-            AnalyzeRequest request = new AnalyzeRequest();
-            request.index("my_index");              // <1>
-            request.analyzer("my_analyzer");        // <2>
-            request.text("some text to analyze");
+            AnalyzeRequest request = AnalyzeRequest.withIndexAnalyzer(
+                "my_index",         // <1>
+                "my_analyzer",        // <2>
+                "some text to analyze"
+            );
             // end::analyze-index-request
 
             // tag::analyze-execute-listener
@@ -2505,10 +2507,7 @@ public void onFailure(Exception e) {
             // end::analyze-execute-listener
 
             // use a built-in analyzer in the test
-            request = new AnalyzeRequest();
-            request.index("my_index");
-            request.field("my_field");
-            request.text("some text to analyze");
+            request = AnalyzeRequest.withField("my_index", "my_field", "some text to analyze");
             // Use a blocking listener in the test
             final CountDownLatch latch = new CountDownLatch(1);
             listener = new LatchedActionListener<>(listener, latch);
@@ -2522,19 +2521,17 @@ public void onFailure(Exception e) {
 
         {
             // tag::analyze-index-normalizer-request
-            AnalyzeRequest request = new AnalyzeRequest();
-            request.index("my_index");                  // <1>
-            request.normalizer("my_normalizer");        // <2>
-            request.text("some text to analyze");
+            AnalyzeRequest request = AnalyzeRequest.withNormalizer(
+                "my_index",             // <1>
+                "my_normalizer",        // <2>
+                "some text to analyze"
+            );
             // end::analyze-index-normalizer-request
         }
 
         {
             // tag::analyze-field-request
-            AnalyzeRequest request = new AnalyzeRequest();
-            request.index("my_index");
-            request.field("my_field");
-            request.text("some text to analyze");
+            AnalyzeRequest request = AnalyzeRequest.withField("my_index", "my_field", "some text to analyze");
             // end::analyze-field-request
         }
 
@@ -2755,4 +2752,77 @@ public void onFailure(Exception e) {
 
         assertTrue(latch.await(30L, TimeUnit.SECONDS));
     }
+
+    public void testReloadSearchAnalyzers() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+        {
+            CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("index"), RequestOptions.DEFAULT);
+            assertTrue(createIndexResponse.isAcknowledged());
+        }
+
+        {
+            // tag::reload-analyzers-request
+            ReloadAnalyzersRequest request = new ReloadAnalyzersRequest("index"); // <1>
+            // end::reload-analyzers-request
+
+            // tag::reload-analyzers-request-indicesOptions
+            request.setIndicesOptions(IndicesOptions.strictExpandOpen()); // <1>
+            // end::reload-analyzers-request-indicesOptions
+
+            // tag::reload-analyzers-execute
+            ReloadAnalyzersResponse reloadResponse = client.indices().reloadAnalyzers(request, RequestOptions.DEFAULT);
+            // end::reload-analyzers-execute
+
+            // tag::reload-analyzers-response
+            Shards shards = reloadResponse.shards(); // <1>
+            Map<String, ReloadDetails> reloadDetails = reloadResponse.getReloadedDetails(); // <2>
+            ReloadDetails details = reloadDetails.get("index"); // <3>
+            String indexName = details.getIndexName(); // <4>
+            Set<String> indicesNodes = details.getReloadedIndicesNodes(); // <5>
+            Set<String> analyzers = details.getReloadedAnalyzers();  // <6>
+            // end::reload-analyzers-response
+            assertNotNull(shards);
+            assertEquals("index", indexName);
+            assertEquals(1, indicesNodes.size());
+            assertEquals(0, analyzers.size());
+
+            // tag::reload-analyzers-execute-listener
+            ActionListener<ReloadAnalyzersResponse> listener =
+                new ActionListener<ReloadAnalyzersResponse>() {
+                    @Override
+                    public void onResponse(ReloadAnalyzersResponse reloadResponse) {
+                        // <1>
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        // <2>
+                    }
+                };
+            // end::reload-analyzers-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::reload-analyzers-execute-async
+            client.indices().reloadAnalyzersAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::reload-analyzers-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+
+        {
+            // tag::reload-analyzers-notfound
+            try {
+                ReloadAnalyzersRequest request = new ReloadAnalyzersRequest("does_not_exist");
+                client.indices().reloadAnalyzers(request, RequestOptions.DEFAULT);
+            } catch (ElasticsearchException exception) {
+                if (exception.status() == RestStatus.BAD_REQUEST) {
+                    // <1>
+                }
+            }
+            // end::reload-analyzers-notfound
+        }
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java
index aab21fb14dfe1..0d8fe0a5cd9a7 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/LicensingDocumentationIT.java
@@ -194,7 +194,7 @@ public void testGetLicense() throws Exception {
             //end::get-license-response
 
             assertThat(currentLicense, containsString("trial"));
-            assertThat(currentLicense, containsString("client_rest-high-level_integTestCluster"));
+            assertThat(currentLicense, containsString("integTest"));
         }
         {
             GetLicenseRequest request = new GetLicenseRequest();
@@ -233,7 +233,7 @@ public void onFailure(Exception e) {
             String currentLicense = response.getLicenseDefinition();
             assertThat(currentLicense, startsWith("{"));
             assertThat(currentLicense, containsString("trial"));
-            assertThat(currentLicense, containsString("client_rest-high-level_integTestCluster"));
+            assertThat(currentLicense, containsString("integTest"));
             assertThat(currentLicense, endsWith("}"));
         }
     }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
index fe7d04a4e0a8d..93f196212ca0c 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.client.ml.DeleteCalendarEventRequest;
 import org.elasticsearch.client.ml.DeleteCalendarJobRequest;
 import org.elasticsearch.client.ml.DeleteCalendarRequest;
+import org.elasticsearch.client.ml.DeleteDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.DeleteDatafeedRequest;
 import org.elasticsearch.client.ml.DeleteExpiredDataRequest;
 import org.elasticsearch.client.ml.DeleteExpiredDataResponse;
@@ -47,6 +48,8 @@
 import org.elasticsearch.client.ml.DeleteJobRequest;
 import org.elasticsearch.client.ml.DeleteJobResponse;
 import org.elasticsearch.client.ml.DeleteModelSnapshotRequest;
+import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
+import org.elasticsearch.client.ml.EvaluateDataFrameResponse;
 import org.elasticsearch.client.ml.FindFileStructureRequest;
 import org.elasticsearch.client.ml.FindFileStructureResponse;
 import org.elasticsearch.client.ml.FlushJobRequest;
@@ -61,8 +64,10 @@
 import org.elasticsearch.client.ml.GetCalendarsResponse;
 import org.elasticsearch.client.ml.GetCategoriesRequest;
 import org.elasticsearch.client.ml.GetCategoriesResponse;
-import org.elasticsearch.client.ml.GetModelSnapshotsRequest;
-import org.elasticsearch.client.ml.GetModelSnapshotsResponse;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsRequest;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsResponse;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsStatsRequest;
+import org.elasticsearch.client.ml.GetDataFrameAnalyticsStatsResponse;
 import org.elasticsearch.client.ml.GetDatafeedRequest;
 import org.elasticsearch.client.ml.GetDatafeedResponse;
 import org.elasticsearch.client.ml.GetDatafeedStatsRequest;
@@ -75,6 +80,8 @@
 import org.elasticsearch.client.ml.GetJobResponse;
 import org.elasticsearch.client.ml.GetJobStatsRequest;
 import org.elasticsearch.client.ml.GetJobStatsResponse;
+import org.elasticsearch.client.ml.GetModelSnapshotsRequest;
+import org.elasticsearch.client.ml.GetModelSnapshotsResponse;
 import org.elasticsearch.client.ml.GetOverallBucketsRequest;
 import org.elasticsearch.client.ml.GetOverallBucketsResponse;
 import org.elasticsearch.client.ml.GetRecordsRequest;
@@ -92,6 +99,8 @@
 import org.elasticsearch.client.ml.PutCalendarJobRequest;
 import org.elasticsearch.client.ml.PutCalendarRequest;
 import org.elasticsearch.client.ml.PutCalendarResponse;
+import org.elasticsearch.client.ml.PutDataFrameAnalyticsRequest;
+import org.elasticsearch.client.ml.PutDataFrameAnalyticsResponse;
 import org.elasticsearch.client.ml.PutDatafeedRequest;
 import org.elasticsearch.client.ml.PutDatafeedResponse;
 import org.elasticsearch.client.ml.PutFilterRequest;
@@ -101,8 +110,11 @@
 import org.elasticsearch.client.ml.RevertModelSnapshotRequest;
 import org.elasticsearch.client.ml.RevertModelSnapshotResponse;
 import org.elasticsearch.client.ml.SetUpgradeModeRequest;
+import org.elasticsearch.client.ml.StartDataFrameAnalyticsRequest;
 import org.elasticsearch.client.ml.StartDatafeedRequest;
 import org.elasticsearch.client.ml.StartDatafeedResponse;
+import org.elasticsearch.client.ml.StopDataFrameAnalyticsRequest;
+import org.elasticsearch.client.ml.StopDataFrameAnalyticsResponse;
 import org.elasticsearch.client.ml.StopDatafeedRequest;
 import org.elasticsearch.client.ml.StopDatafeedResponse;
 import org.elasticsearch.client.ml.UpdateDatafeedRequest;
@@ -118,6 +130,21 @@
 import org.elasticsearch.client.ml.datafeed.DatafeedStats;
 import org.elasticsearch.client.ml.datafeed.DatafeedUpdate;
 import org.elasticsearch.client.ml.datafeed.DelayedDataCheckConfig;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalysis;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsDest;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsSource;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsStats;
+import org.elasticsearch.client.ml.dataframe.OutlierDetection;
+import org.elasticsearch.client.ml.dataframe.QueryConfig;
+import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.AucRocMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.BinarySoftClassification;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetric.ConfusionMatrix;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.PrecisionMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.RecallMetric;
 import org.elasticsearch.client.ml.filestructurefinder.FileStructure;
 import org.elasticsearch.client.ml.job.config.AnalysisConfig;
 import org.elasticsearch.client.ml.job.config.AnalysisLimits;
@@ -139,13 +166,18 @@
 import org.elasticsearch.client.ml.job.results.OverallBucket;
 import org.elasticsearch.client.ml.job.stats.JobStats;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.tasks.TaskId;
+import org.hamcrest.CoreMatchers;
 import org.junit.After;
 
 import java.io.IOException;
@@ -870,18 +902,7 @@ public void testPreviewDatafeed() throws Exception {
         client.machineLearning().putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
         String datafeedId = job.getId() + "-feed";
         String indexName = "preview_data_2";
-        CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
-        createIndexRequest.mapping(XContentFactory.jsonBuilder().startObject()
-            .startObject("properties")
-                .startObject("timestamp")
-                    .field("type", "date")
-                .endObject()
-                .startObject("total")
-                    .field("type", "long")
-                .endObject()
-            .endObject()
-        .endObject());
-        highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT);
+        createIndex(indexName);
         DatafeedConfig datafeed = DatafeedConfig.builder(datafeedId, job.getId())
             .setIndices(indexName)
             .build();
@@ -938,18 +959,7 @@ public void testStartDatafeed() throws Exception {
         client.machineLearning().putJob(new PutJobRequest(job), RequestOptions.DEFAULT);
         String datafeedId = job.getId() + "-feed";
         String indexName = "start_data_2";
-        CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
-        createIndexRequest.mapping(XContentFactory.jsonBuilder().startObject()
-            .startObject("properties")
-                .startObject("timestamp")
-                    .field("type", "date")
-                .endObject()
-                .startObject("total")
-                    .field("type", "long")
-                .endObject()
-            .endObject()
-        .endObject());
-        highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT);
+        createIndex(indexName);
         DatafeedConfig datafeed = DatafeedConfig.builder(datafeedId, job.getId())
             .setIndices(indexName)
             .build();
@@ -1067,18 +1077,7 @@ public void testGetDatafeedStats() throws Exception {
         client.machineLearning().putJob(new PutJobRequest(secondJob), RequestOptions.DEFAULT);
         String datafeedId1 = job.getId() + "-feed";
         String indexName = "datafeed_stats_data_2";
-        CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
-        createIndexRequest.mapping(XContentFactory.jsonBuilder().startObject()
-            .startObject("properties")
-                .startObject("timestamp")
-                    .field("type", "date")
-                .endObject()
-                .startObject("total")
-                    .field("type", "long")
-                .endObject()
-            .endObject()
-        .endObject());
-        highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT);
+        createIndex(indexName);
         DatafeedConfig datafeed = DatafeedConfig.builder(datafeedId1, job.getId())
             .setIndices(indexName)
             .build();
@@ -2802,6 +2801,466 @@ public void onFailure(Exception e) {
         }
     }
 
+    public void testGetDataFrameAnalytics() throws Exception {
+        createIndex(DF_ANALYTICS_CONFIG.getSource().getIndex()[0]);
+
+        RestHighLevelClient client = highLevelClient();
+        client.machineLearning().putDataFrameAnalytics(new PutDataFrameAnalyticsRequest(DF_ANALYTICS_CONFIG), RequestOptions.DEFAULT);
+        {
+            // tag::get-data-frame-analytics-request
+            GetDataFrameAnalyticsRequest request = new GetDataFrameAnalyticsRequest("my-analytics-config"); // <1>
+            // end::get-data-frame-analytics-request
+
+            // tag::get-data-frame-analytics-execute
+            GetDataFrameAnalyticsResponse response = client.machineLearning().getDataFrameAnalytics(request, RequestOptions.DEFAULT);
+            // end::get-data-frame-analytics-execute
+
+            // tag::get-data-frame-analytics-response
+            List<DataFrameAnalyticsConfig> configs = response.getAnalytics();
+            // end::get-data-frame-analytics-response
+
+            assertThat(configs.size(), equalTo(1));
+        }
+        {
+            GetDataFrameAnalyticsRequest request = new GetDataFrameAnalyticsRequest("my-analytics-config");
+
+            // tag::get-data-frame-analytics-execute-listener
+            ActionListener<GetDataFrameAnalyticsResponse> listener = new ActionListener<>() {
+                @Override
+                public void onResponse(GetDataFrameAnalyticsResponse response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::get-data-frame-analytics-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::get-data-frame-analytics-execute-async
+            client.machineLearning().getDataFrameAnalyticsAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::get-data-frame-analytics-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+
+    public void testGetDataFrameAnalyticsStats() throws Exception {
+        createIndex(DF_ANALYTICS_CONFIG.getSource().getIndex()[0]);
+
+        RestHighLevelClient client = highLevelClient();
+        client.machineLearning().putDataFrameAnalytics(new PutDataFrameAnalyticsRequest(DF_ANALYTICS_CONFIG), RequestOptions.DEFAULT);
+        {
+            // tag::get-data-frame-analytics-stats-request
+            GetDataFrameAnalyticsStatsRequest request = new GetDataFrameAnalyticsStatsRequest("my-analytics-config"); // <1>
+            // end::get-data-frame-analytics-stats-request
+
+            // tag::get-data-frame-analytics-stats-execute
+            GetDataFrameAnalyticsStatsResponse response =
+                client.machineLearning().getDataFrameAnalyticsStats(request, RequestOptions.DEFAULT);
+            // end::get-data-frame-analytics-stats-execute
+
+            // tag::get-data-frame-analytics-stats-response
+            List<DataFrameAnalyticsStats> stats = response.getAnalyticsStats();
+            // end::get-data-frame-analytics-stats-response
+
+            assertThat(stats.size(), equalTo(1));
+        }
+        {
+            GetDataFrameAnalyticsStatsRequest request = new GetDataFrameAnalyticsStatsRequest("my-analytics-config");
+
+            // tag::get-data-frame-analytics-stats-execute-listener
+            ActionListener<GetDataFrameAnalyticsStatsResponse> listener = new ActionListener<>() {
+                @Override
+                public void onResponse(GetDataFrameAnalyticsStatsResponse response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::get-data-frame-analytics-stats-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::get-data-frame-analytics-stats-execute-async
+            client.machineLearning().getDataFrameAnalyticsStatsAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::get-data-frame-analytics-stats-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+
+    public void testPutDataFrameAnalytics() throws Exception {
+        createIndex(DF_ANALYTICS_CONFIG.getSource().getIndex()[0]);
+
+        RestHighLevelClient client = highLevelClient();
+        {
+            // tag::put-data-frame-analytics-query-config
+            QueryConfig queryConfig = new QueryConfig(new MatchAllQueryBuilder());
+            // end::put-data-frame-analytics-query-config
+
+            // tag::put-data-frame-analytics-source-config
+            DataFrameAnalyticsSource sourceConfig = DataFrameAnalyticsSource.builder() // <1>
+                .setIndex("put-test-source-index") // <2>
+                .setQueryConfig(queryConfig) // <3>
+                .build();
+            // end::put-data-frame-analytics-source-config
+
+            // tag::put-data-frame-analytics-dest-config
+            DataFrameAnalyticsDest destConfig = DataFrameAnalyticsDest.builder() // <1>
+                .setIndex("put-test-dest-index") // <2>
+                .build();
+            // end::put-data-frame-analytics-dest-config
+
+            // tag::put-data-frame-analytics-analysis-default
+            DataFrameAnalysis outlierDetection = OutlierDetection.createDefault(); // <1>
+            // end::put-data-frame-analytics-analysis-default
+
+            // tag::put-data-frame-analytics-analysis-customized
+            DataFrameAnalysis outlierDetectionCustomized = OutlierDetection.builder() // <1>
+                .setMethod(OutlierDetection.Method.DISTANCE_KNN) // <2>
+                .setNNeighbors(5) // <3>
+                .build();
+            // end::put-data-frame-analytics-analysis-customized
+
+            // tag::put-data-frame-analytics-analyzed-fields
+            FetchSourceContext analyzedFields =
+                new FetchSourceContext(
+                    true,
+                    new String[] { "included_field_1", "included_field_2" },
+                    new String[] { "excluded_field" });
+            // end::put-data-frame-analytics-analyzed-fields
+
+            // tag::put-data-frame-analytics-config
+            DataFrameAnalyticsConfig config = DataFrameAnalyticsConfig.builder("my-analytics-config") // <1>
+                .setSource(sourceConfig) // <2>
+                .setDest(destConfig) // <3>
+                .setAnalysis(outlierDetection) // <4>
+                .setAnalyzedFields(analyzedFields) // <5>
+                .setModelMemoryLimit(new ByteSizeValue(5, ByteSizeUnit.MB)) // <6>
+                .build();
+            // end::put-data-frame-analytics-config
+
+            // tag::put-data-frame-analytics-request
+            PutDataFrameAnalyticsRequest request = new PutDataFrameAnalyticsRequest(config); // <1>
+            // end::put-data-frame-analytics-request
+
+            // tag::put-data-frame-analytics-execute
+            PutDataFrameAnalyticsResponse response = client.machineLearning().putDataFrameAnalytics(request, RequestOptions.DEFAULT);
+            // end::put-data-frame-analytics-execute
+
+            // tag::put-data-frame-analytics-response
+            DataFrameAnalyticsConfig createdConfig = response.getConfig();
+            // end::put-data-frame-analytics-response
+
+            assertThat(createdConfig.getId(), equalTo("my-analytics-config"));
+        }
+        {
+            PutDataFrameAnalyticsRequest request = new PutDataFrameAnalyticsRequest(DF_ANALYTICS_CONFIG);
+            // tag::put-data-frame-analytics-execute-listener
+            ActionListener<PutDataFrameAnalyticsResponse> listener = new ActionListener<>() {
+                @Override
+                public void onResponse(PutDataFrameAnalyticsResponse response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::put-data-frame-analytics-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::put-data-frame-analytics-execute-async
+            client.machineLearning().putDataFrameAnalyticsAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::put-data-frame-analytics-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+
+    public void testDeleteDataFrameAnalytics() throws Exception {
+        createIndex(DF_ANALYTICS_CONFIG.getSource().getIndex()[0]);
+
+        RestHighLevelClient client = highLevelClient();
+        client.machineLearning().putDataFrameAnalytics(new PutDataFrameAnalyticsRequest(DF_ANALYTICS_CONFIG), RequestOptions.DEFAULT);
+        {
+            // tag::delete-data-frame-analytics-request
+            DeleteDataFrameAnalyticsRequest request = new DeleteDataFrameAnalyticsRequest("my-analytics-config"); // <1>
+            // end::delete-data-frame-analytics-request
+
+            // tag::delete-data-frame-analytics-execute
+            AcknowledgedResponse response = client.machineLearning().deleteDataFrameAnalytics(request, RequestOptions.DEFAULT);
+            // end::delete-data-frame-analytics-execute
+
+            // tag::delete-data-frame-analytics-response
+            boolean acknowledged = response.isAcknowledged();
+            // end::delete-data-frame-analytics-response
+
+            assertThat(acknowledged, is(true));
+        }
+        client.machineLearning().putDataFrameAnalytics(new PutDataFrameAnalyticsRequest(DF_ANALYTICS_CONFIG), RequestOptions.DEFAULT);
+        {
+            DeleteDataFrameAnalyticsRequest request = new DeleteDataFrameAnalyticsRequest("my-analytics-config");
+
+            // tag::delete-data-frame-analytics-execute-listener
+            ActionListener<AcknowledgedResponse> listener = new ActionListener<>() {
+                @Override
+                public void onResponse(AcknowledgedResponse response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::delete-data-frame-analytics-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::delete-data-frame-analytics-execute-async
+            client.machineLearning().deleteDataFrameAnalyticsAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::delete-data-frame-analytics-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+
+    public void testStartDataFrameAnalytics() throws Exception {
+        createIndex(DF_ANALYTICS_CONFIG.getSource().getIndex()[0]);
+        highLevelClient().index(
+            new IndexRequest(DF_ANALYTICS_CONFIG.getSource().getIndex()[0]).source(XContentType.JSON, "total", 10000)
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT);
+        RestHighLevelClient client = highLevelClient();
+        client.machineLearning().putDataFrameAnalytics(new PutDataFrameAnalyticsRequest(DF_ANALYTICS_CONFIG), RequestOptions.DEFAULT);
+        {
+            // tag::start-data-frame-analytics-request
+            StartDataFrameAnalyticsRequest request = new StartDataFrameAnalyticsRequest("my-analytics-config"); // <1>
+            // end::start-data-frame-analytics-request
+
+            // tag::start-data-frame-analytics-execute
+            AcknowledgedResponse response = client.machineLearning().startDataFrameAnalytics(request, RequestOptions.DEFAULT);
+            // end::start-data-frame-analytics-execute
+
+            // tag::start-data-frame-analytics-response
+            boolean acknowledged = response.isAcknowledged();
+            // end::start-data-frame-analytics-response
+
+            assertThat(acknowledged, is(true));
+        }
+        assertBusy(
+            () -> assertThat(getAnalyticsState(DF_ANALYTICS_CONFIG.getId()), equalTo(DataFrameAnalyticsState.STOPPED)),
+            30, TimeUnit.SECONDS);
+        {
+            StartDataFrameAnalyticsRequest request = new StartDataFrameAnalyticsRequest("my-analytics-config");
+
+            // tag::start-data-frame-analytics-execute-listener
+            ActionListener<AcknowledgedResponse> listener = new ActionListener<>() {
+                @Override
+                public void onResponse(AcknowledgedResponse response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::start-data-frame-analytics-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::start-data-frame-analytics-execute-async
+            client.machineLearning().startDataFrameAnalyticsAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::start-data-frame-analytics-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+        assertBusy(
+            () -> assertThat(getAnalyticsState(DF_ANALYTICS_CONFIG.getId()), equalTo(DataFrameAnalyticsState.STOPPED)),
+            30, TimeUnit.SECONDS);
+    }
+
+    public void testStopDataFrameAnalytics() throws Exception {
+        createIndex(DF_ANALYTICS_CONFIG.getSource().getIndex()[0]);
+        highLevelClient().index(
+            new IndexRequest(DF_ANALYTICS_CONFIG.getSource().getIndex()[0]).source(XContentType.JSON, "total", 10000)
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT);
+        RestHighLevelClient client = highLevelClient();
+        client.machineLearning().putDataFrameAnalytics(new PutDataFrameAnalyticsRequest(DF_ANALYTICS_CONFIG), RequestOptions.DEFAULT);
+        {
+            // tag::stop-data-frame-analytics-request
+            StopDataFrameAnalyticsRequest request = new StopDataFrameAnalyticsRequest("my-analytics-config"); // <1>
+            request.setForce(false); // <2>
+            // end::stop-data-frame-analytics-request
+
+            // tag::stop-data-frame-analytics-execute
+            StopDataFrameAnalyticsResponse response = client.machineLearning().stopDataFrameAnalytics(request, RequestOptions.DEFAULT);
+            // end::stop-data-frame-analytics-execute
+
+            // tag::stop-data-frame-analytics-response
+            boolean acknowledged = response.isStopped();
+            // end::stop-data-frame-analytics-response
+
+            assertThat(acknowledged, is(true));
+        }
+        assertBusy(
+            () -> assertThat(getAnalyticsState(DF_ANALYTICS_CONFIG.getId()), equalTo(DataFrameAnalyticsState.STOPPED)),
+            30, TimeUnit.SECONDS);
+        {
+            StopDataFrameAnalyticsRequest request = new StopDataFrameAnalyticsRequest("my-analytics-config");
+
+            // tag::stop-data-frame-analytics-execute-listener
+            ActionListener<StopDataFrameAnalyticsResponse> listener = new ActionListener<>() {
+                @Override
+                public void onResponse(StopDataFrameAnalyticsResponse response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::stop-data-frame-analytics-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::stop-data-frame-analytics-execute-async
+            client.machineLearning().stopDataFrameAnalyticsAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::stop-data-frame-analytics-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+        assertBusy(
+            () -> assertThat(getAnalyticsState(DF_ANALYTICS_CONFIG.getId()), equalTo(DataFrameAnalyticsState.STOPPED)),
+            30, TimeUnit.SECONDS);
+    }
+
+    public void testEvaluateDataFrame() throws Exception {
+        String indexName = "evaluate-test-index";
+        CreateIndexRequest createIndexRequest =
+            new CreateIndexRequest(indexName)
+                .mapping(XContentFactory.jsonBuilder().startObject()
+                    .startObject("properties")
+                        .startObject("label")
+                            .field("type", "keyword")
+                        .endObject()
+                        .startObject("p")
+                            .field("type", "double")
+                        .endObject()
+                    .endObject()
+                .endObject());
+        BulkRequest bulkRequest =
+            new BulkRequest(indexName)
+                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                .add(new IndexRequest().source(XContentType.JSON, "label", false, "p", 0.1)) // #0
+                .add(new IndexRequest().source(XContentType.JSON, "label", false, "p", 0.2)) // #1
+                .add(new IndexRequest().source(XContentType.JSON, "label", false, "p", 0.3)) // #2
+                .add(new IndexRequest().source(XContentType.JSON, "label", false, "p", 0.4)) // #3
+                .add(new IndexRequest().source(XContentType.JSON, "label", false, "p", 0.7)) // #4
+                .add(new IndexRequest().source(XContentType.JSON, "label", true,  "p", 0.2)) // #5
+                .add(new IndexRequest().source(XContentType.JSON, "label", true,  "p", 0.3)) // #6
+                .add(new IndexRequest().source(XContentType.JSON, "label", true,  "p", 0.4)) // #7
+                .add(new IndexRequest().source(XContentType.JSON, "label", true,  "p", 0.8)) // #8
+                .add(new IndexRequest().source(XContentType.JSON, "label", true,  "p", 0.9)); // #9
+        RestHighLevelClient client = highLevelClient();
+        client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
+        client.bulk(bulkRequest, RequestOptions.DEFAULT);
+        {
+            // tag::evaluate-data-frame-request
+            EvaluateDataFrameRequest request = new EvaluateDataFrameRequest( // <1>
+                indexName, // <2>
+                new BinarySoftClassification( // <3>
+                    "label", // <4>
+                    "p", // <5>
+                    // Evaluation metrics // <6>
+                    PrecisionMetric.at(0.4, 0.5, 0.6), // <7>
+                    RecallMetric.at(0.5, 0.7), // <8>
+                    ConfusionMatrixMetric.at(0.5), // <9>
+                    AucRocMetric.withCurve())); // <10>
+            // end::evaluate-data-frame-request
+
+            // tag::evaluate-data-frame-execute
+            EvaluateDataFrameResponse response = client.machineLearning().evaluateDataFrame(request, RequestOptions.DEFAULT);
+            // end::evaluate-data-frame-execute
+
+            // tag::evaluate-data-frame-response
+            List<EvaluationMetric.Result> metrics = response.getMetrics(); // <1>
+
+            PrecisionMetric.Result precisionResult = response.getMetricByName(PrecisionMetric.NAME); // <2>
+            double precision = precisionResult.getScoreByThreshold("0.4"); // <3>
+
+            ConfusionMatrixMetric.Result confusionMatrixResult = response.getMetricByName(ConfusionMatrixMetric.NAME); // <4>
+            ConfusionMatrix confusionMatrix = confusionMatrixResult.getScoreByThreshold("0.5"); // <5>
+            // end::evaluate-data-frame-response
+
+            assertThat(
+                metrics.stream().map(m -> m.getMetricName()).collect(Collectors.toList()),
+                containsInAnyOrder(PrecisionMetric.NAME, RecallMetric.NAME, ConfusionMatrixMetric.NAME, AucRocMetric.NAME));
+            assertThat(precision, closeTo(0.6, 1e-9));
+            assertThat(confusionMatrix.getTruePositives(), CoreMatchers.equalTo(2L));  // docs #8 and #9
+            assertThat(confusionMatrix.getFalsePositives(), CoreMatchers.equalTo(1L));  // doc #4
+            assertThat(confusionMatrix.getTrueNegatives(), CoreMatchers.equalTo(4L));  // docs #0, #1, #2 and #3
+            assertThat(confusionMatrix.getFalseNegatives(), CoreMatchers.equalTo(3L));  // docs #5, #6 and #7
+        }
+        {
+            EvaluateDataFrameRequest request = new EvaluateDataFrameRequest(
+                indexName,
+                new BinarySoftClassification(
+                    "label",
+                    "p",
+                    PrecisionMetric.at(0.4, 0.5, 0.6),
+                    RecallMetric.at(0.5, 0.7),
+                    ConfusionMatrixMetric.at(0.5),
+                    AucRocMetric.withCurve()));
+
+            // tag::evaluate-data-frame-execute-listener
+            ActionListener<EvaluateDataFrameResponse> listener = new ActionListener<>() {
+                @Override
+                public void onResponse(EvaluateDataFrameResponse response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::evaluate-data-frame-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::evaluate-data-frame-execute-async
+            client.machineLearning().evaluateDataFrameAsync(request, RequestOptions.DEFAULT, listener); // <1>
+            // end::evaluate-data-frame-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
 
     public void testCreateFilter() throws Exception {
         RestHighLevelClient client = highLevelClient();
@@ -3140,4 +3599,39 @@ private String createFilter(RestHighLevelClient client) throws IOException {
         assertThat(createdFilter.getId(), equalTo("my_safe_domains"));
         return createdFilter.getId();
     }
+
+    private void createIndex(String indexName) throws IOException {
+        CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
+        createIndexRequest.mapping(XContentFactory.jsonBuilder().startObject()
+            .startObject("properties")
+                .startObject("timestamp")
+                    .field("type", "date")
+                .endObject()
+                .startObject("total")
+                    .field("type", "long")
+                .endObject()
+            .endObject()
+        .endObject());
+        highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT);
+    }
+
+    private DataFrameAnalyticsState getAnalyticsState(String configId) throws IOException {
+        GetDataFrameAnalyticsStatsResponse statsResponse =
+            highLevelClient().machineLearning().getDataFrameAnalyticsStats(
+                new GetDataFrameAnalyticsStatsRequest(configId), RequestOptions.DEFAULT);
+        assertThat(statsResponse.getAnalyticsStats(), hasSize(1));
+        DataFrameAnalyticsStats stats = statsResponse.getAnalyticsStats().get(0);
+        return stats.getState();
+    }
+
+    private static final DataFrameAnalyticsConfig DF_ANALYTICS_CONFIG =
+        DataFrameAnalyticsConfig.builder("my-analytics-config")
+            .setSource(DataFrameAnalyticsSource.builder()
+                .setIndex("put-test-source-index")
+                .build())
+            .setDest(DataFrameAnalyticsDest.builder()
+                .setIndex("put-test-dest-index")
+                .build())
+            .setAnalysis(OutlierDetection.createDefault())
+            .build();
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/QueryDSLDocumentationTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/QueryDSLDocumentationTests.java
index 51670b29de1b6..3530e63e47e1d 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/QueryDSLDocumentationTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/QueryDSLDocumentationTests.java
@@ -42,7 +42,6 @@
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.boostingQuery;
-import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery;
 import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
 import static org.elasticsearch.index.query.QueryBuilders.disMaxQuery;
 import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
@@ -106,13 +105,6 @@ public void testBoosting() {
         // end::boosting
     }
 
-    public void testCommonTerms() {
-        // tag::common_terms
-        commonTermsQuery("name",                                     // <1>
-                         "kimchy");                                  // <2>
-        // end::common_terms
-    }
-
     public void testConstantScore() {
         // tag::constant_score
         constantScoreQuery(
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java
index f9a1c5c6571eb..9bbc3b2ea9072 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java
@@ -50,6 +50,7 @@
 import org.elasticsearch.client.security.ExpressionRoleMapping;
 import org.elasticsearch.client.security.GetApiKeyRequest;
 import org.elasticsearch.client.security.GetApiKeyResponse;
+import org.elasticsearch.client.security.GetBuiltinPrivilegesResponse;
 import org.elasticsearch.client.security.GetPrivilegesRequest;
 import org.elasticsearch.client.security.GetPrivilegesResponse;
 import org.elasticsearch.client.security.GetRoleMappingsRequest;
@@ -97,7 +98,6 @@
 
 import javax.crypto.SecretKeyFactory;
 import javax.crypto.spec.PBEKeySpec;
-
 import java.io.IOException;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -120,6 +120,7 @@
 import static org.hamcrest.Matchers.emptyIterable;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.isIn;
 import static org.hamcrest.Matchers.iterableWithSize;
@@ -1500,6 +1501,60 @@ public void onFailure(Exception e) {
         }
     }
 
+    public void testGetBuiltinPrivileges() throws Exception {
+        final RestHighLevelClient client = highLevelClient();
+        {
+            //tag::get-builtin-privileges-execute
+            GetBuiltinPrivilegesResponse response = client.security().getBuiltinPrivileges(RequestOptions.DEFAULT);
+            //end::get-builtin-privileges-execute
+
+            assertNotNull(response);
+            //tag::get-builtin-privileges-response
+            final Set<String> cluster = response.getClusterPrivileges();
+            final Set<String> index = response.getIndexPrivileges();
+            //end::get-builtin-privileges-response
+
+            assertThat(cluster, hasItem("all"));
+            assertThat(cluster, hasItem("manage"));
+            assertThat(cluster, hasItem("monitor"));
+            assertThat(cluster, hasItem("manage_security"));
+
+            assertThat(index, hasItem("all"));
+            assertThat(index, hasItem("manage"));
+            assertThat(index, hasItem("monitor"));
+            assertThat(index, hasItem("read"));
+            assertThat(index, hasItem("write"));
+        }
+        {
+            // tag::get-builtin-privileges-execute-listener
+            ActionListener<GetBuiltinPrivilegesResponse> listener = new ActionListener<GetBuiltinPrivilegesResponse>() {
+                @Override
+                public void onResponse(GetBuiltinPrivilegesResponse response) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::get-builtin-privileges-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final PlainActionFuture<GetBuiltinPrivilegesResponse> future = new PlainActionFuture<>();
+            listener = future;
+
+            // tag::get-builtin-privileges-execute-async
+            client.security().getBuiltinPrivilegesAsync(RequestOptions.DEFAULT, listener); // <1>
+            // end::get-builtin-privileges-execute-async
+
+            final GetBuiltinPrivilegesResponse response = future.get(30, TimeUnit.SECONDS);
+            assertNotNull(response);
+            assertThat(response.getClusterPrivileges(), hasItem("manage_security"));
+            assertThat(response.getIndexPrivileges(), hasItem("read"));
+        }
+    }
+
     public void testGetPrivileges() throws Exception {
         final RestHighLevelClient client = highLevelClient();
         final ApplicationPrivilege readTestappPrivilege =
@@ -1559,9 +1614,9 @@ public void testGetPrivileges() throws Exception {
 
             assertNotNull(response);
             assertThat(response.getPrivileges().size(), equalTo(3));
-            final GetPrivilegesResponse exptectedResponse =
+            final GetPrivilegesResponse expectedResponse =
                 new GetPrivilegesResponse(Arrays.asList(readTestappPrivilege, writeTestappPrivilege, allTestappPrivilege));
-            assertThat(response, equalTo(exptectedResponse));
+            assertThat(response, equalTo(expectedResponse));
             //tag::get-privileges-response
             Set<ApplicationPrivilege> privileges = response.getPrivileges();
             //end::get-privileges-response
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java
index d80c24be6618a..b8149c99a5473 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java
@@ -452,7 +452,7 @@ public void testSnapshotVerifyRepository() throws IOException {
         List<VerifyRepositoryResponse.NodeView> repositoryMetaDataResponse = response.getNodes();
         // end::verify-repository-response
         assertThat(1, equalTo(repositoryMetaDataResponse.size()));
-        assertThat("node-0", equalTo(repositoryMetaDataResponse.get(0).getName()));
+        assertThat("integTest-0", equalTo(repositoryMetaDataResponse.get(0).getName()));
     }
 
     public void testSnapshotVerifyRepositoryAsync() throws InterruptedException {
@@ -590,7 +590,7 @@ public void testSnapshotGetSnapshots() throws IOException {
         // end::get-snapshots-request
 
         // tag::get-snapshots-request-repositoryName
-        request.repository(repositoryName); // <1>
+        request.repositories(repositoryName); // <1>
         // end::get-snapshots-request-repositoryName
 
         // tag::get-snapshots-request-snapshots
@@ -616,7 +616,7 @@ public void testSnapshotGetSnapshots() throws IOException {
         // end::get-snapshots-execute
 
         // tag::get-snapshots-response
-        List<SnapshotInfo> snapshotsInfos = response.getSnapshots();
+        List<SnapshotInfo> snapshotsInfos = response.getSnapshots(repositoryName);
         SnapshotInfo snapshotInfo = snapshotsInfos.get(0);
         RestStatus restStatus = snapshotInfo.status(); // <1>
         SnapshotId snapshotId = snapshotInfo.snapshotId(); // <2>
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/AnalyzeGlobalRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/AnalyzeGlobalRequestTests.java
new file mode 100644
index 0000000000000..a18971d28fe2d
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/AnalyzeGlobalRequestTests.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indices;
+
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class AnalyzeGlobalRequestTests extends AnalyzeRequestTests {
+
+    @Override
+    protected AnalyzeRequest createClientTestInstance() {
+        int option = random().nextInt(3);
+        switch (option) {
+            case 0:
+                return AnalyzeRequest.withGlobalAnalyzer("my_analyzer", "some text", "some more text");
+            case 1:
+                return AnalyzeRequest.buildCustomAnalyzer("my_tokenizer")
+                    .addCharFilter("my_char_filter")
+                    .addCharFilter(Map.of("type", "html_strip"))
+                    .addTokenFilter("my_token_filter")
+                    .addTokenFilter(Map.of("type", "synonym"))
+                    .build("some text", "some more text");
+            case 2:
+                return AnalyzeRequest.buildCustomNormalizer()
+                    .addCharFilter("my_char_filter")
+                    .addCharFilter(Map.of("type", "html_strip"))
+                    .addTokenFilter("my_token_filter")
+                    .addTokenFilter(Map.of("type", "synonym"))
+                    .build("some text", "some more text");
+        }
+        throw new IllegalStateException("nextInt(3) has returned a value greater than 2");
+    }
+
+    @Override
+    protected AnalyzeAction.Request doParseToServerInstance(XContentParser parser) throws IOException {
+        return AnalyzeAction.Request.fromXContent(parser, null);
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/AnalyzeIndexRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/AnalyzeIndexRequestTests.java
new file mode 100644
index 0000000000000..7cf271e89ac48
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/AnalyzeIndexRequestTests.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indices;
+
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class AnalyzeIndexRequestTests extends AnalyzeRequestTests {
+
+    @Override
+    protected AnalyzeRequest createClientTestInstance() {
+        int option = random().nextInt(5);
+        switch (option) {
+            case 0:
+                return AnalyzeRequest.withField("index", "field", "some text", "some more text");
+            case 1:
+                return AnalyzeRequest.withIndexAnalyzer("index", "my_analyzer", "some text", "some more text");
+            case 2:
+                return AnalyzeRequest.withNormalizer("index", "my_normalizer", "text", "more text");
+            case 3:
+                return AnalyzeRequest.buildCustomAnalyzer("index", "my_tokenizer")
+                    .addCharFilter("my_char_filter")
+                    .addCharFilter(Map.of("type", "html_strip"))
+                    .addTokenFilter("my_token_filter")
+                    .addTokenFilter(Map.of("type", "synonym"))
+                    .build("some text", "some more text");
+            case 4:
+                return AnalyzeRequest.buildCustomNormalizer("index")
+                    .addCharFilter("my_char_filter")
+                    .addCharFilter(Map.of("type", "html_strip"))
+                    .addTokenFilter("my_token_filter")
+                    .addTokenFilter(Map.of("type", "synonym"))
+                    .build("some text", "some more text");
+        }
+        throw new IllegalStateException("nextInt(5) has returned a value greater than 4");
+    }
+
+    @Override
+    protected AnalyzeAction.Request doParseToServerInstance(XContentParser parser) throws IOException {
+        return AnalyzeAction.Request.fromXContent(parser, "index");
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/AnalyzeRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/AnalyzeRequestTests.java
new file mode 100644
index 0000000000000..50a339fc8058a
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/AnalyzeRequestTests.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indices;
+
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
+import org.elasticsearch.client.AbstractRequestTestCase;
+
+public abstract class AnalyzeRequestTests extends AbstractRequestTestCase<AnalyzeRequest, AnalyzeAction.Request> {
+
+    @Override
+    protected void assertInstances(AnalyzeAction.Request serverInstance, AnalyzeRequest clientTestInstance) {
+        assertEquals(serverInstance.index(), clientTestInstance.index());
+        assertArrayEquals(serverInstance.text(), clientTestInstance.text());
+        assertEquals(serverInstance.analyzer(), clientTestInstance.analyzer());
+        assertEquals(serverInstance.normalizer(), clientTestInstance.normalizer());
+        assertEquals(serverInstance.charFilters().size(), clientTestInstance.charFilters().size());
+        for (int i = 0; i < serverInstance.charFilters().size(); i++) {
+            assertEquals(serverInstance.charFilters().get(i).name, clientTestInstance.charFilters().get(i).name);
+            assertEquals(serverInstance.charFilters().get(i).definition, clientTestInstance.charFilters().get(i).definition);
+        }
+        assertEquals(serverInstance.tokenFilters().size(), clientTestInstance.tokenFilters().size());
+        for (int i = 0; i < serverInstance.tokenFilters().size(); i++) {
+            assertEquals(serverInstance.tokenFilters().get(i).name, clientTestInstance.tokenFilters().get(i).name);
+            assertEquals(serverInstance.tokenFilters().get(i).definition, clientTestInstance.tokenFilters().get(i).definition);
+        }
+        if (serverInstance.tokenizer() != null) {
+            assertEquals(serverInstance.tokenizer().name, clientTestInstance.tokenizer().name);
+            assertEquals(serverInstance.tokenizer().definition, clientTestInstance.tokenizer().definition);
+        }
+        else {
+            assertNull(clientTestInstance.tokenizer());
+        }
+        assertEquals(serverInstance.field(), clientTestInstance.field());
+        assertEquals(serverInstance.explain(), clientTestInstance.explain());
+        assertArrayEquals(serverInstance.attributes(), clientTestInstance.attributes());
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/AnalyzeResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/AnalyzeResponseTests.java
new file mode 100644
index 0000000000000..e29fa88d7fe3e
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/AnalyzeResponseTests.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indices;
+
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
+import org.elasticsearch.client.AbstractResponseTestCase;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class AnalyzeResponseTests extends AbstractResponseTestCase<AnalyzeAction.Response, AnalyzeResponse> {
+
+    @Override
+    protected AnalyzeAction.Response createServerTestInstance() {
+        int tokenCount = randomIntBetween(1, 30);
+        AnalyzeAction.AnalyzeToken[] tokens = new AnalyzeAction.AnalyzeToken[tokenCount];
+        for (int i = 0; i < tokenCount; i++) {
+            tokens[i] = randomToken();
+        }
+        if (randomBoolean()) {
+            AnalyzeAction.CharFilteredText[] charfilters = null;
+            AnalyzeAction.AnalyzeTokenList[] tokenfilters = null;
+            if (randomBoolean()) {
+                charfilters = new AnalyzeAction.CharFilteredText[]{
+                    new AnalyzeAction.CharFilteredText("my_charfilter", new String[]{"one two"})
+                };
+            }
+            if (randomBoolean()) {
+                tokenfilters = new AnalyzeAction.AnalyzeTokenList[]{
+                    new AnalyzeAction.AnalyzeTokenList("my_tokenfilter_1", tokens),
+                    new AnalyzeAction.AnalyzeTokenList("my_tokenfilter_2", tokens)
+                };
+            }
+            AnalyzeAction.DetailAnalyzeResponse dar = new AnalyzeAction.DetailAnalyzeResponse(
+                charfilters,
+                new AnalyzeAction.AnalyzeTokenList("my_tokenizer", tokens),
+                tokenfilters);
+            return new AnalyzeAction.Response(null, dar);
+        }
+        return new AnalyzeAction.Response(Arrays.asList(tokens), null);
+    }
+
+    private AnalyzeAction.AnalyzeToken randomToken() {
+        String token = randomAlphaOfLengthBetween(1, 20);
+        int position = randomIntBetween(0, 1000);
+        int startOffset = randomIntBetween(0, 1000);
+        int endOffset = randomIntBetween(0, 1000);
+        int posLength = randomIntBetween(1, 5);
+        String type = randomAlphaOfLengthBetween(1, 20);
+        Map<String, Object> extras = new HashMap<>();
+        if (randomBoolean()) {
+            int entryCount = randomInt(6);
+            for (int i = 0; i < entryCount; i++) {
+                switch (randomInt(6)) {
+                    case 0:
+                    case 1:
+                    case 2:
+                    case 3:
+                        String key = randomAlphaOfLength(5);
+                        String value = randomAlphaOfLength(10);
+                        extras.put(key, value);
+                        break;
+                    case 4:
+                        String objkey = randomAlphaOfLength(5);
+                        Map<String, String> obj = new HashMap<>();
+                        obj.put(randomAlphaOfLength(5), randomAlphaOfLength(10));
+                        extras.put(objkey, obj);
+                        break;
+                    case 5:
+                        String listkey = randomAlphaOfLength(5);
+                        List<String> list = new ArrayList<>();
+                        list.add(randomAlphaOfLength(4));
+                        list.add(randomAlphaOfLength(6));
+                        extras.put(listkey, list);
+                        break;
+                }
+            }
+        }
+        return new AnalyzeAction.AnalyzeToken(token, position, startOffset, endOffset, posLength, type, extras);
+    }
+
+    @Override
+    protected AnalyzeResponse doParseToClientInstance(XContentParser parser) throws IOException {
+        return AnalyzeResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected void assertInstances(AnalyzeAction.Response serverTestInstance, AnalyzeResponse clientInstance) {
+        if (serverTestInstance.detail() != null) {
+            assertNotNull(clientInstance.detail());
+            assertInstances(serverTestInstance.detail(), clientInstance.detail());
+        }
+        else {
+            assertEquals(serverTestInstance.getTokens().size(), clientInstance.getTokens().size());
+            for (int i = 0; i < serverTestInstance.getTokens().size(); i++) {
+                assertEqualTokens(serverTestInstance.getTokens().get(0), clientInstance.getTokens().get(0));
+            }
+        }
+    }
+
+    private static void assertEqualTokens(AnalyzeAction.AnalyzeToken serverToken, AnalyzeResponse.AnalyzeToken clientToken) {
+        assertEquals(serverToken.getTerm(), clientToken.getTerm());
+        assertEquals(serverToken.getPosition(), clientToken.getPosition());
+        assertEquals(serverToken.getPositionLength(), clientToken.getPositionLength());
+        assertEquals(serverToken.getStartOffset(), clientToken.getStartOffset());
+        assertEquals(serverToken.getEndOffset(), clientToken.getEndOffset());
+        assertEquals(serverToken.getType(), clientToken.getType());
+        assertEquals(serverToken.getAttributes(), clientToken.getAttributes());
+    }
+
+    private static void assertInstances(AnalyzeAction.DetailAnalyzeResponse serverResponse, DetailAnalyzeResponse clientResponse) {
+        assertInstances(serverResponse.analyzer(), clientResponse.analyzer());
+        assertInstances(serverResponse.tokenizer(), clientResponse.tokenizer());
+        if (serverResponse.tokenfilters() == null) {
+            assertNull(clientResponse.tokenfilters());
+        }
+        else {
+            assertEquals(serverResponse.tokenfilters().length, clientResponse.tokenfilters().length);
+            for (int i = 0; i < serverResponse.tokenfilters().length; i++) {
+                assertInstances(serverResponse.tokenfilters()[i], clientResponse.tokenfilters()[i]);
+            }
+        }
+        if (serverResponse.charfilters() == null) {
+            assertNull(clientResponse.charfilters());
+        }
+        else {
+            assertEquals(serverResponse.charfilters().length, clientResponse.charfilters().length);
+            for (int i = 0; i < serverResponse.charfilters().length; i++) {
+                assertInstances(serverResponse.charfilters()[i], clientResponse.charfilters()[i]);
+            }
+        }
+    }
+
+    private static void assertInstances(AnalyzeAction.AnalyzeTokenList serverTokens,
+                                        DetailAnalyzeResponse.AnalyzeTokenList clientTokens) {
+        if (serverTokens == null) {
+            assertNull(clientTokens);
+        }
+        else {
+            assertEquals(serverTokens.getName(), clientTokens.getName());
+            assertEquals(serverTokens.getTokens().length, clientTokens.getTokens().length);
+            for (int i = 0; i < serverTokens.getTokens().length; i++) {
+                assertEqualTokens(serverTokens.getTokens()[i], clientTokens.getTokens()[i]);
+            }
+        }
+    }
+
+    private static void assertInstances(AnalyzeAction.CharFilteredText serverText, DetailAnalyzeResponse.CharFilteredText clientText) {
+        assertEquals(serverText.getName(), clientText.getName());
+        assertArrayEquals(serverText.getTexts(), clientText.getTexts());
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/ReloadAnalyzersResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/ReloadAnalyzersResponseTests.java
new file mode 100644
index 0000000000000..6719e10808e42
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/ReloadAnalyzersResponseTests.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.indices;
+
+import org.elasticsearch.action.support.DefaultShardOperationFailedException;
+import org.elasticsearch.client.AbstractResponseTestCase;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.seqno.RetentionLeaseNotFoundException;
+import org.elasticsearch.xpack.core.action.ReloadAnalyzersResponse.ReloadDetails;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.isIn;
+
+public class ReloadAnalyzersResponseTests
+        extends AbstractResponseTestCase<org.elasticsearch.xpack.core.action.ReloadAnalyzersResponse, ReloadAnalyzersResponse> {
+
+    private String index;
+    private String id;
+    private Set<Integer> shardIds;
+
+    @Override
+    protected org.elasticsearch.xpack.core.action.ReloadAnalyzersResponse createServerTestInstance() {
+        index = randomAlphaOfLength(8);
+        id = randomAlphaOfLength(8);
+        final int total = randomIntBetween(1, 16);
+        final int successful = total - scaledRandomIntBetween(0, total);
+        final int failed = scaledRandomIntBetween(0, total - successful);
+        final List<DefaultShardOperationFailedException> failures = new ArrayList<>();
+        shardIds = new HashSet<>();
+        for (int i = 0; i < failed; i++) {
+            final DefaultShardOperationFailedException failure = new DefaultShardOperationFailedException(
+                index,
+                randomValueOtherThanMany(shardIds::contains, () -> randomIntBetween(0, total - 1)),
+                new RetentionLeaseNotFoundException(id));
+            failures.add(failure);
+            shardIds.add(failure.shardId());
+        }
+        Map<String, ReloadDetails> reloadedDetailsMap = new HashMap<>();
+        int randomIndices = randomIntBetween(0, 5);
+        for (int i = 0; i < randomIndices; i++) {
+            String indexName = randomAlphaOfLengthBetween(5, 10);
+            Set<String> randomNodeIds = new HashSet<>(Arrays.asList(generateRandomStringArray(5, 5, false, true)));
+            Set<String> randomAnalyzers = new HashSet<>(Arrays.asList(generateRandomStringArray(5, 5, false, true)));
+
+            ReloadDetails reloadedDetails = new ReloadDetails(indexName, randomNodeIds, randomAnalyzers);
+            reloadedDetailsMap.put(indexName, reloadedDetails);
+        }
+        return new org.elasticsearch.xpack.core.action.ReloadAnalyzersResponse(total, successful, failed, failures, reloadedDetailsMap);
+    }
+
+    @Override
+    protected ReloadAnalyzersResponse doParseToClientInstance(XContentParser parser) throws IOException {
+        return ReloadAnalyzersResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected void assertInstances(org.elasticsearch.xpack.core.action.ReloadAnalyzersResponse serverTestInstance,
+            ReloadAnalyzersResponse clientInstance) {
+        assertThat(clientInstance.shards().total(), equalTo(serverTestInstance.getTotalShards()));
+        assertThat(clientInstance.shards().successful(), equalTo(serverTestInstance.getSuccessfulShards()));
+        assertThat(clientInstance.shards().skipped(), equalTo(0));
+        assertThat(clientInstance.shards().failed(), equalTo(serverTestInstance.getFailedShards()));
+        assertThat(clientInstance.shards().failures(), hasSize(clientInstance.shards().failed() == 0 ? 0 : 1)); // failures are grouped
+        if (clientInstance.shards().failed() > 0) {
+            final DefaultShardOperationFailedException groupedFailure = clientInstance.shards().failures().iterator().next();
+            assertThat(groupedFailure.index(), equalTo(index));
+            assertThat(groupedFailure.shardId(), isIn(shardIds));
+            assertThat(groupedFailure.reason(), containsString("reason=retention lease with ID [" + id + "] not found"));
+        }
+        Map<String, ReloadDetails> serverDetails = serverTestInstance.getReloadDetails();
+        assertThat(clientInstance.getReloadedDetails().size(), equalTo(serverDetails.size()));
+        for (Entry<String, org.elasticsearch.client.indices.ReloadAnalyzersResponse.ReloadDetails> entry : clientInstance
+                .getReloadedDetails().entrySet()) {
+            String indexName = entry.getKey();
+            assertTrue(serverDetails.keySet().contains(indexName));
+            assertEquals(serverDetails.get(indexName).getIndexName(), entry.getValue().getIndexName());
+            assertEquals(serverDetails.get(indexName).getReloadedAnalyzers(), entry.getValue().getReloadedAnalyzers());
+            assertEquals(serverDetails.get(indexName).getReloadedIndicesNodes(), entry.getValue().getReloadedIndicesNodes());
+        }
+    }
+
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/license/PutLicenseResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/license/PutLicenseResponseTests.java
index efb7ed98b366f..2422403659199 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/license/PutLicenseResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/license/PutLicenseResponseTests.java
@@ -18,8 +18,9 @@
  */
 package org.elasticsearch.client.license;
 
+import org.elasticsearch.client.AbstractHlrcWriteableXContentTestCase;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.client.AbstractHlrcStreamableXContentTestCase;
 import org.elasticsearch.protocol.xpack.license.LicensesStatus;
 
 import java.io.IOException;
@@ -29,8 +30,8 @@
 import java.util.function.Function;
 import java.util.function.Predicate;
 
-public class PutLicenseResponseTests extends AbstractHlrcStreamableXContentTestCase<
-    org.elasticsearch.protocol.xpack.license.PutLicenseResponse, org.elasticsearch.client.license.PutLicenseResponse> {
+public class PutLicenseResponseTests extends AbstractHlrcWriteableXContentTestCase<
+    org.elasticsearch.protocol.xpack.license.PutLicenseResponse, PutLicenseResponse> {
 
     @Override
     public org.elasticsearch.client.license.PutLicenseResponse doHlrcParseInstance(XContentParser parser) throws IOException {
@@ -96,8 +97,8 @@ private static Map<String, String[]> randomAckMessages() {
     }
 
     @Override
-    protected org.elasticsearch.protocol.xpack.license.PutLicenseResponse createBlankInstance() {
-        return new org.elasticsearch.protocol.xpack.license.PutLicenseResponse();
+    protected Writeable.Reader<org.elasticsearch.protocol.xpack.license.PutLicenseResponse> instanceReader() {
+        return org.elasticsearch.protocol.xpack.license.PutLicenseResponse::new;
     }
 
     @Override
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/license/StartBasicResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/license/StartBasicResponseTests.java
index 69f831cff188e..a22b40d7e6e63 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/license/StartBasicResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/license/StartBasicResponseTests.java
@@ -18,10 +18,11 @@
  */
 package org.elasticsearch.client.license;
 
+import org.elasticsearch.client.AbstractHlrcWriteableXContentTestCase;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.license.PostStartBasicResponse;
-import org.elasticsearch.client.AbstractHlrcStreamableXContentTestCase;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -29,8 +30,8 @@
 import java.util.Map;
 import java.util.function.Predicate;
 
-public class StartBasicResponseTests extends
-        AbstractHlrcStreamableXContentTestCase<PostStartBasicResponse, org.elasticsearch.client.license.StartBasicResponse> {
+public class StartBasicResponseTests extends AbstractHlrcWriteableXContentTestCase<
+    PostStartBasicResponse, StartBasicResponse> {
 
     @Override
     public org.elasticsearch.client.license.StartBasicResponse doHlrcParseInstance(XContentParser parser) throws IOException {
@@ -44,8 +45,8 @@ public PostStartBasicResponse convertHlrcToInternal(org.elasticsearch.client.lic
     }
 
     @Override
-    protected PostStartBasicResponse createBlankInstance() {
-        return new PostStartBasicResponse();
+    protected Writeable.Reader<PostStartBasicResponse> instanceReader() {
+        return PostStartBasicResponse::new;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/AliasesExistResponse.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/AucRocMetricAucRocPointTests.java
similarity index 51%
rename from server/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/AliasesExistResponse.java
rename to client/rest-high-level/src/test/java/org/elasticsearch/client/ml/AucRocMetricAucRocPointTests.java
index 2d430bf9820d2..825adcd2060f8 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/AliasesExistResponse.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/AucRocMetricAucRocPointTests.java
@@ -16,43 +16,32 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+package org.elasticsearch.client.ml;
 
-package org.elasticsearch.action.admin.indices.alias.exists;
-
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.AucRocMetric;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
 
 import java.io.IOException;
 
-public class AliasesExistResponse extends ActionResponse {
-
-    private boolean exists;
-
-    public AliasesExistResponse(boolean exists) {
-        this.exists = exists;
-    }
+public class AucRocMetricAucRocPointTests extends AbstractXContentTestCase<AucRocMetric.AucRocPoint> {
 
-    AliasesExistResponse() {
+    static AucRocMetric.AucRocPoint randomPoint() {
+        return new AucRocMetric.AucRocPoint(randomDouble(), randomDouble(), randomDouble());
     }
 
-    public boolean exists() {
-        return exists;
-    }
-
-    public boolean isExists() {
-        return exists();
+    @Override
+    protected AucRocMetric.AucRocPoint createTestInstance() {
+        return randomPoint();
     }
 
     @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        exists = in.readBoolean();
+    protected AucRocMetric.AucRocPoint doParseInstance(XContentParser parser) throws IOException {
+        return AucRocMetric.AucRocPoint.fromXContent(parser);
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeBoolean(exists);
+    protected boolean supportsUnknownFields() {
+        return true;
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/AucRocMetricResultTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/AucRocMetricResultTests.java
new file mode 100644
index 0000000000000..9ea7689d60f32
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/AucRocMetricResultTests.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.AucRocMetric;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.client.ml.AucRocMetricAucRocPointTests.randomPoint;
+
+public class AucRocMetricResultTests extends AbstractXContentTestCase<AucRocMetric.Result> {
+
+    static AucRocMetric.Result randomResult() {
+        return new AucRocMetric.Result(
+            randomDouble(),
+            Stream
+                .generate(() -> randomPoint())
+                .limit(randomIntBetween(1, 10))
+                .collect(Collectors.toList()));
+    }
+
+    @Override
+    protected AucRocMetric.Result createTestInstance() {
+        return randomResult();
+    }
+
+    @Override
+    protected AucRocMetric.Result doParseInstance(XContentParser parser) throws IOException {
+        return AucRocMetric.Result.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        // allow unknown fields in the root of the object only
+        return field -> !field.isEmpty();
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/ConfusionMatrixMetricConfusionMatrixTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/ConfusionMatrixMetricConfusionMatrixTests.java
new file mode 100644
index 0000000000000..28eb221b318c6
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/ConfusionMatrixMetricConfusionMatrixTests.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetric;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class ConfusionMatrixMetricConfusionMatrixTests extends AbstractXContentTestCase<ConfusionMatrixMetric.ConfusionMatrix> {
+
+    static ConfusionMatrixMetric.ConfusionMatrix randomConfusionMatrix() {
+        return new ConfusionMatrixMetric.ConfusionMatrix(randomInt(), randomInt(), randomInt(), randomInt());
+    }
+
+    @Override
+    protected ConfusionMatrixMetric.ConfusionMatrix createTestInstance() {
+        return randomConfusionMatrix();
+    }
+
+    @Override
+    protected ConfusionMatrixMetric.ConfusionMatrix doParseInstance(XContentParser parser) throws IOException {
+        return ConfusionMatrixMetric.ConfusionMatrix.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/ConfusionMatrixMetricResultTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/ConfusionMatrixMetricResultTests.java
new file mode 100644
index 0000000000000..c4b299a96b536
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/ConfusionMatrixMetricResultTests.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.ConfusionMatrixMetric;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.client.ml.ConfusionMatrixMetricConfusionMatrixTests.randomConfusionMatrix;
+
+public class ConfusionMatrixMetricResultTests extends AbstractXContentTestCase<ConfusionMatrixMetric.Result> {
+
+    static ConfusionMatrixMetric.Result randomResult() {
+        return new ConfusionMatrixMetric.Result(
+            Stream
+                .generate(() -> randomConfusionMatrix())
+                .limit(randomIntBetween(1, 5))
+                .collect(Collectors.toMap(v -> String.valueOf(randomDouble()), v -> v)));
+    }
+
+    @Override
+    protected ConfusionMatrixMetric.Result createTestInstance() {
+        return randomResult();
+    }
+
+    @Override
+    protected ConfusionMatrixMetric.Result doParseInstance(XContentParser parser) throws IOException {
+        return ConfusionMatrixMetric.Result.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        // disallow unknown fields in the root of the object as field names must be parsable as numbers
+        return field -> field.isEmpty();
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/DeleteDataFrameAnalyticsRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/DeleteDataFrameAnalyticsRequestTests.java
new file mode 100644
index 0000000000000..bc2ca2d954e76
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/DeleteDataFrameAnalyticsRequestTests.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Optional;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class DeleteDataFrameAnalyticsRequestTests extends ESTestCase {
+
+    public void testValidate_Ok() {
+        assertEquals(Optional.empty(), new DeleteDataFrameAnalyticsRequest("valid-id").validate());
+        assertEquals(Optional.empty(), new DeleteDataFrameAnalyticsRequest("").validate());
+    }
+
+    public void testValidate_Failure() {
+        assertThat(new DeleteDataFrameAnalyticsRequest(null).validate().get().getMessage(),
+            containsString("data frame analytics id must not be null"));
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/EvaluateDataFrameResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/EvaluateDataFrameResponseTests.java
new file mode 100644
index 0000000000000..b41d113686ccf
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/EvaluateDataFrameResponseTests.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric;
+import org.elasticsearch.client.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Predicate;
+
+public class EvaluateDataFrameResponseTests extends AbstractXContentTestCase<EvaluateDataFrameResponse> {
+
+    public static EvaluateDataFrameResponse randomResponse() {
+        List<EvaluationMetric.Result> metrics = new ArrayList<>();
+        if (randomBoolean()) {
+            metrics.add(AucRocMetricResultTests.randomResult());
+        }
+        if (randomBoolean()) {
+            metrics.add(PrecisionMetricResultTests.randomResult());
+        }
+        if (randomBoolean()) {
+            metrics.add(RecallMetricResultTests.randomResult());
+        }
+        if (randomBoolean()) {
+            metrics.add(ConfusionMatrixMetricResultTests.randomResult());
+        }
+        return new EvaluateDataFrameResponse(randomAlphaOfLength(5), metrics);
+    }
+
+    @Override
+    protected EvaluateDataFrameResponse createTestInstance() {
+        return randomResponse();
+    }
+
+    @Override
+    protected EvaluateDataFrameResponse doParseInstance(XContentParser parser) throws IOException {
+        return EvaluateDataFrameResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        // allow unknown fields in the metrics map (i.e. alongside named metrics like "precision" or "recall")
+        return field -> field.isEmpty() || field.contains(".");
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(new MlEvaluationNamedXContentProvider().getNamedXContentParsers());
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/FindFileStructureRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/FindFileStructureRequestTests.java
index 4cb8bf0a7c166..752d0593bef95 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/FindFileStructureRequestTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/FindFileStructureRequestTests.java
@@ -35,6 +35,7 @@ public class FindFileStructureRequestTests extends AbstractXContentTestCase<Find
 
     static {
         PARSER.declareInt(FindFileStructureRequest::setLinesToSample, FindFileStructureRequest.LINES_TO_SAMPLE);
+        PARSER.declareInt(FindFileStructureRequest::setLineMergeSizeLimit, FindFileStructureRequest.LINE_MERGE_SIZE_LIMIT);
         PARSER.declareString((p, c) -> p.setTimeout(TimeValue.parseTimeValue(c, FindFileStructureRequest.TIMEOUT.getPreferredName())),
             FindFileStructureRequest.TIMEOUT);
         PARSER.declareString(FindFileStructureRequest::setCharset, FindFileStructureRequest.CHARSET);
@@ -72,6 +73,9 @@ public static FindFileStructureRequest createTestRequestWithoutSample() {
         if (randomBoolean()) {
             findFileStructureRequest.setLinesToSample(randomIntBetween(1000, 2000));
         }
+        if (randomBoolean()) {
+            findFileStructureRequest.setLineMergeSizeLimit(randomIntBetween(10000, 20000));
+        }
         if (randomBoolean()) {
             findFileStructureRequest.setTimeout(TimeValue.timeValueSeconds(randomIntBetween(10, 20)));
         }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequestTests.java
new file mode 100644
index 0000000000000..56d87ea6bef49
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequestTests.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Optional;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class GetDataFrameAnalyticsRequestTests extends ESTestCase {
+
+    public void testValidate_Ok() {
+        assertEquals(Optional.empty(), new GetDataFrameAnalyticsRequest("valid-id").validate());
+        assertEquals(Optional.empty(), new GetDataFrameAnalyticsRequest("").validate());
+    }
+
+    public void testValidate_Failure() {
+        assertThat(new GetDataFrameAnalyticsRequest(new String[0]).validate().get().getMessage(),
+            containsString("data frame analytics id must not be null"));
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsStatsRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsStatsRequestTests.java
new file mode 100644
index 0000000000000..4e08d99eaa659
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsStatsRequestTests.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Optional;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class GetDataFrameAnalyticsStatsRequestTests extends ESTestCase {
+
+    public void testValidate_Ok() {
+        assertEquals(Optional.empty(), new GetDataFrameAnalyticsStatsRequest("valid-id").validate());
+        assertEquals(Optional.empty(), new GetDataFrameAnalyticsStatsRequest("").validate());
+    }
+
+    public void testValidate_Failure() {
+        assertThat(new GetDataFrameAnalyticsStatsRequest(new String[0]).validate().get().getMessage(),
+            containsString("data frame analytics id must not be null"));
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PrecisionMetricResultTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PrecisionMetricResultTests.java
new file mode 100644
index 0000000000000..607adacebb827
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PrecisionMetricResultTests.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.PrecisionMetric;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class PrecisionMetricResultTests extends AbstractXContentTestCase<PrecisionMetric.Result> {
+
+    static PrecisionMetric.Result randomResult() {
+        return new PrecisionMetric.Result(
+            Stream
+                .generate(() -> randomDouble())
+                .limit(randomIntBetween(1, 5))
+                .collect(Collectors.toMap(v -> String.valueOf(randomDouble()), v -> v)));
+    }
+
+    @Override
+    protected PrecisionMetric.Result createTestInstance() {
+        return randomResult();
+    }
+
+    @Override
+    protected PrecisionMetric.Result doParseInstance(XContentParser parser) throws IOException {
+        return PrecisionMetric.Result.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        // disallow unknown fields in the root of the object as field names must be parsable as numbers
+        return field -> field.isEmpty();
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PutDataFrameAnalyticsRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PutDataFrameAnalyticsRequestTests.java
new file mode 100644
index 0000000000000..7387ba8ddeb65
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PutDataFrameAnalyticsRequestTests.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.ValidationException;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfigTests;
+import org.elasticsearch.client.ml.dataframe.MlDataFrameAnalysisNamedXContentProvider;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class PutDataFrameAnalyticsRequestTests extends AbstractXContentTestCase<PutDataFrameAnalyticsRequest> {
+
+    public void testValidate_Ok() {
+        assertFalse(createTestInstance().validate().isPresent());
+    }
+
+    public void testValidate_Failure() {
+        Optional<ValidationException> exception = new PutDataFrameAnalyticsRequest(null).validate();
+        assertTrue(exception.isPresent());
+        assertThat(exception.get().getMessage(), containsString("put requires a non-null data frame analytics config"));
+    }
+
+    @Override
+    protected PutDataFrameAnalyticsRequest createTestInstance() {
+        return new PutDataFrameAnalyticsRequest(DataFrameAnalyticsConfigTests.randomDataFrameAnalyticsConfig());
+    }
+
+    @Override
+    protected PutDataFrameAnalyticsRequest doParseInstance(XContentParser parser) throws IOException {
+        return new PutDataFrameAnalyticsRequest(DataFrameAnalyticsConfig.fromXContent(parser));
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
+        namedXContent.addAll(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents());
+        namedXContent.addAll(new MlDataFrameAnalysisNamedXContentProvider().getNamedXContentParsers());
+        return new NamedXContentRegistry(namedXContent);
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/RecallMetricResultTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/RecallMetricResultTests.java
new file mode 100644
index 0000000000000..138875007e30d
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/RecallMetricResultTests.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.client.ml.dataframe.evaluation.softclassification.RecallMetric;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class RecallMetricResultTests extends AbstractXContentTestCase<RecallMetric.Result> {
+
+    static RecallMetric.Result randomResult() {
+        return new RecallMetric.Result(
+            Stream
+                .generate(() -> randomDouble())
+                .limit(randomIntBetween(1, 5))
+                .collect(Collectors.toMap(v -> String.valueOf(randomDouble()), v -> v)));
+    }
+
+    @Override
+    protected RecallMetric.Result createTestInstance() {
+        return randomResult();
+    }
+
+    @Override
+    protected RecallMetric.Result doParseInstance(XContentParser parser) throws IOException {
+        return RecallMetric.Result.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        // disallow unknown fields in the root of the object as field names must be parsable as numbers
+        return field -> field.isEmpty();
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/StartDataFrameAnalyticsRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/StartDataFrameAnalyticsRequestTests.java
new file mode 100644
index 0000000000000..6e43b50bcd12b
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/StartDataFrameAnalyticsRequestTests.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Optional;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class StartDataFrameAnalyticsRequestTests extends ESTestCase {
+
+    public void testValidate_Ok() {
+        assertEquals(Optional.empty(), new StartDataFrameAnalyticsRequest("foo").validate());
+        assertEquals(Optional.empty(), new StartDataFrameAnalyticsRequest("foo").setTimeout(null).validate());
+        assertEquals(Optional.empty(), new StartDataFrameAnalyticsRequest("foo").setTimeout(TimeValue.ZERO).validate());
+    }
+
+    public void testValidate_Failure() {
+        assertThat(new StartDataFrameAnalyticsRequest(null).validate().get().getMessage(),
+            containsString("data frame analytics id must not be null"));
+        assertThat(new StartDataFrameAnalyticsRequest(null).setTimeout(TimeValue.ZERO).validate().get().getMessage(),
+            containsString("data frame analytics id must not be null"));
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/StopDataFrameAnalyticsRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/StopDataFrameAnalyticsRequestTests.java
new file mode 100644
index 0000000000000..57af2083743ae
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/StopDataFrameAnalyticsRequestTests.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Optional;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class StopDataFrameAnalyticsRequestTests extends ESTestCase {
+
+    public void testValidate_Ok() {
+        assertEquals(Optional.empty(), new StopDataFrameAnalyticsRequest("foo").validate());
+        assertEquals(Optional.empty(), new StopDataFrameAnalyticsRequest("foo").setTimeout(null).validate());
+        assertEquals(Optional.empty(), new StopDataFrameAnalyticsRequest("foo").setTimeout(TimeValue.ZERO).validate());
+    }
+
+    public void testValidate_Failure() {
+        assertThat(new StopDataFrameAnalyticsRequest(null).validate().get().getMessage(),
+            containsString("data frame analytics id must not be null"));
+        assertThat(new StopDataFrameAnalyticsRequest(null).setTimeout(TimeValue.ZERO).validate().get().getMessage(),
+            containsString("data frame analytics id must not be null"));
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/IndicesExistsResponse.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/StopDataFrameAnalyticsResponseTests.java
similarity index 53%
rename from server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/IndicesExistsResponse.java
rename to client/rest-high-level/src/test/java/org/elasticsearch/client/ml/StopDataFrameAnalyticsResponseTests.java
index fb2d5c0fa9b9c..55ef1aed7534a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/IndicesExistsResponse.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/StopDataFrameAnalyticsResponseTests.java
@@ -16,39 +16,27 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+package org.elasticsearch.client.ml;
 
-package org.elasticsearch.action.admin.indices.exists.indices;
-
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
 
 import java.io.IOException;
 
-public class IndicesExistsResponse extends ActionResponse {
-
-    private boolean exists;
-
-    IndicesExistsResponse() {
-    }
+public class StopDataFrameAnalyticsResponseTests extends AbstractXContentTestCase<StopDataFrameAnalyticsResponse> {
 
-    public IndicesExistsResponse(boolean exists) {
-        this.exists = exists;
-    }
-
-    public boolean isExists() {
-        return this.exists;
+    @Override
+    protected StopDataFrameAnalyticsResponse createTestInstance() {
+        return new StopDataFrameAnalyticsResponse(randomBoolean());
     }
 
     @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        exists = in.readBoolean();
+    protected StopDataFrameAnalyticsResponse doParseInstance(XContentParser parser) throws IOException {
+        return StopDataFrameAnalyticsResponse.fromXContent(parser);
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeBoolean(exists);
+    protected boolean supportsUnknownFields() {
+        return true;
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java
new file mode 100644
index 0000000000000..957afc69dd0c2
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsSourceTests.randomSourceConfig;
+import static org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsDestTests.randomDestConfig;
+import static org.elasticsearch.client.ml.dataframe.OutlierDetectionTests.randomOutlierDetection;
+
+public class DataFrameAnalyticsConfigTests extends AbstractXContentTestCase<DataFrameAnalyticsConfig> {
+
+    public static DataFrameAnalyticsConfig randomDataFrameAnalyticsConfig() {
+        DataFrameAnalyticsConfig.Builder builder =
+            DataFrameAnalyticsConfig.builder(randomAlphaOfLengthBetween(1, 10))
+                .setSource(randomSourceConfig())
+                .setDest(randomDestConfig())
+                .setAnalysis(randomOutlierDetection());
+        if (randomBoolean()) {
+            builder.setAnalyzedFields(new FetchSourceContext(true,
+                generateRandomStringArray(10, 10, false, false),
+                generateRandomStringArray(10, 10, false, false)));
+        }
+        if (randomBoolean()) {
+            builder.setModelMemoryLimit(new ByteSizeValue(randomIntBetween(1, 16), randomFrom(ByteSizeUnit.MB, ByteSizeUnit.GB)));
+        }
+        if (randomBoolean()) {
+            builder.setCreateTime(Instant.now());
+        }
+        if (randomBoolean()) {
+            builder.setVersion(Version.CURRENT);
+        }
+        return builder.build();
+    }
+
+    @Override
+    protected DataFrameAnalyticsConfig createTestInstance() {
+        return randomDataFrameAnalyticsConfig();
+    }
+
+    @Override
+    protected DataFrameAnalyticsConfig doParseInstance(XContentParser parser) throws IOException {
+        return DataFrameAnalyticsConfig.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        // allow unknown fields in the root of the object only
+        return field -> !field.isEmpty();
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
+        namedXContent.addAll(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents());
+        namedXContent.addAll(new MlDataFrameAnalysisNamedXContentProvider().getNamedXContentParsers());
+        return new NamedXContentRegistry(namedXContent);
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsDestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsDestTests.java
new file mode 100644
index 0000000000000..dce7ca5204d57
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsDestTests.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+public class DataFrameAnalyticsDestTests extends AbstractXContentTestCase<DataFrameAnalyticsDest> {
+
+    public static DataFrameAnalyticsDest randomDestConfig() {
+        return DataFrameAnalyticsDest.builder()
+            .setIndex(randomAlphaOfLengthBetween(1, 10))
+            .setResultsField(randomBoolean() ? null : randomAlphaOfLengthBetween(1, 10))
+            .build();
+    }
+
+    @Override
+    protected DataFrameAnalyticsDest doParseInstance(XContentParser parser) throws IOException {
+        return DataFrameAnalyticsDest.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected DataFrameAnalyticsDest createTestInstance() {
+        return randomDestConfig();
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsSourceTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsSourceTests.java
new file mode 100644
index 0000000000000..d82e1999f3034
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsSourceTests.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+import java.util.function.Predicate;
+
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.client.ml.dataframe.QueryConfigTests.randomQueryConfig;
+
+
+public class DataFrameAnalyticsSourceTests extends AbstractXContentTestCase<DataFrameAnalyticsSource> {
+
+    public static DataFrameAnalyticsSource randomSourceConfig() {
+        return DataFrameAnalyticsSource.builder()
+            .setIndex(generateRandomStringArray(10, 10, false, false))
+            .setQueryConfig(randomBoolean() ? null : randomQueryConfig())
+            .build();
+    }
+
+    @Override
+    protected DataFrameAnalyticsSource doParseInstance(XContentParser parser) throws IOException {
+        return DataFrameAnalyticsSource.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        // allow unknown fields in the root of the object only as QueryConfig stores a Map<String, Object>
+        return field -> !field.isEmpty();
+    }
+
+    @Override
+    protected DataFrameAnalyticsSource createTestInstance() {
+        return randomSourceConfig();
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
+        return new NamedXContentRegistry(searchModule.getNamedXContents());
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStatsTests.java
new file mode 100644
index 0000000000000..fad02eac161c7
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsStatsTests.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe;
+
+import org.elasticsearch.client.ml.NodeAttributesTests;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
+
+public class DataFrameAnalyticsStatsTests extends ESTestCase {
+
+    public void testFromXContent() throws IOException {
+        xContentTester(this::createParser,
+            DataFrameAnalyticsStatsTests::randomDataFrameAnalyticsStats,
+            DataFrameAnalyticsStatsTests::toXContent,
+            DataFrameAnalyticsStats::fromXContent)
+            .supportsUnknownFields(true)
+            .randomFieldsExcludeFilter(field -> field.startsWith("node.attributes"))
+            .test();
+    }
+
+    public static DataFrameAnalyticsStats randomDataFrameAnalyticsStats() {
+        return new DataFrameAnalyticsStats(
+            randomAlphaOfLengthBetween(1, 10),
+            randomFrom(DataFrameAnalyticsState.values()),
+            randomBoolean() ? null : randomAlphaOfLength(10),
+            randomBoolean() ? null : randomIntBetween(0, 100),
+            randomBoolean() ? null : NodeAttributesTests.createRandom(),
+            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 20));
+    }
+
+    public static void toXContent(DataFrameAnalyticsStats stats, XContentBuilder builder) throws IOException {
+        builder.startObject();
+        builder.field(DataFrameAnalyticsStats.ID.getPreferredName(), stats.getId());
+        builder.field(DataFrameAnalyticsStats.STATE.getPreferredName(), stats.getState().value());
+        if (stats.getFailureReason() != null) {
+            builder.field(DataFrameAnalyticsStats.FAILURE_REASON.getPreferredName(), stats.getFailureReason());
+        }
+        if (stats.getProgressPercent() != null) {
+            builder.field(DataFrameAnalyticsStats.PROGRESS_PERCENT.getPreferredName(), stats.getProgressPercent());
+        }
+        if (stats.getNode() != null) {
+            builder.field(DataFrameAnalyticsStats.NODE.getPreferredName(), stats.getNode());
+        }
+        if (stats.getAssignmentExplanation() != null) {
+            builder.field(DataFrameAnalyticsStats.ASSIGNMENT_EXPLANATION.getPreferredName(), stats.getAssignmentExplanation());
+        }
+        builder.endObject();
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/OutlierDetectionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/OutlierDetectionTests.java
new file mode 100644
index 0000000000000..7307999a2bf71
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/OutlierDetectionTests.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.equalTo;
+
+public class OutlierDetectionTests extends AbstractXContentTestCase<OutlierDetection> {
+
+    public static OutlierDetection randomOutlierDetection() {
+        return OutlierDetection.builder()
+            .setNNeighbors(randomBoolean() ? null : randomIntBetween(1, 20))
+            .setMethod(randomBoolean() ? null : randomFrom(OutlierDetection.Method.values()))
+            .setFeatureInfluenceThreshold(randomBoolean() ? null : randomDoubleBetween(0.0, 1.0, true))
+            .build();
+    }
+
+    @Override
+    protected OutlierDetection doParseInstance(XContentParser parser) throws IOException {
+        return OutlierDetection.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected OutlierDetection createTestInstance() {
+        return randomOutlierDetection();
+    }
+
+    public void testGetParams_GivenDefaults() {
+        OutlierDetection outlierDetection = OutlierDetection.createDefault();
+        assertNull(outlierDetection.getNNeighbors());
+        assertNull(outlierDetection.getMethod());
+        assertNull(outlierDetection.getFeatureInfluenceThreshold());
+    }
+
+    public void testGetParams_GivenExplicitValues() {
+        OutlierDetection outlierDetection =
+            OutlierDetection.builder()
+                .setNNeighbors(42)
+                .setMethod(OutlierDetection.Method.LDOF)
+                .setFeatureInfluenceThreshold(0.5)
+                .build();
+        assertThat(outlierDetection.getNNeighbors(), equalTo(42));
+        assertThat(outlierDetection.getMethod(), equalTo(OutlierDetection.Method.LDOF));
+        assertThat(outlierDetection.getFeatureInfluenceThreshold(), closeTo(0.5, 1E-9));
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/QueryConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/QueryConfigTests.java
new file mode 100644
index 0000000000000..7413bc936a215
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/QueryConfigTests.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.ml.dataframe;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.MatchNoneQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.io.IOException;
+
+import static java.util.Collections.emptyList;
+
+public class QueryConfigTests extends AbstractXContentTestCase<QueryConfig> {
+
+    public static QueryConfig randomQueryConfig() {
+        QueryBuilder queryBuilder = randomBoolean() ? new MatchAllQueryBuilder() : new MatchNoneQueryBuilder();
+        return new QueryConfig(queryBuilder);
+    }
+
+    @Override
+    protected QueryConfig createTestInstance() {
+        return randomQueryConfig();
+    }
+
+    @Override
+    protected QueryConfig doParseInstance(XContentParser parser) throws IOException {
+        return QueryConfig.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
+        return new NamedXContentRegistry(searchModule.getNamedXContents());
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/filestructurefinder/FieldStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/filestructurefinder/FieldStatsTests.java
index daf6c4af90ddc..0d6e5e70290d9 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/filestructurefinder/FieldStatsTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/filestructurefinder/FieldStatsTests.java
@@ -43,6 +43,8 @@ static FieldStats createTestFieldStats() {
         Double maxValue = null;
         Double meanValue = null;
         Double medianValue = null;
+        String earliestTimestamp = null;
+        String latestTimestamp = null;
         boolean isMetric = randomBoolean();
         if (isMetric) {
             if (randomBoolean()) {
@@ -54,6 +56,12 @@ static FieldStats createTestFieldStats() {
             }
             meanValue = randomDouble();
             medianValue = randomDouble();
+        } else {
+            boolean isDate = randomBoolean();
+            if (isDate) {
+                earliestTimestamp = randomAlphaOfLength(20);
+                latestTimestamp = randomAlphaOfLength(20);
+            }
         }
 
         List<Map<String, Object>> topHits = new ArrayList<>();
@@ -68,7 +76,7 @@ static FieldStats createTestFieldStats() {
             topHits.add(topHit);
         }
 
-        return new FieldStats(count, cardinality, minValue, maxValue, meanValue, medianValue, topHits);
+        return new FieldStats(count, cardinality, minValue, maxValue, meanValue, medianValue, earliestTimestamp, latestTimestamp, topHits);
     }
 
     @Override
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/AnalysisConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/AnalysisConfigTests.java
index 3914697b768ba..a72de967d86b7 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/AnalysisConfigTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/AnalysisConfigTests.java
@@ -256,7 +256,7 @@ private static AnalysisConfig.Builder createValidCategorizationConfig() {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java
index 70e8b4296b0df..8794389066314 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java
@@ -168,7 +168,7 @@ public static Job createRandomizedJob() {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/process/TimingStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/process/TimingStatsTests.java
new file mode 100644
index 0000000000000..386bc6ac0b3dd
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/process/TimingStatsTests.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client.ml.job.process;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class TimingStatsTests extends AbstractXContentTestCase<TimingStats> {
+
+    private static final String JOB_ID = "my-job-id";
+
+    public static TimingStats createTestInstance(String jobId) {
+        return new TimingStats(
+            jobId,
+            randomLong(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble());
+    }
+
+    @Override
+    public TimingStats createTestInstance() {
+        return createTestInstance(randomAlphaOfLength(10));
+    }
+
+    @Override
+    protected TimingStats doParseInstance(XContentParser parser) {
+        return TimingStats.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    public void testConstructor() {
+        TimingStats stats = new TimingStats(JOB_ID, 7, 1.0, 2.0, 1.23, 7.89);
+
+        assertThat(stats.getJobId(), equalTo(JOB_ID));
+        assertThat(stats.getBucketCount(), equalTo(7L));
+        assertThat(stats.getMinBucketProcessingTimeMs(), equalTo(1.0));
+        assertThat(stats.getMaxBucketProcessingTimeMs(), equalTo(2.0));
+        assertThat(stats.getAvgBucketProcessingTimeMs(), equalTo(1.23));
+        assertThat(stats.getExponentialAvgBucketProcessingTimeMs(), equalTo(7.89));
+    }
+
+    public void testConstructor_NullValues() {
+        TimingStats stats = new TimingStats(JOB_ID, 7, null, null, null, null);
+
+        assertThat(stats.getJobId(), equalTo(JOB_ID));
+        assertThat(stats.getBucketCount(), equalTo(7L));
+        assertThat(stats.getMinBucketProcessingTimeMs(), nullValue());
+        assertThat(stats.getMaxBucketProcessingTimeMs(), nullValue());
+        assertThat(stats.getAvgBucketProcessingTimeMs(), nullValue());
+        assertThat(stats.getExponentialAvgBucketProcessingTimeMs(), nullValue());
+    }
+
+    public void testEquals() {
+        TimingStats stats1 = new TimingStats(JOB_ID, 7, 1.0, 2.0, 1.23, 7.89);
+        TimingStats stats2 = new TimingStats(JOB_ID, 7, 1.0, 2.0, 1.23, 7.89);
+        TimingStats stats3 = new TimingStats(JOB_ID, 7, 1.0, 3.0, 1.23, 7.89);
+
+        assertTrue(stats1.equals(stats1));
+        assertTrue(stats1.equals(stats2));
+        assertFalse(stats2.equals(stats3));
+    }
+
+    public void testHashCode() {
+        TimingStats stats1 = new TimingStats(JOB_ID, 7, 1.0, 2.0, 1.23, 7.89);
+        TimingStats stats2 = new TimingStats(JOB_ID, 7, 1.0, 2.0, 1.23, 7.89);
+        TimingStats stats3 = new TimingStats(JOB_ID, 7, 1.0, 3.0, 1.23, 7.89);
+
+        assertEquals(stats1.hashCode(), stats1.hashCode());
+        assertEquals(stats1.hashCode(), stats2.hashCode());
+        assertNotEquals(stats2.hashCode(), stats3.hashCode());
+    }
+}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/stats/JobStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/stats/JobStatsTests.java
index 5d00f879140e0..e7aa33f1d7a3b 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/stats/JobStatsTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/stats/JobStatsTests.java
@@ -24,6 +24,8 @@
 import org.elasticsearch.client.ml.job.process.DataCountsTests;
 import org.elasticsearch.client.ml.job.process.ModelSizeStats;
 import org.elasticsearch.client.ml.job.process.ModelSizeStatsTests;
+import org.elasticsearch.client.ml.job.process.TimingStats;
+import org.elasticsearch.client.ml.job.process.TimingStatsTests;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.client.ml.job.config.JobState;
@@ -42,12 +44,14 @@ public static JobStats createRandomInstance() {
         DataCounts dataCounts = DataCountsTests.createTestInstance(jobId);
 
         ModelSizeStats modelSizeStats = randomBoolean() ? ModelSizeStatsTests.createRandomized() : null;
+        TimingStats timingStats = randomBoolean() ? TimingStatsTests.createTestInstance(jobId) : null;
         ForecastStats forecastStats = randomBoolean() ? ForecastStatsTests.createRandom(1, 22) : null;
         NodeAttributes nodeAttributes = randomBoolean() ? NodeAttributesTests.createRandom() : null;
         String assigmentExplanation = randomBoolean() ? randomAlphaOfLength(10) : null;
         TimeValue openTime = randomBoolean() ? TimeValue.timeValueMillis(randomIntBetween(1, 10000)) : null;
 
-        return new JobStats(jobId, dataCounts, state, modelSizeStats, forecastStats, nodeAttributes, assigmentExplanation, openTime);
+        return new JobStats(
+            jobId, dataCounts, state, modelSizeStats, timingStats, forecastStats, nodeAttributes, assigmentExplanation, openTime);
     }
 
     @Override
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateTokenRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateTokenRequestTests.java
index 53f3e1d0f368c..760d5e52cb38f 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateTokenRequestTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateTokenRequestTests.java
@@ -66,31 +66,54 @@ public void testCreateTokenFromClientCredentials() {
         assertThat(Strings.toString(request), equalTo("{\"grant_type\":\"client_credentials\"}"));
     }
 
+    public void testCreateTokenFromKerberosTicket() {
+        final CreateTokenRequest request = CreateTokenRequest.kerberosGrant("top secret kerberos ticket".toCharArray());
+        assertThat(request.getGrantType(), equalTo("_kerberos"));
+        assertThat(request.getScope(), nullValue());
+        assertThat(request.getUsername(), nullValue());
+        assertThat(request.getPassword(), nullValue());
+        assertThat(request.getRefreshToken(), nullValue());
+        assertThat(new String(request.getKerberosTicket()), equalTo("top secret kerberos ticket"));
+        assertThat(Strings.toString(request), equalTo("{\"grant_type\":\"_kerberos\"," +
+                "\"kerberos_ticket\":\"top secret kerberos ticket\"}"));
+    }
+
     public void testEqualsAndHashCode() {
         final String grantType = randomAlphaOfLength(8);
         final String scope = randomBoolean() ? null : randomAlphaOfLength(6);
         final String username = randomBoolean() ? null : randomAlphaOfLengthBetween(4, 10);
         final char[] password = randomBoolean() ? null : randomAlphaOfLengthBetween(8, 12).toCharArray();
         final String refreshToken = randomBoolean() ? null : randomAlphaOfLengthBetween(12, 24);
-        final CreateTokenRequest request = new CreateTokenRequest(grantType, scope, username, password, refreshToken);
+        final char[] kerberosTicket = randomBoolean() ? null : randomAlphaOfLengthBetween(8, 12).toCharArray();
+        final CreateTokenRequest request = new CreateTokenRequest(grantType, scope, username, password, refreshToken, kerberosTicket);
         EqualsHashCodeTestUtils.checkEqualsAndHashCode(request,
-            r -> new CreateTokenRequest(r.getGrantType(), r.getScope(), r.getUsername(), r.getPassword(), r.getRefreshToken()),
+            r -> new CreateTokenRequest(r.getGrantType(), r.getScope(), r.getUsername(), r.getPassword(),
+                                        r.getRefreshToken(), r.getKerberosTicket()),
             this::mutate);
     }
 
     private CreateTokenRequest mutate(CreateTokenRequest req) {
-        switch (randomIntBetween(1, 5)) {
-            case 1:
-                return new CreateTokenRequest("g", req.getScope(), req.getUsername(), req.getPassword(), req.getRefreshToken());
-            case 2:
-                return new CreateTokenRequest(req.getGrantType(), "s", req.getUsername(), req.getPassword(), req.getRefreshToken());
-            case 3:
-                return new CreateTokenRequest(req.getGrantType(), req.getScope(), "u", req.getPassword(), req.getRefreshToken());
-            case 4:
-                final char[] password = {'p'};
-                return new CreateTokenRequest(req.getGrantType(), req.getScope(), req.getUsername(), password, req.getRefreshToken());
-            case 5:
-                return new CreateTokenRequest(req.getGrantType(), req.getScope(), req.getUsername(), req.getPassword(), "r");
+        switch (randomIntBetween(1, 6)) {
+        case 1:
+            return new CreateTokenRequest("g", req.getScope(), req.getUsername(), req.getPassword(), req.getRefreshToken(),
+                    req.getKerberosTicket());
+        case 2:
+            return new CreateTokenRequest(req.getGrantType(), "s", req.getUsername(), req.getPassword(), req.getRefreshToken(),
+                    req.getKerberosTicket());
+        case 3:
+            return new CreateTokenRequest(req.getGrantType(), req.getScope(), "u", req.getPassword(), req.getRefreshToken(),
+                    req.getKerberosTicket());
+        case 4:
+            final char[] password = { 'p' };
+            return new CreateTokenRequest(req.getGrantType(), req.getScope(), req.getUsername(), password, req.getRefreshToken(),
+                    req.getKerberosTicket());
+        case 5:
+            final char[] kerberosTicket = { 'k' };
+            return new CreateTokenRequest(req.getGrantType(), req.getScope(), req.getUsername(), req.getPassword(), req.getRefreshToken(),
+                    kerberosTicket);
+        case 6:
+            return new CreateTokenRequest(req.getGrantType(), req.getScope(), req.getUsername(), req.getPassword(), "r",
+                    req.getKerberosTicket());
         }
         throw new IllegalStateException("Bad random number");
     }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateTokenResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateTokenResponseTests.java
index f99ea668665dd..34a03647f6060 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateTokenResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateTokenResponseTests.java
@@ -37,6 +37,7 @@ public void testFromXContent() throws IOException {
         final String refreshToken = randomBoolean() ? null : randomAlphaOfLengthBetween(12, 24);
         final String scope = randomBoolean() ? null : randomAlphaOfLength(4);
         final String type = randomAlphaOfLength(6);
+        final String kerberosAuthenticationResponseToken = randomBoolean() ? null : randomAlphaOfLength(7);
 
         final XContentType xContentType = randomFrom(XContentType.values());
         final XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
@@ -50,6 +51,9 @@ public void testFromXContent() throws IOException {
         if (scope != null || randomBoolean()) {
             builder.field("scope", scope);
         }
+        if (kerberosAuthenticationResponseToken != null) {
+            builder.field("kerberos_authentication_response_token", kerberosAuthenticationResponseToken);
+        }
         builder.endObject();
         BytesReference xContent = BytesReference.bytes(builder);
 
@@ -59,5 +63,6 @@ public void testFromXContent() throws IOException {
         assertThat(response.getScope(), equalTo(scope));
         assertThat(response.getType(), equalTo(type));
         assertThat(response.getExpiresIn(), equalTo(expiresIn));
+        assertThat(response.getKerberosAuthenticationResponseToken(), equalTo(kerberosAuthenticationResponseToken));
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/ExecuteWatchResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/ExecuteWatchResponseTests.java
deleted file mode 100644
index 3e0ef4c8a5e5f..0000000000000
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/ExecuteWatchResponseTests.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client.watcher;
-
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.ObjectPath;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
-import java.util.Map;
-
-import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
-import static org.hamcrest.Matchers.is;
-
-public class ExecuteWatchResponseTests extends ESTestCase {
-
-    public static final String WATCH_ID_VALUE = "my_watch";
-    public static final String NODE_VALUE = "my_node";
-    public static final String TRIGGER_TYPE_VALUE = "manual";
-    public static final String STATE_VALUE = "executed";
-    public static final String STATE_KEY = "state";
-    public static final String TRIGGER_EVENT_KEY = "trigger_event";
-    public static final String TRIGGER_EVENT_TYPE_KEY = "type";
-    public static final String MESSAGES_KEY = "messages";
-    public static final String NODE_KEY = "node";
-    public static final String WATCH_ID_KEY = "watch_id";
-
-    public void testFromXContent() throws IOException {
-        xContentTester(this::createParser,
-            ExecuteWatchResponseTests::createTestInstance,
-            this::toXContent,
-            ExecuteWatchResponse::fromXContent)
-            .supportsUnknownFields(true)
-            .assertEqualsConsumer(this::assertEqualInstances)
-            .assertToXContentEquivalence(false)
-            .test();
-    }
-
-    private void assertEqualInstances(ExecuteWatchResponse expected, ExecuteWatchResponse actual) {
-        assertThat(expected.getRecordId(), is(actual.getRecordId()));
-
-        // This may have extra json, so lets just assume that if all of the original fields from the creation are there, then its equal
-        // This is the same code that is in createTestInstance in this class.
-        Map<String, Object> actualMap = actual.getRecordAsMap();
-        assertThat(ObjectPath.eval(WATCH_ID_KEY, actualMap), is(WATCH_ID_VALUE));
-        assertThat(ObjectPath.eval(NODE_KEY, actualMap), is(NODE_VALUE));
-        List<Object> messages = ObjectPath.eval(MESSAGES_KEY, actualMap);
-        assertThat(messages.size(), is(0));
-        assertThat(ObjectPath.eval(TRIGGER_EVENT_KEY + "." + TRIGGER_EVENT_TYPE_KEY, actualMap), is(TRIGGER_TYPE_VALUE));
-        assertThat(ObjectPath.eval(STATE_KEY, actualMap), is(STATE_VALUE));
-    }
-
-    private XContentBuilder toXContent(BytesReference bytes, XContentBuilder builder) throws IOException {
-        // EMPTY is safe here because we never use namedObject
-        try (InputStream stream = bytes.streamInput();
-             XContentParser parser = createParser(JsonXContent.jsonXContent, stream)) {
-            parser.nextToken();
-            builder.generator().copyCurrentStructure(parser);
-            return builder;
-        }
-    }
-
-    private XContentBuilder toXContent(ExecuteWatchResponse response, XContentBuilder builder) throws IOException {
-        builder.startObject();
-        builder.field("_id", response.getRecordId());
-        builder.field("watch_record");
-        toXContent(response.getRecord(), builder);
-        return builder.endObject();
-    }
-
-    private static ExecuteWatchResponse createTestInstance() {
-        String id = "my_watch_0-2015-06-02T23:17:55.124Z";
-        try {
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            builder.field(WATCH_ID_KEY, WATCH_ID_VALUE);
-            builder.field(NODE_KEY, NODE_VALUE);
-            builder.startArray(MESSAGES_KEY);
-            builder.endArray();
-            builder.startObject(TRIGGER_EVENT_KEY);
-            builder.field(TRIGGER_EVENT_TYPE_KEY, TRIGGER_TYPE_VALUE);
-            builder.endObject();
-            builder.field(STATE_KEY, STATE_VALUE);
-            builder.endObject();
-            BytesReference bytes = BytesReference.bytes(builder);
-            return new ExecuteWatchResponse(id, bytes);
-        }
-        catch (IOException e) {
-            throw new AssertionError(e);
-        }
-    }
-}
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/GetWatchResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/GetWatchResponseTests.java
index 7d919ef5f68c1..b69ea90a49e4a 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/GetWatchResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/GetWatchResponseTests.java
@@ -19,17 +19,13 @@
 package org.elasticsearch.client.watcher;
 
 import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.client.AbstractResponseTestCase;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.time.DateUtils;
-import org.elasticsearch.common.xcontent.DeprecationHandler;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.client.AbstractHlrcStreamableXContentTestCase;
 import org.elasticsearch.xpack.core.watcher.actions.ActionStatus;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
@@ -37,70 +33,17 @@
 import org.elasticsearch.xpack.core.watcher.watch.WatchStatus;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.time.ZonedDateTime;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.function.Predicate;
 
-public class GetWatchResponseTests extends
-    AbstractHlrcStreamableXContentTestCase<GetWatchResponse, org.elasticsearch.client.watcher.GetWatchResponse> {
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 
-    private static final String[] SHUFFLE_FIELDS_EXCEPTION = new String[] { "watch" };
+public class GetWatchResponseTests extends AbstractResponseTestCase<GetWatchResponse, org.elasticsearch.client.watcher.GetWatchResponse> {
 
     @Override
-    protected String[] getShuffleFieldsExceptions() {
-        return SHUFFLE_FIELDS_EXCEPTION;
-    }
-
-    @Override
-    protected ToXContent.Params getToXContentParams() {
-        return new ToXContent.MapParams(Collections.singletonMap("hide_headers", "false"));
-    }
-
-    @Override
-    protected Predicate<String> getRandomFieldsExcludeFilter() {
-        return f -> f.contains("watch") || f.contains("actions") || f.contains("headers");
-    }
-
-    @Override
-    protected void assertEqualInstances(GetWatchResponse expectedInstance, GetWatchResponse newInstance) {
-        if (expectedInstance.isFound() &&
-                expectedInstance.getSource().getContentType() != newInstance.getSource().getContentType()) {
-            /**
-             * The {@link GetWatchResponse#getContentType()} depends on the content type that
-             * was used to serialize the main object so we use the same content type than the
-             * <code>expectedInstance</code> to translate the watch of the <code>newInstance</code>.
-             */
-            XContent from = XContentFactory.xContent(newInstance.getSource().getContentType());
-            XContent to = XContentFactory.xContent(expectedInstance.getSource().getContentType());
-            final BytesReference newSource;
-            // It is safe to use EMPTY here because this never uses namedObject
-            try (InputStream stream = newInstance.getSource().getBytes().streamInput();
-                 XContentParser parser = XContentFactory.xContent(from.type()).createParser(NamedXContentRegistry.EMPTY,
-                     DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream)) {
-                parser.nextToken();
-                XContentBuilder builder = XContentFactory.contentBuilder(to.type());
-                builder.copyCurrentStructure(parser);
-                newSource = BytesReference.bytes(builder);
-            } catch (IOException e) {
-                throw new AssertionError(e);
-            }
-            newInstance = new GetWatchResponse(newInstance.getId(), newInstance.getVersion(),
-                newInstance.getSeqNo(), newInstance.getPrimaryTerm(),
-                newInstance.getStatus(), new XContentSource(newSource, expectedInstance.getSource().getContentType()));
-        }
-        super.assertEqualInstances(expectedInstance, newInstance);
-    }
-
-    @Override
-    protected GetWatchResponse createBlankInstance() {
-        return new GetWatchResponse();
-    }
-
-    @Override
-    protected GetWatchResponse createTestInstance() {
+    protected GetWatchResponse createServerTestInstance() {
         String id = randomAlphaOfLength(10);
         if (LuceneTestCase.rarely()) {
             return new GetWatchResponse(id);
@@ -113,6 +56,34 @@ protected GetWatchResponse createTestInstance() {
         return new GetWatchResponse(id, version, seqNo, primaryTerm, status, new XContentSource(source, XContentType.JSON));
     }
 
+    @Override
+    protected org.elasticsearch.client.watcher.GetWatchResponse doParseToClientInstance(XContentParser parser) throws IOException {
+        return org.elasticsearch.client.watcher.GetWatchResponse.fromXContent(parser);
+    }
+
+    @Override
+    protected void assertInstances(GetWatchResponse serverTestInstance, org.elasticsearch.client.watcher.GetWatchResponse clientInstance) {
+        assertThat(clientInstance.getId(), equalTo(serverTestInstance.getId()));
+        assertThat(clientInstance.getSeqNo(), equalTo(serverTestInstance.getSeqNo()));
+        assertThat(clientInstance.getPrimaryTerm(), equalTo(serverTestInstance.getPrimaryTerm()));
+        assertThat(clientInstance.getVersion(), equalTo(serverTestInstance.getVersion()));
+        if (serverTestInstance.getStatus() != null) {
+            assertThat(convertWatchStatus(clientInstance.getStatus()), equalTo(serverTestInstance.getStatus()));
+        } else {
+            assertThat(clientInstance.getStatus(), nullValue());
+        }
+        if (serverTestInstance.getSource() != null) {
+            assertThat(clientInstance.getSourceAsMap(), equalTo(serverTestInstance.getSource().getAsMap()));
+        } else {
+            assertThat(clientInstance.getSource(), nullValue());
+        }
+    }
+
+    @Override
+    protected ToXContent.Params getParams() {
+        return new ToXContent.MapParams(Map.of("hide_headers", "false"));
+    }
+
     private static BytesReference simpleWatch() {
         try {
             XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent());
@@ -179,58 +150,45 @@ private static ActionStatus.Execution randomExecution() {
         }
     }
 
-    @Override
-    public org.elasticsearch.client.watcher.GetWatchResponse doHlrcParseInstance(XContentParser parser) throws IOException {
-        return org.elasticsearch.client.watcher.GetWatchResponse.fromXContent(parser);
-    }
-
-    @Override
-    public GetWatchResponse convertHlrcToInternal(org.elasticsearch.client.watcher.GetWatchResponse instance) {
-        if (instance.isFound()) {
-            return new GetWatchResponse(instance.getId(), instance.getVersion(), instance.getSeqNo(), instance.getPrimaryTerm(),
-                convertHlrcToInternal(instance.getStatus()), new XContentSource(instance.getSource(), instance.getContentType()));
-        } else {
-            return new GetWatchResponse(instance.getId());
-        }
-    }
-
-    private static WatchStatus convertHlrcToInternal(org.elasticsearch.client.watcher.WatchStatus status) {
+    private static WatchStatus convertWatchStatus(org.elasticsearch.client.watcher.WatchStatus status) {
         final Map<String, ActionStatus> actions = new HashMap<>();
         for (Map.Entry<String, org.elasticsearch.client.watcher.ActionStatus> entry : status.getActions().entrySet()) {
-            actions.put(entry.getKey(), convertHlrcToInternal(entry.getValue()));
+            actions.put(entry.getKey(), convertActionStatus(entry.getValue()));
         }
         return new WatchStatus(status.version(),
-            convertHlrcToInternal(status.state()),
-            status.getExecutionState() == null ? null : convertHlrcToInternal(status.getExecutionState()),
+            convertWatchStatusState(status.state()),
+            status.getExecutionState() == null ? null : convertWatchStatus(status.getExecutionState()),
             status.lastChecked(), status.lastMetCondition(), actions, status.getHeaders()
         );
     }
 
-    private static ActionStatus convertHlrcToInternal(org.elasticsearch.client.watcher.ActionStatus actionStatus) {
-        return new ActionStatus(convertHlrcToInternal(actionStatus.ackStatus()),
-            actionStatus.lastExecution() == null ? null : convertHlrcToInternal(actionStatus.lastExecution()),
-            actionStatus.lastSuccessfulExecution() == null ? null : convertHlrcToInternal(actionStatus.lastSuccessfulExecution()),
-            actionStatus.lastThrottle() == null ? null : convertHlrcToInternal(actionStatus.lastThrottle())
+    private static ActionStatus convertActionStatus(org.elasticsearch.client.watcher.ActionStatus actionStatus) {
+        return new ActionStatus(convertAckStatus(actionStatus.ackStatus()),
+            actionStatus.lastExecution() == null ? null : convertActionStatusExecution(actionStatus.lastExecution()),
+            actionStatus.lastSuccessfulExecution() == null ? null : convertActionStatusExecution(actionStatus.lastSuccessfulExecution()),
+            actionStatus.lastThrottle() == null ? null : convertActionStatusThrottle(actionStatus.lastThrottle())
         );
     }
 
-    private static ActionStatus.AckStatus convertHlrcToInternal(org.elasticsearch.client.watcher.ActionStatus.AckStatus ackStatus) {
-        return new ActionStatus.AckStatus(ackStatus.timestamp(), convertHlrcToInternal(ackStatus.state()));
+    private static ActionStatus.AckStatus convertAckStatus(org.elasticsearch.client.watcher.ActionStatus.AckStatus ackStatus) {
+        return new ActionStatus.AckStatus(ackStatus.timestamp(), convertAckStatusState(ackStatus.state()));
     }
 
-    private static ActionStatus.AckStatus.State convertHlrcToInternal(org.elasticsearch.client.watcher.ActionStatus.AckStatus.State state) {
+    private static ActionStatus.AckStatus.State convertAckStatusState(
+        org.elasticsearch.client.watcher.ActionStatus.AckStatus.State state) {
         return ActionStatus.AckStatus.State.valueOf(state.name());
     }
 
-    private static WatchStatus.State convertHlrcToInternal(org.elasticsearch.client.watcher.WatchStatus.State state) {
+    private static WatchStatus.State convertWatchStatusState(org.elasticsearch.client.watcher.WatchStatus.State state) {
         return new WatchStatus.State(state.isActive(), state.getTimestamp());
     }
 
-    private static ExecutionState convertHlrcToInternal(org.elasticsearch.client.watcher.ExecutionState executionState) {
+    private static ExecutionState convertWatchStatus(org.elasticsearch.client.watcher.ExecutionState executionState) {
         return ExecutionState.valueOf(executionState.name());
     }
 
-    private static ActionStatus.Execution convertHlrcToInternal(org.elasticsearch.client.watcher.ActionStatus.Execution execution) {
+    private static ActionStatus.Execution convertActionStatusExecution(
+        org.elasticsearch.client.watcher.ActionStatus.Execution execution) {
         if (execution.successful()) {
             return ActionStatus.Execution.successful(execution.timestamp());
         } else {
@@ -238,7 +196,7 @@ private static ActionStatus.Execution convertHlrcToInternal(org.elasticsearch.cl
         }
     }
 
-    private static ActionStatus.Throttle convertHlrcToInternal(org.elasticsearch.client.watcher.ActionStatus.Throttle throttle) {
+    private static ActionStatus.Throttle convertActionStatusThrottle(org.elasticsearch.client.watcher.ActionStatus.Throttle throttle) {
         return new ActionStatus.Throttle(throttle.timestamp(), throttle.reason());
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/hlrc/DeleteWatchResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/hlrc/DeleteWatchResponseTests.java
index eebf2c9cef184..493375c451745 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/hlrc/DeleteWatchResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/hlrc/DeleteWatchResponseTests.java
@@ -18,17 +18,19 @@
  */
 package org.elasticsearch.client.watcher.hlrc;
 
+import org.elasticsearch.client.AbstractResponseTestCase;
 import org.elasticsearch.client.watcher.DeleteWatchResponse;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.client.AbstractHlrcXContentTestCase;
 
 import java.io.IOException;
 
-public class DeleteWatchResponseTests extends AbstractHlrcXContentTestCase<
+import static org.hamcrest.Matchers.equalTo;
+
+public class DeleteWatchResponseTests extends AbstractResponseTestCase<
     org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse, DeleteWatchResponse> {
 
     @Override
-    protected org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse createTestInstance() {
+    protected org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse createServerTestInstance() {
         String id = randomAlphaOfLength(10);
         long version = randomLongBetween(1, 10);
         boolean found = randomBoolean();
@@ -36,23 +38,15 @@ protected org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse createTes
     }
 
     @Override
-    protected org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse doParseInstance(XContentParser parser) throws IOException {
-        return org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse.fromXContent(parser);
-    }
-
-    @Override
-    public DeleteWatchResponse doHlrcParseInstance(XContentParser parser) throws IOException {
+    protected DeleteWatchResponse doParseToClientInstance(XContentParser parser) throws IOException {
         return DeleteWatchResponse.fromXContent(parser);
     }
 
     @Override
-    public org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse convertHlrcToInternal(DeleteWatchResponse instance) {
-        return new org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse(instance.getId(), instance.getVersion(),
-            instance.isFound());
-    }
-
-    @Override
-    protected boolean supportsUnknownFields() {
-        return false;
+    protected void assertInstances(org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse serverTestInstance,
+                                   DeleteWatchResponse clientInstance) {
+        assertThat(clientInstance.getId(), equalTo(serverTestInstance.getId()));
+        assertThat(clientInstance.getVersion(), equalTo(serverTestInstance.getVersion()));
+        assertThat(clientInstance.isFound(), equalTo(serverTestInstance.isFound()));
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/hlrc/ExecuteWatchResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/hlrc/ExecuteWatchResponseTests.java
index ace75517a9333..c1492eb53020f 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/hlrc/ExecuteWatchResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/hlrc/ExecuteWatchResponseTests.java
@@ -19,31 +19,23 @@
 
 package org.elasticsearch.client.watcher.hlrc;
 
+import org.elasticsearch.client.AbstractResponseTestCase;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.client.AbstractHlrcXContentTestCase;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
 
 import java.io.IOException;
 
-public class ExecuteWatchResponseTests
-    extends AbstractHlrcXContentTestCase<ExecuteWatchResponse, org.elasticsearch.client.watcher.ExecuteWatchResponse> {
+import static org.hamcrest.Matchers.equalTo;
 
-    @Override
-    public org.elasticsearch.client.watcher.ExecuteWatchResponse doHlrcParseInstance(XContentParser parser) throws IOException {
-        return org.elasticsearch.client.watcher.ExecuteWatchResponse.fromXContent(parser);
-    }
+public class ExecuteWatchResponseTests extends AbstractResponseTestCase<
+    ExecuteWatchResponse, org.elasticsearch.client.watcher.ExecuteWatchResponse> {
 
     @Override
-    public ExecuteWatchResponse convertHlrcToInternal(org.elasticsearch.client.watcher.ExecuteWatchResponse instance) {
-        return new ExecuteWatchResponse(instance.getRecordId(), instance.getRecord(), XContentType.JSON);
-    }
-
-    @Override
-    protected ExecuteWatchResponse createTestInstance() {
+    protected ExecuteWatchResponse createServerTestInstance() {
         String id = "my_watch_0-2015-06-02T23:17:55.124Z";
         try {
             XContentBuilder builder = XContentFactory.jsonBuilder();
@@ -66,12 +58,14 @@ protected ExecuteWatchResponse createTestInstance() {
     }
 
     @Override
-    protected ExecuteWatchResponse doParseInstance(XContentParser parser) throws IOException {
-        return ExecuteWatchResponse.fromXContent(parser);
+    protected org.elasticsearch.client.watcher.ExecuteWatchResponse doParseToClientInstance(XContentParser parser) throws IOException {
+        return org.elasticsearch.client.watcher.ExecuteWatchResponse.fromXContent(parser);
     }
 
     @Override
-    protected boolean supportsUnknownFields() {
-        return false;
+    protected void assertInstances(ExecuteWatchResponse serverTestInstance,
+                                   org.elasticsearch.client.watcher.ExecuteWatchResponse clientInstance) {
+        assertThat(clientInstance.getRecordId(), equalTo(serverTestInstance.getRecordId()));
+        assertThat(clientInstance.getRecordAsMap(), equalTo(serverTestInstance.getRecordSource().getAsMap()));
     }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/hlrc/PutWatchResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/hlrc/PutWatchResponseTests.java
index 9b65618cafc46..a47de0d15fda6 100644
--- a/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/hlrc/PutWatchResponseTests.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/watcher/hlrc/PutWatchResponseTests.java
@@ -18,17 +18,19 @@
  */
 package org.elasticsearch.client.watcher.hlrc;
 
+import org.elasticsearch.client.AbstractResponseTestCase;
 import org.elasticsearch.client.watcher.PutWatchResponse;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.client.AbstractHlrcXContentTestCase;
 
 import java.io.IOException;
 
-public class PutWatchResponseTests extends AbstractHlrcXContentTestCase<
+import static org.hamcrest.Matchers.equalTo;
+
+public class PutWatchResponseTests extends AbstractResponseTestCase<
     org.elasticsearch.protocol.xpack.watcher.PutWatchResponse, PutWatchResponse> {
 
     @Override
-    protected org.elasticsearch.protocol.xpack.watcher.PutWatchResponse createTestInstance() {
+    protected org.elasticsearch.protocol.xpack.watcher.PutWatchResponse createServerTestInstance() {
         String id = randomAlphaOfLength(10);
         long seqNo = randomNonNegativeLong();
         long primaryTerm = randomLongBetween(1, 20);
@@ -38,23 +40,17 @@ protected org.elasticsearch.protocol.xpack.watcher.PutWatchResponse createTestIn
     }
 
     @Override
-    protected org.elasticsearch.protocol.xpack.watcher.PutWatchResponse doParseInstance(XContentParser parser) throws IOException {
-        return org.elasticsearch.protocol.xpack.watcher.PutWatchResponse.fromXContent(parser);
-    }
-
-    @Override
-    public PutWatchResponse doHlrcParseInstance(XContentParser parser) throws IOException {
-        return org.elasticsearch.client.watcher.PutWatchResponse.fromXContent(parser);
-    }
-
-    @Override
-    public org.elasticsearch.protocol.xpack.watcher.PutWatchResponse convertHlrcToInternal(PutWatchResponse instance) {
-        return new org.elasticsearch.protocol.xpack.watcher.PutWatchResponse(instance.getId(), instance.getVersion(),
-            instance.getSeqNo(), instance.getPrimaryTerm(), instance.isCreated());
+    protected PutWatchResponse doParseToClientInstance(XContentParser parser) throws IOException {
+        return PutWatchResponse.fromXContent(parser);
     }
 
     @Override
-    protected boolean supportsUnknownFields() {
-        return false;
+    protected void assertInstances(org.elasticsearch.protocol.xpack.watcher.PutWatchResponse serverTestInstance,
+                                   PutWatchResponse clientInstance) {
+        assertThat(clientInstance.getId(), equalTo(serverTestInstance.getId()));
+        assertThat(clientInstance.getSeqNo(), equalTo(serverTestInstance.getSeqNo()));
+        assertThat(clientInstance.getPrimaryTerm(), equalTo(serverTestInstance.getPrimaryTerm()));
+        assertThat(clientInstance.getVersion(), equalTo(serverTestInstance.getVersion()));
+        assertThat(clientInstance.isCreated(), equalTo(serverTestInstance.isCreated()));
     }
 }
diff --git a/client/rest/build.gradle b/client/rest/build.gradle
index ee0317457118d..352b15699d081 100644
--- a/client/rest/build.gradle
+++ b/client/rest/build.gradle
@@ -44,7 +44,7 @@ dependencies {
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
 
-  testCompile "org.elasticsearch.client:test:${version}"
+  testCompile project(":client:test")
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest:${versions.hamcrest}"
@@ -68,7 +68,7 @@ forbiddenApisTest {
 }
 
 // JarHell is part of es server, which we don't want to pull in
-// TODO: Not anymore. Now in :libs:core
+// TODO: Not anymore. Now in :libs:elasticsearch-core
 jarHell.enabled=false
 
 testingConventions {
diff --git a/client/rest/src/main/java/org/elasticsearch/client/Request.java b/client/rest/src/main/java/org/elasticsearch/client/Request.java
index 2e4733201b12c..0bf9202486427 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/Request.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/Request.java
@@ -81,6 +81,10 @@ public void addParameter(String name, String value) {
         }
     }
 
+    public void addParameters(Map<String, String> paramSource){
+        paramSource.forEach(this::addParameter);
+    }
+
     /**
      * Query string parameters. The returned map is an unmodifiable view of the
      * map in the request so calls to {@link #addParameter(String, String)}
diff --git a/client/rest/src/main/java/org/elasticsearch/client/RequestOptions.java b/client/rest/src/main/java/org/elasticsearch/client/RequestOptions.java
index 0b2cdce3d52f7..3758b4c489ca2 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/RequestOptions.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/RequestOptions.java
@@ -170,10 +170,11 @@ public RequestOptions build() {
         /**
          * Add the provided header to the request.
          */
-        public void addHeader(String name, String value) {
+        public Builder addHeader(String name, String value) {
             Objects.requireNonNull(name, "header name cannot be null");
             Objects.requireNonNull(value, "header value cannot be null");
             this.headers.add(new ReqHeader(name, value));
+            return this;
         }
 
         /**
@@ -182,9 +183,10 @@ public void addHeader(String name, String value) {
          * response body gets streamed from a non-blocking HTTP connection on the
          * client side.
          */
-        public void setHttpAsyncResponseConsumerFactory(HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory) {
+        public Builder setHttpAsyncResponseConsumerFactory(HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory) {
             this.httpAsyncResponseConsumerFactory =
                     Objects.requireNonNull(httpAsyncResponseConsumerFactory, "httpAsyncResponseConsumerFactory cannot be null");
+            return this;
         }
 
         /**
@@ -204,8 +206,9 @@ public void setHttpAsyncResponseConsumerFactory(HttpAsyncResponseConsumerFactory
          * fail the request if the warnings returned don't
          * <strong>exactly</strong> match some set.
          */
-        public void setWarningsHandler(WarningsHandler warningsHandler) {
+        public Builder setWarningsHandler(WarningsHandler warningsHandler) {
             this.warningsHandler = warningsHandler;
+            return this;
         }
     }
 
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java
index 4aa69d3b3e9c1..8bc8d73fd2b26 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java
@@ -134,7 +134,7 @@ private static SSLContext getSslContext() throws Exception {
      * 12.0.1 so we pin to TLSv1.2 when running on an earlier JDK
      */
     private static String getProtocol() {
-        String version = AccessController.doPrivileged((PrivilegedAction<String>) () -> System.getProperty("java.version"));
+        String version = AccessController.doPrivileged((PrivilegedAction<String>) () -> System.getProperty("java.specification.version"));
         String[] components = version.split("\\.");
         if (components.length > 0) {
             final int major = Integer.valueOf(components[0]);
diff --git a/client/sniffer/build.gradle b/client/sniffer/build.gradle
index 2f9eeca6020c4..8b4d21537a109 100644
--- a/client/sniffer/build.gradle
+++ b/client/sniffer/build.gradle
@@ -35,14 +35,14 @@ publishing {
 }
 
 dependencies {
-  compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
+  compile project(":client:rest")
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
 
-  testCompile "org.elasticsearch.client:test:${version}"
+  testCompile project(":client:test")
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.elasticsearch:securemock:${versions.securemock}"
@@ -68,7 +68,7 @@ dependencyLicenses {
 }
 
 // JarHell is part of es server, which we don't want to pull in
-// TODO: Not anymore. Now in :libs:core
+// TODO: Not anymore. Now in :libs:elasticsearch-core
 jarHell.enabled=false
 
 testingConventions {
diff --git a/client/test/build.gradle b/client/test/build.gradle
index 184606e360791..169172736ecd3 100644
--- a/client/test/build.gradle
+++ b/client/test/build.gradle
@@ -44,7 +44,7 @@ forbiddenApisTest {
 }
 
 // JarHell is part of es server, which we don't want to pull in
-// TODO: Not anymore. Now in :libs:core
+// TODO: Not anymore. Now in :libs:elasticsearch-core
 jarHell.enabled=false
 
 // TODO: should we have licenses for our test deps?
diff --git a/client/transport/build.gradle b/client/transport/build.gradle
deleted file mode 100644
index c1e4503445bb7..0000000000000
--- a/client/transport/build.gradle
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-apply plugin: 'elasticsearch.build'
-apply plugin: 'nebula.maven-base-publish'
-apply plugin: 'nebula.maven-scm'
-
-group = 'org.elasticsearch.client'
-
-dependencies {
-  compile "org.elasticsearch:elasticsearch:${version}"
-  compile "org.elasticsearch.plugin:transport-netty4-client:${version}"
-  compile "org.elasticsearch.plugin:reindex-client:${version}"
-  compile "org.elasticsearch.plugin:lang-mustache-client:${version}"
-  compile "org.elasticsearch.plugin:percolator-client:${version}"
-  compile "org.elasticsearch.plugin:parent-join-client:${version}"
-  compile "org.elasticsearch.plugin:rank-eval-client:${version}"
-  testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
-  testCompile "junit:junit:${versions.junit}"
-  testCompile "org.hamcrest:hamcrest:${versions.hamcrest}"
-}
-
-dependencyLicenses {
-  dependencies = project.configurations.runtime.fileCollection {
-    it.group.startsWith('org.elasticsearch') == false
-  }
-}
-
-forbiddenApisTest {
-  // we don't use the core test-framework, no lucene classes present so we don't want the es-test-signatures to
-  // be pulled in
-  replaceSignatureFiles 'jdk-signatures', 'es-all-signatures'
-}
-
-testingConventions {
-  naming.clear()
-  naming {
-    Tests {
-      baseClass 'com.carrotsearch.randomizedtesting.RandomizedTest'
-    }
-  }
-}
diff --git a/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java b/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
deleted file mode 100644
index 4a78c38d06cc6..0000000000000
--- a/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.transport.client;
-
-import io.netty.util.ThreadDeathWatcher;
-import io.netty.util.concurrent.GlobalEventExecutor;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.network.NetworkModule;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.reindex.ReindexPlugin;
-import org.elasticsearch.join.ParentJoinPlugin;
-import org.elasticsearch.percolator.PercolatorPlugin;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.mustache.MustachePlugin;
-import org.elasticsearch.transport.Netty4Plugin;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-/**
- * A builder to create an instance of {@link TransportClient}. This class pre-installs the
- * {@link Netty4Plugin},
- * {@link ReindexPlugin},
- * {@link PercolatorPlugin},
- * {@link MustachePlugin},
- * {@link ParentJoinPlugin}
- * plugins for the client. These plugins are all the required modules for Elasticsearch.
- *
- * @deprecated {@link TransportClient} is deprecated in favour of the High Level REST client and will
- * be removed in Elasticsearch 8.0.
- */
-@SuppressWarnings({"unchecked","varargs"})
-@Deprecated
-public class PreBuiltTransportClient extends TransportClient {
-
-    static {
-        // initialize Netty system properties before triggering any Netty class loads
-        initializeNetty();
-    }
-
-    /**
-     * Netty wants to do some unwelcome things like use unsafe and replace a private field, or use a poorly considered buffer recycler. This
-     * method disables these things by default, but can be overridden by setting the corresponding system properties.
-     */
-    private static void initializeNetty() {
-        /*
-         * We disable three pieces of Netty functionality here:
-         *  - we disable Netty from being unsafe
-         *  - we disable Netty from replacing the selector key set
-         *  - we disable Netty from using the recycler
-         *
-         * While permissions are needed to read and set these, the permissions needed here are innocuous and thus should simply be granted
-         * rather than us handling a security exception here.
-         */
-        setSystemPropertyIfUnset("io.netty.noUnsafe", Boolean.toString(true));
-        setSystemPropertyIfUnset("io.netty.noKeySetOptimization", Boolean.toString(true));
-        setSystemPropertyIfUnset("io.netty.recycler.maxCapacityPerThread", Integer.toString(0));
-    }
-
-    @SuppressForbidden(reason = "set system properties to configure Netty")
-    private static void setSystemPropertyIfUnset(final String key, final String value) {
-        final String currentValue = System.getProperty(key);
-        if (currentValue == null) {
-            System.setProperty(key, value);
-        }
-    }
-
-    private static final Collection<Class<? extends Plugin>> PRE_INSTALLED_PLUGINS =
-            List.of(Netty4Plugin.class, ReindexPlugin.class, PercolatorPlugin.class, MustachePlugin.class, ParentJoinPlugin.class);
-
-    /**
-     * Creates a new transport client with pre-installed plugins.
-     *
-     * @param settings the settings passed to this transport client
-     * @param plugins  an optional array of additional plugins to run with this client
-     */
-    @SafeVarargs
-    public PreBuiltTransportClient(Settings settings, Class<? extends Plugin>... plugins) {
-        this(settings, Arrays.asList(plugins));
-    }
-
-    /**
-     * Creates a new transport client with pre-installed plugins.
-     *
-     * @param settings the settings passed to this transport client
-     * @param plugins  a collection of additional plugins to run with this client
-     */
-    public PreBuiltTransportClient(Settings settings, Collection<Class<? extends Plugin>> plugins) {
-        this(settings, plugins, null);
-    }
-
-    /**
-     * Creates a new transport client with pre-installed plugins.
-     *
-     * @param settings            the settings passed to this transport client
-     * @param plugins             a collection of additional plugins to run with this client
-     * @param hostFailureListener a failure listener that is invoked if a node is disconnected; this can be <code>null</code>
-     */
-    public PreBuiltTransportClient(
-        Settings settings,
-        Collection<Class<? extends Plugin>> plugins,
-        HostFailureListener hostFailureListener) {
-        super(settings, Settings.EMPTY, addPlugins(plugins, PRE_INSTALLED_PLUGINS), hostFailureListener);
-    }
-
-    @Override
-    public void close() {
-        super.close();
-        if (NetworkModule.TRANSPORT_TYPE_SETTING.exists(settings) == false
-            || NetworkModule.TRANSPORT_TYPE_SETTING.get(settings).equals(Netty4Plugin.NETTY_TRANSPORT_NAME)) {
-            try {
-                GlobalEventExecutor.INSTANCE.awaitInactivity(5, TimeUnit.SECONDS);
-            } catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-            }
-            try {
-                ThreadDeathWatcher.awaitInactivity(5, TimeUnit.SECONDS);
-            } catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-            }
-        }
-    }
-
-}
diff --git a/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java b/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java
deleted file mode 100644
index dbcfa8715f5f4..0000000000000
--- a/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.transport.client;
-
-import com.carrotsearch.randomizedtesting.RandomizedTest;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.network.NetworkModule;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.reindex.ReindexPlugin;
-import org.elasticsearch.join.ParentJoinPlugin;
-import org.elasticsearch.percolator.PercolatorPlugin;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.mustache.MustachePlugin;
-import org.elasticsearch.transport.Netty4Plugin;
-import org.junit.Test;
-
-import java.util.Arrays;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-public class PreBuiltTransportClientTests extends RandomizedTest {
-
-    @Test
-    public void testPluginInstalled() {
-        try (TransportClient client = new PreBuiltTransportClient(Settings.EMPTY)) {
-            Settings settings = client.settings();
-            assertEquals(Netty4Plugin.NETTY_TRANSPORT_NAME, NetworkModule.HTTP_DEFAULT_TYPE_SETTING.get(settings));
-            assertEquals(Netty4Plugin.NETTY_TRANSPORT_NAME, NetworkModule.TRANSPORT_DEFAULT_TYPE_SETTING.get(settings));
-        }
-    }
-
-    @Test
-    public void testInstallPluginTwice() {
-        for (Class<? extends Plugin> plugin :
-                Arrays.asList(ParentJoinPlugin.class, ReindexPlugin.class, PercolatorPlugin.class,
-                    MustachePlugin.class)) {
-            try {
-                new PreBuiltTransportClient(Settings.EMPTY, plugin);
-                fail("exception expected");
-            } catch (IllegalArgumentException ex) {
-                assertTrue("Expected message to start with [plugin already exists: ] but was instead [" + ex.getMessage() + "]",
-                        ex.getMessage().startsWith("plugin already exists: "));
-            }
-        }
-    }
-
-}
diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle
index 14a7e566587c4..ad4c77aaa5e5b 100644
--- a/distribution/archives/build.gradle
+++ b/distribution/archives/build.gradle
@@ -36,13 +36,13 @@ apply plugin: 'base'
 // parent to copy to the root of the distribution
 ext.logsDir = new File(buildDir, 'logs-hack/logs')
 task createLogsDir(type: EmptyDirTask) {
-  dir "${logsDir}"
-  dirMode 0755
+  dir = "${logsDir}"
+  dirMode = 0755
 }
 ext.pluginsDir= new File(buildDir, 'plugins-hack/plugins')
 task createPluginsDir(type: EmptyDirTask) {
-  dir "${pluginsDir}"
-  dirMode 0755
+  dir = "${pluginsDir}"
+  dirMode = 0755
 }
 
 CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, boolean oss, boolean jdk) {
diff --git a/distribution/build.gradle b/distribution/build.gradle
index 9c98101da5be3..9606604036101 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -117,7 +117,7 @@ task buildTransportModules {
 
 void copyModule(Sync copyTask, Project module) {
   copyTask.configure {
-    dependsOn { module.bundlePlugin }
+    dependsOn "${module.path}:bundlePlugin"
     from({ zipTree(module.bundlePlugin.outputs.files.singleFile) }) {
       includeEmptyDirs false
 
@@ -167,6 +167,7 @@ buildDefaultLog4jConfig.doLast(writeLog4jProperties)
 
 // copy log4j2.properties from modules that have it
 void copyLog4jProperties(Task buildTask, Project module) {
+  buildTask.dependsOn "${module.path}:bundlePlugin"
   buildTask.doFirst {
     FileTree tree = zipTree(module.bundlePlugin.outputs.files.singleFile)
     FileTree filtered = tree.matching {
@@ -241,7 +242,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
         // delay by using closures, since they have not yet been configured, so no jar task exists yet
         from { project(':server').jar }
         from { project(':server').configurations.runtime }
-        from { project(':libs:plugin-classloader').jar }
+        from { project(':libs:elasticsearch-plugin-classloader').jar }
         from { project(':distribution:tools:java-version-checker').jar }
         from { project(':distribution:tools:launchers').jar }
         into('tools/plugin-cli') {
@@ -381,6 +382,9 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
           if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') {
             details.mode = 0755
           }
+          if (details.name == 'src.zip') {
+            details.exclude()
+          }
         }
       }
     }
@@ -397,11 +401,11 @@ task run(type: RunTask) {
       setting 'xpack.graph.enabled', 'true'
       setting 'xpack.watcher.enabled', 'true'
       setting 'xpack.license.self_generated.type', 'trial'
-      setupCommand 'setupTestAdmin',
-        'bin/elasticsearch-users', 'useradd', 'elastic-admin', '-p', 'elastic-password', '-r', 'superuser'
     } else if (licenseType != 'basic') {
       throw new IllegalArgumentException("Unsupported self-generated license type: [" + licenseType + "[basic] or [trial].")
     }
+    setupCommand 'setupTestAdmin',
+            'bin/elasticsearch-users', 'useradd', 'elastic-admin', '-p', 'elastic-password', '-r', 'superuser'
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.monitoring.enabled', 'true'
     setting 'xpack.sql.enabled', 'true'
diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle
index ab55c737783e5..46f456c8c0493 100644
--- a/distribution/docker/build.gradle
+++ b/distribution/docker/build.gradle
@@ -2,6 +2,7 @@ import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin
 
 apply plugin: 'base'
 apply plugin: 'elasticsearch.test.fixtures'
@@ -44,7 +45,12 @@ project.ext {
       }
 
       into('config') {
+        /*
+         * Oss and default distribution can have different configuration, therefore we want to allow overriding the default configuration
+         * by creating config files in oss or default build-context sub-modules. 
+         */
         from project.projectDir.toPath().resolve("src/docker/config")
+        from project.projectDir.toPath().resolve(oss ? "oss-docker-build-context" : "docker-build-context").resolve("src/docker/config")
       }
 
       from(project.projectDir.toPath().resolve("src/docker/Dockerfile")) {
@@ -72,7 +78,10 @@ void addCopyDockerContextTask(final boolean oss) {
 }
 
 preProcessFixture {
-  dependsOn assemble
+  // don't add the tasks to build the docker images if we have no way of testing them
+  if (TestFixturesPlugin.dockerComposeSupported()) {
+    dependsOn assemble
+  }
 }
 
 postProcessFixture.doLast {
diff --git a/distribution/docker/docker-build-context/src/docker/config/log4j2.properties b/distribution/docker/docker-build-context/src/docker/config/log4j2.properties
new file mode 100644
index 0000000000000..40be55d2e1c3a
--- /dev/null
+++ b/distribution/docker/docker-build-context/src/docker/config/log4j2.properties
@@ -0,0 +1,121 @@
+status = error
+
+# log action execution errors for easier debugging
+logger.action.name = org.elasticsearch.action
+logger.action.level = debug
+
+appender.rolling.type = Console
+appender.rolling.name = rolling
+appender.rolling.layout.type = ESJsonLayout
+appender.rolling.layout.type_name = server
+
+rootLogger.level = info
+rootLogger.appenderRef.rolling.ref = rolling
+
+appender.deprecation_rolling.type = Console
+appender.deprecation_rolling.name = deprecation_rolling
+appender.deprecation_rolling.layout.type = ESJsonLayout
+appender.deprecation_rolling.layout.type_name = deprecation
+
+logger.deprecation.name = org.elasticsearch.deprecation
+logger.deprecation.level = warn
+logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
+logger.deprecation.additivity = false
+
+appender.index_search_slowlog_rolling.type = Console
+appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
+appender.index_search_slowlog_rolling.layout.type = ESJsonLayout
+appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog
+
+logger.index_search_slowlog_rolling.name = index.search.slowlog
+logger.index_search_slowlog_rolling.level = trace
+logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling
+logger.index_search_slowlog_rolling.additivity = false
+
+appender.index_indexing_slowlog_rolling.type = Console
+appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
+appender.index_indexing_slowlog_rolling.layout.type = ESJsonLayout
+appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog
+
+logger.index_indexing_slowlog.name = index.indexing.slowlog.index
+logger.index_indexing_slowlog.level = trace
+logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
+logger.index_indexing_slowlog.additivity = false
+
+appender.audit_rolling.type = Console
+appender.audit_rolling.name = audit_rolling
+appender.audit_rolling.layout.type = PatternLayout
+appender.audit_rolling.layout.pattern = {\
+"type": "audit", \
+"timestamp":"%d{yyyy-MM-dd'T'HH:mm:ss,SSSZ}"\
+%varsNotEmpty{, "node.name":"%enc{%map{node.name}}{JSON}"}\
+%varsNotEmpty{, "node.id":"%enc{%map{node.id}}{JSON}"}\
+%varsNotEmpty{, "host.name":"%enc{%map{host.name}}{JSON}"}\
+%varsNotEmpty{, "host.ip":"%enc{%map{host.ip}}{JSON}"}\
+%varsNotEmpty{, "event.type":"%enc{%map{event.type}}{JSON}"}\
+%varsNotEmpty{, "event.action":"%enc{%map{event.action}}{JSON}"}\
+%varsNotEmpty{, "user.name":"%enc{%map{user.name}}{JSON}"}\
+%varsNotEmpty{, "user.run_by.name":"%enc{%map{user.run_by.name}}{JSON}"}\
+%varsNotEmpty{, "user.run_as.name":"%enc{%map{user.run_as.name}}{JSON}"}\
+%varsNotEmpty{, "user.realm":"%enc{%map{user.realm}}{JSON}"}\
+%varsNotEmpty{, "user.run_by.realm":"%enc{%map{user.run_by.realm}}{JSON}"}\
+%varsNotEmpty{, "user.run_as.realm":"%enc{%map{user.run_as.realm}}{JSON}"}\
+%varsNotEmpty{, "user.roles":%map{user.roles}}\
+%varsNotEmpty{, "origin.type":"%enc{%map{origin.type}}{JSON}"}\
+%varsNotEmpty{, "origin.address":"%enc{%map{origin.address}}{JSON}"}\
+%varsNotEmpty{, "realm":"%enc{%map{realm}}{JSON}"}\
+%varsNotEmpty{, "url.path":"%enc{%map{url.path}}{JSON}"}\
+%varsNotEmpty{, "url.query":"%enc{%map{url.query}}{JSON}"}\
+%varsNotEmpty{, "request.method":"%enc{%map{request.method}}{JSON}"}\
+%varsNotEmpty{, "request.body":"%enc{%map{request.body}}{JSON}"}\
+%varsNotEmpty{, "request.id":"%enc{%map{request.id}}{JSON}"}\
+%varsNotEmpty{, "action":"%enc{%map{action}}{JSON}"}\
+%varsNotEmpty{, "request.name":"%enc{%map{request.name}}{JSON}"}\
+%varsNotEmpty{, "indices":%map{indices}}\
+%varsNotEmpty{, "opaque_id":"%enc{%map{opaque_id}}{JSON}"}\
+%varsNotEmpty{, "x_forwarded_for":"%enc{%map{x_forwarded_for}}{JSON}"}\
+%varsNotEmpty{, "transport.profile":"%enc{%map{transport.profile}}{JSON}"}\
+%varsNotEmpty{, "rule":"%enc{%map{rule}}{JSON}"}\
+%varsNotEmpty{, "event.category":"%enc{%map{event.category}}{JSON}"}\
+}%n
+# "node.name" node name from the `elasticsearch.yml` settings
+# "node.id" node id which should not change between cluster restarts
+# "host.name" unresolved hostname of the local node
+# "host.ip" the local bound ip (i.e. the ip listening for connections)
+# "event.type" a received REST request is translated into one or more transport requests. This indicates which processing layer generated the event "rest" or "transport" (internal)
+# "event.action" the name of the audited event, eg. "authentication_failed", "access_granted", "run_as_granted", etc.
+# "user.name" the subject name as authenticated by a realm
+# "user.run_by.name" the original authenticated subject name that is impersonating another one.
+# "user.run_as.name" if this "event.action" is of a run_as type, this is the subject name to be impersonated as.
+# "user.realm" the name of the realm that authenticated "user.name"
+# "user.run_by.realm" the realm name of the impersonating subject ("user.run_by.name")
+# "user.run_as.realm" if this "event.action" is of a run_as type, this is the realm name the impersonated user is looked up from
+# "user.roles" the roles array of the user; these are the roles that are granting privileges
+# "origin.type" it is "rest" if the event is originating (is in relation to) a REST request; possible other values are "transport" and "ip_filter"
+# "origin.address" the remote address and port of the first network hop, i.e. a REST proxy or another cluster node
+# "realm" name of a realm that has generated an "authentication_failed" or an "authentication_successful"; the subject is not yet authenticated
+# "url.path" the URI component between the port and the query string; it is percent (URL) encoded
+# "url.query" the URI component after the path and before the fragment; it is percent (URL) encoded
+# "request.method" the method of the HTTP request, i.e. one of GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH, TRACE, CONNECT
+# "request.body" the content of the request body entity, JSON escaped
+# "request.id" a synthentic identifier for the incoming request, this is unique per incoming request, and consistent across all audit events generated by that request
+# "action" an action is the most granular operation that is authorized and this identifies it in a namespaced way (internal)
+# "request.name" if the event is in connection to a transport message this is the name of the request class, similar to how rest requests are identified by the url path (internal)
+# "indices" the array of indices that the "action" is acting upon
+# "opaque_id" opaque value conveyed by the "X-Opaque-Id" request header
+# "x_forwarded_for" the addresses from the "X-Forwarded-For" request header, as a verbatim string value (not an array)
+# "transport.profile" name of the transport profile in case this is a "connection_granted" or "connection_denied" event
+# "rule" name of the applied rulee if the "origin.type" is "ip_filter"
+# "event.category" fixed value "elasticsearch-audit"
+
+logger.xpack_security_audit_logfile.name = org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail
+logger.xpack_security_audit_logfile.level = info
+logger.xpack_security_audit_logfile.appenderRef.audit_rolling.ref = audit_rolling
+logger.xpack_security_audit_logfile.additivity = false
+
+logger.xmlsig.name = org.apache.xml.security.signature.XMLSignature
+logger.xmlsig.level = error
+logger.samlxml_decrypt.name = org.opensaml.xmlsec.encryption.support.Decrypter
+logger.samlxml_decrypt.level = fatal
+logger.saml2_decrypt.name = org.opensaml.saml.saml2.encryption.Decrypter
+logger.saml2_decrypt.level = fatal
diff --git a/distribution/docker/src/docker/config/log4j2.properties b/distribution/docker/oss-docker-build-context/src/docker/config/log4j2.properties
similarity index 100%
rename from distribution/docker/src/docker/config/log4j2.properties
rename to distribution/docker/oss-docker-build-context/src/docker/config/log4j2.properties
diff --git a/distribution/src/bin/elasticsearch.bat b/distribution/src/bin/elasticsearch.bat
index 8ef77ac4c7fe9..03dc48728d52c 100644
--- a/distribution/src/bin/elasticsearch.bat
+++ b/distribution/src/bin/elasticsearch.bat
@@ -54,7 +54,7 @@ if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (
   exit /b 1
 )
 
-%JAVA% %ES_JAVA_OPTS% -Delasticsearch -Des.path.home="%ES_HOME%" -Des.path.conf="%ES_PATH_CONF%" -Des.distribution.flavor="%ES_DISTRIBUTION_FLAVOR%" -Des.distribution.type="%ES_DISTRIBUTION_TYPE%" -Des.bundled_jd="%ES_BUNDLED_JDK%" -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" !newparams!
+%JAVA% %ES_JAVA_OPTS% -Delasticsearch -Des.path.home="%ES_HOME%" -Des.path.conf="%ES_PATH_CONF%" -Des.distribution.flavor="%ES_DISTRIBUTION_FLAVOR%" -Des.distribution.type="%ES_DISTRIBUTION_TYPE%" -Des.bundled_jdk="%ES_BUNDLED_JDK%" -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" !newparams!
 
 endlocal
 endlocal
diff --git a/distribution/tools/keystore-cli/build.gradle b/distribution/tools/keystore-cli/build.gradle
index 5d57ef2e05597..de09e78e37493 100644
--- a/distribution/tools/keystore-cli/build.gradle
+++ b/distribution/tools/keystore-cli/build.gradle
@@ -20,9 +20,9 @@
 apply plugin: 'elasticsearch.build'
 
 dependencies {
-  compileOnly "org.elasticsearch:elasticsearch:${version}"
-  compileOnly "org.elasticsearch:elasticsearch-cli:${version}"
-  testCompile "org.elasticsearch.test:framework:${version}"
+  compileOnly project(":server")
+  compileOnly project(":libs:elasticsearch-cli")
+  testCompile project(":test:framework")
   testCompile 'com.google.jimfs:jimfs:1.1'
   testCompile 'com.google.guava:guava:18.0'
 }
diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
index f68a731edf8f7..568ddfe97df16 100644
--- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
+++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java
@@ -51,10 +51,12 @@
 import java.nio.file.Path;
 import java.security.GeneralSecurityException;
 import java.security.KeyStore;
+import java.security.MessageDigest;
 import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.Base64;
 import java.util.List;
+import java.util.Locale;
 import java.util.Set;
 
 import static org.hamcrest.Matchers.containsString;
@@ -126,6 +128,27 @@ public void testCannotReadStringFromClosedKeystore() throws Exception {
         assertThat(exception.getMessage(), containsString("closed"));
     }
 
+    public void testValueSHA256Digest() throws Exception {
+        final KeyStoreWrapper keystore = KeyStoreWrapper.create();
+        final String stringSettingKeyName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT) + "1";
+        final String stringSettingValue = randomAlphaOfLength(32);
+        keystore.setString(stringSettingKeyName, stringSettingValue.toCharArray());
+        final String fileSettingKeyName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT) + "2";
+        final byte[] fileSettingValue = randomByteArrayOfLength(32);
+        keystore.setFile(fileSettingKeyName, fileSettingValue);
+
+        final byte[] stringSettingHash = MessageDigest.getInstance("SHA-256").digest(stringSettingValue.getBytes(StandardCharsets.UTF_8));
+        assertThat(keystore.getSHA256Digest(stringSettingKeyName), equalTo(stringSettingHash));
+        final byte[] fileSettingHash = MessageDigest.getInstance("SHA-256").digest(fileSettingValue);
+        assertThat(keystore.getSHA256Digest(fileSettingKeyName), equalTo(fileSettingHash));
+
+        keystore.close();
+
+        // value hashes accessible even when the keystore is closed
+        assertThat(keystore.getSHA256Digest(stringSettingKeyName), equalTo(stringSettingHash));
+        assertThat(keystore.getSHA256Digest(fileSettingKeyName), equalTo(fileSettingHash));
+    }
+
     public void testUpgradeNoop() throws Exception {
         KeyStoreWrapper keystore = KeyStoreWrapper.create();
         SecureString seed = keystore.getString(KeyStoreWrapper.SEED_SETTING.getKey());
diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle
index 48bc899cd29b4..3db958c6ec41e 100644
--- a/distribution/tools/plugin-cli/build.gradle
+++ b/distribution/tools/plugin-cli/build.gradle
@@ -22,11 +22,11 @@ apply plugin: 'elasticsearch.build'
 archivesBaseName = 'elasticsearch-plugin-cli'
 
 dependencies {
-  compileOnly "org.elasticsearch:elasticsearch:${version}"
-  compileOnly "org.elasticsearch:elasticsearch-cli:${version}"
+  compileOnly project(":server")
+  compileOnly project(":libs:elasticsearch-cli")
   compile "org.bouncycastle:bcpg-jdk15on:${versions.bouncycastle}"
   compile "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}"
-  testCompile "org.elasticsearch.test:framework:${version}"
+  testCompile project(":test:framework")
   testCompile 'com.google.jimfs:jimfs:1.1'
   testCompile 'com.google.guava:guava:18.0'
 }
diff --git a/docs/README.asciidoc b/docs/README.asciidoc
index 0a2840b02e7e4..b7fad903f54b0 100644
--- a/docs/README.asciidoc
+++ b/docs/README.asciidoc
@@ -62,9 +62,9 @@ for its modifiers:
     them" which looks like `// TESTRESPONSE[s/\d+/$body.$_path/]`.
   * You can't use `// TESTRESPONSE` immediately after `// TESTSETUP`. Instead,
   consider using `// TEST[continued]` or rearrange your snippets.
-  * `// TESTRESPONSE[_cat]`: Add substitutions for testing `_cat` responses. Use
-  this after all other substitutions so it doesn't make other substitutions
-  difficult.
+  * `// TESTRESPONSE[non_json]`: Add substitutions for testing responses in a
+  format other than JSON. Use this after all other substitutions so it doesn't
+  make other substitutions difficult.
   * `// TESTRESPONSE[skip:reason]`: Skip the assertions specified by this
   response.
 * `// TESTSETUP`: Marks this snippet as the "setup" for all other snippets in
diff --git a/docs/build.gradle b/docs/build.gradle
index feda444301ec7..08cb2de971320 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -1137,3 +1137,12 @@ buildRestTests.setups['seats'] = '''
             {"theatre": "Graye", "cost": 33, "row": 2, "number": 6, "sold": false}
             {"index":{"_id": "4"}}
             {"theatre": "Skyline", "cost": 20, "row": 5, "number": 2, "sold": false}'''
+buildRestTests.setups['kibana_sample_data_ecommerce'] = '''
+  - do:
+        indices.create:
+          index: kibana_sample_data_ecommerce
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+'''
diff --git a/docs/groovy-api/anatomy.asciidoc b/docs/groovy-api/anatomy.asciidoc
deleted file mode 100644
index ba7cf83bb00cf..0000000000000
--- a/docs/groovy-api/anatomy.asciidoc
+++ /dev/null
@@ -1,102 +0,0 @@
-[[anatomy]]
-== API Anatomy
-
-Once a <<client,GClient>> has been
-obtained, all of Elasticsearch APIs can be executed on it. Each Groovy
-API is exposed using three different mechanisms.
-
-
-[[closure]]
-=== Closure Request
-
-The first type is to simply provide the request as a Closure, which
-automatically gets resolved into the respective request instance (for
-the index API, its the `IndexRequest` class). The API returns a special
-future, called `GActionFuture`. This is a groovier version of
-Elasticsearch Java `ActionFuture` (in turn a nicer extension to Java own
-`Future`) which allows to register listeners (closures) on it for
-success and failures, as well as blocking for the response. For example:
-
-[source,groovy]
---------------------------------------------------
-def indexR = client.index {
-    index "test"
-    type "_doc"
-    id "1"
-    source {
-        test = "value"
-        complex {
-            value1 = "value1"
-            value2 = "value2"
-        }
-    }
-}
-
-println "Indexed $indexR.response.id into $indexR.response.index/$indexR.response.type"
---------------------------------------------------
-
-In the above example, calling `indexR.response` will simply block for
-the response. We can also block for the response for a specific timeout:
-
-[source,groovy]
---------------------------------------------------
-IndexResponse response = indexR.response "5s" // block for 5 seconds, same as:
-response = indexR.response 5, TimeValue.SECONDS //
---------------------------------------------------
-
-We can also register closures that will be called on success and on
-failure:
-
-[source,groovy]
---------------------------------------------------
-indexR.success = {IndexResponse response ->
-    println "Indexed $response.id into $response.index/$response.type"
-}
-indexR.failure = {Throwable t ->
-    println "Failed to index: $t.message"
-}
---------------------------------------------------
-
-
-[[request]]
-=== Request
-
-This option allows to pass the actual instance of the request (instead
-of a closure) as a parameter. The rest is similar to the closure as a
-parameter option (the `GActionFuture` handling). For example:
-
-[source,groovy]
---------------------------------------------------
-def indexR = client.index (new IndexRequest(
-        index: "test",
-        type: "_doc",
-        id: "1",
-        source: {
-            test = "value"
-            complex {
-                value1 = "value1"
-                value2 = "value2"
-            }
-        }))
-
-println "Indexed $indexR.response.id into $indexR.response.index/$indexR.response.type"
---------------------------------------------------
-
-
-[[java-like]]
-=== Java Like
-
-The last option is to provide an actual instance of the API request, and
-an `ActionListener` for the callback. This is exactly like the Java API
-with the added `gexecute` which returns the `GActionFuture`:
-
-[source,groovy]
---------------------------------------------------
-def indexR = node.client.prepareIndex("test", "_doc", "1").setSource({
-    test = "value"
-    complex {
-        value1 = "value1"
-        value2 = "value2"
-    }
-}).gexecute()
---------------------------------------------------
diff --git a/docs/groovy-api/client.asciidoc b/docs/groovy-api/client.asciidoc
deleted file mode 100644
index c3c89e71bc535..0000000000000
--- a/docs/groovy-api/client.asciidoc
+++ /dev/null
@@ -1,59 +0,0 @@
-[[client]]
-== Client
-
-Obtaining an Elasticsearch Groovy `GClient` (a `GClient` is a simple
-wrapper on top of the Java `Client`) is simple. The most common way to
-get a client is by starting an embedded `Node` which acts as a node
-within the cluster.
-
-
-[[node-client]]
-=== Node Client
-
-A Node based client is the simplest form to get a `GClient` to start
-executing operations against Elasticsearch.
-
-[source,groovy]
---------------------------------------------------
-import org.elasticsearch.groovy.client.GClient
-import org.elasticsearch.groovy.node.GNode
-import static org.elasticsearch.groovy.node.GNodeBuilder.nodeBuilder
-
-// on startup
-
-GNode node = nodeBuilder().node();
-GClient client = node.client();
-
-// on shutdown
-
-node.close();
---------------------------------------------------
-
-Since Elasticsearch allows to configure it using JSON based settings,
-the configuration itself can be done using a closure that represent the
-JSON:
-
-[source,groovy]
---------------------------------------------------
-import org.elasticsearch.groovy.node.GNode
-import org.elasticsearch.groovy.node.GNodeBuilder
-import static org.elasticsearch.groovy.node.GNodeBuilder.*
-
-// on startup
-
-GNodeBuilder nodeBuilder = nodeBuilder();
-nodeBuilder.settings {
-    node {
-        client = true
-    }
-    cluster {
-        name = "test"
-    }
-}
-
-GNode node = nodeBuilder.node()
-
-// on shutdown
-
-node.stop().close()
---------------------------------------------------
diff --git a/docs/groovy-api/delete.asciidoc b/docs/groovy-api/delete.asciidoc
deleted file mode 100644
index 3d6547820041a..0000000000000
--- a/docs/groovy-api/delete.asciidoc
+++ /dev/null
@@ -1,16 +0,0 @@
-[[delete]]
-== Delete API
-
-The delete API is very similar to the
-// {javaclient}/java-docs-delete.html[]
-Java delete API, here is an
-example:
-
-[source,groovy]
---------------------------------------------------
-def deleteF = node.client.delete {
-    index "test"
-    type "_doc"
-    id "1"
-}
---------------------------------------------------
diff --git a/docs/groovy-api/get.asciidoc b/docs/groovy-api/get.asciidoc
deleted file mode 100644
index 2cac8429c3e79..0000000000000
--- a/docs/groovy-api/get.asciidoc
+++ /dev/null
@@ -1,19 +0,0 @@
-[[get]]
-== Get API
-
-The get API is very similar to the
-// {javaclient}/java-docs-get.html[]
-Java get API. The main benefit
-of using groovy is handling the source content. It can be automatically
-converted to a `Map` which means using Groovy to navigate it is simple:
-
-[source,groovy]
---------------------------------------------------
-def getF = node.client.get {
-    index "test"
-    type "_doc"
-    id "1"
-}
-
-println "Result of field2: $getF.response.source.complex.field2"
---------------------------------------------------
diff --git a/docs/groovy-api/index.asciidoc b/docs/groovy-api/index.asciidoc
deleted file mode 100644
index e1bb81856f15a..0000000000000
--- a/docs/groovy-api/index.asciidoc
+++ /dev/null
@@ -1,48 +0,0 @@
-= Groovy API
-
-include::../Versions.asciidoc[]
-
-[preface]
-== Preface
-
-This section describes the http://groovy-lang.org/[Groovy] API
-Elasticsearch provides. All Elasticsearch APIs are executed using a
-<<client,GClient>>, and are completely
-asynchronous in nature (they either accept a listener, or return a
-future).
-
-The Groovy API is a wrapper on top of the
-{javaclient}[Java API] exposing it in a groovier
-manner. The execution options for each API follow a similar manner and
-covered in <<anatomy>>.
-
-
-[[maven]]
-=== Maven Repository
-
-The Groovy API is hosted on
-http://search.maven.org/#search%7Cga%7C1%7Ca%3A%22elasticsearch-groovy%22[Maven
-Central].
-
-For example, you can define the latest version in your `pom.xml` file:
-
-["source","xml",subs="attributes"]
---------------------------------------------------
-<dependency>
-    <groupId>org.elasticsearch</groupId>
-    <artifactId>elasticsearch-groovy</artifactId>
-    <version>{version}</version>
-</dependency>
---------------------------------------------------
-
-include::anatomy.asciidoc[]
-
-include::client.asciidoc[]
-
-include::index_.asciidoc[]
-
-include::get.asciidoc[]
-
-include::delete.asciidoc[]
-
-include::search.asciidoc[]
diff --git a/docs/groovy-api/index_.asciidoc b/docs/groovy-api/index_.asciidoc
deleted file mode 100644
index deefb30e031a3..0000000000000
--- a/docs/groovy-api/index_.asciidoc
+++ /dev/null
@@ -1,32 +0,0 @@
-[[index_]]
-== Index API
-
-The index API is very similar to the
-// {javaclient}/java-docs-index.html[]
-Java index API. The Groovy
-extension to it is the ability to provide the indexed source using a
-closure. For example:
-
-[source,groovy]
---------------------------------------------------
-def indexR = client.index {
-    index "test"
-    type "_doc"
-    id "1"
-    source {
-        test = "value"
-        complex {
-            value1 = "value1"
-            value2 = "value2"
-        }
-    }
-}
---------------------------------------------------
-
-In the above example, the source closure itself gets transformed into an
-XContent (defaults to JSON). In order to change how the source closure
-is serialized, a global (static) setting can be set on the `GClient` by
-changing the `indexContentType` field.
-
-Note also that the `source` can be set using the typical Java based
-APIs, the `Closure` option is a Groovy extension.
diff --git a/docs/groovy-api/search.asciidoc b/docs/groovy-api/search.asciidoc
deleted file mode 100644
index 7834e45abc81a..0000000000000
--- a/docs/groovy-api/search.asciidoc
+++ /dev/null
@@ -1,116 +0,0 @@
-[[search]]
-== Search API
-
-The search API is very similar to the
-// {javaclient}/java-search.html[]
-Java search API. The Groovy
-extension allows to provide the search source to execute as a `Closure`
-including the query itself (similar to GORM criteria builder):
-
-[source,groovy]
---------------------------------------------------
-def search = node.client.search {
-    indices "test"
-    types "_doc"
-    source {
-        query {
-            term(test: "value")
-        }
-    }
-}
-
-search.response.hits.each {SearchHit hit ->
-    println "Got hit $hit.id from $hit.index/$hit.type"
-}
---------------------------------------------------
-
-It can also be executed using the "Java API" while still using a closure
-for the query:
-
-[source,groovy]
---------------------------------------------------
-def search = node.client.prepareSearch("test").setQuery({
-        term(test: "value")
-}).gexecute();
-
-search.response.hits.each {SearchHit hit ->
-    println "Got hit $hit.id from $hit.index/$hit.type"
-}
---------------------------------------------------
-
-The format of the search `Closure` follows the same JSON syntax as the
-{ref}/search-search.html[Search API] request.
-
-
-[[more-examples]]
-=== More examples
-
-Term query where multiple values are provided (see
-{ref}/query-dsl-terms-query.html[terms]):
-
-[source,groovy]
---------------------------------------------------
-def search = node.client.search {
-    indices "test"
-    types "_doc"
-    source {
-        query {
-            terms(test: ["value1", "value2"])
-        }
-    }
-}
---------------------------------------------------
-
-Query string (see
-{ref}/query-dsl-query-string-query.html[query string]):
-
-[source,groovy]
---------------------------------------------------
-def search = node.client.search {
-    indices "test"
-    types "_doc"
-    source {
-        query {
-            query_string(
-                fields: ["test"],
-                query: "value1 value2")
-        }
-    }
-}
---------------------------------------------------
-
-Pagination (see
-{ref}/search-request-from-size.html[from/size]):
-
-[source,groovy]
---------------------------------------------------
-def search = node.client.search {
-    indices "test"
-    types "_doc"
-    source {
-        from = 0
-        size = 10
-        query {
-            term(test: "value")
-        }
-    }
-}
---------------------------------------------------
-
-Sorting (see {ref}/search-request-sort.html[sort]):
-
-[source,groovy]
---------------------------------------------------
-def search = node.client.search {
-    indices "test"
-    types "_doc"
-    source {
-        query {
-            term(test: "value")
-        }
-        sort = [
-            date : [ order: "desc"]
-        ]
-    }
-}
---------------------------------------------------
diff --git a/docs/java-api/admin/cluster/health.asciidoc b/docs/java-api/admin/cluster/health.asciidoc
deleted file mode 100644
index 615a011cf72c9..0000000000000
--- a/docs/java-api/admin/cluster/health.asciidoc
+++ /dev/null
@@ -1,76 +0,0 @@
-[[java-admin-cluster-health]]
-==== Cluster Health
-
-[[java-admin-cluster-health-health]]
-===== Health
-
-The cluster health API allows to get a very simple status on the health of the cluster and also can give you
-some technical information about the cluster status per index:
-
-[source,java]
---------------------------------------------------
-ClusterHealthResponse healths = client.admin().cluster().prepareHealth().get(); <1>
-String clusterName = healths.getClusterName();              <2>
-int numberOfDataNodes = healths.getNumberOfDataNodes();     <3>
-int numberOfNodes = healths.getNumberOfNodes();             <4>
-
-for (ClusterIndexHealth health : healths.getIndices().values()) { <5>
-    String index = health.getIndex();                       <6>
-    int numberOfShards = health.getNumberOfShards();        <7>
-    int numberOfReplicas = health.getNumberOfReplicas();    <8>
-    ClusterHealthStatus status = health.getStatus();        <9>
-}
---------------------------------------------------
-<1> Get information for all indices
-<2> Access the cluster name
-<3> Get the total number of data nodes
-<4> Get the total number of nodes
-<5> Iterate over all indices
-<6> Index name
-<7> Number of shards
-<8> Number of replicas
-<9> Index status
-
-[[java-admin-cluster-health-wait-status]]
-===== Wait for status
-
-You can use the cluster health API to wait for a specific status for the whole cluster or for a given index:
-
-[source,java]
---------------------------------------------------
-client.admin().cluster().prepareHealth()            <1>
-        .setWaitForYellowStatus()                   <2>
-        .get();
-client.admin().cluster().prepareHealth("company")   <3>
-        .setWaitForGreenStatus()                    <4>
-        .get();
-
-client.admin().cluster().prepareHealth("employee")  <5>
-        .setWaitForGreenStatus()                    <6>
-        .setTimeout(TimeValue.timeValueSeconds(2))  <7>
-        .get();
---------------------------------------------------
-<1> Prepare a health request
-<2> Wait for the cluster being yellow
-<3> Prepare the health request for index `company`
-<4> Wait for the index being green
-<5> Prepare the health request for index `employee`
-<6> Wait for the index being green
-<7> Wait at most for 2 seconds
-
-If the index does not have the expected status and you want to fail in that case, you need
-to explicitly interpret the result:
-
-[source,java]
---------------------------------------------------
-ClusterHealthResponse response = client.admin().cluster().prepareHealth("company")
-        .setWaitForGreenStatus()    <1>
-        .get();
-
-ClusterHealthStatus status = response.getIndices().get("company").getStatus();
-if (!status.equals(ClusterHealthStatus.GREEN)) {
-    throw new RuntimeException("Index is in " + status + " state"); <2>
-}
---------------------------------------------------
-<1> Wait for the index being green
-<2> Throw an exception if not `GREEN`
diff --git a/docs/java-api/admin/cluster/index.asciidoc b/docs/java-api/admin/cluster/index.asciidoc
deleted file mode 100644
index 4e1850a34fe47..0000000000000
--- a/docs/java-api/admin/cluster/index.asciidoc
+++ /dev/null
@@ -1,16 +0,0 @@
-[[java-admin-cluster]]
-=== Cluster Administration
-
-To access cluster Java API, you need to call `cluster()` method from an <<java-admin,`AdminClient`>>:
-
-[source,java]
---------------------------------------------------
-ClusterAdminClient clusterAdminClient = client.admin().cluster();
---------------------------------------------------
-
-[NOTE]
-In the rest of this guide, we will use `client.admin().cluster()`.
-
-include::health.asciidoc[]
-
-include::stored-scripts.asciidoc[]
diff --git a/docs/java-api/admin/cluster/stored-scripts.asciidoc b/docs/java-api/admin/cluster/stored-scripts.asciidoc
deleted file mode 100644
index 5ebf89e92be55..0000000000000
--- a/docs/java-api/admin/cluster/stored-scripts.asciidoc
+++ /dev/null
@@ -1,29 +0,0 @@
-[[stored-scripts]]
-==== Stored Scripts API
-
-The stored script API allows one to interact with scripts and templates
-stored in Elasticsearch. It can be used to create, update, get,
-and delete stored scripts and templates.
-
-[source,java]
---------------------------------------------------
-PutStoredScriptResponse response = client.admin().cluster().preparePutStoredScript()
-                .setId("script1")
-                .setContent(new BytesArray("{\"script\": {\"lang\": \"painless\", \"source\": \"_score * doc['my_numeric_field'].value\"} }"), XContentType.JSON)
-                .get();
-
-GetStoredScriptResponse response = client().admin().cluster().prepareGetStoredScript()
-                .setId("script1")
-                .get();
-
-DeleteStoredScriptResponse response = client().admin().cluster().prepareDeleteStoredScript()
-                .setId("script1")
-                .get();
---------------------------------------------------
-
-To store templates simply use "mustache" for the scriptLang.
-
-===== Script Language
-
-The put stored script API allows one to set the language of the stored script.
-If one is not provided the default scripting language will be used.
diff --git a/docs/java-api/admin/index.asciidoc b/docs/java-api/admin/index.asciidoc
deleted file mode 100644
index 41599a82c7b3a..0000000000000
--- a/docs/java-api/admin/index.asciidoc
+++ /dev/null
@@ -1,18 +0,0 @@
-[[java-admin]]
-== Java API Administration
-
-Elasticsearch provides a full Java API to deal with administration tasks.
-
-To access them, you need to call `admin()` method from a client to get an `AdminClient`:
-
-[source,java]
---------------------------------------------------
-AdminClient adminClient = client.admin();
---------------------------------------------------
-
-[NOTE]
-In the rest of this guide, we will use `client.admin()`.
-
-include::indices/index.asciidoc[]
-
-include::cluster/index.asciidoc[]
diff --git a/docs/java-api/admin/indices/create-index.asciidoc b/docs/java-api/admin/indices/create-index.asciidoc
deleted file mode 100644
index 34b776bd04e23..0000000000000
--- a/docs/java-api/admin/indices/create-index.asciidoc
+++ /dev/null
@@ -1,28 +0,0 @@
-[[java-admin-indices-create-index]]
-==== Create Index
-
-Using an <<java-admin-indices,`IndicesAdminClient`>>, you can create an index with all default settings and no mapping:
-
-[source,java]
---------------------------------------------------
-client.admin().indices().prepareCreate("twitter").get();
---------------------------------------------------
-
-[float]
-[[java-admin-indices-create-index-settings]]
-===== Index Settings
-
-Each index created can have specific settings associated with it.
-
-[source,java]
---------------------------------------------------
-client.admin().indices().prepareCreate("twitter")
-        .setSettings(Settings.builder()             <1>
-                .put("index.number_of_shards", 3)
-                .put("index.number_of_replicas", 2)
-        )
-        .get();                                     <2>
---------------------------------------------------
-<1> Settings for this index
-<2> Execute the action and wait for the result
-
diff --git a/docs/java-api/admin/indices/get-settings.asciidoc b/docs/java-api/admin/indices/get-settings.asciidoc
deleted file mode 100644
index 844aaf65ec9b5..0000000000000
--- a/docs/java-api/admin/indices/get-settings.asciidoc
+++ /dev/null
@@ -1,22 +0,0 @@
-[[java-admin-indices-get-settings]]
-==== Get Settings
-
-The get settings API allows to retrieve settings of index/indices:
-
-[source,java]
---------------------------------------------------
-GetSettingsResponse response = client.admin().indices()
-        .prepareGetSettings("company", "employee").get();                           <1>
-for (ObjectObjectCursor<String, Settings> cursor : response.getIndexToSettings()) { <2>
-    String index = cursor.key;                                                      <3>
-    Settings settings = cursor.value;                                               <4>
-    Integer shards = settings.getAsInt("index.number_of_shards", null);             <5>
-    Integer replicas = settings.getAsInt("index.number_of_replicas", null);         <6>
-}
---------------------------------------------------
-<1> Get settings for indices `company` and `employee`
-<2> Iterate over results
-<3> Index name
-<4> Settings for the given index
-<5> Number of shards for this index
-<6> Number of replicas for this index
diff --git a/docs/java-api/admin/indices/index.asciidoc b/docs/java-api/admin/indices/index.asciidoc
deleted file mode 100644
index bbd365076c72e..0000000000000
--- a/docs/java-api/admin/indices/index.asciidoc
+++ /dev/null
@@ -1,21 +0,0 @@
-[[java-admin-indices]]
-=== Indices Administration
-
-To access indices Java API, you need to call `indices()` method from an <<java-admin,`AdminClient`>>:
-
-[source,java]
---------------------------------------------------
-IndicesAdminClient indicesAdminClient = client.admin().indices();
---------------------------------------------------
-
-[NOTE]
-In the rest of this guide, we will use `client.admin().indices()`.
-
-include::create-index.asciidoc[]
-
-include::put-mapping.asciidoc[]
-
-include::refresh.asciidoc[]
-
-include::get-settings.asciidoc[]
-include::update-settings.asciidoc[]
diff --git a/docs/java-api/admin/indices/put-mapping.asciidoc b/docs/java-api/admin/indices/put-mapping.asciidoc
deleted file mode 100644
index d63a498d994d2..0000000000000
--- a/docs/java-api/admin/indices/put-mapping.asciidoc
+++ /dev/null
@@ -1,30 +0,0 @@
-[[java-admin-indices-put-mapping]]
-
-==== Put Mapping
-
-You can add mappings at index creation time:
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-tests}/IndicesDocumentationIT.java[index-with-mapping]
---------------------------------------------------
-<1> <<java-admin-indices-create-index,Creates an index>> called `twitter`
-<2> Add a `_doc` type with a field called `message` that has the datatype `text`.
-
-There are several variants of the above `addMapping` method, some taking an
-`XContentBuilder` or a `Map` with the mapping definition as arguments. Make sure
-to check the javadocs to pick the simplest one for your use case.
-
-The PUT mapping API also allows for updating the mapping after index
-creation. In this case you can provide the mapping as a String similar
-to the REST API syntax:
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-tests}/IndicesDocumentationIT.java[putMapping-request-source]
---------------------------------------------------
-<1> Puts a mapping on existing index called `twitter`
-<2> Adds a new field `name` to the mapping
-<3> The type can be also provided within the source
-
-:base-dir!:
diff --git a/docs/java-api/admin/indices/refresh.asciidoc b/docs/java-api/admin/indices/refresh.asciidoc
deleted file mode 100644
index 856c270daf368..0000000000000
--- a/docs/java-api/admin/indices/refresh.asciidoc
+++ /dev/null
@@ -1,19 +0,0 @@
-[[java-admin-indices-refresh]]
-==== Refresh
-
-The refresh API allows to explicitly refresh one or more index:
-
-[source,java]
---------------------------------------------------
-client.admin().indices().prepareRefresh().get(); <1>
-client.admin().indices()
-        .prepareRefresh("twitter")               <2>
-        .get();
-client.admin().indices()
-        .prepareRefresh("twitter", "company")   <3>
-        .get();
---------------------------------------------------
-<1> Refresh all indices
-<2> Refresh one index
-<3> Refresh many indices
-
diff --git a/docs/java-api/admin/indices/update-settings.asciidoc b/docs/java-api/admin/indices/update-settings.asciidoc
deleted file mode 100644
index 9c2cba2adf03b..0000000000000
--- a/docs/java-api/admin/indices/update-settings.asciidoc
+++ /dev/null
@@ -1,16 +0,0 @@
-[[java-admin-indices-update-settings]]
-==== Update Indices Settings
-
-You can change index settings by calling:
-
-[source,java]
---------------------------------------------------
-client.admin().indices().prepareUpdateSettings("twitter")   <1>
-        .setSettings(Settings.builder()                     <2>
-                .put("index.number_of_replicas", 0)
-        )
-        .get();
---------------------------------------------------
-<1> Index to update
-<2> Settings
-
diff --git a/docs/java-api/aggregations/bucket.asciidoc b/docs/java-api/aggregations/bucket.asciidoc
deleted file mode 100644
index fe2e0ea9be309..0000000000000
--- a/docs/java-api/aggregations/bucket.asciidoc
+++ /dev/null
@@ -1,33 +0,0 @@
-[[java-aggregations-bucket]]
-
-include::bucket/global-aggregation.asciidoc[]
-
-include::bucket/filter-aggregation.asciidoc[]
-
-include::bucket/filters-aggregation.asciidoc[]
-
-include::bucket/missing-aggregation.asciidoc[]
-
-include::bucket/nested-aggregation.asciidoc[]
-
-include::bucket/reverse-nested-aggregation.asciidoc[]
-
-include::bucket/children-aggregation.asciidoc[]
-
-include::bucket/terms-aggregation.asciidoc[]
-
-include::bucket/significantterms-aggregation.asciidoc[]
-
-include::bucket/range-aggregation.asciidoc[]
-
-include::bucket/daterange-aggregation.asciidoc[]
-
-include::bucket/iprange-aggregation.asciidoc[]
-
-include::bucket/histogram-aggregation.asciidoc[]
-
-include::bucket/datehistogram-aggregation.asciidoc[]
-
-include::bucket/geodistance-aggregation.asciidoc[]
-
-include::bucket/geohashgrid-aggregation.asciidoc[]
diff --git a/docs/java-api/aggregations/bucket/children-aggregation.asciidoc b/docs/java-api/aggregations/bucket/children-aggregation.asciidoc
deleted file mode 100644
index f6a23fdafe976..0000000000000
--- a/docs/java-api/aggregations/bucket/children-aggregation.asciidoc
+++ /dev/null
@@ -1,35 +0,0 @@
-[[java-aggs-bucket-children]]
-==== Children Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-bucket-children-aggregation.html[Children Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilder aggregation =
-    AggregationBuilders
-        .children("agg", "reseller"); <1>
---------------------------------------------------
-1. `"agg"` is the name of the aggregation and `"reseller"` is the child type
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.join.aggregations.Children;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Children agg = sr.getAggregations().get("agg");
-agg.getDocCount(); // Doc count
---------------------------------------------------
diff --git a/docs/java-api/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/java-api/aggregations/bucket/datehistogram-aggregation.asciidoc
deleted file mode 100644
index 610262b046c21..0000000000000
--- a/docs/java-api/aggregations/bucket/datehistogram-aggregation.asciidoc
+++ /dev/null
@@ -1,73 +0,0 @@
-[[java-aggs-bucket-datehistogram]]
-==== Date Histogram Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-bucket-datehistogram-aggregation.html[Date Histogram Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilder aggregation =
-        AggregationBuilders
-                .dateHistogram("agg")
-                .field("dateOfBirth")
-                .calendarInterval(DateHistogramInterval.YEAR);
---------------------------------------------------
-
-Or if you want to set an interval of 10 days:
-
-[source,java]
---------------------------------------------------
-AggregationBuilder aggregation =
-        AggregationBuilders
-                .dateHistogram("agg")
-                .field("dateOfBirth")
-                .fixedInterval(DateHistogramInterval.days(10));
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Histogram agg = sr.getAggregations().get("agg");
-
-// For each entry
-for (Histogram.Bucket entry : agg.getBuckets()) {
-    DateTime key = (DateTime) entry.getKey();    // Key
-    String keyAsString = entry.getKeyAsString(); // Key as String
-    long docCount = entry.getDocCount();         // Doc count
-
-    logger.info("key [{}], date [{}], doc_count [{}]", keyAsString, key.getYear(), docCount);
-}
---------------------------------------------------
-
-This will basically produce for the first example:
-
-[source,text]
---------------------------------------------------
-key [1942-01-01T00:00:00.000Z], date [1942], doc_count [1]
-key [1945-01-01T00:00:00.000Z], date [1945], doc_count [1]
-key [1946-01-01T00:00:00.000Z], date [1946], doc_count [1]
-...
-key [2005-01-01T00:00:00.000Z], date [2005], doc_count [1]
-key [2007-01-01T00:00:00.000Z], date [2007], doc_count [2]
-key [2008-01-01T00:00:00.000Z], date [2008], doc_count [3]
---------------------------------------------------
-
-===== Order
-
-Supports the same order functionality as the <<java-aggs-bucket-terms,`Terms Aggregation`>>.
diff --git a/docs/java-api/aggregations/bucket/daterange-aggregation.asciidoc b/docs/java-api/aggregations/bucket/daterange-aggregation.asciidoc
deleted file mode 100644
index fa8f31e8cd0b7..0000000000000
--- a/docs/java-api/aggregations/bucket/daterange-aggregation.asciidoc
+++ /dev/null
@@ -1,59 +0,0 @@
-[[java-aggs-bucket-daterange]]
-==== Date Range Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-bucket-daterange-aggregation.html[Date Range Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilder aggregation =
-        AggregationBuilders
-                .dateRange("agg")
-                .field("dateOfBirth")
-                .format("yyyy")
-                .addUnboundedTo("1950")    // from -infinity to 1950 (excluded)
-                .addRange("1950", "1960")  // from 1950 to 1960 (excluded)
-                .addUnboundedFrom("1960"); // from 1960 to +infinity
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.range.Range;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Range agg = sr.getAggregations().get("agg");
-
-// For each entry
-for (Range.Bucket entry : agg.getBuckets()) {
-    String key = entry.getKeyAsString();                // Date range as key
-    DateTime fromAsDate = (DateTime) entry.getFrom();   // Date bucket from as a Date
-    DateTime toAsDate = (DateTime) entry.getTo();       // Date bucket to as a Date
-    long docCount = entry.getDocCount();                // Doc count
-
-    logger.info("key [{}], from [{}], to [{}], doc_count [{}]", key, fromAsDate, toAsDate, docCount);
-}
---------------------------------------------------
-
-This will basically produce:
-
-[source,text]
---------------------------------------------------
-key [*-1950], from [null], to [1950-01-01T00:00:00.000Z], doc_count [8]
-key [1950-1960], from [1950-01-01T00:00:00.000Z], to [1960-01-01T00:00:00.000Z], doc_count [5]
-key [1960-*], from [1960-01-01T00:00:00.000Z], to [null], doc_count [37]
---------------------------------------------------
-
diff --git a/docs/java-api/aggregations/bucket/filter-aggregation.asciidoc b/docs/java-api/aggregations/bucket/filter-aggregation.asciidoc
deleted file mode 100644
index 3ffb05202bbef..0000000000000
--- a/docs/java-api/aggregations/bucket/filter-aggregation.asciidoc
+++ /dev/null
@@ -1,34 +0,0 @@
-[[java-aggs-bucket-filter]]
-==== Filter Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-bucket-filter-aggregation.html[Filter Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilders
-    .filter("agg", QueryBuilders.termQuery("gender", "male"));
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.filter.Filter;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Filter agg = sr.getAggregations().get("agg");
-agg.getDocCount(); // Doc count
---------------------------------------------------
diff --git a/docs/java-api/aggregations/bucket/filters-aggregation.asciidoc b/docs/java-api/aggregations/bucket/filters-aggregation.asciidoc
deleted file mode 100644
index 0b782304dacc0..0000000000000
--- a/docs/java-api/aggregations/bucket/filters-aggregation.asciidoc
+++ /dev/null
@@ -1,51 +0,0 @@
-[[java-aggs-bucket-filters]]
-==== Filters Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-bucket-filters-aggregation.html[Filters Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilder aggregation =
-    AggregationBuilders
-        .filters("agg",
-            new FiltersAggregator.KeyedFilter("men", QueryBuilders.termQuery("gender", "male")),
-            new FiltersAggregator.KeyedFilter("women", QueryBuilders.termQuery("gender", "female")));
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.filters.Filters;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Filters agg = sr.getAggregations().get("agg");
-
-// For each entry
-for (Filters.Bucket entry : agg.getBuckets()) {
-    String key = entry.getKeyAsString();            // bucket key
-    long docCount = entry.getDocCount();            // Doc count
-    logger.info("key [{}], doc_count [{}]", key, docCount);
-}
---------------------------------------------------
-
-This will basically produce:
-
-[source,text]
---------------------------------------------------
-key [men], doc_count [4982]
-key [women], doc_count [5018]
---------------------------------------------------
diff --git a/docs/java-api/aggregations/bucket/geodistance-aggregation.asciidoc b/docs/java-api/aggregations/bucket/geodistance-aggregation.asciidoc
deleted file mode 100644
index 472c3ac59bf48..0000000000000
--- a/docs/java-api/aggregations/bucket/geodistance-aggregation.asciidoc
+++ /dev/null
@@ -1,58 +0,0 @@
-[[java-aggs-bucket-geodistance]]
-==== Geo Distance Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-bucket-geodistance-aggregation.html[Geo Distance Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilder aggregation =
-        AggregationBuilders
-                .geoDistance("agg", new GeoPoint(48.84237171118314,2.33320027692004))
-                .field("address.location")
-                .unit(DistanceUnit.KILOMETERS)
-                .addUnboundedTo(3.0)
-                .addRange(3.0, 10.0)
-                .addRange(10.0, 500.0);
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.range.Range;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Range agg = sr.getAggregations().get("agg");
-
-// For each entry
-for (Range.Bucket entry : agg.getBuckets()) {
-    String key = entry.getKeyAsString();    // key as String
-    Number from = (Number) entry.getFrom(); // bucket from value
-    Number to = (Number) entry.getTo();     // bucket to value
-    long docCount = entry.getDocCount();    // Doc count
-
-    logger.info("key [{}], from [{}], to [{}], doc_count [{}]", key, from, to, docCount);
-}
---------------------------------------------------
-
-This will basically produce:
-
-[source,text]
---------------------------------------------------
-key [*-3.0], from [0.0], to [3.0], doc_count [161]
-key [3.0-10.0], from [3.0], to [10.0], doc_count [460]
-key [10.0-500.0], from [10.0], to [500.0], doc_count [4925]
---------------------------------------------------
diff --git a/docs/java-api/aggregations/bucket/geohashgrid-aggregation.asciidoc b/docs/java-api/aggregations/bucket/geohashgrid-aggregation.asciidoc
deleted file mode 100644
index 19e3f03349397..0000000000000
--- a/docs/java-api/aggregations/bucket/geohashgrid-aggregation.asciidoc
+++ /dev/null
@@ -1,57 +0,0 @@
-[[java-aggs-bucket-geohashgrid]]
-==== Geo Hash Grid Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-bucket-geohashgrid-aggregation.html[Geo Hash Grid Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilder aggregation =
-        AggregationBuilders
-                .geohashGrid("agg")
-                .field("address.location")
-                .precision(4);
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-GeoHashGrid agg = sr.getAggregations().get("agg");
-
-// For each entry
-for (GeoHashGrid.Bucket entry : agg.getBuckets()) {
-    String keyAsString = entry.getKeyAsString(); // key as String
-    GeoPoint key = (GeoPoint) entry.getKey();    // key as geo point
-    long docCount = entry.getDocCount();         // Doc count
-
-    logger.info("key [{}], point {}, doc_count [{}]", keyAsString, key, docCount);
-}
---------------------------------------------------
-
-This will basically produce:
-
-[source,text]
---------------------------------------------------
-key [gbqu], point [47.197265625, -1.58203125], doc_count [1282]
-key [gbvn], point [50.361328125, -4.04296875], doc_count [1248]
-key [u1j0], point [50.712890625, 7.20703125], doc_count [1156]
-key [u0j2], point [45.087890625, 7.55859375], doc_count [1138]
-...
---------------------------------------------------
-
diff --git a/docs/java-api/aggregations/bucket/global-aggregation.asciidoc b/docs/java-api/aggregations/bucket/global-aggregation.asciidoc
deleted file mode 100644
index e0a731159adf5..0000000000000
--- a/docs/java-api/aggregations/bucket/global-aggregation.asciidoc
+++ /dev/null
@@ -1,35 +0,0 @@
-[[java-aggs-bucket-global]]
-==== Global Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-bucket-global-aggregation.html[Global Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilders
-    .global("agg")
-    .subAggregation(AggregationBuilders.terms("genders").field("gender"));
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.global.Global;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Global agg = sr.getAggregations().get("agg");
-agg.getDocCount(); // Doc count
---------------------------------------------------
diff --git a/docs/java-api/aggregations/bucket/histogram-aggregation.asciidoc b/docs/java-api/aggregations/bucket/histogram-aggregation.asciidoc
deleted file mode 100644
index 59bb555401c5b..0000000000000
--- a/docs/java-api/aggregations/bucket/histogram-aggregation.asciidoc
+++ /dev/null
@@ -1,48 +0,0 @@
-[[java-aggs-bucket-histogram]]
-==== Histogram Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-bucket-histogram-aggregation.html[Histogram Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilder aggregation =
-        AggregationBuilders
-                .histogram("agg")
-                .field("height")
-                .interval(1);
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Histogram agg = sr.getAggregations().get("agg");
-
-// For each entry
-for (Histogram.Bucket entry : agg.getBuckets()) {
-    Number key = (Number) entry.getKey();   // Key
-    long docCount = entry.getDocCount();    // Doc count
-
-    logger.info("key [{}], doc_count [{}]", key, docCount);
-}
---------------------------------------------------
-
-===== Order
-
-Supports the same order functionality as the <<java-aggs-bucket-terms,`Terms Aggregation`>>.
diff --git a/docs/java-api/aggregations/bucket/iprange-aggregation.asciidoc b/docs/java-api/aggregations/bucket/iprange-aggregation.asciidoc
deleted file mode 100644
index a2c07df1b26e7..0000000000000
--- a/docs/java-api/aggregations/bucket/iprange-aggregation.asciidoc
+++ /dev/null
@@ -1,79 +0,0 @@
-[[java-aggs-bucket-iprange]]
-==== Ip Range Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-bucket-iprange-aggregation.html[Ip Range Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilder<?> aggregation =
-        AggregationBuilders
-                .ipRange("agg")
-                .field("ip")
-                .addUnboundedTo("192.168.1.0")             // from -infinity to 192.168.1.0 (excluded)
-                .addRange("192.168.1.0", "192.168.2.0")    // from 192.168.1.0 to 192.168.2.0 (excluded)
-                .addUnboundedFrom("192.168.2.0");          // from 192.168.2.0 to +infinity
---------------------------------------------------
-
-Note that you could also use ip masks as ranges:
-
-[source,java]
---------------------------------------------------
-AggregationBuilder<?> aggregation =
-        AggregationBuilders
-                .ipRange("agg")
-                .field("ip")
-                .addMaskRange("192.168.0.0/32")
-                .addMaskRange("192.168.0.0/24")
-                .addMaskRange("192.168.0.0/16");
---------------------------------------------------
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.range.Range;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Range agg = sr.getAggregations().get("agg");
-
-// For each entry
-for (Range.Bucket entry : agg.getBuckets()) {
-    String key = entry.getKeyAsString();            // Ip range as key
-    String fromAsString = entry.getFromAsString();  // Ip bucket from as a String
-    String toAsString = entry.getToAsString();      // Ip bucket to as a String
-    long docCount = entry.getDocCount();            // Doc count
-
-    logger.info("key [{}], from [{}], to [{}], doc_count [{}]", key, fromAsString, toAsString, docCount);
-}
---------------------------------------------------
-
-This will basically produce for the first example:
-
-[source,text]
---------------------------------------------------
-key [*-192.168.1.0], from [null], to [192.168.1.0], doc_count [13]
-key [192.168.1.0-192.168.2.0], from [192.168.1.0], to [192.168.2.0], doc_count [14]
-key [192.168.2.0-*], from [192.168.2.0], to [null], doc_count [23]
---------------------------------------------------
-
-And for the second one (using Ip masks):
-
-[source,text]
---------------------------------------------------
-key [192.168.0.0/32], from [192.168.0.0], to [192.168.0.1], doc_count [0]
-key [192.168.0.0/24], from [192.168.0.0], to [192.168.1.0], doc_count [13]
-key [192.168.0.0/16], from [192.168.0.0], to [192.169.0.0], doc_count [50]
---------------------------------------------------
-
diff --git a/docs/java-api/aggregations/bucket/missing-aggregation.asciidoc b/docs/java-api/aggregations/bucket/missing-aggregation.asciidoc
deleted file mode 100644
index 31d21604dc57a..0000000000000
--- a/docs/java-api/aggregations/bucket/missing-aggregation.asciidoc
+++ /dev/null
@@ -1,34 +0,0 @@
-[[java-aggs-bucket-missing]]
-==== Missing Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-bucket-missing-aggregation.html[Missing Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilders.missing("agg").field("gender");
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.missing.Missing;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Missing agg = sr.getAggregations().get("agg");
-agg.getDocCount(); // Doc count
---------------------------------------------------
-
diff --git a/docs/java-api/aggregations/bucket/nested-aggregation.asciidoc b/docs/java-api/aggregations/bucket/nested-aggregation.asciidoc
deleted file mode 100644
index b1ebad7a63bfa..0000000000000
--- a/docs/java-api/aggregations/bucket/nested-aggregation.asciidoc
+++ /dev/null
@@ -1,34 +0,0 @@
-[[java-aggs-bucket-nested]]
-==== Nested Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-bucket-nested-aggregation.html[Nested Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilders
-    .nested("agg", "resellers");
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.nested.Nested;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Nested agg = sr.getAggregations().get("agg");
-agg.getDocCount(); // Doc count
---------------------------------------------------
diff --git a/docs/java-api/aggregations/bucket/range-aggregation.asciidoc b/docs/java-api/aggregations/bucket/range-aggregation.asciidoc
deleted file mode 100644
index b30c856ebeada..0000000000000
--- a/docs/java-api/aggregations/bucket/range-aggregation.asciidoc
+++ /dev/null
@@ -1,58 +0,0 @@
-[[java-aggs-bucket-range]]
-==== Range Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-bucket-range-aggregation.html[Range Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilder aggregation =
-        AggregationBuilders
-                .range("agg")
-                .field("height")
-                .addUnboundedTo(1.0f)               // from -infinity to 1.0 (excluded)
-                .addRange(1.0f, 1.5f)               // from 1.0 to 1.5 (excluded)
-                .addUnboundedFrom(1.5f);            // from 1.5 to +infinity
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.range.Range;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Range agg = sr.getAggregations().get("agg");
-
-// For each entry
-for (Range.Bucket entry : agg.getBuckets()) {
-    String key = entry.getKeyAsString();             // Range as key
-    Number from = (Number) entry.getFrom();          // Bucket from
-    Number to = (Number) entry.getTo();              // Bucket to
-    long docCount = entry.getDocCount();    // Doc count
-
-    logger.info("key [{}], from [{}], to [{}], doc_count [{}]", key, from, to, docCount);
-}
---------------------------------------------------
-
-This will basically produce for the first example:
-
-[source,text]
---------------------------------------------------
-key [*-1.0], from [-Infinity], to [1.0], doc_count [9]
-key [1.0-1.5], from [1.0], to [1.5], doc_count [21]
-key [1.5-*], from [1.5], to [Infinity], doc_count [20]
---------------------------------------------------
-
diff --git a/docs/java-api/aggregations/bucket/reverse-nested-aggregation.asciidoc b/docs/java-api/aggregations/bucket/reverse-nested-aggregation.asciidoc
deleted file mode 100644
index 635b0e8cf77ee..0000000000000
--- a/docs/java-api/aggregations/bucket/reverse-nested-aggregation.asciidoc
+++ /dev/null
@@ -1,50 +0,0 @@
-[[java-aggs-bucket-reverse-nested]]
-==== Reverse Nested Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-bucket-reverse-nested-aggregation.html[Reverse Nested Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilder aggregation =
-    AggregationBuilders
-        .nested("agg", "resellers")
-        .subAggregation(
-                AggregationBuilders
-                        .terms("name").field("resellers.name")
-                        .subAggregation(
-                                AggregationBuilders
-                                        .reverseNested("reseller_to_product")
-                        )
-        );
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.nested.Nested;
-import org.elasticsearch.search.aggregations.bucket.nested.ReverseNested;
-import org.elasticsearch.search.aggregations.bucket.terms.Terms;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Nested agg = sr.getAggregations().get("agg");
-Terms name = agg.getAggregations().get("name");
-for (Terms.Bucket bucket : name.getBuckets()) {
-    ReverseNested resellerToProduct = bucket.getAggregations().get("reseller_to_product");
-    resellerToProduct.getDocCount(); // Doc count
-}
---------------------------------------------------
-
diff --git a/docs/java-api/aggregations/bucket/significantterms-aggregation.asciidoc b/docs/java-api/aggregations/bucket/significantterms-aggregation.asciidoc
deleted file mode 100644
index 4450c324c8209..0000000000000
--- a/docs/java-api/aggregations/bucket/significantterms-aggregation.asciidoc
+++ /dev/null
@@ -1,47 +0,0 @@
-[[java-aggs-bucket-significantterms]]
-==== Significant Terms Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-bucket-significantterms-aggregation.html[Significant Terms Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilder aggregation =
-        AggregationBuilders
-                .significantTerms("significant_countries")
-                .field("address.country");
-
-// Let say you search for men only
-SearchResponse sr = client.prepareSearch()
-        .setQuery(QueryBuilders.termQuery("gender", "male"))
-        .addAggregation(aggregation)
-        .get();
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-SignificantTerms agg = sr.getAggregations().get("significant_countries");
-
-// For each entry
-for (SignificantTerms.Bucket entry : agg.getBuckets()) {
-    entry.getKey();      // Term
-    entry.getDocCount(); // Doc count
-}
---------------------------------------------------
diff --git a/docs/java-api/aggregations/bucket/terms-aggregation.asciidoc b/docs/java-api/aggregations/bucket/terms-aggregation.asciidoc
deleted file mode 100644
index db584fd4cedd2..0000000000000
--- a/docs/java-api/aggregations/bucket/terms-aggregation.asciidoc
+++ /dev/null
@@ -1,97 +0,0 @@
-[[java-aggs-bucket-terms]]
-==== Terms Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-bucket-terms-aggregation.html[Terms Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilders
-    .terms("genders")
-    .field("gender");
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.terms.Terms;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Terms genders = sr.getAggregations().get("genders");
-
-// For each entry
-for (Terms.Bucket entry : genders.getBuckets()) {
-    entry.getKey();      // Term
-    entry.getDocCount(); // Doc count
-}
---------------------------------------------------
-
-===== Order
-
-Import bucket ordering strategy classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.BucketOrder;
---------------------------------------------------
-
-Ordering the buckets by their `doc_count` in an ascending manner:
-
-[source,java]
---------------------------------------------------
-AggregationBuilders
-    .terms("genders")
-    .field("gender")
-    .order(BucketOrder.count(true))
---------------------------------------------------
-
-Ordering the buckets alphabetically by their terms in an ascending manner:
-
-[source,java]
---------------------------------------------------
-AggregationBuilders
-    .terms("genders")
-    .field("gender")
-    .order(BucketOrder.key(true))
---------------------------------------------------
-
-Ordering the buckets by single value metrics sub-aggregation (identified by the aggregation name):
-
-[source,java]
---------------------------------------------------
-AggregationBuilders
-    .terms("genders")
-    .field("gender")
-    .order(BucketOrder.aggregation("avg_height", false))
-    .subAggregation(
-        AggregationBuilders.avg("avg_height").field("height")
-    )
---------------------------------------------------
-
-Ordering the buckets by multiple criteria:
-
-[source,java]
---------------------------------------------------
-AggregationBuilders
-    .terms("genders")
-    .field("gender")
-    .order(BucketOrder.compound( // in order of priority:
-        BucketOrder.aggregation("avg_height", false), // sort by sub-aggregation first
-        BucketOrder.count(true))) // then bucket count as a tie-breaker
-    .subAggregation(
-        AggregationBuilders.avg("avg_height").field("height")
-    )
---------------------------------------------------
diff --git a/docs/java-api/aggregations/metrics.asciidoc b/docs/java-api/aggregations/metrics.asciidoc
deleted file mode 100644
index c9afb4c39d484..0000000000000
--- a/docs/java-api/aggregations/metrics.asciidoc
+++ /dev/null
@@ -1,27 +0,0 @@
-[[java-aggregations-metrics]]
-
-include::metrics/min-aggregation.asciidoc[]
-
-include::metrics/max-aggregation.asciidoc[]
-
-include::metrics/sum-aggregation.asciidoc[]
-
-include::metrics/avg-aggregation.asciidoc[]
-
-include::metrics/stats-aggregation.asciidoc[]
-
-include::metrics/extendedstats-aggregation.asciidoc[]
-
-include::metrics/valuecount-aggregation.asciidoc[]
-
-include::metrics/percentile-aggregation.asciidoc[]
-
-include::metrics/percentile-rank-aggregation.asciidoc[]
-
-include::metrics/cardinality-aggregation.asciidoc[]
-
-include::metrics/geobounds-aggregation.asciidoc[]
-
-include::metrics/tophits-aggregation.asciidoc[]
-
-include::metrics/scripted-metric-aggregation.asciidoc[]
diff --git a/docs/java-api/aggregations/metrics/avg-aggregation.asciidoc b/docs/java-api/aggregations/metrics/avg-aggregation.asciidoc
deleted file mode 100644
index 511cbabf5c848..0000000000000
--- a/docs/java-api/aggregations/metrics/avg-aggregation.asciidoc
+++ /dev/null
@@ -1,37 +0,0 @@
-[[java-aggs-metrics-avg]]
-==== Avg Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-metrics-avg-aggregation.html[Avg Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AvgAggregationBuilder aggregation =
-        AggregationBuilders
-                .avg("agg")
-                .field("height");
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.metrics.avg.Avg;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Avg agg = sr.getAggregations().get("agg");
-double value = agg.getValue();
---------------------------------------------------
-
diff --git a/docs/java-api/aggregations/metrics/cardinality-aggregation.asciidoc b/docs/java-api/aggregations/metrics/cardinality-aggregation.asciidoc
deleted file mode 100644
index 8a854e553f4a3..0000000000000
--- a/docs/java-api/aggregations/metrics/cardinality-aggregation.asciidoc
+++ /dev/null
@@ -1,38 +0,0 @@
-[[java-aggs-metrics-cardinality]]
-==== Cardinality Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-metrics-cardinality-aggregation.html[Cardinality Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-CardinalityAggregationBuilder aggregation =
-        AggregationBuilders
-                .cardinality("agg")
-                .field("tags");
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Cardinality agg = sr.getAggregations().get("agg");
-long value = agg.getValue();
---------------------------------------------------
-
-
diff --git a/docs/java-api/aggregations/metrics/extendedstats-aggregation.asciidoc b/docs/java-api/aggregations/metrics/extendedstats-aggregation.asciidoc
deleted file mode 100644
index 8f2f12ede6849..0000000000000
--- a/docs/java-api/aggregations/metrics/extendedstats-aggregation.asciidoc
+++ /dev/null
@@ -1,44 +0,0 @@
-[[java-aggs-metrics-extendedstats]]
-==== Extended Stats Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-metrics-extendedstats-aggregation.html[Extended Stats Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-ExtendedStatsAggregationBuilder aggregation =
-        AggregationBuilders
-                .extendedStats("agg")
-                .field("height");
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-ExtendedStats agg = sr.getAggregations().get("agg");
-double min = agg.getMin();
-double max = agg.getMax();
-double avg = agg.getAvg();
-double sum = agg.getSum();
-long count = agg.getCount();
-double stdDeviation = agg.getStdDeviation();
-double sumOfSquares = agg.getSumOfSquares();
-double variance = agg.getVariance();
---------------------------------------------------
-
diff --git a/docs/java-api/aggregations/metrics/geobounds-aggregation.asciidoc b/docs/java-api/aggregations/metrics/geobounds-aggregation.asciidoc
deleted file mode 100644
index 571a61f12e7cc..0000000000000
--- a/docs/java-api/aggregations/metrics/geobounds-aggregation.asciidoc
+++ /dev/null
@@ -1,46 +0,0 @@
-[[java-aggs-metrics-geobounds]]
-==== Geo Bounds Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-metrics-geobounds-aggregation.html[Geo Bounds Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-GeoBoundsAggregationBuilder aggregation =
-        GeoBoundsAggregationBuilder
-                .geoBounds("agg")
-                .field("address.location")
-                .wrapLongitude(true);
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBounds;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-GeoBounds agg = sr.getAggregations().get("agg");
-GeoPoint bottomRight = agg.bottomRight();
-GeoPoint topLeft = agg.topLeft();
-logger.info("bottomRight {}, topLeft {}", bottomRight, topLeft);
---------------------------------------------------
-
-This will basically produce:
-
-[source,text]
---------------------------------------------------
-bottomRight [40.70500764381921, 13.952946866893775], topLeft [53.49603022435221, -4.190029308156676]
---------------------------------------------------
diff --git a/docs/java-api/aggregations/metrics/max-aggregation.asciidoc b/docs/java-api/aggregations/metrics/max-aggregation.asciidoc
deleted file mode 100644
index 9bd393698429b..0000000000000
--- a/docs/java-api/aggregations/metrics/max-aggregation.asciidoc
+++ /dev/null
@@ -1,37 +0,0 @@
-[[java-aggs-metrics-max]]
-==== Max Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-metrics-max-aggregation.html[Max Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-MaxAggregationBuilder aggregation =
-        AggregationBuilders
-                .max("agg")
-                .field("height");
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.metrics.max.Max;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Max agg = sr.getAggregations().get("agg");
-double value = agg.getValue();
---------------------------------------------------
-
diff --git a/docs/java-api/aggregations/metrics/min-aggregation.asciidoc b/docs/java-api/aggregations/metrics/min-aggregation.asciidoc
deleted file mode 100644
index 0205cae44d8f8..0000000000000
--- a/docs/java-api/aggregations/metrics/min-aggregation.asciidoc
+++ /dev/null
@@ -1,37 +0,0 @@
-[[java-aggs-metrics-min]]
-==== Min Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-metrics-min-aggregation.html[Min Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-MinAggregationBuilder aggregation =
-        AggregationBuilders
-                .min("agg")
-                .field("height");
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.metrics.min.Min;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Min agg = sr.getAggregations().get("agg");
-double value = agg.getValue();
---------------------------------------------------
-
diff --git a/docs/java-api/aggregations/metrics/percentile-aggregation.asciidoc b/docs/java-api/aggregations/metrics/percentile-aggregation.asciidoc
deleted file mode 100644
index ad54fbf5a46be..0000000000000
--- a/docs/java-api/aggregations/metrics/percentile-aggregation.asciidoc
+++ /dev/null
@@ -1,68 +0,0 @@
-[[java-aggs-metrics-percentile]]
-==== Percentile Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-metrics-percentile-aggregation.html[Percentile Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-PercentilesAggregationBuilder aggregation =
-        AggregationBuilders
-                .percentiles("agg")
-                .field("height");
---------------------------------------------------
-
-You can provide your own percentiles instead of using defaults:
-
-[source,java]
---------------------------------------------------
-PercentilesAggregationBuilder aggregation =
-        AggregationBuilders
-                .percentiles("agg")
-                .field("height")
-                .percentiles(1.0, 5.0, 10.0, 20.0, 30.0, 75.0, 95.0, 99.0);
---------------------------------------------------
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
-import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Percentiles agg = sr.getAggregations().get("agg");
-// For each entry
-for (Percentile entry : agg) {
-    double percent = entry.getPercent();    // Percent
-    double value = entry.getValue();        // Value
-
-    logger.info("percent [{}], value [{}]", percent, value);
-}
---------------------------------------------------
-
-
-This will basically produce for the first example:
-
-[source,text]
---------------------------------------------------
-percent [1.0], value [0.814338896154595]
-percent [5.0], value [0.8761912455821302]
-percent [25.0], value [1.173346540141847]
-percent [50.0], value [1.5432023318692198]
-percent [75.0], value [1.923915462033674]
-percent [95.0], value [2.2273644908535335]
-percent [99.0], value [2.284989339108279]
---------------------------------------------------
-
diff --git a/docs/java-api/aggregations/metrics/percentile-rank-aggregation.asciidoc b/docs/java-api/aggregations/metrics/percentile-rank-aggregation.asciidoc
deleted file mode 100644
index a846d59f82029..0000000000000
--- a/docs/java-api/aggregations/metrics/percentile-rank-aggregation.asciidoc
+++ /dev/null
@@ -1,55 +0,0 @@
-[[java-aggs-metrics-percentile-rank]]
-==== Percentile Ranks Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-metrics-percentile-rank-aggregation.html[Percentile Ranks Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-PercentileRanksAggregationBuilder aggregation =
-        AggregationBuilders
-                .percentileRanks("agg")
-                .field("height")
-                .values(1.24, 1.91, 2.22);
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
-import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanks;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-PercentileRanks agg = sr.getAggregations().get("agg");
-// For each entry
-for (Percentile entry : agg) {
-    double percent = entry.getPercent();    // Percent
-    double value = entry.getValue();        // Value
-
-    logger.info("percent [{}], value [{}]", percent, value);
-}
---------------------------------------------------
-
-
-This will basically produce:
-
-[source,text]
---------------------------------------------------
-percent [29.664353095090945], value [1.24]
-percent [73.9335313461868], value [1.91]
-percent [94.40095147327283], value [2.22]
---------------------------------------------------
-
diff --git a/docs/java-api/aggregations/metrics/scripted-metric-aggregation.asciidoc b/docs/java-api/aggregations/metrics/scripted-metric-aggregation.asciidoc
deleted file mode 100644
index 5b68fa7be451f..0000000000000
--- a/docs/java-api/aggregations/metrics/scripted-metric-aggregation.asciidoc
+++ /dev/null
@@ -1,100 +0,0 @@
-[[java-aggs-metrics-scripted-metric]]
-==== Scripted Metric Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-metrics-scripted-metric-aggregation.html[Scripted Metric Aggregation]
-with Java API.
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-ScriptedMetricAggregationBuilder aggregation = AggregationBuilders
-    .scriptedMetric("agg")
-    .initScript(new Script("state.heights = []"))
-    .mapScript(new Script("state.heights.add(doc.gender.value == 'male' ? doc.height.value : -1.0 * doc.height.value)"));
---------------------------------------------------
-
-You can also specify a `combine` script which will be executed on each shard:
-
-[source,java]
---------------------------------------------------
-ScriptedMetricAggregationBuilder aggregation = AggregationBuilders
-    .scriptedMetric("agg")
-    .initScript(new Script("state.heights = []"))
-    .mapScript(new Script("state.heights.add(doc.gender.value == 'male' ? doc.height.value : -1.0 * doc.height.value)"))
-    .combineScript(new Script("double heights_sum = 0.0; for (t in state.heights) { heights_sum += t } return heights_sum"));
---------------------------------------------------
-
-You can also specify a `reduce` script which will be executed on the node which gets the request:
-
-[source,java]
---------------------------------------------------
-ScriptedMetricAggregationBuilder aggregation = AggregationBuilders
-    .scriptedMetric("agg")
-    .initScript(new Script("state.heights = []"))
-    .mapScript(new Script("state.heights.add(doc.gender.value == 'male' ? doc.height.value : -1.0 * doc.height.value)"))
-    .combineScript(new Script("double heights_sum = 0.0; for (t in state.heights) { heights_sum += t } return heights_sum"))
-    .reduceScript(new Script("double heights_sum = 0.0; for (a in states) { heights_sum += a } return heights_sum"));
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.aggregations.metrics.tophits.TopHits;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-ScriptedMetric agg = sr.getAggregations().get("agg");
-Object scriptedResult = agg.aggregation();
-logger.info("scriptedResult [{}]", scriptedResult);
---------------------------------------------------
-
-Note that the result depends on the script you built.
-For the first example, this will basically produce:
-
-[source,text]
---------------------------------------------------
-scriptedResult object [ArrayList]
-scriptedResult [ {
-"heights" : [ 1.122218480146643, -1.8148918111233887, -1.7626731575142909, ... ]
-}, {
-"heights" : [ -0.8046067304119863, -2.0785486707864553, -1.9183567430207953, ... ]
-}, {
-"heights" : [ 2.092635728868694, 1.5697545960886536, 1.8826954461968808, ... ]
-}, {
-"heights" : [ -2.1863201099468403, 1.6328549117346856, -1.7078288405893842, ... ]
-}, {
-"heights" : [ 1.6043904836424177, -2.0736538674414025, 0.9898266674373053, ... ]
-} ]
---------------------------------------------------
-
-The second example will produce:
-
-[source,text]
---------------------------------------------------
-scriptedResult object [ArrayList]
-scriptedResult [-41.279615707402876,
-                -60.88007362339038,
-                38.823270659734256,
-                14.840192739445632,
-                11.300902755741326]
---------------------------------------------------
-
-The last example will produce:
-
-[source,text]
---------------------------------------------------
-scriptedResult object [Double]
-scriptedResult [2.171917696507009]
---------------------------------------------------
-
diff --git a/docs/java-api/aggregations/metrics/stats-aggregation.asciidoc b/docs/java-api/aggregations/metrics/stats-aggregation.asciidoc
deleted file mode 100644
index 260d9c01cb944..0000000000000
--- a/docs/java-api/aggregations/metrics/stats-aggregation.asciidoc
+++ /dev/null
@@ -1,41 +0,0 @@
-[[java-aggs-metrics-stats]]
-==== Stats Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-metrics-stats-aggregation.html[Stats Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-StatsAggregationBuilder aggregation =
-        AggregationBuilders
-                .stats("agg")
-                .field("height");
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.metrics.stats.Stats;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Stats agg = sr.getAggregations().get("agg");
-double min = agg.getMin();
-double max = agg.getMax();
-double avg = agg.getAvg();
-double sum = agg.getSum();
-long count = agg.getCount();
---------------------------------------------------
-
diff --git a/docs/java-api/aggregations/metrics/sum-aggregation.asciidoc b/docs/java-api/aggregations/metrics/sum-aggregation.asciidoc
deleted file mode 100644
index 453616916d755..0000000000000
--- a/docs/java-api/aggregations/metrics/sum-aggregation.asciidoc
+++ /dev/null
@@ -1,37 +0,0 @@
-[[java-aggs-metrics-sum]]
-==== Sum Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-metrics-sum-aggregation.html[Sum Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-SumAggregationBuilder aggregation =
-        AggregationBuilders
-                .sum("agg")
-                .field("height");
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.metrics.sum.Sum;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Sum agg = sr.getAggregations().get("agg");
-double value = agg.getValue();
---------------------------------------------------
-
diff --git a/docs/java-api/aggregations/metrics/tophits-aggregation.asciidoc b/docs/java-api/aggregations/metrics/tophits-aggregation.asciidoc
deleted file mode 100644
index 2473b4b89d77b..0000000000000
--- a/docs/java-api/aggregations/metrics/tophits-aggregation.asciidoc
+++ /dev/null
@@ -1,79 +0,0 @@
-[[java-aggs-metrics-tophits]]
-==== Top Hits Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-metrics-top-hits-aggregation.html[Top Hits Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-AggregationBuilder aggregation =
-    AggregationBuilders
-        .terms("agg").field("gender")
-        .subAggregation(
-            AggregationBuilders.topHits("top")
-        );
---------------------------------------------------
-
-You can use most of the options available for standard search such as `from`, `size`, `sort`, `highlight`, `explain`...
-
-[source,java]
---------------------------------------------------
-AggregationBuilder aggregation =
-    AggregationBuilders
-        .terms("agg").field("gender")
-        .subAggregation(
-            AggregationBuilders.topHits("top")
-                .explain(true)
-                .size(1)
-                .from(10)
-        );
---------------------------------------------------
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.aggregations.metrics.tophits.TopHits;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-Terms agg = sr.getAggregations().get("agg");
-
-// For each entry
-for (Terms.Bucket entry : agg.getBuckets()) {
-    String key = entry.getKey();                    // bucket key
-    long docCount = entry.getDocCount();            // Doc count
-    logger.info("key [{}], doc_count [{}]", key, docCount);
-
-    // We ask for top_hits for each bucket
-    TopHits topHits = entry.getAggregations().get("top");
-    for (SearchHit hit : topHits.getHits().getHits()) {
-        logger.info(" -> id [{}], _source [{}]", hit.getId(), hit.getSourceAsString());
-    }
-}
---------------------------------------------------
-
-This will basically produce for the first example:
-
-[source,text]
---------------------------------------------------
-key [male], doc_count [5107]
- -> id [AUnzSZze9k7PKXtq04x2], _source [{"gender":"male",...}]
- -> id [AUnzSZzj9k7PKXtq04x4], _source [{"gender":"male",...}]
- -> id [AUnzSZzl9k7PKXtq04x5], _source [{"gender":"male",...}]
-key [female], doc_count [4893]
- -> id [AUnzSZzM9k7PKXtq04xy], _source [{"gender":"female",...}]
- -> id [AUnzSZzp9k7PKXtq04x8], _source [{"gender":"female",...}]
- -> id [AUnzSZ0W9k7PKXtq04yS], _source [{"gender":"female",...}]
---------------------------------------------------
diff --git a/docs/java-api/aggregations/metrics/valuecount-aggregation.asciidoc b/docs/java-api/aggregations/metrics/valuecount-aggregation.asciidoc
deleted file mode 100644
index b180d22af33cd..0000000000000
--- a/docs/java-api/aggregations/metrics/valuecount-aggregation.asciidoc
+++ /dev/null
@@ -1,37 +0,0 @@
-[[java-aggs-metrics-valuecount]]
-==== Value Count Aggregation
-
-Here is how you can use
-{ref}/search-aggregations-metrics-valuecount-aggregation.html[Value Count Aggregation]
-with Java API.
-
-
-===== Prepare aggregation request
-
-Here is an example on how to create the aggregation request:
-
-[source,java]
---------------------------------------------------
-ValueCountAggregationBuilder aggregation =
-        AggregationBuilders
-                .count("agg")
-                .field("height");
---------------------------------------------------
-
-
-===== Use aggregation response
-
-Import Aggregation definition classes:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-// sr is here your SearchResponse object
-ValueCount agg = sr.getAggregations().get("agg");
-long value = agg.getValue();
---------------------------------------------------
-
diff --git a/docs/java-api/aggs.asciidoc b/docs/java-api/aggs.asciidoc
deleted file mode 100644
index c2e09b4901e87..0000000000000
--- a/docs/java-api/aggs.asciidoc
+++ /dev/null
@@ -1,63 +0,0 @@
-[[java-aggs]]
-== Aggregations
-
-Elasticsearch provides a full Java API to play with aggregations. See the
-{ref}/search-aggregations.html[Aggregations guide].
-
-Use the factory for aggregation builders (`AggregationBuilders`) and add each aggregation
-you want to compute when querying and add it to your search request:
-
-[source,java]
---------------------------------------------------
-SearchResponse sr = node.client().prepareSearch()
-        .setQuery( /* your query */ )
-        .addAggregation( /* add an aggregation */ )
-        .execute().actionGet();
---------------------------------------------------
-
-Note that you can add more than one aggregation. See
-{ref}/search-search.html[Search Java API] for details.
-
-To build aggregation requests, use `AggregationBuilders` helpers. Just import them
-in your class:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.search.aggregations.AggregationBuilders;
---------------------------------------------------
-
-=== Structuring aggregations
-
-As explained in the
-{ref}/search-aggregations.html[Aggregations guide], you can define
-sub aggregations inside an aggregation.
-
-An aggregation could be a metrics aggregation or a bucket aggregation.
-
-For example, here is a 3 levels aggregation composed of:
-
-* Terms aggregation (bucket)
-* Date Histogram aggregation (bucket)
-* Average aggregation (metric)
-
-[source,java]
---------------------------------------------------
-SearchResponse sr = node.client().prepareSearch()
-    .addAggregation(
-        AggregationBuilders.terms("by_country").field("country")
-        .subAggregation(AggregationBuilders.dateHistogram("by_year")
-            .field("dateOfBirth")
-            .calendarInterval(DateHistogramInterval.YEAR)
-            .subAggregation(AggregationBuilders.avg("avg_children").field("children"))
-        )
-    )
-    .execute().actionGet();
---------------------------------------------------
-
-=== Metrics aggregations
-
-include::aggregations/metrics.asciidoc[]
-
-=== Bucket aggregations
-
-include::aggregations/bucket.asciidoc[]
diff --git a/docs/java-api/client.asciidoc b/docs/java-api/client.asciidoc
deleted file mode 100644
index 811d7c398d940..0000000000000
--- a/docs/java-api/client.asciidoc
+++ /dev/null
@@ -1,110 +0,0 @@
-[[client]]
-== Client
-
-You can use the *Java client* in multiple ways:
-
-* Perform standard <<java-docs-index,index>>, <<java-docs-get,get>>,
-  <<java-docs-delete,delete>> and <<java-search,search>> operations on an
-  existing cluster
-* Perform administrative tasks on a running cluster
-
-Obtaining an Elasticsearch `Client` is simple. The most common way to
-get a client is by creating a <<transport-client,`TransportClient`>>
-that connects to a cluster.
-
-[IMPORTANT]
-==============================
-
-The client must have the same major version (e.g. `2.x`, or `5.x`) as the
-nodes in the cluster. Clients may connect to clusters which have a different
-minor version (e.g. `2.3.x`) but it is possible that new functionality may not
-be supported.  Ideally, the client should have the same version as the
-cluster.
-
-==============================
-
-[[transport-client]]
-=== Transport Client
-
-deprecated[7.0.0, The `TransportClient` is deprecated in favour of the {java-rest}/java-rest-high.html[Java High Level REST Client] and will be removed in Elasticsearch 8.0. The {java-rest}/java-rest-high-level-migration.html[migration guide] describes all the steps needed to migrate.]
-
-The `TransportClient` connects remotely to an Elasticsearch cluster
-using the transport module. It does not join the cluster, but simply
-gets one or more initial transport addresses and communicates with them
-in round robin fashion on each action (though most actions will probably
-be "two hop" operations).
-
-[source,java]
---------------------------------------------------
-// on startup
-
-TransportClient client = new PreBuiltTransportClient(Settings.EMPTY)
-        .addTransportAddress(new TransportAddress(InetAddress.getByName("host1"), 9300))
-        .addTransportAddress(new TransportAddress(InetAddress.getByName("host2"), 9300));
-
-// on shutdown
-
-client.close();
---------------------------------------------------
-
-Note that you have to set the cluster name if you use one different than
-"elasticsearch":
-
-[source,java]
---------------------------------------------------
-Settings settings = Settings.builder()
-        .put("cluster.name", "myClusterName").build();
-TransportClient client = new PreBuiltTransportClient(settings);
-//Add transport addresses and do something with the client...
---------------------------------------------------
-
-The Transport client comes with a cluster sniffing feature which
-allows it to dynamically add new hosts and remove old ones.
-When sniffing is enabled, the transport client will connect to the nodes in its
-internal node list, which is built via calls to `addTransportAddress`.
-After this, the client will call the internal cluster state API on those nodes
-to discover available data nodes. The internal node list of the client will
-be replaced with those data nodes only. This list is refreshed every five seconds by default.
-Note that the IP addresses the sniffer connects to are the ones declared as the 'publish'
-address in those node's Elasticsearch config.
-
-Keep in mind that the list might possibly not include the original node it connected to
-if that node is not a data node. If, for instance, you initially connect to a
-master node, after sniffing, no further requests will go to that master node,
-but rather to any data nodes instead. The reason the transport client excludes non-data
-nodes is to avoid sending search traffic to master only nodes.
-
-In order to enable sniffing, set `client.transport.sniff` to `true`:
-
-[source,java]
---------------------------------------------------
-Settings settings = Settings.builder()
-        .put("client.transport.sniff", true).build();
-TransportClient client = new PreBuiltTransportClient(settings);
---------------------------------------------------
-
-Other transport client level settings include:
-
-[cols="<,<",options="header",]
-|=======================================================================
-|Parameter |Description
-|`client.transport.ignore_cluster_name` |Set to `true` to ignore cluster
-name validation of connected nodes. (since 0.19.4)
-
-|`client.transport.ping_timeout` |The time to wait for a ping response
-from a node. Defaults to `5s`.
-
-|`client.transport.nodes_sampler_interval` |How often to sample / ping
-the nodes listed and connected. Defaults to `5s`.
-|=======================================================================
-
-
-[[client-connected-to-client-node]]
-=== Connecting a Client to a Coordinating Only Node
-
-You can start locally a {ref}/modules-node.html#coordinating-only-node[Coordinating Only Node]
-and then simply create a <<transport-client,`TransportClient`>> in your
-application which connects to this Coordinating Only Node.
-
-This way, the coordinating only node will be able to load whatever plugin you
-need (think about discovery plugins for example).
diff --git a/docs/java-api/docs.asciidoc b/docs/java-api/docs.asciidoc
deleted file mode 100644
index 181c5d8e0bd99..0000000000000
--- a/docs/java-api/docs.asciidoc
+++ /dev/null
@@ -1,36 +0,0 @@
-[[java-docs]]
-== Document APIs
-
-This section describes the following CRUD APIs:
-
-.Single document APIs
-* <<java-docs-index>>
-* <<java-docs-get>>
-* <<java-docs-delete>>
-* <<java-docs-update>>
-
-.Multi-document APIs
-* <<java-docs-multi-get>>
-* <<java-docs-bulk>>
-* <<java-docs-reindex>>
-* <<java-docs-update-by-query>>
-* <<java-docs-delete-by-query>>
-
-NOTE: All CRUD APIs are single-index APIs. The `index` parameter accepts a single
-index name, or an `alias` which points to a single index.
-
-include::docs/index_.asciidoc[]
-
-include::docs/get.asciidoc[]
-
-include::docs/delete.asciidoc[]
-
-include::docs/update.asciidoc[]
-
-include::docs/multi-get.asciidoc[]
-
-include::docs/bulk.asciidoc[]
-
-include::docs/update-by-query.asciidoc[]
-
-include::docs/reindex.asciidoc[]
\ No newline at end of file
diff --git a/docs/java-api/docs/bulk.asciidoc b/docs/java-api/docs/bulk.asciidoc
deleted file mode 100644
index 1c2882d9c07e7..0000000000000
--- a/docs/java-api/docs/bulk.asciidoc
+++ /dev/null
@@ -1,190 +0,0 @@
-[[java-docs-bulk]]
-=== Bulk API
-
-The bulk API allows one to index and delete several documents in a
-single request. Here is a sample usage:
-
-[source,java]
---------------------------------------------------
-import static org.elasticsearch.common.xcontent.XContentFactory.*;
-
-BulkRequestBuilder bulkRequest = client.prepareBulk();
-
-// either use client#prepare, or use Requests# to directly build index/delete requests
-bulkRequest.add(client.prepareIndex("twitter", "_doc", "1")
-        .setSource(jsonBuilder()
-                    .startObject()
-                        .field("user", "kimchy")
-                        .field("postDate", new Date())
-                        .field("message", "trying out Elasticsearch")
-                    .endObject()
-                  )
-        );
-
-bulkRequest.add(client.prepareIndex("twitter", "_doc", "2")
-        .setSource(jsonBuilder()
-                    .startObject()
-                        .field("user", "kimchy")
-                        .field("postDate", new Date())
-                        .field("message", "another post")
-                    .endObject()
-                  )
-        );
-        
-BulkResponse bulkResponse = bulkRequest.get();
-if (bulkResponse.hasFailures()) {
-    // process failures by iterating through each bulk response item
-}
---------------------------------------------------
-
-[[java-docs-bulk-processor]]
-=== Using Bulk Processor
-
-The `BulkProcessor` class offers a simple interface to flush bulk operations automatically based on the number or size
-of requests, or after a given period.
-
-To use it, first create a `BulkProcessor` instance:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.action.bulk.BackoffPolicy;
-import org.elasticsearch.action.bulk.BulkProcessor;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.unit.TimeValue;
-
-BulkProcessor bulkProcessor = BulkProcessor.builder(
-        client,  <1>
-        new BulkProcessor.Listener() {
-            @Override
-            public void beforeBulk(long executionId,
-                                   BulkRequest request) { ... } <2>
-
-            @Override
-            public void afterBulk(long executionId,
-                                  BulkRequest request,
-                                  BulkResponse response) { ... } <3>
-
-            @Override
-            public void afterBulk(long executionId,
-                                  BulkRequest request,
-                                  Throwable failure) { ... } <4>
-        })
-        .setBulkActions(10000) <5>
-        .setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB)) <6>
-        .setFlushInterval(TimeValue.timeValueSeconds(5)) <7>
-        .setConcurrentRequests(1) <8>
-        .setBackoffPolicy(
-            BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 3)) <9>
-        .build();
---------------------------------------------------
-<1> Add your Elasticsearch client
-<2> This method is called just before bulk is executed. You can for example see the numberOfActions with
-    `request.numberOfActions()`
-<3> This method is called after bulk execution. You can for example check if there was some failing requests
-    with `response.hasFailures()`
-<4> This method is called when the bulk failed and raised a `Throwable`
-<5> We want to execute the bulk every 10 000 requests
-<6> We want to flush the bulk every 5mb
-<7> We want to flush the bulk every 5 seconds whatever the number of requests
-<8> Set the number of concurrent requests. A value of 0 means that only a single request will be allowed to be
-    executed. A value of 1 means 1 concurrent request is allowed to be executed while accumulating new bulk requests.
-<9> Set a custom backoff policy which will initially wait for 100ms, increase exponentially and retries up to three
-    times. A retry is attempted whenever one or more bulk item requests have failed with an `EsRejectedExecutionException`
-    which indicates that there were too little compute resources available for processing the request. To disable backoff,
-    pass `BackoffPolicy.noBackoff()`.
-
-By default, `BulkProcessor`:
-
-* sets bulkActions to `1000`
-* sets bulkSize to `5mb`
-* does not set flushInterval
-* sets concurrentRequests to 1, which means an asynchronous execution of the flush operation.
-* sets backoffPolicy to an exponential backoff with 8 retries and a start delay of 50ms. The total wait time is roughly 5.1 seconds.
-
-[[java-docs-bulk-processor-requests]]
-==== Add requests
-
-Then you can simply add your requests to the `BulkProcessor`:
-
-[source,java]
---------------------------------------------------
-bulkProcessor.add(new IndexRequest("twitter", "_doc", "1").source(/* your doc here */));
-bulkProcessor.add(new DeleteRequest("twitter", "_doc", "2"));
---------------------------------------------------
-
-[[java-docs-bulk-processor-close]]
-==== Closing the Bulk Processor
-
-When all documents are loaded to the `BulkProcessor` it can be closed by using `awaitClose` or `close` methods:
-
-[source,java]
---------------------------------------------------
-bulkProcessor.awaitClose(10, TimeUnit.MINUTES);
---------------------------------------------------
-
-or
-
-[source,java]
---------------------------------------------------
-bulkProcessor.close();
---------------------------------------------------
-
-Both methods flush any remaining documents and disable all other scheduled flushes if they were scheduled by setting
-`flushInterval`. If concurrent requests were enabled the `awaitClose` method waits for up to the specified timeout for
-all bulk requests to complete then returns `true`, if the specified waiting time elapses before all bulk requests complete,
-`false` is returned. The `close` method doesn't wait for any remaining bulk requests to complete and exits immediately.
-
-[[java-docs-bulk-processor-tests]]
-==== Using Bulk Processor in tests
-
-If you are running tests with Elasticsearch and are using the `BulkProcessor` to populate your dataset
-you should better set the number of concurrent requests to `0` so the flush operation of the bulk will be executed
-in a synchronous manner:
-
-[source,java]
---------------------------------------------------
-BulkProcessor bulkProcessor = BulkProcessor.builder(client, new BulkProcessor.Listener() { /* Listener methods */ })
-        .setBulkActions(10000)
-        .setConcurrentRequests(0)
-        .build();
-
-// Add your requests
-bulkProcessor.add(/* Your requests */);
-
-// Flush any remaining requests
-bulkProcessor.flush();
-
-// Or close the bulkProcessor if you don't need it anymore
-bulkProcessor.close();
-
-// Refresh your indices
-client.admin().indices().prepareRefresh().get();
-
-// Now you can start searching!
-client.prepareSearch().get();
---------------------------------------------------
-
-
-[[java-docs-bulk-global-parameters]]
-==== Global Parameters
-
-Global parameters can be specified on the BulkRequest as well as BulkProcessor, similar to the REST API. These global
- parameters serve as defaults and can be overridden by local parameters specified on each sub request. Some parameters
- have to be set before any sub request is added - index, type - and you have to specify them during  BulkRequest or
- BulkProcessor creation. Some are optional - pipeline, routing - and can be specified at any point before the bulk is sent.
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{hlrc-tests}/BulkProcessorIT.java[bulk-processor-mix-parameters]
---------------------------------------------------
-<1> global parameters from the BulkRequest will be applied on a sub request
-<2> local pipeline parameter on a sub request will override global parameters from BulkRequest
-
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{hlrc-tests}/BulkRequestWithGlobalParametersIT.java[bulk-request-mix-pipeline]
---------------------------------------------------
-<1> local pipeline parameter on a sub request will override global pipeline from the BulkRequest
-<2> global parameter from the BulkRequest will be applied on a sub request
diff --git a/docs/java-api/docs/delete.asciidoc b/docs/java-api/docs/delete.asciidoc
deleted file mode 100644
index 004edc84b3d65..0000000000000
--- a/docs/java-api/docs/delete.asciidoc
+++ /dev/null
@@ -1,42 +0,0 @@
-[[java-docs-delete]]
-=== Delete API
-
-The delete API allows one to delete a typed JSON document from a specific
-index based on its id. The following example deletes the JSON document
-from an index called twitter, under a type called `_doc`, with id valued
-1:
-
-[source,java]
---------------------------------------------------
-DeleteResponse response = client.prepareDelete("twitter", "_doc", "1").get();
---------------------------------------------------
-
-For more information on the delete operation, check out the
-{ref}/docs-delete.html[delete API] docs.
-
-[[java-docs-delete-by-query]]
-=== Delete By Query API
-
-The delete by query API allows one to delete a given set of documents based on
-the result of a query:
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-reindex-tests}/ReindexDocumentationIT.java[delete-by-query-sync]
---------------------------------------------------
-<1> query
-<2> index
-<3> execute the operation
-<4> number of deleted documents
-
-As it can be a long running operation, if you wish to do it asynchronously, you can call `execute` instead of `get`
-and provide a listener like:
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-reindex-tests}/ReindexDocumentationIT.java[delete-by-query-async]
---------------------------------------------------
-<1> query
-<2> index
-<3> listener
-<4> number of deleted documents
diff --git a/docs/java-api/docs/get.asciidoc b/docs/java-api/docs/get.asciidoc
deleted file mode 100644
index ae03eb971004f..0000000000000
--- a/docs/java-api/docs/get.asciidoc
+++ /dev/null
@@ -1,14 +0,0 @@
-[[java-docs-get]]
-=== Get API
-
-The get API allows to get a typed JSON document from the index based on
-its id. The following example gets a JSON document from an index called
-twitter, under a type called `_doc``, with id valued 1:
-
-[source,java]
---------------------------------------------------
-GetResponse response = client.prepareGet("twitter", "_doc", "1").get();
---------------------------------------------------
-
-For more information on the get operation, check out the REST
-{ref}/docs-get.html[get] docs.
diff --git a/docs/java-api/docs/index_.asciidoc b/docs/java-api/docs/index_.asciidoc
deleted file mode 100644
index 80d187d5a77f3..0000000000000
--- a/docs/java-api/docs/index_.asciidoc
+++ /dev/null
@@ -1,167 +0,0 @@
-[[java-docs-index]]
-=== Index API
-
-The index API allows one to index a typed JSON document into a specific
-index and make it searchable.
-
-
-[[java-docs-index-generate]]
-==== Generate JSON document
-
-There are several different ways of generating a JSON document:
-
-* Manually (aka do it yourself) using native `byte[]` or as a `String`
-
-* Using a `Map` that will be automatically converted to its JSON
-equivalent
-
-* Using a third party library to serialize your beans such as
-https://github.com/FasterXML/jackson[Jackson]
-
-* Using built-in helpers XContentFactory.jsonBuilder()
-
-Internally, each type is converted to `byte[]` (so a String is converted
-to a `byte[]`). Therefore, if the object is in this form already, then
-use it. The `jsonBuilder` is highly optimized JSON generator that
-directly constructs a `byte[]`.
-
-
-[[java-docs-index-generate-diy]]
-===== Do It Yourself
-
-Nothing really difficult here but note that you will have to encode
-dates according to the
-{ref}/mapping-date-format.html[Date Format].
-
-[source,java]
---------------------------------------------------
-String json = "{" +
-        "\"user\":\"kimchy\"," +
-        "\"postDate\":\"2013-01-30\"," +
-        "\"message\":\"trying out Elasticsearch\"" +
-    "}";
---------------------------------------------------
-
-
-[[java-docs-index-generate-using-map]]
-===== Using Map
-
-Map is a key:values pair collection. It represents a JSON structure:
-
-[source,java]
---------------------------------------------------
-Map<String, Object> json = new HashMap<String, Object>();
-json.put("user","kimchy");
-json.put("postDate",new Date());
-json.put("message","trying out Elasticsearch");
---------------------------------------------------
-
-
-[[java-docs-index-generate-beans]]
-===== Serialize your beans
-
-You can use https://github.com/FasterXML/jackson[Jackson] to serialize
-your beans to JSON. Please add http://search.maven.org/#search%7Cga%7C1%7Cjackson-databind[Jackson Databind]
- to your project. Then you can use `ObjectMapper` to serialize your beans:
-
-[source,java]
---------------------------------------------------
-import com.fasterxml.jackson.databind.*;
-
-// instance a json mapper
-ObjectMapper mapper = new ObjectMapper(); // create once, reuse
-
-// generate json
-byte[] json = mapper.writeValueAsBytes(yourbeaninstance);
---------------------------------------------------
-
-
-[[java-docs-index-generate-helpers]]
-===== Use Elasticsearch helpers
-
-Elasticsearch provides built-in helpers to generate JSON content.
-
-[source,java]
---------------------------------------------------
-import static org.elasticsearch.common.xcontent.XContentFactory.*;
-
-XContentBuilder builder = jsonBuilder()
-    .startObject()
-        .field("user", "kimchy")
-        .field("postDate", new Date())
-        .field("message", "trying out Elasticsearch")
-    .endObject()
---------------------------------------------------
-
-Note that you can also add arrays with `startArray(String)` and
-`endArray()` methods. By the way, the `field` method +
- accepts many object types. You can directly pass numbers, dates and even
-other XContentBuilder objects.
-
-If you need to see the generated JSON content, you can use the
-`Strings.toString()` method.
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.common.Strings;
-
-String json = Strings.toString(builder);
---------------------------------------------------
-
-
-[[java-docs-index-doc]]
-==== Index document
-
-The following example indexes a JSON document into an index called
-twitter, under a type called `_doc``, with id valued 1:
-
-[source,java]
---------------------------------------------------
-import static org.elasticsearch.common.xcontent.XContentFactory.*;
-
-IndexResponse response = client.prepareIndex("twitter", "_doc", "1")
-        .setSource(jsonBuilder()
-                    .startObject()
-                        .field("user", "kimchy")
-                        .field("postDate", new Date())
-                        .field("message", "trying out Elasticsearch")
-                    .endObject()
-                  )
-        .get();
---------------------------------------------------
-
-Note that you can also index your documents as JSON String and that you
-don't have to give an ID:
-
-[source,java]
---------------------------------------------------
-String json = "{" +
-        "\"user\":\"kimchy\"," +
-        "\"postDate\":\"2013-01-30\"," +
-        "\"message\":\"trying out Elasticsearch\"" +
-    "}";
-
-IndexResponse response = client.prepareIndex("twitter", "_doc")
-        .setSource(json, XContentType.JSON)
-        .get();
---------------------------------------------------
-
-`IndexResponse` object will give you a report:
-
-[source,java]
---------------------------------------------------
-// Index name
-String _index = response.getIndex();
-// Type name
-String _type = response.getType();
-// Document ID (generated or not)
-String _id = response.getId();
-// Version (if it's the first time you index this document, you will get: 1)
-long _version = response.getVersion();
-// status has stored current instance statement.
-RestStatus status = response.status();
---------------------------------------------------
-
-For more information on the index operation, check out the REST
-{ref}/docs-index_.html[index] docs.
-
diff --git a/docs/java-api/docs/multi-get.asciidoc b/docs/java-api/docs/multi-get.asciidoc
deleted file mode 100644
index 8ed2bede2927c..0000000000000
--- a/docs/java-api/docs/multi-get.asciidoc
+++ /dev/null
@@ -1,30 +0,0 @@
-[[java-docs-multi-get]]
-=== Multi Get API
-
-The multi get API allows to get a list of documents based on their `index` and `id`:
-
-[source,java]
---------------------------------------------------
-MultiGetResponse multiGetItemResponses = client.prepareMultiGet()
-    .add("twitter", "_doc", "1")            <1>
-    .add("twitter", "_doc", "2", "3", "4")  <2>
-    .add("another", "_doc", "foo")          <3>
-    .get();
-
-for (MultiGetItemResponse itemResponse : multiGetItemResponses) { <4>
-    GetResponse response = itemResponse.getResponse();
-    if (response.isExists()) {                      <5>
-        String json = response.getSourceAsString(); <6>
-    }
-}
---------------------------------------------------
-<1> get by a single id
-<2> or by a list of ids for the same index
-<3> you can also get from another index
-<4> iterate over the result set
-<5> you can check if the document exists
-<6> access to the `_source` field
-
-For more information on the multi get operation, check out the REST
-{ref}/docs-multi-get.html[multi get] docs.
-
diff --git a/docs/java-api/docs/reindex.asciidoc b/docs/java-api/docs/reindex.asciidoc
deleted file mode 100644
index 842e763f74d71..0000000000000
--- a/docs/java-api/docs/reindex.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-docs-reindex]]
-=== Reindex API
-
-See {ref}/docs-reindex.html[reindex API].
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-reindex-tests}/ReindexDocumentationIT.java[reindex1]
---------------------------------------------------
-<1> Optionally a query can provided to filter what documents should be
-    re-indexed from the source to the target index.
diff --git a/docs/java-api/docs/update-by-query.asciidoc b/docs/java-api/docs/update-by-query.asciidoc
deleted file mode 100644
index ef58d3754276e..0000000000000
--- a/docs/java-api/docs/update-by-query.asciidoc
+++ /dev/null
@@ -1,166 +0,0 @@
-[[java-docs-update-by-query]]
-=== Update By Query API
-
-The simplest usage of `updateByQuery` updates each
-document in an index without changing the source. This usage enables
-picking up a new property or another online mapping change.
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-reindex-tests}/ReindexDocumentationIT.java[update-by-query]
---------------------------------------------------
-
-Calls to the `updateByQuery` API start by getting a snapshot of the index, indexing
-any documents found using the `internal` versioning.
-
-NOTE: Version conflicts happen when a document changes between the time of the
-snapshot and the time the index request processes.
-
-When the versions match, `updateByQuery` updates the document
-and increments the version number.
-
-All update and query failures cause `updateByQuery` to abort. These failures are
-available from the `BulkByScrollResponse#getIndexingFailures` method. Any
-successful updates remain and are not rolled back. While the first failure
-causes the abort, the response contains all of the failures generated by the
-failed bulk request.
-
-To prevent version conflicts from causing `updateByQuery` to abort, set
-`abortOnVersionConflict(false)`. The first example does this because it is
-trying to pick up an online mapping change and a version conflict means that
-the conflicting document was updated between the start of the `updateByQuery`
-and the time when it attempted to update the document. This is fine because
-that update will have picked up the online mapping update.
-
-The `UpdateByQueryRequestBuilder` API supports filtering the updated documents,
-limiting the total number of documents to update, and updating documents
-with a script:
-
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-reindex-tests}/ReindexDocumentationIT.java[update-by-query-filter]
---------------------------------------------------
-
-`UpdateByQueryRequestBuilder` also enables direct access to the query used
-to select the documents. You can use this access to change the default scroll size or
-otherwise modify the request for matching documents.
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-reindex-tests}/ReindexDocumentationIT.java[update-by-query-size]
---------------------------------------------------
-
-You can also combine `size` with sorting to limit the documents updated:
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-reindex-tests}/ReindexDocumentationIT.java[update-by-query-sort]
---------------------------------------------------
-
-In addition to changing the `_source` field for the document, you can use a
-script to change the action, similar to the Update API:
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-reindex-tests}/ReindexDocumentationIT.java[update-by-query-script]
---------------------------------------------------
-
-As in the <<java-docs-update,Update API>>, you can set the value of `ctx.op` to change the
-operation that executes:
-
-`noop`::
-
-Set `ctx.op = "noop"` if your script doesn't make any
-changes. The `updateByQuery` operation then omits that document from the updates.
-This behavior increments the `noop` counter in the response body.
-
-`delete`::
-
-Set `ctx.op = "delete"` if your script decides that the document must be
-deleted. The deletion will be reported in the `deleted` counter in the
-response body.
-
-Setting `ctx.op` to any other value generates an error. Setting any
-other field in `ctx` generates an error.
-
-This API doesn't allow you to move the documents it touches, just modify their
-source. This is intentional! We've made no provisions for removing the document
-from its original location.
-
-You can also perform these operations on multiple indices at once, similar to the search API:
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-reindex-tests}/ReindexDocumentationIT.java[update-by-query-multi-index]
---------------------------------------------------
-
-If you provide a `routing` value then the process copies the routing value to the scroll query,
-limiting the process to the shards that match that routing value:
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-reindex-tests}/ReindexDocumentationIT.java[update-by-query-routing]
---------------------------------------------------
-
-`updateByQuery` can also use the ingest node by
-specifying a `pipeline` like this:
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-reindex-tests}/ReindexDocumentationIT.java[update-by-query-pipeline]
---------------------------------------------------
-
-[float]
-[[java-docs-update-by-query-task-api]]
-=== Works with the Task API
-
-You can fetch the status of all running update-by-query requests with the Task API:
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-reindex-tests}/ReindexDocumentationIT.java[update-by-query-list-tasks]
---------------------------------------------------
-
-With the `TaskId` shown above you can look up the task directly:
-
-// provide API Example
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-reindex-tests}/ReindexDocumentationIT.java[update-by-query-get-task]
---------------------------------------------------
-
-[float]
-[[java-docs-update-by-query-cancel-task-api]]
-=== Works with the Cancel Task API
-
-Any Update By Query can be canceled using the Task Cancel API:
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-reindex-tests}/ReindexDocumentationIT.java[update-by-query-cancel-task]
---------------------------------------------------
-
-Use the `list tasks` API to find the value of `taskId`.
-
-Cancelling a request is typically a very fast process but can take up to a few seconds.
-The task status API continues to list the task until the cancellation is complete.
-
-[float]
-[[java-docs-update-by-query-rethrottle]]
-=== Rethrottling
-
-Use the `_rethrottle` API to change the value of `requests_per_second` on a running update:
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{client-reindex-tests}/ReindexDocumentationIT.java[update-by-query-rethrottle]
---------------------------------------------------
-
-Use the `list tasks` API to find the value of `taskId`.
-
-As with the `updateByQuery` API, the value of `requests_per_second`
-can be any positive float value to set the level of the throttle, or `Float.POSITIVE_INFINITY` to disable throttling.
-A value of `requests_per_second` that speeds up the process takes
-effect immediately. `requests_per_second` values that slow the query take effect
-after completing the current batch in order to prevent scroll timeouts.
diff --git a/docs/java-api/docs/update.asciidoc b/docs/java-api/docs/update.asciidoc
deleted file mode 100644
index 0935c9f11eca4..0000000000000
--- a/docs/java-api/docs/update.asciidoc
+++ /dev/null
@@ -1,118 +0,0 @@
-[[java-docs-update]]
-=== Update API
-
-
-You can either create an `UpdateRequest` and send it to the client:
-
-[source,java]
---------------------------------------------------
-UpdateRequest updateRequest = new UpdateRequest();
-updateRequest.index("index");
-updateRequest.type("_doc");
-updateRequest.id("1");
-updateRequest.doc(jsonBuilder()
-        .startObject()
-            .field("gender", "male")
-        .endObject());
-client.update(updateRequest).get();
---------------------------------------------------
-
-Or you can use `prepareUpdate()` method:
-
-[source,java]
---------------------------------------------------
-client.prepareUpdate("ttl", "doc", "1")
-        .setScript(new Script(
-            "ctx._source.gender = \"male\"", <1>
-            ScriptService.ScriptType.INLINE, null, null))
-        .get();
-
-client.prepareUpdate("ttl", "doc", "1")
-        .setDoc(jsonBuilder()               <2>
-            .startObject()
-                .field("gender", "male")
-            .endObject())
-        .get();
---------------------------------------------------
-<1> Your script. It could also be a locally stored script name.
-In that case, you'll need to use `ScriptService.ScriptType.FILE`
-<2> Document which will be merged to the existing one.
-
-Note that you can't provide both `script` and `doc`.
-
-[[java-docs-update-api-script]]
-==== Update by script
-
-The update API allows to update a document based on a script provided:
-
-[source,java]
---------------------------------------------------
-UpdateRequest updateRequest = new UpdateRequest("ttl", "doc", "1")
-        .script(new Script("ctx._source.gender = \"male\""));
-client.update(updateRequest).get();
---------------------------------------------------
-
-
-[[java-docs-update-api-merge-docs]]
-==== Update by merging documents
-
-The update API also support passing a partial document, which will be merged into the existing document (simple
-recursive merge, inner merging of objects, replacing core "keys/values" and arrays). For example:
-
-[source,java]
---------------------------------------------------
-UpdateRequest updateRequest = new UpdateRequest("index", "type", "1")
-        .doc(jsonBuilder()
-            .startObject()
-                .field("gender", "male")
-            .endObject());
-client.update(updateRequest).get();
---------------------------------------------------
-
-
-[[java-docs-update-api-upsert]]
-==== Upsert
-
-There is also support for `upsert`. If the document does not exist, the content of the `upsert`
-element will be used to index the fresh doc:
-
-[source,java]
---------------------------------------------------
-IndexRequest indexRequest = new IndexRequest("index", "type", "1")
-        .source(jsonBuilder()
-            .startObject()
-                .field("name", "Joe Smith")
-                .field("gender", "male")
-            .endObject());
-UpdateRequest updateRequest = new UpdateRequest("index", "type", "1")
-        .doc(jsonBuilder()
-            .startObject()
-                .field("gender", "male")
-            .endObject())
-        .upsert(indexRequest);              <1>
-client.update(updateRequest).get();
---------------------------------------------------
-<1> If the document does not exist, the one in `indexRequest` will be added
-
-If the document `index/_doc/1` already exists, we will have after this operation a document like:
-
-[source,js]
---------------------------------------------------
-{
-    "name"  : "Joe Dalton",
-    "gender": "male"        <1>
-}
---------------------------------------------------
-// NOTCONSOLE
-<1> This field is added by the update request
-
-If it does not exist, we will have a new document:
-
-[source,js]
---------------------------------------------------
-{
-    "name" : "Joe Smith",
-    "gender": "male"
-}
---------------------------------------------------
-// NOTCONSOLE
diff --git a/docs/java-api/index.asciidoc b/docs/java-api/index.asciidoc
deleted file mode 100644
index 4a7fd7482d26e..0000000000000
--- a/docs/java-api/index.asciidoc
+++ /dev/null
@@ -1,149 +0,0 @@
-= Java API
-
-include::../Versions.asciidoc[]
-
-[[java-api]]
-[preface]
-== Preface
-
-deprecated[7.0.0, The `TransportClient` is deprecated in favour of the {java-rest}/java-rest-high.html[Java High Level REST Client] and will be removed in Elasticsearch 8.0. The {java-rest}/java-rest-high-level-migration.html[migration guide] describes all the steps needed to migrate.]
-
-This section describes the Java API that Elasticsearch provides. All
-Elasticsearch operations are executed using a
-<<client,Client>> object. All
-operations are completely asynchronous in nature (either accepts a
-listener, or returns a future).
-
-Additionally, operations on a client may be accumulated and executed in
-<<java-docs-bulk,Bulk>>.
-
-Note, all the APIs are exposed through the
-Java API (actually, the Java API is used internally to execute them).
-
-== Javadoc
-
-The javadoc for the transport client can be found at {transport-client-javadoc}/index.html.
-
-== Maven Repository
-
-Elasticsearch is hosted on
-http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.elasticsearch.client%22[Maven
-Central].
-
-For example, you can define the latest version in your `pom.xml` file:
-
-["source","xml",subs="attributes"]
---------------------------------------------------
-<dependency>
-    <groupId>org.elasticsearch.client</groupId>
-    <artifactId>transport</artifactId>
-    <version>{version}</version>
-</dependency>
---------------------------------------------------
-
-[[java-transport-usage-maven-lucene]]
-=== Lucene Snapshot repository
-
-The very first releases of any major version (like a beta), might have been built on top of a Lucene Snapshot version.
-In such a case you will be unable to resolve the Lucene dependencies of the client.
-
-For example, if you want to use the `6.0.0-beta1` version which depends on Lucene `7.0.0-snapshot-00142c9`, you must
-define the following repository.
-
-For Maven:
-
-["source","xml",subs="attributes"]
---------------------------------------------------
-<repository>
-    <id>elastic-lucene-snapshots</id>
-    <name>Elastic Lucene Snapshots</name>
-    <url>https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/00142c9</url>
-    <releases><enabled>true</enabled></releases>
-    <snapshots><enabled>false</enabled></snapshots>
-</repository>
---------------------------------------------------
-
-For Gradle:
-
-["source","groovy",subs="attributes"]
---------------------------------------------------
-maven {
-    name "lucene-snapshots"
-    url 'https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/00142c9'
-}
---------------------------------------------------
-
-=== Log4j 2 Logger
-
-You need to also include Log4j 2 dependencies:
-
-["source","xml",subs="attributes"]
---------------------------------------------------
-<dependency>
-    <groupId>org.apache.logging.log4j</groupId>
-    <artifactId>log4j-core</artifactId>
-    <version>2.11.1</version>
-</dependency>
---------------------------------------------------
-
-And also provide a Log4j 2 configuration file in your classpath.
-For example, you can add in your `src/main/resources` project dir a `log4j2.properties` file like:
-
-
-["source","properties",subs="attributes"]
---------------------------------------------------
-appender.console.type = Console
-appender.console.name = console
-appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
-
-rootLogger.level = info
-rootLogger.appenderRef.console.ref = console
---------------------------------------------------
-
-=== Using another Logger
-
-If you want to use another logger than Log4j 2, you can use http://www.slf4j.org/[SLF4J] bridge to do that:
-
-["source","xml",subs="attributes"]
---------------------------------------------------
-<dependency>
-    <groupId>org.apache.logging.log4j</groupId>
-    <artifactId>log4j-to-slf4j</artifactId>
-    <version>2.11.1</version>
-</dependency>
-<dependency>
-    <groupId>org.slf4j</groupId>
-    <artifactId>slf4j-api</artifactId>
-    <version>1.7.24</version>
-</dependency>
---------------------------------------------------
-
-http://www.slf4j.org/manual.html[This page] lists implementations you can use. Pick your favorite logger
-and add it as a dependency. As an example, we will use the `slf4j-simple` logger:
-
-["source","xml",subs="attributes"]
---------------------------------------------------
-<dependency>
-    <groupId>org.slf4j</groupId>
-    <artifactId>slf4j-simple</artifactId>
-    <version>1.7.21</version>
-</dependency>
---------------------------------------------------
-
-:client-tests: {docdir}/../../server/src/test/java/org/elasticsearch/client/documentation
-:hlrc-tests: {docdir}/../../client/rest-high-level/src/test/java/org/elasticsearch/client
-
-:client-reindex-tests: {docdir}/../../modules/reindex/src/test/java/org/elasticsearch/client/documentation
-
-include::client.asciidoc[]
-
-include::docs.asciidoc[]
-
-include::search.asciidoc[]
-
-include::aggs.asciidoc[]
-
-include::query-dsl.asciidoc[]
-
-include::admin/index.asciidoc[]
diff --git a/docs/java-api/query-dsl.asciidoc b/docs/java-api/query-dsl.asciidoc
deleted file mode 100644
index f4823fa08ab51..0000000000000
--- a/docs/java-api/query-dsl.asciidoc
+++ /dev/null
@@ -1,40 +0,0 @@
-[[java-query-dsl]]
-== Query DSL
-
-Elasticsearch provides a full Java query dsl in a similar manner to the
-REST {ref}/query-dsl.html[Query DSL]. The factory for query
-builders is `QueryBuilders`. Once your query is ready, you can use the
-<<java-search,Search API>>.
-
-To use `QueryBuilders` just import them in your class:
-
-[source,java]
---------------------------------------------------
-import static org.elasticsearch.index.query.QueryBuilders.*;
---------------------------------------------------
-
-Note that you can easily print (aka debug) JSON generated queries using
-`toString()` method on `QueryBuilder` object.
-
-The `QueryBuilder` can then be used with any API that accepts a query,
-such as `count` and `search`.
-
-:query-dsl-test: {docdir}/../../client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/QueryDSLDocumentationTests.java
-
-include::query-dsl/match-all-query.asciidoc[]
-
-include::query-dsl/full-text-queries.asciidoc[]
-
-include::query-dsl/term-level-queries.asciidoc[]
-
-include::query-dsl/compound-queries.asciidoc[]
-
-include::query-dsl/joining-queries.asciidoc[]
-
-include::query-dsl/geo-queries.asciidoc[]
-
-include::query-dsl/special-queries.asciidoc[]
-
-include::query-dsl/span-queries.asciidoc[]
-
-:query-dsl-test!:
diff --git a/docs/java-api/query-dsl/bool-query.asciidoc b/docs/java-api/query-dsl/bool-query.asciidoc
deleted file mode 100644
index da9ca0ad0cc8c..0000000000000
--- a/docs/java-api/query-dsl/bool-query.asciidoc
+++ /dev/null
@@ -1,13 +0,0 @@
-[[java-query-dsl-bool-query]]
-==== Bool Query
-
-See {ref}/query-dsl-bool-query.html[Bool Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[bool]
---------------------------------------------------
-<1> must query
-<2> must not query
-<3> should query
-<4> a query that must appear in the matching documents but doesn't contribute to scoring.
diff --git a/docs/java-api/query-dsl/boosting-query.asciidoc b/docs/java-api/query-dsl/boosting-query.asciidoc
deleted file mode 100644
index 2a3c4437d1f89..0000000000000
--- a/docs/java-api/query-dsl/boosting-query.asciidoc
+++ /dev/null
@@ -1,12 +0,0 @@
-[[java-query-dsl-boosting-query]]
-==== Boosting Query
-
-See {ref}/query-dsl-boosting-query.html[Boosting Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[boosting]
---------------------------------------------------
-<1> query that will promote documents
-<2> query that will demote documents
-<3> negative boost
diff --git a/docs/java-api/query-dsl/common-terms-query.asciidoc b/docs/java-api/query-dsl/common-terms-query.asciidoc
deleted file mode 100644
index 2c8dfc7a88cfe..0000000000000
--- a/docs/java-api/query-dsl/common-terms-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-common-terms-query]]
-==== Common Terms Query
-
-See {ref}/query-dsl-common-terms-query.html[Common Terms Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[common_terms]
---------------------------------------------------
-<1> field
-<2> value
diff --git a/docs/java-api/query-dsl/compound-queries.asciidoc b/docs/java-api/query-dsl/compound-queries.asciidoc
deleted file mode 100644
index b93e3b694a5ef..0000000000000
--- a/docs/java-api/query-dsl/compound-queries.asciidoc
+++ /dev/null
@@ -1,45 +0,0 @@
-[[java-compound-queries]]
-=== Compound queries
-
-Compound queries wrap other compound or leaf queries, either to combine their
-results and scores, to change their behaviour, or to switch from query to
-filter context.
-
-The queries in this group are:
-
-<<java-query-dsl-constant-score-query,`constant_score` query>>::
-
-A query which wraps another query, but executes it in filter context.  All
-matching documents are given the same ``constant'' `_score`.
-
-<<java-query-dsl-bool-query,`bool` query>>::
-
-The default query for combining multiple leaf or compound query clauses, as
-`must`, `should`, `must_not`, or `filter` clauses.  The `must` and `should`
-clauses have their scores combined -- the more matching clauses, the better --
-while the `must_not` and `filter` clauses are executed in filter context.
-
-<<java-query-dsl-dis-max-query,`dis_max` query>>::
-
-A query which accepts multiple queries, and returns any documents which match
-any of the query clauses.  While the `bool` query combines the scores from all
-matching queries, the `dis_max` query uses the score of the single best-
-matching query clause.
-
-<<java-query-dsl-function-score-query,`function_score` query>>::
-
-Modify the scores returned by the main query with functions to take into
-account factors like popularity, recency, distance, or custom algorithms
-implemented with scripting.
-
-<<java-query-dsl-boosting-query,`boosting` query>>::
-
-Return documents which match a `positive` query, but reduce the score of
-documents which also match a `negative` query.
-
-
-include::constant-score-query.asciidoc[]
-include::bool-query.asciidoc[]
-include::dis-max-query.asciidoc[]
-include::function-score-query.asciidoc[]
-include::boosting-query.asciidoc[]
diff --git a/docs/java-api/query-dsl/constant-score-query.asciidoc b/docs/java-api/query-dsl/constant-score-query.asciidoc
deleted file mode 100644
index 49c5adbee6a73..0000000000000
--- a/docs/java-api/query-dsl/constant-score-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-constant-score-query]]
-==== Constant Score Query
-
-See {ref}/query-dsl-constant-score-query.html[Constant Score Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[constant_score]
---------------------------------------------------
-<1> your query
-<2> query score
diff --git a/docs/java-api/query-dsl/dis-max-query.asciidoc b/docs/java-api/query-dsl/dis-max-query.asciidoc
deleted file mode 100644
index 8c91bcb99011a..0000000000000
--- a/docs/java-api/query-dsl/dis-max-query.asciidoc
+++ /dev/null
@@ -1,13 +0,0 @@
-[[java-query-dsl-dis-max-query]]
-==== Dis Max Query
-
-See {ref}/query-dsl-dis-max-query.html[Dis Max Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[dis_max]
---------------------------------------------------
-<1> add your queries
-<2> add your queries
-<3> boost factor
-<4> tie breaker
diff --git a/docs/java-api/query-dsl/exists-query.asciidoc b/docs/java-api/query-dsl/exists-query.asciidoc
deleted file mode 100644
index 6fa5ba6a6f257..0000000000000
--- a/docs/java-api/query-dsl/exists-query.asciidoc
+++ /dev/null
@@ -1,10 +0,0 @@
-[[java-query-dsl-exists-query]]
-==== Exists Query
-
-See {ref}/query-dsl-exists-query.html[Exists Query].
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[exists]
---------------------------------------------------
-<1> field
diff --git a/docs/java-api/query-dsl/full-text-queries.asciidoc b/docs/java-api/query-dsl/full-text-queries.asciidoc
deleted file mode 100644
index 27ce4bee1ba64..0000000000000
--- a/docs/java-api/query-dsl/full-text-queries.asciidoc
+++ /dev/null
@@ -1,44 +0,0 @@
-[[java-full-text-queries]]
-=== Full text queries
-
-The high-level full text queries are usually used for running full text
-queries on full text fields like the body of an email. They understand how the
-field being queried is analyzed and will apply each field's
-`analyzer` (or `search_analyzer`) to the query string before executing.
-
-The queries in this group are:
-
-<<java-query-dsl-match-query,`match` query>>::
-
-    The standard query for performing full text queries, including fuzzy matching
-    and phrase or proximity queries.
-
-<<java-query-dsl-multi-match-query,`multi_match` query>>::
-
-    The multi-field version of the `match` query.
-
-<<java-query-dsl-common-terms-query,`common_terms` query>>::
-
-    A more specialized query which gives more preference to uncommon words.
-
-<<java-query-dsl-query-string-query,`query_string` query>>::
-
-    Supports the compact Lucene query string syntax,
-    allowing you to specify AND|OR|NOT conditions and multi-field search
-    within a single query string. For expert users only.
-
-<<java-query-dsl-simple-query-string-query,`simple_query_string`>>::
-
-    A simpler, more robust version of the `query_string` syntax suitable
-    for exposing directly to users.
-
-include::match-query.asciidoc[]
-
-include::multi-match-query.asciidoc[]
-
-include::common-terms-query.asciidoc[]
-
-include::query-string-query.asciidoc[]
-
-include::simple-query-string-query.asciidoc[]
-
diff --git a/docs/java-api/query-dsl/function-score-query.asciidoc b/docs/java-api/query-dsl/function-score-query.asciidoc
deleted file mode 100644
index fcd5f2dc473f5..0000000000000
--- a/docs/java-api/query-dsl/function-score-query.asciidoc
+++ /dev/null
@@ -1,19 +0,0 @@
-[[java-query-dsl-function-score-query]]
-==== Function Score Query
-
-See {ref}/query-dsl-function-score-query.html[Function Score Query].
-
-To use `ScoreFunctionBuilders` just import them in your class:
-
-[source,java]
---------------------------------------------------
-import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.*;
---------------------------------------------------
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[function_score]
---------------------------------------------------
-<1> Add a first function based on a query
-<2> And randomize the score based on a given seed
-<3> Add another function based on the age field
diff --git a/docs/java-api/query-dsl/fuzzy-query.asciidoc b/docs/java-api/query-dsl/fuzzy-query.asciidoc
deleted file mode 100644
index 4a7bde82cdfb7..0000000000000
--- a/docs/java-api/query-dsl/fuzzy-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-fuzzy-query]]
-==== Fuzzy Query
-
-See {ref}/query-dsl-fuzzy-query.html[Fuzzy Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[fuzzy]
---------------------------------------------------
-<1> field
-<2> text
diff --git a/docs/java-api/query-dsl/geo-bounding-box-query.asciidoc b/docs/java-api/query-dsl/geo-bounding-box-query.asciidoc
deleted file mode 100644
index 4983a21213376..0000000000000
--- a/docs/java-api/query-dsl/geo-bounding-box-query.asciidoc
+++ /dev/null
@@ -1,12 +0,0 @@
-[[java-query-dsl-geo-bounding-box-query]]
-==== Geo Bounding Box Query
-
-See {ref}/query-dsl-geo-bounding-box-query.html[Geo Bounding Box Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[geo_bounding_box]
---------------------------------------------------
-<1> field
-<2> bounding box top left point
-<3> bounding box bottom right point
diff --git a/docs/java-api/query-dsl/geo-distance-query.asciidoc b/docs/java-api/query-dsl/geo-distance-query.asciidoc
deleted file mode 100644
index cc8c89ca61eea..0000000000000
--- a/docs/java-api/query-dsl/geo-distance-query.asciidoc
+++ /dev/null
@@ -1,12 +0,0 @@
-[[java-query-dsl-geo-distance-query]]
-==== Geo Distance Query
-
-See {ref}/query-dsl-geo-distance-query.html[Geo Distance Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[geo_distance]
---------------------------------------------------
-<1> field
-<2> center point
-<3> distance from center point
diff --git a/docs/java-api/query-dsl/geo-polygon-query.asciidoc b/docs/java-api/query-dsl/geo-polygon-query.asciidoc
deleted file mode 100644
index 7dbf49b8d1afd..0000000000000
--- a/docs/java-api/query-dsl/geo-polygon-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-geo-polygon-query]]
-==== Geo Polygon Query
-
-See {ref}/query-dsl-geo-polygon-query.html[Geo Polygon Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[geo_polygon]
---------------------------------------------------
-<1> add your polygon of points a document should fall within
-<2> initialise the query with field and points
diff --git a/docs/java-api/query-dsl/geo-queries.asciidoc b/docs/java-api/query-dsl/geo-queries.asciidoc
deleted file mode 100644
index 10df4ff5e8716..0000000000000
--- a/docs/java-api/query-dsl/geo-queries.asciidoc
+++ /dev/null
@@ -1,34 +0,0 @@
-[[java-geo-queries]]
-=== Geo queries
-
-Elasticsearch supports two types of geo data:
-`geo_point` fields which support lat/lon pairs, and
-`geo_shape` fields, which support points, lines, circles, polygons, multi-polygons etc.
-
-The queries in this group are:
-
-<<java-query-dsl-geo-shape-query,`geo_shape`>> query::
-
-    Find document with geo-shapes which either intersect, are contained by, or
-    do not intersect with the specified geo-shape.
-
-<<java-query-dsl-geo-bounding-box-query,`geo_bounding_box`>> query::
-
-    Finds documents with geo-points that fall into the specified rectangle.
-
-<<java-query-dsl-geo-distance-query,`geo_distance`>> query::
-
-    Finds document with geo-points within the specified distance of a central
-    point.
-
-<<java-query-dsl-geo-polygon-query,`geo_polygon`>> query::
-
-    Find documents with geo-points within the specified polygon.
-
-include::geo-shape-query.asciidoc[]
-
-include::geo-bounding-box-query.asciidoc[]
-
-include::geo-distance-query.asciidoc[]
-
-include::geo-polygon-query.asciidoc[]
diff --git a/docs/java-api/query-dsl/geo-shape-query.asciidoc b/docs/java-api/query-dsl/geo-shape-query.asciidoc
deleted file mode 100644
index c2cd4c14e3adc..0000000000000
--- a/docs/java-api/query-dsl/geo-shape-query.asciidoc
+++ /dev/null
@@ -1,56 +0,0 @@
-[[java-query-dsl-geo-shape-query]]
-==== GeoShape Query
-
-See {ref}/query-dsl-geo-shape-query.html[Geo Shape Query]
-
-Note: the `geo_shape` type uses `Spatial4J` and `JTS`, both of which are
-optional dependencies. Consequently you must add `Spatial4J` and `JTS`
-to your classpath in order to use this type:
-
-[source,xml]
------------------------------------------------
-<dependency>
-    <groupId>org.locationtech.spatial4j</groupId>
-    <artifactId>spatial4j</artifactId>
-    <version>0.7</version>                        <1>
-</dependency>
-
-<dependency>
-    <groupId>org.locationtech.jts</groupId>
-    <artifactId>jts-core</artifactId>
-    <version>1.15.0</version>                         <2>
-    <exclusions>
-        <exclusion>
-            <groupId>xerces</groupId>
-            <artifactId>xercesImpl</artifactId>
-        </exclusion>
-    </exclusions>
-</dependency>
------------------------------------------------
-<1> check for updates in http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.locationtech.spatial4j%22%20AND%20a%3A%22spatial4j%22[Maven Central]
-<2> check for updates in http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.locationtech.jts%22%20AND%20a%3A%22jts-core%22[Maven Central]
-
-[source,java]
---------------------------------------------------
-// Import ShapeRelation and ShapeBuilder
-import org.elasticsearch.common.geo.ShapeRelation;
-import org.elasticsearch.common.geo.builders.ShapeBuilder;
---------------------------------------------------
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[geo_shape]
---------------------------------------------------
-<1> field
-<2> shape
-<3> relation can be `ShapeRelation.CONTAINS`, `ShapeRelation.WITHIN`, `ShapeRelation.INTERSECTS` or `ShapeRelation.DISJOINT`
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[indexed_geo_shape]
---------------------------------------------------
-<1> field
-<2> The ID of the document that containing the pre-indexed shape.
-<3> relation
-<4> Name of the index where the pre-indexed shape is. Defaults to 'shapes'.
-<5> The field specified as path containing the pre-indexed shape. Defaults to 'shape'.
diff --git a/docs/java-api/query-dsl/has-child-query.asciidoc b/docs/java-api/query-dsl/has-child-query.asciidoc
deleted file mode 100644
index f47f3af487dfe..0000000000000
--- a/docs/java-api/query-dsl/has-child-query.asciidoc
+++ /dev/null
@@ -1,23 +0,0 @@
-[[java-query-dsl-has-child-query]]
-==== Has Child Query
-
-See {ref}/query-dsl-has-child-query.html[Has Child Query]
-
-When using the `has_child` query it is important to use the `PreBuiltTransportClient` instead of the regular client:
-
-[source,java]
---------------------------------------------------
-Settings settings = Settings.builder().put("cluster.name", "elasticsearch").build();
-TransportClient client = new PreBuiltTransportClient(settings);
-client.addTransportAddress(new TransportAddress(new InetSocketAddress(InetAddresses.forString("127.0.0.1"), 9300)));
---------------------------------------------------
-
-Otherwise the parent-join module doesn't get loaded and the `has_child` query can't be used from the transport client.
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[has_child]
---------------------------------------------------
-<1> child type to query against
-<2> query
-<3> score mode can be `ScoreMode.Avg`, `ScoreMode.Max`, `ScoreMode.Min`, `ScoreMode.None` or `ScoreMode.Total`
diff --git a/docs/java-api/query-dsl/has-parent-query.asciidoc b/docs/java-api/query-dsl/has-parent-query.asciidoc
deleted file mode 100644
index 6a83fe2b0698f..0000000000000
--- a/docs/java-api/query-dsl/has-parent-query.asciidoc
+++ /dev/null
@@ -1,23 +0,0 @@
-[[java-query-dsl-has-parent-query]]
-==== Has Parent Query
-
-See {ref}/query-dsl-has-parent-query.html[Has Parent]
-
-When using the `has_parent` query it is important to use the `PreBuiltTransportClient` instead of the regular client:
-
-[source,java]
---------------------------------------------------
-Settings settings = Settings.builder().put("cluster.name", "elasticsearch").build();
-TransportClient client = new PreBuiltTransportClient(settings);
-client.addTransportAddress(new TransportAddress(new InetSocketAddress(InetAddresses.forString("127.0.0.1"), 9300)));
---------------------------------------------------
-
-Otherwise the parent-join module doesn't get loaded and the `has_parent` query can't be used from the transport client.
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[has_parent]
---------------------------------------------------
-<1> parent type to query against
-<2> query
-<3> whether the score from the parent hit should propagate to the child hit
diff --git a/docs/java-api/query-dsl/ids-query.asciidoc b/docs/java-api/query-dsl/ids-query.asciidoc
deleted file mode 100644
index ba12a5df38b0e..0000000000000
--- a/docs/java-api/query-dsl/ids-query.asciidoc
+++ /dev/null
@@ -1,10 +0,0 @@
-[[java-query-dsl-ids-query]]
-==== Ids Query
-
-
-See {ref}/query-dsl-ids-query.html[Ids Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[ids]
---------------------------------------------------
diff --git a/docs/java-api/query-dsl/joining-queries.asciidoc b/docs/java-api/query-dsl/joining-queries.asciidoc
deleted file mode 100644
index fcefef5f6245b..0000000000000
--- a/docs/java-api/query-dsl/joining-queries.asciidoc
+++ /dev/null
@@ -1,28 +0,0 @@
-[[java-joining-queries]]
-=== Joining queries
-
-Performing full SQL-style joins in a distributed system like Elasticsearch is
-prohibitively expensive.  Instead, Elasticsearch offers two forms of join
-which are designed to scale horizontally.
-
-<<java-query-dsl-nested-query,`nested` query>>::
-
-Documents may contains fields of type `nested`. These
-fields are used to index arrays of objects, where each object can be queried
-(with the `nested` query) as an independent document.
-
-<<java-query-dsl-has-child-query,`has_child`>> and <<java-query-dsl-has-parent-query,`has_parent`>> queries::
-
-A parent-child relationship can exist between two
-document types within a single index. The `has_child` query returns parent
-documents whose child documents match the specified query, while the
-`has_parent` query returns child documents whose parent document matches the
-specified query.
-
-include::nested-query.asciidoc[]
-
-include::has-child-query.asciidoc[]
-
-include::has-parent-query.asciidoc[]
-
-
diff --git a/docs/java-api/query-dsl/match-all-query.asciidoc b/docs/java-api/query-dsl/match-all-query.asciidoc
deleted file mode 100644
index 85d847528f5b8..0000000000000
--- a/docs/java-api/query-dsl/match-all-query.asciidoc
+++ /dev/null
@@ -1,9 +0,0 @@
-[[java-query-dsl-match-all-query]]
-=== Match All Query
-
-See {ref}/query-dsl-match-all-query.html[Match All Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[match_all]
---------------------------------------------------
diff --git a/docs/java-api/query-dsl/match-query.asciidoc b/docs/java-api/query-dsl/match-query.asciidoc
deleted file mode 100644
index 6884deb5f1f24..0000000000000
--- a/docs/java-api/query-dsl/match-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-match-query]]
-==== Match Query
-
-See {ref}/query-dsl-match-query.html[Match Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[match]
---------------------------------------------------
-<1> field
-<2> text
diff --git a/docs/java-api/query-dsl/mlt-query.asciidoc b/docs/java-api/query-dsl/mlt-query.asciidoc
deleted file mode 100644
index 11e5c7ef40482..0000000000000
--- a/docs/java-api/query-dsl/mlt-query.asciidoc
+++ /dev/null
@@ -1,13 +0,0 @@
-[[java-query-dsl-mlt-query]]
-==== More Like This Query
-
-See {ref}/query-dsl-mlt-query.html[More Like This Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[more_like_this]
---------------------------------------------------
-<1> fields
-<2> text
-<3> ignore threshold
-<4> max num of Terms in generated queries
diff --git a/docs/java-api/query-dsl/multi-match-query.asciidoc b/docs/java-api/query-dsl/multi-match-query.asciidoc
deleted file mode 100644
index 86b384d44d3c0..0000000000000
--- a/docs/java-api/query-dsl/multi-match-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-multi-match-query]]
-==== Multi Match Query
-
-See {ref}/query-dsl-multi-match-query.html[Multi Match Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[multi_match]
---------------------------------------------------
-<1> text
-<2> fields
diff --git a/docs/java-api/query-dsl/nested-query.asciidoc b/docs/java-api/query-dsl/nested-query.asciidoc
deleted file mode 100644
index 9b675ea72acfd..0000000000000
--- a/docs/java-api/query-dsl/nested-query.asciidoc
+++ /dev/null
@@ -1,12 +0,0 @@
-[[java-query-dsl-nested-query]]
-==== Nested Query
-
-See {ref}/query-dsl-nested-query.html[Nested Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[nested]
---------------------------------------------------
-<1> path to nested document
-<2> your query. Any fields referenced inside the query must use the complete path (fully qualified).
-<3> score mode could be `ScoreMode.Max`, `ScoreMode.Min`, `ScoreMode.Total`, `ScoreMode.Avg` or `ScoreMode.None`
diff --git a/docs/java-api/query-dsl/percolate-query.asciidoc b/docs/java-api/query-dsl/percolate-query.asciidoc
deleted file mode 100644
index 18cdd4a14e5a9..0000000000000
--- a/docs/java-api/query-dsl/percolate-query.asciidoc
+++ /dev/null
@@ -1,61 +0,0 @@
-[[java-query-percolate-query]]
-==== Percolate Query
-
-See:
- * {ref}/query-dsl-percolate-query.html[Percolate Query]
-
-
-[source,java]
---------------------------------------------------
-Settings settings = Settings.builder().put("cluster.name", "elasticsearch").build();
-TransportClient client = new PreBuiltTransportClient(settings);
-client.addTransportAddress(new TransportAddress(new InetSocketAddress(InetAddresses.forString("127.0.0.1"), 9300)));
---------------------------------------------------
-
-Before the `percolate` query can be used an `percolator` mapping should be added and
-a document containing a percolator query should be indexed:
-
-[source,java]
---------------------------------------------------
-// create an index with a percolator field with the name 'query':
-client.admin().indices().prepareCreate("myIndexName")
-                        .addMapping("_doc", "query", "type=percolator", "content", "type=text")
-                        .get();
-
-//This is the query we're registering in the percolator
-QueryBuilder qb = termQuery("content", "amazing");
-
-//Index the query = register it in the percolator
-client.prepareIndex("myIndexName", "_doc", "myDesignatedQueryName")
-    .setSource(jsonBuilder()
-        .startObject()
-            .field("query", qb) // Register the query
-        .endObject())
-    .setRefreshPolicy(RefreshPolicy.IMMEDIATE) // Needed when the query shall be available immediately
-    .get();
---------------------------------------------------
-
-This indexes the above term query under the name
-*myDesignatedQueryName*.
-
-In order to check a document against the registered queries, use this
-code:
-
-[source,java]
---------------------------------------------------
-//Build a document to check against the percolator
-XContentBuilder docBuilder = XContentFactory.jsonBuilder().startObject();
-docBuilder.field("content", "This is amazing!");
-docBuilder.endObject(); //End of the JSON root object
-
-PercolateQueryBuilder percolateQuery = new PercolateQueryBuilder("query", "_doc", BytesReference.bytes(docBuilder));
-
-// Percolate, by executing the percolator query in the query dsl:
-SearchResponse response = client().prepareSearch("myIndexName")
-        .setQuery(percolateQuery))
-        .get();
-//Iterate over the results
-for(SearchHit hit : response.getHits()) {
-    // Percolator queries as hit
-}
---------------------------------------------------
diff --git a/docs/java-api/query-dsl/prefix-query.asciidoc b/docs/java-api/query-dsl/prefix-query.asciidoc
deleted file mode 100644
index eb15c4426f633..0000000000000
--- a/docs/java-api/query-dsl/prefix-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-prefix-query]]
-==== Prefix Query
-
-See {ref}/query-dsl-prefix-query.html[Prefix Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[prefix]
---------------------------------------------------
-<1> field
-<2> prefix
diff --git a/docs/java-api/query-dsl/query-string-query.asciidoc b/docs/java-api/query-dsl/query-string-query.asciidoc
deleted file mode 100644
index 7d8bead2e340a..0000000000000
--- a/docs/java-api/query-dsl/query-string-query.asciidoc
+++ /dev/null
@@ -1,9 +0,0 @@
-[[java-query-dsl-query-string-query]]
-==== Query String Query
-
-See {ref}/query-dsl-query-string-query.html[Query String Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[query_string]
---------------------------------------------------
diff --git a/docs/java-api/query-dsl/range-query.asciidoc b/docs/java-api/query-dsl/range-query.asciidoc
deleted file mode 100644
index 2d58fbd3a34ef..0000000000000
--- a/docs/java-api/query-dsl/range-query.asciidoc
+++ /dev/null
@@ -1,22 +0,0 @@
-[[java-query-dsl-range-query]]
-==== Range Query
-
-See {ref}/query-dsl-range-query.html[Range Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[range]
---------------------------------------------------
-<1> field
-<2> from
-<3> to
-<4> include lower value means that `from` is `gt` when `false` or `gte` when `true`
-<5> include upper value means that `to` is `lt` when `false` or `lte` when `true`
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[range_simplified]
---------------------------------------------------
-<1> field
-<2> set `from` to 10 and `includeLower` to `true`
-<3> set `to` to 20 and `includeUpper` to `false`
diff --git a/docs/java-api/query-dsl/regexp-query.asciidoc b/docs/java-api/query-dsl/regexp-query.asciidoc
deleted file mode 100644
index f9cd8cd72d9d5..0000000000000
--- a/docs/java-api/query-dsl/regexp-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-regexp-query]]
-==== Regexp Query
-
-See {ref}/query-dsl-regexp-query.html[Regexp Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[regexp]
---------------------------------------------------
-<1> field
-<2> regexp
diff --git a/docs/java-api/query-dsl/script-query.asciidoc b/docs/java-api/query-dsl/script-query.asciidoc
deleted file mode 100644
index a8c60f1d8eb0d..0000000000000
--- a/docs/java-api/query-dsl/script-query.asciidoc
+++ /dev/null
@@ -1,29 +0,0 @@
-[[java-query-dsl-script-query]]
-==== Script Query
-
-See {ref}/query-dsl-script-query.html[Script Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[script_inline]
---------------------------------------------------
-<1> inlined script
-
-
-If you have stored on each data node a script named `myscript.painless` with:
-
-[source,painless]
---------------------------------------------------
-doc['num1'].value > params.param1
---------------------------------------------------
-
-You can use it then with:
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[script_file]
---------------------------------------------------
-<1> Script type: either `ScriptType.FILE`, `ScriptType.INLINE` or `ScriptType.INDEXED`
-<2> Scripting engine
-<3> Script name
-<4> Parameters as a `Map<String, Object>`
diff --git a/docs/java-api/query-dsl/simple-query-string-query.asciidoc b/docs/java-api/query-dsl/simple-query-string-query.asciidoc
deleted file mode 100644
index c3b32ecd1cbb2..0000000000000
--- a/docs/java-api/query-dsl/simple-query-string-query.asciidoc
+++ /dev/null
@@ -1,9 +0,0 @@
-[[java-query-dsl-simple-query-string-query]]
-==== Simple Query String Query
-
-See {ref}/query-dsl-simple-query-string-query.html[Simple Query String Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[simple_query_string]
---------------------------------------------------
diff --git a/docs/java-api/query-dsl/span-containing-query.asciidoc b/docs/java-api/query-dsl/span-containing-query.asciidoc
deleted file mode 100644
index 173e26952c265..0000000000000
--- a/docs/java-api/query-dsl/span-containing-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-span-containing-query]]
-==== Span Containing Query
-
-See {ref}/query-dsl-span-containing-query.html[Span Containing Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[span_containing]
---------------------------------------------------
-<1> `big` part
-<2> `little` part
diff --git a/docs/java-api/query-dsl/span-first-query.asciidoc b/docs/java-api/query-dsl/span-first-query.asciidoc
deleted file mode 100644
index d02c164754c53..0000000000000
--- a/docs/java-api/query-dsl/span-first-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-span-first-query]]
-==== Span First Query
-
-See {ref}/query-dsl-span-first-query.html[Span First Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[span_first]
---------------------------------------------------
-<1> query
-<2> max end position
diff --git a/docs/java-api/query-dsl/span-multi-term-query.asciidoc b/docs/java-api/query-dsl/span-multi-term-query.asciidoc
deleted file mode 100644
index eea00f61fe7e1..0000000000000
--- a/docs/java-api/query-dsl/span-multi-term-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-span-multi-term-query]]
-==== Span Multi Term Query
-
-See {ref}/query-dsl-span-multi-term-query.html[Span Multi Term Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[span_multi]
---------------------------------------------------
-<1> Can be any builder extending the `MultiTermQueryBuilder` class. For example: `FuzzyQueryBuilder`,
-`PrefixQueryBuilder`, `RangeQueryBuilder`, `RegexpQueryBuilder` or `WildcardQueryBuilder`.
diff --git a/docs/java-api/query-dsl/span-near-query.asciidoc b/docs/java-api/query-dsl/span-near-query.asciidoc
deleted file mode 100644
index 6f4661e34c9d1..0000000000000
--- a/docs/java-api/query-dsl/span-near-query.asciidoc
+++ /dev/null
@@ -1,12 +0,0 @@
-[[java-query-dsl-span-near-query]]
-==== Span Near Query
-
-See {ref}/query-dsl-span-near-query.html[Span Near Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[span_near]
---------------------------------------------------
-<1> span term queries
-<2> slop factor: the maximum number of intervening unmatched positions
-<3> whether matches are required to be in-order
diff --git a/docs/java-api/query-dsl/span-not-query.asciidoc b/docs/java-api/query-dsl/span-not-query.asciidoc
deleted file mode 100644
index 001c2ca025e6d..0000000000000
--- a/docs/java-api/query-dsl/span-not-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-span-not-query]]
-==== Span Not Query
-
-See {ref}/query-dsl-span-not-query.html[Span Not Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[span_not]
---------------------------------------------------
-<1> span query whose matches are filtered
-<2> span query whose matches must not overlap those returned
diff --git a/docs/java-api/query-dsl/span-or-query.asciidoc b/docs/java-api/query-dsl/span-or-query.asciidoc
deleted file mode 100644
index 787628b59342f..0000000000000
--- a/docs/java-api/query-dsl/span-or-query.asciidoc
+++ /dev/null
@@ -1,10 +0,0 @@
-[[java-query-dsl-span-or-query]]
-==== Span Or Query
-
-See {ref}/query-dsl-span-or-query.html[Span Or Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[span_or]
---------------------------------------------------
-<1> span term queries
diff --git a/docs/java-api/query-dsl/span-queries.asciidoc b/docs/java-api/query-dsl/span-queries.asciidoc
deleted file mode 100644
index 0ccbe30638c6a..0000000000000
--- a/docs/java-api/query-dsl/span-queries.asciidoc
+++ /dev/null
@@ -1,65 +0,0 @@
-[[java-span-queries]]
-=== Span queries
-
-Span queries are low-level positional queries which provide expert control
-over the order and proximity of the specified terms. These are typically used
-to implement very specific queries on legal documents or patents.
-
-Span queries cannot be mixed with non-span queries (with the exception of the `span_multi` query).
-
-The queries in this group are:
-
-<<java-query-dsl-span-term-query,`span_term` query>>::
-
-The equivalent of the <<java-query-dsl-term-query,`term` query>> but for use with
-other span queries.
-
-<<java-query-dsl-span-multi-term-query,`span_multi` query>>::
-
-Wraps a <<java-query-dsl-term-query,`term`>>, <<java-query-dsl-range-query,`range`>>,
-<<java-query-dsl-prefix-query,`prefix`>>, <<java-query-dsl-wildcard-query,`wildcard`>>,
-<<java-query-dsl-regexp-query,`regexp`>>, or <<java-query-dsl-fuzzy-query,`fuzzy`>> query.
-
-<<java-query-dsl-span-first-query,`span_first` query>>::
-
-Accepts another span query whose matches must appear within the first N
-positions of the field.
-
-<<java-query-dsl-span-near-query,`span_near` query>>::
-
-Accepts multiple span queries whose matches must be within the specified distance of each other, and possibly in the same order.
-
-<<java-query-dsl-span-or-query,`span_or` query>>::
-
-Combines multiple span queries -- returns documents which match any of the
-specified queries.
-
-<<java-query-dsl-span-not-query,`span_not` query>>::
-
-Wraps another span query, and excludes any documents which match that query.
-
-<<java-query-dsl-span-containing-query,`span_containing` query>>::
-
-Accepts a list of span queries, but only returns those spans which also match a second span query.
-
-<<java-query-dsl-span-within-query,`span_within` query>>::
-
-The result from a single span query is returned as long is its span falls
-within the spans returned by a list of other span queries.
-
-
-include::span-term-query.asciidoc[]
-
-include::span-multi-term-query.asciidoc[]
-
-include::span-first-query.asciidoc[]
-
-include::span-near-query.asciidoc[]
-
-include::span-or-query.asciidoc[]
-
-include::span-not-query.asciidoc[]
-
-include::span-containing-query.asciidoc[]
-
-include::span-within-query.asciidoc[]
diff --git a/docs/java-api/query-dsl/span-term-query.asciidoc b/docs/java-api/query-dsl/span-term-query.asciidoc
deleted file mode 100644
index 2bdf9276515dc..0000000000000
--- a/docs/java-api/query-dsl/span-term-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-span-term-query]]
-==== Span Term Query
-
-See {ref}/query-dsl-span-term-query.html[Span Term Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[span_term]
---------------------------------------------------
-<1> field
-<2> value
diff --git a/docs/java-api/query-dsl/span-within-query.asciidoc b/docs/java-api/query-dsl/span-within-query.asciidoc
deleted file mode 100644
index afa527c0b67fb..0000000000000
--- a/docs/java-api/query-dsl/span-within-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-span-within-query]]
-==== Span Within Query
-
-See {ref}/query-dsl-span-within-query.html[Span Within Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[span_within]
---------------------------------------------------
-<1> `big` part
-<2> `little` part
diff --git a/docs/java-api/query-dsl/special-queries.asciidoc b/docs/java-api/query-dsl/special-queries.asciidoc
deleted file mode 100644
index bca3bde3b3f62..0000000000000
--- a/docs/java-api/query-dsl/special-queries.asciidoc
+++ /dev/null
@@ -1,31 +0,0 @@
-[[java-specialized-queries]]
-
-=== Specialized queries
-
-This group contains queries which do not fit into the other groups:
-
-<<java-query-dsl-mlt-query,`more_like_this` query>>::
-
-This query finds documents which are similar to the specified text, document,
-or collection of documents.
-
-<<java-query-dsl-script-query,`script` query>>::
-
-This query allows a script to act as a filter.  Also see the
-<<java-query-dsl-function-score-query,`function_score` query>>.
-
-<<java-query-percolate-query,`percolate` query>>::
-
-This query finds percolator queries based on documents.
-
-<<java-query-dsl-wrapper-query,`wrapper` query>>::
-
-A query that accepts other queries as json or yaml string.
-
-include::mlt-query.asciidoc[]
-
-include::script-query.asciidoc[]
-
-include::percolate-query.asciidoc[]
-
-include::wrapper-query.asciidoc[]
diff --git a/docs/java-api/query-dsl/term-level-queries.asciidoc b/docs/java-api/query-dsl/term-level-queries.asciidoc
deleted file mode 100644
index 7d3649e372bbd..0000000000000
--- a/docs/java-api/query-dsl/term-level-queries.asciidoc
+++ /dev/null
@@ -1,77 +0,0 @@
-[[java-term-level-queries]]
-=== Term level queries
-
-While the <<java-full-text-queries,full text queries>> will analyze the query
-string before executing, the _term-level queries_ operate on the exact terms
-that are stored in the inverted index.
-
-These queries are usually used for structured data like numbers, dates, and
-enums, rather than full text fields.  Alternatively, they allow you to craft
-low-level queries, foregoing the analysis process.
-
-The queries in this group are:
-
-<<java-query-dsl-term-query,`term` query>>::
-
-    Find documents which contain the exact term specified in the field
-    specified.
-
-<<java-query-dsl-terms-query,`terms` query>>::
-
-    Find documents which contain any of the exact terms specified in the field
-    specified.
-
-<<java-query-dsl-range-query,`range` query>>::
-
-    Find documents where the field specified contains values (dates, numbers,
-    or strings) in the range specified.
-
-<<java-query-dsl-exists-query,`exists` query>>::
-
-    Find documents where the field specified contains any non-null value.
-
-<<java-query-dsl-prefix-query,`prefix` query>>::
-
-    Find documents where the field specified contains terms which being with
-    the exact prefix specified.
-
-<<java-query-dsl-wildcard-query,`wildcard` query>>::
-
-    Find documents where the field specified contains terms which match the
-    pattern specified, where the pattern supports single character wildcards
-    (`?`) and multi-character wildcards (`*`)
-
-<<java-query-dsl-regexp-query,`regexp` query>>::
-
-    Find documents where the field specified contains terms which match the
-    regular expression specified.
-
-<<java-query-dsl-fuzzy-query,`fuzzy` query>>::
-
-    Find documents where the field specified contains terms which are fuzzily
-    similar to the specified term.  Fuzziness is measured as a
-    http://en.wikipedia.org/wiki/Damerau%E2%80%93Levenshtein_distance[Levenshtein edit distance]
-    of 1 or 2.
-
-<<java-query-dsl-ids-query,`ids` query>>::
-
-    Find documents with the specified type and IDs.
-
-
-include::term-query.asciidoc[]
-
-include::terms-query.asciidoc[]
-
-include::range-query.asciidoc[]
-
-include::exists-query.asciidoc[]
-
-include::prefix-query.asciidoc[]
-
-include::wildcard-query.asciidoc[]
-
-include::regexp-query.asciidoc[]
-
-include::fuzzy-query.asciidoc[]
-
-include::ids-query.asciidoc[]
diff --git a/docs/java-api/query-dsl/term-query.asciidoc b/docs/java-api/query-dsl/term-query.asciidoc
deleted file mode 100644
index 7c8549dbed403..0000000000000
--- a/docs/java-api/query-dsl/term-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-term-query]]
-==== Term Query
-
-See {ref}/query-dsl-term-query.html[Term Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[term]
---------------------------------------------------
-<1> field
-<2> text
diff --git a/docs/java-api/query-dsl/terms-query.asciidoc b/docs/java-api/query-dsl/terms-query.asciidoc
deleted file mode 100644
index 587968ba18e77..0000000000000
--- a/docs/java-api/query-dsl/terms-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-terms-query]]
-==== Terms Query
-
-See {ref}/query-dsl-terms-query.html[Terms Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[terms]
---------------------------------------------------
-<1> field
-<2> values
diff --git a/docs/java-api/query-dsl/wildcard-query.asciidoc b/docs/java-api/query-dsl/wildcard-query.asciidoc
deleted file mode 100644
index f9ace822aac9d..0000000000000
--- a/docs/java-api/query-dsl/wildcard-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-wildcard-query]]
-==== Wildcard Query
-
-See {ref}/query-dsl-wildcard-query.html[Wildcard Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[wildcard]
---------------------------------------------------
-<1> field
-<2> wildcard expression
diff --git a/docs/java-api/query-dsl/wrapper-query.asciidoc b/docs/java-api/query-dsl/wrapper-query.asciidoc
deleted file mode 100644
index 3bdf3cc69d30a..0000000000000
--- a/docs/java-api/query-dsl/wrapper-query.asciidoc
+++ /dev/null
@@ -1,11 +0,0 @@
-[[java-query-dsl-wrapper-query]]
-==== Wrapper Query
-
-See {ref}/query-dsl-wrapper-query.html[Wrapper Query]
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{query-dsl-test}[wrapper]
---------------------------------------------------
-
-<1> query defined as query builder
diff --git a/docs/java-api/search.asciidoc b/docs/java-api/search.asciidoc
deleted file mode 100644
index ecf8415f4dcbe..0000000000000
--- a/docs/java-api/search.asciidoc
+++ /dev/null
@@ -1,250 +0,0 @@
-[[java-search]]
-== Search API
-
-The search API allows one to execute a search query and get back search hits
-that match the query. It can be executed across one or more indices and
-across one or more types. The query can be provided using the <<java-query-dsl,query Java API>>.
-The body of the search request is built using the `SearchSourceBuilder`. Here is an example:
-
-[source,java]
---------------------------------------------------
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.index.query.QueryBuilders.*;
---------------------------------------------------
-
-[source,java]
---------------------------------------------------
-SearchResponse response = client.prepareSearch("index1", "index2")
-        .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
-        .setQuery(QueryBuilders.termQuery("multi", "test"))                 // Query
-        .setPostFilter(QueryBuilders.rangeQuery("age").from(12).to(18))     // Filter
-        .setFrom(0).setSize(60).setExplain(true)
-        .get();
---------------------------------------------------
-
-Note that all parameters are optional. Here is the smallest search call
-you can write:
-
-[source,java]
---------------------------------------------------
-// MatchAll on the whole cluster with all default options
-SearchResponse response = client.prepareSearch().get();
---------------------------------------------------
-
-NOTE:   Although the Java API defines the additional search types QUERY_AND_FETCH and
-        DFS_QUERY_AND_FETCH, these modes are internal optimizations and should not
-        be specified explicitly by users of the API.
-
-For more information on the search operation, check out the REST
-{ref}/search.html[search] docs.
-
-
-[[java-search-scrolling]]
-=== Using scrolls in Java
-
-Read the {ref}/search-request-scroll.html[scroll documentation]
-first!
-
-[source,java]
---------------------------------------------------
-import static org.elasticsearch.index.query.QueryBuilders.*;
-
-QueryBuilder qb = termQuery("multi", "test");
-
-SearchResponse scrollResp = client.prepareSearch(test)
-        .addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC)
-        .setScroll(new TimeValue(60000))
-        .setQuery(qb)
-        .setSize(100).get(); //max of 100 hits will be returned for each scroll
-//Scroll until no hits are returned
-do {
-    for (SearchHit hit : scrollResp.getHits().getHits()) {
-        //Handle the hit...
-    }
-
-    scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet();
-} while(scrollResp.getHits().getHits().length != 0); // Zero hits mark the end of the scroll and the while loop.
---------------------------------------------------
-
-[[java-search-msearch]]
-=== MultiSearch API
-
-See {ref}/search-multi-search.html[MultiSearch API Query]
-documentation
-
-[source,java]
---------------------------------------------------
-SearchRequestBuilder srb1 = client
-    .prepareSearch().setQuery(QueryBuilders.queryStringQuery("elasticsearch")).setSize(1);
-SearchRequestBuilder srb2 = client
-    .prepareSearch().setQuery(QueryBuilders.matchQuery("name", "kimchy")).setSize(1);
-
-MultiSearchResponse sr = client.prepareMultiSearch()
-        .add(srb1)
-        .add(srb2)
-        .get();
-
-// You will get all individual responses from MultiSearchResponse#getResponses()
-long nbHits = 0;
-for (MultiSearchResponse.Item item : sr.getResponses()) {
-    SearchResponse response = item.getResponse();
-    nbHits += response.getHits().getTotalHits().value;
-}
---------------------------------------------------
-
-
-[[java-search-aggs]]
-=== Using Aggregations
-
-The following code shows how to add two aggregations within your search:
-
-[source,java]
---------------------------------------------------
-SearchResponse sr = client.prepareSearch()
-    .setQuery(QueryBuilders.matchAllQuery())
-    .addAggregation(
-            AggregationBuilders.terms("agg1").field("field")
-    )
-    .addAggregation(
-            AggregationBuilders.dateHistogram("agg2")
-                    .field("birth")
-                    .calendarInterval(DateHistogramInterval.YEAR)
-    )
-    .get();
-
-// Get your facet results
-Terms agg1 = sr.getAggregations().get("agg1");
-Histogram agg2 = sr.getAggregations().get("agg2");
---------------------------------------------------
-
-See <<java-aggs,Aggregations Java API>>
-documentation for details.
-
-
-[[java-search-terminate-after]]
-=== Terminate After
-
-The maximum number of documents to collect for each shard, upon reaching which the query execution will terminate early.
-If set, you will be able to check if the operation terminated early by asking for `isTerminatedEarly()` in the
-`SearchResponse` object:
-
-[source,java]
---------------------------------------------------
-SearchResponse sr = client.prepareSearch(INDEX)
-    .setTerminateAfter(1000)    <1>
-    .get();
-
-if (sr.isTerminatedEarly()) {
-    // We finished early
-}
---------------------------------------------------
-<1> Finish after 1000 docs
-
-[[java-search-template]]
-=== Search Template
-
-See {ref}/search-template.html[Search Template] documentation
-
-Define your template parameters as a `Map<String,Object>`:
-
-[source,java]
---------------------------------------------------
-Map<String, Object> template_params = new HashMap<>();
-template_params.put("param_gender", "male");
---------------------------------------------------
-
-You can use your stored search templates in `config/scripts`.
-For example, if you have a file named `config/scripts/template_gender.mustache` containing:
-
-[source,js]
---------------------------------------------------
-{
-    "query" : {
-        "match" : {
-            "gender" : "{{param_gender}}"
-        }
-    }
-}
---------------------------------------------------
-// NOTCONSOLE
-
-Create your search template request:
-
-[source,java]
---------------------------------------------------
-SearchResponse sr = new SearchTemplateRequestBuilder(client)
-    .setScript("template_gender")                 <1>
-    .setScriptType(ScriptService.ScriptType.FILE) <2>
-    .setScriptParams(template_params)             <3>
-    .setRequest(new SearchRequest())              <4>
-    .get()                                        <5>
-    .getResponse();                               <6>
---------------------------------------------------
-<1> template name
-<2> template stored on disk in `gender_template.mustache`
-<3> parameters
-<4> set the execution context (ie. define the index name here)
-<5> execute and get the template response
-<6> get from the template response the search response itself
-
-You can also store your template in the cluster state:
-
-[source,java]
---------------------------------------------------
-client.admin().cluster().preparePutStoredScript()
-    .setScriptLang("mustache")
-    .setId("template_gender")
-    .setSource(new BytesArray(
-        "{\n" +
-        "    \"query\" : {\n" +
-        "        \"match\" : {\n" +
-        "            \"gender\" : \"{{param_gender}}\"\n" +
-        "        }\n" +
-        "    }\n" +
-        "}")).get();
---------------------------------------------------
-
-To execute a stored templates, use `ScriptService.ScriptType.STORED`:
-
-[source,java]
---------------------------------------------------
-SearchResponse sr = new SearchTemplateRequestBuilder(client)
-        .setScript("template_gender")         <1>
-        .setScriptType(ScriptType.STORED)     <2>
-        .setScriptParams(template_params)     <3>
-        .setRequest(new SearchRequest())      <4>
-        .get()                                <5>
-        .getResponse();                       <6>
---------------------------------------------------
-<1> template name
-<2> template stored in the cluster state
-<3> parameters
-<4> set the execution context (ie. define the index name here)
-<5> execute and get the template response
-<6> get from the template response the search response itself
-
-You can also execute inline templates:
-
-[source,java]
---------------------------------------------------
-sr = new SearchTemplateRequestBuilder(client)
-        .setScript("{\n" +                   <1>
-                "        \"query\" : {\n" +
-                "            \"match\" : {\n" +
-                "                \"gender\" : \"{{param_gender}}\"\n" +
-                "            }\n" +
-                "        }\n" +
-                "}")
-        .setScriptType(ScriptType.INLINE)    <2>
-        .setScriptParams(template_params)    <3>
-        .setRequest(new SearchRequest())     <4>
-        .get()                               <5>
-        .getResponse();                      <6>
---------------------------------------------------
-<1> template's body
-<2> template is passed inline
-<3> parameters
-<4> set the execution context (ie. define the index name here)
-<5> execute and get the template response
-<6> get from the template response the search response itself
diff --git a/docs/java-rest/high-level/dataframe/get_data_frame.asciidoc b/docs/java-rest/high-level/dataframe/get_data_frame.asciidoc
index ec2253b2c25f4..995d9d2c08963 100644
--- a/docs/java-rest/high-level/dataframe/get_data_frame.asciidoc
+++ b/docs/java-rest/high-level/dataframe/get_data_frame.asciidoc
@@ -32,6 +32,7 @@ include-tagged::{doc-tests-file}[{api}-request-options]
 <1> The page parameters `from` and `size`. `from` specifies the number of
 {dataframe-transforms} to skip. `size` specifies the maximum number of
 {dataframe-transforms} to get. Defaults to `0` and `100` respectively.
+<2> Whether to ignore if a wildcard expression matches no transforms.
 
 
 include::../execution.asciidoc[]
diff --git a/docs/java-rest/high-level/dataframe/get_data_frame_stats.asciidoc b/docs/java-rest/high-level/dataframe/get_data_frame_stats.asciidoc
index cdc6254a4e443..8a3e1a96acb14 100644
--- a/docs/java-rest/high-level/dataframe/get_data_frame_stats.asciidoc
+++ b/docs/java-rest/high-level/dataframe/get_data_frame_stats.asciidoc
@@ -22,6 +22,19 @@ include-tagged::{doc-tests-file}[{api}-request]
 --------------------------------------------------
 <1> Constructing a new GET Stats request referencing an existing {dataframe-transform}
 
+==== Optional Arguments
+
+The following arguments are optional.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request-options]
+--------------------------------------------------
+<1> The page parameters `from` and `size`. `from` specifies the number of data frame transform stats to skip.
+`size` specifies the maximum number of data frame transform stats to get.
+Defaults to `0` and `100` respectively.
+<2> Whether to ignore if a wildcard expression matches no transforms.
+
 
 include::../execution.asciidoc[]
 
@@ -39,4 +52,5 @@ include-tagged::{doc-tests-file}[{api}-response]
 <3> The running state of the transform indexer e.g `started`, `indexing`, etc.
 <4> The overall transform statistics recording the number of documents indexed etc.
 <5> The progress of the current run in the transform. Supplies the number of docs left until the next checkpoint
-and the total number of docs expected.
\ No newline at end of file
+and the total number of docs expected.
+<6> The assigned node information if the task is currently assigned to a node and running.
\ No newline at end of file
diff --git a/docs/java-rest/high-level/dataframe/put_data_frame.asciidoc b/docs/java-rest/high-level/dataframe/put_data_frame.asciidoc
index 567449c9c25b1..19c7fe443dbcd 100644
--- a/docs/java-rest/high-level/dataframe/put_data_frame.asciidoc
+++ b/docs/java-rest/high-level/dataframe/put_data_frame.asciidoc
@@ -33,7 +33,7 @@ include-tagged::{doc-tests-file}[{api}-config]
 --------------------------------------------------
 <1> The {dataframe-transform} ID
 <2> The source indices and query from which to gather data
-<3> The destination index
+<3> The destination index and optional pipeline
 <4> The PivotConfig
 <5> Optional free text description of the transform
 
@@ -49,6 +49,16 @@ If query is not set, a `match_all` query is used by default.
 include-tagged::{doc-tests-file}[{api}-source-config]
 --------------------------------------------------
 
+==== DestConfig
+
+The index where to write the data and the optional pipeline
+through which the docs should be indexed
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-dest-config]
+--------------------------------------------------
+
 ===== QueryConfig
 
 The query with which to select data from the source.
diff --git a/docs/java-rest/high-level/dataframe/stop_data_frame.asciidoc b/docs/java-rest/high-level/dataframe/stop_data_frame.asciidoc
index 9b05687c00875..1de4af5c5d592 100644
--- a/docs/java-rest/high-level/dataframe/stop_data_frame.asciidoc
+++ b/docs/java-rest/high-level/dataframe/stop_data_frame.asciidoc
@@ -32,6 +32,7 @@ include-tagged::{doc-tests-file}[{api}-request-options]
 --------------------------------------------------
 <1> If true wait for the data frame task to stop before responding
 <2> Controls the amount of time to wait until the {dataframe-job} stops.
+<3> Whether to ignore if a wildcard expression matches no transforms.
 
 include::../execution.asciidoc[]
 
diff --git a/docs/java-rest/high-level/document/delete-by-query.asciidoc b/docs/java-rest/high-level/document/delete-by-query.asciidoc
index 37be8e2f3f4c8..f4ef87741e639 100644
--- a/docs/java-rest/high-level/document/delete-by-query.asciidoc
+++ b/docs/java-rest/high-level/document/delete-by-query.asciidoc
@@ -39,11 +39,11 @@ include-tagged::{doc-tests-file}[{api}-request-query]
 --------------------------------------------------
 <1> Only copy documents which have field `user` set to `kimchy`
 
-It’s also possible to limit the number of processed documents by setting size.
+It’s also possible to limit the number of processed documents by setting `maxDocs`.
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------
-include-tagged::{doc-tests-file}[{api}-request-size]
+include-tagged::{doc-tests-file}[{api}-request-maxDocs]
 --------------------------------------------------
 <1> Only copy 10 documents
 
diff --git a/docs/java-rest/high-level/document/reindex.asciidoc b/docs/java-rest/high-level/document/reindex.asciidoc
index d78851f3d6a86..ccfb0c9db1a56 100644
--- a/docs/java-rest/high-level/document/reindex.asciidoc
+++ b/docs/java-rest/high-level/document/reindex.asciidoc
@@ -65,11 +65,11 @@ include-tagged::{doc-tests-file}[{api}-request-query]
 --------------------------------------------------
 <1> Only copy documents which have field `user` set to `kimchy`
 
-It’s also possible to limit the number of processed documents by setting size.
+It’s also possible to limit the number of processed documents by setting `maxDocs`.
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------
-include-tagged::{doc-tests-file}[{api}-request-size]
+include-tagged::{doc-tests-file}[{api}-request-maxDocs]
 --------------------------------------------------
 <1> Only copy 10 documents
 
@@ -90,7 +90,7 @@ include-tagged::{doc-tests-file}[{api}-request-pipeline]
 <1> set pipeline to `my_pipeline`
 
 If you want a particular set of documents from the source index you’ll need to use sort. If possible, prefer a more
-selective query to size and sort.
+selective query to maxDocs and sort.
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------
diff --git a/docs/java-rest/high-level/document/term-vectors.asciidoc b/docs/java-rest/high-level/document/term-vectors.asciidoc
index e8d4a25a2cac0..36c7553d4885e 100644
--- a/docs/java-rest/high-level/document/term-vectors.asciidoc
+++ b/docs/java-rest/high-level/document/term-vectors.asciidoc
@@ -15,7 +15,7 @@ artificially provided by the user.
 [id="{upid}-{api}-request"]
 ==== Term Vectors Request
 
-A +{request}+ expects an `index`, a `type` and an `id` to specify
+A +{request}+ expects an `index` and an `id` to specify
 a certain document, and fields for which the information is retrieved.
 
 ["source","java",subs="attributes,callouts,macros"]
@@ -71,9 +71,8 @@ include::../execution.asciidoc[]
 include-tagged::{doc-tests-file}[{api}-response]
 --------------------------------------------------
 <1> The index name of the document.
-<2> The type name of the document.
-<3> The id of the document.
-<4> Indicates whether or not the document found.
+<2> The id of the document.
+<3> Indicates whether or not the document found.
 
 
 ===== Inspecting Term Vectors
diff --git a/docs/java-rest/high-level/document/update-by-query.asciidoc b/docs/java-rest/high-level/document/update-by-query.asciidoc
index be1692c4e9f6a..26a6bc362b191 100644
--- a/docs/java-rest/high-level/document/update-by-query.asciidoc
+++ b/docs/java-rest/high-level/document/update-by-query.asciidoc
@@ -40,11 +40,11 @@ include-tagged::{doc-tests-file}[{api}-request-query]
 --------------------------------------------------
 <1> Only copy documents which have field `user` set to `kimchy`
 
-It’s also possible to limit the number of processed documents by setting size.
+It’s also possible to limit the number of processed documents by setting `maxDocs`.
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------
-include-tagged::{doc-tests-file}[{api}-request-size]
+include-tagged::{doc-tests-file}[{api}-request-maxDocs]
 --------------------------------------------------
 <1> Only copy 10 documents
 
diff --git a/docs/java-rest/high-level/indices/analyze.asciidoc b/docs/java-rest/high-level/indices/analyze.asciidoc
index 4978c9ebcca64..9464394fd1eb9 100644
--- a/docs/java-rest/high-level/indices/analyze.asciidoc
+++ b/docs/java-rest/high-level/indices/analyze.asciidoc
@@ -19,18 +19,18 @@ The simplest version uses a built-in analyzer:
 ---------------------------------------------------
 include-tagged::{doc-tests-file}[{api}-builtin-request]
 ---------------------------------------------------
-<1> The text to include.  Multiple strings are treated as a multi-valued field
-<2> A built-in analyzer
+<1> A built-in analyzer
+<2> The text to include.  Multiple strings are treated as a multi-valued field
 
 You can configure a custom analyzer:
 ["source","java",subs="attributes,callouts,macros"]
 ---------------------------------------------------
 include-tagged::{doc-tests-file}[{api}-custom-request]
 ---------------------------------------------------
-<1> Configure char filters
+<1> Configuration for a custom tokenfilter
 <2> Configure the tokenizer
-<3> Add a built-in tokenfilter
-<4> Configuration for a custom tokenfilter
+<3> Configure char filters
+<4> Add a built-in tokenfilter
 <5> Add the custom tokenfilter
 
 You can also build a custom normalizer, by including only charfilters and
diff --git a/docs/java-rest/high-level/indices/reload_analyzers.asciidoc b/docs/java-rest/high-level/indices/reload_analyzers.asciidoc
new file mode 100644
index 0000000000000..29db206bf1402
--- /dev/null
+++ b/docs/java-rest/high-level/indices/reload_analyzers.asciidoc
@@ -0,0 +1,50 @@
+--
+:api: reload-analyzers
+:request: ReloadAnalyzersRequest
+:response: ReloadAnalyzersResponse
+--
+
+[id="{upid}-{api}"]
+=== Reload Search Analyzers API
+
+[id="{upid}-{api}-request"]
+==== Reload Search Analyzers Request
+
+An +{request}+ requires an `index` argument:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request]
+--------------------------------------------------
+<1> The index to reload
+
+==== Optional arguments
+The following arguments can optionally be provided:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request-indicesOptions]
+--------------------------------------------------
+<1> Setting `IndicesOptions` controls how unavailable indices are resolved and
+how wildcard expressions are expanded
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Reload Search Analyzers Response
+
+The returned +{response}+ allows to retrieve information about the
+executed operation as follows:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------
+<1> Shard statistics. Note that reloading does not happen on each shard of an
+index, but once on each node the index has shards on. The reported shard count
+can therefore differ from the number of index shards
+<2> Reloading details of all indices the request was executed on
+<3> Details can be retrieved by index name
+<4> The reloaded index name
+<5> The nodes the index was reloaded on
+<6> The analyzer names that were reloaded
diff --git a/docs/java-rest/high-level/ml/delete-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/delete-data-frame-analytics.asciidoc
new file mode 100644
index 0000000000000..2e5ade37107cf
--- /dev/null
+++ b/docs/java-rest/high-level/ml/delete-data-frame-analytics.asciidoc
@@ -0,0 +1,28 @@
+--
+:api: delete-data-frame-analytics
+:request: DeleteDataFrameAnalyticsRequest
+:response: AcknowledgedResponse
+--
+[id="{upid}-{api}"]
+=== Delete Data Frame Analytics API
+
+The Delete Data Frame Analytics API is used to delete an existing {dataframe-analytics-config}.
+The API accepts a +{request}+ object as a request and returns a +{response}+.
+
+[id="{upid}-{api}-request"]
+==== Delete Data Frame Analytics Request
+
+A +{request}+ object requires a {dataframe-analytics-config} id.
+
+["source","java",subs="attributes,callouts,macros"]
+---------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request]
+---------------------------------------------------
+<1> Constructing a new request referencing an existing {dataframe-analytics-config}
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Response
+
+The returned +{response}+ object acknowledges the {dataframe-analytics-config} deletion.
diff --git a/docs/java-rest/high-level/ml/evaluate-data-frame.asciidoc b/docs/java-rest/high-level/ml/evaluate-data-frame.asciidoc
new file mode 100644
index 0000000000000..660603d2e38e7
--- /dev/null
+++ b/docs/java-rest/high-level/ml/evaluate-data-frame.asciidoc
@@ -0,0 +1,45 @@
+--
+:api: evaluate-data-frame
+:request: EvaluateDataFrameRequest
+:response: EvaluateDataFrameResponse
+--
+[id="{upid}-{api}"]
+=== Evaluate Data Frame API
+
+The Evaluate Data Frame API is used to evaluate an ML algorithm that ran on a {dataframe}.
+The API accepts an +{request}+ object and returns an +{response}+.
+
+[id="{upid}-{api}-request"]
+==== Evaluate Data Frame Request
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request]
+--------------------------------------------------
+<1> Constructing a new evaluation request
+<2> Reference to an existing index
+<3> Kind of evaluation to perform
+<4> Name of the field in the index. Its value denotes the actual (i.e. ground truth) label for an example. Must be either true or false
+<5> Name of the field in the index. Its value denotes the probability (as per some ML algorithm) of the example being classified as positive
+<6> The remaining parameters are the metrics to be calculated based on the two fields described above.
+<7> https://en.wikipedia.org/wiki/Precision_and_recall[Precision] calculated at thresholds: 0.4, 0.5 and 0.6
+<8> https://en.wikipedia.org/wiki/Precision_and_recall[Recall] calculated at thresholds: 0.5 and 0.7
+<9> https://en.wikipedia.org/wiki/Confusion_matrix[Confusion matrix] calculated at threshold 0.5
+<10> https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve[AuC ROC] calculated and the curve points returned
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Response
+
+The returned +{response}+ contains the requested evaluation metrics.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------
+<1> Fetching all the calculated metrics results
+<2> Fetching precision metric by name
+<3> Fetching precision at a given (0.4) threshold
+<4> Fetching confusion matrix metric by name
+<5> Fetching confusion matrix at a given (0.5) threshold
\ No newline at end of file
diff --git a/docs/java-rest/high-level/ml/get-data-frame-analytics-stats.asciidoc b/docs/java-rest/high-level/ml/get-data-frame-analytics-stats.asciidoc
new file mode 100644
index 0000000000000..e1047e9b3e002
--- /dev/null
+++ b/docs/java-rest/high-level/ml/get-data-frame-analytics-stats.asciidoc
@@ -0,0 +1,34 @@
+--
+:api: get-data-frame-analytics-stats
+:request: GetDataFrameAnalyticsStatsRequest
+:response: GetDataFrameAnalyticsStatsResponse
+--
+[id="{upid}-{api}"]
+=== Get Data Frame Analytics Stats API
+
+The Get Data Frame Analytics Stats API is used to read the operational statistics of one or more {dataframe-analytics-config}s.
+The API accepts a +{request}+ object and returns a +{response}+.
+
+[id="{upid}-{api}-request"]
+==== Get Data Frame Analytics Stats Request
+
+A +{request}+ requires either a {dataframe-analytics-config} id, a comma separated list of ids or
+the special wildcard `_all` to get the statistics for all {dataframe-analytics-config}s
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request]
+--------------------------------------------------
+<1> Constructing a new GET Stats request referencing an existing {dataframe-analytics-config}
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Response
+
+The returned +{response}+ contains the requested {dataframe-analytics-config} statistics.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------
\ No newline at end of file
diff --git a/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc
new file mode 100644
index 0000000000000..c6d368efbcae9
--- /dev/null
+++ b/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc
@@ -0,0 +1,34 @@
+--
+:api: get-data-frame-analytics
+:request: GetDataFrameAnalyticsRequest
+:response: GetDataFrameAnalyticsResponse
+--
+[id="{upid}-{api}"]
+=== Get Data Frame Analytics API
+
+The Get Data Frame Analytics API is used to get one or more {dataframe-analytics-config}s.
+The API accepts a +{request}+ object and returns a +{response}+.
+
+[id="{upid}-{api}-request"]
+==== Get Data Frame Analytics Request
+
+A +{request}+ requires either a {dataframe-analytics-config} id, a comma separated list of ids or
+the special wildcard `_all` to get all {dataframe-analytics-config}s.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request]
+--------------------------------------------------
+<1> Constructing a new GET request referencing an existing {dataframe-analytics-config}
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Response
+
+The returned +{response}+ contains the requested {dataframe-analytics-config}s.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------
diff --git a/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc
new file mode 100644
index 0000000000000..05fbd5bc3922a
--- /dev/null
+++ b/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc
@@ -0,0 +1,115 @@
+--
+:api: put-data-frame-analytics
+:request: PutDataFrameAnalyticsRequest
+:response: PutDataFrameAnalyticsResponse
+--
+[id="{upid}-{api}"]
+=== Put Data Frame Analytics API
+
+The Put Data Frame Analytics API is used to create a new {dataframe-analytics-config}.
+The API accepts a +{request}+ object as a request and returns a +{response}+.
+
+[id="{upid}-{api}-request"]
+==== Put Data Frame Analytics Request
+
+A +{request}+ requires the following argument:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request]
+--------------------------------------------------
+<1> The configuration of the {dataframe-job} to create
+
+[id="{upid}-{api}-config"]
+==== Data Frame Analytics Configuration
+
+The `DataFrameAnalyticsConfig` object contains all the details about the {dataframe-job}
+configuration and contains the following arguments:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-config]
+--------------------------------------------------
+<1> The {dataframe-analytics-config} id
+<2> The source index and query from which to gather data
+<3> The destination index
+<4> The analysis to be performed
+<5> The fields to be included in / excluded from the analysis
+<6> The memory limit for the model created as part of the analysis process
+
+[id="{upid}-{api}-query-config"]
+
+==== SourceConfig
+
+The index and the query from which to collect data.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-source-config]
+--------------------------------------------------
+<1> Constructing a new DataFrameAnalyticsSource
+<2> The source index
+<3> The query from which to gather the data. If query is not set, a `match_all` query is used by default.
+
+===== QueryConfig
+
+The query with which to select data from the source.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-query-config]
+--------------------------------------------------
+
+==== DestinationConfig
+
+The index to which data should be written by the {dataframe-job}.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-dest-config]
+--------------------------------------------------
+<1> Constructing a new DataFrameAnalyticsDest
+<2> The destination index
+
+==== Analysis
+
+The analysis to be performed.
+Currently, only one analysis is supported: +OutlierDetection+.
+
++OutlierDetection+ analysis can be created in one of two ways:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-analysis-default]
+--------------------------------------------------
+<1> Constructing a new OutlierDetection object with default strategy to determine outliers
+
+or
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-analysis-customized]
+--------------------------------------------------
+<1> Constructing a new OutlierDetection object
+<2> The method used to perform the analysis
+<3> Number of neighbors taken into account during analysis
+
+==== Analyzed fields
+
+FetchContext object containing fields to be included in / excluded from the analysis
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-analyzed-fields]
+--------------------------------------------------
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Response
+
+The returned +{response}+ contains the newly created {dataframe-analytics-config}.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------
\ No newline at end of file
diff --git a/docs/java-rest/high-level/ml/start-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/start-data-frame-analytics.asciidoc
new file mode 100644
index 0000000000000..610607daba1f8
--- /dev/null
+++ b/docs/java-rest/high-level/ml/start-data-frame-analytics.asciidoc
@@ -0,0 +1,28 @@
+--
+:api: start-data-frame-analytics
+:request: StartDataFrameAnalyticsRequest
+:response: AcknowledgedResponse
+--
+[id="{upid}-{api}"]
+=== Start Data Frame Analytics API
+
+The Start Data Frame Analytics API is used to start an existing {dataframe-analytics-config}.
+It accepts a +{request}+ object and responds with a +{response}+ object.
+
+[id="{upid}-{api}-request"]
+==== Start Data Frame Analytics Request
+
+A +{request}+ object requires a {dataframe-analytics-config} id.
+
+["source","java",subs="attributes,callouts,macros"]
+---------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request]
+---------------------------------------------------
+<1> Constructing a new start request referencing an existing {dataframe-analytics-config}
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Response
+
+The returned +{response}+ object acknowledges the {dataframe-job} has started.
\ No newline at end of file
diff --git a/docs/java-rest/high-level/ml/stop-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/stop-data-frame-analytics.asciidoc
new file mode 100644
index 0000000000000..3a06f268836f6
--- /dev/null
+++ b/docs/java-rest/high-level/ml/stop-data-frame-analytics.asciidoc
@@ -0,0 +1,29 @@
+--
+:api: stop-data-frame-analytics
+:request: StopDataFrameAnalyticsRequest
+:response: StopDataFrameAnalyticsResponse
+--
+[id="{upid}-{api}"]
+=== Stop Data Frame Analytics API
+
+The Stop Data Frame Analytics API is used to stop a running {dataframe-analytics-config}.
+It accepts a +{request}+ object and responds with a +{response}+ object.
+
+[id="{upid}-{api}-request"]
+==== Stop Data Frame Analytics Request
+
+A +{request}+ object requires a {dataframe-analytics-config} id.
+
+["source","java",subs="attributes,callouts,macros"]
+---------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request]
+---------------------------------------------------
+<1> Constructing a new stop request referencing an existing {dataframe-analytics-config}
+<2> Optionally used to stop a failed task
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Response
+
+The returned +{response}+ object acknowledges the {dataframe-job} has stopped.
\ No newline at end of file
diff --git a/docs/java-rest/high-level/query-builders.asciidoc b/docs/java-rest/high-level/query-builders.asciidoc
index 53d9b9af97d12..f07e81c2c413c 100644
--- a/docs/java-rest/high-level/query-builders.asciidoc
+++ b/docs/java-rest/high-level/query-builders.asciidoc
@@ -23,12 +23,11 @@ This page lists all the available search queries with their corresponding `Query
 | {ref}/query-dsl-match-query-phrase.html[Match Phrase]               | {query-ref}/MatchPhraseQueryBuilder.html[MatchPhraseQueryBuilder]                   | {query-ref}/QueryBuilders.html#matchPhraseQuery-java.lang.String-java.lang.Object-[QueryBuilders.matchPhraseQuery()]
 | {ref}/query-dsl-match-query-phrase-prefix.html[Match Phrase Prefix] | {query-ref}/MatchPhrasePrefixQueryBuilder.html[MatchPhrasePrefixQueryBuilder]       | {query-ref}/QueryBuilders.html#matchPhrasePrefixQuery-java.lang.String-java.lang.Object-[QueryBuilders.matchPhrasePrefixQuery()]
 | {ref}/query-dsl-multi-match-query.html[Multi Match]                 | {query-ref}/MultiMatchQueryBuilder.html[MultiMatchQueryBuilder]                     | {query-ref}/QueryBuilders.html#multiMatchQuery-java.lang.Object-java.lang.String\…-[QueryBuilders.multiMatchQuery()]
-| {ref}/query-dsl-common-terms-query.html[Common Terms]               | {query-ref}/CommonTermsQueryBuilder.html[CommonTermsQueryBuilder]                   | {query-ref}/QueryBuilders.html#commonTermsQuery-java.lang.String-java.lang.Object-[QueryBuilders.commonTermsQuery()]
 | {ref}/query-dsl-query-string-query.html[Query String]               | {query-ref}/QueryStringQueryBuilder.html[QueryStringQueryBuilder]                   | {query-ref}/QueryBuilders.html#queryStringQuery-java.lang.String-[QueryBuilders.queryStringQuery()]
 | {ref}/query-dsl-simple-query-string-query.html[Simple Query String] | {query-ref}/SimpleQueryStringBuilder.html[SimpleQueryStringBuilder]                 | {query-ref}/QueryBuilders.html#simpleQueryStringQuery-java.lang.String-[QueryBuilders.simpleQueryStringQuery()]
 |======
 
-==== Term level queries
+==== Term-level queries
 [options="header"]
 |======
 | Search Query                                                        | QueryBuilder Class | Method in QueryBuilders
diff --git a/docs/java-rest/high-level/security/get-builtin-privileges.asciidoc b/docs/java-rest/high-level/security/get-builtin-privileges.asciidoc
new file mode 100644
index 0000000000000..e8eeb7b3c9417
--- /dev/null
+++ b/docs/java-rest/high-level/security/get-builtin-privileges.asciidoc
@@ -0,0 +1,27 @@
+--
+:api: get-builtin-privileges
+:request: GetBuiltinPrivilegesRequest
+:response: GetBuiltinPrivilegesResponse
+--
+
+[id="{upid}-{api}"]
+=== Get Builtin Privileges API
+
+include::../execution-no-req.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Get Builtin Privileges Response
+
+The returned +{response}+ contains the following properties
+
+`clusterPrivileges`::
+A `Set` of all _cluster_ privileges that are understood by this node.
+
+`indexPrivileges`::
+A `Set` of all _index_ privileges that are understood by this node.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------
+
diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc
index 4e28efc2941db..27f5f38136d2e 100644
--- a/docs/java-rest/high-level/supported-apis.asciidoc
+++ b/docs/java-rest/high-level/supported-apis.asciidoc
@@ -155,6 +155,7 @@ include::indices/get_index.asciidoc[]
 include::indices/freeze_index.asciidoc[]
 include::indices/unfreeze_index.asciidoc[]
 include::indices/delete_template.asciidoc[]
+include::indices/reload_analyzers.asciidoc[]
 
 == Cluster APIs
 
@@ -285,6 +286,13 @@ The Java High Level REST Client supports the following Machine Learning APIs:
 * <<{upid}-put-calendar-job>>
 * <<{upid}-delete-calendar-job>>
 * <<{upid}-delete-calendar>>
+* <<{upid}-get-data-frame-analytics>>
+* <<{upid}-get-data-frame-analytics-stats>>
+* <<{upid}-put-data-frame-analytics>>
+* <<{upid}-delete-data-frame-analytics>>
+* <<{upid}-start-data-frame-analytics>>
+* <<{upid}-stop-data-frame-analytics>>
+* <<{upid}-evaluate-data-frame>>
 * <<{upid}-put-filter>>
 * <<{upid}-get-filters>>
 * <<{upid}-update-filter>>
@@ -329,6 +337,13 @@ include::ml/delete-calendar-event.asciidoc[]
 include::ml/put-calendar-job.asciidoc[]
 include::ml/delete-calendar-job.asciidoc[]
 include::ml/delete-calendar.asciidoc[]
+include::ml/get-data-frame-analytics.asciidoc[]
+include::ml/get-data-frame-analytics-stats.asciidoc[]
+include::ml/put-data-frame-analytics.asciidoc[]
+include::ml/delete-data-frame-analytics.asciidoc[]
+include::ml/start-data-frame-analytics.asciidoc[]
+include::ml/stop-data-frame-analytics.asciidoc[]
+include::ml/evaluate-data-frame.asciidoc[]
 include::ml/put-filter.asciidoc[]
 include::ml/get-filters.asciidoc[]
 include::ml/update-filter.asciidoc[]
@@ -404,6 +419,7 @@ The Java High Level REST Client supports the following Security APIs:
 * <<java-rest-high-security-delete-role-mapping>>
 * <<java-rest-high-security-create-token>>
 * <<{upid}-invalidate-token>>
+* <<{upid}-get-builtin-privileges>>
 * <<{upid}-get-privileges>>
 * <<{upid}-put-privileges>>
 * <<{upid}-delete-privileges>>
@@ -421,6 +437,7 @@ include::security/put-role.asciidoc[]
 include::security/get-roles.asciidoc[]
 include::security/delete-role.asciidoc[]
 include::security/delete-privileges.asciidoc[]
+include::security/get-builtin-privileges.asciidoc[]
 include::security/get-privileges.asciidoc[]
 include::security/clear-roles-cache.asciidoc[]
 include::security/clear-realm-cache.asciidoc[]
@@ -568,4 +585,4 @@ include::dataframe/put_data_frame.asciidoc[]
 include::dataframe/delete_data_frame.asciidoc[]
 include::dataframe/preview_data_frame.asciidoc[]
 include::dataframe/start_data_frame.asciidoc[]
-include::dataframe/stop_data_frame.asciidoc[]
\ No newline at end of file
+include::dataframe/stop_data_frame.asciidoc[]
diff --git a/docs/painless/painless-api-reference/painless-api-reference-score/packages.asciidoc b/docs/painless/painless-api-reference/painless-api-reference-score/packages.asciidoc
index 10f0f1b6daeab..a1beaeb5bc520 100644
--- a/docs/painless/painless-api-reference/painless-api-reference-score/packages.asciidoc
+++ b/docs/painless/painless-api-reference/painless-api-reference-score/packages.asciidoc
@@ -30,6 +30,8 @@ See the <<painless-api-reference-score, Score API>> for a high-level overview of
 * List findResults(Function)
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * def {java11-javadoc}/java.base/java/util/List.html#get(int)[get](int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * int getLength()
 * Map groupBy(Function)
 * int {java11-javadoc}/java.base/java/util/List.html#hashCode()[hashCode]()
@@ -84,6 +86,8 @@ See the <<painless-api-reference-score, Score API>> for a high-level overview of
 * List findResults(Function)
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * def {java11-javadoc}/java.base/java/util/List.html#get(int)[get](int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * int getLength()
 * Map groupBy(Function)
 * int {java11-javadoc}/java.base/java/util/List.html#hashCode()[hashCode]()
@@ -138,6 +142,8 @@ See the <<painless-api-reference-score, Score API>> for a high-level overview of
 * List findResults(Function)
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * def {java11-javadoc}/java.base/java/util/List.html#get(int)[get](int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * int getLength()
 * Map groupBy(Function)
 * int {java11-javadoc}/java.base/java/util/List.html#hashCode()[hashCode]()
diff --git a/docs/painless/painless-api-reference/painless-api-reference-shared/packages.asciidoc b/docs/painless/painless-api-reference/painless-api-reference-shared/packages.asciidoc
index 75ad21ddc93f2..dd8141c2e96d0 100644
--- a/docs/painless/painless-api-reference/painless-api-reference-shared/packages.asciidoc
+++ b/docs/painless/painless-api-reference/painless-api-reference-shared/packages.asciidoc
@@ -4335,6 +4335,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(Function)
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * def {java11-javadoc}/java.base/java/util/List.html#get(int)[get](int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * int getLength()
 * Map groupBy(Function)
 * int {java11-javadoc}/java.base/java/util/List.html#hashCode()[hashCode]()
@@ -4386,6 +4388,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(BiFunction)
 * void {java11-javadoc}/java.base/java/util/Map.html#forEach(java.util.function.BiConsumer)[forEach](BiConsumer)
 * def {java11-javadoc}/java.base/java/util/Map.html#get(java.lang.Object)[get](def)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * def {java11-javadoc}/java.base/java/util/Map.html#getOrDefault(java.lang.Object,java.lang.Object)[getOrDefault](def, def)
 * Map groupBy(BiFunction)
 * int {java11-javadoc}/java.base/java/lang/Object.html#hashCode()[hashCode]()
@@ -4500,6 +4504,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(Function)
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * def {java11-javadoc}/java.base/java/util/List.html#get(int)[get](int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * int getLength()
 * Map groupBy(Function)
 * int {java11-javadoc}/java.base/java/util/List.html#hashCode()[hashCode]()
@@ -4666,6 +4672,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(Function)
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * def {java11-javadoc}/java.base/java/util/List.html#get(int)[get](int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * int getLength()
 * Map groupBy(Function)
 * int {java11-javadoc}/java.base/java/util/List.html#hashCode()[hashCode]()
@@ -5367,6 +5375,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(BiFunction)
 * void {java11-javadoc}/java.base/java/util/Map.html#forEach(java.util.function.BiConsumer)[forEach](BiConsumer)
 * def {java11-javadoc}/java.base/java/util/Map.html#get(java.lang.Object)[get](def)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * def {java11-javadoc}/java.base/java/util/Map.html#getOrDefault(java.lang.Object,java.lang.Object)[getOrDefault](def, def)
 * Map groupBy(BiFunction)
 * int {java11-javadoc}/java.base/java/lang/Object.html#hashCode()[hashCode]()
@@ -5457,6 +5467,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(BiFunction)
 * void {java11-javadoc}/java.base/java/util/Map.html#forEach(java.util.function.BiConsumer)[forEach](BiConsumer)
 * def {java11-javadoc}/java.base/java/util/Map.html#get(java.lang.Object)[get](def)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * def {java11-javadoc}/java.base/java/util/Map.html#getOrDefault(java.lang.Object,java.lang.Object)[getOrDefault](def, def)
 * Map groupBy(BiFunction)
 * int {java11-javadoc}/java.base/java/lang/Object.html#hashCode()[hashCode]()
@@ -5502,6 +5514,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(BiFunction)
 * void {java11-javadoc}/java.base/java/util/Map.html#forEach(java.util.function.BiConsumer)[forEach](BiConsumer)
 * def {java11-javadoc}/java.base/java/util/Map.html#get(java.lang.Object)[get](def)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * def {java11-javadoc}/java.base/java/util/Map.html#getOrDefault(java.lang.Object,java.lang.Object)[getOrDefault](def, def)
 * Map groupBy(BiFunction)
 * int {java11-javadoc}/java.base/java/lang/Object.html#hashCode()[hashCode]()
@@ -5668,6 +5682,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(BiFunction)
 * void {java11-javadoc}/java.base/java/util/Map.html#forEach(java.util.function.BiConsumer)[forEach](BiConsumer)
 * def {java11-javadoc}/java.base/java/util/Map.html#get(java.lang.Object)[get](def)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * def {java11-javadoc}/java.base/java/util/Map.html#getOrDefault(java.lang.Object,java.lang.Object)[getOrDefault](def, def)
 * Map groupBy(BiFunction)
 * int {java11-javadoc}/java.base/java/lang/Object.html#hashCode()[hashCode]()
@@ -5764,6 +5780,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(Function)
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * def {java11-javadoc}/java.base/java/util/List.html#get(int)[get](int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * def {java11-javadoc}/java.base/java/util/Deque.html#getFirst()[getFirst]()
 * def {java11-javadoc}/java.base/java/util/Deque.html#getLast()[getLast]()
 * int getLength()
@@ -5836,6 +5854,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(Function)
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * def {java11-javadoc}/java.base/java/util/List.html#get(int)[get](int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * int getLength()
 * Map groupBy(Function)
 * int {java11-javadoc}/java.base/java/util/List.html#hashCode()[hashCode]()
@@ -6056,6 +6076,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(BiFunction)
 * void {java11-javadoc}/java.base/java/util/Map.html#forEach(java.util.function.BiConsumer)[forEach](BiConsumer)
 * def {java11-javadoc}/java.base/java/util/Map.html#get(java.lang.Object)[get](def)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * def {java11-javadoc}/java.base/java/util/Map.html#getOrDefault(java.lang.Object,java.lang.Object)[getOrDefault](def, def)
 * Map groupBy(BiFunction)
 * int {java11-javadoc}/java.base/java/lang/Object.html#hashCode()[hashCode]()
@@ -6157,6 +6179,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * def {java11-javadoc}/java.base/java/util/NavigableMap.html#floorKey(java.lang.Object)[floorKey](def)
 * void {java11-javadoc}/java.base/java/util/Map.html#forEach(java.util.function.BiConsumer)[forEach](BiConsumer)
 * def {java11-javadoc}/java.base/java/util/Map.html#get(java.lang.Object)[get](def)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * def {java11-javadoc}/java.base/java/util/Map.html#getOrDefault(java.lang.Object,java.lang.Object)[getOrDefault](def, def)
 * Map groupBy(BiFunction)
 * int {java11-javadoc}/java.base/java/lang/Object.html#hashCode()[hashCode]()
@@ -6642,6 +6666,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * def {java11-javadoc}/java.base/java/util/SortedMap.html#firstKey()[firstKey]()
 * void {java11-javadoc}/java.base/java/util/Map.html#forEach(java.util.function.BiConsumer)[forEach](BiConsumer)
 * def {java11-javadoc}/java.base/java/util/Map.html#get(java.lang.Object)[get](def)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * def {java11-javadoc}/java.base/java/util/Map.html#getOrDefault(java.lang.Object,java.lang.Object)[getOrDefault](def, def)
 * Map groupBy(BiFunction)
 * int {java11-javadoc}/java.base/java/lang/Object.html#hashCode()[hashCode]()
@@ -6844,6 +6870,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * def {java11-javadoc}/java.base/java/util/Vector.html#firstElement()[firstElement]()
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * def {java11-javadoc}/java.base/java/util/List.html#get(int)[get](int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * int getLength()
 * Map groupBy(Function)
 * int {java11-javadoc}/java.base/java/util/List.html#hashCode()[hashCode]()
@@ -6988,6 +7016,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * def {java11-javadoc}/java.base/java/util/NavigableMap.html#floorKey(java.lang.Object)[floorKey](def)
 * void {java11-javadoc}/java.base/java/util/Map.html#forEach(java.util.function.BiConsumer)[forEach](BiConsumer)
 * def {java11-javadoc}/java.base/java/util/Map.html#get(java.lang.Object)[get](def)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * def {java11-javadoc}/java.base/java/util/Map.html#getOrDefault(java.lang.Object,java.lang.Object)[getOrDefault](def, def)
 * Map groupBy(BiFunction)
 * int {java11-javadoc}/java.base/java/lang/Object.html#hashCode()[hashCode]()
@@ -7158,6 +7188,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * def {java11-javadoc}/java.base/java/util/Vector.html#firstElement()[firstElement]()
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * def {java11-javadoc}/java.base/java/util/List.html#get(int)[get](int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * int getLength()
 * Map groupBy(Function)
 * int {java11-javadoc}/java.base/java/util/List.html#hashCode()[hashCode]()
@@ -8016,6 +8048,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(Function)
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * Boolean get(int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * int getLength()
 * boolean getValue()
 * Map groupBy(Function)
@@ -8071,6 +8105,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(Function)
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * BytesRef get(int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * int getLength()
 * BytesRef getValue()
 * Map groupBy(Function)
@@ -8126,6 +8162,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(Function)
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * JodaCompatibleZonedDateTime get(int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * int getLength()
 * JodaCompatibleZonedDateTime getValue()
 * Map groupBy(Function)
@@ -8181,6 +8219,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(Function)
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * Double get(int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * int getLength()
 * double getValue()
 * Map groupBy(Function)
@@ -8240,6 +8280,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * double geohashDistance(String)
 * double geohashDistanceWithDefault(String, double)
 * GeoPoint get(int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * double getLat()
 * double[] getLats()
 * int getLength()
@@ -8301,6 +8343,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(Function)
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * Long get(int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * int getLength()
 * long getValue()
 * Map groupBy(Function)
@@ -8356,6 +8400,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(Function)
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * String get(int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * int getLength()
 * String getValue()
 * Map groupBy(Function)
@@ -8415,6 +8461,8 @@ See the <<painless-api-reference-shared, Shared API>> for a high-level overview
 * List findResults(Function)
 * void {java11-javadoc}/java.base/java/lang/Iterable.html#forEach(java.util.function.Consumer)[forEach](Consumer)
 * String get(int)
+* Object getByPath(String)
+* Object getByPath(String, Object)
 * int getLength()
 * String getValue()
 * Map groupBy(Function)
diff --git a/docs/painless/painless-guide/index.asciidoc b/docs/painless/painless-guide/index.asciidoc
index b45406a4e7273..f7e5de693df79 100644
--- a/docs/painless/painless-guide/index.asciidoc
+++ b/docs/painless/painless-guide/index.asciidoc
@@ -1,7 +1,11 @@
 include::painless-walkthrough.asciidoc[]
 
+include::painless-datetime.asciidoc[]
+
 include::painless-method-dispatch.asciidoc[]
 
 include::painless-debugging.asciidoc[]
 
 include::painless-execute-script.asciidoc[]
+
+include::../redirects.asciidoc[]
diff --git a/docs/painless/painless-guide/painless-datetime.asciidoc b/docs/painless/painless-guide/painless-datetime.asciidoc
new file mode 100644
index 0000000000000..45eb34b75726d
--- /dev/null
+++ b/docs/painless/painless-guide/painless-datetime.asciidoc
@@ -0,0 +1,720 @@
+[[painless-datetime]]
+=== Using Datetime in Painless
+
+==== Datetime API
+
+Datetimes in Painless use the standard Java libraries and are available through
+the Painless <<painless-api-reference-shared, Shared API>>. Most of the classes
+from the following Java packages are available to use in Painless scripts:
+
+* <<painless-api-reference-shared-java-time, java.time>>
+* <<painless-api-reference-shared-java-time-chrono, java.time.chrono>>
+* <<painless-api-reference-shared-java-time-format, java.time.format>>
+* <<painless-api-reference-shared-java-time-temporal, java.time.temporal>>
+* <<painless-api-reference-shared-java-time-zone, java.time.zone>>
+
+==== Datetime Representation
+
+Datetimes in Painless are most commonly represented as a numeric value, a
+string value, or a complex value.
+
+numeric:: a datetime representation as a number from a starting offset called
+an epoch; in Painless this is typically a <<primitive-types, long>> as
+milliseconds since an epoch of 1970-01-01 00:00:00 Zulu Time
+string:: a datetime representation as a sequence of characters defined by
+a standard format or a custom format; in Painless this is typically a
+<<string-type, String>> of the standard format
+https://en.wikipedia.org/wiki/ISO_8601[ISO 8601]
+complex:: a datetime representation as a complex type
+(<<reference-types, object>>) that abstracts away internal details of how the
+datetime is stored and often provides utilities for modification and
+comparison; in Painless this is typically a
+<<painless-api-reference-shared-ZonedDateTime, ZonedDateTime>>
+
+Switching between different representations of datetimes is often necessary to
+achieve a script's objective(s). A typical pattern in a script is to switch a
+numeric or string datetime to a complex datetime, modify or compare the complex
+datetime, and then switch it back to a numeric or string datetime for storage
+or to return a result.
+
+==== Datetime Parsing and Formatting
+
+Datetime parsing is a switch from a string datetime to a complex datetime, and
+datetime formatting is a switch from a complex datetime to a string datetime.
+
+A <<painless-api-reference-shared-DateTimeFormatter, DateTimeFormatter>> is a
+complex type (<<reference-types, object>>) that defines the allowed sequence
+of characters for a string datetime. Datetime parsing and formatting often
+requires a DateTimeFormatter. For more information about how to use a
+DateTimeFormatter see the
+{java11-javadoc}/java.base/java/time/format/DateTimeFormatter.html[Java documentation].
+
+===== Datetime Parsing Examples
+
+* parse from milliseconds
++
+[source,Painless]
+----
+String milliSinceEpochString = "434931330000";
+long milliSinceEpoch = Long.parseLong(milliSinceEpochString);
+Instant instant = Instant.ofEpochMilli(milliSinceEpoch);
+ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.of('Z'));
+----
++
+* parse from ISO 8601
++
+[source,Painless]
+----
+String datetime = '1983-10-13T22:15:30Z';
+ZonedDateTime zdt = ZonedDateTime.parse(datetime); <1>
+----
+<1> Note the parse method uses ISO 8601 by default.
++
+* parse from RFC 1123
++
+[source,Painless]
+----
+String datetime = 'Thu, 13 Oct 1983 22:15:30 GMT';
+ZonedDateTime zdt = ZonedDateTime.parse(datetime,
+        DateTimeFormatter.RFC_1123_DATE_TIME); <1>
+----
+<1> Note the use of a built-in DateTimeFormatter.
++
+* parse from a custom format
++
+[source,Painless]
+----
+String datetime = 'custom y 1983 m 10 d 13 22:15:30 Z';
+DateTimeFormatter dtf = DateTimeFormatter.ofPattern(
+        "'custom' 'y' yyyy 'm' MM 'd' dd HH:mm:ss VV");
+ZonedDateTime zdt = ZonedDateTime.parse(datetime, dtf); <1>
+----
+<1> Note the use of a custom DateTimeFormatter.
+
+===== Datetime Formatting Examples
+
+* format to ISO 8601
++
+[source,Painless]
+----
+ZonedDateTime zdt =
+        ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));
+String datetime = zdt.format(DateTimeFormatter.ISO_INSTANT); <1>
+----
+<1> Note the use of a built-in DateTimeFormatter.
++
+* format to a custom format
++
+[source,Painless]
+----
+ZonedDateTime zdt =
+        ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));
+DateTimeFormatter dtf = DateTimeFormatter.ofPattern(
+        "'date:' yyyy/MM/dd 'time:' HH:mm:ss");
+String datetime = zdt.format(dtf); <1>
+----
+<1> Note the use of a custom DateTimeFormatter.
+
+==== Datetime Conversion
+
+Datetime conversion is a switch from a numeric datetime to a complex datetime
+and vice versa.
+
+===== Datetime Conversion Examples
+
+* convert from milliseconds
++
+[source,Painless]
+----
+long milliSinceEpoch = 434931330000L;
+Instant instant = Instant.ofEpochMilli(milliSinceEpoch);
+ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.of('Z'));
+----
++
+* convert to milliseconds
++
+[source,Painless]
+-----
+ZonedDateTime zdt =
+        ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));
+long milliSinceEpoch = zdt.toInstant().toEpochMilli();
+-----
+
+==== Datetime Pieces
+
+Datetime representations often contain the data to extract individual datetime
+pieces such as year, hour, timezone, etc. Use individual pieces of a datetime
+to create a complex datetime, and use a complex datetime to extract individual
+pieces.
+
+===== Datetime Pieces Examples
+
+* create a complex datetime from pieces
++
+[source,Painless]
+----
+int year = 1983;
+int month = 10;
+int day = 13;
+int hour = 22;
+int minutes = 15;
+int seconds = 30;
+int nanos = 0;
+ZonedDateTime zdt = ZonedDateTime.of(
+        year, month, day, hour, minutes, seconds, nanos, ZoneId.of('Z'));
+----
++
+* extract pieces from a complex datetime
++
+[source,Painless]
+----
+ZonedDateTime zdt =
+        ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 100, ZoneId.of(tz));
+int year = zdt.getYear();
+int month = zdt.getMonthValue();
+int day = zdt.getDayOfMonth();
+int hour = zdt.getHour();
+int minutes = zdt.getMinute();
+int seconds = zdt.getSecond();
+int nanos = zdt.getNano();
+----
+
+==== Datetime Modification
+
+Use either a numeric datetime or a complex datetime to do modification such as
+adding several seconds to a datetime or subtracting several days from a
+datetime. Use standard <<painless-operators-numeric, numeric operators>> to
+modify a numeric datetime. Use
+<<painless-api-reference-shared-ZonedDateTime, methods>> (or fields) to modify
+a complex datetime. Note many complex datetimes are immutable so upon
+modification a new complex datetime is created that requires
+<<variable-assignment, assignment>> or immediate use.
+
+===== Datetime Modification Examples
+
+* Subtract three seconds from a numeric datetime in milliseconds
++
+[source,Painless]
+----
+long milliSinceEpoch = 434931330000L;
+milliSinceEpoch = milliSinceEpoch - 1000L*3L;
+----
++
+* Add three days to a complex datetime
++
+[source,Painless]
+----
+ZonedDateTime zdt =
+        ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));
+ZonedDateTime updatedZdt = zdt.plusDays(3);
+----
++
+* Subtract 125 minutes from a complex datetime
++
+[source,Painless]
+----
+ZonedDateTime zdt =
+        ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));
+ZonedDateTime updatedZdt = zdt.minusMinutes(125);
+----
++
+* Set the year on a complex datetime
++
+[source,Painless]
+----
+ZonedDateTime zdt =
+        ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));
+ZonedDateTime updatedZdt = zdt.withYear(1976);
+----
+
+==== Datetime Difference (Elapsed Time)
+
+Use either two numeric datetimes or two complex datetimes to calculate the
+difference (elapsed time) between two different datetimes. Use
+<<subtraction-operator, subtraction>> to calculate the difference between
+between two numeric datetimes of the same time unit such as milliseconds. For
+complex datetimes there is often a method or another complex type
+(<<reference-types, object>>) available to calculate the difference. Use
+<<painless-api-reference-shared-ChronoUnit, ChronoUnit>>
+to calculate the difference between two complex datetimes if supported.
+
+===== Datetime Difference Examples
+
+* Difference in milliseconds between two numeric datetimes
++
+[source,Painless]
+----
+long startTimestamp = 434931327000L;
+long endTimestamp = 434931330000L;
+long differenceInMillis = endTimestamp - startTimestamp;
+----
++
+* Difference in milliseconds between two complex datetimes
++
+[source,Painless]
+----
+ZonedDateTime zdt1 =
+        ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 11000000, ZoneId.of('Z'));
+ZonedDateTime zdt2 =
+        ZonedDateTime.of(1983, 10, 13, 22, 15, 35, 0, ZoneId.of('Z'));
+long differenceInMillis = ChronoUnit.MILLIS.between(zdt1, zdt2);
+----
++
+* Difference in days between two complex datetimes
++
+[source,Painless]
+----
+ZonedDateTime zdt1 =
+        ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 11000000, ZoneId.of('Z'));
+ZonedDateTime zdt2 =
+        ZonedDateTime.of(1983, 10, 17, 22, 15, 35, 0, ZoneId.of('Z'));
+long differenceInDays = ChronoUnit.DAYS.between(zdt1, zdt2);
+----
+
+==== Datetime Comparison
+
+Use either two numeric datetimes or two complex datetimes to do a datetime
+comparison. Use standard <<painless-operators-boolean, comparison operators>>
+to compare two numeric datetimes of the same time unit such as milliseconds.
+For complex datetimes there is often a method or another complex type
+(<<reference-types, object>>) available to do the comparison.
+
+===== Datetime Comparison Examples
+
+* Greater than comparison of two numeric datetimes in milliseconds
++
+[source,Painless]
+----
+long timestamp1 = 434931327000L;
+long timestamp2 = 434931330000L;
+
+if (timestamp1 > timestamp2) {
+   // handle condition
+}
+----
++
+* Equality comparision of two complex datetimes
++
+[source,Painless]
+----
+ZonedDateTime zdt1 =
+        ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));
+ZonedDateTime zdt2 =
+        ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));
+
+if (zdt1.equals(zdt2)) {
+    // handle condition
+}
+----
++
+* Less than comparision of two complex datetimes
++
+[source,Painless]
+----
+ZonedDateTime zdt1 =
+        ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));
+ZonedDateTime zdt2 =
+        ZonedDateTime.of(1983, 10, 17, 22, 15, 35, 0, ZoneId.of('Z'));
+
+if (zdt1.isBefore(zdt2)) {
+    // handle condition
+}
+----
++
+* Greater than comparision of two complex datetimes
++
+[source,Painless]
+----
+ZonedDateTime zdt1 =
+        ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));
+ZonedDateTime zdt2 =
+        ZonedDateTime.of(1983, 10, 17, 22, 15, 35, 0, ZoneId.of('Z'));
+
+if (zdt1.isAfter(zdt2)) {
+    // handle condition
+}
+----
+
+==== Datetime Zone
+
+Both string datetimes and complex datetimes have a timezone with a default of
+`UTC`. Numeric datetimes do not have enough explicit information to
+have a timezone, so `UTC` is always assumed. Use
+<<painless-api-reference-shared-ZonedDateTime, methods>> (or fields) in
+conjunction with a <<painless-api-reference-shared-ZoneId, ZoneId>> to change
+the timezone for a complex datetime. Parse a string datetime into a complex
+datetime to change the timezone, and then format the complex datetime back into
+a desired string datetime. Note many complex datetimes are immutable so upon
+modification a new complex datetime is created that requires
+<<variable-assignment, assignment>> or immediate use.
+
+===== Datetime Zone Examples
+
+* Modify the timezone for a complex datetime
++
+[source,Painless]
+----
+ZonedDateTime utc =
+        ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));
+ZonedDateTime pst = utc.withZoneSameInstant(ZoneId.of('America/Los_Angeles'));
+----
++
+* Modify the timezone for a string datetime
++
+[source,Painless]
+----
+String gmtString = 'Thu, 13 Oct 1983 22:15:30 GMT';
+ZonedDateTime gmtZdt = ZonedDateTime.parse(gmtString,
+        DateTimeFormatter.RFC_1123_DATE_TIME); <1>
+ZonedDateTime pstZdt =
+        gmtZdt.withZoneSameInstant(ZoneId.of('America/Los_Angeles'));
+String pstString = pstZdt.format(DateTimeFormatter.RFC_1123_DATE_TIME);
+----
+<1> Note the use of a built-in DateTimeFormatter.
+
+==== Datetime Input
+
+There are several common ways datetimes are used as input for a script
+determined by the <<painless-contexts, Painless context>>. Typically, datetime
+input will be accessed from parameters specified by the user, from an original
+source document, or from an indexed document.
+
+===== Datetime Input From User Parameters
+
+Use the {ref}/modules-scripting-using.html#_script_parameters[params section]
+during script specification to pass in a numeric datetime or string datetime as
+a script input. Access to user-defined parameters within a script is dependent
+on the Painless context, though, the parameters are most commonly accessible
+through an input called `params`.
+
+*Examples*
+
+* Parse a numeric datetime from user parameters to a complex datetime
++
+** Input:
++
+[source,JSON]
+----
+...
+"script": {
+    ...
+    "params": {
+        "input_datetime": 434931327000
+    }
+}
+...
+----
++
+** Script:
++
+[source,Painless]
+----
+long inputDateTime = params['input_datetime'];
+Instant instant = Instant.ofEpochMilli(inputDateTime);
+ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.of('Z'));
+----
++
+* Parse a string datetime from user parameters to a complex datetime
++
+** Input:
++
+[source,JSON]
+----
+...
+"script": {
+    ...
+    "params": {
+        "input_datetime": "custom y 1983 m 10 d 13 22:15:30 Z"
+    }
+}
+...
+----
++
+** Script:
++
+[source,Painless]
+----
+String datetime = params['input_datetime'];
+DateTimeFormatter dtf = DateTimeFormatter.ofPattern(
+        "'custom' 'y' yyyy 'm' MM 'd' dd HH:mm:ss VV");
+ZonedDateTime zdt = ZonedDateTime.parse(datetime, dtf); <1>
+----
+<1> Note the use of a custom DateTimeFormatter.
+
+===== Datetime Input From a Source Document
+
+Use an original {ref}/mapping-source-field.html[source] document as a script
+input to access a numeric datetime or string datetime for a specific field
+within that document. Access to an original source document within a script is
+dependent on the Painless context and is not always available. An original
+source document is most commonly accessible through an input called
+`ctx['_source']` or `params['_source']`.
+
+*Examples*
+
+* Parse a numeric datetime from a sourced document to a complex datetime
++
+** Input:
++
+[source,JSON]
+----
+{
+    ...
+    "input_datetime": 434931327000
+    ...
+}
+----
++
+** Script:
++
+[source,Painless]
+----
+long inputDateTime = ctx['_source']['input_datetime']; <1>
+Instant instant = Instant.ofEpochMilli(inputDateTime);
+ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.of('Z'));
+----
+<1> Note access to `_source` is dependent on the Painless context.
++
+* Parse a string datetime from a sourced document to a complex datetime
++
+** Input:
++
+[source,JSON]
+----
+{
+    ...
+    "input_datetime": "1983-10-13T22:15:30Z"
+    ...
+}
+----
++
+** Script:
++
+[source,Painless]
+----
+String datetime = params['_source']['input_datetime']; <1>
+ZonedDateTime zdt = ZonedDateTime.parse(datetime); <2>
+----
+<1> Note access to `_source` is dependent on the Painless context.
+<2> Note the parse method uses ISO 8601 by default.
+
+===== Datetime Input From an Indexed Document
+
+Use an indexed document as a script input to access a complex datetime for a
+specific field within that document where the field is mapped as a
+{ref}/date.html[standard date] or a {ref}/date_nanos.html[nanosecond date].
+Numeric datetime fields mapped as {ref}/number.html[numeric] and string
+datetime fields mapped as {ref}/keyword.html[keyword] are accessible through an
+indexed document as well. Access to an indexed document within a script is
+dependent on the Painless context and is not always available. An indexed
+document is most commonly accessible through an input called `doc`.
+
+*Examples*
+
+* Format a complex datetime from an indexed document to a string datetime
++
+** Assumptions:
++
+*** The field `input_datetime` exists in all indexes as part of the query
+*** All indexed documents contain the field `input_datetime`
++
+** Mappings:
++
+[source,JSON]
+----
+{
+    "mappings": {
+        ...
+        "properties": {
+            ...
+            "input_datetime": {
+                "type": "date"
+            }
+            ...
+        }
+        ...
+    }
+}
+----
++
+** Script:
++
+[source,Painless]
+----
+def input = doc['input_datetime'].value;
+String output = input.format(DateTimeFormatter.ISO_INSTANT); <1>
+----
+<1> Note the use of a built-in DateTimeFormatter.
++
+* Find the difference between two complex datetimes from an indexed document
++
+** Assumptions:
++
+*** The fields `start` and `end` may *not* exist in all indexes as part of the
+query
+*** The fields `start` and `end` may *not* have values in all indexed documents
++
+** Mappings:
++
+[source,JSON]
+----
+{
+    "mappings": {
+        ...
+        "properties": {
+            ...
+            "start": {
+                "type": "date"
+            },
+            "end": {
+                "type": "date"
+            }
+            ...
+        }
+        ...
+    }
+}
+----
++
+** Script:
++
+[source,Painless]
+----
+if (doc.containsKey('start') && doc.containsKey('end')) { <1>
+
+    if (doc['start'].size() > 0 && doc['end'].size() > 0) { <2>
+
+        def start = doc['start'].value;
+        def end = doc['end'].value;
+        long differenceInMillis = ChronoUnit.MILLIS.between(start, end);
+
+        // handle difference in times
+    } else {
+        // handle fields without values
+    }
+} else {
+    // handle index with missing fields
+}
+----
+<1> When a query's results span multiple indexes, some indexes may not
+contain a specific field. Use the `containsKey` method call on the `doc` input
+to ensure a field exists as part of the index for the current document.
+<2> Some fields within a document may have no values. Use the `size` method
+call on a field within the `doc` input to ensure that field has at least one
+value for the current document.
+
+==== Datetime Now
+
+Under most Painless contexts the current datetime, `now`, is not supported.
+There are two primary reasons for this. The first is scripts are often run once
+per document, so each time the script is run a different `now` is returned. The
+second is scripts are often run in a distributed fashion without a way to
+appropriately synchronize `now`. Instead, pass in a user-defined parameter with
+either a string datetime or numeric datetime for `now`. A numeric datetime is
+preferred as there is no need to parse it for comparision.
+
+===== Datetime Now Examples
+
+* Use a numeric datetime as `now`
++
+** Assumptions:
++
+*** The field `input_datetime` exists in all indexes as part of the query
+*** All indexed documents contain the field `input_datetime`
++
+** Mappings:
++
+[source,JSON]
+----
+{
+    "mappings": {
+        ...
+        "properties": {
+            ...
+            "input_datetime": {
+                "type": "date"
+            }
+            ...
+        }
+        ...
+    }
+}
+----
++
+** Input:
++
+[source,JSON]
+----
+...
+"script": {
+    ...
+    "params": {
+        "now": <generated numeric datetime in milliseconds since epoch>
+    }
+}
+...
+----
++
+** Script:
++
+[source,Painless]
+----
+long now = params['now'];
+def inputDateTime = doc['input_datetime'];
+long millisDateTime = zdt.toInstant().toEpochMilli();
+long elapsedTime = now - millisDateTime;
+----
++
+* Use a string datetime as `now`
++
+** Assumptions:
++
+*** The field `input_datetime` exists in all indexes as part of the query
+*** All indexed documents contain the field `input_datetime`
++
+** Mappings:
++
+[source,JSON]
+----
+{
+    "mappings": {
+        ...
+        "properties": {
+            ...
+            "input_datetime": {
+                "type": "date"
+            }
+            ...
+        }
+        ...
+    }
+}
+----
++
+** Input:
++
+[source,JSON]
+----
+...
+"script": {
+    ...
+    "params": {
+        "now": "<generated string datetime in ISO-8601>"
+    }
+}
+...
+----
++
+** Script:
++
+[source,Painless]
+----
+String nowString = params['now'];
+ZonedDateTime nowZdt = ZonedDateTime.parse(datetime); <1>
+long now = ZonedDateTime.toInstant().toEpochMilli();
+def inputDateTime = doc['input_datetime'];
+long millisDateTime = zdt.toInstant().toEpochMilli();
+long elapsedTime = now - millisDateTime;
+----
+<1> Note this parses the same string datetime every time the script runs. Use a
+numeric datetime to avoid a significant performance hit.
diff --git a/docs/painless/redirects.asciidoc b/docs/painless/redirects.asciidoc
new file mode 100644
index 0000000000000..94dd5524e9acd
--- /dev/null
+++ b/docs/painless/redirects.asciidoc
@@ -0,0 +1,9 @@
+["appendix",role="exclude",id="redirects"]
+= Deleted pages
+
+The following pages have moved or been deleted.
+
+[role="exclude",id="painless-examples"]
+=== Painless examples
+
+See <<painless-walkthrough, Painless walkthrough>>.
\ No newline at end of file
diff --git a/docs/plugins/analysis-phonetic.asciidoc b/docs/plugins/analysis-phonetic.asciidoc
index e22f819e1eb3e..3627751670a32 100644
--- a/docs/plugins/analysis-phonetic.asciidoc
+++ b/docs/plugins/analysis-phonetic.asciidoc
@@ -65,6 +65,14 @@ GET phonetic_sample/_analyze
 
 <1> Returns: `J`, `joe`, `BLKS`, `bloggs`
 
+It is important to note that `"replace": false` can lead to unexpected behavior since
+the original and the phonetically analyzed version are both kept at the same token position.
+Some queries handle these stacked tokens in special ways. For example, the fuzzy `match`
+query does not apply {ref}/common-options.html#fuzziness[fuzziness] to stacked synonym tokens.
+This can lead to issues that are difficult to diagnose and reason about. For this reason, it
+is often beneficial to use separate fields for analysis with and without phonetic filtering.
+That way searches can be run against both fields with differing boosts and trade-offs (e.g.
+only run a fuzzy `match` query on the original text field, but not on the phonetic version).
 
 [float]
 ===== Double metaphone settings
diff --git a/docs/plugins/analysis-smartcn.asciidoc b/docs/plugins/analysis-smartcn.asciidoc
index 4c019ebf2732c..3786cbc5e2b8b 100644
--- a/docs/plugins/analysis-smartcn.asciidoc
+++ b/docs/plugins/analysis-smartcn.asciidoc
@@ -17,7 +17,415 @@ include::install_remove.asciidoc[]
 [float]
 ==== `smartcn` tokenizer and token filter
 
-The plugin provides the `smartcn` analyzer and `smartcn_tokenizer` tokenizer,
-which are not configurable.
+The plugin provides the `smartcn` analyzer, `smartcn_tokenizer` tokenizer, and
+`smartcn_stop` token filter which are not configurable.
 
 NOTE: The `smartcn_word` token filter and `smartcn_sentence` have been deprecated.
+
+==== Reimplementing and extending the analyzers
+
+The `smartcn` analyzer could be reimplemented as a `custom` analyzer that can
+then be extended and configured as follows:
+
+[source,js]
+----------------------------------------------------
+PUT smartcn_example
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "rebuilt_smartcn": {
+          "tokenizer":  "smartcn_tokenizer",
+          "filter": [
+            "porter_stem",
+            "smartcn_stop"
+          ]
+        }
+      }
+    }
+  }
+}
+----------------------------------------------------
+// CONSOLE
+// TEST[s/\n$/\nstartyaml\n  - compare_analyzers: {index: smartcn_example, first: smartcn, second: rebuilt_smartcn}\nendyaml\n/]
+
+[[analysis-smartcn_stop]]
+==== `smartcn_stop` token filter
+
+The `smartcn_stop` token filter filters out stopwords defined by `smartcn`
+analyzer (`_smartcn_`), and any other custom stopwords specified by the user.
+This filter only supports the predefined `_smartcn_` stopwords list.
+If you want to use a different predefined list, then use the
+{ref}/analysis-stop-tokenfilter.html[`stop` token filter] instead.
+
+[source,js]
+--------------------------------------------------
+PUT smartcn_example
+{
+  "settings": {
+    "index": {
+      "analysis": {
+        "analyzer": {
+          "smartcn_with_stop": {
+            "tokenizer": "smartcn_tokenizer",
+            "filter": [
+              "porter_stem",
+              "my_smartcn_stop"
+            ]
+          }
+        },
+        "filter": {
+          "my_smartcn_stop": {
+            "type": "smartcn_stop",
+            "stopwords": [
+              "_smartcn_",
+              "stack",
+              "的"
+            ]
+          }
+        }
+      }
+    }
+  }
+}
+
+GET smartcn_example/_analyze
+{
+  "analyzer": "smartcn_with_stop",
+  "text": "哈喽，我们是 Elastic   我们是 Elastic Stack（Elasticsearch、Kibana、Beats 和 Logstash）的开发公司。从股票行情到 Twitter 消息流，从 Apache 日志到 WordPress 博文，我们可以帮助人们体验搜索的强大力量，帮助他们以截然不同的方式探索和分析数据"
+}
+--------------------------------------------------
+// CONSOLE
+
+The above request returns:
+
+[source,js]
+--------------------------------------------------
+{
+    "tokens": [
+        {
+            "token": "哈",
+            "start_offset": 0,
+            "end_offset": 1,
+            "type": "word",
+            "position": 0
+        },
+        {
+            "token": "喽",
+            "start_offset": 1,
+            "end_offset": 2,
+            "type": "word",
+            "position": 1
+        },
+        {
+            "token": "我们",
+            "start_offset": 3,
+            "end_offset": 5,
+            "type": "word",
+            "position": 3
+        },
+        {
+            "token": "是",
+            "start_offset": 5,
+            "end_offset": 6,
+            "type": "word",
+            "position": 4
+        },
+        {
+            "token": "elast",
+            "start_offset": 7,
+            "end_offset": 14,
+            "type": "word",
+            "position": 5
+        },
+        {
+            "token": "我们",
+            "start_offset": 17,
+            "end_offset": 19,
+            "type": "word",
+            "position": 6
+        },
+        {
+            "token": "是",
+            "start_offset": 19,
+            "end_offset": 20,
+            "type": "word",
+            "position": 7
+        },
+        {
+            "token": "elast",
+            "start_offset": 21,
+            "end_offset": 28,
+            "type": "word",
+            "position": 8
+        },
+        {
+            "token": "elasticsearch",
+            "start_offset": 35,
+            "end_offset": 48,
+            "type": "word",
+            "position": 11
+        },
+        {
+            "token": "kibana",
+            "start_offset": 49,
+            "end_offset": 55,
+            "type": "word",
+            "position": 13
+        },
+        {
+            "token": "beat",
+            "start_offset": 56,
+            "end_offset": 61,
+            "type": "word",
+            "position": 15
+        },
+        {
+            "token": "和",
+            "start_offset": 62,
+            "end_offset": 63,
+            "type": "word",
+            "position": 16
+        },
+        {
+            "token": "logstash",
+            "start_offset": 64,
+            "end_offset": 72,
+            "type": "word",
+            "position": 17
+        },
+        {
+            "token": "开发",
+            "start_offset": 74,
+            "end_offset": 76,
+            "type": "word",
+            "position": 20
+        },
+        {
+            "token": "公司",
+            "start_offset": 76,
+            "end_offset": 78,
+            "type": "word",
+            "position": 21
+        },
+        {
+            "token": "从",
+            "start_offset": 79,
+            "end_offset": 80,
+            "type": "word",
+            "position": 23
+        },
+        {
+            "token": "股票",
+            "start_offset": 80,
+            "end_offset": 82,
+            "type": "word",
+            "position": 24
+        },
+        {
+            "token": "行情",
+            "start_offset": 82,
+            "end_offset": 84,
+            "type": "word",
+            "position": 25
+        },
+        {
+            "token": "到",
+            "start_offset": 84,
+            "end_offset": 85,
+            "type": "word",
+            "position": 26
+        },
+        {
+            "token": "twitter",
+            "start_offset": 86,
+            "end_offset": 93,
+            "type": "word",
+            "position": 27
+        },
+        {
+            "token": "消息",
+            "start_offset": 94,
+            "end_offset": 96,
+            "type": "word",
+            "position": 28
+        },
+        {
+            "token": "流",
+            "start_offset": 96,
+            "end_offset": 97,
+            "type": "word",
+            "position": 29
+        },
+        {
+            "token": "从",
+            "start_offset": 98,
+            "end_offset": 99,
+            "type": "word",
+            "position": 31
+        },
+        {
+            "token": "apach",
+            "start_offset": 100,
+            "end_offset": 106,
+            "type": "word",
+            "position": 32
+        },
+        {
+            "token": "日志",
+            "start_offset": 107,
+            "end_offset": 109,
+            "type": "word",
+            "position": 33
+        },
+        {
+            "token": "到",
+            "start_offset": 109,
+            "end_offset": 110,
+            "type": "word",
+            "position": 34
+        },
+        {
+            "token": "wordpress",
+            "start_offset": 111,
+            "end_offset": 120,
+            "type": "word",
+            "position": 35
+        },
+        {
+            "token": "博",
+            "start_offset": 121,
+            "end_offset": 122,
+            "type": "word",
+            "position": 36
+        },
+        {
+            "token": "文",
+            "start_offset": 122,
+            "end_offset": 123,
+            "type": "word",
+            "position": 37
+        },
+        {
+            "token": "我们",
+            "start_offset": 124,
+            "end_offset": 126,
+            "type": "word",
+            "position": 39
+        },
+        {
+            "token": "可以",
+            "start_offset": 126,
+            "end_offset": 128,
+            "type": "word",
+            "position": 40
+        },
+        {
+            "token": "帮助",
+            "start_offset": 128,
+            "end_offset": 130,
+            "type": "word",
+            "position": 41
+        },
+        {
+            "token": "人们",
+            "start_offset": 130,
+            "end_offset": 132,
+            "type": "word",
+            "position": 42
+        },
+        {
+            "token": "体验",
+            "start_offset": 132,
+            "end_offset": 134,
+            "type": "word",
+            "position": 43
+        },
+        {
+            "token": "搜索",
+            "start_offset": 134,
+            "end_offset": 136,
+            "type": "word",
+            "position": 44
+        },
+        {
+            "token": "强大",
+            "start_offset": 137,
+            "end_offset": 139,
+            "type": "word",
+            "position": 46
+        },
+        {
+            "token": "力量",
+            "start_offset": 139,
+            "end_offset": 141,
+            "type": "word",
+            "position": 47
+        },
+        {
+            "token": "帮助",
+            "start_offset": 142,
+            "end_offset": 144,
+            "type": "word",
+            "position": 49
+        },
+        {
+            "token": "他们",
+            "start_offset": 144,
+            "end_offset": 146,
+            "type": "word",
+            "position": 50
+        },
+        {
+            "token": "以",
+            "start_offset": 146,
+            "end_offset": 147,
+            "type": "word",
+            "position": 51
+        },
+        {
+            "token": "截然不同",
+            "start_offset": 147,
+            "end_offset": 151,
+            "type": "word",
+            "position": 52
+        },
+        {
+            "token": "方式",
+            "start_offset": 152,
+            "end_offset": 154,
+            "type": "word",
+            "position": 54
+        },
+        {
+            "token": "探索",
+            "start_offset": 154,
+            "end_offset": 156,
+            "type": "word",
+            "position": 55
+        },
+        {
+            "token": "和",
+            "start_offset": 156,
+            "end_offset": 157,
+            "type": "word",
+            "position": 56
+        },
+        {
+            "token": "分析",
+            "start_offset": 157,
+            "end_offset": 159,
+            "type": "word",
+            "position": 57
+        },
+        {
+            "token": "数据",
+            "start_offset": 159,
+            "end_offset": 161,
+            "type": "word",
+            "position": 58
+        }
+    ]
+}
+--------------------------------------------------
+// TESTRESPONSE
\ No newline at end of file
diff --git a/docs/plugins/analysis-stempel.asciidoc b/docs/plugins/analysis-stempel.asciidoc
index a5526129a97d3..cd234c8d42756 100644
--- a/docs/plugins/analysis-stempel.asciidoc
+++ b/docs/plugins/analysis-stempel.asciidoc
@@ -12,7 +12,107 @@ include::install_remove.asciidoc[]
 
 [[analysis-stempel-tokenizer]]
 [float]
-==== `stempel` tokenizer and token filter
+==== `stempel` tokenizer and token filters
 
-The plugin provides the `polish` analyzer and `polish_stem` token filter,
+The plugin provides the `polish` analyzer and the `polish_stem` and `polish_stop` token filters,
 which are not configurable.
+
+==== Reimplementing and extending the analyzers
+
+The `polish` analyzer could be reimplemented as a `custom` analyzer that can
+then be extended and configured differently as follows:
+
+[source,js]
+----------------------------------------------------
+PUT /stempel_example
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "rebuilt_stempel": {
+          "tokenizer":  "standard",
+          "filter": [
+            "lowercase",
+            "polish_stop",
+            "polish_stem"
+          ]
+        }
+      }
+    }
+  }
+}
+----------------------------------------------------
+// CONSOLE
+// TEST[s/\n$/\nstartyaml\n  - compare_analyzers: {index: stempel_example, first: polish, second: rebuilt_stempel}\nendyaml\n/]
+
+[[analysis-polish-stop]]
+==== `polish_stop` token filter
+
+The `polish_stop` token filter filters out Polish stopwords (`_polish_`), and
+any other custom stopwords specified by the user. This filter only supports
+the predefined `_polish_` stopwords list.  If you want to use a different
+predefined list, then use the
+{ref}/analysis-stop-tokenfilter.html[`stop` token filter] instead.
+
+[source,js]
+--------------------------------------------------
+PUT /polish_stop_example
+{
+  "settings": {
+    "index": {
+      "analysis": {
+        "analyzer": {
+          "analyzer_with_stop": {
+            "tokenizer": "standard",
+            "filter": [
+              "lowercase",
+              "polish_stop"
+            ]
+          }
+        },
+        "filter": {
+          "polish_stop": {
+            "type": "polish_stop",
+            "stopwords": [
+              "_polish_",
+              "jeść"
+            ]
+          }
+        }
+      }
+    }
+  }
+}
+
+GET polish_stop_example/_analyze
+{
+  "analyzer": "analyzer_with_stop",
+  "text": "Gdzie kucharek sześć, tam nie ma co jeść."
+}
+--------------------------------------------------
+// CONSOLE
+
+The above request returns:
+
+[source,js]
+--------------------------------------------------
+{
+  "tokens" : [
+    {
+      "token" : "kucharek",
+      "start_offset" : 6,
+      "end_offset" : 14,
+      "type" : "<ALPHANUM>",
+      "position" : 1
+    },
+    {
+      "token" : "sześć",
+      "start_offset" : 15,
+      "end_offset" : 20,
+      "type" : "<ALPHANUM>",
+      "position" : 2
+    }
+  ]
+}
+--------------------------------------------------
+// TESTRESPONSE
diff --git a/docs/plugins/api.asciidoc b/docs/plugins/api.asciidoc
index 74fbba25810d8..7eeba28b22266 100644
--- a/docs/plugins/api.asciidoc
+++ b/docs/plugins/api.asciidoc
@@ -23,9 +23,6 @@ A number of plugins have been contributed by our community:
 * https://github.com/zentity-io/zentity[Entity Resolution Plugin] (https://zentity.io[zentity]):
   Real-time entity resolution with pure Elasticsearch (by Dave Moore)
 
-* https://github.com/NLPchina/elasticsearch-sql/[SQL language Plugin]:
-  Allows Elasticsearch to be queried with SQL (by nlpcn)
-
 * https://github.com/ritesh-kapoor/elasticsearch-pql[PQL language Plugin]:
   Allows Elasticsearch to be queried with simple pipeline query syntax.
 
diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc
index 61dcadd6e10d6..4269cd4f5a524 100644
--- a/docs/plugins/repository-azure.asciidoc
+++ b/docs/plugins/repository-azure.asciidoc
@@ -19,7 +19,11 @@ bin/elasticsearch-keystore add azure.client.default.account
 bin/elasticsearch-keystore add azure.client.default.key
 ----------------------------------------------------------------
 
-Where `account` is the azure account name and `key` the azure secret key.
+Where `account` is the azure account name and `key` the azure secret key. Instead of an azure secret key under `key`, you can alternatively
+define a shared access signatures (SAS) token under `sas_token` to use for authentication instead. When using an SAS token instead of an
+account key, the SAS token must have read (r), write (w), list (l), and delete (d) permissions for the repository base path and
+all its contents. These permissions need to be granted for the blob service (b) and apply to resource types service (s), container (c), and
+object (o).
 These settings are used by the repository's internal azure client.
 
 Note that you can also define more than one account:
@@ -29,14 +33,14 @@ Note that you can also define more than one account:
 bin/elasticsearch-keystore add azure.client.default.account
 bin/elasticsearch-keystore add azure.client.default.key
 bin/elasticsearch-keystore add azure.client.secondary.account
-bin/elasticsearch-keystore add azure.client.secondary.key
+bin/elasticsearch-keystore add azure.client.secondary.sas_token
 ----------------------------------------------------------------
 
 `default` is the default account name which will be used by a repository,
 unless you set an explicit one in the
 <<repository-azure-repository-settings, repository settings>>.
 
-Both `account` and `key` storage settings are
+The `account`, `key`, and `sas_token` storage settings are
 {ref}/secure-settings.html#reloadable-secure-settings[reloadable]. After you
 reload the settings, the internal azure clients, which are used to transfer the
 snapshot, will utilize the latest settings from the keystore.
diff --git a/docs/reference/administering.asciidoc b/docs/reference/administering.asciidoc
new file mode 100644
index 0000000000000..7170185c6ef77
--- /dev/null
+++ b/docs/reference/administering.asciidoc
@@ -0,0 +1,25 @@
+[[administer-elasticsearch]]
+= Administering {es}
+
+[partintro]
+--
+Elasticsearch is a complex piece of software, with many moving parts. There are
+many APIs and features that are designed to help you manage your Elasticsearch
+cluster.
+
+--
+
+[[backup-cluster]]
+== Back up a cluster
+
+As with any software that stores data, it is important to routinely back up your
+data. {es} replicas provide high availability during runtime; they enable you to
+tolerate sporadic node loss without an interruption of service.
+
+Replicas do not provide protection from catastrophic failure, however. For that,
+you need a real backup of your cluster—a complete copy in case something goes
+wrong.
+
+To back up your cluster, you can use the <<modules-snapshots,snapshot API>>.
+
+include::{es-repo-dir}/modules/snapshots.asciidoc[tag=snapshot-intro]
diff --git a/docs/reference/aggregations/bucket.asciidoc b/docs/reference/aggregations/bucket.asciidoc
index b9fbddc65c125..9f186ef1ffbaa 100644
--- a/docs/reference/aggregations/bucket.asciidoc
+++ b/docs/reference/aggregations/bucket.asciidoc
@@ -55,6 +55,8 @@ include::bucket/parent-aggregation.asciidoc[]
 
 include::bucket/range-aggregation.asciidoc[]
 
+include::bucket/rare-terms-aggregation.asciidoc[]
+
 include::bucket/reverse-nested-aggregation.asciidoc[]
 
 include::bucket/sampler-aggregation.asciidoc[]
@@ -64,3 +66,4 @@ include::bucket/significantterms-aggregation.asciidoc[]
 include::bucket/significanttext-aggregation.asciidoc[]
 
 include::bucket/terms-aggregation.asciidoc[]
+
diff --git a/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc
index dfc4f62a91b09..c09d1a67bf253 100644
--- a/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc
@@ -258,6 +258,39 @@ Like with the normal <<search-aggregations-bucket-datehistogram-aggregation, `da
 scripts and value level scripts are supported. This aggregation does not however, support the `min_doc_count`, 
 `extended_bounds` and `order` parameters.  
 
+==== Minimum Interval parameter
+
+The `minimum_interval` allows the caller to specify the minimum rounding interval that should be used.
+This can make the collection process more efficient, as the aggregation will not attempt to round at
+any interval lower than `minimum_interval`.
+
+The accepted units for `minimum_interval` are:
+
+* year
+* month
+* day
+* hour
+* minute
+* second
+
+[source,js]
+--------------------------------------------------
+POST /sales/_search?size=0
+{
+    "aggs" : {
+        "sale_date" : {
+             "auto_date_histogram" : {
+                 "field" : "date",
+                 "buckets": 10,
+                 "minimum_interval": "minute"
+             }
+         }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
+
 ==== Missing value
 
 The `missing` parameter defines how documents that are missing a value should be treated.
diff --git a/docs/reference/aggregations/bucket/rare-terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/rare-terms-aggregation.asciidoc
new file mode 100644
index 0000000000000..e2537b61aefda
--- /dev/null
+++ b/docs/reference/aggregations/bucket/rare-terms-aggregation.asciidoc
@@ -0,0 +1,357 @@
+[[search-aggregations-bucket-rare-terms-aggregation]]
+=== Rare Terms Aggregation
+
+A multi-bucket value source based aggregation which finds "rare" terms -- terms that are at the long-tail
+of the distribution and are not frequent.  Conceptually, this is like a `terms` aggregation that is
+sorted by `_count` ascending.  As noted in the <<search-aggregations-bucket-terms-aggregation-order,terms aggregation docs>>,
+actually ordering a `terms` agg by count ascending has unbounded error.  Instead, you should use the `rare_terms`
+aggregation
+
+//////////////////////////
+
+[source,js]
+--------------------------------------------------
+PUT /products
+{
+    "mappings": {
+        "properties": {
+            "genre": {
+                "type": "keyword"
+            },
+            "product": {
+                "type": "keyword"
+            }
+        }
+    }
+}
+
+POST /products/_doc/_bulk?refresh
+{"index":{"_id":0}}
+{"genre": "rock", "product": "Product A"}
+{"index":{"_id":1}}
+{"genre": "rock"}
+{"index":{"_id":2}}
+{"genre": "rock"}
+{"index":{"_id":3}}
+{"genre": "jazz", "product": "Product Z"}
+{"index":{"_id":4}}
+{"genre": "jazz"}
+{"index":{"_id":5}}
+{"genre": "electronic"}
+{"index":{"_id":6}}
+{"genre": "electronic"}
+{"index":{"_id":7}}
+{"genre": "electronic"}
+{"index":{"_id":8}}
+{"genre": "electronic"}
+{"index":{"_id":9}}
+{"genre": "electronic"}
+{"index":{"_id":10}}
+{"genre": "swing"}
+
+-------------------------------------------------
+// NOTCONSOLE
+// TESTSETUP
+
+//////////////////////////
+
+==== Syntax
+
+A `rare_terms` aggregation looks like this in isolation:
+
+[source,js]
+--------------------------------------------------
+{
+    "rare_terms": {
+        "field": "the_field",
+        "max_doc_count": 1
+    }
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+.`rare_terms` Parameters
+|===
+|Parameter Name |Description |Required |Default Value
+|`field` |The field we wish to find rare terms in |Required |
+|`max_doc_count` |The maximum number of documents a term should appear in. |Optional |`1`
+|`precision` |The precision of the internal CuckooFilters.  Smaller precision leads to
+better approximation, but higher memory usage. Cannot be smaller than `0.00001` |Optional |`0.01`
+|`include` |Terms that should be included in the aggregation|Optional |
+|`exclude` |Terms that should be excluded from the aggregation|Optional |
+|`missing` |The value that should be used if a document does not have the field being aggregated|Optional |
+|===
+
+
+Example:
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "aggs" : {
+        "genres" : {
+            "rare_terms" : {
+                "field" : "genre"
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[s/_search/_search\?filter_path=aggregations/]
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "aggregations" : {
+        "genres" : {
+            "buckets" : [
+                {
+                    "key" : "swing",
+                    "doc_count" : 1
+                }
+            ]
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\.//]
+
+In this example, the only bucket that we see is the "swing" bucket, because it is the only term that appears in
+one document.  If we increase the `max_doc_count` to `2`, we'll see some more buckets:
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "aggs" : {
+        "genres" : {
+            "rare_terms" : {
+                "field" : "genre",
+                "max_doc_count": 2
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[s/_search/_search\?filter_path=aggregations/]
+
+This now shows the "jazz" term which has a `doc_count` of 2":
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "aggregations" : {
+        "genres" : {
+            "buckets" : [
+                {
+                    "key" : "swing",
+                    "doc_count" : 1
+                },
+                {
+                    "key" : "jazz",
+                    "doc_count" : 2
+                }
+            ]
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\.//]
+
+[[search-aggregations-bucket-rare-terms-aggregation-max-doc-count]]
+==== Maximum document count
+
+The `max_doc_count` parameter is used to control the upper bound of document counts that a term can have.  There
+is not a size limitation on the `rare_terms` agg like `terms` agg has.  This means that terms
+which match the `max_doc_count` criteria will be returned.  The aggregation functions in this manner to avoid
+the order-by-ascending issues that afflict the `terms` aggregation.
+
+This does, however, mean that  a large number of results can be returned if chosen incorrectly.
+To limit the danger of this setting, the maximum `max_doc_count` is 100.
+
+[[search-aggregations-bucket-rare-terms-aggregation-max-buckets]]
+==== Max Bucket Limit
+
+The Rare Terms aggregation is more liable to trip the `search.max_buckets` soft limit than other aggregations due
+to how it works.  The `max_bucket` soft-limit is evaluated on a per-shard basis while the aggregation is collecting
+results.  It is possible for a term to be "rare" on a shard but become "not rare" once all the shard results are
+merged together.  This means that individual shards tend to collect more buckets than are truly rare, because
+they only have their own local view.  This list is ultimately pruned to the correct, smaller list of rare
+terms on the coordinating node... but a shard may have already tripped the `max_buckets` soft limit and aborted
+the request.
+
+When aggregating on fields that have potentially many "rare" terms, you may need to increase the `max_buckets` soft
+limit.  Alternatively, you might need to find a way to filter the results to return fewer rare values (smaller time
+span, filter by category, etc), or re-evaluate your definition of "rare" (e.g. if something
+appears 100,000 times, is it truly "rare"?)
+
+[[search-aggregations-bucket-rare-terms-aggregation-approximate-counts]]
+==== Document counts are approximate
+
+The naive way to determine the "rare" terms in a dataset is to place all the values in a map, incrementing counts
+as each document is visited, then return the bottom `n` rows.  This does not scale beyond even modestly sized data
+sets.  A sharded approach where only the "top n" values are retained from each shard (ala the `terms` aggregation)
+fails because the long-tail nature of the problem means it is impossible to find the "top n" bottom values without
+simply collecting all the values from all shards.
+
+Instead, the Rare Terms aggregation uses a different approximate algorithm:
+
+1. Values are placed in a map the first time they are seen.
+2. Each addition occurrence of the term increments a counter in the map
+3. If the counter > the `max_doc_count` threshold, the term is removed from the map and placed in a
+https://www.cs.cmu.edu/~dga/papers/cuckoo-conext2014.pdf[CuckooFilter]
+4. The CuckooFilter is consulted on each term.  If the value is inside the filter, it is known to be above the
+threshold already and skipped.
+
+After execution, the map of values is the map of "rare" terms under the `max_doc_count` threshold.  This map and CuckooFilter
+are then merged with all other shards.  If there are terms that are greater than the threshold (or appear in
+a different shard's CuckooFilter) the term is removed from the merged list.  The final map of values is returned
+to the user as the "rare" terms.
+
+CuckooFilters have the possibility of returning false positives (they can say a value exists in their collection when
+it actually does not).  Since the CuckooFilter is being used to see if a term is over threshold, this means a false positive
+from the CuckooFilter will mistakenly say a value is common when it is not (and thus exclude it from it final list of buckets).
+Practically, this means the aggregations exhibits false-negative behavior since the filter is being used "in reverse"
+of how people generally think of approximate set membership sketches.
+
+CuckooFilters are described in more detail in the paper:
+
+https://www.cs.cmu.edu/~dga/papers/cuckoo-conext2014.pdf[Fan, Bin, et al. "Cuckoo filter: Practically better than bloom."]
+Proceedings of the 10th ACM International on Conference on emerging Networking Experiments and Technologies. ACM, 2014.
+
+==== Precision
+
+Although the internal CuckooFilter is approximate in nature, the false-negative rate can be controlled with a
+`precision` parameter.  This allows the user to trade more runtime memory for more accurate results.
+
+The default precision is `0.001`, and the smallest (e.g. most accurate and largest memory overhead) is `0.00001`.
+Below are some charts which demonstrate how the accuracy of the aggregation is affected by precision and number
+of distinct terms.
+
+The X-axis shows the number of distinct values the aggregation has seen, and the Y-axis shows the percent error.
+Each line series represents one "rarity" condition (ranging from one rare item to 100,000 rare items).  For example,
+the orange "10" line means ten of the values were "rare" (`doc_count == 1`), out of 1-20m distinct values (where the
+rest of the values had `doc_count > 1`)
+
+This first chart shows precision `0.01`:
+
+image:images/rare_terms/accuracy_01.png[]
+
+And precision `0.001` (the default):
+
+image:images/rare_terms/accuracy_001.png[]
+
+And finally `precision 0.0001`:
+
+image:images/rare_terms/accuracy_0001.png[]
+
+The default precision of `0.001` maintains an accuracy of < 2.5% for the tested conditions, and accuracy slowly
+degrades in a controlled, linear fashion as the number of distinct values increases.
+
+The default precision of `0.001` has a memory profile of `1.748⁻⁶ * n` bytes, where `n` is the number
+of distinct values the aggregation has seen (it can also be roughly eyeballed, e.g. 20 million unique values is about
+30mb of memory).  The memory usage is linear to the number of distinct values regardless of which precision is chosen,
+the precision only affects the slope of the memory profile as seen in this chart:
+
+image:images/rare_terms/memory.png[]
+
+For comparison, an equivalent terms aggregation at 20 million buckets would be roughly
+`20m * 69b == ~1.38gb` (with 69 bytes being a very optimistic estimate of an empty bucket cost, far lower than what
+the circuit breaker accounts for).  So although the `rare_terms` agg is relatively heavy, it is still orders of
+magnitude smaller than the equivalent terms aggregation
+
+==== Filtering Values
+
+It is possible to filter the values for which buckets will be created. This can be done using the `include` and
+`exclude` parameters which are based on regular expression strings or arrays of exact values. Additionally,
+`include` clauses can filter using `partition` expressions.
+
+===== Filtering Values with regular expressions
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "aggs" : {
+        "genres" : {
+            "rare_terms" : {
+                "field" : "genre",
+                "include" : "swi*",
+                "exclude" : "electro*"
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+In the above example, buckets will be created for all the tags that starts with `swi`, except those starting
+with `electro` (so the tag `swing` will be aggregated but not `electro_swing`). The `include` regular expression will determine what
+values are "allowed" to be aggregated, while the `exclude` determines the values that should not be aggregated. When
+both are defined, the `exclude` has precedence, meaning, the `include` is evaluated first and only then the `exclude`.
+
+The syntax is the same as <<regexp-syntax,regexp queries>>.
+
+===== Filtering Values with exact values
+
+For matching based on exact values the `include` and `exclude` parameters can simply take an array of
+strings that represent the terms as they are found in the index:
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "aggs" : {
+        "genres" : {
+             "rare_terms" : {
+                 "field" : "genre",
+                 "include" : ["swing", "rock"],
+                 "exclude" : ["jazz"]
+             }
+         }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+
+==== Missing value
+
+The `missing` parameter defines how documents that are missing a value should be treated.
+By default they will be ignored but it is also possible to treat them as if they
+had a value.
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "aggs" : {
+        "genres" : {
+             "rare_terms" : {
+                 "field" : "genre",
+                 "missing": "N/A" <1>
+             }
+         }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+<1> Documents without a value in the `tags` field will fall into the same bucket as documents that have the value `N/A`.
+
+==== Nested, RareTerms, and scoring sub-aggregations
+
+The RareTerms aggregation has to operate in `breadth_first` mode, since it needs to prune terms as doc count thresholds
+are breached.  This requirement means the RareTerms aggregation is incompatible with certain combinations of aggregations
+that require `depth_first`. In particular, scoring sub-aggregations that are inside a `nested` force the entire aggregation tree to run
+in `depth_first` mode.  This will throw an exception since RareTerms is unable to process `depth_first`.
+
+As a concrete example, if `rare_terms` aggregation is the child of a `nested` aggregation, and one of the child aggregations of `rare_terms`
+needs document scores (like a `top_hits` aggregation), this will throw an exception.
\ No newline at end of file
diff --git a/docs/reference/analysis/tokenfilters/elision-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/elision-tokenfilter.asciidoc
index 924903b9f65a8..34646a0413e36 100644
--- a/docs/reference/analysis/tokenfilters/elision-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/elision-tokenfilter.asciidoc
@@ -4,8 +4,12 @@
 A token filter which removes elisions. For example, "l'avion" (the
 plane) will tokenized as "avion" (plane).
 
-Accepts `articles` setting which is a set of stop words articles. For
-example:
+Requires either an `articles` parameter which is a set of stop word articles, or
+`articles_path` which points to a text file containing the stop set. Also optionally
+accepts `articles_case`, which indicates whether the filter treats those articles as
+case sensitive.
+
+For example:
 
 [source,js]
 --------------------------------------------------
@@ -22,6 +26,7 @@ PUT /elision_example
             "filter" : {
                 "elision" : {
                     "type" : "elision",
+                    "articles_case": true,
                     "articles" : ["l", "m", "t", "qu", "n", "s", "j"]
                 }
             }
diff --git a/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc
index 139f7c3ab0ad0..f47e97d27ea7f 100644
--- a/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc
@@ -43,6 +43,8 @@ Additional settings are:
 * `expand` (defaults to `true`).
 * `lenient` (defaults to `false`). If `true` ignores exceptions while parsing the synonym configuration. It is important
 to note that only those synonym rules which cannot get parsed are ignored. For instance consider the following request:
+ 
+
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc
index 009b027b9ef2d..9b1b0b0ce098f 100644
--- a/docs/reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/word-delimiter-tokenfilter.asciidoc
@@ -18,7 +18,7 @@ Parameters include:
 
 `generate_word_parts`::
     If `true` causes parts of words to be
-    generated: "PowerShot" => "Power" "Shot". Defaults to `true`.
+    generated: "Power-Shot", "(Power,Shot)" => "Power" "Shot". Defaults to `true`.
 
 `generate_number_parts`::
     If `true` causes number subwords to be
diff --git a/docs/reference/analysis/tokenizers.asciidoc b/docs/reference/analysis/tokenizers.asciidoc
index d6f15ded05fab..628afebfdcbb8 100644
--- a/docs/reference/analysis/tokenizers.asciidoc
+++ b/docs/reference/analysis/tokenizers.asciidoc
@@ -155,3 +155,7 @@ include::tokenizers/simplepattern-tokenizer.asciidoc[]
 include::tokenizers/simplepatternsplit-tokenizer.asciidoc[]
 
 include::tokenizers/pathhierarchy-tokenizer.asciidoc[]
+
+include::tokenizers/pathhierarchy-tokenizer-examples.asciidoc[]
+
+
diff --git a/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer-examples.asciidoc b/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer-examples.asciidoc
new file mode 100644
index 0000000000000..ee02d66e4034b
--- /dev/null
+++ b/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer-examples.asciidoc
@@ -0,0 +1,191 @@
+[[analysis-pathhierarchy-tokenizer-examples]]
+=== Path Hierarchy Tokenizer Examples
+
+A common use-case for the `path_hierarchy` tokenizer is filtering results by 
+file paths. If indexing a file path along with the data, the use of the 
+`path_hierarchy` tokenizer to analyze the path allows filtering the results 
+by different parts of the file path string.
+
+
+This example configures an index to have two custom analyzers and applies
+those analyzers to multifields of the `file_path` text field that will 
+store filenames. One of the two analyzers uses reverse tokenization.
+Some sample documents are then indexed to represent some file paths
+for photos inside photo folders of two different users.
+
+
+[source,js]
+--------------------------------------------------
+PUT file-path-test
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "custom_path_tree": {
+          "tokenizer": "custom_hierarchy"
+        },
+        "custom_path_tree_reversed": {
+          "tokenizer": "custom_hierarchy_reversed"
+        }
+      },
+      "tokenizer": {
+        "custom_hierarchy": {
+          "type": "path_hierarchy",
+          "delimiter": "/"
+        },
+        "custom_hierarchy_reversed": {
+          "type": "path_hierarchy",
+          "delimiter": "/",
+          "reverse": "true"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties": {
+      "file_path": {
+        "type": "text",
+        "fields": {
+          "tree": {
+            "type": "text",
+            "analyzer": "custom_path_tree"
+          },
+          "tree_reversed": {
+            "type": "text",
+            "analyzer": "custom_path_tree_reversed"
+          }
+        }
+      }
+    }
+  }
+}
+
+POST file-path-test/_doc/1
+{
+  "file_path": "/User/alice/photos/2017/05/16/my_photo1.jpg"
+}
+
+POST file-path-test/_doc/2
+{
+  "file_path": "/User/alice/photos/2017/05/16/my_photo2.jpg"
+}
+
+POST file-path-test/_doc/3
+{
+  "file_path": "/User/alice/photos/2017/05/16/my_photo3.jpg"
+}
+
+POST file-path-test/_doc/4
+{
+  "file_path": "/User/alice/photos/2017/05/15/my_photo1.jpg"
+}
+
+POST file-path-test/_doc/5
+{
+  "file_path": "/User/bob/photos/2017/05/16/my_photo1.jpg"
+}
+--------------------------------------------------
+// CONSOLE
+// TESTSETUP
+
+
+A search for a particular file path string against the text field matches all 
+the example documents, with Bob's documents ranking highest due to `bob` also 
+being one of the terms created by the standard analyzer boosting relevance for
+Bob's documents.
+
+[source,js]
+--------------------------------------------------
+GET file-path-test/_search
+{
+  "query": {
+    "match": {
+      "file_path": "/User/bob/photos/2017/05"
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+
+It's simple to match or filter documents with file paths that exist within a
+particular directory using the `file_path.tree` field.
+
+[source,js]
+--------------------------------------------------
+GET file-path-test/_search
+{
+  "query": {
+    "term": {
+      "file_path.tree": "/User/alice/photos/2017/05/16"
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+With the reverse parameter for this tokenizer, it's also possible to match
+from the other end of the file path, such as individual file names or a deep
+level subdirectory. The following example shows a search for all files named
+`my_photo1.jpg` within any directory via the `file_path.tree_reversed` field 
+configured to use the reverse parameter in the mapping.
+
+
+[source,js]
+--------------------------------------------------
+GET file-path-test/_search
+{
+  "query": {
+    "term": {
+      "file_path.tree_reversed": {
+        "value": "my_photo1.jpg"
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+
+Viewing the tokens generated with both forward and reverse is instructive
+in showing the tokens created for the same file path value.
+
+
+[source,js]
+--------------------------------------------------
+POST file-path-test/_analyze
+{
+  "analyzer": "custom_path_tree",
+  "text": "/User/alice/photos/2017/05/16/my_photo1.jpg"
+}
+
+POST file-path-test/_analyze
+{
+  "analyzer": "custom_path_tree_reversed",
+  "text": "/User/alice/photos/2017/05/16/my_photo1.jpg"
+}
+--------------------------------------------------
+// CONSOLE
+
+
+It's also useful to be able to filter with file paths when combined with other
+types of searches, such as this example looking for any files paths with `16` 
+that also must be in Alice's photo directory.
+
+[source,js]
+--------------------------------------------------
+GET file-path-test/_search
+{
+  "query": {
+    "bool" : {
+      "must" : {
+        "match" : { "file_path" : "16" }
+      },
+      "filter": {
+        "term" : { "file_path.tree" : "/User/alice" }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer.asciidoc
index 55aa7d66da343..8d425197a2a6d 100644
--- a/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer.asciidoc
@@ -170,3 +170,7 @@ If we were to set `reverse` to `true`, it would produce the following:
 ---------------------------
 [ one/two/three/, two/three/, three/ ]
 ---------------------------
+
+[float]
+=== Detailed Examples
+See <<analysis-pathhierarchy-tokenizer-examples, detailed examples here>>. 
diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc
index 0b7d9ce403466..b46bd21a32bb1 100644
--- a/docs/reference/api-conventions.asciidoc
+++ b/docs/reference/api-conventions.asciidoc
@@ -86,6 +86,10 @@ Where:
 `date_format`:: is the optional format in which the computed date should be rendered. Defaults to `yyyy.MM.dd`. Format should be compatible with java-time https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html
 `time_zone`:: is the optional time zone. Defaults to `utc`.
 
+NOTE: Pay attention to the usage of small vs capital letters used in the `date_format`. For example:
+`mm` denotes minute of hour, while `MM` denotes month of year. Similarly `hh` denotes the hour in the
+`1-12` range in combination with `AM/PM`, while `HH` denotes the hour in the `0-23` 24-hour range.
+
 Date math expressions are resolved locale-independent. Consequently, it is not possible to use any other
 calendars than the Gregorian calendar.
 
diff --git a/docs/reference/cat.asciidoc b/docs/reference/cat.asciidoc
index 0da752aba42d2..743bae7b85679 100644
--- a/docs/reference/cat.asciidoc
+++ b/docs/reference/cat.asciidoc
@@ -40,7 +40,7 @@ Might respond with:
 id                     host      ip        node
 u_n93zwxThWHi1PDBJAGAg 127.0.0.1 127.0.0.1 u_n93zw
 --------------------------------------------------
-// TESTRESPONSE[s/u_n93zw(xThWHi1PDBJAGAg)?/.+/ _cat]
+// TESTRESPONSE[s/u_n93zw(xThWHi1PDBJAGAg)?/.+/ non_json]
 
 [float]
 [[help]]
@@ -64,7 +64,7 @@ host | h | host name
 ip   |   | ip address
 node | n | node name
 --------------------------------------------------
-// TESTRESPONSE[s/[|]/[|]/ _cat]
+// TESTRESPONSE[s/[|]/[|]/ non_json]
 
 NOTE: `help` is not supported if any optional url parameter is used.
 For example `GET _cat/shards/twitter?help` or `GET _cat/indices/twi*?help`
@@ -90,7 +90,7 @@ Responds with:
 --------------------------------------------------
 127.0.0.1 9300 27 sLBaIGK
 --------------------------------------------------
-// TESTRESPONSE[s/9300 27 sLBaIGK/\\d+ \\d+ .+/ _cat]
+// TESTRESPONSE[s/9300 27 sLBaIGK/\\d+ \\d+ .+/ non_json]
 
 You can also request multiple columns using simple wildcards like
 `/_cat/thread_pool?h=ip,queue*` to get all headers (or aliases) starting
diff --git a/docs/reference/cat/alias.asciidoc b/docs/reference/cat/alias.asciidoc
index 394231e448dc0..a46a4c658d4b0 100644
--- a/docs/reference/cat/alias.asciidoc
+++ b/docs/reference/cat/alias.asciidoc
@@ -49,7 +49,7 @@ alias2 test1 *      -            -
 alias3 test1 -      1            1
 alias4 test1 -      2            1,2
 --------------------------------------------------
-// TESTRESPONSE[s/[*]/[*]/ _cat]
+// TESTRESPONSE[s/[*]/[*]/ non_json]
 
 The output shows that `alias2` has configured a filter, and specific routing
 configurations in `alias3` and `alias4`.
diff --git a/docs/reference/cat/allocation.asciidoc b/docs/reference/cat/allocation.asciidoc
index 0ff745c3e5cb3..8d2c931665ba1 100644
--- a/docs/reference/cat/allocation.asciidoc
+++ b/docs/reference/cat/allocation.asciidoc
@@ -19,7 +19,7 @@ shards disk.indices disk.used disk.avail disk.total disk.percent host      ip
      1         260b    47.3gb     43.4gb    100.7gb           46 127.0.0.1 127.0.0.1 CSUXak2
 --------------------------------------------------
 // TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/ s/46/\\d+/]
-// TESTRESPONSE[s/CSUXak2/.+/ _cat]
+// TESTRESPONSE[s/CSUXak2/.+/ non_json]
 
 Here we can see that the single shard created has been allocated to the single
 node available.
diff --git a/docs/reference/cat/count.asciidoc b/docs/reference/cat/count.asciidoc
index 28dc39adc8dff..01fec6df73180 100644
--- a/docs/reference/cat/count.asciidoc
+++ b/docs/reference/cat/count.asciidoc
@@ -19,7 +19,7 @@ Looks like:
 epoch      timestamp count
 1475868259 15:24:19  121
 --------------------------------------------------
-// TESTRESPONSE[s/1475868259 15:24:19/\\d+ \\d+:\\d+:\\d+/ _cat]
+// TESTRESPONSE[s/1475868259 15:24:19/\\d+ \\d+:\\d+:\\d+/ non_json]
 
 Or for a single index:
 
@@ -35,7 +35,7 @@ GET /_cat/count/twitter?v
 epoch      timestamp count
 1475868259 15:24:20  120
 --------------------------------------------------
-// TESTRESPONSE[s/1475868259 15:24:20/\\d+ \\d+:\\d+:\\d+/ _cat]
+// TESTRESPONSE[s/1475868259 15:24:20/\\d+ \\d+:\\d+:\\d+/ non_json]
 
 
 NOTE: The document count indicates the number of live documents and does not include deleted documents which have not yet been cleaned up by the merge process.
diff --git a/docs/reference/cat/fielddata.asciidoc b/docs/reference/cat/fielddata.asciidoc
index ff71728a4f50b..f94bcd6fe5d78 100644
--- a/docs/reference/cat/fielddata.asciidoc
+++ b/docs/reference/cat/fielddata.asciidoc
@@ -52,7 +52,7 @@ Nqk-6inXQq-OxUfOUI8jNQ 127.0.0.1 127.0.0.1 Nqk-6in body    544b
 Nqk-6inXQq-OxUfOUI8jNQ 127.0.0.1 127.0.0.1 Nqk-6in soul    480b
 --------------------------------------------------
 // TESTRESPONSE[s/544b|480b/\\d+(\\.\\d+)?[tgmk]?b/]
-// TESTRESPONSE[s/Nqk-6in[^ ]*/.+/ s/soul|body/\\w+/ _cat]
+// TESTRESPONSE[s/Nqk-6in[^ ]*/.+/ s/soul|body/\\w+/ non_json]
 
 Fields can be specified either as a query parameter, or in the URL path:
 
@@ -71,7 +71,7 @@ id                     host      ip        node    field   size
 Nqk-6inXQq-OxUfOUI8jNQ 127.0.0.1 127.0.0.1 Nqk-6in body    544b
 --------------------------------------------------
 // TESTRESPONSE[s/544b|480b/\\d+(\\.\\d+)?[tgmk]?b/]
-// TESTRESPONSE[s/Nqk-6in[^ ]*/.+/ _cat]
+// TESTRESPONSE[s/Nqk-6in[^ ]*/.+/ non_json]
 
 And it accepts a comma delimited list:
 
@@ -91,6 +91,6 @@ Nqk-6inXQq-OxUfOUI8jNQ 127.0.0.1 127.0.0.1 Nqk-6in body    544b
 Nqk-6inXQq-OxUfOUI8jNQ 127.0.0.1 127.0.0.1 Nqk-6in soul    480b
 --------------------------------------------------
 // TESTRESPONSE[s/544b|480b/\\d+(\\.\\d+)?[tgmk]?b/]
-// TESTRESPONSE[s/Nqk-6in[^ ]*/.+/ s/soul|body/\\w+/ _cat]
+// TESTRESPONSE[s/Nqk-6in[^ ]*/.+/ s/soul|body/\\w+/ non_json]
 
 The output shows the individual fielddata for the `body` and `soul` fields, one row per field per node.
diff --git a/docs/reference/cat/health.asciidoc b/docs/reference/cat/health.asciidoc
index 5f053edf30866..883119925fd5f 100644
--- a/docs/reference/cat/health.asciidoc
+++ b/docs/reference/cat/health.asciidoc
@@ -17,7 +17,7 @@ epoch      timestamp cluster       status node.total node.data shards pri relo i
 1475871424 16:17:04  elasticsearch green           1         1      1   1    0    0        0             0                  -                100.0%
 --------------------------------------------------
 // TESTRESPONSE[s/1475871424 16:17:04/\\d+ \\d+:\\d+:\\d+/]
-// TESTRESPONSE[s/elasticsearch/[^ ]+/ s/0                  -/\\d+ (-|\\d+(\\.\\d+)?[ms]+)/ _cat]
+// TESTRESPONSE[s/elasticsearch/[^ ]+/ s/0                  -/\\d+ (-|\\d+(\\.\\d+)?[ms]+)/ non_json]
 
 It has one option `ts` to disable the timestamping:
 
@@ -35,7 +35,7 @@ which looks like:
 cluster       status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
 elasticsearch green           1         1      1   1    0    0        0             0                  -                100.0%
 --------------------------------------------------
-// TESTRESPONSE[s/elasticsearch/[^ ]+/ s/0                  -/\\d+ (-|\\d+(\\.\\d+)?[ms]+)/ _cat]
+// TESTRESPONSE[s/elasticsearch/[^ ]+/ s/0                  -/\\d+ (-|\\d+(\\.\\d+)?[ms]+)/ non_json]
 
 A common use of this command is to verify the health is consistent
 across nodes:
diff --git a/docs/reference/cat/indices.asciidoc b/docs/reference/cat/indices.asciidoc
index 2a5b865fefa47..653889dac8315 100644
--- a/docs/reference/cat/indices.asciidoc
+++ b/docs/reference/cat/indices.asciidoc
@@ -21,7 +21,7 @@ yellow open   twitter  u8FNjxh8Rfy_awN11oDKYQ   1   1       1200            0
 green  open   twitter2 nYFWZEO7TUiOjLQXBaYJpA   1   0          0            0       260b           260b
 --------------------------------------------------
 // TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/]
-// TESTRESPONSE[s/u8FNjxh8Rfy_awN11oDKYQ|nYFWZEO7TUiOjLQXBaYJpA/.+/ _cat]
+// TESTRESPONSE[s/u8FNjxh8Rfy_awN11oDKYQ|nYFWZEO7TUiOjLQXBaYJpA/.+/ non_json]
 
 We can tell quickly how many shards make up an index, the number of
 docs, deleted docs, primary store size, and total store size (all shards including replicas).
@@ -64,7 +64,7 @@ health status index    uuid                   pri rep docs.count docs.deleted st
 yellow open   twitter  u8FNjxh8Rfy_awN11oDKYQ   1   1       1200            0     88.1kb         88.1kb
 --------------------------------------------------
 // TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/]
-// TESTRESPONSE[s/u8FNjxh8Rfy_awN11oDKYQ/.+/ _cat]
+// TESTRESPONSE[s/u8FNjxh8Rfy_awN11oDKYQ/.+/ non_json]
 
 Which index has the largest number of documents?
 
@@ -84,7 +84,7 @@ yellow open   twitter  u8FNjxh8Rfy_awN11oDKYQ   1   1       1200            0
 green  open   twitter2 nYFWZEO7TUiOjLQXBaYJpA   1   0          0            0       260b           260b
 --------------------------------------------------
 // TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/]
-// TESTRESPONSE[s/u8FNjxh8Rfy_awN11oDKYQ|nYFWZEO7TUiOjLQXBaYJpA/.+/ _cat]
+// TESTRESPONSE[s/u8FNjxh8Rfy_awN11oDKYQ|nYFWZEO7TUiOjLQXBaYJpA/.+/ non_json]
 
 How many merge operations have the shards for the `twitter` completed?
 
@@ -102,7 +102,7 @@ Might look like:
 health index   pri rep docs.count mt pri.mt
 yellow twitter   1   1 1200       16     16
 --------------------------------------------------
-// TESTRESPONSE[s/16/\\d+/ _cat]
+// TESTRESPONSE[s/16/\\d+/ non_json]
 
 How much memory is used per index?
 
@@ -122,4 +122,4 @@ twitter   8.1gb
 twitter2  30.5kb
 --------------------------------------------------
 // TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/]
-// TESTRESPONSE[_cat]
+// TESTRESPONSE[non_json]
diff --git a/docs/reference/cat/master.asciidoc b/docs/reference/cat/master.asciidoc
index dfa10d6e3a4e6..a38baa455caee 100644
--- a/docs/reference/cat/master.asciidoc
+++ b/docs/reference/cat/master.asciidoc
@@ -17,7 +17,7 @@ might respond:
 id                     host      ip        node
 YzWoH_2BT-6UjVGDyPdqYg 127.0.0.1 127.0.0.1 YzWoH_2
 --------------------------------------------------
-// TESTRESPONSE[s/YzWoH_2.+/.+/ _cat]
+// TESTRESPONSE[s/YzWoH_2.+/.+/ non_json]
 
 This information is also available via the `nodes` command, but this
 is slightly shorter when all you want to do, for example, is verify
diff --git a/docs/reference/cat/nodeattrs.asciidoc b/docs/reference/cat/nodeattrs.asciidoc
index 2b893a4c79b11..e5c335f7c375f 100644
--- a/docs/reference/cat/nodeattrs.asciidoc
+++ b/docs/reference/cat/nodeattrs.asciidoc
@@ -22,7 +22,7 @@ node-0 127.0.0.1 127.0.0.1 testattr test
 ...
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\.\n$/\n(.+ xpack\\.installed true\n)?\n/]
-// TESTRESPONSE[s/\.\.\.\n/(.+ ml\\..+\n)*/ _cat]
+// TESTRESPONSE[s/\.\.\.\n/(.+ ml\\..+\n)*/ non_json]
 // If xpack is not installed then neither ... with match anything
 // If xpack is installed then the first ... contains ml attributes
 // and the second contains xpack.installed=true
@@ -68,7 +68,7 @@ node-0 19566 testattr test
 --------------------------------------------------
 // TESTRESPONSE[s/19566/\\d*/]
 // TESTRESPONSE[s/\.\.\.\n$/\n(.+ xpack\\.installed true\n)?\n/]
-// TESTRESPONSE[s/\.\.\.\n/(.+ ml\\..+\n)*/ _cat]
+// TESTRESPONSE[s/\.\.\.\n/(.+ ml\\..+\n)*/ non_json]
 // If xpack is not installed then neither ... with match anything
 // If xpack is installed then the first ... contains ml attributes
 // and the second contains xpack.installed=true
diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc
index 151ce80196b50..a882f5f9fa398 100644
--- a/docs/reference/cat/nodes.asciidoc
+++ b/docs/reference/cat/nodes.asciidoc
@@ -14,11 +14,11 @@ Might look like:
 [source,txt]
 --------------------------------------------------
 ip        heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
-127.0.0.1           65          99  42    3.07                  mdi       *      mJw06l1
+127.0.0.1           65          99  42    3.07                  dim       *      mJw06l1
 --------------------------------------------------
 // TESTRESPONSE[s/3.07/(\\d+\\.\\d+( \\d+\\.\\d+ (\\d+\\.\\d+)?)?)?/]
 // TESTRESPONSE[s/65          99  42/\\d+ \\d+ \\d+/]
-// TESTRESPONSE[s/[*]/[*]/ s/mJw06l1/.+/ _cat]
+// TESTRESPONSE[s/[*]/[*]/ s/mJw06l1/.+/ non_json]
 
 The first few columns (`ip`, `heap.percent`, `ram.percent`, `cpu`, `load_*`) tell
 you where your nodes live and give a quick picture of performance stats.
@@ -63,7 +63,7 @@ Might look like:
 id   ip        port  v         m
 veJR 127.0.0.1 59938 {version} *
 --------------------------------------------------
-// TESTRESPONSE[s/veJR/.+/ s/59938/\\d+/ s/[*]/[*]/ _cat]
+// TESTRESPONSE[s/veJR/.+/ s/59938/\\d+/ s/[*]/[*]/ non_json]
 
 [cols="<,<,<,<,<",options="header",subs="normal"]
 |=======================================================================
diff --git a/docs/reference/cat/pending_tasks.asciidoc b/docs/reference/cat/pending_tasks.asciidoc
index d5216c1eb0023..ec923f270c571 100644
--- a/docs/reference/cat/pending_tasks.asciidoc
+++ b/docs/reference/cat/pending_tasks.asciidoc
@@ -24,6 +24,6 @@ insertOrder timeInQueue priority source
        1690       787ms HIGH     update-mapping [foo][t]
        1691       773ms HIGH     update-mapping [foo][t]
 --------------------------------------------------
-// TESTRESPONSE[s/(\n.+)+/(\\n.+)*/ _cat]
+// TESTRESPONSE[s/(\n.+)+/(\\n.+)*/ non_json]
 // We can't assert anything about the tasks in progress here because we don't
 // know what might be in progress....
diff --git a/docs/reference/cat/plugins.asciidoc b/docs/reference/cat/plugins.asciidoc
index 5e516fee72cf4..9800b4fef0a64 100644
--- a/docs/reference/cat/plugins.asciidoc
+++ b/docs/reference/cat/plugins.asciidoc
@@ -31,6 +31,6 @@ U7321H6 mapper-size             {version_qualified} The Mapper Size plugin allow
 U7321H6 store-smb               {version_qualified} The Store SMB plugin adds support for SMB stores.
 U7321H6 transport-nio           {version_qualified} The nio transport.
 ------------------------------------------------------------------------------
-// TESTRESPONSE[s/([.()])/\\$1/ s/U7321H6/.+/ _cat]
+// TESTRESPONSE[s/([.()])/\\$1/ s/U7321H6/.+/ non_json]
 
 We can tell quickly how many plugins per node we have and which versions.
diff --git a/docs/reference/cat/recovery.asciidoc b/docs/reference/cat/recovery.asciidoc
index c4288f882e21e..1477dfb676f9f 100644
--- a/docs/reference/cat/recovery.asciidoc
+++ b/docs/reference/cat/recovery.asciidoc
@@ -30,7 +30,7 @@ twitter 0     13ms store done  n/a         n/a         127.0.0.1   node-0      n
 // TESTRESPONSE[s/100%/0.0%/]
 // TESTRESPONSE[s/9928/0/]
 // TESTRESPONSE[s/13ms/\\d+m?s/]
-// TESTRESPONSE[s/13/\\d+/ _cat]
+// TESTRESPONSE[s/13/\\d+/ non_json]
 
 In the above case, the source and target nodes are the same because the recovery
 type was store, i.e. they were read from local storage on node start.
@@ -57,7 +57,7 @@ twitter 0 1252ms peer done  192.168.1.1 192.168.1.2 0     100.0%  0 100.0%
 // TESTRESPONSE[s/192.168.1.2/127.0.0.1/]
 // TESTRESPONSE[s/192.168.1.1/n\/a/]
 // TESTRESPONSE[s/100.0%/0.0%/]
-// TESTRESPONSE[s/1252/\\d+/ _cat]
+// TESTRESPONSE[s/1252/\\d+/ non_json]
 
 We can see in the above listing that our thw twitter shard was recovered from another node.
 Notice that the recovery type is shown as `peer`. The files and bytes copied are
@@ -81,4 +81,4 @@ This will show a recovery of type snapshot in the response
 i       s t      ty       st    rep     snap   f  fp   b     bp
 twitter 0 1978ms snapshot done  twitter snap_1 79 8.0% 12086 9.0%
 --------------------------------------------------------------------------------
-// TESTRESPONSE[_cat]
+// TESTRESPONSE[non_json]
diff --git a/docs/reference/cat/repositories.asciidoc b/docs/reference/cat/repositories.asciidoc
index 8caf3c5fd6f73..89daf7748a5d4 100644
--- a/docs/reference/cat/repositories.asciidoc
+++ b/docs/reference/cat/repositories.asciidoc
@@ -19,6 +19,6 @@ id    type
 repo1   fs
 repo2   s3
 --------------------------------------------------
-// TESTRESPONSE[s/\nrepo2   s3// _cat]
+// TESTRESPONSE[s/\nrepo2   s3// non_json]
 
 We can quickly see which repositories are registered and their type.
diff --git a/docs/reference/cat/segments.asciidoc b/docs/reference/cat/segments.asciidoc
index a4c2c54d8eefd..5fa2f66e384a0 100644
--- a/docs/reference/cat/segments.asciidoc
+++ b/docs/reference/cat/segments.asciidoc
@@ -20,7 +20,7 @@ index shard prirep ip        segment generation docs.count docs.deleted size siz
 test  0     p      127.0.0.1 _0               0          1            0  3kb        2042 false     true       {lucene_version}   true
 test1 0     p      127.0.0.1 _0               0          1            0  3kb        2042 false     true       {lucene_version}   true
 --------------------------------------------------
-// TESTRESPONSE[s/3kb/\\d+(\\.\\d+)?[mk]?b/ s/2042/\\d+/ _cat]
+// TESTRESPONSE[s/3kb/\\d+(\\.\\d+)?[mk]?b/ s/2042/\\d+/ non_json]
 
 The output shows information about index names and shard numbers in the first
 two columns.
diff --git a/docs/reference/cat/shards.asciidoc b/docs/reference/cat/shards.asciidoc
index f32c553ba72f6..345e493375400 100644
--- a/docs/reference/cat/shards.asciidoc
+++ b/docs/reference/cat/shards.asciidoc
@@ -23,7 +23,7 @@ twitter 0 p STARTED 3014 31.1mb 192.168.56.10 H5dfFeA
 // TESTRESPONSE[s/3014/\\d+/]
 // TESTRESPONSE[s/31.1mb/\\d+(\.\\d+)?[kmg]?b/]
 // TESTRESPONSE[s/192.168.56.10/.*/]
-// TESTRESPONSE[s/H5dfFeA/node-0/ _cat]
+// TESTRESPONSE[s/H5dfFeA/node-0/ non_json]
 
 [float]
 [[index-pattern]]
@@ -49,7 +49,7 @@ twitter 0 p STARTED 3014 31.1mb 192.168.56.10 H5dfFeA
 // TESTRESPONSE[s/3014/\\d+/]
 // TESTRESPONSE[s/31.1mb/\\d+(\.\\d+)?[kmg]?b/]
 // TESTRESPONSE[s/192.168.56.10/.*/]
-// TESTRESPONSE[s/H5dfFeA/node-0/ _cat]
+// TESTRESPONSE[s/H5dfFeA/node-0/ non_json]
 
 
 [float]
@@ -72,7 +72,7 @@ A relocating shard will be shown as follows
 ---------------------------------------------------------------------------
 twitter 0 p RELOCATING 3014 31.1mb 192.168.56.10 H5dfFeA -> -> 192.168.56.30 bGG90GE
 ---------------------------------------------------------------------------
-// TESTRESPONSE[_cat]
+// TESTRESPONSE[non_json]
 
 [float]
 [[states]]
@@ -95,7 +95,7 @@ You can get the initializing state in the response like this
 twitter 0 p STARTED      3014 31.1mb 192.168.56.10 H5dfFeA
 twitter 0 r INITIALIZING    0 14.3mb 192.168.56.30 bGG90GE
 ---------------------------------------------------------------------------
-// TESTRESPONSE[_cat]
+// TESTRESPONSE[non_json]
 
 If a shard cannot be assigned, for example you've overallocated the
 number of replicas for the number of nodes in the cluster, the shard
@@ -119,7 +119,7 @@ twitter 0 r STARTED    3014 31.1mb 192.168.56.30 bGG90GE
 twitter 0 r STARTED    3014 31.1mb 192.168.56.20 I8hydUG
 twitter 0 r UNASSIGNED ALLOCATION_FAILED
 ---------------------------------------------------------------------------
-// TESTRESPONSE[_cat]
+// TESTRESPONSE[non_json]
 
 [float]
 [[reason-unassigned]]
diff --git a/docs/reference/cat/snapshots.asciidoc b/docs/reference/cat/snapshots.asciidoc
index 5677a0f2a7cd4..37d57292e0488 100644
--- a/docs/reference/cat/snapshots.asciidoc
+++ b/docs/reference/cat/snapshots.asciidoc
@@ -1,7 +1,8 @@
 [[cat-snapshots]]
 == cat snapshots
 
-The `snapshots` command shows all snapshots that belong to a specific repository.
+The `snapshots` command shows all snapshots that belong to a specific repository
+or multiple repositories.
 To find a list of available repositories to query, the command `/_cat/repositories` can be used.
 Querying the snapshots of a repository named `repo1` then looks as follows.
 
@@ -18,17 +19,31 @@ Which looks like:
 
 [source,txt]
 --------------------------------------------------
-id     status start_epoch start_time end_epoch  end_time duration indices successful_shards failed_shards total_shards
-snap1  FAILED 1445616705  18:11:45   1445616978 18:16:18     4.6m       1                 4             1            5
-snap2 SUCCESS 1445634298  23:04:58   1445634672 23:11:12     6.2m       2                10             0           10
+id     repository status start_epoch start_time end_epoch  end_time duration indices successful_shards failed_shards total_shards
+snap1  repo1      FAILED 1445616705  18:11:45   1445616978 18:16:18     4.6m       1                 4             1            5
+snap2  repo1      SUCCESS 1445634298  23:04:58   1445634672 23:11:12     6.2m       2                10             0           10
 --------------------------------------------------
 // TESTRESPONSE[s/FAILED/SUCCESS/ s/14456\d+/\\d+/ s/\d+(\.\d+)?(m|s|ms)/\\d+(\\.\\d+)?(m|s|ms)/]
 // TESTRESPONSE[s/\d+:\d+:\d+/\\d+:\\d+:\\d+/]
 // TESTRESPONSE[s/1                 4             1            5/\\d+ \\d+ \\d+ \\d+/]
 // TESTRESPONSE[s/2                10             0           10/\\d+ \\d+ \\d+ \\d+/]
-// TESTRESPONSE[_cat]
+// TESTRESPONSE[non_json]
 
 Each snapshot contains information about when it was started and stopped.
 Start and stop timestamps are available in two formats.
 The `HH:MM:SS` output is simply for quick human consumption.
 The epoch time retains more information, including date, and is machine sortable if the snapshot process spans days.
+
+It is also possible to get the list of snapshots from multiple repositories.
+Here are some examples:
+
+[source,js]
+--------------------------------------------------
+GET /_cat/snapshots/_all
+GET /_cat/snapshots/repo1,repo2
+GET /_cat/snapshots/repo*
+--------------------------------------------------
+// CONSOLE
+// TEST[skip:no repo2]
+
+Please note that if one of the repositories fails during the request you will get an exception instead of the table.
\ No newline at end of file
diff --git a/docs/reference/cat/templates.asciidoc b/docs/reference/cat/templates.asciidoc
index 304459ba96edb..41d27cfbf0306 100644
--- a/docs/reference/cat/templates.asciidoc
+++ b/docs/reference/cat/templates.asciidoc
@@ -28,7 +28,7 @@ template0 [te*]          0
 template1 [tea*]         1
 template2 [teak*]        2     7
 --------------------------------------------------
-// TESTRESPONSE[s/\*/\\*/ s/\[/\\[/ s/\]/\\]/ _cat]
+// TESTRESPONSE[s/\*/\\*/ s/\[/\\[/ s/\]/\\]/ non_json]
 
 The output shows that there are three existing templates,
 with template2 having a version value.
diff --git a/docs/reference/cat/thread_pool.asciidoc b/docs/reference/cat/thread_pool.asciidoc
index 03854fae2f61f..f63abc8903650 100644
--- a/docs/reference/cat/thread_pool.asciidoc
+++ b/docs/reference/cat/thread_pool.asciidoc
@@ -23,7 +23,7 @@ node-0 flush               0 0 0
 node-0 write               0 0 0
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./(node-0 \\S+ 0 0 0\n)*/]
-// TESTRESPONSE[s/\d+/\\d+/ _cat]
+// TESTRESPONSE[s/\d+/\\d+/ non_json]
 // The substitutions do two things:
 // 1. Expect any number of extra thread pools. This allows us to only list a
 //    few thread pools. The list would be super long otherwise. In addition,
@@ -107,7 +107,7 @@ which looks like:
 id                     name    active rejected completed
 0EWUhXeBQtaVGlexUeVwMg generic      0        0        70
 --------------------------------------------------
-// TESTRESPONSE[s/0EWUhXeBQtaVGlexUeVwMg/[\\w-]+/ s/\d+/\\d+/ _cat]
+// TESTRESPONSE[s/0EWUhXeBQtaVGlexUeVwMg/[\\w-]+/ s/\d+/\\d+/ non_json]
 
 Here the host columns and the active, rejected and completed suggest thread pool statistics are displayed.
 
diff --git a/docs/reference/ccr/apis/follow-request-body.asciidoc b/docs/reference/ccr/apis/follow-request-body.asciidoc
index e7e6ae2e26a05..d8fb725f02b14 100644
--- a/docs/reference/ccr/apis/follow-request-body.asciidoc
+++ b/docs/reference/ccr/apis/follow-request-body.asciidoc
@@ -1,4 +1,3 @@
-[role="xpack"]
 [testenv="platinum"]
 `max_read_request_operation_count`::
   (integer) the maximum number of operations to pull per read from the remote
diff --git a/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc b/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc
index 73bdd9494d1c8..db4bf910a05c6 100644
--- a/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc
+++ b/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc
@@ -114,6 +114,9 @@ The `shards` array consists of objects containing the following fields:
 `indices[].shards[].follower_settings_version`::
   (long) the index settings version the follower is synced up to
 
+`indices[].shards[].follower_aliases_version`::
+  (long) the index aliases version the follower is synced up to
+
 `indices[].shards[].total_read_time_millis`::
   (long) the total time reads were outstanding, measured from the time a read
   was sent to the leader to the time a reply was returned to the follower
@@ -217,6 +220,7 @@ The API returns the following results:
           "write_buffer_operation_count" : 64,
           "follower_mapping_version" : 4,
           "follower_settings_version" : 2,
+          "follower_aliases_version" : 8,
           "total_read_time_millis" : 32768,
           "total_read_remote_exec_time_millis" : 16384,
           "successful_read_requests" : 32,
@@ -246,6 +250,7 @@ The API returns the following results:
 // TESTRESPONSE[s/"write_buffer_operation_count" : 64/"write_buffer_operation_count" : $body.indices.0.shards.0.write_buffer_operation_count/]
 // TESTRESPONSE[s/"follower_mapping_version" : 4/"follower_mapping_version" : $body.indices.0.shards.0.follower_mapping_version/]
 // TESTRESPONSE[s/"follower_settings_version" : 2/"follower_settings_version" : $body.indices.0.shards.0.follower_settings_version/]
+// TESTRESPONSE[s/"follower_aliases_version" : 8/"follower_aliases_version" : $body.indices.0.shards.0.follower_aliases_version/]
 // TESTRESPONSE[s/"total_read_time_millis" : 32768/"total_read_time_millis" : $body.indices.0.shards.0.total_read_time_millis/]
 // TESTRESPONSE[s/"total_read_remote_exec_time_millis" : 16384/"total_read_remote_exec_time_millis" : $body.indices.0.shards.0.total_read_remote_exec_time_millis/]
 // TESTRESPONSE[s/"successful_read_requests" : 32/"successful_read_requests" : $body.indices.0.shards.0.successful_read_requests/]
diff --git a/docs/reference/ccr/apis/get-ccr-stats.asciidoc b/docs/reference/ccr/apis/get-ccr-stats.asciidoc
index 03f2f3eee0c0b..d1acbb0064a9f 100644
--- a/docs/reference/ccr/apis/get-ccr-stats.asciidoc
+++ b/docs/reference/ccr/apis/get-ccr-stats.asciidoc
@@ -126,6 +126,7 @@ The API returns the following results:
             "write_buffer_operation_count" : 64,
             "follower_mapping_version" : 4,
             "follower_settings_version" : 2,
+            "follower_aliases_version" : 8,
             "total_read_time_millis" : 32768,
             "total_read_remote_exec_time_millis" : 16384,
             "successful_read_requests" : 32,
@@ -161,6 +162,7 @@ The API returns the following results:
 // TESTRESPONSE[s/"write_buffer_operation_count" : 64/"write_buffer_operation_count" : $body.follow_stats.indices.0.shards.0.write_buffer_operation_count/]
 // TESTRESPONSE[s/"follower_mapping_version" : 4/"follower_mapping_version" : $body.follow_stats.indices.0.shards.0.follower_mapping_version/]
 // TESTRESPONSE[s/"follower_settings_version" : 2/"follower_settings_version" : $body.follow_stats.indices.0.shards.0.follower_settings_version/]
+// TESTRESPONSE[s/"follower_aliases_version" : 8/"follower_aliases_version" : $body.follow_stats.indices.0.shards.0.follower_aliases_version/]
 // TESTRESPONSE[s/"total_read_time_millis" : 32768/"total_read_time_millis" : $body.follow_stats.indices.0.shards.0.total_read_time_millis/]
 // TESTRESPONSE[s/"total_read_remote_exec_time_millis" : 16384/"total_read_remote_exec_time_millis" : $body.follow_stats.indices.0.shards.0.total_read_remote_exec_time_millis/]
 // TESTRESPONSE[s/"successful_read_requests" : 32/"successful_read_requests" : $body.follow_stats.indices.0.shards.0.successful_read_requests/]
diff --git a/docs/reference/ccr/overview.asciidoc b/docs/reference/ccr/overview.asciidoc
index def19d80729de..be009b4aefaba 100644
--- a/docs/reference/ccr/overview.asciidoc
+++ b/docs/reference/ccr/overview.asciidoc
@@ -112,13 +112,18 @@ governed by settings that you can configure when you create the follower index
 in {kib} or by using the {ref}/ccr-put-follow.html[create follower API].
 
 Mapping updates applied to the leader index are automatically retrieved
-as-needed by the follower index.
+as-needed by the follower index. It is not possible to manually modify the
+mapping of a follower index.
 
 Settings updates applied to the leader index that are needed by the follower
 index are automatically retried as-needed by the follower index. Not all
 settings updates are needed by the follower index. For example, changing the
 number of replicas on the leader index is not replicated by the follower index.
 
+Alias updates applied to the leader index are automatically retrieved by the
+follower index. It is not possible to manually modify an alias of a follower
+index.
+
 NOTE: If you apply a non-dynamic settings change to the leader index that is
 needed by the follower index, the follower index will go through a cycle of
 closing itself, applying the settings update, and then re-opening itself. The
diff --git a/docs/reference/cluster.asciidoc b/docs/reference/cluster.asciidoc
index cfa2d5a6488d7..da5ce2b410281 100644
--- a/docs/reference/cluster.asciidoc
+++ b/docs/reference/cluster.asciidoc
@@ -22,12 +22,14 @@ one of the following:
 * an IP address or hostname, to add all matching nodes to the subset.
 * a pattern, using `*` wildcards, which adds all nodes to the subset
   whose name, address or hostname matches the pattern.
-* `master:true`, `data:true`, `ingest:true` or `coordinating_only:true`, which
-  respectively add to the subset all master-eligible nodes, all data nodes,
-  all ingest nodes, and all coordinating-only nodes.
-* `master:false`, `data:false`, `ingest:false` or `coordinating_only:false`,
-  which respectively remove from the subset all master-eligible nodes, all data
-  nodes, all ingest nodes, and all coordinating-only nodes.
+* `master:true`, `data:true`, `ingest:true`, `voting_only:true` or
+  `coordinating_only:true`, which respectively add to the subset all
+  master-eligible nodes, all data nodes, all ingest nodes, all voting-only
+  nodes, and all coordinating-only nodes.
+* `master:false`, `data:false`, `ingest:false`, `voting_only:true`, or
+  `coordinating_only:false`, which respectively remove from the subset all
+  master-eligible nodes, all data nodes, all ingest nodes, all voting-only
+  nodes and all coordinating-only nodes.
 * a pair of patterns, using `*` wildcards, of the form `attrname:attrvalue`,
   which adds to the subset all nodes with a custom node attribute whose name
   and value match the respective patterns. Custom node attributes are
@@ -46,6 +48,9 @@ means that filters such as `master:false` which remove nodes from the chosen
 subset are only useful if they come after some other filters. When used on its
 own, `master:false` selects no nodes.
 
+NOTE: The `voting_only` role requires the {default-dist} of Elasticsearch and
+is not supported in the {oss-dist}.
+
 Here are some examples of the use of node filters with the
 <<cluster-nodes-info,Nodes Info>> APIs.
 
@@ -69,6 +74,7 @@ GET /_nodes/10.0.0.*
 GET /_nodes/_all,master:false
 GET /_nodes/data:true,ingest:true
 GET /_nodes/coordinating_only:true
+GET /_nodes/master:true,voting_only:false
 # Select nodes by custom attribute (e.g. with something like `node.attr.rack: 2` in the configuration file)
 GET /_nodes/rack:2
 GET /_nodes/ra*:2
diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc
index 791fc2414f331..259d8ae2bba11 100644
--- a/docs/reference/cluster/stats.asciidoc
+++ b/docs/reference/cluster/stats.asciidoc
@@ -109,7 +109,8 @@ Will return, for example:
          "data": 1,
          "coordinating_only": 0,
          "master": 1,
-         "ingest": 1
+         "ingest": 1,
+         "voting_only": 0
       },
       "versions": [
          "{version}"
@@ -207,6 +208,7 @@ Will return, for example:
 // TESTRESPONSE[s/"plugins": \[[^\]]*\]/"plugins": $body.$_path/]
 // TESTRESPONSE[s/"network_types": \{[^\}]*\}/"network_types": $body.$_path/]
 // TESTRESPONSE[s/"discovery_types": \{[^\}]*\}/"discovery_types": $body.$_path/]
+// TESTRESPONSE[s/"count": \{[^\}]*\}/"count": $body.$_path/]
 // TESTRESPONSE[s/"packaging_types": \[[^\]]*\]/"packaging_types": $body.$_path/]
 // TESTRESPONSE[s/: true|false/: $body.$_path/]
 // TESTRESPONSE[s/: (\-)?[0-9]+/: $body.$_path/]
@@ -217,7 +219,10 @@ Will return, for example:
 //    see an exhaustive list anyway.
 // 2. Similarly, ignore the contents of `network_types`, `discovery_types`, and
 //    `packaging_types`.
-// 3. All of the numbers and strings on the right hand side of *every* field in
+// 3. Ignore the contents of the (nodes) count object, as what's shown here
+//    depends on the license. Voting-only nodes are e.g. only shown when this
+//    test runs with a basic license.
+// 4. All of the numbers and strings on the right hand side of *every* field in
 //    the response are ignored. So we're really only asserting things about the
 //    the shape of this response, not the values in it.
 
diff --git a/docs/reference/commands/certutil.asciidoc b/docs/reference/commands/certutil.asciidoc
index 6f4d3224d7aeb..07a3f96738dac 100644
--- a/docs/reference/commands/certutil.asciidoc
+++ b/docs/reference/commands/certutil.asciidoc
@@ -103,7 +103,7 @@ which prompts you for details about each instance. Alternatively, you can use
 the `--in` parameter to specify a YAML file that contains details about the
 instances.
 
-The `cert` mode produces a single zip file which contains the CSRs and the
+The `csr` mode produces a single zip file which contains the CSRs and the
 private keys for each instance. Each CSR is provided as a standard PEM
 encoding of a PKCS#10 CSR. Each key is provided as a PEM encoding of an RSA
 private key.
diff --git a/docs/reference/commands/shard-tool.asciidoc b/docs/reference/commands/shard-tool.asciidoc
index 6fca1355a27be..c13c8d3db6a36 100644
--- a/docs/reference/commands/shard-tool.asciidoc
+++ b/docs/reference/commands/shard-tool.asciidoc
@@ -51,14 +51,14 @@ $ bin/elasticsearch-shard remove-corrupted-data --index twitter --shard-id 0
   Please make a complete backup of your index before using this tool.
 
 
-Opening Lucene index at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
+Opening Lucene index at /var/lib/elasticsearchdata/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
 
- >> Lucene index is corrupted at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
+ >> Lucene index is corrupted at /var/lib/elasticsearchdata/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
 
-Opening translog at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/
+Opening translog at /var/lib/elasticsearchdata/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/
 
 
- >> Translog is clean at /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/
+ >> Translog is clean at /var/lib/elasticsearchdata/indices/P45vf_YQRhqjfwLMUvSqDw/0/translog/
 
 
   Corrupted Lucene index segments found - 32 documents will be lost.
@@ -93,7 +93,7 @@ POST /_cluster/reroute
 
 You must accept the possibility of data loss by changing parameter `accept_data_loss` to `true`.
 
-Deleted corrupt marker corrupted_FzTSBSuxT7i3Tls_TgwEag from /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
+Deleted corrupt marker corrupted_FzTSBSuxT7i3Tls_TgwEag from /var/lib/elasticsearchdata/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/
 
 --------------------------------------------------
 
diff --git a/docs/reference/data-frames/apis/delete-transform.asciidoc b/docs/reference/data-frames/apis/delete-transform.asciidoc
index 9fb3d8558c0ea..d772bc3c15d89 100644
--- a/docs/reference/data-frames/apis/delete-transform.asciidoc
+++ b/docs/reference/data-frames/apis/delete-transform.asciidoc
@@ -8,25 +8,42 @@
 <titleabbrev>Delete {dataframe-transforms}</titleabbrev>
 ++++
 
-Deletes an existing {dataframe-transform}.
+beta[]
 
+Deletes an existing {dataframe-transform}.
 
-==== Request
+[discrete]
+[[delete-data-frame-transform-request]]
+==== {api-request-title}
 
 `DELETE _data_frame/transforms/<data_frame_transform_id>`
 
-//==== Description
+[discrete]
+[[delete-data-frame-transform-prereqs]]
+==== {api-prereq-title}
 
-==== Path Parameters
+* If the {es} {security-features} are enabled, you must have
+`manage_data_frame_transforms` cluster privileges to use this API. The built-in
+`data_frame_transforms_admin` role has these privileges. For more information,
+see {stack-ov}/security-privileges.html[Security privileges] and
+{stack-ov}/built-in-roles.html[Built-in roles].
 
-`data_frame_transform_id` (required)::
-  (string) Identifier for the {dataframe-transform}.
+[discrete]
+[[delete-data-frame-transform-desc]]
+==== {api-description-title}
 
-//===== Authorization
+NOTE: Before you can delete the {dataframe-transform}, you must stop it.
 
-==== Examples
+[discrete]
+[[delete-data-frame-transform-path-parms]]
+==== {api-path-parms-title}
+
+`<data_frame_transform_id>` (Required)::
+  (string) Identifier for the {dataframe-transform}.
 
-The following example deletes the `ecommerce_transform` {dataframe-transform}:
+[discrete]
+[[delete-data-frame-transform-examples]]
+==== {api-examples-title}
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/data-frames/apis/get-transform-stats.asciidoc b/docs/reference/data-frames/apis/get-transform-stats.asciidoc
index 09c383f249488..889a109b8a376 100644
--- a/docs/reference/data-frames/apis/get-transform-stats.asciidoc
+++ b/docs/reference/data-frames/apis/get-transform-stats.asciidoc
@@ -8,10 +8,13 @@
 <titleabbrev>Get {dataframe-transform} statistics</titleabbrev>
 ++++
 
-Retrieves usage information for {dataframe-transforms}.
+beta[]
 
+Retrieves usage information for {dataframe-transforms}.
 
-==== Request
+[discrete]
+[[get-data-frame-transform-stats-request]]
+==== {api-request-title}
 
 
 `GET _data_frame/transforms/<data_frame_transform_id>/_stats`
@@ -24,27 +27,90 @@ Retrieves usage information for {dataframe-transforms}.
 
 `GET _data_frame/transforms/*/_stats` +
 
-//===== Description
+[discrete]
+[[get-data-frame-transform-stats-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have
+`monitor_data_frame_transforms` cluster privileges to use this API. The built-in
+`data_frame_transforms_user` role has these privileges. For more information,
+see {stack-ov}/security-privileges.html[Security privileges] and
+{stack-ov}/built-in-roles.html[Built-in roles].
+
+[discrete]
+[[get-data-frame-transform-stats-desc]]
+==== {api-description-title}
 
-==== Path Parameters
+You can get statistics for multiple {dataframe-transforms} in a single API
+request by using a comma-separated list of identifiers or a wildcard expression.
+You can get statistics for all {dataframe-transforms} by using `_all`, by
+specifying `*` as the `<data_frame_transform_id>`, or by omitting the
+`<data_frame_transform_id>`.
 
-`data_frame_transform_id`::
+[discrete]
+[[get-data-frame-transform-stats-path-parms]]
+==== {api-path-parms-title}
+
+`<data_frame_transform_id>` (Optional)::
   (string) Identifier for the {dataframe-transform}. It can be a
   {dataframe-transform} identifier or a wildcard expression. If you do not
   specify one of these options, the API returns information for all
   {dataframe-transforms}.
-
-
-==== Results
-
-The API returns the following information:
+  
+[discrete]
+[[get-data-frame-transform-stats-query-parms]]
+==== {api-query-parms-title}
+
+`allow_no_match` (Optional)::
+  (boolean) Specifies what to do when the request:
++
+--
+* Contains wildcard expressions and there are no {dataframe-transforms} that match.
+* Contains the `_all` string or no identifiers and there are no matches.
+* Contains wildcard expressions and there are only partial matches. 
+
+The default value is `true`, which returns an empty `transforms` array when
+there are no matches and the subset of results when there are partial matches.
+If this parameter is `false`, the request returns a `404` status code when there
+are no matches or only partial matches.
+--
+
+`from` (Optional)::
+  (integer) Skips the specified number of {dataframe-transforms}. The
+  default value is `0`.
+
+`size` (Optional)::
+  (integer) Specifies the maximum number of {dataframe-transforms} to obtain.
+  The default value is `100`.
+
+[discrete]
+[[get-data-frame-transform-stats-response]]
+==== {api-response-body-title}
 
 `transforms`::
-  (array) An array of statistics objects for {dataframe-transforms}.
+  (array) An array of statistics objects for {dataframe-transforms}, which are
+  sorted by the `id` value in ascending order.
+  
+[[get-data-frame-transform-stats-response-codes]]
+==== {api-response-codes-title}
+
+`404` (Missing resources)::
+  If `allow_no_match` is `false`, this code indicates that there are no
+  resources that match the request or only partial matches for the request. 
+
+[discrete]
+[[get-data-frame-transform-stats-example]]
+==== Examples
 
-//==== Authorization
+The following example skips for the first five {dataframe-transforms} and
+gets usage information for a maximum of ten results: 
 
-==== Examples
+[source,js]
+--------------------------------------------------
+GET _data_frame/transforms/_stats?from=5&size=10
+--------------------------------------------------
+// CONSOLE
+// TEST[skip:todo]
 
 The following example gets usage information for the `ecommerce_transform`
 {dataframe-transform}:
diff --git a/docs/reference/data-frames/apis/get-transform.asciidoc b/docs/reference/data-frames/apis/get-transform.asciidoc
index e2b5c5eccb7da..bf7901c191e33 100644
--- a/docs/reference/data-frames/apis/get-transform.asciidoc
+++ b/docs/reference/data-frames/apis/get-transform.asciidoc
@@ -8,10 +8,13 @@
 <titleabbrev>Get {dataframe-transforms}</titleabbrev>
 ++++
 
-Retrieves configuration information for {dataframe-transforms}.
+beta[]
 
+Retrieves configuration information for {dataframe-transforms}.
 
-==== Request
+[discrete]
+[[get-data-frame-transform-request]]
+==== {api-request-title}
 
 `GET _data_frame/transforms/<data_frame_transform_id>` +
 
@@ -23,39 +26,89 @@ Retrieves configuration information for {dataframe-transforms}.
 
 `GET _data_frame/transforms/*`
 
-//===== Description
+[discrete]
+[[get-data-frame-transform-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have
+`monitor_data_frame_transforms` cluster privileges to use this API. The built-in
+`data_frame_transforms_user` role has these privileges. For more information,
+see {stack-ov}/security-privileges.html[Security privileges] and
+{stack-ov}/built-in-roles.html[Built-in roles].
+
+[discrete]
+[[get-data-frame-transform-desc]]
+==== {api-description-title}
 
-==== Path Parameters
+You can get information for multiple {dataframe-transforms} in a single API
+request by using a comma-separated list of identifiers or a wildcard expression.
+You can get information for all {dataframe-transforms} by using `_all`, by
+specifying `*` as the `<data_frame_transform_id>`, or by omitting the
+`<data_frame_transform_id>`.
 
-`data_frame_transform_id`::
+[discrete]
+[[get-data-frame-transform-path-parms]]
+==== {api-path-parms-title}
+
+`<data_frame_transform_id>` (Optional)::
   (string) Identifier for the {dataframe-transform}. It can be a
   {dataframe-transform} identifier or a wildcard expression. If you do not
   specify one of these options, the API returns information for all
   {dataframe-transforms}.
+  
+[discrete]
+[[get-data-frame-transform-query-parms]]
+==== {api-query-parms-title}
+
+`allow_no_match` (Optional)::
+(boolean) Specifies what to do when the request:
++
+--
+* Contains wildcard expressions and there are no {dataframe-transforms} that match.
+* Contains the `_all` string or no identifiers and there are no matches.
+* Contains wildcard expressions and there are only partial matches. 
+
+The default value is `true`, which returns an empty `transforms` array when
+there are no matches and the subset of results when there are partial matches.
+If this parameter is `false`, the request returns a `404` status code when there
+are no matches or only partial matches.
+--
+
+`from` (Optional)::
+  (integer) Skips the specified number of {dataframe-transforms}. The
+  default value is `0`.
+
+`size` (Optional)::
+  (integer) Specifies the maximum number of {dataframe-transforms} to obtain.
+  The default value is `100`.
+
+[discrete]
+[[get-data-frame-transform-response]]
+==== {api-response-body-title}
 
-////
-==== Request Body
-
-  `page`::
-  `from`:::
-      (integer) Skips the specified number of {dataframe-transforms}. The
-      default value is `0`.
-
-  `size`:::
-      (integer) Specifies the maximum number of {dataframe-transforms} to obtain.
-      The default value is `100`.
-////
-
-==== Results
+`transforms`::
+  (array) An array of transform resources, which are sorted by the `id` value in
+  ascending order.
+  
+[[get-data-frame-transform-response-codes]]
+==== {api-response-codes-title}
 
-The API returns the following information:
+`404` (Missing resources)::
+  If `allow_no_match` is `false`, this code indicates that there are no
+  resources that match the request or only partial matches for the request.  
 
-`transforms`::
-  (array) An array of transform resources.
+[discrete]
+[[get-data-frame-transform-example]]
+==== {api-examples-title}
 
-//==== Authorization
+The following example retrieves information about a maximum of ten transforms:
 
-==== Examples
+[source,js]
+--------------------------------------------------
+GET _data_frame/transforms?size=10
+--------------------------------------------------
+// CONSOLE
+// TEST[skip:setup kibana sample data]
 
 The following example gets configuration information for the
 `ecommerce_transform` {dataframe-transform}:
diff --git a/docs/reference/data-frames/apis/index.asciidoc b/docs/reference/data-frames/apis/index.asciidoc
index a75f44e995158..b3fa17e3c48be 100644
--- a/docs/reference/data-frames/apis/index.asciidoc
+++ b/docs/reference/data-frames/apis/index.asciidoc
@@ -8,11 +8,11 @@
 === {dataframe-transforms-cap}
 
 * <<preview-data-frame-transform,Preview {dataframe-transforms}>>
-* <<put-data-frame-transform,Create {dataframe-transforms}>>,
+* <<put-data-frame-transform,Create>> or 
 <<delete-data-frame-transform,delete {dataframe-transforms}>>
-* <<get-data-frame-transform,Get {dataframe-transform} info>>,
-<<get-data-frame-transform-stats,Get {dataframe-transform} statistics>>
-* <<start-data-frame-transform,Start {dataframe-transforms}>>, <<stop-data-frame-transform,Stop {dataframe-transforms}>>
+* <<get-data-frame-transform,Get {dataframe-transform} info>> or
+<<get-data-frame-transform-stats,statistics>>
+* <<start-data-frame-transform,Start>> or <<stop-data-frame-transform,stop {dataframe-transforms}>>
 
 //CREATE
 include::put-transform.asciidoc[]
diff --git a/docs/reference/data-frames/apis/pivotresource.asciidoc b/docs/reference/data-frames/apis/pivotresource.asciidoc
new file mode 100644
index 0000000000000..64c49af03ea77
--- /dev/null
+++ b/docs/reference/data-frames/apis/pivotresource.asciidoc
@@ -0,0 +1,26 @@
+[role="xpack"]
+[testenv="basic"]
+[[data-frame-transform-pivot]]
+=== Pivot resources
+
+A pivot configuration object has the following properties:
+
+`group_by` (required):: (object) Defines how to group the data. More than one grouping can be defined per pivot. The following groupings are supported:
+* {ref}/search-aggregations-bucket-composite-aggregation.html#_terms[Terms]
+* {ref}/search-aggregations-bucket-composite-aggregation.html#_histogram[Histogram]
+* {ref}/search-aggregations-bucket-composite-aggregation.html#_date_histogram[Date Histogram]
+
+`aggregations` (required):: (object) Defines how to aggregate the grouped data.
+The following aggregations are supported:
+* {ref}/search-aggregations-metrics-avg-aggregation.html[Average]
+* {ref}/search-aggregations-metrics-weight-avg-aggregation.html[Weighted Average]
+* {ref}/search-aggregations-metrics-cardinality-aggregation.html[Cardinality]
+* {ref}/search-aggregations-metrics-geocentroid-aggregation.html[Geo Centroid]
+* {ref}/search-aggregations-metrics-max-aggregation.html[Max]
+* {ref}/search-aggregations-metrics-min-aggregation.html[Min]
+* {ref}/search-aggregations-metrics-scripted-metric-aggregation.html[Scripted Metric]
+* {ref}/search-aggregations-metrics-sum-aggregation.html[Sum]
+* {ref}/search-aggregations-metrics-valuecount-aggregation.html[Value Count]
+* {ref}/search-aggregations-pipeline-bucket-script-aggregation.html[Bucket Script]
+
+//For more information, see {stack-ov}/ml-dataframes.html[dataframes-cap}].
\ No newline at end of file
diff --git a/docs/reference/data-frames/apis/preview-transform.asciidoc b/docs/reference/data-frames/apis/preview-transform.asciidoc
index cf3f3cb96f9a7..a4338d0ef60b3 100644
--- a/docs/reference/data-frames/apis/preview-transform.asciidoc
+++ b/docs/reference/data-frames/apis/preview-transform.asciidoc
@@ -8,29 +8,41 @@
 <titleabbrev>Preview {dataframe-transforms}</titleabbrev>
 ++++
 
-Previews a {dataframe-transform}.
+beta[]
 
+Previews a {dataframe-transform}.
 
-==== Request
+[discrete]
+[[preview-data-frame-transform-request]]
+==== {api-request-title}
 
 `POST _data_frame/transforms/_preview`
 
-//==== Description
-//==== Path Parameters
-//==== Authorization
+[discrete]
+[[preview-data-frame-transform-prereq]]
+==== {api-prereq-title}
 
-==== Request Body
+* If the {es} {security-features} are enabled, you must have
+`manage_data_frame_transforms` cluster privileges to use this API. The built-in
+`data_frame_transforms_admin` role has these privileges. You must also have
+`read` and `view_index_metadata` privileges on the source index for the
+{dataframe-transform}. For more information, see
+{stack-ov}/security-privileges.html[Security privileges] and
+{stack-ov}/built-in-roles.html[Built-in roles].
 
-`source`:: The source index or index pattern.
+[discrete]
+[[preview-data-frame-transform-request-body]]
+==== {api-request-body-title}
 
-`pivot`:: Defines the pivot function `group by` fields and the aggregation to
-reduce the data.
+`source` (Required)::
+  (object) The source index or index pattern.
 
+`pivot` (Required)::
+  (object) Defines the pivot function `group by` fields and the aggregation to
+  reduce the data. See <<data-frame-transform-pivot>>.
 
-==== Examples
-
-The following example obtains a preview of a {dataframe-transform} on the {kib}
-eCommerce sample data:
+[discrete]
+==== {api-examples-title}
 
 [source,js]
 --------------------------------------------------
@@ -78,7 +90,17 @@ The data that is returned for this example is as follows:
       "customer_id" : "12"
     }
     ...
-  ]
+  ],
+  "mappings": {
+    "properties": {
+      "max_price": {
+        "type": "double"
+      },
+      "customer_id": {
+        "type": "keyword"
+      }
+    }
+  }
 }
 ----
 // NOTCONSOLE
diff --git a/docs/reference/data-frames/apis/put-transform.asciidoc b/docs/reference/data-frames/apis/put-transform.asciidoc
index f452c38ab4c94..6910cb85a25f5 100644
--- a/docs/reference/data-frames/apis/put-transform.asciidoc
+++ b/docs/reference/data-frames/apis/put-transform.asciidoc
@@ -8,14 +8,31 @@
 <titleabbrev>Create {dataframe-transforms}</titleabbrev>
 ++++
 
-Instantiates a {dataframe-transform}.
+beta[]
 
+Instantiates a {dataframe-transform}.
 
-==== Request
+[discrete]
+[[put-data-frame-transform-request]]
+==== {api-request-title}
 
 `PUT _data_frame/transforms/<data_frame_transform_id>`
 
-===== Description
+[discrete]
+[[put-data-frame-transform-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have
+`manage_data_frame_transforms` cluster privileges to use this API. The built-in
+`data_frame_transforms_admin` role has these privileges. You must also
+have `read` and `view_index_metadata` privileges on the source index and `read`,
+`create_index`, and `index` privileges on the destination index. For more
+information, see {stack-ov}/security-privileges.html[Security privileges] and
+{stack-ov}/built-in-roles.html[Built-in roles].
+
+[discrete]
+[[put-data-frame-transform-desc]]
+==== {api-description-title}
 
 IMPORTANT:  You must use {kib} or this API to create a {dataframe-transform}.
             Do not put a {dataframe-transform} directly into any
@@ -23,33 +40,37 @@ IMPORTANT:  You must use {kib} or this API to create a {dataframe-transform}.
             If {es} {security-features} are enabled, do not give users any
             privileges on `.data-frame-internal*` indices.
 
-==== Path Parameters
+[discrete]
+[[put-data-frame-transform-path-parms]]
+==== {api-path-parms-title}
 
-`data_frame_transform_id` (required)::
+`<data_frame_transform_id>` (Required)::
   (string) Identifier for the {dataframe-transform}. This identifier can contain
   lowercase alphanumeric characters (a-z and 0-9), hyphens, and underscores. It
   must start and end with alphanumeric characters.
 
+[discrete]
+[[put-data-frame-transform-request-body]]
+==== {api-request-body-title}
 
-==== Request Body
-
-`source` (required):: (object) The source configuration, consisting of `index` and optionally
-a `query`.
-
-`dest` (required):: (object) The destination configuration, consisting of `index`.
-
-`pivot`:: (object) Defines the pivot function `group by` fields and the aggregation to
-reduce the data.
-
-`description`:: Optional free text description of the data frame transform
+`description` (Optional)::
+  (string) Free text description of the {dataframe-transform}.
 
+`dest` (Required)::
+  (object) The destination configuration, which consists of `index` and
+  optionally a `pipeline` id.
 
-//==== Authorization
+`pivot` (Optional)::
+  (object) Defines the pivot function `group by` fields and the aggregation to
+  reduce the data. See <<data-frame-transform-pivot>>.
 
-==== Examples
+`source` (Required)::
+  (object) The source configuration, which consists of `index` and optionally
+  a `query`.
 
-The following example creates a {dataframe-transform} for the {kib} eCommerce
-sample data:
+[discrete]
+[[put-data-frame-transform-example]]
+==== {api-examples-title}
 
 [source,js]
 --------------------------------------------------
@@ -66,7 +87,8 @@ PUT _data_frame/transforms/ecommerce_transform
     }
   },
   "dest": {
-    "index": "kibana_sample_data_ecommerce_transform"
+    "index": "kibana_sample_data_ecommerce_transform",
+    "pipeline": "add_timestamp_pipeline"
   },
   "pivot": {
     "group_by": {
@@ -88,7 +110,7 @@ PUT _data_frame/transforms/ecommerce_transform
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[skip:add sample kibana data]
+// TEST[skip: https://github.com/elastic/elasticsearch/issues/43271]
 
 When the transform is created, you receive the following results:
 [source,js]
@@ -97,4 +119,4 @@ When the transform is created, you receive the following results:
   "acknowledged" : true
 }
 ----
-// NOTCONSOLE
+// TESTRESPONSE
diff --git a/docs/reference/data-frames/apis/start-transform.asciidoc b/docs/reference/data-frames/apis/start-transform.asciidoc
index d200ef34f587d..b76bcb0dd4796 100644
--- a/docs/reference/data-frames/apis/start-transform.asciidoc
+++ b/docs/reference/data-frames/apis/start-transform.asciidoc
@@ -8,27 +8,39 @@
 <titleabbrev>Start {dataframe-transforms}</titleabbrev>
 ++++
 
+beta[]
+
 Starts one or more {dataframe-transforms}.
 
-==== Request
+[discrete]
+[[start-data-frame-transform-request]]
+==== {api-request-title}
 
 `POST _data_frame/transforms/<data_frame_transform_id>/_start`
 
-//==== Description
+[discrete]
+[[start-data-frame-transform-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have
+`manage_data_frame_transforms` cluster privileges to use this API. You must also
+have `view_index_metadata` privileges on the source index for the
+{dataframe-transform}. For more information, see
+{stack-ov}/security-privileges.html[Security privileges] and
+{stack-ov}/built-in-roles.html[Built-in roles].
 
-==== Path Parameters
+[discrete]
+[[start-data-frame-transform-path-parms]]
+==== {api-path-parms-title}
 
-`data_frame_transform_id` (required)::
+`<data_frame_transform_id>` (Required)::
   (string) Identifier for the {dataframe-transform}. This identifier can contain
   lowercase alphanumeric characters (a-z and 0-9), hyphens, and underscores. It
   must start and end with alphanumeric characters.
 
-//==== Request Body
-//==== Authorization
-
-==== Examples
-
-The following example starts the `ecommerce_transform` {dataframe-transform}:
+[discrete]
+[[start-data-frame-transform-example]]
+==== {api-examples-title}
 
 [source,js]
 --------------------------------------------------
@@ -41,7 +53,7 @@ When the {dataframe-transform} starts, you receive the following results:
 [source,js]
 ----
 {
-  "started" : true
+  "acknowledged" : true
 }
 ----
 // TESTRESPONSE
diff --git a/docs/reference/data-frames/apis/stop-transform.asciidoc b/docs/reference/data-frames/apis/stop-transform.asciidoc
index 4e89579b52ec1..80c2654babe0d 100644
--- a/docs/reference/data-frames/apis/stop-transform.asciidoc
+++ b/docs/reference/data-frames/apis/stop-transform.asciidoc
@@ -8,9 +8,13 @@
 <titleabbrev>Stop {dataframe-transforms}</titleabbrev>
 ++++
 
+beta[]
+
 Stops one or more {dataframe-transforms}.
 
-==== Request
+[discrete]
+[[stop-data-frame-transform-request]]
+==== {api-request-title}
 
 `POST _data_frame/transforms/<data_frame_transform_id>/_stop` +
 
@@ -18,37 +22,81 @@ Stops one or more {dataframe-transforms}.
 
 `POST _data_frame/transforms/_all/_stop`
 
+[discrete]
+[[stop-data-frame-transform-prereq]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have
+`manage_data_frame_transforms` cluster privileges to use this API. The built-in
+`data_frame_transforms_admin` role has these privileges. For more information,
+see {stack-ov}/security-privileges.html[Security privileges] and
+{stack-ov}/built-in-roles.html[Built-in roles].
+
+[discrete]
+[[stop-data-frame-transform-desc]]
+==== {api-description-title}
 
-==== Description
 You can stop multiple {dataframe-transforms} in a single API request by using a
 comma-separated list of {dataframe-transforms} or a wildcard expression.
-All {dataframe-transforms} can be stopped by using `_all` or `*` as the `<data_frame_transform_id>`.
+All {dataframe-transforms} can be stopped by using `_all` or `*` as the
+`<data_frame_transform_id>`.
 
-==== Path Parameters
+[discrete]
+[[stop-data-frame-transform-path-parms]]
+==== {api-path-parms-title}
 
-`data_frame_transform_id` (required)::
+`<data_frame_transform_id>` (Required)::
   (string) Identifier for the {dataframe-transform}. This identifier can contain
   lowercase alphanumeric characters (a-z and 0-9), hyphens, and underscores. It
   must start and end with alphanumeric characters.
 
-==== Query Parameters
-
-`wait_for_completion`::
-  (boolean) If set to true, causes the API to block until the indexer state completely stops. If set to false, the API returns immediately and the indexer will be stopped asynchronously in the background. Defaults to `false`.
-
- `timeout`::
-   (time value) If `wait_for_completion=true`, the API blocks for (at maximum)
-   the specified duration while waiting for the job to stop. If more than
-   `timeout` time has passed, the API throws a timeout exception. Even if a
-   timeout exception is thrown, the stop request is still processing and
-   eventually moves the job to `STOPPED`. The timeout simply means the API call itself timed out while waiting for the status change. Defaults to `30s`
-    
-//==== Request Body
-//==== Authorization
-
-==== Examples
-
-The following example stops the `ecommerce_transform` {dataframe-transform}:
+[discrete]
+[[stop-data-frame-transform-query-parms]]
+==== {api-query-parms-title}
+
+`allow_no_match` (Optional)::
+(boolean) Specifies what to do when the request:
++
+--
+* Contains wildcard expressions and there are no {dataframe-transforms} that match.
+* Contains the `_all` string or no identifiers and there are no matches.
+* Contains wildcard expressions and there are only partial matches. 
+
+The default value is `true`, which returns a successful acknowledgement message
+when there are no matches. When there are only partial matches, the API stops
+the appropriate {dataframe-transforms}. For example, if the request contains
+`test-id1*,test-id2*` as the identifiers and there are no {dataframe-transforms}
+that match `test-id2*`, the API nonetheless stops the {dataframe-transforms}
+that match `test-id1*`.
+
+If this parameter is `false`, the request returns a `404` status code when there
+are no matches or only partial matches.
+--
+
+`timeout` (Optional)::
+  (time value) If `wait_for_completion=true`, the API blocks for (at maximum)
+  the specified duration while waiting for the transform to stop. If more than
+  `timeout` time has passed, the API throws a timeout exception. Even if a
+  timeout exception is thrown, the stop request is still processing and
+  eventually moves the transform to `STOPPED`. The timeout simply means the API
+  call itself timed out while waiting for the status change. Defaults to `30s`
+
+`wait_for_completion` (Optional)::
+  (boolean) If set to `true`, causes the API to block until the indexer state
+  completely stops. If set to `false`, the API returns immediately and the
+  indexer will be stopped asynchronously in the background. Defaults to `false`.
+
+[discrete]
+[[stop-data-frame-transform-response-codes]]
+==== {api-response-codes-title}
+
+`404` (Missing resources)::
+  If `allow_no_match` is `false`, this code indicates that there are no
+  resources that match the request or only partial matches for the request. 
+
+[discrete]
+[[stop-data-frame-transform-example]]
+==== {api-examples-title}
 
 [source,js]
 --------------------------------------------------
@@ -61,7 +109,7 @@ When the {dataframe-transform} stops, you receive the following results:
 [source,js]
 ----
 {
-  "stopped" : true
+  "acknowledged" : true
 }
 ----
 // TESTRESPONSE
diff --git a/docs/reference/docs/data-replication.asciidoc b/docs/reference/docs/data-replication.asciidoc
index b83858cecfd8d..28389fb05ba94 100644
--- a/docs/reference/docs/data-replication.asciidoc
+++ b/docs/reference/docs/data-replication.asciidoc
@@ -5,12 +5,12 @@
 [float]
 === Introduction
 
-Each index in Elasticsearch is <<getting-started-shards-and-replicas,divided into shards>>
+Each index in Elasticsearch is <<scalability,divided into shards>>
 and each shard can have multiple copies. These copies are known as a _replication group_ and must be kept in sync when documents
 are added or removed. If we fail to do so, reading from one copy will result in very different results than reading from another.
 The process of keeping the shard copies in sync and serving reads from them is what we call the _data replication model_.
 
-Elasticsearch’s data replication model is based on the _primary-backup model_ and is described very well in the 
+Elasticsearch’s data replication model is based on the _primary-backup model_ and is described very well in the
 https://www.microsoft.com/en-us/research/publication/pacifica-replication-in-log-based-distributed-storage-systems/[PacificA paper] of
 Microsoft Research. That model is based on having a single copy from the replication group that acts as the primary shard.
 The other copies are called _replica shards_. The primary serves as the main entry point for all indexing operations. It is in charge of
@@ -23,7 +23,7 @@ it has for various interactions between write and read operations.
 [float]
 === Basic write model
 
-Every indexing operation in Elasticsearch is first resolved to a replication group using <<index-routing,routing>>, 
+Every indexing operation in Elasticsearch is first resolved to a replication group using <<index-routing,routing>>,
 typically based on the document ID. Once the replication group has been determined,
 the operation is forwarded internally to the current _primary shard_ of the group. The primary shard is responsible
 for validating the operation and forwarding it to the other replicas. Since replicas can be offline, the primary
@@ -50,7 +50,7 @@ configuration mistake could cause an operation to fail on a replica despite it b
 are infrequent but the primary has to respond to them.
 
 In the case that the primary itself fails, the node hosting the primary will send a message to the master about it. The indexing
-operation will wait (up to 1 minute, by <<dynamic-index-settings,default>>) for the master to promote one of the replicas to be a 
+operation will wait (up to 1 minute, by <<dynamic-index-settings,default>>) for the master to promote one of the replicas to be a
 new primary. The operation will then be forwarded to the new primary for processing. Note that the master also monitors the
 health of the nodes and may decide to proactively demote a primary. This typically happens when the node holding the primary
 is isolated from the cluster by a networking issue. See <<demoted-primary,here>> for more details.
@@ -60,8 +60,8 @@ when executing it on the replica shards. This may be caused by an actual failure
 issue preventing the operation from reaching the replica (or preventing the replica from responding). All of these
 share the same end result: a replica which is part of the in-sync replica set misses an operation that is about to
 be acknowledged. In order to avoid violating the invariant, the primary sends a message to the master requesting
-that the problematic shard be removed from the in-sync replica set. Only once removal of the shard has been acknowledged 
-by the master does the primary acknowledge the operation. Note that the master will also instruct another node to start 
+that the problematic shard be removed from the in-sync replica set. Only once removal of the shard has been acknowledged
+by the master does the primary acknowledge the operation. Note that the master will also instruct another node to start
 building a new shard copy in order to restore the system to a healthy state.
 
 [[demoted-primary]]
@@ -72,13 +72,13 @@ will be rejected by the replicas. When the primary receives a response from the
 it is no longer the primary then it will reach out to the master and will learn that it has been replaced. The
 operation is then routed to the new primary.
 
-.What happens if there are no replicas? 
+.What happens if there are no replicas?
 ************
 This is a valid scenario that can happen due to index configuration or simply
 because all the replicas have failed. In that case the primary is processing operations without any external validation,
 which may seem problematic. On the other hand, the primary cannot fail other shards on its own but request the master to do
-so on its behalf. This means that the master knows that the primary is the only single good copy. We are therefore guaranteed 
-that the master will not promote any other (out-of-date) shard copy to be a  new primary and that any operation indexed 
+so on its behalf. This means that the master knows that the primary is the only single good copy. We are therefore guaranteed
+that the master will not promote any other (out-of-date) shard copy to be a  new primary and that any operation indexed
 into the primary will not be lost. Of course, since at that point we are running with only single copy of the data, physical hardware
 issues can cause data loss. See <<index-wait-for-active-shards>> for some mitigation options.
 ************
@@ -91,7 +91,7 @@ take non-trivial CPU power. One of the beauties of the primary-backup model is t
 (with the exception of in-flight operations). As such, a single in-sync copy is sufficient to serve read requests.
 
 When a read request is received by a node, that node is responsible for forwarding it to the nodes that hold the relevant shards,
-collating the responses, and responding to the client. We call that node the _coordinating node_ for that request. The basic flow 
+collating the responses, and responding to the client. We call that node the _coordinating node_ for that request. The basic flow
 is as follows:
 
 . Resolve the read requests to the relevant shards. Note that since most searches will be sent to one or more indices,
@@ -153,8 +153,8 @@ Dirty reads:: An isolated primary can expose writes that will not be acknowledge
 [float]
 === The Tip of the Iceberg
 
-This document provides a high level overview of how Elasticsearch deals with data. Of course, there is much much more 
-going on under the hood. Things like primary terms, cluster state publishing, and master election all play a role in 
+This document provides a high level overview of how Elasticsearch deals with data. Of course, there is much much more
+going on under the hood. Things like primary terms, cluster state publishing, and master election all play a role in
 keeping this system behaving correctly. This document also doesn't cover known and important
 bugs (both closed and open). We recognize that https://github.com/elastic/elasticsearch/issues?q=label%3Aresiliency[GitHub is hard to keep up with].
 To help people stay on top of those, we maintain a dedicated https://www.elastic.co/guide/en/elasticsearch/resiliency/current/index.html[resiliency page]
diff --git a/docs/reference/docs/delete-by-query.asciidoc b/docs/reference/docs/delete-by-query.asciidoc
index f8cb84ab790fa..4dbdd18ca02d3 100644
--- a/docs/reference/docs/delete-by-query.asciidoc
+++ b/docs/reference/docs/delete-by-query.asciidoc
@@ -571,11 +571,11 @@ sub-request proportionally.
 * Due to the nature of `slices` each sub-request won't get a perfectly even
 portion of the documents. All documents will be addressed, but some slices may
 be larger than others. Expect larger slices to have a more even distribution.
-* Parameters like `requests_per_second` and `size` on a request with `slices`
-are distributed proportionally to each sub-request. Combine that with the point
-above about distribution being uneven and you should conclude that the using
-`size` with `slices` might not result in exactly `size` documents being
-deleted.
+* Parameters like `requests_per_second` and `max_docs` on a request with
+slices` are distributed proportionally to each sub-request. Combine that with
+the point above about distribution being uneven and you should conclude that
+using `max_docs` with `slices` might not result in exactly `max_docs` documents
+being deleted.
 * Each sub-request gets a slightly different snapshot of the source index
 though these are all taken at approximately the same time.
 
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index e96c262d67bb4..7717506a2d23e 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -190,14 +190,14 @@ not a good idea to rely on this behavior. Instead, make sure that IDs are unique
 using a script.
 
 It's also possible to limit the number of processed documents by setting
-`size`. This will only copy a single document from `twitter` to
+`max_docs`. This will only copy a single document from `twitter` to
 `new_twitter`:
 
 [source,js]
 --------------------------------------------------
 POST _reindex
 {
-  "size": 1,
+  "max_docs": 1,
   "source": {
     "index": "twitter"
   },
@@ -211,14 +211,14 @@ POST _reindex
 
 If you want a particular set of documents from the `twitter` index you'll
 need to use `sort`. Sorting makes the scroll less efficient but in some contexts
-it's worth it. If possible, prefer a more selective query to `size` and `sort`.
+it's worth it. If possible, prefer a more selective query to `max_docs` and `sort`.
 This will copy 10000 documents from `twitter` into `new_twitter`:
 
 [source,js]
 --------------------------------------------------
 POST _reindex
 {
-  "size": 10000,
+  "max_docs": 10000,
   "source": {
     "index": "twitter",
     "sort": { "date": "desc" }
@@ -1115,11 +1115,11 @@ sub-request proportionally.
 * Due to the nature of `slices` each sub-request won't get a perfectly even
 portion of the documents. All documents will be addressed, but some slices may
 be larger than others. Expect larger slices to have a more even distribution.
-* Parameters like `requests_per_second` and `size` on a request with `slices`
-are distributed proportionally to each sub-request. Combine that with the point
-above about distribution being uneven and you should conclude that the using
-`size` with `slices` might not result in exactly `size` documents being
-reindexed.
+* Parameters like `requests_per_second` and `max_docs` on a request with
+`slices` are distributed proportionally to each sub-request. Combine that with
+the point above about distribution being uneven and you should conclude that
+using `max_docs` with `slices` might not result in exactly `max_docs` documents
+being reindexed.
 * Each sub-request gets a slightly different snapshot of the source index,
 though these are all taken at approximately the same time.
 
@@ -1236,7 +1236,7 @@ to load only the existing data into the new index and rename any fields if neede
 ----------------------------------------------------------------
 POST _reindex
 {
-  "size": 10,
+  "max_docs": 10,
   "source": {
     "index": "twitter",
     "query": {
diff --git a/docs/reference/docs/termvectors.asciidoc b/docs/reference/docs/termvectors.asciidoc
index 1e2748916005c..8ec4019fe8861 100644
--- a/docs/reference/docs/termvectors.asciidoc
+++ b/docs/reference/docs/termvectors.asciidoc
@@ -207,7 +207,6 @@ Response:
 {
     "_id": "1",
     "_index": "twitter",
-    "_type": "_doc",
     "_version": 1,
     "found": true,
     "took": 6,
@@ -344,7 +343,6 @@ Response:
 --------------------------------------------------
 {
   "_index": "twitter",
-  "_type": "_doc",
   "_version": 0,
   "found": true,
   "took": 6,
@@ -415,7 +413,6 @@ Response:
 --------------------------------------------------
 {
    "_index": "imdb",
-   "_type": "_doc",
    "_version": 0,
    "found": true,
    "term_vectors": {
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index 883f6ad2a29e3..163ce17194268 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -602,11 +602,11 @@ sub-request proportionally.
 * Due to the nature of `slices` each sub-request won't get a perfectly even
 portion of the documents. All documents will be addressed, but some slices may
 be larger than others. Expect larger slices to have a more even distribution.
-* Parameters like `requests_per_second` and `size` on a request with `slices`
-are distributed proportionally to each sub-request. Combine that with the point
-above about distribution being uneven and you should conclude that the using
-`size` with `slices` might not result in exactly `size` documents being
-updated.
+* Parameters like `requests_per_second` and `max_docs` on a request with
+`slices` are distributed proportionally to each sub-request. Combine that with
+the point above about distribution being uneven and you should conclude that
+using `max_docs` with `slices` might not result in exactly `max_docs` documents
+being updated.
 * Each sub-request gets a slightly different snapshot of the source index
 though these are all taken at approximately the same time.
 
diff --git a/docs/reference/frozen-indices.asciidoc b/docs/reference/frozen-indices.asciidoc
index b37c3426b1701..60f07b58bcb7d 100644
--- a/docs/reference/frozen-indices.asciidoc
+++ b/docs/reference/frozen-indices.asciidoc
@@ -118,5 +118,5 @@ The response looks like:
 i         sth
 twitter   true
 --------------------------------------------------
-// TESTRESPONSE[_cat]
+// TESTRESPONSE[non_json]
 
diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc
index 7df9bdfe7aa6c..77b380dd1dfed 100755
--- a/docs/reference/getting-started.asciidoc
+++ b/docs/reference/getting-started.asciidoc
@@ -1,106 +1,33 @@
 [[getting-started]]
-= Getting started
+= Getting started with {es}
 
 [partintro]
 --
+Ready to take {es} for a test drive and see for yourself how you can use the
+REST APIs to store, search, and analyze data?
 
-Elasticsearch is a highly scalable open-source full-text search and analytics engine. It allows you to store, search, and analyze big volumes of data quickly and in near real time. It is generally used as the underlying engine/technology that powers applications that have complex search features and requirements.
+Follow this getting started tutorial to:
 
-Here are a few sample use-cases that Elasticsearch could be used for:
+. Get an {es} instance up and running
+. Index some sample documents
+. Search for documents using the {es} query language
+. Analyze the results using bucket and metrics aggregations
 
-* You run an online web store where you allow your customers to search for products that you sell. In this case, you can use Elasticsearch to store your entire product catalog and inventory and provide search and autocomplete suggestions for them.
-* You want to collect log or transaction data and you want to analyze and mine this data to look for trends, statistics, summarizations, or anomalies. In this case, you can use Logstash (part of the Elasticsearch/Logstash/Kibana stack) to collect, aggregate, and parse your data, and then have Logstash feed this data into Elasticsearch. Once the data is in Elasticsearch, you can run searches and aggregations to mine any information that is of interest to you.
-* You run a price alerting platform which allows price-savvy customers to specify a rule like "I am interested in buying a specific electronic gadget and I want to be notified if the price of gadget falls below $X from any vendor within the next month". In this case you can scrape vendor prices, push them into Elasticsearch and use its reverse-search (Percolator) capability to match price movements against customer queries and eventually push the alerts out to the customer once matches are found.
-* You have analytics/business-intelligence needs and want to quickly investigate, analyze, visualize, and ask ad-hoc questions on a lot of data (think millions or billions of records). In this case, you can use Elasticsearch to store your data and then use Kibana (part of the Elasticsearch/Logstash/Kibana stack) to build custom dashboards that can visualize aspects of your data that are important to you. Additionally, you can use the Elasticsearch aggregations functionality to perform complex business intelligence queries against your data.
 
-For the rest of this tutorial, you will be guided through the process of getting Elasticsearch up and running, taking a peek inside it, and performing basic operations like indexing, searching, and modifying your data. At the end of this tutorial, you should have a good idea of what Elasticsearch is, how it works, and hopefully be inspired to see how you can use it to either build sophisticated search applications or to mine intelligence from your data.
---
-
-[[getting-started-concepts]]
-== Basic Concepts
-
-There are a few concepts that are core to Elasticsearch. Understanding these concepts from the outset will tremendously help ease the learning process.
-
-[float]
-=== Near Realtime (NRT)
-
-Elasticsearch is a near-realtime search platform. What this means is there is a slight latency (normally one second) from the time you index a document until the time it becomes searchable.
-
-[float]
-=== Cluster
-
-A cluster is a collection of one or more nodes (servers) that together holds your entire data and provides federated indexing and search capabilities across all nodes. A cluster is identified by a unique name which by default is "elasticsearch". This name is important because a node can only be part of a cluster if the node is set up to join the cluster by its name.
-
-Make sure that you don't reuse the same cluster names in different
-environments, otherwise you might end up with nodes joining the wrong cluster.
-For instance you could use `logging-dev`, `logging-stage`, and `logging-prod`
-for the development, staging, and production clusters.
-
-Note that it is valid and perfectly fine to have a cluster with only a single node in it. Furthermore, you may also have multiple independent clusters each with its own unique cluster name.
-
-[float]
-=== Node
-
-A node is a single server that is part of your cluster, stores your data, and participates in the cluster's indexing and search
-capabilities. Just like a cluster, a node is identified by a name which by default is a random Universally Unique IDentifier (UUID) that is assigned to the node at startup. You can define any node name you want if you do not want the default.  This name is important for administration purposes where you want to identify which servers in your network correspond to which nodes in your Elasticsearch cluster.
-
-A node can be configured to join a specific cluster by the cluster name. By default, each node is set up to join a cluster named `elasticsearch` which means that if you start up a number of nodes on your network and--assuming they can discover each other--they will all automatically form and join a single cluster named `elasticsearch`.
-
-In a single cluster, you can have as many nodes as you want. Furthermore, if there are no other Elasticsearch nodes currently running on your network, starting a single node will by default form a new single-node cluster named `elasticsearch`.
-
-[float]
-=== Index
-
-An index is a collection of documents that have somewhat similar characteristics. For example, you can have an index for customer data, another index for a product catalog, and yet another index for order data. An index is identified by a name (that must be all lowercase) and this name is used to refer to the index when performing indexing, search, update, and delete operations against the documents in it.
-
-In a single cluster, you can define as many indexes as you want.
-
-[float]
-=== Type
+Need more context?
 
-deprecated[6.0.0,See <<removal-of-types>>]
+Check out the <<elasticsearch-intro,
+Elasticsearch Introduction>> to learn the lingo and understand the basics of
+how {es} works. If you're already familiar with {es} and want to see how it works
+with the rest of the stack, you might want to jump to the
+{stack-gs}/get-started-elastic-stack.html[Elastic Stack
+Tutorial] to see how to set up a system monitoring solution with {es}, {kib},
+{beats},  and {ls}.
 
-A type used to be a logical category/partition of your index to allow you to store different types of documents in the same index, e.g. one type for users, another type for blog posts.  It is no longer possible to create multiple types in an index, and the whole concept of types will be removed in a later version.  See <<removal-of-types>> for more.
-
-[float]
-=== Document
-
-A document is a basic unit of information that can be indexed. For example, you can have a document for a single customer, another document for a single product, and yet another for a single order. This document is expressed in http://json.org/[JSON] (JavaScript Object Notation) which is a ubiquitous internet data interchange format. Within an index, you can store as many documents as you want.
-
-[[getting-started-shards-and-replicas]]
-[float]
-=== Shards & Replicas
-
-An index can potentially store a large amount of data that can exceed the hardware limits of a single node. For example, a single index of a billion documents taking up 1TB of disk space may not fit on the disk of a single node or may be too slow to serve search requests from a single node alone.
-
-To solve this problem, Elasticsearch provides the ability to subdivide your index into multiple pieces called shards. When you create an index, you can simply define the number of shards that you want. Each shard is in itself a fully-functional and independent "index" that can be hosted on any node in the cluster.
-
-Sharding is important for two primary reasons:
-
-* It allows you to horizontally split/scale your content volume
-* It allows you to distribute and parallelize operations across shards (potentially on multiple nodes) thus increasing performance/throughput
-
-
-The mechanics of how a shard is distributed and also how its documents are aggregated back into search requests are completely managed by Elasticsearch and is transparent to you as the user.
-
-In a network/cloud environment where failures can be expected anytime, it is very useful and highly recommended to have a failover mechanism in case a shard/node somehow goes offline or disappears for whatever reason. To this end, Elasticsearch allows you to make one or more copies of your index's shards into what are called replica shards, or replicas for short.
-
-Replication is important for two primary reasons:
-
-* It provides high availability in case a shard/node fails. For this reason, it is important to note that a replica shard is never allocated on the same node as the original/primary shard that it was copied from.
-* It allows you to scale out your search volume/throughput since searches can be executed on all replicas in parallel.
-
-
-To summarize, each index can be split into multiple shards. An index can also be replicated zero (meaning no replicas) or more times. Once replicated, each index will have primary shards (the original shards that were replicated from) and replica shards (the copies of the primary shards).
-
-The number of shards and replicas can be defined per index at the time the index is created. After the index is created, you may also change the number of replicas dynamically anytime. You can change the number of shards for an existing index using the {ref}/indices-shrink-index.html[`_shrink`] and {ref}/indices-split-index.html[`_split`] APIs, however this is not a trivial task and pre-planning for the correct number of shards is the optimal approach.
-
-By default, each index in Elasticsearch is allocated one primary shard and one replica which means that if you have at least two nodes in your cluster, your index will have one primary shard and another replica shard (one complete replica) for a total of two shards per index.
-
-NOTE: Each Elasticsearch shard is a Lucene index.  There is a maximum number of documents you can have in a single Lucene index.  As of https://issues.apache.org/jira/browse/LUCENE-5843[`LUCENE-5843`], the limit is `2,147,483,519` (= Integer.MAX_VALUE - 128) documents.
-You can monitor shard sizes using the {ref}/cat-shards.html[`_cat/shards`] API.
-
-With that out of the way, let's get started with the fun part...
+TIP: The fastest way to get started with {es} is to
+https://www.elastic.co/cloud/elasticsearch-service/signup[start a free 14-day
+trial of Elasticsearch Service] in the cloud.
+--
 
 [[getting-started-install]]
 == Installation
@@ -118,10 +45,11 @@ NOTE: Elasticsearch includes a bundled version of http://openjdk.java.net[OpenJD
 from the JDK maintainers (GPLv2+CE). To use your own version of Java,
 see the <<jvm-version, JVM version requirements>>
 
-The binaries are available from http://www.elastic.co/downloads[`www.elastic.co/downloads`]
-along with all the releases that have been made in the past. For each release, platform
-dependent archive versions are available for Windows, Linux and MacOS, as well as `DEB` and `RPM`
-packages for Linux, and `MSI` installation packages for Windows.
+The binaries are available from http://www.elastic.co/downloads[`www.elastic.co/downloads`].
+Platform dependent archives are available for Windows, Linux and macOS. In addition,
+`DEB` and `RPM` packages are available for Linux, and an `MSI` installation package
+is available for Windows. You can also use the Elastic Homebrew tap to <<brew,install
+using the brew package manager>> on macOS.
 
 [float]
 === Installation example on Linux
@@ -303,7 +231,7 @@ epoch      timestamp cluster       status node.total node.data shards pri relo i
 --------------------------------------------------
 // TESTRESPONSE[s/1475247709 17:01:49  elasticsearch/\\d+ \\d+:\\d+:\\d+ integTest/]
 // TESTRESPONSE[s/0             0                  -/0             \\d+                  -/]
-// TESTRESPONSE[_cat]
+// TESTRESPONSE[non_json]
 
 We can see that our cluster named "elasticsearch" is up with a green status.
 
@@ -330,10 +258,10 @@ And the response:
 [source,txt]
 --------------------------------------------------
 ip        heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
-127.0.0.1           10           5   5    4.46                        mdi      *      PB2SGZY
+127.0.0.1           10           5   5    4.46                        dim      *      PB2SGZY
 --------------------------------------------------
 // TESTRESPONSE[s/10           5   5    4.46/\\d+ \\d+ \\d+ (\\d+\\.\\d+)? (\\d+\\.\\d+)? (\\d+\.\\d+)?/]
-// TESTRESPONSE[s/[*]/[*]/ s/PB2SGZY/.+/ _cat]
+// TESTRESPONSE[s/[*]/[*]/ s/PB2SGZY/.+/ non_json]
 
 Here, we can see our one node named "PB2SGZY", which is the single node that is currently in our cluster.
 
@@ -354,7 +282,7 @@ And the response:
 --------------------------------------------------
 health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
 --------------------------------------------------
-// TESTRESPONSE[_cat]
+// TESTRESPONSE[non_json]
 
 Which simply means we have no indices yet in the cluster.
 
@@ -379,7 +307,7 @@ And the response:
 health status index    uuid                   pri rep docs.count docs.deleted store.size pri.store.size
 yellow open   customer 95SQ4TSUT7mWBT7VNHH67A   1   1          0            0       260b           260b
 --------------------------------------------------
-// TESTRESPONSE[s/95SQ4TSUT7mWBT7VNHH67A/.+/ s/260b/\\d+\\.?\\d?k?b/ _cat]
+// TESTRESPONSE[s/95SQ4TSUT7mWBT7VNHH67A/.+/ s/260b/\\d+\\.?\\d?k?b/ non_json]
 
 The results of the second command tells us that we now have one index named customer and it has one primary shard and one replica (the defaults) and it contains zero documents in it.
 
@@ -471,7 +399,7 @@ And the response:
 --------------------------------------------------
 health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
 --------------------------------------------------
-// TESTRESPONSE[_cat]
+// TESTRESPONSE[non_json]
 
 Which means that the index was deleted successfully and we are now back to where we started with nothing in our cluster.
 
@@ -715,7 +643,7 @@ health status index uuid                   pri rep docs.count docs.deleted store
 yellow open   bank  l7sSYV2cQXmu6_4rJWVIww   5   1       1000            0    128.6kb        128.6kb
 --------------------------------------------------
 // TESTRESPONSE[s/128.6kb/\\d+(\\.\\d+)?[mk]?b/]
-// TESTRESPONSE[s/l7sSYV2cQXmu6_4rJWVIww/.+/ _cat]
+// TESTRESPONSE[s/l7sSYV2cQXmu6_4rJWVIww/.+/ non_json]
 
 Which means that we just successfully bulk indexed 1000 documents into the bank index.
 
diff --git a/docs/reference/how-to/recipes/stemming.asciidoc b/docs/reference/how-to/recipes/stemming.asciidoc
index d7ddda116327e..d435ce1c94d4b 100644
--- a/docs/reference/how-to/recipes/stemming.asciidoc
+++ b/docs/reference/how-to/recipes/stemming.asciidoc
@@ -228,3 +228,6 @@ In the above case, since `ski` was in-between quotes, it was searched on the
 `body.exact` field due to the `quote_field_suffix` parameter, so only document
 `1` matched. This allows users to mix exact search with stemmed search as they
 like.
+
+NOTE: If the choice of field passed in `quote_field_suffix` does not exist 
+the search will fall back to using the default field for the query string.
diff --git a/docs/reference/ilm/policy-definitions.asciidoc b/docs/reference/ilm/policy-definitions.asciidoc
index 00578ce8c050f..4949c43e6ce65 100644
--- a/docs/reference/ilm/policy-definitions.asciidoc
+++ b/docs/reference/ilm/policy-definitions.asciidoc
@@ -120,9 +120,9 @@ configuring allocation rules is optional. When configuring allocation rules,
 setting number of replicas is optional. Although this action can be treated as
 two separate index settings updates, both can be configured at once.
 
-Read more about index replicas <<getting-started-shards-and-replicas,here>>.
-Read more about shard allocation filtering in
-the <<shard-allocation-filtering,Shard allocation filtering documentation>>.
+For more information about how {es} uses replicas for scaling, see
+<<scalability>>. See <<shard-allocation-filtering>> for more information about
+controlling where Elasticsearch allocates shards of a particular index.
 
 [[ilm-allocate-options]]
 .Allocate Options
diff --git a/docs/reference/images/rare_terms/accuracy_0001.png b/docs/reference/images/rare_terms/accuracy_0001.png
new file mode 100644
index 0000000000000000000000000000000000000000..0c13a3938cde2e2ca119f098aabb1ee8c4d1ce8c
GIT binary patch
literal 25315
zcmd432T)Y)wk_JAD7KOWL_wk?K|zrQ$q0xjIcE?RB!lFfBr6#~o1AmE<P3@;B9cRs
z5s)03jD)7&w{Y)!_P+O=`)<8k_3Bvl7k@+dTHpG@oMVnLW{0ULJtBeA!eKBN$zwSg
zH5lxS3=BpvM0^hXMw|1oGWc=9UQX8$1|$6f{`*X@tNUXf27C7IvCIPvx8${n3&)zy
zDfmsYqT)=ZvQXkcrZ;aX$RCuBvD~>!NEkaq_)|qP(ER-mOaEZM-s}<LA*$B_XV}Z)
zb*z=b1<r<s`UhWy*QBqm_)Z$*7ce-(^`<FDqwcm@j!DNZvHe^7uD(<IINP~y9OD@<
z6h9bO8W#E=n4cFjIRW@de2b6*dijavzt78Gj@=UYhQVUp{?RI%B&;-nUw||d;V<s(
z`OfmeBJD&ytz3m`MVdtk2)oH~+h&|g{Z5~}qr)x7g)V2JUbh5Z<J8w#?U505f}?M*
z-CJEQLc8EIE2P}Xj>npDN!|w?##J1tZY)LUGqBcB$+MrdPqq@9hr_A4SI4GKT-wXP
zbw*BbM>g|6KkR*BJn8W{8k^FKVh}6yxkY2_Rf;h0N&mfhxVy5pi!rW_x$(#hSLryH
z5yDWGXynnZRibB_^(d;UgQx!1REUPk)+R&G(a|G`!x-F>vM-lzmC2WL^}hlxmhB;7
zDnE&3U|xCFBV~L2X(u}kbMS|jd^;0ZA54|)gr4QZQh(kWZhy*^XqZd0P-bG^6WjQb
z#?UE7vrwbp#YlmA-uRZ~U}5coXD|ufypQN%!bybS<i|&h-u%laS!>)ev(aBd$i_I*
zyq-C)3^`*SC?(%FVf5N~P+*!7$slH#=5ttJ=+Mrlg)*qkM_*gV`r_-wj#kU}zTb!%
zrgt4TiIO;}QAG0JoT(BHK69Q_Cq>w~SSkp9(=IMw4#N@SncV(7+Pd-lvx#yrFTE*N
zoD)apPF?587@QUinvM@z1Hc7_yHdqgkmCe?j1-};v%$z8-n&J{&Yi&;pVaTG_^B}O
z7UZQm5UDN=6vT0l8#@nbuD>cb>nv7I6X$`69{i50dit5Lel9_O_2-9sF5P1L!_D}k
z=CRV}ZR4|hYhz>Gz9)`H#|si^1Ep+bX!Ew-LQU?YQB2c3yxcM~XS-%5oZEWt$2(N*
z5Sh=$8P`;<9v;1l@2JK-z3OHfBe&1rBI&KrjeAJX;tl(LMZsFLkgx&v;~AP}`|fLe
zFG6wM_~R&Co9}VR1ep9BDNK7Dhs}7+t96|yMv3*v%^`ie+q<-3*8TaaH(Vp@7#Ic>
zoY%)k28oxOWb4Jl#%JBy(rH`CB#wIo>?RGCX^+=ko6A>F^{C%h`(nO(GEIuDjFji$
zrpbKmD>H>d&NF)Te2ZelcBx~YEZmy(l=Xj7nG4tPXhMqlU_GZC#}<W`CsF#<sNYE3
zpEijq9>cnt{a!zUGkwCwFixtoEHM8vD+~{&xP!Z5%6%aP6&3@JF$RybR#H7#=6kZ>
zTSaHr(0c+Vtlp+#4T-QSel`0I1sg9jm9IWtugAI&_2Q4a@fCJH+exT%Pl)WP`86++
z7X@<|upEWETwlJC({j=&IJKIZ=eRgjk~Creg~T?g`D7Q3y7G%|>Q8V{`$(DTn~Uf0
zTfF$0F3lqin{cG&lF;r7ruwFO?pIMG>LKdTb0oAcG>fz<$8766=eyIeQaaelM)|7#
zUBept3=whaY-67T)B2Ocm6B(z;cS&|Vb)o+WQ}H=)Q6P{(*i@@PkuABiJS|%VK}92
z<lZ`3X{GIO3BPNMH{D$xEH`TOVDmy}h3zw_M5s1+=B8(oXX&99Qa!tDD3=bG#I|?(
zmFr<SQ(-^eagV*f`+OBuzh35l=6s!*4>&BYhywT^T)kU#`)&QXQRQqTD~waiE0bU&
zr_pmi!J{}=jqBsen5}om+R{{WbCITszcQ*}>xp)a_GcBvAdPg5Y3y3{l#4f?hCP*c
z7RqjGkml=AuE7KOsZT{NvP5$|J_@C*b!fS^>E5~{Z1YqQ;UK%y+UVe*9_QKoE1yKP
z2VtC&4clEJD2tSUIOLNscw3=!0eGhI=UVEuu2bF^YUD$VawU0T=LTU+*Gg#{gCl~}
z`)JR(Lqh~%HFRsN{H1cs7%SK4ZkTHnwZ{zoYpDRFUg5d+i!$&J9SOG`z4KTQ4;5rt
zu>NRY+{!Dg?%HUjVMz?~C}__xTqZ+Dao_2L&vJm1=3AA`*gTTT(*E)w&oV-0S{vaH
z_OZR*0vr|;45Fs%4$0nJqOs59R=4-M<6d0a@mFuk^<KdvBg!H$pjs&u{QV3O%RviG
zlS*N9iBuN66iy?baO>u-Fv4=+qhgW(n^!~oD);eyQA<K<Jx=Y2*uw}nlkfVuNJ2!L
z8nZr;C!$&Hwja{Fz%1LMKlcKC`60;?Ld0cNr%d6-AI0SRy+OAPeGw~a+vOYcuknXy
zyxtZ*%sTEZh)t7XS~gD_o^K#?Z{9Iyx&U4cXGn(_tf|&rb|+hO>#Kb^jjEkT#@=By
zipcGHj8GF!*s#j{C=b$%AUH)$SxWQZMx5@&qb$4F9>MVp6S)DI$A~i}`qeH(GT!%4
zcGF*i1bw%wHj&SQ`UJPb2FkKQ`03nek@iE%1d-LGn`Y*k)yXbLWs0-Eq2YB2GFX#7
z12fkxp4Fx-n!=y#qjF6*etl-^kT_htK_jbKlrgW*rSnL*O67k%fzr-Z4bLpe5<TXW
zYGhtPRqo?Bb_+&VuyRi8x~#qD5_*oLaVS==v<w0|*l_?@l0>Oy1O{J(eG8AU6rWR6
zDll{6;w&<gRt#zQlrDZ_Ig8?l{z7l{9-qx<g<4i<4$BAcAjU_+8pA2#-gQT+{dcH0
zIFwUYk!luxf#w6Zln@sue_djrPS8A)wJ-@bA6t+`ggwp5)fpSg2uqEYAmcQd^;el7
z!rdjgm9EFKJKH<{pj7saylf`cUjwUK*6`8k_xBjJ4$Y)<FdvY@$%$!6^%QdlTxB|=
z2HKu2bfutVrE#)S7yFjqGDKO@?W{Ef-zVk$R5;rxrJad;!dHuyv3PHas9&CJtoI!M
zXd;ctlXd*Pct2Cs@!oEh>cWYfbP_k~qQ^3AAvR2wr~h|J#zGD)9m%=$qYTHW>Ieh3
z=^!?Q5QAZo+vVD}FoxIp3FcozDb{K8yJHDm*jcw2dN^GR#>+MG_mufzkD7gOPZEl-
zS7Coh{D~!*@;@Gwr{R7^ROI>-QAH07JnoE6;4xg5-Rf)Lp$!<OL+WP5Skx?-AN8wr
zjpiX0EoAseCW_mQD}B-ANsFbXmd0Uy8@+|os`<TgW`z{PP2Tl%PAzm^OqX}<RNt#{
zhRR^H6&p0(Wb08;uls_uSZh*Lxjv|E?2Mp=Yh+R^&)D41HukQx%FUBZ|4cVftdP~x
z8n-B>^!uvJjXDcA+ue7EL)A_1N^^a}8qQONUaoyv_^^-D-IA+tCF!Sm2m^^K+FytY
z-n^5;Tf3izgnLNapYcm0hw`xOl9~id#JAi46nuos-jcP%7=WO4Iy}~EK?K!<?mAd@
z<$u|goPXr`%m2JVo8zEx!Iwr$7JeOnOR89sJ0HfiS@?TCt&Wr%8mINB_Gyr7OMEcr
ze;*(ftSwKWG%fx55~tJgnw_uXHcNrl<ArPjdJXw3lMHF&8!<Lc$smQ}t-vOH73J)!
zp@T^$oQ{hQ?vt|t2~#HW0hK|6yRQ?P?`F)tB+I^n^mF{aFTR>?SZaZ`fSs2DiIzX=
zL#Ra*8a)C(yPYD~V81E-iT%*|j1P;OsFocmEL#FEpnCF|vk_@;n^b8!C$hx3Jc?Uh
zvU0H&GL;Zk%WCykjXtXI5nr~uT+;oR<)}~YMU0Fds>S;Lp!H%~Ruh7@bhUAR(lM*}
zrf4A9S8^rA?=z|rS8;(1Z{U5)#@QHR>`!zwg<aB>+j=!I-TZpV!Z=UqCo6cdUX(J+
z{{5KQo&2TLmUwnWMXfrlzs&jv%bEekCs`LPui9b_q#xF46}n$+Of3{rkBAP)49FJD
zf4%5T@QD||K|*Dst&NN~n#8;CADj7-%e%F5+U)!9h^ZYMRci*bGdH`qCk5&_3K*<1
z_wP^MEjFwl0=arBepx%-Vsr9L*!aDa5TCvbUP69!9P-B!;$eoWbT79kfuAv0cUY+O
z>pS_z1BIGJ<6jFt&?;m^TihWck<Bm*w=B#1Ovi5*Z~2rGR|bQH6ZjLu{O<F&m$2WY
z7k0`gxqko7R7f1<^2;jEeOu4@4=)WU#YoE1O##a8+Rq@6lm+k(T;4l)hmwE!OLi|7
z_Nf~@4){m-M)EB9!SwOJ-OIB-CmTK2-f0_^+Ih{=xqK#~iPx*Pn-bTrb;$8PT*}*P
z<@M<mc3HjZEMeD-tMUeQK$YvHd)o=vzGEQo|FCBC7}cw^G?h4>CMz*+_7QS#l#QSr
zxe+BggK`#Nb!@&oB0jNuc?I{*u6;bpdaB;be6*sVGf9w_dm(;p<z46V?)fGHs`G>t
z<V?+ngXq=zDICiCc(V;<uU~CD5$o*x=L=)8-Bi<_GJZ!HWq;42kChH#8Q+;KbZ@4K
z!DEqmtkNoRY{4|qFri3Yu4C$CPf*0`kA0UBcG44D33exI#ac_yMF>a<JpJSsSf#w@
z(>nHdOq#^8wT$i7`nPu2zJV&HtPE>yjJ9U*6xb(8zfT_O$Ktp;e0RUo6Ck=ckI6G+
zqQBr1hx^6=X?5$PO}_+_l>35%=^AD1y{l<8)znCGjqfXUz!z|CJcm$*BHHTY%fH*>
zT=rIVi!=(<zRvat+fdpxR+Hg0inMOe9QNla*Qc2u-S;`JsTYS|o&FJjbM{1WPdN4d
zD;CR$M-`3?;$CiF*KRcKsVj?7R|j)wl<jj*?k%CB_D{muhAJa=e8TomLS67TCdeA2
zek4lZl})m3W*e*72B(z6oKTySE1O2@mF{&)lSkqPEDPBmRdSxZi}jWf?lQ;j>dZ{E
zxwWMh%b-j%FCV-O-)SOLdH=_TDBRETDTB`+lOs^9S@Jgh$>e5w%OEy7*vI)^6o0&}
znkM43s8Ws7ZrY-z6SPeRF>I;9#>jIG;kn%@xK_K6%B{N>#Ow809DmSVKW{%xv<ks2
z)Hlf^!$yxv+-8(-1vq@8-(zmr0N{nP*FqbKX?xn^b}{t}0|n}B)Ct(8BU`J<hFVXh
z8VK*nzP)N<HB?;cF!!T&JN4en$lay>wvs83OZ3r>JywDieP4a{%F}fB*C!?p#XR?h
z0n%UID`_zS1*`SR@j;H|NAcqgX{gKqJG5!Q`BI)#TI4~S#7U{x;bImKY#?9t3%kPm
zr<5E@PGb$bL;82S4`StcN{KFBsX@Ne*;iSpwjAU-!eM=}Rlc(kf|--u8td8(Ynt}*
zFB^?`hmN@4S<x-wE@N>N*praSDEyx&{6`4e4!*D$1GUwV&*mHB8Vt#lz@&GivmGVH
zlEG{vX0~y@&Us~ZKFQ7{jSQ-)J5$B#H9pGyPOd=fmYa>>Pq6qp?^~c&YfNNUPPI<+
z+6sdz$4aopt9jf!Fh!uG<>3s0O2hFc)=BTTuB#bMdzENi0CFd`#YXW6jKOTvi=N$Z
z*V|(7dEqNWo+(MmLXO`i80~Z$o=5FN3r;K}Npat4G1@9Kn=r)k1r$h!tDAOj*DvKJ
zt&Z0^?tP_HmhoN(RrMmA093!>brJAO)La@LA;_DP0F|q@hWDuh&XF;!Hd$!;s}r$M
zNAfoAsEsxIHX8!)VOxvzZ3OHD$}gKR9!|;ydsO(Dd{A;xXUZ4vtu3i#6S$Z0T!mS7
zA#?9!G%-?}Xpzpg<}-D&#{J&yB5kzodD83lN2+T<sJND-M@Iv@YBmas&Kd+<@$h?a
z<^Ej7@*MfZx`5OwjMbun=6yBeY@LPU&*vFyq_i45^HnnDWx0Mrb)YI%z3}hhNz}oj
z?Ke>+@Y<M3-pxtI&Glwii2<1>CXxioI9YK3Mzvwu<-F8J($#}l?S`Z4E0e*^XqLa{
zsnkf_dv52bKr(-~u)PMiH)crwPNw^@zb5lyI5*1v!DEc}*y1667wzl3u=n#`_wXnG
zGx8{H5GriOs%+R6a(S(G8bC4g?$#S;M&9P*t@mvV>vbzAM*^9)<-zYzxxbFDcUp?=
zsoYttTNK?v_9nCnyELL_8__T-c0vjjR*l6CWZn{41H2pHbNiqW9fVAn%Jd^eiI$}C
z{PgSq<@-cTHbsm<EZ$$5O*^g}rQr`4>lZWMj0;#wW!W|2SDH^cs6T3KsAMcAO&yzk
zeXddx+T@JePQx3Nk$KOS*nO^y@?5WN10>SAWo83AFIV!<6cLQ~SP47Vp_ySf*qpig
z=$3GXB9&S1FU2c!2*qtwmXV0sXU%Ta&+XjVh4xtwu2B*1Mz}RNlG3a(ELZO8aeZau
zBY*IZ@TL5)*s4g9>I8~ji&M?MQv1fF45Et(1=rr^Uy(x76<m?(R{kyCR#&|#b2t&Y
zYot9bK4irp<}om+GVY8q<bf?d3UVL{deQZm$lLB?JV;?<fV;9j+*w)|E9?QZ**#3u
zonwQBrsu-JBI`9@n(#Pmz22P6dX}ZlGkVYaK}JN^gNz2hh3;k9F9xN62*@%+Uq4wL
z)&wT`3N*=5%_$g0ouJFR@4la<zCOUunw-GaM{3krAcHuU;q<fK70cs(y{NykI_hY>
z9z9^StZ#_E0P#`Q%L4`Lbmr8Snj4gc=e<|3jb@80As9=Uy2$0%6cKQr3=PVRn1Rb;
z3vTF?A7TpsNJa(WE;eEr*hDuh8SnendVJ=*&mp^UB{={d_Cyv$riey(;-a_G{$w!T
z_NxcF%~O$tfra+Cg;E&)B6A@d$*c@09z$XkVp78^Yu9%tRpituC4=%eFp(~dy9}Yu
ztJ4}CIiq?+>=Ta~$@qVmn3uS-VE8CDBXQ>?{yb>Ga>IgxB+pWXnWt13`{H)rds8!b
zu9nu&Hw^kp(!bY~Bwn*GW@po*iP}%zz+Awa@oT*_L)&MHD|4B<59~{>?F@c>f)Gg&
zajm*`&rAm@s6V4Y{<anIzDIXx$KURwPuJAw8X1=An7xq8J#s{CcjSc0;~K5p4Q$eC
z*bj+(I5Xjkb@n~kPOBfi==~a<@zr-Hi?LSkjQ1?#($$%X1t+Y{V(m9i=+V*cx{G&%
z1%%{xlVgS20IO1SBD?HMHG1pzq&EmzFMmFdlBs+{yQ%WUrzltUb)%b!NUzm3#l@qW
zlcgNnAnNIWs29`EINxKZ{)O!_OvA6o#rLp3?Fk%i8IXH&He+gCcGkP7$WCQ<t^QG`
zMMKxOO*cjDos!DK7&ncpY7bUNUZy$Y=m)x`rr>Xo%kH#-wyP(a{y(g?vh3;yzdlq9
zQ!8mFMz8fY$S)^FhdC-~{&VMRR#?$J2G5JJW9vOai!)np<;-gxsyd1xu3hA<nV%K6
z8?U)L?<tlF$klpZ#P>j&OAYDFBjOw+q?sETP5jf+TW$4GlOO-|Mv7fVAO^k%^EPY*
z6x~$2zs2f!dr(xljeIoZdy$R&?_svyeQLiJDJH`3<d*2s;kK;am;tA+N9=|{%HFdD
zN{h>6Sto?Fb5EwM3)kvpW7WzNAG|)-I^mimU}M0nF%|4Vk3?|A!F3J=5ERQ+8MsA*
zE4)UlqtiucuMZi_dkD~-P4FjN!}+k)h+8>b9kg=_of-qG##!=9`W6TiRg))KQaMjj
z1_zs^f)sXDN9vr5xPQHvNtRyH_jDfEI|hWaQ_x-KWU|EXIJFR`2}i?jx3MBeH{lhw
zp`#+eAwM=8lbdOj(6|`Ah|_lDa{h|35-{)iL~ZW8cB3;XJ`4nABiYN?Muw<E>H>=-
zOLmRt8B%qTQ$+>sJD!5^hxmMaWQ(7yqlWzzBFjdfhNG+4;Tyf!$jP)iS-AVM_V35@
zddr9b1r_5abMgy~RRJk9xwG{3hjwg-|JWjufS0P&IB!nP@bJcci+mmL*M%+7NXdey
z>Ak5Me>6?*Qy}BjiI|<uwa<G2ieTds?wl|KHP^uLLn3o(qX7Y9B5rDzL8pgA^a$_X
zyOxc}w*0h~=Hrg$nwCl<<tR;JHD=kV-<C;Ei^%+^c|Nkf_@gPC)!|aMd!sLBlEt|E
z3aTCkNm6qmZq+okM{0#ex;FDCm%)9FbRw(0>sDqPn`0ZRV!nF#%4O~wgS4}p<h3SW
zd6|p&5najMiKFXSqu?Vj&DZxELD9BMORt3aaZ!|3P5(8Tv%+j9Ia9SH+VTaJwe**g
zNybT)qs{f_02Rn&trYC3eg00TAsyS8AUqFIltC<w-M(9KX*DtAjZbM-hCwPz8aRSW
zPBW1cJp5!Od)ZF+)~4*4{^}Fjhfk)Eg?@!C(#7*Gr&O0tkg~JcP;?uukM`7I$exhg
zZ@0k$eMN(Tb>p`5T9F33m(I5iujt=MwZI!~(;&vwI4ta5aO~>#RF`=expm5LVN^R^
z__I;))xtI6H4O%Avh)$-S5KQkYff@={PK0QkiJIE8}>;YoHy_k*B87MpjjD}turc}
zWX$xx@N5F#bF}$qHmY8ngR5bs^2=-A#j1jaYU#w*`ow9FXBru>2<ltT4Ruv;;=+fj
zA$X1M$G8_elux?%1C7vbaTVX{qc*{5BSOjH+Ah~77!6ZIjOV3iBb985rdoUSwshK1
z0CfLN&wgqe8-vP)0~SJt8B-g<Z!|t_v_x~n#W8tE3xw<k5x)y_yn?XVUesUG2{Kpy
zXH<}7!3f0aV96fuoD{)|auvg#?22=E@P1+2GKy+jQKnMNovbeib!D8T_vDMMO@QCu
zoTMp@6dM5t{P+erfh0knBy8jSIvb2i0BBh<8Ic>CO|7z)HB~S_sRvAiew?v3KZ3r#
zvLS#40g~+huS%5ab8pt8k|*!DC@}QmUc)8a`c>AEQIJq5nM=e%ej6ZL2zAGCC|6P3
zkd>LRZ4IXu0Nf@`Xe6VAEu<g;&a>8;PQd!3$(P^}P@t`X`at0Mw@Y+F4%ze~E<XY2
zw0_y0S_u)^P>ttx(9N@wD&}eZJ({`7tdkb1x|F)8p*zlaF@^$N!472ZDge7j!zej2
zZCy8hJ)pbu)bF-#*;hb<3ws`|Rj(nfP^AEC=(I)BPt>_@&yN5IXLGc-HdLU_v0n}l
zl~rGMSm%d(ZNX%Ww%ppNiYD))I&ZE!H&rNt!JYhCejiB=pp7L~`f}a?I-Yyp#gZ^@
zls<^rVjw@p(0%S5&(#!B_dd&k{OOkH3hR-+pw!kkH1!LByZ(6_(=6h;?u3`{J@K*{
z{+O}5I&$yL%k$8R(KnIFz@@|T0s6YAoN8F_GLmH1WHC|iTKzKBJ(qRIZO089%OOBn
z$09ip<wb_|Rcm9_-Gf(4^eTVkDy1wVM<AkN8G}wM-6jNAj253dpQ(`_W|$`Ky$We}
z+^CusBHCRtc#c0c|EB=L{;y$_hH-Us9f@NBL>H|=l_cOeuaLxV5nE}|{|$%^FJGS{
zDFebr+P<C#DY#<zn;+pD=LuEcgB#U7xXb`_HbJHUjS{^<QhH$xOA#@YRteI1vF9_%
zK>9}3(`@mFK+CL#&>K)8tk=hD*Gms63ziJQK9FDI=;Mu(y4?H%TsJ6;w*nU=Nj1|-
zW9)<wQWo2dr@QmHzro!xNzBuwz{YiT*aSSK10MY=I}NCGH4m<W@_Qx0szTUKk_cBs
zvD9+rRN_6p;KZa?=TvY}gqHO|=M0tQ!0+4qmmXtKhK6{i2+6$MSG1<KV9ijzDx4(!
zVx~W+FpQ6p0yfKsEI{)7q?9JUcFqh82&Qu1i$1g~S!fnnA2v^_))_H>Lyc09B%?$!
zE2E4$l>dCpJjqt@&%Jz!=yP&C2C~v2q|V*ij<uk!(|vR0_LC}b6h0)q$@j&duys&H
z^1H@Z2IVTJr6$Ees(JJs<MWwNwTnzKheWe2JgwRh05+=CYd~?jsBpB8tp=P+3T8*~
z#z0V{8VBvNzTzTPd_%h)BxqBSK3RIK@~Kc8ZKT%SEBf#d`lX87e>?68jOxD;BSvu@
z#3-ODKQrw}P_(26gY%oY94c91-pi77$0R^8RdfUp5fsk#By<A9ARw&*0>fo_;AYa@
z=fM%zh4|*{p0`Dg{QCTQ4N$^bwGO8GK+1c^iytwaD1R9RF+)Sey3gNnY5r)hg^r(G
zGWJaGpUqhtKwD^4+Zu|znvoe;gCgIVbC*8;&vB=}!lFNarOJBb8Bn83g&gOHH>SVX
z0L(vBZD(AqOq2^k<=^l_IQKt?pOvAKa$1CC&8Hi;^{bpdn{Cdt4%Ip?<gW}kp}>Z;
ztv#8jb<8`$A8aq^gF>|gL{wc-5ims9(uOya+SMmtBk4t+1VlWFs<s(3tF#;puS0=q
zmJOj?Xj=Hc2{Rw!2w_wxn}}`ivuPGMt)?g@S1Gi6bQhhZ>>qJ*wkH`6z5S#tHqMe{
zl3N*qS+JS;8ism2u{GCW^ErS>04Q)tLXNrMdD}-TES>`>Rt^B_2mr>8?oVKTOzLvP
z&+Zd1yPuJ}=D&z9(yJ_lSlR(TD9Y&lbcQOYGYHOO&Z`B=s^CnQxMC*-%zj=?5^+U&
z7N5ufnWq92jR~VsQW~ti31?t%{WRGF@c>`qA5*a#Rt)jm=c2Ej&BUX@QV4kNt*Uhl
zyhQ=c_AVw&HnzVT=%vMvq8O@N*7T~67kjhV&TuUnN667Ign}y6ak6yrwP>FMOp;<)
zfPLzb3d<c|`4*J>(=Ix9>BYT7$@7^IYw<Db!R0!awdF$?lz<Sv{c;X~@o@3>1NFpp
zXTyAuKF8v3su!DeBG*f0^B|)0U&GK@1)d~3>L++M7$-{;sSMC6UUg2$IG!(<m))i)
z2ulj?(Ms=C8G*|be(v>p2!)`r{Kw<Py5+?W-&|QgC>?4qGwm2Rkii}8^Xlf>XV<>b
z;O6%}+!;?5g?4`p2yq~>E?Bx`j(|{G&^A%$>=;+DKGi%Gaf@cO;r5Vl@5bceg`AA0
z!9b-4?McPduwT!o5&X65<7mu9v0$c4cgPF0ZV@k#Fg@qY2~iD&B2wQ<=Kn=VdLh-c
zzy$Z6NdNhe%7Qer$Js~B4tDJ!6Lg@+b%1OM{`vo(zwFcGbR-XIYMNSL33utl`Aow2
zBLBbQqwK7nd8VyD?mU?~nDK7#%-sLC0XMr)g~N3B&#Txz8B{G}HytMvNF8MtDA|Vr
zscZv8yaVy%3P^K+!B}4$|Jj6Oi6Y>PNCEUQ5Gp86rz73(-NEwPhKBXpsR{4NG9TQb
zZX%CiF<1%r`<bf%zk^T9g|<XbLD~omcJ>MoJpf#KdU{<y!Yc*5WPBn2m=i+EP*O&5
zYv8>p1v}^4#2(o9wqs8FXJ8wpQfaTh&*lla$JWmK2-~GgLOf%kR>|@S9$+@;ln64$
z%(EZ`?Tk{Rc$@*2`4?C>9`Vm0I#UHD_X)>tF_-Beno(c|d0IX0@J8_iAKSbSh(czs
z&y!V^zwFink<VtjB}l93_+_GVcdD4ci|^$6<z^9JY&W&@E`djQVNmNZw+aI6f%x0E
z+s;gVBoNY*F9!Ia0stfZJf)OzWe^(WZH}fx7)KQaCtjc%w!#g)cL(=2+Ze}yCSqZH
z@I$*8c%fAPceoG8_z`y?P))@N(l+2^bt7mI2B9$txAp5jmbq=tY!mhZh&bMS3Q>%l
zdLIq}s|*A+4R@bb;PRIYfZJ!>0ax$1ptO(QUg)j{UM4;8OpXpHa8+3B!ir=C$StuS
zXL@Hr+)b%2kx~M9bPdusATJAm2&+<R5Q=N(NPAnC1|T2=Z^!ut<pxoPV`9n(Sx`@x
z0dlI&%w_glL^%kdFR?G6m1ph5_{%+z!FfnoNS-BmOO8_bz~|CN?>anO`rHP*6=k-#
zH)dBY&)dj9?^EZBwS_2h#brPZ+FrbJ>!WIx><BO=+4N;U9?Dguv%;WKNND-L521~H
zCs&q*Vr2A5VVsw|Y2W!6d`}x|bt*1p^<?l7vV)Dxtw?68-Z_PZ9~iWYwALDI25A<n
z#%vuc#S;x(KG5CIkWb{{A#WX34k4u<|4s{+fXj|5|8L{CWqQlf84)Pz5{wAs4txnX
z8RyA;9>F%Uz0u*G^@|*ZYsL8I|4TUUTH&Bt&Pg<K==?<$oDFPUdpW$`?MA?Qg9+pr
z^5Kt7_94A~|EZYAj=eN_crbSFoJLPGZ48g->8bh72$maVSYPKO1`6F*ilh9(<I96Z
zhRqDR5X4kmmdK8ot2jw<oh<F(Z5|&aheGl|WHn<qI5rb^CIH@diUMSdWxdN<888G5
z1Fze}C$M!HITM6JMSb4KalEry4*_Y&8ASKOce)kkF(mZD1B$dcX0~9<ltIetE}Wm1
zOW!{q><^bI@BLa|MBO+5)Q040<${CDUq_>E=~w;id^phTi;p#hKSI{ab%TxK(bdfe
zR{uL<w_MxTGvb@UB@pYgA-*nP0AvGwuv4EPbU`}G0NgZBJuu@;R29ytme<Pc-M}60
zI!a`C>@LqCdv0>N$5leHw^f0fsF3x&sC(IpJa*)x$?Kv^J$^4NZ4jVX!$dXp5M{=r
zbe&fQixPXPXl+nRDZ&=mwyYNhx0bGgPluZRNA$97AGnvf&bF+_XE!M-=$cpe-f4ND
z9T(V**EJ{n8|<=`Z)fsfF4vO^(yO*jS>Z+BVO@^x12{;(!R_wf@I~t}Z}Aj`2<g(#
zzC#rz`*bL@Gh;_beUwW0uh+98e@X>}5c%8vpWw7lGkX4U;T+&4v}K)F@h|{0AWE+~
z1BGzbtYHw?skW0vmkhR_iQA=zn%j1dl)p^W%QbTX@B7hD!S*+6P?ZsTyTc>qu>*3B
z(3Z&Rw1}?-B~HD&#6|kQ12cfKwBoGH_N5XaTX#Ont_|fMubSte1<*x5ci!P2>-zg3
ze-;$>(A)r9XW<o2r0qoA2vphPPjFg5JS@^FeMSlb>ED!hEu%*RZPe(+D695gCmqSm
zaA#5$<-1+&abZ?QZ$XB7`Cq>oK=bGS66@J3d5j${ho*XMJ@&-o_92q0)=dZu52jo8
z?*Lh~%nM*h0qfyAA`ZVpfnfB*?B{!v8;>I2*7Hn(OXyWuTR<@m5_y(EHS;DB3Z2m=
zvu#n;pqQ=a!-3HCB!N-FXZ+lyt4|@rD}Y*~!01?_o~N{)2XL+`rqXsoIO^s0hvc)F
zz-KiA4&iE92T!>?PwiJOjeM0dTi~Xvy$@X&W&stv&P#o*V8?1&9)Qxn80gbH8=+h>
zhkI*Qz?f9kl_K)UkmkMQK$Ewpe%Xt#cf87pY%s|fu03!JTlqPZ+0pKIQv}I*j$N6x
zS&kz)FFh=K-bM-}N!<cdj|Q7D0pP^d8gLF`;<zw3^M=<&0>YM%atU-!vq~#WSF1ea
zIR4JJOHv53Qh_P0=!4gsU-h(!9kfidl(fmj|D+%JI%litw*Xge3y|C27fC~dk0ydl
zf&jU_)5~ouBo3=Yk^g2TZ~Qa!W-mnI__Jn?;j30^NM1hGCS?<bfr*ia6xueD3H-nr
zY}TD>X!+sC3D~9KwWpuO&GGt{o>F?uy*0DOz$dy2vR@S#!KX@YXu`-JQ0%y_j8+!W
zSt!k(GnWyHV6FDpwM-IrE=1Ul-(RkiijI>6n@#Kpm~>x)<SgG_rYwOQnDkh=4XI2H
zuOR#Y>$9K#c@1-{I6!0ufV-_YOXtIE=1PP_Z%>Bw1Ro+S31O8`Y2Mpn!)RsLG{;JA
z=sMvHu?t7QEB8yu;DT!0y@^+C^pG#t;5rmx{P^?7Q4U`G0ECu98ZU4s8(#bf06cre
z>AnK4B0<{<$mBTbHcKK{_%-`+Tro(^wQlavc!bh>vWHCCQ5|_;&j34c-y~2U=|o(t
zd5lP5em}Q5uFIr80}+=F({L3OC|zJk76PN7A{rwR9DfRudS|OVGz>`tp$jq~`IbWj
z4DcM7B^o${X#$<`#RLeuj(;ECJ{8;q?AUsfy&s9)BcdOn5s+&uT@DW(&m}u$BBZ1l
z{{dG2{pU5uKN}{3SaaYwxL3uK<PfXE)AZ-rzT^71kf6vK$W@l6KjPb={S?j6-1*@5
z&EG=qW$v994;uVgO8*Z}qJotn&+p<N^U%TrP%zrY=zA(Ii*0?s0YMYG`}R*!e|Hh>
zKPWSoS;FM|uQFd8Y|ggXeDWgzkV90v36kKQq3FzSGs>Ui44i>o?LuJYfF+<EXa&|u
zf_7Daq#C(hzxj=>1XN2dK;#F`lDRUUOYIt;AqJ02tN7|>&6egnh~xUL5A0{fAny->
zGDxihQ=<P8_#`<CR(sMdA%o|&bb!TfmTYS>dF&2q8#?QONF$$M;YY;Q0&t-o2sUMV
zqu|0zI>Ds$mJpir^y;`>Jq$`7gY72sBULwAQMZpNVxB`>+9>ObG(5MKxGP_FGtdE>
zbBS*GH()W3uXF^bz|WT?)z{s~$hFr7n_R76!Wj$3ssm#4liU3e1k(ivN=+}38|($R
z1h-bPy?;wquA^PkK`sDXB35X+qGcYn8I4aCfGI37sO>|n)bEX28J1`P0H3-366o{m
zpoITfI{hW2et~Py;|Rb2S`+BtH03@yqe&X5R>W0_dg5$|0>|BZmTu}J$T#)9^BPu}
zdha2N{W`E&6bp#G(R2m{CxlUeEj*d-oIx8vmo&AxKAB;rS!<8RJPNe%fL80rHP0KW
zMd$!bS?=Nk?jp&Y|L^7Uj}-?t4u3ht??!IZMEby{(FnZ9>t`9=ei76!7Z#Nr)|D2n
zw-n^&f%;(&7$<ree2-rxY7G9w0`2-uEd)6eoH*4Z;w>Bav9XqKK?Oc{E>>lW9vDQ%
zB)}bi2?EQHD>YMwMzHN*p(b>{msvjx7`rgdC(iqE++fwbcX}V85*C$x594q<c@mXi
zJB_A4cx@2+rZR1PqJHfEjg4H`s|2GfPUQu+JLXU`PL=8hzG!Ws^x8+7JX$oA@3J(7
zdTL(H#t+SfqhCHz%!RK9T;#FeW`%s@E}g&lf4x2@<F;IoXTX?Yezeq+@w(`-+8@<B
zYKrRrIqY=G!UU;>hO>?W#A&;0p>bH6CZ2Y2cf8EoAjo4<GDF01{w=rhn{(In>s?Bc
zqU20Xb}KQ*GRw4Crd-@s_I$goi5i$WDJ8-fOEo8gK{AekH2m>P%<o(Ju>}F@l=mle
z=<_h>#}DQKUVOEm^(&ik@QwxNt41N2?p+9pPTEgq{F7`Df?#jt!}m-T%C@rpoiXA$
zU8TxdcMsCC`8hBRux`4CZ28Q3>V5eF91xVK-6Y(@PGM!EF5ceHt>d8qiFqftH+l$=
z?mW7=CWUr&OYDL*QdPo=t%Y@Q|Kuum(1eiz-TTdWZ%m*qh)_^#$enc`QQ>Bw!RmA(
z5fje``9QeupyfAzng{w9c*&ud>63YiPTrzAs*2CEAgGTz?R;jDZwhFCW+ATzMQ=0-
zeaP1PprdL37t6}=heQ*0KaUcduNvGQ*nfmeo6(%RDSGqwE_SN<#fO%dU|Q`Tz^>~t
z)I|yHWy>;#P%0i&^NEM&{)Foq<Y{PkK*-mFENM9WJr`UT`J6MJunQP*%&^V)rlPq5
zl?*AVBZ|(^qBIrcXYGHbP9u}!`y7SJ3|QN>J`V!I|HcsoQL}YdC6pSOoWXfV<}FN}
ztW8;i;>b4LUj<1FsSB_zW)A<wrM$FLZ=4Tq#kl^!6Z(BRl&>Kloq+u>Nx_$7(C?_i
zL3TK8M5te{>xdY30Y`RjhK&FwryO`-@AY_r+5>fn_e!ziLAx`j2>Lbrc+4~k)YGKm
z3YWD}m!+J<1X|rlaT#zaKVOKnbdC$-|EMD{l*7&8pAX{X-ne78sduOibhDA110jUq
z!aoQDwhlq-;a~H=ME?VtP0zD-4%I2NKW6Ill0Fo2T~AJG<7h}YB>|@CBzzjyCL6~9
zP+O6W-MjuDpv>|4dF!dB31R2u-#&mKM{b@*|6!py>#Cj*MxS9II9M|J9+!G=wR1qt
zG&)(b5##NrfY#dK1G~HXM8SefT6HmS3=ZYU(}2oAyI}4gL}h&Z8lcB5z>`!0VMHHz
zr0+GM772TE6_dvO?Is&WY?}_us#m|!*;N3YW*D>}VcA8Az{9ExQw6h(ls*purByK?
z0w;k?34#sx><Zsaus)Ejc<mzK9dybdxac_e6!hs-K#wqU2b!xF{)c}w0N@zlBMC_V
zOkP&X69>)(m5)}j&a|+M`+(Cr==*TG(G5KXr_sM4GjNZNA9V^~oesYVHkkl^!X>M7
zZKS*nN<+X46$=p>y4QrsV15;6O#udH{@ETEWn~1IF;=G2Kn3_U?MbqnzxBl5OK}-D
zH8gItT&x1<%M;{H!$Qc1yjoa<`qA2P3MWsC&zSjyTe>r{!Tq83Tt`+amER)hUV0g_
z&soRULLEs&Sy*qZ=lUj~5uw(uD!KqVFnUQ7C^lJrV@Fb&|Al7})ae<e2#FZI0E-XE
zDsYrL^eW7!tGd6PZjHkgW4xtw@TF1!PcvV9=re-I2GIG?)yfP_oZM^KV6AK7r-5bt
zF49%w##hZ^ox4RVd?2YT9xS9)Lna)O^0g0u&vjlM?iK&@>x`=d^wIDnDfLE=U56?j
zN-)>JaPJ)|Rq$g4YWa%0eF|j7!3ip`7?&m!`)wJ>|Dg|%qBc;40fa&f`V?z|!)GC5
zg9EGsRyN2D`18ik%0r@J5xPm3hWGT$Q2<mT2E0`$_hfJp4gkT3xXm5O-M54|l*Q3P
zjuZs`iE)wx06r&4_%=a3J%Ey*Flb<f#`yJJoMZ*?#6tY5AwYdD?QDrUL2&9PLu(WD
zf;1t}dteLT1X%(LXp%_+0TuKMs@P=K_&2;w!?<J-*Dl4vY?GRf_?vNW5>FK#=@2r;
zDu6e1Kp5R_;-L^KDs%rc9byCAL#ON$)Ug&Nx)loW&-Zhr*Z)}$<O1e)B=7>|cz*8<
z*|(g!C<|vm7MVSYyj}{bqY=>SXARm7+;tP*f*1qP@5oH+8yiqVxd3Zsy{X`&Th{Jk
z?-wXm3Lf18gY;_=@HuuHYxJz9<6Hv?maPLcb^TL-sD(%U526eh?Hub+00%!#QCA}D
z1yj4+_xzvmE9Lv=*XK(>QD1u>V&2jh$D40Tpgrx5wiAIm*j?we^bD*qSEUyK?^+-b
z#10wLn$`hNFG?*NAUYvK?P!0w>6bwELp|yE7a#>qt)$n3l-J+mqJ-Sx)5v=?6w!Br
zGUQx<;0N|7ccsEVC5azPbuS>y&NuWIDebL}JO_1C<v+nvyoki<vAwk1rVaq8S_glR
z|2fg`yMI4bvj#Q70VApny7{CIoa|19e;^-+%`Ip#%klv;;Ii10q1LgZGk0Qume&Ha
z0M6PP5QkAEQvk*6{@pFVF2)>gQ6c9$lj^USWc-y3gp3o5LFOaz-Up6A9Uz(-IM#x)
zuuci;jfEX&0N%_tyOvEpp{93=e>q7n#9}lBTzkREu&j(W6ciE*q}D*AL>(?KNZ@Os
z!wGuE(mE5Nm=|qw$|wxVY@g(3N^etG@LQ1hmK=ofYxyM<93X$-fXr?`w%}pN2QI}g
zdqC6tr!}_93uEM2f;+@Ccg>%cHQxz4=?nEfn1k|oUN3MBqWB55OP{qil*r3!or#Hk
zzUaS%uY?xY56{UDf;F0Hh>dH-I}~c7^xR)}#^V7SIRaV(<r&@_ULk-*y9V81RAT-<
zmn<6O)cN5JF)^5dQ_&+$1$0cgx8u`9TzSU+o*fEP1i3}U1?2bBCDPC(kpV@{tGKe$
zf{PihpeN+@=c8tUTI&zMsgfT-?~3;QcM92HyAc@AYTBPY@2vt@-RVo9=e=LDW8I)j
z(i$`wom#LBYRhgL)O^g=QuqQR{TDs%@UI>hi9Qqa?J?$OC6d$f*C&75b7RnsP8%xG
zf8;$0SUd~JfJ?{BJL&UZ$Z-3j|0cr$))vqdMXFiB7w`6g$U>t9YNG21axq|{2F?cd
zdSEo3xHG-E2X;JIQ^$^7EMOug@ZD(=Q?vrs=Ag>`xo1HLBR7A&fPcO-`mnk&uI)y9
zNqVYav;|-Uq3)~;+ze2b)Bn0gMlbvV94)}hC%mKO5*_jZjl)(PO3C&e8gUx+E;eJ;
zc4<X-fol9tF-3R~Q4U$R2Z2lB%S&R|_fRP$SswZCXG*EJ3>||4<G(zh=zbVK<bH+#
zaL?IHnLc6}{7eDoWJ6^#Oo8q&1b7WKf6k%*l@Yejumml`0>Gnh)t?&yQ6EsJ;|c~X
zfko^l%Vpj^4|`#hCY!^!eJvSh72Ap9zV5Zs_ejmdwzPr9r_p!7y`xXaj_Es~BO&p?
z(pzfj6%ZV|#`b{Yu2OKys~;2|o7sE)8PexzGO2v1-VWsD17>Ha-qnuop4r<@U{n>;
z^34R{U+>gK0HM%@##C{y8_=!WtM2A7Y+bV+X=cAR>T6zJs<4XInY`GX>g&Vg(1GXW
zR8^1n4x){Zp#{_~IKqi<FE6NW3l749JZHi<R)GoH9_RDBUs<B^-pj6<HQ-ZUe}B{~
z=Wr-w|H}`Gtbc6>D(C(u+rg$Y(=wn`Rxsr++Am|W6|=sS7&VUWu23M}sGIjOT>u1%
zQ^5kDByd%(SR+qR2u+ATTWh=Lv16XZXBrOmfs5IKwuyhi)-Gr6fy21E?U;r%uY8uV
zg7a9H6!;?y1UIwZ%!dNNq7SLoR`UX&BhsojGe{r(*P75b5@pVG?%x>UN-5BY(B1VI
zY&Dh500J0m3lIgMNWd;P0ZiNg-4toWRf75roCDC>khI~G4Pe_k2Shm<Pzn>V*_CO5
z6G`V&d^3!S&^dCt1Lw9bS#>ru-UwwN1Y(7e(O#B&_znOq(ei;TOn{>W;PsR7<&vA5
zBD@=CH^==IpjP~Vh{w7&I_6n(gc{}_V`8Z`eiv^Nk|{3u4M5tIWN%gv8IvYNc&cbT
z8;t)`nZT+?`B|>x*zH78;FQ4iic~o(^N0zAjfb}z(t$@JR>mqs3Ul`pVf1~7_x0=k
zbDN5AMCSAzH+qp|5|~(WOTV8`<G-II=Qr9|Cn@?$8Odj0e15;4k!B8;-Q`dYM!Rdl
z_lZPlRN2YH{raG>r*pFE>#MCISldZ+$-~l;NC`gxpCt^)V9&^cC99ypNUA=`mc-`K
zUM9R0O@BJ*4ZdrZ;TbNh{mhYiEKH|Fw4@~Ub55i|ZkVE2wa8Qm0V6aGewlF!%uU0V
zkNEDsmtt+5^ip}ixr4v(@H-Xs44mez2BEzCtyf6d_=yQdB*AUZQaX~evBx=X{oX`!
z7>cM#I#N(O=(OjWKUj=tQ`X%c<5rOmeEav(u?2oAp@<u``6S#sdYL*1DY*1xbC5#=
zr_U*q{9BBw)rxi@h?%MX^f{lJmVL|av63xAfT0BOf0APH4rB=eHcMAdTj&FRupcia
z6`axobt<I-nP0#mlA9{R>Eh=PsB7(u#O6_nK8xo_Ys5BjFD?E2t%EaPplD2pYfmSO
z=%%(}of--3!zJkti;R>@ZZkpI28a~-dA+6g&*>>mh(E|eb1J7+`ZI#kCpm^tO(3a@
zjV`bO6B&Fb&>E61144vCW8=bv-1+P~?`d9Dx(|~P!YYJh$W=;P@+6Xkoz1pxP~y)(
zE0aq#o$jH}H{&UKS{`yQ=26~aNwMNOl4r|z@3yd|?z(7JFmO5NGPrs;bajY)_xsfT
zV(NnerYxHl#zOz7u2MHFI8)<w^geAbd2sZ_zl-XidPgZomUZGz{2KD(#_Stvg8>Cz
zwe3co*NAM&z`#3!QqU5?h;metI)Xa1@JZI;G|?<UEMlbCei`xrs1*?=$<#_`@F3`<
zK>vn>Af*YX8s~TCt8!Hr^o5TUU*H3of1f#>*9tGH*yn?ORL?tA_*jTq<tk-jzSau_
zGE=cY8-uvmyi$hN`F5-VSf70QCkaxj=4-n7a9&>*!}TdD+`(9tESYo(+J|G_@kVX6
zx5L<(Ug+kZTdvC#JbuZO$%=B%*kwd|IbTbv+Z~s<lFfHsog!QSo<^hB0E^EzH<5cl
z{9W+$Ruu}QpHsT-Qo(t5IEw?%UlUVe|N6Cvnh*}P%)zLN4bBo_j#oDOIvT|9p~4K@
zNVJh#^i5K1V+yygf$?5E-N?iT#PA1nqRb;Os*(gwBNfw)vs9F)f6;5F+V~<e9TJ=&
zw~fu_Q?(MUWl&(&M@W{UNKA-Z+3)x-0{4qn&J$|LQaEk{`h1qF6kP$&#N^xy?=HBq
zC7tG4DE*A`&9czGmeSg;!)^8Hg@)d-QI4UuaHha#z#>xtzJvmlg74{t9KM1)Z+g${
z=SSc$lpS~Z&s>PD<Npbg==qU|UoBgQ>by(Wo<SW-!0R}-noI!W6xf81e<)rGy%NlV
zAnVvG&E^|-VrLU}a&CBReL<Sj45V*3;y?BIfVitR#vtm}V^a@u1z-qPRzW*`l@+S?
zo5#+g3BWc$;RQY(oj7(yYv4c=0KINdv)=C77-mh~|FM5K(4r-`$|$HZgEGv&$kqD|
zFyCv<k!jJYGJU?z&_W)bf<DvxPZ{K^9N;1ib#xotHr)aJ_e-Nb>kccBugkd64Shse
zAdSjSM5SUtLq)a0UnUWMe6U>t?3hEK$~8#gfiW@6Gd)q}a-_&{>x`ohx)l@PQ$avA
z(^=q%FRfmk{DHquhHFlseywmW9qiB2s`KIg;Qo-Is3rW=P~<XM1<LgbK;69)odgwD
z<H3B=P@|_S)L@ypo(I@gmzl2@sud7w08W4g#JyVDEP&8QfUUsrUOo8k23?>?-t&!>
zlmByGh;eeH4RM~FXRvqf^6ga)M65yJ3Ezi&wr~xR_@#`za@C_axHv}D8sG~mnE^CM
z{5hGPPsF9Gfa3s6+H*3oooGl>xx1hBi68i}Fxj>TTXWF&Z~UX>u%BHK`2x_}rGSYU
z0pz4Lxc4=QM=!JlWT{#tWVm+hJN=(NsT49O3p%kTDN0#=b!ifLA`X85*8M>A(cLX)
zrow+`UI!Me$7^~?VmE%hCaWW$m|4pMWsAVGuf#O}6}VwRG(zfM0ghAV;bj{kz6H$D
zqlX44vO_z=L%GPPZ}?0@yJ_SN?L&Sp-23j)-)x>FU#gW^iGwi72HPhRMmIj_R~2A~
z^H)YCj^gs$P~BNNX}W3jk36vSjk2Xp;Nxk}#5v9;q0gv-%FzZ?QaPYA#}HTuSZwh?
z-l{_;=cRi5AbB6X=tGsf<M`_nfgtf0U<aLWh|pRf$xru1uo${AsVQpQ_kJ&<7x^;`
zBPz|f<JIv_D2zV@jEtiy;^uC2_b3%B7+ICF#?olqZ>)M9Lb^GXQG}drdopb2n$9Y=
zabtzDqT4PYtr@@?XYdUXyDPolUs?9$_D+LJZS8RPdtZ*c;cze01<NaWDp~Ay877=D
zI5&QUeL!{?__A66z_jvaApOA^5%0qay7vUFUyjaPsUSO0cv58y!(878>CsDfbNnN-
ztRH8qUS@8>nZoYC8PRR=Ck#{qmZm^ne*u+{z;ILZzX<Mfj^q}TOm^R`qYlr2uL{sP
ziUGj8wl?%|A8TBuFfHduQ=nxIwp6W<#aC*Zgbni`*Tb4@_85VA1IHKgZvOW<iKe|$
zT*Vi9FC-S+p+ch-GBa_V1gE}INQmcITm?lKC^|I@+eZNsl|M0-wuDNi99*bu{zdv6
zrGHW2GwmtD`8aw>tHHJPDq22PdszlXuC;P$#eI|;d%<%3=AS^!pX|%QQdWK~=jr4Y
z_EV9}umWP4n~gw?1asNgu=;45YUz9s^Q53NRZoB7>Y}uPV>WlpQzrH5gp)q2twGZh
zZ1V?_OC#}%?%zCp8n)lk*vEc|<8Ttm+r^6s{x)H^e9EeSqdmw+oW$p#;x?S#LnKD%
zeda2qq{#(<w^(~68cLCly=`oDPQfN~$Q#s4=K|f1I$lct*r|Ou>e|pQchQgq!#Tw)
zsP0cD=|T}oZe9u>88BewQFm0c?~Dn3(GsA|q{Q}A#zUzJKKubL*`ACoAjJ=}PuNpA
zlLu}uJcznEiFM=JXMoKxlQWa6tqnXO5NlFU?rB3jVr@ScywUQ}v!Ucwe2Kb5q+HOt
z%;7*_aE!)7_}xjX5S0Ay$_ASmqO90oaW8tpSbIm~G+Lgqe^DuX&3)%Rp^7N6%toVA
z(k&IqC?5UtcaELY?Pd{@fA&5UK7^A|e;}5K*-Kj8j=dmzfHHhs`R&$J8|WVf&mh-+
zC4YAJ)df8QG5rUeeLgkwpW~x)RX-5R#g>ewDXiU#neEtFbP6-~@gz#}^7OHKo!S1C
zeB8^7AP97f8+SOx1)2L&y&l+<pfTa}N_)^e{~*S;yhdSrk8$)Yk9wupbeYhn&+a%+
zDU?2+v@4C;nmjYTJnN-g7!?g6MRYck;KC2*tMCa|6|_4!)w4$j<~TpT!Rsg$_)hz!
z7L~#w&d!5mwF{Sm96Dit$oz?$-gT0eJh+v<z<nvGNwx=7J2l5o97L8+F5s)c7565b
zP|b}*fth3~CI7{v+5YQFIrGlpKp!7Q@FKANv+eZS%p2mbXF4B#eF~S{WG4Td=+JAx
zEJw6szVKni@lqdRzevjBN}?q!HeXr+?;E5MsysBnn$&L^en|FE_#Esz5g`TP4Sk&c
zSG2dmmLWIQiZjlm6IV!{)XK(X1luagwu)%6`j*LG<T<;ke%BZsUE7`T5sB{MeV$KG
z7!JMxp(!p(Le8hSmh<Z!DxXU}_46aq9d-x(?+OTCWsjzLa&uHir3a_WIjCA`Trwf{
z^Q8#&yR@lJ(Ae`Ox=k4)8W(=^?Bl|qX(wc3&<z+H`-ona%%6k(f$oKKe=dNl$71K<
zKAh<^@>jS1Oe0?gq)Dr=zm5ul#WT3#Th15C{Sxw_j~qoW5S=9g?^fmXjSjDjk}oC7
zP-*q(?ANqrmU~8&VSN^OZ@?b?jM>s9=E+6Ac}6Oo5x#`X;~FH_kn`p*qxWJNI<f9p
zw_<7WyyZ{JDsikNJG<a`<`{hEjPZ9@^;+}r48QfnJHKxRTcs8|Nk<cgGFlm%?`avp
zbFMJLS|~zQcHp=uYi61r+?8O_Ag@l!cb3F`G8qibf$0^mm6IRq#;_Q+^;ppfKeouW
zI&m=Z)#A9d+JI19`IV@gIGTM`{5k9xV9q8DbKkd@muA&C;cKHgg*|BZ?3Ls&a*dyS
zy0erXQJPO9s=Bc7gZ<j*C=$t!<e@49@X7)M{!&XG_ycC+dJXDj#J*)=z}0}ViGdPw
ztZNgMd<5s|H&uKumAajNRNJGGk1=Q)80TVb@GfNLl=HTMZ;CJIGY4ieO(c35Tfy3I
zkY|`v>#wQgOL3>ubU4dvWb4roMF}!UybH++Q*0YJ-doJz93iL5-k?N9`{cnDq0j!J
zJ<}Q{8JPJyB!JBK{_~D+*F{PL9-(C5rjvZwKd~1cJRp_K$n<8fnJWKCdh+1U1ow_m
zEqwYaU%TDHnX}-FPnte6X18d+r3^DeN-0+$xo+EeJMZk@onG+rp}%?8(C4ZJ1<WA`
z+(G@P;#@$$FG~Gll?t1cIkt>Y7!{ofr+^d%j5QeieI_K=!AJFbM&4D~r&cy;L1}Uo
z$t{-F7eX@px*x>vs!&<*WNu(KXRnKu&OaqoaUz!C^l17stM_S(+AExSAp6VUNkx=|
z_GK(~TlHRivdsMNnK}YuK0=D{bBc3i+%%s#7@6~~B2#)RZoayEURhG1p&P~o{v3UN
z-|h#_Ds?%!9+_%=KJ6K?P<!S_TDnT94>8#NYx{6XW)|`cPAaQ4m8#duJkm-v)`1KX
z=l-8s&O93G{r}^VN>p@hxss()5%mozm1RN^HKaz?ZX4Mq$rdwHqLe}^iewF4%S6^N
zA~odNx|SNtWNXx5FfnGv@_oJCZ|B^5&$shE_jk_kpE<`l4)a;&Gw=87`FK1N&J2`2
zx6KsZ|L6^Wg@75H{7+Im1Cfuy`?xD~d#zUL8H(vVyVXHo?-F{f8tXfzxA+Q45&q6#
zf27Z^RT)v$7O?&kOI9JJ5W*GG4v&Q-57!U!ZwD?-!op42JkesUVKufT1}jnD>OHrv
zupu(ItKepp#KlduEKLD%`0!^%F<ML|m8kQ<$88#1a<H!$ie@IZtQTXu12BnlJD1%h
zZ<ks9Nlr*vGTXduRcQOP<RJ?&!)3|AKizUsj&(8&NL*w9TYV9${KZ>pzS8fp#EnV}
zRi+6SEvm7<)V9N05l)JNx1#lJnRd(7l{@5vg|#<J5@)0nw_E$qOvCZGz$MsqcTTvN
zqDx$3rY*GE7wbM_ZF6=kq<S!eBP&pYQN%bJiRyF~sdQLJh=(MpPF1TGV%{i8urx<E
zUBMhs#JoLoI@ro`1iKj{KKAI;2+nAYs>bq)CSB}f0c^P-d$}(l9$8jTZ#_`A+nM8W
z?_;FX8MUF1aO*t-p-LEWafg4kNw2<s3zrT5O;VG-EaeBn!55Fz9GC$<T)A#9YM66-
zpoS|76_v5&P9!?CBSa=jX}>!R7LPx(diprQTeq*F;b`w**V5b8!Qx87hSn>RH)s43
zwmU3enZ`F~uMF+@sr2lUF-pDaXJx^ls*bBez|KNrh5%3;PoVpxxd>191jOQ^J_PfJ
z{W~FzGjS;Xu?_|4Lyk)|R{6bbg}Rn`cPHiB%Y&qo@pMO}UV=pMmFiW-E1rB(Xq8+v
zCif$3)-u}zY^b$932}`-lA~}L@vDE&mA_k&K-WwiAhGt~2aB8NbF>=2eIr?#eRv@u
zObGqyJgd%_JG1X*kp0I!>0>bhK_+L~xg2(#F$4dA&auxObF7c`t}KHQj6mX;k0LGa
z^Z`?w@u6`fQ_+e5W&q^Oqq7~#1AAkh#XN(>Kb#Cb-01?Sg+OGK+c_Ozqq1NscD+d)
zrqc$xf8Aizxy$`C!=WkAVc(``mwFg=8=P4FsGn(IUFdoY6>k7>hTz1sqC17T|7)l4
zw=t-EII(KIdehBB_l71W%p^7_;m>Ydf17*fFDw1?_OQ~ArtR$cGynP=#TA1<@TkjT
z4d+2-A2YCanfSO)hite#au@4lZ~3+LVT`Sh{hkX=miKOh?Je+8@a2uN$+&{^mr4)?
zN!*uz>5G+&uhv0Kwg=3g*Lv$e`5%e{%=(aNWv%uC*d{q(BrP6puhRDZPLJi_{tz6-
z;cOg+Y~b@}<*(2uP`4v=eS<@qZ88wGjHv;X7t{c7ya5Re5@`zpgR~6o3<}1U<K^5E
zzClPe8v^WZtQ#gHX~3c#If_Xa{xra(b*j8ji2S_e{gEBPH0fCOD(S{OHo^WqhI^gi
z&_mhR6@>n14>j1ZFh~2bgpeY5>5q@U+56PrrOE)fc*(v#58L|RRyvUH%3TZLOgjfp
z%Tg=Er1kn|Hno>78P{yU3mYvt18JM5atDx<A5hJ}+uO1`FAmfODI9OlvB|535=W+<
z`=7kYLH_fLCY&<F1&<VV(7;Jz6Ly{O0pU@nY8-a=ee38K>(xD@`*)muH^8mUXF>^O
zKB|4tC?CV{VKdql;MRAp^1V@eT+QW)fjZcdhD8q&=Wd7lHWKH9!hL(zRT3%2d+?Q{
zxA3Y@di#baKgD*oZ59s`2_e?~S9lZhL#|$Vc8h{o0w;Rop<#fBUKG=bolr#gvrcl3
z%IlTEdsyYa>v0DpIKMFcc>7z@OeN5CV~?{`w3gun#9!PX1>5!@h)R7?SEb;42=zS>
zXn@4ZsR!hsH<zb|AF*E7mjJ2O3D{5$&@M1QxcS%jo;)+nm1dUD&dAD)EW9%PYz*n-
z#PfzmimKw@g()Z1^B?mUCM(E%XF6>?wr!hzNi(7OAfg$TJ=41s{0XJoC}@Sw8CO;X
z$B`noDeF6s1@Ul4Hz!=<Wo<$k9ir5x*-oGlIOIk<mUktE$_EVJ;pL?~R0m=^%4jNa
zVGMJcid`5Z<4lHK0egz?Y!9z&XZvh=FN|V%Q<~~wa3`DsGZom=7m!6F<ugDxr3+c1
z{HV!cU~hVY*cMD+5$9H^9ss`ndnoh~Hp}P^Ku>IVP*E0oc!2!t0u*jI&fcHoGGj*C
z04Wh3%8SU|5TlQFo|00W(XB5PF~2*_)zqNv=FhU#X#wGoPo*Rj)Ajq|HBrC+)a)vK
z_HUY$S#wckLCdRH{!xPeb*p9jd|E|Cj=f<n{`McY6geqAfX#T5woc~6(M49=4T7L!
zRIXJYYGG0PYb#W~x+0|%JuFm~5}#5Q`*nHUaJxqF>Nnvu^uZj=6Y|{i6fyOT;5k_;
zBYu8dmceQ&8Kd_mk<z&<uVxxpgKSDjl{Sk+<+vH^->DPApjzYuQc;2OI|x^Y`Ghca
zf}?(qV!Yy(s7A>v$lRRu=f)!)G(+Gk?PXw2V+m1X>yTYPqGSnf{y=$_MileXkA;S!
zW;nQiA1|M9UYm%odgc6>q~EVRyJ{I%vmp{LObWOKj61JVRa6|~?ezo4-^g?qyAbD7
zSAV^^>B)0Q-(N$nH?)ofJdiKov3^Z$UDc7T_YFv!eV+6*r@>)!jyW?AI$r*=(Qj0B
z^Fd$?h7URMqAO-5>M2{cZcxrl^a`RZ>1SI^-)1x5=78fSrgNp>xS^?_?M`*tU2`qh
zn<i?O+I$#~u?a3Go~k+%I`dCf1;3J%9soc|V{&?q&+?P^QUKfz2=}QkkvHbBLt?cH
zR9CsttSAxswm$Ql^UYE<zHqqT?FFfa2(O@ctcX@5z8&XIJyHy!EKl^B<KGUS+_Sc(
zNlSmdxKtR-KTF4FwMkdsPU8x*K5Mu@6a`l+ON%lO?0Q%2*5p=MKNwmD=&GIxH(|CY
zL{QfS{#HSqBzEZFiRsBFSY<3d4%uY0RTY+Wka}D89&3O7F6f#(w;=`7f<{b?N4#hk
z7h8Se+c&z-+Hw={K`9R$!HqX|k`ulD^ZITxamHL!RnT%#OSvTP4Jsl+gigc75x<{v
ztvqtC9ZqEnU7f?01Cu!gXQQkou(d#07^x0A3gN`!*`OY91H9=EP5P-#CH{=zj;3ON
zhWxuDiTMBif3CLNI1GTp1YwQAaRiOgZ$x?jd~-_}Lq4Kumyn@2spA??mg*d*kLK-v
zcdY)VT>XFjsQ4QwpY9MKBEpv=;R-A$^2HG8jZ#JQ0fE97>M&}_g(UqkXC6wsoDLP$
zWx%?UhHy0+s)~i%(W=Jd{q9lBZh#k}LAATG9RP$pxV#`^bffj=({|Jn=i3VpT?vHB
zd;o&_Q;dS{B?Om-!UXbCe9(KZBjD453n1$gZygZ`%!-E4*}lCMgpDuP3S2RVY{x(d
zt!ELJ-WI3j+4Vsk1=tl}q8!gVWz4bBHWe|+3&DGXfNPg`Dzp;^1Z_4TFrHlKem(;~
z<db+Aq<m20-y#Sfq?TgzVbAICnn49*NFjsNz${{mbpe444hPVXcv#*PqPk`<^DQSH
zgD3U@RMZ@+r)KTIl4nxozf2a==`){oM!f(@J5&_ur=feeB0a*i>IFy%Pp8^4jwo=}
zvOesOWl}arK{Q;pP=f|{E$*u}0B+831jS_HWY(w`V89C%GTMX=2fqZ-Abk1wb%(%_
z!ciZIiXs7ld00yr2&r~uy*-UYSb(dG%`F<OpW`gTGwZtiD)|mfP$gPtLp)RCGRQ%L
zYQ7EH9Up+~FpdXE+lSTDjqp3bG714OwgcVIzGNDD{C=r~8l-M!))g`XD4!lFTHxf@
zvk*NObVhD!r$|XlUVxgk*H%glkiXg?V3*T!i4t7_9)n*^f5a3|WFyiV0JNsTBNGna
zN5gTbg!eTI$mw&VYXFX@v*N!m^~b~l97aPuWFHxNoZTJ6(Avy@vU1J#R~4hEoDV0O
z?vqSS8i9td`n3KQ9;`o5)q+FGu{F2+prK+8Qok$0l8OlOL+vH*N7K!bZb*Yur-pmc
zyCmhzK2Dc+?uGfG)X0}qk);5FKR|$8Zd7hu0z82~LQ8-<#0G`&o-gY>Kv~{d1{K)$
zS1V!Ii2(G@P#MREW0V_^*&txT&a?Cl@T%B}wQY|xP0j+oFq-FjbF^}PjvcXw#&mO{
za~u8KzYVoay;CFKcUNHAqv^A27^<8rrT$uH)zGF>$!&-xgo-%i(v;gS%tE#Fw%rGo
z!~%N)o3TNM5+p>m<=Az|zi>_Z+?e17@rBQ#Bl?iK`!yZgb=2uO&^YR&g-Zmc<sd&x
z!CrMr?t`nGp!OjvNnWoOv!m5otw1bNq%Xx*ycDX`x|v}j<02v{>5i^D)B7!tlJNwl
zU9PE33O}M<2*pJj=Xa{OI!c=sbl`F<v|1XCsjxc2#kgl+*XK<Q1|XU+CxYi2AQ>eY
zWHkHmQsiF1_`S#@^81H;noqYyZM^vXLagyD8`;SRY2-W#Y3%S7+BpP7;=oGNal*!;
zz}hZlZU^9JUfODRT`G8enV-n%&cBWXpvqM5JfkW3J3P#@<8}R^F-A`MS`S0QeKo!=
zPE(8)k-ri5a?7g_s}f76Oc5#7n!VQ#Qt#J(KL@9Vz}j*1O;>??tHI38$(5L)Vh6~a
z?-DPmS8W+jVF!ku+yEq?48Y&gDHghoX@KfD93?kn0<#GRQi*$^tNA$-m$pa@wb!dm
z@_y~MBq?ZbGrzy|GWOp7J1!Yj@~wDNa?Uw?4+V2AJWlm;y-x%n#EHy92&4Z99mRe!
z5u?yW9pa?l;*E+{F2EC9aj8^HTeexUVWHfY=1e!VqFF4P$&Fbmh;@qbZ@upU@X%EA
zelfT?EUc*WmpD)gv1koE9Y^Bg&e!Y=y<_C$WYGsE>}c=<K?`LNA2|oe&*HXDo)zhi
z<+(ni-woM@+ts$))nI7@VPJB(*)in~X@O68*jNas(vq4_CfVM{I^GEr3=tVj5hSL~
zi*=i5@#0;zv?k(t<^GapT03mhs9&jRaiZFyN2w%jdiVw!#hBnx{Y(Y1A-P2+jO=u0
z@DFygN={eILG7(fhtT0dtDo!CdrE5;O5z`1?r!d1(T#(eEe#y@y`rdVUpnJ!Noqwp
z?_hw|c9Trlu-u;^Exa)&cw@Eyk}7|__>l4`ZPFoUdNTW`jeOqa3Y*#cDVC(fvHsgd
z<gr0l6B%89jLL?EduxHg_`fLI|H$wDr@uToFK~Kmm8pYZtP=X7HZ?wMl#4kT{BLY4
BfgAt;

literal 0
HcmV?d00001

diff --git a/docs/reference/images/rare_terms/accuracy_001.png b/docs/reference/images/rare_terms/accuracy_001.png
new file mode 100644
index 0000000000000000000000000000000000000000..2aa1be316c382bd8b6ae492148cda4264c917a5c
GIT binary patch
literal 27086
zcmeFZbyU=C*ET#NAPyo(h|(b4D2N~(0t3>ew1Uzo&Cp0IA>Ac8z|bI#AW{lQ4UN)`
zG)TSYkNbZ5d7t%u|9r95cdd0TWcbAy``mjU`#AQ5s4B}5z$xG`7>qzcUPc`TyZRIc
z!+MN|3qIi`d?N#XTz@0~%mD_w*$V#ucIB;1gg*>s;iDiUsp*=sF@@itY1VkLO`E|=
zE-gvguZkzhZU4dl#`}5$(oj-t#`YVJUPxpQr4P(!;ZXQJ$s39`PTkAsm&BK4=d_KB
zq?VEzk|2D+YVK>_e10NU?OgG?ujrYBUd5{)OJ~Rp^6yDin4%tDkr}tNr5fc&=Y-I>
zVDc;e>RZr1U@~_zuR}jQh?1ZmE!X~E`>PN9&zjN2IEA3CwLzUD+ddj5F-FgSC7;z{
z@q1a0V!XDVcA++x)j)Pgl@+G9X)lLo?C$H7Tis2Q4t=6%`sA+J_V?1GlwYR^1SgGY
zFOE7kGJ|N?|Lrl}e+Sdo3-Z#+DeW4{EXTIzI&3<^sUN=lcH`urrrAdP;<&F^C0$H*
zCVo))@sD1ylXX-FhYq@u+{okSlL#7vJ<OAp70-jzEsqYbi<6n4@(ry*?J)(GRLwsa
zEFXk~IhN{MOAOj(=HmR&ic{y!Kz~k&#{$PeD?X2=`)23Idj}(WcC{HA&0c4f6JEPN
z1UH5>RH~v?#A+?aezyfp?F14~*3BG#xWm_a<~l_3XF!o>%=2(6-uYyun#@$(v@cEc
zV6%SfIoSD0x5#(b2*)39HE7s8%kn%q7<1ohe4Ur>RX1Fq!Lnh+jBnq&H~!dVv+_Zo
z$e^;w_SY25b@pL5wjr-qYh$m|ok~&>k!$NN2I@9?7(r_4nF+h*jqojh63d4K&Ku)q
zU|k&APGA|tw~}DL()l^JqSH^aDVwr!v#|W70$aboj1XO<C?9BacX9vo<=V#SZl6oU
zM7538*~wu|^Z9<s!Sbhw@dmfu^}025#V5Ilaju%l7vKn7n4BDLxkS`EZ&)7AcsF|<
zwbAxk&iGk#OnEu3kNy<fFHoudRvL)C`lwfEiAt~BG=!4o=!c!xAax$a*WIPQ8bzMQ
zA)Bql*J)*7`zF#aPG2ADO3o!&q2EeVogq2>QYxf_ZjSk1Cz%wZdGw}bV}iW-sNHhn
zW_fSzQrcSh)1@!Fz*@$b;`L{dq)wuPsqs?lc}T_!U;txvPHU*tcSh$s@y=sr@md?w
z<xQR^<91%Vl$f3Zn`tMEf%D)~L7ckYM6_a^&02z^p`*uaBM)iaP#cOUcP3rTv(9tB
z(8y_RIfa1o;dkvKT{S1Krjx0y=MglblkR5y8IpIWYhgr${>(7>Qn%e#kJpMSP)Plz
zgQ}9+Ol3=duf0#SRXRqV!(u06edEJAMmFB3bF@>!&t1mK%|=jK92nir{>*oJ=V5jp
zZ33njYt839Vu!VTwP~CJBg3j%5m81eYUOkjh#K0YiX7uN8&#OPnL)n2)!`m=-70%c
zFPIxSX7Ep9pT{@brsG$7&0dZ8x4<GUO_f<DucrR|!+oaw{x+9&cMO}wp+Kk5pm2o9
zPVfYeKntem*-IV$w*kY&dNn9=-cI4wr)s70*`*m~@Fp)z9??kwENAy`kof5r?z-hH
zl;_db#krl;bVKd_8ir><Nd1@U9ks1tVCyTrc9CmEX?aQ*rm5=azSwDUaWGl^&ViFA
z!$#yo+sQ{Ke=*Irz`C8K<>bPtR*(i{XtK{~uCv!&8pf4TG;FY74zX>RX12{lwU&F)
z?`IB6y-qXvYsrp%u{wq;NVXy4nBCsTedy+MB&KvIl1V^**wq6p#pLNN0TJ1m@xmI3
z-RK6pIkIniT+K+gp?u{OoduTVYW0mExxT5l9n{<s`5jrW`_j<lGQN_c1k+`xcuAAF
zeT<Py+Yu?6-&Aqvvo=pJTVBd+eRE=OB38#$LXls{2e>!x{2Ec)8u_01z-DcvSgRw&
zA~j>+(!g2+0v%8+-S|)884=>=pTnphKU{btXE9z`utq<iDsE?}7rA!2qT($)=cG`z
z;Yb=TK&9Ei{_@+0x4t%QfdsbsS)sZ4Eb1ln(t$V1X3jQUoVOYeUpW!LGA6+R)ui%N
zJpV%IRr+m*&~sV(pmEEy&vRl^O~qHR$m--^11(vEi*n6W5Zn0}p=lLa__N?iDOP7|
z0DfVk@~0<Z)*IthzK4RzHcVSVa?)kQlz&ze2dKz-c-?x2C^Jn=EVDVV&F>^2Dfow!
zM7eCnE4w~J(?&}04{^B#&o*D2H0zuhNcY_@jt<jY${{*gAqdmx|9rApKlPHpeX`NL
zW-Xxm3D|T@?|xmj{qF#BZ5-xpq<GcE>exUNBkjdPypb~GJMo^v*u$BB*nwqPEkmaA
zZ^_JLvEVZQRAt-ecU&Fn65!q6sBc<Pi4z&>)$5=<kCmS}f4y4V<)pU~c*=AQm*5A2
zn>f=f>$i=tT+-uD$GrHrIp2ht+DHd*RDQR+c#!%)pi^T*Ba3d$^d4}9U%f(+fuz;L
z_56;@s)8cQqLRS2do5|-=`VSp71>wdM$4iZLPk*yKfY!Vz+(OSu`?K$ydIP^TX9EF
zq*9bGOr2^-9}MM>G$!-=cj1#I^<*70hBPdRtlo{Z7|hiwjEsM8T^-v%d#w8M_+Y}$
zMVul$mzURV$5b$TO-}r;TKCL#``(5ppV<$N8=Z5V2Gs1Hme231=bw@8&lX7dA(Qzr
z@`uAag+2+&{c9;bPF(@y`3IX*dN#GlU$Be;5YK<n+SYG86jV}lc+P(Vu`6llb!;XW
zqq6&qm~tvfKRpD=Gjug$ps$O-n2bP`S5a^#Ct(7PSjZDNEp*wM-hA5=$2Fx-J0*uW
z`KTP}kUOUqTRmK;qw|<%|1>|}7cCv>G=G)$NJ1+jQf`ScT52OtuJI|b7<-3g{vgEf
z$9j_J3ika4djtDSA;YXzx~tc27Ck?IZ*-cKQF0Zw;M0!{J2)pQqCNd}4egf_O3x}|
z;Tjt#&Go#pQ;TO%^sQ#n7>7!+r4_qds;xWw>tY+r{5W=Wfk%~XM&X8g*sqKe?R*=Y
z9O7|yWJVgfmYqgP+qW=EgUlNxZ=~2I`x2$TsCey==#=X%ogOcxyEHPPZm<wg@OOR=
z*MIW`=b+H&tQ+Tg?9fQ*L#_IrY}Jz$wSD1O=5&6GD9_*cMtJ;1s-jO&-|~KE3L6s8
ze|JnJOY+l?*PQ5$L;NW;4wjnj5If*Iq!}cVDLH)v(<GK6ft^OPM_U+1qrMlY72(!W
z6rpGi=4dZQ_vt*2dyJf;_q22+b~9e}rKLK%yG*;~CCisgbCBVy_65Q@^Me>BA~iZ~
za55RLg9vWVR76tPQb1pF{CFjQTo0`NiEP0izi`=hmZ6DH_s2p&%nCo)^1e9qE(RWM
zv&>LB%!5o$-eqo7YAcVG(d`g1AOM%jslQnAKJOr7`f0_1hf~`tiB?3}EJ+=54puvg
z4V|DcKdT+R@Tcmp!_=ODGp5GB)Mhswv>3#g^yqDtm1EQPH_FYX$A3o*9qsP@Nd5Uz
zz)(Z&o<^9hK(AJ--kK%l$|AJ|jZ$4LuVi)3s#QVubM&`2_xEF#J3a}CqXrHZlkMu*
z?co$%?f29nWA;?Nsvo;nxiO!G4QEIkv6yLUEZoUucV##H6+1dI^8k25U_@=$WS<=$
zjMEJ{<Q%J`wDa%(CY~3*R?A&bT|9?~S#8L#;ohDssCg=?2oHzH8Pdalu3<^!(D7f9
zz_n~(iFee#H)5svIt#a0P@K^yU{agNl5uI^(M@EpW5tM4N3;dCjUWcsBlE!FcxU0@
zPaC05@)D_kbz!<`BHAHxEy{Lcte+gKt|K;ZVMRceUOnf4tQ3}^4&}$4AK`M7-WO*F
zdxI+Vlmm<EFoteTm{j|H{!_KKt6FcQ4*S*ZZ2EZ!V01TuyMoQj3*wFZFbX21g(4RB
z{w8V4Vztd!c{>!Z1gTU5=qrmrl9)3s0K+2#iMJ1x#m#81StSRQ<}e|Q{r@wm`9JZ4
zW`)hf!;?1JbK~^m@2}Uhqm@wylLs3UHeFFn2h;9bV~|^`Kbn0%XuDK}F{}WI_D@6a
zGpCdEi(^`ltnP+3uDPv^mVNR(nV6~ey?QNh2f38yjz0SnB<{FWcc;A8P;!K@T(tR6
zrNGRg>Aj~7BJ*kT!-ubXQ?jTu%KCKp4jU6DaEWL^2Bqb-6P-S))3if2QDqfxMJH%A
zQnXq-bGj(_(6|M5QsaFhh>@t<sGhL?p1=o^{0}h`bs#3$>@NPWO@5u~q^%ni84qID
zMTMyQf&T%rW*S*oWj7<%UkBwpAkV?z5>SwupIMja<nF(-?%O&e!5K^0vL4|g|I6@l
zYr0W(%=tfzi0;+F*34GZ?-GN0XYFQ}rBs)?z}oZELvVyC`mDx+Z{1rLKc5qyoLfrw
zo~{EKFo^691S_U?O~<(gjz4A5atuE+rGwT-%U(L~bg|@35{l+I4Tx^pNQ;AXQt9~g
z<2|`al{y<Lkk_Wt21J@xSq|HKzjCJ`q!CG|e$6z+7eOU-f8lR|;YnEsP0y%C4C};N
zti_p9-)rQ>(e`7Pf@#7;w0GTF@~X$0r+w>nfeQ|$D%;#c^!(o~6?JN>+(Y|ZjAltj
zZ}ef4^6mKqQr1;WiBVOU=1Jma<2uWr)A!1)F}OYVJNVAsm{m`l43BLdir_2z^OFF4
z9Gq==&!l-CJi~O}dF))YwxQ;FykFv-df);M$L7ezR~9i75@w}R0NqA(m-@WVD;jsY
zIS%HdRkXQII_1P((4PMZn(D8ge{z&3erADrkh#HaP&ecS4CVPdTmtL8<)76a$M%>O
zD5+e_`8*rSF}aoEwwQ?iat5Wt<5gBmUMyq4;!Hq<DmUwidzLEfH2D^Qris6YTU%)}
zpWl9RH|)s3*iO||o&CBdUOL_AUcXdhJ7sBfI!iuPJ>0TBm>2OQ^>NK}mC>2=1HrY4
znv!Du+VXKXrQE<z2gmg@XXq)H8BcB*e#@awX5|#~Xy&K(YYl%_3!6GT{;uRr0K}tD
z7(OE~v@LqJH+UI<iZ1|+%V|uj)h|(6E8LE^aQE6u*Y7nz9ZKh}rLX*DPTQhATb6U~
zpmzC8!&mISk`q7feZJ&fhk0r!IOw!j?<|=uJfgI<pj1-(q8(m;Fm5%~d;tN}S3i=p
zxID0hm~E^Uzb76*d6+FFz(_7zv0c$D6ZKn7R_UknEE`MW7q$3#)5ohr1=09}UFiBv
z;To<kN9qR~?pZ3Hs~RJj($@}8XlyP3PIWZmU-mE)j;9H5*?8aD9R4(%MsyV&QNuad
z^crb&h1JNPnGV-^YkDJ4vEXM`T51rj*J#t90G?3+jQ-1x2-=z;8YJt+mZ9hVFq-~n
zqx)gyTK><svmq?wL1KHEmOXvq7aoWD_XV;8Z&I%|{nYxT6~-=5+V^Pg#zfi6AkSY{
zY2Wk+46OsK<`Bjy&^mOadQVtOB{)A%`s6PJo}W}BMJow5-cd*i-~OS}UtQU3>2KsT
ztcBjr!43Dih|y3kOMUE8w~-$&v!rmP?~~1{M}ZE@wg~E0oIC*oP7%n<#<FDJ>%UZ6
zTN7|BvNX1^jhW$V26zFT8Mi{go?yNljr9cs5LqRL4FjDXx`|!)3&tMD%^@P?eXmZ(
zvj+x7TOL-A;;`OHWh|DR0<(7sb0px?xGMP8;Oqpf>lxZ$=Y%;{vm<Gxas4nUDLK#E
z8y<!<6lE~_I}oq08=KLQty-Aj2oUaP-kjY+C&%-Po%rU@FWtd9S^xR<MQf=Lm~XX7
zTOjJ?Ao;_;jb3MewYYZc!=XgV^H3CkQ-ejR(1YQ%F}J;+a#J(zd&{?$^^pLNQXzjh
z_M9FVY^^D6aljVd_hWzcr{JDeeI!6U)G}RZ5pCsjw9k)|i5nA`$w+vi_EIq{&lIlk
zTt^(PysDrJ@E`ye%@rCdqQk|XevABdK7-84;NH_Pa)BZUYUWH{+ea@?Oo^XwM`%s-
z3$FdGHkkOl5T|FoJ=>0!>!I}egxf26I62p(g?V$=^Ige|-Q?#ocS8aiQSG+Q6K1`o
z9!lJ$RJlfH)VC5ZxiFrR%tG8KxB@HxYVx5{M+)bZz>w8i)ezAQb$$9v_hnr}?mttj
zVUF+;H?2F@F?2J_SmUs3KC9NtrGK3@eWTG9HGSjED{^hwW#hiW{`8?c4+re1aS>p5
zeU%PO^SKi_@!qGv`c|2T9of9U$llbR?(}RxWgt-Vm}<O6_I<qCW^Iv+nfAS4{L_;+
zpe{3UapHZ^w=Z_BHhlGLd*aX({tYe(|F$}O%SP_X0KSZ5uNhGySmcQeXpw`!BFnts
z@i~vye)DrOTuOCLx=l06;#uU|6N+)B)kOJh&*^eA#o?FDA@kD@oA!(CwA`P1x~~)H
zJoY#$eVQsf>^WE*SVj?%wVN@Gvn4=cBwl;SIP5l*^3id~RVkZ>P%9E2`90C_alBQ+
z#(sCDR})4&)p4me`UtYe#y@c@e?tRrqvRwS?@rY@IfU5+m=WwEZtn>%x^G0yoOnUL
zqri_j%a1zk>uRiIgo3D28ghTNHX|b=<um6L-)dtfUEc3N`17uk)J@@4(O)%NT|;`g
zh6Vxb26ByrnvHyAE~1^aHy8$+ymymlZk#q@se0i(l~Bq~c0Hec&-1sp*!7q1wfoA?
zF&@{Rc5C0e_m;RSux7bdsbNOJD|Oz^_>9@yoq+1c7xkAvKGhN@>16yiT&1s@k5aV$
zySFk@X`z8}T^}xt*Hf8+v79F=SI@v%O17w9_P<o#uRrcSK$<-lpIJ*+u5(yY@F-To
z7@aUAqDv0zpc2v=j+%;3cfR-1<C(XOgsVdGRUpW{Y{GDF>pp0Vf6$mK7{m3JmL#ZV
z0jV@bH5bFCF?q`JBM3o-6~7|f1WcjnBrO!lrq!j!)Hgi$35(y5ju6Q`sNf-PHhyS5
ztpn>O%)A~fbheasRBTgDM=u#hRD5_cbJ2t<%lvJdbP;*PWiLEk)A2V~duGR-zp^7)
zA~i(t!;_hCtLgW01ndHfUfb`DJ|EvrBG0l*mu<rmEq-2Q?~PJhBoH5{S*5W3Wb(B3
zAc66aVx>tlNl`8MWv^B>hGX#WvEhZ|Kx)0_eFJYhny3A1g+lBUhPjyre}DE%5U42H
zPR7j&qYWF!4;xbyK+R?{o&Ce_fNMM-<jW!ya1=>Z`D<8?ov$XMKk4j<Pc`LbnS`N6
zkG?<X`%J@v!>aOdNnQF%6=8@&QiEYV-3hnk%)iVOhxy0#P8qFC{txXZ-wX+KJu6Tl
z`$_4ajtNkkId-Z7+D2rR{MfHMBkAwfOw~HP#zYR+pr;vav_7fddpa|Yn^{Xg9v*J?
z#Sh1J+Go+s9PJNha&@>dG^#!)A7);X6~}B^SeG35J^#oDWw|n8N06uXPHDK*KIi&T
z5@A*#YH_IHPb)t9qh|0+<?QJzPsI*Tk2=Ji353b6%HN(xT0}LXYkObk?hE`D9XvRl
z7Jr(=-?ciK6SeZUP>_Ckf4HBEjIiGdWFlkepM?&|t4-<wpToT{?6FV`<?mByTOMl5
zOd}MSuevadP|2K04&kR&*bO1QqZy$7`#EXd{$F4}xt=Pr7Z3N&hlJFLhD2DP<toXf
z5BZDrB!Fjc9PVxStG9A1N#*t)$bwD*aCiiA$fAb@dIYL!3+%B!hXt!FRV^OHQIMnT
zb8ssuh9sB2<yk6OL^XxN^S=~-vzis0O3XLO%6w$$Uw9?E7TYqV;7gGSza^)|o$v#-
zKCArjMJ<o>-E<cM$Hp|8Li(J*$R@rK;p2WGwK>(@v~y4*8$a5fGgv5+qOOn&d$hlr
zPhSvNfXG^-+;t?_R>COe<PQ7nVL9dQKM+k{Ig8$?Rczn-s|d^pgqxvz88!tj(qRjZ
zTFkqR;fZLi@(56AygSq5HI$$;{5X|^hqNH&GrW~`^!9ARfZA3R(wWqY<Z9_O?X(T{
zeS;1qaAr`rD00o;T^!g9V`+*TyP!O_<rh7@G;=b2s2?E5b9%ht9?xx14rL8IP}a~u
z&y^T=+@*>0v=JYbmSgvNxKr!uS3C0=s1PNpeN~&JQtRgPoRH!A$VYMvo#&QWb%7J+
zX?_Qihhp1JI3rEV6ZTrj!Gp%kM-)=IAv1=u96!qXYnuNy&eKHd_kIc_Dwr&#I9t_H
zNMTK1Ng!ELG)3C%XRvCTpCtV$ai7ljO;FBVi=B+@sr1TSawy1Ja@kZ*C2U^fVbA^L
z;z`nX_>NF)d9fwmw*;en^CDzzacBCee%#7)rVFpg%gp#_=zMjOT_#`fi7eb+l=;WR
zPZfr<CYEgvv$~%)xoZg)Ga72FiUR3eowa*dAM+V9TVs$)Rp|3#ri0v*jrx%vd%$rv
z0oYic^{v9oJ||@K7;(xh^bGYRlsrKspy6k>POq@r=94TbstWTvsd6)3CjwtZ#|%N@
zxa1DdhmcFbk%9Mhq~~zwFfS^@zA;>~M5~+*n03SP(-qRy#>DqZ7(S4x-Ai^TAc|S>
z{9Us()>HzExdtj1G4y1D5^leqUy(Sz2Y_@qf35gn&4PTG!!;Ny3xHFdYSnGse|S<J
z!z6?N=>h<s4<Wb>{C5O^_y0Y=<|p)50t8lQ)a*4qF;?+9VnP|zCh<#kpkZWjA7rjT
zVH*_w>-!ylwOpYDI0K+Vu7$eS>7Ld2vLPdA11kW#(twBUe)8dtiTmMZRa+3@#NNuF
z4TK-OFT5y)9rLLlJAYptE_@B)ywo?XCwWewT>H>^lvl6XIx*E_r%P5`8+u~^`C9)T
zDBMkCNWQcF-5LO~0%KMsjjzZJT{HJr20K6y)v}p_=nVinevb(4et%FHm+m;Az-?{u
z7$U8{yt_65Dpti{Ae*nx{_c;|Iawu(x>tu$2@S+?K6`N$_r{y%Le(6fC&pOdh21(u
z($Rta1xs?i3LuHn$5>?>$5>;~2Ae84x$}_~Apf@-%KwP3Gyqa|@(;H;b$=Ot0%dU9
z+H;TNh4`tNrQTF4F1_kXko(+TS%&7|Pd}1b@OJLo!#BTLP#wvFcGVz*YDXS!8oV?J
zU?wPZU(O7&24#{>7r8Ks1!V3+n{iFD4<g70lQ!J7ft>f$BCh>h8WEa<xle7NQu5Nu
zN?v+(fks{(y(a|cZgV{+-FJoTgVeA{S|ba*UH9VrYy<FA)egNvdEa*7B1wbU^hk<V
zHLre92%g&RuMTtXnK!93kiF|aWx**`h-R++?f#yer`m0IaV&xFrS-QDcbz9}YMz1F
zY-GLXE(cZb(e79ds|K{!HSnVCPt`25nG90Vei5MplYo^$^GdUWJ*@=HN!4mu;BX@-
zKpgyfM_6Z=Gga@Rqh}OB`Ow-NN{vL-;qq{mq&dfKw!BeLcD9}S7FOZDS=%8xa+(W>
zqi5ivLK~JTapnNF^Gq<S7?!3x5%Hrzxxm6k_IE@MMqi?rSgFbUREY*kMN|O(n=A)>
z*;vuD3RH0ky*vDQ7F}*S1VI=`D5hCZqf0*`2GUWL@l==OtBfAB!NY9d%{%o`EtTd!
z8OVb*M<sV7AW-Ihe!OIGB?#<iZAQ*-4-iot+YU7YRNDS%irTFV<k*5Wi+zxO-wdJk
z$8s<?ELkqc#2YXqZ`&E6osNmu4Pu`9q;QDXLHM}yFUbc`kRtiL6*3jtR%}22#??9c
z{oz6Ho5m-C)2^Q=M_3>>4X}<AUqeU?wgv%dQ4DBAtqG6Q?T)F&By_`0H_8mIT2~W|
zjJIt*J9u-G0=z^*vR#<JKn8@KjzO_zfTS%W$gG)F(yT#kC%n)v&wiF>3Zrnv8t{j<
zqlP=YkF{15i-uT$={vOTljb&1PD6Ii%2PQX%{s|AJ^4@#fb=4;@y~2jzHsn)K!yZ`
z3-AN>N75X+7zP!4{{LV9pIr?JP9^m4_?giDQ5^Bs_1QpB*@qFyQ9eIg+e$U$n(`t|
zB#()Tzm=jY4{esuXAPfFwKoW|7@rpmQ1JNr^aYQ`Z2A40%44`p$mD-q1(Q)niz57)
zNGUEA2_Gd@rAPsr1%u5p4)-^FU1E476OrMc526(eM(2pl6hi{VBMBxp6FU;g>kFZm
zV<7jE-bMIl_&>S~QeP91u~IiRa&918iUjJbtfrH%pe}>iSFJh|S6Qm8cpGDO&4{R0
z_1T&0%B+l+52T85H8*@{QqV8GMSPV+^~OJDpeP_yBGLF%vQ3?Dk@Yy>a@1rbNub#I
z8c>>qm78gz?vr(p5Mfma%(xRTQ_3*6MYHI|O-W=No)T*|a2k1O$nI|uv|Cbxpwwv%
z3VXo2R>rKM|NHRN)VDFyJOc_Nmf-YMfY>s2FGA!6OUj>f3&<@&lyV%G-Yb@L4}0^D
z8!#BVQe>tZ4kzBNY?lH}-;h%O7r{>E$dip48$dKyzxgdyzh2(64wYf?c^diw(0bX3
z+(%WTv0_J3AG{7Hpb;4bbgMaVkuKt3TnZ<XfA`mlK`eaeeqc>0WS`CTto$oPYE~Eg
zON9Cuvq03y8pK4;o2mPPI*H$3SyUXZWWQOvH14*Py40#sPR60F3+iwBtWh_qg+KqP
zF#dv74{+XqSRTfFqPkcQEp7xjs>8!tho$F$eJZw_X`-iM1KaAyaO1~IRvPwaAc_F5
zZ1bzdS1Ur$ZrZw}VJCEdbE^JZUGhU46kMD}bh^~!oq)vv<Awu}iIjo5d!5;8u_a&)
zZDyLL_C=u5+e4>SO~GutR{!fGpxo0+3p`}L_1KNt9+P0h0%Sa%VsMmhP_7IJ_+{s3
zCljEmXS0;zpd7YDX3fvS_7N_Y|L5I|1Kb!Y@U<P@<}tJeW#bCIm)}M|+<mZKJz-nX
zoBDV{mfXM;Vq{}EbVgha&Xk3{`hcy<UpQEC?^;O@>0Z#%TW(u9SQgbdw;X*anc63x
z={Z8y>bktT_^_sbH1gt8fq$u{jQR6MWEIEA8;pTVZR?E!e)IkoP+~K%Nu&|99_>G@
z1#WgMU+t5=gu3ehRi1{jP&;??sXe8jZORl7zHF8)3Q`L@cCL;T>)AYp+ZQaiu}lFm
z5=`sihr;vEgM}l_8n@T<ejLcV-DrRB{5XWCd5#IBQjiLyQ6xh2Ac12tQW+>RtVfET
z4aZq@8Upk4OeewNJ6oTu9VAQCZSttqLx*}A-oiU8K)827r<Lw?t!CS9?Q0V0ZJ!~?
z5{i^fo=7jM<K)V|zg;Vl-Fg!WUk*{47An#kn)H|9OIf{8yLdn>UVh<wo>JoMqIw8w
z(}HFgjKk(%qgYlxX``Y)Q_TDSCYEV3>t`y6O8&dJHT9=H@xkloATOEPIud1psP%fC
zL7)P7R;6?Wap<hRHLy`LL(?mH73R}TR%vN?8;3rNkCc$L7(6}}Pu%O&b$kg^tf#Y4
zwmFp@S@p8xrVby^Th`PWhb(r_kl4T3AQQFyoYorS(xwLvxgTqPn5C$O(IeYvo_sCO
zyN!xI(u!gV{}RH)<HI-SvT4DE`TeR51w7f_X{Feja!?cteg~7@C0^nx&{1eq44Ir`
zn7kvXVa&iuzY4KPu!USa!F(=UWIPf1rT4kBi?39GOoscG`hULkUorKh(yLfA4s1U}
zlkMA8MZVsGb|E7L`%Y4aXuMDBKfG+Pkul<KocdG+ThBBzc$mxO!|daTY|C#dxiN5N
zUUFQpZrP+vH@Bc1&Bmu<cS3^@($EN5U&^i-ARfr@>@+I6-7bfz|Kq)Eu{Tum`rEd<
zvs}8?xOUGOMKUG+F=V>;za4~dnA81;K5iLAX)XmBvXJ6k+6Hqg_W@Bz>3mR&Kh{5<
zk~t4N)pez3r2fI0#)zsuY-UoT%S%`9V<*1Y<8`ww#&mCvUX;Ly3rZQE-O$6fm%Usz
zUagMgzf!A9p=#^DN02M&5&Fi6AVX%-JC}E74aAaQ`WvKhbHBPNu4<`osb_TtG6gQs
zGhR*{ZwG5|h&&wQiq`dID`wTse5p$#C3E>)qUPk`+v*vmnzmWD*NG@EFAv5eP1w1C
z>)RUDCz{OQYvP7?Nzc(?0jt)O-rZ5VEm-iw-^7je%4OIHA#;~EjGWG)@MU}N{xFuR
zQHk~bKmKqOz@)QIFH|{CbQ3;auenN(ywM{aaw)$<(5<z^<mldLSJdQgGz@0CyrJ_x
zZm!5YKPgXYKDmEzw-?>P<qFex@w;>h^5Wp(m8}`<@NyC|0R|Ta!;{o*RD5<*mcx8Y
zy<DB#-&mK10b<%=s;2D};W|U5^Lazkd&_20otICM)r@S3AtgrKj%)o^Dk6h@X*YQG
zHx<R&e){<{vAG(v%Sz&bHIur$94AJn>>4fEVEsH;=9tU<%z)hwVCrJxohrMam86Gj
zZMSmbeF2ZUak&SClC4qLaHVH~QBjASWG{nh+IEN}FV~Zbl%LbAVBUnydhdSb*UP6r
z!+XG4o9U*2phyTcAo4SD3ue7s0a*JDMfROHR-zXs7G#)uY?*%;Qdowhf<1v^q6{}D
zSs@*Eqf#*M<>gVV{!BQ5dKJ_5A@h%AB4!DLA3>kDh{v*Xno%U_d<3)iht^x7ox@o-
zN}^G5AuAGx{!U!beIU|<!N@p1AY6q4auC-Ez|JH>se?}_No?tSs7Q>}8&++}V8rzg
ztQ^oRRQVXl;$~TsFZb{+WX(RK?-9CDAAAdl{iJIT(3vlwoJFDtm^Tx?;sOl~Tu7#b
zxRs&%HnconwR2lYL(+0Sck#<a0Mn6%o=K$W<;zxX!nXN4bJBf8wG3JsI_IdX(Na-y
zK=awoBr)&am;3ozJ6yo~+#?wf^-t48#-4;xXsu3lQC>b4foh34dR=`dP>1eDPvITG
zDaUAtbxsnndUcJEh7<76mmB@{NnJ}!p=wW1yn|1tCK1C|!>UXNar500Z_!H3`_Sj<
zDTYxP(eoa=Azc6vCDwj^3IfSZ3AVc?)#X6(18ykE5-PsEo3z9-NYscGD1(zT-w}H!
zB}>Ye%@!JW#yS~0uhw3e#rqujww<VV#Hvb{%QgbPm4Kj$>vl6wV=<5n#_DEf{W;+;
z1^FqT@>n}#c7=~Yh&$l1Jw6Fm5$ZYMh&jAheRO$m+|G3+&959$N;vdbxXgaUS(oW%
zw=&w7Ey{%Lrt~hHF}_qrH8c(&Z-^V5oFSwYv+jxKaXGXaXiNfRn8s{l4>hz<F|qns
z8m6u4OoTraNp4<8Y^&!fQ$Ge%EpYkt>vH_orj!W`BysEtD0-xVZ%Br}{iQoJ%e><Y
zY`1_oaPe&S&UFOq>&pdk8^g}+49tlPS|pnldm<DR_cfz3u+)F&(#e0+?rhQANy{N3
z-N`L6nU;vl@MndxSD#KeMMC#|PO>-Vh6%rOBk>Tdmu{X&K@touXqJg%pufTXh9XFD
z??HAPoFavtEAOAf(8bKTK%x#O>jrmR`f?(+L@bT5J?`T!6LZ7rE4f6H0i{<PBB0aH
zaBrrCw5v2`slTBV8SBHlF;>w5=vzCGp>pq|v85op|M-g>8pi_2740AXC)29(7)EJ5
zeDM`;qTZ#1TFhfKS0&vREHL-)G~CM-VbW3aWozpfX(jHSdMY?&3gu3Kf*b=ij>0%v
zz)1tT-1I*eJ@Pd2l$j>yzT-W8#$JP#fvPwL#-Xyh@JIY0w{O_D(M-Y^RM&sY0tqst
zk}LuNcQ~#@x2%;yz;Yfd+Htx63rInYfs%`kUskei^NZ<!to0a<qrU>t=|q2~^adc#
zwMz^|-RD2bt^1Qqivu~AZ53eAfOxWJ9T3gjt9)26)$kY830QQT^x>ol>SKTrs{(*~
z{63UaO5drEg|NQk)VnUgds^BxYzH<1zSVg#@wGOW_)2!PM?EoM#O=U?t3i2c0!Vrx
z<pVe0d#T|{i$Odk7%V%Iowr?#^UT4V#^ided@A9pseJY={jSv8YNRLqF=%(*F|<Mb
zWI9}_i@u|TL#=JE;cL-<&53cu7(hA5NESPnRC*UrG2ax(no6hY3kjD7=YUE$&OE~J
zy{9<>BbuND@ck+ca+zKaq-g@SlcNL+^`L(4Sx;Q=xS~G%E{CQd_i6Gtz%2%SgM%vK
zHeg4b3XuQIjQwaqy0XA5#GCs0n>cT*f$mPe4qK1?95&NDXZvJUz6A!I`Xfo8aQuE%
z#bf7n-7HZ!D8sR_;?VrC7$6cQr~oqbFiNi)<i6z$`YRyRC$bfyC;v3IzbA*t@PA4#
zZqx%AMiEdkIscR;AB&`ywqEFt6)@Ckjt-vznQF|Zs;l30+>2V~ms)FX>(!pSj9yH>
z9i|~PaA4f1GkP=^w5f8wC6}_lJ~miYOU|voJl>nMQx2ww=^}FS&-GSs1$?u2ENr;6
z4$^AVc>IyX6vm)A%76i&<q%gCek(YnC<`r+)&suBX7ck}=RVQH&mBoKz`W=SAuHL~
z+nNzyIA4~20Hw|`pEO04R)9vXz~h`pi3%y$thdjsUDqPhr|zL;3DZu65wwwzsSM(A
zlbLF`Kij{enPGGE=eKR*_~3)ry_I0C<6TQ0+~Yrs<}T$<30!bX;<uQMGh0Py28diH
z)c}mmO;eQv@Pj&5`>PRq5G4)H{o8n|I7VIL&}zrs==(Up@y!+~rwXG5*;0WrwiZ4J
zHsFoThP?%)j-&bWO7oBSGVJRa$8rERq>5gg?xKC#s%<9Lf%c;UaQ~0r=*)rxVFRSR
zT3-r5nG}@QUJ0&m0h>i90Dlq7nA}3TUrL%C{LTm@qF1&wN-d`w+zMRoag^PBnpk&@
z?0;K4s=pR(x{CO(g(BiTDBIrrauud3RnQuh4OddTsrXE*UJ<Uuj?E`Nez=BBqzd3&
z?&cSOGJ$s2_!H%?CWw27r*((8-*;j{Rn~eD7ksY`Tw_ym6KueN(QOWZ$dGkAq5Qf+
zh&m24Y+F)METr}6><=4~g@M@oU$J+ZfWs_QGGQx=XJa>E!$dM_wQWT8gpwrqj9!N8
zhFC>oc6oV(j4>8F78Dcal>~D{C%HV0*`0K|Sr|j?wsEmYRQWEWFb5nlOCI+XUnytx
zSvM0e4*HE8xD5{7hCK_ipdpaX2`Nfm)lfETxvnHJeXu_E5=2ZDp~IQ2+JB_XMv`q&
zN}_4!xm70H_)2aw93d^(*jW?%%i5wSrQ(S*SMB2G3$|SqE?b5!iWw_|c@xV&KRpoB
zvCQ<(m810>nntd1QHpzc+~EPmPB<MOr^3)x4Yjr?q68la+qxCziE^{|0#+m6Z_<e9
zR_WJZQSh5b)*G$<`tr^O;N}q^$v*_e+T_YU0D=EfrUeiuIqr@ExIM0=W3Zr116C_{
zK7&z>P*KVlD@*#2djkah`qvg&|BBp+&HS%i#U1s-!C#M3<W@M*Z}-pLn5dap{E@s-
zRMA@woX8Z^QGi+k=63rVH^48nV7o~fdC7#{5GBA0kHrWSL%j*%N(I(HEDYFnJpg6x
zMoYhdV-R&v%ZkUu@3b*Kd>b5OTPl1I6YdR_#xyj*e!pwMjv@S^h0-Z8tTJOi%4j9@
z>XVgZWm~KqRCW*#x=CFQYL{&kpqLBnnP4vG3QQu(QL-!A#TfoK%!TH6!Lo@0o|1&5
zz`r4A4$31@a-2-3vwZt+1};+z=V;42N$dOl(u8B}<VI8@VDzd#LHr*$Gi*@>E7Gm;
zUy$&7KUJy0ias%`=HnlBlfLPotnIu$%6Bqc;<bRlLmM@VeJ}ut&<iuTVw8?C`;dMp
z>^Io+Y&G1iK~N;BgCmowKOx*uku|_$B?~*DfNqO@!$FM*VGbHn&P_2!-i`Q5S(I8p
zi<4dsq*Rz`w<T)9ZLl(=+L5J<UcLo%CTz8Q@hgV|{w!D|M4oaHG)~eRGM{?lc#JkT
zG1=|{fUVn{;rBRtQ=`<j^{O|;0&JMe+KK3;Re_D0;PF0Itlz9#L&Y^7PP-qh7M6?y
zE`<WxJykyVtvE&-<8A>?CLY2&X@O{=DCbzefE>*eX=AK%ZICfxUdAYw7&cl1o}?Op
zWXxN2Ae)lti+#?Txl-Q4U2*rpt5G?zom=s@ofw=tW|Hli#!_`*u*l{Nla&$lt7xl*
z{`wd@T||vx;70G(ftLf5kjSkBpPlT3W?`*zPjrmb^{0rCLY<OoP~93ZM6NLDi2eCz
zl_h9<2jtPut4%D*`-Iu9YGk&l*UjJ)r8WElCz>c1?9HbLF-yC~{kSk%aj_5aS78|p
zrX+D+OI#Ezv2$Mp<v31kyBU;V3XDN~#)51Q<P?j5DA6%+iUkLW^Fur~jIR0xd0dp{
zKJzh7w$vnDd<!;R26!?gZ3KlM=aFZx#{lnP%aZ2$V%PyRy5PSg2awr4fGTjQ;C2Fn
zIxY$|M<xK~1T&?N=^v`H($R4^^Cyw2j|U4}M$E_?Yo$xvY(Kl{<iM%1S#LXgaHU0#
zmF>yGx5Pngn8ZPOEuRh8YAs(jEE5H+DQ~u(`wJF_iC`GlzFT0y==|;n<R~5LBA6^q
z6v~@SY{Vc7qm#S*n-rWnF6-fqzGx?1M4XK);_w89cmZT@n-686<ZslW{c8$99Skiz
zoD0K+jJJT9x)>u!`9*U6Qp02){2%6ly=%3+qyp{h<5fKn`&{?iEicvidDznHA4z7Q
z0K&cqCQ4)xa6?gzgE|cXJY-e|G)B^Y$raxY+FyFC(-z;W56%1OlK-0$WY<b<X}OG=
z^dRWOS#LVobWqTE0y;oKB^@^>$3)M2U7F^U*+J-HHqh;U%&LIh7DdX%%d~^9g3U}v
z3dHA?G@?kLVLC%`8I`>V5C9@Q99p%GZl5bxt$`fDxr-@&eovoMN~64H#%lu9CrK9-
zv0y}O1+D7SZCfI-RB=%Y_pJ$8UpASYyk{M5HUXgrRBTN_f>>Ow6ak8H#*o+m43f7C
z59UKe%D@RK&BQ-T8JVstg)AFjF-Y%z1jM3_O=75U{%2`)UqApz%_VLz1?BkdvnwP9
zGF@lly%HP)GUZWF(JbV;Oy(9(^>ssOl7M*BNK0Ekx*0eZwf!RyyW73oU@!#3OlizR
zRt0}30>H8d&<2ln@VGw)t~&uAw6xT!zSqCEn|?8Q<{HOs@O&NEa)#;7-M}Tv82?gi
zx-k2r!{Lh8DwvQgy;}toX{cXpluAHx^t{S4cF@6u&C+}zM2*@B)ZEeeCJKJoxx~FQ
z^t@LhLE>Nc+cAr(6i6;znwh5x6`0zzwA?R~oWRi^DMEweHFjGgefyA6@B`%Vo^l)u
z_JRvjO3KQr@U<-}reXA|PiMYHUicarA`dw9YNs`AK{Gk6ZBVC7K6Egl`w9HRJN>N+
zO*Y7B!_?I-^AE{%j%rn@=<X}1LS?}m#9oK^`OiAX72-=rMRYtF^YCWdACW!oEBTBd
zUV=2Uo^m3b`f;^oBKwko!w-Q2(A^dAoSO*dqo0khs1E8g!Q^g#`s=YXh{;6x!)Qdu
zX}pe19y=~aCVyT~>x=Hb#ypb`hxxG5bDrV*UYE#WN17vnm}xzf16>M58%Xq~#RW9I
z)v3<o7HD0NI;lmg=tkM|(53~KVrL8l+!xsbZrt$>&sCVj^l59<{8YW>lgl{~shbPC
z{>hkn(?(DYGUSOGJEN+94Vu3=W=a{GK;0f2X2ucIzgLld_We>T4f=pAA&Oy+=?V;w
zQA0P%FZR|U$;VPgY-U(KH@FfKT7z&8!5fnXDj4i;07cv>G7(P^#vW;_kljl<EQB{#
z>j5JFWgi(Nfup={8k~w;Eih?dgu&Rz7&sNC8x?^k01;1B9@31)R<HmT4^-|^&znGa
zqU3wwzcE#hxe0*vV4jN0aP|9r{Y-!2BtzHVIG0GnW_+mk(cA}1vfTvW!SJMk_rjqK
zjYq{7e83_xxh_%t?{k&mhkqc|@;?#c+fV)fo_Jt8a)2DjC?6OwB#TxNJ7!#HzXD^r
zl?Zk()8%?_L|4^)??5tXRA2jlixXSJ{vA#%RphSC{?!+2U_Nk`u;*}k^cF}<<-(?c
z3k;=~*9DZ#;<qQgCTy*ZO2CX*Z~wQ49E>S|6k#3(MdMuMR7;Q)lmq22#tSqojstb7
zUM&0yj5Wr(b-*;;X?CQ75qcEa521giE|VEpH``s&ksz#DVbD$nrXX-u0Oi04S3M95
zLrtd6?PNN-VA`#t1TYDlPSONWi|cyKD_a^5oMuK0Na~F1UAEA`?EvmtH)aXW1~wkF
zS=OWO0*bxpITVs=L-ZgWWGWCro2=u)=)NJuNO@ZE6eUh$+g|JZr#$BUjy^qDVA9Am
z<SLsEy#JF45@J>g^RK}q&~O1#75K<<uqqD;I0Kq{0BC9T>wSKcS?HRFa8ZoscG7SK
zE<@|jKgI#|4>Cz;{hWYkDO$0~H1|!RdY}OfwcllfNtgwHS8c!pkb*S;awGH(S?<g$
z)zWoF(N9tfI6<D0&6#0F@kG#JQOOGQhv+rm=_5mcM8cm$x%73K+qax12*ic)H2`D)
zb(Rv&EnI=|MYD0bEwm{T$1(KxfWWWHj?&6VOLqh`*@Cc|rkY;t7DRvyEy0;8SKa!=
zPKxkVjLuedrsmi*a!11ir<c1UuEL09`b$Y=sNCcI)n{Q@%3n}`dCnBo!2_<O`E-E?
z(ttpM2vi)KFL1{*fDj>p98Y(Ok#VS!F1ra2Nuwf6RRmmabnzGC45Xs<#B*s-_5+g1
zOAdVP$yFF#Y&o7H(~~V?7^x0xrh@bz{yesi%odINBYGo%$KiFp0#gkHxF*AdZ74(2
z&k&UAVRTfP{>%hz^`Jw+pidoS7=X+2tcFgaxK~5+5{M#MnkFHJIjg``5~ZmUzh~Di
z(ytxR>VpL1MNsD?=qs!L<%i$u>IRcYWSuA_wdWye+lJABV|$4!2--FJqBh6Clgj5?
zG;f=*nF5=YumERT*m2nyfb*(CbX_;7Am084qGGqqSWr$RG7&^IpMmR4{1CPF5U_Ee
zaUw3&>+gy)&?;+-)_~9mO^1lDHZVsXJBR3JJvLqVKS5CqkmSdo*3yAP06HS70`SSz
z_pZQxx%w8YBV(h;YA-Ea|3|v`43N?<uip7UA%L>14gC@Z`kz6Yn-%CAsQxEBQryvh
zD5by$GLEDZA+OWVpw!bAOIHFqYwmznoZ!e&{~RJ-Dng0cx|Q5yYrreoKJWdX>rwzf
zWQt*cj-@D^trDI7zdPq^H$hfoS9U{DcGWD)<+=p-sV%mlQjK$Qtu-JhM?T3Bf^w-*
z)dGih(VIi_{qYNwJFjsIOs~|~*TBAuDN~HRtH08D<9Ut4|Lg%e_tIS-N-8kW#)Sq?
zGr`_ww8V4+5H~5RN3yaF3O+?ZC6g%@Ukq~6-9GWHZ}+RM$I2}eL#6V2)&c0Q23_6{
zK1>0*5936srn6*0%cXP6$4=78KB;0>e~U|Ifp=1C{jUx{@OsZb9e{w%I6s76{Eh9o
zkg?^m;ApY^NAGjzufrIeiGL-q8fsh9+N%Z~6F^l}0~&XFZEF|eQhLtGH28YRup}l;
zDPyvW+SISOC*BFamZ7TtgDW%%$YW}iBITtIgq_#_`KW@SBQW5Xz^_z6M?I$=v<40K
zieH@a8j9I;wVBq&a7;FPHxp0`4}sXeSa(6=HvbVdunG?bEi)BB8$Z^?l1|lKh9#l9
ztkq2o&Lq1UNO%o!M5jXKwKg<=C?89-&jN=<wt6}AVjz=s-ke-v*YyC1>*gF&VK#?b
z0w;zz80=mRa2pfrV-+^@ogb@ByNGpSrAbj>?SUkVh>{G{k&0lKix~{{vDcZr%?b}(
z=!viI#dp%r=PK^K5hgr%pCC3_$RQVW4^0=-9ypA#BF)v`xPlY1zFyJiEXpJNmi>@C
z&XI{Qdf_=1RPsYn{oMufBWLgQ$1Z3NkDoPmGYmA;KIKC-Kby~+tX=-@Gx!_kw6Z=k
zMtA^}mDsR^&wyBJB-R>m=aSMS(@VXz0yTA;VNKO*4R1h>uY9RoZK%J}^TZj0b6M$O
z+&Vwma#>OKW#C*v&@lH@VZQ)!Ocb(o4_D0upSlK_mge=>0&|x4Z#u5U4A|Eo3awjb
z5uy_rj<ZQzW_-C~uVw3aqBS)MHT-9GVXc|Q!=S7uFG8-V282xQo}gj97^V6aYFbeN
zU$5|@$+Xv=OJ}s+#V#3Oeafa9*X<WTbe0x&_rbSU-xEw~rks<VKr#=S#2KKR1ZuDO
zwzldru<d?R;&?TE?-Al+1atFF`u-Lxr@_&Pc=&Hx@ms<%eyiS_pJRF|YMsTW_;SCM
z^Q3II71cz)pFh+ukX;cqQc`=o6`QL*{TfdW4D1^pv$OBY*7Mu4wQ(HlAm60?RN#~S
zCtFB4FU3C8!0UJc>NBqg20ZzaKtlwy|AujYIX~0z&R(TO)-AzeYR86YpZuOdayKG&
zO44!$Xr<L`<ZmSjH(H)uomLeaE^SbCkAL;rkV#3+wJ(U_#Zu!y%wNwZ=I`UP`Yb?Z
zoDTfrkF&EMf-{a^zPNKHt&!p!$8bQ^zx+{I@I3*O4O~CV+-2$Z^4K>Pgu{s?KE#@T
zR_B(bZYX6eqX~^??K5%f@?|5I-fs13=Bruq1mr-s_D|_SAZD0U$s+XnV|3w51tP;}
zePrB$qOW`awLO<_f;l8HG3zF^f?6($Q0_^6ImWK@owsz9up|QBOU&_M44G8g{d2H_
zWsDJCeX&`;(HN!42t)#gG+_5;hH)rG5~&PAh>Ay}OFDoi_PUO_x{9jU0-V<S^sP%t
zE=NfN@Htp9n;Y5FIiPt$bn@LbAI35N98gapyBoK&Y!(V4GmI1<I1M1dMA_~=hbJOv
z(0VoK@i~ufvVULQi`@|ciVXjpKL9e*8Jc&ousPOBO?4=U!-%RuT+~vH3MEt&I=B!0
z8T1;!h;VRH<|K95sj*04Ur9hcu2p@Wfe1{cJ2Rdw%9jn%j>lebM<%BCv5z0rD4KYW
zGuDvtDhVvZ7~HLzi>D|XWd>z8uvv&MF$*jurH1lpNbSQK%emxA6ekradwBsn^bIMk
zksV*(0B{u}YaIH5&kw<hd#io-CY>F0d4Q%Ug~AR2NS?c;S`2f`K}h!T1I(7hSE9JA
z_bu44*bwlFTLAUniVLL+I)NsUVGsDojF?Ru>x0;4Ezy+mCF<l;rRbPK;y46H859a(
zu@G2nj}zh3mt?~S><@l_PmBmB&rX*92fE&Nx-5_BEA!ALOuWZ_&o}!Dc&JqBZ<?@C
zBy@oK1$cXBL(BY`r)BzTA4!~4Oy=b?i2$EOs^K#*+w~8I-N@7RjtQFwn7I#VTY<F`
z^YTvdy{wrMW;5abs&oFTX8OeI4Ff0HN0rN^hzy1<`X$gw!ak?)!c3&u#5IeC<-tWc
zhP9!bP-#K+16_DKUH9*XtDO*64wpZCw!^VWm)NQ<05`h3N$yXTy<*0G0w%`_heqP#
zqA~7=%S`x%jT8()LKv)O=R5K^Mj_rD)Q5OPx0@ndA*7KWJFivYXAi&QkB^oS*TLe-
zK-WpW(2dgcWs9~nVVj~N^wU^|mfaNki+W^Bwu$R7`|Q|>){cm8;q1>jP2Ic5#BMR(
zSPeeEM`Z6<^T6IgDrY=aAx;H*WA(&8B$(`(C(2A;uPbM=UJj4~4A3-^ol^nTGS5=t
z&fU)RXyHx8%!?g>tN^hi4(^t}XP_HpeN{s;#^Ft<t}awkhM59p&4<y;L?QHcg5URT
z)|y|Wze}?XnSq_s;ekgovdS3SxBUqf;%lJt(*P*`>wVtXmta(+?^rv)sI0(qOTUAn
z&$V1$8G%PQ1d6hwnEB+vzaa+DSprSN60-QDap86aw{WAbYM7g_sm1tdTyIr>NDtbH
zp)d8(%m!7q@Tx`11O#y$ajfv804l(sB0zQrgH#l%e16zW5w^;OEsj==kykai9Yhef
zQl#LD{){jQQgHv*=OFVyA>%ncjl1u}A^7-2zW)Nn+=ZiY|7@yQ2R=W0Zd|qAC29$6
zaFSlIOk^+`g7OP*ta{lyja#k6U>K@UFZ2ug_zcD_69u4_kPpg{X~$E>fhz{EPD-d^
z!D2w+U1F@8h1HF?EsF6PbAYd&12UGFUGojQgl@MW>#Zn&s-f@Caki)Siih?H)uKrp
zkTCaA<xTge&a2=ZL~>xE%rc2271RTmLX?czv#sB0gqwu8DFC@4U|EBUUR+~R0d=P#
zlm*Up<!6CR$Y=&;%dJ}dn3p(>k5XJ<zJREgL1g+xnZgx`duVb^SNt??n{?si2<)+5
zfw2RpBGE;bl_8x)XQIH)!0V=tlXnBuLU+!M?C~xaD+Mf8j1s+jETg$Dv5u67bgN%J
zPVS2vI`GkYO|rOh88FkY(1t0Pk|dn^MPY9~Lo{I2U$_Q8H)dx}{}p4S%fJZUgaa<g
z_Y2IC!eJ_S`*J+P!L3{im!z$c;pT4r+1i|VGDL1C+;{E6(N=D!n{}MN0@1fbNlLzK
z8mdtZtzZRn;mbe<yb+!O()l-FVPwfeN*T$B6Iid`gh`lyiIQqHL1klwZa!W!KK&5r
zx@|Cg(l8@qcEsBtN9c12c~Dst0e9oXTWzU-0UHuPmLQdMCrw3{fwf(U>iw-4X};h`
zX)dt4&=Y}TMN#&;qWYq!5?p?F_sCO<7--~yNLQ7pc8{Bki&Nn)Zmvhr<aJ44n5|$-
zI`yi33qV<^o0ZoLj*t%WZdnvRwH<zWX3&G8Apu*?DlA3flR{SL!j_jY&-SiG<`$ql
zy79sCD2M*6kI~VM0>%U*X~XB+yO~eC^a5Lr8~4`I?#&i*aEUg<-8jZ9QKee{912kd
zLP10$piJ?gwKWDLeSd3|3@@)4W!xQm0B{|__)RsJK?x<7(@#Anhyx(7E5K$wWUE~i
z815w8&y)jW-T^#!q;@~$a3D=w_E+DowWkQqYnyQ_F;`V&bfvy%O7Z!z9-y^b-L6Ff
zafbwfq<cI#aoPU9A+-n?Op^vo#?o((JC>7EAp}d~p~>Z{H-J@t1SzDz(Ua^BAV-j1
zzMUSt9SbKn3X}x-c;DW84vJyG*J_r6*r#F4ZmAm;iKQ_j{U_m0cqvHwxDkzovA_<p
zfqIEuHMf+J1>H4Suen#sD0@?h)EO`^i9|_0@LP<mA+72}3p5F=>)<meCh?K;b1Fo(
zkOIjH@G$aZPMumYfzb3Np$5KK8}Ko9u1Ba4kk|mBLKF&gV7%_50Ua1k!=TbgJaHv7
zr>HGz{!YS9v1`1O>WdeTTD3#AYFYQt+OuxgRHfIY2T@vu)?b5fRRconvY|0uYzll^
zR;v9J;7le0vSdI7qzXWJo(*S!sqKKu9_Z}Cp8+lE`bRmMG7@Fii%Oty90n-o6+mtk
z(Dz|2cZ7D|=I4~n@^BS?hRckM1`3-yKxTN8>X9)Je3T10EROyM!V3U7%QaB_9h!EX
zBX9>__OlK;-HM)j4Jfc2+x`!o7ZCcvA_-+qHq6ObXk$Y8hD3q_9|AN@ShR<d0knRP
z+%X#rnY*7;w>0C4IRt4>G<?~F=8tUT+V-H%(YUp~cq6Z|GtbS*+7Zy_3A)DZ_y}*2
zu}_pTek4$gV&=USr5@b+h6z&a7D+?g911H;4k{<B{>L7Sk<V{mQVLkc7`XlpXqo_u
z|No=zi!EcdTQ6wi&P|q+?!B^lp}GPf>(?is&!9DQ@h+44t)O`~#jKW3LQg5O@=Tn;
zmn6Jx+X6C)jr!F8OLV=gsNq)~CV!^4zHInR_z0yNa-*(&vA9drE#jah;2(?|`us{>
z2hb@OmDvIf>Bg|mg+P#0+p3|*wk-5DVFo4YbsLUrBR}@#+346rEW!uDcPRa@=FU8v
z%Jtpj+9fPo=B-VpiVQ^=ijWMMWggmwD3u5aiDigT*}F)FP-JCg3QguQLn~B<%q*)#
znP-+|cJ7Dv-oM{<e*0Y4`Rg2itm}HO<$c!kKI8q|_vik;=b-V5?(rP^3=lCN)p9No
z{PGScAt#@gHrsPGN$rBAf}P(}O7VI>)+~^jEmp#BY>jIEigXU01iA;7h%JHNb};B6
zOrVV~Zj_|8kxH<i(#$x^PtF>4UT?ddy)jPS@Li1%c(cggP-WN*JDDtmxFNIX&6zhB
z&8(iMT*qkDnbNDO0KudP9)9-*DHl!7+4(mfL#M|;5`d@g03fDW3H*3vv&)k5GL)(0
z)d#C5{12yLcg@7==0@#c(47sKGPWmK#SiiUD^4Q)J{nm9XNa(Mx4hQeh-tHBS;@8;
zi)9~8Tl%gUwy~yG@W~$=cZH-P*Tu(~eTP8l;FEOJpX_WxpeNC*qYIt%F>+VnD8Yvv
zKZ0CC(yjz)pfD2u$^midEA3)hQMmT)j=1o)RQA)B^#(^ne|^oMV=HOmpk~}^`14OR
zHw{tFhWDlQ)up2lAw)dcm~6t2^QuyA)3YUAj6U~i?`o!Fg;mU#9=nH)$rghy3{mM(
zjhn*k;y^?RR@TPndyH8F$5T(Y0~85jz{&)fPqAJj2h5%$ZlY6(H5b2R%{Pkq%qd*3
zWQJg92u#*g@cK^*BW&bq^#t2(0{E+Qx{EPKA09Bs2XP#N**g>Y8G)V$;L<HlR{z%&
zktC52yx(?XoOKoKFel~H@5K83U~Gfh%?h<ki(GPgW<+iXg-HrHTI%f5G5(MQo9Xj#
zMqSe{s%ThV)DnYU4i1Mkj-y8Kpjo0v^z$n>nB3`MoqmdAO}M!iae=^%{!zP|Ubp$=
z5qcULRapJQ_b0#HZ@(ZxoB5ljB?1K@$$-OBK2&IiNl)cg6iX~rM-OiQ=|dV@Qd-mZ
z>bX%H2m-yYxf*hh+FKDKzdr^{I))U64{GvZLfSl!z$Z^bBMWQuo)FI1{n33wRmMKJ
z-LS-@fW;oQhr4CREx&5xrl<QKU;zJuSnNA`{q!p%s<#SOlkEln=--ue1p*2jTjhWO
zV;CPC7M4kITFTL|%%s_xgl!z}BTeDl!vVJ*?D|O5K1>N6m~5=$GG#Wv)9XfgA-n^y
zhd-J^>Lbrv>I{aad6IFvYkApNV-2_03T*Ae{fF!v)0=O73w@^7v!>Z8ay>WdKvtzX
zY6X$qs_gxuzc)o@g0YSZ4oR~bizmJ!Df^-Jps9z=cqR%^3Pa9DMy_A;vCd`GWP@D>
zfOHyjLd9!qH(STrw{}ldJ#3vj;ol9vzBGMSS*?7V>_ZmeRm%Mu4H1=Ksst-4@S29B
zs!zk~+d^N~Vx@?Y?PJ;y;hQXCiZwA5y)ngRpCPq5s&RW|QWQDm4r-sXt2gZx)ViTl
zuHq^jgax=C3Q2))OTQiVK9Jal9~URnKAMXbRNO2ccdsdAmZj)rbibGM#DH=#fAOH!
z*N$sJclR>me^m8i?QT@nueX{)9K}^F*(48h&hzPR6QB4rwXm+NP%#bG9yms!W#m=a
zaiWC2n5SjBkSTK1Y&z<87d+AvPjcO%0|y1IIBvm!bEMb56T3H5$jDC2!LnY4tBz4k
zrP{<U1p51?A!`B)ulzy|$7}C8$%C|_bV;++pTg4q*n9{f6=(^1`GeoNBcsep7_tvI
zx+UCxX?lFC-)SvA2xfp?`zuQV3)4uj`Ys6I?W*mWu38~G4l~AgeA<8Eq>hl?JNnI8
zB*T(m=$lccmQcx(>aA^G$Eas3A{JIDC<=829?;qdtrR!`nUuFb?F{p@AHmZxTVSoz
z?Y%O-@kX@>g564^4v9K=c7elYp-<`5I+J&tV$_0`7EuThj@B@?yT&=#&XF4@!du1K
zg3N^XSP5+CW;?Rbq%e?mIUj|G7{bWE=?9OT1IEZ{uGXJVskrJf*8H+L2!Y|PdZ=dS
zM8&gbtp|YnseFtLjR%yG6SB|FPUmNBKJ+LwI(M5w`WueG=^8FrXhVcKELc&@>6E7I
zA()5Ole6&zf)5=FAm+7DAoy)XXu?&XAVc_9&7ke@uW%z?)lrp9ORuMLdtkF5Lnrvw
zJa#i62L%5<nUt#-78t)G^1@;Ju9ObV#x*FOfN6FR8ge%ToPQJ93H-@BmW0R(#UDk2
z#FlD+)DGm&hhLkd*Z*L{_oKXP(3P(AdUy716^1#G=;H@{cV@4QAa)zNUF67S6<I=r
z9d1{mX2PgBs$P}fA-ws>^+2<}Tio&7b1n`YMrrPBS7n!C+^imUYcz1$MsAiUFCg=d
zKZrrZ4GlfyuSf-g3GSfR*V?DHwGtqyD?^inN_B`(fKdXe1Z}iG>xBr_VA!i!kM?=g
z4jhY!fOQN`T&ds>J{NqB1(uJ{rVuAU^fQUS<(lPfEet1xpq033pVj3##VcuT{!B<y
z2scM`x0mz8`hK+PJZ*tR>sD|JI9p6HThAUY*``=-I!1HyB$N+F=yPG4<^KV$2KYMx
zhB5rX-23ZNjEj91)r+j1M2K76L*ozx-^FazD)pQyf^1%ZM2a~?yh__YT~t8#*Ps1o
z%al?)lGO9JfB7P0q`uMQ^Qk#4M_xndfjNZS9zm4t<LMJi-<t0K3dNly*BqDO{X2yD
zZ<)Oaat-Qm0n4s?sQtg=p-7IZKtD7Sv}eE(1U)PD%UILfKXZAHZ|3qkLN2f2KAEL#
zEI<mW{-^)hw6wg<D8Qrl0a;(6l4eKfVMOkMx<LY0hFH^$`5WLNKa-6pOaQ<}T55s0
z7>>;-KtG<3oWpbTplsIDnqfkWEN5UBwglw}j9*Bt(b&T;x@C+fXWHUT*WB@?Q{t4u
zXE#00azCbqd#%QiL?{vh8nCBxn9DSZJP4Odf24MFY+0$}f92?%rbF}YX*!g%6=ATL
z3hO{P>&3hrk_Y6ty)k;GQS?ua+D~o}r;0@Squtjz-Xe!F6gJX)UsVvy8<dBK^pITQ
z!^jAX$nui@HxusA+49~)B#6J8{h7p+5xB7~O10T&`l4jvNy4Vc<N|MHC?MplxyGJy
z$QJ78yD&a7`0F9-!X~C<leQ8NATIXNc>N=1_njLNIKHL7?6Gi!-YS{5m3>M)*7J8t
zQ@x5PfhJDoVC-&!tq;LAgK%h%0>M^Wj8ce?ir__-U~1o!rMaP=X>zqof6ZN2E#i+%
z+)1A;4{ZHgXq*4R2txxZ7!v=Y5F?uSaS|e9X76<O@ByGzTnSxt0Js=W<-3sDH}n@(
z2@h&3h7&pqWEZGu`Sgule8U?7sY}UM%$zK)<M|4%j>n`KM_9=%)SG7fsC7SXFL<-y
z@7reL<$s*0@GZ<+4i)Ue_$%~-re3+d0HQVWDu@vv&CZ4)&@AnROeu*(xWqRVq*Hat
z){it5K(MGw_Hb8m43C6(UQr?$$e2I5gM#K~nsL4bsxu&z+yzO;M7qylE&iEFX*^|~
z?n#s^Up!Yb-=gaR(OP`p6C4&YAc;)RHb7FYkbv0OlFOdpjn*l>R7>qVAJK)Bi1`;Q
z>BgUU_qh{t672~wuDf!0ZW;W@vB)cBJ2BV7ixu)4#4B7kuvUf(@6C&a!ob0$HEd2u
zW04M2mws&@53rY?;-v{Ynfb0jKHgiXdPy=cyD`4haGi|@!<_&tJI2{ukKe?|LHV?p
zXZzUms+fM~vnvT@V9-yh6|fB}-$Nm&C?VZm_bhTM&*?yb41n3^Y(pYsC4&jh*3~1y
zj1n&)irNKfmY0$jYYw~sb{OAR7F_Zp;0X^et#D)7u5DIl@uRNyc+7Ns++-(v?TmEe
zRN=rnx`59i>eQLDn3^?@oG6xrz2|-$aj6kHC<F{*1TSeBf2mpLk1fvR_xZ+3wcm%t
z`BqN0(HY<uLCnns9pxoK&j#r}#`=1PyF!XSl5m4_a8EBqnbK-PoUsUoE-z6AvwwKK
z_0^aXe+P91n6GSr7GZI$aB2`IOG^@8SwQa>pQqMP;KF_DB4XuI5~*Scj@z~CKWYeT
zYCDwe)Udg*33Wf4mVtTZ-|35D(SK%!dMa?@Ad9sJv;)Rx2>b<bjEeE~R(%z)=Ih*y
z3ta<k=R7)oza~JlhQhAh*chpaU;hFiTNcrGEc-g8&7#CDp)LK+*pq>_J39D(?ftuR
zP{{%4@KR3#AGJ}jBCydwaYlVyc}R>QMRc!ByBR<3u8h4QJDvF5;suI|FO}%sBR}-m
zWR%EIi$tgcla_yxaAmsVxxP-VC~>n51l{l+YQ=^>76!u=(9p>TCiK`9!Ot>*WN6dt
z10*;!O_bkTZHX&)fTNg3hW8;;zrnJIM#U3ziywcF_!ijc_qmo4ejJw2ujT%_sp!Cc
zu$eE^fFCIs<N@T_086SlssVHrBlTW%xzCIJ2cCFF@hS)OB01{f6eid%+&iovh7R=I
zY&!NC5vVKAX8`WK#(s;!l+De8M6h+suqkBeh{jShQCF4cjMuP(9ul_n`mD{6!ob~j
zOUAF3XZs#+1Y4}MXS1zu*PcgQEl6}b!0zc8LBdWcDfrH(mn?9<XSLu?db%O!^}=?k
z4Ki4T%-O5H_`s3BcP48_0I*7&|M+?SH-KZ-WNv%FknH0(xcO15QXJSAvo6ndEadr}
z;jTAnS;F{H@yxe4jQD7N@^LSx4DamWR{w0b3jn!V7~P{=>S_!ypP1qG?j;LjNPz6X
zn7KNv%(j-6vZ1$50%0%L;qbv@ejqD|rr48A%@GoHEVsfHX{*_&IYDB=Bu_m&F<-LJ
z=EwKXQzQ?ez|Xa;(B@zeICuwcCswOa<qWCWlcXk4+AyB_t$Ad9C(u}4oEO+G<E9~)
ztoznN<fs`>OW<B+#p#>aim|d>x6%I7LXH(6|5c=Hc0rp-V~ud!wYL!Nem7#}#bm%L
z8;x+@6C|1BQ$jZeR^x}f#~~g!)L5$5p_C)V>-|`HPKp<cKJ)DRSLH1(Lr>sVJcjxE
zAV`h?fyw*in3H%p*>;}6Ohz-zTG|Xq|8*gr+`?Z=?WkBSBEh^PxYw*A=3W8&K=5x!
zIQ@62yjYx^|KNq=lX8LZxqv>Ms%w@!a#~=m(BM~uolTPBN|S<QPQX?WN(ncP@g2Xr
z_)!g?(HcmNS&b(QSu29O6aM}`yk$V34pb^!K-=Uxj5F{spga#GPXeUCCe#aiLl-D_
z+vS!oXX+=PD0@#rGJK}lCOW#Y2VZS0=We_P;gl{U0$lJ>;dF*sRxCnFA-wAbv&pz^
z>O#pz?eRV&#|^?W3J`R4&-;!hII;xI2+=zI6tKWirkB>W`C%iZCJB}%xHzrMCyJ0V
zD($9*KjMlJifmPmrMhAv)yQd8$^D6OO90xCU|lxM(<5|&owc$$r0j-FRhX?r<Wegk
zhw)qiQAcF=R<0=f&O37HVBAal^4ZXqg!J>d0@#S~0s#q*rO@X!V{lrL0OL9`ShWUt
zivjL$NQkT<-;FdGd<u*&;(1D|h~kpA?GV@^3$Wo9xu8gWvG>$uY!<-*++_=H8w+j=
zHS7kKU}R3zi|4)=r8EbuqU9BNbRn*txgcs(?4k?__y`jRM`$wK)<6*vTJEEkUqYE%
zD=Qmoq>V0sA13e5Sdo^^@2wf+@prFRGb@ktodsmZZ0yo;;fX1Dl>@O{G&LXW*6w_y
z-x;6@vM>Wxoj|*|B29!q8;n1*(tIT1hm-<94t*S$khc|U6K?HzT`r_S>->11%@+iz
zNPrMRbVQzhX#XbA2`cEFkh%X2I}Tal5Oli9(K>Ch&mRmSfPG~EpzmoNwurdrc%u)A
z%W_GzXj^MC%^LXivtNfryBkT|BDi5}#3=VO>`hmI8$lotm*V=I^2V0{cmqID2ND}y
zSxa;+u9NLtp^RNZ8uz`|6$5(AZx}5aJ^H{g+nV^SFVQ)t=sk4Jnn3|SW;e=jKwvV6
zcz@5Ntlh6;<m0lw8U*=i?Gt0bxc}y2bzC^&zJ8e{<_t3G8aP-C6eeU=Kpo8$66jyS
zqTCHXbVdPkfrw*H%DENm%T5F;^i$z?TP+>G7asE&2LAP+!^vEfl$8q=MI5%bqDsy-
z8iDf7k-^>6VT;Xnjx>mus%0`;sWS-Z=bo^2tdI3o;h+aepA%He6cwgbE7zjhPol=m
z55iJJ>C@BH-<<+-?rYT_VV&^y2|`X`KW3$p?gYz?9vCMcO0qD~Y~bAl{ACLunqZS_
zit}7aaR~sy?EF{f3EfEd6+P}{aCqmWjnFAC24P<$9ifEyPkJP0b(>2u@vJsS1ekIr
z%zN|34R?YT@Q81kdYKU;m&6O!gjL66&yOGk7}zFnD#8tUGf$o2cq|FFYHG^ir(b6p
z&aSt}E+@)5n?F0OeiTv*@J=4#t@K=!oDcBw5VEZ%+DBDG(z(>t+TSY2+4^Q6ZSa>M
zEdDU4;K{^cDT;>ZbSoGx-Lp__+Z4|h-iHfRU=y=`GuU=&H6LF|LGP!a*ohXV-wE~I
zP+!TkJu(pi^7Y>gI>dPZ!63&aGz=QuX~_?^20|FTA2gM>eZ>Oc%?>4*E=YQIt~-`{
zfo|8Hbl(^Dwxcf1rB6UOwfJdNcv8n>0ogv;QvEV<aG0j?9RJ0kr)znsF%WEfudIZU
zn9Is+Y8tOS390hu;k_D-P8F?hq#Zu690Q3*?d6h=o;l>awao{+`V1&pSXVY5uGCw3
zC}(X%Nb@mVCaPnw%Vl~U;6wpkSDNXR6g%$z=NiRff#22C)=WRx!AhAEl=Hq|uo2Fe
zC^0GfZAAe(0NKf;T?7^!z1gG_y*|Q;>3W3=9bWJ*ylg|V*ZMhu4|{MrAU#fNE@FN~
zB||GTOg{7V6O4)901xRZ?Oqv<&pqaumH3yqj`o24VS71WHm!=#3eyTQ;q`ugz!!S<
zi8Jj{+GCta)5Cj4JKIdWMl;g7fw;8bdWznaL0i0&=H?;<w{9}aM$5|RGP%pXo=XwC
z26|Li%jCGF(!v5V`yw5Ry;~qgWD~U0C^tHsh2vZg?k5cQ<nxy}tl(vdNR0FjA~RFP
zmnqS=BJ(@Q*;(>x+!|>~M`-hFTtpWg`Al)lV29^;YX|8BQbl?{C&)H%UNamvVx{|}
zZQ>Rg>m#$@@RX=$IV9qdF?z#!WdkNB+vQE&F+>j?mDZ`wZ8R~+O`+YIl^Rws1ju&w
zLTSe{+_hFG%ip;-GS-s{@26)z@s%)bKHc_J5!~C?JiLsR+DpYKSJLHfRq4+%*-E=u
zMg4yK1|dGfTf`kS$p-+jdN@{QBpEmPCVJ$N`rN~}6{9XXy`&l!{4A2Bd@uEh+@gOw
z&n-a{U&T9W)Dkc{b+_YWt)i)uwV-7+cbVP(;-y@Sk>l>9g*tIJzIko{{}HPqX!1-!
zzI6wyL%K~)&u4Gt`QoCF$;W4E*qz7Sa(Y^(gnkKZ%Ky@QxZIQDbqnW%4;Ksf(Fptp
mPtDc=@Gr;B|K;Vt#`f-`U&0wbxGf=rMElff^&C~Z;Qs=?Q~QVj

literal 0
HcmV?d00001

diff --git a/docs/reference/images/rare_terms/accuracy_01.png b/docs/reference/images/rare_terms/accuracy_01.png
new file mode 100644
index 0000000000000000000000000000000000000000..7182b7d3c537e2b376a27d9c68a6117dd2d3ed3d
GIT binary patch
literal 25075
zcmd432UJtr+AbVGEK!OAq98?4s(^(eNC!bcIs#HHAiY<C1Vm{fMXH7lQX&v~hk#fR
zkX}MZdI`OS&YcVQK4+ivo$q|-zhnIOUdPxY+nBYo)?9PG@6%=iRh8w);B;^p3`Qn@
zUq&4UBT<FHh=xuc2k%gQdd~=6PTAdm<OqY2*Fpag(XcCc!(dO+<Ygo^-QpI8PVH$r
zS0AoWF#q6$r+5cQoGkdkPke>lMeo=fE@oy9i`xBX60gl78!c%0j=i-(8GNqLmil~}
zwIEW*YIv~Xb->A&eiz_nN%Qy-0<MMccFm)fpSC3<A8>0cTcVF!glcY^ei->l5FJ^F
zD@?ovt_CKdK0UDu{SPcDn^}Sgyn3q~V~75}90((YUd7HPK`*sz#MIF1mW($s^qP1Q
z+!_q_i;4)kU6|*U|DO-OH`iyHSgc!Y@o1pLdJVx2Q+-aE1f%wSu{PB=*PX=@9mS>H
zVu8b0H^&N=E_qJhb!opxJ2{~<)e?_~V;Zg-VMR{JMKI6{+7w;9{k%6y+i>>a-bbky
zWP3G-gv0qJqk{s`ote}}2u9e+bHvo%Da$c6d*8IOb)J+CyIY#|^%a_0k6;l3j#CeI
zS3<q4YW5dwbL2u9ABjhw<|(VC<1?dluR`v-d+cxEtS5fFd*rpKUw(M7yWmS}nD;PC
zt8b<~bs=1lr-5*pCepL&`1;NP!Ryd<w=?m8TDRoMm*aN{gN}upaWG05F#h*aJ-1gk
z3lsOVzdHD`^FP@cCSXl-u{#!DD>O2c`#hGar{PuLtihE_t|JIsi};~umVUW|Szkj0
z+t4Z)kIE6k8G3=A`)-=2m%r27_DWNTQoA0mY)($!nP^Lv-dw3?(i^I99&!#FuMZvA
zU8-TqzdrbmMQ%PqMJ)d`Z&e$oW+o*;D0P2-$*VLY5!uh^xhz?^P%<$8PL45OK7#ez
zpjLgJ*CDcUYb+?|>9;rLw&U8_`0;5*)71-C^>Saz5aze16J4k2lnG*t@oO(hP7Deq
zIx07e)yXj-j7?kO?3G|dN$;U&D9S&yJEvVUhq{TD$h?tUn=G|Jr6Mv4$8?}I*l3Sq
z%EsV?vPXr_doom*XfH5&?b++)J$^al)T!jMvu<9w)^c}2{9s&s9*pp!{qE+%0&UHf
zFXCV=QDwGwZ=P$|zR04x5xZHOTV9uodOIt^Q#pT2uiPQcQfxDoY5zxj2D~=tJa<7>
zF1CMRseC%2F#lP7zh!Rq6eE|_pxqZT%*Bt1{cns8R$~@j{mxu`dbnG2XnDAucxdiA
z<g6`o|E~Sm>tK4p76v>xaPa1^=UVgG)=|?~*<iYI$F7a#%&9EL=@!g)H3I+e#)@NT
z1j8$f>*8C#NCt|`HNNt14J{!}ar%x*GZz%Tv|2LviQ@YTGxkR*yzGiG!v4F9m95`H
znU-2Ka0=7chb?7^9uybAjdu|IU-k|NHA^M!&Rx|8r8WjLsQn2x@q8YGiZWcgHxY?~
zGZv;g1D5GBI{vO}#q;Nemk&1wY+aQSG-vk4Xlu-DvD@j0wb+p&d~VI~!QQlZrFGE)
zir+HZe*8P8cCf%iv2@R0d@q@+>;UJrAA`W_jX3b@bLM3|{-o<C!+*d1XmB$_W9mK!
z3l7g9Ri@i=^aLvxEAWE?Ui)|>1MV+^*cCEujiMK8&uOn7Y#1GO$7gM#`|0@CI+nU}
zjYiTRd<b85dswFJ4c1Q>n9t2N<&)=Q#l5hnt*bo<<klJZO1n`hS5Hp28ZggX7K(AY
zX$nzN#jlYiYQb+3ZpTUaM01>Bt588M+7~=iec!Qx*q)5ZF{pG&ezw(Yi+r{**Ij5>
z?Y@o0tz7jW%arBp6V=F5n^EKLzuhV_UJ~fMzm9sb+Q4S;t7Akaiu0k>&-XW7>KR;n
zYhRKW6n+`~;$Ml<^|kZO%}8)>G$2`E(^8^=9V1(Eg1u=>5cRmim^B*>-`Wu{7)kNi
zAuKIfaBE6LHu2UB`V!Gx?RJtURi!yU;*RIWWyp{!wF_>3;!rg@zZC06dK>X2x1Wh7
z`^)Z=Yh$iP`5|9Yok0Z%J$?35&P;!CRu+w#nJV`ySInm;s=wTJ7fKB@#7VjNhtAQY
zKg`ldl@0z_pPlZpRJ9huWgLV_v=GwN8zW_p$9<s?pSR%K*<8R!qc<&kvLBUr5Nt*U
zwrG}iu2$B=<r_jIlckizA4*S>vio+JG)0$Vjq*{l>2<A^zwkzf?j0Ul8OjXWTxnDO
zqlnz6Y5`grQt>r!>G&IcOS$z*%2-3_-rg&YaVJ;P(NY>Gp%791<VfmW-~%H~GMk}a
zRphNM`nuYRSajS3m8}cWxv19s5;tMFG+d2Q%_c^>5h-AoeR*Z58z27AOxAv4+z@{4
zY^z4$b0v@2v=~n&Th84#V&tJHV_({_htJZGyoPS8(P&nw<}aG;+LWq}H@a_SUPM_K
zq6Dnx@b9~q<Xii|qHDVqY<I_gOoq0_X}|mDgXYnc*xLsKHT!xqhsYB7d)fGidB&^~
z*1nr=N|V$#`0V_^-xhv-euZ)QRJ2!h+({&@+_W82G51N;?Og76&FPPHybOx#{Azbt
zC5YJ1L-klXey-Gd;JkZwKl_GCyWq^|sC<{P0zXBLUa1SI0%8qgyPbMZuzsy%1?A`4
zDR_vqu<z*8u6q8mJy}wzDCNZawp(hQf(9%b8EQ-)z6H@%O66}R6HydhZ(qr3z%6+l
zdT2-7c3b|{wn`zs*XFC;jmi}I03XW_;f_nRRzHh}9v(7OvRE4a>yvZGi>&fEi&rQ<
zbM?9L%{4mqX3?N)!xD7vh|H1qGE28b!bl|AK-;E{TDG0s@Y>i{yR6Y%UezGeovlJ@
z*&Ub35kY&SYx*AILEQ_y$S0aG^Um~pZwsPBG&gAwm(r5!R@P@bheA=GI4B<^i1hI<
zba+#)WPV7n;Hmn|F2bpw<tJC;K^SuNhmX(^-Dtmgb%2$zyH_R%D~4RmA9<!49D7_{
zl{<3hviV}VOzLb}-I68qkViBpuRepA=bfkJ7I;{a4pjO1$#Qd8_2+g|rc~!x3J|+4
z+RsvuyrZPH&R7-M;)^?*M$)o!v~K)u-cqO$<joI*F-CMHy3Mr}u1kxIf>yn48>c!h
z-4hOR>)KP=_vU*qnVmdF!|5dIOxEox%Tj4xFtog;9vpUDylua&oNYk1<?d3Is?s|-
z9P34KLhsVe0E=YVw--h+FL(4g!c2V<ao(W1PA8;E7YC&f$?a_5X>~DJTbSx~P(|OM
z;2GkF+pR<Qo#^gMzEn`k?YeF;+mUAT@pf}o>gD+AWDLJ<tY2$LsW@414&u|r7+I@M
z)m~J#_AUWljSS`1*j-ipVxM(f{_=g6YV`Lf_?{e_Ff!XbU!5*-xlzT-SI!$4@e65i
zM_HItA4h`vS^1>4co6^<dwy2<<CL>|;npnLhA+}A1u|jDpE<jJDy2?c8z(UC3LS3`
zaHU(+M^s}0C^5~RO$IkY9GU2muFDyGB{My%0*iioe6~I8kS;*x@~WcIPPoTDdNxJy
zqY#Xm1j0HQYCpRu6jHm<kF%x9;hA*i^V&5#QCVtRi>@_zDWBsksx@dK$EmMFBh`dq
zyl{V(mLkcS%yj8jw==(A>8_1vKx`+!q<P+&u&~4F8+s+fr*dceYwBkfYK1D@Ff%w8
zou=gFNn5Yo*w`3_Vl81dvd77~%w&ZO2bYi~?e+&Fhx>`e=AGX~d-l~v4PUX#sL;_X
zv-EA3*CRtDshz4`#5m>NYG}2rZ58cQ9Y0(XKO8^4+E)Jk6UVyrTAI00h3u#N?lm)`
zim4Bnv!vcq0HUf=cTrqbGoswMcuMd|-7)PSU15W$R=JVhpebc_CVEk=C;aDWI3F%+
z5gD+wn(HmF7>ZxphzZ1*es9I{sq0<OkIq`ASK?>eS;@2wlXZe-`w+VmoW7kzMMfi{
z#JDR|PJFaJ#iw*fZdNGe1g|Q)e2Ef35raSM=#*4mU2>Cl!XGBM;p$sOHyz^``LAhi
zbzCh)b1pzPp+Sb)aWpTwM`pPgZ&{{;Q{eX)#cA+6ezI53(_CIZo=+*t3p>UDkoA}<
zk(fKX8I5kWkMk|gT6cu30$0TDs<iKzKD8hJDXW4tg_7V!`HK2E0wdd9zhX;V|Fclh
z^X<Hxo#EBT_Xcmr`y&;kjV_rPg4WUL_(m$-Z&SOM*gJ^@!1XXzGT~&TL6%mI1J%%T
zi{${R^v^7OTYe~|Pp6WPXSG!sd>Y-@og{`unn0AG<D+~aKYAXZN9$j;FSo_w;#9@U
zL8tB0P!XL(ov~0o#cerQW{;&HgE@e90Fyv<P0NdalSSn52{Xf}S^n?xhyRGVVCVY^
z>kX0HCcEDSu}`KGJZ!#J036ca7{#>!0b^d|N>C5zH)Xpo@HwUPJ$kmqr6b6JgK4jW
zdY+oSRVyZUbTI&%$h~nUy&B}c!>VU69e=?ICeJnlyw&|l<o1wQJDz1ky*!~0ySd#4
z@L#N;t)W&6#{iBiwmzX5L2^<6EXa@0@*f~z38K3*siFE$Dr|--lAXP_zA$Cmj$k)=
z4_A3HH~7svuF~^cL^j6>-JZfO)*K=S${jNXrmw~5hi7T9M?nn5Wu`4TPISb;b;yKU
zw`jR|q0aYoj(VDcz~fq&(LjcK&7M`_%!b-Ya6xD*J*^qVpfCOurY0kOSi?S%O{+zE
ziM+E(-dF6`XVkQBt71L3BautP9!t)?PW!vIhnu$7dMx|$o_v{y@b}<2s7~MdOKokX
z{m$6?E}AW2cQIf_wsaf6h9GjhkmGkwZOo;$J4MO^U-gz2q{X|lv_s1ey3iVnY{Oz9
z40oq6p~Uh<4wH@LUK-UQ^`T6mdMs2+asCucjE=iHt@~f8RIcVo!ei$46-p2VIJql<
z7^Upp`1ynHm0tOn48s%CaT=>EH+o7M3}eF=M`FGT<BW5=ja$+EQU+{G&h*W#LLB;<
zIlBBsp*r;!mx@vM3wGbn_i9W|>+VcWi?a6M;Szbv+T_RHTeva0{UX}!Yehb96+f)v
z3VX{a+TTsg@k3;HzR=#w6^yIRZm#Dfmw?US9SFK8Sgirh&<Ac}{)1`4Dem1mTI}P*
z!@X&PwW=B7ZCuTPvvz9oKE|-R|6r?L91z)VD?k%VTo!b-FJx=yCS(njZT4FZE$Mr%
ze8bosd|^6hLkZe0BH8*u(|X1t`!<i?vPZ`kPBZwT^~3G@o%PxA9^0Xc?i`%ZOnEGy
z`BSJl<#-SlZT1(3Di@|=ZC!S?o)iI!D&Tdn7eH4)A+}ZPzBzA>$K{5Wg|ut_=(>3b
z5e9!o_aCGo0QdL5K1F3U_BGHZR&d1g_Nb7N+o*5J<Ilv}YY{Vyb9-7jx+XP;`%6Qc
zwpA;#gS#JE_E!rLDL6S{9Ba?MvWr*5P|Y$P(4y@`msU|i8S0j`$40mI0I%1!BB82%
zfVEO3a50!EzlFcglE2RpqgUEu15QLBAU>1Q_JWni;uDbQ*nxfCj50X{+<1C3->m%#
zLHE7Livq;nO8sVcF0vcBJ1|m_G-r_8lcQJpm|-D3*2ZBs+?a5)FW-N+ABTMoFsf6{
zQu{$}#T+Lg%?G1hU^%N6hRdG`g2l(ii6Q!jm~0;vJX6`MAf~*2jGeiboKa*3rH}Sb
zrAw@5^!N$2&;c^R!N;bp3B`b_@+~eIdhKsu+S<_?8D#^R-s<^J@9hpwi_zxqz2U;x
z;Zx9U{BFyS2}*hJGEQ&w4J6PV`ml)I`Kq-R$ENN}kN{vudW)u4Q&sKdT`jV}Oh1|W
z9k7VXiOVe%izpag!Igw*H(cAWPF0+FMu1XclG5wKkTrllmH>iX%G8aS5$2eGQKVwP
z+r9bPNRFnr4!!&_kKekVPp`rW&81`EA?n}q(D&v>nu1fyQq3WjtLs{&Km4YJSlY53
zpbqubDzn49;sb2%2pL5eYfbhL=bDJ25o1DYWr9UP+H#UaCX$y4xs}Y7mvs>ZuN)ni
zTVm-$C^*@s7}4hcUPLmjXsr9ruDW-Zdv_#|S4QkVKio@*AjU{_n#34P`9xuIOWHRM
z0fm@rSwLqb6adEWhtdAxM`qm=&7JJn8FPE58Za^6=xY_BF^2Bub<aXwn)z{MwJFaw
zEIc7rPdEhJb*mzp(SSDIKjHRpflNIH=UjlJ@qT<+f`}+XDg>vmObVwd0Z4ob&J}2p
zw5R$+QRz`iEG7DJx{nqgDXVoBlP`O>gw=3WA0)3F2_vP!sAIEw+ar5Zf*6SmpAh(y
zk)58>r;M*G4Ayg&p(WZTwjNxCO~Z*TR;?-PVLi*UW>-<mnK$@c7xf9&Rj9$4Sg(C#
zj&AXwJ59*CjxYNXy`2GXE;`PYVPh~K-nhd+(^Vyv57(f3Ek&e0(#v*t+H7SpIYF;A
zO)53<CB1sJ5!z-+F2ynxMuUJx{$PK1x5CTeS1nQLgXRvI;!EzsI>ncE5aI-m_G)?7
z9?QG!mJgpWw>vTlIehK*9<;A#;JGm)=j%oqq}d|>xt4w~&2?$G<moqftQexgBbehO
zs0}|ip1l>L&0UHO87&#GDP-FFbpq4V*NV{0QB8RknjV%d@i8pBpRpZD&01|Y0V>|R
z`QD9+ja{%1&VYsRtIm>Q{UwGn6?^4FlFu7dg-quy@qJjA$OFyPv&!{VmN-n?xJ-Ok
zHDVA!7&_aEwvDf5wPsUEC;|(k!uH3*04+%kCk9QfQBl*aH91_x*NTw2SLZ_Es8LOg
zly`G`362HR%F~SL?llS?UgLqT)*_>XyM6l`KKcc_Y!=8&6mhqM(@fiEnkt&N-PG4n
zBfFaY9#s4N%ImsHn{jmWGJL)MY@U&dHPataY$|}O>zk>Xkpnh;X`^2?QzJsg7c(;*
z;>xr^ci(BgT{9@XU!U~)kX$u6AKch2ry|jER<7XLVS6ZYV%M3J`NBz6iP~@Yu7e$X
zF57(8%JLTqwgt-E1FE^@9AbI56W09i+AEB!MgoxuGqyHV&Ig-t>AAu*pLjV0Z<mUs
z(sxbAu5w4y*qhJ%y3rc-G<=oL_^Fak!TQm|?-6sQ4ebLS(=&v)XZtqhX@9xn)~NY|
zIi0J0`)dgVNT$kSP$PTk_Z+HsuG(D5biO+EVe?{9o5QaYEemiK1=H)ON?)dDZd7GX
ztsGH~akos~;Ca&yG)9{z!iVY?21?mPHgz;T&}T36zpb$F;ky$|+p=%i<W^^-(hwpV
z1+5(*>)ul!eYtGwk-B~A3}>EMrTG4=O0rKN>ML6O-d1Vrira`Vp5CyV+^tLG2R<(T
zEsk1l<Wm+?u$(V1&0Yd>#~e2zz5if6r#N#qDFcW@i$*lRs#qWF2pB54bn$Txp`vH{
zuQq$6VK08TGwt;E+kGnU;#G|lL@p)n6lI=5p}c10noo4U=B`MJ4c#pQCwdK<^BgXB
z>7YA_&M<7*C(w8|j_-Y7-J2cadn#&RAly5TeZ9M>)3vU~A}$8x5e%!slg5a~VogU$
z25C1Y_FRG1*8HH6zuo4jJmO4v`;3MKd>ql4TZh5EdzgXPm{#*T6?`S#*06iih{YLw
zu{BBZRrq41Qmg=`Ez8!jn*0~th61TDVgT>IgEA8rj)4d|^5oofGM1Lk_%psR$>#%H
zozu>P#T8vxD*H~Oke<vcsl9SNryMnrxty+6+L6|>KWCI8m9sNbwKJE6k%$U(B+t+#
zb+#hx0cq30SDcKGvbl9%{RDEz(Lm*>6n9|^?xtB%DHiNz=^n2ks5DpdEqf;vO1?Ud
zfn{6LS`L4DFqk1jxDpLPm@Zxgr3KBkWV*97bIyB)N~-Bb9;01e^82CGjC{IjJ|*nl
z`0a?HdgFS=Aj4gE#2g(5^(Ui9wkS*Vv})l!L|=;J?e#&pkKvZG%lw?z6yf(0fSsG`
zFSeBNx2Ms%$6{<gV~F^YRTqPDLNlkd7isqP+<%BPJEuic#Ru~~eqD7`pLA&1_D6);
zW!dHznvb=g$RnsU!!$;9mFzY1)HN6xQhTz>LwC&g@HGcnTtD&KpjPtGpw^WNOE}03
zsDL1*G6$rN%3y}OJ;7~JXvfXW+Khf?8sUcl-8b+%>7s_oqJ8@v%RQV{?xVten+=9c
zXv0{Yb3;Ze9*0H2m{1FN5pTD9=C4b(e2pE$8ODSw{XnIk{FvJ<qnU@Bc~PNdDA7SV
zNq1fLvLMwob*M$W=6EXeB}TuEo#k}K`z}lo3?A3%5URzgaP?v4E8%m+T{lM@nz(sn
zW}Ocpy(*Y`mh`hs{cwr(K%}8ItWk`Zn)0z>;d|qzbMgZHD2r=+3NhI3hc9(}qhy*7
zqpv?Yx1!@~;C#?S*+obGsQBC{f@kkiuJGrqhL9c6nG1Y`H-v@!kb5HqmNG&MBbF_x
z&&}pE7lY%{#OQZ=bZTe~9K&0jHE&-fv(NTU>VLniyw%VmRCRc-U|Op=h&W&~_wzBn
z=@59t%GFAvc3-UVK*nu4qZCUjyeHL=QM<&Mz(u324bO>pa{7rEX*M?o+a<+42b>IJ
zN1MWjf@VetYf$NgrwAJw=#j#^JIuUaIkn2~d#+O{*DRf?F>GM%R{A|CaI^fKn(gvv
z!#n4Ktuh~{p){?tYSNZ<*Fg`qj;O}Kj-4igOUPy><s>?xBV5gWy+|0iwO)Qdt&;FS
z*?pEQnPG_bsy=JWqpq%&RNH1MntoCAaBj!&h#eB6bUkn}cdOcqbNLfG%4l|&t%h&~
zm58&rW$2aZc59<wQSLVie+Skzv%+QtDBox-1{P@mSm#MfvN-R$+G?2hW8g4J_?iFm
zHXbRU<-(8w!wQgC3BR;{zMS`V)-ad}M410iSW$pVYVr}y{dmWs1yKBZJUN5#L{Aj2
zk=QKqoONs?61s>;(U+%GAy?rP6>INALxlC$fV0VyTw2+AfIe2hoacLQ(u=wmffCf5
zGm!V>6&WyV$eF~fpneMZQ{d#!7RB33R(cpj-MfxcFj@i2B%kx)r(VF|7UCSmDkmyk
zmr5>6`}IPKJ)lgFu)@$6-?~gmWU;rkQeuhY$<iyuFo+{XiuQn-4N*S50^>jf$7cRb
zz+C$tY8#k{y07N}%2MIHHq~4R5L-Wh)j7anTFh|9_nh6;UVc+*-I=a5jv1(WQ3wd<
z*QuW$o&kzj3fwkbV22Hf(d1E2oKyi{EYWv10M?yrB#!S$3j(q`*VA%T2f9aYRGyCC
z!Z^kVS$<lbO*t+PTy8}h3k1_gpRbI4yGGnq;fx=%0o--E48OkDjN*4}+F06L7#KR7
zZcVfW15*Tq_UB6}(B1tglQ_oio1tUT{b{l;%5=IVzT{awIexghQgowBBiU}eeH@%Q
zvbPm1;4vG$N|%K)e1C~G{im}U+*<L%&Q_2PEeno*DBwJ&iJP|tf`t|!>Q2fG&@GUY
zUXj>NW<?@)*7JcUHUc?CwEA7V9C63J+axI4!Q(ZH_v!x{NM%4(Guad~;6r-asw-0s
zas>ee3x`B3P)|)?9VaW;z~f@v4EM>wdEtqTAK1z7olOE{apt&V86d`kwPfmmhu#Iu
z;j4?gezp6?xtc_?0U5lo)6B2qMXEaLVd{yQVB~bb6<Y&SLIA>|I{V38AM*fzKf4=9
z@V}o2oLKhjoQY>s-bnu<=~`<QU$SsZa2q=%nD;6J7^4CnoAUwI-I}?;S-H0P&TPPV
zSM70UD?pz^zi0rSb8h12OcL~deb60%XeZ)4djNU3@DoOMw4o9w*!goc+|)YDmzS>s
zk;i#IoKGqd)oZ?91xR}(Qog`xR-HD4k6k9k8hEO4cH)d=iDD*=)Hq&6KKH~OV}7%?
zTIG1*$>T@QO|`et(_0v^Q)DU2hI%E>FM^^_P@3i|dp)JM>PJq<WC}m-&^D+h?N9DJ
zkkvPo{irbeUeI|PdQxUGQ4B1NE=6X5RhJDA2}eiCjinLsq6beD9OckG=70wGVIM$K
z<5tqo6Pa)t&N8|rx0VD^dRO3y4e8ecJtC1_iMQ(1P`XmAr!9DXW>;&M+kv8w$&yd>
zi4rX50S9~6X{}##CCzi+n2T~B(A*qv7g5`R+;CL^YQ%5DyXOC};r$;8R^ST%AM@$|
zD2_lS1jX?oml4)e0bfwe1TGY2qVkxyE9G90Ro&}T0xwpL>2E%jG28YdqqpkKd*ZUS
z{N(P7)sM~ZZ+?*tx%9!wpvY;{5$I0zB5rw*bo5CR{q!^~PmyEG_Ix;46a$bI3k<G&
zAkLX?&p}`C<F&-G%$4!_0gORqHjvC*KpkvbeO4dR2kcP5310}<{gN1{aL#g`>uT8N
z1N?ch0u(&CdGVorh%y30<*DP;&wSwe4S=)e0Ikg?5v~IYAd3hz8BFdo6KjHMk*yfR
z)03s08!!np{6eUpJw7UEL{LHNIVxl+B$^ewY8sH3Q9ilj3S8;M!?o$wQgFArtqG!q
zu4<|llUQMv^~1uN-6xEp$6%__%iBc9^q7ew>ol%|jKgWt3!#j*eFfh@sDZ)NAbqnF
z)HK!L_ZIncUFc^(=W*s&2hwc;#J-+;jeid=1b#(^=kN~(#D&$%Ct>$#5}72Hk6n@Y
zmTH%Rl^f;z@#?t8{URQX-C0mBmE!KS^RGZ$ms=;V7Dx>yw@q7$fUo~3R=_$NEO-I4
zHfA7UA_Awom%c`rF%-PAB!}HU0nV9sOjrW4YJqTaA^$-r7>5Kq=<<@KPY5RY)`j~-
z?#Og!s(o=MELuPN{x%<s2hgV}oX5VN%F;xue?@{dOSM+DE_=zA;nqw>uB^xeiUj64
zx|Yg*HZn*qi9{1LBkRrizOzkTm!Q&h3TNUVPG_znzDrmtWfXPq1toT9l|?S(_3^N_
z8oGR5;iK#c`}LRPs!*vygaTkThH5u|eN?Nka=@GqsZ@^UF_^1+cifk~87Yz~7h1xh
zk*))A`e(tzrH<3emTgHdL5FB0lYt&}QeM<H%o)TBaBr3QPDEI^@i9fBTFlMtn<@C6
zW>iZbDB&TU3W%z918lk(u#d_0G)bD$k&8&<z5){)o=i|={i8V^m@l$-3o9jxiE783
z7V`~FqQ<@rsfBsVZ0*5(a{peiKnt8=%^8Af)lJ^25Be$2;4w7*K+1f{#<Ruw>1#)b
zk#gBO7(lb!H-}we5&j6YSj86xCI3*W?j4mX?+5>5l85qI&n3oD{QD*nSQw|6xc9H4
zBa9_dcz?l;o%r9xQ4;TGghZ<3DZa$qzIX-3Ob6!5%KvyJ&>j7YOwpFVo1>fCp(zgq
zOyUzXSpHBNNfjQ5^(LVdf$WI?FO~0xmvq~$m=iE&22UjHv()c`{yV{B{`8yyb2(@>
zeUSROI?*8T^lN}8lQQ~u?8*4s*2=gw4r8DN5*Bxndu~_?HkRL=pdiJl2HBIQNQ(y@
z5leCH+0ok)-ZFyR%~3i^Iy4DBe*Zc`^>5Lm>K*lqKG6-7M6lUkYb51gEx9U*nUbKc
zew1_ps$wBw#B0Q+VwN=)r~!JVAUi}aVD%nYtG&tzBIdWB|I7nHp6`PyDc0jTciq=B
z$rwd?TjGSCJ^ZAZ?|YiIUufFXr2*(i)<CDvheYMc#wbSzZh(KLZk(VF>z(b)0PDhH
z@sN`x*Pybe`9xZI(o}P7%yg=vQ4&cPa5EkY4%)q{-dk;QnT*ozhXOqizC%oM8RfUc
z-yHzecCeZLj$M*9(5Ho+=O&Q4r8W%tiJ%gH8!7>2vpvrGe=fJoVIYI_tFc{@Y6e#9
zx@1VO1?6tAW;)Re%Wu*3V;H0=Vgf(|n3W1Gvg+&bjRmYd=A<ep$w8GlJ~~y-7LB;<
zN4HD$4{b)UL;FAk-|oqKA`QYdrklJ_hRyr!DJO6B_vRzm>LF*GfeT1(4FHg!pE=i=
z5h{AHHU8h@NtFGg*-8L!7PlX=01m(zEA0Hqa#Ij^0a;*D__abwKZ$fwVF@UeW>dK!
zK!4XI+j_99y;}qUwA1_`o{Ar&v>S@>3?gPeN|_=E-r%Y|Tx0Lpr&x}adTu*F5hvgs
zH9l{NGml_ZDiE;l*8y<Wsx3*f1i8QS(+FAtu!}S^ur#szrGpMZS!5Lx;fngx{LOyZ
zurzY(6BLYZdkvxTZ1_|>utcc{1JD&FA`U9hAsL|J_f)#t(%*jmwmJSTPE^Y)lZs7Q
z2ZU>`Y^Ju+sH8Zv$7|+t6Nq8VV99*D+(tA9>DbM))CnI!+G{h77tvtoxpEKp(H5%I
z16k+<oDru!sNCZ=5XSl_x<5ZsGvdBJGb+n;mMm+Lj5*nh|EUoeaE&iI>wfgM3?0jz
z+ANH$)g@>f3fM_jF>y*i34IY_h9th#1leyMK(}7#P~>&ySGC~GNd5-yeQxI(_oITZ
z{P~?O;I3Tu!;OLc&T|RP^zcs%n$VsqS2@~g3D^On<vJF`o^$~ep9cn$4KsYgQ24J^
zfN}=bo&QX(66C9fYUmKQpeys7L2t)dc%0|fHHMJrc>CX>=%Oo)H{W_0aim4S5ec}i
zSMsx&w7bUZu<e#FM>9j(_<J#gyAAId@KC&qK!<E#a2<RC%)v^a(Y1CTWL`|n&?~jE
zk<!cQG;<$Sn&EqDRCWYv_jfiN#YX|~oJ6g2=)8!)6T`d<UdlVY9D7K8%{BPD>&iFU
zdi{)3Sdi+QwuGLfi8y03RT>1Tl61`1ErOpKXf=KV>;k+2oA9rw%*<*6jC1H@cLE?!
zWjx%d^Y@?801ELXSlIx|gMmK<Pyd-B>c7e}hm}BSO28F|B&<LybPxyWmX`6eY;s2h
z)90gUf~>2dW8T**!6Zp9bh;142SMK&WbcBjq`Vvekt53>T3(Qt0q18|hO6Y84Qn>z
z=#)cCt@^@)zHrhIQ@_zNmsNcV)pTJ{)5TJeDxf9>>ZHHC%9kXQ40+N3#GwH_E8o`m
zGHVZ7+?tcj!O@+m*IrU(ce}ZBg$4y`(Ewx@MWY`Xfs(NL=SN@@nxIe*-S<?CzHXno
zVor?(g?Rux{56D`{2%3f5EM594bgTH6^Q?=Nt*MplNn*Qll{mU=AzNt!+$W37h-c0
zzenm8M#>kCiRMTEc(M-|YfuPOtJ1|9f->ZxJc!?|=tXka!(78^PbOJ~--rDr7n{s(
z^7N!GtMoD0$5*WsNmjpa^N*Nq;(Hld*gy4Kp;M}%;s<3QYT+T^GT*B&0&!FCTVwx#
zsaLPD5q$#jEnx0>LXkTN@0PnMfKS3mAGfF669B1)Sm2xmU$}Xt$P?Hw0#>~@?s{(J
z#f!Y?c>eSK6M&x{RPc;AoJ-nV9J0LYx>%&hQ~s5wYFR2vC;zG05Fp(S_9vn9IavKM
zS+oNk{xd{9y<#O65drJQB0M0;A>NVP02Ai*;gV!vxd`m2uRSVGvqvVBZp2<Y4~TMW
zh12X)z=7ZBU~PtkvmO<upT3`R-V(99SqzH09-#T-7%mjd85o6SiJ(sPLY63axTR2Y
z8AC(UW+L!6Y%T<_?B4ZGNCwbZ=r0Dx9TUnrryg4m<tX=!7A`#GCjd`fw?2fy(fR21
z;NCj|57Fw^o_;tU)RuHMu!qJc?pP*hGs6eV{^DgHU-X&8^nJK(I!~xMtj3>&)`<Qk
z&_5*5Hv)OWgn*V%`050O?DfuP#hQvm7|MyO68n?9Uio@9<==eS+Hbrz0><S0%M*-w
zP~U#oc{ItrFCgB*mHq~FHTya~J<NKIhf69GYl4B8cdglthZ3&ICW)S_fz{hnu`UPO
z)6-*=&;y|k00gJv-4<V4oa|udeGgC#O8^bf1tj3;eov`_IB77*1$07LI~^#NLfYQD
zSUThq`!G}WGOa&!5@vtpM6m_bc;I&l&{%R}DHI$f%>x|%hHuxwKu#9;T-A-S1rrk3
zEd>BZL#fGU!2b2a9}@ua14}@ScoL49T?jfwhK6m$3E$J~0-yyTJdOtLkr(%}n@YiG
z)-!nx0?Shrv_UJL1o8kbdIs+g5cnhq;ImaKx?=ZZ1y3TFi+tPNd)sRZ6?lweAted8
zOo>O45l8JG3O5)yG+c8T^`+JQ_J&3p4;oKIkTi5J-s(;A-WH&b)88A9b94}}<pl?R
z2Ve%KqI@pc??wg%HT0>15V1c(>xd-iuv)gAqmtvTA|%g8t8e#mSIA>CJN67dC5Wd0
zNNSV6rxtY{=mw9#bIt*w?2CFaN?<~{|9W}C3J|f)b~&a#aCcU~T~C5w$#L>a@3W*O
z-Zf2~RCwSAXc(YO?8pC$5(p3*$W8e50jjwGinYMA?`P<R9E{ODlTymWT_8ni3~Ff!
zkO%vqXl=9su%LTV<tQqlh`{wP(8c?h{IMzp!v+E8m2L`Uxci*s_?bK~Kg;1!pmghI
zD8~<~fJfA+ivxu^E=~iSnk%<ztm8ST2hvS^Nw0Z8u|F|C7R1}KbMq1q8iBw_eLFi%
zeP27EI{<u%x9Rep;@WCekj;zAuLS^iJc{8l0BHrovrV`hJyQT4&lJmcW)6Xs>Kw%b
z`mM61K@woF9dBs3^dH8`DQ$xS_5`BabB<g|WDLSxP`ZY&1#J5QkmxQK(?M$<rfCFc
zwg~_w?5^X~Td)$po#)ZNKzIKcVR1;n_y?t4;q%~!7=58E9Jrv5p`bIabBKH^upSVA
z;;mnAi4GT-1a$*5CC9luJOZ{6Z0q?2ilWJmYuKqKy#Ui?9_0Rhx?2bPJOD__3ZPxg
zQg^qu(E{pRiCKG{>xNOq!?z%Z15yj7EtlxUJj?DLzh=Bt2C$&48??~d)xN#ryikX}
z7NhHwR>`9c=BXxrb<nXD+Ig_Bfio%vuLGc7TWagu-_uHTJv`Xo{CT^ztUR5%3(Dr>
zm=uwO%8_l%fnw<!wbt2d#`~2Z?MU!a;>@0UE9$ZNb8Z`&?7o=*ce^bA`W#^5o*fsq
zSfSt0$L?uyi?EUFHMBqR#=;x8Wwu@*U9<pV`1a-x5&4+%Db(w3zxIxtr&d5U$SjPr
z$nVlui-2bx$(%NtQfV{5X)7N;2F|a*rDgO{s<Y~aBTkAtu5l2+d<72!%rW=p15oU=
zTM8J1UImH!^QN0~S-JhdCev5a;q3OB)nWmQdWcmyZU(i*ZG;^>e_iJ>$RHPhve^Go
zJ|Y`T)E@8zTpxgN3GYhPPQ2T>oLAy~SieD*BWUN{frm29k99yd;Nr@nf7>|o>9H9Q
zUv=>gpphv&83Tz^3|g-Y3bA0KS@l{(3HpLBmQ;|dRJb_h{U|=WFVLQV9oFp}Fotyb
z5SIO}_(Wb(f&VF3c6r0q9s}=hKC@aJEMLsuk&eG(Ul)At0!AfK>}IQ?ZT3pf<q~}-
znAq;D=qaxRtT-Yn<U5S=SRyvjziHj{Mk3Q^iDhp2Nw27=dn7p^J`Rzd)%@=iZF{P8
z^pZ`Q$Q}t>x(ta9gP7Bx7`jWE^Hl+;=^t%wTtRPB_L+?JU5-GIg+fMbTI@8LR&uQa
zi1tK&I#d8NyEdJv`?dKnSGAo?{VxawS3l_iEx?<(ZC&yc_nRYB=v@vLP`S;`QIk<8
z+@w?wNu-4~qsRPB?Jp^jGisS?TLCr)(enK@K5vOzD3Hq}0CfYEVRM}6@XnLs<m(0%
zUHjPMzV}Pt+>EOmkL1wE23eiOe2XtGi-YE5bbNV0(T~cEhh~_M^5ra$!nkM<WBrIi
z6+HAj1CNE`O|U?EOR*a{Z4TWh4eUOX+SmnJbqp%P1%$~TuAnPa5FzVTfu9bk|HQ!1
zjkTrImHu7jxHbilmFWf@xU+Roc)iVp%<E#B-VstbepG>?wFw1Y`V&D}@awtu>7Y|z
zZ4^4k#rGRj%7r)oL5Yoy2(R>8J@7hQI_&!%Om8rfXopJWpyk1CjlFNuKp!+Y@uygf
z7KpyP0q)29XM?=AI|%SEER>Fv93Ctkx=4FMH+s}JHX7Ixp(OuM-Y|Fzrm8i0RpMyJ
z!`;QQFJ!}ZM=h1(WthGTtOxrgQROh>%S{;bMbMK9SlEc>K#g!-V}k5`o{E{2fOiCk
z?h}y3Brz)FG!urOqO!E|w#w$p0SylL6$79*Mr>Seg8Do4d8O^tStP4Uf)z+^o#Y+?
z7Ef%?^z!Hzm(>Y*C^Qf&Eue?L1<n06(b;JLXKgZ<W^|8qjnB7%&M;7H8PlDkhYS9>
zAmlRtJ~qo4=$AIxum4)tZ(}jQrnz`GBTh+Ty_STctifD-XNF~WYdmz}^9iORXah<F
z7z}p42Pvl`6Q-mVfHf0K!VkgrY|!45ueX6s9{7RMTyDdvUSZe8xD{s;9*OGp4#mY<
zexZfGwH})*ePBOKPjfO<M3UB7V=TjCrsLgRv;$25&Ife|xHgvOffxv&{!=LOw^i|=
z5j+>54(NN-1I^Pf8fx7b)B<=I_H5<Q#xhEgxl~QSHv>%Q)+Jww8$UB(1y6_LdX7$h
zo%76UCdUXcrUrj_f{()9$N`c)4+8sVo7T$!ReJO&Mjougk*)87RsaRq@L`84T?YU@
zKH6;sz~OB!>C+I!XV(YY!CbeQ<4_wAP3OYj;sDPz&aovQggaZbat*C>YYr+DUjnF|
zYheG4rrgb)I^WX?_WYL25l4|o|2h|>X;uNqr|cL6W$A@fSf)6*%s~1$nBLSAT(cE5
z7Q%|8Xy9z`F0O#p2jxPYrGcgigPn}-%uuO(?wlg@0n|tQ)T@r(lXMpPYtm>FR5Tj>
zH=|?MD!VBXOrx5Eja;bbd91g~NjF83@2<~2<Hc@0<spY=l?1u@N9(BV@!6i7Sp0D6
zi>LfPhF}B>DtI(NOG-d&K7w{=4dzWIfVvDJ+=YF@h%4I(aA;B)yD<RFyr%eDm%*HC
zUhg`=8TY9R!eoMdG~m>ipPVy*tpMzbMh}CdmMr4<K6C>e$qJ$lF7rA#-Xn9h(I@UB
zZLz3+4YHb^bR0|&J1Tu8utxw>wc4mJA_~3<W-J3>#(1dsA_|ZGF7GIHbmb6=>jskz
z+S>!tsLnu68`2lK7KYoz|C}WHEi8`$5Z7Pnn!7Mq-pNxwajtT??liQuEZG_OSPkDU
zLnoI&gGo|sJ2KpS@~8J%=$*!bxT`2=3*LU5s7H^IMcR|*rYNqMpbTi!rD`oOXN`e|
zT)aBjG>wR7`k$d2IO_fzp(O90x<zL1l-}R?99$;~B*8WZB}=~lN}+x>RcL}igO*H*
z+sae0qbp*_Z9Lef51;~D$Xz{JH|HT)fLRA@<FW-hh672%Z&$WA{~6@U0_c&DrP}vb
zUm^n!6tnmaA#k(RtBqVd))0FudO)HuBSU`a&SP35PkSEnqcTVSujteBOn#K?!XjAM
zSx&Qq<&MfY8e?yD9o_t=zFD`{elM+TAYYZ=Ia<){gKO%@1OjIBTFk#jRvCEps0dhE
zgCht<0$epv?NNG!<=jR|AyKk(v5NGn`pE^{pO9YlOcw=OA&T>C6JOq|<8D7TDm1~J
zm~XBAb=3jFJhWI%FsvMV!{xBD4n_zDzyJ4jt|N3Acl5Qz_T#bxHC{EA9$NqgiPEZ^
z0-sEh;I|zfELJ_8@f-nAOyhQirirM8xmhi=PYY}&`-NJ9+PHH+R0YCb<U_i_Z_)&L
z1G4p<nQj(2>E@wyz`N{pMmJg<&AhvL0K!(x<r{(haR6Oj2eTC^ka!-PLLwm^WJmeq
zshTT4zM%%_puL<wu58L5tY_WyDW}F$M^3=By8&kbT@v5aF35I$Qyr*-_7om~?Sgk>
zp{ounk-G5u<Vkq*#wXoGH%Rl>&4lEk;BalUdRkSE;yo>r$nfV|FJ}*Rq1g*NQu(<8
zoRKEg6kX6_1R~YqA(X2I0O!iOPM6>xi!%E}W}0G5MWP*)teiOnkEwa{_j-{v=9e&c
zB16qz#=+71!V#B=VI7~RPpFoG>jWOxG;2FljJ=tehoCR~%jVw=^j}y{TE9TlKKSQ4
z_4^1jB5F*^DOC{4E`g#04W7`Xh_>#{`wR;GRF8RH(4PT@1LS`rLHq5zHb0m9m+(S)
z`4JUYNHZhu0*%~;$!jkk>=ohTTK!kfT@ero#$2ha!D{O)!wc9BTPdOV^1%j`UeI8z
z0lx1kwKXbLA<YI-^VtOGwr+i7C+;e26|>~6UJoAv8#HZh89f3ktULQrr!S*<UqRYw
zK|r{8&L*9+{+&eVip&KB#ismV2#cuPJp`mQ2k^C?Leti`IC1EdKTiTj+if`s5^1Bk
z_0Hc+UIGT4)adurdc6`Wj{H!W0SFQ=R)BZ=0YMwO)%_HL5<qRiTYQAV)Bu$^Hy=QK
z1X{yg9JIw!Ry8g9j4*E}U<FdA*PdYajX299s{=%?E5wmLK|9xcJ7JV`C$H$HSX1Rd
zsfT~G@qEkw36Il*U6naTHQ29fbFO3ZMdXK?nK-m+=xNxpitm=8K|uX6>R5>rnb7`T
zh^K{$tlkfF21M7fSY!04(Q<&mgMmLegT9-_f15~9j=S>&$OCVB0oX=7O^6v58PDNz
z7f0}qJ^8GkAtlNKs}-ji^Kib*NcHr};CT=&$OMi-*Rb0-8E)jh*!Bd~T__k3P5_bj
z8+>$$a!n64XmS9HwJ%jFQs(POKyEIL9Mw{QEGj1He%8@TNF7>y#m>N>shb4(k)qoZ
z5obfe=3T~tLeo&u^)_jYN;vY!2K;+uA!P~)m!qRj0m`3!DG=lB>-ODbLm90#(jVjl
z-c?>tS8WTVrk*58lDp1xk@Nvj?T~nK+R0l%rdSN3@X6BtP)J_CuoBFTN06@K{yU)%
zWd))eG9)@?|JC>_Wut(>u27FL%fR3oYNVM!tm7K92kme0bqS=Bq|W*ZCPRH=2|UvX
zK(Wgc?SKO0ptM7sUDe<Wbx-P&6y9>}Ox4eTW@^t_YxIT6t;x(nZC#>Ib)5+QS?xzN
z?~#`W$tiWx-q3~rM?3W2=w1#TgC{M6&3NF?U-tM*F-Q7N(f%o5NIU@nZ191<lm}?&
z|M9m0*mX^wP+o!g`W`)&zy4ceRYBLD&!5sC`=$TfzDr_{N>kG(q9*^_<{_yMTIXv-
zjsMh;;Uxbgp8lJ2vkRNpQ)|O~U;ft%%q|$*^ZC<r`yu|fZ~K=6$qAImi63XPz?}K%
z3>Jq2zA=V3(doA|4-HFWLhXr68ZNC~(BKWcg#mmI2D?XqoG{h%N11#MRnG`EmHxLE
zZ$sNaZW1EQ2Hfx=jG`7mODq9iybf4WR-<*kE-gZd2CERf4ZAFD1#q@jxx?h3J^&P|
z>2mg@VP3%WK0$sdkBH=WZ}>IEd6Xq4AN*oL6WGnliEmH!2T8N`90*~MUAog&Vl!m%
zr?4N=8|tH7Wet30I@ci}spkOmU*He@BS*iy1KL8oiS|xomV-19SYrqj2};w*39XCZ
z13DMj?&LuG5nB5|1aeB2(=D-@t!p0p7NmGA6Oa<%^MO6@_PE*YSboc?X#5E1bCrj!
ziQ*%40#;^_;?dxIGKJaG<Hw)%ZbBP4|Dcs;0|PC&9@3*3?z-pz4G7vz0QaRo3PYrQ
zMS^mA-gN*(0O}2JCDwYtlP%Lr*Ubh>a4!%641&9WkP0btn7)xR`|)hBpe_nqLQkdd
z+zgDy1~>%J?+8>vKh03k#N+t+-hOzUIqW3av9sLmC4oI0ng3W*|4lSvN(Hc8n1n3N
zJ@5W9`&9a$quAX=M-sumftZ0K3qx@av_Vj=+mc@aF}}|C8Y#y{;3u_Izm3JAf$wiG
zJ_7^g9NNvKpakav@>@QWF9xV5T(XkojFLhWFm^?s<zf2Qj<CizU{*m?qkCCo<tvrV
z;}a@>>c<10(1xH%D9p38mo#jq)}|-!?-D9Mb3vRV+EP=(7Itbs>+Eg{&=r0@ZI3ms
za=teX1uw1_;#k<HiC`x3l*1mc?8u3v1GpK%gt9QqSWZcI1e=^8=CgCn{IwWx-DY0=
z)pj#i4Uidtc0Y|GMd-v3jgcG}qiC~rv5f|ce@apS8`PYfy}5LPjA2nt{3_W*9@y7I
zFYJ^Fp=<t1FhQb=fevS#`nMDX6omV4aug@Dk6^DkbW*bw(2l<wjDTMqpv?med|I~i
zw5?RBc_-@#6c?jAb;FZSVzxUg#<C;h@uU*wG%&unPIf+q{mA{71r>d&!=K6V1=#8c
z5K@@<J*FgR-qiB``*QHtUp`T#_TeKt?!9+ZE{}bS{{AjhujWJIIE?r$%=gx{fDy)9
zP&9(!rnvZXIqL~hP9N&9n^4IDJ2LMRT|BDX?B^xP)P-}C;;G>0q2iX*d+hT4dj!H2
znagrV=8}mLyU5$?<vb!XGX8Jfb~zM3v-AlOJ(7jWKD**M(tsJ}2~i%rWTUNF%6Yhu
z#L@d)9XSY&AW6=6NAB0-vcDiZOyU>mQ@Xbg_zV>J@B1DqV?Mekyk?!aKm<3DWft2T
zd(6WQlaR8Py4Rn`gnXsBCD3<<)4WH3EWr!i7Zl@np1Wdl;S3`+B|QiRe@x(KVagSH
zpE%BbUVmxqEFw4P;mxY=j_$eQ2n?KRT->VwAHT#L_0*1TZw&1vc9KFl6k+wsKOqL4
zN_-BC;8unX(~hpz8PWvLU+%ONb)Dj*`}~Gxg#M+C7LzbeC9^k@qa5o?+_Mbl!r)+s
zWb4(PHVXHbY)oat-%{J)UB1kjxhqoDIJExK^)oLq&s~CqKN?OvI3M@*w2B0Pb+E17
zHVTVBKCgNjeBKOvz6O`bXjGvz819$obquZZQm!Ue!EKcaCAAv@(i@u<Qj*}(zVSp+
zAjIA)gyIcE#)j}ut1~2R3Q!Fm3>iJ9t5K5**FDDHMI`Q<s!ceMrs<4c8A3`KX3C#`
z-GMfFgA6fiuF<$TwTc!U4y{gQu1Zi{1{+WE6*R*LV%|RbzZ!}=^p}>cRfXGqJFVVw
zPE8xqvc!9k8;;v(3IsiT&-CHlzJIJfMt;y|Z(`aa7EvKaqhTaU?_qR!QYE=|LOO+V
zGDjt9ggI*NfI1wdH=jh!ZDUu^irMaZd+A~rc$Vz4$kT3taFK8Nln)-7=a!aIU|#Hu
zZA|`H*B5-9@wLM<*dv)eVVM|_U56-Nc{e&cKkqL-G~LP){Vi)YK#p`9E&4Rqag-wp
z@$HoInACR`j)ld<8hSPvd2`tuE~PMduKXH3N~EEVDvawGd*ms}1d_&x!tx|_?q8EL
zdgHZ?T^pO5-!?XRyI+_n(5J(%i(EU+fL7pt{8jrdQnFW0JpMs9^7WG3OJ>6b<?OW<
zV@WY{x78<YzY+(=gKWq|dk)$54`LD-RDAF`ln+tAZf-LjvI_gRmE1%zf4{XckLl$K
z-$SUaz1D{w)zu+lM&F%vyXrNT_)|hu3tv>;H>$ZcI<YI<<7}=Ezi4$}A$yA^Iy{hE
zV>G4pZhNLJ`t{ib!S?O#Qz|*Rd($8DLwVoqrR0wV9iF*D@Z534eeKOv^Ud;jE{m02
zvPfJ<KT~Xzaj{r^OJnpBAw)!(<o#5wU%7>PP9q`^&tNOAztnr{Y+U$)pw|1wS{U*6
z8QJY~aQT&;SrOGN&E3$<m9qZBr5DMpQvnXA>LTtYCbO|f1)DZyU0bph9yM?0zD8Im
zzdL6iNE!J{^qX(igQ|>)0+q{4FMojT)09ph_NGE8qx92jIMPm;JVpDZ>yTDyZk`oe
zQi2mIE|92SIXjyCB|~!7gDG<Ghu##+{cUT;=r=REXtz<0iiRiId68x#XLsSJl>_V4
z(UDOEYKwT)S}-5OOvXt9Q17-1HeRaOadSkqMa=81XtI)TTrI<4Bn)6L8}OCm8D0v*
zqLgehfzkf{nHLDRAFbt?tL>Xf?=HWn+4sx%xGUdvb*1x}ZkKS{P+D%=h{j13b~~`K
zEVE#r=<7+uyH)4v>S>;6%LFOAis>0~cmr?R>aoM?aU&0}wK#s`n$dHYmd?1+E<t|H
zNBz-B<<W58OslHz`>&_()JM)PM>_iMBuKeQT?FOKCzFp!0_|DIlv_J4>cz2pV<JuU
zU5Gy7muX6E#QeeYEtHC~J!71XdXD6_sUxOCl<-Tg4yEzJcjVZpd@f;m`@w!l5-U=!
zjEkyRezwEXHh3~umC1lBs;-*+urjb)zwlRGL`Itw_2?;+V?^+iC@0~;==3!Nr;@EY
z%~BbhLuaz$m16;GWExqHJ?4eL&ikto3Oek~E87<hc+{BQyWvEt+T!L`NQK$uQ))&%
z1*w#KYPfs$$#*L`+Fd!?cP%eh#Z2zH`-^K|KgLGQ3?A)Ov?t8O;cJ1FU#1THnvTgX
ziFmu{8`tJS`)Y3FOR*^3^EK)hD}>kg%$51(v_Fk<=x%MCrqB>NrQoZ2Uq#Ljbt*$7
z^f7oJl}PB3s$$!=idF)HCG9QsnafY!Nz_~=9beifUE=p5THilY$6-UsmX744v58o$
z%;iSCM3c1m`tPksh03Ll`+Qc;6<&IePAmE9u=D2VhhCVTt}ZHCN_|wJa`D64YRX(e
zRc|n79>vU)un$q}I*Co-_Qpqa>LZoemLD9)-q}?s4@*?23DCcPvyAob$b;jeM$-Hl
z8Uv9m*e?+uZ-*NR`&<YQZ+IxqP*cpDR*o|nJpg0jQ4gcOK}@Y0ea+G$JePuSY~-n5
ztJ{((Hd4ny?MEnwyjvq}xBEBlc`x^$=(p$3yArp08aOr~?cNBDD4WAwKb+JsNluDa
z?BnUm7<hd>%EKUf(Hx5@B@9Zu1c?iihvoyb9vxe%HEa!eZ9EsgadcUJ-|ZmZhzpWJ
zo_cXo1%1q4=CVRl(**xvz2$9F$8on?kIEFaYIziW=RBKalf|?N71xKSt$f+*sEEB`
zq|{-pel^wn-wW=iEwnKTU3q_)oUs2^>pYKP@~~d?tdQe5(hzl;^>N<}Y99^Z)UW)d
zUgXxIpk7VX*fohC1glBd<3w=25l$9Hs|Ia9pPO!aChAcdJA5s(IzREsA@{bZ6w|AF
z%*<e<$-lv+(f51VSmd9GUC}pw@wFgey4j)mewmEp?1Pd!m#?Z_7FWyBc*C@%;V8uE
zjJfMmPnApgD1iT_CzoMK<R*X1zA_n(K0v}-s$H??68DD)hAWnK>N#mET=C11cxA%M
z)zfMbiCGICMyD*F+62~cV%$gx1qUyU%&3h0np}0>X!R^P=PMAP6H(ykPhOTl%>5;d
zf`%eVKeA;mvkU)<)C%;DT-Y}Ce7kp^d*n06^9)bl*gH%gH9V?}beKL$ZrAcoP(Eb$
zaR2}z-Ee%+(o4O1cn@(o>5Z_&f5VM9Z0T)>fZJvbd{LLFpBJ;#)826Fm$kJBO&2O+
z!LJ+9-!cx!c~l7fxIkQ?V@piK<T?NMZfhx#^h__rTUwC4R!v{7l~iUa_{7(s)|b^I
zhs0y=F))VQvE4t+Zhl*Buj2+^dX|d4-IsYP=;B*-HTK)XZ0Cw7Jf$=}B`a~?I6qrf
zeHHP^cvcm2e8$k!m?zVC51X+}@P%>p|K7=S<FPmJ8Crq&Zw5kM#gMX^2!iFi!~aRx
zkxOHgV~BFSui^od(z`Vu4IPclxXpZy9M2ZF#QUmu@5K22LRm^y<GwYjB#X8$UFL5X
z)(GEcRQ*n4?G35K|2rC%e!PwBgGzUqv5j#@JIX**0ja$25C2Kd2XFN$D|+6IfWTAx
z^u)FWV=J06lq13iiA4X>J*H8$OQ-mbdMed&^cH6no?B>Zx6v(_o>IE~c&7D0iEmx7
z+)>!8qXZ~5A(a^L9*E%Emp&-g0TF<F@JkvMbZ8*P_km27L17KBr5}Ox+ig!jAdaVB
zIi&(CNHrt*tIiemFJ-QOL&^S3tHWRc*4qT8PJc5j1<50+?UUuIMMR&D%=O0;f~rRw
zZX*B4M1sKYN!n=Yk6mn^{#_~(7&(dW>oJ1<Zuo+^jO~RV)P-vxW+GPB8peNuJ0>yV
z#dcN8nyE8(GR<?H;&thm8u&(_VjXS5F|qiB1H2S`WeHPoQ3gcM^g<a$tw2J_W*Z2*
zQY80hjDmY<8Is^iVgodlf@xykIw$JrDX!Q)5ej{?66d?d!}!guCH!!WCj@)bx9tS`
zA%X4(CWY)M@xCAaPg2)^*_{I=BL{vv!>zI^oihlCU>_%=>jk-?|5eJl$3vO#aeS;$
zO3GzxNo;M1q-_bgmr7eB*FlK2T|{G)dkoSR66NT!$RN3kT*`G!TS+Im=2B)dg&Map
zuJbU{aK1l#wr$VudF|`G&Ogt8^SqvUe$Vgn{eIq`55ok}F*tFjHc{11-j>7V&wz&W
zlvY0I(=-Mu;UeTd@+<ymTZ)=gC^f7q9pcUu4`*-7s>-w-(E`U5g92<9z>hM-h<_j$
zZ5a{!Q%Uw0up`tgr}ns>DYndrgzi>x2R*w~@ydso_CxuXN?cBVODU7-4_?!+x18l-
z0Pi7dZA;DC7Drvu5|=iyMYVK1_D}5`^lv_6%8$K(zFi?ug(u^B?QbZ6r9M?Bk^^ah
z-fFnGQ;r9N;?}VE@)$K}y!T6knC81{nJt%=3FTh~SIYU6AnY^IHg28Xc4pLd|KEIk
zI(f)k0(T<zl%Je4D!kQ79eY)h=|Z0AM8ggkd6$B&&qKVqV6suOra|Y8Ut-F&Y6xl!
zHYy!G+ZUsP$dqzWP+EG!7s!FV$sYRAT0@n}>(YKB_Y9tKNE6L^gB;>SGhO-#RuRNn
zH;p|AVkrz@56r@8reVg9jaR2X#|ciaT)XWd^l?rIi>)mGq!I%SJw*<3%#>6}#s0HB
zq$5<Kthut9PNKarnxFj<6LncJE%frverXhrs85nD)XJD)0gw@@c(^Nc{1>r+#}Yb~
zk)4o8byAT$o!}BS_nv#bIJ6!@MUps=8~^OQQKHf6^;zk(ih6yHk7-Ew0l#L`Fj8z_
zFI*2yV+EQbk-V^r=1X$7yFnHu0Yc!g22}v}${{|Z2U<zO0<k<#4j}8*I;tl-auY&?
zyHf004IzPr3uN%?CLQJ}&_cWc$sa*2;ueFc%WMend4quyj+q+X%6?PREuiPysbj^>
zPQ_D7bx-*-BPU#{{Yn6V*t11)lJ8XOyOI~;R6E68G9l3D)e{p?HF|@^=BOHNOiqln
z$`|~9KOdkq02`G7ms1O0zk!04H}shrsd^y@vCt0~y$5J0i<5rlEr`@RJUdiD0!Y$u
zsn%jfB}zm0?E!dm=!y%o`$E90TL{Wn(dKAONY<9;13c*``n-meWjDUG?-W<<XuZ_H
z?_;fDN)C?=Y&H%5{oV09Yz~QV1uGJkS{{q&X&c7t+}f#}f8M>sQd)>Q0bi3K0I<dP
ze4kQJXmpPW6POH<yoiKnQHo)Mb$J5tIv+KLbemB?9PWeT9Dz(OFQd|}6{G3iHBI-k
zpm&zZ7oE~)&c(a<we6WRD!;Xvi&eOvB_+B;dIMGC6^GasPOF~}BOHWBFah0;Q4#Q~
zXb`;fHlsa-18;&24Y4wic?RG)hKF*}0+>gKQl7h?8LMLT0EW+2F`kR@THgjSNP)$j
z%k-XISV7h&H)>Q=l2kN70PcS0z#PoKIspNhI?Sk>()Azb2?seu>&qNcZG{L%)#zf9
zzNMwf%70<|RcmeVlvJh+v?jVA=su%_&j({fmsaCb23L%DxzvfL@`(M%W>Jb0lH~b!
zm6c4hlvna1I;+m^uItLR+PF>OML(j&7*z!kUqkn{T$4O>fD3G#X!g~FgC%t|S$ar{
zOg~?qL;NL_6pLX_g`5-h(KtI%KXy-NoNd->SFP&Uz5Z4+3%@eQj#>FW<bpD{Um|aB
zb~E^-P;}4DW<;%j{&MyEi$;FaLYb13#i)+gu{`sSkPOs}WgNS{H>aIB1rw79dS0Ho
z2WU9G<O0VnmnH<TI??Z}Wio?E%X>qdZqu0Vp~mV2m_q3QEP7K!nl@%seQB#usP_%h
zuQ|IL)tbug!68W}60hrTLiz551tnhm^=+%T>2p(%c~R3{d4$8esQFB5fc++r$)4{t
zw@^DcEF1*9I1Kvb_6m+?&jvMfB&Jet=4oEO$d~BjJKgpr*kq^IRcxy1HoZ1_>pZXb
z#%L|N7M_xNf_-<*EJAIgmQM7%8*Bk5;4lzSmL!&XQOJ%y;hw$v@}&z#Zbw4c+7M%l
zF;w??PFk%PtL$H?qY^*=;QZNMSv6Z9=JTg$wL8jM*MU7a%2L>>${~R@K9@uMXR|)K
zK36skHQv1+{<s;50HAmy_5;q*A6R)7f7Ka_jRe{L;HZ9qgsJ`I@cw#U?^C|xIO=)*
zLFD+$?*8ZPy#K{xgzg>s-+!!h4tkY6(6+v7mAQ9Jg@I6V)Qi6FWsk^Kc=u#3j0Zi1
zREI1)suK%{1<4>zo&-IU>$))Gy5snZwfy#*buzvw?dt+FZsq)x1`<#b)WZ==0Och@
zQyqlS>eqncZ0LG(cI58X%j=B87>MXbfH-3kl!YGO7DM~Kg?IFU#P|>}J9U*FYpkT-
z@nrxF?q|A5nl;2OD1X!<j9c`OnMfha5Tcelk861|UTM&Q&Y%jobOPYgn1i;RxOI#f
z$rp$wCm;*x_Q*O2qNev8tTG-*X%#fmKE@(vHi3H9e}qY+rG|`Ec~j&2uY`ZhM5a#g
zhpgV2?K8op0-Zw#`y1MwjeubS2K@M7AX?QMZ)#S6Q0)ZhX5ae2LRNQ}B9L)dC3MX*
z*ompJ@ir}aKU!O&9r#HdO2rD>w-^#NWoaV)CJ`~qI<z@Fe<1M?3g#+7DOlhL_X2TY
z%V~VM0J4tVQ26eP2JuGmCn!&q{BF7Hy#RL4CW8|=lhZJwCQ{<U`Bsi#$N|i;NyA23
zS460DAUb)R%n4zZf#}fi=Fsd8wZd@Ze<~=deqJ|QuZc|JD2)sG%#kK3p=iVa%DwN=
zGn9wLhgf9}B^-9=F$`kPIO0<v`IV}qg|@RB5<QsWTH<Ena_=$*;b_qXzDlIydgVj>
z+x)f<ryD}TF2o}U9?;!Wpde<jrBh?q>;(endjMjS^q&HbY~b#?b4?3)Z5eec2Y-IM
z9<Z=Xs6~|5M1UWf6WJka@BEH(s38#6S~KSbco&uLkGP447kW+JFD5q`VO1?v$K#j_
z^BD1AgwUA5M`vH$qL<G^O3j%Gd<z!<_t%i#>kcOP3fBc*chQF@%PsS)i7~q#?Upng
zSg&rLJ08wT4Ll5Lm2*=YEY72m%eVSyt0CtY-5Z?TXn(3Y9t4wpk&%IekO7mw9!fbR
zs3Uh97&1)<15x}891AIkW;Y~gbQod`o4(3^LmGDmwuNq_-)e#s<-s>&m7bmPv=^S4
zN?TwU67Wf$PXTBoUEs=E&~irDBDp94YAndV2pj@u?%SP#@(~GU-!I8E^V^c2z^~T~
zTU~uz+#F7HGyY9Nxcc*Ot~63rQvK;8q+t4*ro-IYBzoPvUY=CDx=n1qh@TZZ)*1H3
zvEVyUSZsBO_B_y0i*N8nnP(L=0dX<YI)piqOrg$nITg*_$^n<*w-vG2N5DQ+daDL?
z;lU?c1H?6?NUmooF!5h&=OE$Pi<`r=O1INU>I!M}54WCxgrXYdq~VbdR_X0^pN1mh
zP=4DiM4}yot@n(`Tsxq{`RJpbcECTjmCu|v0*Ns|Tc^Z>%9F}-E6(=G$5#X9=HmOE
zF+YAExym-Ne_~fWlO-v+zO64cmUkm>DBU$*@D<pt4MAQSbFkZ(H4etQ?e=sOPhQ|s
z^I*BY=Z2q0bswc{8?NmjeNU6OICqG4a}-FoOUN@C3UG3omuQZ%yXSU-oaahiGS5#R
zKuPu50=J@h9aI9s189zM;9%5oL1E^fUx5Zhg}5BxLyPywe2`}Aub<hC$a?48R-B~V
zy|1sc?D1E5>UmX}u5zn(;U~4MDWYYTG<Cy@<fdT$rIm)_c)2i_^Lrti_9L4V?%Pm2
zey87l>iIZs$ZEXvzMY3n&`USjcip37HJku#2V@;Yf#!if3_}n$_WLO3t-AZcS?x{%
z?ueY+b#FK@dA}9aF0fe|ZRaJ0$B&@89#%nGNMVnfgokAhSsi!sVGK&3`#_RAXYs=$
z&Qw9n5fI~Cu3F(HAJYlsxanK@n{8YR$H|*l<T0=9%j`!9c7)8m5*2Z(uOOa)F7ZXi
zk6pM`!)4irH)u@dR;7!Uw$9`Mt0C{PyJaZ#=A)9LB+>F#XM%V}Sg{^1JqRRvC85q8
z7sQrzw*NHc=`5pbJNTtYq{D(FbFbD;il?sDc8ZQxh3XTJ>1u~7ostK%mE_Ji<9loi
zqz=@QV|iRN8~l~MRW`({&5rnP$nk(_ByG3=vF2;+YcdaOHuYZ{bspGf%uCR>FeJ^M
zzQ|Ee7WH46V{|0B)uG|eJi)UoCs|n`ZC){~Nqao%qNY8LV6#v2n;6drT0s|*hld>e
z6s)U0;3E3aj$~kc&S2-;*@RrZBPY2EMw+(yEyb(@9W81UK!9)PHrrxZ?8jp9tw#M;
zHnbp_%03B`cP#jjBCq^iHHY@u&)ijiizWP_6#PGaJa3$y?7Xo&+Di%DWd9xuv!Y#Y
Gk^cfCuV!ul

literal 0
HcmV?d00001

diff --git a/docs/reference/images/rare_terms/memory.png b/docs/reference/images/rare_terms/memory.png
new file mode 100644
index 0000000000000000000000000000000000000000..e0de5c2163913181dba940847b04564fea2f9b8e
GIT binary patch
literal 20465
zcmc({XINBOw>7!}5d;ZJumK51l3*ezu~f2>K@<d3Kr$F8awr2LAVC2UkRY+hk~1g<
z5S5&vK$64)Bo$DEZ*IE#efxa(z2`Z9?mhiH76o0q_S$Q&Ip>&TjJ5r>G?W?WIOz}s
zVNg|3&_NI?Ed-%xr`-*I5li{n2|xDSRk{8EK^Q;7zgh}f5%+TlqM4+sa6!-Y4Uw?t
zHETx|>9>5~%v1AF^*zIs^@)jJR8Hm7+Jt{#rdB*n!CLxA`a;m<qxdpB#oiBG{$&C=
z7cecSgh~}@|L}D^DRg)@wZgN8<BzEx;VMZk3!TXomhP$UsTG+OQ)IU|)tTAo**0;)
z=e3UW<EBMS8@yC7Bt$R6xtRfdgY&$AqeS1t)55?J<R=RSdNGLSVGITQz-jIhK!2V0
zN2t-Ca{Cg{AMg1oS<#=X3jg08T(8?#Y+o?;Ez+nhRh`&v7Q#?|{`&I5x08J?)r;KH
zlgVMNTm7Fue_n~K+4!t+evH+&C*PtselX+{DXov1n%a5TpMQ62oYS|_!Q71fq<f;p
z`!EiC7>Yxuuv2<Q?`}Sfz4wK~`G<H{ctsFTmvgKXHos!7TKb!aLV5P}f-l&mm6;y1
z@iz~GWtZD^RZBK!3YuoN$a5vfF2CxQ-(J(qG%CsA_Ff&5xcd{sCF6RxsBhS7OGezL
z{oP|Gxo#<^iJx)X^+!UoR&0-!Z!CPvw(ZJUfvZT_EbGcKwv}J|s%badU)EXU<KtqQ
zO1hPZ4qJX>h{^uTa}H}Ck+u1cOhi)6cGV`)(dWU3jnGSsc_Sh614_^TxW)hG?!{iy
zlVLtJ3jXm;3WM8QBw}%&gK$AeL5~u$=8I52dEe()V}G)8{!!K1uNVoNc7aUYG>s_x
z*(#EW$7D*R9eIV=<H*-K+vKxVz$Lg=C%<*ezOPuloG{gqY0N+0H@H>Iz$NiL_}nmU
zU)h7SAMx^oqGhA|Nl|NrIq%5U>8){}t+cA&U$|08lZ&K{#iqWAmLvj^iiWW}RXxUT
z_>;ebk=Jbd=^qA@1Ah8>uD@>|=sh#^LYXIT=%b%e!JQv7kLX3;UClP^NsITkdGPCd
z9(wH(ZVRKy{C5@}lGYBMzi(a^B78#4f?QhJB!{ztMS196|Gl~%@?6c_)(hTKH!=NY
zToQJ7W4q0DwO3{d(~a*WZQASAudAjeCx3FD>AodmSokhl+_s!7MwySD?kpwW_S#(T
zT3H*9H~RRLfw1)2d})0up)=E<si`l!*to*EC(_h=QcAPPrbEaj#*4gCR%{zDX4Ra~
zZ_lby0~1L%7&EvTgHQAtrydHZ4a1w|lFHz-k{wM+jiYP^H-@B|O8V~yiaQSU8+e%d
zY+_4H_fI<d40@416G*@7c#SHYi8CcNoAy!2nC0M9L6O_yovz}?j8gbo()E4vWYgTF
z8nYSic|8l>b?b_Fub(X>VUDWKS0YCJwjDXfEAh(#a!HpXMT@?PR(y9S&Dv|qZ#vzO
zo~>Nes8}5iwEzD4bZ54ar1aWon6RCh8&R!pu&L>Ea=!i2+8YUbL*HE87P(jE@m{~l
z+)XR@7%_L46L6tgG-Kb-HWkwFdcM1Y)1lF{{Lv9*T;gCnxk)C?xp3EPujg*(=T@&b
z5U%p8TE<pcOGhubbY8bC3yzYiof5E`dZ|=dsPyw!jHB}hwb)ltXD!REpCmT6!P0u*
zG4a;Vjx<Zip-LOq;jgf4igVy9N_+RNS>4}#;VVC(*e0{2n7=ddWc5U9#^ja!0ZX^u
zLhF&<j1><tsk>9@X|J{hGIOhK3{1U>sz?JKv-hy(-s^4g<<@coJ(g9x>Vr|@wy!*D
z5*bIACfkUw&4TZhbo>ewEw5)6G1OaMb@X23njWaC=wn*vTJ5PBOSc=Gtn)D5G<IK_
zxRqPEpjOtDqMh+2h|4JKxO#(3YE4?4uI}uHCr|s=&R)KtQ<K=avT>fSG7Y787OdLx
z4IX*&)z~51+^XWG&FLbs1s1ZTS~9LJ)8Hh@r0`_*a<=1I?>47(%b;4U88%<qdB#eT
z;G`QtdK+I%?xFE_s*$VfRjQmz(LgrVyK0>IcP(wO9KZ5zI;W!1eq{V?MriV&$Hwov
zZ%@t<w%=1Rb(f9DO;=|Qk4wMGsfmv_dlqlMq1IoQ_tI-+)ALJ%bRz?|lvqkJMZL%_
z0luT{``Vli$DST2y_dx8+<hx0#9E_fwo5Ph+LS<s?&3sJNN<SGhM6Q$ZTns3<E0NY
zyrpN;r<(Ceq=NX(@3J||ho)XzWrRq@7)rm@_}*g~Pn0?{SGgGM;eZ|H@m{?}>YtLa
z3RjyJTwfthD>WFFI1r_>rcMa<3-V2_iE$=(t&<109R|q*B(=WvH2KZ1A`^3MX_~EG
z2?Zag-ha~ZQzoUGrFbP5iA*t1VX?=rwK*@DFv>1zkvnp|$Gpear*mD7xS1THsIuG)
zUK!fnXxc8d7aN--xC~+s>BQz(#(Q-Z8Rd=zQT}x&f5gf5VZ+5mbRU)`S}W5Py2o93
zyt}M1ziZ#CPudR~dNVajq((XQU1!V2`?oTm)lTupd)^-T5+q#J)KuLep(*qG3AsPL
zAXoe4Vxx4Ey1M$ovHjTYWtWaZDPL-A_2#S((O|QLGF+;9-DlH2vz%$OkkBNj6298g
zNz$1fXwTb*InDCkTzBr!H|SxI?El$mVa;h`5w9(qR!y`1JmLVOB8iFP%u#FUqbt&{
z8vnk7LUUDa|GI+|^^QCD&cEf6%jU-%Mqj0;3V{P`_m@J-UE(L67d1xPaK4nhozk3A
zkjr^q>*nvyLVQ6=aFP_IRA$CLrda!{G|fixcemEYD>SO4w53=wU%b0~vZ@{XMqy|i
z?st+0L(mz5`*ilK8yixXis8m+k84hgIKAqm*P^p(l39Fi>^2rLropmwIo3`6f*!Jz
zYLrfuadtw;{b!4Zw&j&rjTW)<RR^#dRg+qV;W?Y>gOT(nE1N~$$DgrKYamO$l6v92
z65I7vPr2+cr?FAl{S>3R#GfgQDN;p{+RLX4J-97vh#k*jFJ2m+wu#@EP?o>Ho!yj}
zDr7x{r(>C@S|yZBncKEs;vRviEVF05Sr})txxQ>M5N~q%E#}a0uFt0JB&)RioHF<A
zpya8!)yKRW@9mSklI8-G*fR3ru2kiFX<a|HJn;mVn&M{5Q?#Ycy~ZYdwX`L<K`v=(
zCiBNvrK*hDIl1|xWb)b?&xH7fr-2OTnsCU9soS)YS-P=*yjRU&)I#tBWPyu<j&>iG
zQg7dtp^x;J*Bz<jKB-wvnWs}F1n05`*0nmNSbTdGG0++Qy8CBhK}K}+m@ogFT<iyZ
z*Vd(TzC0q8At{YPO74}~LM`Q4-5S1<DH_}#RKt#!^-Yc1a7Lt3==({D(^*8<rk!1k
z-j5HSZ&BvuE{RnttDKaG9!?=8h8~;xIVDFd`DE=XHBH27==Rj)>GZgG3wGN$y~E?{
zeNC`>`q!_JH&^-`OY+R>u9j8h-B#YaOoDq{Vz;sS+rd3eX)QV5EA#9|bU;L@<MxRB
zR#JagyWZ&_C&52HJ(Yr))Ze>Y+R1d>gBKE-mXEZ&eXbqvDub4Bxs(6&s>hM&UJ~af
ze&+0yy7ZOe-S31pW~nsuVfoHo5E^6)I`KK~sDH{n9ml2fZ0Vks`$PVkHVzTmu0$7g
zUu|WrHY=xBVlR|#e~M9Kz04ie-KaDyzcJ-{O!U!;l$li0jI4{BZhK*UuZK*Jgj7-{
zDM6cGJ*IVuBZyPK<Y2K>z*Ls?<|C#3?JwFr+E|Yf#;YvVzqa%c8)Ky_G%^~0)fM+h
zhsAau3S<aRpBcQCK2xEGx97b6yVFDfld+gFd3CDaO*c}<K3AE-C@I`u>}67VN^eN?
z`qiFl{c+O`CkgQgwVF)k7w5`<T5j7Cg7EX0EOUx4b8=Z(#0Vl)%}chpW|!tBOFezk
zQmmHhhLb93FgJ1M>b>N`BF^Ma-8&XB=PLBs3>lT9WMw`h6_ayzI=VVhP%lyNlE2ea
zJ*O2J=b%ecu1(punf(iuE_6lycKsmy3M%DK<0P%<!zl$@5&F$)N|w*e(sduyal4fA
zstf9zYvZs|_gkOa-f}efdO6mJTDvN>OD>x*PbcomZJ>TOAf<&xQ=jB0L~dllrY;Zc
zPZOA|t#F^4Qpj8!S1(Z`jbCU}(_g=SM8e%Mf@qZ6mQFI{kR7b`r6;h82HmwLydjaj
zS86(IREGyo|1R2`y%aR7*Xb-DpX)Tg$f57L5Fl^WE^yuA9QWlyCBf^@f;WRx$a4lQ
zB)*Q3D~i+nnTO%3-RjOGYk`0<L!~U&m5sUSvpodnt`s$TPoKzR|G~dCk{kT|;x+v)
zE<q&yY^wG7yryg{qfklKc+PXBw}pS)&}A-qno!C7T4nlC<=LM+RR=;$jFNU|tQ<4^
zmWbsyw9DD#E6;hUw4YVrwzRC2(d=(EKihy%m$&*+l8z>mJ_gwBRI@?1>eXR-=_!0t
ze|dg)hBc4+5f);}6)&E`8b!;Wn=?j5mlRJII-e}&<}A|w*-|JOqLwuM<l(F?c`2%;
zoMp~U$V1-bG;NVKxqgINSQX2A6=s3V$GzdHq-g6T){r^rKNh5yNsL~|Af=e8S`6kn
zZEkLOGPSN5rw3f&xkNE40k{2P#;+8{7gCo^_KYa3`<bSM7uB1#jTF)vb-bG`pB60{
z_&AN|`BKIR;jAB)baG67uWYQ$EJ!UVHWV}Ah<(wygg4z7KHxz2I$rPs=f39Uoc2SV
zwXQ*Qc+S_#DJ?|_#+GDphRZ~~dkFD}<As{k#fk45{bdh~kBO)+>O@)P_7@7J3gWDd
zqojO=<PtWMC!gk#%QMK5+~mfj=ERG8Y#vM9pBnWdRSaT}>^6*d>a#B?TUK4KT(!?t
zqSez^Pl}0N|D~?pDiD-W<tDc9yJnkQlcRIfkCD6IqXmz#AYZk}@6ZTx$E!Tfkj}{F
zJk~Ct;>LPc?;M8Pv&eYbexsT<`(c~J`VFgwOu4hsD~1tX_)YEWeVd-ug-jWjlWs?N
zDJhb#_()dC8<$-670?sWh(BM&q(9$2uH6TddRm_Rz6lU2i7$4LO}>FVV>$H)9bd`5
z1RQfKC*!7BPH>=sah8nBpjU|zk}W7E)Sqy6M5yp*?7kdEeL^dxV{92I6*Kufo9?33
z3m1!aij|xuR-@0^hYe|SG%5^OG1EFr<f)^X?|m~qrN4|W4$PiN^GXpnNE9-hLegpd
zGB|9OE+4quarmBWe9nU;t%TqdDXa4Ah8ymT)Mu++kt+2GH5~+(y!PYR-np9Xty1ey
z&STA6T3kgx2OK??<FFs}eQAjYa!(X_H&yI@jenfJPtHDxlAcp=zQPe}f2_)<=z@=A
zP`kh<SD&N>fxDS!bDOhUe;OK%j&EA^rb?}>SNJPAEJ;m;db~Xz;dKpWZuT)6iEO?H
z*lug}>2o(H;gSQ~IrpsSt#B_EFg557{54$>4dJAorqR=^m|3(jO3=7@*!YjLFFI{M
z>7=Q}a$_g-hK*uPuu`|v*wb6}4r9OXPajF*4RP70AzRbfZ8H@jMEWc=S7v6N5ek<q
z7sAUqskMAsJyXZBA#*Bad&O$gKlNz(_u@`3{H2)El}p^-*ia`~5h7D=J+9${?dXS!
z?1Hta%5;_)Haznj=MC=_N4Jd`X{`CC9%GxWwU~~);Zn}oC7aw8^<nYn8-)f(=r~n%
ziWGB43*~&qymo&ZaT%OCo64Ax(QfT?T;o#5ia+b!rLYFF_RZFDt)!EQ&1rRJ@@La$
zE>Ukc^v+uH3j8!!^8QAi6`222Y;@#0tSs5+l}z5CXOo+gj?>w{{l~_oe_8UbuAvov
zwdc#+_}M&l|B`Ry8G~h5R5d_elp;gfQ7V$yH@8p9OhvLs3h(-o6E%N{JMvTGt;5)(
z!hK4y?>{a4a(Bor$YjjPAtqXDn{ZfRImOo_U5#TiW(PB7E6m%=mU4LYUjVwKvs_Ql
zXjrkInNA*gFbQ>}{XJ6+`CWc7nLyL=qQSB~$=W7D|HjfHci3OaW7Ammc#oxgSe+J9
z=7&)8rCFV>VxMk;-rD$0tLE=#d{(!r=_p+Fne?As>`G~gnssu1^e|)iEFBUBOiie+
z*rw8fp?2zrX2MBhoDmA0%MaqbR<28OYtgS@<Jo^wmEoEM5cAy>bQq16X#UMns%)$l
zn<$^bZw_kQ(vkl)QYYXFt`|52EU~wv@!ZJX7#X+xid_B7t5<b&_LmUyExu&|^>9IK
zACO9z;J_=m@~!o0`?_H6V%siJ7mw|&&Ek6_N=79PvCvmnI#rZor+0TKpcK@3(ft%t
zW09uLM=nD@U(h5+{!S?IB0H0|=`@$8I_TeF(N8TXPEhIHyO$TlDc%t0_KPD%+QngV
zlTNG1wyT)q?453?j@bzb?@?j}@Xwx{JoKRZh7(lnMO`M-kF?V?VuvZ)X9*PqP6721
zE;-Lq5##c97{#5-uTJE#Xq-Fro*gCD>h<3;YAycyQP7cT&<EY<H<?sSku$NkwXe#9
zR7@K58MOC6nE-nuJ-zVX#lXDrZP3xvKX#zZ5NxMVYg?3C{Yhm@C@-qmglj4v-j{hb
zJKw;>?X&t^Tf3|#G^qVv{x7|n*|Jvm!|C2LX*Bp<!c(>DYRa;?kQ=4v3W2fdF0^i&
zfv&OO?IpinBgd*S%2%`T&?2ixiwP!lqHJG17KX_-*;kTl?7cRYW2FFOj2#SH!hOk#
zOUz2ESslHY(4@m`2@X}uouz#){rCKWxFj2k9SAn+F%m_<sO)Y(1D`?G_;e^TWP~hu
zJPsM7zpETy_>)20fSd%s|F1EKe0sZOR|%zkXHRY(O?buk{J-#<Z$C`1oC*3LaGJE#
zbSivY|2lSRJ3Ku6+T+~}S$BR!UywvOQ4Aw2*Z;@bcKkj3S;D-YnbbW{<zZj@WRJxC
z(R!Q_%B}JlC?3Zc`)n>3o7FM)71>5sz}j%&I&~AL9xK(coAIEPut83t1EIVu9<VM=
zzabohlnMN~%!{G(=y`YRBQTM24@f|6)pJTZ%+!WC&2+b{FHc(*ST-F;=Y5m|#%*+w
zHz6P}(7qk7y#n-3H{gcLbEZW$5wPq<9gGMjdAAjdV`veCnbZHmZrYe;8a&V%ooEyQ
z6&gUVWk}Mi@!l-8*H^hneQZA)8{09jtpFZa8<<lQNr!YVawq)y5o4KYkXt<2mKN!j
zb?f6(2PdId4#2_TzZd)Z`i{xfRKNa;9`P)(j^`ko6U&<$GfDfVY7ezj*Z|470u02p
z>CS8d@np?L>&o2VT<RRbsZBklV^yZy^oTxOdA(_t&&A`uQW%tjy;lNkght=HTaRHC
z{?H$4YkG73!3}Xa{UMai4dRuT+gadGm~QR9#WIEHzi?lk>iHgZ)_(ry2MOnyYm#1$
zKudN)U?ju<MIRc8f8r<bOb=JS+-Wr8K$_`k>&!KEjFGU<Kdv5q4JLuoh6Bcc^JGp(
z^y3VRY_5NMCGyw-W%qG1%xixxAn*S|if05-?te!2Kj6m`3<dZe3LHZ3))R%>p%kIs
z|A}i4m4M6d-Z{g?fKyb-yl8as?ZQ8yF{k+-Q1uH3sgL4jc24r&ha!e23AZcM7##vm
zOm**Bq?Z2-g$>Zuoe38nC&Zwk%opM-hd~LKFTvclKpqzmDqQ-oUB;Fr1MZm~S$}8q
z&3|QcHaG|qOpnC@PZZ0*apu$3LS#+1VA`A8=#pX4lxcz6V*#9N5#*L~`)I)S>(W5d
zGQ0z=gwJ%Wc1gBVo+9eN6p|w70qN<%kg8a|Q@gfuS+mpk({-g});OT5515~&Krkx%
ztSDP6?rjha0v=n_T!d_zalPP9XN8L++;bT0RXsg(qhh<fvyJCI5^{_yihsO5{gGYt
zh84&J61P9mPPe99ZvFu`_7KLEvK9_+SX^q;tVs8Sl93N|^B{HRj~K}cajobtlAfz~
zjmn+UVWdA^ow$A*rQ-YNgk6v6wwS@V%O{TPN;@?q;X|LSdz7-B`oN8!-UHrb`{}OS
zc<ys|-=h%@LVyTC>4-ic%4+p^Il?T+4yA)9Ph8pH+zPCIz7sJm+5_wS>$DVtmg^=t
z8^>tx4>7|;B<`bAwTyF1t?&Ylun)4lEj{~*cPM!Ukxux#M!>`)T5(!50_sg|Dq(72
zCAM9SC>x$zwGwUY++!i(v*pPp<%EaiZq!+hwM8T9N3{4iSoN$<1u&g)W}7U`CQv10
zJyr^19IJb>?WIOTFYTnABU%PG?)i44>(+_ZY}_{xt2JT~|GRLYGw62oKtm)|J$m{w
zPzQ)8sYjvOw#+Csk<eN<TROtXCF$_So>1NqEn#mXZri!{T?#tj`^FPg;R1hrHaQ{r
zN-c`tV=qjm{*^&`<}RO$><T;OgzwSSgTHgP=PHYbi8xl~1UM?=*|H{P34M#1V7Ai=
zXyH%h*ohJKYume??1anTa#k`$k|xFFtoM>Be6OAPme8yYuaf%Vf2b`mOvi^(psSx*
z$e;QsttQg?UnDPPQ3%7`J7M^<N%-@Hl7!pB!L;Sn&ybx+Z%{kIHzX-DruOVDZr7P7
zXkNJZm~ubzUuz7r=#{}se|||QkN<Q$?HSrb|4bBbF%<q{88*~%e3Uz5%ijNZFY;v1
zPMCMpt@A7F`bWg$ns05g;I5#djeCChJ3`BXew7NB(2y^}wu+<M8Ed}a>4eepPs7aK
zS{TYh>`@4s`E8pME|i5tgol^b*>q+_ja@~9yPp3olNMBuVxW_HtG&o5WA9ABcJ*YJ
z3{3GDRl4S(0qi{6Z|y;vE!x9!yu1Fz5h{n%a47Bu+Z0mdK#)2R`X2+FKi26b!yl*K
zr)tF7fH+hbA#4B%fzYox-WdDXK?-i%d@5`5mNZOGJ>+?7!+}b-#T%|Ki|zY*wUxQY
z9z`g{{+B8@u9St&PjWEje*N(D!wh5D#)tcn4)!UeBkTIvX+lLwOf`xDg@uIjMU0BO
zIV|7YI&0Zj2te$!eQuxd)ht?k5qj?*W!+VFrEER?AEI422-A^kD(@g2bS*t$7mU#)
zQ*BHJsfKlEeD?$O>^QMx(7A1i@!+|e0J#c>hgS>a%CkcwRE`*Y)m!=lLA1Zrys%(9
z-rv|RCuFC)BDckZaepF(Nw@mcTVzXwa_m$RBERJ-P3nJO|KSm}1F>!NYuHFJTAr~}
z?djhaS=o>C%x)>tB$|g%Ak5++vgd?mCAO7bjSQ^DaP%n)=LFOc+_Ks7KS|$x(tVc}
zi|Yl_pI!|L)Pw1me+GMJk}{J)r)T4v^EsUH8S{eI&^kGBhYAUG5#W1q>2)=YxgMTM
zuPIKJhzd9<UzyI2GLs?H`)S)j(7FY~x#xRyHxtiH4ok|U2MfPOAtSfcE`l8d-?7u*
z`IL8E$53K=;o`mqNe)<bnfg@O4^;Nh(b3H$JZ2Q~x<X+KTyC*-+i~00WPf^nSTb)f
zF3a*|13XCv9MuW8`#x``e&(wpyPh-OL;#`IBdK!~90$-4#ORIg)PYT(2w#p`#?VOg
z);~G;@OUi@JeeCJ0>^wrh63Zhv*6DVqwRl}q_LWBiF0^So_U0Rfy!v@dRo@y6AW=D
zX(cfO|HP#i1C}+BKMU_gzL#yJg$^{f5sdgwKJiw&u^le_;pp#mjVu?+i=e3@7~0cb
zOvstXXoUYIhu1&I@}c~*#)c|Ai8}bAlu=Ls{&W%AI|Ng&aC=iXhdmTwcK=6t$2?QG
zy(DOQ%8QBi7#g-53JH6Vu3MC$7p^mC(J&r-2k-{U*N-fvAkH37YXmtXnupspBzOsO
z8Xx95y_Rn3mAv00b$u(uKD148-rgh`0c^4R`Tk*#&5oe2clj!1JCtHO(6Ea=*Y1;h
zn!JC_q7oqy3R?+UF>E?AB7*Lq7lJ7=KZZez0a^;7|G-Xq`EfhH=Jr}@0mR8}3lS+1
z4QZk_80PED31JGgTazH6mK?hDG<+Z}-4m_np!wcO+Jk-@C3JPS892pI;dM75T<Dt`
zP72nO{le&1XHfk50H~sXYz0<ZuSiM%%PwgqutebLRF1M+*ITfLA`7*{p92Xu8vy^>
z&ka;b+^XHh1zK<`(X#Q4wOL(o(Y=u`Wl?rzM};XeAgEWSGIM($QPa<Ck_JoA83Vb<
zzV6_ID-tF==*3~2=LYPlcJF<cuA5d)dH<jAI<m(b1k>U>KR9h$5`Bq11^hU6gA=y;
zt9r#7%JS<k$Y^e4W(Z^#$p?in^6KDhcV5!+bXOBwl<-N+^4#EHxyzh`*ZR^|XI-za
zYPj{q_-(r`V;AEnTlq<6h(-I=`Jt4KHTV|KC>^b0D}@sP%!h5>oImugOKyHxJ!QKJ
z)MLN{#{DDZRkuF}-1jjU5{$GshFP9y`KaG7J)KoBlN_dTzw4p<`eYg*SQ-lG%oi?4
zGzLq>b)xbiqnF$SGe@#O2UdyAmlJuBgw=#0hE!;7-e(K1SH8YS>txl^%h(5v=Xbmg
z18KLU8NQ(~J12Z?I=3e71(Gu5gUYM)TG2kB1R3-~a+~F`?a0uNl7EwPy#q3FVF?ui
zfIyV*!i(KVNBg^5mxM)w<zGz{PaPgj%GxDB9SIoj3xU*OVA@m1>thGHaS4!il1HeI
zv3@VSX%E>)^nJEbNn^Rw<gn;Zk6!DvGKtCR?rE(KP^*bV1Tj~>9eUvjgjYQWv9H91
z4Tvd2Q)s*?ZM2uov!Acomk{pwkoyMvCxSDqtv;oLki0VF$6J2J_gmyiP4aYx!a6|8
zq1rHLf4@{!Uu?^(<C&h-b<HryHR(0wLU1a>7mG2C&y4+8kBW@9-~x7wi<Q0=^2}(I
z_1<tLt)i@Iz0N&<&(H{@BUzQDM8yu4y+eDtMywP-rpW8LCRKKW)n1|x@7$)%1-3x{
z_q^v)i}LJnmAkQVy}?+A<@eXO=nsfG^<{jiRKmk8N0R+^3ya-MwEZyEs}Wr-d2d+3
zfa(HkvYws>g<3(I*qEkMmB4b_vh-A1=3aNvdP9x}BCHkjwe%yi6`%*tJmqqzZ`e@o
zczN=~i9!&Ndf<8;=Bh|>MpP-*p6icM5~3r~G~e7m_HON-do?Jlu;eD}z87JZjTEbp
z&)v2cxKbKB<soRNTQx|-tP$;2?7ivUS7;r^bpvLA-}$&<wqa4fmA>dVL!A5k2O46O
zW!$~b2Nk51KDvOzrHP<daYaXfQ_L!q@SsW7xkd3n8Ch)nKC#zFQ+NUjZAJYYD!Nyb
zX9LMz)VT{2yuN6;%Ad=lQT|M5oW9L?p-gh?PQ!L%c4>xt4bzir?V17t2PgL{t%(X#
zOhLVPH@LM@;;^cINGXCOhMff|tO(2y0!g{qMOQi3AcMT(y0u{Cs`b$?x_n%lQzMI+
z5~*i;oAKm8_|(;(08??a^=dd&`0Dp{!sgwe=KtZ-PI)FaLg%6Xl^{&|uS%siJnj>3
zOXrU+jyIJ+>sMY3V5L{WuKVZ3SE;LLfMeY9_a{^@#?Ez1&sC0W8C`M#-_e63@^l(8
zl8(W`N8&WfcZdHnoQl@XgY;qnCWL<wCpV;(LU@R&>;L*9)|*bhdwI9!{T$j=w+G7}
zSdxD}X3DZ|OXct&Jfx!Og?}&Qyg~2jF1<Hm2i<QDca%vWT2T0EAnv(6=ReHeE)!D3
zBPl|zLY6rTz8C!1BhECn(*mqOaCg5NvdKj-$PYjB5fRl7ZN<J8i)BJ^Q;9f*N1tH6
zWtgasB2V-n`}^^FTaCy2R97%}0))4&S^a#ne@TjvM&|{my4htHSR<Q~ut7xlhT-yA
z-{e%S7`hA@Mx5&EO#(MYH`<(;dj9qmJ_8K_{@5#88XT>VpkUr^2F_K(FJTSEJ>e0E
zev%RcDarO^)k1?XYTaqAs|J&dofyo9Iei-mIeP6c=xBUL<(u2_E3a=JQ4Y>(Ee1U1
z)D@|Jx^&+WCG%;K@CG$pt4f&nukTT2&=6wT=HfhN&-dk91Xj3;03JB~gb%UK2cE=*
zp&QNO7BY~<ow}?#LWCZq<r80@HnW5Nr87~HzOTSilW&e6O4R2gPU@+k0bhflC3v}i
z1a#Z@<r@`p@j2ywj(N3v%_0c%;lDC_1?DKXw6jy>c^!6@y;>3Wec;JLO(V;bZT!=7
zAH#}-Jlz-9f4^}VaO-imMlkG0cXQ|+W;v8nnc?>QC`Aj~Xz#bzX8jrZnOQLWdrZ^C
zyAAg17iZV%HpASCttSr^D;RDsff#>;OC4taoPX>eO0I~)v(7Up%$${D^V53qY|4%+
zJZl$TSL6KWJKtYxyBoEUel^8A0s#Ap4xCy2xg143#b0=nMY)`S*G{srM~@L;3s5=1
z-hv{%Ys27{5ekEwh7VB5^-=5k{D6Y*M{kKEuYs?Tn0_vpz&d&*1g;xVMW}q34AYd$
zgAoIG;%m~br$Fv&1z$PP`QATcI{k2o&`f&%02EeKQl9-?_F%mHXR`aA2>Td%N+YUe
z004c~X|YNi;BGwgaMA<0uTbT_++P&40~H681>OFFiludI!B3fZXDRg*`=+Avqlfv+
z<=Rm}Qlgwaydlo=GM{aNlQ!YeBT8F{JG5xyewqUv0n2B!G)~r|7X=L90CYf(6|>ha
zjD7R6H_E&!A}PgI_`VYZ;8<-38seVmAU3E0u;fSa&fjr+Z>{qn=6eAnYKZ`>KonG!
zsM;a?`fumbacRZur9m*gr`C5N91$%`i4A`M59dr@^HOA0r=AkwW8L2+S$3+e^bHwU
zPN5`<DRA-?0`1mkJU0nuHx&DHLq3lC8o3MMb1_e%<VE;0&u}I{Z)2~hsJJqj7T=o`
zB4-00vLbLIb)&3Z<znLqWjTUtKlT*(+k63G`cS<`^N{B6$~#uCrOIy1e{>#?btxBS
z(2S7~L05`70?~|Uhjq{~4tg#IdHw!;U{=|HDYy#9jwDQ>4I)y82d^8X=$p_=XLS~*
z%V?HE8p$@U=!Cp?$Yb<{@(h@w@=!-e&_xs_zxc4ukL)W3L!JTCEoj0V_$>ZnD8oy#
zXWZ`m8)Wr7+jr#5f(cv|c*-mzjh&;=wq3OJCnf2r5p*i(wJWIS><YUtJWqk>1$n&3
zQAmKdlIz^vgzOt-t-z4~@|JR$o$=^{sVJ`<2eSP+YOU<J(G79>WkB-_RXcEwzXB8m
zs0W<;>@A9WN*qm($3V!x%*ygf%-}{=BVBLv$K2L9{5RWqG2|a?rzlH;Q11#5;MwzT
z>!dkH;G|0Ayw=T^vMt0g=_(4x+f0Q3B98r1eZ^mYJ{$?%^Frhov$^FV_9RV23lOk=
zC$K@KWG9c$0c?WyeaWg}bn(hVf?ROTz+&D4e%5P`Zxq;uS|Ov$cXK&bzW92)d$CF0
z5k*WM>I{2K05Um~i6s?vI#Gdpkg@<ZSLlgTtDT3(0>xb4UU;rotUl&;Mle%<b-;^c
z?D6v<%?x0Z0#W1gZhZseh?Xui&&BYcw`@Ep4Q|u^@QYQmuzDOIE|+Bwu%HF1L(Hzb
zIUsN(?Z&iol2VXPuSCT7*W~9kc+`{!J=B;|jNI0$8ET2OgZ3Wn2Q1f92;r6dajvz#
z{7`&?itPJ3snc_7wt7i7t^gc0Tj#;ucyXjI33`Y=wD+8C)2jlklTv6e)D))5!JvgY
zzhbeS*Bw4|sY?<@AAN@<WDZL^%CkoYXfub?QR8q|^Fiw{0&Fw`cCnq|hSK04(IDql
zvHJ?cl?rObsIHu?VLx&0=15(LOGs0!beHcD1}nWn;y&er+Cz)xmzAi(JLM-0H937-
zV+fFUkDKOP6tcc{?V1(n9r+)b<Yg>>X;C7ci4b9<DNvQlrq43)$aEVgskY4?*LY*a
zE#-7KL>OV<mFuU*Hx#R{yuSZ5b=BX7>%@)HX8R=D3)WvXq+mH%^INqmC&@x4c|d(o
zL*};vdDf-IXzgTL%gbYz`wKsK%$C)OmQNgZF%^0sAk2CCTc;H#5<Z@%o;2)W+B&n<
z%o9n+;=o_sR4c(I&&XG@L?%>l_yoBb@W%Y^!pWml8Q|_dXoPNv%QED<zB*|->(OT(
z0*mY$L$~>*FI^aG)+xvxF7mwIt8eLb&ffWcS9dI&){lYfRZAC00$s#f`jbx6@AUX;
z!x|2%1>J~H-adyiVfjOOG$T}c_z+dnm%$T@CI&b3e~%#dFSAqX<#5v^df!$)lm^m3
z#{jx6o@8AeYU_9`{(WWIbi0?$R_k?xIj&VqHt3w}QuC#IL#jeCuukibK$D7?t6rrF
zPXo2V`RlRB?#~BI-?VhKh$o1BN}F>38|*mxE($|E59JVVe8cR-f)o1YifgS87QK;)
zI|C-l)>Df><9DY-noM=_TLh?u^PaoQH#x%gBNydORt~!UPECkGR$sOtDRh58HpJS7
zqLx4C=#r}d>#ht3p6q?Hhs6#Wu_#YB(?@Za_8|J=6CYL|$^0qam{S0urFBH+P`o+B
zw7_GbSs<p2XICT#J<ZT~pYxjYD0Z6EanUwHXdiz%6&AVw5*=jDHd5#ugfVn4ccAkS
zts<nVb4f!ZO8m+n|9$iuB~mMzx6_b8{Sru`$f#UaMrDQ3%cw3F0?ql_@e#$NkUcC=
zkMVXDcKWJ;VSj}PK;{M~7T}LAR&5Ekgg;6!LJ|UASNpwuxgWT-hd@FV81`LV&U31F
z#YIcI<dFSS=qKHUGG5YvTvqt`ppHnpR!d>I9aN`K)*<Q(hJQgMhE#m>8@h({_ct=&
z@Y5~Fo;$52LaU2r;fxJ53p?c*it`|$BN;|x!Sp?pc*@i{#g8CM+W(bzU7`*d|H6Y#
zI-B-jys^4Oc6}l#bLq9orH-1etm|f|85&lp-t;C+WCvdRqfHm%s9_bFV3Mh?AChpE
z_>!m?m<6X4?N$L=flzb{x-8@482v(GdRm-eO5GACID%a|-jkOmb%`XUa#$9$WjEeb
z*G=d;O|&Wzdy4`qG)oM!uDxh22E8%8)huy7TojlO{L|(fA4(uW{xA>hJo*>rans84
z(<{!NGMjwWfusa@#(S@)dz=NHz3H08UI1BnTYKuIpYWUOg0!1mdd}NE8<my>QtUI=
zuc0*#`otyx77jq9l^U_6)b0liSb+8u&L5}`nD3AgFTw+18E}`K5l)1qha%Kc@}D3`
z(B7z{u2lrO5J+6uE^kz%Ks~cLPH0o?03u(0{KhE1bMplnizE&Nx1d)72?$}+vT)Gv
ziaBc;oEe@*=Z3QHK*UtkVZR=s=4125TlF0YK9{H#l41l19QBe9JOh`wRJ@SIK~PbP
zftIuddo#+WSbv5xy!LF;qrMnTYPP3#lA52Y?)1Nt+Nf1=G^L(J-_@}^nD`1o@5HFP
zJ1ypfp_Y(GTm~S&2>QzPsZ1iIm6HsYp%pFx990WN1M?JCa9JYUz`uLG%-06cPq)^E
z({DH!Mbts@8&gVaJ(fhFX0YY8++jd?7N&C(I1A(7wAucxT$fLX(<1o9PZt?4$R40R
z%1rTOe}WCGc{oi*g}pxS<;dr|@i7kNU!nVnUAo7R21|iY8aO#UQza@r;G$?%fwig^
z&h`M#3c-lMinD%r71$iY#z*r-M*SB|2y>`y@#n!f9j)c@+v_}s`Qp>&POtAuRtAs;
zy9BJk>|5*wd{GYWQ|O^fGq4$=2|^PyzyI5#ah8kBp=TcLVHSZh1+iV6n0S~Bv=$L<
zAA!9#1Xyneh|77vZTvcrk7w5wkWjWOgH(xu^Scx@uQt5S-2Vl2>v|7(E~<O14*3y5
zJa;<K0E&d6L^Q&eT*}UZv->&@sJ0b)iH5}=l7rm`Og)FZo%UsRmlF{ju1e>i6FuM*
zH{e%*zbW$Laj)lb8{H@U>$TZT>*bT!BrRC?FG9cGeChh*c;_jK>mQe7`J(>=mO|Ne
zbiYB^$vwyiP!c}i8JX!|us^@L;@pWvT{rTcp;-V3kgIplEeRI+K(p~N?YoAIW<1|5
zz~+x)spF=Um?zp8N{klV0|f)P2cTSKITN!w(;>dyf?N2JvJ?9!??8vOql5<C{q$^K
z3dvUXs;a7Le=*$l0yI%p_#+F(w@>>u%r@<!Q$enSP|j=*(tz2Q=XroHOX*j?Xz=&f
z_|(|<m$qcL*#Jc{xru6aiBU-H&p*l6U2TkfArX@63O6Z2F9XT@4_x+%Q1cISMuG)$
z@kmS(pb<VTI0v2Pm^Q9;J?kHi4D~rwm~-Z$O;7`_8aL@dE2tEYj7Ec0Z4Rs+u@hs>
zIXFM`(SEWu1>N>w^fwpM$BwGiaiCTw{)7Vl$l(#os7u~E^)=Fg?3m~(rL2)_Pd2*d
zT;^Vth4@%fj7Hb*#sm22=`GrpGwTD{&;uZp{rb|R^PheVfFrXHCKBRiPV))zc$|`?
z)Vn$OnJ#tio+qNCr-2*xcyem!3Fe%F1Sb8N0ys@#%EzsN?XG~%{M|P{oguiLp@E0F
z63~9`k{@FcXr|xLP5~;*J79Wy0Czre;KZ?xeOgcZMCX}%_snhtcZv>+YZXAVAHipi
zP|?K}*u0#ecj7ka1I}w>krj#i(1Zv165V-Y54ew_gtOj}gZ^w47r)QUZ#o{T(8xc$
zC_^(rsYjzdK=J5~&opov3jW{f+xOekxDDlFGJNu5>r1k=EVvsWq>TBgMb(+896)da
zhYJy${0k;6B-9x+u((4q6gVGP*lU6k$$DC9GMS~XMHVnQnuAoh8_M8NBJZbH86Bo&
z8rlPl8|<~h?}ueH^nOVeQDJ~aEF-Xme{>J9`<P{DMMg_LEo(mE*!4V=5*bp$LD!WC
zs~1WChfzuJn+=7(C;mOnhR>RqR|qU)JSQb<Xx;wDS4Q7$?Qz2;z@Zn>Lob19$1I1k
z?5%w9a0VnJ7@<xuEX9d}{a`<MH4i`G;B$ebh}43xFq)oqwKh}b_8~u~@;gQwL_zAW
zrXiI3X_=wx<ddS+!mu8DwVUQX^V?lWXyA_A#DV^wN_e}#3-1ujD-a1r{XlPapvV{a
zGk5J=#DZ24P}w;?85*ZlPfWa5VLtQhCU92NFfvRdNEOI^YM6*n-32%3PWd+A3p1WV
zo<aB?!CTy$FECXYQPB0=j$6XiaiCL&$vJ}AjQ=;jI6INb*L3tL2Vd7dP($!>D!cHo
z=cczi&kVxy9+3FYTFD1an%G@>x;HOY+NJj&CKLorD9=9ceCHl79OqyCm|>ySEsvEM
z!73ln{UmO_M`EO~84L%{HDArSVGVk~19|uw#&@4_K4)u8E9_4D>FJHTm_fRMrdk2K
zRp8Ld24S!>O*6ifd_cs|9N1vzAyp~Z{Q~K?pr^o+C4pJvPs#}uUr((`?A%L?D4iEh
z$5p85Z{~zFxe~x8oZoWd38pCEeDdXTrBLSUPe3BkVOd8ao_EUr1sz^$UGP_cUPLuM
zEwPSS<bo$VU7z{pv2Ox~k{x#3gO5GM=BCWfKczrI6CtvnGH@B*yXU#blHdSlm?(RC
zdirl(j98hMvQOtdW<F@&D=FBzuCjKqc+au#ZG{f$p&YqxZ!0G~4%Q%i73Zwk7+)}G
z0oW+{8pdBoNVE0=X9yi$5%MaC*5*`tAw04-j#Zz3@T=6uGo!ZT<_AhHd2i2V^-DYg
zE}Oc;iZ%tJzFp&#<uB@b`9U;04(1n_Fo%q|2`Npqns+^JElSX<zR~5!DdYOCLbHwZ
z7XqLy?Lb0J)~>>u-3d$ZScphAgkJ%z&i)^o8ffwSf!S2dS9h_4Mk5SzCX;U#>=@fk
z*vsefy83_{WFKG)4)Z49^wz4kbdNt)YhYUq6#+U4VRnGMC)^OUry3p~{5%1`q2m&c
z61Vs8I}knqxg~Jk@vd?1nRL<_@QSKks(SMQ)(%n|g7{WOl(M6GPrVo=tCpg}(jGh@
zZoq@Q?l=tAxIkoUCH&D$l|&oq{%-W|&vrxDIy{Mf&X<5{#u(g1lLar4)GJ@_yr3OD
zhQ=$i1{%D7?l$EcAM0C^g<Ox&o`mCsQs>5YT?Fe3EhPmVJ{ZlAtTb)G&mbEixCG{e
zP!D&>9H+1-e-{~(wK+&-a4?J?uI*o6n2Sa}vI{~#nSMIHHFziFy5Laq{zv{U`v_5k
zO%FpH$BRJQSED@+hv|n=$!hT_-KOwAIwZ(HYVF!!iOwT(;Rxof+8H+U{h8CpJYvrz
zK>{0uEDLccMahb*e2>l{8tr_;J72Q}US?X9h(a*Sa5i?dr;*iMa`oKje?Q*k^S<9D
z*8w0QxD$skhb;Vkxm(%egwo&pO$j<K`J-b*(*L&aykOLV)+7Jr%a^5OxVv}2GRc5f
z#H^qc77|(x>l`p*ALvv-(qK_lwUrF)GS|(_&?bRq_#$R+SjK%NjxO7|^Lo&+-{cDO
z8x}Q>xz>KjcthqSN;Dy%ig1?y7h7bx^?4DHAs1NC+X>4!pTY*Rg#$x)>Tk67ekE#b
z11By(cgODGhfZ-Zar|#U5G>S?HtF#MRzEHy&<+O$EG*lYJU40jw=XhPGv0wja53#K
z{yV7es}5B?Zd;H!Tea;fBcf>_qBxxzo&AllDG!nT*x_Q?!q{2BUprM*Pa(0iv|F99
zC}!RIi4ci@_wHT)xR$b5kJ<V0fdsD5!@RaJbkde@xxF`M38gX57hoI;Z$9nygE?DU
z9Pev>dx;1}hkSq#eUK~cI<u}zK>1FHc`6eI(Fj+oB@1(hQhDD_*7UOlS%Gl^6}bWY
z+C!&6S8nZ^G9$o;P9Qy0`B9MR-l3QczAziem#d)VCd7<h))yN3aivt&YQ7|6WBuN-
z@4h<2_t1g|(i%*AqRy|p1FZ~6Q6Y2BD%4%oc+sFh6fTcavEJvRKm~#_u$Cn6+Zj-8
z5W+?YYbkk{fyePp?+V-XdkP&s2jD~X4|d^Y!meQw4Q~n5{T)SU#76iH)1B9|P!KRq
zQlVfC<x7SS^Fir&SMg^7{7`s=0ct1&87BqZdBMS*)AMoCYT=`wnQo(@-lBkOI3mRN
zV%^tnM0kG8c7A7FBljub^L{&vwO)T0{uy3-`oQ)&Q!RRkmc=goscAy#g8jlydO$*j
zX|*O_`o24(yS?Z-ToqB{Fo{MCg0q1DNP`9d-rl%~3*fuBbGY~}d?-*=@PSadU3jM`
zolrv8qK7Ph#w^SYVtol0@W;+L5q)JG-Dy!3SQ1F`0SX~?o97%+V+KbLq95x~!UZt@
z`7t!R!C%k~@B0AU!aFUmU3jrW3Fse(0KuR?|GgXjKRZ61aDVsg|J4!GQysT+DW3a!
zkfQ(2)k%;+Pjg-XLZ~HCv9t7^*@3uZP`1H-C?6y1AprmZ@<kaeCD<Qr@$a^JpH?7x
z;3x#UK(<GQX^nTkm(bC-kM{cq1O&X4(^L#WFPAp*Up4Puzt@gTaHfx)o&6R0Af4t{
zo<O>l4>j+2yJ>h<sL1a>df()A?Pi*xV&Wh8cqpGD<KKO9HYyfS6w#t%$w9-mCCM-I
zlO*tz20@zotACMtOy$<-nC|OAB?JM$4(%{p=WlXJJ7>l0Xr9v$UjMQbb0sAIqa_uG
zc!gxZ;uMB{#ppW~8iDjbjUf4IHQ#QltUUDFd@ncue-C!(KYK6#!vhI20@T#hvS3r_
zpCPy(52R|@CgKV0?yL@yVaK8G$=fh|ZSAC|#LuB?9NBZ16z3vF%(f0=Qmp>+@S>aF
ziGaTU60i1_U%$Sy+a08CBIDi)m$_o$+{#vmn$4Aj7&clj_Ht*UjpdtrPu1gO&QJU@
zC!kxmsncPDeoaLmK!5CklpO6OrS~G=;!xlZ28i2(>zyU25a__WF=bFg2Pb$-|8n~F
zC~Q<M0DUZcfVarZB7c)P1zx>=`ZwgA>G6Kn3a@hTnx8o5-5wlrO97|$up0Tx=Ft28
zA(j1DLj>9rk=j7HNr;IQUsuMEZ-Ef++<1Px8>sm#*hcOQrh<w<7UdtyU8at(S@|x_
zSAo_G=u-;{516=qAktuifyQ(|J&w)l-vXfI^l195>vjoB!IKQ^!C2W@Y;Pn@ENshi
znH#tuc`^xHy=I_8<e`!WXg6weA-(??E}O5Jg(CToP+fyLnNO{*A)uNp<<FclVs>p`
zg3ddumczFFrFrD{mw|lBhQ%!bm{i4ULcD$B6N_f)<435p&%Fzadln7MN)K8C|1$lb
zKqnf^$VAll4gy(e(`R&#eTjVHXq%=yc6$&I5^1}aW?e&Jh}`@Sac_1D&n+BA^b<f|
zsWkP1BfhR&3P)crz$=><sRni;sBrUs3_v0t0sOiS+xLfO+lZhm?rKlG$;!%FN|pRi
z8vK#19`b*gt{*#S@BtCQEe-4|wDTqf;FdN5_(pJInBxr+XqNW|c_AO&pz93sPQ}@V
zDFKBco5iW45&SmjL>+>OGACmaowBYJ^X|Md!E4q;#Wt9<l%`hyxVj6^YL%10=$==&
zsu_qRseN3MS_S&;E2I3zd_qi^ewT?yHkn4tvaNPNP~{hhyLT<G4!VvofbEn#18$ij
zbUQDE^Uu^H3J*I^ZfuOpyF4I7p13~Tc*+FK4^R&a5@Oi@2#&D0mo<9rYu=gYx`t14
za&>ig`7kL7;QOAvVt8T)Tw_G&UNPcO_Utb@JrNxM>fW^sb2@&yVX8CR7IiocZv2vT
zXs89khA2mGC~z=B<6+|oQ}@Y~*_okRe%!y<cTMljlA;EysbcdF`dxJmt_OwBU%j98
z{8wIn<-g91vLQk=))@hk@sesL@U#V&l@n?-+t+|uf5#-Z`W{+rqJt0g%Edn0-XynX
z)*yUyA=Ty)JZU!Kmh9@!$Cs*9WLiKSpImt?wd+1`eLYOx^HgB(wYHL`ju^VO=7^q_
zuC8GM7Kj(=yxdbN3L$D)_ocfXDSG;fY<ha<6+g~_v;kT0ddiSxtg0J4@8EFHq0qEA
zccIypSeJoGb{bU8OJJwIw9Pz`kn6M8q4Kwqq`KntLGR6JQyM~HmbSPcH@)22C=04c
zOiQWyz~2ei>+on|LeKAh?yA!Cwd*Y|@grvEm=koga!q7i?5T|FE_6vYhi2vsL52lO
z`z!tO;twE}8h5<DW_Z*SoFcp1g?1@MJY5QbCke>>tY%sFp2#xN!yhG{^jW)1+Ri9U
zZhX7-uyQIR^O1w0(&Ux$)Wl~}7tx-iPB9y+?KXfvC@kE|6dlJ3fol=(i^to^D-2z`
z4w<JRFF)T~M-MWO5>D+!4&m*|&!Bu{flX!BW>`C^H75F7kdLnZTRfx_cy2>Mll*qS
zK2K}mb9~v*Q?7n-2d5j)8szYh?WF^wuR!2MKor8+`e^cMJ+rd@<9RYDx%mu>?Kqsy
z;@-3Zz)gr@7SMkkur8<aDQ~}k@KoKN;E3(o->1>7Uhv=lU(cz5Q~mP^Iq&_@XP}^a
zia<`7<?lr0Yt)?p>O2`97cc|XhCCV`*Y}`2+QOqB1QrKD4!Sidalw7a_rMHFVP@u(
zH+auK<_)h<$M(hw&*|PC1d!N)KC6cpc1bv*s2LT;q44Y{bp!ch{0FqyA>8-t524g|
zDLF7OkO=ijWN|RBZ<i{O2MpKPneNk*68hF8Fzy;_sn(H~;xfzs{k`#1UAc~+9AJ|;
zln>r}QZWa4vq)2J4K2?j4z|f^7cvg}_su{ZjPu!YBi#W1${W-HPuK?0s2hxiGpi#Z
zCFlbhIK^$uQQHX^ySij^oGVv)ZC6nG6eLZP-qdSf<2<%D)=)bry7|R^u$lno-W%)E
zV6{PMT(A<xal4JaQ0+4wn4Wgy_1U=RLI!<TgixJNN_3tQ?>J7~0kWgiY#HpFBFxpt
zZ*N$V<XD>(e8SMq1GU7PdatF(g0U6|O$q7WpIFcb?^vw(7yAH>OMLU7QIJ$zn@{;0
zIwc}l>+(_kh^^WZ!4v|c^Pb<AA_y-7sd`7#C5usOT)-STpD5_rb>l(MGO<r`a`BTx
zQsX*UAMU_ru{_kR;r%<%^cw091EWOot&h9W9b(wI0fO%f`K{j}E5HVqb(z9r5y+dT
z&Aya2ddACn&48eCziA|WqKwUo^%<+!?Mp2nC|m42$Ojbxb=e%tCsXhD#m+3VNmXyH
z+oB@z<_Pb$0HHdOH0=ArEJU^wti<EJtD|9>Ut}W(0xbb~frJ^%J?+p0yBkpXk+K=I
zu9-*er|S-mm}n9!mRc{5tZc)s3Bsny&48+Gulw*6mmbt*ohokPLcpTb4@qug<^h6P
zg&2CzkInH}DTwc3@>#uvKBL0|IN(BM9=A_HZ(J#a^f^VeRbETjM^|L?y=~k^p$t6y
z=<X3^fQ?0Sp5xLE3S;mTkn}zW5Z9f7Wv{Sy1)@72HJ|Q0D8+jgeNGGNsX{qXfVvS|
zjS$vZAZ0p(i?Xbb)3+Qx{H;*BXL`CUr*xRE0(?fqIiGEh$rZM$NNRfa(tUVz&Ss01
z8pO`5!2FahmEE2s71mTC%jgqGiVWtk=%a`V)nGS^Jv?Pd5F7)EV8on7uW3?F7gG-M
zi5Ky&E9Rr7qoq4^vU)X<o)Y9T0bRYrY`mWH7y3DmVw)9{PPT%>oFvr>{>K9+6q1fb
z52UWVK_4Q8s;A(cn<STosmr1p6=USaCm}1B^&Ru@3SvJNHdBR<1@t6B(0Hr(QZ~}W
z666U7DFTQU{e|+I!IarbUzbW<YPRIkU4eNfgkQ<VE(&N(mi_Bq@F{Y(xWXKw8x^sm
znMr_Yi-B@$?ThA2ZTTac)B`Y;t4uJ2r_1m#c1D~{BS{tAH67#dYrHJ1#ZWY8)U`5n
zgtTn-SUJ&rPSN(87`FLj(AHi~v3k@%(3`PO&^HaXmV5~+7BQ*pjux{Xt_@D5z1kVv
zE%@Bs)KD^IFev4H+F+(#JQdnaQX_&>gEciVyVC@3N1wUM9nI(pIpLbBs{eFlrdiIi
zBqYBgNcu99b3ahgpF8_22e4aJ<k^aq0gt)Q*k8i#v*v}2)(zo;qI&rEkLe_m7uGrV
zw7MTtcUihjeMsz>X2gQt_lwgxOTEz&<FW>~C$z0}*05|r&0?}^kZp{~Y;o=0>bcJu
zLV2i7iAKH>zLIR2&j^+^8)GL4w4EpdB#+B)Pu8{rx}CHA*SK}~4cp4vRSjq=rSu|Z
zdu?;EzFcD28ix`pwZjym4JF}mUNK_UFDtTJgxJLJvk!!k^q&e~lp!oh>L=*Fo+qzI
zCw5hPc^oij#cj+NcNiY2N>yqoWEWm3V|`la(2z9c3Q&6IAz=1L=!(PmHJugt+V8O8
znxn;o_D{-%q|#@E`mQv+_2ZeMv!4*JBBk8T@=ATgUj8cLB7V?|5uud_Tli{1OKLaY
zTkYs;1J42}{iFzq7_9-HI)@_}!D;&#^}j5tJqWz`JfkbZ#c2AS7-izik?7V`?TCe=
z9u6(NaSEK1+7Z%0yC-FBr+T3P8>JH1s<z;nU_>Psv8j~lbNVYLqp0DZ*gpNG`0PsA
z{&f7M6WPC8se?L>XP2}V$_0s~d#%rPd*#&f3Yq30S?R$hD*J=%(p`O1r_x1wnbOfT
zQ3`2-j|P7!?vJ}0cq=IrY@GB;xmJ30wMlLw_UhMzPR$oAdZVTuSd+VJ_qjKki*=mj
rpw2O;7T8(XuK%lr`oDn)ZQE3A0&+)klAX0tB%!LPp^$U&=A-`u_Why_

literal 0
HcmV?d00001

diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc
index ef7c40b11d048..e34a4483ab6e9 100644
--- a/docs/reference/index-modules.asciidoc
+++ b/docs/reference/index-modules.asciidoc
@@ -36,7 +36,7 @@ specific index module:
 
 `index.number_of_shards`::
 
-    The number of primary shards that an index should have.  Defaults to 5.
+    The number of primary shards that an index should have.  Defaults to 1.
     This setting can only be set at index creation time.  It cannot be
     changed on a closed index. Note: the number of shards are limited to `1024` per
     index. This limitation is a safety limit to prevent accidental creation of indices
@@ -199,6 +199,7 @@ specific index module:
      This setting is only applicable when highlighting is requested on a text that was indexed without offsets or term vectors.
      Defaults to `1000000`.
 
+[[index-max-terms-count]]
  `index.max_terms_count`::
 
     The maximum number of terms that can be used in Terms Query.
diff --git a/docs/reference/index-modules/slowlog.asciidoc b/docs/reference/index-modules/slowlog.asciidoc
index 235256bdce7c0..a96c8fe995b75 100644
--- a/docs/reference/index-modules/slowlog.asciidoc
+++ b/docs/reference/index-modules/slowlog.asciidoc
@@ -26,7 +26,26 @@ index.search.slowlog.threshold.fetch.trace: 200ms
 index.search.slowlog.level: info
 --------------------------------------------------
 
-All of the above settings are _dynamic_ and are set per-index.
+All of the above settings are _dynamic_ and can be set for each index using the
+<<indices-update-settings, update indices settings>> API. For example: 
+
+[source,js]
+--------------------------------------------------
+PUT /twitter/_settings
+{
+    "index.search.slowlog.threshold.query.warn": "10s",
+    "index.search.slowlog.threshold.query.info": "5s",
+    "index.search.slowlog.threshold.query.debug": "2s",
+    "index.search.slowlog.threshold.query.trace": "500ms",
+    "index.search.slowlog.threshold.fetch.warn": "1s",
+    "index.search.slowlog.threshold.fetch.info": "800ms",
+    "index.search.slowlog.threshold.fetch.debug": "500ms",
+    "index.search.slowlog.threshold.fetch.trace": "200ms",
+    "index.search.slowlog.level": "info"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 By default, none are enabled (set to `-1`). Levels (`warn`, `info`,
 `debug`, `trace`) allow to control under which logging level the log
@@ -83,7 +102,23 @@ index.indexing.slowlog.level: info
 index.indexing.slowlog.source: 1000
 --------------------------------------------------
 
-All of the above settings are _dynamic_ and are set per-index.
+All of the above settings are _dynamic_ and can be set for each index using the
+<<indices-update-settings, update indices settings>> API. For example: 
+
+[source,js]
+--------------------------------------------------
+PUT /twitter/_settings
+{
+    "index.indexing.slowlog.threshold.index.warn": "10s",
+    "index.indexing.slowlog.threshold.index.info": "5s",
+    "index.indexing.slowlog.threshold.index.debug": "2s",
+    "index.indexing.slowlog.threshold.index.trace": "500ms",
+    "index.indexing.slowlog.level": "info",
+    "index.indexing.slowlog.source": "1000"
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 By default Elasticsearch will log the first 1000 characters of the _source in
 the slowlog. You can change that with `index.indexing.slowlog.source`. Setting
diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc
index 8e5fea810fb7b..f9d4b5a4e0528 100644
--- a/docs/reference/index.asciidoc
+++ b/docs/reference/index.asciidoc
@@ -10,6 +10,8 @@
 
 include::../Versions.asciidoc[]
 
+include::intro.asciidoc[]
+
 include::getting-started.asciidoc[]
 
 include::setup.asciidoc[]
@@ -39,6 +41,8 @@ include::cluster.asciidoc[]
 
 include::query-dsl.asciidoc[]
 
+include::scripting.asciidoc[]
+
 include::mapping.asciidoc[]
 
 include::analysis.asciidoc[]
@@ -59,7 +63,7 @@ include::rollup/index.asciidoc[]
 
 include::frozen-indices.asciidoc[]
 
-include::rest-api/index.asciidoc[]
+include::administering.asciidoc[]
 
 include::commands/index.asciidoc[]
 
@@ -70,6 +74,8 @@ include::testing.asciidoc[]
 
 include::glossary.asciidoc[]
 
+include::rest-api/index.asciidoc[]
+
 include::release-notes/highlights.asciidoc[]
 
 include::migration/index.asciidoc[]
diff --git a/docs/reference/indices/apis/reload-analyzers.asciidoc b/docs/reference/indices/apis/reload-analyzers.asciidoc
new file mode 100644
index 0000000000000..657f6556df4b2
--- /dev/null
+++ b/docs/reference/indices/apis/reload-analyzers.asciidoc
@@ -0,0 +1,105 @@
+[role="xpack"]
+[testenv="basic"]
+[[indices-reload-analyzers]]
+== Reload Search Analyzers
+
+experimental[]
+
+Reloads search analyzers and its resources.
+
+Synonym filters (both `synonym` and `synonym_graph`) can be declared as
+updateable if they are only used in <<search-analyzer,search analyzers>> 
+with the `updateable` flag:
+
+[source,js]
+--------------------------------------------------
+PUT /my_index
+{
+    "settings": {
+        "index" : {
+            "analysis" : {
+                "analyzer" : {
+                    "my_synonyms" : {
+                        "tokenizer" : "whitespace",
+                        "filter" : ["synonym"]
+                    }
+                },
+                "filter" : {
+                    "synonym" : {
+                        "type" : "synonym",
+                        "synonyms_path" : "analysis/synonym.txt",
+                        "updateable" : true <1>
+                    }
+                }
+            }
+        }
+    },
+    "mappings": {
+        "properties": {
+            "text": {
+                "type": "text",
+                "analyzer" : "standard",
+                "search_analyzer": "my_synonyms" <2>
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+<1> Mark the synonym filter as updateable.
+<2> Synonym analyzer is usable as a search_analyzer.
+
+NOTE: Trying to use the above analyzer as an index analyzer will result in an error.
+
+Using the <<indices-reload-analyzers,analyzer reload API>>, you can trigger reloading of the
+synonym definition. The contents of the configured synonyms file will be reloaded and the
+synonyms definition the filter uses will be updated. 
+
+The `_reload_search_analyzers` API can be run on one or more indices and will trigger 
+reloading of the synonyms from the configured file.
+
+NOTE: Reloading will happen on every node the index has shards, so its important
+to update the synonym file contents on every data node (even the ones that don't currently
+hold shard copies; shards might be relocated there in the future) before calling
+reload to ensure the new state of the file is reflected everywhere in the cluster.
+
+[source,js]
+--------------------------------------------------
+POST /my_index/_reload_search_analyzers
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+The reload request returns information about the nodes it was executed on and the
+analyzers that were reloaded:
+
+[source,js]
+--------------------------------------------------
+{
+    "_shards" : {
+        "total" : 2,
+        "successful" : 2,
+        "failed" : 0
+    },
+    "reload_details" : [
+        {
+            "index" : "my_index",
+            "reloaded_analyzers" : [
+                "my_synonyms" 
+            ],
+            "reloaded_node_ids" : [
+                "mfdqTXn_T7SGr2Ho2KT8uw"
+            ]
+        }
+    ]
+}
+--------------------------------------------------
+// TEST[continued]
+// TESTRESPONSE[s/"total" : 2/"total" : $body._shards.total/]
+// TESTRESPONSE[s/"successful" : 2/"successful" : $body._shards.successful/]
+// TESTRESPONSE[s/mfdqTXn_T7SGr2Ho2KT8uw/$body.reload_details.0.reloaded_node_ids.0/]
+
+NOTE: Reloading does not happen on each shard of an index, but once on each node
+the index has shards on. The total shard count can therefore differ from the number
+of index shards.
diff --git a/docs/reference/indices/create-index.asciidoc b/docs/reference/indices/create-index.asciidoc
index 54efcd9005ac5..3038265891458 100644
--- a/docs/reference/indices/create-index.asciidoc
+++ b/docs/reference/indices/create-index.asciidoc
@@ -12,7 +12,7 @@ PUT twitter
 --------------------------------------------------
 // CONSOLE
 
-This create an index named `twitter` with all default setting.
+This creates an index named `twitter` with all default setting.
 
 [NOTE]
 .Index name limitations
diff --git a/docs/reference/indices/open-close.asciidoc b/docs/reference/indices/open-close.asciidoc
index 6d0866d303b88..4ba434ecbbb6e 100644
--- a/docs/reference/indices/open-close.asciidoc
+++ b/docs/reference/indices/open-close.asciidoc
@@ -2,22 +2,68 @@
 == Open / Close Index API
 
 The open and close index APIs allow to close an index, and later on
-opening it. A closed index has almost no overhead on the cluster (except
-for maintaining its metadata), and is blocked for read/write operations.
-A closed index can be opened which will then go through the normal
-recovery process.
+opening it.
 
-The REST endpoint is `/{index}/_close` and `/{index}/_open`. For
-example:
+A closed index is blocked for read/write operations and does not allow
+all operations that opened indices allow. It is not possible to index
+documents or to search for documents in a closed index. This allows
+closed indices to not have to maintain internal data structures for
+indexing or searching documents, resulting in a smaller overhead on
+the cluster.
+
+When opening or closing an index, the master is responsible for
+restarting the index shards to reflect the new state of the index.
+The shards will then go through the normal recovery process. The
+data of opened/closed indices is automatically replicated by the
+cluster to ensure that enough shard copies are safely kept around
+at all times.
+
+The REST endpoint is `/{index}/_close` and `/{index}/_open`.
+
+The following example shows how to close an index:
 
 [source,js]
 --------------------------------------------------
 POST /my_index/_close
+--------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT my_index\n/]
+
+This will return the following response:
+
+[source,js]
+--------------------------------------------------
+{
+    "acknowledged" : true,
+    "shards_acknowledged" : true,
+    "indices" : {
+        "my_index" : {
+            "closed" : true
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+A closed index can be reopened like this:
 
+[source,js]
+--------------------------------------------------
 POST /my_index/_open
 --------------------------------------------------
 // CONSOLE
-// TEST[s/^/PUT my_index\n/]
+// TEST[s/^/PUT my_index\nPOST my_index\/_close\n/]
+
+which will yield the following response:
+
+[source,js]
+--------------------------------------------------
+{
+    "acknowledged" : true,
+    "shards_acknowledged" : true
+}
+--------------------------------------------------
+// TESTRESPONSE
 
 It is possible to open and close multiple indices. An error will be thrown
 if the request explicitly refers to a missing index. This behaviour can be
@@ -36,6 +82,6 @@ API by setting `cluster.indices.close.enable` to `false`. The default is `true`.
 [float]
 === Wait For Active Shards
 
-Because opening an index allocates its shards, the
+Because opening or closing an index allocates its shards, the
 <<create-index-wait-for-active-shards,`wait_for_active_shards`>> setting on
-index creation applies to the index opening action as well.
+index creation applies to the `_open` and `_close` index actions as well.
diff --git a/docs/reference/indices/recovery.asciidoc b/docs/reference/indices/recovery.asciidoc
index 0929b36e7742d..6e03ddd16b71b 100644
--- a/docs/reference/indices/recovery.asciidoc
+++ b/docs/reference/indices/recovery.asciidoc
@@ -249,7 +249,7 @@ Response:
 }
 --------------------------------------------------
 // TESTRESPONSE[s/"source" : \{[^}]*\}/"source" : $body.$_path/]
-// TESTRESPONSE[s/"details" : \[[^\]]*\]//]
+// TESTRESPONSE[s/"details" : \[[^\]]*\]/"details" : $body.$_path/]
 // TESTRESPONSE[s/: (\-)?[0-9]+/: $body.$_path/]
 // TESTRESPONSE[s/: "[^"]*"/: $body.$_path/]
 ////
diff --git a/docs/reference/indices/shrink-index.asciidoc b/docs/reference/indices/shrink-index.asciidoc
index 24a67208f7289..aa73d3d1350bb 100644
--- a/docs/reference/indices/shrink-index.asciidoc
+++ b/docs/reference/indices/shrink-index.asciidoc
@@ -17,7 +17,10 @@ Shrinking works as follows:
 
 * Then it hard-links segments from the source index into the target index. (If
   the file system doesn't support hard-linking, then all segments are copied
-  into the new index, which is a much more time consuming process.)
+  into the new index, which is a much more time consuming process. Also if using 
+  multiple data paths, shards on different data paths require a full copy of 
+  segment files if they are not on the same disk since hardlinks don’t work across
+  disks)
 
 * Finally, it recovers the target index as though it were a closed index which
   had just been re-opened.
diff --git a/docs/reference/ingest.asciidoc b/docs/reference/ingest.asciidoc
index 6fa2e8c796df6..a945a56c1942b 100644
--- a/docs/reference/ingest.asciidoc
+++ b/docs/reference/ingest.asciidoc
@@ -23,7 +23,30 @@ another processor that renames a field. The <<cluster-state,cluster state>> then
 the configured pipelines.
 
 To use a pipeline, simply specify the `pipeline` parameter on an index or bulk request. This
-way, the ingest node knows which pipeline to use. For example:
+way, the ingest node knows which pipeline to use. 
+
+For example:
+Create a pipeline
+
+[source,js]
+--------------------------------------------------
+PUT _ingest/pipeline/my_pipeline_id
+{
+  "description" : "describe pipeline",
+  "processors" : [
+    {
+      "set" : {
+        "field": "foo",
+        "value": "new"
+      }
+    }
+  ]
+}
+--------------------------------------------------
+// CONSOLE
+// TEST
+
+Index with defined pipeline
 
 [source,js]
 --------------------------------------------------
@@ -33,7 +56,27 @@ PUT my-index/_doc/my-id?pipeline=my_pipeline_id
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[catch:bad_request]
+// TEST[continued]
+
+Response：
+[source,js]
+--------------------------------------------------
+{
+  "_index" : "my-index",
+  "_type" : "_doc",
+  "_id" : "my-id",
+  "_version" : 1,
+  "result" : "created",
+  "_shards" : {
+    "total" : 2,
+    "successful" : 2,
+    "failed" : 0
+  },
+  "_seq_no" : 0,
+  "_primary_term" : 1
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"successful" : 2/"successful" : 1/]
 
 An index may also declare a <<dynamic-index-settings,default pipeline>> that will be used in the
 absence of the `pipeline` parameter.
diff --git a/docs/reference/ingest/processors/date-index-name.asciidoc b/docs/reference/ingest/processors/date-index-name.asciidoc
index fa749fd342056..783ecc9b2b1b0 100644
--- a/docs/reference/ingest/processors/date-index-name.asciidoc
+++ b/docs/reference/ingest/processors/date-index-name.asciidoc
@@ -140,6 +140,6 @@ understands this to mean `2016-04-01` as is explained in the <<date-math-index-n
 | `date_formats`         | no        | yyyy-MM-dd+++'T'+++HH:mm:ss.SSSXX  | An array of the expected date formats for parsing dates / timestamps in the document being preprocessed. Can be a java time pattern or one of the following formats: ISO8601, UNIX, UNIX_MS, or TAI64N.
 | `timezone`             | no        | UTC                          | The timezone to use when parsing the date and when date math index supports resolves expressions into concrete index names.
 | `locale`               | no        | ENGLISH                      | The locale to use when parsing the date from the document being preprocessed, relevant when parsing month names or week days.
-| `index_name_format`    | no        | yyyy-MM-dd                   | The format to be used when printing the parsed date into the index name. An valid java time pattern is expected here. Supports <<accessing-template-fields,template snippets>>.
+| `index_name_format`    | no        | yyyy-MM-dd                   | The format to be used when printing the parsed date into the index name. A valid java time pattern is expected here. Supports <<accessing-template-fields,template snippets>>.
 include::common-options.asciidoc[]
 |======
diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc
index 4e866624309a7..7ce7b430ef610 100644
--- a/docs/reference/ingest/processors/geoip.asciidoc
+++ b/docs/reference/ingest/processors/geoip.asciidoc
@@ -10,7 +10,7 @@ under the CCA-ShareAlike 4.0 license. For more details see, http://dev.maxmind.c
 
 The `geoip` processor can run with other GeoIP2 databases from Maxmind. The files must be copied into the `ingest-geoip` config directory,
 and the `database_file` option should be used to specify the filename of the custom database. Custom database files must be stored
-uncompressed. The `ingest-geoip` config directory is located at `$ES_HOME/config/ingest-geoip`.
+uncompressed. The `ingest-geoip` config directory is located at `$ES_CONFIG/ingest-geoip`.
 
 [[using-ingest-geoip]]
 ==== Using the `geoip` Processor in a Pipeline
diff --git a/docs/reference/ingest/processors/set.asciidoc b/docs/reference/ingest/processors/set.asciidoc
index c964182ef6573..499e49c29b482 100644
--- a/docs/reference/ingest/processors/set.asciidoc
+++ b/docs/reference/ingest/processors/set.asciidoc
@@ -17,7 +17,7 @@ include::common-options.asciidoc[]
 [source,js]
 --------------------------------------------------
 {
-  "description" : "sets the value of count to 1"
+  "description" : "sets the value of count to 1",
   "set": {
     "field": "count",
     "value": 1
diff --git a/docs/reference/intro.asciidoc b/docs/reference/intro.asciidoc
new file mode 100644
index 0000000000000..c16f1bd78a1d5
--- /dev/null
+++ b/docs/reference/intro.asciidoc
@@ -0,0 +1,270 @@
+[[elasticsearch-intro]]
+= Elasticsearch introduction
+[partintro]
+--
+_**You know, for search (and analysis)**_
+
+{es} is the distributed search and analytics engine at the heart of
+the {stack}. {ls} and {beats} facilitate collecting, aggregating, and
+enriching your data and storing it in {es}. {kib} enables you to
+interactively explore, visualize, and share insights into your data and manage
+and monitor the stack. {es} is where the indexing, search, and analysis
+magic happen.
+
+{es} provides real-time search and analytics for all types of data. Whether you
+have structured or unstructured text, numerical data, or geospatial data,
+{es} can efficiently store and index it in a way that supports fast searches.
+You can go far beyond simple data retrieval and aggregate information to discover
+trends and patterns in your data. And as your data and query volume grows, the
+distributed nature of {es} enables your deployment to grow seamlessly right
+along with it.
+
+While not _every_ problem is a search problem, {es} offers speed and flexibility
+to handle data in a wide variety of use cases:
+
+* Add a search box to an app or website
+* Store and analyze logs, metrics, and security event data
+* Use machine learning to automatically model the behavior of your data in real
+  time
+* Automate business workflows using {es} as a storage engine
+* Manage, integrate, and analyze spatial information using {es} as a geographic
+  information system (GIS)
+* Store and process genetic data using {es} as a bioinformatics research tool
+
+We’re continually amazed by the novel ways people use search. But whether
+your use case is similar to one of these, or you're using {es} to tackle a new
+problem, the way you work with your data, documents, and indices in {es} is
+the same.
+--
+
+[[documents-indices]]
+== Data in: documents and indices
+
+{es} is a distributed document store. Instead of storing information as rows of
+columnar data, {es} stores complex data structures that have been serialized
+as JSON documents. When you have multiple {es} nodes in a cluster, stored
+documents are distributed across the cluster and can be accessed immediately
+from any node.
+
+When a document is stored, it is indexed and fully searchable in near
+real-time--within 1 second. {es} uses a data structure called an
+inverted index that supports very fast full-text searches. An inverted index
+lists every unique word that appears in any document and identifies all of the
+documents each word occurs in.
+
+An index can be thought of as an optimized collection of documents and each
+document is a collection of fields, which are the key-value pairs that contain
+your data. By default, {es} indexes all data in every field and each indexed
+field has a dedicated, optimized data structure. For example, text fields are
+stored in inverted indices, and numeric and geo fields are stored in BKD trees.
+The ability to use the per-field data structures to assemble and return search
+results is what makes {es} so fast.
+
+{es} also has the ability to be schema-less, which means that documents can be
+indexed without explicitly specifying how to handle each of the different fields
+that might occur in a document. When dynamic mapping is enabled, {es}
+automatically detects and adds new fields to the index. This default
+behavior makes it easy to index and explore your data--just start
+indexing documents and {es} will detect and map booleans, floating point and
+integer values, dates, and strings to the appropriate {es} datatypes.
+
+Ultimately, however, you know more about your data and how you want to use it
+than {es} can. You can define rules to control dynamic mapping and explicitly
+define mappings to take full control of how fields are stored and indexed.
+
+Defining your own mappings enables you to:
+
+* Distinguish between full-text string fields and exact value string fields
+* Perform language-specific text analysis
+* Optimize fields for partial matching
+* Use custom date formats
+* Use data types such as `geo_point` and `geo_shape` that cannot be automatically
+detected
+
+It’s often useful to index the same field in different ways for different
+purposes. For example, you might want to index a string field as both a text
+field for full-text search and as a keyword field for sorting or aggregating
+your data. Or, you might choose to use more than one language analyzer to
+process the contents of a string field that contains user input.
+
+The analysis chain that is applied to a full-text field during indexing is also
+used at search time. When you query a full-text field, the query text undergoes
+the same analysis before the terms are looked up in the index.
+
+[[search-analyze]]
+== Information out: search and analyze
+
+While you can use {es} as a document store and retrieve documents and their
+metadata, the real power comes from being able to easily access the full suite
+of search capabilities built on the Apache Lucene search engine library.
+
+{es} provides a simple, coherent REST API for managing your cluster and indexing
+and searching your data.  For testing purposes, you can easily submit requests
+directly from the command line or through the Developer Console in {kib}. From
+your applications, you can use the
+https://www.elastic.co/guide/en/elasticsearch/client/index.html[{es} client]
+for your language of choice: Java, JavaScript, Go, .NET, PHP, Perl, Python
+or Ruby.
+
+[float]
+[[search-data]]
+=== Searching your data
+
+The {es} REST APIs support structured queries, full text queries, and complex
+queries that combine the two. Structured queries are
+similar to the types of queries you can construct in SQL. For example, you
+could search the `gender` and `age` fields in your `employee` index and sort the
+matches by the `hire_date` field. Full-text queries find all documents that
+match the query string and return them sorted by _relevance_&mdash;how good a
+match they are for your search terms.
+
+In addition to searching for individual terms, you can perform phrase searches,
+similarity searches, and prefix searches, and get autocomplete suggestions.
+
+Have geospatial or other numerical data that you want to search? {es} indexes
+non-textual data in optimized data structures that support
+high-performance geo and numerical queries.
+
+You can access all of these search capabilities using {es}'s
+comprehensive JSON-style query language (<<query-dsl, Query DSL>>). You can also
+construct <<sql-overview, SQL-style queries>> to search and aggregate data
+natively inside {es}, and JDBC and ODBC drivers enable a broad range of
+third-party applications to interact with {es} via SQL.
+
+[float]
+[[analyze-data]]
+=== Analyzing your data
+
+{es} aggregations enable you to build complex summaries of your data and gain
+insight into key metrics, patterns, and trends. Instead of just finding the
+proverbial “needle in a haystack”, aggregations enable you to answer questions
+like:
+
+* How many needles are in the haystack?
+* What is the average length of the needles?
+* What is the median length of the needles, broken down by manufacturer?
+* How many needles were added to the haystack in each of the last six months?
+
+You can also use aggregations to answer more subtle questions, such as:
+
+* What are your most popular needle manufacturers?
+* Are there any unusual or anomalous clumps of needles?
+
+Because aggregations leverage the same data-structures used for search, they are
+also very fast. This enables you to analyze and visualize your data in real time.
+Your reports and dashboards update as your data changes so you can take action
+based on the latest information.
+
+What’s more, aggregations operate alongside search requests. You can search
+documents, filter results, and perform analytics at the same time, on the same
+data, in a single request. And because aggregations are calculated in the
+context of a particular search, you’re not just displaying a count of all
+size 70 needles, you’re displaying a count of the size 70 needles
+that match your users' search criteria--for example, all size 70 _non-stick
+embroidery_ needles.
+
+[float]
+[[more-features]]
+==== But wait, there’s more
+
+Want to automate the analysis of your time-series data? You can use
+{stack-ov}/ml-overview.html[machine learning] features to create accurate
+baselines of normal behavior in your data and identify anomalous patterns. With
+machine learning, you can detect:
+
+* Anomalies related to temporal deviations in values, counts, or frequencies
+* Statistical rarity
+* Unusual behaviors for a member of a population
+
+And the best part? You can do this without having to specify algorithms, models,
+or other data science-related configurations.
+
+[[scalability]]
+== Scalability and resilience: clusters, nodes, and shards
+++++
+<titleabbrev>Scalability and resilience</titleabbrev>
+++++
+
+{es} is built to be always available and to scale with your needs. It does this
+by being distributed by nature. You can add servers (nodes) to a cluster to
+increase capacity and {es} automatically distributes your data and query load
+across all of the available nodes. No need to overhaul your application, {es}
+knows how to balance multi-node clusters to provide scale and high availability.
+The more nodes, the merrier.
+
+How does this work? Under the covers, an {es} index is really just a logical
+grouping of one or more physical shards, where each shard is actually a
+self-contained index. By distributing the documents in an index across multiple
+shards, and distributing those shards across multiple nodes, {es} can ensure
+redundancy, which both protects against hardware failures and increases
+query capacity as nodes are added to a cluster. As the cluster grows (or shrinks),
+{es} automatically migrates shards to rebalance the cluster.
+
+There are two types of shards: primaries and replicas. Each document in an index
+belongs to one primary shard. A replica shard is a copy of a primary shard.
+Replicas provide redundant copies of your data to protect against hardware
+failure and increase capacity to serve read requests
+like searching or retrieving a document.
+
+The number of primary shards in an index is fixed at the time that an index is
+created, but the number of replica shards can be changed at any time, without
+interrupting indexing or query operations.
+
+[float]
+[[it-depends]]
+=== It depends...
+
+There are a number of performance considerations and trade offs with respect
+to shard size and the number of primary shards configured for an index. The more
+shards, the more overhead there is simply in maintaining those indices. The
+larger the shard size, the longer it takes to move shards around when {es}
+needs to rebalance a cluster.
+
+Querying lots of small shards makes the processing per shard faster, but more
+queries means more overhead, so querying a smaller
+number of larger shards might be faster. In short...it depends.
+
+As a starting point:
+
+* Aim to keep the average shard size between a few GB and a few tens of GB. For
+  use cases with time-based data, it is common to see shards in the 20GB to 40GB
+  range.
+
+* Avoid the gazillion shards problem. The number of shards a node can hold is
+  proportional to the available heap space. As a general rule, the number of
+  shards per GB of heap space should be less than 20.
+
+The best way to determine the optimal configuration for your use case is
+through https://www.elastic.co/elasticon/conf/2016/sf/quantitative-cluster-sizing[
+testing with your own data and queries].
+
+[float]
+[[disaster-ccr]]
+=== In case of disaster
+
+For performance reasons, the nodes within a cluster need to be on the same
+network. Balancing shards in a cluster across nodes in different data centers
+simply takes too long. But high-availability architectures demand that you avoid
+putting all of your eggs in one basket. In the event of a major outage in one
+location, servers in another location need to be able to take over. Seamlessly.
+The answer? {ccr-cap} (CCR).
+
+CCR provides a way to automatically synchronize indices from your primary cluster
+to a secondary remote cluster that can serve as a hot backup. If the primary
+cluster fails, the secondary cluster can take over. You can also use CCR to
+create secondary clusters to serve read requests in geo-proximity to your users.
+
+{ccr-cap} is active-passive. The index on the primary cluster is
+the active leader index and handles all write requests. Indices replicated to
+secondary clusters are read-only followers.
+
+[float]
+[[admin]]
+=== Care and feeding
+
+As with any enterprise system, you need tools to secure, manage, and
+monitor your {es} clusters. Security, monitoring, and administrative features
+that are integrated into {es} enable you to use {kibana-ref}/introduction.html[{kib}]
+as a control center for managing a cluster. Features like <<rollup-overview,
+data rollups>> and <<index-lifecycle-management, index lifecycle management>>
+help you intelligently manage your data over time.
diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc
index 2e09a0a8ca24a..d0a3c6e06cd66 100644
--- a/docs/reference/mapping.asciidoc
+++ b/docs/reference/mapping.asciidoc
@@ -87,15 +87,11 @@ causing a mapping explosion:
     `2`, etc. The default is `20`.
 
 `index.mapping.nested_fields.limit`::
-    The maximum number of `nested` fields in an index, defaults to `50`.
-    Indexing 1 document with 100 nested fields actually indexes 101 documents
-    as each nested document is indexed as a separate hidden document.
+    The maximum number of distinct `nested` mappings in an index, defaults to `50`.
 
 `index.mapping.nested_objects.limit`::
-    The maximum number of `nested` json objects within a single document across
-    all nested fields, defaults to 10000. Indexing one document with an array of
-    100 objects within a nested field, will actually create 101 documents, as
-    each nested object will be indexed as a separate hidden document.
+    The maximum number of `nested` JSON objects within a single document across
+    all nested types, defaults to 10000.
 
 `index.mapping.field_name_length.limit`::
     Setting for the maximum length of a field name. The default value is
diff --git a/docs/reference/mapping/params/coerce.asciidoc b/docs/reference/mapping/params/coerce.asciidoc
index 55f31262351fd..be5b2a648c600 100644
--- a/docs/reference/mapping/params/coerce.asciidoc
+++ b/docs/reference/mapping/params/coerce.asciidoc
@@ -47,8 +47,7 @@ PUT my_index/_doc/2
 <1> The `number_one` field will contain the integer `10`.
 <2> This document will be rejected because coercion is disabled.
 
-TIP: The `coerce` setting is allowed to have different settings for fields of
-the same name in the same index.  Its value can be updated on existing fields
+TIP: The `coerce` setting value can be updated on existing fields
 using the <<indices-put-mapping,PUT mapping API>>.
 
 [[coerce-setting]]
diff --git a/docs/reference/mapping/params/enabled.asciidoc b/docs/reference/mapping/params/enabled.asciidoc
index 7193c6aa9f6e3..edbfb1f77d71e 100644
--- a/docs/reference/mapping/params/enabled.asciidoc
+++ b/docs/reference/mapping/params/enabled.asciidoc
@@ -89,8 +89,8 @@ GET my_index/_mapping <3>
 <2> The document can be retrieved.
 <3> Checking the mapping reveals that no fields have been added.
 
-TIP: The `enabled` setting can be updated on existing fields
-using the <<indices-put-mapping,PUT mapping API>>.
+The `enabled` setting for existing fields and the top-level mapping
+definition cannot be updated.
 
 Note that because Elasticsearch completely skips parsing the field
 contents, it is possible to add non-object data to a disabled field:
diff --git a/docs/reference/mapping/params/ignore-above.asciidoc b/docs/reference/mapping/params/ignore-above.asciidoc
index daf5c92bcf34d..33c0eaf339f24 100644
--- a/docs/reference/mapping/params/ignore-above.asciidoc
+++ b/docs/reference/mapping/params/ignore-above.asciidoc
@@ -30,7 +30,7 @@ PUT my_index/_doc/2 <3>
   "message": "Syntax error with some long stacktrace"
 }
 
-GET _search <4>
+GET my_index/_search <4>
 {
   "aggs": {
     "messages": {
diff --git a/docs/reference/mapping/params/ignore-malformed.asciidoc b/docs/reference/mapping/params/ignore-malformed.asciidoc
index 8c91bb48ee7cd..d84a7290eb7c9 100644
--- a/docs/reference/mapping/params/ignore-malformed.asciidoc
+++ b/docs/reference/mapping/params/ignore-malformed.asciidoc
@@ -46,8 +46,7 @@ PUT my_index/_doc/2
 <1> This document will have the `text` field indexed, but not the `number_one` field.
 <2> This document will be rejected because `number_two` does not allow malformed values.
 
-TIP: The `ignore_malformed` setting is allowed to have different settings for
-fields of the same name in the same index.  Its value can be updated on
+TIP: The `ignore_malformed` setting value can be updated on
 existing fields using the <<indices-put-mapping,PUT mapping API>>.
 
 
diff --git a/docs/reference/mapping/params/multi-fields.asciidoc b/docs/reference/mapping/params/multi-fields.asciidoc
index ee1bc02c7fd8d..448f7fd2e81ec 100644
--- a/docs/reference/mapping/params/multi-fields.asciidoc
+++ b/docs/reference/mapping/params/multi-fields.asciidoc
@@ -60,8 +60,7 @@ GET my_index/_search
 
 NOTE: Multi-fields do not change the original `_source` field.
 
-TIP: The `fields` setting is allowed to have different settings for fields of
-the same name in the same index.  New multi-fields can be added to existing
+TIP: New multi-fields can be added to existing
 fields using the <<indices-put-mapping,PUT mapping API>>.
 
 ==== Multi-fields with multiple analyzers
diff --git a/docs/reference/mapping/params/normalizer.asciidoc b/docs/reference/mapping/params/normalizer.asciidoc
index da0298abda228..0f8c09552f4c6 100644
--- a/docs/reference/mapping/params/normalizer.asciidoc
+++ b/docs/reference/mapping/params/normalizer.asciidoc
@@ -7,7 +7,7 @@ produces a single token.
 
 The `normalizer` is applied prior to indexing the keyword, as well as at
 search-time when the `keyword` field is searched via a query parser such as
-the <<query-dsl-match-query,`match`>> query or via a term level query 
+the <<query-dsl-match-query,`match`>> query or via a term-level query 
 such as the <<query-dsl-term-query,`term`>> query.
 
 [source,js]
diff --git a/docs/reference/mapping/params/norms.asciidoc b/docs/reference/mapping/params/norms.asciidoc
index 8a7be4baef8c5..6a250d296a2dc 100644
--- a/docs/reference/mapping/params/norms.asciidoc
+++ b/docs/reference/mapping/params/norms.asciidoc
@@ -11,11 +11,10 @@ don't need scoring on a specific field, you should disable norms on that
 field. In  particular, this is the case for fields that are used solely for
 filtering or aggregations.
 
-TIP: The `norms` setting must have the same setting for fields of the
-same name in the same index.  Norms can be disabled on existing fields using
+TIP: Norms can be disabled on existing fields using
 the <<indices-put-mapping,PUT mapping API>>.
 
-Norms can be disabled (but not reenabled) after the fact, using the
+Norms can be disabled (but not reenabled after the fact), using the
 <<indices-put-mapping,PUT mapping API>> like so:
 
 [source,js]
diff --git a/docs/reference/mapping/removal_of_types.asciidoc b/docs/reference/mapping/removal_of_types.asciidoc
index 47952d61c4954..4aa178d443f55 100644
--- a/docs/reference/mapping/removal_of_types.asciidoc
+++ b/docs/reference/mapping/removal_of_types.asciidoc
@@ -258,7 +258,7 @@ Elasticsearch 6.x::
 
 * The `_default_` mapping type is deprecated.
 
-* In 6.7, the index creation, index template, and mapping APIs support a query
+* In 6.8, the index creation, index template, and mapping APIs support a query
   string parameter (`include_type_name`) which indicates whether requests and
   responses should include a type name. It defaults to `true`, and should be set
   to an explicit value to prepare to upgrade to 7.0. Not setting `include_type_name`
@@ -442,12 +442,12 @@ documents to it using typeless `index` calls, and load documents with typeless
 
 Index creation, index template, and mapping APIs support a new `include_type_name`
 URL parameter that specifies whether mapping definitions in requests and responses
-should contain the type name. The parameter defaults to `true` in version 6.7 to
+should contain the type name. The parameter defaults to `true` in version 6.8 to
 match the pre-7.0 behavior of using type names in mappings. It defaults to `false`
 in version 7.0 and will be removed in version 8.0.
 
-It should be set explicitly in 6.7 to prepare to upgrade to 7.0. To avoid deprecation
-warnings in 6.7, the parameter can be set to either `true` or `false`. In 7.0, setting
+It should be set explicitly in 6.8 to prepare to upgrade to 7.0. To avoid deprecation
+warnings in 6.8, the parameter can be set to either `true` or `false`. In 7.0, setting
 `include_type_name` at all will result in a deprecation warning.
 
 See some examples of interactions with Elasticsearch with this option set to `false`:
@@ -717,12 +717,12 @@ indices.
 [float]
 ==== Mixed-version clusters
 
-In a cluster composed of both 6.7 and 7.0 nodes, the parameter
+In a cluster composed of both 6.8 and 7.0 nodes, the parameter
 `include_type_name` should be specified in indices APIs like index
 creation. This is because the parameter has a different default between
-6.7 and 7.0, so the same mapping definition will not be valid for both
+6.8 and 7.0, so the same mapping definition will not be valid for both
 node versions.
 
 Typeless document APIs such as `bulk` and `update` are only available as of
-7.0, and will not work with 6.7 nodes. This also holds true for the typeless
+7.0, and will not work with 6.8 nodes. This also holds true for the typeless
 versions of queries that perform document lookups, such as `terms`.
diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc
index c0db156dc3a1c..bdfcf1128a061 100644
--- a/docs/reference/mapping/types.asciidoc
+++ b/docs/reference/mapping/types.asciidoc
@@ -42,8 +42,6 @@ string::         <<text,`text`>> and <<keyword,`keyword`>>
 
 <<parent-join>>::   Defines parent/child relation for documents within the same index
 
-<<alias>>::         Defines an alias to an existing field.
-
 <<rank-feature>>::  Record numeric feature to boost hits at query time.
 
 <<rank-features>>:: Record numeric features to boost hits at query time.
@@ -54,6 +52,11 @@ string::         <<text,`text`>> and <<keyword,`keyword`>>
 
 <<search-as-you-type>>:: A text-like field optimized for queries to implement as-you-type completion
 
+<<alias>>::         Defines an alias to an existing field.
+
+<<flattened>>:: Allows an entire JSON object to be indexed as a single field.
+
+
 [float]
 === Multi-fields
 
@@ -82,6 +85,8 @@ include::types/date.asciidoc[]
 
 include::types/date_nanos.asciidoc[]
 
+include::types/flattened.asciidoc[]
+
 include::types/geo-point.asciidoc[]
 
 include::types/geo-shape.asciidoc[]
diff --git a/docs/reference/mapping/types/alias.asciidoc b/docs/reference/mapping/types/alias.asciidoc
index a7bba54df56e4..c70d96a7e572d 100644
--- a/docs/reference/mapping/types/alias.asciidoc
+++ b/docs/reference/mapping/types/alias.asciidoc
@@ -16,7 +16,7 @@ PUT trips
       },
       "route_length_miles": {
         "type": "alias",
-        "path": "distance" // <1>
+        "path": "distance" <1>
       },
       "transit_mode": {
         "type": "keyword"
diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc
index 335c8f16ba9f1..7ea04b42330c2 100644
--- a/docs/reference/mapping/types/dense-vector.asciidoc
+++ b/docs/reference/mapping/types/dense-vector.asciidoc
@@ -1,3 +1,5 @@
+[role="xpack"]
+[testenv="basic"]
 [[dense-vector]]
 === Dense vector datatype
 
@@ -5,9 +7,7 @@ experimental[]
 
 A `dense_vector` field stores dense vectors of float values.
 The maximum number of dimensions that can be in a vector should
-not exceed 1024. The number of dimensions can be
-different across documents. A `dense_vector` field is
-a single-valued field.
+not exceed 1024. A `dense_vector` field is a single-valued field.
 
 These vectors can be used for <<vector-functions,document scoring>>.
 For example, a document score can represent a distance between
@@ -22,7 +22,8 @@ PUT my_index
   "mappings": {
     "properties": {
       "my_vector": {
-        "type": "dense_vector"
+        "type": "dense_vector",
+        "dims": 3  <1>
       },
       "my_text" : {
         "type" : "keyword"
@@ -40,13 +41,14 @@ PUT my_index/_doc/1
 PUT my_index/_doc/2
 {
   "my_text" : "text2",
-  "my_vector" : [-0.5, 10, 10, 4]
+  "my_vector" : [-0.5, 10, 10]
 }
 
 --------------------------------------------------
 // CONSOLE
 
+<1> dims—the number of dimensions in the vector, required parameter.
+
 Internally, each document's dense vector is encoded as a binary
 doc value. Its size in bytes is equal to
-`4 * NUMBER_OF_DIMENSIONS`, where `NUMBER_OF_DIMENSIONS` -
-number of the vector's dimensions.
\ No newline at end of file
+`4 * dims`, where `dims`—the number of the vector's dimensions.
\ No newline at end of file
diff --git a/docs/reference/mapping/types/flattened.asciidoc b/docs/reference/mapping/types/flattened.asciidoc
new file mode 100644
index 0000000000000..80fd72c3dcc1f
--- /dev/null
+++ b/docs/reference/mapping/types/flattened.asciidoc
@@ -0,0 +1,188 @@
+[role="xpack"]
+[testenv="basic"]
+
+[[flattened]]
+=== Flattened datatype
+
+By default, each subfield in an object is mapped and indexed separately. If
+the names or types of the subfields are not known in advance, then they are
+<<dynamic-mapping, mapped dynamically>>.
+
+The `flattened` type provides an alternative approach, where the entire
+object is mapped as a single field. Given an object, the `flattened`
+mapping will parse out its leaf values and index them into one field as
+keywords. The object's contents can then be searched through simple queries
+and aggregations.
+
+This data type can be useful for indexing objects with a large or unknown
+number of unique keys. Only one field mapping is created for the whole JSON
+object, which can help prevent a <<mapping-limit-settings, mappings explosion>>
+from having too many distinct field mappings.
+
+On the other hand, flattened object fields present a trade-off in terms of
+search functionality. Only basic queries are allowed, with no support for
+numeric range queries or highlighting. Further information on the limitations
+can be found in the <<supported-operations, Supported operations>> section.
+
+NOTE: The `flattened` mapping type should **not** be used for indexing all
+document content, as it treats all values as keywords and does not provide full
+search functionality. The default approach, where each subfield has its own
+entry in the mappings, works well in the majority of cases.
+
+An flattened object field can be created as follows:
+[source,js]
+--------------------------------
+PUT bug_reports
+{
+  "mappings": {
+    "properties": {
+      "title": {
+        "type": "text"
+      },
+      "labels": {
+        "type": "flattened"
+      }
+    }
+  }
+}
+
+POST bug_reports/_doc/1
+{
+  "title": "Results are not sorted correctly.",
+  "labels": {
+    "priority": "urgent",
+    "release": ["v1.2.5", "v1.3.0"],
+    "timestamp": {
+      "created": 1541458026,
+      "closed": 1541457010
+    }
+  }
+}
+--------------------------------
+// CONSOLE
+// TESTSETUP
+
+During indexing, tokens are created for each leaf value in the JSON object. The
+values are indexed as string keywords, without analysis or special handling for
+numbers or dates.
+
+Querying the top-level `flattened` field searches all leaf values in the
+object:
+
+[source,js]
+--------------------------------
+POST bug_reports/_search
+{
+  "query": {
+    "term": {"labels": "urgent"}
+  }
+}
+--------------------------------
+// CONSOLE
+
+To query on a specific key in the flattened object, object dot notation is used:
+[source,js]
+--------------------------------
+POST bug_reports/_search
+{
+  "query": {
+    "term": {"labels.release": "v1.3.0"}
+  }
+}
+--------------------------------
+// CONSOLE
+
+[[supported-operations]]
+==== Supported operations
+
+Because of the similarities in the way values are indexed, `flattened`
+fields share much of the same mapping and search functionality as
+<<keyword, `keyword`>> fields.
+
+Currently, flattened object fields can be used with the following query types:
+
+- `term`, `terms`, and `terms_set`
+- `prefix`
+- `range`
+- `match` and `multi_match`
+- `query_string` and `simple_query_string`
+- `exists`
+
+When querying, it is not possible to refer to field keys using wildcards, as in
+`{ "term": {"labels.time*": 1541457010}}`. Note that all queries, including
+`range`, treat the values as string keywords. Highlighting is not supported on
+`flattened` fields.
+
+It is possible to sort on an flattened object field, as well as perform simple
+keyword-style aggregations such as `terms`. As with queries, there is no
+special support for numerics -- all values in the JSON object are treated as
+keywords. When sorting, this implies that values are compared
+lexicographically.
+
+Flattened object fields currently cannot be stored. It is not possible to
+specify the <<mapping-store, `store`>> parameter in the mapping.
+
+[[flattened-params]]
+==== Parameters for flattened object fields
+
+The following mapping parameters are accepted:
+
+[horizontal]
+
+<<mapping-boost,`boost`>>::
+
+    Mapping field-level query time boosting. Accepts a floating point number,
+    defaults to `1.0`.
+
+`depth_limit`::
+
+    The maximum allowed depth of the flattened object field, in terms of nested
+    inner objects. If a flattened object field exceeds this limit, then an
+    error will be thrown. Defaults to `20`.
+
+<<doc-values,`doc_values`>>::
+
+    Should the field be stored on disk in a column-stride fashion, so that it
+    can later be used for sorting, aggregations, or scripting? Accepts `true`
+    (default) or `false`.
+
+<<eager-global-ordinals,`eager_global_ordinals`>>::
+
+    Should global ordinals be loaded eagerly on refresh? Accepts `true` or
+    `false` (default). Enabling this is a good idea on fields that are
+    frequently used for terms aggregations.
+
+<<ignore-above,`ignore_above`>>::
+
+    Leaf values longer than this limit will not be indexed. By default, there
+    is no limit and all values will be indexed. Note that this limit applies
+    to the leaf values within the flattened object field, and not the length of
+    the entire field.
+
+<<mapping-index,`index`>>::
+
+    Determines if the field should be searchable. Accepts `true` (default) or
+    `false`.
+
+<<index-options,`index_options`>>::
+
+    What information should be stored in the index for scoring purposes.
+    Defaults to `docs` but can also be set to `freqs` to take term frequency
+    into account when computing scores.
+
+<<null-value,`null_value`>>::
+
+    A string value which is substituted for any explicit `null` values within
+    the flattened object field. Defaults to `null`, which means null sields are
+    treated as if it were missing.
+
+<<similarity,`similarity`>>::
+
+    Which scoring algorithm or _similarity_ should be used. Defaults
+    to `BM25`.
+
+`split_queries_on_whitespace`::
+
+    Whether <<full-text-queries,full text queries>> should split the input on
+    whitespace when building a query for this field. Accepts `true` or `false`
+    (default).
diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc
index 26f59e1058c09..be5308db41a3d 100644
--- a/docs/reference/mapping/types/geo-shape.asciidoc
+++ b/docs/reference/mapping/types/geo-shape.asciidoc
@@ -108,6 +108,8 @@ geo-points containing any more than latitude and longitude (two dimensions) valu
 and reject the whole document.
 | `true`
 
+|`coerce` |If `true` unclosed linear rings in polygons will be automatically closed.
+| `false`
 
 |=======================================================================
 
diff --git a/docs/reference/mapping/types/nested.asciidoc b/docs/reference/mapping/types/nested.asciidoc
index 5969dcfd6956b..63bb4591369e5 100644
--- a/docs/reference/mapping/types/nested.asciidoc
+++ b/docs/reference/mapping/types/nested.asciidoc
@@ -159,6 +159,22 @@ Nested documents can be:
 * sorted with <<nested-sorting,nested sorting>>.
 * retrieved and highlighted with <<nested-inner-hits,nested inner hits>>.
 
+[IMPORTANT]
+=============================================
+
+Because nested documents are indexed as separate documents, they can only be
+accessed  within the scope of the `nested` query, the
+`nested`/`reverse_nested` aggregations, or <<nested-inner-hits,nested inner hits>>.
+
+For instance, if a string field within a nested document has
+<<index-options,`index_options`>> set to `offsets` to allow use of the postings
+during the highlighting, these offsets will not be available during the main highlighting
+phase.  Instead, highlighting needs to be performed via
+<<nested-inner-hits,nested inner hits>>. The same consideration applies when loading
+fields during a search through <<search-request-docvalue-fields, `docvalue_fields`>>
+or <<search-request-stored-fields, `stored_fields`>>.
+
+=============================================
 
 [[nested-params]]
 ==== Parameters for `nested` fields
@@ -178,35 +194,32 @@ The following parameters are accepted by `nested` fields:
     may be added to an existing nested object.
 
 
-[IMPORTANT]
-=============================================
+[float]
+=== Limits on `nested` mappings and objects
 
-Because nested documents are indexed as separate documents, they can only be
-accessed  within the scope of the `nested` query, the
-`nested`/`reverse_nested` aggregations, or <<nested-inner-hits,nested inner hits>>.
+As described earlier, each nested object is indexed as a separate document under the hood.
+Continuing with the example above, if we indexed a single document containing 100 `user` objects,
+then 101 Lucene documents would be created -- one for the parent document, and one for each
+nested object. Because of the expense associated with `nested` mappings, Elasticsearch puts
+settings in place to guard against performance problems:
 
-For instance, if a string field within a nested document has
-<<index-options,`index_options`>> set to `offsets` to allow use of the postings
-during the highlighting, these offsets will not be available during the main highlighting
-phase.  Instead, highlighting needs to be performed via
-<<nested-inner-hits,nested inner hits>>.
+`index.mapping.nested_fields.limit`::
 
-=============================================
+    The `nested` type should only be used in special cases, when arrays of objects need to be
+    queried independently of each other. To safeguard against poorly designed mappings, this setting
+    limits the number of unique `nested` types per index. In our example, the `user` mapping would
+    count as only 1 towards this limit. Defaults to 50.
 
+`index.mapping.nested_objects.limit`::
 
-==== Limiting the number of `nested` fields
+    This setting limits the number of nested objects that a single document may contain across all
+    `nested` types, in order to prevent out of memory errors when a document contains too many nested
+    objects. To illustrate how the setting works, say we added another `nested` type called `comments`
+    to our example mapping above. Then for each document, the combined number of `user` and `comment`
+    objects it contains must be below the limit. Defaults to 10000.
 
-Indexing a document with 100 nested fields actually indexes 101 documents as each nested
-document is indexed as a separate document. To safeguard against ill-defined mappings
-the number of nested fields that can be defined per index has been limited to 50. See
-<<mapping-limit-settings>>.
+Additional background on these settings, including information on their default values, can be found
+in <<mapping-limit-settings>>.
 
-[[limit-nested-json-objects-number]]
-==== Limiting the number of `nested` json objects
-Indexing a document with an array of 100 objects within a nested field, will actually
-create 101 documents, as each nested object will be indexed as a separate document.
-To prevent out of memory errors when a single document contains too many nested json
-objects, the number of nested json objects that a single document may contain across all fields
-has been limited to 10000. See <<mapping-limit-settings>>.
 
 
diff --git a/docs/reference/mapping/types/sparse-vector.asciidoc b/docs/reference/mapping/types/sparse-vector.asciidoc
index 70b2ce4ed3198..de63a1a822a0a 100644
--- a/docs/reference/mapping/types/sparse-vector.asciidoc
+++ b/docs/reference/mapping/types/sparse-vector.asciidoc
@@ -1,3 +1,5 @@
+[role="xpack"]
+[testenv="basic"]
 [[sparse-vector]]
 === Sparse vector datatype
 
diff --git a/docs/reference/migration/migrate_8_0.asciidoc b/docs/reference/migration/migrate_8_0.asciidoc
index b697abf9a9f25..ff3f5030ed9fb 100644
--- a/docs/reference/migration/migrate_8_0.asciidoc
+++ b/docs/reference/migration/migrate_8_0.asciidoc
@@ -15,6 +15,7 @@ coming[8.0.0]
 * <<breaking_80_discovery_changes>>
 * <<breaking_80_mappings_changes>>
 * <<breaking_80_packaging_changes>>
+* <<breaking_80_rollup_changes>>
 * <<breaking_80_snapshots_changes>>
 * <<breaking_80_security_changes>>
 * <<breaking_80_ilm_changes>>
@@ -24,6 +25,7 @@ coming[8.0.0]
 * <<breaking_80_transport_changes>>
 * <<breaking_80_http_changes>>
 * <<breaking_80_reindex_changes>>
+* <<breaking_80_search_changes>>
 * <<breaking_80_settings_changes>>
 
 //NOTE: The notable-breaking-changes tagged regions are re-used in the
@@ -51,6 +53,7 @@ include::migrate_8_0/analysis.asciidoc[]
 include::migrate_8_0/discovery.asciidoc[]
 include::migrate_8_0/mappings.asciidoc[]
 include::migrate_8_0/packaging.asciidoc[]
+include::migrate_8_0/rollup.asciidoc[]
 include::migrate_8_0/snapshots.asciidoc[]
 include::migrate_8_0/security.asciidoc[]
 include::migrate_8_0/ilm.asciidoc[]
@@ -60,4 +63,5 @@ include::migrate_8_0/node.asciidoc[]
 include::migrate_8_0/transport.asciidoc[]
 include::migrate_8_0/http.asciidoc[]
 include::migrate_8_0/reindex.asciidoc[]
+include::migrate_8_0/search.asciidoc[]
 include::migrate_8_0/settings.asciidoc[]
diff --git a/docs/reference/migration/migrate_8_0/mappings.asciidoc b/docs/reference/migration/migrate_8_0/mappings.asciidoc
index 371e9fc44c415..16e75473885c6 100644
--- a/docs/reference/migration/migrate_8_0/mappings.asciidoc
+++ b/docs/reference/migration/migrate_8_0/mappings.asciidoc
@@ -14,3 +14,12 @@
 
 The number of completion contexts within a single completion field
 has been limited to 10.
+
+[float]
+==== Defining multi-fields within multi-fields
+
+Previously, it was possible to define a multi-field within a multi-field.
+Defining chained multi-fields was deprecated in 7.3 and is now no longer
+supported. To migrate the mappings, all instances of `fields` that occur within
+a `fields` block should be removed, either by flattening the chained `fields`
+blocks into a single level, or by switching to `copy_to` if appropriate.
\ No newline at end of file
diff --git a/docs/reference/migration/migrate_8_0/node.asciidoc b/docs/reference/migration/migrate_8_0/node.asciidoc
index a1dcd654807e1..b1187e88b5d90 100644
--- a/docs/reference/migration/migrate_8_0/node.asciidoc
+++ b/docs/reference/migration/migrate_8_0/node.asciidoc
@@ -14,3 +14,25 @@
 The `node.max_local_storage_nodes` setting was deprecated in 7.x and
 has been removed in 8.0. Nodes should be run on separate data paths
 to ensure that each node is consistently assigned to the same data path.
+
+[float]
+==== Change of data folder layout
+
+Each node's data is now stored directly in the data directory set by the
+`path.data` setting, rather than in `${path.data}/nodes/0`, because the removal
+of the `node.max_local_storage_nodes` setting means that nodes may no longer
+share a data path. At startup, Elasticsearch will automatically migrate the data
+path to the new layout. This automatic migration will not proceed if the data
+path contains data for more than one node. You should move to a configuration in
+which each node has its own data path before upgrading.
+
+If you try to upgrade a configuration in which there is data for more than one
+node in a data path then the automatic migration will fail and Elasticsearch
+will refuse to start. To resolve this you will need to perform the migration
+manually. The data for the extra nodes are stored in folders named
+`${path.data}/nodes/1`, `${path.data}/nodes/2` and so on, and you should move
+each of these folders to an appropriate location and then configure the
+corresponding node to use this location for its data path. If your nodes each
+have more than one data path in their `path.data` settings then you should move
+all the corresponding subfolders in parallel. Each node uses the same subfolder
+(e.g. `nodes/2`) across all its data paths.
\ No newline at end of file
diff --git a/docs/reference/migration/migrate_8_0/reindex.asciidoc b/docs/reference/migration/migrate_8_0/reindex.asciidoc
index ef4f5aed147ca..912f0f9dbf11b 100644
--- a/docs/reference/migration/migrate_8_0/reindex.asciidoc
+++ b/docs/reference/migration/migrate_8_0/reindex.asciidoc
@@ -12,4 +12,18 @@ Instead, please specify the index-name without any encoding.
 [float]
 ==== Removal of types
 
-The `/{index}/{type}/_delete_by_query` and `/{index}/{type}/_update_by_query` REST endpoints have been removed in favour of `/{index}/_delete_by_query` and `/{index}/_update_by_query`, since indexes no longer contain types, these typed endpoints are obsolete.
\ No newline at end of file
+The `/{index}/{type}/_delete_by_query` and `/{index}/{type}/_update_by_query` REST endpoints have been removed in favour of `/{index}/_delete_by_query` and `/{index}/_update_by_query`, since indexes no longer contain types, these typed endpoints are obsolete.
+
+[float]
+==== Removal of size parameter
+
+Previously, a `_reindex` request had two different size specifications in the body:
+
+- Outer level, determining the maximum number of documents to process
+- Inside the `source` element, determining the scroll/batch size.
+
+The outer level `size` parameter has now been renamed to `max_docs` to
+avoid confusion and clarify its semantics.
+
+Similarly, the `size` parameter has been renamed to `max_docs` for
+`_delete_by_query` and `_update_by_query` to keep the 3 interfaces consistent.
\ No newline at end of file
diff --git a/docs/reference/migration/migrate_8_0/rollup.asciidoc b/docs/reference/migration/migrate_8_0/rollup.asciidoc
new file mode 100644
index 0000000000000..71c9b38d444d6
--- /dev/null
+++ b/docs/reference/migration/migrate_8_0/rollup.asciidoc
@@ -0,0 +1,20 @@
+[float]
+[[breaking_80_rollup_changes]]
+=== Rollup changes
+
+//NOTE: The notable-breaking-changes tagged regions are re-used in the
+//Installation and Upgrade Guide
+
+//tag::notable-breaking-changes[]
+
+// end::notable-breaking-changes[]
+
+[float]
+==== StartRollupJob endpoint returns success if job already started
+
+Previously, attempting to start an already-started rollup job would
+result in a `500 InternalServerError: Cannot start task for Rollup Job
+[job] because state was [STARTED]` exception.
+
+Now, attempting to start a job that is already started will just
+return a successful `200 OK: started` response.
\ No newline at end of file
diff --git a/docs/reference/migration/migrate_8_0/search.asciidoc b/docs/reference/migration/migrate_8_0/search.asciidoc
index 82886d35bc6a5..97796a10fca22 100644
--- a/docs/reference/migration/migrate_8_0/search.asciidoc
+++ b/docs/reference/migration/migrate_8_0/search.asciidoc
@@ -7,4 +7,21 @@
 
 The `/{index}/{type}/_search`, `/{index}/{type}/_msearch`, `/{index}/{type}/_search/template` and `/{index}/{type}/_msearch/template` REST endpoints have been removed in favour of `/{index}/_search`, `/{index}/_msearch`, `/{index}/_search/template` and `/{index}/_msearch/template`, since indexes no longer contain types, these typed endpoints are obsolete..
 
-The `/{index}/{type}/_termvectors`, `/{index}/{type}/{id}/_termvectors` and `/{index}/{type}/_mtermvectors` REST endpoints have been removed in favour of `/{index}/_termvectors`, `/{index}/{id}/_termvectors` and `/{index}/_mtermvectors`, since indexes no longer contain types, these typed endpoints are obsolete..
\ No newline at end of file
+The `/{index}/{type}/_termvectors`, `/{index}/{type}/{id}/_termvectors` and `/{index}/{type}/_mtermvectors` REST endpoints have been removed in favour of `/{index}/_termvectors`, `/{index}/{id}/_termvectors` and `/{index}/_mtermvectors`, since indexes no longer contain types, these typed endpoints are obsolete..
+
+[float]
+==== Removal of queries
+
+The `common` query was deprecated in 7.x and  has been removed in 8.0.
+The same functionality can be achieved by the `match` query if the total number of hits is not tracked.
+
+[float]
+===== Removal of query parameters
+
+The `cutoff_frequency` parameter was deprecated in 7.x and has been removed in 8.0 from `match` and `multi_match` queries.
+The same functionality can be achieved without any configuration provided that the total number of hits is not tracked.
+
+[float]
+===== Removal of sort parameters
+
+The `nested_filter` and `nested_path` options, deprecated in 6.x, have been removed in favor of the `nested` context.
diff --git a/docs/reference/migration/migrate_8_0/security.asciidoc b/docs/reference/migration/migrate_8_0/security.asciidoc
index a7cacef8ff017..f3ee4fd97ef85 100644
--- a/docs/reference/migration/migrate_8_0/security.asciidoc
+++ b/docs/reference/migration/migrate_8_0/security.asciidoc
@@ -33,3 +33,11 @@ The `elasticsearch-migrate` tool provided a way to convert file
 realm users and roles into the native realm. It has been deprecated
 since 7.2.0. Users and roles should now be created in the native
 realm directly.
+
+[float]
+[[separating-node-and-client-traffic]]
+==== The `transport.profiles.*.xpack.security.type` setting has been removed
+
+The `transport.profiles.*.xpack.security.type` setting has been removed since
+the Transport Client has been removed and therefore all client traffic now uses
+the HTTP transport. Transport profiles using this setting should be removed.
diff --git a/docs/reference/migration/migrate_8_0/snapshots.asciidoc b/docs/reference/migration/migrate_8_0/snapshots.asciidoc
index 791e5b28da057..27a0afa92b71e 100644
--- a/docs/reference/migration/migrate_8_0/snapshots.asciidoc
+++ b/docs/reference/migration/migrate_8_0/snapshots.asciidoc
@@ -9,6 +9,59 @@
 
 // end::notable-breaking-changes[]
 
+[float]
+=== Get snapshots response format is changed
+It's possible to get snapshots from multiple repositories in one go. The response format has changed
+and now contains separate response for each repository.
+
+For example, requesting one snapshot from particular repository
+
+[source,js]
+-----------------------------------
+GET _snapshot/repo1/snap1
+-----------------------------------
+// CONSOLE
+// TEST[skip:no repo and snapshots are created]
+
+produces the following response
+
+[source,js]
+-----------------------------------
+{
+    "responses": [
+        {
+            "repository": "repo1",
+            "snapshots": [
+                {
+                    "snapshot": "snap1",
+                    "uuid": "cEzdqUKxQ5G6MyrJAcYwmA",
+                    "version_id": 8000099,
+                    "version": "8.0.0",
+                    "indices": [],
+                    "include_global_state": true,
+                    "state": "SUCCESS",
+                    "start_time": "2019-05-10T17:01:57.868Z",
+                    "start_time_in_millis": 1557507717868,
+                    "end_time": "2019-05-10T17:01:57.909Z",
+                    "end_time_in_millis": 1557507717909,
+                    "duration_in_millis": 41,
+                    "failures": [],
+                    "shards": {
+                        "total": 0,
+                        "failed": 0,
+                        "successful": 0
+                    }
+                }
+            ]
+        }
+    ]
+}
+-----------------------------------
+// TESTRESPONSE
+// TEST[skip:no repo and snapshots are created]
+
+See <<modules-snapshots>> for more information.
+
 [float]
 ==== Deprecated node level compress setting removed
 
diff --git a/docs/reference/ml/apis/close-job.asciidoc b/docs/reference/ml/apis/close-job.asciidoc
index c5f9b5fc2444a..2a38648e48fe5 100644
--- a/docs/reference/ml/apis/close-job.asciidoc
+++ b/docs/reference/ml/apis/close-job.asciidoc
@@ -12,8 +12,8 @@ A job can be opened and closed multiple times throughout its lifecycle.
 A closed job cannot receive data or perform analysis
 operations, but you can still explore and navigate results.
 
-
-==== Request
+[[ml-close-job-request]]
+==== {api-request-title}
 
 `POST _ml/anomaly_detectors/<job_id>/_close` +
 
@@ -21,8 +21,15 @@ operations, but you can still explore and navigate results.
 
 `POST _ml/anomaly_detectors/_all/_close` +
 
+[[ml-close-job-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-==== Description
+[[ml-close-job-desc]]
+==== {api-description-title}
 
 You can close multiple jobs in a single API request by using a group name, a
 comma-separated list of jobs, or a wildcard expression. You can close all jobs
@@ -47,32 +54,26 @@ after the close job API returns.  The `force` query parameter should only be use
 situations where the job has already failed, or where you are not interested in
 results the job might have recently produced or might produce in the future.
 
+[[ml-close-job-path-parms]]
+==== {api-path-parms-title}
 
-==== Path Parameters
-
-`job_id`::
+`<job_id>` (Required)::
   (string) Identifier for the job. It can be a job identifier, a group name, or
   a wildcard expression.
 
+[[ml-close-job-query-parms]]
+==== {api-query-parms-title}
 
-==== Query Parameters
-
-`force`::
+`force` (Optional)::
   (boolean) Use to close a failed job, or to forcefully close a job which has not
   responded to its initial close request.
 
-`timeout`::
+`timeout` (Optional)::
   (time units) Controls the time to wait until a job has closed.
   The default value is 30 minutes.
 
-
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see {xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-close-job-example]]
+==== {api-examples-title}
 
 The following example closes the `total-requests` job:
 
diff --git a/docs/reference/ml/apis/datafeedresource.asciidoc b/docs/reference/ml/apis/datafeedresource.asciidoc
index 33fce3dbf7c9d..5c1e3e74a6ae8 100644
--- a/docs/reference/ml/apis/datafeedresource.asciidoc
+++ b/docs/reference/ml/apis/datafeedresource.asciidoc
@@ -61,12 +61,12 @@ A {dfeed} resource has the following properties:
 
 `delayed_data_check_config`::
   (object) Specifies whether the data feed checks for missing data and 
-  and the size of the window. For example:
+  the size of the window. For example:
   `{"enabled": true, "check_window": "1h"}` See
   <<ml-datafeed-delayed-data-check-config>>.
 
 [[ml-datafeed-chunking-config]]
-==== Chunking Configuration Objects
+==== Chunking configuration objects
 
 {dfeeds-cap} might be required to search over long time periods, for several months
 or years. This search is split into time chunks in order to ensure the load
@@ -88,31 +88,33 @@ A chunking configuration object has the following properties:
   For example: `3h`.
 
 [[ml-datafeed-delayed-data-check-config]]
-==== Delayed Data Check Configuration Objects
+==== Delayed data check configuration objects
 
 The {dfeed} can optionally search over indices that have already been read in
-an effort to find if any data has since been added to the index. If missing data
-is found, it is a good indication that the `query_delay` option is set too low and
-the data is being indexed after the {dfeed} has passed that moment in time. See 
+an effort to determine whether any data has subsequently been added to the index.
+If missing data is found, it is a good indication that the `query_delay` option
+is set too low and the data is being indexed after the {dfeed} has passed that
+moment in time. See 
 {stack-ov}/ml-delayed-data-detection.html[Working with delayed data].
 
-This check only runs on real-time {dfeeds}
+This check runs only on real-time {dfeeds}.
 
 The configuration object has the following properties:
 
 `enabled`::
-  (boolean) Should the {dfeed} periodically check for data being indexed after reading.
-  Defaults to `true`
+  (boolean) Specifies whether the {dfeed} periodically checks for delayed data.
+  Defaults to `true`.
 
 `check_window`::
-  (time units) The window of time before the latest finalized bucket that should be searched
-  for late data. Defaults to `null` which causes an appropriate `check_window` to be calculated
-  when the real-time {dfeed} runs.
-  The default `check_window` span calculation is the max between `2h` or `8 * bucket_span`.
+  (time units) The window of time that is searched for late data. This window of
+  time ends with the latest finalized bucket. It defaults to `null`, which
+  causes an appropriate `check_window` to be calculated when the real-time
+  {dfeed} runs. In particular, the default `check_window` span calculation is
+  based on the maximum of `2h` or `8 * bucket_span`.
 
 [float]
 [[ml-datafeed-counts]]
-==== {dfeed-cap} Counts
+==== {dfeed-cap} counts
 
 The get {dfeed} statistics API provides information about the operational
 progress of a {dfeed}. All of these properties are informational; you cannot
diff --git a/docs/reference/ml/apis/delete-calendar-event.asciidoc b/docs/reference/ml/apis/delete-calendar-event.asciidoc
index 68f7a0738375d..0aa9ce5cc8d92 100644
--- a/docs/reference/ml/apis/delete-calendar-event.asciidoc
+++ b/docs/reference/ml/apis/delete-calendar-event.asciidoc
@@ -8,34 +8,37 @@
 
 Deletes scheduled events from a calendar.
 
-
-==== Request
+[[ml-delete-calendar-event-request]]
+==== {api-request-title}
 
 `DELETE _ml/calendars/<calendar_id>/events/<event_id>`
 
+[[ml-delete-calendar-event-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-==== Description
+[[ml-delete-calendar-event-desc]]
+==== {api-description-title}
 
 This API removes individual events from a calendar. To remove all scheduled
 events and delete the calendar, see the
 <<ml-delete-calendar,delete calendar API>>.
 
-==== Path Parameters
+[[ml-delete-calendar-event-path-parms]]
+==== {api-path-parms-title}
 
-`calendar_id`(required)::
+`<calendar_id>` (Required)::
   (string) Identifier for the calendar.
 
-`event_id` (required)::
+`<event_id>` (Required)::
   (string) Identifier for the scheduled event. You can obtain this identifier
   by using the <<ml-get-calendar-event,get calendar events API>>.
 
-
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see {xpack-ref}/security-privileges.html[Security Privileges].
-
-==== Examples
+[[ml-delete-calendar-event-example]]
+==== {api-examples-title}
 
 The following example deletes a scheduled event from the `planned-outages`
 calendar:
diff --git a/docs/reference/ml/apis/delete-calendar-job.asciidoc b/docs/reference/ml/apis/delete-calendar-job.asciidoc
index 118a706d29460..a555b3d3b922b 100644
--- a/docs/reference/ml/apis/delete-calendar-job.asciidoc
+++ b/docs/reference/ml/apis/delete-calendar-job.asciidoc
@@ -8,28 +8,30 @@
 
 Deletes jobs from a calendar.
 
-
-==== Request
+[[ml-delete-calendar-job-request]]
+==== {api-request-title}
 
 `DELETE _ml/calendars/<calendar_id>/jobs/<job_id>`
 
+[[ml-delete-calendar-job-prereqs]]
+==== {api-prereq-title}
 
-==== Path Parameters
-
-`calendar_id`(required)::
-  (string) Identifier for the calendar.
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-`job_id` (required)::
-  (string) An identifier for the job. It can be a job identifier, a group name, or a
-           comma-separated list of jobs or groups.
+[[ml-delete-calendar-job-path-parms]]
+==== {api-path-parms-title}
 
+`<calendar_id>` (Required)::
+  (string) Identifier for the calendar.
 
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see {xpack-ref}/security-privileges.html[Security Privileges].
+`<job_id>` (Required)::
+  (string) An identifier for the job. It can be a job identifier, a group name,
+  or a comma-separated list of jobs or groups.
 
-==== Examples
+[[ml-delete-calendar-job-example]]
+==== {api-examples-title}
 
 The following example removes the association between the `planned-outages`
 calendar and `total-requests` job:
diff --git a/docs/reference/ml/apis/delete-calendar.asciidoc b/docs/reference/ml/apis/delete-calendar.asciidoc
index 2707f3175e04c..065c117c49c63 100644
--- a/docs/reference/ml/apis/delete-calendar.asciidoc
+++ b/docs/reference/ml/apis/delete-calendar.asciidoc
@@ -8,31 +8,32 @@
 
 Deletes a calendar.
 
-
-==== Request
+[[ml-delete-calendar-request]]
+==== {api-request-title}
 
 `DELETE _ml/calendars/<calendar_id>`
 
+[[ml-delete-calendar-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-==== Description
+[[ml-delete-calendar-desc]]
+==== {api-description-title}
 
 This API removes all scheduled events from the calendar then deletes the
 calendar.
 
+[[ml-delete-calendar-path-parms]]
+==== {api-path-parms-title}
 
-==== Path Parameters
-
-`calendar_id` (required)::
+`<calendar_id>` (Required)::
   (string) Identifier for the calendar.
 
-
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see {xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-delete-calendar-example]]
+==== {api-examples-title}
 
 The following example deletes the `planned-outages` calendar:
 
diff --git a/docs/reference/ml/apis/delete-datafeed.asciidoc b/docs/reference/ml/apis/delete-datafeed.asciidoc
index d2a7845be7584..23917bf9e3365 100644
--- a/docs/reference/ml/apis/delete-datafeed.asciidoc
+++ b/docs/reference/ml/apis/delete-datafeed.asciidoc
@@ -10,38 +10,39 @@
 
 Deletes an existing {dfeed}.
 
-
-==== Request
+[[ml-delete-datafeed-request]]
+==== {api-request-title}
 
 `DELETE _ml/datafeeds/<feed_id>`
 
+[[ml-delete-datafeed-prereqs]]
+==== {api-prereq-title}
 
-==== Description
-
-NOTE: Unless the `force` parameter is used, the {dfeed} must be stopped before it can be deleted.
-
-
-==== Path Parameters
-
-`feed_id` (required)::
-  (string) Identifier for the {dfeed}
-
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-===== Query Parameters
+[[ml-delete-datafeed-desc]]
+==== {api-description-title}
 
-`force`::
-  (boolean) Use to forcefully delete a started {dfeed}; this method is quicker than
-  stopping and deleting the {dfeed}.
+NOTE: Unless you use the `force` parameter, you must stop the {dfeed} before you
+can delete it.
 
+[[ml-delete-datafeed-path-parms]]
+==== {api-path-parms-title}
 
-===== Authorization
+`<feed_id>` (Required)::
+  (string) Identifier for the {dfeed}.
 
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see {xpack-ref}/security-privileges.html[Security Privileges].
-//<<privileges-list-cluster>>.
+[[ml-delete-datafeed-query-parms]]
+==== {api-query-parms-title}
 
+`force` (Optional)::
+  (boolean) Use to forcefully delete a started {dfeed}; this method is quicker
+  than stopping and deleting the {dfeed}.
 
-==== Examples
+[[ml-delete-datafeed-example]]
+==== {api-examples-title}
 
 The following example deletes the `datafeed-total-requests` {dfeed}:
 
diff --git a/docs/reference/ml/apis/delete-expired-data.asciidoc b/docs/reference/ml/apis/delete-expired-data.asciidoc
index 8814a1686736e..ada9ec1c8c34e 100644
--- a/docs/reference/ml/apis/delete-expired-data.asciidoc
+++ b/docs/reference/ml/apis/delete-expired-data.asciidoc
@@ -8,25 +8,27 @@
 
 Deletes expired and unused machine learning data.
 
-==== Request
+[[ml-delete-expired-data-request]]
+==== {api-request-title}
 
 `DELETE _ml/_delete_expired_data`
 
-==== Description
+[[ml-delete-expired-data-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
+
+[[ml-delete-expired-data-desc]]
+==== {api-description-title}
 
 Deletes all job results, model snapshots and forecast data that have exceeded
 their `retention days` period. Machine learning state documents that are not
 associated with any job are also deleted.
 
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see
-{stack-ov}/security-privileges.html[Security Privileges] and
-{stack-ov}/built-in-roles.html[Built-in Roles].
-
-
-==== Examples
+[[ml-delete-expired-data-example]]
+==== {api-examples-title}
 
 The endpoint takes no arguments:
 
diff --git a/docs/reference/ml/apis/delete-filter.asciidoc b/docs/reference/ml/apis/delete-filter.asciidoc
index b2cbb7ef2832d..1962db29ad74e 100644
--- a/docs/reference/ml/apis/delete-filter.asciidoc
+++ b/docs/reference/ml/apis/delete-filter.asciidoc
@@ -8,32 +8,33 @@
 
 Deletes a filter.
 
-
-==== Request
+[[ml-delete-filter-request]]
+==== {api-request-title}
 
 `DELETE _ml/filters/<filter_id>`
 
+[[ml-delete-filter-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-==== Description
+[[ml-delete-filter-desc]]
+==== {api-description-title}
 
 This API deletes a {stack-ov}/ml-rules.html[filter]. 
 If a {ml} job references the filter, you cannot delete the filter. You must 
 update or delete the job before you can delete the filter.
 
+[[ml-delete-filter-path-parms]]
+==== {api-path-parms-title}
 
-==== Path Parameters
-
-`filter_id` (required)::
+`<filter_id>` (Required)::
   (string) Identifier for the filter.
 
-
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see {xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-delete-filter-example]]
+==== {api-examples-title}
 
 The following example deletes the `safe_domains` filter:
 
diff --git a/docs/reference/ml/apis/delete-forecast.asciidoc b/docs/reference/ml/apis/delete-forecast.asciidoc
index 133b9105e478a..aac054217fced 100644
--- a/docs/reference/ml/apis/delete-forecast.asciidoc
+++ b/docs/reference/ml/apis/delete-forecast.asciidoc
@@ -8,7 +8,8 @@
 
 Deletes forecasts from a {ml} job.  
 
-==== Request
+[[ml-delete-forecast-request]]
+==== {api-request-title}
 
 `DELETE _ml/anomaly_detectors/<job_id>/_forecast` +
 
@@ -16,48 +17,54 @@ Deletes forecasts from a {ml} job.
 
 `DELETE _ml/anomaly_detectors/<job_id>/_forecast/_all`
 
+[[ml-delete-forecast-prereqs]]
+==== {api-prereq-title}
 
-==== Description
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-By default, forecasts are retained for 14 days. You can specify a different 
-retention period with the `expires_in` parameter in the <<ml-forecast,forecast jobs API>>. The delete forecast API enables you to delete one or more forecasts before they expire.
+[[ml-delete-forecast-desc]]
+==== {api-description-title}
 
-NOTE: When you delete a job its associated forecasts are deleted. 
+By default, forecasts are retained for 14 days. You can specify a different 
+retention period with the `expires_in` parameter in the
+<<ml-forecast,forecast jobs API>>. The delete forecast API enables you to delete
+one or more forecasts before they expire.
 
-For more information, see {stack-ov}/ml-overview.html#ml-forecasting[Forecasting the Future].
+NOTE: When you delete a job, its associated forecasts are deleted. 
 
+For more information, see
+{stack-ov}/ml-overview.html#ml-forecasting[Forecasting the future].
 
-==== Path Parameters
+[[ml-delete-forecast-path-parms]]
+==== {api-path-parms-title}
 
-`job_id` (required)::
+`<job_id>` (Required)::
   (string) Identifier for the job.
 
-`forecast_id`::
+`forecast_id` (Optional)::
   (string) A comma-separated list of forecast identifiers. 
   If you do not specify this optional parameter or if you specify `_all`, the 
   API deletes all forecasts from the job. 
 
-==== Request Parameters
+[[ml-delete-forecast-query-parms]]
+==== {api-query-parms-title}
 
-`allow_no_forecasts`::
+`allow_no_forecasts` (Optional)::
   (boolean) Specifies whether an error occurs when there are no forecasts. In 
   particular, if this parameter is set to `false` and there are no forecasts 
   associated with the job, attempts to delete all forecasts return an error. 
   The default value is `true`.
 
-`timeout`::
+`timeout` (Optional)::
   (time units) Specifies the period of time to wait for the completion of the 
   delete operation. When this period of time elapses, the API fails and returns 
   an error. The default value is `30s`. For more information about time units, 
   see <<time-units>>.
-  
-
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see {stack-ov}/security-privileges.html[Security Privileges].
 
-==== Examples
+[[ml-delete-forecast-example]]
+==== {api-examples-title}
 
 The following example deletes all forecasts from the `total-requests` job:
 
diff --git a/docs/reference/ml/apis/delete-job.asciidoc b/docs/reference/ml/apis/delete-job.asciidoc
index a52c434f93cfe..efd172ef5fb60 100644
--- a/docs/reference/ml/apis/delete-job.asciidoc
+++ b/docs/reference/ml/apis/delete-job.asciidoc
@@ -8,13 +8,20 @@
 
 Deletes an existing anomaly detection job.
 
-
-==== Request
+[[ml-delete-job-request]]
+==== {api-request-title}
 
 `DELETE _ml/anomaly_detectors/<job_id>`
 
+[[ml-delete-job-prereqs]]
+==== {api-prereq-title}
+
+* If {es} {security-features} are enabled, you must have `manage_ml` or `manage`
+cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-==== Description
+[[ml-delete-job-desc]]
+==== {api-description-title}
 
 All job configuration, model state and results are deleted.
 
@@ -30,29 +37,25 @@ is used the job must be closed before it can be deleted.
 It is not currently possible to delete multiple jobs using wildcards or a comma
 separated list.
 
-==== Path Parameters
+[[ml-delete-job-path-parms]]
+==== {api-path-parms-title}
 
-`job_id` (required)::
-  (string) Identifier for the job
+`<job_id>` (Required)::
+  (string) Identifier for the job.
 
-===== Query Parameters
+[[ml-delete-job-query-parms]]
+==== {api-query-parms-title}
 
-`force`::
+`force` (Optional)::
   (boolean) Use to forcefully delete an opened job; this method is quicker than
   closing and deleting the job.
 
-`wait_for_completion`::
+`wait_for_completion` (Optional)::
   (boolean) Specifies whether the request should return immediately or wait
   until the job deletion completes. Defaults to `true`.
 
-==== Authorization
-
-If {es} {security-features} are enabled, you must have `manage_ml`, or `manage`
-cluster privileges to use this API.
-For more information, see {stack-ov}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-delete-job-example]]
+==== {api-examples-title}
 
 The following example deletes the `total-requests` job:
 
diff --git a/docs/reference/ml/apis/delete-snapshot.asciidoc b/docs/reference/ml/apis/delete-snapshot.asciidoc
index 18092ff8e89c1..0e696f2a01139 100644
--- a/docs/reference/ml/apis/delete-snapshot.asciidoc
+++ b/docs/reference/ml/apis/delete-snapshot.asciidoc
@@ -8,34 +8,36 @@
 
 Deletes an existing model snapshot.
 
-
-==== Request
+[[ml-delete-snapshot-request]]
+==== {api-request-title}
 
 `DELETE _ml/anomaly_detectors/<job_id>/model_snapshots/<snapshot_id>`
 
+[[ml-delete-snapshot-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-==== Description
+[[ml-delete-snapshot-desc]]
+==== {api-description-title}
 
 IMPORTANT: You cannot delete the active model snapshot. To delete that snapshot,
 first revert to a different one. To identify the active model snapshot, refer to
 the `model_snapshot_id` in the results from the get jobs API.
 
-==== Path Parameters
-
-`job_id` (required)::
-  (string) Identifier for the job
-
-`snapshot_id` (required)::
-  (string) Identifier for the model snapshot
-
-
-==== Authorization
+[[ml-delete-snapshot-path-parms]]
+==== {api-path-parms-title}
 
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see {xpack-ref}/security-privileges.html[Security Privileges].
+`<job_id>` (Required)::
+  (string) Identifier for the job.
 
+`<snapshot_id>` (Required)::
+  (string) Identifier for the model snapshot.
 
-==== Examples
+[[ml-delete-snapshot-example]]
+==== {api-examples-title}
 
 The following example deletes the `1491948163` snapshot:
 
diff --git a/docs/reference/ml/apis/find-file-structure.asciidoc b/docs/reference/ml/apis/find-file-structure.asciidoc
index e9d9da479c0f2..212e80c7e1bd2 100644
--- a/docs/reference/ml/apis/find-file-structure.asciidoc
+++ b/docs/reference/ml/apis/find-file-structure.asciidoc
@@ -11,12 +11,20 @@ experimental[]
 Finds the structure of a text file. The text file must contain data that is
 suitable to be ingested into {es}.
 
-==== Request
+[[ml-find-file-structure-request]]
+==== {api-request-title}
 
 `POST _ml/find_file_structure`
 
+[[ml-find-file-structure-prereqs]]
+==== {api-prereq-title}
 
-==== Description
+* If the {es} {security-features} are enabled, you must have `monitor_ml` or
+`monitor` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
+
+[[ml-find-file-structure-desc]]
+==== {api-description-title}
 
 This API provides a starting point for ingesting data into {es} in a format that
 is suitable for subsequent use with other {ml} functionality.
@@ -47,38 +55,39 @@ specify the `explain` query parameter. It causes an `explanation` to appear in
 the response, which should help in determining why the returned structure was
 chosen.
 
-==== Query Parameters
+[[ml-find-file-structure-query-parms]]
+==== {api-query-parms-title}
 
-`charset`::
+`charset` (Optional)::
   (string) The file's character set. It must be a character set that is supported
   by the JVM that {es} uses. For example, `UTF-8`, `UTF-16LE`, `windows-1252`, or
   `EUC-JP`. If this parameter is not specified, the structure finder chooses an
   appropriate character set.
 
-`column_names`::
+`column_names` (Optional)::
   (string) If you have set `format` to `delimited`, you can specify the column names
   in a comma-separated list. If this parameter is not specified, the structure
   finder uses the column names from the header row of the file. If the file does
   not have a header role, columns are named "column1", "column2", "column3", etc.
 
-`delimiter`::
+`delimiter` (Optional)::
   (string) If you have set `format` to `delimited`, you can specify the character used
   to delimit the values in each row. Only a single character is supported; the
   delimiter cannot have multiple characters. If this parameter is not specified,
   the structure finder considers the following possibilities: comma, tab,
   semi-colon, and pipe (`|`).
 
-`explain`::
+`explain` (Optional)::
   (boolean) If this parameter is set to `true`, the response includes a field
   named `explanation`, which is an array of strings that indicate how the
   structure finder produced its result. The default value is `false`.
 
-`format`::
+`format` (Optional)::
   (string) The high level structure of the file. Valid values are `ndjson`, `xml`,
   `delimited`, and `semi_structured_text`. If this parameter is not specified,
   the structure finder chooses one.
 
-`grok_pattern`::
+`grok_pattern` (Optional)::
   (string) If you have set `format` to `semi_structured_text`, you can specify a Grok
   pattern that is used to extract fields from every message in the file. The
   name of the timestamp field in the Grok pattern must match what is specified
@@ -86,13 +95,20 @@ chosen.
   name of the timestamp field in the Grok pattern must match "timestamp". If
   `grok_pattern` is not specified, the structure finder creates a Grok pattern.
 
-`has_header_row`::
+`has_header_row` (Optional)::
   (boolean) If you have set `format` to `delimited`, you can use this parameter to
   indicate whether the column names are in the first row of the file. If this
   parameter is not specified, the structure finder guesses based on the similarity of
   the first row of the file to other rows.
 
-`lines_to_sample`::
+`line_merge_size_limit` (Optional)::
+  (unsigned integer) The maximum number of characters in a message when lines are
+  merged to form messages while analyzing semi-structured files. The default
+  is 10000. If you have extremely long messages you may need to increase this, but
+  be aware that this may lead to very long processing times if the way to group
+  lines into messages is misdetected.
+
+`lines_to_sample` (Optional)::
   (unsigned integer) The number of lines to include in the structural analysis,
   starting from the beginning of the file. The minimum is 2; the default
   is 1000. If the value of this parameter is greater than the number of lines in
@@ -108,7 +124,7 @@ efficient to upload a sample file with more variety in the first 1000 lines than
 to request analysis of 100000 lines to achieve some variety.
 --
 
-`quote`::
+`quote` (Optional)::
   (string) If you have set `format` to `delimited`, you can specify the character used
   to quote the values in each row if they contain newlines or the delimiter
   character. Only a single character is supported. If this parameter is not
@@ -116,18 +132,18 @@ to request analysis of 100000 lines to achieve some variety.
   format does not use quoting, a workaround is to set this argument to a
   character that does not appear anywhere in the sample.
 
-`should_trim_fields`::
+`should_trim_fields` (Optional)::
   (boolean) If you have set `format` to `delimited`, you can specify whether values
   between delimiters should have whitespace trimmed from them. If this parameter
   is not specified and the delimiter is pipe (`|`), the default value is `true`.
   Otherwise, the default value is `false`.
 
-`timeout`::
+`timeout` (Optional)::
   (time) Sets the maximum amount of time that the structure analysis make take.
   If the analysis is still running when the timeout expires then it will be
   aborted. The default value is 25 seconds.
 
-`timestamp_field`::
+`timestamp_field` (Optional)::
   (string) The name of the field that contains the primary timestamp of each
   record in the file. In particular, if the file were ingested into an index,
   this is the field that would be used to populate the `@timestamp` field. +
@@ -146,7 +162,7 @@ field (if any) is the primary timestamp field. For structured file formats, it
 is not compulsory to have a timestamp in the file.
 --
 
-`timestamp_format`::
+`timestamp_format` (Optional)::
   (string) The Java time format of the timestamp field in the file. +
 +
 --
@@ -190,22 +206,16 @@ format from a built-in set.
 
 --
 
-==== Request Body
+[[ml-find-file-structure-request-body]]
+==== {api-request-body-title}
 
 The text file that you want to analyze. It must contain data that is suitable to
 be ingested into {es}. It does not need to be in JSON format and it does not
 need to be UTF-8 encoded. The size is limited to the {es} HTTP receive buffer
 size, which defaults to 100 Mb.
 
-
-==== Authorization
-
-You must have `monitor_ml`, or `monitor` cluster privileges to use this API.
-For more information, see {stack-ov}/security-privileges.html[Security Privileges].
-
-
 [[ml-find-file-structure-examples]]
-==== Examples
+==== {api-examples-title}
 
 Suppose you have a newline-delimited JSON file that contains information about
 some books. You can send the contents to the `find_file_structure` endpoint:
@@ -438,6 +448,8 @@ If the request does not encounter errors, you receive the following result:
     "release_date" : {
       "count" : 24,
       "cardinality" : 20,
+      "earliest" : "1932-06-01",
+      "latest" : "2011-06-02",
       "top_hits" : [
         {
           "value" : "1985-06-01",
@@ -1145,6 +1157,8 @@ If the request does not encounter errors, you receive the following result:
     "tpep_dropoff_datetime" : {
       "count" : 19998,
       "cardinality" : 9066,
+      "earliest" : "2018-05-31 06:18:15",
+      "latest" : "2018-06-02 02:25:44",
       "top_hits" : [
         {
           "value" : "2018-06-01 01:12:12",
@@ -1191,6 +1205,8 @@ If the request does not encounter errors, you receive the following result:
     "tpep_pickup_datetime" : {
       "count" : 19998,
       "cardinality" : 8760,
+      "earliest" : "2018-05-31 06:08:31",
+      "latest" : "2018-06-02 01:21:21",
       "top_hits" : [
         {
           "value" : "2018-06-01 00:01:23",
@@ -1450,6 +1466,8 @@ this:
     "timestamp" : {
       "count" : 53,
       "cardinality" : 28,
+      "earliest" : "2018-09-27T14:39:28,518",
+      "latest" : "2018-09-27T14:39:37,012",
       "top_hits" : [
         {
           "value" : "2018-09-27T14:39:29,859",
@@ -1712,6 +1730,8 @@ this:
     "timestamp" : {
       "count" : 53,
       "cardinality" : 28,
+      "earliest" : "2018-09-27T14:39:28,518",
+      "latest" : "2018-09-27T14:39:37,012",
       "top_hits" : [
         {
           "value" : "2018-09-27T14:39:29,859",
diff --git a/docs/reference/ml/apis/flush-job.asciidoc b/docs/reference/ml/apis/flush-job.asciidoc
index e2793b2c1a174..590f866ca1799 100644
--- a/docs/reference/ml/apis/flush-job.asciidoc
+++ b/docs/reference/ml/apis/flush-job.asciidoc
@@ -8,13 +8,20 @@
 
 Forces any buffered data to be processed by the job.
 
-
-==== Request
+[[ml-flush-job-request]]
+==== {api-request-title}
 
 `POST _ml/anomaly_detectors/<job_id>/_flush`
 
+[[ml-flush-job-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-==== Description
+[[ml-flush-job-desc]]
+==== {api-description-title}
 
 The flush jobs API is only applicable when sending data for analysis using the
 <<ml-post-data,post data API>>. Depending on the content of the buffer, then it
@@ -26,44 +33,38 @@ remains open and is available to continue analyzing data. A close operation
 additionally prunes and persists the model state to disk and the job must be
 opened again before analyzing further data.
 
+[[ml-flush-job-path-parms]]
+==== {api-path-parms-title}
 
-==== Path Parameters
-
-`job_id` (required)::
-(string) Identifier for the job
-
+`<job_id>` (Required)::
+(string) Identifier for the job.
 
-==== Query Parameters
+[[ml-flush-job-query-parms]]
+==== {api-query-parms-title}
 
-`advance_time`::
+`advance_time` (Optional)::
   (string) Specifies to advance to a particular time value. Results are
   generated and the model is updated for data from the specified time interval.
 
-`calc_interim`::
+`calc_interim` (Optional)::
   (boolean) If true, calculates the interim results for the most recent bucket
   or all buckets within the latency period.
 
-`end`::
+`end` (Optional)::
   (string) When used in conjunction with `calc_interim`, specifies the range
   of buckets on which to calculate interim results.
 
-`skip_time`::
+`skip_time` (Optional)::
   (string) Specifies to skip to a particular time value. Results are not
   generated and the model is not updated for data from the specified time
   interval.
 
-`start`::
+`start` (Optional)::
   (string) When used in conjunction with `calc_interim`, specifies the range of
   buckets on which to calculate interim results.
 
-
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see {xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-flush-job-example]]
+==== {api-examples-title}
 
 The following example flushes the `total-requests` job:
 
diff --git a/docs/reference/ml/apis/forecast.asciidoc b/docs/reference/ml/apis/forecast.asciidoc
index 71a7e1db2b185..d137b2e1be3ce 100644
--- a/docs/reference/ml/apis/forecast.asciidoc
+++ b/docs/reference/ml/apis/forecast.asciidoc
@@ -8,14 +8,22 @@
 
 Predicts the future behavior of a time series by using its historical behavior. 
 
-==== Request
+[[ml-forecast-request]]
+==== {api-request-title}
 
 `POST _ml/anomaly_detectors/<job_id>/_forecast`
 
+[[ml-forecast-prereqs]]
+==== {api-prereq-title}
 
-==== Description
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-See {xpack-ref}/ml-overview.html#ml-forecasting[Forecasting the Future].
+[[ml-forecast-desc]]
+==== {api-description-title}
+
+See {stack-ov}/ml-overview.html#ml-forecasting[Forecasting the future].
 
 [NOTE]
 ===============================
@@ -25,33 +33,29 @@ forecast. For more information about this property, see <<ml-job-resource>>.
 * The job must be open when you create a forecast. Otherwise, an error occurs.
 ===============================
 
-==== Path Parameters
+[[ml-forecast-path-parms]]
+==== {api-path-parms-title}
 
-`job_id`::
+`<job_id>` (Required)::
   (string) Identifier for the job.
 
+[[ml-forecast-request-body]]
+==== {api-request-body-title}
 
-==== Request Parameters
-
-`duration`::
+`duration` (Optional)::
   (time units) A period of time that indicates how far into the future to
   forecast. For example, `30d` corresponds to 30 days. The default value is 1
   day. The forecast starts at the last record that was processed. For more
   information about time units, see <<time-units>>.
 
-`expires_in`::
+`expires_in` (Optional)::
   (time units) The period of time that forecast results are retained.
   After a forecast expires, the results are deleted. The default value is 14 days.
   If set to a value of `0`, the forecast is never automatically deleted.
   For more information about time units, see <<time-units>>.
 
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see {xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-forecast-example]]
+==== {api-examples-title}
 
 The following example requests a 10 day forecast for the `total-requests` job:
 
diff --git a/docs/reference/ml/apis/get-bucket.asciidoc b/docs/reference/ml/apis/get-bucket.asciidoc
index 39c548dd64e8d..2a73d0f5d3538 100644
--- a/docs/reference/ml/apis/get-bucket.asciidoc
+++ b/docs/reference/ml/apis/get-bucket.asciidoc
@@ -8,64 +8,75 @@
 
 Retrieves job results for one or more buckets.
 
-
-==== Request
+[[ml-get-bucket-request]]
+==== {api-request-title}
 
 `GET _ml/anomaly_detectors/<job_id>/results/buckets` +
 
 `GET _ml/anomaly_detectors/<job_id>/results/buckets/<timestamp>`
 
+[[ml-get-bucket-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `monitor_ml`,
+`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. You also
+need `read` index privilege on the index that stores the results. The
+`machine_learning_admin` and `machine_learning_user` roles provide these
+privileges. For more information, see
+{stack-ov}/security-privileges.html[Security privileges] and
+{stack-ov}/built-in-roles.html[Built-in roles].
 
-==== Description
+[[ml-get-bucket-desc]]
+==== {api-description-title}
 
 The get buckets API presents a chronological view of the records, grouped by
 bucket.
 
+[[ml-get-bucket-path-parms]]
+==== {api-path-parms-title}
 
-==== Path Parameters
-
-`job_id`::
+`<job_id>` (Required)::
   (string) Identifier for the job
 
-`timestamp`::
+`<timestamp>` (Optional)::
   (string) The timestamp of a single bucket result.
-  If you do not specify this optional parameter, the API returns information
+  If you do not specify this parameter, the API returns information
   about all buckets.
 
+[[ml-get-bucket-request-body]]
+==== {api-request-body-title}
 
-==== Request Body
-
-`anomaly_score`::
+`anomaly_score` (Optional)::
   (double) Returns buckets with anomaly scores greater or equal than this value.
 
-`desc`::
+`desc` (Optional)::
   (boolean) If true, the buckets are sorted in descending order.
 
-`end`::
+`end` (Optional)::
   (string) Returns buckets with timestamps earlier than this time.
 
-`exclude_interim`::
+`exclude_interim` (Optional)::
   (boolean) If true, the output excludes interim results.
   By default, interim results are included.
 
-`expand`::
+`expand` (Optional)::
   (boolean) If true, the output includes anomaly records.
 
-`page`::
+`page` (Optional)::
 `from`:::
   (integer) Skips the specified number of buckets.
 `size`:::
   (integer) Specifies the maximum number of buckets to obtain.
 
-`sort`::
+`sort` (Optional)::
   (string) Specifies the sort field for the requested buckets.
   By default, the buckets are sorted by the `timestamp` field.
 
-`start`::
+`start` (Optional)::
   (string) Returns buckets with timestamps after this time.
 
-
-===== Results
+[[ml-get-bucket-results]]
+==== {api-response-body-title}
 
 The API returns the following information:
 
@@ -73,18 +84,8 @@ The API returns the following information:
   (array) An array of bucket objects. For more information, see
   <<ml-results-buckets,Buckets>>.
 
-
-==== Authorization
-
-You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster
-privileges to use this API. You also need `read` index privilege on the index
-that stores the results. The `machine_learning_admin` and `machine_learning_user`
-roles provide these privileges. For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges] and
-{xpack-ref}/built-in-roles.html[Built-in Roles].
-
-
-==== Examples
+[[ml-get-bucket-example]]
+==== {api-examples-title}
 
 The following example gets bucket information for the `it-ops-kpi` job:
 
diff --git a/docs/reference/ml/apis/get-calendar-event.asciidoc b/docs/reference/ml/apis/get-calendar-event.asciidoc
index a890f67db0d23..173a249488684 100644
--- a/docs/reference/ml/apis/get-calendar-event.asciidoc
+++ b/docs/reference/ml/apis/get-calendar-event.asciidoc
@@ -9,39 +9,49 @@
 Retrieves information about the scheduled events in
 calendars.
 
-
-==== Request
+[[ml-get-calendar-event-request]]
+==== {api-request-title}
 
 `GET _ml/calendars/<calendar_id>/events` +
 
 `GET _ml/calendars/_all/events`
 
+[[ml-get-calendar-event-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `monitor_ml`,
+`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-===== Description
+[[ml-get-calendar-event-desc]]
+==== {api-description-title}
 
 You can get scheduled event information for a single calendar or for all
 calendars by using `_all`.
 
-==== Path Parameters
+[[ml-get-calendar-event-path-parms]]
+==== {api-path-parms-title}
 
-`calendar_id` (required)::
+`<calendar_id>` (Required)::
   (string) Identifier for the calendar.
 
-==== Request Body
+[[ml-get-calendar-event-request-body]]
+==== {api-request-body-title}
 
-`end`::
+`end` (Optional)::
     (string) Specifies to get events with timestamps earlier than this time.
 
-`from`::
+`from` (Optional)::
     (integer) Skips the specified number of events.
 
-`size`::
+`size` (Optional)::
     (integer) Specifies the maximum number of events to obtain.
 
-`start`::
+`start` (Optional)::
     (string) Specifies to get events with timestamps after this time.
 
-==== Results
+[[ml-get-calendar-event-results]]
+==== {api-response-body-title}
 
 The API returns the following information:
 
@@ -49,15 +59,8 @@ The API returns the following information:
   (array) An array of scheduled event resources.
   For more information, see <<ml-event-resource>>.
 
-
-==== Authorization
-
-You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster
-privileges to use this API. For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-get-calendar-event-example]]
+==== {api-examples-title}
 
 The following example gets information about the scheduled events in the
 `planned-outages` calendar:
diff --git a/docs/reference/ml/apis/get-calendar.asciidoc b/docs/reference/ml/apis/get-calendar.asciidoc
index 09e429b1f6de3..3d55f825bdb86 100644
--- a/docs/reference/ml/apis/get-calendar.asciidoc
+++ b/docs/reference/ml/apis/get-calendar.asciidoc
@@ -8,37 +8,44 @@
 
 Retrieves configuration information for calendars.
 
-
-==== Request
+[[ml-get-calendar-request]]
+==== {api-request-title}
 
 `GET _ml/calendars/<calendar_id>` +
 
 `GET _ml/calendars/_all`
 
+[[ml-get-calendar-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `monitor_ml`,
+`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-===== Description
+[[ml-get-calendar-desc]]
+==== {api-description-title}
 
 You can get information for a single calendar or for all calendars by using
 `_all`.
 
+[[ml-get-calendar-path-parms]]
+==== {api-path-parms-title}
 
-==== Path Parameters
-
-`calendar_id`::
+`<calendar_id>` (Required)::
   (string) Identifier for the calendar.
 
+[[ml-get-calendar-request-body]]
+==== {api-request-body-title}
 
-==== Request Body
-
-`page`::
+`page` (Optional)::
 `from`:::
     (integer) Skips the specified number of calendars.
 
-`size`:::
+`size` (Optional):::
     (integer) Specifies the maximum number of calendars to obtain.
 
-
-==== Results
+[[ml-get-calendar-results]]
+==== {api-response-body-title}
 
 The API returns the following information:
 
@@ -46,15 +53,8 @@ The API returns the following information:
   (array) An array of calendar resources.
   For more information, see <<ml-calendar-resource>>.
 
-
-==== Authorization
-
-You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster
-privileges to use this API. For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-get-calendar-example]]
+==== {api-examples-title}
 
 The following example gets configuration information for the `planned-outages`
 calendar:
diff --git a/docs/reference/ml/apis/get-category.asciidoc b/docs/reference/ml/apis/get-category.asciidoc
index 1fbfda20eccc0..6301eaf13a538 100644
--- a/docs/reference/ml/apis/get-category.asciidoc
+++ b/docs/reference/ml/apis/get-category.asciidoc
@@ -8,38 +8,50 @@
 
 Retrieves job results for one or more categories.
 
-
-==== Request
+[[ml-get-category-request]]
+==== {api-request-title}
 
 `GET _ml/anomaly_detectors/<job_id>/results/categories` +
 
 `GET _ml/anomaly_detectors/<job_id>/results/categories/<category_id>`
 
-==== Description
+[[ml-get-category-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `monitor_ml`,
+`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. You also
+need `read` index privilege on the index that stores the results. The
+`machine_learning_admin` and `machine_learning_user` roles provide these
+privileges. See {stack-ov}/security-privileges.html[Security privileges] and
+{stack-ov}/built-in-roles.html[Built-in roles].
+
+[[ml-get-category-desc]]
+==== {api-description-title}
 
 For more information about categories, see
-{xpack-ref}/ml-configuring-categories.html[Categorizing Log Messages].
+{stack-ov}/ml-configuring-categories.html[Categorizing log messages].
 
-==== Path Parameters
+[[ml-get-category-path-parms]]
+==== {api-path-parms-title}
 
-`job_id`::
+`<job_id>` (Required)::
   (string) Identifier for the job.
 
-`category_id`::
-  (long) Identifier for the category. If you do not specify this optional parameter,
+`<category_id>` (Optional)::
+  (long) Identifier for the category. If you do not specify this parameter,
   the API returns information about all categories in the job.
 
+[[ml-get-category-request-body]]
+==== {api-request-body-title}
 
-==== Request Body
-
-`page`::
+`page` (Optional)::
 `from`:::
   (integer) Skips the specified number of categories.
 `size`:::
   (integer) Specifies the maximum number of categories to obtain.
 
-
-==== Results
+[[ml-get-category-results]]
+==== {api-response-body-title}
 
 The API returns the following information:
 
@@ -47,18 +59,8 @@ The API returns the following information:
   (array) An array of category objects. For more information, see
   <<ml-results-categories,Categories>>.
 
-
-==== Authorization
-
-You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster
-privileges to use this API. You also need `read` index privilege on the index
-that stores the results. The `machine_learning_admin` and `machine_learning_user`
-roles provide these privileges. For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges] and
-{xpack-ref}/built-in-roles.html[Built-in Roles].
-
-
-==== Examples
+[[ml-get-category-example]]
+==== {api-examples-title}
 
 The following example gets information about one category for the
 `esxi_log` job:
diff --git a/docs/reference/ml/apis/get-datafeed-stats.asciidoc b/docs/reference/ml/apis/get-datafeed-stats.asciidoc
index d1b842509b408..6ce99785912af 100644
--- a/docs/reference/ml/apis/get-datafeed-stats.asciidoc
+++ b/docs/reference/ml/apis/get-datafeed-stats.asciidoc
@@ -10,9 +10,8 @@
 
 Retrieves usage information for {dfeeds}.
 
-
-==== Request
-
+[[ml-get-datafeed-stats-request]]
+==== {api-request-title}
 
 `GET _ml/datafeeds/<feed_id>/_stats` +
 
@@ -20,11 +19,17 @@ Retrieves usage information for {dfeeds}.
 
 `GET _ml/datafeeds/_stats`  +
 
-`GET _ml/datafeeds/_all/_stats` +
+`GET _ml/datafeeds/_all/_stats` 
 
+[[ml-get-datafeed-stats-prereqs]]
+==== {api-prereq-title}
 
+* If the {es} {security-features} are enabled, you must have `monitor_ml`,
+`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-==== Description
+[[ml-get-datafeed-stats-desc]]
+==== {api-description-title}
 
 You can get statistics for multiple {dfeeds} in a single API request by using a
 comma-separated list of {dfeeds} or a wildcard expression. You can get
@@ -36,15 +41,16 @@ If the {dfeed} is stopped, the only information you receive is the
 
 IMPORTANT: This API returns a maximum of 10,000 {dfeeds}.
 
-==== Path Parameters
+[[ml-get-datafeed-stats-path-parms]]
+==== {api-path-parms-title}
 
-`feed_id`::
+`<feed_id>` (Optional)::
   (string) Identifier for the {dfeed}. It can be a {dfeed} identifier or a
   wildcard expression. If you do not specify one of these options, the API
   returns statistics for all {dfeeds}.
 
-
-==== Results
+[[ml-get-datafeed-stats-results]]
+==== {api-response-body-title}
 
 The API returns the following information:
 
@@ -52,15 +58,8 @@ The API returns the following information:
   (array) An array of {dfeed} count objects.
   For more information, see <<ml-datafeed-counts>>.
 
-
-==== Authorization
-
-You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster
-privileges to use this API. For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-get-datafeed-stats-example]]
+==== {api-examples-title}
 
 The following example gets usage information for the
 `datafeed-total-requests` {dfeed}:
diff --git a/docs/reference/ml/apis/get-datafeed.asciidoc b/docs/reference/ml/apis/get-datafeed.asciidoc
index 2c9ef7e3aec74..abc79ae5c7d71 100644
--- a/docs/reference/ml/apis/get-datafeed.asciidoc
+++ b/docs/reference/ml/apis/get-datafeed.asciidoc
@@ -10,8 +10,8 @@
 
 Retrieves configuration information for {dfeeds}.
 
-==== Request
-
+[[ml-get-datafeed-request]]
+==== {api-request-title}
 
 `GET _ml/datafeeds/<feed_id>` +
 
@@ -19,10 +19,17 @@ Retrieves configuration information for {dfeeds}.
 
 `GET _ml/datafeeds/` +
 
-`GET _ml/datafeeds/_all` +
+`GET _ml/datafeeds/_all` 
+
+[[ml-get-datafeed-prereqs]]
+==== {api-prereq-title}
 
+* If the {es} {security-features} are enabled, you must have `monitor_ml`,
+`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-===== Description
+[[ml-get-datafeed-desc]]
+==== {api-description-title}
 
 You can get information for multiple {dfeeds} in a single API request by using a
 comma-separated list of {dfeeds} or a wildcard expression. You can get
@@ -31,15 +38,16 @@ information for all {dfeeds} by using `_all`, by specifying `*` as the
 
 IMPORTANT: This API returns a maximum of 10,000 {dfeeds}. 
 
-==== Path Parameters
+[[ml-get-datafeed-path-parms]]
+==== {api-path-parms-title}
 
-`feed_id`::
+`<feed_id>` (Optional)::
   (string) Identifier for the {dfeed}. It can be a {dfeed} identifier or a
   wildcard expression. If you do not specify one of these options, the API
   returns information about all {dfeeds}.
 
-
-==== Results
+[[ml-get-datafeed-results]]
+==== {api-response-body-title}
 
 The API returns the following information:
 
@@ -47,15 +55,8 @@ The API returns the following information:
   (array) An array of {dfeed} objects.
   For more information, see <<ml-datafeed-resource>>.
 
-
-==== Authorization
-
-You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster
-privileges to use this API. For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-get-datafeed-example]]
+==== {api-examples-title}
 
 The following example gets configuration information for the
 `datafeed-total-requests` {dfeed}:
diff --git a/docs/reference/ml/apis/get-filter.asciidoc b/docs/reference/ml/apis/get-filter.asciidoc
index 53dfab2530a10..ad5fee343f6d5 100644
--- a/docs/reference/ml/apis/get-filter.asciidoc
+++ b/docs/reference/ml/apis/get-filter.asciidoc
@@ -8,36 +8,43 @@
 
 Retrieves filters.
 
-
-==== Request
+[[ml-get-filter-request]]
+==== {api-request-title}
 
 `GET _ml/filters/<filter_id>` +
 
 `GET _ml/filters/`
 
+[[ml-get-filter-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `monitor_ml`,
+`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-===== Description
+[[ml-get-filter-desc]]
+==== {api-description-title}
 
 You can get a single filter or all filters. For more information, see 
 {stack-ov}/ml-rules.html[Machine learning custom rules].
 
+[[ml-get-filter-path-parms]]
+==== {api-path-parms-title}
 
-==== Path Parameters
-
-`filter_id`::
+`<filter_id>` (Optional)::
   (string) Identifier for the filter.
 
+[[ml-get-filter-query-parms]]
+==== {api-query-parms-title}
 
-==== Querystring Parameters
-
-`from`:::
+`from` (Optional):::
     (integer) Skips the specified number of filters.
 
-`size`:::
+`size` (Optional):::
     (integer) Specifies the maximum number of filters to obtain.
 
-
-==== Results
+[[ml-get-filter-results]]
+==== {api-response-body-title}
 
 The API returns the following information:
 
@@ -45,15 +52,8 @@ The API returns the following information:
   (array) An array of filter resources.
   For more information, see <<ml-filter-resource>>.
 
-
-==== Authorization
-
-You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster
-privileges to use this API. For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-get-filter-example]]
+==== {api-examples-title}
 
 The following example gets configuration information for the `safe_domains`
 filter:
diff --git a/docs/reference/ml/apis/get-influencer.asciidoc b/docs/reference/ml/apis/get-influencer.asciidoc
index 7425a734ed441..8d7ca889a264f 100644
--- a/docs/reference/ml/apis/get-influencer.asciidoc
+++ b/docs/reference/ml/apis/get-influencer.asciidoc
@@ -8,48 +8,58 @@
 
 Retrieves job results for one or more influencers.
 
-
-==== Request
+[[ml-get-influencer-request]]
+==== {api-request-title}
 
 `GET _ml/anomaly_detectors/<job_id>/results/influencers`
 
-//===== Description
+[[ml-get-influencer-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `monitor_ml`,
+`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. You also
+need `read` index privilege on the index that stores the results. The
+`machine_learning_admin` and `machine_learning_user` roles provide these
+privileges. See {stack-ov}/security-privileges.html[Security privileges] and
+{stack-ov}/built-in-roles.html[Built-in roles].
 
-==== Path Parameters
+[[ml-get-influencer-path-parms]]
+==== {api-path-parms-title}
 
-`job_id`::
+`<job_id>` (Required)::
   (string) Identifier for the job.
 
-==== Request Body
+[[ml-get-influencer-request-body]]
+==== {api-request-body-title}
 
-`desc`::
+`desc` (Optional)::
   (boolean) If true, the results are sorted in descending order.
 
-`end`::
+`end` (Optional)::
   (string) Returns influencers with timestamps earlier than this time.
 
-`exclude_interim`::
+`exclude_interim` (Optional)::
   (boolean) If true, the output excludes interim results.
   By default, interim results are included.
 
-`influencer_score`::
+`influencer_score` (Optional)::
   (double) Returns influencers with anomaly scores greater or equal than this value.
 
-`page`::
+`page` (Optional)::
 `from`:::
     (integer) Skips the specified number of influencers.
 `size`:::
   (integer) Specifies the maximum number of influencers to obtain.
 
-`sort`::
+`sort` (Optional)::
   (string) Specifies the sort field for the requested influencers.
   By default the influencers are sorted by the `influencer_score` value.
 
-`start`::
+`start` (Optional)::
   (string) Returns influencers with timestamps after this time.
 
-
-==== Results
+[[ml-get-influencer-results]]
+==== {api-response-body-title}
 
 The API returns the following information:
 
@@ -57,19 +67,8 @@ The API returns the following information:
   (array) An array of influencer objects.
   For more information, see <<ml-results-influencers,Influencers>>.
 
-
-==== Authorization
-
-You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster
-privileges to use this API. You also need `read` index privilege on the index
-that stores the results. The `machine_learning_admin` and `machine_learning_user`
-roles provide these privileges. For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges] and
-{xpack-ref}/built-in-roles.html[Built-in Roles].
-//<<security-privileges>> and <<built-in-roles>>.
-
-
-==== Examples
+[[ml-get-influencer-example]]
+==== {api-examples-title}
 
 The following example gets influencer information for the `it_ops_new_kpi` job:
 
diff --git a/docs/reference/ml/apis/get-job-stats.asciidoc b/docs/reference/ml/apis/get-job-stats.asciidoc
index b674b01802bee..8a705d7ff9ed1 100644
--- a/docs/reference/ml/apis/get-job-stats.asciidoc
+++ b/docs/reference/ml/apis/get-job-stats.asciidoc
@@ -8,10 +8,8 @@
 
 Retrieves usage information for jobs.
 
-
-==== Request
-
-
+[[ml-get-job-stats-request]]
+==== {api-request-title}
 
 `GET _ml/anomaly_detectors/<job_id>/_stats`
 
@@ -19,10 +17,17 @@ Retrieves usage information for jobs.
 
 `GET _ml/anomaly_detectors/_stats` +
 
-`GET _ml/anomaly_detectors/_all/_stats` +
+`GET _ml/anomaly_detectors/_all/_stats` 
 
+[[ml-get-job-stats-prereqs]]
+==== {api-prereq-title}
 
-===== Description
+* If the {es} {security-features} are enabled, you must have `monitor_ml`,
+`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
+
+[[ml-get-job-stats-desc]]
+==== {api-description-title}
 
 You can get statistics for multiple jobs in a single API request by using a
 group name, a comma-separated list of jobs, or a wildcard expression. You can
@@ -31,16 +36,16 @@ get statistics for all jobs by using `_all`, by specifying `*` as the
 
 IMPORTANT: This API returns a maximum of 10,000 jobs.
 
+[[ml-get-job-stats-path-parms]]
+==== {api-path-parms-title}
 
-==== Path Parameters
-
-`job_id`::
+`<job_id>` (Optional)::
   (string) An identifier for the job. It can be a job identifier, a group name,
   or a wildcard expression. If you do not specify one of these options, the API
   returns statistics for all jobs.
 
-
-==== Results
+[[ml-get-job-stats-results]]
+==== {api-response-body-title}
 
 The API returns the following information:
 
@@ -48,15 +53,8 @@ The API returns the following information:
   (array) An array of job statistics objects.
   For more information, see <<ml-jobstats,Job Statistics>>.
 
-
-==== Authorization
-
-You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster
-privileges to use this API. For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-get-job-stats-example]]
+==== {api-examples-title}
 
 The following example gets usage information for the `farequote` job:
 
@@ -105,7 +103,14 @@ The API returns the following results:
         "log_time": 1491948163000,
         "timestamp": 1455234600000
       },
-      "state": "closed"
+      "state": "closed",
+      "timing_stats": {
+        "job_id": "farequote",
+        "minimum_bucket_processing_time_ms": 0.0,
+        "maximum_bucket_processing_time_ms": 15.0,
+        "average_bucket_processing_time_ms": 8.75,
+        "exponential_average_bucket_processing_time_ms": 6.1435899
+      }
     }
   ]
 }
diff --git a/docs/reference/ml/apis/get-job.asciidoc b/docs/reference/ml/apis/get-job.asciidoc
index 4eb7eaf5a7f07..176ca09fc56c5 100644
--- a/docs/reference/ml/apis/get-job.asciidoc
+++ b/docs/reference/ml/apis/get-job.asciidoc
@@ -8,8 +8,8 @@
 
 Retrieves configuration information for jobs.
 
-
-==== Request
+[[ml-get-job-request]]
+==== {api-request-title}
 
 `GET _ml/anomaly_detectors/<job_id>` +
 
@@ -19,8 +19,15 @@ Retrieves configuration information for jobs.
 
 `GET _ml/anomaly_detectors/_all`
 
+[[ml-get-job-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `monitor_ml`,
+`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-===== Description
+[[ml-get-job-desc]]
+==== {api-description-title}
 
 You can get information for multiple jobs in a single API request by using a
 group name, a comma-separated list of jobs, or a wildcard expression. You can
@@ -29,15 +36,16 @@ get information for all jobs by using `_all`, by specifying `*` as the
 
 IMPORTANT: This API returns a maximum of 10,000 jobs. 
 
+[[ml-get-job-path-parms]]
+==== {api-path-parms-title}
 
-==== Path Parameters
-
-`job_id`::
+`<job_id> (Optional)`::
   (string) Identifier for the job. It can be a job identifier, a group name,
   or a wildcard expression. If you do not specify one of these options, the API
   returns information for all jobs.
 
-==== Results
+[[ml-get-job-results]]
+==== {api-response-body-title}
 
 The API returns the following information:
 
@@ -45,15 +53,8 @@ The API returns the following information:
   (array) An array of job resources.
   For more information, see <<ml-job-resource,Job Resources>>.
 
-
-==== Authorization
-
-You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster
-privileges to use this API. For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-get-job-example]]
+==== {api-examples-title}
 
 The following example gets configuration information for the `total-requests` job:
 
diff --git a/docs/reference/ml/apis/get-ml-info.asciidoc b/docs/reference/ml/apis/get-ml-info.asciidoc
index 00c57c8cf7f5b..2c486741ffd42 100644
--- a/docs/reference/ml/apis/get-ml-info.asciidoc
+++ b/docs/reference/ml/apis/get-ml-info.asciidoc
@@ -10,28 +10,30 @@
 
 Returns defaults and limits used by machine learning.
 
-==== Request
+[[get-ml-info-request]]
+==== {api-request-title}
 
 `GET _ml/info`
 
-==== Description
+[[get-ml-info-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `monitor_ml`,
+`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. The
+`machine_learning_admin` and `machine_learning_user` roles provide these
+privileges. See {stack-ov}/security-privileges.html[Security privileges] and
+{stack-ov}/built-in-roles.html[Built-in roles].
+
+[[get-ml-info-desc]]
+==== {api-description-title}
 
 This endpoint is designed to be used by a user interface that needs to fully
 understand machine learning configurations where some options are not specified,
 meaning that the defaults should be used.  This endpoint may be used to find out
 what those defaults are.
 
-
-==== Authorization
-
-You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster
-privileges to use this API.  The `machine_learning_admin` and `machine_learning_user`
-roles provide these privileges. For more information, see
-{stack-ov}/security-privileges.html[Security Privileges] and
-{stack-ov}/built-in-roles.html[Built-in Roles].
-
-
-==== Examples
+[[get-ml-info-example]]
+==== {api-examples-title}
 
 The endpoint takes no arguments:
 
@@ -57,7 +59,13 @@ This is a possible response:
     }
   },
   "upgrade_mode": false,
+  "native_code" : {
+    "version": "7.0.0",
+    "build_hash": "99a07c016d5a73"
+  },
   "limits" : { }
 }
 ----
 // TESTRESPONSE[s/"upgrade_mode": false/"upgrade_mode": $body.upgrade_mode/]
+// TESTRESPONSE[s/"version": "7.0.0",/"version": "$body.native_code.version",/]
+// TESTRESPONSE[s/"build_hash": "99a07c016d5a73"/"build_hash": "$body.native_code.build_hash"/]
diff --git a/docs/reference/ml/apis/get-overall-buckets.asciidoc b/docs/reference/ml/apis/get-overall-buckets.asciidoc
index d8592e6516bbb..4d8287f9a54f7 100644
--- a/docs/reference/ml/apis/get-overall-buckets.asciidoc
+++ b/docs/reference/ml/apis/get-overall-buckets.asciidoc
@@ -9,7 +9,8 @@
 Retrieves overall bucket results that summarize the
 bucket results of multiple jobs.
 
-==== Request
+[[ml-get-overall-buckets-request]]
+==== {api-request-title}
 
 `GET _ml/anomaly_detectors/<job_id>/results/overall_buckets` +
 
@@ -17,7 +18,18 @@ bucket results of multiple jobs.
 
 `GET _ml/anomaly_detectors/_all/results/overall_buckets`
 
-==== Description
+[[ml-get-overall-buckets-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `monitor_ml`,
+`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. You also
+need `read` index privilege on the index that stores the results. The
+`machine_learning_admin` and `machine_learning_user` roles provide these
+privileges. See {stack-ov}/security-privileges.html[Security privileges] and
+{stack-ov}/built-in-roles.html[Built-in roles].
+
+[[ml-get-overall-buckets-desc]]
+==== {api-description-title}
 
 You can summarize the bucket results for all jobs by using `_all` or by
 specifying `*` as the `<job_id>`.
@@ -41,43 +53,46 @@ to request overall buckets that span longer than the largest job's `bucket_span`
 When set, the `overall_score` will be the max `overall_score` of the corresponding
 overall buckets with a span equal to the largest job's `bucket_span`.
 
-==== Path Parameters
+[[ml-get-overall-buckets-path-parms]]
+==== {api-path-parms-title}
 
-`job_id`::
+`<job_id>` (Required)::
   (string) Identifier for the job. It can be a job identifier, a group name, a
   comma-separated list of jobs or groups, or a wildcard expression.
 
-==== Request Body
+[[ml-get-overall-buckets-request-body]]
+==== {api-request-body-title}
 
-`allow_no_jobs`::
+`allow_no_jobs` (Optional)::
   (boolean) If `false` and the `job_id` does not match any job an error will
   be returned. The default value is `true`.
 
-`bucket_span`::
+`bucket_span` (Optional)::
   (string) The span of the overall buckets. Must be greater or equal
   to the largest job's `bucket_span`. Defaults to the largest job's `bucket_span`.
 
-`end`::
+`end` (Optional)::
   (string) Returns overall buckets with timestamps earlier than this time.
 
-`exclude_interim`::
+`exclude_interim` (Optional)::
   (boolean) If `true`, the output excludes interim overall buckets.
   Overall buckets are interim if any of the job buckets within
   the overall bucket interval are interim.
   By default, interim results are included.
 
-`overall_score`::
-  (double) Returns overall buckets with overall scores greater or equal than this value.
+`overall_score` (Optional)::
+  (double) Returns overall buckets with overall scores greater or equal than
+  this value.
 
-`start`::
+`start` (Optional)::
   (string) Returns overall buckets with timestamps after this time.
 
-`top_n`::
+`top_n` (Optional)::
   (integer) The number of top job bucket scores to be used in the
   `overall_score` calculation. The default value is `1`.
 
-
-===== Results
+[[ml-get-overall-buckets-results]]
+==== {api-response-body-title}
 
 The API returns the following information:
 
@@ -85,18 +100,8 @@ The API returns the following information:
   (array) An array of overall bucket objects. For more information, see
   <<ml-results-overall-buckets,Overall Buckets>>.
 
-
-==== Authorization
-
-You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster
-privileges to use this API. You also need `read` index privilege on the index
-that stores the results. The `machine_learning_admin` and `machine_learning_user`
-roles provide these privileges. For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges] and
-{xpack-ref}/built-in-roles.html[Built-in Roles].
-
-
-==== Examples
+[[ml-get-overall-buckets-example]]
+==== {api-examples-title}
 
 The following example gets overall buckets for jobs with IDs matching `job-*`:
 
diff --git a/docs/reference/ml/apis/get-record.asciidoc b/docs/reference/ml/apis/get-record.asciidoc
index afc7d2733c872..0acc3e0e49fce 100644
--- a/docs/reference/ml/apis/get-record.asciidoc
+++ b/docs/reference/ml/apis/get-record.asciidoc
@@ -8,49 +8,58 @@
 
 Retrieves anomaly records for a job.
 
-
-==== Request
+[[ml-get-record-request]]
+==== {api-request-title}
 
 `GET _ml/anomaly_detectors/<job_id>/results/records`
 
-//===== Description
+[[ml-get-record-prereqs]]
+==== {api-prereq-title}
 
-==== Path Parameters
+* If the {es} {security-features} are enabled, you must have `monitor_ml`,
+`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. You also
+need `read` index privilege on the index that stores the results. The
+`machine_learning_admin` and `machine_learning_user` roles provide these
+privileges. See {stack-ov}/security-privileges.html[Security privileges] and
+{stack-ov}/built-in-roles.html[Built-in roles].
 
-`job_id`::
-  (string) Identifier for the job.
+[[ml-get-record-path-parms]]
+==== {api-path-parms-title}
 
+`job_id` (Required)::
+  (string) Identifier for the job.
 
-==== Request Body
+[[ml-get-record-request-body]]
+==== {api-request-body-title}
 
-`desc`::
+`desc` (Optional)::
   (boolean) If true, the results are sorted in descending order.
 
-`end`::
+`end` (Optional)::
   (string) Returns records with timestamps earlier than this time.
 
-`exclude_interim`::
+`exclude_interim` (Optional)::
   (boolean) If true, the output excludes interim results.
   By default, interim results are included.
 
-`page`::
+`page` (Optional)::
 `from`:::
   (integer) Skips the specified number of records.
 `size`:::
   (integer) Specifies the maximum number of records to obtain.
 
-`record_score`::
+`record_score` (Optional)::
   (double) Returns records with anomaly scores greater or equal than this value.
 
-`sort`::
+`sort` (Optional)::
   (string) Specifies the sort field for the requested records.
   By default, the records are sorted by the `anomaly_score` value.
 
-`start`::
+`start` (Optional)::
   (string) Returns records with timestamps after this time.
 
-
-==== Results
+[[ml-get-record-results]]
+==== {api-response-body-title}
 
 The API returns the following information:
 
@@ -58,19 +67,8 @@ The API returns the following information:
   (array) An array of record objects. For more information, see
   <<ml-results-records,Records>>.
 
-
-==== Authorization
-
-You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster
-privileges to use this API. You also need `read` index privilege on the index
-that stores the results. The `machine_learning_admin` and `machine_learning_user`
-roles provide these privileges. For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges] and
-{xpack-ref}/built-in-roles.html[Built-in Roles].
-//<<security-privileges>> and <<built-in-roles>>.
-
-
-==== Examples
+[[ml-get-record-example]]
+==== {api-examples-title}
 
 The following example gets record information for the `it-ops-kpi` job:
 
diff --git a/docs/reference/ml/apis/get-snapshot.asciidoc b/docs/reference/ml/apis/get-snapshot.asciidoc
index 4935a6e2d238f..ea1b15df33f33 100644
--- a/docs/reference/ml/apis/get-snapshot.asciidoc
+++ b/docs/reference/ml/apis/get-snapshot.asciidoc
@@ -8,47 +8,54 @@
 
 Retrieves information about model snapshots.
 
-
-==== Request
+[[ml-get-snapshot-request]]
+==== {api-request-title}
 
 `GET _ml/anomaly_detectors/<job_id>/model_snapshots` +
 
 `GET _ml/anomaly_detectors/<job_id>/model_snapshots/<snapshot_id>`
 
-//===== Description
+[[ml-get-snapshot-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `monitor_ml`,
+`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-==== Path Parameters
+[[ml-get-snapshot-path-parms]]
+==== {api-path-parms-title}
 
-`job_id`::
+`<job_id>` (Required)::
   (string) Identifier for the job.
 
-`snapshot_id`::
+`<snapshot_id>` (Optional)::
   (string) Identifier for the model snapshot. If you do not specify this
   optional parameter, the API returns information about all model snapshots.
 
-==== Request Body
+[[ml-get-snapshot-request-body]]
+==== {api-request-body-title}
 
-`desc`::
+`desc` (Optional)::
   (boolean) If true, the results are sorted in descending order.
 
-`end`::
+`end` (Optional)::
   (date) Returns snapshots with timestamps earlier than this time.
 
-`from`::
+`from` (Optional)::
   (integer) Skips the specified number of snapshots.
 
-`size`::
+`size` (Optional)::
   (integer) Specifies the maximum number of snapshots to obtain.
 
-`sort`::
+`sort` (Optional)::
   (string) Specifies the sort field for the requested snapshots.
   By default, the snapshots are sorted by their timestamp.
 
-`start`::
+`start` (Optional)::
   (string) Returns snapshots with timestamps after this time.
 
-
-==== Results
+[[ml-get-snapshot-results]]
+==== {api-response-body-title}
 
 The API returns the following information:
 
@@ -56,16 +63,8 @@ The API returns the following information:
   (array) An array of model snapshot objects. For more information, see
   <<ml-snapshot-resource,Model Snapshots>>.
 
-
-==== Authorization
-
-You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster
-privileges to use this API. For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-//<<privileges-list-cluster>>.
-
-
-==== Examples
+[[ml-get-snapshot-example]]
+==== {api-examples-title}
 
 The following example gets model snapshot information for the
 `it_ops_new_logs` job:
diff --git a/docs/reference/ml/apis/jobcounts.asciidoc b/docs/reference/ml/apis/jobcounts.asciidoc
index ed69a8b0a796b..3b92224034b60 100644
--- a/docs/reference/ml/apis/jobcounts.asciidoc
+++ b/docs/reference/ml/apis/jobcounts.asciidoc
@@ -19,11 +19,15 @@ progress of a job.
 
 `model_size_stats`::
   (object) An object that provides information about the size and contents of the model.
-  See <<ml-modelsizestats,model size stats objects>>
+  See <<ml-modelsizestats,model size stats objects>>.
 
 `forecasts_stats`::
   (object) An object that provides statistical information about forecasts
-  of this job. See <<ml-forecastsstats, forecasts stats objects>>
+  of this job. See <<ml-forecastsstats, forecasts stats objects>>.
+
+`timing_stats`::
+  (object) An object that provides statistical information about timing aspect
+  of this job. See <<ml-timingstats, timing stats objects>>.
 
 `node`::
   (object) For open jobs only, contains information about the node where the
@@ -209,6 +213,31 @@ The `forecasts_stats` object shows statistics about forecasts. It has the follow
 NOTE: `memory_bytes`, `records`, `processing_time_ms` and `status` require at least 1 forecast, otherwise
 these fields are omitted.
 
+[float]
+[[ml-timingstats]]
+==== Timing Stats Objects
+
+The `timing_stats` object shows timing-related statistics about the job's progress. It has the following properties:
+
+`job_id`::
+  (string) A numerical character string that uniquely identifies the job.
+
+`bucket_count`::
+  (long) The number of buckets processed.
+
+`minimum_bucket_processing_time_ms`::
+  (double) Minimum among all bucket processing times in milliseconds.
+
+`maximum_bucket_processing_time_ms`::
+  (double) Maximum among all bucket processing times in milliseconds.
+
+`average_bucket_processing_time_ms`::
+  (double) Average of all bucket processing times in milliseconds.
+
+`exponential_average_bucket_processing_time_ms`::
+  (double) Exponential moving average of all bucket processing times in milliseconds.
+
+
 [float]
 [[ml-stats-node]]
 ==== Node Objects
diff --git a/docs/reference/ml/apis/open-job.asciidoc b/docs/reference/ml/apis/open-job.asciidoc
index 08c7b97d9c050..84000cb89b0de 100644
--- a/docs/reference/ml/apis/open-job.asciidoc
+++ b/docs/reference/ml/apis/open-job.asciidoc
@@ -10,41 +10,42 @@ Opens one or more jobs.
 A job must be opened in order for it to be ready to receive and analyze data.
 A job can be opened and closed multiple times throughout its lifecycle.
 
-
-==== Request
+[[ml-open-job-request]]
+==== {api-request-title}
 
 `POST _ml/anomaly_detectors/{job_id}/_open`
 
+[[ml-open-job-prereqs]]
+==== {api-prereq-title}
 
-==== Description
-
-When you open a new job, it starts with an empty model.
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-When you open an existing job, the most recent model state is automatically loaded.
-The job is ready to resume its analysis from where it left off, once new data is received.
+[[ml-open-job-desc]]
+==== {api-description-title}
 
+When you open a new job, it starts with an empty model.
 
-==== Path Parameters
+When you open an existing job, the most recent model state is automatically
+loaded. The job is ready to resume its analysis from where it left off, once new
+data is received.
 
-`job_id` (required)::
-(string) Identifier for the job
+[[ml-open-job-path-parms]]
+==== {api-path-parms-title}
 
+`<job_id>` (Required)::
+  (string) Identifier for the job
 
-==== Request Body
+[[ml-open-job-request-body]]
+==== {api-request-body-title}
 
-`timeout`::
+`timeout` (Optional)::
   (time) Controls the time to wait until a job has opened.
   The default value is 30 minutes.
 
-
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-open-job-example]]
+==== {api-examples-title}
 
 The following example opens the `total-requests` job and sets an optional
 property:
diff --git a/docs/reference/ml/apis/post-calendar-event.asciidoc b/docs/reference/ml/apis/post-calendar-event.asciidoc
index 5d122a5d6d1a8..88d771f3b7f18 100644
--- a/docs/reference/ml/apis/post-calendar-event.asciidoc
+++ b/docs/reference/ml/apis/post-calendar-event.asciidoc
@@ -8,38 +8,40 @@
 
 Posts scheduled events in a calendar.
 
-==== Request
+[[ml-post-calendar-event-request]]
+==== {api-request-title}
 
 `POST _ml/calendars/<calendar_id>/events`
 
+[[ml-post-calendar-event-prereqs]]
+==== {api-prereq-title}
 
-==== Description
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-This API accepts a list of {xpack-ref}/ml-calendars.html[scheduled events], each
+[[ml-post-calendar-event-desc]]
+==== {api-description-title}
+
+This API accepts a list of {stack-ov}/ml-calendars.html[scheduled events], each
 of which must have a start time, end time, and description.
 
-==== Path Parameters
+[[ml-post-calendar-event-path-parms]]
+==== {api-path-parms-title}
 
-`calendar_id` (required)::
+`<calendar_id>` (Required)::
 		(string) Identifier for the calendar.
 
+[[ml-post-calendar-event-request-body]]
+==== {api-request-body-title}
 
-==== Request Body
-
-`events`::
-  (array) A list of one of more scheduled events. The event's start and end times
-  may be specified as integer milliseconds since the epoch or as a string in ISO 8601
-  format. See <<ml-event-resource>>.
-
-
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
+`events` (Required)::
+  (array) A list of one of more scheduled events. The event's start and end
+	times may be specified as integer milliseconds since the epoch or as a string
+	in ISO 8601 format. See <<ml-event-resource>>.
 
-==== Examples
+[[ml-post-calendar-event-example]]
+==== {api-examples-title}
 
 You can add scheduled events to the `planned-outages` calendar as follows:
 
diff --git a/docs/reference/ml/apis/post-data.asciidoc b/docs/reference/ml/apis/post-data.asciidoc
index 2df0df69e9030..3c2d0e49fde93 100644
--- a/docs/reference/ml/apis/post-data.asciidoc
+++ b/docs/reference/ml/apis/post-data.asciidoc
@@ -8,13 +8,20 @@
 
 Sends data to an anomaly detection job for analysis.
 
-
-==== Request
+[[ml-post-data-request]]
+==== {api-request-title}
 
 `POST _ml/anomaly_detectors/<job_id>/_data`
 
+[[ml-post-data-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-==== Description
+[[ml-post-data-desc]]
+==== {api-description-title}
 
 The job must have a state of `open` to receive and process the data.
 
@@ -42,39 +49,32 @@ IMPORTANT:  For each job, data can only be accepted from a single connection at
 a time. It is not currently possible to post data to multiple jobs using wildcards
 or a comma-separated list.
 
+[[ml-post-data-path-parms]]
+==== {api-path-parms-title}
 
-==== Path Parameters
-
-`job_id` (required)::
-		(string) Identifier for the job
-
-
-==== Query Parameters
+`<job_id>` (Required)::
+		(string) Identifier for the job.
 
-`reset_start`::
-		(string) Specifies the start of the bucket resetting range
+[[ml-post-data-query-parms]]
+==== {api-query-parms-title}
 
-`reset_end`::
-		(string) Specifies the end of the bucket resetting range
+`reset_start` (Optional)::
+		(string) Specifies the start of the bucket resetting range.
 
+`reset_end` (Optional)::
+		(string) Specifies the end of the bucket resetting range.
 
-==== Request Body
+[[ml-post-data-request-body]]
+==== {api-request-body-title}
 
 A sequence of one or more JSON documents containing the data to be analyzed.
 Only whitespace characters are permitted in between the documents.
 
+[[ml-post-data-example]]
+==== {api-examples-title}
 
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-//<<privileges-list-cluster>>.
-
-
-==== Examples
-
-The following example posts data from the it_ops_new_kpi.json file to the `it_ops_new_kpi` job:
+The following example posts data from the `it_ops_new_kpi.json` file to the
+`it_ops_new_kpi` job:
 
 [source,js]
 --------------------------------------------------
@@ -83,8 +83,8 @@ $ curl -s -H "Content-type: application/json"
 --data-binary @it_ops_new_kpi.json
 --------------------------------------------------
 
-When the data is sent, you receive information about the operational progress of the job.
-For example:
+When the data is sent, you receive information about the operational progress of
+the job. For example:
 
 [source,js]
 ----
diff --git a/docs/reference/ml/apis/preview-datafeed.asciidoc b/docs/reference/ml/apis/preview-datafeed.asciidoc
index 83af6a78057cc..4ca3ebcd10e40 100644
--- a/docs/reference/ml/apis/preview-datafeed.asciidoc
+++ b/docs/reference/ml/apis/preview-datafeed.asciidoc
@@ -10,45 +10,41 @@
 
 Previews a {dfeed}.
 
-
-==== Request
+[[ml-preview-datafeed-request]]
+==== {api-request-title}
 
 `GET _ml/datafeeds/<datafeed_id>/_preview`
 
+[[ml-preview-datafeed-prereqs]]
+==== {api-prereq-title}
+
+* If {es} {security-features} are enabled, you must have `monitor_ml`, `monitor`,
+`manage_ml`, or `manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-==== Description
+[[ml-preview-datafeed-desc]]
+==== {api-description-title}
 
 The preview {dfeeds} API returns the first "page" of results from the `search`
 that is created by using the current {dfeed} settings. This preview shows the
 structure of the data that will be passed to the anomaly detection engine.
 
-
-==== Path Parameters
-
-`datafeed_id` (required)::
-  (string) Identifier for the {dfeed}
-
-
-==== Authorization
-
-If {es} {security-features} are enabled, you must have `monitor_ml`, `monitor`,
-`manage_ml`, or `manage` cluster privileges to use this API. For more
-information, see
-{stack-ov}/security-privileges.html[Security Privileges].
-
-
-==== Security Integration
-
-When {es} {security-features} are enabled, the {dfeed} query is previewed using
-the credentials of the user calling the preview {dfeed} API.  When the {dfeed}
-is started it runs the query using the roles of the last user to
-create or update it.  If the two sets of roles differ then the preview may
-not accurately reflect what the {dfeed} will return when started.  To avoid
+IMPORTANT: When {es} {security-features} are enabled, the {dfeed} query is
+previewed using the credentials of the user calling the preview {dfeed} API.
+When the {dfeed} is started it runs the query using the roles of the last user
+to create or update it.  If the two sets of roles differ then the preview may
+not accurately reflect what the {dfeed} will return when started. To avoid
 such problems, the same user that creates/updates the {dfeed} should preview
 it to ensure it is returning the expected data.
 
+[[ml-preview-datafeed-path-parms]]
+==== {api-path-parms-title}
+
+`<datafeed_id>` (Required)::
+  (string) Identifier for the {dfeed}.
 
-==== Examples
+[[ml-preview-datafeed-example]]
+==== {api-examples-title}
 
 The following example obtains a preview of the `datafeed-farequote` {dfeed}:
 
diff --git a/docs/reference/ml/apis/put-calendar-job.asciidoc b/docs/reference/ml/apis/put-calendar-job.asciidoc
index cafc5f670627c..0a1ee2fcc6de0 100644
--- a/docs/reference/ml/apis/put-calendar-job.asciidoc
+++ b/docs/reference/ml/apis/put-calendar-job.asciidoc
@@ -8,28 +8,30 @@
 
 Adds a job to a calendar.
 
-==== Request
+[[ml-put-calendar-job-request]]
+==== {api-request-title}
 
 `PUT _ml/calendars/<calendar_id>/jobs/<job_id>`
 
+[[ml-put-calendar-job-prereqs]]
+==== {api-prereq-title}
 
-==== Path Parameters
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-`calendar_id` (required)::
-  (string) Identifier for the calendar.
-
-`job_id` (required)::
-  (string) An identifier for the job. It can be a job identifier, a group name, or a
-           comma-separated list of jobs or groups.
+[[ml-put-calendar-job-path-parms]]
+==== {api-path-parms-title}
 
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
+`<calendar_id>` (Required)::
+  (string) Identifier for the calendar.
 
+`<job_id>` (Required)::
+  (string) An identifier for the job. It can be a job identifier, a group name,
+  or a comma-separated list of jobs or groups.
 
-==== Examples
+[[ml-put-calendar-job-example]]
+==== {api-examples-title}
 
 The following example associates the `planned-outages` calendar with the
 `total-requests` job:
diff --git a/docs/reference/ml/apis/put-calendar.asciidoc b/docs/reference/ml/apis/put-calendar.asciidoc
index 9b1e781e3cc06..f98dd541d6753 100644
--- a/docs/reference/ml/apis/put-calendar.asciidoc
+++ b/docs/reference/ml/apis/put-calendar.asciidoc
@@ -8,35 +8,38 @@
 
 Instantiates a calendar.
 
-==== Request
+[[ml-put-calendar-request]]
+==== {api-request-title}
 
 `PUT _ml/calendars/<calendar_id>`
 
-===== Description
+[[ml-put-calendar-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
+
+[[ml-put-calendar-desc]]
+==== {api-description-title}
 
 For more information, see
-{xpack-ref}/ml-calendars.html[Calendars and Scheduled Events].
+{stack-ov}/ml-calendars.html[Calendars and Scheduled Events].
 
-==== Path Parameters
+[[ml-put-calendar-path-parms]]
+==== {api-path-parms-title}
 
-`calendar_id` (required)::
+`<calendar_id>` (Required)::
   (string) Identifier for the calendar.
 
+[[ml-put-calendar-request-body]]
+==== {api-request-body-title}
 
-==== Request Body
-
-`description`::
+`description` (Optional)::
   (string) A description of the calendar.
 
-
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-put-calendar-example]]
+==== {api-examples-title}
 
 The following example creates the `planned-outages` calendar:
 
diff --git a/docs/reference/ml/apis/put-datafeed.asciidoc b/docs/reference/ml/apis/put-datafeed.asciidoc
index 52728dd093da5..6c4578abb1671 100644
--- a/docs/reference/ml/apis/put-datafeed.asciidoc
+++ b/docs/reference/ml/apis/put-datafeed.asciidoc
@@ -10,101 +10,100 @@
 
 Instantiates a {dfeed}.
 
-
-==== Request
+[[ml-put-datafeed-request]]
+==== {api-request-title}
 
 `PUT _ml/datafeeds/<feed_id>`
 
+[[ml-put-datafeed-prereqs]]
+==== {api-prereq-title}
+
+* If {es} {security-features} are enabled, you must have `manage_ml` or `manage`
+cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-==== Description
+[[ml-put-datafeed-desc]]
+==== {api-description-title}
 
 You must create a job before you create a {dfeed}.  You can associate only one
 {dfeed} to each job.
 
-IMPORTANT:  You must use {kib} or this API to create a {dfeed}. Do not put a {dfeed}
-            directly to the `.ml-config` index using the Elasticsearch index API.
-            If {es} {security-features} are enabled, do not give users `write`
-            privileges on the `.ml-config` index.
-
-
-==== Path Parameters
-
-`feed_id` (required)::
+[IMPORTANT]
+====
+* You must use {kib} or this API to create a {dfeed}. Do not put a
+{dfeed} directly to the `.ml-config` index using the {es} index API. If {es}
+{security-features} are enabled, do not give users `write` privileges on the
+`.ml-config` index.
+* When {es} {security-features} are enabled, your {dfeed} remembers which roles
+the user who created it had at the time of creation and runs the query using
+those same roles.
+====
+
+[[ml-put-datafeed-path-parms]]
+==== {api-path-parms-title}
+
+`<feed_id>` (Required)::
   (string) A numerical character string that uniquely identifies the {dfeed}.
   This identifier can contain lowercase alphanumeric characters (a-z and 0-9),
   hyphens, and underscores. It must start and end with alphanumeric characters.
 
+[[ml-put-datafeed-request-body]]
+==== {api-request-body-title}
 
-==== Request Body
-
-`aggregations`::
+`aggregations` (Optional)::
   (object) If set, the {dfeed} performs aggregation searches.
   For more information, see <<ml-datafeed-resource>>.
 
-`chunking_config`::
+`chunking_config` (Optional)::
   (object) Specifies how data searches are split into time chunks.
   See <<ml-datafeed-chunking-config>>.
 
-`frequency`::
+`delayed_data_check_config` (Optional)::
+  (object) Specifies whether the data feed checks for missing data and 
+  the size of the window. See
+  <<ml-datafeed-delayed-data-check-config>>.
+
+`frequency` (Optional)::
   (time units) The interval at which scheduled queries are made while the {dfeed}
   runs in real time. The default value is either the bucket span for short
   bucket spans, or, for longer bucket spans, a sensible fraction of the bucket
   span. For example: `150s`.
 
-`indices` (required)::
+`indices` (Required)::
   (array) An array of index names. Wildcards are supported. For example:
   `["it_ops_metrics", "server*"]`.
 
-`job_id` (required)::
+`job_id` (Required)::
  (string) A numerical character string that uniquely identifies the job.
 
-`query`::
+`query` (Optional)::
   (object) The {es} query domain-specific language (DSL). This value
   corresponds to the query object in an {es} search POST body. All the
   options that are supported by {Es} can be used, as this object is
   passed verbatim to {es}. By default, this property has the following
   value: `{"match_all": {"boost": 1}}`.
 
-`query_delay`::
+`query_delay` (Optional)::
   (time units) The number of seconds behind real time that data is queried. For
   example, if data from 10:04 a.m. might not be searchable in {es} until
   10:06 a.m., set this property to 120 seconds. The default value is `60s`.
 
-`script_fields`::
+`script_fields` (Optional)::
   (object) Specifies scripts that evaluate custom expressions and returns
   script fields to the {dfeed}.
   The <<ml-detectorconfig,detector configuration objects>> in a job can contain
-  functions that use these script fields.
-  For more information,
+  functions that use these script fields. For more information,
   see {ref}/search-request-script-fields.html[Script Fields].
 
-`scroll_size`::
+`scroll_size` (Optional)::
   (unsigned integer) The `size` parameter that is used in {es} searches.
   The default value is `1000`.
 
-`delayed_data_check_config`::
-  (object) Specifies if and with how large a window should the data feed check
-  for missing data. See <<ml-datafeed-delayed-data-check-config>>.
-
 For more information about these properties,
 see <<ml-datafeed-resource>>.
 
-
-==== Authorization
-
-If {es} {security-features} are enabled, you must have `manage_ml`, or `manage`
-cluster privileges to use this API. For more information, see
-{stack-ov}/security-privileges.html[Security Privileges].
-
-
-==== Security integration
-
-When {es} {security-features} are enabled, your {dfeed} remembers which roles the
-user who created it had at the time of creation and runs the query using those
-same roles.
-
-
-==== Examples
+[[ml-put-datafeed-example]]
+==== {api-examples-title}
 
 The following example creates the `datafeed-total-requests` {dfeed}:
 
diff --git a/docs/reference/ml/apis/put-filter.asciidoc b/docs/reference/ml/apis/put-filter.asciidoc
index abe52dfb13b25..ad0d6d34ea81d 100644
--- a/docs/reference/ml/apis/put-filter.asciidoc
+++ b/docs/reference/ml/apis/put-filter.asciidoc
@@ -8,42 +8,45 @@
 
 Instantiates a filter.
 
-==== Request
+[[ml-put-filter-request]]
+==== {api-request-title}
 
 `PUT _ml/filters/<filter_id>`
 
-===== Description
+[[ml-put-filter-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
+
+[[ml-put-filter-desc]]
+==== {api-description-title}
 
 A {stack-ov}/ml-rules.html[filter] contains a list of strings. 
 It can be used by one or more jobs. Specifically, filters are referenced in 
 the `custom_rules` property of <<ml-detectorconfig,detector configuration objects>>. 
 
-==== Path Parameters
+[[ml-put-filter-path-parms]]
+==== {api-path-parms-title}
 
-`filter_id` (required)::
+`<filter_id>` (Required)::
   (string) Identifier for the filter.
 
+[[ml-put-filter-request-body]]
+==== {api-request-body-title}
 
-==== Request Body
-
-`description`::
+`description` (Optional)::
   (string) A description of the filter.
   
-`items`::
+`items` (Required)::
   (array of strings) The items of the filter.
   A wildcard `*` can be used at the beginning
   or the end of an item. Up to 10000 items
   are allowed in each filter.
 
-
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-put-filter-example]]
+==== {api-examples-title}
 
 The following example creates the `safe_domains` filter:
 
diff --git a/docs/reference/ml/apis/put-job.asciidoc b/docs/reference/ml/apis/put-job.asciidoc
index e3d80c276dc55..c60de488180d9 100644
--- a/docs/reference/ml/apis/put-job.asciidoc
+++ b/docs/reference/ml/apis/put-job.asciidoc
@@ -8,84 +8,87 @@
 
 Instantiates a job.
 
-==== Request
+[[ml-put-job-request]]
+==== {api-request-title}
 
 `PUT _ml/anomaly_detectors/<job_id>`
 
-===== Description
+[[ml-put-job-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
+
+[[ml-put-job-desc]]
+==== {api-description-title}
 
 IMPORTANT: You must use {kib} or this API to create a {ml} job. Do not put a job
             directly to the `.ml-config` index using the Elasticsearch index API.
             If {es} {security-features} are enabled, do not give users `write`
             privileges on the `.ml-config` index.
 
+[[ml-put-job-path-parms]]
+==== {api-path-parms-title}
 
-==== Path Parameters
-
-`job_id` (required)::
+`<job_id>` (Required)::
   (string) Identifier for the job. This identifier can contain lowercase
   alphanumeric characters (a-z and 0-9), hyphens, and underscores. It must
   start and end with alphanumeric characters.
 
+[[ml-put-job-request-body]]
+==== {api-request-body-title}
 
-==== Request Body
-
-`analysis_config`::
+`analysis_config` (Required)::
   (object) The analysis configuration, which specifies how to analyze the data.
   See <<ml-analysisconfig, analysis configuration objects>>.
 
-`analysis_limits`::
+`analysis_limits` (Optional)::
   (object) Specifies runtime limits for the job. See
   <<ml-apilimits,analysis limits>>.
 
-`background_persist_interval`::
+`background_persist_interval` (Optional)::
   (time units) Advanced configuration option. The time between each periodic
   persistence of the model. See <<ml-job-resource>>.
 
-`custom_settings`::
+`custom_settings` (Optional)::
   (object) Advanced configuration option. Contains custom meta data about the
   job. See <<ml-job-resource>>.
 
-`data_description` (required)::
+`data_description` (Required)::
   (object) Describes the format of the input data. This object is required, but
   it can be empty (`{}`). See <<ml-datadescription,data description objects>>.
 
-`description`::
+`description` (Optional)::
   (string) A description of the job.
 
-`groups`::
+`groups` (Optional)::
   (array of strings) A list of job groups. See <<ml-job-resource>>.
 
-`model_plot_config`::
+`model_plot_config` (Optional)::
   (object) Advanced configuration option. Specifies to store model information
   along with the results. This adds overhead to the performance of the system
   and is not feasible for jobs with many entities, see <<ml-apimodelplotconfig>>.
 
-`model_snapshot_retention_days`::
+`model_snapshot_retention_days` (Optional)::
   (long) The time in days that model snapshots are retained for the job.
   Older snapshots are deleted. The default value is `1`, which means snapshots
   are retained for one day (twenty-four hours). 
 
-`renormalization_window_days`::
+`renormalization_window_days` (Optional)::
   (long) Advanced configuration option. The period over which adjustments to the
   score are applied, as new data is seen. See <<ml-job-resource>>.
 
-`results_index_name`::
+`results_index_name` (Optional)::
   (string) A text string that affects the name of the {ml} results index. The 
   default value is `shared`, which generates an index named `.ml-anomalies-shared`. 
 
-`results_retention_days`::
+`results_retention_days` (Optional)::
   (long) Advanced configuration option. The number of days for which job results
   are retained. See <<ml-job-resource>>.
 
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-put-job-example]]
+==== {api-examples-title}
 
 The following example creates the `total-requests` job:
 
diff --git a/docs/reference/ml/apis/revert-snapshot.asciidoc b/docs/reference/ml/apis/revert-snapshot.asciidoc
index b560f7b041206..86d3d4c14a93c 100644
--- a/docs/reference/ml/apis/revert-snapshot.asciidoc
+++ b/docs/reference/ml/apis/revert-snapshot.asciidoc
@@ -8,12 +8,20 @@
 
 Reverts to a specific snapshot.
 
-==== Request
+[[ml-revert-snapshot-request]]
+==== {api-request-title}
 
 `POST _ml/anomaly_detectors/<job_id>/model_snapshots/<snapshot_id>/_revert`
 
+[[ml-revert-snapshot-prereqs]]
+==== {api-prereq-title}
 
-==== Description
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
+
+[[ml-revert-snapshot-desc]]
+==== {api-description-title}
 
 The {ml} feature in {xpack} reacts quickly to anomalous input, learning new
 behaviors in data. Highly anomalous input increases the variance in the models
@@ -25,18 +33,19 @@ Friday or a critical system failure.
 
 IMPORTANT: Before you revert to a saved snapshot, you must close the job.
 
+[[ml-revert-snapshot-path-parms]]
+==== {api-path-parms-title}
 
-==== Path Parameters
-
-`job_id` (required)::
-  (string) Identifier for the job
+`<job_id>` (Required)::
+  (string) Identifier for the job.
 
-`snapshot_id` (required)::
-  (string) Identifier for the model snapshot
+`<snapshot_id>` (Required)::
+  (string) Identifier for the model snapshot.
 
-==== Request Body
+[[ml-revert-snapshot-request-body]]
+==== {api-request-body-title}
 
-`delete_intervening_results`::
+`delete_intervening_results` (Optional)::
   (boolean) If true, deletes the results in the time period between the
   latest results and the time of the reverted snapshot. It also resets the
   model to accept records for this time period. The default value is false.
@@ -45,15 +54,8 @@ NOTE: If you choose not to delete intervening results when reverting a snapshot,
 the job will not accept input data that is older than the current time.
 If you want to resend data, then delete the intervening results.
 
-
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-revert-snapshot-example]]
+==== {api-examples-title}
 
 The following example reverts to the `1491856080` snapshot for the
 `it_ops_new_kpi` job:
diff --git a/docs/reference/ml/apis/set-upgrade-mode.asciidoc b/docs/reference/ml/apis/set-upgrade-mode.asciidoc
index 5434d70d4e61e..6a00656430c66 100644
--- a/docs/reference/ml/apis/set-upgrade-mode.asciidoc
+++ b/docs/reference/ml/apis/set-upgrade-mode.asciidoc
@@ -9,7 +9,8 @@
 Sets a cluster wide upgrade_mode setting that prepares {ml} indices for an
 upgrade. 
 
-==== Request
+[[ml-set-upgrade-mode-request]]
+==== {api-request-title}
 //////////////////////////
 
 [source,js]
@@ -25,7 +26,15 @@ POST /_ml/set_upgrade_mode?enabled=false&timeout=10m
 
 `POST _ml/set_upgrade_mode`
 
-==== Description
+[[ml-set-upgrade-mode-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
+
+[[ml-set-upgrade-mode-desc]]
+==== {api-description-title}
 
 When upgrading your cluster, in some circumstances you must restart your nodes and
 reindex your {ml} indices. In those circumstances, there must be no {ml} jobs running. 
@@ -37,7 +46,6 @@ though stopping jobs is not a requirement in that case.
 
 For more information, see {stack-ref}/upgrading-elastic-stack.html[Upgrading the {stack}].
 
-
 When `enabled=true` this API temporarily halts all job and {dfeed} tasks and
 prohibits new job and {dfeed} tasks from starting.
 
@@ -50,23 +58,18 @@ You can see the current value for the `upgrade_mode` setting by using the
 IMPORTANT:  No new {ml} jobs can be opened while the `upgrade_mode` setting is
 `true`.
 
-==== Query Parameters
+[[ml-set-upgrade-mode-query-parms]]
+==== {api-query-parms-title}
 
-`enabled`::
+`enabled` (Optional)::
   (boolean) When `true`, this enables `upgrade_mode`. Defaults to `false`
 
-`timeout`::
+`timeout` (Optional)::
   (time) The time to wait for the request to be completed.
   The default value is 30 seconds.
 
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see
-{stack-ov}/security-privileges.html[Security privileges].
-
-
-==== Examples
+[[ml-set-upgrade-mode-example]]
+==== {api-examples-title}
 
 The following example enables `upgrade_mode` for the cluster:
 
diff --git a/docs/reference/ml/apis/snapshotresource.asciidoc b/docs/reference/ml/apis/snapshotresource.asciidoc
index 73e2e9b3b8bc1..13a26e26e8d70 100644
--- a/docs/reference/ml/apis/snapshotresource.asciidoc
+++ b/docs/reference/ml/apis/snapshotresource.asciidoc
@@ -3,7 +3,7 @@
 [[ml-snapshot-resource]]
 === Model snapshot resources
 
-Model snapshots are saved to disk periodically.
+Model snapshots are saved to an internal index within the Elasticsearch cluster.
 By default, this is occurs approximately every 3 hours to 4 hours and is
 configurable with the `background_persist_interval` property.
 
diff --git a/docs/reference/ml/apis/start-datafeed.asciidoc b/docs/reference/ml/apis/start-datafeed.asciidoc
index aee237b72c837..05cf0766e9522 100644
--- a/docs/reference/ml/apis/start-datafeed.asciidoc
+++ b/docs/reference/ml/apis/start-datafeed.asciidoc
@@ -12,11 +12,20 @@ Starts one or more {dfeeds}.
 A {dfeed} must be started in order to retrieve data from {es}.
 A {dfeed} can be started and stopped multiple times throughout its lifecycle.
 
-==== Request
+[[ml-start-datafeed-request]]
+==== {api-request-title}
 
 `POST _ml/datafeeds/<feed_id>/_start`
 
-==== Description
+[[ml-start-datafeed-prereqs]]
+==== {api-prereq-title}
+
+* If {es} {security-features} are enabled, you must have `manage_ml` or `manage`
+cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
+
+[[ml-start-datafeed-desc]]
+==== {api-description-title}
 
 NOTE: Before you can start a {dfeed}, the job must be open. Otherwise, an error
 occurs.
@@ -56,42 +65,33 @@ If you specify a `start` value that is earlier than the timestamp of the latest
 processed record, the {dfeed} continues from 1 millisecond after the timestamp
 of the latest processed record.
 
+IMPORTANT: When {es} {security-features} are enabled, your {dfeed} remembers
+which roles the last user to create or update it had at the time of
+creation/update and runs the query using those same roles.
 
-==== Path Parameters
+[[ml-start-datafeed-path-parms]]
+==== {api-path-parms-title}
 
-`feed_id` (required)::
-(string) Identifier for the {dfeed}
+`<feed_id>` (Required)::
+  (string) Identifier for the {dfeed}.
 
-==== Request Body
+[[ml-start-datafeed-request-body]]
+==== {api-request-body-title}
 
-`end`::
+`end` (Optional)::
   (string) The time that the {dfeed} should end. This value is exclusive.
   The default value is an empty string.
 
-`start`::
+`start` (Optional)::
   (string) The time that the {dfeed} should begin. This value is inclusive.
   The default value is an empty string.
 
-`timeout`::
+`timeout` (Optional)::
   (time) Controls the amount of time to wait until a {dfeed} starts.
   The default value is 20 seconds.
 
-
-==== Authorization
-
-If {es} {security-features} are enabled, you must have `manage_ml`, or `manage`
-cluster privileges to use this API. For more information, see
-{stack-ov}/security-privileges.html[Security Privileges].
-
-
-==== Security integration
-
-When {es} {security-features} are enabled, your {dfeed} remembers which roles the
-last user to create or update it had at the time of creation/update and runs the
-query using those same roles.
-
-
-==== Examples
+[[ml-start-datafeed-example]]
+==== {api-examples-title}
 
 The following example starts the `datafeed-it-ops-kpi` {dfeed}:
 
diff --git a/docs/reference/ml/apis/stop-datafeed.asciidoc b/docs/reference/ml/apis/stop-datafeed.asciidoc
index 1489137b9db07..bdac8d51fab04 100644
--- a/docs/reference/ml/apis/stop-datafeed.asciidoc
+++ b/docs/reference/ml/apis/stop-datafeed.asciidoc
@@ -10,10 +10,8 @@
 
 Stops one or more {dfeeds}.
 
-A {dfeed} that is stopped ceases to retrieve data from {es}.
-A {dfeed} can be started and stopped multiple times throughout its lifecycle.
-
-==== Request
+[[ml-stop-datafeed-request]]
+==== {api-request-title}
 
 `POST _ml/datafeeds/<feed_id>/_stop` +
 
@@ -21,39 +19,42 @@ A {dfeed} can be started and stopped multiple times throughout its lifecycle.
 
 `POST _ml/datafeeds/_all/_stop`
 
+[[ml-stop-datafeed-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-===== Description
+[[ml-stop-datafeed-desc]]
+==== {api-description-title}
+
+A {dfeed} that is stopped ceases to retrieve data from {es}.
+A {dfeed} can be started and stopped multiple times throughout its lifecycle.
 
 You can stop multiple {dfeeds} in a single API request by using a
 comma-separated list of {dfeeds} or a wildcard expression. You can close all
 {dfeeds} by using `_all` or by specifying `*` as the `<feed_id>`.
 
+[[ml-stop-datafeed-path-parms]]
+==== {api-path-parms-title}
 
-==== Path Parameters
-
-`feed_id`::
+`<feed_id>` (Required)::
   (string) Identifier for the {dfeed}. It can be a {dfeed} identifier or a
   wildcard expression.
 
+[[ml-stop-datafeed-request-body]]
+==== {api-request-body-title}
 
-==== Request Body
-
-`force`::
+`force` (Optional)::
   (boolean) If true, the {dfeed} is stopped forcefully.
 
-`timeout`::
+`timeout` (Optional)::
   (time) Controls the amount of time to wait until a {dfeed} stops.
   The default value is 20 seconds.
 
-
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-stop-datafeed-example]]
+==== {api-examples-title}
 
 The following example stops the `datafeed-total-requests` {dfeed}:
 
diff --git a/docs/reference/ml/apis/update-datafeed.asciidoc b/docs/reference/ml/apis/update-datafeed.asciidoc
index a370c1acef9d7..b57088673d845 100644
--- a/docs/reference/ml/apis/update-datafeed.asciidoc
+++ b/docs/reference/ml/apis/update-datafeed.asciidoc
@@ -10,55 +10,77 @@
 
 Updates certain properties of a {dfeed}.
 
-==== Request
+[[ml-update-datafeed-request]]
+==== {api-request-title}
 
 `POST _ml/datafeeds/<feed_id>/_update`
 
-//===== Description
+[[ml-update-datafeed-prereqs]]
+==== {api-prereq-title}
 
-==== Path Parameters
+* If {es} {security-features} are enabled, you must have `manage_ml`, or `manage`
+cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-`feed_id` (required)::
-  (string) Identifier for the {dfeed}
+[[ml-update-datafeed-desc]]
+==== {api-description-title}
 
-==== Request Body
+If you update the `delayed_data_check_config` property, you must stop and
+start the {dfeed} for the change to be applied.
+
+IMPORTANT: When {es} {security-features} are enabled, your {dfeed} remembers
+which roles the user who updated it had at the time of update and runs the query
+using those same roles.
+
+[[ml-update-datafeed-path-parms]]
+==== {api-path-parms-title}
+
+`<feed_id>` (Required)::
+  (string) Identifier for the {dfeed}.
+
+[[ml-update-datafeed-request-body]]
+==== {api-request-body-title}
 
 The following properties can be updated after the {dfeed} is created:
 
-`aggregations`::
+`aggregations` (Optional)::
   (object) If set, the {dfeed} performs aggregation searches.
   For more information, see <<ml-datafeed-resource>>.
 
-`chunking_config`::
+`chunking_config` (Optional)::
   (object) Specifies how data searches are split into time chunks.
   See <<ml-datafeed-chunking-config>>.
+  
+`delayed_data_check_config` (Optional)::
+  (object) Specifies whether the data feed checks for missing data and 
+  the size of the window. See <<ml-datafeed-delayed-data-check-config>>.  
 
-`frequency`::
+`frequency` (Optional)::
   (time units) The interval at which scheduled queries are made while the
   {dfeed} runs in real time. The default value is either the bucket span for short
   bucket spans, or, for longer bucket spans, a sensible fraction of the bucket
   span. For example: `150s`.
 
-`indices`::
+`indices` (Optional)::
   (array) An array of index names. Wildcards are supported. For example:
   `["it_ops_metrics", "server*"]`.
 
-`job_id`::
+`job_id` (Optional)::
  (string) A numerical character string that uniquely identifies the job.
 
-`query`::
+`query` (Optional)::
   (object) The {es} query domain-specific language (DSL). This value
   corresponds to the query object in an {es} search POST body. All the
   options that are supported by {es} can be used, as this object is
   passed verbatim to {es}. By default, this property has the following
   value: `{"match_all": {"boost": 1}}`.
 
-`query_delay`::
+`query_delay` (Optional)::
   (time units) The number of seconds behind real-time that data is queried. For
   example, if data from 10:04 a.m. might not be searchable in {es} until
   10:06 a.m., set this property to 120 seconds. The default value is `60s`.
 
-`script_fields`::
+`script_fields` (Optional)::
   (object) Specifies scripts that evaluate custom expressions and returns
   script fields to the {dfeed}.
   The <<ml-detectorconfig,detector configuration objects>> in a job can contain
@@ -66,29 +88,15 @@ The following properties can be updated after the {dfeed} is created:
   For more information,
   see {ref}/search-request-script-fields.html[Script Fields].
 
-`scroll_size`::
+`scroll_size` (Optional)::
   (unsigned integer) The `size` parameter that is used in {es} searches.
   The default value is `1000`.
 
 For more information about these properties,
 see <<ml-datafeed-resource>>.
 
-
-==== Authorization
-
-If {es} {security-features} are enabled, you must have `manage_ml`, or `manage`
-cluster privileges to use this API. For more information, see
-{stack-ov}/security-privileges.html[Security Privileges].
-
-
-==== Security Integration
-
-When {es} {security-features} are enabled, your {dfeed} remembers which roles the
-user who updated it had at the time of update and runs the query using those
-same roles.
-
-
-==== Examples
+[[ml-update-datafeed-example]]
+==== {api-examples-title}
 
 The following example updates the query for the `datafeed-total-requests`
 {dfeed} so that only log entries of error level are analyzed:
diff --git a/docs/reference/ml/apis/update-filter.asciidoc b/docs/reference/ml/apis/update-filter.asciidoc
index 45c294a0b8bc6..df8f3056d12cc 100644
--- a/docs/reference/ml/apis/update-filter.asciidoc
+++ b/docs/reference/ml/apis/update-filter.asciidoc
@@ -8,40 +8,41 @@
 
 Updates the description of a filter, adds items, or removes items. 
 
-==== Request
+[[ml-update-filter-request]]
+==== {api-request-title}
 
 `POST _ml/filters/<filter_id>/_update`
 
-//==== Description
+[[ml-update-filter-prereqs]]
+==== {api-prereq-title}
 
-==== Path Parameters
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-`filter_id` (required)::
-		(string) Identifier for the filter.
+[[ml-update-filter-path-parms]]
+==== {api-path-parms-title}
 
+`<filter_id>` (Required)::
+		(string) Identifier for the filter.
 
-==== Request Body
+[[ml-update-filter-request-body]]
+==== {api-request-body-title}
 
-`description`::
+`description` (Optional)::
   (string) A description for the filter. See <<ml-filter-resource>>.
 	
-`add_items`::
+`add_items` (Optional)::
   (array of strings) The items to add to the filter.
 	
-`remove_items`::
+`remove_items` (Optional)::
   (array of strings) The items to remove from the filter.
 
+[[ml-update-filter-example]]
+==== {api-examples-title}
 
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
-
-You can change the description, add and remove items to the `safe_domains` filter as follows:
+You can change the description, add and remove items to the `safe_domains`
+filter as follows:
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/ml/apis/update-job.asciidoc b/docs/reference/ml/apis/update-job.asciidoc
index 3382e7fe34675..e78bda613d801 100644
--- a/docs/reference/ml/apis/update-job.asciidoc
+++ b/docs/reference/ml/apis/update-job.asciidoc
@@ -8,17 +8,27 @@
 
 Updates certain properties of a job.
 
-==== Request
+[[ml-update-job-request]]
+==== {api-request-title}
 
 `POST _ml/anomaly_detectors/<job_id>/_update`
 
+[[ml-update-job-prereqs]]
+==== {api-prereq-title}
 
-==== Path Parameters
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-`job_id` (required)::
-  (string) Identifier for the job
 
-==== Request Body
+[[ml-update-job-path-parms]]
+==== {api-path-parms-title}
+
+`<job_id>` (Required)::
+  (string) Identifier for the job.
+
+[[ml-update-job-request-body]]
+==== {api-request-body-title}
 
 The following properties can be updated after the job is created:
 
@@ -86,14 +96,8 @@ A detector update object has the following properties:
 
 No other detector property can be updated.
 
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-update-job-example]]
+==== {api-examples-title}
 
 The following example updates the `total-requests` job:
 
diff --git a/docs/reference/ml/apis/update-snapshot.asciidoc b/docs/reference/ml/apis/update-snapshot.asciidoc
index ffd38f590b1e2..1fe2ed5384bc0 100644
--- a/docs/reference/ml/apis/update-snapshot.asciidoc
+++ b/docs/reference/ml/apis/update-snapshot.asciidoc
@@ -8,45 +8,45 @@
 
 Updates certain properties of a snapshot.
 
-==== Request
+[[ml-update-snapshot-request]]
+==== {api-request-title}
 
 `POST _ml/anomaly_detectors/<job_id>/model_snapshots/<snapshot_id>/_update`
 
+[[ml-update-snapshot-prereqs]]
+==== {api-prereq-title}
 
-//==== Description
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
 
-==== Path Parameters
 
-`job_id` (required)::
-  (string) Identifier for the job
+[[ml-update-snapshot-path-parms]]
+==== {api-path-parms-title}
 
-`snapshot_id` (required)::
-  (string) Identifier for the model snapshot
+`<job_id>` (Required)::
+  (string) Identifier for the job.
 
-==== Request Body
+`<snapshot_id>` (Required)::
+  (string) Identifier for the model snapshot.
+
+[[ml-update-snapshot-request-body]]
+==== {api-request-body-title}
 
 The following properties can be updated after the model snapshot is created:
 
-`description`::
-  (string) An optional description of the model snapshot. For example,
+`description` (Optional)::
+  (string) A description of the model snapshot. For example,
   "Before black friday".
 
-`retain`::
+`retain` (Optional)::
   (boolean) If true, this snapshot will not be deleted during automatic cleanup
   of snapshots older than `model_snapshot_retention_days`.
   Note that this snapshot will still be deleted when the job is deleted.
   The default value is false.
 
-
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-//<<privileges-list-cluster>>.
-
-
-==== Examples
+[[ml-update-snapshot-example]]
+==== {api-examples-title}
 
 The following example updates the snapshot identified as `1491852978`:
 
diff --git a/docs/reference/ml/apis/validate-detector.asciidoc b/docs/reference/ml/apis/validate-detector.asciidoc
index 0f9fe9902e36e..2e5896b95cc93 100644
--- a/docs/reference/ml/apis/validate-detector.asciidoc
+++ b/docs/reference/ml/apis/validate-detector.asciidoc
@@ -8,30 +8,32 @@
 
 Validates detector configuration information.
 
-==== Request
+[[ml-valid-detector-request]]
+==== {api-request-title}
 
 `POST _ml/anomaly_detectors/_validate/detector`
 
-==== Description
+[[ml-valid-detector-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
+
+[[ml-valid-detector-desc]]
+==== {api-description-title}
 
 The validate detectors API enables you validate the detector configuration
 before you create a job.
 
-
-==== Request Body
+[[ml-valid-detector-request-body]]
+==== {api-request-body-title}
 
 For a list of the properties that you can specify in the body of this API,
 see <<ml-detectorconfig,detector configuration objects>>.
 
-
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-valid-detector-example]]
+==== {api-examples-title}
 
 The following example validates detector configuration information:
 
diff --git a/docs/reference/ml/apis/validate-job.asciidoc b/docs/reference/ml/apis/validate-job.asciidoc
index 5fbfb62dd28a6..faa7cab2f3928 100644
--- a/docs/reference/ml/apis/validate-job.asciidoc
+++ b/docs/reference/ml/apis/validate-job.asciidoc
@@ -8,30 +8,32 @@
 
 Validates job configuration information.
 
-==== Request
+[[ml-valid-job-request]]
+==== {api-request-title}
 
 `POST _ml/anomaly_detectors/_validate`
 
-==== Description
+[[ml-valid-job-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have `manage_ml` or
+`manage` cluster privileges to use this API. See
+{stack-ov}/security-privileges.html[Security privileges].
+
+[[ml-valid-job-desc]]
+==== {api-description-title}
 
 The validate jobs API enables you validate the job configuration before you
 create the job.
 
-
-==== Request Body
+[[ml-valid-job-request-body]]
+==== {api-request-body-title}
 
 For a list of the properties that you can specify in the body of this API,
 see <<ml-job-resource,Job Resources>>.
 
-
-==== Authorization
-
-You must have `manage_ml`, or `manage` cluster privileges to use this API.
-For more information, see
-{xpack-ref}/security-privileges.html[Security Privileges].
-
-
-==== Examples
+[[ml-valid-job-example]]
+==== {api-examples-title}
 
 The following example validates job configuration information:
 
diff --git a/docs/reference/ml/configuring.asciidoc b/docs/reference/ml/configuring.asciidoc
index 9304a93d360c7..a27818e0cf1f5 100644
--- a/docs/reference/ml/configuring.asciidoc
+++ b/docs/reference/ml/configuring.asciidoc
@@ -5,7 +5,7 @@
 If you want to use {ml-features}, there must be at least one {ml} node in
 your cluster and all master-eligible nodes must have {ml} enabled. By default,
 all nodes are {ml} nodes. For more information about these settings, see 
-{ref}/modules-node.html#modules-node-xpack[{ml} nodes].
+{ref}/modules-node.html#ml-node[{ml} nodes].
 
 To use the {ml-features} to analyze your data, you must create a job and
 send your data to that job.
diff --git a/docs/reference/ml/functions/geo.asciidoc b/docs/reference/ml/functions/geo.asciidoc
index 130e17d85dcfe..ee666c83094f0 100644
--- a/docs/reference/ml/functions/geo.asciidoc
+++ b/docs/reference/ml/functions/geo.asciidoc
@@ -54,8 +54,9 @@ detects anomalies where the geographic location of a credit card transaction is
 unusual for a particular customer’s credit card. An anomaly might indicate fraud.
 
 IMPORTANT: The `field_name` that you supply must be a single string that contains
-two comma-separated numbers of the form `latitude,longitude`. The `latitude` and
-`longitude` must be in the range -180 to 180 and represent a point on the
+two comma-separated numbers of the form `latitude,longitude`, a `geo_point` field,
+a `geo_shape` field that contains point values, or a `geo_centroid` aggregation.
+The `latitude` and `longitude` must be in the range -180 to 180 and represent a point on the
 surface of the Earth.
 
 For example, JSON data might contain the following transaction coordinates:
@@ -71,34 +72,9 @@ For example, JSON data might contain the following transaction coordinates:
 // NOTCONSOLE
 
 In {es}, location data is likely to be stored in `geo_point` fields. For more
-information, see {ref}/geo-point.html[Geo-point datatype]. This data type is not
-supported natively in {ml-features}. You can, however, use Painless scripts
-in `script_fields` in your {dfeed} to transform the data into an appropriate
-format. For example, the following Painless script transforms
-`"coords": {"lat" : 41.44, "lon":90.5}` into `"lat-lon": "41.44,90.5"`:
-
-[source,js]
---------------------------------------------------
-PUT _ml/datafeeds/datafeed-test2
-{
-  "job_id": "farequote",
-  "indices": ["farequote"],
-  "query": {
-    "match_all": {
-          "boost": 1
-    }
-  },
-  "script_fields": {
-    "lat-lon": {
-      "script": {
-        "source": "doc['coords'].lat + ',' + doc['coords'].lon",
-        "lang": "painless"
-      }
-    }
-  }
-}
---------------------------------------------------
-// CONSOLE
-// TEST[skip:setup:farequote_job]
+information, see {ref}/geo-point.html[Geo-point datatype]. This data type is
+supported natively in {ml-features}. Specifically, {dfeed} when pulling data from
+a `geo_point` field, will transform the data into the appropriate `lat,lon` string
+format before sending to the {ml} job.
 
 For more information, see <<ml-configuring-transform>>.
diff --git a/docs/reference/modules.asciidoc b/docs/reference/modules.asciidoc
index 5928c676b1e46..e9f187a98e3c7 100644
--- a/docs/reference/modules.asciidoc
+++ b/docs/reference/modules.asciidoc
@@ -46,20 +46,10 @@ The modules in this section are:
 
     A Java node client joins the cluster, but doesn't hold data or act as a master node.
 
-<<modules-scripting-painless,Painless>>::
-
-    A built-in scripting language for Elasticsearch that's designed to be as secure as possible.
-
 <<modules-plugins,Plugins>>::
 
     Using plugins to extend Elasticsearch.
 
-<<modules-scripting,Scripting>>::
-
-    Custom scripting available in Lucene Expressions, ad Groovy. You can also
-    write scripts in the built-in scripting language,
-    <<modules-scripting-painless, Painless>>.
-
 <<modules-snapshots,Snapshot/Restore>>::
 
     Backup your data with snapshot/restore.
@@ -102,8 +92,6 @@ include::modules/node.asciidoc[]
 :edit_url:
 include::modules/plugins.asciidoc[]
 
-include::modules/scripting.asciidoc[]
-
 include::modules/snapshots.asciidoc[]
 
 include::modules/threadpool.asciidoc[]
diff --git a/docs/reference/modules/cluster/allocation_awareness.asciidoc b/docs/reference/modules/cluster/allocation_awareness.asciidoc
index 4381d0281cc9d..5fc9197d449d5 100644
--- a/docs/reference/modules/cluster/allocation_awareness.asciidoc
+++ b/docs/reference/modules/cluster/allocation_awareness.asciidoc
@@ -46,7 +46,7 @@ You can also set custom attributes when you start a node:
 +
 [source,sh]
 --------------------------------------------------------
-`./bin/elasticsearch -Enode.attr.rack_id=rack_one`
+./bin/elasticsearch -Enode.attr.rack_id=rack_one
 --------------------------------------------------------
 
 . Tell {es} to take one or more awareness attributes into account when
diff --git a/docs/reference/modules/discovery/adding-removing-nodes.asciidoc b/docs/reference/modules/discovery/adding-removing-nodes.asciidoc
index ccc0e99125371..2a62bb5e49dd3 100644
--- a/docs/reference/modules/discovery/adding-removing-nodes.asciidoc
+++ b/docs/reference/modules/discovery/adding-removing-nodes.asciidoc
@@ -35,30 +35,36 @@ four of the nodes at once: to do so would leave only three nodes remaining,
 which is less than half of the voting configuration, which means the cluster
 cannot take any further actions.
 
+More precisely, if you shut down half or more of the master-eligible nodes all
+at the same time then the cluster will normally become unavailable. If this
+happens then you can bring the cluster back online by starting the removed
+nodes again.
+
 As long as there are at least three master-eligible nodes in the cluster, as a
 general rule it is best to remove nodes one-at-a-time, allowing enough time for
 the cluster to <<modules-discovery-quorums,automatically adjust>> the voting
 configuration and adapt the fault tolerance level to the new set of nodes.
 
 If there are only two master-eligible nodes remaining then neither node can be
-safely removed since both are required to reliably make progress. You must first
-inform Elasticsearch that one of the nodes should not be part of the voting
-configuration, and that the voting power should instead be given to other nodes.
-You can then take the excluded node offline without preventing the other node
-from making progress. A node which is added to a voting configuration exclusion
-list still works normally, but Elasticsearch tries to remove it from the voting
-configuration so its vote is no longer required.  Importantly, Elasticsearch
-will never automatically move a node on the voting exclusions list back into the
-voting configuration. Once an excluded node has been successfully
+safely removed since both are required to reliably make progress. To remove one
+of these nodes you must first inform {es} that it should not be part of the
+voting configuration, and that the voting power should instead be given to the
+other node. You can then take the excluded node offline without preventing the
+other node from making progress. A node which is added to a voting
+configuration exclusion list still works normally, but {es} tries to remove it
+from the voting configuration so its vote is no longer required. Importantly,
+{es} will never automatically move a node on the voting exclusions list back
+into the voting configuration. Once an excluded node has been successfully
 auto-reconfigured out of the voting configuration, it is safe to shut it down
 without affecting the cluster's master-level availability. A node can be added
-to the voting configuration exclusion list using the <<voting-config-exclusions>> API. For example:
+to the voting configuration exclusion list using the
+<<voting-config-exclusions>> API. For example:
 
 [source,js]
 --------------------------------------------------
-# Add node to voting configuration exclusions list and wait for the system to
-# auto-reconfigure the node out of the voting configuration up to the default
-# timeout of 30 seconds
+# Add node to voting configuration exclusions list and wait for the system
+# to auto-reconfigure the node out of the voting configuration up to the
+# default timeout of 30 seconds
 POST /_cluster/voting_config_exclusions/node_name
 
 # Add node to voting configuration exclusions list and wait for
diff --git a/docs/reference/modules/discovery/bootstrapping.asciidoc b/docs/reference/modules/discovery/bootstrapping.asciidoc
index 2b17af17ec5da..cc7cb0ea91227 100644
--- a/docs/reference/modules/discovery/bootstrapping.asciidoc
+++ b/docs/reference/modules/discovery/bootstrapping.asciidoc
@@ -6,8 +6,9 @@ set of <<master-node,master-eligible nodes>> to be explicitly defined on one or
 more of the master-eligible nodes in the cluster. This is known as _cluster
 bootstrapping_.  This is only required the very first time the cluster starts
 up: nodes that have already joined a cluster store this information in their
-data folder and freshly-started nodes that are joining an existing cluster
-obtain this information from the cluster's elected master. 
+data folder for use in a <<restart-upgrade,full cluster restart>>, and
+freshly-started nodes that are joining a running cluster obtain this
+information from the cluster's elected master.
 
 The initial set of master-eligible nodes is defined in the
 <<initial_master_nodes,`cluster.initial_master_nodes` setting>>. This should be
@@ -58,19 +59,6 @@ cluster.initial_master_nodes:
   - master-c
 --------------------------------------------------
 
-If it is not possible to use the names of the nodes then you can also use IP
-addresses, or IP addresses and ports, or even a mix of IP addresses and node
-names:
-
-[source,yaml]
---------------------------------------------------
-cluster.initial_master_nodes:
-  - 10.0.10.101
-  - 10.0.10.102:9300
-  - 10.0.10.102:9301
-  - master-node-name
---------------------------------------------------
-
 Like all node settings, it is also possible to specify the initial set of master
 nodes on the command-line that is used to start Elasticsearch:
 
@@ -139,3 +127,29 @@ in the <<modules-discovery-bootstrap-cluster,section on cluster bootstrapping>>:
 * `discovery.seed_providers`
 * `discovery.seed_hosts`
 * `cluster.initial_master_nodes`
+
+[NOTE]
+==================================================
+
+[[modules-discovery-bootstrap-cluster-joining]] If you start an {es} node
+without configuring these settings then it will start up in development mode and
+auto-bootstrap itself into a new cluster. If you start some {es} nodes on
+different hosts then by default they will not discover each other and will form
+a different cluster on each host. {es} will not merge separate clusters together
+after they have formed, even if you subsequently try and configure all the nodes
+into a single cluster. This is because there is no way to merge these separate
+clusters together without a risk of data loss. You can tell that you have formed
+separate clusters by checking the cluster UUID reported by `GET /` on each node.
+If you intended to form a single cluster then you should start again:
+
+* Take a <<modules-snapshots,snapshot>> of each of the single-host clusters if
+  you do not want to lose any data that they hold. Note that each cluster must
+  use its own snapshot repository.
+* Shut down all the nodes.
+* Completely wipe each node by deleting the contents of their
+  <<data-path,data folders>>.
+* Configure `cluster.initial_master_nodes` as described above.
+* Restart all the nodes and verify that they have formed a single cluster.
+* <<modules-snapshots,Restore>> any snapshots as required.
+
+==================================================
diff --git a/docs/reference/modules/discovery/discovery-settings.asciidoc b/docs/reference/modules/discovery/discovery-settings.asciidoc
index 84472552cfced..850f0100c8939 100644
--- a/docs/reference/modules/discovery/discovery-settings.asciidoc
+++ b/docs/reference/modules/discovery/discovery-settings.asciidoc
@@ -16,14 +16,21 @@ Discovery and cluster formation are affected by the following settings:
     to use to obtain the addresses of the seed nodes used to start the
     discovery process. By default, it is the
     <<settings-based-hosts-provider,settings-based seed hosts provider>>.
-
+    
+`discovery.type`::
+ 
+    Specifies whether {es} should form a multiple-node cluster. By default, {es}
+    discovers other nodes when forming a cluster and allows other nodes to join
+    the cluster later. If `discovery.type` is set to `single-node`, {es} forms a
+    single-node cluster. For more information about when you might use this
+    setting, see <<single-node-discovery>>.
+    
 `cluster.initial_master_nodes`::
 
-    Sets a list of the <<node.name,node names>> or transport addresses of the
-    initial set of master-eligible nodes in a brand-new cluster. By default
-    this list is empty, meaning that this node expects to join a cluster that
-    has already been bootstrapped. See <<initial_master_nodes>>.
-
+    Sets the initial set of master-eligible nodes in a brand-new cluster. By
+    default this list is empty, meaning that this node expects to join a cluster
+    that has already been bootstrapped. See <<initial_master_nodes>>.
+            
 [float]
 ==== Expert settings
 
diff --git a/docs/reference/modules/indices/search-settings.asciidoc b/docs/reference/modules/indices/search-settings.asciidoc
index 30137fa382779..071471a1803ca 100644
--- a/docs/reference/modules/indices/search-settings.asciidoc
+++ b/docs/reference/modules/indices/search-settings.asciidoc
@@ -10,8 +10,8 @@ The following _expert_ setting can be set to manage global search limits.
 This setting limits the number of clauses a Lucene BooleanQuery can have. The
 default of 1024 is quite high and should normally be sufficient. This limit does
 not only affect Elasticsearchs `bool` query, but many other queries are rewritten to Lucene's
-BooleanQuery internally. The limit is in place to prevent searches from becoming to large
-and taking up too much CPU and memory. In case you consider to increase this setting,
-make sure you exhausted all other options to avoid having to do this. Higher values can lead 
+BooleanQuery internally. The limit is in place to prevent searches from becoming too large
+and taking up too much CPU and memory. In case you're considering increasing this setting,
+make sure you've exhausted all other options to avoid having to do this. Higher values can lead 
 to performance degradations and memory issues, especially in clusters with a high load or 
 few resources.
diff --git a/docs/reference/modules/memcached.asciidoc b/docs/reference/modules/memcached.asciidoc
deleted file mode 100644
index 508d328671b26..0000000000000
--- a/docs/reference/modules/memcached.asciidoc
+++ /dev/null
@@ -1,69 +0,0 @@
-[[modules-memcached]]
-== memcached
-
-The memcached module allows to expose *Elasticsearch*
-APIs over the memcached protocol (as closely
-as possible).
-
-It is provided as a plugin called `transport-memcached` and installing
-is explained
-https://github.com/elastic/elasticsearch-transport-memcached[here]
-. Another option is to download the memcached plugin and placing it
-under the `plugins` directory.
-
-The memcached protocol supports both the binary and the text protocol,
-automatically detecting the correct one to use.
-
-[float]
-=== Mapping REST to Memcached Protocol
-
-Memcached commands are mapped to REST and handled by the same generic
-REST layer in Elasticsearch. Here is a list of the memcached commands
-supported:
-
-[float]
-==== GET
-
-The memcached `GET` command maps to a REST `GET`. The key used is the
-URI (with parameters). The main downside is the fact that the memcached
-`GET` does not allow body in the request (and `SET` does not allow to
-return a result...). For this reason, most REST APIs (like search) allow
-to accept the "source" as a URI parameter as well.
-
-[float]
-==== SET
-
-The memcached `SET` command maps to a REST `POST`. The key used is the
-URI (with parameters), and the body maps to the REST body.
-
-[float]
-==== DELETE
-
-The memcached `DELETE` command maps to a REST `DELETE`. The key used is
-the URI (with parameters).
-
-[float]
-==== QUIT
-
-The memcached `QUIT` command is supported and disconnects the client.
-
-[float]
-=== Settings
-
-The following are the settings the can be configured for memcached:
-
-[cols="<,<",options="header",]
-|===============================================================
-|Setting |Description
-|`memcached.port` |A bind port range. Defaults to `11211-11311`.
-|===============================================================
-
-It also uses the common
-<<modules-network,network settings>>.
-
-[float]
-=== Disable memcached
-
-The memcached module can be completely disabled and not started using by
-setting `memcached.enabled` to `false`. By default it is enabled once it
-is detected as a plugin.
diff --git a/docs/reference/modules/ml-node.asciidoc b/docs/reference/modules/ml-node.asciidoc
deleted file mode 100644
index 365a08dfe7bb1..0000000000000
--- a/docs/reference/modules/ml-node.asciidoc
+++ /dev/null
@@ -1,116 +0,0 @@
-[float]
-[[modules-node-xpack]]
-== [xpack]#X-Pack node settings#
-
-//This content is referenced from the elastic/elasticsearch/docs/reference/modules/node.asciidoc
-
-If {xpack} is installed, there is an additional node type:
-
-<<ml-node,Machine learning node>>::
-
-A node that has `xpack.ml.enabled` and `node.ml` set to `true`, which is the
-default behavior when {xpack} is installed. If you want to use {ml-features}, there must be at least one {ml} node in your cluster. For more
-information about {ml-features},
-see {stack-ov}/xpack-ml.html[Machine learning in the {stack}].
-
-IMPORTANT: Do not set use the `node.ml` setting unless {xpack} is installed.
-Otherwise, the node fails to start.
-
-If {xpack} is installed, nodes are master-eligible, data, ingest, and {ml}
-nodes by default. As the cluster grows and in particular if you have large
-{ml} jobs, consider separating dedicated master-eligible nodes from dedicated
-data nodes and dedicated {ml} nodes.
-
-To create a dedicated master-eligible node when {xpack} is installed, set:
-
-[source,yaml]
--------------------
-node.master: true <1>
-node.data: false <2>
-node.ingest: false <3>
-node.ml: false <4>
-xpack.ml.enabled: true <5>
--------------------
-<1> The `node.master` role is enabled by default.
-<2> Disable the `node.data` role (enabled by default).
-<3> Disable the `node.ingest` role (enabled by default).
-<4> Disable the `node.ml` role (enabled by default in {xpack}).
-<5> The `xpack.ml.enabled` setting is enabled by default in {xpack}.
-
-To create a dedicated data node when {xpack} is installed, set:
-
-[source,yaml]
--------------------
-node.master: false <1>
-node.data: true <2>
-node.ingest: false <3>
-node.ml: false <4>
--------------------
-<1> Disable the `node.master` role (enabled by default).
-<2> The `node.data` role is enabled by default.
-<3> Disable the `node.ingest` role (enabled by default).
-<4> Disable the `node.ml` role (enabled by default in {xpack}).
-
-To create a dedicated ingest node when {xpack} is installed, set:
-
-[source,yaml]
--------------------
-node.master: false <1>
-node.data: false <2>
-node.ingest: true <3>
-cluster.remote.connect: false <4>
-node.ml: false <5>
--------------------
-<1> Disable the `node.master` role (enabled by default).
-<2> Disable the `node.data` role (enabled by default).
-<3> The `node.ingest` role is enabled by default.
-<4> Disable {ccs} (enabled by default).
-<5> Disable the `node.ml` role (enabled by default in {xpack}).
-
-To create a dedicated coordinating node when {xpack} is installed, set:
-
-[source,yaml]
--------------------
-node.master: false <1>
-node.data: false <2>
-node.ingest: false <3>
-cluster.remote.connect: false <4>
-node.ml: false <5>
--------------------
-<1> Disable the `node.master` role (enabled by default).
-<2> Disable the `node.data` role (enabled by default).
-<3> Disable the `node.ingest` role (enabled by default).
-<4> Disable {ccs} (enabled by default).
-<5> Disable the `node.ml` role (enabled by default in {xpack}).
-
-[float]
-[[ml-node]]
-=== [xpack]#Machine learning node#
-
-The {ml-features} provide {ml} nodes, which run jobs and handle {ml} API
-requests. If `xpack.ml.enabled` is set to true and `node.ml` is set to `false`,
-the node can service API requests but it cannot run jobs.
-
-If you want to use {ml-features} in your cluster, you must enable {ml}
-(set `xpack.ml.enabled` to `true`) on all master-eligible nodes. Do not use
-these settings if you do not have {xpack} installed.
-
-For more information about these settings, see <<ml-settings>>.
-
-To create a dedicated {ml} node, set:
-
-[source,yaml]
--------------------
-node.master: false <1>
-node.data: false <2>
-node.ingest: false <3>
-cluster.remote.connect: false <4>
-node.ml: true <5>
-xpack.ml.enabled: true <6>
--------------------
-<1> Disable the `node.master` role (enabled by default).
-<2> Disable the `node.data` role (enabled by default).
-<3> Disable the `node.ingest` role (enabled by default).
-<4> Disable {ccs} (enabled by default).
-<5> The `node.ml` role is enabled by default in {xpack}.
-<6> The `xpack.ml.enabled` setting is enabled by default in {xpack}.
diff --git a/docs/reference/modules/network.asciidoc b/docs/reference/modules/network.asciidoc
index e60b56fe12d74..4136f07bb9b31 100644
--- a/docs/reference/modules/network.asciidoc
+++ b/docs/reference/modules/network.asciidoc
@@ -24,7 +24,9 @@ this host to other nodes in the cluster. Accepts an IP address, hostname, a
 <<network-interface-values,special value>>, or an array of any combination of
 these. Note that any values containing a `:` (e.g., an IPv6 address or
 containing one of the <<network-interface-values,special values>>) must be
-quoted because `:` is a special character in YAML.
+quoted because `:` is a special character in YAML. `0.0.0.0` is an acceptable
+IP address and will bind to all network interfaces. The value `0` has the
+same effect as the value `0.0.0.0`.
 +
 Defaults to `_local_`.
 
@@ -173,11 +175,10 @@ settings, but may be further configured independently:
 
 TCP Transport::
 
-Used for communication between nodes in the cluster, by the Java
-{javaclient}/transport-client.html[Transport client].
+Used for communication between nodes in the cluster.
 See the <<modules-transport,Transport module>> for more information.
 
 HTTP::
 
-Exposes the JSON-over-HTTP interface used by all clients other than the Java
-clients. See the <<modules-http,HTTP module>> for more information.
+Exposes the JSON-over-HTTP interface used by all clients.
+See the <<modules-http,HTTP module>> for more information.
diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc
index 031138dada3f1..a25ab22260d3d 100644
--- a/docs/reference/modules/node.asciidoc
+++ b/docs/reference/modules/node.asciidoc
@@ -2,19 +2,24 @@
 == Node
 
 Any time that you start an instance of Elasticsearch, you are starting a
-_node_. A collection of connected nodes is  called a
+_node_. A collection of connected nodes is called a
 <<modules-cluster,cluster>>. If you are running a single node of Elasticsearch,
 then you have a cluster of one node.
 
 Every node in the cluster can handle <<modules-http,HTTP>> and
 <<modules-transport,Transport>> traffic by default. The transport layer
-is used exclusively for communication between nodes and the
-{javaclient}/transport-client.html[Java `TransportClient`]; the HTTP layer is
-used only by external REST clients.
+is used exclusively for communication between nodes; the HTTP layer is
+used by REST clients.
 
 All nodes know about all the other nodes in the cluster and can forward client
-requests to the appropriate node. Besides that, each node serves one or more
-purpose:
+requests to the appropriate node. 
+
+By default, a node is all of the following types: master-eligible, data, ingest,
+and machine learning (if available).
+
+TIP: As the cluster grows and in particular if you have large {ml} jobs,
+consider separating dedicated master-eligible nodes from dedicated data nodes
+and dedicated {ml} nodes.
 
 <<master-node,Master-eligible node>>::
 
@@ -35,6 +40,17 @@ and enrich the document before indexing. With a heavy ingest load, it makes
 sense to use dedicated ingest nodes and to mark the master and data nodes as
 `node.ingest: false`.
 
+<<ml-node,Machine learning node>>::
+
+A node that has `xpack.ml.enabled` and `node.ml` set to `true`, which is the
+default behavior in the {es} {default-dist}. If you want to use {ml-features},
+there must be at least one {ml} node in your cluster. For more information about
+{ml-features}, see
+{stack-ov}/xpack-ml.html[Machine learning in the {stack}].
++
+IMPORTANT: If you use the {oss-dist}, do not set `node.ml`. Otherwise, the node
+fails to start.
+
 [NOTE]
 [[coordinating-node]]
 .Coordinating node
@@ -68,8 +84,9 @@ creating or deleting an index, tracking which nodes are part of the cluster,
 and deciding which shards to allocate to which nodes. It is important for
 cluster health to have a stable master node.
 
-Any master-eligible node (all nodes by default) may be elected to become the
-master node by the <<modules-discovery,master election process>>.
+Any master-eligible node that is not a <<voting-only-node,voting-only node>> may
+be elected to become the master node by the <<modules-discovery,master election
+process>>.
 
 IMPORTANT: Master nodes must have access to the `data/` directory (just like
 `data` nodes) as this is where the cluster state is persisted between node restarts.
@@ -86,7 +103,27 @@ better _not_ to use dedicated master nodes for this purpose. It is important
 for the stability of the cluster that master-eligible nodes do as little work
 as possible.
 
-To create a dedicated master-eligible node, set:
+To create a dedicated master-eligible node in the {default-dist}, set:
+
+[source,yaml]
+-------------------
+node.master: true <1>
+node.voting_only: false <2>
+node.data: false <3>
+node.ingest: false <4>
+node.ml: false <5>
+xpack.ml.enabled: true <6>
+cluster.remote.connect: false <7>
+-------------------
+<1> The `node.master` role is enabled by default.
+<2> The `node.voting_only` role is disabled by default.
+<3> Disable the `node.data` role (enabled by default).
+<4> Disable the `node.ingest` role (enabled by default).
+<5> Disable the `node.ml` role (enabled by default).
+<6> The `xpack.ml.enabled` setting is enabled by default.
+<7> Disable {ccs} (enabled by default).
+
+To create a dedicated master-eligible node in the {oss-dist}, set:
 
 [source,yaml]
 -------------------
@@ -100,10 +137,70 @@ cluster.remote.connect: false <4>
 <3> Disable the `node.ingest` role (enabled by default).
 <4> Disable {ccs} (enabled by default).
 
-ifdef::include-xpack[]
-NOTE: These settings apply only when {xpack} is not installed. To create a
-dedicated master-eligible node when {xpack} is installed, see <<modules-node-xpack,{xpack} node settings>>.
-endif::include-xpack[]
+[float]
+[[voting-only-node]]
+==== Voting-only master-eligible node
+
+A voting-only master-eligible node is a node that participates in
+<<modules-discovery,master elections>> but which will not act as the cluster's
+elected master node. In particular, a voting-only node can serve as a tiebreaker
+in elections.
+
+It may seem confusing to use the term "master-eligible" to describe a
+voting-only node since such a node is not actually eligible to become the master
+at all. This terminology is an unfortunate consequence of history:
+master-eligible nodes are those nodes that participate in elections and perform
+certain tasks during cluster state publications, and voting-only nodes have the
+same responsibilities even if they can never become the elected master.
+
+To configure a master-eligible node as a voting-only node, set the following
+setting:
+
+[source,yaml]
+-------------------
+node.voting_only: true <1>
+-------------------
+<1> The default for `node.voting_only` is `false`.
+
+IMPORTANT: The `voting_only` role requires the {default-dist} of Elasticsearch
+and is not supported in the {oss-dist}. If you use the {oss-dist} and set
+`node.voting_only` then the node will fail to start.  Also note that only
+master-eligible nodes can be marked as voting-only.
+
+High availability (HA) clusters require at least three master-eligible nodes, at
+least two of which are not voting-only nodes. Such a cluster will be able to
+elect a master node even if one of the nodes fails.
+
+Since voting-only nodes never act as the cluster's elected master, they may
+require require less heap and a less powerful CPU than the true master nodes.
+However all master-eligible nodes, including voting-only nodes, require
+reasonably fast persistent storage and a reliable and low-latency network
+connection to the rest of the cluster, since they are on the critical path for
+<<cluster-state-publishing,publishing cluster state updates>>.
+
+Voting-only master-eligible nodes may also fill other roles in your cluster.
+For instance, a node may be both a data node and a voting-only master-eligible
+node. A _dedicated_ voting-only master-eligible nodes is a voting-only
+master-eligible node that fills no other roles in the cluster. To create a
+dedicated voting-only master-eligible node in the {default-dist}, set:
+
+[source,yaml]
+-------------------
+node.master: true <1>
+node.voting_only: true <2>
+node.data: false <3>
+node.ingest: false <4>
+node.ml: false <5>
+xpack.ml.enabled: true <6>
+cluster.remote.connect: false <7>
+-------------------
+<1> The `node.master` role is enabled by default.
+<2> Enable the `node.voting_only` role (disabled by default).
+<3> Disable the `node.data` role (enabled by default).
+<4> Disable the `node.ingest` role (enabled by default).
+<5> Disable the `node.ml` role (enabled by default).
+<6> The `xpack.ml.enabled` setting is enabled by default.
+<7> Disable {ccs} (enabled by default).
 
 [float]
 [[data-node]]
@@ -117,8 +214,24 @@ monitor these resources and to add more data nodes if they are overloaded.
 The main benefit of having dedicated data nodes is the separation of the
 master and data roles.
 
-To create a dedicated data node, set:
+To create a dedicated data node in the {default-dist}, set:
+[source,yaml]
+-------------------
+node.master: false <1>
+node.voting_only: false <2>
+node.data: true <3>
+node.ingest: false <4>
+node.ml: false <5>
+cluster.remote.connect: false <6>
+-------------------
+<1> Disable the `node.master` role (enabled by default).
+<2> The `node.voting_only` role is disabled by default.
+<3> The `node.data` role is enabled by default.
+<4> Disable the `node.ingest` role (enabled by default).
+<5> Disable the `node.ml` role (enabled by default).
+<6> Disable {ccs} (enabled by default).
 
+To create a dedicated data node in the {oss-dist}, set:
 [source,yaml]
 -------------------
 node.master: false <1>
@@ -131,11 +244,6 @@ cluster.remote.connect: false <4>
 <3> Disable the `node.ingest` role (enabled by default).
 <4> Disable {ccs} (enabled by default).
 
-ifdef::include-xpack[]
-NOTE: These settings apply only when {xpack} is not installed. To create a
-dedicated data node when {xpack} is installed, see <<modules-node-xpack,{xpack} node settings>>.
-endif::include-xpack[]
-
 [float]
 [[node-ingest-node]]
 === Ingest Node
@@ -145,7 +253,25 @@ ingest processors. Depending on the type of operations performed by the ingest
 processors and the required resources, it may make sense to have dedicated
 ingest nodes, that will only perform this specific task.
 
-To create a dedicated ingest node, set:
+To create a dedicated ingest node in the {default-dist}, set:
+
+[source,yaml]
+-------------------
+node.master: false <1>
+node.voting_only: false <2>
+node.data: false <3>
+node.ingest: true <4>
+node.ml: false <5>
+cluster.remote.connect: false <6>
+-------------------
+<1> Disable the `node.master` role (enabled by default).
+<2> The `node.voting_only` role is disabled by default.
+<3> Disable the `node.data` role (enabled by default).
+<4> The `node.ingest` role is enabled by default.
+<5> Disable the `node.ml` role (enabled by default).
+<6> Disable {ccs} (enabled by default).
+
+To create a dedicated ingest node in the {oss-dist}, set:
 
 [source,yaml]
 -------------------
@@ -159,11 +285,6 @@ cluster.remote.connect: false <4>
 <3> The `node.ingest` role is enabled by default.
 <4> Disable {ccs} (enabled by default).
 
-ifdef::include-xpack[]
-NOTE: These settings apply only when {xpack} is not installed. To create a
-dedicated ingest node when {xpack} is installed, see <<modules-node-xpack,{xpack} node settings>>.
-endif::include-xpack[]
-
 [float]
 [[coordinating-only-node]]
 === Coordinating only node
@@ -185,7 +306,25 @@ acknowledgement of cluster state updates from every node! The benefit of
 coordinating only nodes should not be overstated -- data nodes can happily
 serve the same purpose.
 
-To create a dedicated coordinating node, set:
+To create a dedicated coordinating node in the {default-dist}, set:
+
+[source,yaml]
+-------------------
+node.master: false <1>
+node.voting_only: false <2>
+node.data: false <3>
+node.ingest: false <4>
+node.ml: false <5>
+cluster.remote.connect: false <6>
+-------------------
+<1> Disable the `node.master` role (enabled by default).
+<2> The `node.voting_only` role is disabled by default.
+<3> Disable the `node.data` role (enabled by default).
+<4> Disable the `node.ingest` role (enabled by default).
+<5> Disable the `node.ml` role (enabled by default).
+<6> Disable {ccs} (enabled by default).
+
+To create a dedicated coordinating node in the {oss-dist}, set:
 
 [source,yaml]
 -------------------
@@ -199,10 +338,39 @@ cluster.remote.connect: false <4>
 <3> Disable the `node.ingest` role (enabled by default).
 <4> Disable {ccs} (enabled by default).
 
-ifdef::include-xpack[]
-NOTE: These settings apply only when {xpack} is not installed. To create a
-dedicated coordinating node when {xpack} is installed, see <<modules-node-xpack,{xpack} node settings>>.
-endif::include-xpack[]
+[float]
+[[ml-node]]
+=== [xpack]#Machine learning node#
+
+The {ml-features} provide {ml} nodes, which run jobs and handle {ml} API
+requests. If `xpack.ml.enabled` is set to true and `node.ml` is set to `false`,
+the node can service API requests but it cannot run jobs.
+
+If you want to use {ml-features} in your cluster, you must enable {ml}
+(set `xpack.ml.enabled` to `true`) on all master-eligible nodes. If you have the
+{oss-dist}, do not use these settings.
+
+For more information about these settings, see <<ml-settings>>.
+
+To create a dedicated {ml} node in the {default-dist}, set:
+
+[source,yaml]
+-------------------
+node.master: false <1>
+node.voting_only: false <2>
+node.data: false <3>
+node.ingest: false <4>
+node.ml: true <5>
+xpack.ml.enabled: true <6>
+cluster.remote.connect: false <7>
+-------------------
+<1> Disable the `node.master` role (enabled by default).
+<2> The `node.voting_only` role is disabled by default.
+<3> Disable the `node.data` role (enabled by default).
+<4> Disable the `node.ingest` role (enabled by default).
+<5> The `node.ml` role is enabled by default.
+<6> The `xpack.ml.enabled` setting is enabled by default.
+<7> Disable {ccs} (enabled by default).
 
 [float]
 [[change-node-role]]
@@ -283,7 +451,3 @@ your data! The RPM and Debian distributions do this for you already.
 More node settings can be found in <<modules,Modules>>.  Of particular note are
 the <<cluster.name,`cluster.name`>>, the <<node.name,`node.name`>> and the
 <<modules-network,network settings>>.
-
-ifdef::include-xpack[]
-include::ml-node.asciidoc[]
-endif::include-xpack[]
diff --git a/docs/reference/modules/remote-clusters.asciidoc b/docs/reference/modules/remote-clusters.asciidoc
index e654a50e6122e..52fe0ba42064e 100644
--- a/docs/reference/modules/remote-clusters.asciidoc
+++ b/docs/reference/modules/remote-clusters.asciidoc
@@ -30,24 +30,25 @@ node to the selected remote _gateway nodes_ only.
 The _gateway nodes_ selection depends on the following criteria:
 
 - *version*: Remote nodes must be compatible with the cluster they are
-registered to. This is subject to the same rules as <<rolling-upgrades>>.
-Any node can communicate with any other node on the same major version (e.g.
-6.0 can talk to any 6.x node). Only nodes on the last minor version of a
-certain major version can communicate with nodes on the following major
-version (e.g. 6.7 can communicate with 7.0, as well as any 7.x node, while
- 6.6 or earlier cannot talk to any 7.x node). Note that version compatibility
- is symmetric, meaning that if 6.7 can communicate with 7.0, 7.0 can also
- communicate with 6.7. The matrix below summarizes compatibility as described
- above.
-
-[cols="^,^,^,^,^,^"]
+registered to. This is subject to rules that are similar to those for
+<<rolling-upgrades>>. Any node can communicate with any other node on the same
+major version (e.g. 7.0 can talk to any 7.x node). Only nodes on the last minor
+version of a certain major version can communicate with nodes on the following
+major version. Note that in the 6.x series, 6.8 can communicate with any 7.x
+node, while 6.7 can only communicate with 7.0. Version compatibility  is
+symmetric, meaning that if 6.7 can communicate with 7.0, 7.0 can also
+communicate with 6.7. The matrix below summarizes compatibility as described above.
+
+[cols="^,^,^,^,^,^,^,^"]
 |====
-| Compatibility | 5.0->5.5 | 5.6 | 6.0->6.6 | 6.7 | 7.x
-| 5.0->5.5      |    Yes   | Yes |    No    | No  | No
-| 5.6           |    Yes   | Yes |    Yes   | Yes | No
-| 6.0->6.6      |    No    | Yes |    Yes   | Yes | No
-| 6.7           |    No    | Yes |    Yes   | Yes | Yes
-| 7.x           |    No    | No  |    No    | Yes | Yes
+| Compatibility | 5.0->5.5 | 5.6 | 6.0->6.6 | 6.7 | 6.8 | 7.0 | 7.1->7.x
+| 5.0->5.5      |    Yes   | Yes |    No    | No  | No  | No  |    No
+| 5.6           |    Yes   | Yes |    Yes   | Yes | Yes | No  |    No
+| 6.0->6.6      |    No    | Yes |    Yes   | Yes | Yes | No  |    No
+| 6.7           |    No    | Yes |    Yes   | Yes | Yes | Yes |    No
+| 6.8           |    No    | Yes |    Yes   | Yes | Yes | Yes |    Yes
+| 7.0           |    No    | No  |    No    | Yes | Yes | Yes |    Yes
+| 7.1->7.x      |    No    | No  |    No    | No  | Yes | Yes |    Yes
 |====
 
 - *role*: Dedicated master nodes never get selected.
diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc
index ec7916d5a3445..ed52c0958fec0 100644
--- a/docs/reference/modules/snapshots.asciidoc
+++ b/docs/reference/modules/snapshots.asciidoc
@@ -1,29 +1,32 @@
 [[modules-snapshots]]
 == Snapshot And Restore
 
+// tag::snapshot-intro[]
 A snapshot is a backup taken from a running Elasticsearch cluster. You can take
 a snapshot of individual indices or of the entire cluster and store it in a
 repository on a shared filesystem, and there are plugins that support remote
 repositories on S3, HDFS, Azure, Google Cloud Storage and more.
 
-Snapshots are taken incrementally. This means that when creating a snapshot of
-an index Elasticsearch will avoid copying any data that is already stored in
-the repository as part of an earlier snapshot of the same index. Therefore it
-can be efficient to take snapshots of your cluster quite frequently.
-
-Snapshots can be restored into a running cluster via the restore API. When
-restoring an index it is possible to alter the name of the restored index as
-well as some of its settings, allowing a great deal of flexibility in how the
-snapshot and restore functionality can be used.
-
-WARNING: It is not possible to back up an Elasticsearch cluster simply by
-taking a copy of the data directories of all of its nodes. Elasticsearch may be
-making changes to the contents of its data directories while it is running, and
-this means that copying its data directories cannot be expected to capture a
-consistent picture of their contents. Attempts to restore a cluster from such a
-backup may fail, reporting corruption and/or missing files, or may appear to
-have succeeded having silently lost some of its data. The only reliable way to
-back up a cluster is by using the snapshot and restore functionality.
+Snapshots are taken incrementally. This means that when it creates a snapshot of
+an index, Elasticsearch avoids copying any data that is already stored in the
+repository as part of an earlier snapshot of the same index. Therefore it can be
+efficient to take snapshots of your cluster quite frequently.
+
+You can restore snapshots into a running cluster via the
+<<restore-snapshot,restore API>>. When you restore an index, you can alter the
+name of the restored index as well as some of its settings. There is a great
+deal of flexibility in how the snapshot and restore functionality can be used.
+
+WARNING: You cannot back up an Elasticsearch cluster by simply taking a copy of
+the data directories of all of its nodes. Elasticsearch may be making changes to
+the contents of its data directories while it is running; copying its data
+directories cannot be expected to capture a consistent picture of their contents.
+If you try to restore a cluster from such a backup, it may fail and report
+corruption and/or missing files. Alternatively, it may appear to have succeeded
+though it silently lost some of its data. The only reliable way to back up a
+cluster is by using the snapshot and restore functionality.
+
+// end::snapshot-intro[]
 
 [float]
 === Version compatibility
@@ -349,7 +352,11 @@ PUT /_snapshot/my_backup/snapshot_2?wait_for_completion=true
 {
   "indices": "index_1,index_2",
   "ignore_unavailable": true,
-  "include_global_state": false
+  "include_global_state": false,
+  "_meta": {
+    "taken_by": "kimchy",
+    "taken_because": "backup before upgrading"
+  }
 }
 -----------------------------------
 // CONSOLE
@@ -363,6 +370,9 @@ By setting `include_global_state` to false it's possible to prevent the cluster
 the snapshot. By default, the entire snapshot will fail if one or more indices participating in the snapshot don't have
 all primary shards available. This behaviour can be changed by setting `partial` to `true`.
 
+The `_meta` field can be used to attach arbitrary metadata to the snapshot. This may be a record of who took the snapshot,
+why it was taken, or any other data that might be useful.
+
 Snapshot names can be automatically derived using <<date-math-index-names,date math expressions>>, similarly as when creating
 new indices. Note that special characters need to be URI encoded.
 
@@ -465,6 +475,16 @@ that setting `verbose` to `false` will omit all other information about the snap
 such as status information, the number of snapshotted shards, etc.  The default
 value of the `verbose` parameter is `true`.
 
+It is also possible to retrieve snapshots from multiple repositories in one go, for example:
+[source,sh]
+-----------------------------------
+GET /_snapshot/_all
+GET /_snapshot/my_backup,my_fs_backup
+GET /_snapshot/my*/snap*
+-----------------------------------
+// CONSOLE
+// TEST[skip:no my_fs_backup]
+
 A currently running snapshot can be retrieved using the following command:
 
 [source,sh]
@@ -502,6 +522,7 @@ When a repository is unregistered, Elasticsearch only removes the reference to t
 the snapshots. The snapshots themselves are left untouched and in place.
 
 [float]
+[[restore-snapshot]]
 === Restore
 
 A snapshot can be restored using the following command:
diff --git a/docs/reference/modules/thrift.asciidoc b/docs/reference/modules/thrift.asciidoc
deleted file mode 100644
index 1ea3f81812693..0000000000000
--- a/docs/reference/modules/thrift.asciidoc
+++ /dev/null
@@ -1,25 +0,0 @@
-[[modules-thrift]]
-== Thrift
-
-The https://thrift.apache.org/[thrift] transport module allows to expose the REST interface of
-Elasticsearch using thrift. Thrift should provide better performance
-over http. Since thrift provides both the wire protocol and the
-transport, it should make using Elasticsearch more efficient (though it has limited
-documentation).
-
-Using thrift requires installing the `transport-thrift` plugin, located
-https://github.com/elastic/elasticsearch-transport-thrift[here].
-
-The thrift
-https://github.com/elastic/elasticsearch-transport-thrift/blob/master/elasticsearch.thrift[schema]
-can be used to generate thrift clients.
-
-[cols="<,<",options="header",]
-|=======================================================================
-|Setting |Description
-|`thrift.port` |The port to bind to. Defaults to 9500-9600
-
-|`thrift.frame` |Defaults to `-1`, which means no framing. Set to a
-higher value to specify the frame size (like `15mb`).
-|=======================================================================
-
diff --git a/docs/reference/monitoring/configuring-filebeat.asciidoc b/docs/reference/monitoring/configuring-filebeat.asciidoc
new file mode 100644
index 0000000000000..fd77dc860ce8a
--- /dev/null
+++ b/docs/reference/monitoring/configuring-filebeat.asciidoc
@@ -0,0 +1,187 @@
+[role="xpack"]
+[testenv="basic"]
+[[configuring-filebeat]]
+=== Collecting {es} log data with {filebeat}
+
+[subs="attributes"]
+++++
+<titleabbrev>Collecting log data with {filebeat}</titleabbrev>
+++++
+
+You can use {filebeat} to monitor the {es} log files, collect log events, and
+ship them to the monitoring cluster. Your recent logs are visible on the
+*Monitoring* page in {kib}.
+
+//NOTE: The tagged regions are re-used in the Stack Overview.
+
+. Verify that {es} is running and that the monitoring cluster is ready to
+receive data from {filebeat}.
++
+--
+TIP: In production environments, we strongly recommend using a separate cluster 
+(referred to as the _monitoring cluster_) to store the data. Using a separate 
+monitoring cluster prevents production cluster outages from impacting your 
+ability to access your monitoring data. It also prevents monitoring activities 
+from impacting the performance of your production cluster. See
+{stack-ov}/monitoring-production.html[Monitoring in a production environment].
+
+--
+
+. Enable the collection of monitoring data on your cluster.
++
+--
+include::configuring-metricbeat.asciidoc[tag=enable-collection]
+
+For more information, see <<monitoring-settings>> and <<cluster-update-settings>>.
+--
+
+. Identify which logs you want to monitor. 
++
+--
+The {filebeat} {es} module can handle
+{stack-ov}/audit-log-output.html[audit logs],
+{ref}/logging.html#deprecation-logging[deprecation logs],
+{ref}/gc-logging.html[gc logs], {ref}/logging.html[server logs], and 
+{ref}/index-modules-slowlog.html[slow logs].
+For more information about the location of your {es} logs, see the
+{ref}/path-settings.html[path.logs] setting.
+
+IMPORTANT: If there are both structured (`*.json`) and unstructured (plain text)
+versions of the logs, you must use the structured logs. Otherwise, they might
+not appear in the appropriate context in {kib}.
+
+--
+
+. {filebeat-ref}/filebeat-installation.html[Install {filebeat}] on the {es}
+nodes that contain logs that you want to monitor.
+
+. Identify where to send the log data.
++
+--
+// tag::output-elasticsearch[]
+For example, specify {es} output information for your monitoring cluster in
+the {filebeat} configuration file (`filebeat.yml`):
+
+[source,yaml]
+----------------------------------
+output.elasticsearch:
+  # Array of hosts to connect to.
+  hosts: ["http://es-mon-1:9200", "http://es-mon2:9200"] <1>
+
+  # Optional protocol and basic auth credentials.
+  #protocol: "https"
+  #username: "elastic"
+  #password: "changeme"
+----------------------------------
+<1> In this example, the data is stored on a monitoring cluster with nodes 
+`es-mon-1` and `es-mon-2`. 
+
+If you configured the monitoring cluster to use encrypted communications, you
+must access it via HTTPS. For example, use a `hosts` setting like
+`https://es-mon-1:9200`.
+
+IMPORTANT: The {es} {monitor-features} use ingest pipelines, therefore the
+cluster that stores the monitoring data must have at least one 
+<<ingest,ingest node>>.
+ 
+If {es} {security-features} are enabled on the monitoring cluster, you must
+provide a valid user ID and password so that {filebeat} can send metrics 
+successfully. 
+
+For more information about these configuration options, see 
+{filebeat-ref}/elasticsearch-output.html[Configure the {es} output].
+// end::output-elasticsearch[]
+--
+
+. Optional: Identify where to visualize the data.
++
+--
+// tag::setup-kibana[]
+{filebeat} provides example {kib} dashboards, visualizations and searches. To
+load the dashboards into the appropriate {kib} instance, specify the
+`setup.kibana` information in the {filebeat} configuration file
+(`filebeat.yml`) on each node:
+
+[source,yaml]
+----------------------------------
+setup.kibana:
+  host: "localhost:5601"
+  #username: "my_kibana_user"
+  #password: "YOUR_PASSWORD"
+----------------------------------
+
+TIP: In production environments, we strongly recommend using a dedicated {kib} 
+instance for your monitoring cluster.
+
+If {security-features} are enabled, you must provide a valid user ID and
+password so that {filebeat} can connect to {kib}: 
+
+.. Create a user on the monitoring cluster that has the 
+{stack-ov}/built-in-roles.html[`kibana_user` built-in role] or equivalent
+privileges.
+
+.. Add the `username` and `password` settings to the {es} output information in 
+the {filebeat} configuration file. The example shows a hard-coded password, but
+you should store sensitive values in the
+{filebeat-ref}/keystore.html[secrets keystore].
+
+See {filebeat-ref}/setup-kibana-endpoint.html[Configure the {kib} endpoint].
+
+// end::setup-kibana[]
+--
+
+. Enable the {es} module and set up the initial {filebeat} environment on each
+node.
++
+--
+// tag::enable-es-module[]
+For example:
+
+["source","sh",subs="attributes,callouts"]
+----------------------------------------------------------------------
+filebeat modules enable elasticsearch
+filebeat setup -e
+----------------------------------------------------------------------
+
+For more information, see  
+{filebeat-ref}/filebeat-module-elasticsearch.html[{es} module]. 
+
+// end::enable-es-module[]
+--
+
+. Configure the {es} module in {filebeat} on each node.
++
+--
+// tag::configure-es-module[]
+If the logs that you want to monitor aren't in the default location, set the
+appropriate path variables in the `modules.d/elasticsearch.yml` file. See
+{filebeat-ref}/filebeat-module-elasticsearch.html#configuring-elasticsearch-module[Configure the {es} module].
+
+IMPORTANT: If there are JSON logs, configure the `var.paths` settings to point
+to them instead of the plain text logs.
+
+// end::configure-es-module[]
+--
+
+. {filebeat-ref}/filebeat-starting.html[Start {filebeat}] on each node. 
++
+--
+NOTE: Depending on how you’ve installed {filebeat}, you might see errors related
+to file ownership or permissions when you try to run {filebeat} modules. See
+{beats-ref}/config-file-permissions.html[Config file ownership and permissions].
+
+--
+
+. Check whether the appropriate indices exist on the monitoring cluster.
++
+--
+For example, use the {ref}/cat-indices.html[cat indices] command to verify
+that there are new `filebeat-*` indices. 
+
+TIP: If you want to use the *Monitoring* UI in {kib}, there must also be 
+`.monitoring-*` indices. Those indices are generated when you collect metrics
+about {stack} products. For example, see <<configuring-metricbeat>>.
+
+--
+
+. {kibana-ref}/monitoring-data.html[View the monitoring data in {kib}].
diff --git a/docs/reference/monitoring/configuring-metricbeat.asciidoc b/docs/reference/monitoring/configuring-metricbeat.asciidoc
index df578e88da614..c768a7782053e 100644
--- a/docs/reference/monitoring/configuring-metricbeat.asciidoc
+++ b/docs/reference/monitoring/configuring-metricbeat.asciidoc
@@ -17,17 +17,16 @@ image::monitoring/images/metricbeat.png[Example monitoring architecture]
 To learn about monitoring in general, see 
 {stack-ov}/xpack-monitoring.html[Monitoring the {stack}]. 
 
-. Enable the collection of monitoring data. Set 
-`xpack.monitoring.collection.enabled` to `true` on each node in the production 
-cluster. By default, it is is disabled (`false`). 
-+ 
+//NOTE: The tagged regions are re-used in the Stack Overview.
+
+. Enable the collection of monitoring data. +
++
 --
-NOTE: You can specify this setting in either the `elasticsearch.yml` on each 
-node or across the cluster as a dynamic cluster setting. If {es} 
-{security-features} are enabled, you must have `monitor` cluster privileges to 
-view the cluster settings and `manage` cluster privileges to change them.
+// tag::enable-collection[]
+Set `xpack.monitoring.collection.enabled` to `true` on the
+production cluster. By default, it is is disabled (`false`). 
 
-For example, you can use the following APIs to review and change this setting:
+You can use the following APIs to review and change this setting:
 
 [source,js]
 ----------------------------------
@@ -42,42 +41,20 @@ PUT _cluster/settings
 ----------------------------------
 // CONSOLE 
 
-For more information, see <<monitoring-settings>> and <<cluster-update-settings>>.
---
-
-. Disable the default collection of {es} monitoring metrics. Set 
-`xpack.monitoring.elasticsearch.collection.enabled` to `false` on each node in 
-the production cluster.
-+ 
---
-NOTE: You can specify this setting in either the `elasticsearch.yml` on each 
-node or across the cluster as a dynamic cluster setting. If {es} 
-{security-features} are enabled, you must have `monitor` cluster privileges to 
+If {es} {security-features} are enabled, you must have `monitor` cluster privileges to 
 view the cluster settings and `manage` cluster privileges to change them.
 
-For example, you can use the following API to change this setting:
-
-[source,js]
-----------------------------------
-PUT _cluster/settings
-{
-  "persistent": {
-    "xpack.monitoring.elasticsearch.collection.enabled": false
-  }
-}
-----------------------------------
-// CONSOLE
-
-Leave `xpack.monitoring.enabled` set to its default value (`true`). 
+// end::enable-collection[]
+For more information, see <<monitoring-settings>> and <<cluster-update-settings>>.
 --
 
-. On each {es} node in the production cluster:
-
-.. {metricbeat-ref}/metricbeat-installation.html[Install {metricbeat}].
+. {metricbeat-ref}/metricbeat-installation.html[Install {metricbeat}] on each
+{es} node in the production cluster.
 
-.. Enable the {es} module in {metricbeat}. +
+. Enable the {es} {xpack} module in {metricbeat} on each {es} node. +
 +
 --
+// tag::enable-es-module[]
 For example, to enable the default configuration in the `modules.d` directory, 
 run the following command:
 
@@ -89,39 +66,74 @@ metricbeat modules enable elasticsearch-xpack
 For more information, see 
 {metricbeat-ref}/configuration-metricbeat.html[Specify which modules to run] and 
 {metricbeat-ref}/metricbeat-module-elasticsearch.html[{es} module]. 
---
-
-.. By default the module will collect {es} monitoring metrics from `http://localhost:9200`.
-If the local {es} node has a different address, you must specify it via the `hosts` setting
-in the `modules.d/elasticsearch-xpack.yml` file.
 
-.. If Elastic {security-features} are enabled, you must also provide a user ID
-and password so that {metricbeat} can collect metrics successfully. 
-
-... Create a user on the production cluster that has the
-{stack-ov}/built-in-roles.html[`remote_monitoring_collector` built-in role]. 
-Alternatively, use the {stack-ov}/built-in-users.html[`remote_monitoring_user` built-in user].
+// end::enable-es-module[]
+--
 
-... Add the `username` and `password` settings to the {es} module configuration
-file.
+. Configure the {es} {xpack} module in {metricbeat} on each {es} node. +
 +
 --
-For example, add the following settings in the `modules.d/elasticsearch-xpack.yml` file:
+// tag::configure-es-module[]
+The `modules.d/elasticsearch-xpack.yml` file contains the following settings:
 
 [source,yaml]
 ----------------------------------
-- module: elasticsearch
-  ...
-  username: remote_monitoring_user
-  password: YOUR_PASSWORD
+  - module: elasticsearch
+    metricsets:
+      - ccr
+      - cluster_stats
+      - index
+      - index_recovery
+      - index_summary
+      - ml_job
+      - node_stats
+      - shard
+    period: 10s
+    hosts: ["http://localhost:9200"]
+    #username: "user"
+    #password: "secret"
+    xpack.enabled: true
 ----------------------------------
+
+By default, the module collects {es} monitoring metrics from
+`http://localhost:9200`. If that host and port number are not correct, you must
+update the `hosts` setting. If you configured {es} to use encrypted
+communications, you must access it via HTTPS. For example, use a `hosts` setting
+like `https://localhost:9200`.
+// end::configure-es-module[]
+
+// tag::remote-monitoring-user[]
+If Elastic {security-features} are enabled, you must also provide a user ID
+and password so that {metricbeat} can collect metrics successfully: 
+
+.. Create a user on the production cluster that has the
+{stack-ov}/built-in-roles.html[`remote_monitoring_collector` built-in role]. 
+Alternatively, use the
+{stack-ov}/built-in-users.html[`remote_monitoring_user` built-in user].
+
+.. Add the `username` and `password` settings to the {es} module configuration
+file. 
+// end::remote-monitoring-user[]
+--
+
+. Optional: Disable the system module in {metricbeat}.
++
 --
+// tag::disable-system-module[]
+By default, the {metricbeat-ref}/metricbeat-module-system.html[system module] is
+enabled. The information it collects, however, is not shown on the *Monitoring*
+page in {kib}. Unless you want to use that information for other purposes, run
+the following command:
 
-.. If you configured {es} to use <<configuring-tls,encrypted communications>>, 
-you must access it via HTTPS. For example, use a `hosts` setting like 
-`https://localhost:9200` in the `modules.d/elasticsearch-xpack.yml` file.
+["source","sh",subs="attributes,callouts"]
+----------------------------------------------------------------------
+metricbeat modules disable system
+----------------------------------------------------------------------
 
-.. Identify where to send the monitoring data. +
+// end::disable-system-module[] 
+--
+
+. Identify where to send the monitoring data. +
 +
 --
 TIP: In production environments, we strongly recommend using a separate cluster 
@@ -136,48 +148,68 @@ configuration file (`metricbeat.yml`):
 [source,yaml]
 ----------------------------------
 output.elasticsearch:
+  # Array of hosts to connect to.
   hosts: ["http://es-mon-1:9200", "http://es-mon2:9200"] <1>
+  
+  # Optional protocol and basic auth credentials.
+  #protocol: "https"
+  #username: "elastic"
+  #password: "changeme"
 ----------------------------------
 <1> In this example, the data is stored on a monitoring cluster with nodes 
 `es-mon-1` and `es-mon-2`. 
 
+If you configured the monitoring cluster to use encrypted communications, you
+must access it via HTTPS. For example, use a `hosts` setting like
+`https://es-mon-1:9200`.
+
 IMPORTANT: The {es} {monitor-features} use ingest pipelines, therefore the
 cluster that stores the monitoring data must have at least one 
 <<ingest,ingest node>>. 
 
-For more information about these configuration options, see 
-{metricbeat-ref}/elasticsearch-output.html[Configure the {es} output].
---
+If {es} {security-features} are enabled on the monitoring cluster, you must
+provide a valid user ID and password so that {metricbeat} can send metrics 
+successfully: 
 
-.. If {es} {security-features} are enabled on the monitoring cluster, you 
-must provide a valid user ID and password so that {metricbeat} can send metrics 
-successfully. 
-
-... Create a user on the monitoring cluster that has the 
+.. Create a user on the monitoring cluster that has the 
 {stack-ov}/built-in-roles.html[`remote_monitoring_agent` built-in role]. 
 Alternatively, use the 
 {stack-ov}/built-in-users.html[`remote_monitoring_user` built-in user].
 
-... Add the `username` and `password` settings to the {es} output information in 
-the {metricbeat} configuration file (`metricbeat.yml`):
+.. Add the `username` and `password` settings to the {es} output information in 
+the {metricbeat} configuration file.
+
+For more information about these configuration options, see 
+{metricbeat-ref}/elasticsearch-output.html[Configure the {es} output].
+--
+
+. {metricbeat-ref}/metricbeat-starting.html[Start {metricbeat}] on each node. 
+
+. Disable the default collection of {es} monitoring metrics. +
 +
 --
-[source,yaml]
+// tag::disable-default-collection[]
+Set `xpack.monitoring.elasticsearch.collection.enabled` to `false` on the 
+production cluster.
+
+You can use the following API to change this setting:
+
+[source,js]
 ----------------------------------
-output.elasticsearch:
-  ...
-  username: remote_monitoring_user
-  password: YOUR_PASSWORD
+PUT _cluster/settings
+{
+  "persistent": {
+    "xpack.monitoring.elasticsearch.collection.enabled": false
+  }
+}
 ----------------------------------
---
-
-.. If you configured the monitoring cluster to use 
-<<configuring-tls,encrypted communications>>, you must access it via 
-HTTPS. For example, use a `hosts` setting like `https://es-mon-1:9200` in the 
-`metricbeat.yml` file. 
+// CONSOLE
 
-. <<starting-elasticsearch,Start {es}>>.
+If {es} {security-features} are enabled, you must have `monitor` cluster
+privileges to  view the cluster settings and `manage` cluster privileges
+to change them.
 
-. {metricbeat-ref}/metricbeat-starting.html[Start {metricbeat}]. 
+// end::disable-default-collection[]
+--
 
 . {kibana-ref}/monitoring-data.html[View the monitoring data in {kib}]. 
diff --git a/docs/reference/monitoring/configuring-monitoring.asciidoc b/docs/reference/monitoring/configuring-monitoring.asciidoc
index 9fe5b71d7a18c..e129999e3a51b 100644
--- a/docs/reference/monitoring/configuring-monitoring.asciidoc
+++ b/docs/reference/monitoring/configuring-monitoring.asciidoc
@@ -12,9 +12,12 @@ methods to collect metrics about {es}:
 * <<collecting-monitoring-data>>
 * <<configuring-metricbeat>>
 
+You can also <<configuring-filebeat,use {filebeat} to collect {es} logs>>. 
+
 To learn about monitoring in general, see 
 {stack-ov}/xpack-monitoring.html[Monitoring the {stack}]. 
 
 include::collecting-monitoring-data.asciidoc[]
 include::configuring-metricbeat.asciidoc[]
+include::configuring-filebeat.asciidoc[]
 include::indices.asciidoc[]
\ No newline at end of file
diff --git a/docs/reference/query-dsl/boosting-query.asciidoc b/docs/reference/query-dsl/boosting-query.asciidoc
index 5cd12ce1f00b7..c57235e71606d 100644
--- a/docs/reference/query-dsl/boosting-query.asciidoc
+++ b/docs/reference/query-dsl/boosting-query.asciidoc
@@ -1,36 +1,60 @@
 [[query-dsl-boosting-query]]
 === Boosting Query
 
-The `boosting` query can be used to effectively demote results that
-match a given query. Unlike the "NOT" clause in bool query, this still
-selects documents that contain undesirable terms, but reduces their
-overall score.
+Returns documents matching a `positive` query while reducing the
+<<query-filter-context, relevance score>> of documents that also match a
+`negative` query.
 
-It accepts a `positive` query and a `negative` query.
-Only documents that match the `positive` query will be included
-in the results list, but documents that also match the `negative` query
-will be downgraded by multiplying the original `_score` of the document
-with the `negative_boost`.
+You can use the `boosting` query to demote certain documents without
+excluding them from the search results.
+
+[[boosting-query-ex-request]]
+==== Example request
 
 [source,js]
---------------------------------------------------
+----
 GET /_search
 {
     "query": {
         "boosting" : {
             "positive" : {
                 "term" : {
-                    "field1" : "value1"
+                    "text" : "apple"
                 }
             },
             "negative" : {
                  "term" : {
-                     "field2" : "value2"
+                     "text" : "pie tart fruit crumble tree"
                 }
             },
-            "negative_boost" : 0.2
+            "negative_boost" : 0.5
         }
     }
 }
---------------------------------------------------
+----
 // CONSOLE
+
+[[boosting-top-level-params]]
+==== Top-level parameters for `boosting`
+
+`positive` (Required)::
+Query you wish to run. Any returned documents must match this query.
+
+`negative` (Required)::
++
+--
+Query used to decrease the <<query-filter-context, relevance score>> of matching
+documents.
+
+If a returned document matches the `positive` query and this query, the
+`boosting` query calculates the final <<query-filter-context, relevance score>>
+for the document as follows:
+
+. Take the original relevance score from the `positive` query.
+. Multiply the score by the `negative_boost` value.
+--
+
+`negative_boost` (Required)::
+Floating point number between `0` and `1.0` used to decrease the
+<<query-filter-context, relevance scores>> of documents matching the `negative`
+query.
\ No newline at end of file
diff --git a/docs/reference/query-dsl/common-terms-query.asciidoc b/docs/reference/query-dsl/common-terms-query.asciidoc
deleted file mode 100644
index 87288778246a6..0000000000000
--- a/docs/reference/query-dsl/common-terms-query.asciidoc
+++ /dev/null
@@ -1,299 +0,0 @@
-[[query-dsl-common-terms-query]]
-=== Common Terms Query
-
-The `common` terms query is a modern alternative to stopwords which
-improves the precision and recall of search results (by taking stopwords
-into account), without sacrificing performance.
-
-[float]
-==== The problem
-
-Every term in a query has a cost. A search for `"The brown fox"`
-requires three term queries, one for each of `"the"`, `"brown"` and
-`"fox"`, all of which are executed against all documents in the index.
-The query for `"the"` is likely to match many documents and thus has a
-much smaller impact on relevance than the other two terms.
-
-Previously, the solution to this problem was to ignore terms with high
-frequency. By treating `"the"` as a _stopword_, we reduce the index size
-and reduce the number of term queries that need to be executed.
-
-The problem with this approach is that, while stopwords have a small
-impact on relevance, they are still important. If we remove stopwords,
-we lose precision, (eg we are unable to distinguish between `"happy"`
-and `"not happy"`) and we lose recall (eg text like `"The The"` or
-`"To be or not to be"` would simply not exist in the index).
-
-[float]
-==== The solution
-
-The `common` terms query divides the query terms into two groups: more
-important (ie _low frequency_ terms) and less important (ie _high
-frequency_ terms which would previously have been stopwords).
-
-First it searches for documents which match the more important terms.
-These are the terms which appear in fewer documents and have a greater
-impact on relevance.
-
-Then, it executes a second query for the less important terms -- terms
-which appear frequently and have a low impact on relevance. But instead
-of calculating the relevance score for *all* matching documents, it only
-calculates the `_score` for documents already matched by the first
-query. In this way the high frequency terms can improve the relevance
-calculation without paying the cost of poor performance.
-
-If a query consists only of high frequency terms, then a single query is
-executed as an `AND` (conjunction) query, in other words all terms are
-required. Even though each individual term will match many documents,
-the combination of terms narrows down the resultset to only the most
-relevant. The single query can also be executed as an `OR` with a
-specific
-<<query-dsl-minimum-should-match,`minimum_should_match`>>,
-in this case a high enough value should probably be used.
-
-Terms are allocated to the high or low frequency groups based on the
-`cutoff_frequency`, which can be specified as an absolute frequency
-(`>=1`) or as a relative frequency (`0.0 .. 1.0`). (Remember that document
-frequencies are computed on a per shard level as explained in the blog post
-{defguide}/relevance-is-broken.html[Relevance is broken].)
-
-Perhaps the most interesting property of this query is that it adapts to
-domain specific stopwords automatically. For example, on a video hosting
-site, common terms like `"clip"` or `"video"` will automatically behave
-as stopwords without the need to maintain a manual list.
-
-[float]
-==== Examples
-
-In this example, words that have a document frequency greater than 0.1%
-(eg `"this"` and `"is"`) will be treated as _common terms_.
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query": {
-        "common": {
-            "body": {
-                "query": "this is bonsai cool",
-                "cutoff_frequency": 0.001
-            }
-        }
-    }
-}
---------------------------------------------------
-// CONSOLE
-
-The number of terms which should match can be controlled with the
-<<query-dsl-minimum-should-match,`minimum_should_match`>>
-(`high_freq`, `low_freq`), `low_freq_operator` (default `"or"`) and
-`high_freq_operator` (default `"or"`) parameters.
-
-For low frequency terms, set the `low_freq_operator` to `"and"` to make
-all terms required:
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query": {
-        "common": {
-            "body": {
-                "query": "nelly the elephant as a cartoon",
-                "cutoff_frequency": 0.001,
-                "low_freq_operator": "and"
-            }
-        }
-    }
-}
---------------------------------------------------
-// CONSOLE
-
-which is roughly equivalent to:
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query": {
-        "bool": {
-            "must": [
-            { "term": { "body": "nelly"}},
-            { "term": { "body": "elephant"}},
-            { "term": { "body": "cartoon"}}
-            ],
-            "should": [
-            { "term": { "body": "the"}},
-            { "term": { "body": "as"}},
-            { "term": { "body": "a"}}
-            ]
-        }
-    }
-}
---------------------------------------------------
-// CONSOLE
-
-Alternatively use
-<<query-dsl-minimum-should-match,`minimum_should_match`>>
-to specify a minimum number or percentage of low frequency terms which
-must be present, for instance:
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query": {
-        "common": {
-            "body": {
-                "query": "nelly the elephant as a cartoon",
-                "cutoff_frequency": 0.001,
-                "minimum_should_match": 2
-            }
-        }
-    }
-}
---------------------------------------------------
-// CONSOLE
-
-which is roughly equivalent to:
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query": {
-        "bool": {
-            "must": {
-                "bool": {
-                    "should": [
-                    { "term": { "body": "nelly"}},
-                    { "term": { "body": "elephant"}},
-                    { "term": { "body": "cartoon"}}
-                    ],
-                    "minimum_should_match": 2
-                }
-            },
-            "should": [
-                { "term": { "body": "the"}},
-                { "term": { "body": "as"}},
-                { "term": { "body": "a"}}
-                ]
-        }
-    }
-}
---------------------------------------------------
-// CONSOLE
-
-A different
-<<query-dsl-minimum-should-match,`minimum_should_match`>>
-can be applied for low and high frequency terms with the additional
-`low_freq` and `high_freq` parameters. Here is an example when providing
-additional parameters (note the change in structure):
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query": {
-        "common": {
-            "body": {
-                "query": "nelly the elephant not as a cartoon",
-                "cutoff_frequency": 0.001,
-                "minimum_should_match": {
-                    "low_freq" : 2,
-                    "high_freq" : 3
-                }
-            }
-        }
-    }
-}
---------------------------------------------------
-// CONSOLE
-
-which is roughly equivalent to:
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query": {
-        "bool": {
-            "must": {
-                "bool": {
-                    "should": [
-                    { "term": { "body": "nelly"}},
-                    { "term": { "body": "elephant"}},
-                    { "term": { "body": "cartoon"}}
-                    ],
-                    "minimum_should_match": 2
-                }
-            },
-            "should": {
-                "bool": {
-                    "should": [
-                    { "term": { "body": "the"}},
-                    { "term": { "body": "not"}},
-                    { "term": { "body": "as"}},
-                    { "term": { "body": "a"}}
-                    ],
-                    "minimum_should_match": 3
-                }
-            }
-        }
-    }
-}
---------------------------------------------------
-// CONSOLE
-
-In this case it means the high frequency terms have only an impact on
-relevance when there are at least three of them. But the most
-interesting use of the
-<<query-dsl-minimum-should-match,`minimum_should_match`>>
-for high frequency terms is when there are only high frequency terms:
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query": {
-        "common": {
-            "body": {
-                "query": "how not to be",
-                "cutoff_frequency": 0.001,
-                "minimum_should_match": {
-                    "low_freq" : 2,
-                    "high_freq" : 3
-                }
-            }
-        }
-    }
-}
---------------------------------------------------
-// CONSOLE
-
-which is roughly equivalent to:
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query": {
-        "bool": {
-            "should": [
-            { "term": { "body": "how"}},
-            { "term": { "body": "not"}},
-            { "term": { "body": "to"}},
-            { "term": { "body": "be"}}
-            ],
-            "minimum_should_match": "3<50%"
-        }
-    }
-}
---------------------------------------------------
-// CONSOLE
-
-The high frequency generated query is then slightly less restrictive
-than with an `AND`.
-
-The `common` terms query also supports `boost` and `analyzer` as
-parameters.
diff --git a/docs/reference/query-dsl/constant-score-query.asciidoc b/docs/reference/query-dsl/constant-score-query.asciidoc
index aa7ee60aa5c6f..bfcece8d62e2f 100644
--- a/docs/reference/query-dsl/constant-score-query.asciidoc
+++ b/docs/reference/query-dsl/constant-score-query.asciidoc
@@ -1,12 +1,12 @@
 [[query-dsl-constant-score-query]]
 === Constant Score Query
 
-A query that wraps another query and simply returns a
-constant score equal to the query boost for every document in the
-filter. Maps to Lucene `ConstantScoreQuery`.
+Wraps a <<query-dsl-bool-query, filter query>> and returns every matching
+document with a <<query-filter-context, relevance score>> equal to the `boost`
+parameter value.
 
 [source,js]
---------------------------------------------------
+----
 GET /_search
 {
     "query": {
@@ -18,8 +18,22 @@ GET /_search
         }
     }
 }
---------------------------------------------------
+----
 // CONSOLE
 
-Filter clauses are executed in <<query-filter-context,filter context>>,
-meaning that scoring is ignored and clauses are considered for caching.
+[[constant-score-top-level-params]]
+==== Top-level parameters for `constant_score`
+`filter`::
++
+--
+<<query-dsl-bool-query, Filter query>> you wish to run. Any returned documents
+must match this query. Required.
+
+Filter queries do not calculate <<query-filter-context, relevance scores>>. To
+speed up performance, {es} automatically caches frequently used filter queries.
+--
+
+`boost`::
+Floating point number used as the constant <<query-filter-context, relevance
+score>> for every document matching the `filter` query. Default is `1.0`.
+Optional.
\ No newline at end of file
diff --git a/docs/reference/query-dsl/dis-max-query.asciidoc b/docs/reference/query-dsl/dis-max-query.asciidoc
index f05f97107a0a4..9a0f1fb7b039c 100644
--- a/docs/reference/query-dsl/dis-max-query.asciidoc
+++ b/docs/reference/query-dsl/dis-max-query.asciidoc
@@ -1,48 +1,63 @@
 [[query-dsl-dis-max-query]]
-=== Dis Max Query
-
-A query that generates the union of documents produced by its
-subqueries, and that scores each document with the maximum score for
-that document as produced by any subquery, plus a tie breaking increment
-for any additional matching subqueries.
-
-This is useful when searching for a word in multiple fields with
-different boost factors (so that the fields cannot be combined
-equivalently into a single search field). We want the primary score to
-be the one associated with the highest boost, not the sum of the field
-scores (as Boolean Query would give). If the query is "albino elephant"
-this ensures that "albino" matching one field and "elephant" matching
-another gets a higher score than "albino" matching both fields. To get
-this result, use both Boolean Query and DisjunctionMax Query: for each
-term a DisjunctionMaxQuery searches for it in each field, while the set
-of these DisjunctionMaxQuery's is combined into a BooleanQuery.
-
-The tie breaker capability allows results that include the same term in
-multiple fields to be judged better than results that include this term
-in only the best of those multiple fields, without confusing this with
-the better case of two different terms in the multiple fields. The
-default `tie_breaker` is `0.0`.
-
-This query maps to Lucene `DisjunctionMaxQuery`.
+=== Disjunction Max Query
+
+Returns documents matching one or more wrapped queries, called query clauses or
+clauses.
+
+If a returned document matches multiple query clauses, the `dis_max` query
+assigns the document the highest relevance score from any matching clause, plus
+a tie breaking increment for any additional matching subqueries.
+
+You can use the `dis_max` to search for a term in fields mapped with different
+<<mapping-boost,boost>> factors.
+
+[[query-dsl-dis-max-query-ex-request]]
+==== Example request
 
 [source,js]
---------------------------------------------------
+----
 GET /_search
 {
     "query": {
         "dis_max" : {
-            "tie_breaker" : 0.7,
-            "boost" : 1.2,
             "queries" : [
-                {
-                    "term" : { "age" : 34 }
-                },
-                {
-                    "term" : { "age" : 35 }
-                }
-            ]
+                { "term" : { "title" : "Quick pets" }},
+                { "term" : { "body" : "Quick pets" }}
+            ],
+            "tie_breaker" : 0.7
         }
     }
 }    
---------------------------------------------------
+----
 // CONSOLE
+
+[[query-dsl-dis-max-query-top-level-params]]
+==== Top-level parameters for `dis_max`
+
+`queries` (Required)::
+(array of query objects) Contains one or more query clauses. Returned documents
+**must match one or more** of these queries. If a document matches multiple
+queries, {es} uses the highest <<query-filter-context, relevance score>>.
+
+`tie_breaker` (Optional)::
++
+--
+(float) Floating point number between `0` and `1.0` used to increase the
+<<query-filter-context, relevance scores>> of documents matching multiple query
+clauses. Defaults to `0.0`.
+
+You can use the `tie_breaker` value to assign higher relevance scores to
+documents that contain the same term in multiple fields than documents that
+contain this term in only the best of those multiple fields, without confusing
+this with the better case of two different terms in the multiple fields.
+
+If a document matches multiple clauses, the `dis_max` query calculates the
+relevance score for the document as follows:
+
+. Take the relevance score from a matching clause with the highest score.
+. Multiply the score from any other matching clauses by the `tie_breaker` value.
+. Add the highest score to the multiplied scores.
+
+If the `tie_breaker` value is greater than `0.0`, all matching clauses count,
+but the clause with the highest score counts most.
+--
\ No newline at end of file
diff --git a/docs/reference/query-dsl/exists-query.asciidoc b/docs/reference/query-dsl/exists-query.asciidoc
index 9c7553e314c43..f35e97db8ec18 100644
--- a/docs/reference/query-dsl/exists-query.asciidoc
+++ b/docs/reference/query-dsl/exists-query.asciidoc
@@ -1,8 +1,14 @@
 [[query-dsl-exists-query]]
 === Exists Query
 
-Returns documents that contain a value other than `null` or `[]` in a provided
-field.
+Returns documents that contain an indexed value for a field.
+
+An indexed value may not exist for a document's field due to a variety of reasons:
+
+* The field in the source JSON is `null` or `[]`
+* The field has `"index" : false` set in the mapping
+* The length of the field value exceeded an `ignore_above` setting in the mapping
+* The field value was malformed and `ignore_malformed` was defined in the mapping
 
 [[exists-query-ex-request]]
 ==== Example request
@@ -25,8 +31,7 @@ GET /_search
 `field`::
 Name of the field you wish to search.
 +
-To return a document, this field must exist and contain a value other
-than `null` or `[]`. These values can include:
+While a field is deemed non-existant if the JSON value is `null` or `[]`, these values will indicate the field does exist:
 +
 * Empty strings, such as `""` or `"-"`
 * Arrays containing `null` and another value, such as `[null, "foo"]`
@@ -36,13 +41,13 @@ than `null` or `[]`. These values can include:
 ==== Notes
 
 [[find-docs-null-values]]
-===== Find documents with null values
-To find documents that contain only `null` values or `[]` in a provided field,
+===== Find documents missing indexed values
+To find documents that are missing an indexed value for a field,
 use the `must_not` <<query-dsl-bool-query, boolean query>> with the `exists`
 query.
 
-The following search returns documents that contain only `null` values or `[]`
-in the `user` field.
+The following search returns documents that are missing an indexed value for
+the `user` field.
 
 [source,js]
 ----
diff --git a/docs/reference/query-dsl/full-text-queries.asciidoc b/docs/reference/query-dsl/full-text-queries.asciidoc
index 0af99b61f194f..8fc53bc7e9b8a 100644
--- a/docs/reference/query-dsl/full-text-queries.asciidoc
+++ b/docs/reference/query-dsl/full-text-queries.asciidoc
@@ -29,10 +29,6 @@ The queries in this group are:
 
     The multi-field version of the `match` query.
 
-<<query-dsl-common-terms-query,`common` terms query>>::
-
-    A more specialized query which gives more preference to uncommon words.
-
 <<query-dsl-query-string-query,`query_string` query>>::
 
     Supports the compact Lucene <<query-string-syntax,query string syntax>>,
@@ -59,8 +55,6 @@ include::match-bool-prefix-query.asciidoc[]
 
 include::multi-match-query.asciidoc[]
 
-include::common-terms-query.asciidoc[]
-
 include::query-string-query.asciidoc[]
 
 include::simple-query-string-query.asciidoc[]
diff --git a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
index 487e944c09e10..1a088a350145f 100644
--- a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
+++ b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
@@ -39,7 +39,7 @@ Then the following simple query can be executed with a
 
 [source,js]
 --------------------------------------------------
-GET /_search
+GET my_locations/_search
 {
     "query": {
         "bool" : {
@@ -94,7 +94,7 @@ representations of the geo point, the filter can accept it as well:
 
 [source,js]
 --------------------------------------------------
-GET /_search
+GET my_locations/_search
 {
     "query": {
         "bool" : {
@@ -129,7 +129,7 @@ conform with http://geojson.org/[GeoJSON].
 
 [source,js]
 --------------------------------------------------
-GET /_search
+GET my_locations/_search
 {
     "query": {
         "bool" : {
@@ -157,7 +157,7 @@ Format in `lat,lon`.
 
 [source,js]
 --------------------------------------------------
-GET /_search
+GET my_locations/_search
 {
     "query": {
         "bool" : {
@@ -183,7 +183,7 @@ GET /_search
 
 [source,js]
 --------------------------------------------------
-GET /_search
+GET my_locations/_search
 {
     "query": {
         "bool" : {
@@ -208,7 +208,7 @@ GET /_search
 
 [source,js]
 --------------------------------------------------
-GET /_search
+GET my_locations/_search
 {
     "query": {
         "bool" : {
@@ -243,7 +243,7 @@ geohash the geohash can be specified in both `top_left` and
 
 [source,js]
 --------------------------------------------------
-GET /_search
+GET my_locations/_search
 {
     "query": {
         "geo_bounding_box" : {
@@ -273,7 +273,7 @@ values separately.
 
 [source,js]
 --------------------------------------------------
-GET /_search
+GET my_locations/_search
 {
     "query": {
         "bool" : {
@@ -323,7 +323,7 @@ are not supported. Here is an example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
+GET my_locations/_search
 {
     "query": {
         "bool" : {
diff --git a/docs/reference/query-dsl/intervals-query.asciidoc b/docs/reference/query-dsl/intervals-query.asciidoc
index f5788783f7e63..7353ca137f3e1 100644
--- a/docs/reference/query-dsl/intervals-query.asciidoc
+++ b/docs/reference/query-dsl/intervals-query.asciidoc
@@ -40,7 +40,6 @@ POST _search
             }
           ]
         },
-        "boost" : 2.0,
         "_name" : "favourite_food"
       }
     }
@@ -83,6 +82,53 @@ to search across multiple fields as if they were all the same field; for example
 you could index the same text into stemmed and unstemmed fields, and search for
 stemmed tokens near unstemmed ones.
 
+[[intervals-prefix]]
+==== `prefix`
+
+The `prefix` rule finds terms that start with a specified prefix.  The prefix will
+expand to match at most 128 terms; if there are more matching terms in the index,
+then an error will be returned.  To avoid this limit, enable the
+<<index-prefixes,`index-prefixes`>> option on the field being searched.
+
+[horizontal]
+`prefix`::
+Match terms starting with this prefix
+`analyzer`::
+Which analyzer should be used to normalize the `prefix`.  By default, the
+search analyzer of the top-level field will be used.
+`use_field`::
+If specified, then match intervals from this field rather than the top-level field.
+The `prefix` will be normalized using the search analyzer from this field, unless
+`analyzer` is specified separately.
+
+[[intervals-wildcard]]
+==== `wildcard`
+
+The `wildcard` rule finds terms that match a wildcard pattern.  The pattern will
+expand to match at most 128 terms; if there are more matching terms in the index,
+then an error will be returned.
+
+[horizontal]
+`pattern`::
+Find terms matching this pattern
++
+--
+This parameter supports two wildcard operators:
+
+* `?`, which matches any single character
+* `*`, which can match zero or more characters, including an empty one
+
+WARNING: Avoid beginning patterns with `*` or `?`. This can increase
+the iterations needed to find matching terms and slow search performance.
+--
+`analyzer`::
+Which analyzer should be used to normalize the `pattern`.  By default, the
+search analyzer of the top-level field will be used.
+`use_field`::
+If specified, then match intervals from this field rather than the top-level field.
+The `pattern` will be normalized using the search analyzer from this field, unless
+`analyzer` is specified separately.
+
 [[intervals-all_of]]
 ==== `all_of`
 
@@ -298,4 +344,4 @@ POST _search
   }
 }
 --------------------------------------------------
-// CONSOLE
\ No newline at end of file
+// CONSOLE
diff --git a/docs/reference/query-dsl/match-query.asciidoc b/docs/reference/query-dsl/match-query.asciidoc
index 89a0a942b79ce..4fcb40a76ec9c 100644
--- a/docs/reference/query-dsl/match-query.asciidoc
+++ b/docs/reference/query-dsl/match-query.asciidoc
@@ -75,7 +75,8 @@ rewritten.
 Fuzzy transpositions (`ab` -> `ba`) are allowed by default but can be disabled
 by setting `fuzzy_transpositions` to `false`.
 
-Note that fuzzy matching is not applied to terms with synonyms, as under the hood
+NOTE: Fuzzy matching is not applied to terms with synonyms or in cases where the
+analysis process produces multiple tokens at the same position. Under the hood
 these terms are expanded to a special synonym query that blends term frequencies,
 which does not support fuzzy expansion.
 
@@ -119,50 +120,6 @@ GET /_search
 --------------------------------------------------
 // CONSOLE
 
-[[query-dsl-match-query-cutoff]]
-===== Cutoff frequency
-
-The match query supports a `cutoff_frequency` that allows
-specifying an absolute or relative document frequency where high
-frequency terms are moved into an optional subquery and are only scored
-if one of the low frequency (below the cutoff) terms in the case of an
-`or` operator or all of the low frequency terms in the case of an `and`
-operator match.
-
-This query allows handling `stopwords` dynamically at runtime, is domain
-independent and doesn't require a stopword file. It prevents scoring /
-iterating high frequency terms and only takes the terms into account if a
-more significant / lower frequency term matches a document. Yet, if all
-of the query terms are above the given `cutoff_frequency` the query is
-automatically transformed into a pure conjunction (`and`) query to
-ensure fast execution.
-
-The `cutoff_frequency` can either be relative to the total number of
-documents if in the range `[0..1)` or absolute if greater or equal to
-`1.0`.
-
-Here is an example showing a query composed of stopwords exclusively:
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query": {
-        "match" : {
-            "message" : {
-                "query" : "to be or not to be",
-                "cutoff_frequency" : 0.001
-            }
-        }
-    }
-}
---------------------------------------------------
-// CONSOLE
-
-IMPORTANT: The `cutoff_frequency` option operates on a per-shard-level. This means
-that when trying it out on test indexes with low document numbers you
-should follow the advice in {defguide}/relevance-is-broken.html[Relevance is broken].
-
 [[query-dsl-match-query-synonyms]]
 ===== Synonyms
 
diff --git a/docs/reference/query-dsl/mlt-query.asciidoc b/docs/reference/query-dsl/mlt-query.asciidoc
index 509b4a9b44094..e59cd24292d48 100644
--- a/docs/reference/query-dsl/mlt-query.asciidoc
+++ b/docs/reference/query-dsl/mlt-query.asciidoc
@@ -232,9 +232,8 @@ The syntax is the same as the <<query-dsl-minimum-should-match,minimum should ma
 `fail_on_unsupported_field`::
 Controls whether the query should fail (throw an exception) if any of the 
 specified fields are not of the supported types
-(`text` or `keyword'). Set this to `false` to ignore the field and continue
-processing. Defaults to
-`true`.
+(`text` or `keyword`). Set this to `false` to ignore the field and continue
+processing. Defaults to `true`.
 
 `boost_terms`::
 Each term in the formed query could be further boosted by their tf-idf score.
diff --git a/docs/reference/query-dsl/multi-match-query.asciidoc b/docs/reference/query-dsl/multi-match-query.asciidoc
index 9f574ed814d3c..e7bcd799df702 100644
--- a/docs/reference/query-dsl/multi-match-query.asciidoc
+++ b/docs/reference/query-dsl/multi-match-query.asciidoc
@@ -151,8 +151,8 @@ follows:
 
 Also, accepts `analyzer`, `boost`, `operator`, `minimum_should_match`,
 `fuzziness`, `lenient`, `prefix_length`, `max_expansions`, `rewrite`, `zero_terms_query`,
- `cutoff_frequency`, `auto_generate_synonyms_phrase_query` and `fuzzy_transpositions`,
-  as explained in <<query-dsl-match-query, match query>>.
+`auto_generate_synonyms_phrase_query` and `fuzzy_transpositions`,
+as explained in <<query-dsl-match-query, match query>>.
 
 [IMPORTANT]
 [[operator-min]]
@@ -247,9 +247,7 @@ The score from each `match` clause is added together, then divided by the
 number of `match` clauses.
 
 Also, accepts `analyzer`, `boost`, `operator`, `minimum_should_match`,
-`fuzziness`, `lenient`, `prefix_length`, `max_expansions`, `rewrite`, `zero_terms_query`
-and `cutoff_frequency`, as explained in <<query-dsl-match-query,match query>>, but
-*see <<operator-min>>*.
+`fuzziness`, `lenient`, `prefix_length`, `max_expansions`, `rewrite`, and `zero_terms_query`.
 
 [[type-phrase]]
 ==== `phrase` and `phrase_prefix`
@@ -292,9 +290,9 @@ GET /_search
 --------------------------------------------------
 // CONSOLE
 
-Also, accepts `analyzer`, `boost`, `lenient`, `slop` and `zero_terms_query`  as explained
-in <<query-dsl-match-query>>.  Type `phrase_prefix` additionally accepts
-`max_expansions`.
+Also, accepts `analyzer`, <<mapping-boost,`boost`>>, `lenient` and `zero_terms_query` as explained
+in <<query-dsl-match-query>>, as well as `slop` which is explained in <<query-dsl-match-query-phrase>>.
+Type `phrase_prefix` additionally accepts `max_expansions`.
 
 [IMPORTANT]
 [[phrase-fuzziness]]
@@ -389,8 +387,7 @@ explanation:
     +blended("smith", fields: [first_name, last_name])
 
 Also, accepts `analyzer`, `boost`, `operator`, `minimum_should_match`,
-`lenient`, `zero_terms_query` and `cutoff_frequency`, as explained in
-<<query-dsl-match-query, match query>>.
+`lenient` and `zero_terms_query`.
 
 [[cross-field-analysis]]
 ===== `cross_field` and analysis
@@ -554,5 +551,4 @@ explained in <<query-dsl-match-query, match query>> are supported. The
 construct term queries, but do not have an effect on the prefix query
 constructed from the final term.
 
-The `slop` and `cutoff_frequency` parameters are not supported by this query
-type.
+The `slop` parameter is not supported by this query type.
diff --git a/docs/reference/query-dsl/query_filter_context.asciidoc b/docs/reference/query-dsl/query_filter_context.asciidoc
index 79f8c4bd960b0..6e7177ed19c18 100644
--- a/docs/reference/query-dsl/query_filter_context.asciidoc
+++ b/docs/reference/query-dsl/query_filter_context.asciidoc
@@ -52,12 +52,12 @@ GET /_search
   "query": { <1>
     "bool": { <2>
       "must": [
-        { "match": { "title":   "Search"        }}, <2>
-        { "match": { "content": "Elasticsearch" }}  <2>
+        { "match": { "title":   "Search"        }},
+        { "match": { "content": "Elasticsearch" }}  
       ],
       "filter": [ <3>
-        { "term":  { "status": "published" }}, <4>
-        { "range": { "publish_date": { "gte": "2015-01-01" }}} <4>
+        { "term":  { "status": "published" }}, 
+        { "range": { "publish_date": { "gte": "2015-01-01" }}} 
       ]
     }
   }
@@ -68,11 +68,16 @@ GET /_search
 <2> The `bool` and two `match` clauses are used in query context,
     which means that they are used to score how well each document
     matches.
-<3> The `filter` parameter indicates filter context.
-<4> The `term` and `range` clauses are used in filter context.
-    They will filter out documents which do not match, but they will
+<3> The `filter` parameter indicates filter context. Its `term` and 
+    `range` clauses are used in filter context. They will filter out 
+    documents which do not match, but they will
     not affect the score for matching documents.
 
+WARNING: Scores calculated for queries in query context are represented 
+as single precision floating point numbers; they have only
+24 bits for significand's precision. Score calculations that exceed the
+significand's precision will be converted to floats with loss of precision.
+
 TIP: Use query clauses in query context for conditions which should affect the
 score of matching documents (i.e. how well does the document match), and use
 all other query clauses in filter context.
diff --git a/docs/reference/query-dsl/range-query.asciidoc b/docs/reference/query-dsl/range-query.asciidoc
index 61c46996949f1..27db882fe1dd3 100644
--- a/docs/reference/query-dsl/range-query.asciidoc
+++ b/docs/reference/query-dsl/range-query.asciidoc
@@ -1,14 +1,16 @@
 [[query-dsl-range-query]]
 === Range Query
 
-Matches documents with fields that have terms within a certain range.
-The type of the Lucene query depends on the field type, for `string`
-fields, the `TermRangeQuery`, while for number/date fields, the query is
-a `NumericRangeQuery`. The following example returns all documents where
-`age` is between `10` and `20`:
+Returns documents that contain terms within a provided range.
+
+[[range-query-ex-request]]
+==== Example request
+
+The following search returns documents where the `age` field contains a term
+between `10` and `20`.
 
 [source,js]
---------------------------------------------------
+----
 GET _search
 {
     "query": {
@@ -21,147 +23,209 @@ GET _search
         }
     }
 }
---------------------------------------------------
+----
 // CONSOLE 
 
-The `range` query accepts the following parameters:
+[[range-query-top-level-params]]
+==== Top-level parameters for `range`
+
+`<field>`::
++
+--
+Field you wish to search.
+--
+
+[[range-query-field-params]]
+==== Parameters for `<field>`
+
+`gt`::
+Greater than. Optional.
+
+`gte`::
+Greater than or equal to. Optional.
+
+`lt`::
+Less than. Optional.
+
+`lte`::
+Less than or equal to. Optional.
+
+`format`::
++
+--
+Date format used to convert `date` values in the query.
+
+By default, {es} uses the <<mapping-date-format,date `format`>> provided in the
+`<field>`'s mapping. This value overrides that mapping format.
 
-[horizontal]
-`gte`:: 	Greater-than or equal to
-`gt`::  	Greater-than
-`lte`:: 	Less-than or equal to
-`lt`::  	Less-than
-`boost`:: 	Sets the boost value of the query, defaults to `1.0`
+For valid syntax, see <<mapping-date-format,`format`>>. Optional.
 
+[WARNING]
+====
+If a `format` and `date` value are incomplete, {es} replaces any missing year,
+month, or date component with the start of
+https://en.wikipedia.org/wiki/Unix_time[Unix time], which is January 1st, 1970.
+
+For example, if the `format` value is `dd`, {es} converts a `gte` value of `10`
+to `1970-01-10T00:00:00.000Z`.
+====
+
+--
+
+[[querying-range-fields]]
+`relation`::
++
+--
+Indicates how the range query matches values for `range` fields. Optional. Valid
+values are:
+
+`INTERSECTS` (Default)::
+Matches documents with a range field value that intersects the query's range.
+
+`CONTAINS`::
+Matches documents with a range field value that entirely contains the query's range.
+
+`WITHIN`::
+Matches documents with a range field value entirely within the query's range.
+--
+
+`time_zone`::
++
+--
+https://en.wikipedia.org/wiki/List_of_UTC_time_offsets[Coordinated Universal
+Time (UTC) offset] or
+https://en.wikipedia.org/wiki/List_of_tz_database_time_zones[IANA time zone]
+used to convert `date` values in the query to UTC. Optional.
+
+Valid values are ISO 8601 UTC offsets, such as `+01:00` or -`08:00`, and IANA
+time zone IDs, such as `America/Los_Angeles`.
+
+For an example query using the `time_zone` parameter, see
+<<range-query-time-zone,Time zone in `range` queries>>.
+
+[NOTE]
+====
+The `time_zone` parameter does **not** affect the <<date-math,date math>> value
+of `now`. `now` is always the current system time in UTC.
+
+However, the `time_zone` parameter does convert dates calculated using `now` and
+<<date-math,date math rounding>>. For example, the `time_zone` parameter will
+convert a value of `now/d`.
+====
+--
+
+`boost`::
++
+--
+Floating point number used to decrease or increase the
+<<query-filter-context, relevance scores>> of a query. Default is `1.0`.
+Optional.
+
+You can use the `boost` parameter to adjust relevance scores for searches
+containing two or more queries.
+
+Boost values are relative to the default value of `1.0`. A boost value between
+`0` and `1.0` decreases the relevance score. A value greater than `1.0`
+increases the relevance score.
+--
+
+[[range-query-notes]]
+==== Notes
 
 [[ranges-on-dates]]
-==== Ranges on date fields
+===== Using the `range` query with `date` fields
+
+When the `<field>` parameter is a <<date,`date`>> field datatype, you can use
+<<date-math,date math>> with the following parameters:
 
-When running `range` queries on fields of type <<date,`date`>>, ranges can be
-specified using <<date-math>>:
+* `gt`
+* `gte`
+* `lt`
+* `lte`
+
+For example, the following search returns documents where the `timestamp` field
+contains a date between today and yesterday.
 
 [source,js]
---------------------------------------------------
+----
 GET _search
 {
     "query": {
         "range" : {
-            "date" : {
+            "timestamp" : {
                 "gte" : "now-1d/d",
                 "lt" :  "now/d"
             }
         }
     }
 }
---------------------------------------------------
+----
 // CONSOLE
 
-===== Date math and rounding
-
-When using <<date-math,date math>> to round dates to the nearest day, month,
-hour, etc, the rounded dates depend on whether the ends of the ranges are
-inclusive or exclusive.
 
-Rounding up moves to the last millisecond of the rounding scope, and rounding
-down to the first millisecond of the rounding scope. For example:
+[[range-query-date-math-rounding]]
+====== Date math and rounding
+{es} rounds <<date-math,date math>> values in parameters as follows:
 
-[horizontal]
 `gt`::
++
+--
+Rounds up to the lastest millisecond.
 
-    Greater than the date rounded up: `2014-11-18||/M` becomes
-    `2014-11-30T23:59:59.999`, ie excluding the entire month.
+For example, `2014-11-18||/M` rounds up to `2014-11-30T23:59:59.999`, including
+the entire month.
+--
 
 `gte`::
++
+--
+Rounds down to the first millisecond.
 
-    Greater than or equal to the date rounded down: `2014-11-18||/M` becomes
-    `2014-11-01`, ie including the entire month.
+For example, `2014-11-18||/M` rounds down to `2014-11-01`, excluding
+the entire month.
+--
 
 `lt`::
++
+--
+Rounds down to the first millisecond.
 
-    Less than the date rounded down: `2014-11-18||/M` becomes `2014-11-01`, ie
-    excluding the entire month.
+For example, `2014-11-18||/M` rounds down to `2014-11-01`, excluding
+the entire month.
+--
 
 `lte`::
++
+--
+Rounds up to the lastest millisecond.
 
-    Less than or equal to the date rounded up: `2014-11-18||/M` becomes
-    `2014-11-30T23:59:59.999`, ie including the entire month.
+For example, `2014-11-18||/M` rounds up to `2014-11-30T23:59:59.999`, including
+the entire month.
+--
 
-===== Date format in range queries
+[[range-query-time-zone]]
+===== Example query using `time_zone` parameter
 
-Formatted dates will be parsed using the <<mapping-date-format,`format`>>
-specified on the <<date,`date`>> field by default, but it can be overridden by
-passing the `format` parameter to the `range` query:
+You can use the `time_zone` parameter to convert `date` values to UTC using a
+UTC offset. For example:
 
 [source,js]
---------------------------------------------------
-GET _search
-{
-    "query": {
-        "range" : {
-            "born" : {
-                "gte": "01/01/2012",
-                "lte": "2013",
-                "format": "dd/MM/yyyy||yyyy"
-            }
-        }
-    }
-}
---------------------------------------------------
-// CONSOLE 
-
-Note that if the date misses some of the year, month and day coordinates, the
-missing parts are filled with the start of
-https://en.wikipedia.org/wiki/Unix_time[unix time], which is January 1st, 1970.
-This means, that when e.g. specifying `dd` as the format, a value like `"gte" : 10`
-will translate to `1970-01-10T00:00:00.000Z`.
-
-===== Time zone in range queries
-
-Dates can be converted from another timezone to UTC either by specifying the
-time zone in the date value itself (if the <<mapping-date-format, `format`>>
-accepts it), or it can be specified as the `time_zone` parameter:
-
-[source,js]
---------------------------------------------------
+----
 GET _search
 {
     "query": {
         "range" : {
             "timestamp" : {
-                "gte": "2015-01-01 00:00:00", <1>
-                "lte": "now", <2>
-                "time_zone": "+01:00"
+                "time_zone": "+01:00", <1>
+                "gte": "2015-01-01 00:00:00", <2>
+                "lte": "now" <3>
             }
         }
     }
 }
---------------------------------------------------
+----
 // CONSOLE
-<1> This date will be converted to `2014-12-31T23:00:00 UTC`.
-<2> `now` is not affected by the `time_zone` parameter, its always the current system time (in UTC).
-However, when using <<date-math,date math rounding>> (e.g. down to the nearest day using `now/d`),
-the provided `time_zone` will be considered.
-
-
-[[querying-range-fields]]
-==== Querying range fields
-
-`range` queries can be used on fields of type <<range,`range`>>, allowing to
-match a range specified in the query with a range field value in the document.
-The `relation` parameter controls how these two ranges are matched:
-
-[horizontal]
-`WITHIN`::
-
-    Matches documents who's range field is entirely within the query's range.
-
-`CONTAINS`::
-
-    Matches documents who's range field entirely contains the query's range.
-
-`INTERSECTS`::
-
-    Matches documents who's range field intersects the query's range.
-    This is the default value when querying range fields.
-
-For examples, see <<range,`range`>> mapping type.
+<1> Indicates that `date` values use a UTC offset of `+01:00`.
+<2> With a UTC offset of `+01:00`, {es} converts this date to
+`2014-12-31T23:00:00 UTC`.
+<3> The `time_zone` parameter does not affect the `now` value.
\ No newline at end of file
diff --git a/docs/reference/query-dsl/regexp-syntax.asciidoc b/docs/reference/query-dsl/regexp-syntax.asciidoc
index 6a929ba98d502..74094b0cab1b0 100644
--- a/docs/reference/query-dsl/regexp-syntax.asciidoc
+++ b/docs/reference/query-dsl/regexp-syntax.asciidoc
@@ -224,7 +224,7 @@ follows a tilde `"~"` is negated.  For instance, `"ab~cd" means:
 
 * Starts with `a`
 * Followed by `b`
-* Followed by a string of any length that it anything but `c`
+* Followed by a string of any length that is anything but `c`
 * Ends with `d`
 
 For the string `"abcdef"`:
diff --git a/docs/reference/query-dsl/script-score-query.asciidoc b/docs/reference/query-dsl/script-score-query.asciidoc
index e8d97a31fa95f..401d323f6fff4 100644
--- a/docs/reference/query-dsl/script-score-query.asciidoc
+++ b/docs/reference/query-dsl/script-score-query.asciidoc
@@ -76,6 +76,8 @@ to be the most efficient by using the internal mechanisms.
 --------------------------------------------------
 // NOTCONSOLE
 
+[role="xpack"]
+[testenv="basic"]
 [[vector-functions]]
 ===== Functions for vector fields
 
@@ -197,8 +199,7 @@ a vector function is executed, 0 is returned as a result
 for this document.
 
 NOTE: If a document's dense vector field has a number of dimensions
-different from the query's vector, 0 is used for missing dimensions
-in the calculations of vector functions.
+different from the query's vector, an error will be thrown.
 
 
 [[random-score-function]]
diff --git a/docs/reference/query-dsl/term-level-queries.asciidoc b/docs/reference/query-dsl/term-level-queries.asciidoc
index dd7ea38819f01..5941aea76c2fa 100644
--- a/docs/reference/query-dsl/term-level-queries.asciidoc
+++ b/docs/reference/query-dsl/term-level-queries.asciidoc
@@ -1,68 +1,59 @@
 [[term-level-queries]]
-== Term level queries
+== Term-level queries
 
-While the <<full-text-queries,full text queries>> will analyze the query
-string before executing, the _term-level queries_ operate on the exact terms
-that are stored in the inverted index, and will normalize terms before executing 
-only for <<keyword,`keyword`>> fields with <<normalizer,`normalizer`>> property.
+You can use **term-level queries** to find documents based on precise values in
+structured data. Examples of structured data include date ranges, IP addresses,
+prices, or product IDs.
 
-These queries are usually used for structured data like numbers, dates, and
-enums, rather than full text fields.  Alternatively, they allow you to craft
-low-level queries, foregoing the analysis process.
+Unlike <<full-text-queries, full-text queries>>, term-level queries do not
+analyze search terms. Instead, term-level queries match the exact terms stored
+in a field.
 
-The queries in this group are:
 
-<<query-dsl-term-query,`term` query>>::
+[NOTE]
+====
+Term-level queries still normalize search terms for `keyword` fields with the
+`normalizer` property. For more details, see <<normalizer, `normalizer`>>.
+====
 
-    Find documents which contain the exact term specified in the field
-    specified.
+[float]
+[[term-level-query-types]]
+=== Types of term-level queries
 
-<<query-dsl-terms-query,`terms` query>>::
+<<query-dsl-term-query,`term` query>>::
+Returns documents that contain an exact term in a provided field.
 
-    Find documents which contain any of the exact terms specified in the field
-    specified.
+<<query-dsl-terms-query,`terms` query>>::
+Returns documents that contain one or more exact terms in a provided field.
 
 <<query-dsl-terms-set-query,`terms_set` query>>::
-
-    Find documents which match with one or more of the specified terms. The
-    number of terms that must match depend on the specified minimum should
-    match field or script.
+Returns documents that contain a minimum number of exact terms in a provided
+field. You can define the minimum number of matching terms using a field or
+script.
 
 <<query-dsl-range-query,`range` query>>::
-
-    Find documents where the field specified contains values (dates, numbers,
-    or strings) in the range specified.
+Returns documents that contain terms within a provided range.
 
 <<query-dsl-exists-query,`exists` query>>::
-
-    Find documents where the field specified contains any non-null value.
+Returns documents that contain any indexed value for a field.
 
 <<query-dsl-prefix-query,`prefix` query>>::
-
-    Find documents where the field specified contains terms which begin with
-    the exact prefix specified.
+Returns documents that contain a specific prefix in a provided field.
 
 <<query-dsl-wildcard-query,`wildcard` query>>::
-
-    Find documents where the field specified contains terms which match the
-    pattern specified, where the pattern supports single character wildcards
-    (`?`) and multi-character wildcards (`*`)
+Returns documents that contain terms matching a wildcard pattern.
 
 <<query-dsl-regexp-query,`regexp` query>>::
-
-    Find documents where the field specified contains terms which match the
-    <<regexp-syntax,regular expression>> specified.
+Returns documents that contain terms matching a
+https://en.wikipedia.org/wiki/Regular_expression[regular expression].
 
 <<query-dsl-fuzzy-query,`fuzzy` query>>::
-
-    Find documents where the field specified contains terms which are fuzzily
-    similar to the specified term.  Fuzziness is measured as a
-    http://en.wikipedia.org/wiki/Damerau%E2%80%93Levenshtein_distance[Levenshtein edit distance]
-    of 1 or 2.
+Returns documents that contain terms similar to the search term. {es} measures
+similarity, or fuzziness, using a
+http://en.wikipedia.org/wiki/Levenshtein_distance[Levenshtein edit distance].
 
 <<query-dsl-ids-query,`ids` query>>::
-
-    Find documents with the specified IDs.
+Returns documents based on their <<mapping-id-field, document IDs>>. 
 
 
 include::term-query.asciidoc[]
diff --git a/docs/reference/query-dsl/terms-query.asciidoc b/docs/reference/query-dsl/terms-query.asciidoc
index db4597fbea504..53ae0163f804f 100644
--- a/docs/reference/query-dsl/terms-query.asciidoc
+++ b/docs/reference/query-dsl/terms-query.asciidoc
@@ -1,121 +1,252 @@
 [[query-dsl-terms-query]]
 === Terms Query
 
-Filters documents that have fields that match any of the provided terms
-(*not analyzed*). For example:
+Returns documents that contain one or more *exact* terms in a provided field.
+
+The `terms` query is the same as the <<query-dsl-term-query, `term` query>>,
+except you can search for multiple values.
+
+[[terms-query-ex-request]]
+==== Example request
+
+The following search returns documents where the `user` field contains `kimchy`
+or `elasticsearch`.
 
 [source,js]
---------------------------------------------------
+----
 GET /_search
 {
-    "query": {
-        "terms" : { "user" : ["kimchy", "elasticsearch"]}
+    "query" : {
+        "terms" : {
+            "user" : ["kimchy", "elasticsearch"],
+            "boost" : 1.0 
+        }
     }
 }
---------------------------------------------------
+----
 // CONSOLE
 
-NOTE: Highlighting `terms` queries is best-effort only, so terms of a `terms`
-query might not be highlighted depending on the highlighter implementation that
-is selected and on the number of terms in the `terms` query.
+[[terms-top-level-params]]
+==== Top-level parameters for `terms`
+`<field>`::
++
+--
+Field you wish to search.
+
+The value of this parameter is an array of terms you wish to find in the
+provided field. To return a document, one or more terms must exactly match a
+field value, including whitespace and capitalization.
+
+By default, {es} limits the `terms` query to a maximum of 65,536
+terms. You can change this limit using the <<index-max-terms-count,
+`index.max_terms_count`>> setting.
+
+[NOTE]
+To use the field values of an existing document as search terms, use the
+<<query-dsl-terms-lookup, terms lookup>> parameters.
+--
+
+`boost`::
++
+--
+Floating point number used to decrease or increase the
+<<query-filter-context, relevance scores>> of a query. Default is `1.0`.
+Optional.
+
+You can use the `boost` parameter to adjust relevance scores for searches
+containing two or more queries.
+
+Boost values are relative to the default value of `1.0`. A boost value between
+`0` and `1.0` decreases the relevance score. A value greater than `1.0`
+increases the relevance score.
+--
+
+[[terms-query-notes]]
+==== Notes
+
+[[query-dsl-terms-query-highlighting]]
+===== Highlighting `terms` queries
+<<search-request-highlighting,Highlighting>> is best-effort only. {es} may not
+return highlight results for `terms` queries depending on:
+
+* Highlighter type
+* Number of terms in the query
 
-[float]
 [[query-dsl-terms-lookup]]
-===== Terms lookup mechanism
+===== Terms lookup
+Terms lookup fetches the field values of an existing document. {es} then uses
+those values as search terms. This can be helpful when searching for a large set
+of terms.
 
-When it's needed to specify a `terms` filter with a lot of terms it can
-be beneficial to fetch those term values from a document in an index. A
-concrete example would be to filter tweets tweeted by your followers.
-Potentially the amount of user ids specified in the terms filter can be
-a lot. In this scenario it makes sense to use the terms filter's terms
-lookup mechanism.
+Because terms lookup fetches values from a document, the <<mapping-source-field,
+`_source`>> mapping field must be enabled to use terms lookup. The `_source`
+field is enabled by default.
 
-The terms lookup mechanism supports the following options:
+[NOTE]
+By default, {es} limits the `terms` query to a maximum of 65,536
+terms. This includes terms fetched using terms lookup. You can change
+this limit using the <<index-max-terms-count, `index.max_terms_count`>> setting.
 
-[horizontal]
+To perform a terms lookup, use the following parameters.
+
+[[query-dsl-terms-lookup-params]]
+====== Terms lookup parameters
 `index`::
-    The index to fetch the term values from.
+Name of the index from which to fetch field values.
 
 `id`::
-    The id of the document to fetch the term values from.
+<<mapping-id-field,ID>> of the document from which to fetch field values.
 
 `path`::
-    The field specified as path to fetch the actual values for the
-    `terms` filter.
++
+--
+Name of the field from which to fetch field values. {es} uses
+these values as search terms for the query.
+
+If the field values include an array of nested inner objects, you can access
+those objects using dot notation syntax.
+--
 
 `routing`::
-    A custom routing value to be used when retrieving the
-    external terms doc.
-
-The values for the `terms` filter will be fetched from a field in a
-document with the specified id in the specified type and index.
-Internally a get request is executed to fetch the values from the
-specified path. At the moment for this feature to work the `_source`
-needs to be stored.
-
-Also, consider using an index with a single shard and fully replicated
-across all nodes if the "reference" terms data is not large. The lookup
-terms filter will prefer to execute the get request on a local node if
-possible, reducing the need for networking.
-
-[WARNING]
-Executing a Terms Query request with a lot of terms can be quite slow,
-as each additional term demands extra processing and memory.
-To safeguard against this, the maximum number of terms that can be used
-in a Terms Query both directly or through lookup has been limited to `65536`.
-This default maximum can be changed for a particular index with the index setting
- `index.max_terms_count`.
-
-[float]
-===== Terms lookup twitter example
-At first we index the information for user with id 2, specifically, its
-followers, then index a tweet from user with id 1. Finally we search on
-all the tweets that match the followers of user 2.
+Custom <<mapping-routing-field, routing value>> of the document from which to
+fetch term values. If a custom routing value was provided when the document was
+indexed, this parameter is required.
+
+[[query-dsl-terms-lookup-example]]
+====== Terms lookup example
+
+To see how terms lookup works, try the following example.  
+
+. Create an index with a `keyword` field named `color`.
++
+--
 
 [source,js]
---------------------------------------------------
-PUT /users/_doc/2
+----
+PUT my_index
 {
-    "followers" : ["1", "3"]
+    "mappings" : {
+        "properties" : {
+            "color" : { "type" : "keyword" }
+        }
+    }
 }
+----
+// CONSOLE
+--
 
-PUT /tweets/_doc/1
+. Index a document with an ID of 1 and values of `["blue", "green"]` in the
+`color` field.
++
+--
+
+[source,js]
+----
+PUT my_index/_doc/1
 {
-    "user" : "1"
+  "color":   ["blue", "green"]
 }
+----
+// CONSOLE
+// TEST[continued]
+--
 
-GET /tweets/_search
+. Index another document with an ID of 2 and value of `blue` in the `color`
+field.
++
+--
+
+[source,js]
+----
+PUT my_index/_doc/2
 {
-    "query" : {
-        "terms" : {
-            "user" : {
-                "index" : "users",
-                "id" : "2",
-                "path" : "followers"
-            }
-        }
-    }
+  "color":   "blue"
 }
---------------------------------------------------
+----
+// CONSOLE
+// TEST[continued]
+--
+
+. Use the `terms` query with terms lookup parameters to find documents
+containing one or more of the same terms as document 2. Include the `pretty`
+parameter so the response is more readable.
++
+--
+
+////
+
+[source,js]
+----
+POST my_index/_refresh
+----
 // CONSOLE
+// TEST[continued]
 
-The structure of the external terms document can also include an array of
-inner objects, for example:
+////
 
 [source,js]
---------------------------------------------------
-PUT /users/_doc/2
+----
+GET my_index/_search?pretty
 {
- "followers" : [
-   {
-     "id" : "1"
-   },
-   {
-     "id" : "2"
-   }
- ]
+  "query": {
+    "terms": {
+        "color" : {
+            "index" : "my_index",
+            "id" : "2",
+            "path" : "color"
+        }
+    }
+  }
 }
---------------------------------------------------
+----
 // CONSOLE
+// TEST[continued]
+
+Because document 2 and document 1 both contain `blue` as a value in the `color`
+field, {es} returns both documents.
 
-In which case, the lookup path will be `followers.id`.
+[source,js]
+----
+{
+  "took" : 17,
+  "timed_out" : false,
+  "_shards" : {
+    "total" : 1,
+    "successful" : 1,
+    "skipped" : 0,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : {
+      "value" : 2,
+      "relation" : "eq"
+    },
+    "max_score" : 1.0,
+    "hits" : [
+      {
+        "_index" : "my_index",
+        "_type" : "_doc",
+        "_id" : "1",
+        "_score" : 1.0,
+        "_source" : {
+          "color" : [
+            "blue",
+            "green"
+          ]
+        }
+      },
+      {
+        "_index" : "my_index",
+        "_type" : "_doc",
+        "_id" : "2",
+        "_score" : 1.0,
+        "_source" : {
+          "color" : "blue"
+        }
+      }
+    ]
+  }
+}
+----
+// TESTRESPONSE[s/"took" : 17/"took" : $body.took/]
+--
\ No newline at end of file
diff --git a/docs/reference/query-dsl/terms-set-query.asciidoc b/docs/reference/query-dsl/terms-set-query.asciidoc
index 3ebfb672e205f..0f097e494bfda 100644
--- a/docs/reference/query-dsl/terms-set-query.asciidoc
+++ b/docs/reference/query-dsl/terms-set-query.asciidoc
@@ -1,121 +1,231 @@
 [[query-dsl-terms-set-query]]
 === Terms Set Query
 
-Returns any documents that match with at least one or more of the
-provided terms. The terms are not analyzed and thus must match exactly.
-The number of terms that must match varies per document and is either
-controlled by a minimum should match field or computed per document in
-a minimum should match script.
+Returns documents that contain a minimum number of *exact* terms in a provided
+field.
 
-The field that controls the number of required terms that must match must
-be a number field:
+The `terms_set` query is the same as the <<query-dsl-terms-query, `terms`
+query>>, except you can define the number of matching terms required to
+return a document. For example:
+
+* A field, `programming_languages`, contains a list of known programming
+languages, such as `c++`, `java`, or `php` for job candidates. You can use the
+`terms_set` query to return documents that match at least two of these
+languages.
+
+* A field, `permissions`, contains a list of possible user permissions for an
+application. You can use the `terms_set` query to return documents that
+match a subset of these permissions.
+
+[[terms-set-query-ex-request]]
+==== Example request
+
+[[terms-set-query-ex-request-index-setup]]
+===== Index setup
+In most cases, you'll need to include a <<number, numeric>> field mapping in
+your index to use the `terms_set` query. This numeric field contains the
+number of matching terms required to return a document.
+
+To see how you can set up an index for the `terms_set` query, try the
+following example.
+
+. Create an index, `job-candidates`, with the following field mappings:
++
+--
+
+* `name`, a <<keyword, `keyword`>> field. This field contains the name of the
+job candidate.
+
+* `programming_languages`, a <<keyword, `keyword`>> field. This field contains
+programming languages known by the job candidate.
+
+* `required_matches`, a <<number, numeric>> `long` field. This field contains
+the number of matching terms required to return a document.
 
 [source,js]
---------------------------------------------------
-PUT /my-index
+----
+PUT /job-candidates
 {
     "mappings": {
         "properties": {
+            "name": {
+                "type": "keyword"
+            },
+            "programming_languages": {
+                "type": "keyword"
+            },
             "required_matches": {
                 "type": "long"
             }
         }
     }
 }
+----
+// CONSOLE
+// TESTSETUP
+
+--
 
-PUT /my-index/_doc/1?refresh
+. Index a document with an ID of `1` and the following values:
++
+--
+
+* `Jane Smith` in the `name` field.
+
+* `["c++", "java"]` in the `programming_languages` field.
+
+* `2` in the `required_matches` field.
+
+Include the `?refresh` parameter so the document is immediately available for
+search.
+
+[source,js]
+----
+PUT /job-candidates/_doc/1?refresh
 {
-    "codes": ["ghi", "jkl"],
+    "name": "Jane Smith",
+    "programming_languages": ["c++", "java"],
     "required_matches": 2
 }
+----
+// CONSOLE
+
+--
+
+. Index another document with an ID of `2` and the following values:
++
+--
+
+* `Jason Response` in the `name` field.
+
+* `["java", "php"]` in the `programming_languages` field.
+
+* `2` in the `required_matches` field.
 
-PUT /my-index/_doc/2?refresh
+[source,js]
+----
+PUT /job-candidates/_doc/2?refresh
 {
-    "codes": ["def", "ghi"],
+    "name": "Jason Response",
+    "programming_languages": ["java", "php"],
     "required_matches": 2
 }
---------------------------------------------------
+----
 // CONSOLE
-// TESTSETUP
 
-An example that uses the minimum should match field:
+--
+
+You can now use the `required_matches` field value as the number of
+matching terms required to return a document in the `terms_set` query.
+
+[[terms-set-query-ex-request-query]]
+===== Example query
+
+The following search returns documents where the `programming_languages` field
+contains at least two of the following terms:
+
+* `c++`
+* `java`
+* `php`
+
+The `minimum_should_match_field` is `required_matches`. This means the
+number of matching terms required is `2`, the value of the `required_matches`
+field.
 
 [source,js]
---------------------------------------------------
-GET /my-index/_search
+----
+GET /job-candidates/_search
 {
     "query": {
         "terms_set": {
-            "codes" : {
-                "terms" : ["abc", "def", "ghi"],
+            "programming_languages": {
+                "terms": ["c++", "java", "php"],
                 "minimum_should_match_field": "required_matches"
             }
         }
     }
 }
---------------------------------------------------
+----
 // CONSOLE
 
-Response:
+[[terms-set-top-level-params]]
+==== Top-level parameters for `terms_set`
 
-[source,js]
---------------------------------------------------
-{
-  "took": 13,
-  "timed_out": false,
-  "_shards": {
-    "total": 1,
-    "successful": 1,
-    "skipped" : 0,
-    "failed": 0
-  },
-  "hits": {
-    "total" : {
-        "value": 1,
-        "relation": "eq"
-    },
-    "max_score": 0.87546873,
-    "hits": [
-      {
-        "_index": "my-index",
-        "_type": "_doc",
-        "_id": "2",
-        "_score": 0.87546873,
-        "_source": {
-          "codes": ["def", "ghi"],
-          "required_matches": 2
-        }
-      }
-    ]
-  }
-}
---------------------------------------------------
-// TESTRESPONSE[s/"took": 13,/"took": "$body.took",/]
+`<field>`::
+Field you wish to search.
+
+[[terms-set-field-params]]
+==== Parameters for `<field>`
 
-Scripts can also be used to control how many terms are required to match
-in a more dynamic way. For example a create date or a popularity field
-can be used as basis for the number of required terms to match.
+`terms`::
++
+--
+Array of terms you wish to find in the provided `<field>`. To return a document,
+a required number of terms must exactly match the field values, including
+whitespace and capitalization.
 
-Also the `params.num_terms` parameter is available in the script to indicate the
-number of terms that have been specified.
+The required number of matching terms is defined in the
+`minimum_should_match_field` or `minimum_should_match_script` parameter.
+--
 
-An example that always limits the number of required terms to match to never
-become larger than the number of terms specified:
+`minimum_should_match_field`::
+<<number, Numeric>> field containing the number of matching terms
+required to return a document.
+
+`minimum_should_match_script`::
++
+--
+Custom script containing the number of matching terms required to return a
+document.
+
+For parameters and valid values, see <<modules-scripting, Scripting>>.
+
+For an example query using the `minimum_should_match_script` parameter, see
+<<terms-set-query-script, How to use the `minimum_should_match_script`
+parameter>>.
+--
+
+[[terms-set-query-notes]]
+==== Notes
+
+[[terms-set-query-script]]
+===== How to use the `minimum_should_match_script` parameter
+You can use `minimum_should_match_script` to define the required number of
+matching terms using a script. This is useful if you need to set the number of
+required terms dynamically.
+
+[[terms-set-query-script-ex]]
+====== Example query using `minimum_should_match_script`
+
+The following search returns documents where the `programming_languages` field
+contains at least two of the following terms:
+
+* `c++`
+* `java`
+* `php`
+
+The `source` parameter of this query indicates:
+
+* The required number of terms to match cannot exceed `params.num_terms`, the
+number of terms provided in the `terms` field.
+* The required number of terms to match is `2`, the value of the
+`required_matches` field.
 
 [source,js]
---------------------------------------------------
-GET /my-index/_search
+----
+GET /job-candidates/_search
 {
     "query": {
         "terms_set": {
-            "codes" : {
-                "terms" : ["abc", "def", "ghi"],
+            "programming_languages": {
+                "terms": ["c++", "java", "php"],
                 "minimum_should_match_script": {
                    "source": "Math.min(params.num_terms, doc['required_matches'].value)"
-                }
+                },
+                "boost": 1.0
             }
         }
     }
 }
---------------------------------------------------
-// CONSOLE
+----
+// CONSOLE
\ No newline at end of file
diff --git a/docs/reference/query-dsl/wildcard-query.asciidoc b/docs/reference/query-dsl/wildcard-query.asciidoc
index ba1c72bb1e53b..b2e8eb0adf772 100644
--- a/docs/reference/query-dsl/wildcard-query.asciidoc
+++ b/docs/reference/query-dsl/wildcard-query.asciidoc
@@ -1,51 +1,67 @@
 [[query-dsl-wildcard-query]]
 === Wildcard Query
+Returns documents that contain terms matching a wildcard pattern.
 
-Matches documents that have fields matching a wildcard expression (*not
-analyzed*). Supported wildcards are `*`, which matches any character
-sequence (including the empty one), and `?`, which matches any single
-character. Note that this query can be slow, as it needs to iterate over many
-terms. In order to prevent extremely slow wildcard queries, a wildcard
-term should not start with one of the wildcards `*` or `?`. The wildcard
-query maps to Lucene `WildcardQuery`.
+A wildcard operator is a placeholder that matches one or more characters. For
+example, the `*` wildcard operator matches zero or more characters. You can
+combine wildcard operators with other characters to create a wildcard pattern.
 
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query": {
-        "wildcard" : { "user" : "ki*y" }
-    }
-}
---------------------------------------------------
-// CONSOLE
+[[wildcard-query-ex-request]]
+==== Example request
 
-A boost can also be associated with the query:
+The following search returns documents where the `user` field contains a term
+that begins with `ki` and ends with `y`. These matching terms can include `kiy`,
+`kity`, or `kimchy`.
 
 [source,js]
---------------------------------------------------
+----
 GET /_search
 {
     "query": {
-        "wildcard" : { "user" : { "value" : "ki*y", "boost" : 2.0 } }
+        "wildcard": {
+            "user": {
+                "value": "ki*y",
+                "boost": 1.0,
+                "rewrite": "constant_score"
+            }
+        }
     }
 }
---------------------------------------------------
+----
 // CONSOLE
 
-Or :
+[[wildcard-top-level-params]]
+==== Top-level parameters for `wildcard`
+`<field>`::
+Field you wish to search.
 
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query": {
-        "wildcard" : { "user" : { "wildcard" : "ki*y", "boost" : 2.0 } }
-    }
-}
---------------------------------------------------
-// CONSOLE
+[[wildcard-query-field-params]]
+==== Parameters for `<field>`
+`value`::
+Wildcard pattern for terms you wish to find in the provided `<field>`.
++
+--
+This parameter supports two wildcard operators:
+
+* `?`, which matches any single character
+* `*`, which can match zero or more characters, including an empty one
+
+WARNING: Avoid beginning patterns with `*` or `?`. This can increase
+the iterations needed to find matching terms and slow search performance.
+-- 
+
+`boost`::
+Floating point number used to decrease or increase the
+<<query-filter-context, relevance scores>> of a query. Default is `1.0`.
+Optional.
++
+You can use the `boost` parameter to adjust relevance scores for searches
+containing two or more queries.
++
+Boost values are relative to the default value of `1.0`. A boost value between
+`0` and `1.0` decreases the relevance score. A value greater than `1.0`
+increases the relevance score.
 
-This multi term query allows to control how it gets rewritten using the
-<<query-dsl-multi-term-rewrite,rewrite>>
-parameter.
+`rewrite` (Expert)::
+Method used to rewrite the query. For valid values and more information, see the
+<<query-dsl-multi-term-rewrite, `rewrite` parameter>>. Optional.
\ No newline at end of file
diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc
index 6f68d781f4856..4d15ee7e25503 100644
--- a/docs/reference/redirects.asciidoc
+++ b/docs/reference/redirects.asciidoc
@@ -82,15 +82,13 @@ The `_uid` field has been removed in favour of the <<mapping-id-field,`_id` fiel
 === memcached
 
 The `memcached` transport is no longer supported.  Instead use the REST
-interface over <<modules-http,HTTP>> or the
-{javaclient}/index.html[Java API].
+interface over <<modules-http,HTTP>>.
 
 [role="exclude",id="modules-thrift"]
 === Thrift
 
 The `thrift` transport is no longer supported.  Instead use the REST
-interface over <<modules-http,HTTP>> or the
-{javaclient}/index.html[Java API].
+interface over <<modules-http,HTTP>>.
 
 // QUERY DSL
 
@@ -610,4 +608,16 @@ The `TransportClient` is deprecated in favour of the
 {java-rest}/java-rest-high.html[Java High Level REST Client] and was removed in
 Elasticsearch 8.0. The
 {java-rest}/java-rest-high-level-migration.html[migration guide] describes all
-the steps needed to migrate.
\ No newline at end of file
+the steps needed to migrate.
+
+[role="exclude",id="query-dsl-common-terms-query"]
+=== Common Terms Query
+
+The `common` terms query is deprecated. Use the <<query-dsl-match-query, `match`
+query>> instead. The `match` query skips blocks of documents efficiently,
+without any configuration, if the total number of hits is not tracked.
+
+[role="exclude",id="xpack-api"]
+=== X-Pack APIs
+
+{es} {xpack} APIs are now documented in <<rest-apis, REST APIs>>. 
\ No newline at end of file
diff --git a/docs/reference/rest-api/defs.asciidoc b/docs/reference/rest-api/defs.asciidoc
index 823b63cbe579d..65a0384a3d3bf 100644
--- a/docs/reference/rest-api/defs.asciidoc
+++ b/docs/reference/rest-api/defs.asciidoc
@@ -12,6 +12,7 @@ These resource definitions are used in APIs related to {ml-features} and
 * <<ml-job-resource,Jobs>>
 * <<ml-jobstats,Job statistics>>
 * <<ml-snapshot-resource,Model snapshots>>
+* <<data-frame-transform-pivot,Pivots>>
 * <<ml-results-resource,Results>>
 * <<role-mapping-resources,Role mappings>>
 * <<ml-event-resource,Scheduled Events>>
@@ -22,6 +23,7 @@ include::{es-repo-dir}/ml/apis/filterresource.asciidoc[]
 include::{es-repo-dir}/ml/apis/jobresource.asciidoc[]
 include::{es-repo-dir}/ml/apis/jobcounts.asciidoc[]
 include::{es-repo-dir}/ml/apis/snapshotresource.asciidoc[]
+include::{es-repo-dir}/data-frames/apis/pivotresource.asciidoc[]
 include::{xes-repo-dir}/rest-api/security/role-mapping-resources.asciidoc[]
 include::{es-repo-dir}/ml/apis/resultsresource.asciidoc[]
 include::{es-repo-dir}/ml/apis/eventresource.asciidoc[]
diff --git a/docs/reference/rest-api/index.asciidoc b/docs/reference/rest-api/index.asciidoc
index a5b93747dfa1a..9f70c2a3cef65 100644
--- a/docs/reference/rest-api/index.asciidoc
+++ b/docs/reference/rest-api/index.asciidoc
@@ -1,17 +1,21 @@
-[role="xpack"]
-[[xpack-api]]
-= {xpack} APIs
+[[rest-apis]]
+= REST APIs
 
 [partintro]
 --
-{xpack} exposes REST APIs that are used by the UI components and can be called
-directly to configure and access {xpack} features.
+{es} exposes REST APIs that are used by the UI components and can be called
+directly to configure and access {es} features.
+
+[NOTE]
+We are working on including more {es} APIs in this section. Some content might
+not be included yet.
 
 * <<info-api,Info API>>
 * <<ccr-apis,{ccr-cap} APIs>>
 * <<data-frame-apis,{dataframe-cap} APIs>>
 * <<graph-explore-api,Graph Explore API>>
 * <<freeze-index-api>>, <<unfreeze-index-api>>
+* <<indices-reload-analyzers,Reload Search Analyzers API>>
 * <<index-lifecycle-management-api,Index lifecycle management APIs>>
 * <<licensing-apis,Licensing APIs>>
 * <<ml-apis,Machine Learning APIs>>
@@ -35,4 +39,5 @@ include::{es-repo-dir}/rollup/rollup-api.asciidoc[]
 include::{xes-repo-dir}/rest-api/security.asciidoc[]
 include::{es-repo-dir}/indices/apis/unfreeze.asciidoc[]
 include::{xes-repo-dir}/rest-api/watcher.asciidoc[]
+include::{es-repo-dir}/indices/apis/reload-analyzers.asciidoc[]
 include::defs.asciidoc[]
diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc
index e626b491da17d..adbf5f01979a0 100644
--- a/docs/reference/rest-api/info.asciidoc
+++ b/docs/reference/rest-api/info.asciidoc
@@ -64,61 +64,58 @@ Example response:
    },
    "features" : {
       "ccr" : {
-        "description" : "Cross Cluster Replication",
         "available" : true,
         "enabled" : true
       },
       "data_frame" : {
-          "description" : "Data Frame for the Elastic Stack",
           "available" : true,
           "enabled" : true
       },
+      "flattened" : {
+         "available" : true,
+         "enabled" : true
+      },
       "graph" : {
-         "description" : "Graph Data Exploration for the Elastic Stack",
          "available" : true,
          "enabled" : true
       },
       "ilm" : {
-         "description" : "Index lifecycle management for the Elastic Stack",
          "available" : true,
          "enabled" : true
       },
       "logstash" : {
-         "description" : "Logstash management component for X-Pack",
          "available" : true,
          "enabled" : true
       },
       "ml" : {
-         "description" : "Machine Learning for the Elastic Stack",
          "available" : true,
-         "enabled" : true,
-         "native_code_info" : {
-           "version" : "7.0.0-alpha1-SNAPSHOT",
-           "build_hash" : "99a07c016d5a73"
-        }
+         "enabled" : true
       },
       "monitoring" : {
-         "description" : "Monitoring for the Elastic Stack",
          "available" : true,
          "enabled" : true
       },
       "rollup": {
-        "description": "Time series pre-aggregation and rollup",
          "available": true,
          "enabled": true
       },
       "security" : {
-         "description" : "Security for the Elastic Stack",
          "available" : true,
          "enabled" : false
       },
       "sql" : {
-         "description" : "SQL access to Elasticsearch",
+         "available" : true,
+         "enabled" : true
+      },
+      "vectors" : {
+         "available" : true,
+         "enabled" : true
+      },
+      "voting_only" : {
          "available" : true,
          "enabled" : true
       },
       "watcher" : {
-         "description" : "Alerting, Notification and Automation for the Elastic Stack",
          "available" : true,
          "enabled" : true
       }
diff --git a/docs/reference/rollup/api-quickref.asciidoc b/docs/reference/rollup/api-quickref.asciidoc
index 21eefefb4b12b..d1ea03b6284d7 100644
--- a/docs/reference/rollup/api-quickref.asciidoc
+++ b/docs/reference/rollup/api-quickref.asciidoc
@@ -5,7 +5,7 @@
 
 experimental[]
 
-Most {rollup} endpoints have the following base:
+Most rollup endpoints have the following base:
 
 [source,js]
 ----
diff --git a/docs/reference/rollup/rollup-api.asciidoc b/docs/reference/rollup/rollup-api.asciidoc
index 099686fb4329d..5981336d0a054 100644
--- a/docs/reference/rollup/rollup-api.asciidoc
+++ b/docs/reference/rollup/rollup-api.asciidoc
@@ -9,7 +9,7 @@
 
 * <<rollup-put-job,Create Job>>, <<rollup-delete-job,Delete Job>>,
 * <<rollup-start-job,Start Job>>, <<rollup-stop-job,Stop Job>>,
-* <<rollup-get-job,Get Job, List all Jobs>>
+* <<rollup-get-job,Get Job+++,+++ List all Jobs>>
 * <<rollup-job-config,Job configuration details>>
 
 [float]
diff --git a/docs/reference/modules/scripting.asciidoc b/docs/reference/scripting.asciidoc
similarity index 86%
rename from docs/reference/modules/scripting.asciidoc
rename to docs/reference/scripting.asciidoc
index 44696ea94bb89..33b8795a58114 100644
--- a/docs/reference/modules/scripting.asciidoc
+++ b/docs/reference/scripting.asciidoc
@@ -1,9 +1,11 @@
 [[modules-scripting]]
-== Scripting
+= Scripting
 
-The scripting module enables you to use scripts to evaluate custom
-expressions. For example, you could use a script to return "script fields"
-as part of a search request or evaluate a custom score for a query.
+[partintro]
+--
+With scripting, you can evaluate custom expressions in {es}. For example, you
+could use a script to return "script fields" as part of a search request or
+evaluate a custom score for a query.
 
 The default scripting language is <<modules-scripting-painless, `Painless`>>.
 Additional `lang` plugins enable you to run scripts written in other languages.
@@ -11,7 +13,7 @@ Everywhere a script can be used, you can include a `lang` parameter
 to specify the language of the script.
 
 [float]
-=== General-purpose languages:
+== General-purpose languages
 
 These languages can be used for any purpose in the scripting APIs,
 and give the most flexibility.
@@ -29,7 +31,7 @@ and give the most flexibility.
 |=======================================================================
 
 [float]
-=== Special-purpose languages:
+== Special-purpose languages
 
 These languages are less flexible, but typically have higher performance for
 certain tasks.
@@ -67,7 +69,7 @@ sandboxed languages can be a security issue, please read
 <<modules-scripting-security, Scripting and security>> for more details.
 
 =================================================
-
+--
 
 include::scripting/using.asciidoc[]
 
diff --git a/docs/reference/modules/scripting/engine.asciidoc b/docs/reference/scripting/engine.asciidoc
similarity index 97%
rename from docs/reference/modules/scripting/engine.asciidoc
rename to docs/reference/scripting/engine.asciidoc
index da3b4529daacc..b4a2cd29fdd72 100644
--- a/docs/reference/modules/scripting/engine.asciidoc
+++ b/docs/reference/scripting/engine.asciidoc
@@ -1,5 +1,5 @@
 [[modules-scripting-engine]]
-=== Advanced scripts using script engines
+== Advanced scripts using script engines
 
 A `ScriptEngine` is a backend for implementing a scripting language. It may also
 be used to write scripts that need to use advanced internals of scripting. For example,
diff --git a/docs/reference/modules/scripting/expression.asciidoc b/docs/reference/scripting/expression.asciidoc
similarity index 99%
rename from docs/reference/modules/scripting/expression.asciidoc
rename to docs/reference/scripting/expression.asciidoc
index de3704a899e52..fe58cbbdf131b 100644
--- a/docs/reference/modules/scripting/expression.asciidoc
+++ b/docs/reference/scripting/expression.asciidoc
@@ -1,5 +1,5 @@
 [[modules-scripting-expression]]
-=== Lucene Expressions Language
+== Lucene expressions language
 
 Lucene's expressions compile a `javascript` expression to bytecode. They are
 designed for high-performance custom ranking and sorting functions and are
diff --git a/docs/reference/modules/scripting/fields.asciidoc b/docs/reference/scripting/fields.asciidoc
similarity index 98%
rename from docs/reference/modules/scripting/fields.asciidoc
rename to docs/reference/scripting/fields.asciidoc
index 681e8d6e54067..cf8905189f799 100644
--- a/docs/reference/modules/scripting/fields.asciidoc
+++ b/docs/reference/scripting/fields.asciidoc
@@ -1,5 +1,5 @@
 [[modules-scripting-fields]]
-=== Accessing document fields and special variables
+== Accessing document fields and special variables
 
 Depending on where a script is used, it will have access to certain special
 variables and document fields.
@@ -17,7 +17,7 @@ API will have access to the `ctx` variable which exposes:
 `ctx._index` etc::  Access to <<mapping-fields,document meta-fields>>, some of which may be read-only.
 
 [float]
-== Search and Aggregation scripts
+== Search and aggregation scripts
 
 With the exception of <<search-request-script-fields,script fields>> which are
 executed once per search hit, scripts used in search and aggregations will be
@@ -80,7 +80,7 @@ GET my_index/_search
 
 [float]
 [[modules-scripting-doc-vals]]
-=== Doc Values
+=== Doc values
 
 By far the fastest most efficient way to access a field value from a
 script is to use the `doc['field_name']` syntax, which retrieves the field
@@ -140,7 +140,7 @@ access `text` fields from scripts.
 
 [float]
 [[modules-scripting-stored]]
-=== Stored Fields and `_source`
+=== Stored fields and `_source`
 
 _Stored fields_ -- fields explicitly marked as
 <<mapping-store,`"store": true`>> -- can be accessed using the
diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/scripting/painless.asciidoc
similarity index 97%
rename from docs/reference/modules/scripting/painless.asciidoc
rename to docs/reference/scripting/painless.asciidoc
index 6dd9b50db51ed..1d615c1426e90 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/scripting/painless.asciidoc
@@ -1,5 +1,5 @@
 [[modules-scripting-painless]]
-=== Painless Scripting Language
+== Painless scripting language
 
 _Painless_ is a simple, secure scripting language designed specifically for use
 with Elasticsearch. It is the default scripting language for Elasticsearch and
diff --git a/docs/reference/modules/scripting/security.asciidoc b/docs/reference/scripting/security.asciidoc
similarity index 99%
rename from docs/reference/modules/scripting/security.asciidoc
rename to docs/reference/scripting/security.asciidoc
index f1946bd0f2876..421cec2ccf760 100644
--- a/docs/reference/modules/scripting/security.asciidoc
+++ b/docs/reference/scripting/security.asciidoc
@@ -1,5 +1,5 @@
 [[modules-scripting-security]]
-=== Scripting and security
+== Scripting and security
 
 While Elasticsearch contributors make every effort to prevent scripts from
 running amok, security is something best done in
diff --git a/docs/reference/modules/scripting/using.asciidoc b/docs/reference/scripting/using.asciidoc
similarity index 97%
rename from docs/reference/modules/scripting/using.asciidoc
rename to docs/reference/scripting/using.asciidoc
index 86202a98dd537..6da631ac9f592 100644
--- a/docs/reference/modules/scripting/using.asciidoc
+++ b/docs/reference/scripting/using.asciidoc
@@ -1,5 +1,5 @@
 [[modules-scripting-using]]
-=== How to use scripts
+== How to use scripts
 
 Wherever scripting is supported in the Elasticsearch API, the syntax follows
 the same pattern:
@@ -45,7 +45,7 @@ GET my_index/_search
 // CONSOLE
 
 [float]
-=== Script Parameters
+=== Script parameters
 
 `lang`::
 
@@ -107,7 +107,7 @@ minute will be compiled. You can change this setting dynamically by setting
 
 [float]
 [[modules-scripting-short-script-form]]
-=== Short Script Form
+=== Short script form
 A short script form can be used for brevity. In the short form, `script` is represented
 by a string instead of an object. This string contains the source of the script.
 
@@ -131,12 +131,13 @@ The same script in the normal form:
 
 [float]
 [[modules-scripting-stored-scripts]]
-=== Stored Scripts
+=== Stored scripts
 
 Scripts may be stored in and retrieved from the cluster state using the
 `_scripts` end-point.
 
-==== Request Examples
+[float]
+==== Request examples
 
 The following are examples of using a stored script that lives at
 `/_scripts/{id}`.
@@ -196,7 +197,7 @@ DELETE _scripts/calculate-score
 
 [float]
 [[modules-scripting-using-caching]]
-=== Script Caching
+=== Script caching
 
 All scripts are cached by default so that they only need to be recompiled
 when updates occur. By default, scripts do not have a time-based expiration, but
diff --git a/docs/reference/search/request/docvalue-fields.asciidoc b/docs/reference/search/request/docvalue-fields.asciidoc
index 6697b5bb3e383..784cc94015366 100644
--- a/docs/reference/search/request/docvalue-fields.asciidoc
+++ b/docs/reference/search/request/docvalue-fields.asciidoc
@@ -67,3 +67,7 @@ on their mappings: `long`, `double` and other numeric fields are formatted as
 numbers, `keyword` fields are formatted as strings, `date` fields are formatted
 with the configured `date` format, etc.
 
+NOTE: On its own, `docvalue_fields` cannot be used to load fields in nested
+objects -- if a field contains a nested object in its path, then no data will
+be returned for that docvalue field. To access nested fields, `docvalue_fields`
+must be used within an <<search-request-inner-hits, `inner_hits`>> block.
\ No newline at end of file
diff --git a/docs/reference/search/request/preference.asciidoc b/docs/reference/search/request/preference.asciidoc
index 5f3fcb2efa6b7..7412f04844c08 100644
--- a/docs/reference/search/request/preference.asciidoc
+++ b/docs/reference/search/request/preference.asciidoc
@@ -6,8 +6,12 @@ default, Elasticsearch selects from the available shard copies in an
 unspecified order, taking the <<allocation-awareness,allocation awareness>> and
 <<search-adaptive-replica,adaptive replica selection>> configuration into
 account. However, it may sometimes be desirable to try and route certain
-searches to certain sets of shard copies, for instance to make better use of
-per-copy caches.
+searches to certain sets of shard copies.
+
+A possible use case would be to make use of per-copy caches like the
+<<shard-request-cache,request cache>>. Doing this, however, runs contrary to the
+idea of search parallelization and can create hotspots on certain nodes because
+the load might not be evenly distributed anymore.
 
 The `preference` is a query string parameter which can be set to:
 
@@ -64,6 +68,10 @@ GET /_search?preference=xyzabc123
 ------------------------------------------------
 // CONSOLE
 
+This can be an effective strategy to increase usage of e.g. the request cache for
+unique users running similar searches repeatedly by always hitting the same cache, while
+requests of different users are still spread across all shard copies.
+
 NOTE: The `_only_local` preference guarantees only to use shard copies on the
 local node, which is sometimes useful for troubleshooting. All other options do
 not _fully_ guarantee that any particular shard copies are used in a search,
diff --git a/docs/reference/search/request/script-fields.asciidoc b/docs/reference/search/request/script-fields.asciidoc
index da5868ea7d65e..1bd61e0048182 100644
--- a/docs/reference/search/request/script-fields.asciidoc
+++ b/docs/reference/search/request/script-fields.asciidoc
@@ -33,7 +33,7 @@ GET /_search
 // CONSOLE
 // TEST[setup:sales]
 
-Script fields can work on fields that are not stored (`my_field_name` in
+Script fields can work on fields that are not stored (`price` in
 the above case), and allow to return custom values to be returned (the
 evaluated value of the script).
 
diff --git a/docs/reference/search/request/sort.asciidoc b/docs/reference/search/request/sort.asciidoc
index c12ec3a679a72..ccbc3da6e063b 100644
--- a/docs/reference/search/request/sort.asciidoc
+++ b/docs/reference/search/request/sort.asciidoc
@@ -252,7 +252,7 @@ field support has a `nested` sort option with the following properties:
     A filter that the inner objects inside the nested path
     should match with in order for its field values to be taken into account
     by sorting. Common case is to repeat the query / filter inside the
-    nested filter or query. By default no `nested_filter` is active.
+    nested filter or query. By default no `filter` is active.
 `max_children`::
     The maximum number of children to consider per root document
     when picking the sort value. Defaults to unlimited.
@@ -260,14 +260,8 @@ field support has a `nested` sort option with the following properties:
     Same as top-level `nested` but applies to another nested path within the
     current nested object.
 
-[WARNING]
-.Nested sort options before Elasticsearch 6.1
-============================================
-
-The `nested_path` and `nested_filter` options have been deprecated in
-favor of the options documented above.
-
-============================================
+NOTE: Elasticsearch will throw an error if a nested field is defined in a sort without
+a `nested` context.
 
 ===== Nested sorting examples
 
@@ -300,7 +294,7 @@ POST /_search
 // CONSOLE
 
 In the below example `parent` and `child` fields are of type `nested`.
-The `nested_path` needs to be specified at each level; otherwise, Elasticsearch doesn't know on what nested level sort values need to be captured.
+The `nested.path` needs to be specified at each level; otherwise, Elasticsearch doesn't know on what nested level sort values need to be captured.
 
 [source,js]
 --------------------------------------------------
@@ -374,7 +368,7 @@ GET /_search
 // CONSOLE
 
 NOTE: If a nested inner object doesn't match with
-the `nested_filter` then a missing value is used.
+the `nested.filter` then a missing value is used.
 
 ==== Ignoring Unmapped Fields
 
diff --git a/docs/reference/search/request/stored-fields.asciidoc b/docs/reference/search/request/stored-fields.asciidoc
index 195dc39f11e73..b55e0fce45757 100644
--- a/docs/reference/search/request/stored-fields.asciidoc
+++ b/docs/reference/search/request/stored-fields.asciidoc
@@ -49,6 +49,11 @@ Script fields can also be automatically detected and used as fields, so
 things like `_source.obj1.field1` can be used, though not recommended, as
 `obj1.field1` will work as well.
 
+NOTE: On its own, `stored_fields` cannot be used to load fields in nested
+objects -- if a field contains a nested object in its path, then no data will
+be returned for that stored field. To access nested fields, `stored_fields`
+must be used within an <<search-request-inner-hits, `inner_hits`>> block.
+
 ==== Disable stored fields entirely
 
 To disable the stored fields (and metadata fields) entirely use: `_none_`:
diff --git a/docs/reference/search/suggesters/phrase-suggest.asciidoc b/docs/reference/search/suggesters/phrase-suggest.asciidoc
index d92c32eddf033..2d0e3e218e62c 100644
--- a/docs/reference/search/suggesters/phrase-suggest.asciidoc
+++ b/docs/reference/search/suggesters/phrase-suggest.asciidoc
@@ -224,7 +224,7 @@ The response contains suggestions scored by the most likely spell correction fir
 
 [source,js]
 --------------------------------------------------
-POST _search
+POST test/_search
 {
   "suggest": {
     "text" : "noble prize",
@@ -267,7 +267,10 @@ POST _search
 
 The `phrase` suggester supports multiple smoothing models to balance
 weight between infrequent grams (grams (shingles) are not existing in
-the index) and frequent grams (appear at least once in the index).
+the index) and frequent grams (appear at least once in the index). The
+smoothing model can be selected by setting the `smoothing` parameter
+to one of the following options. Each smoothing model supports specific
+properties that can be configured.
 
 [horizontal]
 `stupid_backoff`::
@@ -288,6 +291,28 @@ the index) and frequent grams (appear at least once in the index).
     All parameters (`trigram_lambda`, `bigram_lambda`, `unigram_lambda`)
     must be supplied.
 
+[source,js]
+--------------------------------------------------
+POST test/_search
+{
+  "suggest": {
+    "text" : "obel prize",
+    "simple_phrase" : {
+      "phrase" : {
+        "field" : "title.trigram",
+        "size" : 1,
+        "smoothing" : {
+          "laplace" : {
+            "alpha" : 0.7
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
 ==== Candidate Generators
 
 The `phrase` suggester uses candidate generators to produce a list of
@@ -389,7 +414,7 @@ accept ordinary analyzer names.
 
 [source,js]
 --------------------------------------------------
-POST _search
+POST test/_search
 {
   "suggest": {
     "text" : "obel prize",
diff --git a/docs/reference/security/securing-communications/configuring-tls-docker.asciidoc b/docs/reference/security/securing-communications/configuring-tls-docker.asciidoc
index 1d23430e37eec..2190f17e4588f 100644
--- a/docs/reference/security/securing-communications/configuring-tls-docker.asciidoc
+++ b/docs/reference/security/securing-communications/configuring-tls-docker.asciidoc
@@ -43,11 +43,13 @@ instances:
 `.env`:
 [source,yaml]
 ----
-CERTS_DIR=/usr/share/elasticsearch/config/certificates <1>
-ELASTIC_PASSWORD=PleaseChangeMe <2>
+COMPOSE_PROJECT_NAME=es <1>
+CERTS_DIR=/usr/share/elasticsearch/config/certificates <2>
+ELASTIC_PASSWORD=PleaseChangeMe <3>
 ----
-<1> The path, inside the Docker image, where certificates are expected to be found.
-<2> Initial password for the `elastic` user.
+<1> Use an `es_` prefix for all volumes and networks created by docker-compose.
+<2> The path, inside the Docker image, where certificates are expected to be found.
+<3> Initial password for the `elastic` user.
 
 [[getting-starter-tls-create-certs-composefile]]
 `create-certs.yml`:
@@ -69,21 +71,21 @@ services:
     image: {docker-image}
     command: >
       bash -c '
-        if [[ ! -d config/certificates/certs ]]; then
-          mkdir config/certificates/certs;
+        yum install -y -q -e 0 unzip;
+        if [[ ! -f /certs/bundle.zip ]]; then
+          bin/elasticsearch-certutil cert --silent --pem --in config/certificates/instances.yml -out /certs/bundle.zip;
+          unzip /certs/bundle.zip -d /certs; <1>
         fi;
-        if [[ ! -f /local/certs/bundle.zip ]]; then
-          bin/elasticsearch-certgen --silent --in config/certificates/instances.yml --out config/certificates/certs/bundle.zip;
-          unzip config/certificates/certs/bundle.zip -d config/certificates/certs; <1>
-        fi;
-        chgrp -R 0 config/certificates/certs
+        chown -R 1000:0 /certs
       '
-    user: $\{UID:-1000\}
+    user: "0"
     working_dir: /usr/share/elasticsearch
-    volumes: ['.:/usr/share/elasticsearch/config/certificates']
+    volumes: ['certs:/certs', '.:/usr/share/elasticsearch/config/certificates']
+
+volumes: {"certs"}
 ----
 
-<1> The new node certificates and CA certificate+key are placed under the local directory `certs`.
+<1> The new node certificates and CA certificate+key are placed in a docker volume `es_certs`.
 endif::[]
 
 [[getting-starter-tls-create-docker-compose]]
@@ -106,7 +108,7 @@ services:
     image: {docker-image}
     environment:
       - node.name=es01
-      - discovery.seed_hosts=es02
+      - discovery.seed_hosts=es01,es02
       - cluster.initial_master_nodes=es01,es02
       - ELASTIC_PASSWORD=$ELASTIC_PASSWORD <1>
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
@@ -121,7 +123,7 @@ services:
       - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
       - xpack.security.transport.ssl.certificate=$CERTS_DIR/es01/es01.crt
       - xpack.security.transport.ssl.key=$CERTS_DIR/es01/es01.key
-    volumes: ['esdata_01:/usr/share/elasticsearch/data', './certs:$CERTS_DIR']
+    volumes: ['data01:/usr/share/elasticsearch/data', 'certs:$CERTS_DIR']
     ports:
       - 9200:9200
     healthcheck:
@@ -135,7 +137,7 @@ services:
     image: {docker-image}
     environment:
       - node.name=es02
-      - discovery.seed_hosts=es01
+      - discovery.seed_hosts=es01,es02
       - cluster.initial_master_nodes=es01,es02
       - ELASTIC_PASSWORD=$ELASTIC_PASSWORD
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
@@ -150,14 +152,14 @@ services:
       - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
       - xpack.security.transport.ssl.certificate=$CERTS_DIR/es02/es02.crt
       - xpack.security.transport.ssl.key=$CERTS_DIR/es02/es02.key
-    volumes: ['esdata_02:/usr/share/elasticsearch/data', './certs:$CERTS_DIR']
+    volumes: ['data02:/usr/share/elasticsearch/data', 'certs:$CERTS_DIR']
 
   wait_until_ready:
     image: {docker-image}
     command: /usr/bin/true
     depends_on: {"es01": {"condition": "service_healthy"}}
 
-volumes: {"esdata_01": {"driver": "local"}, "esdata_02": {"driver": "local"}}
+volumes: {"data01", "data02", "certs"}
 ----
 
 <1> Bootstrap `elastic` with the password defined in `.env`. See
@@ -175,7 +177,7 @@ endif::[]
 --
 ["source","sh"]
 ----
-docker-compose -f create-certs.yml up
+docker-compose -f create-certs.yml run --rm create_certs
 ----
 --
 . Start two {es} nodes configured for SSL/TLS:
@@ -189,9 +191,9 @@ docker-compose up -d
 . Access the {es} API over SSL/TLS using the bootstrapped password:
 +
 --
-["source","sh"]
+["source","sh",subs="attributes"]
 ----
-curl --cacert certs/ca/ca.crt -u elastic:PleaseChangeMe https://localhost:9200
+docker run --rm -v es_certs:/certs --network=es_default {docker-image} curl --cacert /certs/ca/ca.crt -u elastic:PleaseChangeMe https://es01:9200
 ----
 // NOTCONSOLE
 --
@@ -210,3 +212,13 @@ auto --batch \
 --url https://localhost:9200"
 ----
 --
+
+[float]
+==== Tear everything down
+To remove all the Docker resources created by the example, issue:
+--
+["source","sh"]
+----
+docker-compose down -v
+----
+--
diff --git a/docs/reference/security/securing-communications/separating-node-client-traffic.asciidoc b/docs/reference/security/securing-communications/separating-node-client-traffic.asciidoc
deleted file mode 100644
index 61ebd3e682594..0000000000000
--- a/docs/reference/security/securing-communications/separating-node-client-traffic.asciidoc
+++ /dev/null
@@ -1,68 +0,0 @@
-[role="xpack"]
-[[separating-node-client-traffic]]
-=== Separating node-to-node and client traffic
-
-Elasticsearch has the feature of so called 
-{ref}/modules-transport.html[TCP transport profiles]
-that allows it to bind to several ports and addresses. The {es}
-{security-features} extend on this functionality to enhance the security of the
-cluster by enabling the separation of node-to-node transport traffic from client
-transport traffic. This is important if the client transport traffic is not
-trusted and could potentially be malicious. To separate the node-to-node traffic
-from the client traffic, add the following to `elasticsearch.yml`:
-
-[source, yaml]
---------------------------------------------------
-transport.profiles.client: <1>
-  port: 9500-9600 <2>
-  xpack.security:
-    type: client <3>
---------------------------------------------------
-<1> `client` is the name of this example profile
-<2> The port range that will be used by transport clients to communicate with
-    this cluster
-<3> Categorizes the profile as a `client`. This accounts for additional security
-    filters by denying request attempts on for internal cluster operations
-    (e.g shard level actions and ping requests) from this profile.
-
-If supported by your environment, an internal network can be used for node-to-node
-traffic and public network can be used for client traffic by adding the following
-to `elasticsearch.yml`:
-
-[source, yaml]
---------------------------------------------------
-transport.profiles.default.bind_host: 10.0.0.1 <1>
-transport.profiles.client.bind_host: 1.1.1.1 <2>
---------------------------------------------------
-<1> The bind address for the network that will be used for node-to-node communication
-<2> The bind address for the network used for client communication
-
-If separate networks are not available, then
-{stack-ov}/ip-filtering.html[IP Filtering] can
-be enabled to limit access to the profiles.
-
-When using SSL for transport, a different set of certificates can also be used
-for the client traffic by adding the following to `elasticsearch.yml`:
-
-[source, yaml]
---------------------------------------------------
-transport.profiles.client.xpack.security.ssl.truststore:
-  path: /path/to/another/truststore
-  password: x-pack-test-password
-
-transport.profiles.client.xpack.security.ssl.keystore:
-  path: /path/to/another/keystore
-  password: x-pack-test-password
---------------------------------------------------
-
-To change the default behavior that requires certificates for transport clients,
-set the following value in the `elasticsearch.yml` file:
-
-[source, yaml]
---------------------------------------------------
-transport.profiles.client.xpack.security.ssl.client_authentication: none
---------------------------------------------------
-
-This setting keeps certificate authentication active for node-to-node traffic,
-but removes the requirement to distribute a signed certificate to transport
-clients.
\ No newline at end of file
diff --git a/docs/reference/settings/ml-settings.asciidoc b/docs/reference/settings/ml-settings.asciidoc
index 09fb8adad8523..91afcbe5b3466 100644
--- a/docs/reference/settings/ml-settings.asciidoc
+++ b/docs/reference/settings/ml-settings.asciidoc
@@ -109,3 +109,11 @@ cluster and the job is assigned to run on that node.
 IMPORTANT: This setting assumes some external process is capable of adding ML nodes
 to the cluster. This setting is only useful when used in conjunction with
 such an external process.
+
+`xpack.ml.process_connect_timeout` (<<cluster-update-settings,Dynamic>>)::
+The connection timeout for {ml} processes that run separately from the {es} JVM.
+Defaults to `10s`. Some {ml} processing is done by processes that run separately
+to the {es} JVM. When such processes are started they must connect to the {es}
+JVM. If such a process does not connect within the time period specified by this
+setting then the process is assumed to have failed. Defaults to `10s`. The minimum
+value for this setting is `5s`.
diff --git a/docs/reference/settings/monitoring-settings.asciidoc b/docs/reference/settings/monitoring-settings.asciidoc
index c48b7d8764d2c..18379577e6a3e 100644
--- a/docs/reference/settings/monitoring-settings.asciidoc
+++ b/docs/reference/settings/monitoring-settings.asciidoc
@@ -283,5 +283,6 @@ For example: `["elasticsearch_version_mismatch","xpack_license_expiration"]`.
 :component:              {monitoring}
 :verifies:
 :server!:
+:ssl-context:            monitoring
 
 include::ssl-settings.asciidoc[]
diff --git a/docs/reference/settings/notification-settings.asciidoc b/docs/reference/settings/notification-settings.asciidoc
index 77f755b09e285..a2eb84bc2110e 100644
--- a/docs/reference/settings/notification-settings.asciidoc
+++ b/docs/reference/settings/notification-settings.asciidoc
@@ -37,14 +37,8 @@ required. For more information, see
 {xpack-ref}/encrypting-data.html[Encrypting sensitive data in {watcher}].
 
 `xpack.watcher.history.cleaner_service.enabled`::
-ifdef::asciidoctor[]
 added:[6.3.0,Default changed to `true`.]
 deprecated:[7.0.0,Watcher history indices are now managed by the `watch-history-ilm-policy` ILM policy]
-endif::[]
-ifndef::asciidoctor[]
-added[6.3.0,Default changed to `true`.]
-deprecated[7.0.0,Watcher history indices are now managed by the `watch-history-ilm-policy` ILM policy]
-endif::[]
 +
 Set to `true` (default) to enable the cleaner service. If this setting is
 `true`, the `xpack.monitoring.enabled` setting must also be set to `true` with
@@ -85,6 +79,7 @@ corresponding endpoints are whitelisted as well.
 :component:              {watcher}
 :verifies:
 :server!:
+:ssl-context:            watcher
 
 include::ssl-settings.asciidoc[]
 
diff --git a/docs/reference/settings/security-settings.asciidoc b/docs/reference/settings/security-settings.asciidoc
index 3cfe1d1e58769..00c1941f65256 100644
--- a/docs/reference/settings/security-settings.asciidoc
+++ b/docs/reference/settings/security-settings.asciidoc
@@ -1566,6 +1566,7 @@ a PKCS#12 container includes trusted certificate ("anchor") entries look for
 :client-auth-default:    none
 :verifies!:
 :server:
+:ssl-context:            security-http
 
 include::ssl-settings.asciidoc[]
 
@@ -1575,6 +1576,7 @@ include::ssl-settings.asciidoc[]
 :client-auth-default!:
 :verifies:
 :server:
+:ssl-context:            security-transport
 
 include::ssl-settings.asciidoc[]
 
diff --git a/docs/reference/settings/ssl-settings.asciidoc b/docs/reference/settings/ssl-settings.asciidoc
index f392d0c2fb816..a9c8576a8c4e9 100644
--- a/docs/reference/settings/ssl-settings.asciidoc
+++ b/docs/reference/settings/ssl-settings.asciidoc
@@ -1,4 +1,3 @@
-
 ==== {component} TLS/SSL Settings
 You can configure the following TLS/SSL settings. If the settings are not configured,
 the {ref}/security-settings.html#ssl-tls-settings[Default TLS/SSL Settings]
@@ -39,6 +38,7 @@ endif::verifies[]
 Supported cipher suites can be found in Oracle's http://docs.oracle.com/javase/8/docs/technotes/guides/security/SunProviders.html[
 Java Cryptography Architecture documentation]. Defaults to ``.
 
+[#{ssl-context}-tls-ssl-key-trusted-certificate-settings]
 ===== {component} TLS/SSL Key and Trusted Certificate Settings
 
 The following settings are used to specify a private key, certificate, and the
@@ -105,6 +105,7 @@ Password to the truststore.
 +{ssl-prefix}.ssl.truststore.secure_password+ (<<secure-settings,Secure>>)::
 Password to the truststore.
 
+[#{ssl-context}-pkcs12-files]
 ===== PKCS#12 Files
 
 {es} can be configured to use PKCS#12 container files (`.p12` or `.pfx` files)
@@ -143,6 +144,7 @@ Password to the PKCS#12 file.
 +{ssl-prefix}.ssl.truststore.secure_password+ (<<secure-settings,Secure>>)::
 Password to the PKCS#12 file.
 
+[#{ssl-context}-pkcs11-tokens]
 ===== PKCS#11 Tokens
 
 {es} can be configured to use a PKCS#11 token that contains the private key,
diff --git a/docs/reference/setup/important-settings/discovery-settings.asciidoc b/docs/reference/setup/important-settings/discovery-settings.asciidoc
index 245852b209609..942c076a33a0d 100644
--- a/docs/reference/setup/important-settings/discovery-settings.asciidoc
+++ b/docs/reference/setup/important-settings/discovery-settings.asciidoc
@@ -39,9 +39,9 @@ first election. In <<dev-vs-prod-mode,development mode>>, with no discovery
 settings configured, this step is automatically performed by the nodes
 themselves. As this auto-bootstrapping is <<modules-discovery-quorums,inherently
 unsafe>>, when you start a brand new cluster in <<dev-vs-prod-mode,production
-mode>>, you must explicitly list the names or IP addresses of the
-master-eligible nodes whose votes should be counted in the very first election.
-This list is set using the `cluster.initial_master_nodes` setting.
+mode>>, you must explicitly list the master-eligible nodes whose votes should be
+counted in the very first election.  This list is set using the
+`cluster.initial_master_nodes` setting.
 
 [source,yaml]
 --------------------------------------------------
diff --git a/docs/reference/setup/install.asciidoc b/docs/reference/setup/install.asciidoc
index 800cecb40652a..7e03ad3947b4c 100644
--- a/docs/reference/setup/install.asciidoc
+++ b/docs/reference/setup/install.asciidoc
@@ -59,6 +59,13 @@ downloaded from the Elastic Docker Registry.
 +
 {ref}/docker.html[Install {es} with Docker]
 
+`brew`::
+
+Formulae are available from the Elastic Homebrew tap for installing
+{es} on macOS with the Homebrew package manager.
++
+{ref}/brew.html[Install {es} on macOS with Homebrew]
+
 [float]
 [[config-mgmt-tools]]
 === Configuration Management Tools
@@ -84,3 +91,4 @@ include::install/windows.asciidoc[]
 
 include::install/docker.asciidoc[]
 
+include::install/brew.asciidoc[]
diff --git a/docs/reference/setup/install/brew.asciidoc b/docs/reference/setup/install/brew.asciidoc
new file mode 100644
index 0000000000000..cfc9e4c3e05be
--- /dev/null
+++ b/docs/reference/setup/install/brew.asciidoc
@@ -0,0 +1,69 @@
+[[brew]]
+=== Install {es} on macOS with Homebrew
+
+Elastic publishes Homebrew formulae so you can install {es} with the
+https://brew.sh/[Homebrew] package manager.
+
+To install with Homebrew, you first need to tap the
+Elastic Homebrew repository:
+
+[source,sh]
+-------------------------
+brew tap elastic/tap
+-------------------------
+
+Once you've tapped the Elastic Homebrew repo, you can use `brew install` to
+install the default distribution of {es}:
+
+[source,sh]
+-------------------------
+brew install elastic/tap/elasticsearch-full
+-------------------------
+
+This installs the most recently released default distribution of {es}.
+To install the OSS distribution, specify `elastic/tap/elasticsearch-oss`.
+
+[[brew-layout]]
+==== Directory layout for Homebrew installs
+
+When you install {es} with `brew install` the config files, logs,
+and data directory are stored in the following locations.
+
+[cols="<h,<,<m,<m",options="header",]
+|=======================================================================
+| Type | Description | Default Location | Setting
+| home
+  | Elasticsearch home directory or `$ES_HOME`
+  | /usr/local/var/homebrew/linked/elasticsearch-full
+ d|
+
+| bin
+  | Binary scripts including `elasticsearch` to start a node
+    and `elasticsearch-plugin` to install plugins
+  | /usr/local/var/homebrew/linked/elasticsearch-full/bin
+ d|
+
+| conf
+  | Configuration files including `elasticsearch.yml`
+  | /usr/local/etc/elasticsearch
+  | <<config-files-location,ES_PATH_CONF>>
+
+| data
+  | The location of the data files of each index / shard allocated
+    on the node. Can hold multiple locations.
+  | /usr/local/var/lib/elasticsearch
+  | path.data
+
+| logs
+  | Log files location.
+  | /usr/local/var/log/elasticsearch
+  | path.logs
+
+| plugins
+  | Plugin files location. Each plugin will be contained in a subdirectory.
+  | /usr/local/var/homebrew/linked/elasticsearch/plugins
+  |
+
+|=======================================================================
+
+include::next-steps.asciidoc[]
diff --git a/docs/reference/setup/install/docker.asciidoc b/docs/reference/setup/install/docker.asciidoc
index 1fcc261d68e1f..e8dd1ee95957a 100644
--- a/docs/reference/setup/install/docker.asciidoc
+++ b/docs/reference/setup/install/docker.asciidoc
@@ -332,7 +332,7 @@ data through a bind-mount:
 
 As a last resort, you can also force the container to mutate the ownership of
 any bind-mounts used for the <<path-settings,data and log dirs>> through the
-environment variable `TAKE_FILE_OWNERSHIP`. Inn this case, they will be owned by
+environment variable `TAKE_FILE_OWNERSHIP`. In this case, they will be owned by
 uid:gid `1000:0` providing read/write access to the {es} process as required.
 --
 
@@ -372,6 +372,12 @@ published ports with `--publish-all`, unless you are pinning one container per h
 
 . Use the `ES_JAVA_OPTS` environment variable to set heap size. For example, to
 use 16GB, use `-e ES_JAVA_OPTS="-Xms16g -Xmx16g"` with `docker run`.
++
+--
+NOTE: You still need to <<heap-size,configure the heap size>> even if you are
+https://docs.docker.com/config/containers/resource_constraints/#limit-a-containers-access-to-memory[limiting
+memory access] to the container.
+--
 
 . Pin your deployments to a specific version of the {es} Docker image, for
 example +docker.elastic.co/elasticsearch/elasticsearch:{version}+.
diff --git a/docs/reference/setup/secure-settings.asciidoc b/docs/reference/setup/secure-settings.asciidoc
index 4e3799db75bde..82b61848a846d 100644
--- a/docs/reference/setup/secure-settings.asciidoc
+++ b/docs/reference/setup/secure-settings.asciidoc
@@ -7,8 +7,12 @@ keystore and the `elasticsearch-keystore` tool to manage the settings in the key
 
 NOTE: All commands here should be run as the user which will run Elasticsearch.
 
-NOTE: Only some settings are designed to be read from the keystore. See
-documentation for each setting to see if it is supported as part of the keystore.
+IMPORTANT: Only some settings are designed to be read from the keystore. However,
+the keystore has no validation to block unsupported settings.
+Adding unsupported settings to the keystore will cause {es}
+Additional unsupported settings being added to the keystore will cause Elasticsearch
+to fail to start. See documentation for each setting to see if it is supported
+as part of the keystore.
 
 NOTE: All the modifications to the keystore take affect only after restarting
 Elasticsearch.
diff --git a/docs/reference/sql/endpoints/jdbc.asciidoc b/docs/reference/sql/endpoints/jdbc.asciidoc
index 4e9de8934895a..fdacb8b97a232 100644
--- a/docs/reference/sql/endpoints/jdbc.asciidoc
+++ b/docs/reference/sql/endpoints/jdbc.asciidoc
@@ -207,3 +207,8 @@ connection. For example:
 --------------------------------------------------
 include-tagged::{jdbc-tests}/SimpleExampleTestCase.java[simple_example]
 --------------------------------------------------
+
+NOTE:: {es-sql} doesn't provide a connection pooling mechanism, thus the connections
+the JDBC driver creates are not pooled. In order to achieve pooled connections,
+a third-party connection pooling mechanism is required. Configuring and setting up the
+third-party provider is outside the scope of this documentation.
\ No newline at end of file
diff --git a/docs/reference/sql/endpoints/rest.asciidoc b/docs/reference/sql/endpoints/rest.asciidoc
index e44649f3a8767..8140fe162d9de 100644
--- a/docs/reference/sql/endpoints/rest.asciidoc
+++ b/docs/reference/sql/endpoints/rest.asciidoc
@@ -3,6 +3,16 @@
 [[sql-rest]]
 == SQL REST API
 
+* <<sql-rest-overview>>
+* <<sql-rest-format>>
+* <<sql-pagination>>
+* <<sql-rest-filtering>>
+* <<sql-rest-columnar>>
+* <<sql-rest-fields>>
+
+[[sql-rest-overview]]
+=== Overview
+
 The SQL REST API accepts SQL in a JSON document, executes it,
 and returns the results. 
 For example:
@@ -30,18 +40,17 @@ Alastair Reynolds|Revelation Space    |585            |2000-03-15T00:00:00.000Z
 James S.A. Corey |Leviathan Wakes     |561            |2011-06-02T00:00:00.000Z
 --------------------------------------------------
 // TESTRESPONSE[s/\|/\\|/ s/\+/\\+/]
-// TESTRESPONSE[_cat]
+// TESTRESPONSE[non_json]
 
 [[sql-kibana-console]]
 .Using Kibana Console
-If you are using {kibana-ref}/console-kibana.html[Kibana Console].
+If you are using {kibana-ref}/console-kibana.html[Kibana Console]
 (which is highly recommended), take advantage of the
 triple quotes `"""` when creating the query. This not only automatically escapes double
 quotes (`"`) inside the query string but also support multi-line as shown below:
 image:images/sql/rest/console-triple-quotes.png[]
 
 [[sql-rest-format]]
-[float]
 === Response Data Formats
 
 While the textual format is nice for humans, computers prefer something
@@ -94,6 +103,35 @@ s|Description
 
 |===
 
+Here are some examples for the human readable formats:
+
+==== CSV
+
+[source,js]
+--------------------------------------------------
+POST /_sql?format=csv
+{
+    "query": "SELECT * FROM library ORDER BY page_count DESC",
+    "fetch_size": 5
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:library]
+
+Which returns:
+
+[source,text]
+--------------------------------------------------
+author,name,page_count,release_date
+Peter F. Hamilton,Pandora's Star,768,2004-03-02T00:00:00.000Z
+Vernor Vinge,A Fire Upon the Deep,613,1992-06-01T00:00:00.000Z
+Frank Herbert,Dune,604,1965-06-01T00:00:00.000Z
+Alastair Reynolds,Revelation Space,585,2000-03-15T00:00:00.000Z
+James S.A. Corey,Leviathan Wakes,561,2011-06-02T00:00:00.000Z
+--------------------------------------------------
+// TESTRESPONSE[non_json]
+
+==== JSON
 
 [source,js]
 --------------------------------------------------
@@ -129,8 +167,113 @@ Which returns:
 --------------------------------------------------
 // TESTRESPONSE[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl\+v\/\/\/w8=/$body.cursor/]
 
+==== TSV
+
+[source,js]
+--------------------------------------------------
+POST /_sql?format=tsv
+{
+    "query": "SELECT * FROM library ORDER BY page_count DESC",
+    "fetch_size": 5
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:library]
+
+Which returns:
+
+[source,text]
+--------------------------------------------------
+author	name	page_count	release_date
+Peter F. Hamilton	Pandora's Star	768	2004-03-02T00:00:00.000Z
+Vernor Vinge	A Fire Upon the Deep	613	1992-06-01T00:00:00.000Z
+Frank Herbert	Dune	604	1965-06-01T00:00:00.000Z
+Alastair Reynolds	Revelation Space	585	2000-03-15T00:00:00.000Z
+James S.A. Corey	Leviathan Wakes	561	2011-06-02T00:00:00.000Z
+--------------------------------------------------
+// TESTRESPONSE[s/\t/ /]
+// TESTRESPONSE[non_json]
+
+==== TXT
+
+[source,js]
+--------------------------------------------------
+POST /_sql?format=txt
+{
+    "query": "SELECT * FROM library ORDER BY page_count DESC",
+    "fetch_size": 5
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:library]
+
+Which returns:
+
+[source,text]
+--------------------------------------------------
+     author      |        name        |  page_count   |      release_date      
+-----------------+--------------------+---------------+------------------------
+Peter F. Hamilton|Pandora's Star      |768            |2004-03-02T00:00:00.000Z
+Vernor Vinge     |A Fire Upon the Deep|613            |1992-06-01T00:00:00.000Z
+Frank Herbert    |Dune                |604            |1965-06-01T00:00:00.000Z
+Alastair Reynolds|Revelation Space    |585            |2000-03-15T00:00:00.000Z
+James S.A. Corey |Leviathan Wakes     |561            |2011-06-02T00:00:00.000Z
+--------------------------------------------------
+// TESTRESPONSE[s/\|/\\|/ s/\+/\\+/]
+// TESTRESPONSE[non_json]
+
+==== YAML
+
+[source,js]
+--------------------------------------------------
+POST /_sql?format=yaml
+{
+    "query": "SELECT * FROM library ORDER BY page_count DESC",
+    "fetch_size": 5
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:library]
+
+Which returns:
+
+[source,yaml]
+--------------------------------------------------
+columns:
+- name: "author"
+  type: "text"
+- name: "name"
+  type: "text"
+- name: "page_count"
+  type: "short"
+- name: "release_date"
+  type: "datetime"
+rows:
+- - "Peter F. Hamilton"
+  - "Pandora's Star"
+  - 768
+  - "2004-03-02T00:00:00.000Z"
+- - "Vernor Vinge"
+  - "A Fire Upon the Deep"
+  - 613
+  - "1992-06-01T00:00:00.000Z"
+- - "Frank Herbert"
+  - "Dune"
+  - 604
+  - "1965-06-01T00:00:00.000Z"
+- - "Alastair Reynolds"
+  - "Revelation Space"
+  - 585
+  - "2000-03-15T00:00:00.000Z"
+- - "James S.A. Corey"
+  - "Leviathan Wakes"
+  - 561
+  - "2011-06-02T00:00:00.000Z"
+cursor: "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl+v///w8="
+--------------------------------------------------
+// TESTRESPONSE[s/sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmBmF1dGhvcgFmBG5hbWUBZgpwYWdlX2NvdW50AWYMcmVsZWFzZV9kYXRl\+v\/\/\/w8=/$body.cursor/]
+
 [[sql-pagination]]
-[float]
 === Paginating through a large response
 
 Using the example above, one can continue to the next page by sending back the `cursor` field. In
@@ -198,7 +341,6 @@ Which will like return the
 
 
 [[sql-rest-filtering]]
-[float]
 === Filtering using {es} query DSL
 
 You can filter the results that SQL will run on using a standard
@@ -233,10 +375,9 @@ Which returns:
 Douglas Adams  |The Hitchhiker's Guide to the Galaxy|180            |1979-10-12T00:00:00.000Z
 --------------------------------------------------
 // TESTRESPONSE[s/\|/\\|/ s/\+/\\+/]
-// TESTRESPONSE[_cat]
+// TESTRESPONSE[non_json]
 
 [[sql-rest-columnar]]
-[float]
 === Columnar results
 
 The most well known way of displaying the results of an SQL query result in general is the one where each
@@ -311,7 +452,6 @@ Which looks like:
 // TESTRESPONSE[s/46ToAwFzQERYRjFaWEo1UVc1a1JtVjBZMmdCQUFBQUFBQUFBQUVXWjBaNlFXbzNOV0pVY21Wa1NUZDJhV2t3V2xwblp3PT3\/\/\/\/\/DwQBZgZhdXRob3IBBHRleHQAAAFmBG5hbWUBBHRleHQAAAFmCnBhZ2VfY291bnQBBGxvbmcBAAFmDHJlbGVhc2VfZGF0ZQEIZGF0ZXRpbWUBAAEP/$body.cursor/]
 
 [[sql-rest-fields]]
-[float]
 === Supported REST parameters
 
 In addition to the `query` and `fetch_size`, a request a number of user-defined fields for specifying
diff --git a/docs/reference/sql/functions/conditional.asciidoc b/docs/reference/sql/functions/conditional.asciidoc
index d0b8e7d2ff3f1..0206115c182c1 100644
--- a/docs/reference/sql/functions/conditional.asciidoc
+++ b/docs/reference/sql/functions/conditional.asciidoc
@@ -33,17 +33,17 @@ If the condition’s result is true, the value of the result expression that fol
 the subsequent when clauses will be skipped and not processed.
 
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[case]
 ----
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[caseReturnNull]
 ----
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[caseWithElse]
 ----
@@ -70,12 +70,12 @@ CASE WHEN expression = value1 THEN result1
 END
 ----
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[caseWithOperand]
 ----
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[caseWithOperandAndElse]
 ----
@@ -155,12 +155,12 @@ If all arguments are null, then it returns `null`.
 
 
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[coalesceReturnNonNull]
 ----
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[coalesceReturnNull]
 ----
@@ -199,12 +199,12 @@ If all arguments are null, then it returns `null`.
 
 
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[greatestReturnNonNull]
 ----
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[greatestReturnNull]
 ----
@@ -237,12 +237,12 @@ If all arguments are null, then it returns `null`.
 
 
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[ifNullReturnFirst]
 ----
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[ifNullReturnSecond]
 ----
@@ -277,12 +277,12 @@ logic of programming languages. If the 3rd expression is not provided and the co
 `null` is returned.
 
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[iifWithDefaultValue]
 ----
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[iifWithoutDefaultValue]
 ----
@@ -325,12 +325,12 @@ If all arguments are null, then it returns `null`.
 
 
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[isNullReturnFirst]
 ----
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[isNullReturnSecond]
 ----
@@ -370,12 +370,12 @@ If all arguments are null, then it returns `null`.
 
 
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[leastReturnNonNull]
 ----
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[leastReturnNull]
 ----
@@ -407,12 +407,12 @@ Returns `null` when the two input expressions are equal and
 if not, it returns the 1st expression.
 
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[nullIfReturnFirst]
 ----
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[nullIfReturnNull]
 ----
@@ -446,12 +446,12 @@ If all arguments are null, then it returns `null`.
 
 
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[nvlReturnFirst]
 ----
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[nvlReturnSecond]
 ----
diff --git a/docs/reference/sql/functions/date-time.asciidoc b/docs/reference/sql/functions/date-time.asciidoc
index d9d5e7bcf14e5..45231393521c7 100644
--- a/docs/reference/sql/functions/date-time.asciidoc
+++ b/docs/reference/sql/functions/date-time.asciidoc
@@ -57,32 +57,32 @@ s|Description
 
 Basic arithmetic operators (`+`, `-`, etc) support date/time parameters as indicated below:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[dtIntervalPlusInterval]
 --------------------------------------------------
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[dtDateTimePlusInterval]
 --------------------------------------------------
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[dtMinusInterval]
 --------------------------------------------------
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[dtIntervalMinusInterval]
 --------------------------------------------------
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[dtDateTimeMinusInterval]
 --------------------------------------------------
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[dtIntervalMul]
 --------------------------------------------------
@@ -116,17 +116,17 @@ Unlike CURRENT_DATE, `CURDATE()` can only be used as a function with no argument
 
 This method always returns the same value for its every occurrence within the same query.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[currentDate]
 --------------------------------------------------
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[currentDateFunction]
 --------------------------------------------------
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[curDateFunction]
 --------------------------------------------------
@@ -134,7 +134,7 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[curDateFunction]
 Typically, this function (as well as its twin <<sql-functions-today,TODAY())>> function
 is used for relative date filtering:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[filterToday]
 --------------------------------------------------
@@ -165,29 +165,29 @@ meaning a milliseconds precision current time will be returned.
 
 This method always returns the same value for its every occurrence within the same query.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[currentTime]
 --------------------------------------------------
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[currentTimeFunction]
 --------------------------------------------------
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[curTimeFunction]
 --------------------------------------------------
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[currentTimeFunctionPrecision]
 --------------------------------------------------
 
 Typically, this function is used for relative date/time filtering:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[filterCurrentTime]
 --------------------------------------------------
@@ -221,17 +221,17 @@ meaning a milliseconds precision current date/time will be returned.
 
 This method always returns the same value for its every occurrence within the same query.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[curTs]
 --------------------------------------------------
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[curTsFunction]
 --------------------------------------------------
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[curTsFunctionPrecision]
 --------------------------------------------------
@@ -239,7 +239,7 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[curTsFunctionPrecision]
 Typically, this function (as well as its twin <<sql-functions-now,NOW())>> function is used for
 relative date/time filtering:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[filterNow]
 --------------------------------------------------
@@ -267,7 +267,7 @@ DAY_OF_MONTH(datetime_exp) <1>
 
 Extract the day of the month from a date/datetime.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[dayOfMonth]
 --------------------------------------------------
@@ -291,7 +291,7 @@ DAY_OF_WEEK(datetime_exp) <1>
 
 Extract the day of the week from a date/datetime. Sunday is `1`, Monday is `2`, etc.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[dayOfWeek]
 --------------------------------------------------
@@ -315,7 +315,7 @@ DAY_OF_YEAR(datetime_exp) <1>
 
 Extract the day of the year from a date/datetime.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[dayOfYear]
 --------------------------------------------------
@@ -339,7 +339,7 @@ DAY_NAME(datetime_exp) <1>
 
 Extract the day of the week from a date/datetime in text format (`Monday`, `Tuesday`...).
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[dayName]
 --------------------------------------------------
@@ -363,7 +363,7 @@ HOUR_OF_DAY(datetime_exp) <1>
 
 Extract the hour of the day from a date/datetime.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[hourOfDay]
 --------------------------------------------------
@@ -388,7 +388,7 @@ ISO_DAY_OF_WEEK(datetime_exp) <1>
 Extract the day of the week from a date/datetime, following the https://en.wikipedia.org/wiki/ISO_week_date[ISO 8601 standard].
 Monday is `1`, Tuesday is `2`, etc.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[isoDayOfWeek]
 --------------------------------------------------
@@ -413,7 +413,7 @@ ISO_WEEK_OF_YEAR(datetime_exp) <1>
 Extract the week of the year from a date/datetime, following https://en.wikipedia.org/wiki/ISO_week_date[ISO 8601 standard]. The first week
 of a year is the first week with a majority (4 or more) of its days in January.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[isoWeekOfYear]
 --------------------------------------------------
@@ -437,7 +437,7 @@ MINUTE_OF_DAY(datetime_exp) <1>
 
 Extract the minute of the day from a date/datetime.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[minuteOfDay]
 --------------------------------------------------
@@ -461,7 +461,7 @@ MINUTE_OF_HOUR(datetime_exp) <1>
 
 Extract the minute of the hour from a date/datetime.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[minuteOfHour]
 --------------------------------------------------
@@ -485,7 +485,7 @@ MONTH(datetime_exp) <1>
 
 Extract the month of the year from a date/datetime.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[monthOfYear]
 --------------------------------------------------
@@ -509,7 +509,7 @@ MONTH_NAME(datetime_exp) <1>
 
 Extract the month from a date/datetime in text format (`January`, `February`...).
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[monthName]
 --------------------------------------------------
@@ -533,7 +533,7 @@ This function offers the same functionality as <<sql-functions-current-timestamp
 the datetime when the current query reached the server. This method always returns the same value for its every
 occurrence within the same query.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[nowFunction]
 --------------------------------------------------
@@ -541,7 +541,7 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[nowFunction]
 Typically, this function (as well as its twin <<sql-functions-current-timestamp,CURRENT_TIMESTAMP())>> function is used
 for relative date/time filtering:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[filterNow]
 --------------------------------------------------
@@ -565,7 +565,7 @@ SECOND_OF_MINUTE(datetime_exp) <1>
 
 Extract the second of the minute from a date/datetime.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[secondOfMinute]
 --------------------------------------------------
@@ -589,7 +589,7 @@ QUARTER(datetime_exp) <1>
 
 Extract the year quarter the date/datetime falls in.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[quarter]
 --------------------------------------------------
@@ -613,7 +613,7 @@ This function offers the same functionality as <<sql-functions-current-date,CURR
 the date when the current query reached the server. This method always returns the same value for its every occurrence
 within the same query.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[todayFunction]
 --------------------------------------------------
@@ -621,7 +621,7 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[todayFunction]
 Typically, this function (as well as its twin <<sql-functions-current-timestamp,CURRENT_TIMESTAMP())>> function is used
 for relative date filtering:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[filterToday]
 --------------------------------------------------
@@ -645,7 +645,7 @@ WEEK_OF_YEAR(datetime_exp) <1>
 
 Extract the week of the year from a date/datetime.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[weekOfYear]
 --------------------------------------------------
@@ -669,7 +669,7 @@ YEAR(datetime_exp) <1>
 
 Extract the year from a date/datetime.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[year]
 --------------------------------------------------
@@ -697,14 +697,14 @@ EXTRACT(
 Extract fields from a date/datetime by specifying the name of a <<sql-functions-datetime,datetime function>>.
 The following
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[extractDayOfYear]
 --------------------------------------------------
 
 is the equivalent to
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[dayOfYear]
 --------------------------------------------------
diff --git a/docs/reference/sql/functions/geo.asciidoc b/docs/reference/sql/functions/geo.asciidoc
index 72f69af85529f..fc9a85ce97e4d 100644
--- a/docs/reference/sql/functions/geo.asciidoc
+++ b/docs/reference/sql/functions/geo.asciidoc
@@ -147,7 +147,7 @@ ST_Y(
 
 .Description:
 
-Returns the the latitude of the first point in the geometry.
+Returns the latitude of the first point in the geometry.
 
 ["source","sql",subs="attributes,macros"]
 --------------------------------------------------
@@ -206,4 +206,4 @@ Returns the distance between geometries in meters. Both geometries have to be po
 ["source","sql",subs="attributes,macros"]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/geo.csv-spec[distance]
---------------------------------------------------
\ No newline at end of file
+--------------------------------------------------
diff --git a/docs/reference/sql/functions/grouping.asciidoc b/docs/reference/sql/functions/grouping.asciidoc
index 0a498a1aacef0..6f2f5a1b6e4c2 100644
--- a/docs/reference/sql/functions/grouping.asciidoc
+++ b/docs/reference/sql/functions/grouping.asciidoc
@@ -44,14 +44,14 @@ NOTE:: The histogram in SQL does *NOT* return empty buckets for missing interval
 `Histogram` can be applied on either numeric fields:
 
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[histogramNumeric]
 ----
 
 or date/time fields:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[histogramDateTime]
 ----
@@ -59,14 +59,14 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[histogramDateTime]
 Expressions inside the histogram are also supported as long as the
 return type is numeric:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[histogramNumericExpression]
 ----
 
 Do note that histograms (and grouping functions in general) allow custom expressions but cannot have any functions applied to them in the `GROUP BY`. In other words, the following statement is *NOT* allowed:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[expressionOnHistogramNotAllowed]
 ----
@@ -75,7 +75,7 @@ as it requires two groupings (one for histogram followed by a second for applyin
 
 Instead one can rewrite the query to move the expression on the histogram _inside_ of it:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[histogramDateTimeExpression]
 ----
diff --git a/docs/reference/sql/functions/like-rlike.asciidoc b/docs/reference/sql/functions/like-rlike.asciidoc
index 73212bc113542..2d5ef0b62f93f 100644
--- a/docs/reference/sql/functions/like-rlike.asciidoc
+++ b/docs/reference/sql/functions/like-rlike.asciidoc
@@ -38,7 +38,7 @@ with the `LIKE` operator:
 The percent sign represents zero, one or multiple characters. The underscore represents a single number or character. These symbols can be
 used in combinations.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[simpleLike]
 ----
@@ -75,7 +75,7 @@ and underscore (`_`); the pattern in this case is a regular expression which all
 For more details about the regular expressions syntax, https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/regex/Pattern.html[Java's Pattern class javadoc]
 is a good starting point.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[simpleRLike]
 ----
diff --git a/docs/reference/sql/functions/operators.asciidoc b/docs/reference/sql/functions/operators.asciidoc
index 4b7e8990290dd..02841c84b58e5 100644
--- a/docs/reference/sql/functions/operators.asciidoc
+++ b/docs/reference/sql/functions/operators.asciidoc
@@ -8,7 +8,7 @@ Boolean operator for comparing against one or multiple expressions.
 [[sql-operators-equality]]
 ==== `Equality (=)`
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/filter.sql-spec[whereFieldEquality]
 --------------------------------------------------
@@ -16,12 +16,12 @@ include-tagged::{sql-specs}/filter.sql-spec[whereFieldEquality]
 [[sql-operators-null-safe-equality]]
 ==== `Null safe Equality (<=>)`
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[nullEqualsCompareWithNull]
 --------------------------------------------------
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[nullEqualsCompareTwoNulls]
 --------------------------------------------------
@@ -29,7 +29,7 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[nullEqualsCompareTwoNulls]
 [[sql-operators-inequality]]
 ==== `Inequality (<> or !=)`
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/filter.sql-spec[whereFieldNonEquality]
 --------------------------------------------------
@@ -37,7 +37,7 @@ include-tagged::{sql-specs}/filter.sql-spec[whereFieldNonEquality]
 [[sql-operators-comparison]]
 ==== `Comparison (<, <=, >, >=)`
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/filter.sql-spec[whereFieldLessThan]
 --------------------------------------------------
@@ -45,7 +45,7 @@ include-tagged::{sql-specs}/filter.sql-spec[whereFieldLessThan]
 [[sql-operators-between]]
 ==== `BETWEEN`
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/filter.sql-spec[whereBetween]
 --------------------------------------------------
@@ -53,7 +53,7 @@ include-tagged::{sql-specs}/filter.sql-spec[whereBetween]
 [[sql-operators-is-null]]
 ==== `IS NULL/IS NOT NULL`
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/filter.sql-spec[whereIsNotNullAndIsNull]
 --------------------------------------------------
@@ -61,7 +61,7 @@ include-tagged::{sql-specs}/filter.sql-spec[whereIsNotNullAndIsNull]
 [[sql-operators-in]]
 ==== `IN (<value1>, <value2>, ...)`
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/filter.sql-spec[whereWithInAndMultipleValues]
 --------------------------------------------------
@@ -74,7 +74,7 @@ Boolean operator for evaluating one or two expressions.
 [[sql-operators-and]]
 ==== `AND`
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/filter.sql-spec[whereFieldAndComparison]
 --------------------------------------------------
@@ -82,7 +82,7 @@ include-tagged::{sql-specs}/filter.sql-spec[whereFieldAndComparison]
 [[sql-operators-or]]
 ==== `OR`
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/filter.sql-spec[whereFieldOrComparison]
 --------------------------------------------------
@@ -90,7 +90,7 @@ include-tagged::{sql-specs}/filter.sql-spec[whereFieldOrComparison]
 [[sql-operators-not]]
 ==== `NOT`
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/filter.sql-spec[whereFieldEqualityNot]
 --------------------------------------------------
@@ -104,7 +104,7 @@ The result is a value of numeric type.
 [[sql-operators-plus]]
 ==== `Add (+)`
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/arithmetic.sql-spec[plus]
 --------------------------------------------------
@@ -112,7 +112,7 @@ include-tagged::{sql-specs}/arithmetic.sql-spec[plus]
 [[sql-operators-subtract]]
 ==== `Subtract (infix -)`
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/arithmetic.sql-spec[minus]
 --------------------------------------------------
@@ -120,7 +120,7 @@ include-tagged::{sql-specs}/arithmetic.sql-spec[minus]
 [[sql-operators-negate]]
 ==== `Negate (unary -)`
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/arithmetic.sql-spec[unaryMinus]
 --------------------------------------------------
@@ -128,7 +128,7 @@ include-tagged::{sql-specs}/arithmetic.sql-spec[unaryMinus]
 [[sql-operators-multiply]]
 ==== `Multiply (*)`
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/arithmetic.sql-spec[multiply]
 --------------------------------------------------
@@ -136,7 +136,7 @@ include-tagged::{sql-specs}/arithmetic.sql-spec[multiply]
 [[sql-operators-divide]]
 ==== `Divide (/)`
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/arithmetic.sql-spec[divide]
 --------------------------------------------------
@@ -144,7 +144,7 @@ include-tagged::{sql-specs}/arithmetic.sql-spec[divide]
 [[sql-operators-remainder]]
 ==== `Modulo or Remainder(%)`
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/arithmetic.sql-spec[mod]
 --------------------------------------------------
@@ -157,7 +157,7 @@ include-tagged::{sql-specs}/arithmetic.sql-spec[mod]
 
 `::` provides an alternative syntax to the <<sql-functions-type-conversion-cast>> function.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[conversionStringToLongCastOperator]
 --------------------------------------------------
diff --git a/docs/reference/sql/functions/search.asciidoc b/docs/reference/sql/functions/search.asciidoc
index 6990f6669d69c..34716e070434f 100644
--- a/docs/reference/sql/functions/search.asciidoc
+++ b/docs/reference/sql/functions/search.asciidoc
@@ -33,7 +33,7 @@ and <<query-dsl-multi-match-query,multi_match>> {es} queries.
 
 The first parameter is the field or fields to match against. In case it receives one value only, {es-sql} will use a `match` query to perform the search:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[simpleMatch]
 ----
@@ -41,7 +41,7 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[simpleMatch]
 However, it can also receive a list of fields and their corresponding optional `boost` value. In this case, {es-sql} will use a
 `multi_match` query to match the documents:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[multiFieldsMatch]
 ----
@@ -53,22 +53,17 @@ the final score than the `author` field when searching for `frank dune` text in
 Both options above can be used in combination with the optional third parameter of the `MATCH()` predicate, where one can specify
 additional configuration parameters (separated by semicolon `;`) for either `match` or `multi_match` queries. For example:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[optionalParamsForMatch]
 ----
 
-In the more advanced example above, the `cutoff_frequency` parameter allows specifying an absolute or relative document frequency where
-high frequency terms are moved into an optional subquery and are only scored if one of the low frequency (below the cutoff) terms in the 
-case of an `or` operator or all of the low frequency terms in the case of an `and` operator match. More about this you can find in the
-<<query-dsl-match-query-cutoff>> page.
-
 NOTE: The allowed optional parameters for a single-field `MATCH()` variant (for the `match` {es} query) are: `analyzer`, `auto_generate_synonyms_phrase_query`,
-`cutoff_frequency`, `lenient`, `fuzziness`, `fuzzy_transpositions`, `fuzzy_rewrite`, `minimum_should_match`, `operator`,
+`lenient`, `fuzziness`, `fuzzy_transpositions`, `fuzzy_rewrite`, `minimum_should_match`, `operator`,
 `max_expansions`, `prefix_length`.
 
 NOTE: The allowed optional parameters for a multi-field `MATCH()` variant (for the `multi_match` {es} query) are: `analyzer`, `auto_generate_synonyms_phrase_query`,
-`cutoff_frequency`, `lenient`, `fuzziness`, `fuzzy_transpositions`, `fuzzy_rewrite`, `minimum_should_match`, `operator`,
+`lenient`, `fuzziness`, `fuzzy_transpositions`, `fuzzy_rewrite`, `minimum_should_match`, `operator`,
 `max_expansions`, `prefix_length`, `slop`, `tie_breaker`, `type`.
 
 
@@ -95,14 +90,14 @@ Just like `MATCH`, `QUERY` is a full-text search predicate that gives the user c
 The first parameter is basically the input that will be passed as is to the `query_string` query, which means that anything that `query_string`
 accepts in its `query` field can be used here as well:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[simpleQueryQuery]
 ----
 
 A more advanced example, showing more of the features that `query_string` supports, of course possible with {es-sql}:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[advancedQueryQuery]
 ----
@@ -113,7 +108,7 @@ regex and fuzziness queries for the `name` field.
 If one needs to customize various configuration options that `query_string` exposes, this can be done using the second _optional_ parameter.
 Multiple settings can be specified separated by a semicolon `;`:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[optionalParameterQuery]
 ----
@@ -149,14 +144,14 @@ combined using the same rules as {es}'s
 
 Typically `SCORE` is used for ordering the results of a query based on their relevance:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[orderByScore]
 ----
 
 However, it is perfectly fine to return the score without sorting by it:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[scoreWithMatch]
 ----
diff --git a/docs/reference/sql/functions/string.asciidoc b/docs/reference/sql/functions/string.asciidoc
index 7acc358763512..a82ac66adce1c 100644
--- a/docs/reference/sql/functions/string.asciidoc
+++ b/docs/reference/sql/functions/string.asciidoc
@@ -24,7 +24,7 @@ ASCII(string_exp) <1>
 
 Returns the ASCII code value of the leftmost character of `string_exp` as an integer.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringAscii]
 --------------------------------------------------
@@ -47,7 +47,7 @@ BIT_LENGTH(string_exp) <1>
 
 Returns the length in bits of the `string_exp` input expression.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringBitLength]
 --------------------------------------------------
@@ -70,7 +70,7 @@ CHAR(code) <1>
 
 Returns the character that has the ASCII code value specified by the numeric input. The value should be between 0 and 255; otherwise, the return value is data source–dependent.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringChar]
 --------------------------------------------------
@@ -93,7 +93,7 @@ CHAR_LENGTH(string_exp) <1>
 
 Returns the length in characters of the input, if the string expression is of a character data type; otherwise, returns the length in bytes of the string expression (the smallest integer not less than the number of bits divided by 8).
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringCharLength]
 --------------------------------------------------
@@ -119,7 +119,7 @@ CONCAT(
 
 Returns a character string that is the result of concatenating `string_exp1` to `string_exp2`. If one of the string is `NULL`, the other string will be returned.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringConcat]
 --------------------------------------------------
@@ -149,7 +149,7 @@ INSERT(
 
 Returns a string where `length` characters have been deleted from `source`, beginning at `start`, and where `replacement` has been inserted into `source`, beginning at `start`.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringInsert]
 --------------------------------------------------
@@ -172,7 +172,7 @@ LCASE(string_exp) <1>
 
 Returns a string equal to that in `string_exp`, with all uppercase characters converted to lowercase.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringLCase]
 --------------------------------------------------
@@ -198,7 +198,7 @@ LEFT(
 
 Returns the leftmost count characters of `string_exp`.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringLeft]
 --------------------------------------------------
@@ -221,7 +221,7 @@ LENGTH(string_exp) <1>
 
 Returns the number of characters in `string_exp`, excluding trailing blanks.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringLength]
 --------------------------------------------------
@@ -250,12 +250,12 @@ LOCATE(
 
 Returns the starting position of the first occurrence of `pattern` within `source`. The search for the first occurrence of `pattern` begins with the first character position in `source` unless the optional argument, `start`, is specified. If `start` is specified, the search begins with the character position indicated by the value of `start`. The first character position in `source` is indicated by the value 1. If `pattern` is not found within `source`, the value 0 is returned.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringLocateWoStart]
 --------------------------------------------------
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringLocateWithStart]
 --------------------------------------------------
@@ -278,7 +278,7 @@ LTRIM(string_exp) <1>
 
 Returns the characters of `string_exp`, with leading blanks removed.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringLTrim]
 --------------------------------------------------
@@ -301,7 +301,7 @@ OCTET_LENGTH(string_exp) <1>
 
 Returns the length in bytes of the `string_exp` input expression.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringOctetLength]
 --------------------------------------------------
@@ -327,7 +327,7 @@ POSITION(
 
 Returns the position of the `string_exp1` in `string_exp2`. The result is an exact numeric.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringPosition]
 --------------------------------------------------
@@ -353,7 +353,7 @@ REPEAT(
 
 Returns a character string composed of `string_exp` repeated `count` times.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringRepeat]
 --------------------------------------------------
@@ -381,7 +381,7 @@ REPLACE(
 
 Search `source` for occurrences of `pattern`, and replace with `replacement`.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringReplace]
 --------------------------------------------------
@@ -407,7 +407,7 @@ RIGHT(
 
 Returns the rightmost count characters of `string_exp`.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringRight]
 --------------------------------------------------
@@ -430,7 +430,7 @@ RTRIM(string_exp) <1>
 
 Returns the characters of `string_exp` with trailing blanks removed.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringRTrim]
 --------------------------------------------------
@@ -453,7 +453,7 @@ SPACE(count) <1>
 
 Returns a character string consisting of `count` spaces.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringSpace]
 --------------------------------------------------
@@ -481,7 +481,7 @@ SUBSTRING(
 
 Returns a character string that is derived from `source`, beginning at the character position specified by `start` for `length` characters.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringSubString]
 --------------------------------------------------
@@ -504,7 +504,7 @@ UCASE(string_exp) <1>
 
 Returns a string equal to that of the input, with all lowercase characters converted to uppercase.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringUCase]
 --------------------------------------------------
diff --git a/docs/reference/sql/functions/system.asciidoc b/docs/reference/sql/functions/system.asciidoc
index dfca7d526d3a6..b2d604728c165 100644
--- a/docs/reference/sql/functions/system.asciidoc
+++ b/docs/reference/sql/functions/system.asciidoc
@@ -24,7 +24,7 @@ Returns the name of the database being queried. In the case of Elasticsearch SQL
 is the name of the Elasticsearch cluster. This function should always return a non-null
 value.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[database]
 --------------------------------------------------
@@ -46,7 +46,7 @@ USER()
 Returns the username of the authenticated user executing the query. This function can
 return `null` in case {stack-ov}/elasticsearch-security.html[Security] is disabled.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 --------------------------------------------------
 include-tagged::{sql-specs}/docs/docs.csv-spec[user]
 --------------------------------------------------
diff --git a/docs/reference/sql/functions/type-conversion.asciidoc b/docs/reference/sql/functions/type-conversion.asciidoc
index 7f8488be40f64..c6c761305519e 100644
--- a/docs/reference/sql/functions/type-conversion.asciidoc
+++ b/docs/reference/sql/functions/type-conversion.asciidoc
@@ -25,17 +25,17 @@ Casts the result of the given expression to the target <<sql-data-types, data ty
 If the cast is not possible (for example because of target type is too narrow or because
 the value itself cannot be converted), the query fails.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[conversionStringToIntCast]
 ----
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[conversionIntToStringCast]
 ----
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[conversionStringToDateTimeCast]
 ----
@@ -66,12 +66,12 @@ Works exactly like <<sql-functions-type-conversion-cast>> with slightly differen
 Moreover, apart from the standard <<sql-data-types, data types>> it supports the corresponding
 https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/explicit-data-type-conversion-function?view=sql-server-2017[ODBC data types].
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[conversionStringToIntConvertODBCDataType]
 ----
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[conversionStringToIntConvertESDataType]
 ----
diff --git a/docs/reference/sql/getting-started.asciidoc b/docs/reference/sql/getting-started.asciidoc
index 3ce7b6ebdf3e3..ffd52060e2b0f 100644
--- a/docs/reference/sql/getting-started.asciidoc
+++ b/docs/reference/sql/getting-started.asciidoc
@@ -40,7 +40,7 @@ Dan Simmons    |Hyperion       |482            |1989-05-26T00:00:00.000Z
 Frank Herbert  |Dune           |604            |1965-06-01T00:00:00.000Z
 --------------------------------------------------
 // TESTRESPONSE[s/\|/\\|/ s/\+/\\+/]
-// TESTRESPONSE[_cat]
+// TESTRESPONSE[non_json]
 
 You can also use the <<sql-cli>>. There is a script to start it
 shipped in x-pack's bin directory:
diff --git a/docs/reference/sql/language/data-types.asciidoc b/docs/reference/sql/language/data-types.asciidoc
index ad9b2a320c0c6..7c5b4db045f9b 100644
--- a/docs/reference/sql/language/data-types.asciidoc
+++ b/docs/reference/sql/language/data-types.asciidoc
@@ -1,7 +1,7 @@
 [role="xpack"]
 [testenv="basic"]
 [[sql-data-types]]
-== Data Types
+=== Data Types
 
 [cols="^,^m,^,^"]
 
@@ -89,7 +89,7 @@ s|SQL precision
 
 [[sql-multi-field]]
 [float]
-=== SQL and multi-fields
+==== SQL and multi-fields
 
 A core concept in {es} is that of an `analyzed` field, that is a full-text value that is interpreted in order
 to be effectively indexed. These fields are of type <<text, `text`>> and are not used for sorting or aggregations as their actual value depends on the <<analyzer, `analyzer`>> used hence why {es} also offers the <<keyword, `keyword`>> type for storing the _exact_
diff --git a/docs/reference/sql/language/indices.asciidoc b/docs/reference/sql/language/indices.asciidoc
index 82c7f30fb041e..78834470df5db 100644
--- a/docs/reference/sql/language/indices.asciidoc
+++ b/docs/reference/sql/language/indices.asciidoc
@@ -1,20 +1,20 @@
 [role="xpack"]
 [testenv="basic"]
 [[sql-index-patterns]]
-== Index patterns
+=== Index patterns
 
 {es-sql} supports two types of patterns for matching multiple indices or tables:
 
 [[sql-index-patterns-multi]]
 [float]
-=== {es} multi-index
+==== {es} multi-index
 
 The {es} notation for enumerating, including or excluding <<multi-index,multi index syntax>>
 is supported _as long_ as it is quoted or escaped as a table identifier.
 
 For example:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showTablesEsMultiIndex]
 ----
@@ -28,7 +28,7 @@ The same kind of patterns can also be used to query multiple indices or tables.
 
 For example:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[fromTablePatternQuoted]
 ----
@@ -37,14 +37,14 @@ NOTE: There is the restriction that all resolved concrete tables have the exact
 
 [[sql-index-patterns-like]]
 [float]
-=== SQL `LIKE` notation
+==== SQL `LIKE` notation
 
 The common `LIKE` statement (including escaping if needed) to match a wildcard pattern, based on one `_`
 or multiple `%` characters.
 
 Using `SHOW TABLES` command again:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showTablesLikeWildcard]
 ----
@@ -53,7 +53,7 @@ The pattern matches all tables that start with `emp`.
 
 This command supports _escaping_ as well, for example:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showTablesLikeEscape]
 ----
@@ -86,7 +86,7 @@ NOTE: As the query type of quoting between the two patterns is fairly similar (`
 requires the keyword `LIKE` for SQL `LIKE` pattern.
 
 [[sql-index-frozen]]
-== Frozen Indices
+=== Frozen Indices
 
 {es} <<frozen-indices, frozen indices>> are a useful and powerful tool for hot/warm architecture introduced in {es} 6.6, 
 essentially by trading speed for memory.
@@ -101,13 +101,13 @@ Set to `true` properties `index_include_frozen` in the <<sql-rest>> or `index.in
 dedicated keyword::
 Explicitly perform the inclusion through the dedicated `FROZEN` keyword in the `FROM` clause or `INCLUDE FROZEN` in the `SHOW` commands:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showTablesIncludeFrozen]
 ----
 
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[fromTableIncludeFrozen]
 ----
diff --git a/docs/reference/sql/language/syntax/commands/describe-table.asciidoc b/docs/reference/sql/language/syntax/commands/describe-table.asciidoc
index da02f1fa23817..9aad578da479e 100644
--- a/docs/reference/sql/language/syntax/commands/describe-table.asciidoc
+++ b/docs/reference/sql/language/syntax/commands/describe-table.asciidoc
@@ -30,7 +30,7 @@ DESC
 
 `DESC` and `DESCRIBE` are aliases to <<sql-syntax-show-columns>>.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[describeTable]
 ----
diff --git a/docs/reference/sql/language/syntax/commands/index.asciidoc b/docs/reference/sql/language/syntax/commands/index.asciidoc
index 4af8f19d7034b..78259d07914a8 100644
--- a/docs/reference/sql/language/syntax/commands/index.asciidoc
+++ b/docs/reference/sql/language/syntax/commands/index.asciidoc
@@ -1,7 +1,7 @@
 [role="xpack"]
 [testenv="basic"]
 [[sql-commands]]
-== SQL Commands
+=== SQL Commands
 
 This section contains the list of SQL commands supported by {es-sql} along with their syntax:
 
diff --git a/docs/reference/sql/language/syntax/commands/select.asciidoc b/docs/reference/sql/language/syntax/commands/select.asciidoc
index 08ebe0ae96497..5fe9769774cef 100644
--- a/docs/reference/sql/language/syntax/commands/select.asciidoc
+++ b/docs/reference/sql/language/syntax/commands/select.asciidoc
@@ -36,7 +36,7 @@ The general execution of `SELECT` is as follows:
 
 As with a table, every output column of a `SELECT` has a name which can be either specified per column through the `AS` keyword :
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[selectColumnAlias]
 ----
@@ -46,14 +46,14 @@ which is why it is recommended to specify it.
 
 assigned by {es-sql} if no name is given:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[selectInline]
 ----
 
 or if it's a simple column reference, use its name as the column name:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[selectColumn]
 ----
@@ -63,7 +63,7 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[selectColumn]
 
 To select all the columns in the source, one can use `*`:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[wildcardWithOrder]
 ----
@@ -89,14 +89,14 @@ Represents the name (optionally qualified) of an existing table, either a concre
 
 If the table name contains special SQL characters (such as `.`,`-`,`*`,etc...) use double quotes to escape them:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[fromTableQuoted]
 ----
 
 The name can be a <<multi-index, pattern>> pointing to multiple indices (likely requiring quoting as mentioned above) with the restriction that *all* resolved concrete tables have **exact mapping**.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[fromTablePatternQuoted]
 ----
@@ -104,7 +104,7 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[fromTablePatternQuoted]
 `alias`::
 A substitute name for the `FROM` item containing the alias. An alias is used for brevity or to eliminate ambiguity. When an alias is provided, it completely hides the actual name of the table and must be used in its place.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[fromTableAlias]
 ----
@@ -125,7 +125,7 @@ where:
 
 Represents an expression that evaluates to a `boolean`. Only the rows that match the condition (to `true`) are returned.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[basicWhere]
 ----
@@ -148,34 +148,34 @@ Represents an expression on which rows are being grouped _on_. It can be a colum
 
 A common, group by column name:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[groupByColumn]
 ----
 
 Grouping by output ordinal:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[groupByOrdinal]
 ----
 
 Grouping by alias:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[groupByAlias]
 ----
 
 And grouping by column expression (typically used along-side an alias):
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[groupByExpression]
 ----
 
 Or a mixture of the above:
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[groupByMulti]
 ----
@@ -185,27 +185,27 @@ When a `GROUP BY` clause is used in a `SELECT`, _all_ output expressions must be
 
 To wit:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[groupByAndAgg]
 ----
 
 Expressions over aggregates used in output:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[groupByAndAggExpression]
 ----
 
 Multiple aggregates used:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[groupByAndMultipleAggs]
 ----
 
 [TIP]
-If custom bucketing is required, it can be achieved with the use of `<<sql-functions-conditional-case, CASE>>`,
+If custom bucketing is required, it can be achieved with the use of <<sql-functions-conditional-case, `CASE`>>,
 as shown <<sql-functions-conditional-case-groupby-custom-buckets, here>>.
 
 [[sql-syntax-group-by-implicit]]
@@ -216,14 +216,14 @@ As such, the query emits only a single row (as there is only a single group).
 
 A common example is counting the number of records:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[groupByImplicitCount]
 ----
 
 Of course, multiple aggregations can be applied:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[groupByImplicitMultipleAggs]
 ----
@@ -249,14 +249,14 @@ Both `WHERE` and `HAVING` are used for filtering however there are several signi
 . `WHERE` works on individual *rows*, `HAVING` works on the *groups* created by ``GROUP BY``
 . `WHERE` is evaluated *before* grouping, `HAVING` is evaluated *after* grouping
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[groupByHaving]
 ----
 
 Further more, one can use multiple aggregate expressions inside `HAVING` even ones that are not used in the output (`SELECT`):
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[groupByHavingMultiple]
 ----
@@ -269,14 +269,14 @@ As such, the query emits only a single row (as there is only a single group) and
 
 In this example, `HAVING` matches:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[groupByHavingImplicitMatch]
 ----
 
 //However `HAVING` can also not match, in which case an empty result is returned:
 //
-//["source","sql",subs="attributes,callouts,macros"]
+//[source, sql]
 //----
 //include-tagged::{sql-specs}/docs/docs.csv-spec[groupByHavingImplicitNoMatch]
 //----
@@ -304,7 +304,7 @@ IMPORTANT: When used along-side, `GROUP BY` expression can point _only_ to the c
 
 For example, the following query sorts by an arbitrary input field (`page_count`):
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[orderByBasic]
 ----
@@ -318,26 +318,26 @@ NOTE: With `GROUP BY`, make sure the ordering targets the resulting group - appl
 
 For example, to order groups simply indicate the grouping key:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[orderByGroup]
 ----
 
 Multiple keys can be specified of course:
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[groupByMulti]
 ----
 
 Further more, it is possible to order groups based on aggregations of their values:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[orderByAgg]
 ----
 
-IMPORTANT: Ordering by aggregation is possible for up to 512 entries for memory consumption reasons.
-In cases where the results pass this threshold, use <<`LIMIT`, sql-syntax-limit>> to reduce the number
+IMPORTANT: Ordering by aggregation is possible for up to *10000* entries for memory consumption reasons.
+In cases where the results pass this threshold, use <<sql-syntax-limit,`LIMIT`>> to reduce the number
 of results.
 
 [[sql-syntax-order-by-score]]
@@ -352,7 +352,7 @@ combined using the same rules as {es}'s
 
 To sort based on the `score`, use the special function `SCORE()`:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[orderByScore]
 ----
@@ -360,7 +360,7 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[orderByScore]
 Note that you can return `SCORE()` by using a full-text search predicate in the `WHERE` clause.
 This is possible even if `SCORE()` is not used for sorting:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[orderByScoreWithMatch]
 ----
@@ -387,7 +387,7 @@ ALL:: indicates there is no limit and thus all results are being returned.
 
 To return 
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[limitBasic]
 ----
diff --git a/docs/reference/sql/language/syntax/commands/show-columns.asciidoc b/docs/reference/sql/language/syntax/commands/show-columns.asciidoc
index b21c02358e526..9cb90af6b656f 100644
--- a/docs/reference/sql/language/syntax/commands/show-columns.asciidoc
+++ b/docs/reference/sql/language/syntax/commands/show-columns.asciidoc
@@ -21,7 +21,7 @@ patterns.
 
 List the columns in table and their data type (and other attributes).
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showColumns]
 ----
diff --git a/docs/reference/sql/language/syntax/commands/show-functions.asciidoc b/docs/reference/sql/language/syntax/commands/show-functions.asciidoc
index 47c000e81d9fd..8689788867c20 100644
--- a/docs/reference/sql/language/syntax/commands/show-functions.asciidoc
+++ b/docs/reference/sql/language/syntax/commands/show-functions.asciidoc
@@ -15,7 +15,7 @@ SHOW FUNCTIONS [LIKE pattern?]? <1>
 
 List all the SQL functions and their type. The `LIKE` clause can be used to restrict the list of names to the given pattern.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showFunctions]
 ----
@@ -23,25 +23,25 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[showFunctions]
 The list of functions returned can be customized based on the pattern.
 
 It can be an exact match:
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showFunctionsLikeExact]
 ----
 
 A wildcard for exactly one character:
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showFunctionsLikeChar]
 ----
 
 A wildcard matching zero or more characters:
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showFunctionsLikeWildcard]
 ----
 
 Or of course, a variation of the above:
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showFunctionsWithPattern]
 ----
diff --git a/docs/reference/sql/language/syntax/commands/show-tables.asciidoc b/docs/reference/sql/language/syntax/commands/show-tables.asciidoc
index 554819e24b178..d5a40337713d4 100644
--- a/docs/reference/sql/language/syntax/commands/show-tables.asciidoc
+++ b/docs/reference/sql/language/syntax/commands/show-tables.asciidoc
@@ -24,7 +24,7 @@ patterns.
 
 List the tables available to the current user and their type.
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showTables]
 ----
@@ -32,7 +32,7 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[showTables]
 Match multiple indices by using {es} <<multi-index,multi index syntax>>
 notation:
 
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showTablesEsMultiIndex]
 ----
@@ -40,26 +40,26 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[showTablesEsMultiIndex]
 One can also use the `LIKE` clause to restrict the list of names to the given pattern.
 
 The pattern can be an exact match:
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showTablesLikeExact]
 ----
 
 Multiple chars:
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showTablesLikeWildcard]
 ----
 
 A single char:
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showTablesLikeOneChar]
 ----
 
 
 Or a mixture of single and multiple chars:
-["source","sql",subs="attributes,callouts,macros"]
+[source, sql]
 ----
 include-tagged::{sql-specs}/docs/docs.csv-spec[showTablesLikeMixed]
 ----
diff --git a/docs/reference/sql/language/syntax/lexic/index.asciidoc b/docs/reference/sql/language/syntax/lexic/index.asciidoc
index 014193a0469dd..a668ee724e56f 100644
--- a/docs/reference/sql/language/syntax/lexic/index.asciidoc
+++ b/docs/reference/sql/language/syntax/lexic/index.asciidoc
@@ -1,7 +1,7 @@
 [role="xpack"]
 [testenv="basic"]
 [[sql-lexical-structure]]
-== Lexical Structure
+=== Lexical Structure
 
 This section covers the major lexical structure of SQL, which for the most part, is going to resemble that of ANSI SQL itself hence why low-levels details are not discussed in depth.
 
@@ -10,8 +10,7 @@ This section covers the major lexical structure of SQL, which for the most part,
 A token can be a __key word__, an _identifier_ (_quoted_ or _unquoted_), a _literal_ (or constant) or a special character symbol (typically a delimiter). Tokens are typically separated by whitespace (be it space, tab) though in some cases, where there is no ambiguity (typically due to a character symbol) this is not needed - however for readability purposes this should be avoided.
 
 [[sql-syntax-keywords]]
-[float]
-=== Key Words
+==== Key Words
 
 Take the following example:
 
@@ -35,8 +34,7 @@ Identifiers however are not - as {es} is case sensitive, {es-sql} uses the recei
 To help differentiate between the two, through-out the documentation the SQL key words are upper-cased a convention we find increases readability and thus recommend to others.
 
 [[sql-syntax-identifiers]]
-[float]
-=== Identifiers
+==== Identifiers
 
 Identifiers can be of two types: __quoted__ and __unquoted__:
 
@@ -59,14 +57,13 @@ The first identifier from needs to quoted as otherwise it clashes with the `FROM
 Hence why in general, *especially* when dealing with user input it is *highly* recommended to use quotes for identifiers. It adds minimal increase to your queries and in return offers clarity and disambiguation.
 
 [[sql-syntax-literals]]
-[float]
-=== Literals (Constants)
+==== Literals (Constants)
 
 {es-sql} supports two kind of __implicitly-typed__ literals: strings and numbers.
 
 [[sql-syntax-string-literals]]
 [float]
-==== String Literals
+===== String Literals
 
 A string literal is an arbitrary number of characters bounded by single quotes `'`: `'Giant Robot'`. 
 To include a single quote in the string, escape it using another single quote: `'Captain EO''s Voyage'`. 
@@ -75,7 +72,7 @@ NOTE: An escaped single quote is *not* a double quote (`"`), but a single quote
 
 [sql-syntax-numeric-literals]
 [float]
-==== Numeric Literals
+===== Numeric Literals
 
 Numeric literals are accepted both in decimal and scientific notation with exponent marker (`e` or `E`), starting either with a digit or decimal point `.`:
 
@@ -92,7 +89,7 @@ Numeric literals that contain a decimal point are always interpreted as being of
 
 [[sql-syntax-generic-literals]]
 [float]
-==== Generic Literals
+===== Generic Literals
 
 When dealing with arbitrary type literal, one creates the object by casting, typically, the string representation to the desired type. This can be achieved through the dedicated <<sql-operators-cast, cast operator>> and <<sql-functions-type-conversion, functions>>:
 
@@ -106,8 +103,7 @@ CONVERT('10.0.0.1', IP)                     -- cast '10.0.0.1' to an IP
 Do note that {es-sql} provides functions that out of the box return popular literals (like `E()`) or provide dedicated parsing for certain strings.
 
 [[sql-syntax-single-vs-double-quotes]]
-[float]
-=== Single vs Double Quotes
+==== Single vs Double Quotes
 
 It is worth pointing out that in SQL, single quotes `'` and double quotes `"` have different meaning and *cannot* be used interchangeably.
 Single quotes are used to declare a <<sql-syntax-string-literals, string literal>> while double quotes for <<sql-syntax-identifiers, identifiers>>.
@@ -126,8 +122,7 @@ SELECT "first_name" <1>
 <2> Single quotes `'` used for a string literal
 
 [[sql-syntax-special-chars]]
-[float]
-=== Special characters
+==== Special characters
 
 A few characters that are not alphanumeric have a dedicated meaning different from that of an operator. For completeness these are specified below:
 
@@ -146,8 +141,7 @@ s|Description
 |===
 
 [[sql-syntax-operators]]
-[float]
-=== Operators
+==== Operators
 
 Most operators in {es-sql} have the same precedence and are left-associative. As this is done at parsing time, parenthesis need to be used to enforce a different precedence.
 
@@ -205,8 +199,7 @@ s|Description
 
 
 [[sql-syntax-comments]]
-[float]
-=== Comments
+==== Comments
 
 {es-sql} allows comments which are sequence of characters ignored by the parsers.
 
diff --git a/docs/reference/sql/limitations.asciidoc b/docs/reference/sql/limitations.asciidoc
index c5b334480c993..2d46d016c6b89 100644
--- a/docs/reference/sql/limitations.asciidoc
+++ b/docs/reference/sql/limitations.asciidoc
@@ -87,8 +87,8 @@ It is recommended to use `LIMIT` for queries that use sorting by aggregation, es
 SELECT * FROM test GROUP BY age ORDER BY COUNT(*) LIMIT 100;
 --------------------------------------------------
 
-It is possible to run the same queries without a `LIMIT` however in that case if the maximum size (*512*) is passed, an exception will be
-returned as {es-sql} is unable to track (and sort) all the results returned.
+It is possible to run the same queries without a `LIMIT` however in that case if the maximum size (*10000*) is passed,
+an exception will be returned as {es-sql} is unable to track (and sort) all the results returned.
 
 [float]
 === Using aggregation functions on top of scalar functions
diff --git a/docs/reference/sql/security.asciidoc b/docs/reference/sql/security.asciidoc
index ad946c33e2d98..cbf41b46997a4 100644
--- a/docs/reference/sql/security.asciidoc
+++ b/docs/reference/sql/security.asciidoc
@@ -33,7 +33,7 @@ the API require `cluster:monitor/main`.
 The following example configures a role that can run SQL in JDBC querying the `test` and `bort`
 indices:
 
-["source","yaml",subs="attributes,callouts,macros"]
+[source, yaml]
 --------------------------------------------------
 include-tagged::{sql-tests}/security/roles.yml[cli_drivers]
 --------------------------------------------------
diff --git a/docs/reference/upgrade.asciidoc b/docs/reference/upgrade.asciidoc
index e5e447aff75f6..3bd35197dcd5f 100644
--- a/docs/reference/upgrade.asciidoc
+++ b/docs/reference/upgrade.asciidoc
@@ -7,8 +7,8 @@
 process so upgrading does not interrupt service. Rolling upgrades are supported:
 
 * Between minor versions
-* From 5.6 to 6.7
-* From 6.7 to {version}
+* From 5.6 to 6.8
+* From 6.8 to {version}
 
 {es} can read indices created in the previous major version. If you
 have indices created in 5.x or before, you must reindex or delete them
diff --git a/docs/reference/upgrade/cluster_restart.asciidoc b/docs/reference/upgrade/cluster_restart.asciidoc
index 1865f005dccc8..52b14d642b620 100644
--- a/docs/reference/upgrade/cluster_restart.asciidoc
+++ b/docs/reference/upgrade/cluster_restart.asciidoc
@@ -5,7 +5,7 @@ To upgrade directly to {es} {version} from versions 6.0-6.6, you must shut down
 all nodes in the cluster, upgrade each node to {version}, and restart the cluster.
 
 NOTE: If you are running a version prior to 6.0,
-https://www.elastic.co/guide/en/elastic-stack/6.7/upgrading-elastic-stack.html[upgrade to 6.7]
+{stack-ref-68}/upgrading-elastic-stack.html[upgrade to 6.8]
 and reindex your old indices or bring up a new {version} cluster and
 <<reindex-upgrade-remote, reindex from remote>>.
 
diff --git a/docs/reference/upgrade/reindex_upgrade.asciidoc b/docs/reference/upgrade/reindex_upgrade.asciidoc
index 7fc6c320263c3..faa8fbc1639b4 100644
--- a/docs/reference/upgrade/reindex_upgrade.asciidoc
+++ b/docs/reference/upgrade/reindex_upgrade.asciidoc
@@ -36,7 +36,7 @@ been deleted.
 [[reindex-upgrade-inplace]]
 === Reindex in place
 
-You can use the Upgrade Assistant in {kib} 6.7 to automatically reindex 5.x
+You can use the Upgrade Assistant in {kib} 6.8 to automatically reindex 5.x
 indices you need to carry forward to {version}.
 
 To manually reindex your old indices in place:
@@ -103,7 +103,7 @@ endif::include-xpack[]
 
 You can use <<reindex-from-remote,reindex from remote>> to migrate indices from
 your old cluster to a new {version} cluster. This enables you move to {version}
-from a pre-6.7 cluster without interrupting service.
+from a pre-6.8 cluster without interrupting service.
 
 [WARNING]
 =============================================
@@ -196,4 +196,4 @@ monitor progress of the reindex job with the <<tasks,task API>>:
   `30s` and `1`).
 
 .. Once reindexing is complete and the status of the new index is `green`,
-  you can delete the old index.
\ No newline at end of file
+  you can delete the old index.
diff --git a/docs/reference/upgrade/rolling_upgrade.asciidoc b/docs/reference/upgrade/rolling_upgrade.asciidoc
index 2cf1061e67ba7..7ad4a0bb904c1 100644
--- a/docs/reference/upgrade/rolling_upgrade.asciidoc
+++ b/docs/reference/upgrade/rolling_upgrade.asciidoc
@@ -10,8 +10,8 @@ running the older version.
 Rolling upgrades are supported:
 
 * Between minor versions
-* {stack-ref-67}/upgrading-elastic-stack.html[From 5.6 to 6.7]
-* {stack-ref-70}/upgrading-elastic-stack.html[From 6.7 to 7.0]
+* {stack-ref-68}/upgrading-elastic-stack.html[From 5.6 to 6.8]
+* {stack-ref-70}/upgrading-elastic-stack.html[From 6.8 to 7.0]
 * From {prev-major-version} to {version}
 
 Upgrading directly to {version} from 6.6 or earlier requires a
diff --git a/gradle.properties b/gradle.properties
index 491770edd7c52..63b1dc3cd7288 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,3 +1,3 @@
 org.gradle.daemon=true
-org.gradle.jvmargs=-Xmx2g -XX:+HeapDumpOnOutOfMemoryError -Xss2m
+org.gradle.jvmargs=-Xmx3g -XX:+HeapDumpOnOutOfMemoryError -Xss2m
 options.forkOptions.memoryMaximumSize=2g
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index 47216b872e431..9fabbb9bd78d9 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.5-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=14cd15fc8cc8705bd69dcfa3c8fefb27eb7027f5de4b47a8b279218f76895a91
+distributionSha256Sum=302b7df46730ce75c582542c056c9bf5cac2b94fbf2cc656d0e37e41e8a5d371
diff --git a/gradlew b/gradlew
index b0d6d0ab5deb5..8e25e6c19d574 100755
--- a/gradlew
+++ b/gradlew
@@ -7,7 +7,7 @@
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/gradlew.bat b/gradlew.bat
index 15e1ee37a70d7..24467a141f791 100644
--- a/gradlew.bat
+++ b/gradlew.bat
@@ -5,7 +5,7 @@
 @rem you may not use this file except in compliance with the License.
 @rem You may obtain a copy of the License at
 @rem
-@rem      http://www.apache.org/licenses/LICENSE-2.0
+@rem      https://www.apache.org/licenses/LICENSE-2.0
 @rem
 @rem Unless required by applicable law or agreed to in writing, software
 @rem distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/libs/build.gradle b/libs/build.gradle
index b0924aa1f54f1..03b5d2c611e71 100644
--- a/libs/build.gradle
+++ b/libs/build.gradle
@@ -26,19 +26,19 @@ subprojects {
 
     /*
      * Subprojects may depend on the "core" lib but may not depend on any
-     * other libs. This keeps are dependencies simpler.
+     * other libs. This keeps our dependencies simpler.
      */
     project.afterEvaluate {
         configurations.all { Configuration conf ->
-            dependencies.all { Dependency dep ->
-                Project depProject = dependencyToProject(dep)
+            dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
+                Project depProject = dep.dependencyProject
                 if (depProject != null
-                        && false == depProject.path.equals(':libs:core')
+                        && false == depProject.path.equals(':libs:elasticsearch-core')
                         && false == isEclipse 
                         && depProject.path.startsWith(':libs')) {
                     throw new InvalidUserDataException("projects in :libs "
                             + "may not depend on other projects libs except "
-                            + ":libs:core but "
+                            + ":libs:elasticsearch-core but "
                             + "${project.path} depends on ${depProject.path}")
                 }
             }
diff --git a/libs/cli/build.gradle b/libs/cli/build.gradle
index b1f3b338255c4..a97c62096a512 100644
--- a/libs/cli/build.gradle
+++ b/libs/cli/build.gradle
@@ -23,7 +23,7 @@ apply plugin: 'nebula.maven-scm'
 
 dependencies {
     compile 'net.sf.jopt-simple:jopt-simple:5.0.2'
-    compile "org.elasticsearch:elasticsearch-core:${version}"
+    compile project(':libs:elasticsearch-core')
 }
 
 test.enabled = false
diff --git a/libs/core/build.gradle b/libs/core/build.gradle
index 86778c3d23744..b67b213d7b41c 100644
--- a/libs/core/build.gradle
+++ b/libs/core/build.gradle
@@ -21,30 +21,20 @@ apply plugin: 'nebula.optional-base'
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
 
-archivesBaseName = 'elasticsearch-core'
-
-publishing {
-    publications {
-        nebula {
-            artifactId = archivesBaseName
-        }
-    }
-}
-
 dependencies {
     testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
     testCompile "junit:junit:${versions.junit}"
     testCompile "org.hamcrest:hamcrest:${versions.hamcrest}"
 
-    if (isEclipse == false || project.path == ":libs:core-tests") {
-        testCompile("org.elasticsearch.test:framework:${version}") {
+    if (isEclipse == false || project.path == ":libs:elasticsearch-core-tests") {
+        testCompile(project(":test:framework")) {
             exclude group: 'org.elasticsearch', module: 'elasticsearch-core'
         }
     }
 }
 
 forbiddenApisMain {
-    // :libs:core does not depend on server
+    // :libs:elasticsearch-core does not depend on server
     // TODO: Need to decide how we want to handle for forbidden signatures with the changes to server
     replaceSignatureFiles 'jdk-signatures'
 }
@@ -52,7 +42,7 @@ forbiddenApisMain {
 if (isEclipse) {
     // in eclipse the project is under a fake root, we need to change around the source sets
     sourceSets {
-        if (project.path == ":libs:core") {
+        if (project.path == ":libs:elasticsearch-core") {
             main.java.srcDirs = ['java']
             main.resources.srcDirs = ['resources']
         } else {
diff --git a/libs/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java b/libs/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java
index f40fbbe73a9dc..c208e7d795391 100644
--- a/libs/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java
+++ b/libs/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java
@@ -31,6 +31,26 @@ public class TimeValue implements Comparable<TimeValue> {
     public static final TimeValue MINUS_ONE = timeValueMillis(-1);
     public static final TimeValue ZERO = timeValueMillis(0);
 
+    private static final long C0 = 1L;
+    private static final long C1 = C0 * 1000L;
+    private static final long C2 = C1 * 1000L;
+    private static final long C3 = C2 * 1000L;
+    private static final long C4 = C3 * 60L;
+    private static final long C5 = C4 * 60L;
+    private static final long C6 = C5 * 24L;
+
+    private final long duration;
+    private final TimeUnit timeUnit;
+
+    public TimeValue(long millis) {
+        this(millis, TimeUnit.MILLISECONDS);
+    }
+
+    public TimeValue(long duration, TimeUnit timeUnit) {
+        this.duration = duration;
+        this.timeUnit = timeUnit;
+    }
+
     public static TimeValue timeValueNanos(long nanos) {
         return new TimeValue(nanos, TimeUnit.NANOSECONDS);
     }
@@ -51,17 +71,6 @@ public static TimeValue timeValueHours(long hours) {
         return new TimeValue(hours, TimeUnit.HOURS);
     }
 
-    private final long duration;
-
-    /**
-     * @return the number of {@link #timeUnit()} units this value contains
-     */
-    public long duration() {
-        return duration;
-    }
-
-    private final TimeUnit timeUnit;
-
     /**
      * @return the unit used for the this time value, see {@link #duration()}
      */
@@ -69,13 +78,11 @@ public TimeUnit timeUnit() {
         return timeUnit;
     }
 
-    public TimeValue(long millis) {
-        this(millis, TimeUnit.MILLISECONDS);
-    }
-
-    public TimeValue(long duration, TimeUnit timeUnit) {
-        this.duration = duration;
-        this.timeUnit = timeUnit;
+    /**
+     * @return the number of {@link #timeUnit()} units this value contains
+     */
+    public long duration() {
+        return duration;
     }
 
     public long nanos() {
@@ -187,9 +194,31 @@ public double getDaysFrac() {
      *
      * Note that this method might produce fractional time values (ex 1.6m) which cannot be
      * parsed by method like {@link TimeValue#parse(String, String, String)}.
+     *
+     * Also note that the maximum string value that will be generated is
+     * {@code 106751.9d} due to the way that values are internally converted
+     * to nanoseconds (106751.9 days is Long.MAX_VALUE nanoseconds)
      */
     @Override
     public String toString() {
+        return this.toHumanReadableString(1);
+    }
+
+    /**
+     * Returns a {@link String} representation of the current {@link TimeValue}.
+     *
+     * Note that this method might produce fractional time values (ex 1.6m) which cannot be
+     * parsed by method like {@link TimeValue#parse(String, String, String)}. The number of
+     * fractional decimals (up to 10 maximum) are truncated to the number of fraction pieces
+     * specified.
+     *
+     * Also note that the maximum string value that will be generated is
+     * {@code 106751.9d} due to the way that values are internally converted
+     * to nanoseconds (106751.9 days is Long.MAX_VALUE nanoseconds)
+     *
+     * @param fractionPieces the number of decimal places to include
+     */
+    public String toHumanReadableString(int fractionPieces) {
         if (duration < 0) {
             return Long.toString(duration);
         }
@@ -218,25 +247,68 @@ public String toString() {
             value = microsFrac();
             suffix = "micros";
         }
-        return formatDecimal(value) + suffix;
+        // Limit fraction pieces to a min of 0 and maximum of 10
+        return formatDecimal(value, Math.min(10, Math.max(0, fractionPieces))) + suffix;
     }
 
-    private static String formatDecimal(double value) {
+    private static String formatDecimal(double value, int fractionPieces) {
         String p = String.valueOf(value);
+        int totalLength = p.length();
         int ix = p.indexOf('.') + 1;
         int ex = p.indexOf('E');
-        char fraction = p.charAt(ix);
-        if (fraction == '0') {
+        // Location where the fractional values end
+        int fractionEnd = ex == -1 ? Math.min(ix + fractionPieces, totalLength) : ex;
+
+        // Determine the value of the fraction, so if it were .000 the
+        // actual long value is 0, in which case, it can be elided.
+        long fractionValue;
+        try {
+            fractionValue = Long.parseLong(p.substring(ix, fractionEnd));
+        } catch (NumberFormatException e) {
+            fractionValue = 0;
+        }
+
+        if (fractionValue == 0 || fractionPieces <= 0) {
+            // Either the part of the fraction we were asked to report is
+            // zero, or the user requested 0 fraction pieces, so return
+            // only the integral value
             if (ex != -1) {
                 return p.substring(0, ix - 1) + p.substring(ex);
             } else {
                 return p.substring(0, ix - 1);
             }
         } else {
+            // Build up an array of fraction characters, without going past
+            // the end of the string. This keeps track of trailing '0' chars
+            // that should be truncated from the end to avoid getting a
+            // string like "1.3000d" (returning "1.3d" instead) when the
+            // value is 1.30000009
+            char[] fractions = new char[fractionPieces];
+            int fracCount = 0;
+            int truncateCount = 0;
+            for (int i = 0; i < fractionPieces; i++) {
+                int position = ix + i;
+                if (position >= fractionEnd) {
+                    // No more pieces, the fraction has ended
+                    break;
+                }
+                char fraction = p.charAt(position);
+                if (fraction == '0') {
+                    truncateCount++;
+                } else {
+                    truncateCount = 0;
+                }
+                fractions[i] = fraction;
+                fracCount++;
+            }
+
+            // Generate the fraction string from the char array, truncating any trailing zeros
+            String fractionStr = new String(fractions, 0, fracCount - truncateCount);
+
             if (ex != -1) {
-                return p.substring(0, ix) + fraction + p.substring(ex);
+                return p.substring(0, ix) + fractionStr + p.substring(ex);
             } else {
-                return p.substring(0, ix) + fraction;
+                return p.substring(0, ix) + fractionStr;
             }
         }
     }
@@ -317,14 +389,6 @@ private static long parse(final String initialInput, final String normalized, fi
         }
     }
 
-    private static final long C0 = 1L;
-    private static final long C1 = C0 * 1000L;
-    private static final long C2 = C1 * 1000L;
-    private static final long C3 = C2 * 1000L;
-    private static final long C4 = C3 * 60L;
-    private static final long C5 = C4 * 60L;
-    private static final long C6 = C5 * 24L;
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/libs/core/src/main/java/org/elasticsearch/core/internal/io/IOUtils.java b/libs/core/src/main/java/org/elasticsearch/core/internal/io/IOUtils.java
index 46d19d2a814fe..d3e9afd4970df 100644
--- a/libs/core/src/main/java/org/elasticsearch/core/internal/io/IOUtils.java
+++ b/libs/core/src/main/java/org/elasticsearch/core/internal/io/IOUtils.java
@@ -24,6 +24,7 @@
 import java.nio.file.FileVisitResult;
 import java.nio.file.FileVisitor;
 import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.nio.file.attribute.BasicFileAttributes;
@@ -249,6 +250,7 @@ public FileVisitResult visitFileFailed(final Path file, final IOException exc) t
     }
 
     // TODO: replace with constants class if needed (cf. org.apache.lucene.util.Constants)
+    private static final boolean WINDOWS = System.getProperty("os.name").startsWith("Windows");
     private static final boolean LINUX = System.getProperty("os.name").startsWith("Linux");
     private static final boolean MAC_OS_X = System.getProperty("os.name").startsWith("Mac OS X");
 
@@ -263,18 +265,28 @@ public FileVisitResult visitFileFailed(final Path file, final IOException exc) t
      *                   systems and operating systems allow to fsync on a directory)
      */
     public static void fsync(final Path fileToSync, final boolean isDir) throws IOException {
+        if (isDir && WINDOWS) {
+            // opening a directory on Windows fails, directories can not be fsynced there
+            if (Files.exists(fileToSync) == false) {
+                // yet do not suppress trying to fsync directories that do not exist
+                throw new NoSuchFileException(fileToSync.toString());
+            }
+            return;
+        }
         try (FileChannel file = FileChannel.open(fileToSync, isDir ? StandardOpenOption.READ : StandardOpenOption.WRITE)) {
-            file.force(true);
-        } catch (final IOException ioe) {
-            if (isDir) {
-                assert (LINUX || MAC_OS_X) == false :
-                        "on Linux and MacOSX fsyncing a directory should not throw IOException, "+
-                                "we just don't want to rely on that in production (undocumented); got: " + ioe;
-                // ignore exception if it is a directory
-                return;
+            try {
+                file.force(true);
+            } catch (final IOException e) {
+                if (isDir) {
+                    assert (LINUX || MAC_OS_X) == false :
+                            "on Linux and MacOSX fsyncing a directory should not throw IOException, "+
+                                    "we just don't want to rely on that in production (undocumented); got: " + e;
+                    // ignore exception if it is a directory
+                    return;
+                }
+                // throw original exception
+                throw e;
             }
-            // throw original exception
-            throw ioe;
         }
     }
 }
diff --git a/libs/core/src/test/eclipse-build.gradle b/libs/core/src/test/eclipse-build.gradle
index b5fe04174287f..9fb1d2cac390f 100644
--- a/libs/core/src/test/eclipse-build.gradle
+++ b/libs/core/src/test/eclipse-build.gradle
@@ -2,5 +2,5 @@
 apply from: '../../build.gradle'
 
 dependencies {
-  testCompile project(':libs:core')
+  testCompile project(':libs:elasticsearch-core')
 }
diff --git a/libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java b/libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
index af6b89be5fffe..3dec990b7b251 100644
--- a/libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
+++ b/libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
@@ -47,6 +47,26 @@ public void testToString() {
         assertThat("1.5m", equalTo(new TimeValue(90, TimeUnit.SECONDS).toString()));
         assertThat("1.5h", equalTo(new TimeValue(90, TimeUnit.MINUTES).toString()));
         assertThat("1.5d", equalTo(new TimeValue(36, TimeUnit.HOURS).toString()));
+        assertThat("1d", equalTo(new TimeValue(36, TimeUnit.HOURS).toHumanReadableString(0)));
+        assertThat("1d", equalTo(new TimeValue(36, TimeUnit.HOURS).toHumanReadableString(-4)));
+        assertThat("1.5d", equalTo(new TimeValue(36, TimeUnit.HOURS).toHumanReadableString(2)));
+        assertThat("1.45d", equalTo(new TimeValue(35, TimeUnit.HOURS).toHumanReadableString(2)));
+        assertThat("1.4583333333d", equalTo(new TimeValue(35, TimeUnit.HOURS).toHumanReadableString(10)));
+        assertThat("1d", equalTo(new TimeValue(103723200, TimeUnit.MILLISECONDS).toHumanReadableString(0)));
+        assertThat("1.2d", equalTo(new TimeValue(103723200, TimeUnit.MILLISECONDS).toHumanReadableString(1)));
+        assertThat("1.2d", equalTo(new TimeValue(103723200, TimeUnit.MILLISECONDS).toHumanReadableString(2)));
+        assertThat("1.2d", equalTo(new TimeValue(103723200, TimeUnit.MILLISECONDS).toHumanReadableString(3)));
+        assertThat("1.2005d", equalTo(new TimeValue(103723200, TimeUnit.MILLISECONDS).toHumanReadableString(4)));
+        assertThat("1d", equalTo(new TimeValue(86400077, TimeUnit.MILLISECONDS).toHumanReadableString(0)));
+        assertThat("1d", equalTo(new TimeValue(86400077, TimeUnit.MILLISECONDS).toHumanReadableString(1)));
+        assertThat("1d", equalTo(new TimeValue(86400077, TimeUnit.MILLISECONDS).toHumanReadableString(2)));
+        assertThat("1d", equalTo(new TimeValue(86400077, TimeUnit.MILLISECONDS).toHumanReadableString(3)));
+        assertThat("1d", equalTo(new TimeValue(86400077, TimeUnit.MILLISECONDS).toHumanReadableString(4)));
+        assertThat("1d", equalTo(new TimeValue(86400077, TimeUnit.MILLISECONDS).toHumanReadableString(5)));
+        assertThat("1d", equalTo(new TimeValue(86400077, TimeUnit.MILLISECONDS).toHumanReadableString(6)));
+        assertThat("1.0000008d", equalTo(new TimeValue(86400077, TimeUnit.MILLISECONDS).toHumanReadableString(7)));
+        assertThat("1.00000089d", equalTo(new TimeValue(86400077, TimeUnit.MILLISECONDS).toHumanReadableString(8)));
+        assertThat("1.4583333333d", equalTo(new TimeValue(35, TimeUnit.HOURS).toHumanReadableString(Integer.MAX_VALUE)));
         assertThat("1000d", equalTo(new TimeValue(1000, TimeUnit.DAYS).toString()));
     }
 
diff --git a/libs/core/src/test/java/org/elasticsearch/core/internal/io/IOUtilsTests.java b/libs/core/src/test/java/org/elasticsearch/core/internal/io/IOUtilsTests.java
index ee5af323b5219..8af0f2a707e24 100644
--- a/libs/core/src/test/java/org/elasticsearch/core/internal/io/IOUtilsTests.java
+++ b/libs/core/src/test/java/org/elasticsearch/core/internal/io/IOUtilsTests.java
@@ -19,6 +19,7 @@
 
 import org.apache.lucene.mockfile.FilterFileSystemProvider;
 import org.apache.lucene.mockfile.FilterPath;
+import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.test.ESTestCase;
@@ -27,14 +28,20 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.URI;
+import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.AccessDeniedException;
 import java.nio.file.FileSystem;
 import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.OpenOption;
 import java.nio.file.Path;
+import java.nio.file.attribute.FileAttribute;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Objects;
+import java.util.Set;
 import java.util.function.Function;
 
 import static org.hamcrest.Matchers.arrayWithSize;
@@ -214,6 +221,43 @@ public void testFsyncDirectory() throws Exception {
         // no exception
     }
 
+    private static final class AccessDeniedWhileOpeningDirectoryFileSystem extends FilterFileSystemProvider {
+
+        AccessDeniedWhileOpeningDirectoryFileSystem(final FileSystem delegate) {
+            super("access_denied://", Objects.requireNonNull(delegate));
+        }
+
+        @Override
+        public FileChannel newFileChannel(
+                final Path path,
+                final Set<? extends OpenOption> options,
+                final FileAttribute<?>... attrs) throws IOException {
+            if (Files.isDirectory(path)) {
+                throw new AccessDeniedException(path.toString());
+            }
+            return delegate.newFileChannel(path, options, attrs);
+        }
+
+    }
+
+    public void testFsyncAccessDeniedOpeningDirectory() throws Exception {
+        final Path path = createTempDir().toRealPath();
+        final FileSystem fs = new AccessDeniedWhileOpeningDirectoryFileSystem(path.getFileSystem()).getFileSystem(URI.create("file:///"));
+        final Path wrapped = new FilterPath(path, fs);
+        if (Constants.WINDOWS) {
+            // no exception, we early return and do not even try to open the directory
+            IOUtils.fsync(wrapped, true);
+        } else {
+            expectThrows(AccessDeniedException.class, () -> IOUtils.fsync(wrapped, true));
+        }
+    }
+
+    public void testFsyncNonExistentDirectory() throws Exception {
+        final Path dir = FilterPath.unwrap(createTempDir()).toRealPath();
+        final Path nonExistentDir = dir.resolve("non-existent");
+        expectThrows(NoSuchFileException.class, () -> IOUtils.fsync(nonExistentDir, true));
+    }
+
     public void testFsyncFile() throws IOException {
         final Path path = createTempDir().toRealPath();
         final Path subPath = path.resolve(randomAlphaOfLength(8));
diff --git a/libs/dissect/build.gradle b/libs/dissect/build.gradle
index 853c78646c25b..7e71f86f64f44 100644
--- a/libs/dissect/build.gradle
+++ b/libs/dissect/build.gradle
@@ -17,17 +17,15 @@
  * under the License.
  */
 
-archivesBaseName = 'elasticsearch-dissect'
-
 dependencies {
-     if (isEclipse == false || project.path == ":libs:dissect-tests") {
-        testCompile("org.elasticsearch.test:framework:${version}") {
-            exclude group: 'org.elasticsearch', module: 'dissect'
+     if (isEclipse == false || project.path == ":libs:elasticsearch-dissect-tests") {
+        testCompile(project(":test:framework")) {
+            exclude group: 'org.elasticsearch', module: 'elasticsearch-dissect'
         }
     }
     testCompile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-    testCompile("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
-    testCompile("com.fasterxml.jackson.core:jackson-databind:${versions.jackson}")
+    testCompile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+    testCompile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
 }
 
 forbiddenApisMain {
@@ -37,7 +35,7 @@ forbiddenApisMain {
 if (isEclipse) {
     // in eclipse the project is under a fake root, we need to change around the source sets
     sourceSets {
-        if (project.path == ":libs:dissect") {
+        if (project.path == ":libs:elasticsearch-dissect") {
             main.java.srcDirs = ['java']
             main.resources.srcDirs = ['resources']
         } else {
diff --git a/libs/dissect/src/test/eclipse-build.gradle b/libs/dissect/src/test/eclipse-build.gradle
index 56d632f23b129..c10fea5e2b748 100644
--- a/libs/dissect/src/test/eclipse-build.gradle
+++ b/libs/dissect/src/test/eclipse-build.gradle
@@ -3,5 +3,5 @@
 apply from: '../../build.gradle'
 
 dependencies {
-  testCompile project(':libs:dissect')
+  testCompile project(':libs:elasticsearch-dissect')
 }
diff --git a/libs/geo/build.gradle b/libs/geo/build.gradle
index ab3419b93b9b8..e2e5a11d53586 100644
--- a/libs/geo/build.gradle
+++ b/libs/geo/build.gradle
@@ -22,8 +22,8 @@ apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
 
 dependencies {
-    if (isEclipse == false || project.path == ":libs:geo-tests") {
-        testCompile("org.elasticsearch.test:framework:${version}") {
+    if (isEclipse == false || project.path == ":libs:elasticsearch-geo-tests") {
+        testCompile(project(":test:framework")) {
             exclude group: 'org.elasticsearch', module: 'elasticsearch-geo'
         }
     }
@@ -38,7 +38,7 @@ forbiddenApisMain {
 if (isEclipse) {
     // in eclipse the project is under a fake root, we need to change around the source sets
     sourceSets {
-        if (project.path == ":libs:geo") {
+        if (project.path == ":libs:elasticsearch-geo") {
             main.java.srcDirs = ['java']
             main.resources.srcDirs = ['resources']
         } else {
diff --git a/libs/geo/src/main/java/org/elasticsearch/geo/geometry/Circle.java b/libs/geo/src/main/java/org/elasticsearch/geo/geometry/Circle.java
index cb8e2c4cb33e1..ad9881ab72fba 100644
--- a/libs/geo/src/main/java/org/elasticsearch/geo/geometry/Circle.java
+++ b/libs/geo/src/main/java/org/elasticsearch/geo/geometry/Circle.java
@@ -49,8 +49,6 @@ public Circle(final double lat, final double lon, final double alt, final double
         if (radiusMeters < 0 ) {
             throw new IllegalArgumentException("Circle radius [" + radiusMeters + "] cannot be negative");
         }
-        GeometryUtils.checkLatitude(lat);
-        GeometryUtils.checkLongitude(lon);
     }
 
     @Override
diff --git a/libs/geo/src/main/java/org/elasticsearch/geo/geometry/GeometryUtils.java b/libs/geo/src/main/java/org/elasticsearch/geo/geometry/GeometryUtils.java
deleted file mode 100644
index 9a7d4b99d3e53..0000000000000
--- a/libs/geo/src/main/java/org/elasticsearch/geo/geometry/GeometryUtils.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.geo.geometry;
-
-/**
- * Geometry-related utility methods
- */
-final class GeometryUtils {
-    /**
-     * Minimum longitude value.
-     */
-    static final double MIN_LON_INCL = -180.0D;
-
-    /**
-     * Maximum longitude value.
-     */
-    static final double MAX_LON_INCL = 180.0D;
-
-    /**
-     * Minimum latitude value.
-     */
-    static final double MIN_LAT_INCL = -90.0D;
-
-    /**
-     * Maximum latitude value.
-     */
-    static final double MAX_LAT_INCL = 90.0D;
-
-    // No instance:
-    private GeometryUtils() {
-    }
-
-    /**
-     * validates latitude value is within standard +/-90 coordinate bounds
-     */
-    static void checkLatitude(double latitude) {
-        if (Double.isNaN(latitude) || latitude < MIN_LAT_INCL || latitude > MAX_LAT_INCL) {
-            throw new IllegalArgumentException(
-                "invalid latitude " + latitude + "; must be between " + MIN_LAT_INCL + " and " + MAX_LAT_INCL);
-        }
-    }
-
-    /**
-     * validates longitude value is within standard +/-180 coordinate bounds
-     */
-    static void checkLongitude(double longitude) {
-        if (Double.isNaN(longitude) || longitude < MIN_LON_INCL || longitude > MAX_LON_INCL) {
-            throw new IllegalArgumentException(
-                "invalid longitude " + longitude + "; must be between " + MIN_LON_INCL + " and " + MAX_LON_INCL);
-        }
-    }
-
-}
diff --git a/libs/geo/src/main/java/org/elasticsearch/geo/geometry/Line.java b/libs/geo/src/main/java/org/elasticsearch/geo/geometry/Line.java
index c2c9cb4b83a18..20f4314246950 100644
--- a/libs/geo/src/main/java/org/elasticsearch/geo/geometry/Line.java
+++ b/libs/geo/src/main/java/org/elasticsearch/geo/geometry/Line.java
@@ -59,10 +59,6 @@ public Line(double[] lats, double[] lons, double[] alts) {
         if (alts != null && alts.length != lats.length) {
             throw new IllegalArgumentException("alts and lats must be equal length");
         }
-        for (int i = 0; i < lats.length; i++) {
-            GeometryUtils.checkLatitude(lats[i]);
-            GeometryUtils.checkLongitude(lons[i]);
-        }
     }
 
     public int length() {
diff --git a/libs/geo/src/main/java/org/elasticsearch/geo/geometry/Point.java b/libs/geo/src/main/java/org/elasticsearch/geo/geometry/Point.java
index 248f433b96a13..88fd5eb06fe79 100644
--- a/libs/geo/src/main/java/org/elasticsearch/geo/geometry/Point.java
+++ b/libs/geo/src/main/java/org/elasticsearch/geo/geometry/Point.java
@@ -42,8 +42,6 @@ public Point(double lat, double lon) {
     }
 
     public Point(double lat, double lon, double alt) {
-        GeometryUtils.checkLatitude(lat);
-        GeometryUtils.checkLongitude(lon);
         this.lat = lat;
         this.lon = lon;
         this.alt = alt;
diff --git a/libs/geo/src/main/java/org/elasticsearch/geo/geometry/Rectangle.java b/libs/geo/src/main/java/org/elasticsearch/geo/geometry/Rectangle.java
index ca7ec2e57c98d..75ba25721e755 100644
--- a/libs/geo/src/main/java/org/elasticsearch/geo/geometry/Rectangle.java
+++ b/libs/geo/src/main/java/org/elasticsearch/geo/geometry/Rectangle.java
@@ -71,10 +71,6 @@ public Rectangle(double minLat, double maxLat, double minLon, double maxLon) {
      * Constructs a bounding box by first validating the provided latitude and longitude coordinates
      */
     public Rectangle(double minLat, double maxLat, double minLon, double maxLon, double minAlt, double maxAlt) {
-        GeometryUtils.checkLatitude(minLat);
-        GeometryUtils.checkLatitude(maxLat);
-        GeometryUtils.checkLongitude(minLon);
-        GeometryUtils.checkLongitude(maxLon);
         this.minLon = minLon;
         this.maxLon = maxLon;
         this.minLat = minLat;
@@ -90,17 +86,6 @@ public Rectangle(double minLat, double maxLat, double minLon, double maxLon, dou
         }
     }
 
-    public double getWidth() {
-        if (crossesDateline()) {
-            return GeometryUtils.MAX_LON_INCL - minLon + maxLon - GeometryUtils.MIN_LON_INCL;
-        }
-        return maxLon - minLon;
-    }
-
-    public double getHeight() {
-        return maxLat - minLat;
-    }
-
     public double getMinLat() {
         return minLat;
     }
@@ -156,21 +141,6 @@ public String toString() {
         return b.toString();
     }
 
-    /**
-     * Returns true if this bounding box crosses the dateline
-     */
-    public boolean crossesDateline() {
-        return maxLon < minLon;
-    }
-
-    /** returns true if rectangle (defined by minLat, maxLat, minLon, maxLon) contains the lat lon point */
-    public boolean containsPoint(final double lat, final double lon) {
-        if (lat >= minLat && lat <= maxLat) {
-            return crossesDateline() ? lon >= minLon || lon <= maxLon : lon >= minLon && lon <= maxLon;
-        }
-        return false;
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/libs/geo/src/main/java/org/elasticsearch/geo/utils/GeographyValidator.java b/libs/geo/src/main/java/org/elasticsearch/geo/utils/GeographyValidator.java
new file mode 100644
index 0000000000000..756792358abd4
--- /dev/null
+++ b/libs/geo/src/main/java/org/elasticsearch/geo/utils/GeographyValidator.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.geo.utils;
+
+import org.elasticsearch.geo.geometry.Circle;
+import org.elasticsearch.geo.geometry.Geometry;
+import org.elasticsearch.geo.geometry.GeometryCollection;
+import org.elasticsearch.geo.geometry.GeometryVisitor;
+import org.elasticsearch.geo.geometry.Line;
+import org.elasticsearch.geo.geometry.LinearRing;
+import org.elasticsearch.geo.geometry.MultiLine;
+import org.elasticsearch.geo.geometry.MultiPoint;
+import org.elasticsearch.geo.geometry.MultiPolygon;
+import org.elasticsearch.geo.geometry.Point;
+import org.elasticsearch.geo.geometry.Polygon;
+import org.elasticsearch.geo.geometry.Rectangle;
+
+/**
+ * Validator that checks that lats are between -90 and +90 and lons are between -180 and +180 and altitude is present only if
+ * ignoreZValue is set to true
+ */
+public class GeographyValidator implements GeometryValidator {
+
+    /**
+     * Minimum longitude value.
+     */
+    private static final double MIN_LON_INCL = -180.0D;
+
+    /**
+     * Maximum longitude value.
+     */
+    private static final double MAX_LON_INCL = 180.0D;
+
+    /**
+     * Minimum latitude value.
+     */
+    private static final double MIN_LAT_INCL = -90.0D;
+
+    /**
+     * Maximum latitude value.
+     */
+    private static final double MAX_LAT_INCL = 90.0D;
+
+    private final boolean ignoreZValue;
+
+    public GeographyValidator(boolean ignoreZValue) {
+        this.ignoreZValue = ignoreZValue;
+    }
+
+    /**
+     * validates latitude value is within standard +/-90 coordinate bounds
+     */
+    protected void checkLatitude(double latitude) {
+        if (Double.isNaN(latitude) || latitude < MIN_LAT_INCL || latitude > MAX_LAT_INCL) {
+            throw new IllegalArgumentException(
+                "invalid latitude " + latitude + "; must be between " + MIN_LAT_INCL + " and " + MAX_LAT_INCL);
+        }
+    }
+
+    /**
+     * validates longitude value is within standard +/-180 coordinate bounds
+     */
+    protected void checkLongitude(double longitude) {
+        if (Double.isNaN(longitude) || longitude < MIN_LON_INCL || longitude > MAX_LON_INCL) {
+            throw new IllegalArgumentException(
+                "invalid longitude " + longitude + "; must be between " + MIN_LON_INCL + " and " + MAX_LON_INCL);
+        }
+    }
+
+    protected void checkAltitude(double zValue) {
+        if (ignoreZValue == false && Double.isNaN(zValue) == false) {
+            throw new IllegalArgumentException("found Z value [" + zValue + "] but [ignore_z_value] "
+                + "parameter is [" + ignoreZValue + "]");
+        }
+    }
+
+    @Override
+    public void validate(Geometry geometry) {
+        geometry.visit(new GeometryVisitor<Void, RuntimeException>() {
+
+            @Override
+            public Void visit(Circle circle) throws RuntimeException {
+                checkLatitude(circle.getLat());
+                checkLongitude(circle.getLon());
+                checkAltitude(circle.getAlt());
+                return null;
+            }
+
+            @Override
+            public Void visit(GeometryCollection<?> collection) throws RuntimeException {
+                for (Geometry g : collection) {
+                    g.visit(this);
+                }
+                return null;
+            }
+
+            @Override
+            public Void visit(Line line) throws RuntimeException {
+                for (int i = 0; i < line.length(); i++) {
+                    checkLatitude(line.getLat(i));
+                    checkLongitude(line.getLon(i));
+                    checkAltitude(line.getAlt(i));
+                }
+                return null;
+            }
+
+            @Override
+            public Void visit(LinearRing ring) throws RuntimeException {
+                for (int i = 0; i < ring.length(); i++) {
+                    checkLatitude(ring.getLat(i));
+                    checkLongitude(ring.getLon(i));
+                    checkAltitude(ring.getAlt(i));
+                }
+                return null;
+            }
+
+            @Override
+            public Void visit(MultiLine multiLine) throws RuntimeException {
+                return visit((GeometryCollection<?>) multiLine);
+            }
+
+            @Override
+            public Void visit(MultiPoint multiPoint) throws RuntimeException {
+                return visit((GeometryCollection<?>) multiPoint);
+            }
+
+            @Override
+            public Void visit(MultiPolygon multiPolygon) throws RuntimeException {
+                return visit((GeometryCollection<?>) multiPolygon);
+            }
+
+            @Override
+            public Void visit(Point point) throws RuntimeException {
+                checkLatitude(point.getLat());
+                checkLongitude(point.getLon());
+                checkAltitude(point.getAlt());
+                return null;
+            }
+
+            @Override
+            public Void visit(Polygon polygon) throws RuntimeException {
+                polygon.getPolygon().visit(this);
+                for (int i = 0; i < polygon.getNumberOfHoles(); i++) {
+                    polygon.getHole(i).visit(this);
+                }
+                return null;
+            }
+
+            @Override
+            public Void visit(Rectangle rectangle) throws RuntimeException {
+                checkLatitude(rectangle.getMinLat());
+                checkLatitude(rectangle.getMaxLat());
+                checkLongitude(rectangle.getMinLon());
+                checkLongitude(rectangle.getMaxLon());
+                checkAltitude(rectangle.getMinAlt());
+                checkAltitude(rectangle.getMaxAlt());
+                return null;
+            }
+        });
+    }
+}
diff --git a/libs/geo/src/main/java/org/elasticsearch/geo/utils/GeometryValidator.java b/libs/geo/src/main/java/org/elasticsearch/geo/utils/GeometryValidator.java
new file mode 100644
index 0000000000000..2caf6738ed469
--- /dev/null
+++ b/libs/geo/src/main/java/org/elasticsearch/geo/utils/GeometryValidator.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.geo.utils;
+
+import org.elasticsearch.geo.geometry.Geometry;
+
+/**
+ * Generic geometry validator that can be used by the parser to verify the validity of the parsed geometry
+ */
+public interface GeometryValidator {
+
+    /**
+     * Validates the geometry and throws IllegalArgumentException if the geometry is not valid
+     */
+    void validate(Geometry geometry);
+
+}
diff --git a/libs/geo/src/main/java/org/elasticsearch/geo/utils/WellKnownText.java b/libs/geo/src/main/java/org/elasticsearch/geo/utils/WellKnownText.java
index e1af54e3383e0..4fd4bdb6fd150 100644
--- a/libs/geo/src/main/java/org/elasticsearch/geo/utils/WellKnownText.java
+++ b/libs/geo/src/main/java/org/elasticsearch/geo/utils/WellKnownText.java
@@ -52,17 +52,25 @@ public class WellKnownText {
     public static final String COMMA = ",";
     public static final String NAN = "NaN";
 
-    private static final String NUMBER = "<NUMBER>";
-    private static final String EOF = "END-OF-STREAM";
-    private static final String EOL = "END-OF-LINE";
+    private final String NUMBER = "<NUMBER>";
+    private final String EOF = "END-OF-STREAM";
+    private final String EOL = "END-OF-LINE";
 
-    public static String toWKT(Geometry geometry) {
+    private final boolean coerce;
+    private final GeometryValidator validator;
+
+    public WellKnownText(boolean coerce, GeometryValidator validator) {
+        this.coerce = coerce;
+        this.validator = validator;
+    }
+
+    public String toWKT(Geometry geometry) {
         StringBuilder builder = new StringBuilder();
         toWKT(geometry, builder);
         return builder.toString();
     }
 
-    public static void toWKT(Geometry geometry, StringBuilder sb) {
+    public void toWKT(Geometry geometry, StringBuilder sb) {
         sb.append(getWKTName(geometry));
         sb.append(SPACE);
         if (geometry.isEmpty()) {
@@ -216,7 +224,7 @@ public Void visit(Rectangle rectangle) {
         }
     }
 
-    public static Geometry fromWKT(String wkt) throws IOException, ParseException {
+    public Geometry fromWKT(String wkt) throws IOException, ParseException {
         StringReader reader = new StringReader(wkt);
         try {
             // setup the tokenizer; configured to read words w/o numbers
@@ -234,7 +242,9 @@ public static Geometry fromWKT(String wkt) throws IOException, ParseException {
             tokenizer.whitespaceChars('\r', '\r');
             tokenizer.whitespaceChars('\n', '\n');
             tokenizer.commentChar('#');
-            return parseGeometry(tokenizer);
+            Geometry geometry = parseGeometry(tokenizer);
+            validator.validate(geometry);
+            return geometry;
         } finally {
             reader.close();
         }
@@ -243,7 +253,7 @@ public static Geometry fromWKT(String wkt) throws IOException, ParseException {
     /**
      * parse geometry from the stream tokenizer
      */
-    private static Geometry parseGeometry(StreamTokenizer stream) throws IOException, ParseException {
+    private Geometry parseGeometry(StreamTokenizer stream) throws IOException, ParseException {
         final String type = nextWord(stream).toLowerCase(Locale.ROOT);
         switch (type) {
             case "point":
@@ -268,7 +278,7 @@ private static Geometry parseGeometry(StreamTokenizer stream) throws IOException
         throw new IllegalArgumentException("Unknown geometry type: " + type);
     }
 
-    private static GeometryCollection<Geometry> parseGeometryCollection(StreamTokenizer stream) throws IOException, ParseException {
+    private GeometryCollection<Geometry> parseGeometryCollection(StreamTokenizer stream) throws IOException, ParseException {
         if (nextEmptyOrOpen(stream).equals(EMPTY)) {
             return GeometryCollection.EMPTY;
         }
@@ -280,7 +290,7 @@ private static GeometryCollection<Geometry> parseGeometryCollection(StreamTokeni
         return new GeometryCollection<>(shapes);
     }
 
-    private static Point parsePoint(StreamTokenizer stream) throws IOException, ParseException {
+    private Point parsePoint(StreamTokenizer stream) throws IOException, ParseException {
         if (nextEmptyOrOpen(stream).equals(EMPTY)) {
             return Point.EMPTY;
         }
@@ -296,7 +306,7 @@ private static Point parsePoint(StreamTokenizer stream) throws IOException, Pars
         return pt;
     }
 
-    private static void parseCoordinates(StreamTokenizer stream, ArrayList<Double> lats, ArrayList<Double> lons, ArrayList<Double> alts)
+    private void parseCoordinates(StreamTokenizer stream, ArrayList<Double> lats, ArrayList<Double> lons, ArrayList<Double> alts)
         throws IOException, ParseException {
         parseCoordinate(stream, lats, lons, alts);
         while (nextCloserOrComma(stream).equals(COMMA)) {
@@ -304,7 +314,7 @@ private static void parseCoordinates(StreamTokenizer stream, ArrayList<Double> l
         }
     }
 
-    private static void parseCoordinate(StreamTokenizer stream, ArrayList<Double> lats, ArrayList<Double> lons, ArrayList<Double> alts)
+    private void parseCoordinate(StreamTokenizer stream, ArrayList<Double> lats, ArrayList<Double> lons, ArrayList<Double> alts)
         throws IOException, ParseException {
         lons.add(nextNumber(stream));
         lats.add(nextNumber(stream));
@@ -316,7 +326,7 @@ private static void parseCoordinate(StreamTokenizer stream, ArrayList<Double> la
         }
     }
 
-    private static MultiPoint parseMultiPoint(StreamTokenizer stream) throws IOException, ParseException {
+    private MultiPoint parseMultiPoint(StreamTokenizer stream) throws IOException, ParseException {
         String token = nextEmptyOrOpen(stream);
         if (token.equals(EMPTY)) {
             return MultiPoint.EMPTY;
@@ -336,7 +346,7 @@ private static MultiPoint parseMultiPoint(StreamTokenizer stream) throws IOExcep
         return new MultiPoint(Collections.unmodifiableList(points));
     }
 
-    private static Line parseLine(StreamTokenizer stream) throws IOException, ParseException {
+    private Line parseLine(StreamTokenizer stream) throws IOException, ParseException {
         String token = nextEmptyOrOpen(stream);
         if (token.equals(EMPTY)) {
             return Line.EMPTY;
@@ -352,7 +362,7 @@ private static Line parseLine(StreamTokenizer stream) throws IOException, ParseE
         }
     }
 
-    private static MultiLine parseMultiLine(StreamTokenizer stream) throws IOException, ParseException {
+    private MultiLine parseMultiLine(StreamTokenizer stream) throws IOException, ParseException {
         String token = nextEmptyOrOpen(stream);
         if (token.equals(EMPTY)) {
             return MultiLine.EMPTY;
@@ -365,12 +375,13 @@ private static MultiLine parseMultiLine(StreamTokenizer stream) throws IOExcepti
         return new MultiLine(Collections.unmodifiableList(lines));
     }
 
-    private static LinearRing parsePolygonHole(StreamTokenizer stream) throws IOException, ParseException {
+    private LinearRing parsePolygonHole(StreamTokenizer stream) throws IOException, ParseException {
         nextOpener(stream);
         ArrayList<Double> lats = new ArrayList<>();
         ArrayList<Double> lons = new ArrayList<>();
         ArrayList<Double> alts = new ArrayList<>();
         parseCoordinates(stream, lats, lons, alts);
+        closeLinearRingIfCoerced(lats, lons, alts);
         if (alts.isEmpty()) {
             return new LinearRing(toArray(lats), toArray(lons));
         } else {
@@ -378,7 +389,7 @@ private static LinearRing parsePolygonHole(StreamTokenizer stream) throws IOExce
         }
     }
 
-    private static Polygon parsePolygon(StreamTokenizer stream) throws IOException, ParseException {
+    private Polygon parsePolygon(StreamTokenizer stream) throws IOException, ParseException {
         if (nextEmptyOrOpen(stream).equals(EMPTY)) {
             return Polygon.EMPTY;
         }
@@ -391,6 +402,7 @@ private static Polygon parsePolygon(StreamTokenizer stream) throws IOException,
         while (nextCloserOrComma(stream).equals(COMMA)) {
             holes.add(parsePolygonHole(stream));
         }
+        closeLinearRingIfCoerced(lats, lons, alts);
         LinearRing shell;
         if (alts.isEmpty()) {
             shell = new LinearRing(toArray(lats), toArray(lons));
@@ -404,7 +416,25 @@ private static Polygon parsePolygon(StreamTokenizer stream) throws IOException,
         }
     }
 
-    private static MultiPolygon parseMultiPolygon(StreamTokenizer stream) throws IOException, ParseException {
+    /**
+     * Treats supplied arrays as coordinates of a linear ring. If the ring is not closed and coerce is set to true,
+     * the first set of coordinates (lat, lon and alt if available) are added to the end of the arrays.
+     */
+    private void closeLinearRingIfCoerced(ArrayList<Double> lats, ArrayList<Double> lons, ArrayList<Double> alts) {
+        if (coerce && lats.isEmpty() == false && lons.isEmpty() == false) {
+            int last = lats.size() - 1;
+            if (!lats.get(0).equals(lats.get(last)) || !lons.get(0).equals(lons.get(last)) ||
+                (alts.isEmpty() == false && !alts.get(0).equals(alts.get(last)))) {
+                lons.add(lons.get(0));
+                lats.add(lats.get(0));
+                if (alts.isEmpty() == false) {
+                    alts.add(alts.get(0));
+                }
+            }
+        }
+    }
+
+    private MultiPolygon parseMultiPolygon(StreamTokenizer stream) throws IOException, ParseException {
         String token = nextEmptyOrOpen(stream);
         if (token.equals(EMPTY)) {
             return MultiPolygon.EMPTY;
@@ -417,7 +447,7 @@ private static MultiPolygon parseMultiPolygon(StreamTokenizer stream) throws IOE
         return new MultiPolygon(Collections.unmodifiableList(polygons));
     }
 
-    private static Rectangle parseBBox(StreamTokenizer stream) throws IOException, ParseException {
+    private Rectangle parseBBox(StreamTokenizer stream) throws IOException, ParseException {
         if (nextEmptyOrOpen(stream).equals(EMPTY)) {
             return Rectangle.EMPTY;
         }
@@ -434,7 +464,7 @@ private static Rectangle parseBBox(StreamTokenizer stream) throws IOException, P
     }
 
 
-    private static Circle parseCircle(StreamTokenizer stream) throws IOException, ParseException {
+    private Circle parseCircle(StreamTokenizer stream) throws IOException, ParseException {
         if (nextEmptyOrOpen(stream).equals(EMPTY)) {
             return Circle.EMPTY;
         }
@@ -453,7 +483,7 @@ private static Circle parseCircle(StreamTokenizer stream) throws IOException, Pa
     /**
      * next word in the stream
      */
-    private static String nextWord(StreamTokenizer stream) throws ParseException, IOException {
+    private String nextWord(StreamTokenizer stream) throws ParseException, IOException {
         switch (stream.nextToken()) {
             case StreamTokenizer.TT_WORD:
                 final String word = stream.sval;
@@ -468,7 +498,7 @@ private static String nextWord(StreamTokenizer stream) throws ParseException, IO
         throw new ParseException("expected word but found: " + tokenString(stream), stream.lineno());
     }
 
-    private static double nextNumber(StreamTokenizer stream) throws IOException, ParseException {
+    private double nextNumber(StreamTokenizer stream) throws IOException, ParseException {
         if (stream.nextToken() == StreamTokenizer.TT_WORD) {
             if (stream.sval.equalsIgnoreCase(NAN)) {
                 return Double.NaN;
@@ -483,7 +513,7 @@ private static double nextNumber(StreamTokenizer stream) throws IOException, Par
         throw new ParseException("expected number but found: " + tokenString(stream), stream.lineno());
     }
 
-    private static String tokenString(StreamTokenizer stream) {
+    private String tokenString(StreamTokenizer stream) {
         switch (stream.ttype) {
             case StreamTokenizer.TT_WORD:
                 return stream.sval;
@@ -497,13 +527,13 @@ private static String tokenString(StreamTokenizer stream) {
         return "'" + (char) stream.ttype + "'";
     }
 
-    private static boolean isNumberNext(StreamTokenizer stream) throws IOException {
+    private boolean isNumberNext(StreamTokenizer stream) throws IOException {
         final int type = stream.nextToken();
         stream.pushBack();
         return type == StreamTokenizer.TT_WORD;
     }
 
-    private static String nextEmptyOrOpen(StreamTokenizer stream) throws IOException, ParseException {
+    private String nextEmptyOrOpen(StreamTokenizer stream) throws IOException, ParseException {
         final String next = nextWord(stream);
         if (next.equals(EMPTY) || next.equals(LPAREN)) {
             return next;
@@ -512,28 +542,28 @@ private static String nextEmptyOrOpen(StreamTokenizer stream) throws IOException
             + " but found: " + tokenString(stream), stream.lineno());
     }
 
-    private static String nextCloser(StreamTokenizer stream) throws IOException, ParseException {
+    private String nextCloser(StreamTokenizer stream) throws IOException, ParseException {
         if (nextWord(stream).equals(RPAREN)) {
             return RPAREN;
         }
         throw new ParseException("expected " + RPAREN + " but found: " + tokenString(stream), stream.lineno());
     }
 
-    private static String nextComma(StreamTokenizer stream) throws IOException, ParseException {
+    private String nextComma(StreamTokenizer stream) throws IOException, ParseException {
         if (nextWord(stream).equals(COMMA) == true) {
             return COMMA;
         }
         throw new ParseException("expected " + COMMA + " but found: " + tokenString(stream), stream.lineno());
     }
 
-    private static String nextOpener(StreamTokenizer stream) throws IOException, ParseException {
+    private String nextOpener(StreamTokenizer stream) throws IOException, ParseException {
         if (nextWord(stream).equals(LPAREN)) {
             return LPAREN;
         }
         throw new ParseException("expected " + LPAREN + " but found: " + tokenString(stream), stream.lineno());
     }
 
-    private static String nextCloserOrComma(StreamTokenizer stream) throws IOException, ParseException {
+    private String nextCloserOrComma(StreamTokenizer stream) throws IOException, ParseException {
         String token = nextWord(stream);
         if (token.equals(COMMA) || token.equals(RPAREN)) {
             return token;
@@ -542,7 +572,7 @@ private static String nextCloserOrComma(StreamTokenizer stream) throws IOExcepti
             + " but found: " + tokenString(stream), stream.lineno());
     }
 
-    public static String getWKTName(Geometry geometry) {
+    private static String getWKTName(Geometry geometry) {
         return geometry.visit(new GeometryVisitor<String, RuntimeException>() {
             @Override
             public String visit(Circle circle) {
@@ -596,7 +626,7 @@ public String visit(Rectangle rectangle) {
         });
     }
 
-    private static double[] toArray(ArrayList<Double> doubles) {
+    private double[] toArray(ArrayList<Double> doubles) {
         return doubles.stream().mapToDouble(i -> i).toArray();
     }
 
diff --git a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/BaseGeometryTestCase.java b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/BaseGeometryTestCase.java
index cc7dcd340c734..073bff4cb7575 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/BaseGeometryTestCase.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/BaseGeometryTestCase.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.geo.utils.GeographyValidator;
 import org.elasticsearch.geo.utils.WellKnownText;
 import org.elasticsearch.test.AbstractWireTestCase;
 
@@ -53,9 +54,10 @@ protected Writeable.Reader<T> instanceReader() {
     @SuppressWarnings("unchecked")
     @Override
     protected T copyInstance(T instance, Version version) throws IOException {
-        String text = WellKnownText.toWKT(instance);
+        WellKnownText wkt = new WellKnownText(true, new GeographyValidator(true));
+        String text = wkt.toWKT(instance);
         try {
-            return (T) WellKnownText.fromWKT(text);
+            return (T) wkt.fromWKT(text);
         } catch (ParseException e) {
             throw new ElasticsearchException(e);
         }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/CircleTests.java b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/CircleTests.java
index a38a29af24b54..e8912a39fb435 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/CircleTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/CircleTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.geo.geometry;
 
+import org.elasticsearch.geo.utils.GeographyValidator;
+import org.elasticsearch.geo.utils.GeometryValidator;
 import org.elasticsearch.geo.utils.WellKnownText;
 
 import java.io.IOException;
@@ -36,24 +38,26 @@ protected Circle createTestInstance(boolean hasAlt) {
     }
 
     public void testBasicSerialization() throws IOException, ParseException {
-        assertEquals("circle (20.0 10.0 15.0)", WellKnownText.toWKT(new Circle(10, 20, 15)));
-        assertEquals(new Circle(10, 20, 15), WellKnownText.fromWKT("circle (20.0 10.0 15.0)"));
+        WellKnownText wkt = new WellKnownText(true, new GeographyValidator(true));
+        assertEquals("circle (20.0 10.0 15.0)", wkt.toWKT(new Circle(10, 20, 15)));
+        assertEquals(new Circle(10, 20, 15), wkt.fromWKT("circle (20.0 10.0 15.0)"));
 
-        assertEquals("circle (20.0 10.0 15.0 25.0)", WellKnownText.toWKT(new Circle(10, 20, 25, 15)));
-        assertEquals(new Circle(10, 20, 25, 15), WellKnownText.fromWKT("circle (20.0 10.0 15.0 25.0)"));
+        assertEquals("circle (20.0 10.0 15.0 25.0)", wkt.toWKT(new Circle(10, 20, 25, 15)));
+        assertEquals(new Circle(10, 20, 25, 15), wkt.fromWKT("circle (20.0 10.0 15.0 25.0)"));
 
-        assertEquals("circle EMPTY", WellKnownText.toWKT(Circle.EMPTY));
-        assertEquals(Circle.EMPTY, WellKnownText.fromWKT("circle EMPTY)"));
+        assertEquals("circle EMPTY", wkt.toWKT(Circle.EMPTY));
+        assertEquals(Circle.EMPTY, wkt.fromWKT("circle EMPTY)"));
     }
 
     public void testInitValidation() {
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> new Circle(10, 20, -1));
+        GeometryValidator validator = new GeographyValidator(true);
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> validator.validate(new Circle(10, 20, -1)));
         assertEquals("Circle radius [-1.0] cannot be negative", ex.getMessage());
 
-        ex = expectThrows(IllegalArgumentException.class, () -> new Circle(100, 20, 1));
+        ex = expectThrows(IllegalArgumentException.class, () -> validator.validate(new Circle(100, 20, 1)));
         assertEquals("invalid latitude 100.0; must be between -90.0 and 90.0", ex.getMessage());
 
-        ex = expectThrows(IllegalArgumentException.class, () -> new Circle(10, 200, 1));
+        ex = expectThrows(IllegalArgumentException.class, () -> validator.validate(new Circle(10, 200, 1)));
         assertEquals("invalid longitude 200.0; must be between -180.0 and 180.0", ex.getMessage());
     }
 }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/GeometryCollectionTests.java b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/GeometryCollectionTests.java
index 8b6415270635e..c78c47dfbcd96 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/GeometryCollectionTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/GeometryCollectionTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.geo.geometry;
 
+import org.elasticsearch.geo.utils.GeographyValidator;
 import org.elasticsearch.geo.utils.WellKnownText;
 
 import java.io.IOException;
@@ -35,14 +36,15 @@ protected GeometryCollection<Geometry> createTestInstance(boolean hasAlt) {
 
 
     public void testBasicSerialization() throws IOException, ParseException {
+        WellKnownText wkt = new WellKnownText(true, new GeographyValidator(true));
         assertEquals("geometrycollection (point (20.0 10.0),point EMPTY)",
-            WellKnownText.toWKT(new GeometryCollection<Geometry>(Arrays.asList(new Point(10, 20), Point.EMPTY))));
+            wkt.toWKT(new GeometryCollection<Geometry>(Arrays.asList(new Point(10, 20), Point.EMPTY))));
 
         assertEquals(new GeometryCollection<Geometry>(Arrays.asList(new Point(10, 20), Point.EMPTY)),
-            WellKnownText.fromWKT("geometrycollection (point (20.0 10.0),point EMPTY)"));
+            wkt.fromWKT("geometrycollection (point (20.0 10.0),point EMPTY)"));
 
-        assertEquals("geometrycollection EMPTY", WellKnownText.toWKT(GeometryCollection.EMPTY));
-        assertEquals(GeometryCollection.EMPTY, WellKnownText.fromWKT("geometrycollection EMPTY)"));
+        assertEquals("geometrycollection EMPTY", wkt.toWKT(GeometryCollection.EMPTY));
+        assertEquals(GeometryCollection.EMPTY, wkt.fromWKT("geometrycollection EMPTY)"));
     }
 
     @SuppressWarnings("ConstantConditions")
diff --git a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/GeometryValidatorTests.java b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/GeometryValidatorTests.java
new file mode 100644
index 0000000000000..c747fc2df50a7
--- /dev/null
+++ b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/GeometryValidatorTests.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.geo.geometry;
+
+import org.elasticsearch.geo.utils.GeographyValidator;
+import org.elasticsearch.geo.utils.GeometryValidator;
+import org.elasticsearch.geo.utils.WellKnownText;
+import org.elasticsearch.test.ESTestCase;
+
+public class GeometryValidatorTests extends ESTestCase {
+
+    public static class NoopValidator implements GeometryValidator {
+
+        @Override
+        public void validate(Geometry geometry) {
+
+        }
+    }
+
+    public static class OneValidator extends GeographyValidator {
+        /**
+         * Minimum longitude value.
+         */
+        private static final double MIN_LON_INCL = -1D;
+
+        /**
+         * Maximum longitude value.
+         */
+        private static final double MAX_LON_INCL = 1D;
+
+        /**
+         * Minimum latitude value.
+         */
+        private static final double MIN_LAT_INCL = -1D;
+
+        /**
+         * Maximum latitude value.
+         */
+        private static final double MAX_LAT_INCL = 1D;
+
+        /**
+         * Minimum altitude value.
+         */
+        private static final double MIN_ALT_INCL = -1D;
+
+        /**
+         * Maximum altitude value.
+         */
+        private static final double MAX_ALT_INCL = 1D;
+
+        public OneValidator() {
+            super(true);
+        }
+
+        @Override
+        protected void checkLatitude(double latitude) {
+            if (Double.isNaN(latitude) || latitude < MIN_LAT_INCL || latitude > MAX_LAT_INCL) {
+                throw new IllegalArgumentException(
+                    "invalid latitude " + latitude + "; must be between " + MIN_LAT_INCL + " and " + MAX_LAT_INCL);
+            }
+        }
+
+        @Override
+        protected void checkLongitude(double longitude) {
+            if (Double.isNaN(longitude) || longitude < MIN_LON_INCL || longitude > MAX_LON_INCL) {
+                throw new IllegalArgumentException(
+                    "invalid longitude " + longitude + "; must be between " + MIN_LON_INCL + " and " + MAX_LON_INCL);
+            }
+        }
+
+        @Override
+        protected void checkAltitude(double zValue) {
+            if (Double.isNaN(zValue) == false && (zValue < MIN_ALT_INCL || zValue > MAX_ALT_INCL)) {
+                throw new IllegalArgumentException(
+                    "invalid altitude " + zValue + "; must be between " + MIN_ALT_INCL + " and " + MAX_ALT_INCL);
+            }
+        }
+    }
+
+    public void testNoopValidator() throws Exception {
+        WellKnownText parser = new WellKnownText(true, new NoopValidator());
+        parser.fromWKT("CIRCLE (10000 20000 30000)");
+        parser.fromWKT("POINT (10000 20000)");
+        parser.fromWKT("LINESTRING (10000 20000, 0 0)");
+        parser.fromWKT("POLYGON ((300 100, 400 200, 500 300, 300 100), (50 150, 250 150, 200 100))");
+        parser.fromWKT("MULTIPOINT (10000 20000, 20000 30000)");
+    }
+
+    public void testOneValidator() throws Exception {
+        WellKnownText parser = new WellKnownText(true, new OneValidator());
+        parser.fromWKT("POINT (0 1)");
+        parser.fromWKT("POINT (0 1 0.5)");
+        IllegalArgumentException ex;
+        ex = expectThrows(IllegalArgumentException.class, () -> parser.fromWKT("CIRCLE (1 2 3)"));
+        assertEquals("invalid latitude 2.0; must be between -1.0 and 1.0", ex.getMessage());
+        ex = expectThrows(IllegalArgumentException.class, () -> parser.fromWKT("POINT (2 1)"));
+        assertEquals("invalid longitude 2.0; must be between -1.0 and 1.0", ex.getMessage());
+        ex = expectThrows(IllegalArgumentException.class, () -> parser.fromWKT("LINESTRING (1 -1 0, 0 0 2)"));
+        assertEquals("invalid altitude 2.0; must be between -1.0 and 1.0", ex.getMessage());
+        ex = expectThrows(IllegalArgumentException.class, () -> parser.fromWKT("POLYGON ((0.3 0.1, 0.4 0.2, 5 0.3, 0.3 0.1))"));
+        assertEquals("invalid longitude 5.0; must be between -1.0 and 1.0", ex.getMessage());
+        ex = expectThrows(IllegalArgumentException.class, () -> parser.fromWKT(
+            "POLYGON ((0.3 0.1, 0.4 0.2, 0.5 0.3, 0.3 0.1), (0.5 1.5, 2.5 1.5, 2.0 1.0))"));
+        assertEquals("invalid latitude 1.5; must be between -1.0 and 1.0", ex.getMessage());
+        ex = expectThrows(IllegalArgumentException.class, () -> parser.fromWKT("MULTIPOINT (0 1, -2 1)"));
+        assertEquals("invalid longitude -2.0; must be between -1.0 and 1.0", ex.getMessage());
+    }
+
+
+}
diff --git a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/LineTests.java b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/LineTests.java
index 7156039f9085f..b9f8cb37f5422 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/LineTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/LineTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.geo.geometry;
 
+import org.elasticsearch.geo.utils.GeographyValidator;
+import org.elasticsearch.geo.utils.GeometryValidator;
 import org.elasticsearch.geo.utils.WellKnownText;
 
 import java.io.IOException;
@@ -31,26 +33,37 @@ protected Line createTestInstance(boolean hasAlt) {
     }
 
     public void testBasicSerialization() throws IOException, ParseException {
-        assertEquals("linestring (3.0 1.0, 4.0 2.0)", WellKnownText.toWKT(new Line(new double[]{1, 2}, new double[]{3, 4})));
-        assertEquals(new Line(new double[]{1, 2}, new double[]{3, 4}), WellKnownText.fromWKT("linestring (3 1, 4 2)"));
+        WellKnownText wkt = new WellKnownText(true, new GeographyValidator(true));
+        assertEquals("linestring (3.0 1.0, 4.0 2.0)", wkt.toWKT(new Line(new double[]{1, 2}, new double[]{3, 4})));
+        assertEquals(new Line(new double[]{1, 2}, new double[]{3, 4}), wkt.fromWKT("linestring (3 1, 4 2)"));
 
-        assertEquals("linestring (3.0 1.0 5.0, 4.0 2.0 6.0)", WellKnownText.toWKT(new Line(new double[]{1, 2}, new double[]{3, 4},
+        assertEquals("linestring (3.0 1.0 5.0, 4.0 2.0 6.0)", wkt.toWKT(new Line(new double[]{1, 2}, new double[]{3, 4},
             new double[]{5, 6})));
         assertEquals(new Line(new double[]{1, 2}, new double[]{3, 4}, new double[]{6, 5}),
-            WellKnownText.fromWKT("linestring (3 1 6, 4 2 5)"));
+            wkt.fromWKT("linestring (3 1 6, 4 2 5)"));
 
-        assertEquals("linestring EMPTY", WellKnownText.toWKT(Line.EMPTY));
-        assertEquals(Line.EMPTY, WellKnownText.fromWKT("linestring EMPTY)"));
+        assertEquals("linestring EMPTY", wkt.toWKT(Line.EMPTY));
+        assertEquals(Line.EMPTY, wkt.fromWKT("linestring EMPTY)"));
     }
 
     public void testInitValidation() {
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> new Line(new double[]{1}, new double[]{3}));
+        GeometryValidator validator = new GeographyValidator(true);
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+            () -> validator.validate(new Line(new double[]{1}, new double[]{3})));
         assertEquals("at least two points in the line is required", ex.getMessage());
 
-        ex = expectThrows(IllegalArgumentException.class, () -> new Line(new double[]{1, 2, 3, 1}, new double[]{3, 4, 500, 3}));
+        ex = expectThrows(IllegalArgumentException.class,
+            () -> validator.validate(new Line(new double[]{1, 2, 3, 1}, new double[]{3, 4, 500, 3})));
         assertEquals("invalid longitude 500.0; must be between -180.0 and 180.0", ex.getMessage());
 
-        ex = expectThrows(IllegalArgumentException.class, () -> new Line(new double[]{1, 100, 3, 1}, new double[]{3, 4, 5, 3}));
+        ex = expectThrows(IllegalArgumentException.class,
+            () -> validator.validate(new Line(new double[]{1, 100, 3, 1}, new double[]{3, 4, 5, 3})));
         assertEquals("invalid latitude 100.0; must be between -90.0 and 90.0", ex.getMessage());
     }
+
+    public void testWKTValidation() {
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+            () -> new WellKnownText(randomBoolean(), new GeographyValidator(false)).fromWKT("linestring (3 1 6, 4 2 5)"));
+        assertEquals("found Z value [6.0] but [ignore_z_value] parameter is [false]", ex.getMessage());
+    }
 }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/LinearRingTests.java b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/LinearRingTests.java
index d5b708f558cac..07e9e866233e7 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/LinearRingTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/LinearRingTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.geo.geometry;
 
+import org.elasticsearch.geo.utils.GeographyValidator;
+import org.elasticsearch.geo.utils.GeometryValidator;
 import org.elasticsearch.geo.utils.WellKnownText;
 import org.elasticsearch.test.ESTestCase;
 
@@ -26,30 +28,35 @@ public class LinearRingTests extends ESTestCase {
 
     public void testBasicSerialization() {
         UnsupportedOperationException ex = expectThrows(UnsupportedOperationException.class,
-            () -> WellKnownText.toWKT(new LinearRing(new double[]{1, 2, 3, 1}, new double[]{3, 4, 5, 3})));
+            () -> new WellKnownText(true, new GeographyValidator(true))
+                .toWKT(new LinearRing(new double[]{1, 2, 3, 1}, new double[]{3, 4, 5, 3})));
         assertEquals("line ring cannot be serialized using WKT", ex.getMessage());
     }
 
     public void testInitValidation() {
+        GeometryValidator validator = new GeographyValidator(true);
         IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-            () -> new LinearRing(new double[]{1, 2, 3}, new double[]{3, 4, 5}));
+            () -> validator.validate(new LinearRing(new double[]{1, 2, 3}, new double[]{3, 4, 5})));
         assertEquals("first and last points of the linear ring must be the same (it must close itself): lats[0]=1.0 lats[2]=3.0 " +
                 "lons[0]=3.0 lons[2]=5.0",
             ex.getMessage());
 
         ex = expectThrows(IllegalArgumentException.class,
-            () -> new LinearRing(new double[]{1, 2, 1}, new double[]{3, 4, 3}, new double[]{1, 2, 3}));
+            () -> validator.validate(new LinearRing(new double[]{1, 2, 1}, new double[]{3, 4, 3}, new double[]{1, 2, 3})));
         assertEquals("first and last points of the linear ring must be the same (it must close itself): lats[0]=1.0 lats[2]=1.0 " +
                 "lons[0]=3.0 lons[2]=3.0 alts[0]=1.0 alts[2]=3.0",
             ex.getMessage());
 
-        ex = expectThrows(IllegalArgumentException.class, () -> new LinearRing(new double[]{1}, new double[]{3}));
+        ex = expectThrows(IllegalArgumentException.class,
+            () -> validator.validate(new LinearRing(new double[]{1}, new double[]{3})));
         assertEquals("at least two points in the line is required", ex.getMessage());
 
-        ex = expectThrows(IllegalArgumentException.class, () -> new LinearRing(new double[]{1, 2, 3, 1}, new double[]{3, 4, 500, 3}));
+        ex = expectThrows(IllegalArgumentException.class,
+            () -> validator.validate(new LinearRing(new double[]{1, 2, 3, 1}, new double[]{3, 4, 500, 3})));
         assertEquals("invalid longitude 500.0; must be between -180.0 and 180.0", ex.getMessage());
 
-        ex = expectThrows(IllegalArgumentException.class, () -> new LinearRing(new double[]{1, 100, 3, 1}, new double[]{3, 4, 5, 3}));
+        ex = expectThrows(IllegalArgumentException.class,
+            () -> validator.validate(new LinearRing(new double[]{1, 100, 3, 1}, new double[]{3, 4, 5, 3})));
         assertEquals("invalid latitude 100.0; must be between -90.0 and 90.0", ex.getMessage());
     }
 
diff --git a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/MultiLineTests.java b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/MultiLineTests.java
index 3fcb84d93b7d2..9ed782e65cc06 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/MultiLineTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/MultiLineTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.geo.geometry;
 
+import org.elasticsearch.geo.utils.GeographyValidator;
 import org.elasticsearch.geo.utils.WellKnownText;
 
 import java.io.IOException;
@@ -40,12 +41,13 @@ protected MultiLine createTestInstance(boolean hasAlt) {
     }
 
     public void testBasicSerialization() throws IOException, ParseException {
-        assertEquals("multilinestring ((3.0 1.0, 4.0 2.0))", WellKnownText.toWKT(
+        WellKnownText wkt = new WellKnownText(true, new GeographyValidator(true));
+        assertEquals("multilinestring ((3.0 1.0, 4.0 2.0))", wkt.toWKT(
             new MultiLine(Collections.singletonList(new Line(new double[]{1, 2}, new double[]{3, 4})))));
         assertEquals(new MultiLine(Collections.singletonList(new Line(new double[]{1, 2}, new double[]{3, 4}))),
-            WellKnownText.fromWKT("multilinestring ((3 1, 4 2))"));
+            wkt.fromWKT("multilinestring ((3 1, 4 2))"));
 
-        assertEquals("multilinestring EMPTY", WellKnownText.toWKT(MultiLine.EMPTY));
-        assertEquals(MultiLine.EMPTY, WellKnownText.fromWKT("multilinestring EMPTY)"));
+        assertEquals("multilinestring EMPTY", wkt.toWKT(MultiLine.EMPTY));
+        assertEquals(MultiLine.EMPTY, wkt.fromWKT("multilinestring EMPTY)"));
     }
 }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/MultiPointTests.java b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/MultiPointTests.java
index ecdcc0815a82b..c170adf9c9411 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/MultiPointTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/MultiPointTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.geo.geometry;
 
+import org.elasticsearch.geo.utils.GeographyValidator;
 import org.elasticsearch.geo.utils.WellKnownText;
 
 import java.io.IOException;
@@ -41,22 +42,23 @@ protected MultiPoint createTestInstance(boolean hasAlt) {
     }
 
     public void testBasicSerialization() throws IOException, ParseException {
-        assertEquals("multipoint (2.0 1.0)", WellKnownText.toWKT(
+        WellKnownText wkt = new WellKnownText(true, new GeographyValidator(true));
+        assertEquals("multipoint (2.0 1.0)", wkt.toWKT(
             new MultiPoint(Collections.singletonList(new Point(1, 2)))));
         assertEquals(new MultiPoint(Collections.singletonList(new Point(1 ,2))),
-            WellKnownText.fromWKT("multipoint (2 1)"));
+            wkt.fromWKT("multipoint (2 1)"));
 
         assertEquals("multipoint (2.0 1.0, 3.0 4.0)",
-            WellKnownText.toWKT(new MultiPoint(Arrays.asList(new Point(1, 2), new Point(4, 3)))));
+            wkt.toWKT(new MultiPoint(Arrays.asList(new Point(1, 2), new Point(4, 3)))));
         assertEquals(new MultiPoint(Arrays.asList(new Point(1, 2), new Point(4, 3))),
-            WellKnownText.fromWKT("multipoint (2 1, 3 4)"));
+            wkt.fromWKT("multipoint (2 1, 3 4)"));
 
         assertEquals("multipoint (2.0 1.0 10.0, 3.0 4.0 20.0)",
-            WellKnownText.toWKT(new MultiPoint(Arrays.asList(new Point(1, 2, 10), new Point(4, 3, 20)))));
+            wkt.toWKT(new MultiPoint(Arrays.asList(new Point(1, 2, 10), new Point(4, 3, 20)))));
         assertEquals(new MultiPoint(Arrays.asList(new Point(1, 2, 10), new Point(4, 3, 20))),
-            WellKnownText.fromWKT("multipoint (2 1 10, 3 4 20)"));
+            wkt.fromWKT("multipoint (2 1 10, 3 4 20)"));
 
-        assertEquals("multipoint EMPTY", WellKnownText.toWKT(MultiPoint.EMPTY));
-        assertEquals(MultiPoint.EMPTY, WellKnownText.fromWKT("multipoint EMPTY)"));
+        assertEquals("multipoint EMPTY", wkt.toWKT(MultiPoint.EMPTY));
+        assertEquals(MultiPoint.EMPTY, wkt.fromWKT("multipoint EMPTY)"));
     }
 }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/MultiPolygonTests.java b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/MultiPolygonTests.java
index 1b82cfd3871a5..9918dfa546c82 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/MultiPolygonTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/MultiPolygonTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.geo.geometry;
 
+import org.elasticsearch.geo.utils.GeographyValidator;
 import org.elasticsearch.geo.utils.WellKnownText;
 
 import java.io.IOException;
@@ -40,14 +41,15 @@ protected MultiPolygon createTestInstance(boolean hasAlt) {
     }
 
     public void testBasicSerialization() throws IOException, ParseException {
+        WellKnownText wkt = new WellKnownText(true, new GeographyValidator(true));
         assertEquals("multipolygon (((3.0 1.0, 4.0 2.0, 5.0 3.0, 3.0 1.0)))",
-            WellKnownText.toWKT(new MultiPolygon(Collections.singletonList(
+            wkt.toWKT(new MultiPolygon(Collections.singletonList(
                 new Polygon(new LinearRing(new double[]{1, 2, 3, 1}, new double[]{3, 4, 5, 3}))))));
         assertEquals(new MultiPolygon(Collections.singletonList(
             new Polygon(new LinearRing(new double[]{1, 2, 3, 1}, new double[]{3, 4, 5, 3})))),
-            WellKnownText.fromWKT("multipolygon (((3.0 1.0, 4.0 2.0, 5.0 3.0, 3.0 1.0)))"));
+            wkt.fromWKT("multipolygon (((3.0 1.0, 4.0 2.0, 5.0 3.0, 3.0 1.0)))"));
 
-        assertEquals("multipolygon EMPTY", WellKnownText.toWKT(MultiPolygon.EMPTY));
-        assertEquals(MultiPolygon.EMPTY, WellKnownText.fromWKT("multipolygon EMPTY)"));
+        assertEquals("multipolygon EMPTY", wkt.toWKT(MultiPolygon.EMPTY));
+        assertEquals(MultiPolygon.EMPTY, wkt.fromWKT("multipolygon EMPTY)"));
     }
 }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/PointTests.java b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/PointTests.java
index d480f44e30a44..82e8fc40e75e9 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/PointTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/PointTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.geo.geometry;
 
+import org.elasticsearch.geo.utils.GeographyValidator;
+import org.elasticsearch.geo.utils.GeometryValidator;
 import org.elasticsearch.geo.utils.WellKnownText;
 
 import java.io.IOException;
@@ -31,21 +33,29 @@ protected Point createTestInstance(boolean hasAlt) {
     }
 
     public void testBasicSerialization() throws IOException, ParseException {
-        assertEquals("point (20.0 10.0)", WellKnownText.toWKT(new Point(10, 20)));
-        assertEquals(new Point(10, 20), WellKnownText.fromWKT("point (20.0 10.0)"));
+        WellKnownText wkt = new WellKnownText(true, new GeographyValidator(true));
+        assertEquals("point (20.0 10.0)", wkt.toWKT(new Point(10, 20)));
+        assertEquals(new Point(10, 20), wkt.fromWKT("point (20.0 10.0)"));
 
-        assertEquals("point (20.0 10.0 100.0)", WellKnownText.toWKT(new Point(10, 20, 100)));
-        assertEquals(new Point(10, 20, 100), WellKnownText.fromWKT("point (20.0 10.0 100.0)"));
+        assertEquals("point (20.0 10.0 100.0)", wkt.toWKT(new Point(10, 20, 100)));
+        assertEquals(new Point(10, 20, 100), wkt.fromWKT("point (20.0 10.0 100.0)"));
 
-        assertEquals("point EMPTY", WellKnownText.toWKT(Point.EMPTY));
-        assertEquals(Point.EMPTY, WellKnownText.fromWKT("point EMPTY)"));
+        assertEquals("point EMPTY", wkt.toWKT(Point.EMPTY));
+        assertEquals(Point.EMPTY, wkt.fromWKT("point EMPTY)"));
     }
 
     public void testInitValidation() {
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> new Point(100, 10));
+        GeometryValidator validator = new GeographyValidator(true);
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> validator.validate(new Point(100, 10)));
         assertEquals("invalid latitude 100.0; must be between -90.0 and 90.0", ex.getMessage());
 
-        ex = expectThrows(IllegalArgumentException.class, () -> new Point(10, 500));
+        ex = expectThrows(IllegalArgumentException.class, () -> validator.validate(new Point(10, 500)));
         assertEquals("invalid longitude 500.0; must be between -180.0 and 180.0", ex.getMessage());
     }
+
+    public void testWKTValidation() {
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+            () -> new WellKnownText(randomBoolean(), new GeographyValidator(false)).fromWKT("point (20.0 10.0 100.0)"));
+        assertEquals("found Z value [100.0] but [ignore_z_value] parameter is [false]", ex.getMessage());
+    }
 }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/PolygonTests.java b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/PolygonTests.java
index fd3032435891c..adbe1f38cdcc0 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/PolygonTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/PolygonTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.geo.geometry;
 
+import org.elasticsearch.geo.utils.GeographyValidator;
 import org.elasticsearch.geo.utils.WellKnownText;
 
 import java.io.IOException;
@@ -32,18 +33,28 @@ protected Polygon createTestInstance(boolean hasAlt) {
     }
 
     public void testBasicSerialization() throws IOException, ParseException {
+        WellKnownText wkt = new WellKnownText(true, new GeographyValidator(true));
         assertEquals("polygon ((3.0 1.0, 4.0 2.0, 5.0 3.0, 3.0 1.0))",
-            WellKnownText.toWKT(new Polygon(new LinearRing(new double[]{1, 2, 3, 1}, new double[]{3, 4, 5, 3}))));
+            wkt.toWKT(new Polygon(new LinearRing(new double[]{1, 2, 3, 1}, new double[]{3, 4, 5, 3}))));
         assertEquals(new Polygon(new LinearRing(new double[]{1, 2, 3, 1}, new double[]{3, 4, 5, 3})),
-            WellKnownText.fromWKT("polygon ((3 1, 4 2, 5 3, 3 1))"));
+            wkt.fromWKT("polygon ((3 1, 4 2, 5 3, 3 1))"));
 
         assertEquals("polygon ((3.0 1.0 5.0, 4.0 2.0 4.0, 5.0 3.0 3.0, 3.0 1.0 5.0))",
-            WellKnownText.toWKT(new Polygon(new LinearRing(new double[]{1, 2, 3, 1}, new double[]{3, 4, 5, 3}, new double[]{5, 4, 3, 5}))));
+            wkt.toWKT(new Polygon(new LinearRing(new double[]{1, 2, 3, 1}, new double[]{3, 4, 5, 3}, new double[]{5, 4, 3, 5}))));
         assertEquals(new Polygon(new LinearRing(new double[]{1, 2, 3, 1}, new double[]{3, 4, 5, 3}, new double[]{5, 4, 3, 5})),
-            WellKnownText.fromWKT("polygon ((3 1 5, 4 2 4, 5 3 3, 3 1 5))"));
+            wkt.fromWKT("polygon ((3 1 5, 4 2 4, 5 3 3, 3 1 5))"));
 
-        assertEquals("polygon EMPTY", WellKnownText.toWKT(Polygon.EMPTY));
-        assertEquals(Polygon.EMPTY, WellKnownText.fromWKT("polygon EMPTY)"));
+        // Auto closing in coerce mode
+        assertEquals(new Polygon(new LinearRing(new double[]{1, 2, 3, 1}, new double[]{3, 4, 5, 3})),
+            wkt.fromWKT("polygon ((3 1, 4 2, 5 3))"));
+        assertEquals(new Polygon(new LinearRing(new double[]{1, 2, 3, 1}, new double[]{3, 4, 5, 3}, new double[]{5, 4, 3, 5})),
+            wkt.fromWKT("polygon ((3 1 5, 4 2 4, 5 3 3))"));
+        assertEquals(new Polygon(new LinearRing(new double[]{1, 2, 3, 1}, new double[]{3, 4, 5, 3}),
+            Collections.singletonList(new LinearRing(new double[]{1.5, 1.5, 1.0, 1.5}, new double[]{0.5, 2.5, 2.0, 0.5}))),
+            wkt.fromWKT("polygon ((3 1, 4 2, 5 3, 3 1), (0.5 1.5, 2.5 1.5, 2.0 1.0))"));
+
+        assertEquals("polygon EMPTY", wkt.toWKT(Polygon.EMPTY));
+        assertEquals(Polygon.EMPTY, wkt.fromWKT("polygon EMPTY)"));
     }
 
     public void testInitValidation() {
@@ -60,4 +71,21 @@ public void testInitValidation() {
                 Collections.singletonList(new LinearRing(new double[]{1, 2, 3, 1}, new double[]{3, 4, 5, 3}))));
         assertEquals("holes must have the same number of dimensions as the polygon", ex.getMessage());
     }
+
+    public void testWKTValidation() {
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+            () -> new WellKnownText(false, new GeographyValidator(true)).fromWKT("polygon ((3 1 5, 4 2 4, 5 3 3))"));
+        assertEquals("first and last points of the linear ring must be the same (it must close itself): " +
+            "lats[0]=1.0 lats[2]=3.0 lons[0]=3.0 lons[2]=5.0 alts[0]=5.0 alts[2]=3.0", ex.getMessage());
+
+        ex = expectThrows(IllegalArgumentException.class,
+            () -> new WellKnownText(randomBoolean(), new GeographyValidator(false)).fromWKT("polygon ((3 1 5, 4 2 4, 5 3 3, 3 1 5))"));
+        assertEquals("found Z value [5.0] but [ignore_z_value] parameter is [false]", ex.getMessage());
+
+        ex = expectThrows(IllegalArgumentException.class,
+            () -> new WellKnownText(false, new GeographyValidator(randomBoolean())).fromWKT(
+                "polygon ((3 1, 4 2, 5 3, 3 1), (0.5 1.5, 2.5 1.5, 2.0 1.0))"));
+        assertEquals("first and last points of the linear ring must be the same (it must close itself): " +
+            "lats[0]=1.5 lats[2]=1.0 lons[0]=0.5 lons[2]=2.0", ex.getMessage());
+    }
 }
diff --git a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/RectangleTests.java b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/RectangleTests.java
index 14cb777f94bb9..8bd1494eb34a9 100644
--- a/libs/geo/src/test/java/org/elasticsearch/geo/geometry/RectangleTests.java
+++ b/libs/geo/src/test/java/org/elasticsearch/geo/geometry/RectangleTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.geo.geometry;
 
+import org.elasticsearch.geo.utils.GeographyValidator;
+import org.elasticsearch.geo.utils.GeometryValidator;
 import org.elasticsearch.geo.utils.WellKnownText;
 
 import java.io.IOException;
@@ -32,24 +34,30 @@ protected Rectangle createTestInstance(boolean hasAlt) {
     }
 
     public void testBasicSerialization() throws IOException, ParseException {
-        assertEquals("bbox (10.0, 20.0, 40.0, 30.0)", WellKnownText.toWKT(new Rectangle(30, 40, 10, 20)));
-        assertEquals(new Rectangle(30, 40, 10, 20), WellKnownText.fromWKT("bbox (10.0, 20.0, 40.0, 30.0)"));
+        WellKnownText wkt = new WellKnownText(true, new GeographyValidator(true));
+        assertEquals("bbox (10.0, 20.0, 40.0, 30.0)", wkt.toWKT(new Rectangle(30, 40, 10, 20)));
+        assertEquals(new Rectangle(30, 40, 10, 20), wkt.fromWKT("bbox (10.0, 20.0, 40.0, 30.0)"));
 
-        assertEquals("bbox EMPTY", WellKnownText.toWKT(Rectangle.EMPTY));
-        assertEquals(Rectangle.EMPTY, WellKnownText.fromWKT("bbox EMPTY)"));
+        assertEquals("bbox EMPTY", wkt.toWKT(Rectangle.EMPTY));
+        assertEquals(Rectangle.EMPTY, wkt.fromWKT("bbox EMPTY)"));
     }
 
     public void testInitValidation() {
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> new Rectangle(100, 1, 2, 3));
+        GeometryValidator validator = new GeographyValidator(true);
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+            () -> validator.validate(new Rectangle(1, 100, 2, 3)));
         assertEquals("invalid latitude 100.0; must be between -90.0 and 90.0", ex.getMessage());
 
-        ex = expectThrows(IllegalArgumentException.class, () -> new Rectangle(1, 2, 200, 3));
+        ex = expectThrows(IllegalArgumentException.class,
+            () -> validator.validate(new Rectangle(1, 2, 200, 3)));
         assertEquals("invalid longitude 200.0; must be between -180.0 and 180.0", ex.getMessage());
 
-        ex = expectThrows(IllegalArgumentException.class, () -> new Rectangle(2, 1, 2, 3));
+        ex = expectThrows(IllegalArgumentException.class,
+            () -> validator.validate(new Rectangle(2, 1, 2, 3)));
         assertEquals("max lat cannot be less than min lat", ex.getMessage());
 
-        ex = expectThrows(IllegalArgumentException.class, () -> new Rectangle(1, 2, 2, 3, 5, Double.NaN));
+        ex = expectThrows(IllegalArgumentException.class,
+            () -> validator.validate(new Rectangle(1, 2, 2, 3, 5, Double.NaN)));
         assertEquals("only one altitude value is specified", ex.getMessage());
     }
 }
diff --git a/libs/grok/build.gradle b/libs/grok/build.gradle
index 9ca02df35aabe..b7ae54e54c3af 100644
--- a/libs/grok/build.gradle
+++ b/libs/grok/build.gradle
@@ -17,16 +17,14 @@
  * under the License.
  */
 
-archivesBaseName = 'elasticsearch-grok'
-
 dependencies {
     compile 'org.jruby.joni:joni:2.1.6'
     // joni dependencies:
-    compile 'org.jruby.jcodings:jcodings:1.0.12'
+    compile 'org.jruby.jcodings:jcodings:1.0.44'
 
-    if (isEclipse == false || project.path == ":libs:grok-tests") {
-        testCompile("org.elasticsearch.test:framework:${version}") {
-            exclude group: 'org.elasticsearch', module: 'grok'
+    if (isEclipse == false || project.path == ":libs:elasticsearch-grok-tests") {
+        testCompile(project(":test:framework")) {
+            exclude group: 'org.elasticsearch', module: 'elasticsearch-grok'
         }
     }
 }
@@ -38,7 +36,7 @@ forbiddenApisMain {
 if (isEclipse) {
     // in eclipse the project is under a fake root, we need to change around the source sets
     sourceSets {
-        if (project.path == ":libs:grok") {
+        if (project.path == ":libs:elasticsearch-grok") {
             main.java.srcDirs = ['java']
             main.resources.srcDirs = ['resources']
         } else {
diff --git a/libs/grok/licenses/jcodings-1.0.12.jar.sha1 b/libs/grok/licenses/jcodings-1.0.12.jar.sha1
deleted file mode 100644
index b097e32ece493..0000000000000
--- a/libs/grok/licenses/jcodings-1.0.12.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6bc17079fcaa8823ea8cd0d4c66516335b558db8
\ No newline at end of file
diff --git a/libs/grok/licenses/jcodings-1.0.44.jar.sha1 b/libs/grok/licenses/jcodings-1.0.44.jar.sha1
new file mode 100644
index 0000000000000..4449009d3395e
--- /dev/null
+++ b/libs/grok/licenses/jcodings-1.0.44.jar.sha1
@@ -0,0 +1 @@
+a6884b2fd8fd9a56874db05afaa22435043a2e3e
\ No newline at end of file
diff --git a/libs/grok/src/test/eclipse-build.gradle b/libs/grok/src/test/eclipse-build.gradle
index c5d791c166324..606aaffa121ab 100644
--- a/libs/grok/src/test/eclipse-build.gradle
+++ b/libs/grok/src/test/eclipse-build.gradle
@@ -3,5 +3,5 @@
 apply from: '../../build.gradle'
 
 dependencies {
-  testCompile project(':libs:grok')
+  testCompile project(':libs:elasticsearch-grok')
 }
diff --git a/libs/nio/build.gradle b/libs/nio/build.gradle
index 66436bb040e18..d6d0eaea0ab83 100644
--- a/libs/nio/build.gradle
+++ b/libs/nio/build.gradle
@@ -19,25 +19,15 @@
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
 
-archivesBaseName = 'elasticsearch-nio'
-
-publishing {
-  publications {
-    nebula {
-      artifactId = archivesBaseName
-    }
-  }
-}
-
 dependencies {
-  compile "org.elasticsearch:elasticsearch-core:${version}"
+  compile project(':libs:elasticsearch-core')
 
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest:${versions.hamcrest}"
 
-  if (isEclipse == false || project.path == ":libs:nio-tests") {
-      testCompile("org.elasticsearch.test:framework:${version}") {
+  if (isEclipse == false || project.path == ":libs:elasticsearch-nio-tests") {
+      testCompile(project(":test:framework")) {
         exclude group: 'org.elasticsearch', module: 'elasticsearch-nio'
       }
   }
@@ -46,7 +36,7 @@ dependencies {
 if (isEclipse) {
   // in eclipse the project is under a fake root, we need to change around the source sets
   sourceSets {
-    if (project.path == ":libs:nio") {
+    if (project.path == ":libs:elasticsearch-nio") {
       main.java.srcDirs = ['java']
       main.resources.srcDirs = ['resources']
     } else {
diff --git a/libs/nio/src/main/java/org/elasticsearch/nio/BytesChannelContext.java b/libs/nio/src/main/java/org/elasticsearch/nio/BytesChannelContext.java
index 8b174eac468e1..211e609ba4c0d 100644
--- a/libs/nio/src/main/java/org/elasticsearch/nio/BytesChannelContext.java
+++ b/libs/nio/src/main/java/org/elasticsearch/nio/BytesChannelContext.java
@@ -21,19 +21,12 @@
 
 import java.io.IOException;
 import java.util.function.Consumer;
-import java.util.function.Predicate;
 
 public class BytesChannelContext extends SocketChannelContext {
 
     public BytesChannelContext(NioSocketChannel channel, NioSelector selector, Consumer<Exception> exceptionHandler,
-                               ReadWriteHandler handler, InboundChannelBuffer channelBuffer) {
-        this(channel, selector, exceptionHandler, handler, channelBuffer, ALWAYS_ALLOW_CHANNEL);
-    }
-
-    public BytesChannelContext(NioSocketChannel channel, NioSelector selector, Consumer<Exception> exceptionHandler,
-                               ReadWriteHandler handler, InboundChannelBuffer channelBuffer,
-                               Predicate<NioSocketChannel> allowChannelPredicate) {
-        super(channel, selector, exceptionHandler, handler, channelBuffer, allowChannelPredicate);
+                               NioChannelHandler handler, InboundChannelBuffer channelBuffer) {
+        super(channel, selector, exceptionHandler, handler, channelBuffer);
     }
 
     @Override
diff --git a/libs/nio/src/main/java/org/elasticsearch/nio/BytesWriteHandler.java b/libs/nio/src/main/java/org/elasticsearch/nio/BytesWriteHandler.java
index 07333aa2eebc4..48d83d2169248 100644
--- a/libs/nio/src/main/java/org/elasticsearch/nio/BytesWriteHandler.java
+++ b/libs/nio/src/main/java/org/elasticsearch/nio/BytesWriteHandler.java
@@ -24,7 +24,7 @@
 import java.util.List;
 import java.util.function.BiConsumer;
 
-public abstract class BytesWriteHandler implements ReadWriteHandler {
+public abstract class BytesWriteHandler implements NioChannelHandler {
 
     private static final List<FlushOperation> EMPTY_LIST = Collections.emptyList();
 
@@ -48,6 +48,11 @@ public List<FlushOperation> pollFlushOperations() {
         return EMPTY_LIST;
     }
 
+    @Override
+    public boolean closeNow() {
+        return false;
+    }
+
     @Override
     public void close() {}
 }
diff --git a/libs/nio/src/main/java/org/elasticsearch/nio/ChannelFactory.java b/libs/nio/src/main/java/org/elasticsearch/nio/ChannelFactory.java
index 97750af2432c3..423168b35d09b 100644
--- a/libs/nio/src/main/java/org/elasticsearch/nio/ChannelFactory.java
+++ b/libs/nio/src/main/java/org/elasticsearch/nio/ChannelFactory.java
@@ -177,7 +177,9 @@ SocketChannel openNioChannel(InetSocketAddress remoteAddress) throws IOException
 
         SocketChannel acceptNioChannel(ServerChannelContext serverContext) throws IOException {
             ServerSocketChannel rawChannel = serverContext.getChannel().getRawChannel();
+            assert rawChannel.isBlocking() == false;
             SocketChannel socketChannel = accept(rawChannel);
+            assert rawChannel.isBlocking() == false;
             if (socketChannel == null) {
                 return null;
             }
diff --git a/libs/nio/src/main/java/org/elasticsearch/nio/DelegatingHandler.java b/libs/nio/src/main/java/org/elasticsearch/nio/DelegatingHandler.java
new file mode 100644
index 0000000000000..d928b0bf9349d
--- /dev/null
+++ b/libs/nio/src/main/java/org/elasticsearch/nio/DelegatingHandler.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.nio;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.function.BiConsumer;
+
+public abstract class DelegatingHandler implements NioChannelHandler {
+
+    private NioChannelHandler delegate;
+
+    public DelegatingHandler(NioChannelHandler delegate) {
+        this.delegate = delegate;
+    }
+
+    @Override
+    public void channelRegistered() {
+        this.delegate.channelRegistered();
+    }
+
+    @Override
+    public WriteOperation createWriteOperation(SocketChannelContext context, Object message, BiConsumer<Void, Exception> listener) {
+        return delegate.createWriteOperation(context, message, listener);
+    }
+
+    @Override
+    public List<FlushOperation> writeToBytes(WriteOperation writeOperation) {
+        return delegate.writeToBytes(writeOperation);
+    }
+
+    @Override
+    public List<FlushOperation> pollFlushOperations() {
+        return delegate.pollFlushOperations();
+    }
+
+    @Override
+    public int consumeReads(InboundChannelBuffer channelBuffer) throws IOException {
+        return delegate.consumeReads(channelBuffer);
+    }
+
+    @Override
+    public boolean closeNow() {
+        return delegate.closeNow();
+    }
+
+    @Override
+    public void close() throws IOException {
+        delegate.close();
+    }
+}
diff --git a/libs/nio/src/main/java/org/elasticsearch/nio/ReadWriteHandler.java b/libs/nio/src/main/java/org/elasticsearch/nio/NioChannelHandler.java
similarity index 90%
rename from libs/nio/src/main/java/org/elasticsearch/nio/ReadWriteHandler.java
rename to libs/nio/src/main/java/org/elasticsearch/nio/NioChannelHandler.java
index 92b276ad2d6da..61bda9a450768 100644
--- a/libs/nio/src/main/java/org/elasticsearch/nio/ReadWriteHandler.java
+++ b/libs/nio/src/main/java/org/elasticsearch/nio/NioChannelHandler.java
@@ -24,9 +24,9 @@
 import java.util.function.BiConsumer;
 
 /**
- * Implements the application specific logic for handling inbound and outbound messages for a channel.
+ * Implements the application specific logic for handling channel operations.
  */
-public interface ReadWriteHandler {
+public interface NioChannelHandler {
 
     /**
      * This method is called when the channel is registered with its selector.
@@ -72,5 +72,12 @@ public interface ReadWriteHandler {
      */
     int consumeReads(InboundChannelBuffer channelBuffer) throws IOException;
 
+    /**
+     * This method indicates if the underlying channel should be closed.
+     *
+     * @return if the channel should be closed
+     */
+    boolean closeNow();
+
     void close() throws IOException;
 }
diff --git a/libs/nio/src/main/java/org/elasticsearch/nio/NioSelector.java b/libs/nio/src/main/java/org/elasticsearch/nio/NioSelector.java
index 1d0af24ae2cba..fff07f625ff13 100644
--- a/libs/nio/src/main/java/org/elasticsearch/nio/NioSelector.java
+++ b/libs/nio/src/main/java/org/elasticsearch/nio/NioSelector.java
@@ -61,6 +61,7 @@ public class NioSelector implements Closeable {
     private final AtomicBoolean isClosed = new AtomicBoolean(false);
     private final CompletableFuture<Void> isRunningFuture = new CompletableFuture<>();
     private final AtomicReference<Thread> thread = new AtomicReference<>(null);
+    private final AtomicBoolean wokenUp = new AtomicBoolean(false);
 
     public NioSelector(EventHandler eventHandler) throws IOException {
         this(eventHandler, Selector.open());
@@ -153,7 +154,7 @@ void singleLoop() {
             preSelect();
             long nanosUntilNextTask = taskScheduler.nanosUntilNextTask(System.nanoTime());
             int ready;
-            if (nanosUntilNextTask == 0) {
+            if (wokenUp.getAndSet(false) || nanosUntilNextTask == 0) {
                 ready = selector.selectNow();
             } else {
                 long millisUntilNextTask = TimeUnit.NANOSECONDS.toMillis(nanosUntilNextTask);
@@ -221,13 +222,10 @@ void processKey(SelectionKey selectionKey) {
         if (selectionKey.isAcceptable()) {
             assert context instanceof ServerChannelContext : "Only server channels can receive accept events";
             ServerChannelContext serverChannelContext = (ServerChannelContext) context;
-            int ops = selectionKey.readyOps();
-            if ((ops & SelectionKey.OP_ACCEPT) != 0) {
-                try {
-                    eventHandler.acceptChannel(serverChannelContext);
-                } catch (IOException e) {
-                    eventHandler.acceptException(serverChannelContext, e);
-                }
+            try {
+                eventHandler.acceptChannel(serverChannelContext);
+            } catch (IOException e) {
+                eventHandler.acceptException(serverChannelContext, e);
             }
         } else {
             assert context instanceof SocketChannelContext : "Only sockets channels can receive non-accept events";
@@ -279,29 +277,36 @@ private void handleTask(Runnable task) {
 
     /**
      * Queues a write operation to be handled by the event loop. This can be called by any thread and is the
-     * api available for non-selector threads to schedule writes.
+     * api available for non-selector threads to schedule writes. When invoked from the selector thread the write will be executed
+     * right away.
      *
      * @param writeOperation to be queued
      */
     public void queueWrite(WriteOperation writeOperation) {
-        queuedWrites.offer(writeOperation);
-        if (isOpen() == false) {
-            boolean wasRemoved = queuedWrites.remove(writeOperation);
-            if (wasRemoved) {
-                writeOperation.getListener().accept(null, new ClosedSelectorException());
-            }
+        if (isOnCurrentThread()) {
+            writeToChannel(writeOperation);
         } else {
-            wakeup();
+            queuedWrites.offer(writeOperation);
+            if (isOpen() == false) {
+                boolean wasRemoved = queuedWrites.remove(writeOperation);
+                if (wasRemoved) {
+                    writeOperation.getListener().accept(null, new ClosedSelectorException());
+                }
+            } else {
+                wakeup();
+            }
         }
     }
 
     public void queueChannelClose(NioChannel channel) {
         ChannelContext<?> context = channel.getContext();
         assert context.getSelector() == this : "Must schedule a channel for closure with its selector";
-        channelsToClose.offer(context);
         if (isOnCurrentThread() == false) {
+            channelsToClose.offer(context);
             ensureSelectorOpenForEnqueuing(channelsToClose, context);
             wakeup();
+        } else {
+            closeChannel(context);
         }
     }
 
@@ -313,9 +318,13 @@ public void queueChannelClose(NioChannel channel) {
      */
     public void scheduleForRegistration(NioChannel channel) {
         ChannelContext<?> context = channel.getContext();
-        channelsToRegister.add(context);
-        ensureSelectorOpenForEnqueuing(channelsToRegister, context);
-        wakeup();
+        if (isOnCurrentThread() == false) {
+            channelsToRegister.add(context);
+            ensureSelectorOpenForEnqueuing(channelsToRegister, context);
+            wakeup();
+        } else {
+            registerChannel(context);
+        }
     }
 
     /**
@@ -326,7 +335,7 @@ public void scheduleForRegistration(NioChannel channel) {
      *
      * @param writeOperation to be queued in a channel's buffer
      */
-    public void writeToChannel(WriteOperation writeOperation) {
+    private void writeToChannel(WriteOperation writeOperation) {
         assertOnSelectorThread();
         SocketChannelContext context = writeOperation.getChannel();
         // If the channel does not currently have anything that is ready to flush, we should flush after
@@ -380,8 +389,10 @@ private void cleanupPendingWrites() {
     }
 
     private void wakeup() {
-        // TODO: Do we need the wakeup optimizations that some other libraries use?
-        selector.wakeup();
+        assert isOnCurrentThread() == false;
+        if (wokenUp.compareAndSet(false, true)) {
+            selector.wakeup();
+        }
     }
 
     private void handleWrite(SocketChannelContext context) {
@@ -414,30 +425,38 @@ private void attemptConnect(SocketChannelContext context, boolean connectEvent)
     private void setUpNewChannels() {
         ChannelContext<?> newChannel;
         while ((newChannel = this.channelsToRegister.poll()) != null) {
-            assert newChannel.getSelector() == this : "The channel must be registered with the selector with which it was created";
-            try {
-                if (newChannel.isOpen()) {
-                    eventHandler.handleRegistration(newChannel);
-                    if (newChannel instanceof SocketChannelContext) {
-                        attemptConnect((SocketChannelContext) newChannel, false);
-                    }
-                } else {
-                    eventHandler.registrationException(newChannel, new ClosedChannelException());
+            registerChannel(newChannel);
+        }
+    }
+
+    private void registerChannel(ChannelContext<?> newChannel) {
+        assert newChannel.getSelector() == this : "The channel must be registered with the selector with which it was created";
+        try {
+            if (newChannel.isOpen()) {
+                eventHandler.handleRegistration(newChannel);
+                if (newChannel instanceof SocketChannelContext) {
+                    attemptConnect((SocketChannelContext) newChannel, false);
                 }
-            } catch (Exception e) {
-                eventHandler.registrationException(newChannel, e);
+            } else {
+                eventHandler.registrationException(newChannel, new ClosedChannelException());
             }
+        } catch (Exception e) {
+            eventHandler.registrationException(newChannel, e);
         }
     }
 
     private void closePendingChannels() {
         ChannelContext<?> channelContext;
         while ((channelContext = channelsToClose.poll()) != null) {
-            try {
-                eventHandler.handleClose(channelContext);
-            } catch (Exception e) {
-                eventHandler.closeException(channelContext, e);
-            }
+            closeChannel(channelContext);
+        }
+    }
+
+    private void closeChannel(final ChannelContext<?> channelContext) {
+        try {
+            eventHandler.handleClose(channelContext);
+        } catch (Exception e) {
+            eventHandler.closeException(channelContext, e);
         }
     }
 
@@ -470,7 +489,7 @@ private void handleQueuedWrites() {
      * @param <O> the object type
      */
     private <O> void ensureSelectorOpenForEnqueuing(ConcurrentLinkedQueue<O> queue, O objectAdded) {
-        if (isOpen() == false && isOnCurrentThread() == false) {
+        if (isOpen() == false) {
             if (queue.remove(objectAdded)) {
                 throw new IllegalStateException("selector is already closed");
             }
diff --git a/libs/nio/src/main/java/org/elasticsearch/nio/SocketChannelContext.java b/libs/nio/src/main/java/org/elasticsearch/nio/SocketChannelContext.java
index 22d85472126c1..f77ccb17aef39 100644
--- a/libs/nio/src/main/java/org/elasticsearch/nio/SocketChannelContext.java
+++ b/libs/nio/src/main/java/org/elasticsearch/nio/SocketChannelContext.java
@@ -32,7 +32,6 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
-import java.util.function.Predicate;
 
 /**
  * This context should implement the specific logic for a channel. When a channel receives a notification
@@ -45,13 +44,10 @@
  */
 public abstract class SocketChannelContext extends ChannelContext<SocketChannel> {
 
-    protected static final Predicate<NioSocketChannel> ALWAYS_ALLOW_CHANNEL = (c) -> true;
-
     protected final NioSocketChannel channel;
     protected final InboundChannelBuffer channelBuffer;
     protected final AtomicBoolean isClosing = new AtomicBoolean(false);
-    private final ReadWriteHandler readWriteHandler;
-    private final Predicate<NioSocketChannel> allowChannelPredicate;
+    private final NioChannelHandler readWriteHandler;
     private final NioSelector selector;
     private final CompletableContext<Void> connectContext = new CompletableContext<>();
     private final LinkedList<FlushOperation> pendingFlushes = new LinkedList<>();
@@ -59,14 +55,12 @@ public abstract class SocketChannelContext extends ChannelContext<SocketChannel>
     private Exception connectException;
 
     protected SocketChannelContext(NioSocketChannel channel, NioSelector selector, Consumer<Exception> exceptionHandler,
-                                   ReadWriteHandler readWriteHandler, InboundChannelBuffer channelBuffer,
-                                   Predicate<NioSocketChannel> allowChannelPredicate) {
+                                   NioChannelHandler readWriteHandler, InboundChannelBuffer channelBuffer) {
         super(channel.getRawChannel(), exceptionHandler);
         this.selector = selector;
         this.channel = channel;
         this.readWriteHandler = readWriteHandler;
         this.channelBuffer = channelBuffer;
-        this.allowChannelPredicate = allowChannelPredicate;
     }
 
     @Override
@@ -135,13 +129,7 @@ public void sendMessage(Object message, BiConsumer<Void, Exception> listener) {
 
         WriteOperation writeOperation = readWriteHandler.createWriteOperation(this, message, listener);
 
-        NioSelector selector = getSelector();
-        if (selector.isOnCurrentThread() == false) {
-            selector.queueWrite(writeOperation);
-            return;
-        }
-
-        selector.writeToChannel(writeOperation);
+        getSelector().queueWrite(writeOperation);
     }
 
     public void queueWriteOperation(WriteOperation writeOperation) {
@@ -171,9 +159,6 @@ protected FlushOperation getPendingFlush() {
     protected void register() throws IOException {
         super.register();
         readWriteHandler.channelRegistered();
-        if (allowChannelPredicate.test(channel) == false) {
-            closeNow = true;
-        }
     }
 
     @Override
@@ -233,7 +218,7 @@ public boolean readyForFlush() {
     public abstract boolean selectorShouldClose();
 
     protected boolean closeNow() {
-        return closeNow;
+        return closeNow || readWriteHandler.closeNow();
     }
 
     protected void setCloseNow() {
@@ -278,7 +263,7 @@ protected int readFromChannel(InboundChannelBuffer channelBuffer) throws IOExcep
 
     // Currently we limit to 64KB. This is a trade-off which means more syscalls, in exchange for less
     // copying.
-    private final int WRITE_LIMIT = 1 << 16;
+    private static final int WRITE_LIMIT = 1 << 16;
 
     protected int flushToChannel(FlushOperation flushOperation) throws IOException {
         ByteBuffer ioBuffer = getSelector().getIoBuffer();
diff --git a/libs/nio/src/main/java/org/elasticsearch/nio/WriteOperation.java b/libs/nio/src/main/java/org/elasticsearch/nio/WriteOperation.java
index 3d17519be7e1f..b5f60bd28a1d6 100644
--- a/libs/nio/src/main/java/org/elasticsearch/nio/WriteOperation.java
+++ b/libs/nio/src/main/java/org/elasticsearch/nio/WriteOperation.java
@@ -23,7 +23,7 @@
 /**
  * This is a basic write operation that can be queued with a channel. The only requirements of a write
  * operation is that is has a listener and a reference to its channel. The actual conversion of the write
- * operation implementation to bytes will be performed by the {@link ReadWriteHandler}.
+ * operation implementation to bytes will be performed by the {@link NioChannelHandler}.
  */
 public interface WriteOperation {
 
diff --git a/libs/nio/src/test/eclipse-build.gradle b/libs/nio/src/test/eclipse-build.gradle
index e30e76b0da59f..f7e152311ad5d 100644
--- a/libs/nio/src/test/eclipse-build.gradle
+++ b/libs/nio/src/test/eclipse-build.gradle
@@ -3,5 +3,5 @@
 apply from: '../../build.gradle'
 
 dependencies {
-  testCompile project(':libs:nio')
+  testCompile project(':libs:elasticsearch-nio')
 }
diff --git a/libs/nio/src/test/java/org/elasticsearch/nio/EventHandlerTests.java b/libs/nio/src/test/java/org/elasticsearch/nio/EventHandlerTests.java
index 1b42e8be60d7c..578890b152ff1 100644
--- a/libs/nio/src/test/java/org/elasticsearch/nio/EventHandlerTests.java
+++ b/libs/nio/src/test/java/org/elasticsearch/nio/EventHandlerTests.java
@@ -44,7 +44,7 @@ public class EventHandlerTests extends ESTestCase {
     private Consumer<Exception> channelExceptionHandler;
     private Consumer<Exception> genericExceptionHandler;
 
-    private ReadWriteHandler readWriteHandler;
+    private NioChannelHandler readWriteHandler;
     private EventHandler handler;
     private DoNotRegisterSocketContext context;
     private DoNotRegisterServerContext serverContext;
@@ -56,7 +56,7 @@ public class EventHandlerTests extends ESTestCase {
     public void setUpHandler() throws IOException {
         channelExceptionHandler = mock(Consumer.class);
         genericExceptionHandler = mock(Consumer.class);
-        readWriteHandler = mock(ReadWriteHandler.class);
+        readWriteHandler = mock(NioChannelHandler.class);
         channelFactory = mock(ChannelFactory.class);
         NioSelector selector = mock(NioSelector.class);
         ArrayList<NioSelector> selectors = new ArrayList<>();
@@ -260,7 +260,7 @@ private class DoNotRegisterSocketContext extends BytesChannelContext {
 
 
         DoNotRegisterSocketContext(NioSocketChannel channel, NioSelector selector, Consumer<Exception> exceptionHandler,
-                                   ReadWriteHandler handler) {
+                                   NioChannelHandler handler) {
             super(channel, selector, exceptionHandler, handler, InboundChannelBuffer.allocatingInstance());
         }
 
diff --git a/libs/nio/src/test/java/org/elasticsearch/nio/NioSelectorTests.java b/libs/nio/src/test/java/org/elasticsearch/nio/NioSelectorTests.java
index 7a641315fe285..55d2e645cadee 100644
--- a/libs/nio/src/test/java/org/elasticsearch/nio/NioSelectorTests.java
+++ b/libs/nio/src/test/java/org/elasticsearch/nio/NioSelectorTests.java
@@ -19,7 +19,9 @@
 
 package org.elasticsearch.nio;
 
+import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 import org.mockito.ArgumentCaptor;
@@ -108,14 +110,14 @@ public void testQueueChannelForClosed() throws IOException {
     }
 
     @SuppressWarnings({"unchecked", "rawtypes"})
-    public void testCloseException() throws IOException {
+    public void testCloseException() throws IOException, InterruptedException {
         IOException ioException = new IOException();
         NioChannel channel = mock(NioChannel.class);
         ChannelContext context = mock(ChannelContext.class);
         when(channel.getContext()).thenReturn(context);
         when(context.getSelector()).thenReturn(selector);
 
-        selector.queueChannelClose(channel);
+        executeOnNewThread(() -> selector.queueChannelClose(channel));
 
         doThrow(ioException).when(eventHandler).handleClose(context);
 
@@ -198,9 +200,10 @@ public void testIOExceptionWhileSelect() throws IOException {
         verify(eventHandler).selectorException(ioException);
     }
 
-    public void testSelectorClosedIfOpenAndEventLoopNotRunning() throws IOException {
+    public void testSelectorClosedIfOpenAndEventLoopNotRunning() throws Exception {
         when(rawSelector.isOpen()).thenReturn(true);
-        selector.close();
+        executeOnNewThread(() -> selector.close());
+
         verify(rawSelector).close();
     }
 
@@ -222,8 +225,7 @@ public void testClosedServerChannelWillNotBeRegistered() {
     }
 
     public void testRegisterServerChannelFailsDueToException() throws Exception {
-        selector.scheduleForRegistration(serverChannel);
-
+        executeOnNewThread(() -> selector.scheduleForRegistration(serverChannel));
         ClosedChannelException closedChannelException = new ClosedChannelException();
         doThrow(closedChannelException).when(eventHandler).handleRegistration(serverChannelContext);
 
@@ -242,16 +244,18 @@ public void testClosedSocketChannelWillNotBeRegistered() throws Exception {
         verify(eventHandler, times(0)).handleConnect(channelContext);
     }
 
-    public void testRegisterSocketChannelFailsDueToException() throws Exception {
-        selector.scheduleForRegistration(channel);
+    public void testRegisterSocketChannelFailsDueToException() throws InterruptedException {
+        executeOnNewThread(() -> {
+            selector.scheduleForRegistration(channel);
 
-        ClosedChannelException closedChannelException = new ClosedChannelException();
-        doThrow(closedChannelException).when(eventHandler).handleRegistration(channelContext);
+            ClosedChannelException closedChannelException = new ClosedChannelException();
+            doThrow(closedChannelException).when(eventHandler).handleRegistration(channelContext);
 
-        selector.preSelect();
+            selector.preSelect();
 
-        verify(eventHandler).registrationException(channelContext, closedChannelException);
-        verify(eventHandler, times(0)).handleConnect(channelContext);
+            verify(eventHandler).registrationException(channelContext, closedChannelException);
+            verify(eventHandler, times(0)).handleConnect(channelContext);
+        });
     }
 
     public void testAcceptEvent() throws IOException {
@@ -292,17 +296,17 @@ public void testSuccessfullyRegisterChannelWillAttemptConnect() throws Exception
     }
 
     public void testQueueWriteWhenNotRunning() throws Exception {
-        selector.close();
-
-        selector.queueWrite(new FlushReadyWrite(channelContext, buffers, listener));
-
+        executeOnNewThread(() -> {
+            selector.close();
+            selector.queueWrite(new FlushReadyWrite(channelContext, buffers, listener));
+        });
         verify(listener).accept(isNull(Void.class), any(ClosedSelectorException.class));
     }
 
     public void testQueueWriteChannelIsClosed() throws Exception {
         WriteOperation writeOperation = new FlushReadyWrite(channelContext, buffers, listener);
-        selector.queueWrite(writeOperation);
 
+        executeOnNewThread(() -> selector.queueWrite(writeOperation));
         when(channelContext.isOpen()).thenReturn(false);
         selector.preSelect();
 
@@ -315,7 +319,7 @@ public void testQueueWriteSelectionKeyThrowsException() throws Exception {
 
         WriteOperation writeOperation = new FlushReadyWrite(channelContext, buffers, listener);
         CancelledKeyException cancelledKeyException = new CancelledKeyException();
-        selector.queueWrite(writeOperation);
+        executeOnNewThread(() -> selector.queueWrite(writeOperation));
 
         when(channelContext.getSelectionKey()).thenReturn(selectionKey);
         when(selectionKey.interestOps(anyInt())).thenThrow(cancelledKeyException);
@@ -327,7 +331,7 @@ public void testQueueWriteSelectionKeyThrowsException() throws Exception {
 
     public void testQueueWriteSuccessful() throws Exception {
         WriteOperation writeOperation = new FlushReadyWrite(channelContext, buffers, listener);
-        selector.queueWrite(writeOperation);
+        executeOnNewThread(() -> selector.queueWrite(writeOperation));
 
         assertTrue((selectionKey.interestOps() & SelectionKey.OP_WRITE) == 0);
 
@@ -343,7 +347,7 @@ public void testQueueDirectlyInChannelBufferSuccessful() throws Exception {
         assertEquals(0, (selectionKey.interestOps() & SelectionKey.OP_WRITE));
 
         when(channelContext.readyForFlush()).thenReturn(true);
-        selector.writeToChannel(writeOperation);
+        selector.queueWrite(writeOperation);
 
         verify(channelContext).queueWriteOperation(writeOperation);
         verify(eventHandler, times(0)).handleWrite(channelContext);
@@ -357,7 +361,7 @@ public void testShouldFlushIfNoPendingFlushes() throws Exception {
         assertEquals(0, (selectionKey.interestOps() & SelectionKey.OP_WRITE));
 
         when(channelContext.readyForFlush()).thenReturn(false);
-        selector.writeToChannel(writeOperation);
+        selector.queueWrite(writeOperation);
 
         verify(channelContext).queueWriteOperation(writeOperation);
         verify(eventHandler).handleWrite(channelContext);
@@ -374,7 +378,7 @@ public void testQueueDirectlyInChannelBufferSelectionKeyThrowsException() throws
         when(channelContext.getSelectionKey()).thenReturn(selectionKey);
         when(channelContext.readyForFlush()).thenReturn(false);
         when(selectionKey.interestOps(anyInt())).thenThrow(cancelledKeyException);
-        selector.writeToChannel(writeOperation);
+        selector.queueWrite(writeOperation);
 
         verify(channelContext, times(0)).queueWriteOperation(writeOperation);
         verify(eventHandler, times(0)).handleWrite(channelContext);
@@ -477,14 +481,17 @@ public void testWillCallPostHandleAfterChannelHandling() throws Exception {
     public void testCleanup() throws Exception {
         NioSocketChannel unregisteredChannel = mock(NioSocketChannel.class);
         SocketChannelContext unregisteredContext = mock(SocketChannelContext.class);
+        when(unregisteredContext.getSelector()).thenReturn(selector);
         when(unregisteredChannel.getContext()).thenReturn(unregisteredContext);
 
-        selector.scheduleForRegistration(channel);
+        executeOnNewThread(() -> selector.scheduleForRegistration(channel));
 
         selector.preSelect();
 
-        selector.queueWrite(new FlushReadyWrite(channelContext, buffers, listener));
-        selector.scheduleForRegistration(unregisteredChannel);
+        executeOnNewThread(() -> {
+            selector.queueWrite(new FlushReadyWrite(channelContext, buffers, listener));
+            selector.scheduleForRegistration(unregisteredChannel);
+        });
 
         TestSelectionKey testSelectionKey = new TestSelectionKey(0);
         testSelectionKey.attach(channelContext);
@@ -496,4 +503,20 @@ public void testCleanup() throws Exception {
         verify(eventHandler).handleClose(channelContext);
         verify(eventHandler).handleClose(unregisteredContext);
     }
+
+    private static void executeOnNewThread(CheckedRunnable<Exception> runnable) throws InterruptedException {
+        final Thread thread = new Thread(new AbstractRunnable() {
+            @Override
+            protected void doRun() throws Exception {
+                runnable.run();
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new AssertionError(e);
+            }
+        });
+        thread.start();
+        thread.join();
+    }
 }
diff --git a/libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java b/libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java
index 0040f70df85ae..5563ccc43063b 100644
--- a/libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java
+++ b/libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java
@@ -35,7 +35,6 @@
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.IntFunction;
-import java.util.function.Predicate;
 
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyInt;
@@ -54,7 +53,7 @@ public class SocketChannelContextTests extends ESTestCase {
     private NioSocketChannel channel;
     private BiConsumer<Void, Exception> listener;
     private NioSelector selector;
-    private ReadWriteHandler readWriteHandler;
+    private NioChannelHandler readWriteHandler;
     private ByteBuffer ioBuffer = ByteBuffer.allocate(1024);
 
     @SuppressWarnings("unchecked")
@@ -68,7 +67,7 @@ public void setup() throws Exception {
         when(channel.getRawChannel()).thenReturn(rawChannel);
         exceptionHandler = mock(Consumer.class);
         selector = mock(NioSelector.class);
-        readWriteHandler = mock(ReadWriteHandler.class);
+        readWriteHandler = mock(NioChannelHandler.class);
         InboundChannelBuffer channelBuffer = InboundChannelBuffer.allocatingInstance();
         context = new TestSocketChannelContext(channel, selector, exceptionHandler, readWriteHandler, channelBuffer);
 
@@ -102,22 +101,6 @@ public void testSignalWhenPeerClosed() throws IOException {
         assertTrue(context.closeNow());
     }
 
-    public void testValidateInRegisterCanSucceed() throws IOException {
-        InboundChannelBuffer channelBuffer = InboundChannelBuffer.allocatingInstance();
-        context = new TestSocketChannelContext(channel, selector, exceptionHandler, readWriteHandler, channelBuffer, (c) -> true);
-        assertFalse(context.closeNow());
-        context.register();
-        assertFalse(context.closeNow());
-    }
-
-    public void testValidateInRegisterCanFail() throws IOException {
-        InboundChannelBuffer channelBuffer = InboundChannelBuffer.allocatingInstance();
-        context = new TestSocketChannelContext(channel, selector, exceptionHandler, readWriteHandler, channelBuffer, (c) -> false);
-        assertFalse(context.closeNow());
-        context.register();
-        assertTrue(context.closeNow());
-    }
-
     public void testConnectSucceeds() throws IOException {
         AtomicBoolean listenerCalled = new AtomicBoolean(false);
         when(rawChannel.finishConnect()).thenReturn(false, true);
@@ -192,7 +175,7 @@ public void testSendMessageFromSameThreadIsQueuedInChannel() {
         when(readWriteHandler.createWriteOperation(context, buffers, listener)).thenReturn(writeOperation);
         context.sendMessage(buffers, listener);
 
-        verify(selector).writeToChannel(writeOpCaptor.capture());
+        verify(selector).queueWrite(writeOpCaptor.capture());
         WriteOperation writeOp = writeOpCaptor.getValue();
 
         assertSame(writeOperation, writeOp);
@@ -394,14 +377,8 @@ public void testFlushBuffersHandlesIOExceptionSecondTimeThroughLoop() throws IOE
     private static class TestSocketChannelContext extends SocketChannelContext {
 
         private TestSocketChannelContext(NioSocketChannel channel, NioSelector selector, Consumer<Exception> exceptionHandler,
-                                         ReadWriteHandler readWriteHandler, InboundChannelBuffer channelBuffer) {
-            this(channel, selector, exceptionHandler, readWriteHandler, channelBuffer, ALWAYS_ALLOW_CHANNEL);
-        }
-
-        private TestSocketChannelContext(NioSocketChannel channel, NioSelector selector, Consumer<Exception> exceptionHandler,
-                                         ReadWriteHandler readWriteHandler, InboundChannelBuffer channelBuffer,
-                                         Predicate<NioSocketChannel> allowChannelPredicate) {
-            super(channel, selector, exceptionHandler, readWriteHandler, channelBuffer, allowChannelPredicate);
+                                         NioChannelHandler readWriteHandler, InboundChannelBuffer channelBuffer) {
+            super(channel, selector, exceptionHandler, readWriteHandler, channelBuffer);
         }
 
         @Override
diff --git a/libs/secure-sm/build.gradle b/libs/secure-sm/build.gradle
index bbd44afc70a6d..3e79d9ee2e84b 100644
--- a/libs/secure-sm/build.gradle
+++ b/libs/secure-sm/build.gradle
@@ -19,16 +19,6 @@
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
 
-archivesBaseName = 'elasticsearch-secure-sm'
-
-publishing {
-    publications {
-        nebula {
-            artifactId = archivesBaseName
-        }
-    }
-}
-
 dependencies {
     // do not add non-test compile dependencies to secure-sm without a good reason to do so
 
@@ -36,9 +26,9 @@ dependencies {
     testCompile "junit:junit:${versions.junit}"
     testCompile "org.hamcrest:hamcrest:${versions.hamcrest}"
 
-    if (isEclipse == false || project.path == ":libs:secure-sm-tests") {
-        testCompile("org.elasticsearch.test:framework:${version}") {
-            exclude group: 'org.elasticsearch', module: 'secure-sm'
+    if (isEclipse == false || project.path == ":libs:elasticsearch-secure-sm-tests") {
+        testCompile(project(":test:framework")) {
+            exclude group: 'org.elasticsearch', module: 'elasticsearch-secure-sm'
         }
     }
 }
@@ -50,7 +40,7 @@ forbiddenApisMain {
 if (isEclipse) {
     // in Eclipse the project is under a fake root so we need to change around the source sets
     sourceSets {
-        if (project.path == ":libs:secure-sm") {
+        if (project.path == ":libs:elasticsearch-secure-sm") {
             main.java.srcDirs = ['java']
             main.resources.srcDirs = ['resources']
         } else {
diff --git a/libs/secure-sm/src/test/eclipse-build.gradle b/libs/secure-sm/src/test/eclipse-build.gradle
index 56dcdcbac27fa..63d610b75c1db 100644
--- a/libs/secure-sm/src/test/eclipse-build.gradle
+++ b/libs/secure-sm/src/test/eclipse-build.gradle
@@ -3,5 +3,5 @@
 apply from: '../../build.gradle'
 
 dependencies {
-  testCompile project(':libs:secure-sm')
+  testCompile project(':libs:elasticsearch-secure-sm')
 }
diff --git a/libs/ssl-config/build.gradle b/libs/ssl-config/build.gradle
index 860cdcd9e6fc8..71ebd642a4339 100644
--- a/libs/ssl-config/build.gradle
+++ b/libs/ssl-config/build.gradle
@@ -19,10 +19,10 @@
 apply plugin: "nebula.maven-scm"
 
 dependencies {
-    compile "org.elasticsearch:elasticsearch-core:${version}"
+    compile project(':libs:elasticsearch-core')
 
-    if (isEclipse == false || project.path == ":libs:ssl-config-tests") {
-        testCompile("org.elasticsearch.test:framework:${version}") {
+    if (isEclipse == false || project.path == ":libs:elasticsearch-ssl-config-tests") {
+        testCompile(project(":test:framework")) {
             exclude group: 'org.elasticsearch', module: 'elasticsearch-ssl-config'
         }
     }
@@ -35,7 +35,7 @@ dependencies {
 if (isEclipse) {
   // in eclipse the project is under a fake root, we need to change around the source sets
   sourceSets {
-    if (project.path == ":libs:ssl-config") {
+    if (project.path == ":libs:elasticsearch-ssl-config") {
       main.java.srcDirs = ['java']
       main.resources.srcDirs = ['resources']
     } else {
diff --git a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/PemTrustConfigTests.java b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/PemTrustConfigTests.java
index 44a39c2e183d6..8d87c1599352b 100644
--- a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/PemTrustConfigTests.java
+++ b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/PemTrustConfigTests.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
-import javax.net.ssl.X509ExtendedTrustManager;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
@@ -37,6 +36,8 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import javax.net.ssl.X509ExtendedTrustManager;
+
 public class PemTrustConfigTests extends ESTestCase {
 
     public void testBuildTrustConfigFromSinglePemFile() throws Exception {
@@ -57,7 +58,7 @@ public void testBuildTrustConfigFromMultiplePemFiles() throws Exception {
 
     public void testBadFileFormatFails() throws Exception {
         final Path ca = createTempFile("ca", ".crt");
-        Files.write(ca, randomByteArrayOfLength(128), StandardOpenOption.APPEND);
+        Files.write(ca, generateRandomByteArrayOfLength(128), StandardOpenOption.APPEND);
         final PemTrustConfig trustConfig = new PemTrustConfig(Collections.singletonList(ca));
         assertThat(trustConfig.getDependentFiles(), Matchers.containsInAnyOrder(ca));
         assertInvalidFileFormat(trustConfig, ca);
@@ -106,7 +107,7 @@ public void testTrustConfigReloadsFileContents() throws Exception {
         Files.delete(ca1);
         assertFileNotFound(trustConfig, ca1);
 
-        Files.write(ca1, randomByteArrayOfLength(128), StandardOpenOption.CREATE);
+        Files.write(ca1, generateRandomByteArrayOfLength(128), StandardOpenOption.CREATE);
         assertInvalidFileFormat(trustConfig, ca1);
     }
 
@@ -148,4 +149,24 @@ private void assertFileNotFound(PemTrustConfig trustConfig, Path file) {
         assertThat(exception.getMessage(), Matchers.containsString(file.toAbsolutePath().toString()));
         assertThat(exception.getCause(), Matchers.instanceOf(NoSuchFileException.class));
     }
+
+    private byte[] generateRandomByteArrayOfLength(int length) {
+        byte[] bytes = randomByteArrayOfLength(length);
+        /*
+         * If the bytes represent DER encoded value indicating ASN.1 SEQUENCE followed by length byte if it is zero then while trying to
+         * parse PKCS7 block from the encoded stream, it failed parsing the content type. The DerInputStream.getSequence() method in this
+         * case returns an empty DerValue array but ContentType does not check the length of array before accessing the array resulting in a
+         * ArrayIndexOutOfBoundsException. This check ensures that when we create random stream of bytes we do not create ASN.1 SEQUENCE
+         * followed by zero length which fails the test intermittently.
+         */
+        while(checkRandomGeneratedBytesRepresentZeroLengthDerSequenceCausingArrayIndexOutOfBound(bytes)) {
+            bytes = randomByteArrayOfLength(length);
+        }
+        return bytes;
+    }
+
+    private static boolean checkRandomGeneratedBytesRepresentZeroLengthDerSequenceCausingArrayIndexOutOfBound(byte[] bytes) {
+        // Tag value indicating an ASN.1 "SEQUENCE". Reference: sun.security.util.DerValue.tag_Sequence = 0x30
+        return bytes[0] == 0x30 && bytes[1] == 0x00;
+    }
 }
diff --git a/libs/x-content/build.gradle b/libs/x-content/build.gradle
index 0e99d80da1e00..e54427c058304 100644
--- a/libs/x-content/build.gradle
+++ b/libs/x-content/build.gradle
@@ -21,18 +21,8 @@ apply plugin: 'elasticsearch.build'
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
 
-archivesBaseName = 'elasticsearch-x-content'
-
-publishing {
-    publications {
-        nebula {
-            artifactId = archivesBaseName
-        }
-    }
-}
-
 dependencies {
-    compile "org.elasticsearch:elasticsearch-core:${version}"
+    compile project(':libs:elasticsearch-core')
 
     compile "org.yaml:snakeyaml:${versions.snakeyaml}"
     compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
@@ -44,8 +34,8 @@ dependencies {
     testCompile "junit:junit:${versions.junit}"
     testCompile "org.hamcrest:hamcrest:${versions.hamcrest}"
 
-    if (isEclipse == false || project.path == ":libs:x-content-tests") {
-        testCompile("org.elasticsearch.test:framework:${version}") {
+    if (isEclipse == false || project.path == ":libs:elasticsearch-x-content-tests") {
+        testCompile(project(":test:framework")) {
             exclude group: 'org.elasticsearch', module: 'elasticsearch-x-content'
         }
     }
@@ -61,7 +51,7 @@ forbiddenApisMain {
 if (isEclipse) {
     // in eclipse the project is under a fake root, we need to change around the source sets
     sourceSets {
-        if (project.path == ":libs:x-content") {
+        if (project.path == ":libs:elasticsearch-x-content") {
             main.java.srcDirs = ['java']
             main.resources.srcDirs = ['resources']
         } else {
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
index ee5e3347f8d99..c80c5bdb0d09a 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
@@ -78,14 +78,63 @@ public static <Value, ElementValue> BiConsumer<Value, List<ElementValue>> fromLi
         };
     }
 
+    private interface UnknownFieldParser<Value, Context> {
+
+        void acceptUnknownField(String parserName, String field, XContentLocation location, XContentParser parser,
+                                Value value, Context context) throws IOException;
+    }
+
+    private static <Value, Context> UnknownFieldParser<Value, Context> ignoreUnknown() {
+      return (n, f, l, p, v, c) -> p.skipChildren();
+    }
+
+    private static <Value, Context> UnknownFieldParser<Value, Context> errorOnUnknown() {
+        return (n, f, l, p, v, c) -> {
+            throw new XContentParseException(l, "[" + n + "] unknown field [" + f + "], parser not found");
+        };
+    }
+
+    /**
+     * Defines how to consume a parsed undefined field
+     */
+    public interface UnknownFieldConsumer<Value> {
+        void accept(Value target, String field, Object value);
+    }
+
+    private static <Value, Context> UnknownFieldParser<Value, Context> consumeUnknownField(UnknownFieldConsumer<Value> consumer) {
+        return (parserName, field, location, parser, value, context) -> {
+            XContentParser.Token t = parser.currentToken();
+            switch (t) {
+                case VALUE_STRING:
+                    consumer.accept(value, field, parser.text());
+                    break;
+                case VALUE_NUMBER:
+                    consumer.accept(value, field, parser.numberValue());
+                    break;
+                case VALUE_BOOLEAN:
+                    consumer.accept(value, field, parser.booleanValue());
+                    break;
+                case VALUE_NULL:
+                    consumer.accept(value, field, null);
+                    break;
+                case START_OBJECT:
+                    consumer.accept(value, field, parser.map());
+                    break;
+                case START_ARRAY:
+                    consumer.accept(value, field, parser.list());
+                    break;
+                default:
+                    throw new XContentParseException(parser.getTokenLocation(),
+                        "[" + parserName + "] cannot parse field [" + field + "] with value type [" + t + "]");
+            }
+        };
+    }
+
     private final Map<String, FieldParser> fieldParserMap = new HashMap<>();
     private final String name;
     private final Supplier<Value> valueSupplier;
-    /**
-     * Should this parser ignore unknown fields? This should generally be set to true only when parsing responses from external systems,
-     * never when parsing requests from users.
-     */
-    private final boolean ignoreUnknownFields;
+
+    private final UnknownFieldParser<Value, Context> unknownFieldParser;
 
     /**
      * Creates a new ObjectParser instance with a name. This name is used to reference the parser in exceptions and messages.
@@ -95,25 +144,45 @@ public ObjectParser(String name) {
     }
 
     /**
-     * Creates a new ObjectParser instance which a name.
+     * Creates a new ObjectParser instance with a name.
      * @param name the parsers name, used to reference the parser in exceptions and messages.
      * @param valueSupplier a supplier that creates a new Value instance used when the parser is used as an inner object parser.
      */
     public ObjectParser(String name, @Nullable Supplier<Value> valueSupplier) {
-        this(name, false, valueSupplier);
+        this(name, errorOnUnknown(), valueSupplier);
     }
 
     /**
-     * Creates a new ObjectParser instance which a name.
+     * Creates a new ObjectParser instance with a name.
      * @param name the parsers name, used to reference the parser in exceptions and messages.
      * @param ignoreUnknownFields Should this parser ignore unknown fields? This should generally be set to true only when parsing
      *      responses from external systems, never when parsing requests from users.
      * @param valueSupplier a supplier that creates a new Value instance used when the parser is used as an inner object parser.
      */
     public ObjectParser(String name, boolean ignoreUnknownFields, @Nullable Supplier<Value> valueSupplier) {
+        this(name, ignoreUnknownFields ? ignoreUnknown() : errorOnUnknown(), valueSupplier);
+    }
+
+    /**
+     * Creates a new ObjectParser instance with a name.
+     * @param name the parsers name, used to reference the parser in exceptions and messages.
+     * @param unknownFieldConsumer how to consume parsed unknown fields
+     * @param valueSupplier a supplier that creates a new Value instance used when the parser is used as an inner object parser.
+     */
+    public ObjectParser(String name, UnknownFieldConsumer<Value> unknownFieldConsumer, @Nullable Supplier<Value> valueSupplier) {
+        this(name, consumeUnknownField(unknownFieldConsumer), valueSupplier);
+    }
+
+    /**
+     * Creates a new ObjectParser instance with a name.
+     * @param name the parsers name, used to reference the parser in exceptions and messages.
+     * @param unknownFieldParser how to parse unknown fields
+     * @param valueSupplier a supplier that creates a new Value instance used when the parser is used as an inner object parser.
+     */
+    private ObjectParser(String name, UnknownFieldParser<Value, Context> unknownFieldParser, @Nullable Supplier<Value> valueSupplier) {
         this.name = name;
         this.valueSupplier = valueSupplier;
-        this.ignoreUnknownFields = ignoreUnknownFields;
+        this.unknownFieldParser = unknownFieldParser;
     }
 
     /**
@@ -152,17 +221,18 @@ public Value parse(XContentParser parser, Value value, Context context) throws I
 
         FieldParser fieldParser = null;
         String currentFieldName = null;
+        XContentLocation currentPosition = null;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
-                fieldParser = getParser(currentFieldName, parser);
+                currentPosition = parser.getTokenLocation();
+                fieldParser = fieldParserMap.get(currentFieldName);
             } else {
                 if (currentFieldName == null) {
                     throw new XContentParseException(parser.getTokenLocation(), "[" + name  + "] no field found");
                 }
                 if (fieldParser == null) {
-                    assert ignoreUnknownFields : "this should only be possible if configured to ignore known fields";
-                    parser.skipChildren(); // noop if parser points to a value, skips children if parser is start object or start array
+                    unknownFieldParser.acceptUnknownField(name, currentFieldName, currentPosition, parser, value, context);
                 } else {
                     fieldParser.assertSupports(name, parser, currentFieldName);
                     parseSub(parser, fieldParser, currentFieldName, value, context);
@@ -363,15 +433,6 @@ private void parseSub(XContentParser parser, FieldParser fieldParser, String cur
         }
     }
 
-    private FieldParser getParser(String fieldName, XContentParser xContentParser) {
-        FieldParser parser = fieldParserMap.get(fieldName);
-        if (parser == null && false == ignoreUnknownFields) {
-            throw new XContentParseException(xContentParser.getTokenLocation(),
-                    "[" + name + "] unknown field [" + fieldName + "], parser not found");
-        }
-        return parser;
-    }
-
     private class FieldParser {
         private final Parser<Value, Context> parser;
         private final EnumSet<XContentParser.Token> supportedTokens;
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java
index 6d4da08bfaa59..82a663bd9dc5d 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java
@@ -117,7 +117,7 @@ public boolean isValue() {
     }
 
     enum NumberType {
-        INT, LONG, FLOAT, DOUBLE
+        INT, BIG_INTEGER, LONG, FLOAT, DOUBLE, BIG_DECIMAL
     }
 
     XContentType contentType();
@@ -136,11 +136,9 @@ enum NumberType {
 
     Map<String, String> mapStrings() throws IOException;
 
-    Map<String, String> mapStringsOrdered() throws IOException;
-
     /**
      * Returns an instance of {@link Map} holding parsed map.
-     * Serves as a replacement for the "map", "mapOrdered", "mapStrings" and "mapStringsOrdered" methods above.
+     * Serves as a replacement for the "map", "mapOrdered" and "mapStrings" methods above.
      *
      * @param mapFactory factory for creating new {@link Map} objects
      * @param mapValueParser parser for parsing a single map value
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java
index 252bfea7ca9c0..9a8686001e2dc 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java
@@ -108,11 +108,6 @@ public Map<String, String> mapStrings() throws IOException {
         return parser.mapStrings();
     }
 
-    @Override
-    public Map<String, String> mapStringsOrdered() throws IOException {
-        return parser.mapStringsOrdered();
-    }
-
     @Override
     public <T> Map<String, T> map(
             Supplier<Map<String, T>> mapFactory, CheckedFunction<XContentParser, T, IOException> mapValueParser) throws IOException {
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java
index be0d01763b893..7489222df2e76 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java
@@ -199,12 +199,16 @@ private NumberType convertNumberType(JsonParser.NumberType numberType) {
         switch (numberType) {
             case INT:
                 return NumberType.INT;
+            case BIG_INTEGER:
+                return NumberType.BIG_INTEGER;
             case LONG:
                 return NumberType.LONG;
             case FLOAT:
                 return NumberType.FLOAT;
             case DOUBLE:
                 return NumberType.DOUBLE;
+            case BIG_DECIMAL:
+                return NumberType.BIG_DECIMAL;
         }
         throw new IllegalStateException("No matching token for number_type [" + numberType + "]");
     }
diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
index 68e03e34a1a17..264af205e488b 100644
--- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
+++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
@@ -276,11 +276,6 @@ public Map<String, String> mapStrings() throws IOException {
         return readMapStrings(this);
     }
 
-    @Override
-    public Map<String, String> mapStringsOrdered() throws IOException {
-        return readOrderedMapStrings(this);
-    }
-
     @Override
     public <T> Map<String, T> map(
             Supplier<Map<String, T>> mapFactory, CheckedFunction<XContentParser, T, IOException> mapValueParser) throws IOException {
@@ -303,8 +298,6 @@ public List<Object> listOrderedMap() throws IOException {
 
     static final Supplier<Map<String, String>> SIMPLE_MAP_STRINGS_FACTORY = HashMap::new;
 
-    static final Supplier<Map<String, String>> ORDERED_MAP_STRINGS_FACTORY = LinkedHashMap::new;
-
     static Map<String, Object> readMap(XContentParser parser) throws IOException {
         return readMap(parser, SIMPLE_MAP_FACTORY);
     }
@@ -314,11 +307,7 @@ static Map<String, Object> readOrderedMap(XContentParser parser) throws IOExcept
     }
 
     static Map<String, String> readMapStrings(XContentParser parser) throws IOException {
-        return readMapStrings(parser, SIMPLE_MAP_STRINGS_FACTORY);
-    }
-
-    static Map<String, String> readOrderedMapStrings(XContentParser parser) throws IOException {
-        return readMapStrings(parser, ORDERED_MAP_STRINGS_FACTORY);
+        return readGenericMap(parser, SIMPLE_MAP_STRINGS_FACTORY, XContentParser::text);
     }
 
     static List<Object> readList(XContentParser parser) throws IOException {
@@ -333,10 +322,6 @@ static Map<String, Object> readMap(XContentParser parser, Supplier<Map<String, O
         return readGenericMap(parser, mapFactory, p -> readValue(p, mapFactory));
     }
 
-    static Map<String, String> readMapStrings(XContentParser parser, Supplier<Map<String, String>> mapFactory) throws IOException {
-        return readGenericMap(parser, mapFactory, XContentParser::text);
-    }
-
     static <T> Map<String, T> readGenericMap(
             XContentParser parser,
             Supplier<Map<String, T>> mapFactory,
@@ -382,7 +367,7 @@ static List<Object> readList(XContentParser parser, Supplier<Map<String, Object>
         return list;
     }
 
-    static Object readValue(XContentParser parser, Supplier<Map<String, Object>> mapFactory) throws IOException {
+    public static Object readValue(XContentParser parser, Supplier<Map<String, Object>> mapFactory) throws IOException {
         switch (parser.currentToken()) {
             case VALUE_STRING: return parser.text();
             case VALUE_NUMBER: return parser.numberValue();
diff --git a/libs/x-content/src/test/eclipse-build.gradle b/libs/x-content/src/test/eclipse-build.gradle
index f456f71a4c310..a8d81f9f95831 100644
--- a/libs/x-content/src/test/eclipse-build.gradle
+++ b/libs/x-content/src/test/eclipse-build.gradle
@@ -3,5 +3,5 @@
 apply from: '../../build.gradle'
 
 dependencies {
-  testCompile project(':libs:x-content')
+  testCompile project(':libs:elasticsearch-x-content')
 }
diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
index e089b8a956ac8..6002c6bd35076 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
@@ -33,7 +33,9 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.hamcrest.Matchers.containsString;
@@ -733,4 +735,41 @@ public void setFoo(int foo) {
             this.foo = foo;
         }
     }
+
+    private static class ObjectWithArbitraryFields {
+        String name;
+        Map<String, Object> fields = new HashMap<>();
+        void setField(String key, Object value) {
+            fields.put(key, value);
+        }
+        void setName(String name) {
+            this.name = name;
+        }
+    }
+
+    public void testConsumeUnknownFields() throws IOException {
+        XContentParser parser = createParser(JsonXContent.jsonXContent,
+            "{\n"
+                + "  \"test\" : \"foo\",\n"
+                + "  \"test_number\" : 2,\n"
+                + "  \"name\" : \"geoff\",\n"
+                + "  \"test_boolean\" : true,\n"
+                + "  \"test_null\" : null,\n"
+                + "  \"test_array\":  [1,2,3,4],\n"
+                + "  \"test_nested\": { \"field\" : \"value\", \"field2\" : [ \"list1\", \"list2\" ] }\n"
+                + "}");
+        ObjectParser<ObjectWithArbitraryFields, Void> op
+            = new ObjectParser<>("unknown", ObjectWithArbitraryFields::setField, ObjectWithArbitraryFields::new);
+        op.declareString(ObjectWithArbitraryFields::setName, new ParseField("name"));
+
+        ObjectWithArbitraryFields o = op.parse(parser, null);
+        assertEquals("geoff", o.name);
+        assertEquals(6, o.fields.size());
+        assertEquals("foo", o.fields.get("test"));
+        assertEquals(2, o.fields.get("test_number"));
+        assertEquals(true, o.fields.get("test_boolean"));
+        assertNull(o.fields.get("test_null"));
+        assertEquals(List.of(1, 2, 3, 4), o.fields.get("test_array"));
+        assertEquals(Map.of("field", "value", "field2", List.of("list1", "list2")), o.fields.get("test_nested"));
+    }
 }
diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
index c519880224ccb..31a00c4025ab2 100644
--- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
+++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
@@ -189,7 +189,7 @@ private Map<String, String> readMapStrings(String source) throws IOException {
             assertThat(parser.currentName(), equalTo("foo"));
             token = parser.nextToken();
             assertThat(token, equalTo(XContentParser.Token.START_OBJECT));
-            return randomBoolean() ? parser.mapStringsOrdered() : parser.mapStrings();
+            return parser.mapStrings();
         }
     }
 
diff --git a/modules/aggs-matrix-stats/build.gradle b/modules/aggs-matrix-stats/build.gradle
index d3af124b8af71..b3060fa178621 100644
--- a/modules/aggs-matrix-stats/build.gradle
+++ b/modules/aggs-matrix-stats/build.gradle
@@ -20,5 +20,4 @@
 esplugin {
     description 'Adds aggregations whose input are a list of numeric fields and output includes a matrix.'
     classname 'org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin'
-    hasClientJar = true
 }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
index 7dcdff426711b..c70ddea7e3b22 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
@@ -256,12 +256,16 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(stats, results);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), stats, results);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalMatrixStats other = (InternalMatrixStats) obj;
         return Objects.equals(this.stats, other.stats) &&
             Objects.equals(this.results, other.results);
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
index 75f24eb78037a..90f51236252ce 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
@@ -81,7 +81,7 @@ public MultiValueMode multiValueMode() {
 
     @Override
     protected MatrixStatsAggregatorFactory innerBuild(SearchContext context, Map<String, ValuesSourceConfig<Numeric>> configs,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
+            AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
         return new MatrixStatsAggregatorFactory(name, configs, multiValueMode, context, parent, subFactoriesBuilder, metaData);
     }
 
@@ -91,16 +91,6 @@ public XContentBuilder doXContentBody(XContentBuilder builder, ToXContent.Params
         return builder;
     }
 
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(Object obj) {
-        return true;
-    }
-
     @Override
     public String getType() {
         return NAME;
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
index fb456d75bb78b..469cd0dad8fcb 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
@@ -32,14 +32,13 @@
 import java.util.List;
 import java.util.Map;
 
-final class MatrixStatsAggregatorFactory
-    extends ArrayValuesSourceAggregatorFactory<ValuesSource.Numeric, MatrixStatsAggregatorFactory> {
+final class MatrixStatsAggregatorFactory extends ArrayValuesSourceAggregatorFactory<ValuesSource.Numeric> {
 
     private final MultiValueMode multiValueMode;
 
     MatrixStatsAggregatorFactory(String name,
             Map<String, ValuesSourceConfig<ValuesSource.Numeric>> configs, MultiValueMode multiValueMode,
-            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+            SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
         super(name, configs, context, parent, subFactoriesBuilder, metaData);
         this.multiValueMode = multiValueMode;
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregationBuilder.java
index eb8152e0fe0b8..a3d7fc5d64290 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregationBuilder.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregationBuilder.java
@@ -239,10 +239,10 @@ public Map<String, Object> missingMap() {
     }
 
     @Override
-    protected final ArrayValuesSourceAggregatorFactory<VS, ?> doBuild(SearchContext context, AggregatorFactory<?> parent,
+    protected final ArrayValuesSourceAggregatorFactory<VS> doBuild(SearchContext context, AggregatorFactory parent,
                                                                       AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
         Map<String, ValuesSourceConfig<VS>> configs = resolveConfig(context);
-        ArrayValuesSourceAggregatorFactory<VS, ?> factory = innerBuild(context, configs, parent, subFactoriesBuilder);
+        ArrayValuesSourceAggregatorFactory<VS> factory = innerBuild(context, configs, parent, subFactoriesBuilder);
         return factory;
     }
 
@@ -255,9 +255,9 @@ protected Map<String, ValuesSourceConfig<VS>> resolveConfig(SearchContext contex
         return configs;
     }
 
-    protected abstract ArrayValuesSourceAggregatorFactory<VS, ?> innerBuild(SearchContext context,
+    protected abstract ArrayValuesSourceAggregatorFactory<VS> innerBuild(SearchContext context,
                                                                 Map<String, ValuesSourceConfig<VS>> configs,
-                                                                AggregatorFactory<?> parent,
+                                                                AggregatorFactory parent,
                                                                 AggregatorFactories.Builder subFactoriesBuilder) throws IOException;
 
     public ValuesSourceConfig<VS> config(SearchContext context, String field, Script script) {
@@ -354,30 +354,21 @@ public final XContentBuilder internalXContent(XContentBuilder builder, Params pa
     protected abstract XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException;
 
     @Override
-    protected final int doHashCode() {
-        return Objects.hash(fields, format, missing, targetValueType, valueType, valuesSourceType,
-            innerHashCode());
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), fields, format, missing, targetValueType, valueType, valuesSourceType);
     }
 
-    protected abstract int innerHashCode();
-
     @Override
-    protected final boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         ArrayValuesSourceAggregationBuilder<?, ?> other = (ArrayValuesSourceAggregationBuilder<?, ?>) obj;
-        if (!Objects.equals(fields, other.fields))
-            return false;
-        if (!Objects.equals(format, other.format))
-            return false;
-        if (!Objects.equals(missing, other.missing))
-            return false;
-        if (!Objects.equals(targetValueType, other.targetValueType))
-            return false;
-        if (!Objects.equals(valueType, other.valueType))
-            return false;
-        if (!Objects.equals(valuesSourceType, other.valuesSourceType))
-            return false;
-        return innerEquals(obj);
+        return Objects.equals(fields, other.fields)
+            && Objects.equals(format, other.format)
+            && Objects.equals(missing, other.missing)
+            && Objects.equals(targetValueType, other.targetValueType)
+            && Objects.equals(valueType, other.valueType)
+            && Objects.equals(valuesSourceType, other.valuesSourceType);
     }
-
-    protected abstract boolean innerEquals(Object obj);
 }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregatorFactory.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregatorFactory.java
index ce8eeecd19036..754cb9576c7dd 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregatorFactory.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/ArrayValuesSourceAggregatorFactory.java
@@ -30,13 +30,13 @@
 import java.util.List;
 import java.util.Map;
 
-public abstract class ArrayValuesSourceAggregatorFactory<VS extends ValuesSource, AF extends ArrayValuesSourceAggregatorFactory<VS, AF>>
-    extends AggregatorFactory<AF> {
+public abstract class ArrayValuesSourceAggregatorFactory<VS extends ValuesSource>
+    extends AggregatorFactory {
 
     protected Map<String, ValuesSourceConfig<VS>> configs;
 
     public ArrayValuesSourceAggregatorFactory(String name, Map<String, ValuesSourceConfig<VS>> configs,
-                                              SearchContext context, AggregatorFactory<?> parent,
+                                              SearchContext context, AggregatorFactory parent,
                                               AggregatorFactories.Builder subFactoriesBuilder,
                                               Map<String, Object> metaData) throws IOException {
         super(name, context, parent, subFactoriesBuilder, metaData);
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
index f095b766ee1d5..a655f42a36c26 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
@@ -83,6 +83,7 @@
 import org.apache.lucene.analysis.miscellaneous.TruncateTokenFilter;
 import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter;
 import org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter;
+import org.apache.lucene.analysis.miscellaneous.WordDelimiterIterator;
 import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
 import org.apache.lucene.analysis.ngram.EdgeNGramTokenizer;
 import org.apache.lucene.analysis.ngram.NGramTokenFilter;
@@ -110,6 +111,7 @@
 import org.apache.lucene.analysis.tr.TurkishAnalyzer;
 import org.apache.lucene.analysis.util.ElisionFilter;
 import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.Version;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -237,7 +239,7 @@ public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
         filters.put("dutch_stem", DutchStemTokenFilterFactory::new);
         filters.put("edge_ngram", EdgeNGramTokenFilterFactory::new);
         filters.put("edgeNGram", EdgeNGramTokenFilterFactory::new);
-        filters.put("elision", ElisionTokenFilterFactory::new);
+        filters.put("elision", requiresAnalysisSettings(ElisionTokenFilterFactory::new));
         filters.put("fingerprint", FingerprintTokenFilterFactory::new);
         filters.put("flatten_graph", FlattenGraphTokenFilterFactory::new);
         filters.put("french_stem", FrenchStemTokenFilterFactory::new);
@@ -320,9 +322,6 @@ public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
     @Override
     public List<PreBuiltAnalyzerProviderFactory> getPreBuiltAnalyzerProviderFactories() {
         List<PreBuiltAnalyzerProviderFactory> analyzers = new ArrayList<>();
-        // TODO remove in 8.0
-        analyzers.add(new PreBuiltAnalyzerProviderFactory("standard_html_strip", CachingStrategy.ELASTICSEARCH,
-            () -> new StandardHtmlStripAnalyzer(CharArraySet.EMPTY_SET)));
         analyzers.add(new PreBuiltAnalyzerProviderFactory("pattern", CachingStrategy.ELASTICSEARCH,
             () -> new PatternAnalyzer(Regex.compile("\\W+" /*PatternAnalyzer.NON_WORD_PATTERN*/, null), true,
             CharArraySet.EMPTY_SET)));
@@ -393,10 +392,6 @@ public List<PreConfiguredTokenFilter> getPreConfiguredTokenFilters() {
                 input -> new CommonGramsFilter(input, CharArraySet.EMPTY_SET)));
         filters.add(PreConfiguredTokenFilter.singleton("czech_stem", false, CzechStemFilter::new));
         filters.add(PreConfiguredTokenFilter.singleton("decimal_digit", true, DecimalDigitFilter::new));
-        filters.add(PreConfiguredTokenFilter.singleton("delimited_payload_filter", false, input ->
-                new DelimitedPayloadTokenFilter(input,
-                        DelimitedPayloadTokenFilterFactory.DEFAULT_DELIMITER,
-                        DelimitedPayloadTokenFilterFactory.DEFAULT_ENCODER)));
         filters.add(PreConfiguredTokenFilter.singleton("delimited_payload", false, input ->
                 new DelimitedPayloadTokenFilter(input,
                         DelimitedPayloadTokenFilterFactory.DEFAULT_DELIMITER,
@@ -455,13 +450,15 @@ public List<PreConfiguredTokenFilter> getPreConfiguredTokenFilters() {
                       | WordDelimiterFilter.SPLIT_ON_CASE_CHANGE
                       | WordDelimiterFilter.SPLIT_ON_NUMERICS
                       | WordDelimiterFilter.STEM_ENGLISH_POSSESSIVE, null)));
-        filters.add(PreConfiguredTokenFilter.singleton("word_delimiter_graph", false, false, input ->
-                new WordDelimiterGraphFilter(input,
+        filters.add(PreConfiguredTokenFilter.singletonWithVersion("word_delimiter_graph", false, false, (input, version) -> {
+            boolean adjustOffsets = version.onOrAfter(Version.V_7_3_0);
+            return new WordDelimiterGraphFilter(input, adjustOffsets, WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE,
                         WordDelimiterGraphFilter.GENERATE_WORD_PARTS
                       | WordDelimiterGraphFilter.GENERATE_NUMBER_PARTS
                       | WordDelimiterGraphFilter.SPLIT_ON_CASE_CHANGE
                       | WordDelimiterGraphFilter.SPLIT_ON_NUMERICS
-                      | WordDelimiterGraphFilter.STEM_ENGLISH_POSSESSIVE, null)));
+                      | WordDelimiterGraphFilter.STEM_ENGLISH_POSSESSIVE, null);
+        }));
         return filters;
     }
 
@@ -475,8 +472,12 @@ public List<PreConfiguredTokenizer> getPreConfiguredTokenizers() {
         tokenizers.add(PreConfiguredTokenizer.singleton("letter", LetterTokenizer::new));
         tokenizers.add(PreConfiguredTokenizer.singleton("whitespace", WhitespaceTokenizer::new));
         tokenizers.add(PreConfiguredTokenizer.singleton("ngram", NGramTokenizer::new));
-        tokenizers.add(PreConfiguredTokenizer.singleton("edge_ngram",
-            () -> new EdgeNGramTokenizer(EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE)));
+        tokenizers.add(PreConfiguredTokenizer.elasticsearchVersion("edge_ngram", (version) -> {
+            if (version.onOrAfter(Version.V_7_3_0)) {
+                return new EdgeNGramTokenizer(NGramTokenizer.DEFAULT_MIN_NGRAM_SIZE, NGramTokenizer.DEFAULT_MAX_NGRAM_SIZE);
+            }
+            return new EdgeNGramTokenizer(EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE);
+        }));
         tokenizers.add(PreConfiguredTokenizer.singleton("pattern", () -> new PatternTokenizer(Regex.compile("\\W+", null), -1)));
         tokenizers.add(PreConfiguredTokenizer.singleton("thai", ThaiTokenizer::new));
         // TODO deprecate and remove in API
@@ -485,8 +486,12 @@ public List<PreConfiguredTokenizer> getPreConfiguredTokenizers() {
 
         // Temporary shim for aliases. TODO deprecate after they are moved
         tokenizers.add(PreConfiguredTokenizer.singleton("nGram", NGramTokenizer::new));
-        tokenizers.add(PreConfiguredTokenizer.singleton("edgeNGram",
-            () -> new EdgeNGramTokenizer(EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE)));
+        tokenizers.add(PreConfiguredTokenizer.elasticsearchVersion("edgeNGram", (version) -> {
+            if (version.onOrAfter(Version.V_7_3_0)) {
+                return new EdgeNGramTokenizer(NGramTokenizer.DEFAULT_MIN_NGRAM_SIZE, NGramTokenizer.DEFAULT_MAX_NGRAM_SIZE);
+            }
+            return new EdgeNGramTokenizer(EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE);
+        }));
         tokenizers.add(PreConfiguredTokenizer.singleton("PathHierarchy", PathHierarchyTokenizer::new));
 
         return tokenizers;
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ElisionTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ElisionTokenFilterFactory.java
index 52cb69952b836..39d042caa8c25 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ElisionTokenFilterFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ElisionTokenFilterFactory.java
@@ -36,6 +36,9 @@ public class ElisionTokenFilterFactory extends AbstractTokenFilterFactory implem
     ElisionTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
         this.articles = Analysis.parseArticles(env, settings);
+        if (this.articles == null) {
+            throw new IllegalArgumentException("elision filter requires [articles] or [articles_path] setting");
+        }
     }
 
     @Override
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StandardHtmlStripAnalyzer.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StandardHtmlStripAnalyzer.java
deleted file mode 100644
index a35a0ea2a4a0b..0000000000000
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/StandardHtmlStripAnalyzer.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.analysis.common;
-
-import org.apache.lucene.analysis.CharArraySet;
-import org.apache.lucene.analysis.LowerCaseFilter;
-import org.apache.lucene.analysis.StopFilter;
-import org.apache.lucene.analysis.StopwordAnalyzerBase;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.en.EnglishAnalyzer;
-import org.apache.lucene.analysis.standard.StandardTokenizer;
-
-public class StandardHtmlStripAnalyzer extends StopwordAnalyzerBase {
-
-    /**
-     * @deprecated use {@link StandardHtmlStripAnalyzer#StandardHtmlStripAnalyzer(CharArraySet)} instead
-     */
-    @Deprecated
-    public StandardHtmlStripAnalyzer() {
-        super(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET);
-    }
-    /**
-     * @deprecated in 6.5, can not create in 7.0, and we remove this in 8.0
-     */
-    @Deprecated
-    StandardHtmlStripAnalyzer(CharArraySet stopwords) {
-        super(stopwords);
-    }
-
-    @Override
-    protected TokenStreamComponents createComponents(final String fieldName) {
-        final Tokenizer src = new StandardTokenizer();
-        TokenStream tok = new LowerCaseFilter(src);
-        if (!stopwords.isEmpty()) {
-            tok = new StopFilter(tok, stopwords);
-        }
-        return new TokenStreamComponents(src, tok);
-    }
-
-}
diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java
index 75d4eca4254f8..5d6135549b882 100644
--- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java
+++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
 import org.elasticsearch.index.analysis.Analysis;
+import org.elasticsearch.index.analysis.AnalysisMode;
 import org.elasticsearch.index.analysis.CharFilterFactory;
 import org.elasticsearch.index.analysis.CustomAnalyzer;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
@@ -50,6 +51,7 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
     private final boolean lenient;
     protected final Settings settings;
     protected final Environment environment;
+    private final boolean updateable;
 
     SynonymTokenFilterFactory(IndexSettings indexSettings, Environment env,
                                       String name, Settings settings) {
@@ -65,9 +67,15 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
         this.expand = settings.getAsBoolean("expand", true);
         this.lenient = settings.getAsBoolean("lenient", false);
         this.format = settings.get("format", "");
+        this.updateable = settings.getAsBoolean("updateable", false);
         this.environment = env;
     }
 
+    @Override
+    public AnalysisMode getAnalysisMode() {
+        return this.updateable ? AnalysisMode.SEARCH_TIME : AnalysisMode.ALL;
+    }
+
     @Override
     public TokenStream create(TokenStream tokenStream) {
         throw new IllegalStateException("Call createPerAnalyzerSynonymFactory to specialize this factory for an analysis chain first");
@@ -98,6 +106,11 @@ public TokenFilterFactory getSynonymFilter() {
                 // which doesn't support stacked input tokens
                 return IDENTITY_FILTER;
             }
+
+            @Override
+            public AnalysisMode getAnalysisMode() {
+                return updateable ? AnalysisMode.SEARCH_TIME : AnalysisMode.ALL;
+            }
         };
     }
 
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java
new file mode 100644
index 0000000000000..0172f7cbc2657
--- /dev/null
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.analysis.common;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.test.ESTokenStreamTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.test.VersionUtils;
+
+import java.io.IOException;
+import java.util.Collections;
+
+public class EdgeNGramTokenizerTests extends ESTokenStreamTestCase {
+
+    private IndexAnalyzers buildAnalyzers(Version version, String tokenizer) throws IOException {
+        Settings settings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+        Settings indexSettings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, version)
+            .put("index.analysis.analyzer.my_analyzer.tokenizer", tokenizer)
+            .build();
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
+        return new AnalysisModule(TestEnvironment.newEnvironment(settings),
+            Collections.singletonList(new CommonAnalysisPlugin())).getAnalysisRegistry().build(idxSettings);
+    }
+
+    public void testPreConfiguredTokenizer() throws IOException {
+
+        // Before 7.3 we return ngrams of length 1 only
+        {
+            Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_0_0,
+                VersionUtils.getPreviousVersion(Version.V_7_3_0));
+            try (IndexAnalyzers indexAnalyzers = buildAnalyzers(version, "edge_ngram")) {
+                NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
+                assertNotNull(analyzer);
+                assertAnalyzesTo(analyzer, "test", new String[]{"t"});
+            }
+        }
+
+        // Check deprecated name as well
+        {
+            Version version = VersionUtils.randomVersionBetween(random(), Version.V_7_0_0,
+                VersionUtils.getPreviousVersion(Version.V_7_3_0));
+            try (IndexAnalyzers indexAnalyzers = buildAnalyzers(version, "edgeNGram")) {
+                NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
+                assertNotNull(analyzer);
+                assertAnalyzesTo(analyzer, "test", new String[]{"t"});
+            }
+        }
+
+        // Afterwards, we return ngrams of length 1 and 2, to match the default factory settings
+        {
+            try (IndexAnalyzers indexAnalyzers = buildAnalyzers(Version.CURRENT, "edge_ngram")) {
+                NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
+                assertNotNull(analyzer);
+                assertAnalyzesTo(analyzer, "test", new String[]{"t", "te"});
+            }
+        }
+
+        // Check deprecated name as well
+        {
+            try (IndexAnalyzers indexAnalyzers = buildAnalyzers(Version.CURRENT, "edgeNGram")) {
+                NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
+                assertNotNull(analyzer);
+                assertAnalyzesTo(analyzer, "test", new String[]{"t", "te"});
+
+            }
+        }
+
+    }
+
+}
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ElisionFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ElisionFilterFactoryTests.java
new file mode 100644
index 0000000000000..dbfd49d5649d5
--- /dev/null
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ElisionFilterFactoryTests.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.analysis.common;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.analysis.AnalysisTestsHelper;
+import org.elasticsearch.test.ESTokenStreamTestCase;
+
+import java.io.IOException;
+
+public class ElisionFilterFactoryTests extends ESTokenStreamTestCase {
+
+    public void testElisionFilterWithNoArticles() throws IOException {
+        Settings settings = Settings.builder()
+            .put("index.analysis.filter.elision.type", "elision")
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings, new CommonAnalysisPlugin()));
+
+        assertEquals("elision filter requires [articles] or [articles_path] setting", e.getMessage());
+    }
+
+}
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java
index 8f58a074cf102..c4cc7589678ad 100644
--- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java
@@ -108,7 +108,7 @@ public void testNgramHighlightingWithBrokenPositions() throws IOException {
         client().prepareIndex("test", "test", "1")
             .setSource("name", "ARCOTEL Hotels Deutschland").get();
         refresh();
-        SearchResponse search = client().prepareSearch("test").setTypes("test")
+        SearchResponse search = client().prepareSearch("test")
                 .setQuery(matchQuery("name.autocomplete", "deut tel").operator(Operator.OR))
                 .highlighter(new HighlightBuilder().field("name.autocomplete")).get();
         assertHighlight(search, 0, "name.autocomplete", 0,
diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java
index d799674f231a1..c8e3699ea840d 100644
--- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java
+++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java
@@ -20,14 +20,24 @@
 
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisTestsHelper;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.test.VersionUtils;
 
 import java.io.IOException;
 import java.io.StringReader;
+import java.util.Collections;
 
 public class WordDelimiterGraphTokenFilterFactoryTests
         extends BaseWordDelimiterTokenFilterFactoryTestCase {
@@ -107,4 +117,51 @@ public void testAdjustingOffsets() throws IOException {
         assertTokenStreamContents(tokenFilter.create(tokenizer), expected, expectedStartOffsets, expectedEndOffsets, null,
             expectedIncr, expectedPosLen, null);
     }
+
+    public void testPreconfiguredFilter() throws IOException {
+        // Before 7.3 we don't adjust offsets
+        {
+            Settings settings = Settings.builder()
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .build();
+            Settings indexSettings = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED,
+                    VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, VersionUtils.getPreviousVersion(Version.V_7_3_0)))
+                .put("index.analysis.analyzer.my_analyzer.tokenizer", "standard")
+                .putList("index.analysis.analyzer.my_analyzer.filter", "word_delimiter_graph")
+                .build();
+            IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
+
+            try (IndexAnalyzers indexAnalyzers = new AnalysisModule(TestEnvironment.newEnvironment(settings),
+                Collections.singletonList(new CommonAnalysisPlugin())).getAnalysisRegistry().build(idxSettings)) {
+
+                NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
+                assertNotNull(analyzer);
+                assertAnalyzesTo(analyzer, "h100", new String[]{"h", "100"}, new int[]{ 0, 0 }, new int[]{ 4, 4 });
+
+            }
+        }
+
+        // Afger 7.3 we do adjust offsets
+        {
+            Settings settings = Settings.builder()
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .build();
+            Settings indexSettings = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put("index.analysis.analyzer.my_analyzer.tokenizer", "standard")
+                .putList("index.analysis.analyzer.my_analyzer.filter", "word_delimiter_graph")
+                .build();
+            IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
+
+            try (IndexAnalyzers indexAnalyzers = new AnalysisModule(TestEnvironment.newEnvironment(settings),
+                Collections.singletonList(new CommonAnalysisPlugin())).getAnalysisRegistry().build(idxSettings)) {
+
+                NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
+                assertNotNull(analyzer);
+                assertAnalyzesTo(analyzer, "h100", new String[]{"h", "100"}, new int[]{ 0, 1 }, new int[]{ 1, 4 });
+
+            }
+        }
+    }
 }
diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yml
index 15de6fe664786..2904cc3e95b58 100644
--- a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yml
+++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yml
@@ -67,15 +67,6 @@
     - length: { tokens: 1 }
     - match:  { tokens.0.token: a1 b2 c3 d4 }
 
----
-"standard_html_strip":
-    - do:
-        catch: /\[standard_html_strip\] analyzer is not supported for new indices, use a custom analyzer using \[standard\] tokenizer and \[html_strip\] char_filter, plus \[lowercase\] filter/
-        indices.analyze:
-          body:
-            text:     <bold/> <italic/>
-            analyzer: standard_html_strip
-
 ---
 "pattern":
     - do:
diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yml
index 3486b9defd9d2..92d0dce7b6201 100644
--- a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yml
+++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yml
@@ -587,6 +587,20 @@
     - length: { tokens: 1 }
     - match:  { tokens.0.token: avion }
 
+    - do:
+        catch: bad_request
+        indices.create:
+          index: test2
+          body:
+            settings:
+              analysis:
+                filter:
+                  my_elision:
+                    type: elision
+    - match: { status: 400 }
+    - match: { error.type: illegal_argument_exception }
+    - match: { error.reason: "elision filter requires [articles] or [articles_path] setting" }
+
 ---
 "stemmer":
     - do:
diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/50_queries_with_synonyms.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/50_queries_with_synonyms.yml
index 784ffd9dd123a..dca56565e6954 100644
--- a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/50_queries_with_synonyms.yml
+++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/50_queries_with_synonyms.yml
@@ -46,117 +46,6 @@
             field1: the quick lazy huge brown fox jumps over the tree
           refresh: true
 
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        body:
-          query:
-            common:
-              field1:
-                query: the fast brown
-                cutoff_frequency: 3
-                low_freq_operator: or
-  - match: { hits.total: 3 }
-  - match: { hits.hits.0._id: "1" }
-  - match: { hits.hits.1._id: "2" }
-  - match: { hits.hits.2._id: "3" }
-
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        body:
-          query:
-            common:
-              field1:
-                query: the fast brown
-                cutoff_frequency: 3
-                low_freq_operator: and
-  - match: { hits.total: 2 }
-  - match: { hits.hits.0._id: "1" }
-  - match: { hits.hits.1._id: "2" }
-
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        body:
-          query:
-            common:
-              field1:
-                query: the fast brown
-                cutoff_frequency: 3
-  - match: { hits.total: 3 }
-  - match: { hits.hits.0._id: "1" }
-  - match: { hits.hits.1._id: "2" }
-  - match: { hits.hits.2._id: "3" }
-
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        body:
-          query:
-            common:
-              field1:
-                query: the fast huge fox
-                minimum_should_match:
-                  low_freq: 3
-  - match: { hits.total: 1 }
-  - match: { hits.hits.0._id: "2" }
-
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        body:
-          query:
-            common:
-              field1:
-                query: the fast lazy fox brown
-                cutoff_frequency: 1
-                minimum_should_match:
-                  high_freq: 5
-  - match: { hits.total: 2 }
-  - match: { hits.hits.0._id: "2" }
-  - match: { hits.hits.1._id: "1" }
-
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        body:
-          query:
-            common:
-              field1:
-                query: the fast lazy fox brown
-                cutoff_frequency: 1
-                minimum_should_match:
-                  high_freq: 6
-  - match: { hits.total: 1 }
-  - match: { hits.hits.0._id: "2" }
-
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        body:
-          query:
-            common:
-              field1:
-                query: the fast lazy fox brown
-                cutoff_frequency: 1
-  - match: { hits.total: 1 }
-  - match: { hits.hits.0._id: "2" }
-
-  - do:
-      search:
-        rest_total_hits_as_int: true
-        body:
-          query:
-            common:
-              field1:
-                query: the quick brown
-                cutoff_frequency: 3
-  - match: { hits.total: 3 }
-  - match: { hits.hits.0._id: "1" }
-  - match: { hits.hits.1._id: "2" }
-  - match: { hits.hits.2._id: "3" }
-
   - do:
       search:
         rest_total_hits_as_int: true
@@ -165,7 +54,6 @@
             match:
               field1:
                 query: the fast brown
-                cutoff_frequency: 3
                 operator: and
   - match: { hits.total: 2 }
   - match: { hits.hits.0._id: "1" }
@@ -179,7 +67,6 @@
             match:
               field1:
                 query: the fast brown
-                cutoff_frequency: 3
                 operator: or
   - match: { hits.total: 3 }
   - match: { hits.hits.0._id: "1" }
@@ -194,13 +81,13 @@
             match:
               field1:
                 query: the fast brown
-                cutoff_frequency: 3
                 minimum_should_match: 3
   - match: { hits.total: 2 }
   - match: { hits.hits.0._id: "1" }
   - match: { hits.hits.1._id: "2" }
 
   - do:
+      warnings:
       search:
         rest_total_hits_as_int: true
         body:
@@ -208,7 +95,6 @@
             multi_match:
               query: the fast brown
               fields: [ "field1", "field2" ]
-              cutoff_frequency: 3
               operator: and
   - match: { hits.total: 3 }
   - match: { hits.hits.0._id: "3" }
diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle
index a94c375afc3e1..00c444e50e87d 100644
--- a/modules/ingest-common/build.gradle
+++ b/modules/ingest-common/build.gradle
@@ -25,6 +25,6 @@ esplugin {
 
 dependencies {
     compileOnly project(':modules:lang-painless')
-    compile project(':libs:grok')
-    compile project(':libs:dissect')
+    compile project(':libs:elasticsearch-grok')
+    compile project(':libs:elasticsearch-dissect')
 }
\ No newline at end of file
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java
index 1141a4cf7e891..f6eadab8014d2 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java
@@ -18,11 +18,11 @@
  */
 package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.node.NodeClient;
@@ -45,7 +45,7 @@
 import static org.elasticsearch.ingest.common.IngestCommonPlugin.GROK_PATTERNS;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
-public class GrokProcessorGetAction extends Action<GrokProcessorGetAction.Response> {
+public class GrokProcessorGetAction extends StreamableResponseActionType<GrokProcessorGetAction.Response> {
 
     static final GrokProcessorGetAction INSTANCE = new GrokProcessorGetAction();
     static final String NAME = "cluster:admin/ingest/processor/grok/get";
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java
index 5b3d88879ba13..5c3b0b0e7f3b5 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java
@@ -132,7 +132,11 @@ public void testRenameExistingFieldNullValue() throws Exception {
         String newFieldName = randomValueOtherThanMany(ingestDocument::hasField, () -> RandomDocumentPicks.randomFieldName(random()));
         Processor processor = createRenameProcessor(fieldName, newFieldName, false);
         processor.execute(ingestDocument);
-        assertThat(ingestDocument.hasField(fieldName), equalTo(false));
+        if (newFieldName.startsWith(fieldName + '.')) {
+            assertThat(ingestDocument.getFieldValue(fieldName, Object.class), instanceOf(Map.class));
+        } else {
+            assertThat(ingestDocument.hasField(fieldName), equalTo(false));
+        }
         assertThat(ingestDocument.hasField(newFieldName), equalTo(true));
         assertThat(ingestDocument.getFieldValue(newFieldName, Object.class), nullValue());
     }
diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml
index 86f4821ddaa23..3d100fad3b027 100644
--- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml
+++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml
@@ -119,6 +119,12 @@ teardown:
           {"doc":{"bytes_source_field":"2kb"}, "doc_as_upsert":true}
           {"update":{"_id":"8","_index":"test"}}
           {"script": "ctx._source.ran_script = true","upsert":{"bytes_source_field":"3kb"}, "scripted_upsert" : true}
+          {"update":{"_id":"6_alias","_index":"test_alias"}}
+          {"script":"ctx._source.ran_script = true","upsert":{"bytes_source_field":"1kb"}}
+          {"update":{"_id":"7_alias","_index":"test_alias"}}
+          {"doc":{"bytes_source_field":"2kb"}, "doc_as_upsert":true}
+          {"update":{"_id":"8_alias","_index":"test_alias"}}
+          {"script": "ctx._source.ran_script = true","upsert":{"bytes_source_field":"3kb"}, "scripted_upsert" : true}
 
   - do:
       mget:
@@ -127,6 +133,9 @@ teardown:
           - { _index: "test", _id: "6" }
           - { _index: "test", _id: "7" }
           - { _index: "test", _id: "8" }
+          - { _index: "test", _id: "6_alias" }
+          - { _index: "test", _id: "7_alias" }
+          - { _index: "test", _id: "8_alias" }
   - match: { docs.0._index: "test" }
   - match: { docs.0._id: "6" }
   - match: { docs.0._source.bytes_source_field: "1kb" }
@@ -141,6 +150,20 @@ teardown:
   - match: { docs.2._source.bytes_source_field: "3kb" }
   - match: { docs.2._source.bytes_target_field: 3072 }
   - match: { docs.2._source.ran_script: true }
+  - match: { docs.3._index: "test" }
+  - match: { docs.3._id: "6_alias" }
+  - match: { docs.3._source.bytes_source_field: "1kb" }
+  - match: { docs.3._source.bytes_target_field: 1024 }
+  - is_false: docs.3._source.ran_script
+  - match: { docs.4._index: "test" }
+  - match: { docs.4._id: "7_alias" }
+  - match: { docs.4._source.bytes_source_field: "2kb" }
+  - match: { docs.4._source.bytes_target_field: 2048 }
+  - match: { docs.5._index: "test" }
+  - match: { docs.5._id: "8_alias" }
+  - match: { docs.5._source.bytes_source_field: "3kb" }
+  - match: { docs.5._source.bytes_target_field: 3072 }
+  - match: { docs.5._source.ran_script: true }
 
 # explicit no default pipeline
   - do:
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionAggregationScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionAggregationScript.java
index 199f52c40319d..ba7b1b68c41e5 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionAggregationScript.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionAggregationScript.java
@@ -38,20 +38,37 @@ class ExpressionAggregationScript implements AggregationScript.LeafFactory {
     final Expression exprScript;
     final SimpleBindings bindings;
     final DoubleValuesSource source;
+    final boolean needsScore;
     final ReplaceableConstDoubleValueSource specialValue; // _value
 
-    ExpressionAggregationScript(Expression e, SimpleBindings b, ReplaceableConstDoubleValueSource v) {
+    ExpressionAggregationScript(Expression e, SimpleBindings b, boolean n, ReplaceableConstDoubleValueSource v) {
         exprScript = e;
         bindings = b;
         source = exprScript.getDoubleValuesSource(bindings);
+        needsScore = n;
         specialValue = v;
     }
 
+    @Override
+    public boolean needs_score() {
+        return needsScore;
+    }
+
     @Override
     public AggregationScript newInstance(final LeafReaderContext leaf) throws IOException {
         return new AggregationScript() {
             // Fake the scorer until setScorer is called.
-            DoubleValues values = source.getValues(leaf, null);
+            DoubleValues values = source.getValues(leaf, new DoubleValues() {
+                @Override
+                public double doubleValue() throws IOException {
+                    return get_score().doubleValue();
+                }
+
+                @Override
+                public boolean advanceExact(int doc) throws IOException {
+                    return true;
+                }
+            });
 
             @Override
             public Object execute() {
@@ -84,10 +101,4 @@ public void setNextAggregationValue(Object value) {
             }
         };
     }
-
-    @Override
-    public boolean needs_score() {
-        return false;
-    }
-
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
index 1ad0fff49b78a..abd8738b0e4c1 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java
@@ -221,10 +221,14 @@ private AggregationScript.LeafFactory newAggregationScript(Expression expr, Sear
         // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings,
         // instead of complicating SimpleBindings (which should stay simple)
         SimpleBindings bindings = new SimpleBindings();
+        boolean needsScores = false;
         ReplaceableConstDoubleValueSource specialValue = null;
         for (String variable : expr.variables) {
             try {
-                if (variable.equals("_value")) {
+                if (variable.equals("_score")) {
+                    bindings.add(new SortField("_score", SortField.Type.SCORE));
+                    needsScores = true;
+                } else if (variable.equals("_value")) {
                     specialValue = new ReplaceableConstDoubleValueSource();
                     bindings.add("_value", specialValue);
                     // noop: _value is special for aggregations, and is handled in ExpressionScriptBindings
@@ -237,6 +241,7 @@ private AggregationScript.LeafFactory newAggregationScript(Expression expr, Sear
                     // delegate valuesource creation based on field's type
                     // there are three types of "fields" to expressions, and each one has a different "api" of variables and methods.
                     final ValueSource valueSource = getDocValueSource(variable, lookup);
+                    needsScores |= valueSource.getSortField(false).needsScores();
                     bindings.add(variable, valueSource.asDoubleValuesSource());
                 }
             } catch (Exception e) {
@@ -244,7 +249,7 @@ private AggregationScript.LeafFactory newAggregationScript(Expression expr, Sear
                 throw convertToScriptException("link error", expr.sourceText, variable, e);
             }
         }
-        return new ExpressionAggregationScript(expr, bindings, specialValue);
+        return new ExpressionAggregationScript(expr, bindings, needsScores, specialValue);
     }
 
     private FieldScript.LeafFactory newFieldScript(Expression expr, SearchLookup lookup, @Nullable Map<String, Object> vars) {
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java
index 205e638314fe3..abc1b8e882389 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java
@@ -64,7 +64,7 @@ public void setUp() throws Exception {
         when(fieldData.load(anyObject())).thenReturn(atomicFieldData);
 
         service = new ExpressionScriptEngine();
-        lookup = new SearchLookup(mapperService, ignored -> fieldData, null);
+        lookup = new SearchLookup(mapperService, ignored -> fieldData);
     }
 
     private FieldScript.LeafFactory compile(String expression) {
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java
index e6bd503bfabe1..94acf6b35ab98 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java
@@ -19,9 +19,6 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.text.ParseException;
-import java.util.Collections;
 import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
@@ -33,6 +30,10 @@
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.Collections;
+
 import static org.mockito.Matchers.anyInt;
 import static org.mockito.Matchers.anyObject;
 import static org.mockito.Mockito.mock;
@@ -63,7 +64,7 @@ public void setUp() throws Exception {
         when(fieldData.load(anyObject())).thenReturn(atomicFieldData);
 
         service = new ExpressionScriptEngine();
-        lookup = new SearchLookup(mapperService, ignored -> fieldData, null);
+        lookup = new SearchLookup(mapperService, ignored -> fieldData);
     }
 
     private NumberSortScript.LeafFactory compile(String expression) {
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java
index 137f8e058cd85..cc67501eba319 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java
@@ -19,9 +19,6 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.text.ParseException;
-import java.util.Collections;
 import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
@@ -33,6 +30,10 @@
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.Collections;
+
 import static org.mockito.Matchers.anyInt;
 import static org.mockito.Matchers.anyObject;
 import static org.mockito.Mockito.mock;
@@ -63,7 +64,7 @@ public void setUp() throws Exception {
         when(fieldData.load(anyObject())).thenReturn(atomicFieldData);
 
         service = new ExpressionScriptEngine();
-        lookup = new SearchLookup(mapperService, ignored -> fieldData, null);
+        lookup = new SearchLookup(mapperService, ignored -> fieldData);
     }
 
     private TermsSetQueryScript.LeafFactory compile(String expression) {
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
index 4f8fa5f463edd..6e1c0efb7ded9 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
@@ -28,8 +28,8 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
+import org.elasticsearch.index.query.functionscore.ScriptScoreFunctionBuilder;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
@@ -120,7 +120,7 @@ public void testScore() throws Exception {
                 client().prepareIndex("test", "doc", "1").setSource("text", "hello goodbye"),
                 client().prepareIndex("test", "doc", "2").setSource("text", "hello hello hello goodbye"),
                 client().prepareIndex("test", "doc", "3").setSource("text", "hello hello goodebye"));
-        ScoreFunctionBuilder<?> score = ScoreFunctionBuilders.scriptFunction(
+        ScriptScoreFunctionBuilder score = ScoreFunctionBuilders.scriptFunction(
                 new Script(ScriptType.INLINE, "expression", "1 / _score", Collections.emptyMap()));
         SearchRequestBuilder req = client().prepareSearch().setIndices("test");
         req.setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("text", "hello"), score).boostMode(CombineFunction.REPLACE));
@@ -132,6 +132,15 @@ public void testScore() throws Exception {
         assertEquals("1", hits.getAt(0).getId());
         assertEquals("3", hits.getAt(1).getId());
         assertEquals("2", hits.getAt(2).getId());
+
+        req = client().prepareSearch().setIndices("test");
+        req.setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("text", "hello"), score).boostMode(CombineFunction.REPLACE));
+        score = ScoreFunctionBuilders.scriptFunction(
+                new Script(ScriptType.INLINE, "expression", "1 / _score", Collections.emptyMap()));
+        req.addAggregation(AggregationBuilders.max("max_score").script((score).getScript()));
+        req.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); // make sure DF is consistent
+        rsp = req.get();
+        assertSearchResponse(rsp);
     }
 
     public void testDateMethods() throws Exception {
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java
index 7f7f30f271acf..29e13eebf5845 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java
@@ -67,7 +67,7 @@ public void testAllOpsDisabledIndexedScripts() throws IOException {
             client().prepareSearch()
                     .setSource(new SearchSourceBuilder().scriptField("test1",
                             new Script(ScriptType.STORED, null, "script1", Collections.emptyMap())))
-                    .setIndices("test").setTypes("scriptTest").get();
+                    .setIndices("test").get();
             fail("search script should have been rejected");
         } catch(Exception e) {
             assertThat(e.toString(), containsString("cannot execute scripts using [field] context"));
diff --git a/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yml b/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yml
index fd8a2ebf1183b..33bad01a1232a 100644
--- a/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yml
+++ b/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yml
@@ -25,9 +25,25 @@ setup:
         rest_total_hits_as_int: true
         body:
           script_fields:
-            my_field :
+            my_field:
               script:
                 lang: expression
                 source: 'doc["age"].value + 19'
 
-  - match:  { hits.hits.0.fields.my_field.0: 42.0 }
+  - match: { hits.hits.0.fields.my_field.0: 42.0 }
+
+---
+"Expressions aggregation score test":
+
+    - do:
+        search:
+          rest_total_hits_as_int: true
+          body:
+            aggs:
+              max_score:
+                max:
+                  script:
+                    lang: expression
+                    source: '_score'
+
+    - match: { aggregations.max_score.value: 1.0 }
diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle
index 2a46bd9ed2efa..f493a85ab67d4 100644
--- a/modules/lang-mustache/build.gradle
+++ b/modules/lang-mustache/build.gradle
@@ -20,7 +20,6 @@
 esplugin {
   description 'Mustache scripting integration for Elasticsearch'
   classname 'org.elasticsearch.script.mustache.MustachePlugin'
-  hasClientJar = true // For the template apis and query
 }
 
 dependencies {
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateAction.java
index a9a44d0471586..a096a89951e7c 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateAction.java
@@ -19,10 +19,10 @@
 
 package org.elasticsearch.script.mustache;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
-public class MultiSearchTemplateAction extends Action<MultiSearchTemplateResponse> {
+public class MultiSearchTemplateAction extends ActionType<MultiSearchTemplateResponse> {
 
     public static final MultiSearchTemplateAction INSTANCE = new MultiSearchTemplateAction();
     public static final String NAME = "indices:data/read/msearch/template";
@@ -31,11 +31,6 @@ private MultiSearchTemplateAction() {
         super(NAME);
     }
 
-    @Override
-    public MultiSearchTemplateResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<MultiSearchTemplateResponse> getResponseReader() {
         return MultiSearchTemplateResponse::new;
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateAction.java
index 5d905ec39e1ab..2e8417c993990 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateAction.java
@@ -19,10 +19,10 @@
 
 package org.elasticsearch.script.mustache;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
-public class SearchTemplateAction extends Action<SearchTemplateResponse> {
+public class SearchTemplateAction extends ActionType<SearchTemplateResponse> {
 
     public static final SearchTemplateAction INSTANCE = new SearchTemplateAction();
     public static final String NAME = "indices:data/read/search/template";
@@ -31,11 +31,6 @@ private SearchTemplateAction() {
         super(NAME);
     }
 
-    @Override
-    public SearchTemplateResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<SearchTemplateResponse> getResponseReader() {
         return SearchTemplateResponse::new;
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java
index 39400197a3871..3b5c7562472e4 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java
@@ -56,13 +56,10 @@ public void testParseRequest() throws Exception {
         assertThat(request.requests().get(0).getRequest().preference(), nullValue());
         assertThat(request.requests().get(1).getRequest().indices()[0], equalTo("test2"));
         assertThat(request.requests().get(1).getRequest().indices()[1], equalTo("test3"));
-        assertThat(request.requests().get(1).getRequest().types()[0], equalTo("type1"));
         assertThat(request.requests().get(1).getRequest().requestCache(), nullValue());
         assertThat(request.requests().get(1).getRequest().preference(), equalTo("_local"));
         assertThat(request.requests().get(2).getRequest().indices()[0], equalTo("test4"));
         assertThat(request.requests().get(2).getRequest().indices()[1], equalTo("test1"));
-        assertThat(request.requests().get(2).getRequest().types()[0], equalTo("type2"));
-        assertThat(request.requests().get(2).getRequest().types()[1], equalTo("type1"));
         assertThat(request.requests().get(2).getRequest().routing(), equalTo("123"));
         assertNotNull(request.requests().get(0).getScript());
         assertNotNull(request.requests().get(1).getScript());
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java
index 0fbc3fa16afd2..3ff2bb649a79b 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java
@@ -185,7 +185,7 @@ public void testIndexedTemplateClient() throws Exception {
         templateParams.put("fieldParam", "foo");
 
         SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
-                .setRequest(new SearchRequest("test").types("type"))
+                .setRequest(new SearchRequest("test"))
                 .setScript("testTemplate").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
                 .get();
         assertHitCount(searchResponse.getResponse(), 4);
@@ -235,7 +235,7 @@ public void testIndexedTemplate() throws Exception {
         templateParams.put("fieldParam", "foo");
 
         SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
-                .setRequest(new SearchRequest().indices("test").types("type"))
+                .setRequest(new SearchRequest().indices("test"))
                 .setScript("1a")
                 .setScriptType(ScriptType.STORED)
                 .setScriptParams(templateParams)
@@ -243,7 +243,7 @@ public void testIndexedTemplate() throws Exception {
         assertHitCount(searchResponse.getResponse(), 4);
 
         expectThrows(ResourceNotFoundException.class, () -> new SearchTemplateRequestBuilder(client())
-                .setRequest(new SearchRequest().indices("test").types("type"))
+                .setRequest(new SearchRequest().indices("test"))
                 .setScript("1000")
                 .setScriptType(ScriptType.STORED)
                 .setScriptParams(templateParams)
@@ -251,7 +251,7 @@ public void testIndexedTemplate() throws Exception {
 
         templateParams.put("fieldParam", "bar");
         searchResponse = new SearchTemplateRequestBuilder(client())
-                .setRequest(new SearchRequest("test").types("type"))
+                .setRequest(new SearchRequest("test"))
                 .setScript("2").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
                 .get();
         assertHitCount(searchResponse.getResponse(), 1);
@@ -297,7 +297,7 @@ public void testIndexedTemplateOverwrite() throws Exception {
             templateParams.put("P_Keyword1", "dev");
 
             IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SearchTemplateRequestBuilder(client())
-                    .setRequest(new SearchRequest("testindex").types("test"))
+                    .setRequest(new SearchRequest("testindex"))
                     .setScript("git01").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
                     .get());
             assertThat(e.getMessage(), containsString("No negative slop allowed"));
@@ -308,7 +308,7 @@ public void testIndexedTemplateOverwrite() throws Exception {
             );
 
             SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
-                    .setRequest(new SearchRequest("testindex").types("test"))
+                    .setRequest(new SearchRequest("testindex"))
                     .setScript("git01").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
                     .get();
             assertHitCount(searchResponse.getResponse(), 1);
@@ -352,7 +352,7 @@ public void testIndexedTemplateWithArray() throws Exception {
         arrayTemplateParams.put("fieldParam", fieldParams);
 
         SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
-                .setRequest(new SearchRequest("test").types("type"))
+                .setRequest(new SearchRequest("test"))
                 .setScript("4").setScriptType(ScriptType.STORED).setScriptParams(arrayTemplateParams)
                 .get();
         assertHitCount(searchResponse.getResponse(), 5);
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java
index 7d4a6479727e2..b91054ee131d2 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java
@@ -63,8 +63,8 @@ protected SearchTemplateRequest mutateInstance(SearchTemplateRequest instance) t
         mutators.add(request -> request.setExplain(!request.isExplain()));
         mutators.add(request -> request.setSimulate(!request.isSimulate()));
 
-        mutators.add(request -> request.setRequest(
-            RandomSearchRequestGenerator.randomSearchRequest(SearchSourceBuilder::searchSource)));
+        mutators.add(request -> request.setRequest(randomValueOtherThan(request.getRequest(),
+                () -> RandomSearchRequestGenerator.randomSearchRequest(SearchSourceBuilder::searchSource))));
 
         SearchTemplateRequest mutatedInstance = copyInstance(instance);
         Consumer<SearchTemplateRequest> mutator = randomFrom(mutators);
diff --git a/modules/lang-mustache/src/test/resources/org/elasticsearch/script/mustache/simple-msearch-template.json b/modules/lang-mustache/src/test/resources/org/elasticsearch/script/mustache/simple-msearch-template.json
index 11a0091492c4d..1809b4012fde1 100644
--- a/modules/lang-mustache/src/test/resources/org/elasticsearch/script/mustache/simple-msearch-template.json
+++ b/modules/lang-mustache/src/test/resources/org/elasticsearch/script/mustache/simple-msearch-template.json
@@ -1,6 +1,6 @@
 {"index":["test0", "test1"], "request_cache": true}
 {"source": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } }
-{"index" : "test2,test3", "type" : "type1", "preference": "_local"}
+{"index" : "test2,test3", "preference": "_local"}
 {"source": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } }
-{"index" : ["test4", "test1"], "type" :  [ "type2", "type1" ], "routing": "123"}
+{"index" : ["test4", "test1"], "routing": "123"}
 {"source": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } }
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml
index c0cc784250b90..f4d4f3a97e5e0 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml
@@ -133,6 +133,7 @@
         index:  test
         id:     1
         body:   {}
+        refresh: true
 
   - do:
       put_script:
@@ -145,5 +146,5 @@
       search_template:
         body: { "id": "template_1", "params": {} }
 
-  - match: { hits.total.value: 0 }
+  - match: { hits.total.value: 1 }
   - match: { hits.total.relation: eq }
diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle
index 1f6b722ec308c..6cce3850232e0 100644
--- a/modules/lang-painless/build.gradle
+++ b/modules/lang-painless/build.gradle
@@ -69,7 +69,7 @@ sourceSets {
 }
 
 dependencies {
-  docCompile "org.elasticsearch:elasticsearch:${project.versions.elasticsearch}"
+  docCompile project(':server')
   docCompile project(':modules:lang-painless')
 }
 
diff --git a/modules/lang-painless/spi/build.gradle b/modules/lang-painless/spi/build.gradle
index 7e43a242a23a9..3f25f247a2b67 100644
--- a/modules/lang-painless/spi/build.gradle
+++ b/modules/lang-painless/spi/build.gradle
@@ -33,7 +33,7 @@ publishing {
 }
 
 dependencies {
-  compile "org.elasticsearch:elasticsearch:${version}"
+  compile project(":server")
 }
 
 // no tests...yet?
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java
index b08b38d2bfc7b..d79c605986880 100644
--- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.painless.spi;
 
+import org.elasticsearch.painless.spi.annotation.WhitelistAnnotationParser;
+
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
@@ -52,7 +54,8 @@ public final class Whitelist {
     };
 
     public static final List<Whitelist> BASE_WHITELISTS =
-        Collections.singletonList(WhitelistLoader.loadFromResourceFiles(Whitelist.class, BASE_WHITELIST_FILES));
+            Collections.singletonList(WhitelistLoader.loadFromResourceFiles(
+                    Whitelist.class, WhitelistAnnotationParser.BASE_ANNOTATION_PARSERS, BASE_WHITELIST_FILES));
 
     /** The {@link ClassLoader} used to look up the whitelisted Java classes, constructors, methods, and fields. */
     public final ClassLoader classLoader;
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClass.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClass.java
index 7b3eb75aa3ecd..3f2bf3e997a27 100644
--- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClass.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClass.java
@@ -19,9 +19,12 @@
 
 package org.elasticsearch.painless.spi;
 
+import java.util.AbstractMap;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 /**
  * Class represents the equivalent of a Java class in Painless complete with super classes,
@@ -46,11 +49,6 @@ public final class WhitelistClass {
     /** The Java class name this class represents. */
     public final String javaClassName;
 
-    /**
-     * Allow the Java class name to only be specified as the fully-qualified name.
-     */
-    public final boolean noImport;
-
     /** The {@link List} of whitelisted ({@link WhitelistConstructor}s) available to this class. */
     public final List<WhitelistConstructor> whitelistConstructors;
 
@@ -60,17 +58,27 @@ public final class WhitelistClass {
     /** The {@link List} of whitelisted ({@link WhitelistField}s) available to this class. */
     public final List<WhitelistField> whitelistFields;
 
+    /** The {@link Map} of annotations for this class. */
+    public final Map<Class<?>, Object> painlessAnnotations;
+
     /** Standard constructor. All values must be not {@code null}. */
-    public WhitelistClass(String origin, String javaClassName, boolean noImport,
-            List<WhitelistConstructor> whitelistConstructors, List<WhitelistMethod> whitelistMethods, List<WhitelistField> whitelistFields)
-    {
+    public WhitelistClass(String origin, String javaClassName,
+            List<WhitelistConstructor> whitelistConstructors, List<WhitelistMethod> whitelistMethods, List<WhitelistField> whitelistFields,
+            List<Object> painlessAnnotations) {
 
         this.origin = Objects.requireNonNull(origin);
         this.javaClassName = Objects.requireNonNull(javaClassName);
-        this.noImport = noImport;
 
         this.whitelistConstructors = Collections.unmodifiableList(Objects.requireNonNull(whitelistConstructors));
         this.whitelistMethods = Collections.unmodifiableList(Objects.requireNonNull(whitelistMethods));
         this.whitelistFields = Collections.unmodifiableList(Objects.requireNonNull(whitelistFields));
+
+        if (painlessAnnotations.isEmpty()) {
+            this.painlessAnnotations = Collections.emptyMap();
+        } else {
+            this.painlessAnnotations = Collections.unmodifiableMap(Objects.requireNonNull(painlessAnnotations).stream()
+                    .map(painlessAnnotation -> new AbstractMap.SimpleEntry<>(painlessAnnotation.getClass(), painlessAnnotation))
+                    .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
+        }
     }
 }
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClassBinding.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClassBinding.java
index da19917464820..f3ba052156dfa 100644
--- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClassBinding.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClassBinding.java
@@ -19,8 +19,12 @@
 
 package org.elasticsearch.painless.spi;
 
+import java.util.AbstractMap;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 /**
  * A class binding represents a method call that stores state. Each class binding's Java class must
@@ -51,9 +55,13 @@ public class WhitelistClassBinding {
      */
     public final List<String> canonicalTypeNameParameters;
 
+    /** The {@link Map} of annotations for this class binding. */
+    public final Map<Class<?>, Object> painlessAnnotations;
+
     /** Standard constructor. All values must be not {@code null}. */
     public WhitelistClassBinding(String origin, String targetJavaClassName,
-            String methodName, String returnCanonicalTypeName, List<String> canonicalTypeNameParameters) {
+            String methodName, String returnCanonicalTypeName, List<String> canonicalTypeNameParameters,
+            List<Object> painlessAnnotations) {
 
         this.origin = Objects.requireNonNull(origin);
         this.targetJavaClassName = Objects.requireNonNull(targetJavaClassName);
@@ -61,5 +69,13 @@ public WhitelistClassBinding(String origin, String targetJavaClassName,
         this.methodName = Objects.requireNonNull(methodName);
         this.returnCanonicalTypeName = Objects.requireNonNull(returnCanonicalTypeName);
         this.canonicalTypeNameParameters = Objects.requireNonNull(canonicalTypeNameParameters);
+
+        if (painlessAnnotations.isEmpty()) {
+            this.painlessAnnotations = Collections.emptyMap();
+        } else {
+            this.painlessAnnotations = Collections.unmodifiableMap(Objects.requireNonNull(painlessAnnotations).stream()
+                    .map(painlessAnnotation -> new AbstractMap.SimpleEntry<>(painlessAnnotation.getClass(), painlessAnnotation))
+                    .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
+        }
     }
 }
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistConstructor.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistConstructor.java
index 032ef397def01..13aec63fceb77 100644
--- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistConstructor.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistConstructor.java
@@ -19,9 +19,12 @@
 
 package org.elasticsearch.painless.spi;
 
+import java.util.AbstractMap;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 /**
  * Constructor represents the equivalent of a Java constructor available as a whitelisted class
@@ -40,9 +43,20 @@ public final class WhitelistConstructor {
      */
     public final List<String> canonicalTypeNameParameters;
 
+    /** The {@link Map} of annotations for this constructor. */
+    public final Map<Class<?>, Object> painlessAnnotations;
+
     /** Standard constructor. All values must be not {@code null}. */
-    public WhitelistConstructor(String origin, List<String> canonicalTypeNameParameters) {
+    public WhitelistConstructor(String origin, List<String> canonicalTypeNameParameters, List<Object> painlessAnnotations) {
         this.origin = Objects.requireNonNull(origin);
         this.canonicalTypeNameParameters = Collections.unmodifiableList(Objects.requireNonNull(canonicalTypeNameParameters));
+
+        if (painlessAnnotations.isEmpty()) {
+            this.painlessAnnotations = Collections.emptyMap();
+        } else {
+            this.painlessAnnotations = Collections.unmodifiableMap(Objects.requireNonNull(painlessAnnotations).stream()
+                    .map(painlessAnnotation -> new AbstractMap.SimpleEntry<>(painlessAnnotation.getClass(), painlessAnnotation))
+                    .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
+        }
     }
 }
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistField.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistField.java
index 44ed31a227e1c..84f12e907584c 100644
--- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistField.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistField.java
@@ -19,7 +19,12 @@
 
 package org.elasticsearch.painless.spi;
 
+import java.util.AbstractMap;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 /**
  * Field represents the equivalent of a Java field available as a whitelisted class field
@@ -37,10 +42,21 @@ public class WhitelistField {
     /** The canonical type name for the field which can be used to look up the Java field through reflection. */
     public final String canonicalTypeNameParameter;
 
+    /** The {@link Map} of annotations for this field. */
+    public final Map<Class<?>, Object> painlessAnnotations;
+
     /** Standard constructor.  All values must be not {@code null}. */
-    public WhitelistField(String origin, String fieldName, String canonicalTypeNameParameter) {
+    public WhitelistField(String origin, String fieldName, String canonicalTypeNameParameter, List<Object> painlessAnnotations) {
         this.origin = Objects.requireNonNull(origin);
         this.fieldName = Objects.requireNonNull(fieldName);
         this.canonicalTypeNameParameter = Objects.requireNonNull(canonicalTypeNameParameter);
+
+        if (painlessAnnotations.isEmpty()) {
+            this.painlessAnnotations = Collections.emptyMap();
+        } else {
+            this.painlessAnnotations = Collections.unmodifiableMap(Objects.requireNonNull(painlessAnnotations).stream()
+                    .map(painlessAnnotation -> new AbstractMap.SimpleEntry<>(painlessAnnotation.getClass(), painlessAnnotation))
+                    .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
+        }
     }
 }
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistInstanceBinding.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistInstanceBinding.java
index 46c2f0f91fe02..2a41f873bf8e5 100644
--- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistInstanceBinding.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistInstanceBinding.java
@@ -19,8 +19,12 @@
 
 package org.elasticsearch.painless.spi;
 
+import java.util.AbstractMap;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 /**
  * An instance binding represents a method call that stores state. Each instance binding must provide
@@ -47,9 +51,13 @@ public class WhitelistInstanceBinding {
      */
     public final List<String> canonicalTypeNameParameters;
 
+    /** The {@link Map} of annotations for this instance binding. */
+    public final Map<Class<?>, Object> painlessAnnotations;
+
     /** Standard constructor. All values must be not {@code null}. */
     public WhitelistInstanceBinding(String origin, Object targetInstance,
-            String methodName, String returnCanonicalTypeName, List<String> canonicalTypeNameParameters) {
+            String methodName, String returnCanonicalTypeName, List<String> canonicalTypeNameParameters,
+            List<Object> painlessAnnotations) {
 
         this.origin = Objects.requireNonNull(origin);
         this.targetInstance = Objects.requireNonNull(targetInstance);
@@ -57,5 +65,13 @@ public WhitelistInstanceBinding(String origin, Object targetInstance,
         this.methodName = Objects.requireNonNull(methodName);
         this.returnCanonicalTypeName = Objects.requireNonNull(returnCanonicalTypeName);
         this.canonicalTypeNameParameters = Objects.requireNonNull(canonicalTypeNameParameters);
+
+        if (painlessAnnotations.isEmpty()) {
+            this.painlessAnnotations = Collections.emptyMap();
+        } else {
+            this.painlessAnnotations = Collections.unmodifiableMap(Objects.requireNonNull(painlessAnnotations).stream()
+                    .map(painlessAnnotation -> new AbstractMap.SimpleEntry<>(painlessAnnotation.getClass(), painlessAnnotation))
+                    .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
+        }
     }
 }
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java
index d896c345a47e9..a9f6efea0e0b6 100644
--- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.painless.spi;
 
+import org.elasticsearch.painless.spi.annotation.WhitelistAnnotationParser;
+
 import java.io.InputStreamReader;
 import java.io.LineNumberReader;
 import java.lang.reflect.Constructor;
@@ -30,11 +32,22 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /** Loads and creates a {@link Whitelist} from one to many text files. */
 public final class WhitelistLoader {
 
+    /**
+     * Loads and creates a {@link Whitelist} from one to many text files using only the base annotation parsers.
+     * See {@link #loadFromResourceFiles(Class, Map, String...)} for information on how to structure a whitelist
+     * text file.
+     */
+    public static Whitelist loadFromResourceFiles(Class<?> resource, String... filepaths) {
+        return loadFromResourceFiles(resource, WhitelistAnnotationParser.BASE_ANNOTATION_PARSERS, filepaths);
+    }
+
     /**
      * Loads and creates a {@link Whitelist} from one to many text files. The file paths are passed in as an array of
      * {@link String}s with a single {@link Class} to be be used to load the resources where each {@link String}
@@ -54,7 +67,7 @@ public final class WhitelistLoader {
      *     a Painless type name with the exception that any dollar symbols used as part of inner classes will
      *     be replaced with dot symbols. </li>
      *     <li> short Java type name - The text after the final dot symbol of any specified Java class. A
-     *     short type Java name may be excluded by using the 'no_import' token during Painless class parsing
+     *     short type Java name may be excluded by using the 'no_import' attribute during Painless class parsing
      *     as described later. </li>
      * </ul>
      *
@@ -65,8 +78,8 @@ public final class WhitelistLoader {
      *   be ignored by the parser. </li>
      *   <li> Primitive types may be specified starting with 'class' and followed by the Java type name,
      *   an opening bracket, a newline, a closing bracket, and a final newline. </li>
-     *   <li> Complex types may be specified starting with 'class' and followed the fully-qualified Java
-     *   class name, optionally followed by an 'no_import' token, an opening bracket, a newline,
+     *   <li> Complex types may be specified starting with 'class' and followed by the fully-qualified Java
+     *   class name, optionally followed by a 'no_import' attribute, an opening bracket, a newline,
      *   constructor/method/field specifications, a closing bracket, and a final newline. Within a complex
      *   type the following may be parsed:
      *   <ul>
@@ -90,6 +103,10 @@ public final class WhitelistLoader {
      *     of the field, followed by the Java name of the field (which all be the Painless name
      *     for the field), and a newline. </li>
      *   </ul>
+     *   <li> Annotations may be added starting with an at, followed by a name, optionally an opening brace,
+     *   a parameter name, an equals, an opening quote, an argument value, a closing quote, (possibly repeated
+     *   for multiple arguments,) and a closing brace. Multiple annotations may be added after a class (before
+     *   the opening bracket), after a method, or after field. </li>
      * </ul>
      *
      * Note there must be a one-to-one correspondence of Painless type names to Java type/class names.
@@ -110,7 +127,7 @@ public final class WhitelistLoader {
      *
      * # complex types
      *
-     * class my.package.Example no_import {
+     * class my.package.Example @no_import {
      *   # constructors
      *   ()
      *   (int)
@@ -120,7 +137,8 @@ public final class WhitelistLoader {
      *   # method
      *   Example add(int, def)
      *   int add(Example, Example)
-     *   void example()
+     *   void example() @deprecated[use example 2 instead]
+     *   void example2()
      *
      *   # augmented
      *   Example some.other.Class sub(Example, int, def)
@@ -132,7 +150,7 @@ public final class WhitelistLoader {
      * }
      * }
      */
-    public static Whitelist loadFromResourceFiles(Class<?> resource, String... filepaths) {
+    public static Whitelist loadFromResourceFiles(Class<?> resource, Map<String, WhitelistAnnotationParser> parsers, String... filepaths) {
         List<WhitelistClass> whitelistClasses = new ArrayList<>();
         List<WhitelistMethod> whitelistStatics = new ArrayList<>();
         List<WhitelistClassBinding> whitelistClassBindings = new ArrayList<>();
@@ -149,10 +167,10 @@ public static Whitelist loadFromResourceFiles(Class<?> resource, String... filep
                 String parseType = null;
                 String whitelistClassOrigin = null;
                 String javaClassName = null;
-                boolean noImport = false;
                 List<WhitelistConstructor> whitelistConstructors = null;
                 List<WhitelistMethod> whitelistMethods = null;
                 List<WhitelistField> whitelistFields = null;
+                List<Object> classAnnotations = null;
 
                 while ((line = reader.readLine()) != null) {
                     number = reader.getLineNumber();
@@ -164,7 +182,7 @@ public static Whitelist loadFromResourceFiles(Class<?> resource, String... filep
                     }
 
                     // Handle a new class by resetting all the variables necessary to construct a new WhitelistClass for the whitelist.
-                    // Expects the following format: 'class' ID 'no_import'? '{' '\n'
+                    // Expects the following format: 'class' ID annotations? '{' '\n'
                     if (line.startsWith("class ")) {
                         // Ensure the final token of the line is '{'.
                         if (line.endsWith("{") == false) {
@@ -176,19 +194,19 @@ public static Whitelist loadFromResourceFiles(Class<?> resource, String... filep
                             throw new IllegalArgumentException("invalid definition: cannot embed class definition [" + line + "]");
                         }
 
-                        // Parse the Java class name.
-                        String[] tokens = line.substring(5, line.length() - 1).trim().split("\\s+");
+                        // Parse the Java class name and annotations if they exist.
+                        int annotationIndex = line.indexOf('@');
 
-                        // Ensure the correct number of tokens.
-                        if (tokens.length == 2 && "no_import".equals(tokens[1])) {
-                            noImport = true;
-                        } else if (tokens.length != 1) {
-                            throw new IllegalArgumentException("invalid class definition: failed to parse class name [" + line + "]");
+                        if (annotationIndex == -1) {
+                            annotationIndex = line.length() - 1;
+                            classAnnotations = Collections.emptyList();
+                        } else {
+                            classAnnotations = parseWhitelistAnnotations(parsers, line.substring(annotationIndex, line.length() - 1));
                         }
 
                         parseType = "class";
                         whitelistClassOrigin = "[" + filepath + "]:[" + number + "]";
-                        javaClassName = tokens[0];
+                        javaClassName = line.substring(5, annotationIndex).trim();
 
                         // Reset all the constructors, methods, and fields to support a new class.
                         whitelistConstructors = new ArrayList<>();
@@ -217,22 +235,22 @@ public static Whitelist loadFromResourceFiles(Class<?> resource, String... filep
                         // Create a new WhitelistClass with all the previously gathered constructors, methods,
                         // augmented methods, and fields, and add it to the list of whitelisted classes.
                         if ("class".equals(parseType)) {
-                            whitelistClasses.add(new WhitelistClass(whitelistClassOrigin, javaClassName, noImport,
-                                    whitelistConstructors, whitelistMethods, whitelistFields));
+                            whitelistClasses.add(new WhitelistClass(whitelistClassOrigin, javaClassName,
+                                    whitelistConstructors, whitelistMethods, whitelistFields, classAnnotations));
 
                             whitelistClassOrigin = null;
                             javaClassName = null;
-                            noImport = false;
                             whitelistConstructors = null;
                             whitelistMethods = null;
                             whitelistFields = null;
+                            classAnnotations = null;
                         }
 
                         // Reset the parseType.
                         parseType = null;
 
                     // Handle static import definition types.
-                    // Expects the following format: ID ID '(' ( ID ( ',' ID )* )? ')' ( 'from_class' | 'bound_to' ) ID '\n'
+                    // Expects the following format: ID ID '(' ( ID ( ',' ID )* )? ')' ( 'from_class' | 'bound_to' ) ID annotations? '\n'
                     } else if ("static_import".equals(parseType)) {
                         // Mark the origin of this parsable object.
                         String origin = "[" + filepath + "]:[" + number + "]";
@@ -274,8 +292,19 @@ public static Whitelist loadFromResourceFiles(Class<?> resource, String... filep
                             canonicalTypeNameParameters = new String[0];
                         }
 
+                        // Parse the annotations if they exist.
+                        List<Object> annotations;
+                        int annotationIndex = line.indexOf('@');
+
+                        if (annotationIndex == -1) {
+                            annotationIndex = line.length();
+                            annotations = Collections.emptyList();
+                        } else {
+                            annotations = parseWhitelistAnnotations(parsers, line.substring(annotationIndex));
+                        }
+
                         // Parse the static import type and class.
-                        tokens = line.substring(parameterEndIndex + 1).trim().split("\\s+");
+                        tokens = line.substring(parameterEndIndex + 1, annotationIndex).trim().split("\\s+");
 
                         String staticImportType;
                         String targetJavaClassName;
@@ -291,10 +320,12 @@ public static Whitelist loadFromResourceFiles(Class<?> resource, String... filep
                         // Add a static import method or binding depending on the static import type.
                         if ("from_class".equals(staticImportType)) {
                             whitelistStatics.add(new WhitelistMethod(origin, targetJavaClassName,
-                                    methodName, returnCanonicalTypeName, Arrays.asList(canonicalTypeNameParameters)));
+                                    methodName, returnCanonicalTypeName, Arrays.asList(canonicalTypeNameParameters),
+                                    annotations));
                         } else if ("bound_to".equals(staticImportType)) {
                             whitelistClassBindings.add(new WhitelistClassBinding(origin, targetJavaClassName,
-                                    methodName, returnCanonicalTypeName, Arrays.asList(canonicalTypeNameParameters)));
+                                    methodName, returnCanonicalTypeName, Arrays.asList(canonicalTypeNameParameters),
+                                    annotations));
                         } else {
                             throw new IllegalArgumentException("invalid static import definition: " +
                                     "unexpected static import type [" + staticImportType + "] [" + line + "]");
@@ -306,36 +337,38 @@ public static Whitelist loadFromResourceFiles(Class<?> resource, String... filep
                         String origin = "[" + filepath + "]:[" + number + "]";
 
                         // Handle the case for a constructor definition.
-                        // Expects the following format: '(' ( ID ( ',' ID )* )? ')' '\n'
+                        // Expects the following format: '(' ( ID ( ',' ID )* )? ')' annotations? '\n'
                         if (line.startsWith("(")) {
-                            // Ensure the final token of the line is ')'.
-                            if (line.endsWith(")") == false) {
+                            // Parse the constructor parameters.
+                            int parameterEndIndex = line.indexOf(')');
+
+                            if (parameterEndIndex == -1) {
                                 throw new IllegalArgumentException(
-                                        "invalid constructor definition: expected a closing parenthesis [" + line + "]");
+                                        "illegal constructor definition: end of constructor parameters not found [" + line + "]");
                             }
 
-                            // Parse the constructor parameters.
-                            String[] tokens = line.substring(1, line.length() - 1).replaceAll("\\s+", "").split(",");
+                            String[] canonicalTypeNameParameters = line.substring(1, parameterEndIndex).replaceAll("\\s+", "").split(",");
 
                             // Handle the case for a constructor with no parameters.
-                            if ("".equals(tokens[0])) {
-                                tokens = new String[0];
+                            if ("".equals(canonicalTypeNameParameters[0])) {
+                                canonicalTypeNameParameters = new String[0];
                             }
 
-                            whitelistConstructors.add(new WhitelistConstructor(origin, Arrays.asList(tokens)));
+                            // Parse the annotations if they exist.
+                            List<Object> annotations;
+                            int annotationIndex = line.indexOf('@');
+                            annotations = annotationIndex == -1 ?
+                                    Collections.emptyList() : parseWhitelistAnnotations(parsers, line.substring(annotationIndex));
 
-                            // Handle the case for a method or augmented method definition.
-                            // Expects the following format: ID ID? ID '(' ( ID ( ',' ID )* )? ')' '\n'
-                        } else if (line.contains("(")) {
-                            // Ensure the final token of the line is ')'.
-                            if (line.endsWith(")") == false) {
-                                throw new IllegalArgumentException(
-                                        "invalid method definition: expected a closing parenthesis [" + line + "]");
-                            }
+                            whitelistConstructors.add(new WhitelistConstructor(
+                                    origin, Arrays.asList(canonicalTypeNameParameters), annotations));
 
+                        // Handle the case for a method or augmented method definition.
+                        // Expects the following format: ID ID? ID '(' ( ID ( ',' ID )* )? ')' annotations? '\n'
+                        } else if (line.contains("(")) {
                             // Parse the tokens prior to the method parameters.
-                            int parameterIndex = line.indexOf('(');
-                            String[] tokens = line.substring(0, parameterIndex).trim().split("\\s+");
+                            int parameterStartIndex = line.indexOf('(');
+                            String[] tokens = line.substring(0, parameterStartIndex).trim().split("\\s+");
 
                             String methodName;
                             String javaAugmentedClassName;
@@ -354,28 +387,54 @@ public static Whitelist loadFromResourceFiles(Class<?> resource, String... filep
                             String returnCanonicalTypeName = tokens[0];
 
                             // Parse the method parameters.
-                            tokens = line.substring(parameterIndex + 1, line.length() - 1).replaceAll("\\s+", "").split(",");
+                            int parameterEndIndex = line.indexOf(')');
+
+                            if (parameterEndIndex == -1) {
+                                throw new IllegalArgumentException(
+                                        "illegal static import definition: end of method parameters not found [" + line + "]");
+                            }
+
+                            String[] canonicalTypeNameParameters =
+                                    line.substring(parameterStartIndex + 1, parameterEndIndex).replaceAll("\\s+", "").split(",");
 
                             // Handle the case for a method with no parameters.
-                            if ("".equals(tokens[0])) {
-                                tokens = new String[0];
+                            if ("".equals(canonicalTypeNameParameters[0])) {
+                                canonicalTypeNameParameters = new String[0];
                             }
 
+                            // Parse the annotations if they exist.
+                            List<Object> annotations;
+                            int annotationIndex = line.indexOf('@');
+                            annotations = annotationIndex == -1 ?
+                                    Collections.emptyList() : parseWhitelistAnnotations(parsers, line.substring(annotationIndex));
+
                             whitelistMethods.add(new WhitelistMethod(origin, javaAugmentedClassName, methodName,
-                                    returnCanonicalTypeName, Arrays.asList(tokens)));
+                                    returnCanonicalTypeName, Arrays.asList(canonicalTypeNameParameters),
+                                    annotations));
 
-                            // Handle the case for a field definition.
-                            // Expects the following format: ID ID '\n'
+                        // Handle the case for a field definition.
+                        // Expects the following format: ID ID annotations? '\n'
                         } else {
+                            // Parse the annotations if they exist.
+                            List<Object> annotations;
+                            int annotationIndex = line.indexOf('@');
+
+                            if (annotationIndex == -1) {
+                                annotationIndex = line.length();
+                                annotations = Collections.emptyList();
+                            } else {
+                                annotations = parseWhitelistAnnotations(parsers, line.substring(annotationIndex));
+                            }
+
                             // Parse the field tokens.
-                            String[] tokens = line.split("\\s+");
+                            String[] tokens = line.substring(0, annotationIndex).split("\\s+");
 
                             // Ensure the correct number of tokens.
                             if (tokens.length != 2) {
                                 throw new IllegalArgumentException("invalid field definition: unexpected format [" + line + "]");
                             }
 
-                            whitelistFields.add(new WhitelistField(origin, tokens[1], tokens[0]));
+                            whitelistFields.add(new WhitelistField(origin, tokens[1], tokens[0], annotations));
                         }
                     } else {
                         throw new IllegalArgumentException("invalid definition: unable to parse line [" + line + "]");
@@ -396,5 +455,85 @@ public static Whitelist loadFromResourceFiles(Class<?> resource, String... filep
         return new Whitelist(loader, whitelistClasses, whitelistStatics, whitelistClassBindings, Collections.emptyList());
     }
 
+    private static List<Object> parseWhitelistAnnotations(
+            Map<String, WhitelistAnnotationParser> parsers, String line) {
+
+        List<Object> annotations;
+
+        if (line.isBlank()) {
+            annotations = Collections.emptyList();
+        } else {
+            line = line.trim();
+
+            if (line.charAt(0) != '@') {
+                throw new IllegalArgumentException("invalid annotation: expected at symbol [" + line + "]");
+            }
+
+            if (line.length() < 2) {
+                throw new IllegalArgumentException("invalid annotation: expected name [" + line + "]");
+            }
+
+            String[] annotationStrings = line.substring(1).split("@");
+            annotations = new ArrayList<>(annotationStrings.length);
+
+            for (String annotationString : annotationStrings) {
+                String name;
+                Map<String, String> arguments;
+
+                annotationString = annotationString.trim();
+                int index = annotationString.indexOf('[');
+
+                if (index == -1) {
+                    name = annotationString;
+                    arguments = Collections.emptyMap();
+                } else {
+                    if (annotationString.charAt(annotationString.length() - 1) != ']') {
+                        throw new IllegalArgumentException("invalid annotation: expected closing brace [" + line + "]");
+                    }
+
+                    name = annotationString.substring(0, index);
+                    arguments = new HashMap<>();
+
+                    String[] argumentsStrings = annotationString.substring(index + 1, annotationString.length() - 1).split(",");
+
+                    for (String argumentString : argumentsStrings) {
+                        String[] argumentKeyValue = argumentString.split("=");
+
+                        if (argumentKeyValue.length != 2) {
+                            throw new IllegalArgumentException("invalid annotation: expected key=\"value\" [" + line + "]");
+                        }
+                        
+                        String argumentKey = argumentKeyValue[0].trim();
+                        
+                        if (argumentKey.isEmpty()) {
+                            throw new IllegalArgumentException("invalid annotation: expected key=\"value\" [" + line + "]");
+                        }
+                        
+                        String argumentValue = argumentKeyValue[1];
+                        
+                        if (argumentValue.length() < 3 || argumentValue.charAt(0) != '"' || 
+                                argumentValue.charAt(argumentValue.length() - 1) != '"') {
+                            throw new IllegalArgumentException("invalid annotation: expected key=\"value\" [" + line + "]");
+                        }
+                        
+                        argumentValue = argumentValue.substring(1, argumentValue.length() - 1);
+
+                        arguments.put(argumentKey, argumentValue);
+                    }
+                }
+
+                WhitelistAnnotationParser parser = parsers.get(name);
+
+                if (parser == null) {
+                    throw new IllegalArgumentException("invalid annotation: parser not found for [" + name + "] [" + line + "]");
+                }
+
+                annotations.add(parser.parse(arguments));
+            }
+        }
+
+        return annotations;
+    }
+
     private WhitelistLoader() {}
 }
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistMethod.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistMethod.java
index f450ee0238d19..838fcba5034ac 100644
--- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistMethod.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistMethod.java
@@ -19,9 +19,12 @@
 
 package org.elasticsearch.painless.spi;
 
+import java.util.AbstractMap;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 /**
  * Method represents the equivalent of a Java method available as a whitelisted class method
@@ -61,18 +64,30 @@ public class WhitelistMethod {
      */
     public final List<String> canonicalTypeNameParameters;
 
+    /** The {@link Map} of annotations for this method. */
+    public final Map<Class<?>, Object> painlessAnnotations;
+
     /**
      * Standard constructor. All values must be not {@code null} with the exception of
      * augmentedCanonicalClassName; augmentedCanonicalClassName will be {@code null} unless the method
      * is augmented as described in the class documentation.
      */
     public WhitelistMethod(String origin, String augmentedCanonicalClassName, String methodName,
-            String returnCanonicalTypeName, List<String> canonicalTypeNameParameters) {
+            String returnCanonicalTypeName, List<String> canonicalTypeNameParameters,
+            List<Object> painlessAnnotations) {
 
         this.origin = Objects.requireNonNull(origin);
         this.augmentedCanonicalClassName = augmentedCanonicalClassName;
         this.methodName = methodName;
         this.returnCanonicalTypeName = Objects.requireNonNull(returnCanonicalTypeName);
         this.canonicalTypeNameParameters = Collections.unmodifiableList(Objects.requireNonNull(canonicalTypeNameParameters));
+
+        if (painlessAnnotations.isEmpty()) {
+            this.painlessAnnotations = Collections.emptyMap();
+        } else {
+            this.painlessAnnotations = Collections.unmodifiableMap(Objects.requireNonNull(painlessAnnotations).stream()
+                    .map(painlessAnnotation -> new AbstractMap.SimpleEntry<>(painlessAnnotation.getClass(), painlessAnnotation))
+                    .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TypesExistsAction.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DeprecatedAnnotation.java
similarity index 63%
rename from server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TypesExistsAction.java
rename to modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DeprecatedAnnotation.java
index 0b508110d7f50..b7162e3d6bf99 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TypesExistsAction.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DeprecatedAnnotation.java
@@ -16,21 +16,20 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.action.admin.indices.exists.types;
 
-import org.elasticsearch.action.Action;
+package org.elasticsearch.painless.spi.annotation;
 
-public class TypesExistsAction extends Action<TypesExistsResponse> {
+public class DeprecatedAnnotation {
 
-    public static final TypesExistsAction INSTANCE = new TypesExistsAction();
-    public static final String NAME = "indices:admin/types/exists";
+    public static final String NAME = "deprecated";
 
-    private TypesExistsAction() {
-        super(NAME);
+    private final String message;
+
+    public DeprecatedAnnotation(String message) {
+        this.message = message;
     }
 
-    @Override
-    public TypesExistsResponse newResponse() {
-        return new TypesExistsResponse();
+    public String getMessage() {
+        return message;
     }
 }
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DeprecatedAnnotationParser.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DeprecatedAnnotationParser.java
new file mode 100644
index 0000000000000..c0992969c2278
--- /dev/null
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/DeprecatedAnnotationParser.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.spi.annotation;
+
+import java.util.Map;
+
+public class DeprecatedAnnotationParser implements WhitelistAnnotationParser {
+
+    public static final DeprecatedAnnotationParser INSTANCE = new DeprecatedAnnotationParser();
+
+    public static final String MESSAGE = "message";
+
+    private DeprecatedAnnotationParser() {
+
+    }
+
+    @Override
+    public Object parse(Map<String, String> arguments) {
+        String message = arguments.getOrDefault(MESSAGE, "");
+
+        if ((arguments.isEmpty() || arguments.size() == 1 && arguments.containsKey(MESSAGE)) == false) {
+            throw new IllegalArgumentException("unexpected parameters for [@deprecation] annotation, found " + arguments);
+        }
+
+        return new DeprecatedAnnotation(message);
+    }
+}
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SparseVectorFieldTypeTests.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NoImportAnnotation.java
similarity index 75%
rename from modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SparseVectorFieldTypeTests.java
rename to modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NoImportAnnotation.java
index 9fe8d17e697e6..ceec4740b3989 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SparseVectorFieldTypeTests.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NoImportAnnotation.java
@@ -17,12 +17,15 @@
  * under the License.
  */
 
-package org.elasticsearch.index.mapper;
+package org.elasticsearch.painless.spi.annotation;
 
-public class SparseVectorFieldTypeTests extends FieldTypeTestCase {
+public class NoImportAnnotation {
+
+    public static final String NAME = "no_import";
+
+    public static final NoImportAnnotation INSTANCE = new NoImportAnnotation();
+
+    private NoImportAnnotation() {
 
-    @Override
-    protected MappedFieldType createDefaultFieldType() {
-        return new SparseVectorFieldMapper.SparseVectorFieldType();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/IndicesExistsAction.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NoImportAnnotationParser.java
similarity index 59%
rename from server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/IndicesExistsAction.java
rename to modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NoImportAnnotationParser.java
index b878994549f5a..e8027b56e4109 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/IndicesExistsAction.java
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/NoImportAnnotationParser.java
@@ -17,21 +17,24 @@
  * under the License.
  */
 
-package org.elasticsearch.action.admin.indices.exists.indices;
+package org.elasticsearch.painless.spi.annotation;
 
-import org.elasticsearch.action.Action;
+import java.util.Map;
 
-public class IndicesExistsAction extends Action<IndicesExistsResponse> {
+public class NoImportAnnotationParser implements WhitelistAnnotationParser {
 
-    public static final IndicesExistsAction INSTANCE = new IndicesExistsAction();
-    public static final String NAME = "indices:admin/exists";
+    public static final NoImportAnnotationParser INSTANCE = new NoImportAnnotationParser();
+
+    private NoImportAnnotationParser() {
 
-    private IndicesExistsAction() {
-        super(NAME);
     }
 
     @Override
-    public IndicesExistsResponse newResponse() {
-        return new IndicesExistsResponse();
+    public Object parse(Map<String, String> arguments) {
+        if (arguments.isEmpty() == false) {
+            throw new IllegalArgumentException("unexpected parameters for [@no_import] annotation, found " + arguments);
+        }
+
+        return NoImportAnnotation.INSTANCE;
     }
 }
diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java
new file mode 100644
index 0000000000000..d5dcbab36f34f
--- /dev/null
+++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.spi.annotation;
+
+import java.util.AbstractMap;
+import java.util.Collections;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * WhitelistAnnotationParser is an interface used to define how to
+ * parse an annotation against any whitelist object while loading.
+ */
+public interface WhitelistAnnotationParser {
+
+    Map<String, WhitelistAnnotationParser> BASE_ANNOTATION_PARSERS = Collections.unmodifiableMap(
+            Stream.of(
+                    new AbstractMap.SimpleEntry<>(NoImportAnnotation.NAME, NoImportAnnotationParser.INSTANCE),
+                    new AbstractMap.SimpleEntry<>(DeprecatedAnnotation.NAME, DeprecatedAnnotationParser.INSTANCE)
+            ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))
+    );
+
+    Object parse(Map<String, String> arguments);
+}
diff --git a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextDocGenerator.java b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextDocGenerator.java
index babc3e10e55db..f09dd4f521ade 100644
--- a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextDocGenerator.java
+++ b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextDocGenerator.java
@@ -798,6 +798,7 @@ private static Map<String, String> getDisplayNames(List<PainlessContextClassInfo
 
     private static boolean isInternalClass(String javaName) {
         return  javaName.equals("org.elasticsearch.script.ScoreScript") ||
+                javaName.equals("org.elasticsearch.xpack.sql.expression.function.scalar.geo.GeoShape") ||
                 javaName.equals("org.elasticsearch.xpack.sql.expression.function.scalar.whitelist.InternalSqlScriptUtils") ||
                 javaName.equals("org.elasticsearch.xpack.sql.expression.literal.IntervalDayTime") ||
                 javaName.equals("org.elasticsearch.xpack.sql.expression.literal.IntervalYearMonth");
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
index 96a03fe5914bd..92cd4f575feec 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
@@ -23,14 +23,19 @@
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.painless.action.PainlessContextAction;
 import org.elasticsearch.painless.action.PainlessExecuteAction;
 import org.elasticsearch.painless.spi.PainlessExtension;
@@ -45,7 +50,10 @@
 import org.elasticsearch.script.ScoreScript;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptEngine;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.pipeline.MovingFunctionScript;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -103,8 +111,13 @@ public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<
     }
 
     @Override
-    public Collection<Module> createGuiceModules() {
-        return Collections.singleton(b -> b.bind(PainlessScriptEngine.class).toInstance(painlessScriptEngine.get()));
+    public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
+                                               ResourceWatcherService resourceWatcherService, ScriptService scriptService,
+                                               NamedXContentRegistry xContentRegistry, Environment environment,
+                                               NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) {
+        // this is a hack to bind the painless script engine in guice (all components are added to guice), so that
+        // the painless context api. this is a temporary measure until transport actions do no require guice
+        return Collections.singletonList(painlessScriptEngine.get());
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextAction.java
index c947de1fd82d3..5e941a2d0e2e0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextAction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessContextAction.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.painless.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
@@ -64,7 +64,7 @@
  *     retrieves all available information about the API for this specific context</li>
  * </ul>
  */
-public class PainlessContextAction extends Action<PainlessContextAction.Response> {
+public class PainlessContextAction extends ActionType<PainlessContextAction.Response> {
 
     public static final PainlessContextAction INSTANCE = new PainlessContextAction();
     private static final String NAME = "cluster:admin/scripts/painless/context";
@@ -75,11 +75,6 @@ private PainlessContextAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException();
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java
index cb407978da83e..d0a5d5cc9611b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java
@@ -30,8 +30,7 @@
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.store.RAMDirectory;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.ActionFilters;
@@ -90,7 +89,7 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
-public class PainlessExecuteAction extends Action<PainlessExecuteAction.Response> {
+public class PainlessExecuteAction extends ActionType<PainlessExecuteAction.Response> {
 
     public static final PainlessExecuteAction INSTANCE = new PainlessExecuteAction();
     private static final String NAME = "cluster:admin/scripts/painless/execute";
@@ -100,8 +99,8 @@ private PainlessExecuteAction() {
     }
 
     @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    public Writeable.Reader<Response> getResponseReader() {
+        return Response::new;
     }
 
     public static class Request extends SingleShardRequest<Request> implements ToXContentObject {
@@ -258,9 +257,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
         }
 
-        private Script script;
-        private ScriptContext<?> context = PainlessTestScript.CONTEXT;
-        private ContextSetup contextSetup;
+        private final Script script;
+        private final ScriptContext<?> context;
+        private final ContextSetup contextSetup;
 
         static Request parse(XContentParser parser) throws IOException {
             return PARSER.parse(parser, null);
@@ -268,16 +267,20 @@ static Request parse(XContentParser parser) throws IOException {
 
         Request(Script script, String scriptContextName, ContextSetup setup) {
             this.script = Objects.requireNonNull(script);
-            if (scriptContextName != null) {
-                this.context = fromScriptContextName(scriptContextName);
-            }
+            this.context = scriptContextName != null ? fromScriptContextName(scriptContextName) : PainlessTestScript.CONTEXT;
             if (setup != null) {
                 this.contextSetup = setup;
                 index(contextSetup.index);
+            } else {
+                contextSetup = null;
             }
         }
 
-        Request() {
+        Request(StreamInput in) throws IOException {
+            super(in);
+            script = new Script(in);
+            context = fromScriptContextName(in.readString());
+            contextSetup = in.readOptionalWriteable(ContextSetup::new);
         }
 
         public Script getScript() {
@@ -309,29 +312,12 @@ public ActionRequestValidationException validate() {
             return validationException;
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            script = new Script(in);
-            if (in.getVersion().before(Version.V_6_4_0)) {
-                byte scriptContextId = in.readByte();
-                assert scriptContextId == 0;
-            } else {
-                context = fromScriptContextName(in.readString());
-                contextSetup = in.readOptionalWriteable(ContextSetup::new);
-            }
-        }
-
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             script.writeTo(out);
-            if (out.getVersion().before(Version.V_6_4_0)) {
-                out.writeByte((byte) 0);
-            } else {
-                out.writeString(context.name);
-                out.writeOptionalWriteable(contextSetup);
-            }
+            out.writeString(context.name);
+            out.writeOptionalWriteable(contextSetup);
         }
 
         // For testing only:
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java
index bbbbc3dfc37cf..d0745dc982c36 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java
@@ -25,6 +25,7 @@
 import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.function.BiConsumer;
@@ -34,6 +35,7 @@
 import java.util.function.Function;
 import java.util.function.ObjIntConsumer;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 import java.util.function.ToDoubleFunction;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -552,4 +554,115 @@ public static String[] splitOnToken(String receiver, String token, int limit) {
         // O(N) or faster depending on implementation
         return result.toArray(new String[0]);
     }
+
+    /**
+     * Access values in nested containers with a dot separated path.  Path elements are treated
+     * as strings for Maps and integers for Lists.
+     * @throws IllegalArgumentException if any of the following:
+     *  - path is empty
+     *  - path contains a trailing '.' or a repeated '.'
+     *  - an element of the path does not exist, ie key or index not present
+     *  - there is a non-container type at a non-terminal path element
+     *  - a path element for a List is not an integer
+     * @return object at path
+     */
+    public static <E> Object getByPath(List<E> receiver, String path) {
+        return getByPathDispatch(receiver, splitPath(path), 0, throwCantFindValue(path));
+    }
+
+    /**
+     * Same as {@link #getByPath(List, String)}, but for Map.
+     */
+    public static <K,V> Object getByPath(Map<K,V> receiver, String path) {
+        return getByPathDispatch(receiver, splitPath(path), 0, throwCantFindValue(path));
+    }
+
+    /**
+     * Same as {@link #getByPath(List, String)}, but with a default value.
+     * @return element at path or {@code defaultValue} if the terminal path element does not exist.
+     */
+    public static <E> Object getByPath(List<E> receiver, String path, Object defaultValue) {
+        return getByPathDispatch(receiver, splitPath(path), 0, () -> defaultValue);
+    }
+
+    /**
+     * Same as {@link #getByPath(List, String, Object)}, but for Map.
+     */
+    public static <K,V> Object getByPath(Map<K,V> receiver, String path, Object defaultValue) {
+        return getByPathDispatch(receiver, splitPath(path), 0, () -> defaultValue);
+    }
+
+    // Dispatches to getByPathMap, getByPathList or returns obj if done. See handleMissing for dealing with missing
+    // elements.
+    private static Object getByPathDispatch(Object obj, String[] elements, int i, Supplier<Object> defaultSupplier) {
+        if (i > elements.length - 1) {
+            return obj;
+        } else if (elements[i].length() == 0 ) {
+            String format = "Extra '.' in path [%s] at index [%d]";
+            throw new IllegalArgumentException(String.format(Locale.ROOT, format, String.join(".", elements), i));
+        } else if (obj instanceof Map<?,?>) {
+            return getByPathMap((Map<?,?>) obj, elements, i, defaultSupplier);
+        } else if (obj instanceof List<?>) {
+            return getByPathList((List<?>) obj, elements, i, defaultSupplier);
+        }
+        return handleMissing(obj, elements, i, defaultSupplier);
+    }
+
+    // lookup existing key in map, call back to dispatch.
+    private static <K,V> Object getByPathMap(Map<K,V> map, String[] elements, int i, Supplier<Object> defaultSupplier) {
+        String element = elements[i];
+        if (map.containsKey(element)) {
+            return getByPathDispatch(map.get(element), elements, i + 1, defaultSupplier);
+        }
+        return handleMissing(map, elements, i, defaultSupplier);
+    }
+
+    // lookup existing index in list, call back to dispatch.  Throws IllegalArgumentException with NumberFormatException
+    // if index can't be parsed as an int.
+    private static <E> Object getByPathList(List<E> list, String[] elements, int i, Supplier<Object> defaultSupplier) {
+        String element = elements[i];
+        try {
+            int elemInt = Integer.parseInt(element);
+            if (list.size() >= elemInt) {
+                return getByPathDispatch(list.get(elemInt), elements, i + 1, defaultSupplier);
+            }
+        } catch (NumberFormatException e) {
+            String format = "Could not parse [%s] as a int index into list at path [%s] and index [%d]";
+            throw new IllegalArgumentException(String.format(Locale.ROOT, format, element, String.join(".", elements), i), e);
+        }
+        return handleMissing(list, elements, i, defaultSupplier);
+    }
+
+    // Split path on '.', throws IllegalArgumentException for empty paths and paths ending in '.'
+    private static String[] splitPath(String path) {
+        if (path.length() == 0) {
+            throw new IllegalArgumentException("Missing path");
+        }
+        if (path.endsWith(".")) {
+            String format = "Trailing '.' in path [%s]";
+            throw new IllegalArgumentException(String.format(Locale.ROOT, format, path));
+        }
+        return path.split("\\.");
+    }
+
+    // A supplier that throws IllegalArgumentException
+    private static Supplier<Object> throwCantFindValue(String path) {
+        return () -> {
+            throw new IllegalArgumentException(String.format(Locale.ROOT, "Could not find value at path [%s]", path));
+        };
+    }
+
+    // Use defaultSupplier if at last path element, otherwise throw IllegalArgumentException
+    private static Object handleMissing(Object obj, String[] elements, int i, Supplier<Object> defaultSupplier) {
+        if (obj instanceof List || obj instanceof Map) {
+            if (elements.length - 1 == i) {
+                return defaultSupplier.get();
+            }
+            String format = "Container does not have [%s], for non-terminal index [%d] in path [%s]";
+            throw new IllegalArgumentException(String.format(Locale.ROOT, format, elements[i], i, String.join(".", elements)));
+        }
+        String format = "Non-container [%s] at [%s], index [%d] in path [%s]";
+        throw new IllegalArgumentException(
+            String.format(Locale.ROOT, format, obj.getClass().getName(), elements[i], i, String.join(".", elements)));
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
index 6f4400cb4fbd4..0cef518e0d607 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.painless.spi.WhitelistField;
 import org.elasticsearch.painless.spi.WhitelistInstanceBinding;
 import org.elasticsearch.painless.spi.WhitelistMethod;
+import org.elasticsearch.painless.spi.annotation.NoImportAnnotation;
 import org.objectweb.asm.ClassWriter;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.commons.GeneratorAdapter;
@@ -119,7 +120,8 @@ public static PainlessLookup buildFromWhitelists(List<Whitelist> whitelists) {
                 for (WhitelistClass whitelistClass : whitelist.whitelistClasses) {
                     origin = whitelistClass.origin;
                     painlessLookupBuilder.addPainlessClass(
-                            whitelist.classLoader, whitelistClass.javaClassName, whitelistClass.noImport == false);
+                            whitelist.classLoader, whitelistClass.javaClassName,
+                            whitelistClass.painlessAnnotations.containsKey(NoImportAnnotation.class) == false);
                 }
             }
 
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/java.util.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/java.util.txt
index 94f302a891d48..958ac927a66dd 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/java.util.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/java.util.txt
@@ -126,6 +126,8 @@ class java.util.List {
   int org.elasticsearch.painless.api.Augmentation getLength()
   void sort(Comparator)
   List subList(int,int)
+  Object org.elasticsearch.painless.api.Augmentation getByPath(String)
+  Object org.elasticsearch.painless.api.Augmentation getByPath(String, Object)
 }
 
 class java.util.ListIterator {
@@ -161,6 +163,8 @@ class java.util.Map {
   void replaceAll(BiFunction)
   int size()
   Collection values()
+  Object org.elasticsearch.painless.api.Augmentation getByPath(String)
+  Object org.elasticsearch.painless.api.Augmentation getByPath(String, Object)
 
   # some adaptations of groovy methods
   List org.elasticsearch.painless.api.Augmentation collect(BiFunction)
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.score.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.score.txt
index 03ec9275aa8b7..fd49cbb2d5222 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.score.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.score.txt
@@ -19,7 +19,7 @@
 
 # This file contains a whitelist for functions to be used in Score context
 
-class org.elasticsearch.script.ScoreScript no_import {
+class org.elasticsearch.script.ScoreScript @no_import {
 }
 
 static_import {
@@ -27,10 +27,10 @@ static_import {
     double sigmoid(double, double, double) from_class org.elasticsearch.script.ScoreScriptUtils
     double randomScore(org.elasticsearch.script.ScoreScript, int, String) bound_to org.elasticsearch.script.ScoreScriptUtils$RandomScoreField
     double randomScore(org.elasticsearch.script.ScoreScript, int) bound_to org.elasticsearch.script.ScoreScriptUtils$RandomScoreDoc
-    double decayGeoLinear(String, String, String, double, GeoPoint)  bound_to org.elasticsearch.script.ScoreScriptUtils$DecayGeoLinear
-    double decayGeoExp(String, String, String, double, GeoPoint)  bound_to org.elasticsearch.script.ScoreScriptUtils$DecayGeoExp
-    double decayGeoGauss(String, String, String, double, GeoPoint)  bound_to org.elasticsearch.script.ScoreScriptUtils$DecayGeoGauss
-    double decayNumericLinear(double, double, double, double, double) bound_to org.elasticsearch.script.ScoreScriptUtils$DecayNumericLinear
+    double decayGeoLinear(String, String, String, double, GeoPoint) bound_to org.elasticsearch.script.ScoreScriptUtils$DecayGeoLinear
+    double decayGeoExp(String, String, String, double, GeoPoint) bound_to org.elasticsearch.script.ScoreScriptUtils$DecayGeoExp
+    double decayGeoGauss(String, String, String, double, GeoPoint) bound_to org.elasticsearch.script.ScoreScriptUtils$DecayGeoGauss
+    double decayNumericLinear(double, double, double, double, double)bound_to org.elasticsearch.script.ScoreScriptUtils$DecayNumericLinear
     double decayNumericExp(double, double, double, double, double) bound_to org.elasticsearch.script.ScoreScriptUtils$DecayNumericExp
     double decayNumericGauss(double, double, double, double, double) bound_to org.elasticsearch.script.ScoreScriptUtils$DecayNumericGauss
     double decayDateLinear(String, String, String, double, JodaCompatibleZonedDateTime) bound_to org.elasticsearch.script.ScoreScriptUtils$DecayDateLinear
@@ -38,4 +38,3 @@ static_import {
     double decayDateGauss(String, String, String, double, JodaCompatibleZonedDateTime) bound_to org.elasticsearch.script.ScoreScriptUtils$DecayDateGauss
 }
 
-
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt
index 412afc62dcf54..031fc342f339b 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt
@@ -24,31 +24,31 @@
 
 #### Primitive types
 
-class void no_import {
+class void @no_import {
 }
 
-class boolean no_import {
+class boolean @no_import {
 }
 
-class byte no_import {
+class byte @no_import {
 }
 
-class short no_import {
+class short @no_import {
 }
 
-class char no_import {
+class char @no_import {
 }
 
-class int no_import {
+class int @no_import {
 }
 
-class long no_import {
+class long @no_import {
 }
 
-class float no_import {
+class float @no_import {
 }
 
-class double no_import {
+class double @no_import {
 }
 
 #### Painless debugging API
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnnotationTestObject.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnnotationTestObject.java
new file mode 100644
index 0000000000000..bc9309bbbb611
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnnotationTestObject.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.painless.spi.annotation.WhitelistAnnotationParser;
+
+import java.util.Map;
+
+public class AnnotationTestObject {
+
+    public static class TestAnnotation {
+
+        public static final String NAME = "test_annotation";
+
+        private final String one;
+        private final String two;
+        private final String three;
+
+        public TestAnnotation(String one, String two, String three) {
+            this.one = one;
+            this.two = two;
+            this.three = three;
+        }
+
+        public String getOne() {
+            return one;
+        }
+
+        public String getTwo() {
+            return two;
+        }
+
+        public String getThree() {
+            return three;
+        }
+    }
+
+    public static class TestAnnotationParser implements WhitelistAnnotationParser {
+
+        public static final TestAnnotationParser INSTANCE = new TestAnnotationParser();
+
+        private TestAnnotationParser() {
+
+        }
+
+        @Override
+        public Object parse(Map<String, String> arguments) {
+            if (arguments.size() != 3) {
+                throw new IllegalArgumentException("expected three arguments");
+            }
+
+            String one = arguments.get("one");
+
+            if (one == null) {
+                throw new IllegalArgumentException("missing one");
+            }
+
+            String two = arguments.get("two");
+
+            if (two == null) {
+                throw new IllegalArgumentException("missing two");
+            }
+
+            String three = arguments.get("three");
+
+            if (three == null) {
+                throw new IllegalArgumentException("missing three");
+            }
+
+            return new TestAnnotation(one, two, three);
+        }
+    }
+
+    public void deprecatedMethod() {
+
+    }
+
+    public void annotatedTestMethod() {
+
+    }
+
+    public void annotatedMultipleMethod() {
+
+    }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java
index 70fbb733e2f8f..e462997444165 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java
@@ -232,7 +232,6 @@ public void testString_SplitOnToken() {
             new SplitCase("1\n1.1.\r\n1\r\n111", "\r\n"),
         };
         for (SplitCase split : cases) {
-            //System.out.println(String.format("Splitting '%s' by '%s' %d times", split.input, split.token, split.count));
             assertArrayEquals(
                 split.input.split(Pattern.quote(split.token), split.count),
                 (String[])exec("return \""+split.input+"\".splitOnToken(\""+split.token+"\", "+split.count+");")
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BindingsTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BindingsTests.java
index 6aca40fc4cdaf..171880abd7907 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BindingsTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BindingsTests.java
@@ -103,9 +103,9 @@ protected Map<ScriptContext<?>, List<Whitelist>> scriptContexts() {
 
         InstanceBindingTestClass instanceBindingTestClass = new InstanceBindingTestClass(1);
         WhitelistInstanceBinding getter = new WhitelistInstanceBinding("test", instanceBindingTestClass,
-                "setInstanceBindingValue", "void", Collections.singletonList("int"));
+                "setInstanceBindingValue", "void", Collections.singletonList("int"), Collections.emptyList());
         WhitelistInstanceBinding setter = new WhitelistInstanceBinding("test", instanceBindingTestClass,
-                "getInstanceBindingValue", "int", Collections.emptyList());
+                "getInstanceBindingValue", "int", Collections.emptyList(), Collections.emptyList());
         List<WhitelistInstanceBinding> instanceBindingsList = new ArrayList<>();
         instanceBindingsList.add(getter);
         instanceBindingsList.add(setter);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DateTimeTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DateTimeTests.java
new file mode 100644
index 0000000000000..38ceae74e053a
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DateTimeTests.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+
+public class DateTimeTests extends ScriptTestCase {
+
+    public void testLongToZonedDateTime() {
+        assertEquals(ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of("Z")), exec(
+                "long milliSinceEpoch = 434931330000L;" +
+                "Instant instant = Instant.ofEpochMilli(milliSinceEpoch);" +
+                "return ZonedDateTime.ofInstant(instant, ZoneId.of('Z'));"
+        ));
+    }
+
+    public void testStringToZonedDateTime() {
+        assertEquals(ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of("Z")), exec(
+                "String milliSinceEpochString = '434931330000';" +
+                "long milliSinceEpoch = Long.parseLong(milliSinceEpochString);" +
+                "Instant instant = Instant.ofEpochMilli(milliSinceEpoch);" +
+                "return ZonedDateTime.ofInstant(instant, ZoneId.of('Z'));"
+        ));
+
+        assertEquals(ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of("Z")), exec(
+                "String datetime = '1983-10-13T22:15:30Z';" +
+                "return ZonedDateTime.parse(datetime);"
+        ));
+
+        assertEquals(ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of("Z")), exec(
+                "String datetime = 'Thu, 13 Oct 1983 22:15:30 GMT';" +
+                "return ZonedDateTime.parse(datetime, DateTimeFormatter.RFC_1123_DATE_TIME);"
+        ));
+
+        assertEquals(ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of("Z")), exec(
+                "String datetime = 'custom y 1983 m 10 d 13 22:15:30 Z';" +
+                "DateTimeFormatter dtf = DateTimeFormatter.ofPattern(" +
+                        "\"'custom' 'y' yyyy 'm' MM 'd' dd HH:mm:ss VV\");" +
+                "return ZonedDateTime.parse(datetime, dtf);"
+        ));
+    }
+
+    public void testPiecesToZonedDateTime() {
+        assertEquals(ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of("Z")), exec(
+                "int year = 1983;" +
+                "int month = 10;" +
+                "int day = 13;" +
+                "int hour = 22;" +
+                "int minutes = 15;" +
+                "int seconds = 30;" +
+                "int nanos = 0;" +
+                "String tz = 'Z';" +
+                "return ZonedDateTime.of(year, month, day, hour, minutes, seconds, nanos, ZoneId.of(tz));"
+        ));
+    }
+
+    public void testZonedDatetimeToLong() {
+        assertEquals(434931330000L, exec(
+                "ZonedDateTime zdt = ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));" +
+                "return zdt.toInstant().toEpochMilli();"
+        ));
+    }
+
+    public void testZonedDateTimeToString() {
+        assertEquals("1983-10-13T22:15:30Z", exec(
+                "ZonedDateTime zdt = ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));" +
+                "return zdt.format(DateTimeFormatter.ISO_INSTANT);"
+        ));
+
+        assertEquals("date: 1983/10/13 time: 22:15:30", exec(
+                "ZonedDateTime zdt = ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));" +
+                "DateTimeFormatter dtf = DateTimeFormatter.ofPattern(" +
+                    "\"'date:' yyyy/MM/dd 'time:' HH:mm:ss\");" +
+                "return zdt.format(dtf);"
+        ));
+    }
+
+    public void testZonedDateTimeToPieces() {
+        assertArrayEquals(new int[] {1983, 10, 13, 22, 15, 30, 100}, (int[])exec(
+                "int[] pieces = new int[7];" +
+                "ZonedDateTime zdt = ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 100, ZoneId.of('Z'));" +
+                "pieces[0] = zdt.year;" +
+                "pieces[1] = zdt.monthValue;" +
+                "pieces[2] = zdt.dayOfMonth;" +
+                "pieces[3] = zdt.hour;" +
+                "pieces[4] = zdt.minute;" +
+                "pieces[5] = zdt.second;" +
+                "pieces[6] = zdt.nano;" +
+                "return pieces;"
+        ));
+    }
+
+    public void testLongManipulation() {
+        assertEquals(ZonedDateTime.of(1983, 10, 13, 22, 15, 27, 0, ZoneId.of("Z")), exec(
+                "long milliSinceEpoch = 434931330000L;" +
+                "milliSinceEpoch = milliSinceEpoch - 1000L*3L;" +
+                "Instant instant = Instant.ofEpochMilli(milliSinceEpoch);" +
+                "return ZonedDateTime.ofInstant(instant, ZoneId.of('Z'))"
+        ));
+    }
+
+    public void testZonedDateTimeManipulation() {
+        assertEquals(ZonedDateTime.of(1983, 10, 16, 22, 15, 30, 0, ZoneId.of("Z")), exec(
+                "ZonedDateTime zdt = ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));" +
+                "return zdt.plusDays(3);"
+        ));
+
+        assertEquals(ZonedDateTime.of(1983, 10, 13, 20, 10, 30, 0, ZoneId.of("Z")), exec(
+                "ZonedDateTime zdt = ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));" +
+                "return zdt.minusMinutes(125);"
+        ));
+
+        assertEquals(ZonedDateTime.of(1976, 10, 13, 22, 15, 30, 0, ZoneId.of("Z")), exec(
+                "ZonedDateTime zdt = ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));" +
+                "return zdt.withYear(1976);"
+        ));
+    }
+
+    public void testLongTimeDifference() {
+        assertEquals(3000L, exec(
+                "long startTimestamp = 434931327000L;" +
+                "long endTimestamp = 434931330000L;" +
+                "return endTimestamp - startTimestamp;"
+        ));
+    }
+
+    public void testZonedDateTimeDifference() {
+        assertEquals(4989L, exec(
+                "ZonedDateTime zdt1 = ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 11000000, ZoneId.of('Z'));" +
+                "ZonedDateTime zdt2 = ZonedDateTime.of(1983, 10, 13, 22, 15, 35, 0, ZoneId.of('Z'));" +
+                "return ChronoUnit.MILLIS.between(zdt1, zdt2);"
+        ));
+
+        assertEquals(4L, exec(
+                "ZonedDateTime zdt1 = ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 11000000, ZoneId.of('Z'));" +
+                "ZonedDateTime zdt2 = ZonedDateTime.of(1983, 10, 17, 22, 15, 35, 0, ZoneId.of('Z'));" +
+                "return ChronoUnit.DAYS.between(zdt1, zdt2);"
+        ));
+    }
+
+    public void compareLongs() {
+        assertEquals(false, exec(
+                "long ts1 = 434931327000L;" +
+                "long ts2 = 434931330000L;" +
+                "return ts1 > ts2;"
+        ));
+    }
+
+    public void compareZonedDateTimes() {
+        assertEquals(true, exec(
+                "ZonedDateTime zdt1 = ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));" +
+                "ZonedDateTime zdt2 = ZonedDateTime.of(1983, 10, 17, 22, 15, 35, 0, ZoneId.of('Z'));" +
+                "return zdt1.isBefore(zdt2);"
+        ));
+
+        assertEquals(false, exec(
+                "ZonedDateTime zdt1 = ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));" +
+                "ZonedDateTime zdt2 = ZonedDateTime.of(1983, 10, 17, 22, 15, 35, 0, ZoneId.of('Z'));" +
+                "return zdt1.isAfter(zdt2);"
+        ));
+    }
+
+    public void testTimeZone() {
+        assertEquals(ZonedDateTime.of(1983, 10, 13, 15, 15, 30, 0, ZoneId.of("America/Los_Angeles")), exec(
+                "ZonedDateTime utc = ZonedDateTime.of(1983, 10, 13, 22, 15, 30, 0, ZoneId.of('Z'));" +
+                "return utc.withZoneSameInstant(ZoneId.of('America/Los_Angeles'));"));
+
+        assertEquals("Thu, 13 Oct 1983 15:15:30 -0700", exec(
+                "String gmtString = 'Thu, 13 Oct 1983 22:15:30 GMT';" +
+                "ZonedDateTime gmtZdt = ZonedDateTime.parse(gmtString," +
+                "DateTimeFormatter.RFC_1123_DATE_TIME);" +
+                "ZonedDateTime pstZdt =" +
+                "gmtZdt.withZoneSameInstant(ZoneId.of('America/Los_Angeles'));" +
+                "return pstZdt.format(DateTimeFormatter.RFC_1123_DATE_TIME);"));
+    }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/GetByPathAugmentationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/GetByPathAugmentationTests.java
new file mode 100644
index 0000000000000..603ab7fd0e60c
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/GetByPathAugmentationTests.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+public class GetByPathAugmentationTests extends ScriptTestCase {
+
+    private final String k001Key = "k011";
+    private final String k001Value = "b";
+    private final Map<String,String> k001Obj = new HashMap<>();
+    private final String k001MapStr = "['" + k001Key + "': '" + k001Value + "']";
+    private final String mapMapList = "['k0': ['k01': [['k010': 'a'], " + k001MapStr + "]], 'k1': ['q']]";
+
+    private final String l2m2l1Index0 = "ll0";
+    private final String l2m2l1Index1 = "ll1";
+    private final List<String> l2m2l1Obj = new ArrayList<>();
+    private final String l2m2l1Str = "['" + l2m2l1Index0 + "', '" + l2m2l1Index1 + "']";
+    private final String listMapListList = "[['m0':'v0'],['m1':'v1'],['m2':['l0','l1', " + l2m2l1Str + "]]]";
+
+    private final String mapList = "['key0': ['a', 'b'], 'key1': ['c', 'd']]";
+    private final String mapMap = "['a': ['b': 'c']]";
+
+    public GetByPathAugmentationTests() {
+        l2m2l1Obj.add(l2m2l1Index0);
+        l2m2l1Obj.add(l2m2l1Index1);
+        k001Obj.put(k001Key, k001Value);
+    }
+
+    private String toScript(String collection, String key) {
+        return String.format(Locale.ROOT, "return %s.getByPath('%s')", collection, key);
+    }
+
+    private String toScript(String collection, String key, String defaultValue) {
+        return String.format(Locale.ROOT, "return %s.getByPath('%s', %s)", collection, key, defaultValue);
+    }
+
+    private String numberFormat(String unparsable, String path, int i) {
+        String format = "Could not parse [%s] as a int index into list at path [%s] and index [%d]";
+        return String.format(Locale.ROOT, format, unparsable, path, i);
+    }
+
+    private String missingValue(String path) {
+        return String.format(Locale.ROOT, "Could not find value at path [%s]", path);
+    }
+
+    private void assertPathValue(String collection, String key, Object value) {
+        assertEquals(value, exec(toScript(collection, key)));
+    }
+
+    private void assertPathDefaultValue(String collection, String key, Object value, String defaultValue) {
+        assertEquals(value, exec(toScript(collection, key, defaultValue)));
+    }
+
+    private IllegalArgumentException assertPathError(String collection, String key, String message) {
+        return assertPathError(toScript(collection, key), message);
+    }
+
+    private IllegalArgumentException assertPathError(String collection, String key, String defaultValue, String message) {
+        return assertPathError(toScript(collection, key, defaultValue), message);
+    }
+
+    private IllegalArgumentException assertPathError(String script, String message) {
+        IllegalArgumentException illegal = expectScriptThrows(
+            IllegalArgumentException.class,
+            () -> exec(script)
+        );
+        assertEquals(message, illegal.getMessage());
+        return illegal;
+    }
+
+    public void testOneLevelMap() {
+        assertPathValue("['k0':'v0']", "k0", "v0");
+    }
+
+    public void testOneLevelList() {
+        assertPathValue("['a','b','c','d']", "2", "c");
+    }
+
+    public void testTwoLevelMapList() {
+        assertPathValue("['key0': ['a', 'b'], 'key1': ['c', 'd']]", "key1.0", "c");
+    }
+
+    public void testMapDiffSizeList() {
+        assertPathValue("['k0': ['a','b','c','d'], 'k1': ['q']]", "k0.3", "d");
+    }
+
+    public void testBiMapList() {
+        assertPathValue(mapMapList, "k0.k01.1.k011", k001Value);
+    }
+
+    public void testBiMapListObject() {
+        assertPathValue(mapMapList, "k0.k01.1", k001Obj);
+    }
+
+    public void testListMap() {
+        assertPathValue("[['key0': 'value0'], ['key1': 'value1']]", "1.key1", "value1");
+    }
+
+    public void testTriList() {
+        assertPathValue("[['a','b'],['c','d'],[['e','f'],['g','h']]]", "2.1.1", "h");
+    }
+
+    public void testMapBiListObject() {
+        assertPathValue(listMapListList, "2.m2.2", l2m2l1Obj);
+    }
+
+    public void testMapBiList() {
+        assertPathValue(listMapListList, "2.m2.2.1", l2m2l1Index1);
+    }
+
+    public void testGetCollection() {
+        List<String> k1List = new ArrayList<>();
+        k1List.add("c");
+        k1List.add("d");
+        assertPathValue("['key0': ['a', 'b'], 'key1': ['c', 'd']]", "key1", k1List);
+    }
+
+    public void testMapListDefaultOneLevel() {
+        assertPathDefaultValue(mapList, "key2", "x", "'x'");
+    }
+
+    public void testMapListDefaultTwoLevel() {
+        assertPathDefaultValue(mapList, "key1.1", "d", "'x'");
+    }
+
+    public void testBiMapListDefault() {
+        assertPathDefaultValue(mapMapList, "k0.k01.1.k012", "foo", "'foo'");
+    }
+
+    public void testBiMapListDefaultExists() {
+        assertPathDefaultValue(mapMapList, "k0.k01.1.k011", "b", "'foo'");
+    }
+
+    public void testBiMapListDefaultObjectExists() {
+        assertPathDefaultValue(mapMapList, "k0.k01.1", k001Obj, "'foo'");
+    }
+
+    public void testBiMapListDefaultObject() {
+        assertPathDefaultValue(mapMapList, "k0.k01.9", k001Obj, k001MapStr);
+    }
+
+    public void testListMapBiListDefaultExists() {
+        assertPathDefaultValue(listMapListList, "2.m2.2", l2m2l1Obj, "'foo'");
+    }
+
+    public void testListMapBiListDefaultObject() {
+        assertPathDefaultValue(listMapListList, "2.m2.9", l2m2l1Obj, l2m2l1Str);
+    }
+
+    public void testBiListBadIndex() {
+        String path = "1.k0";
+        IllegalArgumentException err = assertPathError("[['a','b'],['c','d']]", path, numberFormat("k0", path, 1));
+        assertEquals(err.getCause().getClass(), NumberFormatException.class);
+    }
+
+    public void testBiMapListMissingLast() {
+        String path = "k0.k01.1.k012";
+        assertPathError(mapMapList, path, missingValue(path));
+    }
+
+    public void testBiMapListBadIndex() {
+        String path = "k0.k01.k012";
+        IllegalArgumentException err = assertPathError(mapMapList, path, numberFormat("k012", path, 2));
+        assertEquals(err.getCause().getClass(), NumberFormatException.class);
+    }
+
+    public void testListMapBiListMissingObject() {
+        String path = "2.m2.12";
+        assertPathError(listMapListList, path, missingValue(path));
+    }
+
+    public void testListMapBiListBadIndexAtObject() {
+        String path = "2.m2.a8";
+        IllegalArgumentException err = assertPathError(listMapListList, path, numberFormat("a8", path, 2));
+        assertEquals(err.getCause().getClass(), NumberFormatException.class);
+    }
+
+    public void testNonContainer() {
+        assertPathError(mapMap, "a.b.c", "Non-container [java.lang.String] at [c], index [2] in path [a.b.c]");
+    }
+
+    public void testMissingPath() {
+        assertPathError(mapMap, "", "Missing path");
+    }
+
+    public void testDoubleDot() {
+        assertPathError(mapMap, "a..b", "Extra '.' in path [a..b] at index [1]");
+    }
+
+    public void testTrailingDot() {
+        assertPathError(mapMap, "a.b.", "Trailing '.' in path [a.b.]");
+    }
+
+    public void testBiListDefaultBadIndex() {
+        String path = "1.k0";
+        IllegalArgumentException err = assertPathError(
+            "[['a','b'],['c','d']]",
+            path,
+            "'foo'",
+            numberFormat("k0", path, 1));
+        assertEquals(err.getCause().getClass(), NumberFormatException.class);
+    }
+
+    public void testBiMapListDefaultBadIndex() {
+        String path = "k0.k01.k012";
+        IllegalArgumentException err = assertPathError(
+            mapMapList,
+            path,
+            "'foo'",
+            numberFormat("k012", path, 2));
+        assertEquals(err.getCause().getClass(), NumberFormatException.class);
+    }
+
+    public void testListMapBiListObjectDefaultBadIndex() {
+        String path = "2.m2.a8";
+        IllegalArgumentException err = assertPathError(
+            listMapListList,
+            path,
+            "'foo'",
+            numberFormat("a8", path, 2));
+        assertEquals(err.getCause().getClass(), NumberFormatException.class);
+    }
+
+    public void testNonContainerDefaultBadIndex() {
+        assertPathError(mapMap, "a.b.c", "'foo'",
+            "Non-container [java.lang.String] at [c], index [2] in path [a.b.c]");
+    }
+
+    public void testDoubleDotDefault() {
+        assertPathError(mapMap, "a..b", "'foo'", "Extra '.' in path [a..b] at index [1]");
+    }
+
+    public void testTrailingDotDefault() {
+        assertPathError(mapMap, "a.b.", "'foo'", "Trailing '.' in path [a.b.]");
+    }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
index eeb636d6697c6..92816426eaf47 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
@@ -47,7 +47,7 @@ public void testNeedsScores() {
         PainlessScriptEngine service = new PainlessScriptEngine(Settings.EMPTY, contexts);
 
         QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null);
-        SearchLookup lookup = new SearchLookup(index.mapperService(), shardContext::getForField, null);
+        SearchLookup lookup = new SearchLookup(index.mapperService(), shardContext::getForField);
 
         NumberSortScript.Factory factory = service.compile(null, "1.2", NumberSortScript.CONTEXT, Collections.emptyMap());
         NumberSortScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), lookup);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
deleted file mode 100644
index c1ba6bfbe1c75..0000000000000
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
+++ /dev/null
@@ -1,451 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.core.internal.io.IOUtils;
-import org.elasticsearch.painless.lookup.PainlessClass;
-import org.elasticsearch.painless.lookup.PainlessConstructor;
-import org.elasticsearch.painless.lookup.PainlessField;
-import org.elasticsearch.painless.lookup.PainlessLookup;
-import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
-import org.elasticsearch.painless.lookup.PainlessLookupUtility;
-import org.elasticsearch.painless.lookup.PainlessMethod;
-import org.elasticsearch.painless.lookup.def;
-import org.elasticsearch.painless.spi.Whitelist;
-
-import java.io.IOException;
-import java.io.PrintStream;
-import java.lang.reflect.Modifier;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
-
-import static java.util.Comparator.comparing;
-
-/**
- * Generates an API reference from the method and type whitelists in {@link PainlessLookup}.
- */
-public class PainlessDocGenerator {
-
-    private static final PainlessLookup PAINLESS_LOOKUP = PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS);
-    private static final Logger logger = LogManager.getLogger(PainlessDocGenerator.class);
-    private static final Comparator<PainlessField> FIELD_NAME = comparing(f -> f.javaField.getName());
-    private static final Comparator<PainlessMethod> METHOD_NAME = comparing(m -> m.javaMethod.getName());
-    private static final Comparator<PainlessMethod> METHOD_NUMBER_OF_PARAMS = comparing(m -> m.typeParameters.size());
-    private static final Comparator<PainlessConstructor> CONSTRUCTOR_NUMBER_OF_PARAMS = comparing(m -> m.typeParameters.size());
-
-    public static void main(String[] args) throws IOException {
-        Path apiRootPath = PathUtils.get(args[0]);
-
-        // Blow away the last execution and recreate it from scratch
-        IOUtils.rm(apiRootPath);
-        Files.createDirectories(apiRootPath);
-
-        Path indexPath = apiRootPath.resolve("index.asciidoc");
-        logger.info("Starting to write [index.asciidoc]");
-        try (PrintStream indexStream = new PrintStream(
-            Files.newOutputStream(indexPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE),
-            false, StandardCharsets.UTF_8.name())) {
-            emitGeneratedWarning(indexStream);
-            List<Class<?>> classes = PAINLESS_LOOKUP.getClasses().stream().sorted(
-                    Comparator.comparing(Class::getCanonicalName)).collect(Collectors.toList());
-            for (Class<?> clazz : classes) {
-                PainlessClass struct = PAINLESS_LOOKUP.lookupPainlessClass(clazz);
-                String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(clazz);
-
-                if (clazz.isPrimitive()) {
-                    // Primitives don't have methods to reference
-                    continue;
-                }
-                if (clazz == def.class) {
-                    // def is special but doesn't have any methods all of its own.
-                    continue;
-                }
-                indexStream.print("include::");
-                indexStream.print(canonicalClassName);
-                indexStream.println(".asciidoc[]");
-
-                Path typePath = apiRootPath.resolve(canonicalClassName + ".asciidoc");
-                logger.info("Writing [{}.asciidoc]", canonicalClassName);
-                try (PrintStream typeStream = new PrintStream(
-                        Files.newOutputStream(typePath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE),
-                        false, StandardCharsets.UTF_8.name())) {
-                    emitGeneratedWarning(typeStream);
-                    typeStream.print("[[");
-                    emitAnchor(typeStream, clazz);
-                    typeStream.print("]]++");
-                    typeStream.print(canonicalClassName);
-                    typeStream.println("++::");
-
-                    Consumer<PainlessField> documentField = field -> PainlessDocGenerator.documentField(typeStream, field);
-                    Consumer<PainlessMethod> documentMethod = method -> PainlessDocGenerator.documentMethod(typeStream, method);
-                    Consumer<PainlessConstructor> documentConstructor =
-                            constructor -> PainlessDocGenerator.documentConstructor(typeStream, constructor);
-                    struct.staticFields.values().stream().sorted(FIELD_NAME).forEach(documentField);
-                    struct.fields.values().stream().sorted(FIELD_NAME).forEach(documentField);
-                    struct.staticMethods.values().stream().sorted(
-                            METHOD_NAME.thenComparing(METHOD_NUMBER_OF_PARAMS)).forEach(documentMethod);
-                    struct.constructors.values().stream().sorted(CONSTRUCTOR_NUMBER_OF_PARAMS).forEach(documentConstructor);
-                    Map<String, Class<?>> inherited = new TreeMap<>();
-                    struct.methods.values().stream().sorted(METHOD_NAME.thenComparing(METHOD_NUMBER_OF_PARAMS)).forEach(method -> {
-                        if (method.targetClass == clazz) {
-                            documentMethod(typeStream, method);
-                        } else {
-                            inherited.put(canonicalClassName, method.targetClass);
-                        }
-                    });
-
-                    if (false == inherited.isEmpty()) {
-                        typeStream.print("* Inherits methods from ");
-                        boolean first = true;
-                        for (Class<?> inheritsFrom : inherited.values()) {
-                            if (first) {
-                                first = false;
-                            } else {
-                                typeStream.print(", ");
-                            }
-                            typeStream.print("++");
-                            emitStruct(typeStream, inheritsFrom);
-                            typeStream.print("++");
-                        }
-                        typeStream.println();
-                    }
-                }
-            }
-        }
-        logger.info("Done writing [index.asciidoc]");
-    }
-
-    private static void documentField(PrintStream stream, PainlessField field) {
-        stream.print("** [[");
-        emitAnchor(stream, field);
-        stream.print("]]");
-
-        if (Modifier.isStatic(field.javaField.getModifiers())) {
-            stream.print("static ");
-        }
-
-        emitType(stream, field.typeParameter);
-        stream.print(' ');
-
-        String javadocRoot = javadocRoot(field);
-        emitJavadocLink(stream, javadocRoot, field);
-        stream.print('[');
-        stream.print(field.javaField.getName());
-        stream.print(']');
-
-        if (javadocRoot.equals("java8")) {
-            stream.print(" (");
-            emitJavadocLink(stream, "java9", field);
-            stream.print("[java 9])");
-        }
-
-        stream.println();
-    }
-
-    /**
-     * Document a constructor.
-     */
-    private static void documentConstructor(PrintStream stream, PainlessConstructor constructor) {
-        stream.print("* ++[[");
-        emitAnchor(stream, constructor);
-        stream.print("]]");
-
-        String javadocRoot = javadocRoot(constructor.javaConstructor.getDeclaringClass());
-        emitJavadocLink(stream, javadocRoot, constructor);
-        stream.print('[');
-
-        stream.print(constructorName(constructor));
-
-        stream.print("](");
-        boolean first = true;
-        for (Class<?> arg : constructor.typeParameters) {
-            if (first) {
-                first = false;
-            } else {
-                stream.print(", ");
-            }
-            emitType(stream, arg);
-        }
-        stream.print(")++");
-
-        if (javadocRoot.equals("java8")) {
-            stream.print(" (");
-            emitJavadocLink(stream, "java9", constructor);
-            stream.print("[java 9])");
-        }
-
-        stream.println();
-    }
-
-    /**
-     * Document a method.
-     */
-    private static void documentMethod(PrintStream stream, PainlessMethod method) {
-        stream.print("* ++[[");
-        emitAnchor(stream, method);
-        stream.print("]]");
-
-        if (method.targetClass == method.javaMethod.getDeclaringClass() && Modifier.isStatic(method.javaMethod.getModifiers())) {
-            stream.print("static ");
-        }
-
-        emitType(stream, method.returnType);
-        stream.print(' ');
-
-        String javadocRoot = javadocRoot(method);
-        emitJavadocLink(stream, javadocRoot, method);
-        stream.print('[');
-
-        stream.print(methodName(method));
-
-        stream.print("](");
-        boolean first = true;
-        for (Class<?> arg : method.typeParameters) {
-            if (first) {
-                first = false;
-            } else {
-                stream.print(", ");
-            }
-            emitType(stream, arg);
-        }
-        stream.print(")++");
-
-        if (javadocRoot.equals("java8")) {
-            stream.print(" (");
-            emitJavadocLink(stream, "java9", method);
-            stream.print("[java 9])");
-        }
-
-        stream.println();
-    }
-
-    /**
-     * Anchor text for a {@link PainlessClass}.
-     */
-    private static void emitAnchor(PrintStream stream, Class<?> clazz) {
-        stream.print("painless-api-reference-");
-        stream.print(PainlessLookupUtility.typeToCanonicalTypeName(clazz).replace('.', '-'));
-    }
-
-    /**
-     * Anchor text for a {@link PainlessConstructor}.
-     */
-    private static void emitAnchor(PrintStream stream, PainlessConstructor constructor) {
-        emitAnchor(stream, constructor.javaConstructor.getDeclaringClass());
-        stream.print('-');
-        stream.print(constructorName(constructor));
-        stream.print('-');
-        stream.print(constructor.typeParameters.size());
-    }
-
-    /**
-     * Anchor text for a {@link PainlessMethod}.
-     */
-    private static void emitAnchor(PrintStream stream, PainlessMethod method) {
-        emitAnchor(stream, method.targetClass);
-        stream.print('-');
-        stream.print(methodName(method));
-        stream.print('-');
-        stream.print(method.typeParameters.size());
-    }
-
-    /**
-     * Anchor text for a {@link PainlessField}.
-     */
-    private static void emitAnchor(PrintStream stream, PainlessField field) {
-        emitAnchor(stream, field.javaField.getDeclaringClass());
-        stream.print('-');
-        stream.print(field.javaField.getName());
-    }
-
-    private static String constructorName(PainlessConstructor constructor) {
-        return PainlessLookupUtility.typeToCanonicalTypeName(constructor.javaConstructor.getDeclaringClass());
-    }
-
-    private static String methodName(PainlessMethod method) {
-        return PainlessLookupUtility.typeToCanonicalTypeName(method.targetClass);
-    }
-
-    /**
-     * Emit a {@link Class}. If the type is primitive or an array of primitives this just emits the name of the type. Otherwise this emits
-     an internal link with the text.
-     */
-    private static void emitType(PrintStream stream, Class<?> clazz) {
-        emitStruct(stream, clazz);
-        while ((clazz = clazz.getComponentType()) != null) {
-            stream.print("[]");
-        }
-    }
-
-    /**
-     * Emit a {@link PainlessClass}. If the {@linkplain PainlessClass} is primitive or def this just emits the name of the struct.
-     * Otherwise this emits an internal link with the name.
-     */
-    private static void emitStruct(PrintStream stream, Class<?> clazz) {
-        String canonicalClassName = PainlessLookupUtility.typeToCanonicalTypeName(clazz);
-
-        if (false == clazz.isPrimitive() && clazz != def.class) {
-            stream.print("<<");
-            emitAnchor(stream, clazz);
-            stream.print(',');
-            stream.print(canonicalClassName);
-            stream.print(">>");
-        } else {
-            stream.print(canonicalClassName);
-        }
-    }
-
-    /**
-     * Emit an external link to Javadoc for a {@link PainlessMethod}.
-     *
-     * @param root name of the root uri variable
-     */
-    private static void emitJavadocLink(PrintStream stream, String root, PainlessConstructor constructor) {
-        stream.print("link:{");
-        stream.print(root);
-        stream.print("-javadoc}/");
-        stream.print(classUrlPath(constructor.javaConstructor.getDeclaringClass()));
-        stream.print(".html#");
-        stream.print(constructorName(constructor));
-        stream.print("%2D");
-        boolean first = true;
-        for (Class<?> clazz: constructor.typeParameters) {
-            if (first) {
-                first = false;
-            } else {
-                stream.print("%2D");
-            }
-            stream.print(clazz.getName());
-            if (clazz.isArray()) {
-                stream.print(":A");
-            }
-        }
-        stream.print("%2D");
-    }
-
-    /**
-     * Emit an external link to Javadoc for a {@link PainlessMethod}.
-     *
-     * @param root name of the root uri variable
-     */
-    private static void emitJavadocLink(PrintStream stream, String root, PainlessMethod method) {
-        stream.print("link:{");
-        stream.print(root);
-        stream.print("-javadoc}/");
-        stream.print(classUrlPath(method.javaMethod.getDeclaringClass()));
-        stream.print(".html#");
-        stream.print(methodName(method));
-        stream.print("%2D");
-        boolean first = true;
-        if (method.targetClass != method.javaMethod.getDeclaringClass()) {
-            first = false;
-            stream.print(method.javaMethod.getDeclaringClass().getName());
-        }
-        for (Class<?> clazz: method.typeParameters) {
-            if (first) {
-                first = false;
-            } else {
-                stream.print("%2D");
-            }
-            stream.print(clazz.getName());
-            if (clazz.isArray()) {
-                stream.print(":A");
-            }
-        }
-        stream.print("%2D");
-    }
-
-    /**
-     * Emit an external link to Javadoc for a {@link PainlessField}.
-     *
-     * @param root name of the root uri variable
-     */
-    private static void emitJavadocLink(PrintStream stream, String root, PainlessField field) {
-        stream.print("link:{");
-        stream.print(root);
-        stream.print("-javadoc}/");
-        stream.print(classUrlPath(field.javaField.getDeclaringClass()));
-        stream.print(".html#");
-        stream.print(field.javaField.getName());
-    }
-
-    /**
-     * Pick the javadoc root for a {@link PainlessMethod}.
-     */
-    private static String javadocRoot(PainlessMethod method) {
-        if (method.targetClass != method.javaMethod.getDeclaringClass()) {
-            return "painless";
-        }
-        return javadocRoot(method.targetClass);
-    }
-
-    /**
-     * Pick the javadoc root for a {@link PainlessField}.
-     */
-    private static String javadocRoot(PainlessField field) {
-        return javadocRoot(field.javaField.getDeclaringClass());
-    }
-
-    /**
-     * Pick the javadoc root for a {@link Class}.
-     */
-    private static String javadocRoot(Class<?> clazz) {
-        String classPackage = clazz.getPackage().getName();
-        if (classPackage.startsWith("java")) {
-            return "java8";
-        }
-        if (classPackage.startsWith("org.elasticsearch.painless")) {
-            return "painless";
-        }
-        if (classPackage.startsWith("org.elasticsearch")) {
-            return "elasticsearch";
-        }
-        if (classPackage.startsWith("org.joda.time")) {
-            return "joda-time";
-        }
-        if (classPackage.startsWith("org.apache.lucene")) {
-            return "lucene-core";
-        }
-        throw new IllegalArgumentException("Unrecognized package: " + classPackage);
-    }
-
-    private static void emitGeneratedWarning(PrintStream stream) {
-        stream.println("////");
-        stream.println("Automatically generated by PainlessDocGenerator. Do not edit.");
-        stream.println("Rebuild by running `gradle generatePainlessApi`.");
-        stream.println("////");
-        stream.println();
-    }
-
-    private static String classUrlPath(Class<?> clazz) {
-        return clazz.getName().replace('.', '/').replace('$', '.');
-    }
-}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhitelistLoaderTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhitelistLoaderTests.java
new file mode 100644
index 0000000000000..7fe9dcce3bc09
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhitelistLoaderTests.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.painless.spi.Whitelist;
+import org.elasticsearch.painless.spi.WhitelistClass;
+import org.elasticsearch.painless.spi.WhitelistLoader;
+import org.elasticsearch.painless.spi.WhitelistMethod;
+import org.elasticsearch.painless.spi.annotation.DeprecatedAnnotation;
+import org.elasticsearch.painless.spi.annotation.NoImportAnnotation;
+import org.elasticsearch.painless.spi.annotation.WhitelistAnnotationParser;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class WhitelistLoaderTests extends ScriptTestCase {
+
+    public void testAnnotations() {
+        Map<String, WhitelistAnnotationParser> parsers = new HashMap<>(WhitelistAnnotationParser.BASE_ANNOTATION_PARSERS);
+        parsers.put(AnnotationTestObject.TestAnnotation.NAME, AnnotationTestObject.TestAnnotationParser.INSTANCE);
+        Whitelist whitelist = WhitelistLoader.loadFromResourceFiles(Whitelist.class, parsers, "org.elasticsearch.painless.annotation");
+
+        assertEquals(1, whitelist.whitelistClasses.size());
+
+        WhitelistClass whitelistClass = whitelist.whitelistClasses.get(0);
+
+        assertNotNull(whitelistClass.painlessAnnotations.get(NoImportAnnotation.class));
+        assertEquals(1, whitelistClass.painlessAnnotations.size());
+        assertEquals(3, whitelistClass.whitelistMethods.size());
+
+        int count = 0;
+
+        for (WhitelistMethod whitelistMethod : whitelistClass.whitelistMethods) {
+            if ("deprecatedMethod".equals(whitelistMethod.methodName)) {
+                assertEquals("use another method",
+                        ((DeprecatedAnnotation)whitelistMethod.painlessAnnotations.get(DeprecatedAnnotation.class)).getMessage());
+                assertEquals(1, whitelistMethod.painlessAnnotations.size());
+                ++count;
+            }
+
+            if ("annotatedTestMethod".equals(whitelistMethod.methodName)) {
+                AnnotationTestObject.TestAnnotation ta =
+                        ((AnnotationTestObject.TestAnnotation)whitelistMethod.painlessAnnotations.get(
+                                AnnotationTestObject.TestAnnotation.class));
+                assertEquals("one", ta.getOne());
+                assertEquals("two", ta.getTwo());
+                assertEquals("three", ta.getThree());
+                assertEquals(1, whitelistMethod.painlessAnnotations.size());
+                ++count;
+            }
+
+            if ("annotatedMultipleMethod".equals(whitelistMethod.methodName)) {
+                assertEquals("test",
+                        ((DeprecatedAnnotation)whitelistMethod.painlessAnnotations.get(DeprecatedAnnotation.class)).getMessage());
+                AnnotationTestObject.TestAnnotation ta =
+                        ((AnnotationTestObject.TestAnnotation)whitelistMethod.painlessAnnotations.get(
+                                AnnotationTestObject.TestAnnotation.class));
+                assertEquals("one", ta.getOne());
+                assertEquals("two", ta.getTwo());
+                assertEquals("three", ta.getThree());
+                assertEquals(2, whitelistMethod.painlessAnnotations.size());
+                ++count;
+            }
+        }
+
+        assertEquals(3, count);
+    }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java
index 51801c29095fd..d0996c4c2591c 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -35,7 +36,7 @@
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -43,7 +44,7 @@
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class PainlessExecuteRequestTests extends AbstractStreamableTestCase<PainlessExecuteAction.Request> {
+public class PainlessExecuteRequestTests extends AbstractWireSerializingTestCase<PainlessExecuteAction.Request> {
 
     // Testing XContent serialization manually here, because the xContentType field in ContextSetup determines
     // how the request needs to parse and the xcontent serialization framework randomizes that. The XContentType
@@ -72,12 +73,12 @@ public final void testFromXContent() throws Exception {
 
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        return new NamedWriteableRegistry(new SearchModule(Settings.EMPTY, false, Collections.emptyList()).getNamedWriteables());
+        return new NamedWriteableRegistry(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedWriteables());
     }
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        return new NamedXContentRegistry(new SearchModule(Settings.EMPTY, false, Collections.emptyList()).getNamedXContents());
+        return new NamedXContentRegistry(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents());
     }
 
     @Override
@@ -89,8 +90,8 @@ protected PainlessExecuteAction.Request createTestInstance() {
     }
 
     @Override
-    protected PainlessExecuteAction.Request createBlankInstance() {
-        return new PainlessExecuteAction.Request();
+    protected Writeable.Reader<PainlessExecuteAction.Request> instanceReader() {
+        return PainlessExecuteAction.Request::new;
     }
 
     public void testValidate() {
diff --git a/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.annotation b/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.annotation
new file mode 100644
index 0000000000000..04e22b6dded96
--- /dev/null
+++ b/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.annotation
@@ -0,0 +1,7 @@
+# whitelist for annotation tests
+
+class org.elasticsearch.painless.AnnotationTestObject @no_import {
+  void deprecatedMethod() @deprecated[message="use another method"]
+  void annotatedTestMethod() @test_annotation[one="one",two="two",three="three"]
+  void annotatedMultipleMethod() @test_annotation[one="one",two="two",three="three"] @deprecated[message="test"]
+}
\ No newline at end of file
diff --git a/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.test b/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.test
index 37faf50a70126..3ac8fa0ee0c91 100644
--- a/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.test
+++ b/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.test
@@ -2,7 +2,7 @@
 class org.elasticsearch.painless.BindingsTests$BindingsTestScript {
 }
 
-class org.elasticsearch.painless.FeatureTestObject no_import {
+class org.elasticsearch.painless.FeatureTestObject @no_import {
   int z
   ()
   (int,int)
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/scripts_painless_context.json b/modules/lang-painless/src/test/resources/rest-api-spec/api/scripts_painless_context.json
similarity index 88%
rename from rest-api-spec/src/main/resources/rest-api-spec/api/scripts_painless_context.json
rename to modules/lang-painless/src/test/resources/rest-api-spec/api/scripts_painless_context.json
index 1eb490e8ab59f..cb078ec4a00b8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/scripts_painless_context.json
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/api/scripts_painless_context.json
@@ -1,8 +1,8 @@
 {
   "scripts_painless_context": {
+    "stability": "experimental",
     "methods": ["GET"],
     "url": {
-      "path": "/_scripts/painless/_context",
       "paths": ["/_scripts/painless/_context"],
       "parts": {
       },
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/100_terms_agg.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/100_terms_agg.yml
new file mode 100644
index 0000000000000..774a5dd59b0a0
--- /dev/null
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/100_terms_agg.yml
@@ -0,0 +1,104 @@
+setup:
+  - do:
+      indices.create:
+          index: test_1
+          body:
+            settings:
+              number_of_replicas: 0
+            mappings:
+              properties:
+                str:
+                   type: keyword
+                double:
+                  type: double
+                number:
+                  type: long
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+
+  - do:
+      index:
+        index: test_1
+        id: 1
+        body:
+          str: "abc"
+          double: 1.0
+          number: 1
+
+  - do:
+      index:
+        index: test_1
+        id: 2
+        body:
+          str: "abc"
+          double: 1.0
+          number: 1
+
+  - do:
+      index:
+        index: test_1
+        id: 3
+        body:
+          str: "bcd"
+          double: 2.0
+          number: 2
+
+  - do:
+      indices.refresh: {}
+
+---
+"String Value Script with doc notation":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body: { "size" : 0, "aggs" : { "str_terms" : { "terms" : { "field" : "str", "script": { "source": "return doc.str[0] + \"1\""} } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.str_terms.buckets: 2 }
+  - match: { aggregations.str_terms.buckets.0.key: "abc1" }
+  - is_false: aggregations.str_terms.buckets.0.key_as_string
+  - match: { aggregations.str_terms.buckets.0.doc_count: 2 }
+  - match: { aggregations.str_terms.buckets.1.key: "bcd1" }
+  - is_false: aggregations.str_terms.buckets.1.key_as_string
+  - match: { aggregations.str_terms.buckets.1.doc_count: 1 }
+
+---
+"Long Value Script with doc notation":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body: { "size" : 0, "aggs" : { "long_terms" : { "terms" : { "field" : "number", "script": { "source": "return doc.number[0] + 1"} } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.long_terms.buckets: 2 }
+  - match: { aggregations.long_terms.buckets.0.key: 2.0 }
+  - is_false: aggregations.long_terms.buckets.0.key_as_string
+  - match: { aggregations.long_terms.buckets.0.doc_count: 2 }
+  - match: { aggregations.long_terms.buckets.1.key: 3.0 }
+  - is_false: aggregations.long_terms.buckets.1.key_as_string
+  - match: { aggregations.long_terms.buckets.1.doc_count: 1 }
+
+---
+"Double Value Script with doc notation":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body: { "size" : 0, "aggs" : { "double_terms" : { "terms" : { "field" : "double", "script": { "source": "return doc.double[0] + 1"} } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.double_terms.buckets: 2 }
+  - match: { aggregations.double_terms.buckets.0.key: 2.0 }
+  - is_false: aggregations.double_terms.buckets.0.key_as_string
+  - match: { aggregations.double_terms.buckets.0.doc_count: 2 }
+  - match: { aggregations.double_terms.buckets.1.key: 3.0 }
+  - is_false: aggregations.double_terms.buckets.1.key_as_string
+  - match: { aggregations.double_terms.buckets.1.doc_count: 1 }
+
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/71_context_api.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/71_context_api.yml
index a185ede4db662..0413661fc586c 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/71_context_api.yml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/71_context_api.yml
@@ -17,6 +17,6 @@
     - match: { classes.6.methods.0.parameters.0 : java.lang.CharSequence }
     - match: { classes.6.methods.0.parameters.1 : int }
     - match: { classes.6.methods.0.parameters.2 : int }
-    - match: { imported_methods.0.name: dotProduct }
-    - match: { class_bindings.0.name: cosineSimilarity }
+    - match: { imported_methods.0.name: saturation }
+    - match: { class_bindings.0.name: decayDateExp }
     - match: { instance_bindings: [] }
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java
index 45a067d7994d2..569e20a14161f 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java
+++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java
@@ -39,8 +39,6 @@ public Map<String, Mapper.TypeParser> getMappers() {
         mappers.put(TokenCountFieldMapper.CONTENT_TYPE, new TokenCountFieldMapper.TypeParser());
         mappers.put(RankFeatureFieldMapper.CONTENT_TYPE, new RankFeatureFieldMapper.TypeParser());
         mappers.put(RankFeaturesFieldMapper.CONTENT_TYPE, new RankFeaturesFieldMapper.TypeParser());
-        mappers.put(DenseVectorFieldMapper.CONTENT_TYPE, new DenseVectorFieldMapper.TypeParser());
-        mappers.put(SparseVectorFieldMapper.CONTENT_TYPE, new SparseVectorFieldMapper.TypeParser());
         mappers.put(SearchAsYouTypeFieldMapper.CONTENT_TYPE, new SearchAsYouTypeFieldMapper.TypeParser());
         return Collections.unmodifiableMap(mappers);
     }
diff --git a/modules/mapper-extras/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/modules/mapper-extras/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension
deleted file mode 100644
index f4cc27a362e51..0000000000000
--- a/modules/mapper-extras/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension
+++ /dev/null
@@ -1 +0,0 @@
-org.elasticsearch.index.query.DocValuesWhitelistExtension
\ No newline at end of file
diff --git a/modules/mapper-extras/src/main/resources/org/elasticsearch/index/query/docvalues_whitelist.txt b/modules/mapper-extras/src/main/resources/org/elasticsearch/index/query/docvalues_whitelist.txt
deleted file mode 100644
index 3a8989e20b020..0000000000000
--- a/modules/mapper-extras/src/main/resources/org/elasticsearch/index/query/docvalues_whitelist.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-#
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-class org.elasticsearch.index.query.VectorScriptDocValues {
-}
-class org.elasticsearch.index.query.VectorScriptDocValues$DenseVectorScriptDocValues {
-}
-class org.elasticsearch.index.query.VectorScriptDocValues$SparseVectorScriptDocValues {
-}
-
-static_import {
-    double cosineSimilarity(List, VectorScriptDocValues.DenseVectorScriptDocValues) bound_to org.elasticsearch.index.query.ScoreScriptUtils$CosineSimilarity
-    double dotProduct(List, VectorScriptDocValues.DenseVectorScriptDocValues) from_class org.elasticsearch.index.query.ScoreScriptUtils
-    double dotProductSparse(Map, VectorScriptDocValues.SparseVectorScriptDocValues) bound_to org.elasticsearch.index.query.ScoreScriptUtils$DotProductSparse
-    double cosineSimilaritySparse(Map, VectorScriptDocValues.SparseVectorScriptDocValues) bound_to org.elasticsearch.index.query.ScoreScriptUtils$CosineSimilaritySparse
-}
\ No newline at end of file
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/DenseVectorFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/DenseVectorFieldMapperTests.java
deleted file mode 100644
index cf6fc99657756..0000000000000
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/DenseVectorFieldMapperTests.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.mapper;
-
-import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.IndexService;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.util.Collection;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.instanceOf;
-
-public class DenseVectorFieldMapperTests extends ESSingleNodeTestCase {
-    private DocumentMapper mapper;
-
-    @Before
-    public void setUpMapper() throws Exception {
-        IndexService indexService =  createIndex("test-index");
-        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
-        String mapping = Strings.toString(XContentFactory.jsonBuilder()
-            .startObject()
-                .startObject("_doc")
-                    .startObject("properties")
-                        .startObject("my-dense-vector").field("type", "dense_vector")
-                        .endObject()
-                    .endObject()
-                .endObject()
-            .endObject());
-        mapper = parser.parse("_doc", new CompressedXContent(mapping));
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(MapperExtrasPlugin.class);
-    }
-
-    public void testDefaults() throws Exception {
-        float[] expectedArray = {-12.1f, 100.7f, -4};
-        ParsedDocument doc1 = mapper.parse(new SourceToParse("test-index", "_doc", "1", BytesReference
-            .bytes(XContentFactory.jsonBuilder()
-                .startObject()
-                    .startArray("my-dense-vector").value(expectedArray[0]).value(expectedArray[1]).value(expectedArray[2]).endArray()
-                .endObject()),
-            XContentType.JSON));
-        IndexableField[] fields = doc1.rootDoc().getFields("my-dense-vector");
-        assertEquals(1, fields.length);
-        assertThat(fields[0], instanceOf(BinaryDocValuesField.class));
-
-        // assert that after decoding the indexed value is equal to expected
-        BytesRef vectorBR = ((BinaryDocValuesField) fields[0]).binaryValue();
-        float[] decodedValues = VectorEncoderDecoder.decodeDenseVector(vectorBR);
-        assertArrayEquals(
-            "Decoded dense vector values is not equal to the indexed one.",
-            expectedArray,
-            decodedValues,
-            0.001f
-        );
-    }
-
-    public void testDimensionLimit() throws IOException {
-        float[] validVector = new float[DenseVectorFieldMapper.MAX_DIMS_COUNT];
-        BytesReference validDoc = BytesReference.bytes(
-            XContentFactory.jsonBuilder().startObject()
-                .array("my-dense-vector", validVector)
-            .endObject());
-        mapper.parse(new SourceToParse("test-index", "_doc", "1", validDoc, XContentType.JSON));
-
-        float[] invalidVector = new float[DenseVectorFieldMapper.MAX_DIMS_COUNT + 1];
-        BytesReference invalidDoc = BytesReference.bytes(
-            XContentFactory.jsonBuilder().startObject()
-                .array("my-dense-vector", invalidVector)
-                .endObject());
-        MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapper.parse(
-            new SourceToParse("test-index", "_doc", "1", invalidDoc, XContentType.JSON)));
-        assertThat(e.getDetailedMessage(), containsString("has exceeded the maximum allowed number of dimensions"));
-    }
-}
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java
index b5348ac91465c..d05be9ba2d7f9 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java
+++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperIntegrationIT.java
@@ -21,6 +21,7 @@
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.bulk.BulkResponse;
@@ -180,7 +181,7 @@ private SearchRequestBuilder searchByNumericRange(int low, int high) {
     }
 
     private SearchRequestBuilder prepareSearch() {
-        SearchRequestBuilder request = client().prepareSearch("test").setTypes("test");
+        SearchRequestBuilder request = client().prepareSearch("test");
         request.addStoredField("foo.token_count");
         request.addStoredField("foo.token_count_without_position_increments");
         if (loadCountedFields) {
diff --git a/modules/parent-join/build.gradle b/modules/parent-join/build.gradle
index 67bcc9d54e8e7..756a65a371a9b 100644
--- a/modules/parent-join/build.gradle
+++ b/modules/parent-join/build.gradle
@@ -20,5 +20,4 @@
 esplugin {
   description 'This module adds the support parent-child queries and aggregations'
   classname 'org.elasticsearch.join.ParentJoinPlugin'
-  hasClientJar = true
 }
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java
index e0fac88cb531c..c10b025d2759f 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java
@@ -95,9 +95,9 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<WithOrdinals, ?> innerBuild(SearchContext context,
+    protected ValuesSourceAggregatorFactory<WithOrdinals> innerBuild(SearchContext context,
                                                                         ValuesSourceConfig<WithOrdinals> config,
-                                                                        AggregatorFactory<?> parent,
+                                                                        AggregatorFactory parent,
                                                                         Builder subFactoriesBuilder) throws IOException {
         return new ChildrenAggregatorFactory(name, config, childFilter, parentFilter, context, parent,
                 subFactoriesBuilder, metaData);
@@ -159,12 +159,15 @@ public static ChildrenAggregationBuilder parse(String aggregationName, XContentP
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(childType);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), childType);
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         ChildrenAggregationBuilder other = (ChildrenAggregationBuilder) obj;
         return Objects.equals(childType, other.childType);
     }
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java
index 1f466f1020d18..a8cfe62edd9ed 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregatorFactory.java
@@ -35,7 +35,7 @@
 import java.util.List;
 import java.util.Map;
 
-public class ChildrenAggregatorFactory extends ValuesSourceAggregatorFactory<WithOrdinals, ChildrenAggregatorFactory> {
+public class ChildrenAggregatorFactory extends ValuesSourceAggregatorFactory<WithOrdinals> {
 
     private final Query parentFilter;
     private final Query childFilter;
@@ -45,7 +45,7 @@ public ChildrenAggregatorFactory(String name,
                                         Query childFilter,
                                         Query parentFilter,
                                         SearchContext context,
-                                        AggregatorFactory<?> parent,
+                                        AggregatorFactory parent,
                                         AggregatorFactories.Builder subFactoriesBuilder,
                                         Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregationBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregationBuilder.java
index 495a5c0f9ad3f..83575d6d527e6 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregationBuilder.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregationBuilder.java
@@ -95,9 +95,9 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<WithOrdinals, ?> innerBuild(SearchContext context,
+    protected ValuesSourceAggregatorFactory<WithOrdinals> innerBuild(SearchContext context,
                                                                         ValuesSourceConfig<WithOrdinals> config,
-                                                                        AggregatorFactory<?> parent,
+                                                                        AggregatorFactory parent,
                                                                         Builder subFactoriesBuilder) throws IOException {
         return new ParentAggregatorFactory(name, config, childFilter, parentFilter, context, parent,
                 subFactoriesBuilder, metaData);
@@ -159,12 +159,15 @@ public static ParentAggregationBuilder parse(String aggregationName, XContentPar
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(childType);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), childType);
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         ParentAggregationBuilder other = (ParentAggregationBuilder) obj;
         return Objects.equals(childType, other.childType);
     }
diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java
index 2ae3da7c47af3..fd87744dcf235 100644
--- a/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java
+++ b/modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentAggregatorFactory.java
@@ -35,7 +35,7 @@
 import java.util.List;
 import java.util.Map;
 
-public class ParentAggregatorFactory extends ValuesSourceAggregatorFactory<WithOrdinals, ParentAggregatorFactory> {
+public class ParentAggregatorFactory extends ValuesSourceAggregatorFactory<WithOrdinals> {
 
     private final Query parentFilter;
     private final Query childFilter;
@@ -45,7 +45,7 @@ public ParentAggregatorFactory(String name,
                                    Query childFilter,
                                    Query parentFilter,
                                    SearchContext context,
-                                   AggregatorFactory<?> parent,
+                                   AggregatorFactory parent,
                                    AggregatorFactories.Builder subFactoriesBuilder,
                                    Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java
index 8a50e3a734977..925f85eb684f0 100644
--- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.join.query;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -280,13 +281,9 @@ public void testFromJson() throws IOException {
     }
 
     public void testToQueryInnerQueryType() throws IOException {
-        String[] searchTypes = new String[]{TYPE};
         QueryShardContext shardContext = createShardContext();
-        shardContext.setTypes(searchTypes);
         HasChildQueryBuilder hasChildQueryBuilder = hasChildQuery(CHILD_DOC, new IdsQueryBuilder().addIds("id"), ScoreMode.None);
         Query query = hasChildQueryBuilder.toQuery(shardContext);
-        //verify that the context types are still the same as the ones we previously set
-        assertThat(shardContext.getTypes(), equalTo(searchTypes));
         assertLateParsingQuery(query, CHILD_DOC, "id");
     }
 
diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java
index ea77ad80799ba..a634690dcac85 100644
--- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java
+++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java
@@ -192,14 +192,10 @@ public void testIllegalValues() throws IOException {
     }
 
     public void testToQueryInnerQueryType() throws IOException {
-        String[] searchTypes = new String[]{TYPE};
         QueryShardContext shardContext = createShardContext();
-        shardContext.setTypes(searchTypes);
         HasParentQueryBuilder hasParentQueryBuilder = new HasParentQueryBuilder(PARENT_DOC, new IdsQueryBuilder().addIds("id"),
                 false);
         Query query = hasParentQueryBuilder.toQuery(shardContext);
-        //verify that the context types are still the same as the ones we previously set
-        assertThat(shardContext.getTypes(), equalTo(searchTypes));
         HasChildQueryBuilderTests.assertLateParsingQuery(query, PARENT_DOC, "id");
     }
 
diff --git a/modules/percolator/build.gradle b/modules/percolator/build.gradle
index 0826c91676dea..1296491748c35 100644
--- a/modules/percolator/build.gradle
+++ b/modules/percolator/build.gradle
@@ -20,7 +20,6 @@
 esplugin {
     description 'Percolator module adds capability to index queries and query these queries by specifying documents'
     classname 'org.elasticsearch.percolator.PercolatorPlugin'
-    hasClientJar = true
 }
 
 dependencies {
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index 3021f5b31606e..151dd8b9e61ba 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -178,7 +178,7 @@ public PercolateQueryBuilder(String field, String documentType, List<BytesRefere
         this.documentSupplier = null;
     }
 
-    private PercolateQueryBuilder(String field, String documentType, Supplier<BytesReference> documentSupplier) {
+    protected PercolateQueryBuilder(String field, String documentType, Supplier<BytesReference> documentSupplier) {
         if (field == null) {
             throw new IllegalArgumentException("[field] is a required argument");
         }
@@ -491,8 +491,12 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) {
             if (source == null) {
                 return this; // not executed yet
             } else {
-                return new PercolateQueryBuilder(field, documentType, Collections.singletonList(source),
-                    XContentHelper.xContentType(source));
+                PercolateQueryBuilder rewritten = new PercolateQueryBuilder(field, documentType,
+                    Collections.singletonList(source), XContentHelper.xContentType(source));
+                if (name != null) {
+                    rewritten.setName(name);
+                }
+                return rewritten;
             }
         }
         GetRequest getRequest;
@@ -527,7 +531,12 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) {
                 listener.onResponse(null);
             }, listener::onFailure));
         });
-        return new PercolateQueryBuilder(field, documentType, documentSupplier::get);
+
+        PercolateQueryBuilder rewritten = new PercolateQueryBuilder(field, documentType, documentSupplier::get);
+        if (name != null) {
+            rewritten.setName(name);
+        }
+        return rewritten;
     }
 
     @Override
@@ -626,6 +635,10 @@ XContentType getXContentType() {
         return documentXContentType;
     }
 
+    public String getQueryName() {
+        return name;
+    }
+
     static IndexSearcher createMultiDocumentSearcher(Analyzer analyzer, Collection<ParsedDocument> docs) {
         RAMDirectory ramDirectory = new RAMDirectory();
         try (IndexWriter indexWriter = new IndexWriter(ramDirectory, new IndexWriterConfig(analyzer))) {
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
index c245e2cb3a20b..d3c4bdedde7d2 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
@@ -22,7 +22,6 @@
 import org.apache.lucene.index.PrefixCodedTerms;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.BlendedTermQuery;
-import org.apache.lucene.queries.CommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
@@ -75,7 +74,6 @@ final class QueryAnalyzer {
         entry(BoostQuery.class, boostQuery()),
         entry(TermQuery.class, termQuery()),
         entry(TermInSetQuery.class, termInSetQuery()),
-        entry(CommonTermsQuery.class, commonTermsQuery()),
         entry(BlendedTermQuery.class, blendedTermQuery()),
         entry(PhraseQuery.class, phraseQuery()),
         entry(MultiPhraseQuery.class, multiPhraseQuery()),
@@ -185,13 +183,6 @@ private static BiFunction<Query, Version, Result> synonymQuery() {
         };
     }
 
-    private static BiFunction<Query, Version, Result> commonTermsQuery() {
-        return (query, version) -> {
-            Set<QueryExtraction> terms = ((CommonTermsQuery) query).getTerms().stream().map(QueryExtraction::new).collect(toSet());
-            return new Result(false, terms, Math.min(1, terms.size()));
-        };
-    }
-
     private static BiFunction<Query, Version, Result> blendedTermQuery() {
         return (query, version) -> {
             Set<QueryExtraction> terms = ((BlendedTermQuery) query).getTerms().stream().map(QueryExtraction::new).collect(toSet());
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
index b191dd948c574..e487037afaea7 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
@@ -46,7 +46,6 @@
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.memory.MemoryIndex;
 import org.apache.lucene.queries.BlendedTermQuery;
-import org.apache.lucene.queries.CommonTermsQuery;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
@@ -84,8 +83,8 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.common.geo.ShapeRelation;
+import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
@@ -530,12 +529,6 @@ public void testDuelIdBased() throws Exception {
     public void testDuelSpecificQueries() throws Exception {
         List<ParseContext.Document> documents = new ArrayList<>();
 
-        CommonTermsQuery commonTermsQuery = new CommonTermsQuery(Occur.SHOULD, Occur.SHOULD, 128);
-        commonTermsQuery.add(new Term("field", "quick"));
-        commonTermsQuery.add(new Term("field", "brown"));
-        commonTermsQuery.add(new Term("field", "fox"));
-        addQuery(commonTermsQuery, documents);
-
         BlendedTermQuery blendedTermQuery = BlendedTermQuery.dismaxBlendedQuery(new Term[]{new Term("field", "quick"),
                 new Term("field", "brown"), new Term("field", "fox")}, 1.0f);
         addQuery(blendedTermQuery, documents);
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
index 6053a92b54a20..a86f93ce40549 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
@@ -54,6 +54,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Supplier;
 
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
 import static org.hamcrest.Matchers.equalTo;
@@ -295,9 +296,17 @@ private static BytesReference randomSource(Set<String> usedFields) {
         }
     }
 
+    /**
+     * Test that this query is never cacheable
+     */
     @Override
-    protected boolean isCacheable(PercolateQueryBuilder queryBuilder) {
-        return false;
+    public void testCacheability() throws IOException {
+        PercolateQueryBuilder queryBuilder = createTestQueryBuilder();
+        QueryShardContext context = createShardContext();
+        assert context.isCacheable();
+        QueryBuilder rewritten = rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(rewritten.toQuery(context));
+        assertFalse("query should not be cacheable: " + queryBuilder.toString(), context.isCacheable());
     }
 
     @Override
@@ -331,4 +340,29 @@ public void testFieldAlias() throws IOException {
         assertEquals(query.getVerifiedMatchesQuery(), aliasQuery.getVerifiedMatchesQuery());
     }
 
+    public void testSettingNameWhileRewriting() {
+        String testName = "name1";
+        QueryShardContext shardContext = createShardContext();
+        PercolateQueryBuilder percolateQueryBuilder = doCreateTestQueryBuilder(true);
+        percolateQueryBuilder.setName(testName);
+
+        QueryBuilder rewrittenQueryBuilder = percolateQueryBuilder.doRewrite(shardContext);
+
+        assertEquals(testName, ((PercolateQueryBuilder) rewrittenQueryBuilder).getQueryName());
+        assertNotEquals(rewrittenQueryBuilder, percolateQueryBuilder);
+    }
+
+    public void testSettingNameWhileRewritingWhenDocumentSupplierAndSourceNotNull() {
+        Supplier<BytesReference> supplier = () -> new BytesArray("{\"test\": \"test\"}");
+        String testName = "name1";
+        QueryShardContext shardContext = createShardContext();
+        PercolateQueryBuilder percolateQueryBuilder = new PercolateQueryBuilder(queryField, null, supplier);
+        percolateQueryBuilder.setName(testName);
+
+        QueryBuilder rewrittenQueryBuilder = percolateQueryBuilder.doRewrite(shardContext);
+
+        assertEquals(testName, ((PercolateQueryBuilder) rewrittenQueryBuilder).getQueryName());
+        assertNotEquals(rewrittenQueryBuilder, percolateQueryBuilder);
+    }
+
 }
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
index ee31a81ae168a..57a6ca15ac593 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
@@ -44,7 +44,6 @@
 import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.yamlBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery;
 import static org.elasticsearch.index.query.QueryBuilders.geoBoundingBoxQuery;
 import static org.elasticsearch.index.query.QueryBuilders.geoDistanceQuery;
 import static org.elasticsearch.index.query.QueryBuilders.geoPolygonQuery;
@@ -356,13 +355,10 @@ public void testPercolatorSpecificQueries()  throws Exception {
         );
 
         client().prepareIndex("test", "type", "1")
-                .setSource(jsonBuilder().startObject().field("query", commonTermsQuery("field1", "quick brown fox")).endObject())
-                .get();
-        client().prepareIndex("test", "type", "2")
                 .setSource(jsonBuilder().startObject().field("query", multiMatchQuery("quick brown fox", "field1", "field2")
                         .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)).endObject())
                 .get();
-        client().prepareIndex("test", "type", "3")
+        client().prepareIndex("test", "type", "2")
                 .setSource(jsonBuilder().startObject().field("query",
                         spanNearQuery(spanTermQuery("field1", "quick"), 0)
                                 .addClause(spanTermQuery("field1", "brown"))
@@ -372,7 +368,7 @@ public void testPercolatorSpecificQueries()  throws Exception {
                 .get();
         client().admin().indices().prepareRefresh().get();
 
-        client().prepareIndex("test", "type", "4")
+        client().prepareIndex("test", "type", "3")
                 .setSource(jsonBuilder().startObject().field("query",
                         spanNotQuery(
                                 spanNearQuery(spanTermQuery("field1", "quick"), 0)
@@ -387,7 +383,7 @@ public void testPercolatorSpecificQueries()  throws Exception {
                 .get();
 
         // doesn't match
-        client().prepareIndex("test", "type", "5")
+        client().prepareIndex("test", "type", "4")
                 .setSource(jsonBuilder().startObject().field("query",
                         spanNotQuery(
                                 spanNearQuery(spanTermQuery("field1", "quick"), 0)
@@ -410,15 +406,13 @@ public void testPercolatorSpecificQueries()  throws Exception {
                 .setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON))
                 .addSort("_id", SortOrder.ASC)
                 .get();
-        assertHitCount(response, 4);
+        assertHitCount(response, 3);
         assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
         assertThat(response.getHits().getAt(0).getScore(), equalTo(Float.NaN));
         assertThat(response.getHits().getAt(1).getId(), equalTo("2"));
         assertThat(response.getHits().getAt(1).getScore(), equalTo(Float.NaN));
         assertThat(response.getHits().getAt(2).getId(), equalTo("3"));
         assertThat(response.getHits().getAt(2).getScore(), equalTo(Float.NaN));
-        assertThat(response.getHits().getAt(3).getId(), equalTo("4"));
-        assertThat(response.getHits().getAt(3).getScore(), equalTo(Float.NaN));
     }
 
     public void testPercolatorQueryWithHighlighting() throws Exception {
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
index c07467187f05f..358e9176e19b5 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
@@ -28,8 +28,8 @@
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.BlendedTermQuery;
-import org.apache.lucene.queries.CommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
@@ -44,7 +44,6 @@
 import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TermRangeQuery;
-import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.join.QueryBitSetProducer;
 import org.apache.lucene.search.join.ScoreMode;
 import org.apache.lucene.search.spans.SpanFirstQuery;
@@ -520,27 +519,10 @@ public void testExtractQueryMetadata_boostQuery() {
         assertThat(terms.get(0).bytes(), equalTo(termQuery1.getTerm().bytes()));
     }
 
-    public void testExtractQueryMetadata_commonTermsQuery() {
-        CommonTermsQuery commonTermsQuery = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 100);
-        commonTermsQuery.add(new Term("_field", "_term1"));
-        commonTermsQuery.add(new Term("_field", "_term2"));
-        Result result = analyze(commonTermsQuery, Version.CURRENT);
-        assertThat(result.verified, is(false));
-        assertThat(result.minimumShouldMatch, equalTo(1));
-        List<QueryExtraction> terms = new ArrayList<>(result.extractions);
-        terms.sort(Comparator.comparing(qt -> qt.term));
-        assertThat(terms.size(), equalTo(2));
-        assertThat(result.minimumShouldMatch, equalTo(1));
-        assertThat(terms.get(0).field(), equalTo("_field"));
-        assertThat(terms.get(0).text(), equalTo("_term1"));
-        assertThat(terms.get(1).field(), equalTo("_field"));
-        assertThat(terms.get(1).text(), equalTo("_term2"));
-    }
-
     public void testExtractQueryMetadata_blendedTermQuery() {
         Term[] termsArr = new Term[]{new Term("_field", "_term1"), new Term("_field", "_term2")};
-        BlendedTermQuery commonTermsQuery = BlendedTermQuery.dismaxBlendedQuery(termsArr, 1.0f);
-        Result result = analyze(commonTermsQuery, Version.CURRENT);
+        BlendedTermQuery blendedTermQuery = BlendedTermQuery.dismaxBlendedQuery(termsArr, 1.0f);
+        Result result = analyze(blendedTermQuery, Version.CURRENT);
         assertThat(result.verified, is(true));
         assertThat(result.minimumShouldMatch, equalTo(1));
         List<QueryAnalyzer.QueryExtraction> terms = new ArrayList<>(result.extractions);
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java
index 30cad734023d5..88c2a098deb21 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java
@@ -55,13 +55,13 @@ public class QueryBuilderStoreTests extends ESTestCase {
 
     @Override
     protected NamedWriteableRegistry writableRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedWriteableRegistry(searchModule.getNamedWriteables());
     }
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle
index a0fe3bef0a617..3e8e40ae89c40 100644
--- a/modules/rank-eval/build.gradle
+++ b/modules/rank-eval/build.gradle
@@ -20,7 +20,6 @@
 esplugin {
   description 'The Rank Eval module adds APIs to evaluate ranking quality.'
   classname 'org.elasticsearch.index.rankeval.RankEvalPlugin'
-  hasClientJar = true
 }
 
 testClusters.integTest {
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java
index 54e89fe0e98b8..07de8c8a22cad 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.index.rankeval;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for explaining evaluating search ranking results.
+ * ActionType for explaining evaluating search ranking results.
  */
-public class RankEvalAction extends Action<RankEvalResponse> {
+public class RankEvalAction extends StreamableResponseActionType<RankEvalResponse> {
 
     public static final RankEvalAction INSTANCE = new RankEvalAction();
     public static final String NAME = "indices:data/read/rank_eval";
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java
index 4108a817f046e..1bfb576c37911 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.index.rankeval;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 
 public class RankEvalRequestBuilder extends ActionRequestBuilder<RankEvalRequest, RankEvalResponse> {
 
-    public RankEvalRequestBuilder(ElasticsearchClient client, Action<RankEvalResponse> action,
+    public RankEvalRequestBuilder(ElasticsearchClient client, ActionType<RankEvalResponse> action,
             RankEvalRequest request) {
         super(client, action, request);
     }
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java
index f77951dd58b6e..74683d8135ed7 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java
@@ -64,7 +64,7 @@ public class RatedRequestsTests extends ESTestCase {
     @BeforeClass
     public static void init() {
         xContentRegistry = new NamedXContentRegistry(
-                Stream.of(new SearchModule(Settings.EMPTY, false, emptyList()).getNamedXContents().stream()).flatMap(Function.identity())
+                Stream.of(new SearchModule(Settings.EMPTY, emptyList()).getNamedXContents().stream()).flatMap(Function.identity())
                         .collect(toList()));
     }
 
diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle
index da184deedaa11..78846e2d81dd5 100644
--- a/modules/reindex/build.gradle
+++ b/modules/reindex/build.gradle
@@ -26,7 +26,6 @@ apply plugin: 'elasticsearch.test-with-dependencies'
 esplugin {
   description 'The Reindex module adds APIs to reindex from one index to another or update documents in place.'
   classname 'org.elasticsearch.index.reindex.ReindexPlugin'
-  hasClientJar = true
 }
 
 testClusters.integTest {
@@ -55,8 +54,8 @@ test {
 }
 
 dependencies {
-  compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
-  compile "org.elasticsearch:elasticsearch-ssl-config:${version}"
+  compile project(":client:rest")
+  compile project(":libs:elasticsearch-ssl-config")
   // for http - testing reindex from remote
   testCompile project(path: ':modules:transport-netty4', configuration: 'runtime')
   // for parent/child testing
@@ -77,7 +76,7 @@ forbiddenPatterns {
     exclude '**/*.p12'
 }
 
-// Support for testing reindex-from-remote against old Elaticsearch versions
+// Support for testing reindex-from-remote against old Elasticsearch versions
 configurations {
   oldesFixture
   es2
@@ -125,7 +124,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
       dependsOn unzip
       executable = new File(project.runtimeJavaHome, 'bin/java')
       env 'CLASSPATH', "${ -> project.configurations.oldesFixture.asPath }"
-      env 'JAVA_HOME', getJavaHome(it, 8)
+      env 'JAVA_HOME', "${ -> getJavaHome(it, 8)}"
       args 'oldes.OldElasticsearch',
            baseDir,
            unzip.temporaryDir,
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
index a4b28e6c598d3..c6bfcf13eb113 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
@@ -75,7 +75,7 @@
 import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.action.bulk.BackoffPolicy.exponentialBackoff;
 import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
-import static org.elasticsearch.index.reindex.AbstractBulkByScrollRequest.SIZE_ALL_MATCHES;
+import static org.elasticsearch.index.reindex.AbstractBulkByScrollRequest.MAX_DOCS_ALL_MATCHES;
 import static org.elasticsearch.rest.RestStatus.CONFLICT;
 import static org.elasticsearch.search.sort.SortBuilders.fieldSort;
 
@@ -263,8 +263,8 @@ void onScrollResponse(TimeValue lastBatchStartTime, int lastBatchSize, Scrollabl
             return;
         }
         long total = response.getTotalHits();
-        if (mainRequest.getSize() > 0) {
-            total = min(total, mainRequest.getSize());
+        if (mainRequest.getMaxDocs() > 0) {
+            total = min(total, mainRequest.getMaxDocs());
         }
         worker.setTotal(total);
         AbstractRunnable prepareBulkRequestRunnable = new AbstractRunnable() {
@@ -304,9 +304,9 @@ void prepareBulkRequest(TimeValue thisBatchStartTime, ScrollableHitSource.Respon
         }
         worker.countBatch();
         List<? extends ScrollableHitSource.Hit> hits = response.getHits();
-        if (mainRequest.getSize() != SIZE_ALL_MATCHES) {
-            // Truncate the hits if we have more than the request size
-            long remaining = max(0, mainRequest.getSize() - worker.getSuccessfullyProcessed());
+        if (mainRequest.getMaxDocs() != MAX_DOCS_ALL_MATCHES) {
+            // Truncate the hits if we have more than the request max docs
+            long remaining = max(0, mainRequest.getMaxDocs() - worker.getSuccessfullyProcessed());
             if (remaining < hits.size()) {
                 hits = hits.subList(0, (int) remaining);
             }
@@ -395,7 +395,7 @@ void onBulkResponse(TimeValue thisBatchStartTime, BulkResponse response) {
                 return;
             }
 
-            if (mainRequest.getSize() != SIZE_ALL_MATCHES && worker.getSuccessfullyProcessed() >= mainRequest.getSize()) {
+            if (mainRequest.getMaxDocs() != MAX_DOCS_ALL_MATCHES && worker.getSuccessfullyProcessed() >= mainRequest.getMaxDocs()) {
                 // We've processed all the requested docs.
                 refreshAndFinish(emptyList(), emptyList(), false);
                 return;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
index bf333352dd55c..63451abb7ccd7 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
@@ -38,7 +38,7 @@
 
 public abstract class AbstractBaseReindexRestHandler<
                 Request extends AbstractBulkByScrollRequest<Request>,
-                A extends Action<BulkByScrollResponse>
+                A extends ActionType<BulkByScrollResponse>
             > extends BaseRestHandler {
 
     private final A action;
@@ -105,6 +105,11 @@ protected Request setCommonOptions(RestRequest restRequest, Request request) {
         if (requestsPerSecond != null) {
             request.setRequestsPerSecond(requestsPerSecond);
         }
+
+        if (restRequest.hasParam("max_docs")) {
+            setMaxDocsValidateIdentical(request, restRequest.paramAsInt("max_docs", -1));
+        }
+
         return request;
     }
 
@@ -170,4 +175,13 @@ public static Float parseRequestsPerSecond(RestRequest request) {
         }
         return requestsPerSecond;
     }
+
+    static void setMaxDocsValidateIdentical(AbstractBulkByScrollRequest<?> request, int maxDocs) {
+        if (request.getMaxDocs() != AbstractBulkByScrollRequest.MAX_DOCS_ALL_MATCHES && request.getMaxDocs() != maxDocs) {
+            throw new IllegalArgumentException("[max_docs] set to two different values [" + request.getMaxDocs() + "]" +
+                " and [" + maxDocs + "]");
+        } else {
+            request.setMaxDocs(maxDocs);
+        }
+    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
index fab94494fe13d..3124d9cb0be0c 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
@@ -38,7 +38,7 @@
  */
 public abstract class AbstractBulkByQueryRestHandler<
         Request extends AbstractBulkByScrollRequest<Request>,
-        A extends Action<BulkByScrollResponse>> extends AbstractBaseReindexRestHandler<Request, A> {
+        A extends ActionType<BulkByScrollResponse>> extends AbstractBaseReindexRestHandler<Request, A> {
 
     protected AbstractBulkByQueryRestHandler(Settings settings, A action) {
         super(settings, action);
@@ -52,7 +52,7 @@ protected void parseInternalRequest(Request internal, RestRequest restRequest,
         SearchRequest searchRequest = internal.getSearchRequest();
 
         try (XContentParser parser = extractRequestSpecificFields(restRequest, bodyConsumers)) {
-            RestSearchAction.parseSearchRequest(searchRequest, restRequest, parser, internal::setSize);
+            RestSearchAction.parseSearchRequest(searchRequest, restRequest, parser, size -> failOnSizeSpecified());
         }
 
         searchRequest.source().size(restRequest.paramAsInt("scroll_size", searchRequest.source().size()));
@@ -94,4 +94,8 @@ private XContentParser extractRequestSpecificFields(RestRequest restRequest,
                 parser.getDeprecationHandler(), BytesReference.bytes(builder.map(body)).streamInput());
         }
     }
+
+    private static void failOnSizeSpecified() {
+        throw new IllegalArgumentException("invalid parameter [size], use [max_docs] instead");
+    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java
index 2eca143393567..6df73414e81f3 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
@@ -61,7 +61,7 @@ private BulkByScrollParallelizationHelper() {}
     static <Request extends AbstractBulkByScrollRequest<Request>> void startSlicedAction(
             Request request,
             BulkByScrollTask task,
-            Action<BulkByScrollResponse> action,
+            ActionType<BulkByScrollResponse> action,
             ActionListener<BulkByScrollResponse> listener,
             Client client,
             DiscoveryNode node,
@@ -85,7 +85,7 @@ static <Request extends AbstractBulkByScrollRequest<Request>> void startSlicedAc
     private static <Request extends AbstractBulkByScrollRequest<Request>> void sliceConditionally(
             Request request,
             BulkByScrollTask task,
-            Action<BulkByScrollResponse> action,
+            ActionType<BulkByScrollResponse> action,
             ActionListener<BulkByScrollResponse> listener,
             Client client,
             DiscoveryNode node,
@@ -112,13 +112,13 @@ private static int countSlicesBasedOnShards(ClusterSearchShardsResponse response
             (sum, term) -> sum + term
         ));
         Set<Integer> counts = new HashSet<>(countsByIndex.values());
-        int leastShards = Collections.min(counts);
+        int leastShards = counts.isEmpty() ? 1 : Collections.min(counts);
         return Math.min(leastShards, AUTO_SLICE_CEILING);
     }
 
     private static <Request extends AbstractBulkByScrollRequest<Request>> void sendSubRequests(
             Client client,
-            Action<BulkByScrollResponse> action,
+            ActionType<BulkByScrollResponse> action,
             String localNodeId,
             BulkByScrollTask task,
             Request request,
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
index 5d4d140131a14..74e600f32cd75 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
@@ -58,6 +58,7 @@ protected DeleteByQueryRequest buildRequest(RestRequest request) throws IOExcept
 
         Map<String, Consumer<Object>> consumers = new HashMap<>();
         consumers.put("conflicts", o -> internal.setConflicts((String) o));
+        consumers.put("max_docs", s -> setMaxDocsValidateIdentical(internal, ((Number) s).intValue()));
 
         parseInternalRequest(internal, request, consumers);
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
index ae9ca0be7ca65..e534717e775f5 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
@@ -19,89 +19,21 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.apache.logging.log4j.LogManager;
-import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ObjectParser;
-import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.script.Script;
 
 import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.List;
-import java.util.Map;
 
-import static java.util.Collections.emptyMap;
-import static java.util.Objects.requireNonNull;
 import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 /**
  * Expose reindex over rest.
  */
 public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexRequest, ReindexAction> {
-    static final ObjectParser<ReindexRequest, Void> PARSER = new ObjectParser<>("reindex");
-    static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in reindex requests is deprecated.";
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestReindexAction.class));
-
-    static {
-        ObjectParser.Parser<ReindexRequest, Void> sourceParser = (parser, request, context) -> {
-            // Funky hack to work around Search not having a proper ObjectParser and us wanting to extract query if using remote.
-            Map<String, Object> source = parser.map();
-            String[] indices = extractStringArray(source, "index");
-            if (indices != null) {
-                request.getSearchRequest().indices(indices);
-            }
-            String[] types = extractStringArray(source, "type");
-            if (types != null) {
-                deprecationLogger.deprecatedAndMaybeLog("reindex_with_types", TYPES_DEPRECATION_MESSAGE);
-                request.getSearchRequest().types(types);
-            }
-            request.setRemoteInfo(buildRemoteInfo(source));
-            XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType());
-            builder.map(source);
-            try (InputStream stream = BytesReference.bytes(builder).streamInput();
-                 XContentParser innerParser = parser.contentType().xContent()
-                     .createParser(parser.getXContentRegistry(), parser.getDeprecationHandler(), stream)) {
-                request.getSearchRequest().source().parseXContent(innerParser, false);
-            }
-        };
-
-        ObjectParser<IndexRequest, Void> destParser = new ObjectParser<>("dest");
-        destParser.declareString(IndexRequest::index, new ParseField("index"));
-        destParser.declareString((request, type) -> {
-            deprecationLogger.deprecatedAndMaybeLog("reindex_with_types", TYPES_DEPRECATION_MESSAGE);
-            request.type(type);
-        }, new ParseField("type"));
-        destParser.declareString(IndexRequest::routing, new ParseField("routing"));
-        destParser.declareString(IndexRequest::opType, new ParseField("op_type"));
-        destParser.declareString(IndexRequest::setPipeline, new ParseField("pipeline"));
-        destParser.declareString((s, i) -> s.versionType(VersionType.fromString(i)), new ParseField("version_type"));
-
-        PARSER.declareField(sourceParser::parse, new ParseField("source"), ValueType.OBJECT);
-        PARSER.declareField((p, v, c) -> destParser.parse(p, v.getDestination(), c), new ParseField("dest"), ValueType.OBJECT);
-        PARSER.declareInt(ReindexRequest::setSize, new ParseField("size"));
-        PARSER.declareField((p, v, c) -> v.setScript(Script.parse(p)), new ParseField("script"),
-                ValueType.OBJECT);
-        PARSER.declareString(ReindexRequest::setConflicts, new ParseField("conflicts"));
-    }
 
     public RestReindexAction(Settings settings, RestController controller) {
         super(settings, ReindexAction.INSTANCE);
@@ -124,123 +56,15 @@ protected ReindexRequest buildRequest(RestRequest request) throws IOException {
             throw new IllegalArgumentException("_reindex doesn't support [pipeline] as a query parameter. "
                     + "Specify it in the [dest] object instead.");
         }
-        ReindexRequest internal = new ReindexRequest();
+
+        ReindexRequest internal;
         try (XContentParser parser = request.contentParser()) {
-            PARSER.parse(parser, internal, null);
+            internal = ReindexRequest.fromXContent(parser);
         }
+
         if (request.hasParam("scroll")) {
             internal.setScroll(parseTimeValue(request.param("scroll"), "scroll"));
         }
         return internal;
     }
-
-    static RemoteInfo buildRemoteInfo(Map<String, Object> source) throws IOException {
-        @SuppressWarnings("unchecked")
-        Map<String, Object> remote = (Map<String, Object>) source.remove("remote");
-        if (remote == null) {
-            return null;
-        }
-        String username = extractString(remote, "username");
-        String password = extractString(remote, "password");
-        String hostInRequest = requireNonNull(extractString(remote, "host"), "[host] must be specified to reindex from a remote cluster");
-        URI uri;
-        try {
-            uri = new URI(hostInRequest);
-            // URI has less stringent URL parsing than our code. We want to fail if all values are not provided.
-            if (uri.getPort() == -1) {
-                throw new URISyntaxException(hostInRequest, "The port was not defined in the [host]");
-            }
-        } catch (URISyntaxException ex) {
-            throw new IllegalArgumentException("[host] must be of the form [scheme]://[host]:[port](/[pathPrefix])? but was ["
-                + hostInRequest + "]", ex);
-        }
-
-        String scheme = uri.getScheme();
-        String host = uri.getHost();
-        int port = uri.getPort();
-
-        String pathPrefix = null;
-        if (uri.getPath().isEmpty() == false) {
-            pathPrefix = uri.getPath();
-        }
-
-        Map<String, String> headers = extractStringStringMap(remote, "headers");
-        TimeValue socketTimeout = extractTimeValue(remote, "socket_timeout", RemoteInfo.DEFAULT_SOCKET_TIMEOUT);
-        TimeValue connectTimeout = extractTimeValue(remote, "connect_timeout", RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
-        if (false == remote.isEmpty()) {
-            throw new IllegalArgumentException(
-                    "Unsupported fields in [remote]: [" + Strings.collectionToCommaDelimitedString(remote.keySet()) + "]");
-        }
-        return new RemoteInfo(scheme, host, port, pathPrefix, queryForRemote(source),
-            username, password, headers, socketTimeout, connectTimeout);
-    }
-
-    /**
-     * Yank a string array from a map. Emulates XContent's permissive String to
-     * String array conversions.
-     */
-    private static String[] extractStringArray(Map<String, Object> source, String name) {
-        Object value = source.remove(name);
-        if (value == null) {
-            return null;
-        }
-        if (value instanceof List) {
-            @SuppressWarnings("unchecked")
-            List<String> list = (List<String>) value;
-            return list.toArray(new String[list.size()]);
-        } else if (value instanceof String) {
-            return new String[] {(String) value};
-        } else {
-            throw new IllegalArgumentException("Expected [" + name + "] to be a list of a string but was [" + value + ']');
-        }
-    }
-
-    private static String extractString(Map<String, Object> source, String name) {
-        Object value = source.remove(name);
-        if (value == null) {
-            return null;
-        }
-        if (value instanceof String) {
-            return (String) value;
-        }
-        throw new IllegalArgumentException("Expected [" + name + "] to be a string but was [" + value + "]");
-    }
-
-    private static Map<String, String> extractStringStringMap(Map<String, Object> source, String name) {
-        Object value = source.remove(name);
-        if (value == null) {
-            return emptyMap();
-        }
-        if (false == value instanceof Map) {
-            throw new IllegalArgumentException("Expected [" + name + "] to be an object containing strings but was [" + value + "]");
-        }
-        Map<?, ?> map = (Map<?, ?>) value;
-        for (Map.Entry<?, ?> entry : map.entrySet()) {
-            if (false == entry.getKey() instanceof String || false == entry.getValue() instanceof String) {
-                throw new IllegalArgumentException("Expected [" + name + "] to be an object containing strings but has [" + entry + "]");
-            }
-        }
-        @SuppressWarnings("unchecked") // We just checked....
-        Map<String, String> safe = (Map<String, String>) map;
-        return safe;
-    }
-
-    private static TimeValue extractTimeValue(Map<String, Object> source, String name, TimeValue defaultValue) {
-        String string = extractString(source, name);
-        return string == null ? defaultValue : parseTimeValue(string, name);
-    }
-
-    private static BytesReference queryForRemote(Map<String, Object> source) throws IOException {
-        XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
-        Object query = source.remove("query");
-        if (query == null) {
-            return BytesReference.bytes(matchAllQuery().toXContent(builder, ToXContent.EMPTY_PARAMS));
-        }
-        if (!(query instanceof Map)) {
-            throw new IllegalArgumentException("Expected [query] to be an object but was [" + query + "]");
-        }
-        @SuppressWarnings("unchecked")
-        Map<String, Object> map = (Map<String, Object>) query;
-        return BytesReference.bytes(builder.map(map));
-    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
index 121f8c372c59c..ea715aee7bd3a 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
@@ -66,6 +66,7 @@ protected UpdateByQueryRequest buildRequest(RestRequest request) throws IOExcept
         Map<String, Consumer<Object>> consumers = new HashMap<>();
         consumers.put("conflicts", o -> internal.setConflicts((String) o));
         consumers.put("script", o -> internal.setScript(parseScript(o)));
+        consumers.put("max_docs", s -> setMaxDocsValidateIdentical(internal, ((Number) s).intValue()));
 
         parseInternalRequest(internal, request, consumers);
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleAction.java
index 3cb2c60c62373..cf04d6d856ddb 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleAction.java
@@ -19,11 +19,11 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.common.io.stream.Writeable;
 
-public class RethrottleAction extends Action<ListTasksResponse> {
+public class RethrottleAction extends ActionType<ListTasksResponse> {
     public static final RethrottleAction INSTANCE = new RethrottleAction();
     public static final String NAME = "cluster:admin/reindex/rethrottle";
 
@@ -31,11 +31,6 @@ private RethrottleAction() {
         super(NAME);
     }
 
-    @Override
-    public ListTasksResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<ListTasksResponse> getResponseReader() {
         return ListTasksResponse::new;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequestBuilder.java
index 25407e6dc93d5..648eb6e441b1a 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequestBuilder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.support.tasks.TasksRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -29,7 +29,7 @@
  */
 public class RethrottleRequestBuilder extends TasksRequestBuilder<RethrottleRequest, ListTasksResponse, RethrottleRequestBuilder> {
     public RethrottleRequestBuilder(ElasticsearchClient client,
-            Action<ListTasksResponse> action) {
+            ActionType<ListTasksResponse> action) {
         super(client, action, new RethrottleRequest());
     }
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java
index 90332c7d55c9c..40c4ba757d176 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java
@@ -59,7 +59,6 @@ static Request initialSearch(SearchRequest searchRequest, BytesReference query,
         // It is nasty to build paths with StringBuilder but we'll be careful....
         StringBuilder path = new StringBuilder("/");
         addIndices(path, searchRequest.indices());
-        addTypes(path, searchRequest.types());
         path.append("_search");
         Request request = new Request("POST", path.toString());
 
@@ -75,14 +74,13 @@ static Request initialSearch(SearchRequest searchRequest, BytesReference query,
             request.addParameter("scroll", keepAlive.getStringRep());
         }
         request.addParameter("size", Integer.toString(searchRequest.source().size()));
-        if (searchRequest.source().version() == null || searchRequest.source().version() == true) {
-            /*
-             * Passing `null` here just add the `version` request parameter
-             * without any value. This way of requesting the version works
-             * for all supported versions of Elasticsearch.
-             */
-            request.addParameter("version", null);
+
+        if (searchRequest.source().version() == null || searchRequest.source().version() == false) {
+            request.addParameter("version", Boolean.FALSE.toString());
+        } else {
+            request.addParameter("version", Boolean.TRUE.toString());
         }
+
         if (searchRequest.source().sorts() != null) {
             boolean useScan = false;
             // Detect if we should use search_type=scan rather than a sort
@@ -178,16 +176,6 @@ private static String encodeIndex(String s) {
         }
     }
 
-    private static void addTypes(StringBuilder path, String[] types) {
-        if (types == null || types.length == 0) {
-            return;
-        }
-        for (String indexOrType : types) {
-            checkIndexOrType("Type", indexOrType);
-        }
-        path.append(Strings.arrayToCommaDelimitedString(types)).append('/');
-    }
-
     private static void checkIndexOrType(String name, String indexOrType) {
         if (indexOrType.indexOf(',') >= 0) {
             throw new IllegalArgumentException(name + " containing [,] not supported but got [" + indexOrType + "]");
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java
index 166eef1be369c..664941127b78e 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java
@@ -143,7 +143,7 @@ class Fields {
                 return new SearchFailure(reasonThrowable, index, shardId, nodeId);
             });
     static {
-        SEARCH_FAILURE_PARSER.declareString(optionalConstructorArg(), new ParseField("index"));
+        SEARCH_FAILURE_PARSER.declareStringOrNull(optionalConstructorArg(), new ParseField("index"));
         SEARCH_FAILURE_PARSER.declareInt(optionalConstructorArg(), new ParseField("shard"));
         SEARCH_FAILURE_PARSER.declareString(optionalConstructorArg(), new ParseField("node"));
         SEARCH_FAILURE_PARSER.declareField(constructorArg(), (p, c) -> {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java b/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java
index 7667fbbcf89f8..c2b907acde44e 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/client/documentation/ReindexDocumentationIT.java
@@ -116,7 +116,7 @@ public void testUpdateByQuery() {
               new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE);
             updateByQuery.source("source_index")
                 .filter(QueryBuilders.termQuery("level", "awesome"))
-                .size(1000)
+                .maxDocs(1000)
                 .script(new Script(ScriptType.INLINE,
                     "ctx._source.awesome = 'absolutely'",
                     "painless",
@@ -139,7 +139,7 @@ public void testUpdateByQuery() {
             UpdateByQueryRequestBuilder updateByQuery =
                new UpdateByQueryRequestBuilder(client, UpdateByQueryAction.INSTANCE);
             updateByQuery.source("source_index")
-                .size(100)
+                .maxDocs(100)
                 .source()
                 .addSort("cat", SortOrder.DESC);
             BulkByScrollResponse response = updateByQuery.get();
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
index bdedc65b7a6d3..3d28ce3bcbc96 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -761,7 +761,7 @@ private class MyMockClient extends FilterClient {
         @Override
         @SuppressWarnings("unchecked")
         protected <Request extends ActionRequest, Response extends ActionResponse>
-        void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+        void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
             if (false == expectedHeaders.equals(threadPool().getThreadContext().getHeaders())) {
                 listener.onFailure(
                         new RuntimeException("Expected " + expectedHeaders + " but got " + threadPool().getThreadContext().getHeaders()));
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
index 6d6ae01f0626c..7eaa989b88a32 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
@@ -37,7 +37,6 @@
 import org.elasticsearch.ingest.IngestTestPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.tasks.TaskInfo;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.hamcrest.Matcher;
 import org.junit.Before;
 
@@ -60,7 +59,6 @@
  * different cancellation places - that is the responsibility of AsyncBulkByScrollActionTests which have more precise control to
  * simulate failures but does not exercise important portion of the stack like transport and task management.
  */
-@TestLogging("org.elasticsearch.index.reindex:DEBUG,org.elasticsearch.action.bulk:DEBUG")
 public class CancelTests extends ReindexTestCase {
 
     protected static final String INDEX = "reindex-cancel-index";
@@ -216,7 +214,7 @@ public void testReindexCancel() throws Exception {
             assertThat(response, matcher().created(modified).reasonCancelled(equalTo("by user request")));
 
             refresh("dest");
-            assertHitCount(client().prepareSearch("dest").setTypes(TYPE).setSize(0).get(), modified);
+            assertHitCount(client().prepareSearch("dest").setSize(0).get(), modified);
         }, equalTo("reindex from [" + INDEX + "] to [dest][" + TYPE + "]"));
     }
 
@@ -251,7 +249,7 @@ public void testReindexCancelWithWorkers() throws Exception {
                 (response, total, modified) -> {
                     assertThat(response, matcher().created(modified).reasonCancelled(equalTo("by user request")).slices(hasSize(5)));
                     refresh("dest");
-                    assertHitCount(client().prepareSearch("dest").setTypes(TYPE).setSize(0).get(), modified);
+                    assertHitCount(client().prepareSearch("dest").setSize(0).get(), modified);
                 },
                 equalTo("reindex from [" + INDEX + "] to [dest][" + TYPE + "]"));
     }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java
index 5bef735be5e6e..3118021f9056d 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java
@@ -63,25 +63,25 @@ public void testBasics() throws Exception {
                 client().prepareIndex("test", "test", "7").setSource("foo", "f")
         );
 
-        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 7);
+        assertHitCount(client().prepareSearch("test").setSize(0).get(), 7);
 
         // Deletes two docs that matches "foo:a"
         assertThat(deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).get(), matcher().deleted(2));
-        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 5);
+        assertHitCount(client().prepareSearch("test").setSize(0).get(), 5);
 
         // Deletes the two first docs with limit by size
         DeleteByQueryRequestBuilder request = deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).size(2).refresh(true);
         request.source().addSort("foo.keyword", SortOrder.ASC);
         assertThat(request.get(), matcher().deleted(2));
-        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 3);
+        assertHitCount(client().prepareSearch("test").setSize(0).get(), 3);
 
         // Deletes but match no docs
         assertThat(deleteByQuery().source("test").filter(termQuery("foo", "no_match")).refresh(true).get(), matcher().deleted(0));
-        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 3);
+        assertHitCount(client().prepareSearch("test").setSize(0).get(), 3);
 
         // Deletes all remaining docs
         assertThat(deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).refresh(true).get(), matcher().deleted(3));
-        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 0);
+        assertHitCount(client().prepareSearch("test").setSize(0).get(), 0);
     }
 
     public void testDeleteByQueryWithOneIndex() throws Exception {
@@ -236,7 +236,7 @@ public void testSlices() throws Exception {
                 client().prepareIndex("test", "test", "6").setSource("foo", "e"),
                 client().prepareIndex("test", "test", "7").setSource("foo", "f")
         );
-        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 7);
+        assertHitCount(client().prepareSearch("test").setSize(0).get(), 7);
 
         int slices = randomSlices();
         int expectedSlices = expectedSliceStatuses(slices, "test");
@@ -251,7 +251,7 @@ public void testSlices() throws Exception {
             matcher()
                 .deleted(2)
                 .slices(hasSize(expectedSlices)));
-        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 5);
+        assertHitCount(client().prepareSearch("test").setSize(0).get(), 5);
 
         // Delete remaining docs
         assertThat(
@@ -263,7 +263,7 @@ public void testSlices() throws Exception {
             matcher()
                 .deleted(5)
                 .slices(hasSize(expectedSlices)));
-        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 0);
+        assertHitCount(client().prepareSearch("test").setSize(0).get(), 0);
     }
 
     public void testMultipleSources() throws Exception {
@@ -301,9 +301,18 @@ public void testMultipleSources() throws Exception {
                 .slices(hasSize(expectedSlices)));
 
         for (String index : docs.keySet()) {
-            assertHitCount(client().prepareSearch(index).setTypes("test").setSize(0).get(), 0);
+            assertHitCount(client().prepareSearch(index).setSize(0).get(), 0);
         }
 
     }
 
+    public void testMissingSources() {
+        BulkByScrollResponse response = updateByQuery()
+            .source("missing-index-*")
+            .refresh(true)
+            .setSlices(AbstractBulkByScrollRequest.AUTO_SLICES)
+            .get();
+        assertThat(response, matcher().deleted(0).slices(hasSize(0)));
+    }
+
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java
index b86f28452cc96..fd8cf49cc39ad 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java
@@ -73,18 +73,35 @@ public void testReindexFromRemote() throws IOException {
         Map<?, ?> http = (Map<?, ?>) nodeInfo.get("http");
         String remote = "http://"+ http.get("publish_address");
         Request request = new Request("POST", "/_reindex");
-        request.setJsonEntity(
+        if (randomBoolean()) {
+            request.setJsonEntity(
                 "{\n" +
-                "  \"source\":{\n" +
-                "    \"index\":\"test\",\n" +
-                "    \"remote\":{\n" +
-                "      \"host\":\"" + remote + "\"\n" +
-                "    }\n" +
-                "  }\n," +
-                "  \"dest\":{\n" +
-                "    \"index\":\"des\"\n" +
-                "  }\n" +
-                "}");
+                    "  \"source\":{\n" +
+                    "    \"index\":\"test\",\n" +
+                    "    \"remote\":{\n" +
+                    "      \"host\":\"" + remote + "\"\n" +
+                    "    }\n" +
+                    "  }\n," +
+                    "  \"dest\":{\n" +
+                    "    \"index\":\"des\"\n" +
+                    "  }\n" +
+                    "}");
+        } else {
+            // Test with external version_type
+            request.setJsonEntity(
+                "{\n" +
+                    "  \"source\":{\n" +
+                    "    \"index\":\"test\",\n" +
+                    "    \"remote\":{\n" +
+                    "      \"host\":\"" + remote + "\"\n" +
+                    "    }\n" +
+                    "  }\n," +
+                    "  \"dest\":{\n" +
+                    "    \"index\":\"des\",\n" +
+                    "    \"version_type\": \"external\"\n" +
+                    "  }\n" +
+                    "}");
+        }
         Map<String, Object> response = entityAsMap(client().performRequest(request));
         assertThat(response, hasEntry("total", count));
         assertThat(response, hasEntry("created", count));
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
index 43764bf25fcbf..55634204eb8f9 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
@@ -58,8 +58,8 @@ public void testFiltering() throws Exception {
         assertThat(copy.get(), matcher().created(2));
         assertHitCount(client().prepareSearch("dest_half").setSize(0).get(), 2);
 
-        // Limit with size
-        copy = reindex().source("source").destination("dest_size_one", "type").size(1).refresh(true);
+        // Limit with maxDocs
+        copy = reindex().source("source").destination("dest_size_one", "type").maxDocs(1).refresh(true);
         assertThat(copy.get(), matcher().created(1));
         assertHitCount(client().prepareSearch("dest_size_one").setSize(0).get(), 1);
     }
@@ -86,7 +86,7 @@ public void testCopyMany() throws Exception {
         copy = reindex().source("source").destination("dest_half", "type").refresh(true);
         // Use a small batch size so we have to use more than one batch
         copy.source().setSize(5);
-        copy.size(half); // The real "size" of the request.
+        copy.maxDocs(half);
         assertThat(copy.get(), matcher().created(half).batches(half, 5));
         assertHitCount(client().prepareSearch("dest_half").setSize(0).get(), half);
     }
@@ -109,14 +109,14 @@ public void testCopyManyWithSlices() throws Exception {
         // Use a small batch size so we have to use more than one batch
         copy.source().setSize(5);
         assertThat(copy.get(), matcher().created(max).batches(greaterThanOrEqualTo(max / 5)).slices(hasSize(expectedSlices)));
-        assertHitCount(client().prepareSearch("dest").setTypes("type").setSize(0).get(), max);
+        assertHitCount(client().prepareSearch("dest").setSize(0).get(), max);
 
         // Copy some of the docs
         int half = max / 2;
         copy = reindex().source("source").destination("dest_half", "type").refresh(true).setSlices(slices);
         // Use a small batch size so we have to use more than one batch
         copy.source().setSize(5);
-        copy.size(half); // The real "size" of the request.
+        copy.maxDocs(half);
         BulkByScrollResponse response = copy.get();
         assertThat(response, matcher().created(lessThanOrEqualTo((long) half)).slices(hasSize(expectedSlices)));
         assertHitCount(client().prepareSearch("dest_half").setSize(0).get(), response.getCreated());
@@ -157,5 +157,13 @@ public void testMultipleSources() throws Exception {
         assertHitCount(client().prepareSearch("dest").setSize(0).get(), allDocs.size());
     }
 
+    public void testMissingSources() {
+        BulkByScrollResponse response = updateByQuery()
+            .source("missing-index-*")
+            .refresh(true)
+            .setSlices(AbstractBulkByScrollRequest.AUTO_SLICES)
+            .get();
+        assertThat(response, matcher().created(0).slices(hasSize(0)));
+    }
 
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
index 917d196b6e9fb..4b9a69cfce1c8 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
@@ -21,7 +21,6 @@
 
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -38,7 +37,6 @@
 /**
  * Tests failure capturing and abort-on-failure behavior of reindex.
  */
-@TestLogging("_root:DEBUG")
 public class ReindexFailureTests extends ReindexTestCase {
     public void testFailuresCauseAbortDefault() throws Exception {
         /*
@@ -114,7 +112,7 @@ public void testResponseOnSearchFailure() throws Exception {
                  * so we *try* and wait for the delete to be fully
                  * complete here.
                  */
-                assertBusy(() -> assertFalse(client().admin().indices().prepareExists("source").get().isExists()));
+                assertBusy(() -> assertFalse(indexExists("source")));
             } catch (ExecutionException e) {
                 logger.info("Triggered a reindex failure on the {} attempt: {}", attempt, e.getMessage());
                 assertThat(e.getMessage(),
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java
index b0b35fbaac74e..73d92e239451f 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteBuildRestClientTests.java
@@ -22,9 +22,11 @@
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilderTestCase;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.util.ArrayList;
@@ -38,9 +40,12 @@
 import static org.mockito.Mockito.mock;
 
 public class ReindexFromRemoteBuildRestClientTests extends RestClientBuilderTestCase {
+
+    private final BytesReference matchAll = new BytesArray(new MatchAllQueryBuilder().toString());
+
     public void testBuildRestClient() throws Exception {
         for(final String path: new String[]{"", null, "/", "path"}) {
-            RemoteInfo remoteInfo = new RemoteInfo("https", "localhost", 9200, path, new BytesArray("ignored"), null, null, emptyMap(),
+            RemoteInfo remoteInfo = new RemoteInfo("https", "localhost", 9200, path, matchAll, null, null, emptyMap(),
                 RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
             long taskId = randomLong();
             List<Thread> threads = synchronizedList(new ArrayList<>());
@@ -64,7 +69,7 @@ public void testHeaders() throws Exception {
         for (int i = 0; i < numHeaders; ++i) {
             headers.put("header" + i, Integer.toString(i));
         }
-        RemoteInfo remoteInfo = new RemoteInfo("https", "localhost", 9200, null, new BytesArray("ignored"), null, null,
+        RemoteInfo remoteInfo = new RemoteInfo("https", "localhost", 9200, null, matchAll, null, null,
             headers, RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
         long taskId = randomLong();
         List<Thread> threads = synchronizedList(new ArrayList<>());
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWhitelistTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWhitelistTests.java
index f7a4e74fa19fd..6982a94146eee 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWhitelistTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWhitelistTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.test.ESTestCase;
 
 import java.net.UnknownHostException;
@@ -37,6 +38,9 @@
  * Tests the reindex-from-remote whitelist of remotes.
  */
 public class ReindexFromRemoteWhitelistTests extends ESTestCase {
+
+    private final BytesReference query = new BytesArray("{ \"foo\" : \"bar\" }");
+
     public void testLocalRequestWithoutWhitelist() {
         checkRemoteWhitelist(buildRemoteWhitelist(emptyList()), null);
     }
@@ -49,7 +53,7 @@ public void testLocalRequestWithWhitelist() {
      * Build a {@link RemoteInfo}, defaulting values that we don't care about in this test to values that don't hurt anything.
      */
     private RemoteInfo newRemoteInfo(String host, int port) {
-        return new RemoteInfo(randomAlphaOfLength(5), host, port, null, new BytesArray("test"), null, null, emptyMap(),
+        return new RemoteInfo(randomAlphaOfLength(5), host, port, null, query, null, null, emptyMap(),
                 RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
     }
 
@@ -63,7 +67,7 @@ public void testWhitelistedRemote() {
 
     public void testWhitelistedByPrefix() {
         checkRemoteWhitelist(buildRemoteWhitelist(singletonList("*.example.com:9200")),
-                new RemoteInfo(randomAlphaOfLength(5), "es.example.com", 9200, null, new BytesArray("test"), null, null, emptyMap(),
+                new RemoteInfo(randomAlphaOfLength(5), "es.example.com", 9200, null, query, null, null, emptyMap(),
                         RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT));
         checkRemoteWhitelist(buildRemoteWhitelist(singletonList("*.example.com:9200")),
                 newRemoteInfo("6e134134a1.us-east-1.aws.example.com", 9200));
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java
index a2e00e66fd218..97f803650644f 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRestClientSslTests.java
@@ -199,8 +199,9 @@ public void testClientPassesClientCertificate() throws IOException {
     }
 
     private RemoteInfo getRemoteInfo() {
-        return new RemoteInfo("https", server.getAddress().getHostName(), server.getAddress().getPort(), "/", new BytesArray("test"),
-            "user", "password", Collections.emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
+        return new RemoteInfo("https", server.getAddress().getHostName(), server.getAddress().getPort(), "/",
+            new BytesArray("{\"match_all\":{}}"), "user", "password", Collections.emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT,
+            RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
     }
 
     @SuppressForbidden(reason = "use http server")
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java
index 8264d4342c993..dc9d4a548170c 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
@@ -61,6 +62,8 @@ public class ReindexSourceTargetValidationTests extends ESTestCase {
     private static final AutoCreateIndex AUTO_CREATE_INDEX = new AutoCreateIndex(Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), INDEX_NAME_EXPRESSION_RESOLVER);
 
+    private final BytesReference query = new BytesArray("{ \"foo\" : \"bar\" }");
+
     public void testObviousCases() {
         fails("target", "target");
         fails("target", "foo", "bar", "target", "baz");
@@ -106,10 +109,10 @@ public void testTargetIsWriteAlias() {
 
     public void testRemoteInfoSkipsValidation() {
         // The index doesn't have to exist
-        succeeds(new RemoteInfo(randomAlphaOfLength(5), "test", 9200, null, new BytesArray("test"), null, null, emptyMap(),
+        succeeds(new RemoteInfo(randomAlphaOfLength(5), "test", 9200, null, query, null, null, emptyMap(),
                 RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT), "does_not_exist", "target");
         // And it doesn't matter if they are the same index. They are considered to be different because the remote one is, well, remote.
-        succeeds(new RemoteInfo(randomAlphaOfLength(5), "test", 9200, null, new BytesArray("test"), null, null, emptyMap(),
+        succeeds(new RemoteInfo(randomAlphaOfLength(5), "test", 9200, null, query, null, null, emptyMap(),
                 RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT), "target", "target");
     }
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java
index f0aca38545b4c..3fb85d16a2cc0 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.rest.RestRequest.Method;
@@ -32,12 +31,8 @@
 import org.junit.Before;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
 
 import static java.util.Collections.singletonMap;
-import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 
 public class RestReindexActionTests extends RestActionTestCase {
 
@@ -48,126 +43,6 @@ public void setUpAction() {
         action = new RestReindexAction(Settings.EMPTY, controller());
     }
 
-    public void testBuildRemoteInfoNoRemote() throws IOException {
-        assertNull(RestReindexAction.buildRemoteInfo(new HashMap<>()));
-    }
-
-    public void testBuildRemoteInfoFullyLoaded() throws IOException {
-        Map<String, String> headers = new HashMap<>();
-        headers.put("first", "a");
-        headers.put("second", "b");
-        headers.put("third", "");
-
-        Map<String, Object> remote = new HashMap<>();
-        remote.put("host", "https://example.com:9200");
-        remote.put("username", "testuser");
-        remote.put("password", "testpass");
-        remote.put("headers", headers);
-        remote.put("socket_timeout", "90s");
-        remote.put("connect_timeout", "10s");
-
-        Map<String, Object> query = new HashMap<>();
-        query.put("a", "b");
-
-        Map<String, Object> source = new HashMap<>();
-        source.put("remote", remote);
-        source.put("query", query);
-
-        RemoteInfo remoteInfo = RestReindexAction.buildRemoteInfo(source);
-        assertEquals("https", remoteInfo.getScheme());
-        assertEquals("example.com", remoteInfo.getHost());
-        assertEquals(9200, remoteInfo.getPort());
-        assertEquals("{\n  \"a\" : \"b\"\n}", remoteInfo.getQuery().utf8ToString());
-        assertEquals("testuser", remoteInfo.getUsername());
-        assertEquals("testpass", remoteInfo.getPassword());
-        assertEquals(headers, remoteInfo.getHeaders());
-        assertEquals(timeValueSeconds(90), remoteInfo.getSocketTimeout());
-        assertEquals(timeValueSeconds(10), remoteInfo.getConnectTimeout());
-    }
-
-    public void testBuildRemoteInfoWithoutAllParts() throws IOException {
-        expectThrows(IllegalArgumentException.class, () -> buildRemoteInfoHostTestCase("example.com"));
-        expectThrows(IllegalArgumentException.class, () -> buildRemoteInfoHostTestCase(":9200"));
-        expectThrows(IllegalArgumentException.class, () -> buildRemoteInfoHostTestCase("http://:9200"));
-        expectThrows(IllegalArgumentException.class, () -> buildRemoteInfoHostTestCase("example.com:9200"));
-        expectThrows(IllegalArgumentException.class, () -> buildRemoteInfoHostTestCase("http://example.com"));
-    }
-
-    public void testBuildRemoteInfoWithAllHostParts() throws IOException {
-        RemoteInfo info = buildRemoteInfoHostTestCase("http://example.com:9200");
-        assertEquals("http", info.getScheme());
-        assertEquals("example.com", info.getHost());
-        assertEquals(9200, info.getPort());
-        assertNull(info.getPathPrefix());
-        assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout()); // Didn't set the timeout so we should get the default
-        assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout()); // Didn't set the timeout so we should get the default
-
-        info = buildRemoteInfoHostTestCase("https://other.example.com:9201");
-        assertEquals("https", info.getScheme());
-        assertEquals("other.example.com", info.getHost());
-        assertEquals(9201, info.getPort());
-        assertNull(info.getPathPrefix());
-        assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout());
-        assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout());
-
-        info = buildRemoteInfoHostTestCase("https://[::1]:9201");
-        assertEquals("https", info.getScheme());
-        assertEquals("[::1]", info.getHost());
-        assertEquals(9201, info.getPort());
-        assertNull(info.getPathPrefix());
-        assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout());
-        assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout());
-
-        info = buildRemoteInfoHostTestCase("https://other.example.com:9201/");
-        assertEquals("https", info.getScheme());
-        assertEquals("other.example.com", info.getHost());
-        assertEquals(9201, info.getPort());
-        assertEquals("/", info.getPathPrefix());
-        assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout());
-        assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout());
-
-        info = buildRemoteInfoHostTestCase("https://other.example.com:9201/proxy-path/");
-        assertEquals("https", info.getScheme());
-        assertEquals("other.example.com", info.getHost());
-        assertEquals(9201, info.getPort());
-        assertEquals("/proxy-path/", info.getPathPrefix());
-        assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout());
-        assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout());
-
-        final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
-            () -> buildRemoteInfoHostTestCase("https"));
-        assertEquals("[host] must be of the form [scheme]://[host]:[port](/[pathPrefix])? but was [https]",
-            exception.getMessage());
-    }
-
-    public void testReindexFromRemoteRequestParsing() throws IOException {
-        BytesReference request;
-        try (XContentBuilder b = JsonXContent.contentBuilder()) {
-            b.startObject(); {
-                b.startObject("source"); {
-                    b.startObject("remote"); {
-                        b.field("host", "http://localhost:9200");
-                    }
-                    b.endObject();
-                    b.field("index", "source");
-                }
-                b.endObject();
-                b.startObject("dest"); {
-                    b.field("index", "dest");
-                }
-                b.endObject();
-            }
-            b.endObject();
-            request = BytesReference.bytes(b);
-        }
-        try (XContentParser p = createParser(JsonXContent.jsonXContent, request)) {
-            ReindexRequest r = new ReindexRequest();
-            RestReindexAction.PARSER.parse(p, r, null);
-            assertEquals("localhost", r.getRemoteInfo().getHost());
-            assertArrayEquals(new String[] {"source"}, r.getSearchRequest().indices());
-        }
-    }
-
     public void testPipelineQueryParameterIsError() throws IOException {
         FakeRestRequest.Builder request = new FakeRestRequest.Builder(xContentRegistry());
         try (XContentBuilder body = JsonXContent.contentBuilder().prettyPrint()) {
@@ -206,37 +81,6 @@ public void testSetScrollTimeout() throws IOException {
         }
     }
 
-    private RemoteInfo buildRemoteInfoHostTestCase(String hostInRest) throws IOException {
-        Map<String, Object> remote = new HashMap<>();
-        remote.put("host", hostInRest);
-
-        Map<String, Object> source = new HashMap<>();
-        source.put("remote", remote);
-
-        return RestReindexAction.buildRemoteInfo(source);
-    }
-
-    /**
-     * test deprecation is logged if one or more types are used in source search request inside reindex
-     */
-    public void testTypeInSource() throws IOException {
-        FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(xContentRegistry())
-                .withMethod(Method.POST)
-                .withPath("/_reindex");
-        XContentBuilder b = JsonXContent.contentBuilder().startObject();
-        {
-            b.startObject("source");
-            {
-                b.field("type", randomFrom(Arrays.asList("\"t1\"", "[\"t1\", \"t2\"]", "\"_doc\"")));
-            }
-            b.endObject();
-        }
-        b.endObject();
-        requestBuilder.withContent(new BytesArray(BytesReference.bytes(b).toBytesRef()), XContentType.JSON);
-        dispatchRequest(requestBuilder.build());
-        assertWarnings(RestReindexAction.TYPES_DEPRECATION_MESSAGE);
-    }
-
     /**
      * test deprecation is logged if a type is used in the destination index request inside reindex
      */
@@ -255,6 +99,6 @@ public void testTypeInDestination() throws IOException {
         b.endObject();
         requestBuilder.withContent(new BytesArray(BytesReference.bytes(b).toBytesRef()), XContentType.JSON);
         dispatchRequest(requestBuilder.build());
-        assertWarnings(RestReindexAction.TYPES_DEPRECATION_MESSAGE);
+        assertWarnings(ReindexRequest.TYPES_DEPRECATION_MESSAGE);
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
index 58067cd2cdbbf..4420fe450bd4d 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.tasks.TaskId;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -50,7 +49,6 @@
  * too but this is the only place that tests running against multiple nodes so it is the only integration tests that checks for
  * serialization.
  */
-@TestLogging("org.elasticsearch.index.reindex:TRACE,org.elasticsearch.action.bulk:TRACE,org.elasticsearch.search.SearchService:TRACE")
 public class RethrottleTests extends ReindexTestCase {
 
     public void testReindex() throws Exception {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
index ab1968a2e3582..d401efaae4bf0 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
@@ -51,7 +51,7 @@ public void testReindexRequest() throws IOException {
         reindex.getDestination().index("test");
         if (randomBoolean()) {
             int port = between(1, Integer.MAX_VALUE);
-            BytesReference query = new BytesArray(randomAlphaOfLength(5));
+            BytesReference query = new BytesArray("{\"match_all\":{}}");
             String username = randomBoolean() ? randomAlphaOfLength(5) : null;
             String password = username != null && randomBoolean() ? randomAlphaOfLength(5) : null;
             int headersCount = randomBoolean() ? 0 : between(1, 10);
@@ -107,7 +107,7 @@ private void randomRequest(AbstractBulkByScrollRequest<?> request) {
         request.getSearchRequest().indices("test");
         request.getSearchRequest().source().size(between(1, 1000));
         if (randomBoolean()) {
-            request.setSize(between(1, Integer.MAX_VALUE));
+            request.setMaxDocs(between(1, Integer.MAX_VALUE));
         }
         request.setAbortOnVersionConflict(random().nextBoolean());
         request.setRefresh(rarely());
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java
index 91a92005c2cdc..f55ee1e802e6b 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java
@@ -21,7 +21,6 @@
 
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.search.sort.SortOrder;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -34,14 +33,13 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.hasSize;
 
-@TestLogging("org.elasticsearch.index.reindex:TRACE,org.elasticsearch.action.bulk:TRACE,org.elasticsearch.search.SearchService:TRACE")
 public class UpdateByQueryBasicTests extends ReindexTestCase {
     public void testBasics() throws Exception {
         indexRandom(true, client().prepareIndex("test", "test", "1").setSource("foo", "a"),
                 client().prepareIndex("test", "test", "2").setSource("foo", "a"),
                 client().prepareIndex("test", "test", "3").setSource("foo", "b"),
                 client().prepareIndex("test", "test", "4").setSource("foo", "c"));
-        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 4);
+        assertHitCount(client().prepareSearch("test").setSize(0).get(), 4);
         assertEquals(1, client().prepareGet("test", "test", "1").get().getVersion());
         assertEquals(1, client().prepareGet("test", "test", "4").get().getVersion());
 
@@ -79,7 +77,7 @@ public void testSlices() throws Exception {
             client().prepareIndex("test", "test", "2").setSource("foo", "a"),
             client().prepareIndex("test", "test", "3").setSource("foo", "b"),
             client().prepareIndex("test", "test", "4").setSource("foo", "c"));
-        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 4);
+        assertHitCount(client().prepareSearch("test").setSize(0).get(), 4);
         assertEquals(1, client().prepareGet("test", "test", "1").get().getVersion());
         assertEquals(1, client().prepareGet("test", "test", "4").get().getVersion());
 
@@ -160,4 +158,13 @@ public void testMultipleSources() throws Exception {
             assertEquals(2, client().prepareGet(index, "test", Integer.toString(randomDoc)).get().getVersion());
         }
     }
+
+    public void testMissingSources() {
+        BulkByScrollResponse response = updateByQuery()
+            .source("missing-index-*")
+            .refresh(true)
+            .setSlices(AbstractBulkByScrollRequest.AUTO_SLICES)
+            .get();
+        assertThat(response, matcher().updated(0).slices(hasSize(0)));
+    }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java
index 9feed83595ff1..27d975c4114f9 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java
@@ -56,19 +56,38 @@ private void oldEsTestCase(String portPropertyName, String requestsPerSecond) th
                 }
 
                 Request reindex = new Request("POST", "/_reindex");
-                reindex.setJsonEntity(
+                if (randomBoolean()) {
+                    // Reindex using the external version_type
+                    reindex.setJsonEntity(
                         "{\n"
-                      + "  \"source\":{\n"
-                      + "    \"index\": \"test\",\n"
-                      + "    \"size\": 1,\n"
-                      + "    \"remote\": {\n"
-                      + "      \"host\": \"http://127.0.0.1:" + oldEsPort + "\"\n"
-                      + "    }\n"
-                      + "  },\n"
-                      + "  \"dest\": {\n"
-                      + "    \"index\": \"test\"\n"
-                      + "  }\n"
-                      + "}");
+                            + "  \"source\":{\n"
+                            + "    \"index\": \"test\",\n"
+                            + "    \"size\": 1,\n"
+                            + "    \"remote\": {\n"
+                            + "      \"host\": \"http://127.0.0.1:" + oldEsPort + "\"\n"
+                            + "    }\n"
+                            + "  },\n"
+                            + "  \"dest\": {\n"
+                            + "    \"index\": \"test\",\n"
+                            + "    \"version_type\": \"external\"\n"
+                            + "  }\n"
+                            + "}");
+                } else {
+                    // Reindex using the default internal version_type
+                    reindex.setJsonEntity(
+                        "{\n"
+                            + "  \"source\":{\n"
+                            + "    \"index\": \"test\",\n"
+                            + "    \"size\": 1,\n"
+                            + "    \"remote\": {\n"
+                            + "      \"host\": \"http://127.0.0.1:" + oldEsPort + "\"\n"
+                            + "    }\n"
+                            + "  },\n"
+                            + "  \"dest\": {\n"
+                            + "    \"index\": \"test\"\n"
+                            + "  }\n"
+                            + "}");
+                }
                 reindex.addParameter("refresh", "true");
                 reindex.addParameter("pretty", "true");
                 if (requestsPerSecond != null) {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteInfoTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteInfoTests.java
index de0ade9c47cc3..f0c4ccf476636 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteInfoTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteInfoTests.java
@@ -19,28 +19,28 @@
 
 package org.elasticsearch.index.reindex.remote;
 
-import org.elasticsearch.index.reindex.RemoteInfo;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.index.reindex.RemoteInfo;
 import org.elasticsearch.test.ESTestCase;
 
 import static java.util.Collections.emptyMap;
 
 public class RemoteInfoTests extends ESTestCase {
     private RemoteInfo newRemoteInfo(String scheme, String prefixPath, String username, String password) {
-        return new RemoteInfo(scheme, "testhost", 12344, prefixPath, new BytesArray("testquery"), username, password, emptyMap(),
-                RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
+        return new RemoteInfo(scheme, "testhost", 12344, prefixPath,new BytesArray("{ \"foo\" : \"bar\" }"), username, password,
+            emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
     }
 
     public void testToString() {
-        assertEquals("host=testhost port=12344 query=testquery",
+        assertEquals("host=testhost port=12344 query={ \"foo\" : \"bar\" }",
             newRemoteInfo("http", null, null, null).toString());
-        assertEquals("host=testhost port=12344 query=testquery username=testuser",
+        assertEquals("host=testhost port=12344 query={ \"foo\" : \"bar\" } username=testuser",
             newRemoteInfo("http", null, "testuser", null).toString());
-        assertEquals("host=testhost port=12344 query=testquery username=testuser password=<<>>",
+        assertEquals("host=testhost port=12344 query={ \"foo\" : \"bar\" } username=testuser password=<<>>",
             newRemoteInfo("http", null, "testuser", "testpass").toString());
-        assertEquals("scheme=https host=testhost port=12344 query=testquery username=testuser password=<<>>",
+        assertEquals("scheme=https host=testhost port=12344 query={ \"foo\" : \"bar\" } username=testuser password=<<>>",
             newRemoteInfo("https", null, "testuser", "testpass").toString());
-        assertEquals("scheme=https host=testhost port=12344 pathPrefix=prxy query=testquery username=testuser password=<<>>",
+        assertEquals("scheme=https host=testhost port=12344 pathPrefix=prxy query={ \"foo\" : \"bar\" } username=testuser password=<<>>",
             newRemoteInfo("https", "prxy", "testuser", "testpass").toString());
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java
index bf6856754044d..0005fa921b33b 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java
@@ -63,50 +63,27 @@ public void testIntialSearchPath() {
         SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder());
         assertEquals("/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
         searchRequest.indices("a");
-        searchRequest.types("b");
-        assertEquals("/a/b/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
+        assertEquals("/a/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
         searchRequest.indices("a", "b");
-        searchRequest.types("c", "d");
-        assertEquals("/a,b/c,d/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
+        assertEquals("/a,b/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
         searchRequest.indices("cat,");
-        assertEquals("/cat%2C/c,d/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
+        assertEquals("/cat%2C/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
         searchRequest.indices("cat/");
-        assertEquals("/cat%2F/c,d/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
+        assertEquals("/cat%2F/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
         searchRequest.indices("cat/", "dog");
-        assertEquals("/cat%2F,dog/c,d/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
+        assertEquals("/cat%2F,dog/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
         // test a specific date math + all characters that need escaping.
         searchRequest.indices("<cat{now/d}>", "<>/{}|+:,");
-        assertEquals("/%3Ccat%7Bnow%2Fd%7D%3E,%3C%3E%2F%7B%7D%7C%2B%3A%2C/c,d/_search",
+        assertEquals("/%3Ccat%7Bnow%2Fd%7D%3E,%3C%3E%2F%7B%7D%7C%2B%3A%2C/_search",
             initialSearch(searchRequest, query, remoteVersion).getEndpoint());
 
         // re-escape already escaped (no special handling).
         searchRequest.indices("%2f", "%3a");
-        assertEquals("/%252f,%253a/c,d/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
+        assertEquals("/%252f,%253a/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
         searchRequest.indices("%2fcat,");
-        assertEquals("/%252fcat%2C/c,d/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
+        assertEquals("/%252fcat%2C/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
         searchRequest.indices("%3ccat/");
-        assertEquals("/%253ccat%2F/c,d/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
-
-        searchRequest.indices("ok");
-        searchRequest.types("cat,");
-        expectBadStartRequest(searchRequest, "Type", ",", "cat,");
-        searchRequest.types("cat,", "dog");
-        expectBadStartRequest(searchRequest, "Type", ",", "cat,");
-        searchRequest.types("dog", "cat,");
-        expectBadStartRequest(searchRequest, "Type", ",", "cat,");
-        searchRequest.types("cat/");
-        expectBadStartRequest(searchRequest, "Type", "/", "cat/");
-        searchRequest.types("cat/", "dog");
-        expectBadStartRequest(searchRequest, "Type", "/", "cat/");
-        searchRequest.types("dog", "cat/");
-        expectBadStartRequest(searchRequest, "Type", "/", "cat/");
-    }
-
-    private void expectBadStartRequest(SearchRequest searchRequest, String type, String bad, String failed) {
-        Version remoteVersion = Version.fromId(between(0, Version.CURRENT.id));
-        BytesReference query = new BytesArray("{}");
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> initialSearch(searchRequest, query, remoteVersion));
-        assertEquals(type + " containing [" + bad + "] not supported but got [" + failed + "]", e.getMessage());
+        assertEquals("/%253ccat%2F/_search", initialSearch(searchRequest, query, remoteVersion).getEndpoint());
     }
 
     public void testInitialSearchParamsSort() {
@@ -136,7 +113,7 @@ public void testInitialSearchParamsFields() {
         // Test request without any fields
         Version remoteVersion = Version.fromId(between(2000099, Version.CURRENT.id));
         assertThat(initialSearch(searchRequest, query, remoteVersion).getParameters(),
-                not(either(hasKey("stored_fields")).or(hasKey("fields"))));
+            not(either(hasKey("stored_fields")).or(hasKey("fields"))));
 
         // Test stored_fields for versions that support it
         searchRequest = new SearchRequest().source(new SearchSourceBuilder());
@@ -157,14 +134,14 @@ public void testInitialSearchParamsFields() {
         searchRequest.source().storedField("_source").storedField("_id");
         remoteVersion = Version.fromId(between(0, 2000099 - 1));
         assertThat(initialSearch(searchRequest, query, remoteVersion).getParameters(),
-                hasEntry("fields", "_source,_id,_parent,_routing,_ttl"));
+            hasEntry("fields", "_source,_id,_parent,_routing,_ttl"));
 
         // But only versions before 1.0 force _source to be in the list
         searchRequest = new SearchRequest().source(new SearchSourceBuilder());
         searchRequest.source().storedField("_id");
         remoteVersion = Version.fromId(between(1000099, 2000099 - 1));
         assertThat(initialSearch(searchRequest, query, remoteVersion).getParameters(),
-                hasEntry("fields", "_id,_parent,_routing,_ttl"));
+            hasEntry("fields", "_id,_parent,_routing,_ttl"));
     }
 
     public void testInitialSearchParamsMisc() {
@@ -184,7 +161,7 @@ public void testInitialSearchParamsMisc() {
             fetchVersion = randomBoolean();
             searchRequest.source().version(fetchVersion);
         }
-        
+
         Map<String, String> params = initialSearch(searchRequest, query, remoteVersion).getParameters();
 
         if (scroll == null) {
@@ -193,7 +170,12 @@ public void testInitialSearchParamsMisc() {
             assertScroll(remoteVersion, params, scroll);
         }
         assertThat(params, hasEntry("size", Integer.toString(size)));
-        assertThat(params, fetchVersion == null || fetchVersion == true ? hasEntry("version", null) : not(hasEntry("version", null)));
+        if (fetchVersion != null) {
+            assertThat(params, fetchVersion ? hasEntry("version", Boolean.TRUE.toString()) :
+                hasEntry("version", Boolean.FALSE.toString()));
+        } else {
+            assertThat(params, hasEntry("version", Boolean.FALSE.toString()));
+        }
     }
 
     private void assertScroll(Version remoteVersion, Map<String, String> params, TimeValue requested) {
@@ -220,22 +202,22 @@ public void testInitialSearchEntity() throws IOException {
         assertEquals(ContentType.APPLICATION_JSON.toString(), entity.getContentType().getValue());
         if (remoteVersion.onOrAfter(Version.fromId(1000099))) {
             assertEquals("{\"query\":" + query + ",\"_source\":true}",
-                    Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)));
+                Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)));
         } else {
             assertEquals("{\"query\":" + query + "}",
-                    Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)));
+                Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)));
         }
 
         // Source filtering is included if set up
-        searchRequest.source().fetchSource(new String[] {"in1", "in2"}, new String[] {"out"});
+        searchRequest.source().fetchSource(new String[]{"in1", "in2"}, new String[]{"out"});
         entity = initialSearch(searchRequest, new BytesArray(query), remoteVersion).getEntity();
         assertEquals(ContentType.APPLICATION_JSON.toString(), entity.getContentType().getValue());
         assertEquals("{\"query\":" + query + ",\"_source\":{\"includes\":[\"in1\",\"in2\"],\"excludes\":[\"out\"]}}",
-                Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)));
+            Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)));
 
         // Invalid XContent fails
         RuntimeException e = expectThrows(RuntimeException.class,
-                () -> initialSearch(searchRequest, new BytesArray("{}, \"trailing\": {}"), remoteVersion));
+            () -> initialSearch(searchRequest, new BytesArray("{}, \"trailing\": {}"), remoteVersion));
         assertThat(e.getCause().getMessage(), containsString("Unexpected character (',' (code 44))"));
         e = expectThrows(RuntimeException.class, () -> initialSearch(searchRequest, new BytesArray("{"), remoteVersion));
         assertThat(e.getCause().getMessage(), containsString("Unexpected end-of-input"));
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsersTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsersTests.java
new file mode 100644
index 0000000000000..f5b406f779a9e
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsersTests.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex.remote;
+
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.reindex.ScrollableHitSource;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+public class RemoteResponseParsersTests extends ESTestCase {
+
+    /**
+     * Check that we can parse shard search failures without index information.
+     */
+    public void testFailureWithoutIndex() throws IOException {
+        ShardSearchFailure failure = new ShardSearchFailure(new EsRejectedExecutionException("exhausted"));
+        XContentBuilder builder = jsonBuilder();
+        failure.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        try (XContentParser parser = createParser(builder)) {
+            ScrollableHitSource.SearchFailure parsed = RemoteResponseParsers.SEARCH_FAILURE_PARSER.parse(parser, XContentType.JSON);
+            assertNotNull(parsed.getReason());
+            assertThat(parsed.getReason().getMessage(), Matchers.containsString("exhausted"));
+            assertThat(parsed.getReason(), Matchers.instanceOf(EsRejectedExecutionException.class));
+        }
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java
index 844c6b8351993..0ab100a856fc1 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java
@@ -40,7 +40,6 @@
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.bulk.BackoffPolicy;
-import org.elasticsearch.index.reindex.ScrollableHitSource.Response;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.HeapBufferedAsyncResponseConsumer;
 import org.elasticsearch.client.RestClient;
@@ -53,6 +52,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.index.reindex.ScrollableHitSource.Response;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESTestCase;
@@ -123,41 +123,20 @@ public void tearDown() throws Exception {
     }
 
     public void testLookupRemoteVersion() throws Exception {
+        assertLookupRemoteVersion(Version.fromString("0.20.5"), "main/0_20_5.json");
+        assertLookupRemoteVersion(Version.fromString("0.90.13"), "main/0_90_13.json");
+        assertLookupRemoteVersion(Version.fromString("1.7.5"), "main/1_7_5.json");
+        assertLookupRemoteVersion(Version.fromId(2030399), "main/2_3_3.json");
+        // assert for V_5_0_0 (no qualifier) since we no longer consider qualifier in Version since 7
+        assertLookupRemoteVersion(Version.fromId(5000099), "main/5_0_0_alpha_3.json");
+        // V_5_0_0 since we no longer consider qualifier in Version
+        assertLookupRemoteVersion(Version.fromId(5000099), "main/with_unknown_fields.json");
+    }
+
+    private void assertLookupRemoteVersion(Version expected, String s) throws Exception {
         AtomicBoolean called = new AtomicBoolean();
-        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/0_20_5.json").lookupRemoteVersion(v -> {
-            assertEquals(Version.fromString("0.20.5"), v);
-            called.set(true);
-        });
-        assertTrue(called.get());
-        called.set(false);
-        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/0_90_13.json").lookupRemoteVersion(v -> {
-            assertEquals(Version.fromString("0.90.13"), v);
-            called.set(true);
-        });
-        assertTrue(called.get());
-        called.set(false);
-        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/1_7_5.json").lookupRemoteVersion(v -> {
-            assertEquals(Version.fromString("1.7.5"), v);
-            called.set(true);
-        });
-        assertTrue(called.get());
-        called.set(false);
-        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/2_3_3.json").lookupRemoteVersion(v -> {
-            assertEquals(Version.fromId(2030399), v);
-            called.set(true);
-        });
-        assertTrue(called.get());
-        called.set(false);
-        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/5_0_0_alpha_3.json").lookupRemoteVersion(v -> {
-            // assert for V_5_0_0 (no qualifier) since we no longer consider qualifier in Version since 7
-            assertEquals(Version.fromId(5000099), v);
-            called.set(true);
-        });
-        assertTrue(called.get());
-        called.set(false);
-        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/with_unknown_fields.json").lookupRemoteVersion(v -> {
-            // V_5_0_0 since we no longer consider qualifier in Version
-            assertEquals(Version.fromId(5000099), v);
+        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, s).lookupRemoteVersion(v -> {
+            assertEquals(expected, v);
             called.set(true);
         });
         assertTrue(called.get());
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml
index ded91a095f611..1763baebe0277 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml
@@ -280,7 +280,11 @@
   - match: {count: 1}
 
 ---
-"Limit by size":
+"Limit by size pre 7.3":
+  - skip:
+      version: "7.3.0 - "
+      reason: "7.3 should use max_docs or get deprecation warning"
+
   - do:
       index:
         index:  twitter
@@ -318,6 +322,93 @@
 
   - match: {count: 1}
 
+---
+"Limit by max_docs in URL":
+  - skip:
+      version: " - 7.2.99"
+      reason: "max_docs introduced in 7.3.0"
+
+  - do:
+      index:
+        index:  twitter
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      index:
+        index:  twitter
+        id:     2
+        body:   { "user": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      delete_by_query:
+        index: twitter
+        max_docs: 1
+        body:
+          query:
+            match_all: {}
+
+  - match: {deleted: 1}
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      count:
+        index: twitter
+
+  - match: {count: 1}
+
+---
+"Limit by max_docs in body":
+  - skip:
+      version: " - 7.2.99"
+      reason: "max_docs introduced in 7.3.0"
+
+  - do:
+      index:
+        index:  twitter
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      index:
+        index:  twitter
+        id:     2
+        body:   { "user": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      delete_by_query:
+        index: twitter
+        body:
+          max_docs: 1
+          query:
+            match:
+              user: "kimchy"
+
+  - match: {deleted: 1}
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      count:
+        index: twitter
+
+  - match: {count: 1}
+
 ---
 "Can override scroll_size":
   - do:
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml
index 3caa8c4b43405..45de10b370669 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml
@@ -31,17 +31,21 @@
             match_all: {}
 
 ---
-"invalid size fails":
+"invalid max_docs fails":
+  - skip:
+      version: " - 7.2.99"
+      reason: "max_docs introduced in 7.3.0"
+
   - do:
       index:
         index:   test
         id:      1
         body:    { "text": "test" }
   - do:
-      catch: /\[size\] parameter cannot be negative, found \[-4\]/
+      catch: /\[max_docs\] parameter cannot be negative, found \[-4\]/
       delete_by_query:
         index: test
-        size: -4
+        max_docs: -4
         body:
           query:
             match_all: {}
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml
index d45b14a1edc5f..44971c49c358a 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml
@@ -95,21 +95,90 @@
           conflicts: cat
 
 ---
-"invalid size fails":
+"specifying size fails":
+  - skip:
+      version: " - 7.99.99"
+      reason: "size supported until 8"
+
+  - do:
+      index:
+        index:  test
+        id:     1
+        body:   { "text": "test" }
+
+  - do:
+      catch: /invalid parameter \[size\], use \[max_docs\] instead/
+      reindex:
+        body:
+          source:
+            index: test
+          dest:
+            index: dest
+          size: 1
+
+---
+"invalid max_docs in body fails":
+  - skip:
+      version: " - 7.2.99"
+      reason: "max_docs introduced in 7.3.0"
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      catch: /\[max_docs\] parameter cannot be negative, found \[-4\]/
+      reindex:
+        body:
+          source:
+            index: test
+          dest:
+            index: dest
+          max_docs: -4
+
+---
+"invalid max_docs in URL fails":
+  - skip:
+      version: " - 7.2.99"
+      reason: "max_docs introduced in 7.3.0"
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      catch: /\[max_docs\] parameter cannot be negative, found \[-4\]/
+      reindex:
+        max_docs: -4
+        body:
+          source:
+            index: test
+          dest:
+            index: dest
+
+---
+"inconsistent max_docs in body and max_docs in URL fails":
+  - skip:
+      version: " - 7.2.99"
+      reason: "max_docs introduced in 7.3.0"
+
   - do:
       index:
         index:   test
         id:      1
         body:    { "text": "test" }
   - do:
-      catch: /\[size\] parameter cannot be negative, found \[-4\]/
+      catch: /\[max_docs\] set to two different values \[4\] and \[5\]/
       reindex:
+        max_docs: 5
         body:
           source:
             index: test
           dest:
             index: dest
-          size: -4
+          max_docs: 4
 
 ---
 "requests_per_second cannot be an empty string":
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/30_search.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/30_search.yml
index dff24e4413368..709b9c0d17340 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/30_search.yml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/30_search.yml
@@ -32,7 +32,11 @@
   - match: { hits.total: 1 }
 
 ---
-"Sorting and size combined":
+"Sorting and size combined pre 7.3":
+  - skip:
+      version: "7.3.0 - "
+      reason: "7.3 should use max_docs or get deprecation warning"
+
   - do:
       index:
         index:  test
@@ -69,3 +73,81 @@
         index: target
         q: order:1
   - match: { hits.total: 1 }
+
+---
+"Sorting and max_docs in body combined":
+  - skip:
+      version: " - 7.2.99"
+      reason: "max_docs introduced in 7.3.0"
+
+  - do:
+      index:
+        index:  test
+        id:     1
+        body:   { "order": 1 }
+  - do:
+      index:
+        index:  test
+        id:     2
+        body:   { "order": 2 }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        refresh: true
+        body:
+          max_docs: 1
+          source:
+            index: test
+            sort: order
+          dest:
+            index: target
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: target
+  - match: { hits.total: 1 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: target
+        q: order:1
+  - match: { hits.total: 1 }
+
+---
+"max_docs in URL":
+  - skip:
+      version: " - 7.2.99"
+      reason: "max_docs introduced in 7.3.0"
+
+  - do:
+      index:
+        index:  test
+        id:     1
+        body:   { "value": 17 }
+  - do:
+      index:
+        index:  test
+        id:     2
+        body:   { "value": 17 }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        refresh: true
+        max_docs: 1
+        body:
+          source:
+            index: test
+          dest:
+            index: target
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: target
+  - match: { hits.total: 1 }
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yml
index 65d6438e75d28..8354fc0aaf322 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yml
@@ -218,7 +218,11 @@
   - match: {indices.source.total.search.open_contexts: 0}
 
 ---
-"Reindex from remote with size":
+"Reindex from remote with max_docs":
+  - skip:
+      version: " - 7.2.99"
+      reason: "max_docs introduced in 7.3.0"
+
   - do:
       index:
         index:   source
@@ -245,7 +249,7 @@
       reindex:
         refresh: true
         body:
-          size: 1
+          max_docs: 1
           source:
             remote:
               host: http://${host}
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml
index 40119f22abb5b..2a3696a4005c7 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml
@@ -218,7 +218,11 @@
   - gte: { took: 0 }
 
 ---
-"Limit by size":
+"Limit by size pre 7.3":
+  - skip:
+      version: "7.3.0 - "
+      reason: "7.3 should use max_docs or get deprecation warning"
+
   - do:
       index:
         index:  twitter
@@ -243,6 +247,70 @@
   - match: {throttled_millis: 0}
   - gte: { took: 0 }
 
+---
+"Limit by max_docs in URL":
+  - skip:
+      version: " - 7.2.99"
+      reason: "max_docs introduced in 7.3.0"
+
+  - do:
+      index:
+        index:  twitter
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      index:
+        index:  twitter
+        id:     2
+        body:   { "user": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      update_by_query:
+        index: twitter
+        max_docs: 1
+  - match: {updated: 1}
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+
+---
+"Limit by max_docs in body":
+  - skip:
+      version: " - 7.2.99"
+      reason: "max_docs introduced in 7.3.0"
+
+  - do:
+      index:
+        index:  twitter
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      index:
+        index:  twitter
+        id:     2
+        body:   { "user": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      update_by_query:
+        index: twitter
+        body:
+          max_docs: 1
+          query:
+            match:
+              user: "kimchy"
+  - match: {updated: 1}
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+
 ---
 "Can override scroll_size":
   - do:
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml
index b6823f88c15fc..21644b3932984 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml
@@ -12,17 +12,60 @@
         conflicts: cat
 
 ---
-"invalid size fails":
+"invalid max_docs in URL fails":
+  - skip:
+      version: " - 7.2.99"
+      reason: "max_docs introduced in 7.3.0"
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      catch: /\[max_docs\] parameter cannot be negative, found \[-4\]/
+      update_by_query:
+        index: test
+        max_docs: -4
+
+---
+"invalid max_docs in body fails":
+  - skip:
+      version: " - 7.2.99"
+      reason: "max_docs introduced in 7.3.0"
+
   - do:
       index:
         index:   test
         id:      1
         body:    { "text": "test" }
   - do:
-      catch: /\[size\] parameter cannot be negative, found \[-4\]/
+      catch: /\[max_docs\] parameter cannot be negative, found \[-4\]/
       update_by_query:
         index: test
-        size: -4
+        body:
+          max_docs: -4
+
+---
+"inconsistent max_docs in body and max_docs in URL fails":
+  - skip:
+      version: " - 7.2.99"
+      reason: "max_docs introduced in 7.3.0"
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      catch: /\[max_docs\] set to two different values \[4\] and \[5\]/
+      delete_by_query:
+        index: test
+        max_docs: 5
+        body:
+          max_docs: 4
+          query:
+            match_all: {}
 
 ---
 "invalid scroll_size fails":
diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java
index 91676037043e3..f9406e454474f 100644
--- a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java
+++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.common.blobstore.url;
 
 import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
@@ -74,6 +75,11 @@ public Map<String, BlobMetaData> listBlobs() throws IOException {
         throw new UnsupportedOperationException("URL repository doesn't support this operation");
     }
 
+    @Override
+    public Map<String, BlobContainer> children() throws IOException {
+        throw new UnsupportedOperationException("URL repository doesn't support this operation");
+    }
+
     /**
      * This operation is not supported by URLBlobContainer
      */
@@ -90,6 +96,11 @@ public void deleteBlob(String blobName) throws IOException {
         throw new UnsupportedOperationException("URL repository is read only");
     }
 
+    @Override
+    public void delete() {
+        throw new UnsupportedOperationException("URL repository is read only");
+    }
+
     /**
      * This operation is not supported by URLBlobContainer
      */
diff --git a/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java
index 0ea2a1b72c574..a27b091cfc037 100644
--- a/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java
+++ b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java
@@ -75,8 +75,6 @@ public class URLRepository extends BlobStoreRepository {
 
     private final Environment environment;
 
-    private final BlobPath basePath;
-
     private final URL url;
 
     /**
@@ -84,7 +82,7 @@ public class URLRepository extends BlobStoreRepository {
      */
     public URLRepository(RepositoryMetaData metadata, Environment environment,
                          NamedXContentRegistry namedXContentRegistry, ThreadPool threadPool) {
-        super(metadata, environment.settings(), namedXContentRegistry, threadPool);
+        super(metadata, environment.settings(), namedXContentRegistry, threadPool, BlobPath.cleanPath());
 
         if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(environment.settings()) ==  false) {
             throw new RepositoryException(metadata.name(), "missing url");
@@ -92,7 +90,6 @@ public URLRepository(RepositoryMetaData metadata, Environment environment,
         this.environment = environment;
         supportedProtocols = SUPPORTED_PROTOCOLS_SETTING.get(environment.settings());
         urlWhiteList = ALLOWED_URLS_SETTING.get(environment.settings()).toArray(new URIPattern[]{});
-        basePath = BlobPath.cleanPath();
         url = URL_SETTING.exists(metadata.settings())
             ? URL_SETTING.get(metadata.settings()) : REPOSITORIES_URL_SETTING.get(environment.settings());
     }
@@ -115,11 +112,6 @@ protected BlobStore getBlobStore() {
         return super.getBlobStore();
     }
 
-    @Override
-    protected BlobPath basePath() {
-        return basePath;
-    }
-
     /**
      * Makes sure that the url is white listed or if it points to the local file system it matches one on of the root path in path.repo
      */
diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java
index ab9268b081456..160333d5346ed 100644
--- a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java
+++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java
@@ -88,7 +88,7 @@ public void testUrlRepository() throws Exception {
             .prepareGetSnapshots("test-repo")
             .setSnapshots("test-snap")
             .get()
-            .getSnapshots()
+            .getSnapshots("test-repo")
             .get(0)
             .state();
         assertThat(state, equalTo(SnapshotState.SUCCESS));
@@ -116,8 +116,8 @@ public void testUrlRepository() throws Exception {
 
         logger.info("--> list available shapshots");
         GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
-        assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
-        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1));
+        assertThat(getSnapshotsResponse.getSnapshots("url-repo"), notNullValue());
+        assertThat(getSnapshotsResponse.getSnapshots("url-repo").size(), equalTo(1));
 
         logger.info("--> delete snapshot");
         AcknowledgedResponse deleteSnapshotResponse = client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get();
@@ -125,7 +125,7 @@ public void testUrlRepository() throws Exception {
 
         logger.info("--> list available shapshot again, no snapshots should be returned");
         getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
-        assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
-        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0));
+        assertThat(getSnapshotsResponse.getSnapshots("url-repo"), notNullValue());
+        assertThat(getSnapshotsResponse.getSnapshots("url-repo").size(), equalTo(0));
     }
 }
diff --git a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml
index b932f0d53caad..e76669dc75e01 100644
--- a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml
+++ b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml
@@ -112,6 +112,9 @@ teardown:
 
 ---
 "Restore with repository-url using http://":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   # Ensure that the URL repository is registered
   - do:
@@ -126,9 +129,9 @@ teardown:
         repository: repository-url
         snapshot: snapshot-one,snapshot-two
 
-  - is_true: snapshots
-  - match: { snapshots.0.state : SUCCESS }
-  - match: { snapshots.1.state : SUCCESS }
+  - is_true: responses.0.snapshots
+  - match: { responses.0.snapshots.0.state : SUCCESS }
+  - match: { responses.0.snapshots.1.state : SUCCESS }
 
   # Delete the index
   - do:
@@ -174,6 +177,9 @@ teardown:
 
 ---
 "Restore with repository-url using file://":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   # Ensure that the URL repository is registered
   - do:
@@ -188,9 +194,9 @@ teardown:
         repository: repository-file
         snapshot: snapshot-one,snapshot-two
 
-  - is_true: snapshots
-  - match: { snapshots.0.state : SUCCESS }
-  - match: { snapshots.1.state : SUCCESS }
+  - is_true: responses.0.snapshots
+  - match: { responses.0.snapshots.0.state : SUCCESS }
+  - match: { responses.0.snapshots.1.state : SUCCESS }
 
   # Delete the index
   - do:
@@ -236,13 +242,18 @@ teardown:
 
 ---
 "Get a non existing snapshot":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   - do:
-      catch: /snapshot_missing_exception/
       snapshot.get:
         repository: repository-url
         snapshot: missing
 
+  - is_true: responses.0.error
+  - match: { responses.0.error.type: snapshot_missing_exception }
+
 ---
 "Delete a non existing snapshot":
 
diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle
index d64e0aff7749d..49d9f3693fdc7 100644
--- a/modules/transport-netty4/build.gradle
+++ b/modules/transport-netty4/build.gradle
@@ -26,7 +26,6 @@
 esplugin {
     description 'Netty 4 based transport implementation'
     classname 'org.elasticsearch.transport.Netty4Plugin'
-    hasClientJar = true
 }
 
 compileTestJava.options.compilerArgs << "-Xlint:-cast,-rawtypes,-unchecked"
@@ -76,6 +75,10 @@ thirdPartyAudit {
         'org.apache.commons.logging.Log',
         'org.apache.commons.logging.LogFactory',
 
+        // from Log4j (deliberate, Netty will fallback to Log4j 2)
+        'org.apache.log4j.Level',
+        'org.apache.log4j.Logger',
+
         // from io.netty.handler.ssl.OpenSslEngine (netty)
         'io.netty.internal.tcnative.Buffer',
         'io.netty.internal.tcnative.Library',
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
index 356cfa0bbf99d..ad1dac038f4a9 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
@@ -23,8 +23,8 @@
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelFuture;
 import io.netty.channel.ChannelHandler;
-import io.netty.channel.ChannelHandlerAdapter;
 import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
 import io.netty.channel.ChannelInitializer;
 import io.netty.channel.ChannelOption;
 import io.netty.channel.FixedRecvByteBufAllocator;
@@ -34,7 +34,6 @@
 import io.netty.handler.codec.ByteToMessageDecoder;
 import io.netty.handler.codec.http.HttpContentCompressor;
 import io.netty.handler.codec.http.HttpContentDecompressor;
-import io.netty.handler.codec.http.HttpMethod;
 import io.netty.handler.codec.http.HttpObjectAggregator;
 import io.netty.handler.codec.http.HttpRequestDecoder;
 import io.netty.handler.codec.http.HttpResponseEncoder;
@@ -44,12 +43,10 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
@@ -60,26 +57,14 @@
 import org.elasticsearch.http.HttpHandlingSettings;
 import org.elasticsearch.http.HttpReadTimeoutException;
 import org.elasticsearch.http.HttpServerChannel;
-import org.elasticsearch.http.netty4.cors.Netty4CorsConfig;
-import org.elasticsearch.http.netty4.cors.Netty4CorsConfigBuilder;
 import org.elasticsearch.http.netty4.cors.Netty4CorsHandler;
-import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.netty4.Netty4Utils;
 
 import java.net.InetSocketAddress;
-import java.util.Arrays;
 import java.util.concurrent.TimeUnit;
-import java.util.regex.Pattern;
-import java.util.regex.PatternSyntaxException;
 
 import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_MAX_AGE;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CHUNK_SIZE;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE;
@@ -91,15 +76,10 @@
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_TCP_REUSE_ADDRESS;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_TCP_SEND_BUFFER_SIZE;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_PIPELINING_MAX_EVENTS;
-import static org.elasticsearch.http.netty4.cors.Netty4CorsHandler.ANY_ORIGIN;
 
 public class Netty4HttpServerTransport extends AbstractHttpServerTransport {
     private static final Logger logger = LogManager.getLogger(Netty4HttpServerTransport.class);
 
-    static {
-        Netty4Utils.setup();
-    }
-
     /*
      * Size in bytes of an individual message received by io.netty.handler.codec.MessageAggregator which accumulates the content for an
      * HTTP request. This number is used for estimating the maximum number of allowed buffers before the MessageAggregator's internal
@@ -160,8 +140,6 @@ public class Netty4HttpServerTransport extends AbstractHttpServerTransport {
 
     protected volatile ServerBootstrap serverBootstrap;
 
-    private final Netty4CorsConfig corsConfig;
-
     public Netty4HttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool,
                                      NamedXContentRegistry xContentRegistry, Dispatcher dispatcher) {
         super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher);
@@ -180,8 +158,6 @@ public Netty4HttpServerTransport(Settings settings, NetworkService networkServic
         ByteSizeValue receivePredictor = SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE.get(settings);
         recvByteBufAllocator = new FixedRecvByteBufAllocator(receivePredictor.bytesAsInt());
 
-        this.corsConfig = buildCorsConfig(settings);
-
         logger.debug("using max_chunk_size[{}], max_header_size[{}], max_initial_line_length[{}], max_content_length[{}], " +
                 "receive_predictor[{}], max_composite_buffer_components[{}], pipelining_max_events[{}]",
             maxChunkSize, maxHeaderSize, maxInitialLineLength, maxContentLength, receivePredictor, maxCompositeBufferComponents,
@@ -234,43 +210,6 @@ protected void doStart() {
         }
     }
 
-    // package private for testing
-    static Netty4CorsConfig buildCorsConfig(Settings settings) {
-        if (SETTING_CORS_ENABLED.get(settings) == false) {
-            return Netty4CorsConfigBuilder.forOrigins().disable().build();
-        }
-        String origin = SETTING_CORS_ALLOW_ORIGIN.get(settings);
-        final Netty4CorsConfigBuilder builder;
-        if (Strings.isNullOrEmpty(origin)) {
-            builder = Netty4CorsConfigBuilder.forOrigins();
-        } else if (origin.equals(ANY_ORIGIN)) {
-            builder = Netty4CorsConfigBuilder.forAnyOrigin();
-        } else {
-            try {
-                Pattern p = RestUtils.checkCorsSettingForRegex(origin);
-                if (p == null) {
-                    builder = Netty4CorsConfigBuilder.forOrigins(RestUtils.corsSettingAsArray(origin));
-                } else {
-                    builder = Netty4CorsConfigBuilder.forPattern(p);
-                }
-            } catch (PatternSyntaxException e) {
-                throw new SettingsException("Bad regex in [" + SETTING_CORS_ALLOW_ORIGIN.getKey() + "]: [" + origin + "]", e);
-            }
-        }
-        if (SETTING_CORS_ALLOW_CREDENTIALS.get(settings)) {
-            builder.allowCredentials();
-        }
-        String[] strMethods = Strings.tokenizeToStringArray(SETTING_CORS_ALLOW_METHODS.get(settings), ",");
-        HttpMethod[] methods = Arrays.stream(strMethods)
-            .map(HttpMethod::valueOf)
-            .toArray(HttpMethod[]::new);
-        return builder.allowedRequestMethods(methods)
-            .maxAge(SETTING_CORS_MAX_AGE.get(settings))
-            .allowedRequestHeaders(Strings.tokenizeToStringArray(SETTING_CORS_ALLOW_HEADERS.get(settings), ","))
-            .shortCircuit()
-            .build();
-    }
-
     @Override
     protected HttpServerChannel bind(InetSocketAddress socketAddress) throws Exception {
         ChannelFuture future = serverBootstrap.bind(socketAddress).sync();
@@ -351,7 +290,7 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E
     }
 
     @ChannelHandler.Sharable
-    private static class ServerChannelExceptionHandler extends ChannelHandlerAdapter {
+    private static class ServerChannelExceptionHandler extends ChannelInboundHandlerAdapter {
 
         private final Netty4HttpServerTransport transport;
 
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfig.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfig.java
deleted file mode 100644
index 17e21ce468b4c..0000000000000
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfig.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.http.netty4.cors;
-
-import io.netty.handler.codec.http.DefaultHttpHeaders;
-import io.netty.handler.codec.http.EmptyHttpHeaders;
-import io.netty.handler.codec.http.HttpHeaders;
-import io.netty.handler.codec.http.HttpMethod;
-
-import java.util.Collections;
-import java.util.LinkedHashSet;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.regex.Pattern;
-
-/**
- * Configuration for Cross-Origin Resource Sharing (CORS).
- *
- * This class was lifted from the Netty project:
- *  https://github.com/netty/netty
- */
-public final class Netty4CorsConfig {
-
-    private final Optional<Set<String>> origins;
-    private final Optional<Pattern> pattern;
-    private final boolean anyOrigin;
-    private final boolean enabled;
-    private final boolean allowCredentials;
-    private final long maxAge;
-    private final Set<HttpMethod> allowedRequestMethods;
-    private final Set<String> allowedRequestHeaders;
-    private final Map<CharSequence, Callable<?>> preflightHeaders;
-    private final boolean shortCircuit;
-
-    Netty4CorsConfig(final Netty4CorsConfigBuilder builder) {
-        origins = builder.origins.map(s -> new LinkedHashSet<>(s));
-        pattern = builder.pattern;
-        anyOrigin = builder.anyOrigin;
-        enabled = builder.enabled;
-        allowCredentials = builder.allowCredentials;
-        maxAge = builder.maxAge;
-        allowedRequestMethods = builder.requestMethods;
-        allowedRequestHeaders = builder.requestHeaders;
-        preflightHeaders = builder.preflightHeaders;
-        shortCircuit = builder.shortCircuit;
-    }
-
-    /**
-     * Determines if support for CORS is enabled.
-     *
-     * @return {@code true} if support for CORS is enabled, false otherwise.
-     */
-    public boolean isCorsSupportEnabled() {
-        return enabled;
-    }
-
-    /**
-     * Determines whether a wildcard origin, '*', is supported. This also means that null origins are
-     * supported.
-     *
-     * @return {@code boolean} true if any origin is allowed.
-     */
-    public boolean isAnyOriginSupported() {
-        return anyOrigin;
-    }
-
-    /**
-     * Returns the set of allowed origins.
-     *
-     * @return {@code Set} the allowed origins.
-     */
-    public Optional<Set<String>> origins() {
-        return origins;
-    }
-
-    /**
-     * Returns whether the input origin is allowed by this configuration.
-     *
-     * @return {@code true} if the origin is allowed, otherwise {@code false}
-     */
-    public boolean isOriginAllowed(final String origin) {
-        if (origins.isPresent()) {
-            return origins.get().contains(origin);
-        } else if (pattern.isPresent()) {
-            return pattern.get().matcher(origin).matches();
-        }
-        return false;
-    }
-
-    /**
-     * Determines if credentials are supported for CORS requests.
-     *
-     * By default credentials are not included in CORS requests but if isCredentialsAllowed returns
-     * true credentials will be added to CORS requests. Setting this value to true will set the
-     * CORS 'Access-Control-Allow-Credentials' response header to true.
-     *
-     * Please note that credentials support needs to be enabled on the client side as well.
-     * The client needs to opt-in to send credentials by calling:
-     * <pre>
-     * xhr.withCredentials = true;
-     * </pre>
-     * The default value for 'withCredentials' is false in which case no credentials are sent.
-     * Setting this to true will included credentials in cross origin requests.
-     *
-     * @return {@code true} if credentials are supported.
-     */
-    public boolean isCredentialsAllowed() {
-        return allowCredentials;
-    }
-
-    /**
-     * Gets the maxAge setting.
-     *
-     * When making a preflight request the client has to perform two request with can be inefficient.
-     * This setting will set the CORS 'Access-Control-Max-Age' response header and enables the
-     * caching of the preflight response for the specified time. During this time no preflight
-     * request will be made.
-     *
-     * @return {@code long} the time in seconds that a preflight request may be cached.
-     */
-    public long maxAge() {
-        return maxAge;
-    }
-
-    /**
-     * Returns the allowed set of Request Methods. The Http methods that should be returned in the
-     * CORS 'Access-Control-Request-Method' response header.
-     *
-     * @return {@code Set} of {@link HttpMethod}s that represent the allowed Request Methods.
-     */
-    public Set<HttpMethod> allowedRequestMethods() {
-        return Collections.unmodifiableSet(allowedRequestMethods);
-    }
-
-    /**
-     * Returns the allowed set of Request Headers.
-     *
-     * The header names returned from this method will be used to set the CORS
-     * 'Access-Control-Allow-Headers' response header.
-     *
-     * @return {@code Set<String>} of strings that represent the allowed Request Headers.
-     */
-    public Set<String> allowedRequestHeaders() {
-        return Collections.unmodifiableSet(allowedRequestHeaders);
-    }
-
-    /**
-     * Returns HTTP response headers that should be added to a CORS preflight response.
-     *
-     * @return {@link HttpHeaders} the HTTP response headers to be added.
-     */
-    public HttpHeaders preflightResponseHeaders() {
-        if (preflightHeaders.isEmpty()) {
-            return EmptyHttpHeaders.INSTANCE;
-        }
-        final HttpHeaders preflightHeaders = new DefaultHttpHeaders();
-        for (Map.Entry<CharSequence, Callable<?>> entry : this.preflightHeaders.entrySet()) {
-            final Object value = getValue(entry.getValue());
-            if (value instanceof Iterable) {
-                preflightHeaders.add(entry.getKey().toString(), (Iterable<?>) value);
-            } else {
-                preflightHeaders.add(entry.getKey().toString(), value);
-            }
-        }
-        return preflightHeaders;
-    }
-
-    /**
-     * Determines whether a CORS request should be rejected if it's invalid before being
-     * further processing.
-     *
-     * CORS headers are set after a request is processed. This may not always be desired
-     * and this setting will check that the Origin is valid and if it is not valid no
-     * further processing will take place, and a error will be returned to the calling client.
-     *
-     * @return {@code true} if a CORS request should short-circuit upon receiving an invalid Origin header.
-     */
-    public boolean isShortCircuit() {
-        return shortCircuit;
-    }
-
-    private static <T> T getValue(final Callable<T> callable) {
-        try {
-            return callable.call();
-        } catch (final Exception e) {
-            throw new IllegalStateException("Could not generate value for callable [" + callable + ']', e);
-        }
-    }
-
-    @Override
-    public String toString() {
-        return  "CorsConfig[enabled=" + enabled +
-            ", origins=" + origins +
-            ", anyOrigin=" + anyOrigin +
-            ", isCredentialsAllowed=" + allowCredentials +
-            ", maxAge=" + maxAge +
-            ", allowedRequestMethods=" + allowedRequestMethods +
-            ", allowedRequestHeaders=" + allowedRequestHeaders +
-            ", preflightHeaders=" + preflightHeaders + ']';
-    }
-
-}
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfigBuilder.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfigBuilder.java
deleted file mode 100644
index 3e87d948dabf4..0000000000000
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsConfigBuilder.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.http.netty4.cors;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.regex.Pattern;
-
-/**
- * Builder used to configure and build a {@link Netty4CorsConfig} instance.
- *
- * This class was lifted from the Netty project:
- *  https://github.com/netty/netty
- */
-public final class Netty4CorsConfigBuilder {
-
-    /**
-     * Creates a Builder instance with it's origin set to '*'.
-     *
-     * @return Builder to support method chaining.
-     */
-    public static Netty4CorsConfigBuilder forAnyOrigin() {
-        return new Netty4CorsConfigBuilder();
-    }
-
-    /**
-     * Create a {@link Netty4CorsConfigBuilder} instance with the specified pattern origin.
-     *
-     * @param pattern the regular expression pattern to match incoming origins on.
-     * @return {@link Netty4CorsConfigBuilder} with the configured origin pattern.
-     */
-    public static Netty4CorsConfigBuilder forPattern(final Pattern pattern) {
-        if (pattern == null) {
-            throw new IllegalArgumentException("CORS pattern cannot be null");
-        }
-        return new Netty4CorsConfigBuilder(pattern);
-    }
-
-    /**
-     * Creates a {@link Netty4CorsConfigBuilder} instance with the specified origins.
-     *
-     * @return {@link Netty4CorsConfigBuilder} to support method chaining.
-     */
-    public static Netty4CorsConfigBuilder forOrigins(final String... origins) {
-        return new Netty4CorsConfigBuilder(origins);
-    }
-
-    Optional<Set<String>> origins;
-    Optional<Pattern> pattern;
-    final boolean anyOrigin;
-    boolean enabled = true;
-    boolean allowCredentials;
-    long maxAge;
-    final Set<HttpMethod> requestMethods = new HashSet<>();
-    final Set<String> requestHeaders = new HashSet<>();
-    final Map<CharSequence, Callable<?>> preflightHeaders = new HashMap<>();
-    boolean shortCircuit;
-
-    /**
-     * Creates a new Builder instance with the origin passed in.
-     *
-     * @param origins the origin to be used for this builder.
-     */
-    Netty4CorsConfigBuilder(final String... origins) {
-        this.origins = Optional.of(new LinkedHashSet<>(Arrays.asList(origins)));
-        pattern = Optional.empty();
-        anyOrigin = false;
-    }
-
-    /**
-     * Creates a new Builder instance allowing any origin, "*" which is the
-     * wildcard origin.
-     *
-     */
-    Netty4CorsConfigBuilder() {
-        anyOrigin = true;
-        origins = Optional.empty();
-        pattern = Optional.empty();
-    }
-
-    /**
-     * Creates a new Builder instance allowing any origin that matches the pattern.
-     *
-     * @param pattern the pattern to match against for incoming origins.
-     */
-    Netty4CorsConfigBuilder(final Pattern pattern) {
-        this.pattern = Optional.of(pattern);
-        origins = Optional.empty();
-        anyOrigin = false;
-    }
-
-    /**
-     * Disables CORS support.
-     *
-     * @return {@link Netty4CorsConfigBuilder} to support method chaining.
-     */
-    public Netty4CorsConfigBuilder disable() {
-        enabled = false;
-        return this;
-    }
-
-    /**
-     * By default cookies are not included in CORS requests, but this method will enable cookies to
-     * be added to CORS requests. Calling this method will set the CORS 'Access-Control-Allow-Credentials'
-     * response header to true.
-     *
-     * Please note, that cookie support needs to be enabled on the client side as well.
-     * The client needs to opt-in to send cookies by calling:
-     * <pre>
-     * xhr.withCredentials = true;
-     * </pre>
-     * The default value for 'withCredentials' is false in which case no cookies are sent.
-     * Setting this to true will included cookies in cross origin requests.
-     *
-     * @return {@link Netty4CorsConfigBuilder} to support method chaining.
-     */
-    public Netty4CorsConfigBuilder allowCredentials() {
-        allowCredentials = true;
-        return this;
-    }
-
-    /**
-     * When making a preflight request the client has to perform two request with can be inefficient.
-     * This setting will set the CORS 'Access-Control-Max-Age' response header and enables the
-     * caching of the preflight response for the specified time. During this time no preflight
-     * request will be made.
-     *
-     * @param max the maximum time, in seconds, that the preflight response may be cached.
-     * @return {@link Netty4CorsConfigBuilder} to support method chaining.
-     */
-    public Netty4CorsConfigBuilder maxAge(final long max) {
-        maxAge = max;
-        return this;
-    }
-
-    /**
-     * Specifies the allowed set of HTTP Request Methods that should be returned in the
-     * CORS 'Access-Control-Request-Method' response header.
-     *
-     * @param methods the {@link HttpMethod}s that should be allowed.
-     * @return {@link Netty4CorsConfigBuilder} to support method chaining.
-     */
-    public Netty4CorsConfigBuilder allowedRequestMethods(final HttpMethod... methods) {
-        requestMethods.addAll(Arrays.asList(methods));
-        return this;
-    }
-
-    /**
-     * Specifies the if headers that should be returned in the CORS 'Access-Control-Allow-Headers'
-     * response header.
-     *
-     * If a client specifies headers on the request, for example by calling:
-     * <pre>
-     * xhr.setRequestHeader('My-Custom-Header', "SomeValue");
-     * </pre>
-     * the server will receive the above header name in the 'Access-Control-Request-Headers' of the
-     * preflight request. The server will then decide if it allows this header to be sent for the
-     * real request (remember that a preflight is not the real request but a request asking the server
-     * if it allow a request).
-     *
-     * @param headers the headers to be added to the preflight 'Access-Control-Allow-Headers' response header.
-     * @return {@link Netty4CorsConfigBuilder} to support method chaining.
-     */
-    public Netty4CorsConfigBuilder allowedRequestHeaders(final String... headers) {
-        requestHeaders.addAll(Arrays.asList(headers));
-        return this;
-    }
-
-    /**
-     * Specifies that a CORS request should be rejected if it's invalid before being
-     * further processing.
-     *
-     * CORS headers are set after a request is processed. This may not always be desired
-     * and this setting will check that the Origin is valid and if it is not valid no
-     * further processing will take place, and a error will be returned to the calling client.
-     *
-     * @return {@link Netty4CorsConfigBuilder} to support method chaining.
-     */
-    public Netty4CorsConfigBuilder shortCircuit() {
-        shortCircuit = true;
-        return this;
-    }
-
-    /**
-     * Builds a {@link Netty4CorsConfig} with settings specified by previous method calls.
-     *
-     * @return {@link Netty4CorsConfig} the configured CorsConfig instance.
-     */
-    public Netty4CorsConfig build() {
-        if (preflightHeaders.isEmpty()) {
-            preflightHeaders.put("date", DateValueGenerator.INSTANCE);
-            preflightHeaders.put("content-length", new ConstantValueGenerator("0"));
-        }
-        return new Netty4CorsConfig(this);
-    }
-
-    /**
-     * This class is used for preflight HTTP response values that do not need to be
-     * generated, but instead the value is "static" in that the same value will be returned
-     * for each call.
-     */
-    private static final class ConstantValueGenerator implements Callable<Object> {
-
-        private final Object value;
-
-        /**
-         * Sole constructor.
-         *
-         * @param value the value that will be returned when the call method is invoked.
-         */
-        private ConstantValueGenerator(final Object value) {
-            if (value == null) {
-                throw new IllegalArgumentException("value must not be null");
-            }
-            this.value = value;
-        }
-
-        @Override
-        public Object call() {
-            return value;
-        }
-    }
-
-    /**
-     * This callable is used for the DATE preflight HTTP response HTTP header.
-     * It's value must be generated when the response is generated, hence will be
-     * different for every call.
-     */
-    private static final class DateValueGenerator implements Callable<Date> {
-
-        static final DateValueGenerator INSTANCE = new DateValueGenerator();
-
-        @Override
-        public Date call() throws Exception {
-            return new Date();
-        }
-    }
-
-}
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsHandler.java
index 5f7baffc86aa6..1855d7c8b757e 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsHandler.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/cors/Netty4CorsHandler.java
@@ -32,31 +32,32 @@
 import io.netty.handler.codec.http.HttpResponse;
 import io.netty.handler.codec.http.HttpResponseStatus;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.http.CorsHandler;
 import org.elasticsearch.http.netty4.Netty4HttpResponse;
 
+import java.util.Date;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 /**
  * Handles <a href="http://www.w3.org/TR/cors/">Cross Origin Resource Sharing</a> (CORS) requests.
  * <p>
- * This handler can be configured using a {@link Netty4CorsConfig}, please
+ * This handler can be configured using a {@link CorsHandler.Config}, please
  * refer to this class for details about the configuration options available.
  *
- * This code was borrowed from Netty 4 and refactored to work for Elasticsearch's Netty 3 setup.
  */
 public class Netty4CorsHandler extends ChannelDuplexHandler {
 
     public static final String ANY_ORIGIN = "*";
     private static Pattern SCHEME_PATTERN = Pattern.compile("^https?://");
 
-    private final Netty4CorsConfig config;
+    private final CorsHandler.Config config;
     private FullHttpRequest request;
 
     /**
-     * Creates a new instance with the specified {@link Netty4CorsConfig}.
+     * Creates a new instance with the specified {@link CorsHandler.Config}.
      */
-    public Netty4CorsHandler(final Netty4CorsConfig config) {
+    public Netty4CorsHandler(final CorsHandler.Config config) {
         if (config == null) {
             throw new NullPointerException();
         }
@@ -76,7 +77,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception
                     releaseRequest();
                 }
             }
-            if (config.isShortCircuit() && !validateOrigin()) {
+            if (!validateOrigin()) {
                 try {
                     forbidden(ctx, request);
                     return;
@@ -96,7 +97,7 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise)
         ctx.write(response, promise);
     }
 
-    public static void setCorsResponseHeaders(HttpRequest request, HttpResponse resp, Netty4CorsConfig config) {
+    public static void setCorsResponseHeaders(HttpRequest request, HttpResponse resp, CorsHandler.Config config) {
         if (!config.isCorsSupportEnabled()) {
             return;
         }
@@ -161,7 +162,8 @@ private static boolean isSameOrigin(final String origin, final String host) {
      * @param response the HttpResponse to which the preflight response headers should be added.
      */
     private void setPreflightHeaders(final HttpResponse response) {
-        response.headers().add(config.preflightResponseHeaders());
+        response.headers().add("date", new Date());
+        response.headers().add("content-length", "0");
     }
 
     private boolean setOrigin(final HttpResponse response) {
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java
index 609186fc3c30e..bcfd5e0b326d3 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java
@@ -35,7 +35,6 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.netty4.Netty4Transport;
-import org.elasticsearch.transport.netty4.Netty4Utils;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -45,10 +44,6 @@
 
 public class Netty4Plugin extends Plugin implements NetworkPlugin {
 
-    static {
-        Netty4Utils.setup();
-    }
-
     public static final String NETTY_TRANSPORT_NAME = "netty4";
     public static final String NETTY_HTTP_TRANSPORT_NAME = "netty4";
 
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java
deleted file mode 100644
index 4eca1803b6381..0000000000000
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4InternalESLogger.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.transport.netty4;
-
-import io.netty.util.internal.logging.AbstractInternalLogger;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.SuppressLoggerChecks;
-
-@SuppressLoggerChecks(reason = "safely delegates to logger")
-class Netty4InternalESLogger extends AbstractInternalLogger {
-
-    private final Logger logger;
-
-    Netty4InternalESLogger(final String name) {
-        super(name);
-        this.logger = LogManager.getLogger(name);
-    }
-
-    @Override
-    public boolean isTraceEnabled() {
-        return logger.isTraceEnabled();
-    }
-
-    @Override
-    public void trace(String msg) {
-        logger.trace(msg);
-    }
-
-    @Override
-    public void trace(String format, Object arg) {
-        logger.trace(format, arg);
-    }
-
-    @Override
-    public void trace(String format, Object argA, Object argB) {
-        logger.trace(format, argA, argB);
-    }
-
-    @Override
-    public void trace(String format, Object... arguments) {
-        logger.trace(format, arguments);
-    }
-
-    @Override
-    public void trace(String msg, Throwable t) {
-        logger.trace(msg, t);
-    }
-
-    @Override
-    public boolean isDebugEnabled() {
-        return logger.isDebugEnabled();
-    }
-
-    @Override
-    public void debug(String msg) {
-        logger.debug(msg);
-    }
-
-    @Override
-    public void debug(String format, Object arg) {
-        logger.debug(format, arg);
-    }
-
-    @Override
-    public void debug(String format, Object argA, Object argB) {
-        logger.debug(format, argA, argB);
-    }
-
-    @Override
-    public void debug(String format, Object... arguments) {
-        logger.debug(format, arguments);
-    }
-
-    @Override
-    public void debug(String msg, Throwable t) {
-        logger.debug(msg, t);
-    }
-
-    @Override
-    public boolean isInfoEnabled() {
-        return logger.isInfoEnabled();
-    }
-
-    @Override
-    public void info(String msg) {
-        logger.info(msg);
-    }
-
-    @Override
-    public void info(String format, Object arg) {
-        logger.info(format, arg);
-    }
-
-    @Override
-    public void info(String format, Object argA, Object argB) {
-        logger.info(format, argA, argB);
-    }
-
-    @Override
-    public void info(String format, Object... arguments) {
-        logger.info(format, arguments);
-    }
-
-    @Override
-    public void info(String msg, Throwable t) {
-        logger.info(msg, t);
-    }
-
-    @Override
-    public boolean isWarnEnabled() {
-        return logger.isWarnEnabled();
-    }
-
-    @Override
-    public void warn(String msg) {
-        logger.warn(msg);
-    }
-
-    @Override
-    public void warn(String format, Object arg) {
-        logger.warn(format, arg);
-    }
-
-    @Override
-    public void warn(String format, Object... arguments) {
-        logger.warn(format, arguments);
-    }
-
-    @Override
-    public void warn(String format, Object argA, Object argB) {
-        logger.warn(format, argA, argB);
-    }
-
-    @Override
-    public void warn(String msg, Throwable t) {
-        logger.warn(msg, t);
-    }
-
-    @Override
-    public boolean isErrorEnabled() {
-        return logger.isErrorEnabled();
-    }
-
-    @Override
-    public void error(String msg) {
-        logger.error(msg);
-    }
-
-    @Override
-    public void error(String format, Object arg) {
-        logger.error(format, arg);
-    }
-
-    @Override
-    public void error(String format, Object argA, Object argB) {
-        logger.error(format, argA, argB);
-    }
-
-    @Override
-    public void error(String format, Object... arguments) {
-        logger.error(format, arguments);
-    }
-
-    @Override
-    public void error(String msg, Throwable t) {
-        logger.error(msg, t);
-    }
-
-}
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
index 34fb2173143c8..5f29c51a1ce60 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
@@ -25,8 +25,8 @@
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelFuture;
 import io.netty.channel.ChannelHandler;
-import io.netty.channel.ChannelHandlerAdapter;
 import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
 import io.netty.channel.ChannelInitializer;
 import io.netty.channel.ChannelOption;
 import io.netty.channel.FixedRecvByteBufAllocator;
@@ -78,10 +78,6 @@
 public class Netty4Transport extends TcpTransport {
     private static final Logger logger = LogManager.getLogger(Netty4Transport.class);
 
-    static {
-        Netty4Utils.setup();
-    }
-
     public static final Setting<Integer> WORKER_COUNT =
         new Setting<>("transport.netty.worker_count",
             (s) -> Integer.toString(EsExecutors.numberOfProcessors(s) * 2),
@@ -315,7 +311,7 @@ private void addClosedExceptionLogger(Channel channel) {
     }
 
     @ChannelHandler.Sharable
-    private class ServerChannelExceptionHandler extends ChannelHandlerAdapter {
+    private class ServerChannelExceptionHandler extends ChannelInboundHandlerAdapter {
 
         @Override
         public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java
index 35928b6c3c8a2..211a574dc980d 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java
@@ -23,8 +23,6 @@
 import io.netty.buffer.CompositeByteBuf;
 import io.netty.buffer.Unpooled;
 import io.netty.util.NettyRuntime;
-import io.netty.util.internal.logging.InternalLogger;
-import io.netty.util.internal.logging.InternalLoggerFactory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
 import org.elasticsearch.common.Booleans;
@@ -38,21 +36,6 @@
 
 public class Netty4Utils {
 
-    static {
-        InternalLoggerFactory.setDefaultFactory(new InternalLoggerFactory() {
-
-            @Override
-            public InternalLogger newInstance(final String name) {
-                return new Netty4InternalESLogger(name);
-            }
-
-        });
-    }
-
-    public static void setup() {
-
-    }
-
     private static AtomicBoolean isAvailableProcessorsSet = new AtomicBoolean();
 
     /**
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4CorsTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4CorsTests.java
index 115e6735eb273..0ed380d68f75e 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4CorsTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4CorsTests.java
@@ -29,6 +29,7 @@
 import io.netty.handler.codec.http.HttpVersion;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.http.CorsHandler;
 import org.elasticsearch.http.HttpTransportSettings;
 import org.elasticsearch.http.netty4.cors.Netty4CorsHandler;
 import org.elasticsearch.rest.RestStatus;
@@ -140,7 +141,7 @@ private FullHttpResponse executeRequest(final Settings settings, final String or
         }
         httpRequest.headers().add(HttpHeaderNames.HOST, host);
         EmbeddedChannel embeddedChannel = new EmbeddedChannel();
-        embeddedChannel.pipeline().addLast(new Netty4CorsHandler(Netty4HttpServerTransport.buildCorsConfig(settings)));
+        embeddedChannel.pipeline().addLast(new Netty4CorsHandler(CorsHandler.fromSettings(settings)));
         Netty4HttpRequest nettyRequest = new Netty4HttpRequest(httpRequest, 0);
         embeddedChannel.writeOutbound(nettyRequest.createResponse(RestStatus.OK, new BytesArray("content")));
         return embeddedChannel.readOutbound();
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
index e45f6de92d5fe..eaec286c7b237 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java
@@ -43,7 +43,7 @@
  * This test checks that in-flight requests are limited on HTTP level and that requests that are excluded from limiting can pass.
  *
  * As the same setting is also used to limit in-flight requests on transport level, we avoid transport messages by forcing
- * a single node "cluster". We also force test infrastructure to use the node client instead of the transport client for the same reason.
+ * a single node "cluster".
  */
 @ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numClientNodes = 0, numDataNodes = 1)
 public class Netty4HttpRequestSizeLimitIT extends ESNetty4IntegTestCase {
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
index bc4ebe5672ec7..d398890bf23f7 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
@@ -39,12 +39,10 @@
 import io.netty.handler.codec.http.HttpUtil;
 import io.netty.handler.codec.http.HttpVersion;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
@@ -55,7 +53,6 @@
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.HttpTransportSettings;
 import org.elasticsearch.http.NullDispatcher;
-import org.elasticsearch.http.netty4.cors.Netty4CorsConfig;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
@@ -69,23 +66,11 @@
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.regex.PatternSyntaxException;
-import java.util.stream.Collectors;
-
-import static org.elasticsearch.common.Strings.collectionToDelimitedString;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_MAX_AGE;
+
 import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
 import static org.elasticsearch.rest.RestStatus.OK;
 import static org.hamcrest.Matchers.containsString;
@@ -119,48 +104,6 @@ public void shutdown() throws Exception {
         bigArrays = null;
     }
 
-    public void testCorsConfig() {
-        final Set<String> methods = new HashSet<>(Arrays.asList("get", "options", "post"));
-        final Set<String> headers = new HashSet<>(Arrays.asList("Content-Type", "Content-Length"));
-        final String prefix = randomBoolean() ? " " : ""; // sometimes have a leading whitespace between comma delimited elements
-        final Settings settings = Settings.builder()
-                                      .put(SETTING_CORS_ENABLED.getKey(), true)
-                                      .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "*")
-                                      .put(SETTING_CORS_ALLOW_METHODS.getKey(), collectionToDelimitedString(methods, ",", prefix, ""))
-                                      .put(SETTING_CORS_ALLOW_HEADERS.getKey(), collectionToDelimitedString(headers, ",", prefix, ""))
-                                      .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
-                                      .build();
-        final Netty4CorsConfig corsConfig = Netty4HttpServerTransport.buildCorsConfig(settings);
-        assertTrue(corsConfig.isAnyOriginSupported());
-        assertEquals(headers, corsConfig.allowedRequestHeaders());
-        assertEquals(methods, corsConfig.allowedRequestMethods().stream().map(HttpMethod::name).collect(Collectors.toSet()));
-    }
-
-    public void testCorsConfigWithDefaults() {
-        final Set<String> methods = Strings.commaDelimitedListToSet(SETTING_CORS_ALLOW_METHODS.getDefault(Settings.EMPTY));
-        final Set<String> headers = Strings.commaDelimitedListToSet(SETTING_CORS_ALLOW_HEADERS.getDefault(Settings.EMPTY));
-        final long maxAge = SETTING_CORS_MAX_AGE.getDefault(Settings.EMPTY);
-        final Settings settings = Settings.builder().put(SETTING_CORS_ENABLED.getKey(), true).build();
-        final Netty4CorsConfig corsConfig = Netty4HttpServerTransport.buildCorsConfig(settings);
-        assertFalse(corsConfig.isAnyOriginSupported());
-        assertEquals(Collections.emptySet(), corsConfig.origins().get());
-        assertEquals(headers, corsConfig.allowedRequestHeaders());
-        assertEquals(methods, corsConfig.allowedRequestMethods().stream().map(HttpMethod::name).collect(Collectors.toSet()));
-        assertEquals(maxAge, corsConfig.maxAge());
-        assertFalse(corsConfig.isCredentialsAllowed());
-    }
-
-    public void testCorsConfigWithBadRegex() {
-        final Settings settings = Settings.builder()
-            .put(SETTING_CORS_ENABLED.getKey(), true)
-            .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "/[*/")
-            .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
-            .build();
-        SettingsException e = expectThrows(SettingsException.class, () -> Netty4HttpServerTransport.buildCorsConfig(settings));
-        assertThat(e.getMessage(), containsString("Bad regex in [http.cors.allow-origin]: [/[*/]"));
-        assertThat(e.getCause(), instanceOf(PatternSyntaxException.class));
-    }
-
     /**
      * Test that {@link Netty4HttpServerTransport} supports the "Expect: 100-continue" HTTP header
      * @throws InterruptedException if the client communication with the server is interrupted
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java
index fcb8e75700d0c..664632cb527f8 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java
@@ -47,7 +47,7 @@
 // These tests are here today so they have access to a proper REST client. They cannot be in :server:integTest since the REST client needs a
 // proper transport implementation, and they cannot be REST tests today since they need to restart nodes. When #35599 and friends land we
 // should be able to move these tests to run against a proper cluster instead. TODO do this.
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false)
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class Zen2RestApiIT extends ESNetty4IntegTestCase {
 
     @Override
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java
index 881fe73112c5a..47af90d2bf39e 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java
@@ -19,24 +19,16 @@
 package org.elasticsearch.transport.netty4;
 
 import org.elasticsearch.ESNetty4IntegTestCase;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.network.NetworkAddress;
-import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.junit.annotations.Network;
-import org.elasticsearch.transport.MockTransportClient;
-import org.elasticsearch.transport.Netty4Plugin;
 
-import java.net.InetAddress;
 import java.util.Locale;
 
 import static org.hamcrest.Matchers.allOf;
@@ -69,22 +61,6 @@ protected Settings nodeSettings(int nodeOrdinal) {
         return builder.build();
     }
 
-    public void testThatTransportClientCanConnect() throws Exception {
-        Settings settings = Settings.builder()
-            .put("cluster.name", internalCluster().getClusterName())
-            .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME)
-            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-            .build();
-        // we have to test all the ports that the data node might be bound to
-        try (TransportClient transportClient = new MockTransportClient(settings, Netty4Plugin.class)) {
-            for (int i = 0; i <= 10; i++) {
-                transportClient.addTransportAddress(new TransportAddress(InetAddress.getByName("127.0.0.1"), randomPort + i));
-            }
-            ClusterHealthResponse response = transportClient.admin().cluster().prepareHealth().get();
-            assertThat(response.getStatus(), is(ClusterHealthStatus.GREEN));
-        }
-    }
-
     @Network
     public void testThatInfosAreExposed() throws Exception {
         NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().clear().setTransport(true).get();
diff --git a/plugins/analysis-icu/build.gradle b/plugins/analysis-icu/build.gradle
index 90132e2c58fcd..f818959e706de 100644
--- a/plugins/analysis-icu/build.gradle
+++ b/plugins/analysis-icu/build.gradle
@@ -22,7 +22,6 @@ import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
 esplugin {
   description 'The ICU Analysis plugin integrates Lucene ICU module into elasticsearch, adding ICU relates analysis components.'
   classname 'org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin'
-  hasClientJar = true
 }
 
 tasks.withType(CheckForbiddenApis) {
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java
index 3bcefe0cf5680..8a55d3424c1ff 100644
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java
@@ -18,15 +18,10 @@
  */
 package org.elasticsearch.index.mapper;
 
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertOrderedSearchHits;
-
 import com.ibm.icu.text.Collator;
 import com.ibm.icu.text.RuleBasedCollator;
 import com.ibm.icu.util.ULocale;
+
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -43,6 +38,12 @@
 import java.util.Collection;
 import java.util.Collections;
 
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertOrderedSearchHits;
+
 public class ICUCollationKeywordFieldMapperIT extends ESIntegTestCase {
 
     @Override
@@ -82,7 +83,6 @@ public void testBasicUsage() throws Exception {
         // searching for either of the terms should return both results since they collate to the same value
         SearchRequest request = new SearchRequest()
             .indices(index)
-            .types(type)
             .source(new SearchSourceBuilder()
                 .fetchSource(false)
                 .query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1]))
@@ -122,7 +122,6 @@ public void testMultipleValues() throws Exception {
         // using sort mode = max, values B and C will be used for the sort
         SearchRequest request = new SearchRequest()
             .indices(index)
-            .types(type)
             .source(new SearchSourceBuilder()
                 .fetchSource(false)
                 .query(QueryBuilders.termQuery("collate", "a"))
@@ -139,7 +138,6 @@ public void testMultipleValues() throws Exception {
         // same thing, using different sort mode that will use a for both docs
         request = new SearchRequest()
             .indices(index)
-            .types(type)
             .source(new SearchSourceBuilder()
                 .fetchSource(false)
                 .query(QueryBuilders.termQuery("collate", "a"))
@@ -183,7 +181,6 @@ public void testNormalization() throws Exception {
         // searching for either of the terms should return both results since they collate to the same value
         SearchRequest request = new SearchRequest()
             .indices(index)
-            .types(type)
             .source(new SearchSourceBuilder()
                 .fetchSource(false)
                 .query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1]))
@@ -225,7 +222,6 @@ public void testSecondaryStrength() throws Exception {
 
         SearchRequest request = new SearchRequest()
             .indices(index)
-            .types(type)
             .source(new SearchSourceBuilder()
                 .fetchSource(false)
                 .query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1]))
@@ -268,7 +264,6 @@ public void testIgnorePunctuation() throws Exception {
 
         SearchRequest request = new SearchRequest()
             .indices(index)
-            .types(type)
             .source(new SearchSourceBuilder()
                 .fetchSource(false)
                 .query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1]))
@@ -312,7 +307,6 @@ public void testIgnoreWhitespace() throws Exception {
 
         SearchRequest request = new SearchRequest()
             .indices(index)
-            .types(type)
             .source(new SearchSourceBuilder()
                 .fetchSource(false)
                 .sort("collate", SortOrder.ASC)
@@ -352,7 +346,6 @@ public void testNumerics() throws Exception {
 
         SearchRequest request = new SearchRequest()
             .indices(index)
-            .types(type)
             .source(new SearchSourceBuilder()
                 .fetchSource(false)
                 .sort("collate", SortOrder.ASC)
@@ -394,7 +387,6 @@ public void testIgnoreAccentsButNotCase() throws Exception {
 
         SearchRequest request = new SearchRequest()
             .indices(index)
-            .types(type)
             .source(new SearchSourceBuilder()
                 .fetchSource(false)
                 .sort("collate", SortOrder.ASC)
@@ -435,7 +427,6 @@ public void testUpperCaseFirst() throws Exception {
 
         SearchRequest request = new SearchRequest()
             .indices(index)
-            .types(type)
             .source(new SearchSourceBuilder()
                 .fetchSource(false)
                 .sort("collate", SortOrder.ASC)
@@ -487,7 +478,6 @@ public void testCustomRules() throws Exception {
 
         SearchRequest request = new SearchRequest()
             .indices(index)
-            .types(type)
             .source(new SearchSourceBuilder()
                 .fetchSource(false)
                 .query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1]))
diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseStopTokenFilterFactory.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseStopTokenFilterFactory.java
new file mode 100644
index 0000000000000..b528481811394
--- /dev/null
+++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/index/analysis/SmartChineseStopTokenFilterFactory.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.StopFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
+import org.apache.lucene.search.suggest.analyzing.SuggestStopFilter;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+import java.util.Map;
+import java.util.Set;
+
+import static java.util.Collections.singletonMap;
+
+public class SmartChineseStopTokenFilterFactory extends AbstractTokenFilterFactory {
+    private static final Map<String, Set<?>> NAMED_STOP_WORDS = singletonMap("_smartcn_", SmartChineseAnalyzer.getDefaultStopSet());
+
+    private final CharArraySet stopWords;
+
+    private final boolean ignoreCase;
+
+    private final boolean removeTrailing;
+
+    public SmartChineseStopTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
+        super(indexSettings, name, settings);
+        this.ignoreCase = settings.getAsBoolean("ignore_case", false);
+        this.removeTrailing = settings.getAsBoolean("remove_trailing", true);
+        this.stopWords = Analysis.parseWords(env, settings, "stopwords",
+            SmartChineseAnalyzer.getDefaultStopSet(), NAMED_STOP_WORDS, ignoreCase);
+    }
+
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        if (removeTrailing) {
+            return new StopFilter(tokenStream, stopWords);
+        } else {
+            return new SuggestStopFilter(tokenStream, stopWords);
+        }
+    }
+
+}
diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java
index b11a157c149d6..61d2d75359a95 100644
--- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java
+++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.index.analysis.AnalyzerProvider;
 import org.elasticsearch.index.analysis.SmartChineseAnalyzerProvider;
 import org.elasticsearch.index.analysis.SmartChineseNoOpTokenFilterFactory;
+import org.elasticsearch.index.analysis.SmartChineseStopTokenFilterFactory;
 import org.elasticsearch.index.analysis.SmartChineseTokenizerTokenizerFactory;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenizerFactory;
@@ -38,15 +39,18 @@
 public class AnalysisSmartChinesePlugin extends Plugin implements AnalysisPlugin {
     @Override
     public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
-        // This is a noop token filter; it's here for backwards compat before we had "smartcn_tokenizer"
-        return singletonMap("smartcn_word", SmartChineseNoOpTokenFilterFactory::new);
+        Map<String, AnalysisProvider<TokenFilterFactory>> tokenFilters = new HashMap<>();
+        tokenFilters.put("smartcn_stop", SmartChineseStopTokenFilterFactory::new);
+        // TODO: deprecate and remove, this is a noop token filter; it's here for backwards compat before we had "smartcn_tokenizer"
+        tokenFilters.put("smartcn_word", SmartChineseNoOpTokenFilterFactory::new);
+        return tokenFilters;
     }
 
     @Override
     public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
         Map<String, AnalysisProvider<TokenizerFactory>> extra = new HashMap<>();
         extra.put("smartcn_tokenizer", SmartChineseTokenizerTokenizerFactory::new);
-        // This is an alias to "smartcn_tokenizer"; it's here for backwards compat
+        // TODO: deprecate and remove, this is an alias to "smartcn_tokenizer"; it's here for backwards compat
         extra.put("smartcn_sentence", SmartChineseTokenizerTokenizerFactory::new);
         return extra;
     }
diff --git a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStopTokenFilterFactory.java b/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStopTokenFilterFactory.java
new file mode 100644
index 0000000000000..32897ad29d7b4
--- /dev/null
+++ b/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStopTokenFilterFactory.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis.pl;
+
+
+import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.StopFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.pl.PolishAnalyzer;
+import org.apache.lucene.search.suggest.analyzing.SuggestStopFilter;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+import org.elasticsearch.index.analysis.Analysis;
+
+import java.util.Map;
+import java.util.Set;
+
+import static java.util.Collections.singletonMap;
+
+public class PolishStopTokenFilterFactory extends AbstractTokenFilterFactory {
+    private static final Map<String, Set<?>> NAMED_STOP_WORDS = singletonMap("_polish_", PolishAnalyzer.getDefaultStopSet());
+
+    private final CharArraySet stopWords;
+
+    private final boolean ignoreCase;
+
+    private final boolean removeTrailing;
+
+    public PolishStopTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
+        super(indexSettings, name, settings);
+        this.ignoreCase = settings.getAsBoolean("ignore_case", false);
+        this.removeTrailing = settings.getAsBoolean("remove_trailing", true);
+        this.stopWords = Analysis.parseWords(env, settings, "stopwords",
+                PolishAnalyzer.getDefaultStopSet(), NAMED_STOP_WORDS, ignoreCase);
+    }
+
+    @Override
+    public TokenStream create(TokenStream tokenStream) {
+        if (removeTrailing) {
+            return new StopFilter(tokenStream, stopWords);
+        } else {
+            return new SuggestStopFilter(tokenStream, stopWords);
+        }
+    }
+
+    public Set<?> stopWords() {
+        return stopWords;
+    }
+
+    public boolean ignoreCase() {
+        return ignoreCase;
+    }
+
+}
diff --git a/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java b/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java
index 98dd9634fb961..a523d7dcaa0ce 100644
--- a/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java
+++ b/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.analysis.pl.PolishAnalyzerProvider;
 import org.elasticsearch.index.analysis.pl.PolishStemTokenFilterFactory;
+import org.elasticsearch.index.analysis.pl.PolishStopTokenFilterFactory;
 import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
 import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -35,7 +36,8 @@
 public class AnalysisStempelPlugin extends Plugin implements AnalysisPlugin {
     @Override
     public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
-        return singletonMap("polish_stem", PolishStemTokenFilterFactory::new);
+        return Map.of("polish_stem", PolishStemTokenFilterFactory::new,
+                      "polish_stop", PolishStopTokenFilterFactory::new);
     }
 
     @Override
diff --git a/plugins/discovery-azure-classic/build.gradle b/plugins/discovery-azure-classic/build.gradle
index cb024d8ac00d7..8a9f207f0b467 100644
--- a/plugins/discovery-azure-classic/build.gradle
+++ b/plugins/discovery-azure-classic/build.gradle
@@ -35,6 +35,7 @@ dependencies {
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
+  compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "commons-lang:commons-lang:2.6"
   compile "commons-io:commons-io:2.4"
diff --git a/server/licenses/log4j-1.2-api-2.11.1.jar.sha1 b/plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.11.1.jar.sha1
similarity index 100%
rename from server/licenses/log4j-1.2-api-2.11.1.jar.sha1
rename to plugins/discovery-azure-classic/licenses/log4j-1.2-api-2.11.1.jar.sha1
diff --git a/server/licenses/log4j-LICENSE.txt b/plugins/discovery-azure-classic/licenses/log4j-LICENSE.txt
similarity index 100%
rename from server/licenses/log4j-LICENSE.txt
rename to plugins/discovery-azure-classic/licenses/log4j-LICENSE.txt
diff --git a/server/licenses/log4j-NOTICE.txt b/plugins/discovery-azure-classic/licenses/log4j-NOTICE.txt
similarity index 100%
rename from server/licenses/log4j-NOTICE.txt
rename to plugins/discovery-azure-classic/licenses/log4j-NOTICE.txt
diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
index 82347ca2bf1e4..c5b310c14817a 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
@@ -279,7 +279,8 @@ private static String getProtocol() {
             return "TLSv1.2";
         } else {
             JavaVersion full =
-                AccessController.doPrivileged((PrivilegedAction<JavaVersion>) () -> JavaVersion.parse(System.getProperty("java.version")));
+                AccessController.doPrivileged(
+                        (PrivilegedAction<JavaVersion>) () -> JavaVersion.parse(System.getProperty("java.specification.version")));
             if (full.compareTo(JavaVersion.parse("12.0.1")) < 0) {
                 return "TLSv1.2";
             }
@@ -289,13 +290,14 @@ private static String getProtocol() {
 
     @AfterClass
     public static void stopHttpd() throws IOException {
-        for (int i = 0; i < internalCluster().size(); i++) {
+        try {
             // shut them all down otherwise we get spammed with connection refused exceptions
-            internalCluster().stopRandomDataNode();
+            internalCluster().close();
+        } finally {
+            httpsServer.stop(0);
+            httpsServer = null;
+            logDir = null;
         }
-        httpsServer.stop(0);
-        httpsServer = null;
-        logDir = null;
     }
 
     public void testJoin() throws ExecutionException, InterruptedException {
diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle
index 50dc6ac5d85b5..6935de286b237 100644
--- a/plugins/discovery-ec2/build.gradle
+++ b/plugins/discovery-ec2/build.gradle
@@ -23,7 +23,7 @@ esplugin {
 }
 
 versions << [
-  'aws': '1.11.505'
+  'aws': '1.11.562'
 ]
 
 dependencies {
@@ -32,6 +32,7 @@ dependencies {
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
+  compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile 'com.fasterxml.jackson.core:jackson-databind:2.8.11.3'
   compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.505.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.505.jar.sha1
deleted file mode 100644
index add5db290e8a8..0000000000000
--- a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.505.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d19328c227b2b5ad81d137361ebc9cbcd0396465
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.562.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.562.jar.sha1
new file mode 100644
index 0000000000000..ed8ded6a3608c
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.562.jar.sha1
@@ -0,0 +1 @@
+b5fc47ec1b5afe180f5ebb4eda755acdca7a20ae
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.505.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.505.jar.sha1
deleted file mode 100644
index 857f0888de3aa..0000000000000
--- a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.505.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b669b3c90ea9bf73734ab26f0cb30c5c66addf55
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.562.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.562.jar.sha1
new file mode 100644
index 0000000000000..040d28de70b29
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.562.jar.sha1
@@ -0,0 +1 @@
+0211a055fb3e036033af4b1ca25ada0574a756ec
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/log4j-1.2-api-2.11.1.jar.sha1 b/plugins/discovery-ec2/licenses/log4j-1.2-api-2.11.1.jar.sha1
new file mode 100644
index 0000000000000..575d75dbda8c5
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/log4j-1.2-api-2.11.1.jar.sha1
@@ -0,0 +1 @@
+3aba3398fe064a3eab4331f88161c7480e848418
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/log4j-LICENSE.txt b/plugins/discovery-ec2/licenses/log4j-LICENSE.txt
new file mode 100644
index 0000000000000..6279e5206de13
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/log4j-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 1999-2005 The Apache Software Foundation
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/discovery-ec2/licenses/log4j-NOTICE.txt b/plugins/discovery-ec2/licenses/log4j-NOTICE.txt
new file mode 100644
index 0000000000000..0375732360047
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/log4j-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache log4j
+Copyright 2007 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
\ No newline at end of file
diff --git a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle
index 8397549f384fe..218c9aca2920a 100644
--- a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle
+++ b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle
@@ -22,6 +22,8 @@ import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.test.AntFixture
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
+import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
+
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -74,9 +76,9 @@ integTest.enabled = false
 
         setting 'discovery.seed_providers', 'ec2'
         setting 'network.host', '_ec2_'
-        setting 'discovery.ec2.endpoint', { "http://${-> fixture.addressAndPort}" }
+        setting 'discovery.ec2.endpoint', { "http://${-> fixture.addressAndPort}" }, IGNORE_VALUE
 
-        systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${-> fixture.addressAndPort}" }
+        systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${-> fixture.addressAndPort}" }, IGNORE_VALUE
     }
 }
 
@@ -103,7 +105,7 @@ ec2FixtureContainerCredentials.env 'ACTIVATE_CONTAINER_CREDENTIALS', true
 
 testClusters.integTestContainerCredentials {
     environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI',
-        { "http://${-> tasks.findByName("ec2FixtureContainerCredentials").addressAndPort}/ecs_credentials_endpoint" }
+        { "http://${-> tasks.findByName("ec2FixtureContainerCredentials").addressAndPort}/ecs_credentials_endpoint" }, IGNORE_VALUE
 }
 
 // Extra config for InstanceProfile
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java
index 3135769df5f46..040472723fbc0 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java
@@ -20,679 +20,17 @@
 package org.elasticsearch.discovery.ec2;
 
 import com.amazonaws.AmazonClientException;
-import com.amazonaws.AmazonWebServiceRequest;
 import com.amazonaws.ClientConfiguration;
-import com.amazonaws.ResponseMetadata;
 import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.regions.Region;
-import com.amazonaws.services.ec2.AmazonEC2;
-import com.amazonaws.services.ec2.model.AcceptReservedInstancesExchangeQuoteRequest;
-import com.amazonaws.services.ec2.model.AcceptReservedInstancesExchangeQuoteResult;
-import com.amazonaws.services.ec2.model.AcceptTransitGatewayVpcAttachmentRequest;
-import com.amazonaws.services.ec2.model.AcceptTransitGatewayVpcAttachmentResult;
-import com.amazonaws.services.ec2.model.AcceptVpcEndpointConnectionsRequest;
-import com.amazonaws.services.ec2.model.AcceptVpcEndpointConnectionsResult;
-import com.amazonaws.services.ec2.model.AcceptVpcPeeringConnectionRequest;
-import com.amazonaws.services.ec2.model.AcceptVpcPeeringConnectionResult;
-import com.amazonaws.services.ec2.model.AdvertiseByoipCidrRequest;
-import com.amazonaws.services.ec2.model.AdvertiseByoipCidrResult;
-import com.amazonaws.services.ec2.model.AllocateAddressRequest;
-import com.amazonaws.services.ec2.model.AllocateAddressResult;
-import com.amazonaws.services.ec2.model.AllocateHostsRequest;
-import com.amazonaws.services.ec2.model.AllocateHostsResult;
-import com.amazonaws.services.ec2.model.ApplySecurityGroupsToClientVpnTargetNetworkRequest;
-import com.amazonaws.services.ec2.model.ApplySecurityGroupsToClientVpnTargetNetworkResult;
-import com.amazonaws.services.ec2.model.AssignIpv6AddressesRequest;
-import com.amazonaws.services.ec2.model.AssignIpv6AddressesResult;
-import com.amazonaws.services.ec2.model.AssignPrivateIpAddressesRequest;
-import com.amazonaws.services.ec2.model.AssignPrivateIpAddressesResult;
-import com.amazonaws.services.ec2.model.AssociateAddressRequest;
-import com.amazonaws.services.ec2.model.AssociateAddressResult;
-import com.amazonaws.services.ec2.model.AssociateClientVpnTargetNetworkRequest;
-import com.amazonaws.services.ec2.model.AssociateClientVpnTargetNetworkResult;
-import com.amazonaws.services.ec2.model.AssociateDhcpOptionsRequest;
-import com.amazonaws.services.ec2.model.AssociateDhcpOptionsResult;
-import com.amazonaws.services.ec2.model.AssociateIamInstanceProfileRequest;
-import com.amazonaws.services.ec2.model.AssociateIamInstanceProfileResult;
-import com.amazonaws.services.ec2.model.AssociateRouteTableRequest;
-import com.amazonaws.services.ec2.model.AssociateRouteTableResult;
-import com.amazonaws.services.ec2.model.AssociateSubnetCidrBlockRequest;
-import com.amazonaws.services.ec2.model.AssociateSubnetCidrBlockResult;
-import com.amazonaws.services.ec2.model.AssociateTransitGatewayRouteTableRequest;
-import com.amazonaws.services.ec2.model.AssociateTransitGatewayRouteTableResult;
-import com.amazonaws.services.ec2.model.AssociateVpcCidrBlockRequest;
-import com.amazonaws.services.ec2.model.AssociateVpcCidrBlockResult;
-import com.amazonaws.services.ec2.model.AttachClassicLinkVpcRequest;
-import com.amazonaws.services.ec2.model.AttachClassicLinkVpcResult;
-import com.amazonaws.services.ec2.model.AttachInternetGatewayRequest;
-import com.amazonaws.services.ec2.model.AttachInternetGatewayResult;
-import com.amazonaws.services.ec2.model.AttachNetworkInterfaceRequest;
-import com.amazonaws.services.ec2.model.AttachNetworkInterfaceResult;
-import com.amazonaws.services.ec2.model.AttachVolumeRequest;
-import com.amazonaws.services.ec2.model.AttachVolumeResult;
-import com.amazonaws.services.ec2.model.AttachVpnGatewayRequest;
-import com.amazonaws.services.ec2.model.AttachVpnGatewayResult;
-import com.amazonaws.services.ec2.model.AuthorizeClientVpnIngressRequest;
-import com.amazonaws.services.ec2.model.AuthorizeClientVpnIngressResult;
-import com.amazonaws.services.ec2.model.AuthorizeSecurityGroupEgressRequest;
-import com.amazonaws.services.ec2.model.AuthorizeSecurityGroupEgressResult;
-import com.amazonaws.services.ec2.model.AuthorizeSecurityGroupIngressRequest;
-import com.amazonaws.services.ec2.model.AuthorizeSecurityGroupIngressResult;
-import com.amazonaws.services.ec2.model.BundleInstanceRequest;
-import com.amazonaws.services.ec2.model.BundleInstanceResult;
-import com.amazonaws.services.ec2.model.CancelBundleTaskRequest;
-import com.amazonaws.services.ec2.model.CancelBundleTaskResult;
-import com.amazonaws.services.ec2.model.CancelCapacityReservationRequest;
-import com.amazonaws.services.ec2.model.CancelCapacityReservationResult;
-import com.amazonaws.services.ec2.model.CancelConversionTaskRequest;
-import com.amazonaws.services.ec2.model.CancelConversionTaskResult;
-import com.amazonaws.services.ec2.model.CancelExportTaskRequest;
-import com.amazonaws.services.ec2.model.CancelExportTaskResult;
-import com.amazonaws.services.ec2.model.CancelImportTaskRequest;
-import com.amazonaws.services.ec2.model.CancelImportTaskResult;
-import com.amazonaws.services.ec2.model.CancelReservedInstancesListingRequest;
-import com.amazonaws.services.ec2.model.CancelReservedInstancesListingResult;
-import com.amazonaws.services.ec2.model.CancelSpotFleetRequestsRequest;
-import com.amazonaws.services.ec2.model.CancelSpotFleetRequestsResult;
-import com.amazonaws.services.ec2.model.CancelSpotInstanceRequestsRequest;
-import com.amazonaws.services.ec2.model.CancelSpotInstanceRequestsResult;
-import com.amazonaws.services.ec2.model.ConfirmProductInstanceRequest;
-import com.amazonaws.services.ec2.model.ConfirmProductInstanceResult;
-import com.amazonaws.services.ec2.model.CopyFpgaImageRequest;
-import com.amazonaws.services.ec2.model.CopyFpgaImageResult;
-import com.amazonaws.services.ec2.model.CopyImageRequest;
-import com.amazonaws.services.ec2.model.CopyImageResult;
-import com.amazonaws.services.ec2.model.CopySnapshotRequest;
-import com.amazonaws.services.ec2.model.CopySnapshotResult;
-import com.amazonaws.services.ec2.model.CreateCapacityReservationRequest;
-import com.amazonaws.services.ec2.model.CreateCapacityReservationResult;
-import com.amazonaws.services.ec2.model.CreateClientVpnEndpointRequest;
-import com.amazonaws.services.ec2.model.CreateClientVpnEndpointResult;
-import com.amazonaws.services.ec2.model.CreateClientVpnRouteRequest;
-import com.amazonaws.services.ec2.model.CreateClientVpnRouteResult;
-import com.amazonaws.services.ec2.model.CreateCustomerGatewayRequest;
-import com.amazonaws.services.ec2.model.CreateCustomerGatewayResult;
-import com.amazonaws.services.ec2.model.CreateDefaultSubnetRequest;
-import com.amazonaws.services.ec2.model.CreateDefaultSubnetResult;
-import com.amazonaws.services.ec2.model.CreateDefaultVpcRequest;
-import com.amazonaws.services.ec2.model.CreateDefaultVpcResult;
-import com.amazonaws.services.ec2.model.CreateDhcpOptionsRequest;
-import com.amazonaws.services.ec2.model.CreateDhcpOptionsResult;
-import com.amazonaws.services.ec2.model.CreateEgressOnlyInternetGatewayRequest;
-import com.amazonaws.services.ec2.model.CreateEgressOnlyInternetGatewayResult;
-import com.amazonaws.services.ec2.model.CreateFleetRequest;
-import com.amazonaws.services.ec2.model.CreateFleetResult;
-import com.amazonaws.services.ec2.model.CreateFlowLogsRequest;
-import com.amazonaws.services.ec2.model.CreateFlowLogsResult;
-import com.amazonaws.services.ec2.model.CreateFpgaImageRequest;
-import com.amazonaws.services.ec2.model.CreateFpgaImageResult;
-import com.amazonaws.services.ec2.model.CreateImageRequest;
-import com.amazonaws.services.ec2.model.CreateImageResult;
-import com.amazonaws.services.ec2.model.CreateInstanceExportTaskRequest;
-import com.amazonaws.services.ec2.model.CreateInstanceExportTaskResult;
-import com.amazonaws.services.ec2.model.CreateInternetGatewayRequest;
-import com.amazonaws.services.ec2.model.CreateInternetGatewayResult;
-import com.amazonaws.services.ec2.model.CreateKeyPairRequest;
-import com.amazonaws.services.ec2.model.CreateKeyPairResult;
-import com.amazonaws.services.ec2.model.CreateLaunchTemplateRequest;
-import com.amazonaws.services.ec2.model.CreateLaunchTemplateResult;
-import com.amazonaws.services.ec2.model.CreateLaunchTemplateVersionRequest;
-import com.amazonaws.services.ec2.model.CreateLaunchTemplateVersionResult;
-import com.amazonaws.services.ec2.model.CreateNatGatewayRequest;
-import com.amazonaws.services.ec2.model.CreateNatGatewayResult;
-import com.amazonaws.services.ec2.model.CreateNetworkAclEntryRequest;
-import com.amazonaws.services.ec2.model.CreateNetworkAclEntryResult;
-import com.amazonaws.services.ec2.model.CreateNetworkAclRequest;
-import com.amazonaws.services.ec2.model.CreateNetworkAclResult;
-import com.amazonaws.services.ec2.model.CreateNetworkInterfacePermissionRequest;
-import com.amazonaws.services.ec2.model.CreateNetworkInterfacePermissionResult;
-import com.amazonaws.services.ec2.model.CreateNetworkInterfaceRequest;
-import com.amazonaws.services.ec2.model.CreateNetworkInterfaceResult;
-import com.amazonaws.services.ec2.model.CreatePlacementGroupRequest;
-import com.amazonaws.services.ec2.model.CreatePlacementGroupResult;
-import com.amazonaws.services.ec2.model.CreateReservedInstancesListingRequest;
-import com.amazonaws.services.ec2.model.CreateReservedInstancesListingResult;
-import com.amazonaws.services.ec2.model.CreateRouteRequest;
-import com.amazonaws.services.ec2.model.CreateRouteResult;
-import com.amazonaws.services.ec2.model.CreateRouteTableRequest;
-import com.amazonaws.services.ec2.model.CreateRouteTableResult;
-import com.amazonaws.services.ec2.model.CreateSecurityGroupRequest;
-import com.amazonaws.services.ec2.model.CreateSecurityGroupResult;
-import com.amazonaws.services.ec2.model.CreateSnapshotRequest;
-import com.amazonaws.services.ec2.model.CreateSnapshotResult;
-import com.amazonaws.services.ec2.model.CreateSpotDatafeedSubscriptionRequest;
-import com.amazonaws.services.ec2.model.CreateSpotDatafeedSubscriptionResult;
-import com.amazonaws.services.ec2.model.CreateSubnetRequest;
-import com.amazonaws.services.ec2.model.CreateSubnetResult;
-import com.amazonaws.services.ec2.model.CreateTagsRequest;
-import com.amazonaws.services.ec2.model.CreateTagsResult;
-import com.amazonaws.services.ec2.model.CreateTransitGatewayRequest;
-import com.amazonaws.services.ec2.model.CreateTransitGatewayResult;
-import com.amazonaws.services.ec2.model.CreateTransitGatewayRouteRequest;
-import com.amazonaws.services.ec2.model.CreateTransitGatewayRouteResult;
-import com.amazonaws.services.ec2.model.CreateTransitGatewayRouteTableRequest;
-import com.amazonaws.services.ec2.model.CreateTransitGatewayRouteTableResult;
-import com.amazonaws.services.ec2.model.CreateTransitGatewayVpcAttachmentRequest;
-import com.amazonaws.services.ec2.model.CreateTransitGatewayVpcAttachmentResult;
-import com.amazonaws.services.ec2.model.CreateVolumeRequest;
-import com.amazonaws.services.ec2.model.CreateVolumeResult;
-import com.amazonaws.services.ec2.model.CreateVpcEndpointConnectionNotificationRequest;
-import com.amazonaws.services.ec2.model.CreateVpcEndpointConnectionNotificationResult;
-import com.amazonaws.services.ec2.model.CreateVpcEndpointRequest;
-import com.amazonaws.services.ec2.model.CreateVpcEndpointResult;
-import com.amazonaws.services.ec2.model.CreateVpcEndpointServiceConfigurationRequest;
-import com.amazonaws.services.ec2.model.CreateVpcEndpointServiceConfigurationResult;
-import com.amazonaws.services.ec2.model.CreateVpcPeeringConnectionRequest;
-import com.amazonaws.services.ec2.model.CreateVpcPeeringConnectionResult;
-import com.amazonaws.services.ec2.model.CreateVpcRequest;
-import com.amazonaws.services.ec2.model.CreateVpcResult;
-import com.amazonaws.services.ec2.model.CreateVpnConnectionRequest;
-import com.amazonaws.services.ec2.model.CreateVpnConnectionResult;
-import com.amazonaws.services.ec2.model.CreateVpnConnectionRouteRequest;
-import com.amazonaws.services.ec2.model.CreateVpnConnectionRouteResult;
-import com.amazonaws.services.ec2.model.CreateVpnGatewayRequest;
-import com.amazonaws.services.ec2.model.CreateVpnGatewayResult;
-import com.amazonaws.services.ec2.model.DeleteClientVpnEndpointRequest;
-import com.amazonaws.services.ec2.model.DeleteClientVpnEndpointResult;
-import com.amazonaws.services.ec2.model.DeleteClientVpnRouteRequest;
-import com.amazonaws.services.ec2.model.DeleteClientVpnRouteResult;
-import com.amazonaws.services.ec2.model.DeleteCustomerGatewayRequest;
-import com.amazonaws.services.ec2.model.DeleteCustomerGatewayResult;
-import com.amazonaws.services.ec2.model.DeleteDhcpOptionsRequest;
-import com.amazonaws.services.ec2.model.DeleteDhcpOptionsResult;
-import com.amazonaws.services.ec2.model.DeleteEgressOnlyInternetGatewayRequest;
-import com.amazonaws.services.ec2.model.DeleteEgressOnlyInternetGatewayResult;
-import com.amazonaws.services.ec2.model.DeleteFleetsRequest;
-import com.amazonaws.services.ec2.model.DeleteFleetsResult;
-import com.amazonaws.services.ec2.model.DeleteFlowLogsRequest;
-import com.amazonaws.services.ec2.model.DeleteFlowLogsResult;
-import com.amazonaws.services.ec2.model.DeleteFpgaImageRequest;
-import com.amazonaws.services.ec2.model.DeleteFpgaImageResult;
-import com.amazonaws.services.ec2.model.DeleteInternetGatewayRequest;
-import com.amazonaws.services.ec2.model.DeleteInternetGatewayResult;
-import com.amazonaws.services.ec2.model.DeleteKeyPairRequest;
-import com.amazonaws.services.ec2.model.DeleteKeyPairResult;
-import com.amazonaws.services.ec2.model.DeleteLaunchTemplateRequest;
-import com.amazonaws.services.ec2.model.DeleteLaunchTemplateResult;
-import com.amazonaws.services.ec2.model.DeleteLaunchTemplateVersionsRequest;
-import com.amazonaws.services.ec2.model.DeleteLaunchTemplateVersionsResult;
-import com.amazonaws.services.ec2.model.DeleteNatGatewayRequest;
-import com.amazonaws.services.ec2.model.DeleteNatGatewayResult;
-import com.amazonaws.services.ec2.model.DeleteNetworkAclEntryRequest;
-import com.amazonaws.services.ec2.model.DeleteNetworkAclEntryResult;
-import com.amazonaws.services.ec2.model.DeleteNetworkAclRequest;
-import com.amazonaws.services.ec2.model.DeleteNetworkAclResult;
-import com.amazonaws.services.ec2.model.DeleteNetworkInterfacePermissionRequest;
-import com.amazonaws.services.ec2.model.DeleteNetworkInterfacePermissionResult;
-import com.amazonaws.services.ec2.model.DeleteNetworkInterfaceRequest;
-import com.amazonaws.services.ec2.model.DeleteNetworkInterfaceResult;
-import com.amazonaws.services.ec2.model.DeletePlacementGroupRequest;
-import com.amazonaws.services.ec2.model.DeletePlacementGroupResult;
-import com.amazonaws.services.ec2.model.DeleteRouteRequest;
-import com.amazonaws.services.ec2.model.DeleteRouteResult;
-import com.amazonaws.services.ec2.model.DeleteRouteTableRequest;
-import com.amazonaws.services.ec2.model.DeleteRouteTableResult;
-import com.amazonaws.services.ec2.model.DeleteSecurityGroupRequest;
-import com.amazonaws.services.ec2.model.DeleteSecurityGroupResult;
-import com.amazonaws.services.ec2.model.DeleteSnapshotRequest;
-import com.amazonaws.services.ec2.model.DeleteSnapshotResult;
-import com.amazonaws.services.ec2.model.DeleteSpotDatafeedSubscriptionRequest;
-import com.amazonaws.services.ec2.model.DeleteSpotDatafeedSubscriptionResult;
-import com.amazonaws.services.ec2.model.DeleteSubnetRequest;
-import com.amazonaws.services.ec2.model.DeleteSubnetResult;
-import com.amazonaws.services.ec2.model.DeleteTagsRequest;
-import com.amazonaws.services.ec2.model.DeleteTagsResult;
-import com.amazonaws.services.ec2.model.DeleteTransitGatewayRequest;
-import com.amazonaws.services.ec2.model.DeleteTransitGatewayResult;
-import com.amazonaws.services.ec2.model.DeleteTransitGatewayRouteRequest;
-import com.amazonaws.services.ec2.model.DeleteTransitGatewayRouteResult;
-import com.amazonaws.services.ec2.model.DeleteTransitGatewayRouteTableRequest;
-import com.amazonaws.services.ec2.model.DeleteTransitGatewayRouteTableResult;
-import com.amazonaws.services.ec2.model.DeleteTransitGatewayVpcAttachmentRequest;
-import com.amazonaws.services.ec2.model.DeleteTransitGatewayVpcAttachmentResult;
-import com.amazonaws.services.ec2.model.DeleteVolumeRequest;
-import com.amazonaws.services.ec2.model.DeleteVolumeResult;
-import com.amazonaws.services.ec2.model.DeleteVpcEndpointConnectionNotificationsRequest;
-import com.amazonaws.services.ec2.model.DeleteVpcEndpointConnectionNotificationsResult;
-import com.amazonaws.services.ec2.model.DeleteVpcEndpointServiceConfigurationsRequest;
-import com.amazonaws.services.ec2.model.DeleteVpcEndpointServiceConfigurationsResult;
-import com.amazonaws.services.ec2.model.DeleteVpcEndpointsRequest;
-import com.amazonaws.services.ec2.model.DeleteVpcEndpointsResult;
-import com.amazonaws.services.ec2.model.DeleteVpcPeeringConnectionRequest;
-import com.amazonaws.services.ec2.model.DeleteVpcPeeringConnectionResult;
-import com.amazonaws.services.ec2.model.DeleteVpcRequest;
-import com.amazonaws.services.ec2.model.DeleteVpcResult;
-import com.amazonaws.services.ec2.model.DeleteVpnConnectionRequest;
-import com.amazonaws.services.ec2.model.DeleteVpnConnectionResult;
-import com.amazonaws.services.ec2.model.DeleteVpnConnectionRouteRequest;
-import com.amazonaws.services.ec2.model.DeleteVpnConnectionRouteResult;
-import com.amazonaws.services.ec2.model.DeleteVpnGatewayRequest;
-import com.amazonaws.services.ec2.model.DeleteVpnGatewayResult;
-import com.amazonaws.services.ec2.model.DeprovisionByoipCidrRequest;
-import com.amazonaws.services.ec2.model.DeprovisionByoipCidrResult;
-import com.amazonaws.services.ec2.model.DeregisterImageRequest;
-import com.amazonaws.services.ec2.model.DeregisterImageResult;
-import com.amazonaws.services.ec2.model.DescribeAccountAttributesRequest;
-import com.amazonaws.services.ec2.model.DescribeAccountAttributesResult;
-import com.amazonaws.services.ec2.model.DescribeAddressesRequest;
-import com.amazonaws.services.ec2.model.DescribeAddressesResult;
-import com.amazonaws.services.ec2.model.DescribeAggregateIdFormatRequest;
-import com.amazonaws.services.ec2.model.DescribeAggregateIdFormatResult;
-import com.amazonaws.services.ec2.model.DescribeAvailabilityZonesRequest;
-import com.amazonaws.services.ec2.model.DescribeAvailabilityZonesResult;
-import com.amazonaws.services.ec2.model.DescribeBundleTasksRequest;
-import com.amazonaws.services.ec2.model.DescribeBundleTasksResult;
-import com.amazonaws.services.ec2.model.DescribeByoipCidrsRequest;
-import com.amazonaws.services.ec2.model.DescribeByoipCidrsResult;
-import com.amazonaws.services.ec2.model.DescribeCapacityReservationsRequest;
-import com.amazonaws.services.ec2.model.DescribeCapacityReservationsResult;
-import com.amazonaws.services.ec2.model.DescribeClassicLinkInstancesRequest;
-import com.amazonaws.services.ec2.model.DescribeClassicLinkInstancesResult;
-import com.amazonaws.services.ec2.model.DescribeClientVpnAuthorizationRulesRequest;
-import com.amazonaws.services.ec2.model.DescribeClientVpnAuthorizationRulesResult;
-import com.amazonaws.services.ec2.model.DescribeClientVpnConnectionsRequest;
-import com.amazonaws.services.ec2.model.DescribeClientVpnConnectionsResult;
-import com.amazonaws.services.ec2.model.DescribeClientVpnEndpointsRequest;
-import com.amazonaws.services.ec2.model.DescribeClientVpnEndpointsResult;
-import com.amazonaws.services.ec2.model.DescribeClientVpnRoutesRequest;
-import com.amazonaws.services.ec2.model.DescribeClientVpnRoutesResult;
-import com.amazonaws.services.ec2.model.DescribeClientVpnTargetNetworksRequest;
-import com.amazonaws.services.ec2.model.DescribeClientVpnTargetNetworksResult;
-import com.amazonaws.services.ec2.model.DescribeConversionTasksRequest;
-import com.amazonaws.services.ec2.model.DescribeConversionTasksResult;
-import com.amazonaws.services.ec2.model.DescribeCustomerGatewaysRequest;
-import com.amazonaws.services.ec2.model.DescribeCustomerGatewaysResult;
-import com.amazonaws.services.ec2.model.DescribeDhcpOptionsRequest;
-import com.amazonaws.services.ec2.model.DescribeDhcpOptionsResult;
-import com.amazonaws.services.ec2.model.DescribeEgressOnlyInternetGatewaysRequest;
-import com.amazonaws.services.ec2.model.DescribeEgressOnlyInternetGatewaysResult;
-import com.amazonaws.services.ec2.model.DescribeElasticGpusRequest;
-import com.amazonaws.services.ec2.model.DescribeElasticGpusResult;
-import com.amazonaws.services.ec2.model.DescribeExportTasksRequest;
-import com.amazonaws.services.ec2.model.DescribeExportTasksResult;
-import com.amazonaws.services.ec2.model.DescribeFleetHistoryRequest;
-import com.amazonaws.services.ec2.model.DescribeFleetHistoryResult;
-import com.amazonaws.services.ec2.model.DescribeFleetInstancesRequest;
-import com.amazonaws.services.ec2.model.DescribeFleetInstancesResult;
-import com.amazonaws.services.ec2.model.DescribeFleetsRequest;
-import com.amazonaws.services.ec2.model.DescribeFleetsResult;
-import com.amazonaws.services.ec2.model.DescribeFlowLogsRequest;
-import com.amazonaws.services.ec2.model.DescribeFlowLogsResult;
-import com.amazonaws.services.ec2.model.DescribeFpgaImageAttributeRequest;
-import com.amazonaws.services.ec2.model.DescribeFpgaImageAttributeResult;
-import com.amazonaws.services.ec2.model.DescribeFpgaImagesRequest;
-import com.amazonaws.services.ec2.model.DescribeFpgaImagesResult;
-import com.amazonaws.services.ec2.model.DescribeHostReservationOfferingsRequest;
-import com.amazonaws.services.ec2.model.DescribeHostReservationOfferingsResult;
-import com.amazonaws.services.ec2.model.DescribeHostReservationsRequest;
-import com.amazonaws.services.ec2.model.DescribeHostReservationsResult;
-import com.amazonaws.services.ec2.model.DescribeHostsRequest;
-import com.amazonaws.services.ec2.model.DescribeHostsResult;
-import com.amazonaws.services.ec2.model.DescribeIamInstanceProfileAssociationsRequest;
-import com.amazonaws.services.ec2.model.DescribeIamInstanceProfileAssociationsResult;
-import com.amazonaws.services.ec2.model.DescribeIdFormatRequest;
-import com.amazonaws.services.ec2.model.DescribeIdFormatResult;
-import com.amazonaws.services.ec2.model.DescribeIdentityIdFormatRequest;
-import com.amazonaws.services.ec2.model.DescribeIdentityIdFormatResult;
-import com.amazonaws.services.ec2.model.DescribeImageAttributeRequest;
-import com.amazonaws.services.ec2.model.DescribeImageAttributeResult;
-import com.amazonaws.services.ec2.model.DescribeImagesRequest;
-import com.amazonaws.services.ec2.model.DescribeImagesResult;
-import com.amazonaws.services.ec2.model.DescribeImportImageTasksRequest;
-import com.amazonaws.services.ec2.model.DescribeImportImageTasksResult;
-import com.amazonaws.services.ec2.model.DescribeImportSnapshotTasksRequest;
-import com.amazonaws.services.ec2.model.DescribeImportSnapshotTasksResult;
-import com.amazonaws.services.ec2.model.DescribeInstanceAttributeRequest;
-import com.amazonaws.services.ec2.model.DescribeInstanceAttributeResult;
-import com.amazonaws.services.ec2.model.DescribeInstanceCreditSpecificationsRequest;
-import com.amazonaws.services.ec2.model.DescribeInstanceCreditSpecificationsResult;
-import com.amazonaws.services.ec2.model.DescribeInstanceStatusRequest;
-import com.amazonaws.services.ec2.model.DescribeInstanceStatusResult;
+import com.amazonaws.services.ec2.AbstractAmazonEC2;
 import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
 import com.amazonaws.services.ec2.model.DescribeInstancesResult;
-import com.amazonaws.services.ec2.model.DescribeInternetGatewaysRequest;
-import com.amazonaws.services.ec2.model.DescribeInternetGatewaysResult;
-import com.amazonaws.services.ec2.model.DescribeKeyPairsRequest;
-import com.amazonaws.services.ec2.model.DescribeKeyPairsResult;
-import com.amazonaws.services.ec2.model.DescribeLaunchTemplateVersionsRequest;
-import com.amazonaws.services.ec2.model.DescribeLaunchTemplateVersionsResult;
-import com.amazonaws.services.ec2.model.DescribeLaunchTemplatesRequest;
-import com.amazonaws.services.ec2.model.DescribeLaunchTemplatesResult;
-import com.amazonaws.services.ec2.model.DescribeMovingAddressesRequest;
-import com.amazonaws.services.ec2.model.DescribeMovingAddressesResult;
-import com.amazonaws.services.ec2.model.DescribeNatGatewaysRequest;
-import com.amazonaws.services.ec2.model.DescribeNatGatewaysResult;
-import com.amazonaws.services.ec2.model.DescribeNetworkAclsRequest;
-import com.amazonaws.services.ec2.model.DescribeNetworkAclsResult;
-import com.amazonaws.services.ec2.model.DescribeNetworkInterfaceAttributeRequest;
-import com.amazonaws.services.ec2.model.DescribeNetworkInterfaceAttributeResult;
-import com.amazonaws.services.ec2.model.DescribeNetworkInterfacePermissionsRequest;
-import com.amazonaws.services.ec2.model.DescribeNetworkInterfacePermissionsResult;
-import com.amazonaws.services.ec2.model.DescribeNetworkInterfacesRequest;
-import com.amazonaws.services.ec2.model.DescribeNetworkInterfacesResult;
-import com.amazonaws.services.ec2.model.DescribePlacementGroupsRequest;
-import com.amazonaws.services.ec2.model.DescribePlacementGroupsResult;
-import com.amazonaws.services.ec2.model.DescribePrefixListsRequest;
-import com.amazonaws.services.ec2.model.DescribePrefixListsResult;
-import com.amazonaws.services.ec2.model.DescribePrincipalIdFormatRequest;
-import com.amazonaws.services.ec2.model.DescribePrincipalIdFormatResult;
-import com.amazonaws.services.ec2.model.DescribePublicIpv4PoolsRequest;
-import com.amazonaws.services.ec2.model.DescribePublicIpv4PoolsResult;
-import com.amazonaws.services.ec2.model.DescribeRegionsRequest;
-import com.amazonaws.services.ec2.model.DescribeRegionsResult;
-import com.amazonaws.services.ec2.model.DescribeReservedInstancesListingsRequest;
-import com.amazonaws.services.ec2.model.DescribeReservedInstancesListingsResult;
-import com.amazonaws.services.ec2.model.DescribeReservedInstancesModificationsRequest;
-import com.amazonaws.services.ec2.model.DescribeReservedInstancesModificationsResult;
-import com.amazonaws.services.ec2.model.DescribeReservedInstancesOfferingsRequest;
-import com.amazonaws.services.ec2.model.DescribeReservedInstancesOfferingsResult;
-import com.amazonaws.services.ec2.model.DescribeReservedInstancesRequest;
-import com.amazonaws.services.ec2.model.DescribeReservedInstancesResult;
-import com.amazonaws.services.ec2.model.DescribeRouteTablesRequest;
-import com.amazonaws.services.ec2.model.DescribeRouteTablesResult;
-import com.amazonaws.services.ec2.model.DescribeScheduledInstanceAvailabilityRequest;
-import com.amazonaws.services.ec2.model.DescribeScheduledInstanceAvailabilityResult;
-import com.amazonaws.services.ec2.model.DescribeScheduledInstancesRequest;
-import com.amazonaws.services.ec2.model.DescribeScheduledInstancesResult;
-import com.amazonaws.services.ec2.model.DescribeSecurityGroupReferencesRequest;
-import com.amazonaws.services.ec2.model.DescribeSecurityGroupReferencesResult;
-import com.amazonaws.services.ec2.model.DescribeSecurityGroupsRequest;
-import com.amazonaws.services.ec2.model.DescribeSecurityGroupsResult;
-import com.amazonaws.services.ec2.model.DescribeSnapshotAttributeRequest;
-import com.amazonaws.services.ec2.model.DescribeSnapshotAttributeResult;
-import com.amazonaws.services.ec2.model.DescribeSnapshotsRequest;
-import com.amazonaws.services.ec2.model.DescribeSnapshotsResult;
-import com.amazonaws.services.ec2.model.DescribeSpotDatafeedSubscriptionRequest;
-import com.amazonaws.services.ec2.model.DescribeSpotDatafeedSubscriptionResult;
-import com.amazonaws.services.ec2.model.DescribeSpotFleetInstancesRequest;
-import com.amazonaws.services.ec2.model.DescribeSpotFleetInstancesResult;
-import com.amazonaws.services.ec2.model.DescribeSpotFleetRequestHistoryRequest;
-import com.amazonaws.services.ec2.model.DescribeSpotFleetRequestHistoryResult;
-import com.amazonaws.services.ec2.model.DescribeSpotFleetRequestsRequest;
-import com.amazonaws.services.ec2.model.DescribeSpotFleetRequestsResult;
-import com.amazonaws.services.ec2.model.DescribeSpotInstanceRequestsRequest;
-import com.amazonaws.services.ec2.model.DescribeSpotInstanceRequestsResult;
-import com.amazonaws.services.ec2.model.DescribeSpotPriceHistoryRequest;
-import com.amazonaws.services.ec2.model.DescribeSpotPriceHistoryResult;
-import com.amazonaws.services.ec2.model.DescribeStaleSecurityGroupsRequest;
-import com.amazonaws.services.ec2.model.DescribeStaleSecurityGroupsResult;
-import com.amazonaws.services.ec2.model.DescribeSubnetsRequest;
-import com.amazonaws.services.ec2.model.DescribeSubnetsResult;
-import com.amazonaws.services.ec2.model.DescribeTagsRequest;
-import com.amazonaws.services.ec2.model.DescribeTagsResult;
-import com.amazonaws.services.ec2.model.DescribeTransitGatewayAttachmentsRequest;
-import com.amazonaws.services.ec2.model.DescribeTransitGatewayAttachmentsResult;
-import com.amazonaws.services.ec2.model.DescribeTransitGatewayRouteTablesRequest;
-import com.amazonaws.services.ec2.model.DescribeTransitGatewayRouteTablesResult;
-import com.amazonaws.services.ec2.model.DescribeTransitGatewayVpcAttachmentsRequest;
-import com.amazonaws.services.ec2.model.DescribeTransitGatewayVpcAttachmentsResult;
-import com.amazonaws.services.ec2.model.DescribeTransitGatewaysRequest;
-import com.amazonaws.services.ec2.model.DescribeTransitGatewaysResult;
-import com.amazonaws.services.ec2.model.DescribeVolumeAttributeRequest;
-import com.amazonaws.services.ec2.model.DescribeVolumeAttributeResult;
-import com.amazonaws.services.ec2.model.DescribeVolumeStatusRequest;
-import com.amazonaws.services.ec2.model.DescribeVolumeStatusResult;
-import com.amazonaws.services.ec2.model.DescribeVolumesModificationsRequest;
-import com.amazonaws.services.ec2.model.DescribeVolumesModificationsResult;
-import com.amazonaws.services.ec2.model.DescribeVolumesRequest;
-import com.amazonaws.services.ec2.model.DescribeVolumesResult;
-import com.amazonaws.services.ec2.model.DescribeVpcAttributeRequest;
-import com.amazonaws.services.ec2.model.DescribeVpcAttributeResult;
-import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkDnsSupportRequest;
-import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkDnsSupportResult;
-import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkRequest;
-import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkResult;
-import com.amazonaws.services.ec2.model.DescribeVpcEndpointConnectionNotificationsRequest;
-import com.amazonaws.services.ec2.model.DescribeVpcEndpointConnectionNotificationsResult;
-import com.amazonaws.services.ec2.model.DescribeVpcEndpointConnectionsRequest;
-import com.amazonaws.services.ec2.model.DescribeVpcEndpointConnectionsResult;
-import com.amazonaws.services.ec2.model.DescribeVpcEndpointServiceConfigurationsRequest;
-import com.amazonaws.services.ec2.model.DescribeVpcEndpointServiceConfigurationsResult;
-import com.amazonaws.services.ec2.model.DescribeVpcEndpointServicePermissionsRequest;
-import com.amazonaws.services.ec2.model.DescribeVpcEndpointServicePermissionsResult;
-import com.amazonaws.services.ec2.model.DescribeVpcEndpointServicesRequest;
-import com.amazonaws.services.ec2.model.DescribeVpcEndpointServicesResult;
-import com.amazonaws.services.ec2.model.DescribeVpcEndpointsRequest;
-import com.amazonaws.services.ec2.model.DescribeVpcEndpointsResult;
-import com.amazonaws.services.ec2.model.DescribeVpcPeeringConnectionsRequest;
-import com.amazonaws.services.ec2.model.DescribeVpcPeeringConnectionsResult;
-import com.amazonaws.services.ec2.model.DescribeVpcsRequest;
-import com.amazonaws.services.ec2.model.DescribeVpcsResult;
-import com.amazonaws.services.ec2.model.DescribeVpnConnectionsRequest;
-import com.amazonaws.services.ec2.model.DescribeVpnConnectionsResult;
-import com.amazonaws.services.ec2.model.DescribeVpnGatewaysRequest;
-import com.amazonaws.services.ec2.model.DescribeVpnGatewaysResult;
-import com.amazonaws.services.ec2.model.DetachClassicLinkVpcRequest;
-import com.amazonaws.services.ec2.model.DetachClassicLinkVpcResult;
-import com.amazonaws.services.ec2.model.DetachInternetGatewayRequest;
-import com.amazonaws.services.ec2.model.DetachInternetGatewayResult;
-import com.amazonaws.services.ec2.model.DetachNetworkInterfaceRequest;
-import com.amazonaws.services.ec2.model.DetachNetworkInterfaceResult;
-import com.amazonaws.services.ec2.model.DetachVolumeRequest;
-import com.amazonaws.services.ec2.model.DetachVolumeResult;
-import com.amazonaws.services.ec2.model.DetachVpnGatewayRequest;
-import com.amazonaws.services.ec2.model.DetachVpnGatewayResult;
-import com.amazonaws.services.ec2.model.DisableTransitGatewayRouteTablePropagationRequest;
-import com.amazonaws.services.ec2.model.DisableTransitGatewayRouteTablePropagationResult;
-import com.amazonaws.services.ec2.model.DisableVgwRoutePropagationRequest;
-import com.amazonaws.services.ec2.model.DisableVgwRoutePropagationResult;
-import com.amazonaws.services.ec2.model.DisableVpcClassicLinkDnsSupportRequest;
-import com.amazonaws.services.ec2.model.DisableVpcClassicLinkDnsSupportResult;
-import com.amazonaws.services.ec2.model.DisableVpcClassicLinkRequest;
-import com.amazonaws.services.ec2.model.DisableVpcClassicLinkResult;
-import com.amazonaws.services.ec2.model.DisassociateAddressRequest;
-import com.amazonaws.services.ec2.model.DisassociateAddressResult;
-import com.amazonaws.services.ec2.model.DisassociateClientVpnTargetNetworkRequest;
-import com.amazonaws.services.ec2.model.DisassociateClientVpnTargetNetworkResult;
-import com.amazonaws.services.ec2.model.DisassociateIamInstanceProfileRequest;
-import com.amazonaws.services.ec2.model.DisassociateIamInstanceProfileResult;
-import com.amazonaws.services.ec2.model.DisassociateRouteTableRequest;
-import com.amazonaws.services.ec2.model.DisassociateRouteTableResult;
-import com.amazonaws.services.ec2.model.DisassociateSubnetCidrBlockRequest;
-import com.amazonaws.services.ec2.model.DisassociateSubnetCidrBlockResult;
-import com.amazonaws.services.ec2.model.DisassociateTransitGatewayRouteTableRequest;
-import com.amazonaws.services.ec2.model.DisassociateTransitGatewayRouteTableResult;
-import com.amazonaws.services.ec2.model.DisassociateVpcCidrBlockRequest;
-import com.amazonaws.services.ec2.model.DisassociateVpcCidrBlockResult;
-import com.amazonaws.services.ec2.model.DryRunResult;
-import com.amazonaws.services.ec2.model.DryRunSupportedRequest;
-import com.amazonaws.services.ec2.model.EnableTransitGatewayRouteTablePropagationRequest;
-import com.amazonaws.services.ec2.model.EnableTransitGatewayRouteTablePropagationResult;
-import com.amazonaws.services.ec2.model.EnableVgwRoutePropagationRequest;
-import com.amazonaws.services.ec2.model.EnableVgwRoutePropagationResult;
-import com.amazonaws.services.ec2.model.EnableVolumeIORequest;
-import com.amazonaws.services.ec2.model.EnableVolumeIOResult;
-import com.amazonaws.services.ec2.model.EnableVpcClassicLinkDnsSupportRequest;
-import com.amazonaws.services.ec2.model.EnableVpcClassicLinkDnsSupportResult;
-import com.amazonaws.services.ec2.model.EnableVpcClassicLinkRequest;
-import com.amazonaws.services.ec2.model.EnableVpcClassicLinkResult;
-import com.amazonaws.services.ec2.model.ExportClientVpnClientCertificateRevocationListRequest;
-import com.amazonaws.services.ec2.model.ExportClientVpnClientCertificateRevocationListResult;
-import com.amazonaws.services.ec2.model.ExportClientVpnClientConfigurationRequest;
-import com.amazonaws.services.ec2.model.ExportClientVpnClientConfigurationResult;
-import com.amazonaws.services.ec2.model.ExportTransitGatewayRoutesRequest;
-import com.amazonaws.services.ec2.model.ExportTransitGatewayRoutesResult;
 import com.amazonaws.services.ec2.model.Filter;
-import com.amazonaws.services.ec2.model.GetConsoleOutputRequest;
-import com.amazonaws.services.ec2.model.GetConsoleOutputResult;
-import com.amazonaws.services.ec2.model.GetConsoleScreenshotRequest;
-import com.amazonaws.services.ec2.model.GetConsoleScreenshotResult;
-import com.amazonaws.services.ec2.model.GetHostReservationPurchasePreviewRequest;
-import com.amazonaws.services.ec2.model.GetHostReservationPurchasePreviewResult;
-import com.amazonaws.services.ec2.model.GetLaunchTemplateDataRequest;
-import com.amazonaws.services.ec2.model.GetLaunchTemplateDataResult;
-import com.amazonaws.services.ec2.model.GetPasswordDataRequest;
-import com.amazonaws.services.ec2.model.GetPasswordDataResult;
-import com.amazonaws.services.ec2.model.GetReservedInstancesExchangeQuoteRequest;
-import com.amazonaws.services.ec2.model.GetReservedInstancesExchangeQuoteResult;
-import com.amazonaws.services.ec2.model.GetTransitGatewayAttachmentPropagationsRequest;
-import com.amazonaws.services.ec2.model.GetTransitGatewayAttachmentPropagationsResult;
-import com.amazonaws.services.ec2.model.GetTransitGatewayRouteTableAssociationsRequest;
-import com.amazonaws.services.ec2.model.GetTransitGatewayRouteTableAssociationsResult;
-import com.amazonaws.services.ec2.model.GetTransitGatewayRouteTablePropagationsRequest;
-import com.amazonaws.services.ec2.model.GetTransitGatewayRouteTablePropagationsResult;
-import com.amazonaws.services.ec2.model.ImportClientVpnClientCertificateRevocationListRequest;
-import com.amazonaws.services.ec2.model.ImportClientVpnClientCertificateRevocationListResult;
-import com.amazonaws.services.ec2.model.ImportImageRequest;
-import com.amazonaws.services.ec2.model.ImportImageResult;
-import com.amazonaws.services.ec2.model.ImportInstanceRequest;
-import com.amazonaws.services.ec2.model.ImportInstanceResult;
-import com.amazonaws.services.ec2.model.ImportKeyPairRequest;
-import com.amazonaws.services.ec2.model.ImportKeyPairResult;
-import com.amazonaws.services.ec2.model.ImportSnapshotRequest;
-import com.amazonaws.services.ec2.model.ImportSnapshotResult;
-import com.amazonaws.services.ec2.model.ImportVolumeRequest;
-import com.amazonaws.services.ec2.model.ImportVolumeResult;
 import com.amazonaws.services.ec2.model.Instance;
 import com.amazonaws.services.ec2.model.InstanceState;
 import com.amazonaws.services.ec2.model.InstanceStateName;
-import com.amazonaws.services.ec2.model.ModifyCapacityReservationRequest;
-import com.amazonaws.services.ec2.model.ModifyCapacityReservationResult;
-import com.amazonaws.services.ec2.model.ModifyClientVpnEndpointRequest;
-import com.amazonaws.services.ec2.model.ModifyClientVpnEndpointResult;
-import com.amazonaws.services.ec2.model.ModifyFleetRequest;
-import com.amazonaws.services.ec2.model.ModifyFleetResult;
-import com.amazonaws.services.ec2.model.ModifyFpgaImageAttributeRequest;
-import com.amazonaws.services.ec2.model.ModifyFpgaImageAttributeResult;
-import com.amazonaws.services.ec2.model.ModifyHostsRequest;
-import com.amazonaws.services.ec2.model.ModifyHostsResult;
-import com.amazonaws.services.ec2.model.ModifyIdFormatRequest;
-import com.amazonaws.services.ec2.model.ModifyIdFormatResult;
-import com.amazonaws.services.ec2.model.ModifyIdentityIdFormatRequest;
-import com.amazonaws.services.ec2.model.ModifyIdentityIdFormatResult;
-import com.amazonaws.services.ec2.model.ModifyImageAttributeRequest;
-import com.amazonaws.services.ec2.model.ModifyImageAttributeResult;
-import com.amazonaws.services.ec2.model.ModifyInstanceAttributeRequest;
-import com.amazonaws.services.ec2.model.ModifyInstanceAttributeResult;
-import com.amazonaws.services.ec2.model.ModifyInstanceCapacityReservationAttributesRequest;
-import com.amazonaws.services.ec2.model.ModifyInstanceCapacityReservationAttributesResult;
-import com.amazonaws.services.ec2.model.ModifyInstanceCreditSpecificationRequest;
-import com.amazonaws.services.ec2.model.ModifyInstanceCreditSpecificationResult;
-import com.amazonaws.services.ec2.model.ModifyInstancePlacementRequest;
-import com.amazonaws.services.ec2.model.ModifyInstancePlacementResult;
-import com.amazonaws.services.ec2.model.ModifyLaunchTemplateRequest;
-import com.amazonaws.services.ec2.model.ModifyLaunchTemplateResult;
-import com.amazonaws.services.ec2.model.ModifyNetworkInterfaceAttributeRequest;
-import com.amazonaws.services.ec2.model.ModifyNetworkInterfaceAttributeResult;
-import com.amazonaws.services.ec2.model.ModifyReservedInstancesRequest;
-import com.amazonaws.services.ec2.model.ModifyReservedInstancesResult;
-import com.amazonaws.services.ec2.model.ModifySnapshotAttributeRequest;
-import com.amazonaws.services.ec2.model.ModifySnapshotAttributeResult;
-import com.amazonaws.services.ec2.model.ModifySpotFleetRequestRequest;
-import com.amazonaws.services.ec2.model.ModifySpotFleetRequestResult;
-import com.amazonaws.services.ec2.model.ModifySubnetAttributeRequest;
-import com.amazonaws.services.ec2.model.ModifySubnetAttributeResult;
-import com.amazonaws.services.ec2.model.ModifyTransitGatewayVpcAttachmentRequest;
-import com.amazonaws.services.ec2.model.ModifyTransitGatewayVpcAttachmentResult;
-import com.amazonaws.services.ec2.model.ModifyVolumeAttributeRequest;
-import com.amazonaws.services.ec2.model.ModifyVolumeAttributeResult;
-import com.amazonaws.services.ec2.model.ModifyVolumeRequest;
-import com.amazonaws.services.ec2.model.ModifyVolumeResult;
-import com.amazonaws.services.ec2.model.ModifyVpcAttributeRequest;
-import com.amazonaws.services.ec2.model.ModifyVpcAttributeResult;
-import com.amazonaws.services.ec2.model.ModifyVpcEndpointConnectionNotificationRequest;
-import com.amazonaws.services.ec2.model.ModifyVpcEndpointConnectionNotificationResult;
-import com.amazonaws.services.ec2.model.ModifyVpcEndpointRequest;
-import com.amazonaws.services.ec2.model.ModifyVpcEndpointResult;
-import com.amazonaws.services.ec2.model.ModifyVpcEndpointServiceConfigurationRequest;
-import com.amazonaws.services.ec2.model.ModifyVpcEndpointServiceConfigurationResult;
-import com.amazonaws.services.ec2.model.ModifyVpcEndpointServicePermissionsRequest;
-import com.amazonaws.services.ec2.model.ModifyVpcEndpointServicePermissionsResult;
-import com.amazonaws.services.ec2.model.ModifyVpcPeeringConnectionOptionsRequest;
-import com.amazonaws.services.ec2.model.ModifyVpcPeeringConnectionOptionsResult;
-import com.amazonaws.services.ec2.model.ModifyVpcTenancyRequest;
-import com.amazonaws.services.ec2.model.ModifyVpcTenancyResult;
-import com.amazonaws.services.ec2.model.MonitorInstancesRequest;
-import com.amazonaws.services.ec2.model.MonitorInstancesResult;
-import com.amazonaws.services.ec2.model.MoveAddressToVpcRequest;
-import com.amazonaws.services.ec2.model.MoveAddressToVpcResult;
-import com.amazonaws.services.ec2.model.ProvisionByoipCidrRequest;
-import com.amazonaws.services.ec2.model.ProvisionByoipCidrResult;
-import com.amazonaws.services.ec2.model.PurchaseHostReservationRequest;
-import com.amazonaws.services.ec2.model.PurchaseHostReservationResult;
-import com.amazonaws.services.ec2.model.PurchaseReservedInstancesOfferingRequest;
-import com.amazonaws.services.ec2.model.PurchaseReservedInstancesOfferingResult;
-import com.amazonaws.services.ec2.model.PurchaseScheduledInstancesRequest;
-import com.amazonaws.services.ec2.model.PurchaseScheduledInstancesResult;
-import com.amazonaws.services.ec2.model.RebootInstancesRequest;
-import com.amazonaws.services.ec2.model.RebootInstancesResult;
-import com.amazonaws.services.ec2.model.RegisterImageRequest;
-import com.amazonaws.services.ec2.model.RegisterImageResult;
-import com.amazonaws.services.ec2.model.RejectTransitGatewayVpcAttachmentRequest;
-import com.amazonaws.services.ec2.model.RejectTransitGatewayVpcAttachmentResult;
-import com.amazonaws.services.ec2.model.RejectVpcEndpointConnectionsRequest;
-import com.amazonaws.services.ec2.model.RejectVpcEndpointConnectionsResult;
-import com.amazonaws.services.ec2.model.RejectVpcPeeringConnectionRequest;
-import com.amazonaws.services.ec2.model.RejectVpcPeeringConnectionResult;
-import com.amazonaws.services.ec2.model.ReleaseAddressRequest;
-import com.amazonaws.services.ec2.model.ReleaseAddressResult;
-import com.amazonaws.services.ec2.model.ReleaseHostsRequest;
-import com.amazonaws.services.ec2.model.ReleaseHostsResult;
-import com.amazonaws.services.ec2.model.ReplaceIamInstanceProfileAssociationRequest;
-import com.amazonaws.services.ec2.model.ReplaceIamInstanceProfileAssociationResult;
-import com.amazonaws.services.ec2.model.ReplaceNetworkAclAssociationRequest;
-import com.amazonaws.services.ec2.model.ReplaceNetworkAclAssociationResult;
-import com.amazonaws.services.ec2.model.ReplaceNetworkAclEntryRequest;
-import com.amazonaws.services.ec2.model.ReplaceNetworkAclEntryResult;
-import com.amazonaws.services.ec2.model.ReplaceRouteRequest;
-import com.amazonaws.services.ec2.model.ReplaceRouteResult;
-import com.amazonaws.services.ec2.model.ReplaceRouteTableAssociationRequest;
-import com.amazonaws.services.ec2.model.ReplaceRouteTableAssociationResult;
-import com.amazonaws.services.ec2.model.ReplaceTransitGatewayRouteRequest;
-import com.amazonaws.services.ec2.model.ReplaceTransitGatewayRouteResult;
-import com.amazonaws.services.ec2.model.ReportInstanceStatusRequest;
-import com.amazonaws.services.ec2.model.ReportInstanceStatusResult;
-import com.amazonaws.services.ec2.model.RequestSpotFleetRequest;
-import com.amazonaws.services.ec2.model.RequestSpotFleetResult;
-import com.amazonaws.services.ec2.model.RequestSpotInstancesRequest;
-import com.amazonaws.services.ec2.model.RequestSpotInstancesResult;
 import com.amazonaws.services.ec2.model.Reservation;
-import com.amazonaws.services.ec2.model.ResetFpgaImageAttributeRequest;
-import com.amazonaws.services.ec2.model.ResetFpgaImageAttributeResult;
-import com.amazonaws.services.ec2.model.ResetImageAttributeRequest;
-import com.amazonaws.services.ec2.model.ResetImageAttributeResult;
-import com.amazonaws.services.ec2.model.ResetInstanceAttributeRequest;
-import com.amazonaws.services.ec2.model.ResetInstanceAttributeResult;
-import com.amazonaws.services.ec2.model.ResetNetworkInterfaceAttributeRequest;
-import com.amazonaws.services.ec2.model.ResetNetworkInterfaceAttributeResult;
-import com.amazonaws.services.ec2.model.ResetSnapshotAttributeRequest;
-import com.amazonaws.services.ec2.model.ResetSnapshotAttributeResult;
-import com.amazonaws.services.ec2.model.RestoreAddressToClassicRequest;
-import com.amazonaws.services.ec2.model.RestoreAddressToClassicResult;
-import com.amazonaws.services.ec2.model.RevokeClientVpnIngressRequest;
-import com.amazonaws.services.ec2.model.RevokeClientVpnIngressResult;
-import com.amazonaws.services.ec2.model.RevokeSecurityGroupEgressRequest;
-import com.amazonaws.services.ec2.model.RevokeSecurityGroupEgressResult;
-import com.amazonaws.services.ec2.model.RevokeSecurityGroupIngressRequest;
-import com.amazonaws.services.ec2.model.RevokeSecurityGroupIngressResult;
-import com.amazonaws.services.ec2.model.RunInstancesRequest;
-import com.amazonaws.services.ec2.model.RunInstancesResult;
-import com.amazonaws.services.ec2.model.RunScheduledInstancesRequest;
-import com.amazonaws.services.ec2.model.RunScheduledInstancesResult;
-import com.amazonaws.services.ec2.model.SearchTransitGatewayRoutesRequest;
-import com.amazonaws.services.ec2.model.SearchTransitGatewayRoutesResult;
-import com.amazonaws.services.ec2.model.StartInstancesRequest;
-import com.amazonaws.services.ec2.model.StartInstancesResult;
-import com.amazonaws.services.ec2.model.StopInstancesRequest;
-import com.amazonaws.services.ec2.model.StopInstancesResult;
 import com.amazonaws.services.ec2.model.Tag;
-import com.amazonaws.services.ec2.model.TerminateClientVpnConnectionsRequest;
-import com.amazonaws.services.ec2.model.TerminateClientVpnConnectionsResult;
-import com.amazonaws.services.ec2.model.TerminateInstancesRequest;
-import com.amazonaws.services.ec2.model.TerminateInstancesResult;
-import com.amazonaws.services.ec2.model.UnassignIpv6AddressesRequest;
-import com.amazonaws.services.ec2.model.UnassignIpv6AddressesResult;
-import com.amazonaws.services.ec2.model.UnassignPrivateIpAddressesRequest;
-import com.amazonaws.services.ec2.model.UnassignPrivateIpAddressesResult;
-import com.amazonaws.services.ec2.model.UnmonitorInstancesRequest;
-import com.amazonaws.services.ec2.model.UnmonitorInstancesResult;
-import com.amazonaws.services.ec2.model.UpdateSecurityGroupRuleDescriptionsEgressRequest;
-import com.amazonaws.services.ec2.model.UpdateSecurityGroupRuleDescriptionsEgressResult;
-import com.amazonaws.services.ec2.model.UpdateSecurityGroupRuleDescriptionsIngressRequest;
-import com.amazonaws.services.ec2.model.UpdateSecurityGroupRuleDescriptionsIngressResult;
-import com.amazonaws.services.ec2.model.WithdrawByoipCidrRequest;
-import com.amazonaws.services.ec2.model.WithdrawByoipCidrResult;
-import com.amazonaws.services.ec2.waiters.AmazonEC2Waiters;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -704,7 +42,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-public class AmazonEC2Mock implements AmazonEC2 {
+public class AmazonEC2Mock extends AbstractAmazonEC2 {
 
     private static final Logger logger = LogManager.getLogger(AmazonEC2Mock.class);
 
@@ -830,2250 +168,12 @@ public DescribeInstancesResult describeInstances(DescribeInstancesRequest descri
         );
     }
 
-    // Not implemented methods in Mock
-
     @Override
     public void setEndpoint(String endpoint) throws IllegalArgumentException {
         this.endpoint = endpoint;
     }
 
-    @Override
-    public void setRegion(Region region) throws IllegalArgumentException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AcceptReservedInstancesExchangeQuoteResult acceptReservedInstancesExchangeQuote(
-            AcceptReservedInstancesExchangeQuoteRequest acceptReservedInstancesExchangeQuoteRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AcceptTransitGatewayVpcAttachmentResult acceptTransitGatewayVpcAttachment(
-        AcceptTransitGatewayVpcAttachmentRequest acceptTransitGatewayVpcAttachmentRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AcceptVpcEndpointConnectionsResult acceptVpcEndpointConnections(
-        AcceptVpcEndpointConnectionsRequest acceptVpcEndpointConnectionsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public RebootInstancesResult rebootInstances(RebootInstancesRequest rebootInstancesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeReservedInstancesResult describeReservedInstances(
-            DescribeReservedInstancesRequest describeReservedInstancesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateFlowLogsResult createFlowLogs(CreateFlowLogsRequest createFlowLogsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeAvailabilityZonesResult describeAvailabilityZones(DescribeAvailabilityZonesRequest describeAvailabilityZonesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public RestoreAddressToClassicResult restoreAddressToClassic(RestoreAddressToClassicRequest restoreAddressToClassicRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public RevokeClientVpnIngressResult revokeClientVpnIngress(RevokeClientVpnIngressRequest revokeClientVpnIngressRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DetachVolumeResult detachVolume(DetachVolumeRequest detachVolumeRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteKeyPairResult deleteKeyPair(DeleteKeyPairRequest deleteKeyPairRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteLaunchTemplateResult deleteLaunchTemplate(DeleteLaunchTemplateRequest deleteLaunchTemplateRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteLaunchTemplateVersionsResult deleteLaunchTemplateVersions(
-        DeleteLaunchTemplateVersionsRequest deleteLaunchTemplateVersionsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteNatGatewayResult deleteNatGateway(DeleteNatGatewayRequest deleteNatGatewayRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public UnmonitorInstancesResult unmonitorInstances(UnmonitorInstancesRequest unmonitorInstancesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public UpdateSecurityGroupRuleDescriptionsIngressResult updateSecurityGroupRuleDescriptionsIngress(
-            UpdateSecurityGroupRuleDescriptionsIngressRequest updateSecurityGroupRuleDescriptionsIngressRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public WithdrawByoipCidrResult withdrawByoipCidr(WithdrawByoipCidrRequest withdrawByoipCidrRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public UpdateSecurityGroupRuleDescriptionsEgressResult updateSecurityGroupRuleDescriptionsEgress(
-            UpdateSecurityGroupRuleDescriptionsEgressRequest updateSecurityGroupRuleDescriptionsEgressRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AttachVpnGatewayResult attachVpnGateway(AttachVpnGatewayRequest attachVpnGatewayRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AuthorizeClientVpnIngressResult authorizeClientVpnIngress(AuthorizeClientVpnIngressRequest authorizeClientVpnIngressRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateImageResult createImage(CreateImageRequest createImageRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteSecurityGroupResult deleteSecurityGroup(DeleteSecurityGroupRequest deleteSecurityGroupRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateInstanceExportTaskResult createInstanceExportTask(CreateInstanceExportTaskRequest createInstanceExportTaskRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AuthorizeSecurityGroupEgressResult authorizeSecurityGroupEgress(
-            AuthorizeSecurityGroupEgressRequest authorizeSecurityGroupEgressRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AssociateDhcpOptionsResult associateDhcpOptions(AssociateDhcpOptionsRequest associateDhcpOptionsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public GetPasswordDataResult getPasswordData(GetPasswordDataRequest getPasswordDataRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public GetReservedInstancesExchangeQuoteResult getReservedInstancesExchangeQuote(
-            GetReservedInstancesExchangeQuoteRequest getReservedInstancesExchangeQuoteRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public GetTransitGatewayAttachmentPropagationsResult getTransitGatewayAttachmentPropagations(
-        GetTransitGatewayAttachmentPropagationsRequest getTransitGatewayAttachmentPropagationsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public GetTransitGatewayRouteTableAssociationsResult getTransitGatewayRouteTableAssociations(
-        GetTransitGatewayRouteTableAssociationsRequest getTransitGatewayRouteTableAssociationsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public GetTransitGatewayRouteTablePropagationsResult getTransitGatewayRouteTablePropagations(
-        GetTransitGatewayRouteTablePropagationsRequest getTransitGatewayRouteTablePropagationsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ImportClientVpnClientCertificateRevocationListResult importClientVpnClientCertificateRevocationList(
-        ImportClientVpnClientCertificateRevocationListRequest importClientVpnClientCertificateRevocationListRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public StopInstancesResult stopInstances(StopInstancesRequest stopInstancesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public TerminateClientVpnConnectionsResult terminateClientVpnConnections(
-        TerminateClientVpnConnectionsRequest terminateClientVpnConnectionsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ImportKeyPairResult importKeyPair(ImportKeyPairRequest importKeyPairRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteNetworkInterfaceResult deleteNetworkInterface(DeleteNetworkInterfaceRequest deleteNetworkInterfaceRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyVpcAttributeResult modifyVpcAttribute(ModifyVpcAttributeRequest modifyVpcAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSpotFleetInstancesResult describeSpotFleetInstances(DescribeSpotFleetInstancesRequest describeSpotFleetInstancesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateSecurityGroupResult createSecurityGroup(CreateSecurityGroupRequest createSecurityGroupRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSpotPriceHistoryResult describeSpotPriceHistory(DescribeSpotPriceHistoryRequest describeSpotPriceHistoryRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeNetworkInterfacesResult describeNetworkInterfaces(DescribeNetworkInterfacesRequest describeNetworkInterfacesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeNetworkInterfacePermissionsResult describeNetworkInterfacePermissions(
-            DescribeNetworkInterfacePermissionsRequest describeNetworkInterfacePermissionsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeRegionsResult describeRegions(DescribeRegionsRequest describeRegionsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateDhcpOptionsResult createDhcpOptions(CreateDhcpOptionsRequest createDhcpOptionsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateReservedInstancesListingResult createReservedInstancesListing(
-            CreateReservedInstancesListingRequest createReservedInstancesListingRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteVpcEndpointsResult deleteVpcEndpoints(DeleteVpcEndpointsRequest deleteVpcEndpointsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ResetSnapshotAttributeResult resetSnapshotAttribute(ResetSnapshotAttributeRequest resetSnapshotAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteRouteResult deleteRoute(DeleteRouteRequest deleteRouteRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeInternetGatewaysResult describeInternetGateways(DescribeInternetGatewaysRequest describeInternetGatewaysRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ImportVolumeResult importVolume(ImportVolumeRequest importVolumeRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyCapacityReservationResult modifyCapacityReservation(ModifyCapacityReservationRequest modifyCapacityReservationRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyClientVpnEndpointResult modifyClientVpnEndpoint(ModifyClientVpnEndpointRequest modifyClientVpnEndpointRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyFleetResult modifyFleet(ModifyFleetRequest modifyFleetRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyFpgaImageAttributeResult modifyFpgaImageAttribute(ModifyFpgaImageAttributeRequest modifyFpgaImageAttributeRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyHostsResult modifyHosts(ModifyHostsRequest modifyHostsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyIdFormatResult modifyIdFormat(ModifyIdFormatRequest modifyIdFormatRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSecurityGroupsResult describeSecurityGroups(DescribeSecurityGroupsRequest describeSecurityGroupsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeStaleSecurityGroupsResult describeStaleSecurityGroups(
-            DescribeStaleSecurityGroupsRequest describeStaleSecurityGroupsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSecurityGroupReferencesResult describeSecurityGroupReferences(
-            DescribeSecurityGroupReferencesRequest describeSecurityGroupReferencesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public RejectVpcPeeringConnectionResult rejectVpcPeeringConnection(
-            RejectVpcPeeringConnectionRequest rejectVpcPeeringConnectionRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyVpcPeeringConnectionOptionsResult modifyVpcPeeringConnectionOptions(
-            ModifyVpcPeeringConnectionOptionsRequest modifyVpcPeeringConnectionOptionsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyVpcTenancyResult modifyVpcTenancy(ModifyVpcTenancyRequest modifyVpcTenancyRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteFlowLogsResult deleteFlowLogs(DeleteFlowLogsRequest deleteFlowLogsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteFpgaImageResult deleteFpgaImage(DeleteFpgaImageRequest deleteFpgaImageRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DetachVpnGatewayResult detachVpnGateway(DetachVpnGatewayRequest detachVpnGatewayRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DisableTransitGatewayRouteTablePropagationResult disableTransitGatewayRouteTablePropagation(
-        DisableTransitGatewayRouteTablePropagationRequest disableTransitGatewayRouteTablePropagationRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeregisterImageResult deregisterImage(DeregisterImageRequest deregisterImageRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSpotDatafeedSubscriptionResult describeSpotDatafeedSubscription(
-                DescribeSpotDatafeedSubscriptionRequest describeSpotDatafeedSubscriptionRequest)
-                throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteTagsResult deleteTags(DeleteTagsRequest deleteTagsRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteTransitGatewayResult deleteTransitGateway(DeleteTransitGatewayRequest deleteTransitGatewayRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteTransitGatewayRouteResult deleteTransitGatewayRoute(DeleteTransitGatewayRouteRequest deleteTransitGatewayRouteRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteTransitGatewayRouteTableResult deleteTransitGatewayRouteTable(
-        DeleteTransitGatewayRouteTableRequest deleteTransitGatewayRouteTableRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteTransitGatewayVpcAttachmentResult deleteTransitGatewayVpcAttachment(
-        DeleteTransitGatewayVpcAttachmentRequest deleteTransitGatewayVpcAttachmentRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteSubnetResult deleteSubnet(DeleteSubnetRequest deleteSubnetRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeAccountAttributesResult describeAccountAttributes(DescribeAccountAttributesRequest describeAccountAttributesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AttachClassicLinkVpcResult attachClassicLinkVpc(AttachClassicLinkVpcRequest attachClassicLinkVpcRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateVpnGatewayResult createVpnGateway(CreateVpnGatewayRequest createVpnGatewayRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteClientVpnEndpointResult deleteClientVpnEndpoint(DeleteClientVpnEndpointRequest deleteClientVpnEndpointRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteClientVpnRouteResult deleteClientVpnRoute(DeleteClientVpnRouteRequest deleteClientVpnRouteRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public EnableVolumeIOResult enableVolumeIO(EnableVolumeIORequest enableVolumeIORequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public MoveAddressToVpcResult moveAddressToVpc(MoveAddressToVpcRequest moveAddressToVpcRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ProvisionByoipCidrResult provisionByoipCidr(ProvisionByoipCidrRequest provisionByoipCidrRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteVpnGatewayResult deleteVpnGateway(DeleteVpnGatewayRequest deleteVpnGatewayRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeprovisionByoipCidrResult deprovisionByoipCidr(DeprovisionByoipCidrRequest deprovisionByoipCidrRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AttachVolumeResult attachVolume(AttachVolumeRequest attachVolumeRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVolumeStatusResult describeVolumeStatus(DescribeVolumeStatusRequest describeVolumeStatusRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVolumesModificationsResult describeVolumesModifications(
-            DescribeVolumesModificationsRequest describeVolumesModificationsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeImportSnapshotTasksResult describeImportSnapshotTasks(
-            DescribeImportSnapshotTasksRequest describeImportSnapshotTasksRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpnConnectionsResult describeVpnConnections(DescribeVpnConnectionsRequest describeVpnConnectionsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ResetImageAttributeResult resetImageAttribute(ResetImageAttributeRequest resetImageAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public EnableVgwRoutePropagationResult enableVgwRoutePropagation(EnableVgwRoutePropagationRequest enableVgwRoutePropagationRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateSnapshotResult createSnapshot(CreateSnapshotRequest createSnapshotRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteVolumeResult deleteVolume(DeleteVolumeRequest deleteVolumeRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateNetworkInterfaceResult createNetworkInterface(CreateNetworkInterfaceRequest createNetworkInterfaceRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyReservedInstancesResult modifyReservedInstances(ModifyReservedInstancesRequest modifyReservedInstancesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CancelSpotFleetRequestsResult cancelSpotFleetRequests(CancelSpotFleetRequestsRequest cancelSpotFleetRequestsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public UnassignPrivateIpAddressesResult unassignPrivateIpAddresses(UnassignPrivateIpAddressesRequest unassignPrivateIpAddressesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public UnassignIpv6AddressesResult unassignIpv6Addresses(UnassignIpv6AddressesRequest unassignIpv6AddressesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpcsResult describeVpcs(DescribeVpcsRequest describeVpcsRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CancelConversionTaskResult cancelConversionTask(CancelConversionTaskRequest cancelConversionTaskRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AssociateAddressResult associateAddress(AssociateAddressRequest associateAddressRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AssociateClientVpnTargetNetworkResult associateClientVpnTargetNetwork(
-        AssociateClientVpnTargetNetworkRequest associateClientVpnTargetNetworkRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AssociateIamInstanceProfileResult associateIamInstanceProfile(AssociateIamInstanceProfileRequest associateIamInstanceRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AssociateVpcCidrBlockResult associateVpcCidrBlock(AssociateVpcCidrBlockRequest associateVpcCidrBlockRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AssociateSubnetCidrBlockResult associateSubnetCidrBlock(AssociateSubnetCidrBlockRequest associateSubnetCidrBlockRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AssociateTransitGatewayRouteTableResult associateTransitGatewayRouteTable(
-        AssociateTransitGatewayRouteTableRequest associateTransitGatewayRouteTableRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteCustomerGatewayResult deleteCustomerGateway(DeleteCustomerGatewayRequest deleteCustomerGatewayRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateNetworkAclEntryResult createNetworkAclEntry(CreateNetworkAclEntryRequest createNetworkAclEntryRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AcceptVpcPeeringConnectionResult acceptVpcPeeringConnection(AcceptVpcPeeringConnectionRequest acceptVpcPeeringConnectionRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeExportTasksResult describeExportTasks(DescribeExportTasksRequest describeExportTasksRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeElasticGpusResult describeElasticGpus(DescribeElasticGpusRequest describeElasticGpusRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeFpgaImagesResult describeFpgaImages(DescribeFpgaImagesRequest describeFpgaImagesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeHostReservationOfferingsResult describeHostReservationOfferings(
-            DescribeHostReservationOfferingsRequest describeHostReservationOfferingsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeHostReservationsResult describeHostReservations(DescribeHostReservationsRequest describeHostReservationsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeIdentityIdFormatResult describeIdentityIdFormat(DescribeIdentityIdFormatRequest describeIdentityIdFormatRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DetachInternetGatewayResult detachInternetGateway(DetachInternetGatewayRequest detachInternetGatewayRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateVpcPeeringConnectionResult createVpcPeeringConnection(CreateVpcPeeringConnectionRequest createVpcPeeringConnectionRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateRouteTableResult createRouteTable(CreateRouteTableRequest createRouteTableRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CancelImportTaskResult cancelImportTask(CancelImportTaskRequest cancelImportTaskRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVolumesResult describeVolumes(DescribeVolumesRequest describeVolumesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeReservedInstancesListingsResult describeReservedInstancesListings(
-            DescribeReservedInstancesListingsRequest describeReservedInstancesListingsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ReportInstanceStatusResult reportInstanceStatus(ReportInstanceStatusRequest reportInstanceStatusRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeRouteTablesResult describeRouteTables(DescribeRouteTablesRequest describeRouteTablesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeDhcpOptionsResult describeDhcpOptions(DescribeDhcpOptionsRequest describeDhcpOptionsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-    
-    @Override
-    public DescribeEgressOnlyInternetGatewaysResult describeEgressOnlyInternetGateways(
-            DescribeEgressOnlyInternetGatewaysRequest describeEgressOnlyInternetGatewaysRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public MonitorInstancesResult monitorInstances(MonitorInstancesRequest monitorInstancesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribePrefixListsResult describePrefixLists(DescribePrefixListsRequest describePrefixListsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public RequestSpotFleetResult requestSpotFleet(RequestSpotFleetRequest requestSpotFleetRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeImportImageTasksResult describeImportImageTasks(DescribeImportImageTasksRequest describeImportImageTasksRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeNetworkAclsResult describeNetworkAcls(DescribeNetworkAclsRequest describeNetworkAclsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeBundleTasksResult describeBundleTasks(DescribeBundleTasksRequest describeBundleTasksRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ImportInstanceResult importInstance(ImportInstanceRequest importInstanceRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteVpcPeeringConnectionResult deleteVpcPeeringConnection(DeleteVpcPeeringConnectionRequest deleteVpcPeeringConnectionRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public GetConsoleOutputResult getConsoleOutput(GetConsoleOutputRequest getConsoleOutputRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public GetConsoleScreenshotResult getConsoleScreenshot(GetConsoleScreenshotRequest getConsoleScreenshotRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public GetHostReservationPurchasePreviewResult getHostReservationPurchasePreview(
-            GetHostReservationPurchasePreviewRequest getHostReservationPurchasePreviewRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public GetLaunchTemplateDataResult getLaunchTemplateData(GetLaunchTemplateDataRequest getLaunchTemplateDataRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateInternetGatewayResult createInternetGateway(CreateInternetGatewayRequest createInternetGatewayRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteVpnConnectionRouteResult deleteVpnConnectionRoute(DeleteVpnConnectionRouteRequest deleteVpnConnectionRouteRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DetachNetworkInterfaceResult detachNetworkInterface(DetachNetworkInterfaceRequest detachNetworkInterfaceRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyImageAttributeResult modifyImageAttribute(ModifyImageAttributeRequest modifyImageAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateCustomerGatewayResult createCustomerGateway(CreateCustomerGatewayRequest createCustomerGatewayRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateDefaultSubnetResult createDefaultSubnet(CreateDefaultSubnetRequest createDefaultSubnetRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateEgressOnlyInternetGatewayResult createEgressOnlyInternetGateway(
-            CreateEgressOnlyInternetGatewayRequest createEgressOnlyInternetGatewayRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateFleetResult createFleet(CreateFleetRequest createFleetRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateFpgaImageResult createFpgaImage(CreateFpgaImageRequest createFpgaImageRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateNetworkInterfacePermissionResult createNetworkInterfacePermission(
-            CreateNetworkInterfacePermissionRequest createNetworkInterfacePermissionRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateDefaultVpcResult createDefaultVpc(CreateDefaultVpcRequest createDefaultVpcRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateSpotDatafeedSubscriptionResult createSpotDatafeedSubscription(
-            CreateSpotDatafeedSubscriptionRequest createSpotDatafeedSubscriptionRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AttachInternetGatewayResult attachInternetGateway(AttachInternetGatewayRequest attachInternetGatewayRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteVpnConnectionResult deleteVpnConnection(DeleteVpnConnectionRequest deleteVpnConnectionRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeMovingAddressesResult describeMovingAddresses(DescribeMovingAddressesRequest describeMovingAddressesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeConversionTasksResult describeConversionTasks(DescribeConversionTasksRequest describeConversionTasksRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateVpnConnectionResult createVpnConnection(CreateVpnConnectionRequest createVpnConnectionRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ImportImageResult importImage(ImportImageRequest importImageRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DisableVpcClassicLinkResult disableVpcClassicLink(DisableVpcClassicLinkRequest disableVpcClassicLinkRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DisableVpcClassicLinkDnsSupportResult disableVpcClassicLinkDnsSupport(
-            DisableVpcClassicLinkDnsSupportRequest disableVpcClassicLinkDnsSupportRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeInstanceAttributeResult describeInstanceAttribute(DescribeInstanceAttributeRequest describeInstanceAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeInstanceCreditSpecificationsResult describeInstanceCreditSpecifications(
-        DescribeInstanceCreditSpecificationsRequest describeInstanceCreditSpecificationsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeFlowLogsResult describeFlowLogs(DescribeFlowLogsRequest describeFlowLogsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpcPeeringConnectionsResult describeVpcPeeringConnections(
-            DescribeVpcPeeringConnectionsRequest describeVpcPeeringConnectionsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribePlacementGroupsResult describePlacementGroups(DescribePlacementGroupsRequest describePlacementGroupsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public RunInstancesResult runInstances(RunInstancesRequest runInstancesRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public RunScheduledInstancesResult runScheduledInstances(RunScheduledInstancesRequest runScheduledInstancesRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public SearchTransitGatewayRoutesResult searchTransitGatewayRoutes(
-        SearchTransitGatewayRoutesRequest searchTransitGatewayRoutesRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSubnetsResult describeSubnets(DescribeSubnetsRequest describeSubnetsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AssociateRouteTableResult associateRouteTable(AssociateRouteTableRequest associateRouteTableRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyVolumeAttributeResult modifyVolumeAttribute(ModifyVolumeAttributeRequest modifyVolumeAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteNetworkAclResult deleteNetworkAcl(DeleteNetworkAclRequest deleteNetworkAclRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeImagesResult describeImages(DescribeImagesRequest describeImagesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public StartInstancesResult startInstances(StartInstancesRequest startInstancesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyInstanceAttributeResult modifyInstanceAttribute(ModifyInstanceAttributeRequest modifyInstanceAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyInstanceCapacityReservationAttributesResult modifyInstanceCapacityReservationAttributes(
-        ModifyInstanceCapacityReservationAttributesRequest modifyInstanceCapacityReservationAttributesRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyInstanceCreditSpecificationResult modifyInstanceCreditSpecification(
-        ModifyInstanceCreditSpecificationRequest modifyInstanceCreditSpecificationRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyInstancePlacementResult modifyInstancePlacement(ModifyInstancePlacementRequest modifyInstancePlacementRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyLaunchTemplateResult modifyLaunchTemplate(ModifyLaunchTemplateRequest modifyLaunchTemplateRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyIdentityIdFormatResult modifyIdentityIdFormat(ModifyIdentityIdFormatRequest modifyIdentityIdFormatRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CancelReservedInstancesListingResult cancelReservedInstancesListing(
-            CancelReservedInstancesListingRequest cancelReservedInstancesListingRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteDhcpOptionsResult deleteDhcpOptions(DeleteDhcpOptionsRequest deleteDhcpOptionsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteEgressOnlyInternetGatewayResult deleteEgressOnlyInternetGateway(
-            DeleteEgressOnlyInternetGatewayRequest deleteEgressOnlyInternetGatewayRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteFleetsResult deleteFleets(DeleteFleetsRequest deleteFleetsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteNetworkInterfacePermissionResult deleteNetworkInterfacePermission(
-            DeleteNetworkInterfacePermissionRequest deleteNetworkInterfacePermissionRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AuthorizeSecurityGroupIngressResult authorizeSecurityGroupIngress(
-            AuthorizeSecurityGroupIngressRequest authorizeSecurityGroupIngressRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSpotInstanceRequestsResult describeSpotInstanceRequests(
-            DescribeSpotInstanceRequestsRequest describeSpotInstanceRequestsRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateVpcResult createVpc(CreateVpcRequest createVpcRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeCustomerGatewaysResult describeCustomerGateways(DescribeCustomerGatewaysRequest describeCustomerGatewaysRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CancelExportTaskResult cancelExportTask(CancelExportTaskRequest cancelExportTaskRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateRouteResult createRoute(CreateRouteRequest createRouteRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateVpcEndpointResult createVpcEndpoint(CreateVpcEndpointRequest createVpcEndpointRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateVpcEndpointConnectionNotificationResult createVpcEndpointConnectionNotification(
-        CreateVpcEndpointConnectionNotificationRequest createVpcEndpointConnectionNotificationRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateVpcEndpointServiceConfigurationResult createVpcEndpointServiceConfiguration(
-        CreateVpcEndpointServiceConfigurationRequest createVpcEndpointServiceConfigurationRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CopyImageResult copyImage(CopyImageRequest copyImageRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpcClassicLinkResult describeVpcClassicLink(DescribeVpcClassicLinkRequest describeVpcClassicLinkRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyNetworkInterfaceAttributeResult modifyNetworkInterfaceAttribute(
-            ModifyNetworkInterfaceAttributeRequest modifyNetworkInterfaceAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteRouteTableResult deleteRouteTable(DeleteRouteTableRequest deleteRouteTableRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeNetworkInterfaceAttributeResult describeNetworkInterfaceAttribute(
-            DescribeNetworkInterfaceAttributeRequest describeNetworkInterfaceAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeClassicLinkInstancesResult describeClassicLinkInstances(
-            DescribeClassicLinkInstancesRequest describeClassicLinkInstancesRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public RequestSpotInstancesResult requestSpotInstances(RequestSpotInstancesRequest requestSpotInstancesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ResetFpgaImageAttributeResult resetFpgaImageAttribute(ResetFpgaImageAttributeRequest resetFpgaImageAttributeRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateTagsResult createTags(CreateTagsRequest createTagsRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateTransitGatewayResult createTransitGateway(CreateTransitGatewayRequest createTransitGatewayRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateTransitGatewayRouteResult createTransitGatewayRoute(CreateTransitGatewayRouteRequest createTransitGatewayRouteRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateTransitGatewayRouteTableResult createTransitGatewayRouteTable(
-        CreateTransitGatewayRouteTableRequest createTransitGatewayRouteTableRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateTransitGatewayVpcAttachmentResult createTransitGatewayVpcAttachment(
-        CreateTransitGatewayVpcAttachmentRequest createTransitGatewayVpcAttachmentRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVolumeAttributeResult describeVolumeAttribute(DescribeVolumeAttributeRequest describeVolumeAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AttachNetworkInterfaceResult attachNetworkInterface(AttachNetworkInterfaceRequest attachNetworkInterfaceRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ReplaceRouteResult replaceRoute(ReplaceRouteRequest replaceRouteRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeTagsResult describeTags(DescribeTagsRequest describeTagsRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CancelBundleTaskResult cancelBundleTask(CancelBundleTaskRequest cancelBundleTaskRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CancelCapacityReservationResult cancelCapacityReservation(CancelCapacityReservationRequest cancelCapacityReservationRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DisableVgwRoutePropagationResult disableVgwRoutePropagation(DisableVgwRoutePropagationRequest disableVgwRoutePropagationRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ImportSnapshotResult importSnapshot(ImportSnapshotRequest importSnapshotRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CancelSpotInstanceRequestsResult cancelSpotInstanceRequests(CancelSpotInstanceRequestsRequest cancelSpotInstanceRequestsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSpotFleetRequestsResult describeSpotFleetRequests(DescribeSpotFleetRequestsRequest describeSpotFleetRequestsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public PurchaseReservedInstancesOfferingResult purchaseReservedInstancesOffering(
-            PurchaseReservedInstancesOfferingRequest purchaseReservedInstancesOfferingRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public PurchaseScheduledInstancesResult purchaseScheduledInstances(
-            PurchaseScheduledInstancesRequest purchaseScheduledInstancesRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public PurchaseHostReservationResult purchaseHostReservation(PurchaseHostReservationRequest purchaseHostReservationRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifySnapshotAttributeResult modifySnapshotAttribute(ModifySnapshotAttributeRequest modifySnapshotAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeReservedInstancesModificationsResult describeReservedInstancesModifications(
-            DescribeReservedInstancesModificationsRequest describeReservedInstancesModificationsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public TerminateInstancesResult terminateInstances(TerminateInstancesRequest terminateInstancesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyVpcEndpointResult modifyVpcEndpoint(ModifyVpcEndpointRequest modifyVpcEndpointRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyVpcEndpointConnectionNotificationResult modifyVpcEndpointConnectionNotification(
-        ModifyVpcEndpointConnectionNotificationRequest modifyVpcEndpointConnectionNotificationRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyVpcEndpointServiceConfigurationResult modifyVpcEndpointServiceConfiguration(
-        ModifyVpcEndpointServiceConfigurationRequest modifyVpcEndpointServiceConfigurationRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyVpcEndpointServicePermissionsResult modifyVpcEndpointServicePermissions(
-        ModifyVpcEndpointServicePermissionsRequest modifyVpcEndpointServicePermissionsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteSpotDatafeedSubscriptionResult deleteSpotDatafeedSubscription(
-            DeleteSpotDatafeedSubscriptionRequest deleteSpotDatafeedSubscriptionRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteInternetGatewayResult deleteInternetGateway(DeleteInternetGatewayRequest deleteInternetGatewayRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSnapshotAttributeResult describeSnapshotAttribute(DescribeSnapshotAttributeRequest describeSnapshotAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ReplaceRouteTableAssociationResult replaceRouteTableAssociation(
-            ReplaceRouteTableAssociationRequest replaceRouteTableAssociationRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ReplaceTransitGatewayRouteResult replaceTransitGatewayRoute(
-        ReplaceTransitGatewayRouteRequest replaceTransitGatewayRouteRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeAddressesResult describeAddresses(DescribeAddressesRequest describeAddressesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeImageAttributeResult describeImageAttribute(DescribeImageAttributeRequest describeImageAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeKeyPairsResult describeKeyPairs(DescribeKeyPairsRequest describeKeyPairsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ConfirmProductInstanceResult confirmProductInstance(ConfirmProductInstanceRequest confirmProductInstanceRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CopyFpgaImageResult copyFpgaImage(CopyFpgaImageRequest copyFpgaImageRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DisassociateRouteTableResult disassociateRouteTable(DisassociateRouteTableRequest disassociateRouteTableRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DisassociateIamInstanceProfileResult disassociateIamInstanceProfile(
-            DisassociateIamInstanceProfileRequest disassociateIamInstanceProfileRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DisassociateVpcCidrBlockResult disassociateVpcCidrBlock(DisassociateVpcCidrBlockRequest disassociateVpcCidrBlockRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public EnableTransitGatewayRouteTablePropagationResult enableTransitGatewayRouteTablePropagation(
-        EnableTransitGatewayRouteTablePropagationRequest enableTransitGatewayRouteTablePropagationRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DisassociateSubnetCidrBlockResult disassociateSubnetCidrBlock(
-            DisassociateSubnetCidrBlockRequest disassociateSubnetCidrBlockRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DisassociateTransitGatewayRouteTableResult disassociateTransitGatewayRouteTable(
-        DisassociateTransitGatewayRouteTableRequest disassociateTransitGatewayRouteTableRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpcAttributeResult describeVpcAttribute(DescribeVpcAttributeRequest describeVpcAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public RevokeSecurityGroupEgressResult revokeSecurityGroupEgress(RevokeSecurityGroupEgressRequest revokeSecurityGroupEgressRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteNetworkAclEntryResult deleteNetworkAclEntry(DeleteNetworkAclEntryRequest deleteNetworkAclEntryRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateVolumeResult createVolume(CreateVolumeRequest createVolumeRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyVolumeResult modifyVolume(ModifyVolumeRequest modifyVolumeRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeInstanceStatusResult describeInstanceStatus(DescribeInstanceStatusRequest describeInstanceStatusRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpnGatewaysResult describeVpnGateways(DescribeVpnGatewaysRequest describeVpnGatewaysRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateSubnetResult createSubnet(CreateSubnetRequest createSubnetRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeReservedInstancesOfferingsResult describeReservedInstancesOfferings(
-            DescribeReservedInstancesOfferingsRequest describeReservedInstancesOfferingsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AssignPrivateIpAddressesResult assignPrivateIpAddresses(AssignPrivateIpAddressesRequest assignPrivateIpAddressesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AssignIpv6AddressesResult assignIpv6Addresses(AssignIpv6AddressesRequest assignIpv6AddressesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSpotFleetRequestHistoryResult describeSpotFleetRequestHistory(
-            DescribeSpotFleetRequestHistoryRequest describeSpotFleetRequestHistoryRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteSnapshotResult deleteSnapshot(DeleteSnapshotRequest deleteSnapshotRequest) 
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ReplaceNetworkAclAssociationResult replaceNetworkAclAssociation(
-            ReplaceNetworkAclAssociationRequest replaceNetworkAclAssociationRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DisassociateAddressResult disassociateAddress(DisassociateAddressRequest disassociateAddressRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DisassociateClientVpnTargetNetworkResult disassociateClientVpnTargetNetwork(
-        DisassociateClientVpnTargetNetworkRequest disassociateClientVpnTargetNetworkRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreatePlacementGroupResult createPlacementGroup(CreatePlacementGroupRequest createPlacementGroupRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public BundleInstanceResult bundleInstance(BundleInstanceRequest bundleInstanceRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeletePlacementGroupResult deletePlacementGroup(DeletePlacementGroupRequest deletePlacementGroupRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifySubnetAttributeResult modifySubnetAttribute(ModifySubnetAttributeRequest modifySubnetAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifyTransitGatewayVpcAttachmentResult modifyTransitGatewayVpcAttachment(
-        ModifyTransitGatewayVpcAttachmentRequest modifyTransitGatewayVpcAttachmentRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteVpcResult deleteVpc(DeleteVpcRequest deleteVpcRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteVpcEndpointConnectionNotificationsResult deleteVpcEndpointConnectionNotifications(
-        DeleteVpcEndpointConnectionNotificationsRequest deleteVpcEndpointConnectionNotificationsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteVpcEndpointServiceConfigurationsResult deleteVpcEndpointServiceConfigurations(
-        DeleteVpcEndpointServiceConfigurationsRequest deleteVpcEndpointServiceConfigurationsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CopySnapshotResult copySnapshot(CopySnapshotRequest copySnapshotRequest) throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateCapacityReservationResult createCapacityReservation(CreateCapacityReservationRequest createCapacityReservationRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateClientVpnEndpointResult createClientVpnEndpoint(CreateClientVpnEndpointRequest createClientVpnEndpointRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateClientVpnRouteResult createClientVpnRoute(CreateClientVpnRouteRequest createClientVpnRouteRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpcEndpointServicesResult describeVpcEndpointServices(
-            DescribeVpcEndpointServicesRequest describeVpcEndpointServicesRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AllocateAddressResult allocateAddress(AllocateAddressRequest allocateAddressRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ReleaseAddressResult releaseAddress(ReleaseAddressRequest releaseAddressRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ReleaseHostsResult releaseHosts(ReleaseHostsRequest releaseHostsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ReplaceIamInstanceProfileAssociationResult replaceIamInstanceProfileAssociation(
-            ReplaceIamInstanceProfileAssociationRequest replaceIamInstanceProfileAssociationRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ResetInstanceAttributeResult resetInstanceAttribute(ResetInstanceAttributeRequest resetInstanceAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateKeyPairResult createKeyPair(CreateKeyPairRequest createKeyPairRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateLaunchTemplateResult createLaunchTemplate(CreateLaunchTemplateRequest createLaunchTemplateRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateLaunchTemplateVersionResult createLaunchTemplateVersion(
-        CreateLaunchTemplateVersionRequest createLaunchTemplateVersionRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateNatGatewayResult createNatGateway(CreateNatGatewayRequest createNatGatewayRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ReplaceNetworkAclEntryResult replaceNetworkAclEntry(ReplaceNetworkAclEntryRequest replaceNetworkAclEntryRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSnapshotsResult describeSnapshots(DescribeSnapshotsRequest describeSnapshotsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateNetworkAclResult createNetworkAcl(CreateNetworkAclRequest createNetworkAclRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public RegisterImageResult registerImage(RegisterImageRequest registerImageRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public RejectTransitGatewayVpcAttachmentResult rejectTransitGatewayVpcAttachment(
-        RejectTransitGatewayVpcAttachmentRequest rejectTransitGatewayVpcAttachmentRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public RejectVpcEndpointConnectionsResult rejectVpcEndpointConnections(
-        RejectVpcEndpointConnectionsRequest rejectVpcEndpointConnectionsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ResetNetworkInterfaceAttributeResult resetNetworkInterfaceAttribute(
-            ResetNetworkInterfaceAttributeRequest resetNetworkInterfaceAttributeRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public EnableVpcClassicLinkResult enableVpcClassicLink(EnableVpcClassicLinkRequest enableVpcClassicLinkRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public EnableVpcClassicLinkDnsSupportResult enableVpcClassicLinkDnsSupport(
-            EnableVpcClassicLinkDnsSupportRequest enableVpcClassicLinkDnsSupportRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ExportClientVpnClientCertificateRevocationListResult exportClientVpnClientCertificateRevocationList(
-        ExportClientVpnClientCertificateRevocationListRequest exportClientVpnClientCertificateRevocationListRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ExportClientVpnClientConfigurationResult exportClientVpnClientConfiguration(
-        ExportClientVpnClientConfigurationRequest exportClientVpnClientConfigurationRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ExportTransitGatewayRoutesResult exportTransitGatewayRoutes(
-        ExportTransitGatewayRoutesRequest exportTransitGatewayRoutesRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateVpnConnectionRouteResult createVpnConnectionRoute(CreateVpnConnectionRouteRequest createVpnConnectionRouteRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpcEndpointsResult describeVpcEndpoints(DescribeVpcEndpointsRequest describeVpcEndpointsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DetachClassicLinkVpcResult detachClassicLinkVpc(DetachClassicLinkVpcRequest detachClassicLinkVpcRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeReservedInstancesResult describeReservedInstances() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeAvailabilityZonesResult describeAvailabilityZones() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSpotPriceHistoryResult describeSpotPriceHistory() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeNetworkInterfacesResult describeNetworkInterfaces() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeRegionsResult describeRegions() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeInternetGatewaysResult describeInternetGateways() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSecurityGroupsResult describeSecurityGroups() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSpotDatafeedSubscriptionResult describeSpotDatafeedSubscription() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeAccountAttributesResult describeAccountAttributes() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVolumeStatusResult describeVolumeStatus() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeImportSnapshotTasksResult describeImportSnapshotTasks() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpnConnectionsResult describeVpnConnections() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpcsResult describeVpcs() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AcceptVpcPeeringConnectionResult acceptVpcPeeringConnection() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AdvertiseByoipCidrResult advertiseByoipCidr(AdvertiseByoipCidrRequest advertiseByoipCidrRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeExportTasksResult describeExportTasks() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeFleetHistoryResult describeFleetHistory(DescribeFleetHistoryRequest describeFleetHistoryRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeFleetInstancesResult describeFleetInstances(DescribeFleetInstancesRequest describeFleetInstancesRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeFleetsResult describeFleets(DescribeFleetsRequest describeFleetsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateVpcPeeringConnectionResult createVpcPeeringConnection() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CancelImportTaskResult cancelImportTask() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVolumesResult describeVolumes() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeReservedInstancesListingsResult describeReservedInstancesListings()
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeRouteTablesResult describeRouteTables() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeScheduledInstanceAvailabilityResult describeScheduledInstanceAvailability(
-            DescribeScheduledInstanceAvailabilityRequest describeScheduledInstanceAvailabilityRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeScheduledInstancesResult describeScheduledInstances(
-            DescribeScheduledInstancesRequest describeScheduledInstancesRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeDhcpOptionsResult describeDhcpOptions() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribePrefixListsResult describePrefixLists() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribePrincipalIdFormatResult describePrincipalIdFormat(DescribePrincipalIdFormatRequest describePrincipalIdFormatRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribePublicIpv4PoolsResult describePublicIpv4Pools(DescribePublicIpv4PoolsRequest describePublicIpv4PoolsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeImportImageTasksResult describeImportImageTasks() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeNetworkAclsResult describeNetworkAcls() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeBundleTasksResult describeBundleTasks() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeByoipCidrsResult describeByoipCidrs(DescribeByoipCidrsRequest describeByoipCidrsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeCapacityReservationsResult describeCapacityReservations(
-        DescribeCapacityReservationsRequest describeCapacityReservationsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public RevokeSecurityGroupIngressResult revokeSecurityGroupIngress(RevokeSecurityGroupIngressRequest revokeSecurityGroupIngressRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public RevokeSecurityGroupIngressResult revokeSecurityGroupIngress() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public CreateInternetGatewayResult createInternetGateway() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeMovingAddressesResult describeMovingAddresses() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeNatGatewaysResult describeNatGateways(DescribeNatGatewaysRequest describeNatGatewaysRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeConversionTasksResult describeConversionTasks() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ImportImageResult importImage() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeFlowLogsResult describeFlowLogs() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeFpgaImageAttributeResult describeFpgaImageAttribute(
-        DescribeFpgaImageAttributeRequest describeFpgaImageAttributeRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeHostsResult describeHosts(DescribeHostsRequest describeHostsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeHostsResult describeHosts() {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeIamInstanceProfileAssociationsResult describeIamInstanceProfileAssociations(
-            DescribeIamInstanceProfileAssociationsRequest describeIamInstanceProfileAssociationsRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeIdFormatResult describeIdFormat(DescribeIdFormatRequest describeIdFormatRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeIdFormatResult describeIdFormat() {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpcPeeringConnectionsResult describeVpcPeeringConnections() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribePlacementGroupsResult describePlacementGroups() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSubnetsResult describeSubnets() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeInstancesResult describeInstances() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeImagesResult describeImages() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSpotInstanceRequestsResult describeSpotInstanceRequests() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeCustomerGatewaysResult describeCustomerGateways() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpcClassicLinkResult describeVpcClassicLink() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpcClassicLinkDnsSupportResult describeVpcClassicLinkDnsSupport(
-            DescribeVpcClassicLinkDnsSupportRequest describeVpcClassicLinkDnsSupportRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpcEndpointConnectionNotificationsResult describeVpcEndpointConnectionNotifications(
-        DescribeVpcEndpointConnectionNotificationsRequest describeVpcEndpointConnectionNotificationsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpcEndpointConnectionsResult describeVpcEndpointConnections(
-        DescribeVpcEndpointConnectionsRequest describeVpcEndpointConnectionsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpcEndpointServiceConfigurationsResult describeVpcEndpointServiceConfigurations(
-        DescribeVpcEndpointServiceConfigurationsRequest describeVpcEndpointServiceConfigurationsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpcEndpointServicePermissionsResult describeVpcEndpointServicePermissions(
-        DescribeVpcEndpointServicePermissionsRequest describeVpcEndpointServicePermissionsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeClassicLinkInstancesResult describeClassicLinkInstances() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeClientVpnAuthorizationRulesResult describeClientVpnAuthorizationRules(
-        DescribeClientVpnAuthorizationRulesRequest describeClientVpnAuthorizationRulesRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeClientVpnConnectionsResult describeClientVpnConnections(
-        DescribeClientVpnConnectionsRequest describeClientVpnConnectionsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeClientVpnEndpointsResult describeClientVpnEndpoints(
-        DescribeClientVpnEndpointsRequest describeClientVpnEndpointsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeClientVpnRoutesResult describeClientVpnRoutes(
-        DescribeClientVpnRoutesRequest describeClientVpnRoutesRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeClientVpnTargetNetworksResult describeClientVpnTargetNetworks(
-        DescribeClientVpnTargetNetworksRequest describeClientVpnTargetNetworksRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeTagsResult describeTags() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeTransitGatewayAttachmentsResult describeTransitGatewayAttachments(
-        DescribeTransitGatewayAttachmentsRequest describeTransitGatewayAttachmentsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeTransitGatewayRouteTablesResult describeTransitGatewayRouteTables(
-        DescribeTransitGatewayRouteTablesRequest describeTransitGatewayRouteTablesRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeTransitGatewayVpcAttachmentsResult describeTransitGatewayVpcAttachments(
-        DescribeTransitGatewayVpcAttachmentsRequest describeTransitGatewayVpcAttachmentsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeTransitGatewaysResult describeTransitGateways(DescribeTransitGatewaysRequest describeTransitGatewaysRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ImportSnapshotResult importSnapshot() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSpotFleetRequestsResult describeSpotFleetRequests() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeReservedInstancesModificationsResult describeReservedInstancesModifications()
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DeleteSpotDatafeedSubscriptionResult deleteSpotDatafeedSubscription() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeAddressesResult describeAddresses() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeAggregateIdFormatResult describeAggregateIdFormat(DescribeAggregateIdFormatRequest describeAggregateIdFormatRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeKeyPairsResult describeKeyPairs() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeLaunchTemplateVersionsResult describeLaunchTemplateVersions(
-        DescribeLaunchTemplateVersionsRequest describeLaunchTemplateVersionsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeLaunchTemplatesResult describeLaunchTemplates(DescribeLaunchTemplatesRequest describeLaunchTemplatesRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeInstanceStatusResult describeInstanceStatus() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpnGatewaysResult describeVpnGateways() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeReservedInstancesOfferingsResult describeReservedInstancesOfferings()
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpcEndpointServicesResult describeVpcEndpointServices() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AllocateAddressResult allocateAddress() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public AllocateHostsResult allocateHosts(AllocateHostsRequest allocateHostsRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ApplySecurityGroupsToClientVpnTargetNetworkResult applySecurityGroupsToClientVpnTargetNetwork(
-        ApplySecurityGroupsToClientVpnTargetNetworkRequest applySecurityGroupsToClientVpnTargetNetworkRequest) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeSnapshotsResult describeSnapshots() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public DescribeVpcEndpointsResult describeVpcEndpoints() throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public <X extends AmazonWebServiceRequest> DryRunResult<X> dryRun(DryRunSupportedRequest<X> request)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
     @Override
     public void shutdown() {
     }
-
-    @Override
-    public AmazonEC2Waiters waiters() {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ResponseMetadata getCachedResponseMetadata(AmazonWebServiceRequest request) {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
-
-    @Override
-    public ModifySpotFleetRequestResult modifySpotFleetRequest(ModifySpotFleetRequestRequest modifySpotFleetRequestRequest)
-            throws AmazonClientException {
-        throw new UnsupportedOperationException("Not supported in mock");
-    }
 }
diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle
index 697cc3780a1fd..25baa4b17ce4a 100644
--- a/plugins/discovery-gce/build.gradle
+++ b/plugins/discovery-gce/build.gradle
@@ -17,6 +17,7 @@ dependencies {
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
+  compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
 }
 
diff --git a/plugins/discovery-gce/licenses/log4j-1.2-api-2.11.1.jar.sha1 b/plugins/discovery-gce/licenses/log4j-1.2-api-2.11.1.jar.sha1
new file mode 100644
index 0000000000000..575d75dbda8c5
--- /dev/null
+++ b/plugins/discovery-gce/licenses/log4j-1.2-api-2.11.1.jar.sha1
@@ -0,0 +1 @@
+3aba3398fe064a3eab4331f88161c7480e848418
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/log4j-LICENSE.txt b/plugins/discovery-gce/licenses/log4j-LICENSE.txt
new file mode 100644
index 0000000000000..6279e5206de13
--- /dev/null
+++ b/plugins/discovery-gce/licenses/log4j-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 1999-2005 The Apache Software Foundation
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/discovery-gce/licenses/log4j-NOTICE.txt b/plugins/discovery-gce/licenses/log4j-NOTICE.txt
new file mode 100644
index 0000000000000..0375732360047
--- /dev/null
+++ b/plugins/discovery-gce/licenses/log4j-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache log4j
+Copyright 2007 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
\ No newline at end of file
diff --git a/plugins/discovery-gce/qa/gce/build.gradle b/plugins/discovery-gce/qa/gce/build.gradle
index 16a65c762f6f7..b0231ff7f3c90 100644
--- a/plugins/discovery-gce/qa/gce/build.gradle
+++ b/plugins/discovery-gce/qa/gce/build.gradle
@@ -17,9 +17,12 @@
  * under the License.
  */
 
+
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.test.AntFixture
 
+import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
+
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -54,13 +57,13 @@ testClusters.integTest {
     numberOfNodes = gceNumberOfNodes
     plugin file(project(':plugins:discovery-gce').bundlePlugin.archiveFile)
     // use gce fixture for Auth calls instead of http://metadata.google.internal
-    environment 'GCE_METADATA_HOST', { "http://${gceFixture.addressAndPort}" }
+    environment 'GCE_METADATA_HOST', { "http://${gceFixture.addressAndPort}" }, IGNORE_VALUE
     // allows to configure hidden settings (`cloud.gce.host` and `cloud.gce.root_url`)
     systemProperty 'es.allow_reroute_gce_settings', 'true'
 
     setting 'discovery.seed_providers', 'gce'
     // use gce fixture for metadata server calls instead of http://metadata.google.internal
-    setting 'cloud.gce.host', { "http://${gceFixture.addressAndPort}" }
+    setting 'cloud.gce.host', { "http://${gceFixture.addressAndPort}" }, IGNORE_VALUE
     // use gce fixture for API calls instead of https://www.googleapis.com
-    setting 'cloud.gce.root_url', { "http://${gceFixture.addressAndPort}" }
+    setting 'cloud.gce.root_url', { "http://${gceFixture.addressAndPort}" }, IGNORE_VALUE
 }
diff --git a/plugins/examples/build.gradle b/plugins/examples/build.gradle
index 2b9f3c6433d8f..a251c1bbbe85d 100644
--- a/plugins/examples/build.gradle
+++ b/plugins/examples/build.gradle
@@ -7,3 +7,22 @@ gradle.projectsEvaluated {
     }
   }
 }
+
+configure(project('painless-whitelist')) {
+    configurations.all {
+       resolutionStrategy.dependencySubstitution {
+            substitute module('org.elasticsearch.plugin:elasticsearch-scripting-painless-spi') with project(':modules:lang-painless:spi')
+            substitute module('org.elasticsearch.test:logger-usage') with project(':test:logger-usage')
+        }
+    }
+}
+
+configure(project('security-authorization-engine')) {
+    configurations.all {
+        resolutionStrategy.dependencySubstitution {
+            substitute module('org.elasticsearch.plugin:x-pack-core') with project(':x-pack:plugin:core')
+            substitute module('org.elasticsearch.client:elasticsearch-rest-high-level-client') with project(':client:rest-high-level')
+            substitute module('org.elasticsearch.test:logger-usage') with project(':test:logger-usage')
+        }
+    }
+}
\ No newline at end of file
diff --git a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java
index b6a5b5e8f842f..9220206903eb5 100644
--- a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java
+++ b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.Suggester;
 
+import java.io.IOException;
 import java.util.Locale;
 
 public class CustomSuggester extends Suggester<CustomSuggestionContext> {
@@ -35,15 +36,12 @@ public Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.S
         String name,
         CustomSuggestionContext suggestion,
         IndexSearcher searcher,
-        CharsRefBuilder spare) {
-
-        // Get the suggestion context
-        String text = suggestion.getText().utf8ToString();
+        CharsRefBuilder spare) throws IOException {
 
         // create two suggestions with 12 and 123 appended
-        CustomSuggestion response = new CustomSuggestion(name, suggestion.getSize(), "suggestion-dummy-value");
-
-        CustomSuggestion.Entry entry = new CustomSuggestion.Entry(new Text(text), 0, text.length(), "entry-dummy-value");
+        CustomSuggestion response = emptySuggestion(name, suggestion, spare);
+        CustomSuggestion.Entry entry = response.getEntries().get(0);
+        String text = entry.getText().string();
 
         String firstOption =
             String.format(Locale.ROOT, "%s-%s-%s-%s", text, suggestion.getField(), suggestion.options.get("suffix"), "12");
@@ -55,8 +53,16 @@ public Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.S
         CustomSuggestion.Entry.Option option123 = new CustomSuggestion.Entry.Option(new Text(secondOption), 0.8f, "option-dummy-value-2");
         entry.addOption(option123);
 
-        response.addTerm(entry);
+        return response;
+    }
 
+    @Override
+    protected CustomSuggestion emptySuggestion(String name, CustomSuggestionContext suggestion,
+            CharsRefBuilder spare) throws IOException {
+        String text = suggestion.getText().utf8ToString();
+        CustomSuggestion response = new CustomSuggestion(name, suggestion.getSize(), "suggestion-dummy-value");
+        CustomSuggestion.Entry entry = new CustomSuggestion.Entry(new Text(text), 0, text.length(), "entry-dummy-value");
+        response.addTerm(entry);
         return response;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/AliasesExistRequestBuilder.java b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExamplePainlessAnnotation.java
similarity index 62%
rename from server/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/AliasesExistRequestBuilder.java
rename to plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExamplePainlessAnnotation.java
index d889d15ce9c70..701f18f7720ad 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/AliasesExistRequestBuilder.java
+++ b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExamplePainlessAnnotation.java
@@ -17,14 +17,25 @@
  * under the License.
  */
 
-package org.elasticsearch.action.admin.indices.alias.exists;
+package org.elasticsearch.example.painlesswhitelist;
 
-import org.elasticsearch.action.admin.indices.alias.get.BaseAliasesRequestBuilder;
-import org.elasticsearch.client.ElasticsearchClient;
+public class ExamplePainlessAnnotation {
 
-public class AliasesExistRequestBuilder extends BaseAliasesRequestBuilder<AliasesExistResponse, AliasesExistRequestBuilder> {
+    public static final String NAME = "example_annotation";
 
-    public AliasesExistRequestBuilder(ElasticsearchClient client, AliasesExistAction action, String... aliases) {
-        super(client, action, aliases);
+    public int category;
+    public String message;
+
+    public ExamplePainlessAnnotation(int category, String message) {
+        this.category = category;
+        this.message = message;
+    }
+
+    public int getCategory() {
+        return category;
+    }
+
+    public String getMessage() {
+        return message;
     }
 }
diff --git a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistAnnotationParser.java b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistAnnotationParser.java
new file mode 100644
index 0000000000000..b0a5edd29cffc
--- /dev/null
+++ b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistAnnotationParser.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example.painlesswhitelist;
+
+import org.elasticsearch.painless.spi.annotation.WhitelistAnnotationParser;
+
+import java.util.Map;
+
+public class ExampleWhitelistAnnotationParser implements WhitelistAnnotationParser {
+
+    public static final ExampleWhitelistAnnotationParser INSTANCE = new ExampleWhitelistAnnotationParser();
+
+    private ExampleWhitelistAnnotationParser() {
+
+    }
+
+    @Override
+    public Object parse(Map<String, String> arguments) {
+        if (arguments.size() != 2) {
+            throw new IllegalArgumentException("expected exactly two arguments");
+        }
+
+        String categoryString = arguments.get("category");
+
+        if (categoryString == null) {
+            throw new IllegalArgumentException("expected category argument");
+        }
+
+        int category;
+
+        try {
+            category = Integer.parseInt(categoryString);
+        } catch (NumberFormatException nfe) {
+            throw new IllegalArgumentException("expected category as an int, found [" + categoryString + "]", nfe);
+        }
+
+        String message = arguments.get("message");
+
+        if (categoryString == null) {
+            throw new IllegalArgumentException("expected message argument");
+        }
+
+        return new ExamplePainlessAnnotation(category, message);
+    }
+}
diff --git a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistExtension.java b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistExtension.java
index d0b03708a08b3..2ad4cb3d4de97 100644
--- a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistExtension.java
+++ b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistExtension.java
@@ -23,30 +23,34 @@
 import org.elasticsearch.painless.spi.Whitelist;
 import org.elasticsearch.painless.spi.WhitelistInstanceBinding;
 import org.elasticsearch.painless.spi.WhitelistLoader;
+import org.elasticsearch.painless.spi.annotation.WhitelistAnnotationParser;
 import org.elasticsearch.script.FieldScript;
 import org.elasticsearch.script.ScriptContext;
 
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 /** An extension of painless which adds a whitelist. */
 public class ExampleWhitelistExtension implements PainlessExtension {
 
-    private static final Whitelist WHITELIST =
-        WhitelistLoader.loadFromResourceFiles(ExampleWhitelistExtension.class, "example_whitelist.txt");
-
     @Override
     public Map<ScriptContext<?>, List<Whitelist>> getContextWhitelists() {
+        Map<String, WhitelistAnnotationParser> parsers = new HashMap<>(WhitelistAnnotationParser.BASE_ANNOTATION_PARSERS);
+        parsers.put(ExamplePainlessAnnotation.NAME, ExampleWhitelistAnnotationParser.INSTANCE);
+        Whitelist classWhitelist =
+                WhitelistLoader.loadFromResourceFiles(ExampleWhitelistExtension.class, parsers, "example_whitelist.txt");
+
         ExampleWhitelistedInstance ewi = new ExampleWhitelistedInstance(1);
         WhitelistInstanceBinding addValue = new WhitelistInstanceBinding("example addValue", ewi,
-            "addValue", "int", Collections.singletonList("int"));
+            "addValue", "int", Collections.singletonList("int"), Collections.emptyList());
         WhitelistInstanceBinding getValue = new WhitelistInstanceBinding("example getValue", ewi,
-            "getValue", "int", Collections.emptyList());
+            "getValue", "int", Collections.emptyList(), Collections.emptyList());
         Whitelist instanceWhitelist = new Whitelist(ewi.getClass().getClassLoader(), Collections.emptyList(),
             Collections.emptyList(), Collections.emptyList(), Arrays.asList(addValue, getValue));
 
-        return Collections.singletonMap(FieldScript.CONTEXT, Arrays.asList(WHITELIST, instanceWhitelist));
+        return Collections.singletonMap(FieldScript.CONTEXT, Arrays.asList(classWhitelist, instanceWhitelist));
     }
 }
diff --git a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistedClass.java b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistedClass.java
index 14f15b383d0c8..bb0e71f3dc870 100644
--- a/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistedClass.java
+++ b/plugins/examples/painless-whitelist/src/main/java/org/elasticsearch/example/painlesswhitelist/ExampleWhitelistedClass.java
@@ -54,4 +54,9 @@ public static void staticMethod() {
     public static int toInt(String x) {
         return Integer.parseInt(x);
     }
+
+    // example method to attach annotations in whitelist
+    public void annotate() {
+        // some logic here
+    }
 }
diff --git a/plugins/examples/painless-whitelist/src/main/resources/org/elasticsearch/example/painlesswhitelist/example_whitelist.txt b/plugins/examples/painless-whitelist/src/main/resources/org/elasticsearch/example/painlesswhitelist/example_whitelist.txt
index 99e5521aebf36..9746b968d4df1 100644
--- a/plugins/examples/painless-whitelist/src/main/resources/org/elasticsearch/example/painlesswhitelist/example_whitelist.txt
+++ b/plugins/examples/painless-whitelist/src/main/resources/org/elasticsearch/example/painlesswhitelist/example_whitelist.txt
@@ -33,6 +33,9 @@ class org.elasticsearch.example.painlesswhitelist.ExampleWhitelistedClass {
   # getter and setter for private member
   int getPrivateMemberAccessor()
   void setPrivateMemberAccessor(int)
+
+  # annotation
+  void annotate() @example_annotation[category="1",message="example annotation"]
 }
 
 class java.lang.String {
diff --git a/plugins/examples/rest-handler/src/test/resources/rest-api-spec/api/cat.example.json b/plugins/examples/rest-handler/src/test/resources/rest-api-spec/api/cat.example.json
index e42617eef7bf7..2aa5edd4d5ae1 100644
--- a/plugins/examples/rest-handler/src/test/resources/rest-api-spec/api/cat.example.json
+++ b/plugins/examples/rest-handler/src/test/resources/rest-api-spec/api/cat.example.json
@@ -1,9 +1,9 @@
 {
   "cat.example": {
     "documentation": "",
+    "stability" : "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/example",
       "paths": ["/_cat/example"],
       "parts": {},
       "params": {
diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
index aa0dbe07c8305..969d91c2d6b5d 100644
--- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
+++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
@@ -226,7 +226,7 @@ public void testAgainstTermVectorsAPI() throws IOException {
         }
         bulk.get();
 
-        TermVectorsRequest request = new TermVectorsRequest("test", "type", "0").termStatistics(true);
+        TermVectorsRequest request = new TermVectorsRequest("test", "0").termStatistics(true);
 
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService test = indicesService.indexService(resolveIndex("test"));
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index a7c1af412d949..47ee57d3a3e78 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -71,3 +71,26 @@ testClusters {
         keystore 'azure.client.integration_test.key', 'azure_key'
     }
 }
+
+String azureAccount = System.getenv("azure_storage_account")
+String azureKey = System.getenv("azure_storage_key")
+String azureContainer = System.getenv("azure_storage_container")
+String azureBasePath = System.getenv("azure_storage_base_path")
+String azureSasToken = System.getenv("azure_storage_sas_token")
+
+test {
+  exclude '**/AzureStorageCleanupThirdPartyTests.class'
+}
+
+task thirdPartyTest(type: Test) {
+  include '**/AzureStorageCleanupThirdPartyTests.class'
+  systemProperty 'test.azure.account', azureAccount ? azureAccount : ""
+  systemProperty 'test.azure.key', azureKey ? azureKey : ""
+  systemProperty 'test.azure.sas_token', azureSasToken ? azureSasToken : ""
+  systemProperty 'test.azure.container', azureContainer ? azureContainer : ""
+  systemProperty 'test.azure.base', azureBasePath ? azureBasePath : ""
+}
+
+if (azureAccount || azureKey || azureContainer || azureBasePath || azureSasToken) {
+  check.dependsOn(thirdPartyTest)
+}
diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle
index 8d8aaebac9f3f..5f96fd9dc305b 100644
--- a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle
+++ b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle
@@ -17,9 +17,13 @@
  * under the License.
  */
 
+
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.test.AntFixture
 
+import static org.elasticsearch.gradle.PropertyNormalization.DEFAULT
+import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
+
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -29,12 +33,14 @@ String azureAccount = System.getenv("azure_storage_account")
 String azureKey = System.getenv("azure_storage_key")
 String azureContainer = System.getenv("azure_storage_container")
 String azureBasePath = System.getenv("azure_storage_base_path")
+String azureSasToken = System.getenv("azure_storage_sas_token")
 
-if (!azureAccount && !azureKey && !azureContainer && !azureBasePath) {
+if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) {
     azureAccount = 'azure_integration_test_account'
     azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64
     azureContainer = 'container_test'
     azureBasePath = 'integration_test'
+    azureSasToken = ''
     useFixture = true
 }
 
@@ -63,14 +69,23 @@ integTest {
 testClusters.integTest {
     plugin file(project(':plugins:repository-azure').bundlePlugin.archiveFile)
     keystore 'azure.client.integration_test.account', azureAccount
-    keystore 'azure.client.integration_test.key', azureKey
+    if (azureKey != null && azureKey.isEmpty() == false) {
+        println "Using access key in external service tests."
+        keystore 'azure.client.integration_test.key', azureKey
+    }
+    if (azureSasToken != null && azureSasToken.isEmpty() == false) {
+        println "Using SAS token in external service tests."
+        keystore 'azure.client.integration_test.sas_token', azureSasToken
+    }
 
     if (useFixture) {
         tasks.integTest.dependsOn azureStorageFixture
         // Use a closure on the string to delay evaluation until tests are executed. The endpoint_suffix is used
         // in a hacky way to change the protocol and endpoint. We must fix that.
         setting 'azure.client.integration_test.endpoint_suffix',
-                { "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://${azureStorageFixture.addressAndPort }" }
+                { "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://${azureStorageFixture.addressAndPort }" }, IGNORE_VALUE
+        String firstPartOfSeed = project.rootProject.testSeed.tokenize(':').get(0)
+        setting 'thread_pool.repository_azure.max', (Math.abs(Long.parseUnsignedLong(firstPartOfSeed, 16) % 10) + 1).toString(), System.getProperty('ignore.tests.seed') == null ? DEFAULT : IGNORE_VALUE
     } else {
         println "Using an external service to test the repository-azure plugin"
     }
diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml b/plugins/repository-azure/qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml
index 92866190959e6..fade1f9f1e67d 100644
--- a/plugins/repository-azure/qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml
+++ b/plugins/repository-azure/qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml
@@ -15,6 +15,9 @@ setup:
 
 ---
 "Snapshot/Restore with repository-azure":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   # Get repository
   - do:
@@ -121,9 +124,9 @@ setup:
         repository: repository
         snapshot: snapshot-one,snapshot-two
 
-  - is_true: snapshots
-  - match: { snapshots.0.state : SUCCESS }
-  - match: { snapshots.1.state : SUCCESS }
+  - is_true: responses.0.snapshots
+  - match: { responses.0.snapshots.0.state: SUCCESS }
+  - match: { responses.0.snapshots.1.state: SUCCESS }
 
   # Delete the index
   - do:
@@ -203,13 +206,18 @@ setup:
 
 ---
 "Get a non existing snapshot":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   - do:
-      catch: /snapshot_missing_exception/
       snapshot.get:
         repository: repository
         snapshot: missing
 
+  - is_true: responses.0.error
+  - match: { responses.0.error.type: snapshot_missing_exception }
+
 ---
 "Delete a non existing snapshot":
 
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
index 002907a0a7eb1..12113542dee44 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
@@ -23,29 +23,38 @@
 import com.microsoft.azure.storage.StorageException;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
+import org.elasticsearch.action.support.GroupedActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
 import java.net.URISyntaxException;
 import java.nio.file.NoSuchFileException;
+import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
 
 public class AzureBlobContainer extends AbstractBlobContainer {
 
     private final Logger logger = LogManager.getLogger(AzureBlobContainer.class);
     private final AzureBlobStore blobStore;
-
+    private final ThreadPool threadPool;
     private final String keyPath;
 
-    public AzureBlobContainer(BlobPath path, AzureBlobStore blobStore) {
+    AzureBlobContainer(BlobPath path, AzureBlobStore blobStore, ThreadPool threadPool) {
         super(path);
         this.blobStore = blobStore;
         this.keyPath = path.buildAsString();
+        this.threadPool = threadPool;
     }
 
     @Override
@@ -117,6 +126,43 @@ public void deleteBlob(String blobName) throws IOException {
         }
     }
 
+    @Override
+    public void delete() throws IOException {
+        try {
+            blobStore.deleteBlobDirectory(keyPath, threadPool.executor(AzureRepositoryPlugin.REPOSITORY_THREAD_POOL_NAME));
+        } catch (URISyntaxException | StorageException e) {
+            throw new IOException(e);
+        }
+    }
+
+    @Override
+    public void deleteBlobsIgnoringIfNotExists(List<String> blobNames) throws IOException {
+        final PlainActionFuture<Void> result = PlainActionFuture.newFuture();
+        if (blobNames.isEmpty()) {
+            result.onResponse(null);
+        } else {
+            final GroupedActionListener<Void> listener =
+                new GroupedActionListener<>(ActionListener.map(result, v -> null), blobNames.size());
+            final ExecutorService executor = threadPool.executor(AzureRepositoryPlugin.REPOSITORY_THREAD_POOL_NAME);
+            // Executing deletes in parallel since Azure SDK 8 is using blocking IO while Azure does not provide a bulk delete API endpoint
+            // TODO: Upgrade to newer non-blocking Azure SDK 11 and execute delete requests in parallel that way.
+            for (String blobName : blobNames) {
+                executor.execute(new ActionRunnable<>(listener) {
+                    @Override
+                    protected void doRun() throws IOException {
+                        deleteBlobIgnoringIfNotExists(blobName);
+                        listener.onResponse(null);
+                    }
+                });
+            }
+        }
+        try {
+            result.actionGet();
+        } catch (Exception e) {
+            throw new IOException("Exception during bulk delete", e);
+        }
+    }
+
     @Override
     public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String prefix) throws IOException {
         logger.trace("listBlobsByPrefix({})", prefix);
@@ -135,6 +181,16 @@ public Map<String, BlobMetaData> listBlobs() throws IOException {
         return listBlobsByPrefix(null);
     }
 
+    @Override
+    public Map<String, BlobContainer> children() throws IOException {
+        final BlobPath path = path();
+        try {
+            return blobStore.children(path);
+        } catch (URISyntaxException | StorageException e) {
+            throw new IOException("Failed to list children in path [" + path.buildAsString() + "].", e);
+        }
+    }
+
     protected String buildKey(String blobName) {
         return keyPath + (blobName == null ? "" : blobName);
     }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
index 7eeadc7f6475b..a7d9bb93a5125 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
@@ -28,27 +28,34 @@
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.repositories.azure.AzureRepository.Repository;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URISyntaxException;
 import java.nio.file.FileAlreadyExistsException;
+import java.util.Collections;
 import java.util.Map;
+import java.util.concurrent.Executor;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyMap;
 
 public class AzureBlobStore implements BlobStore {
 
     private final AzureStorageService service;
+    private final ThreadPool threadPool;
 
     private final String clientName;
     private final String container;
     private final LocationMode locationMode;
 
-    public AzureBlobStore(RepositoryMetaData metadata, AzureStorageService service) {
+    public AzureBlobStore(RepositoryMetaData metadata, AzureStorageService service, ThreadPool threadPool) {
         this.container = Repository.CONTAINER_SETTING.get(metadata.settings());
         this.clientName = Repository.CLIENT_NAME.get(metadata.settings());
         this.service = service;
+        this.threadPool = threadPool;
         // locationMode is set per repository, not per client
         this.locationMode = Repository.LOCATION_MODE_SETTING.get(metadata.settings());
         final Map<String, AzureStorageSettings> prevSettings = this.service.refreshAndClearCache(emptyMap());
@@ -70,17 +77,13 @@ public LocationMode getLocationMode() {
 
     @Override
     public BlobContainer blobContainer(BlobPath path) {
-        return new AzureBlobContainer(path, this);
+        return new AzureBlobContainer(path, this, threadPool);
     }
 
     @Override
     public void close() {
     }
 
-    public boolean containerExist() throws URISyntaxException, StorageException {
-        return service.doesContainerExist(clientName, container);
-    }
-
     public boolean blobExists(String blob) throws URISyntaxException, StorageException {
         return service.blobExists(clientName, container, blob);
     }
@@ -89,6 +92,10 @@ public void deleteBlob(String blob) throws URISyntaxException, StorageException
         service.deleteBlob(clientName, container, blob);
     }
 
+    public void deleteBlobDirectory(String path, Executor executor) throws URISyntaxException, StorageException, IOException {
+        service.deleteBlobDirectory(clientName, container, path, executor);
+    }
+
     public InputStream getInputStream(String blob) throws URISyntaxException, StorageException, IOException {
         return service.getInputStream(clientName, container, blob);
     }
@@ -98,6 +105,11 @@ public Map<String, BlobMetaData> listBlobsByPrefix(String keyPath, String prefix
         return service.listBlobsByPrefix(clientName, container, keyPath, prefix);
     }
 
+    public Map<String, BlobContainer> children(BlobPath path) throws URISyntaxException, StorageException {
+        return Collections.unmodifiableMap(service.children(clientName, container, path).stream().collect(
+            Collectors.toMap(Function.identity(), name -> new AzureBlobContainer(path.add(name), this, threadPool))));
+    }
+
     public void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists)
         throws URISyntaxException, StorageException, FileAlreadyExistsException {
         service.writeBlob(this.clientName, container, blobName, inputStream, blobSize, failIfAlreadyExists);
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
index 7c3520918fc58..08b3bb3b45897 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
@@ -20,11 +20,9 @@
 package org.elasticsearch.repositories.azure;
 
 import com.microsoft.azure.storage.LocationMode;
-import com.microsoft.azure.storage.StorageException;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -34,14 +32,9 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
-import org.elasticsearch.snapshots.SnapshotCreationException;
-import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.threadpool.ThreadPool;
 
-import java.net.URISyntaxException;
-import java.util.List;
 import java.util.Locale;
 import java.util.function.Function;
 
@@ -79,17 +72,27 @@ public static final class Repository {
         public static final Setting<Boolean> READONLY_SETTING = Setting.boolSetting("readonly", false, Property.NodeScope);
     }
 
-    private final BlobPath basePath;
     private final ByteSizeValue chunkSize;
     private final AzureStorageService storageService;
     private final boolean readonly;
 
     public AzureRepository(RepositoryMetaData metadata, Environment environment, NamedXContentRegistry namedXContentRegistry,
             AzureStorageService storageService, ThreadPool threadPool) {
-        super(metadata, environment.settings(), namedXContentRegistry, threadPool);
+        super(metadata, environment.settings(), namedXContentRegistry, threadPool, buildBasePath(metadata));
         this.chunkSize = Repository.CHUNK_SIZE_SETTING.get(metadata.settings());
         this.storageService = storageService;
 
+        // If the user explicitly did not define a readonly value, we set it by ourselves depending on the location mode setting.
+        // For secondary_only setting, the repository should be read only
+        final LocationMode locationMode = Repository.LOCATION_MODE_SETTING.get(metadata.settings());
+        if (Repository.READONLY_SETTING.exists(metadata.settings())) {
+            this.readonly = Repository.READONLY_SETTING.get(metadata.settings());
+        } else {
+            this.readonly = locationMode == LocationMode.SECONDARY_ONLY;
+        }
+    }
+
+    private static BlobPath buildBasePath(RepositoryMetaData metadata) {
         final String basePath = Strings.trimLeadingCharacter(Repository.BASE_PATH_SETTING.get(metadata.settings()), '/');
         if (Strings.hasLength(basePath)) {
             // Remove starting / if any
@@ -97,18 +100,9 @@ public AzureRepository(RepositoryMetaData metadata, Environment environment, Nam
             for(final String elem : basePath.split("/")) {
                 path = path.add(elem);
             }
-            this.basePath = path;
+            return path;
         } else {
-            this.basePath = BlobPath.cleanPath();
-        }
-
-        // If the user explicitly did not define a readonly value, we set it by ourselves depending on the location mode setting.
-        // For secondary_only setting, the repository should be read only
-        final LocationMode locationMode = Repository.LOCATION_MODE_SETTING.get(metadata.settings());
-        if (Repository.READONLY_SETTING.exists(metadata.settings())) {
-            this.readonly = Repository.READONLY_SETTING.get(metadata.settings());
-        } else {
-            this.readonly = locationMode == LocationMode.SECONDARY_ONLY;
+            return BlobPath.cleanPath();
         }
     }
 
@@ -119,38 +113,19 @@ protected BlobStore getBlobStore() {
 
     @Override
     protected AzureBlobStore createBlobStore() {
-        final AzureBlobStore blobStore = new AzureBlobStore(metadata, storageService);
+        final AzureBlobStore blobStore = new AzureBlobStore(metadata, storageService, threadPool);
 
         logger.debug(() -> new ParameterizedMessage(
             "using container [{}], chunk_size [{}], compress [{}], base_path [{}]",
-            blobStore, chunkSize, isCompress(), basePath));
+            blobStore, chunkSize, isCompress(), basePath()));
         return blobStore;
     }
 
-    @Override
-    protected BlobPath basePath() {
-        return basePath;
-    }
-
     @Override
     protected ByteSizeValue chunkSize() {
         return chunkSize;
     }
 
-    @Override
-    public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, MetaData clusterMetadata) {
-        try {
-            final AzureBlobStore blobStore = (AzureBlobStore) blobStore();
-            if (blobStore.containerExist() == false) {
-                throw new IllegalArgumentException("The bucket [" + blobStore + "] does not exist. Please create it before "
-                        + " creating an azure snapshot repository backed by it.");
-            }
-        } catch (URISyntaxException | StorageException e) {
-            throw new SnapshotCreationException(metadata.name(), snapshotId, e);
-        }
-        super.initializeSnapshot(snapshotId, indices, clusterMetadata);
-    }
-
     @Override
     public boolean isReadOnly() {
         return readonly;
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
index ab48cf1314ec5..809ba9d515834 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java
@@ -22,12 +22,15 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ReloadablePlugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.repositories.Repository;
+import org.elasticsearch.threadpool.ExecutorBuilder;
+import org.elasticsearch.threadpool.ScalingExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Arrays;
@@ -40,6 +43,8 @@
  */
 public class AzureRepositoryPlugin extends Plugin implements RepositoryPlugin, ReloadablePlugin {
 
+    public static final String REPOSITORY_THREAD_POOL_NAME = "repository_azure";
+
     // protected for testing
     final AzureStorageService azureStoreService;
 
@@ -60,6 +65,7 @@ public List<Setting<?>> getSettings() {
         return Arrays.asList(
             AzureStorageSettings.ACCOUNT_SETTING,
             AzureStorageSettings.KEY_SETTING,
+            AzureStorageSettings.SAS_TOKEN_SETTING,
             AzureStorageSettings.ENDPOINT_SUFFIX_SETTING,
             AzureStorageSettings.TIMEOUT_SETTING,
             AzureStorageSettings.MAX_RETRIES_SETTING,
@@ -69,6 +75,15 @@ public List<Setting<?>> getSettings() {
         );
     }
 
+    @Override
+    public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
+        return Collections.singletonList(executorBuilder());
+    }
+
+    public static ExecutorBuilder<?> executorBuilder() {
+        return new ScalingExecutorBuilder(REPOSITORY_THREAD_POOL_NAME, 0, 32, TimeValue.timeValueSeconds(30L));
+    }
+
     @Override
     public void reload(Settings settings) {
         // secure settings should be readable
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
index 89a78fd8045ee..f4ee7b9dbcad9 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java
@@ -29,21 +29,26 @@
 import com.microsoft.azure.storage.blob.BlobInputStream;
 import com.microsoft.azure.storage.blob.BlobListingDetails;
 import com.microsoft.azure.storage.blob.BlobProperties;
+import com.microsoft.azure.storage.blob.CloudBlob;
 import com.microsoft.azure.storage.blob.CloudBlobClient;
 import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.microsoft.azure.storage.blob.CloudBlobDirectory;
 import com.microsoft.azure.storage.blob.CloudBlockBlob;
 import com.microsoft.azure.storage.blob.DeleteSnapshotsOption;
 import com.microsoft.azure.storage.blob.ListBlobItem;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.blobstore.BlobMetaData;
+import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -52,9 +57,16 @@
 import java.net.URISyntaxException;
 import java.nio.file.FileAlreadyExistsException;
 import java.security.InvalidKeyException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Supplier;
 
 import static java.util.Collections.emptyMap;
@@ -116,7 +128,7 @@ private static CloudBlobClient buildClient(AzureStorageSettings azureStorageSett
     }
 
     private static CloudBlobClient createClient(AzureStorageSettings azureStorageSettings) throws InvalidKeyException, URISyntaxException {
-        final String connectionString = azureStorageSettings.buildConnectionString();
+        final String connectionString = azureStorageSettings.getConnectString();
         return CloudStorageAccount.parse(connectionString).createCloudBlobClient();
     }
 
@@ -140,12 +152,6 @@ public Map<String, AzureStorageSettings> refreshAndClearCache(Map<String, AzureS
         return prevSettings;
     }
 
-    public boolean doesContainerExist(String account, String container) throws URISyntaxException, StorageException {
-        final Tuple<CloudBlobClient, Supplier<OperationContext>> client = client(account);
-        final CloudBlobContainer blobContainer = client.v1().getContainerReference(container);
-        return SocketAccess.doPrivilegedException(() -> blobContainer.exists(null, null, client.v2().get()));
-    }
-
     /**
      * Extract the blob name from a URI like https://myservice.azure.net/container/path/to/myfile
      * It should remove the container part (first part of the path) and gives path/to/myfile
@@ -186,6 +192,50 @@ public void deleteBlob(String account, String container, String blob) throws URI
         });
     }
 
+    void deleteBlobDirectory(String account, String container, String path, Executor executor)
+            throws URISyntaxException, StorageException, IOException {
+        final Tuple<CloudBlobClient, Supplier<OperationContext>> client = client(account);
+        final CloudBlobContainer blobContainer = client.v1().getContainerReference(container);
+        final Collection<Exception> exceptions = Collections.synchronizedList(new ArrayList<>());
+        final AtomicLong outstanding = new AtomicLong(1L);
+        final PlainActionFuture<Void> result = PlainActionFuture.newFuture();
+        SocketAccess.doPrivilegedVoidException(() -> {
+            for (final ListBlobItem blobItem : blobContainer.listBlobs(path, true)) {
+                // uri.getPath is of the form /container/keyPath.* and we want to strip off the /container/
+                // this requires 1 + container.length() + 1, with each 1 corresponding to one of the /
+                final String blobPath = blobItem.getUri().getPath().substring(1 + container.length() + 1);
+                outstanding.incrementAndGet();
+                executor.execute(new AbstractRunnable() {
+                    @Override
+                    protected void doRun() throws Exception {
+                        deleteBlob(account, container, blobPath);
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        exceptions.add(e);
+                    }
+
+                    @Override
+                    public void onAfter() {
+                        if (outstanding.decrementAndGet() == 0) {
+                            result.onResponse(null);
+                        }
+                    }
+                });
+            }
+        });
+        if (outstanding.decrementAndGet() == 0) {
+            result.onResponse(null);
+        }
+        result.actionGet();
+        if (exceptions.isEmpty() == false) {
+            final IOException ex = new IOException("Deleting directory [" + path + "] failed");
+            exceptions.forEach(ex::addSuppressed);
+            throw ex;
+        }
+    }
+
     public InputStream getInputStream(String account, String container, String blob)
         throws URISyntaxException, StorageException, IOException {
         final Tuple<CloudBlobClient, Supplier<OperationContext>> client = client(account);
@@ -214,15 +264,40 @@ public Map<String, BlobMetaData> listBlobsByPrefix(String account, String contai
                 // uri.getPath is of the form /container/keyPath.* and we want to strip off the /container/
                 // this requires 1 + container.length() + 1, with each 1 corresponding to one of the /
                 final String blobPath = uri.getPath().substring(1 + container.length() + 1);
-                final BlobProperties properties = ((CloudBlockBlob) blobItem).getProperties();
-                final String name = blobPath.substring(keyPath.length());
-                logger.trace(() -> new ParameterizedMessage("blob url [{}], name [{}], size [{}]", uri, name, properties.getLength()));
-                blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getLength()));
+                if (blobItem instanceof CloudBlob) {
+                    final BlobProperties properties = ((CloudBlob) blobItem).getProperties();
+                    final String name = blobPath.substring(keyPath.length());
+                    logger.trace(() -> new ParameterizedMessage("blob url [{}], name [{}], size [{}]", uri, name, properties.getLength()));
+                    blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getLength()));
+                }
             }
         });
         return Map.copyOf(blobsBuilder);
     }
 
+    public Set<String> children(String account, String container, BlobPath path) throws URISyntaxException, StorageException {
+        final var blobsBuilder = new HashSet<String>();
+        final Tuple<CloudBlobClient, Supplier<OperationContext>> client = client(account);
+        final CloudBlobContainer blobContainer = client.v1().getContainerReference(container);
+        final String keyPath = path.buildAsString();
+        final EnumSet<BlobListingDetails> enumBlobListingDetails = EnumSet.of(BlobListingDetails.METADATA);
+
+        SocketAccess.doPrivilegedVoidException(() -> {
+            for (ListBlobItem blobItem : blobContainer.listBlobs(keyPath, false, enumBlobListingDetails, null, client.v2().get())) {
+                if (blobItem instanceof CloudBlobDirectory) {
+                    final URI uri = blobItem.getUri();
+                    logger.trace(() -> new ParameterizedMessage("blob url [{}]", uri));
+                    // uri.getPath is of the form /container/keyPath.* and we want to strip off the /container/
+                    // this requires 1 + container.length() + 1, with each 1 corresponding to one of the /.
+                    // Lastly, we add the length of keyPath to the offset to strip this container's path.
+                    final String uriPath = uri.getPath();
+                    blobsBuilder.add(uriPath.substring(1 + container.length() + 1 + keyPath.length(), uriPath.length() - 1));
+                }
+            }
+        });
+        return Set.copyOf(blobsBuilder);
+    }
+
     public void writeBlob(String account, String container, String blobName, InputStream inputStream, long blobSize,
                           boolean failIfAlreadyExists)
         throws URISyntaxException, StorageException, FileAlreadyExistsException {
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java
index e57d855cb0ee5..d87e48542d1c4 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java
@@ -21,6 +21,7 @@
 
 import com.microsoft.azure.storage.LocationMode;
 import com.microsoft.azure.storage.RetryPolicy;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.SecureSetting;
 import org.elasticsearch.common.settings.SecureString;
@@ -53,6 +54,10 @@ final class AzureStorageSettings {
     public static final AffixSetting<SecureString> KEY_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "key",
         key -> SecureSetting.secureString(key, null));
 
+    /** Azure SAS token */
+    public static final AffixSetting<SecureString> SAS_TOKEN_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "sas_token",
+        key -> SecureSetting.secureString(key, null));
+
     /** max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). */
     public static final Setting<Integer> MAX_RETRIES_SETTING =
         Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "max_retries",
@@ -82,7 +87,7 @@ final class AzureStorageSettings {
         PROXY_HOST_SETTING);
 
     private final String account;
-    private final String key;
+    private final String connectString;
     private final String endpointSuffix;
     private final TimeValue timeout;
     private final int maxRetries;
@@ -90,10 +95,10 @@ final class AzureStorageSettings {
     private final LocationMode locationMode;
 
     // copy-constructor
-    private AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries, Proxy proxy,
-            LocationMode locationMode) {
+    private AzureStorageSettings(String account, String connectString, String endpointSuffix, TimeValue timeout, int maxRetries,
+                                 Proxy proxy, LocationMode locationMode) {
         this.account = account;
-        this.key = key;
+        this.connectString = connectString;
         this.endpointSuffix = endpointSuffix;
         this.timeout = timeout;
         this.maxRetries = maxRetries;
@@ -101,10 +106,10 @@ private AzureStorageSettings(String account, String key, String endpointSuffix,
         this.locationMode = locationMode;
     }
 
-    AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries,
-                                Proxy.Type proxyType, String proxyHost, Integer proxyPort) {
+    private AzureStorageSettings(String account, String key, String sasToken, String endpointSuffix, TimeValue timeout, int maxRetries,
+                                 Proxy.Type proxyType, String proxyHost, Integer proxyPort) {
         this.account = account;
-        this.key = key;
+        this.connectString = buildConnectString(account, key, sasToken, endpointSuffix);
         this.endpointSuffix = endpointSuffix;
         this.timeout = timeout;
         this.maxRetries = maxRetries;
@@ -145,13 +150,26 @@ public Proxy getProxy() {
         return proxy;
     }
 
-    public String buildConnectionString() {
+    public String getConnectString() {
+        return connectString;
+    }
+
+    private static String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix) {
+        final boolean hasSasToken = Strings.hasText(sasToken);
+        final boolean hasKey = Strings.hasText(key);
+        if (hasSasToken == false && hasKey == false) {
+            throw new SettingsException("Neither a secret key nor a shared access token was set.");
+        }
+        if (hasSasToken && hasKey) {
+            throw new SettingsException("Both a secret as well as a shared access token were set.");
+        }
         final StringBuilder connectionStringBuilder = new StringBuilder();
-        connectionStringBuilder.append("DefaultEndpointsProtocol=https")
-                .append(";AccountName=")
-                .append(account)
-                .append(";AccountKey=")
-                .append(key);
+        connectionStringBuilder.append("DefaultEndpointsProtocol=https").append(";AccountName=").append(account);
+        if (hasKey) {
+            connectionStringBuilder.append(";AccountKey=").append(key);
+        } else {
+            connectionStringBuilder.append(";SharedAccessSignature=").append(sasToken);
+        }
         if (Strings.hasText(endpointSuffix)) {
             connectionStringBuilder.append(";EndpointSuffix=").append(endpointSuffix);
         }
@@ -166,7 +184,6 @@ public LocationMode getLocationMode() {
     public String toString() {
         final StringBuilder sb = new StringBuilder("AzureStorageSettings{");
         sb.append("account='").append(account).append('\'');
-        sb.append(", key='").append(key).append('\'');
         sb.append(", timeout=").append(timeout);
         sb.append(", endpointSuffix='").append(endpointSuffix).append('\'');
         sb.append(", maxRetries=").append(maxRetries);
@@ -201,8 +218,9 @@ public static Map<String, AzureStorageSettings> load(Settings settings) {
     /** Parse settings for a single client. */
     private static AzureStorageSettings getClientSettings(Settings settings, String clientName) {
         try (SecureString account = getConfigValue(settings, clientName, ACCOUNT_SETTING);
-             SecureString key = getConfigValue(settings, clientName, KEY_SETTING)) {
-            return new AzureStorageSettings(account.toString(), key.toString(),
+             SecureString key = getConfigValue(settings, clientName, KEY_SETTING);
+             SecureString sasToken = getConfigValue(settings, clientName, SAS_TOKEN_SETTING)) {
+            return new AzureStorageSettings(account.toString(), key.toString(), sasToken.toString(),
                 getValue(settings, clientName, ENDPOINT_SUFFIX_SETTING),
                 getValue(settings, clientName, TIMEOUT_SETTING),
                 getValue(settings, clientName, MAX_RETRIES_SETTING),
@@ -228,10 +246,9 @@ static Map<String, AzureStorageSettings> overrideLocationMode(Map<String, AzureS
                                                                   LocationMode locationMode) {
         final var map = new HashMap<String, AzureStorageSettings>();
         for (final Map.Entry<String, AzureStorageSettings> entry : clientsSettings.entrySet()) {
-            final AzureStorageSettings azureSettings = new AzureStorageSettings(entry.getValue().account, entry.getValue().key,
-                    entry.getValue().endpointSuffix, entry.getValue().timeout, entry.getValue().maxRetries, entry.getValue().proxy,
-                    locationMode);
-            map.put(entry.getKey(), azureSettings);
+            map.put(entry.getKey(),
+                new AzureStorageSettings(entry.getValue().account, entry.getValue().connectString, entry.getValue().endpointSuffix,
+                    entry.getValue().timeout, entry.getValue().maxRetries, entry.getValue().proxy, locationMode));
         }
         return Map.copyOf(map);
     }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java
index 13cc487a1c122..07d0a1e18d3bd 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java
@@ -23,13 +23,31 @@
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
 
+import java.util.concurrent.TimeUnit;
 
 public class AzureBlobStoreContainerTests extends ESBlobStoreContainerTestCase {
+
+    private ThreadPool threadPool;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool("AzureBlobStoreTests", AzureRepositoryPlugin.executorBuilder());
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        ThreadPool.terminate(threadPool, 10L, TimeUnit.SECONDS);
+    }
+
     @Override
     protected BlobStore newBlobStore() {
         RepositoryMetaData repositoryMetaData = new RepositoryMetaData("azure", "ittest", Settings.EMPTY);
         AzureStorageServiceMock client = new AzureStorageServiceMock();
-        return new AzureBlobStore(repositoryMetaData, client);
+        return new AzureBlobStore(repositoryMetaData, client, threadPool);
     }
 }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java
index 67d30fda05b69..74bfcb784aed0 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java
@@ -22,13 +22,31 @@
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.repositories.ESBlobStoreTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.concurrent.TimeUnit;
 
 public class AzureBlobStoreTests extends ESBlobStoreTestCase {
 
+    private ThreadPool threadPool;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool("AzureBlobStoreTests", AzureRepositoryPlugin.executorBuilder());
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        ThreadPool.terminate(threadPool, 10L, TimeUnit.SECONDS);
+    }
+
     @Override
     protected BlobStore newBlobStore() {
         RepositoryMetaData repositoryMetaData = new RepositoryMetaData("azure", "ittest", Settings.EMPTY);
         AzureStorageServiceMock client = new AzureStorageServiceMock();
-        return new AzureBlobStore(repositoryMetaData, client);
+        return new AzureBlobStore(repositoryMetaData, client, threadPool);
     }
 }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
new file mode 100644
index 0000000000000..1c5c2dd39fae6
--- /dev/null
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.azure;
+
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.SecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase;
+
+import java.util.Collection;
+
+import static org.hamcrest.Matchers.blankOrNullString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
+public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(AzureRepositoryPlugin.class);
+    }
+
+    @Override
+    protected SecureSettings credentials() {
+        assertThat(System.getProperty("test.azure.account"), not(blankOrNullString()));
+        final boolean hasSasToken = Strings.hasText(System.getProperty("test.azure.sas_token"));
+        if (hasSasToken == false) {
+            assertThat(System.getProperty("test.azure.key"), not(blankOrNullString()));
+        } else {
+            assertThat(System.getProperty("test.azure.key"), blankOrNullString());
+        }
+        assertThat(System.getProperty("test.azure.container"), not(blankOrNullString()));
+        assertThat(System.getProperty("test.azure.base"), not(blankOrNullString()));
+
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("azure.client.default.account", System.getProperty("test.azure.account"));
+        if (hasSasToken) {
+            secureSettings.setString("azure.client.default.sas_token", System.getProperty("test.azure.sas_token"));
+        } else {
+            secureSettings.setString("azure.client.default.key", System.getProperty("test.azure.key"));
+        }
+        return secureSettings;
+    }
+
+    @Override
+    protected void createRepository(String repoName) {
+        AcknowledgedResponse putRepositoryResponse = client().admin().cluster().preparePutRepository(repoName)
+            .setType("azure")
+            .setSettings(Settings.builder()
+                .put("container", System.getProperty("test.azure.container"))
+                .put("base_path", System.getProperty("test.azure.base"))
+            ).get();
+        assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
+    }
+}
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceMock.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceMock.java
index 17502a1d1f982..2217b6743fb2e 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceMock.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceMock.java
@@ -55,11 +55,6 @@ public AzureStorageServiceMock() {
         super(Settings.EMPTY);
     }
 
-    @Override
-    public boolean doesContainerExist(String account, String container) {
-        return true;
-    }
-
     @Override
     public boolean blobExists(String account, String container, String blob) {
         return blobs.containsKey(blob);
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java
index f7b49bd24adf6..128e0e0a2140e 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java
@@ -158,15 +158,21 @@ public void testReinitClientWrongSettings() throws IOException {
         secureSettings2.setString("azure.client.azure1.account", "myaccount1");
         // missing key
         final Settings settings2 = Settings.builder().setSecureSettings(secureSettings2).build();
+        final MockSecureSettings secureSettings3 = new MockSecureSettings();
+        secureSettings3.setString("azure.client.azure1.account", "myaccount3");
+        secureSettings3.setString("azure.client.azure1.key", encodeKey("mykey33"));
+        secureSettings3.setString("azure.client.azure1.sas_token", encodeKey("mysasToken33"));
+        final Settings settings3 = Settings.builder().setSecureSettings(secureSettings3).build();
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) {
             final AzureStorageService azureStorageService = plugin.azureStoreService;
             final CloudBlobClient client11 = azureStorageService.client("azure1").v1();
             assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount1.blob.core.windows.net"));
-            plugin.reload(settings2);
+            final SettingsException e1 = expectThrows(SettingsException.class, () -> plugin.reload(settings2));
+            assertThat(e1.getMessage(), is("Neither a secret key nor a shared access token was set."));
+            final SettingsException e2 = expectThrows(SettingsException.class, () -> plugin.reload(settings3));
+            assertThat(e2.getMessage(), is("Both a secret as well as a shared access token were set."));
             // existing client untouched
             assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount1.blob.core.windows.net"));
-            final SettingsException e = expectThrows(SettingsException.class, () -> azureStorageService.client("azure1"));
-            assertThat(e.getMessage(), is("Invalid azure client settings with name [azure1]"));
         }
     }
 
diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle
index e5af9081ca189..f16a82fc5587f 100644
--- a/plugins/repository-gcs/build.gradle
+++ b/plugins/repository-gcs/build.gradle
@@ -1,3 +1,5 @@
+import java.nio.file.Files
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -23,38 +25,35 @@ esplugin {
 }
 
 dependencies {
-  compile 'com.google.cloud:google-cloud-storage:1.59.0'
-  compile 'com.google.cloud:google-cloud-core:1.59.0'
-  compile 'com.google.guava:guava:20.0'
-  compile "joda-time:joda-time:${versions.joda}"
-  compile 'com.google.http-client:google-http-client:1.24.1'
+  compile 'com.google.cloud:google-cloud-storage:1.77.0'
+  compile 'com.google.cloud:google-cloud-core:1.77.0'
+  compile 'com.google.guava:guava:26.0-jre'
+  compile 'com.google.http-client:google-http-client:1.30.1'
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
+  compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
-  compile 'com.google.api:api-common:1.7.0'
-  compile 'com.google.api:gax:1.30.0'
+  compile 'com.google.api:api-common:1.8.1'
+  compile 'com.google.api:gax:1.45.0'
   compile 'org.threeten:threetenbp:1.3.3'
-  compile 'com.google.protobuf:protobuf-java-util:3.6.0'
-  compile 'com.google.protobuf:protobuf-java:3.6.0'
+  compile 'com.google.protobuf:protobuf-java-util:3.7.1'
+  compile 'com.google.protobuf:protobuf-java:3.7.1'
   compile 'com.google.code.gson:gson:2.7'
-  compile 'com.google.api.grpc:proto-google-common-protos:1.12.0'
+  compile 'com.google.api.grpc:proto-google-common-protos:1.16.0'
   compile 'com.google.api.grpc:proto-google-iam-v1:0.12.0'
-  compile 'com.google.cloud:google-cloud-core-http:1.59.0'
-  compile 'com.google.auth:google-auth-library-credentials:0.10.0'
-  compile 'com.google.auth:google-auth-library-oauth2-http:0.10.0'
-  compile 'com.google.oauth-client:google-oauth-client:1.24.1'
-  compile 'com.google.api-client:google-api-client:1.24.1'
-  compile 'com.google.http-client:google-http-client-appengine:1.24.1'
-  compile 'com.google.http-client:google-http-client-jackson:1.24.1'
-  compile 'org.codehaus.jackson:jackson-core-asl:1.9.11'
-  compile 'com.google.http-client:google-http-client-jackson2:1.24.1'
-  compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  compile 'com.google.api:gax-httpjson:0.47.0'
-  compile 'io.opencensus:opencensus-api:0.15.0'
+  compile 'com.google.cloud:google-cloud-core-http:1.77.0'
+  compile 'com.google.auth:google-auth-library-credentials:0.16.1'
+  compile 'com.google.auth:google-auth-library-oauth2-http:0.16.1'
+  compile 'com.google.oauth-client:google-oauth-client:1.28.0'
+  compile 'com.google.api-client:google-api-client:1.28.0'
+  compile 'com.google.http-client:google-http-client-appengine:1.29.2'
+  compile 'com.google.http-client:google-http-client-jackson2:1.29.2'
+  compile 'com.google.api:gax-httpjson:0.62.0'
   compile 'io.grpc:grpc-context:1.12.0'
-  compile 'io.opencensus:opencensus-contrib-http-util:0.15.0'
-  compile 'com.google.apis:google-api-services-storage:v1-rev135-1.24.1'
+  compile 'io.opencensus:opencensus-api:0.18.0'
+  compile 'io.opencensus:opencensus-contrib-http-util:0.18.0'
+  compile 'com.google.apis:google-api-services-storage:v1-rev20190426-1.28.0'
 }
 
 dependencyLicenses {
@@ -62,7 +61,6 @@ dependencyLicenses {
   mapping from: /google-auth-.*/, to: 'google-auth'
   mapping from: /google-http-.*/, to: 'google-http'
   mapping from: /opencensus.*/, to: 'opencensus'
-  mapping from: /jackson-.*/, to: 'jackson'
   mapping from: /http.*/, to: 'httpclient'
   mapping from: /protobuf.*/, to: 'protobuf'
   mapping from: /proto-google.*/, to: 'proto-google'
@@ -78,6 +76,10 @@ thirdPartyAudit {
         'com.google.common.cache.Striped64',
         'com.google.common.cache.Striped64$1',
         'com.google.common.cache.Striped64$Cell',
+        'com.google.common.hash.Striped64',
+        'com.google.common.hash.Striped64$1',
+        'com.google.common.hash.Striped64$Cell',
+        'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
         'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
         'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
         'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
@@ -122,3 +124,22 @@ check {
   // also execute the QA tests when testing the plugin
   dependsOn 'qa:google-cloud-storage:check'
 }
+
+String gcsServiceAccount = System.getenv("google_storage_service_account")
+String gcsBucket = System.getenv("google_storage_bucket")
+String gcsBasePath = System.getenv("google_storage_base_path")
+
+test {
+  exclude '**/GoogleCloudStorageThirdPartyTests.class'
+}
+
+task thirdPartyTest(type: Test) {
+  include '**/GoogleCloudStorageThirdPartyTests.class'
+  systemProperty 'test.google.account', gcsServiceAccount ? Base64.encoder.encodeToString(Files.readAllBytes(file(gcsServiceAccount).toPath())) : ""
+  systemProperty 'test.google.bucket', gcsBucket ? gcsBucket : ""
+  systemProperty 'test.google.base', gcsBasePath ? gcsBasePath : "/"
+}
+
+if (gcsServiceAccount || gcsBucket || gcsBasePath) {
+  check.dependsOn(thirdPartyTest)
+}
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/api-common-1.7.0.jar.sha1 b/plugins/repository-gcs/licenses/api-common-1.7.0.jar.sha1
deleted file mode 100644
index 67291b658e5c5..0000000000000
--- a/plugins/repository-gcs/licenses/api-common-1.7.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ea59fb8b2450999345035dec8a6f472543391766
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/api-common-1.8.1.jar.sha1 b/plugins/repository-gcs/licenses/api-common-1.8.1.jar.sha1
new file mode 100644
index 0000000000000..7a1c114c6c0fc
--- /dev/null
+++ b/plugins/repository-gcs/licenses/api-common-1.8.1.jar.sha1
@@ -0,0 +1 @@
+e89befb19b08ad84b262b2f226ab79aefcaa9d7f
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/gax-1.30.0.jar.sha1 b/plugins/repository-gcs/licenses/gax-1.30.0.jar.sha1
deleted file mode 100644
index d6d2bb20ed840..0000000000000
--- a/plugins/repository-gcs/licenses/gax-1.30.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-58fa2feb11b092be0a6ebe705a28736f12374230
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/gax-1.45.0.jar.sha1 b/plugins/repository-gcs/licenses/gax-1.45.0.jar.sha1
new file mode 100644
index 0000000000000..8c0cbc659aa60
--- /dev/null
+++ b/plugins/repository-gcs/licenses/gax-1.45.0.jar.sha1
@@ -0,0 +1 @@
+2ade3e3502f9d14e3731347a82ea02372094211f
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/gax-httpjson-0.47.0.jar.sha1 b/plugins/repository-gcs/licenses/gax-httpjson-0.47.0.jar.sha1
deleted file mode 100644
index fdc722d1520d6..0000000000000
--- a/plugins/repository-gcs/licenses/gax-httpjson-0.47.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d096f3142eb3adbf877588d1044895d148d9efcb
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/gax-httpjson-0.62.0.jar.sha1 b/plugins/repository-gcs/licenses/gax-httpjson-0.62.0.jar.sha1
new file mode 100644
index 0000000000000..161ca85ccfc0c
--- /dev/null
+++ b/plugins/repository-gcs/licenses/gax-httpjson-0.62.0.jar.sha1
@@ -0,0 +1 @@
+05a1a4736acd1c4f30304be953532be6aecdc2c9
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-api-client-1.24.1.jar.sha1 b/plugins/repository-gcs/licenses/google-api-client-1.24.1.jar.sha1
deleted file mode 100644
index 27dafe58a0182..0000000000000
--- a/plugins/repository-gcs/licenses/google-api-client-1.24.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-37de23fb9b8b077de4ecec3192d98e752b0e5d72
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-api-client-1.28.0.jar.sha1 b/plugins/repository-gcs/licenses/google-api-client-1.28.0.jar.sha1
new file mode 100644
index 0000000000000..c9b0efd4cdcbf
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-api-client-1.28.0.jar.sha1
@@ -0,0 +1 @@
+8fe155d766ed22480939e3a9db428151e0264d9e
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev135-1.24.1.jar.sha1 b/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev135-1.24.1.jar.sha1
deleted file mode 100644
index e3042ee6ea07e..0000000000000
--- a/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev135-1.24.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-28d3d391dfc7e7e7951760708ad2f48cecacf38f
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20190426-1.28.0.jar.sha1 b/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20190426-1.28.0.jar.sha1
new file mode 100644
index 0000000000000..84c68e6ffd528
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20190426-1.28.0.jar.sha1
@@ -0,0 +1 @@
+34dd008901f382507a572f5242d0e5c5ea4ad713
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-auth-library-credentials-0.10.0.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-credentials-0.10.0.jar.sha1
deleted file mode 100644
index c8258d69326b8..0000000000000
--- a/plugins/repository-gcs/licenses/google-auth-library-credentials-0.10.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f981288bd84fe6d140ed70d1d8dbe994a64fa3cc
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-auth-library-credentials-0.16.1.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-credentials-0.16.1.jar.sha1
new file mode 100644
index 0000000000000..6527ebec6da50
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-auth-library-credentials-0.16.1.jar.sha1
@@ -0,0 +1 @@
+9a15387cc0438ac3f3e625b6050cf39f4e981e13
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-0.10.0.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-0.10.0.jar.sha1
deleted file mode 100644
index f55ef7c9c2150..0000000000000
--- a/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-0.10.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c079a62086121973a23d90f54e2b8c13050fa39d
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-0.16.1.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-0.16.1.jar.sha1
new file mode 100644
index 0000000000000..e6467acf8f595
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-0.16.1.jar.sha1
@@ -0,0 +1 @@
+3407d434678faef3439a7012efa336e751ddc623
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-cloud-core-1.59.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-1.59.0.jar.sha1
deleted file mode 100644
index 20e3b0c782dfd..0000000000000
--- a/plugins/repository-gcs/licenses/google-cloud-core-1.59.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f2d0c00917660b244da514f82cba96f7697f2c82
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-cloud-core-1.77.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-1.77.0.jar.sha1
new file mode 100644
index 0000000000000..d16477c5bd68f
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-cloud-core-1.77.0.jar.sha1
@@ -0,0 +1 @@
+7cd83a789fde368a999c1793c6297e7b4e56b2ac
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-1.59.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-1.59.0.jar.sha1
deleted file mode 100644
index ab4c7b7dca9c5..0000000000000
--- a/plugins/repository-gcs/licenses/google-cloud-core-http-1.59.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e2a094ec3e8acb15b99f2d4bd42ac9bbc7d9f33e
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-cloud-core-http-1.77.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-core-http-1.77.0.jar.sha1
new file mode 100644
index 0000000000000..7efc3167589eb
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-cloud-core-http-1.77.0.jar.sha1
@@ -0,0 +1 @@
+e16acbc935a7762ba9b220860ae45c2c67d17d8c
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-cloud-storage-1.59.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-storage-1.59.0.jar.sha1
deleted file mode 100644
index 0f5a8633bd028..0000000000000
--- a/plugins/repository-gcs/licenses/google-cloud-storage-1.59.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-23dc0edf739ff1fb5a91fbddd7bd1f2cbfe0f827
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-cloud-storage-1.77.0.jar.sha1 b/plugins/repository-gcs/licenses/google-cloud-storage-1.77.0.jar.sha1
new file mode 100644
index 0000000000000..de15e888520d9
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-cloud-storage-1.77.0.jar.sha1
@@ -0,0 +1 @@
+e368e1a8bbc0d0a4354f4e5eec076f38f6966050
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-1.24.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-1.24.1.jar.sha1
deleted file mode 100644
index 46b99f23e470a..0000000000000
--- a/plugins/repository-gcs/licenses/google-http-client-1.24.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-396eac8d3fb1332675f82b208f48a469d64f3b4a
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-1.30.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-1.30.1.jar.sha1
new file mode 100644
index 0000000000000..85323c108f99e
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-http-client-1.30.1.jar.sha1
@@ -0,0 +1 @@
+573aacbda8feb0d43f7056291fbce5496f42a6aa
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-appengine-1.24.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-appengine-1.24.1.jar.sha1
deleted file mode 100644
index e39f63fe33ae3..0000000000000
--- a/plugins/repository-gcs/licenses/google-http-client-appengine-1.24.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8535031ae10bf6a196e68f25e10c0d6382699cb6
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-appengine-1.29.2.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-appengine-1.29.2.jar.sha1
new file mode 100644
index 0000000000000..6973b62e9281c
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-http-client-appengine-1.29.2.jar.sha1
@@ -0,0 +1 @@
+d93f4d1d8c2496d75221e53173e4c503b7096a4d
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-jackson-1.24.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-jackson-1.24.1.jar.sha1
deleted file mode 100644
index f6b9694abaa6c..0000000000000
--- a/plugins/repository-gcs/licenses/google-http-client-jackson-1.24.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-02c88e77c14effdda76f02a0eac968de74e0bd4e
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.24.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.24.1.jar.sha1
deleted file mode 100644
index 634b7d9198c8e..0000000000000
--- a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.24.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2ad1dffd8a450055e68d8004fe003033b751d761
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.29.2.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.29.2.jar.sha1
new file mode 100644
index 0000000000000..5a44d18d2aa1b
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.29.2.jar.sha1
@@ -0,0 +1 @@
+d67891f5a438e1f339387a09628e0ab0af8b612a
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-oauth-client-1.24.1.jar.sha1 b/plugins/repository-gcs/licenses/google-oauth-client-1.24.1.jar.sha1
deleted file mode 100644
index 2d89939674a51..0000000000000
--- a/plugins/repository-gcs/licenses/google-oauth-client-1.24.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7b0e0218b96808868c23a7d0b40566a713931d9f
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-oauth-client-1.28.0.jar.sha1 b/plugins/repository-gcs/licenses/google-oauth-client-1.28.0.jar.sha1
new file mode 100644
index 0000000000000..474df6e026570
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-oauth-client-1.28.0.jar.sha1
@@ -0,0 +1 @@
+9a9e5d0c33b663d6475c96ce79b2949545a113af
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/guava-20.0.jar.sha1 b/plugins/repository-gcs/licenses/guava-20.0.jar.sha1
deleted file mode 100644
index 7b6ae09060b29..0000000000000
--- a/plugins/repository-gcs/licenses/guava-20.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-89507701249388e1ed5ddcf8c41f4ce1be7831ef
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/guava-26.0-jre.jar.sha1 b/plugins/repository-gcs/licenses/guava-26.0-jre.jar.sha1
new file mode 100644
index 0000000000000..63d05007650ce
--- /dev/null
+++ b/plugins/repository-gcs/licenses/guava-26.0-jre.jar.sha1
@@ -0,0 +1 @@
+6a806eff209f36f635f943e16d97491f00f6bfab
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/jackson-LICENSE b/plugins/repository-gcs/licenses/jackson-LICENSE
deleted file mode 100644
index f5f45d26a49d6..0000000000000
--- a/plugins/repository-gcs/licenses/jackson-LICENSE
+++ /dev/null
@@ -1,8 +0,0 @@
-This copy of Jackson JSON processor streaming parser/generator is licensed under the
-Apache (Software) License, version 2.0 ("the License").
-See the License for details about distribution rights, and the
-specific rights regarding derivate works.
-
-You may obtain a copy of the License at:
-
-http://www.apache.org/licenses/LICENSE-2.0
diff --git a/plugins/repository-gcs/licenses/jackson-NOTICE b/plugins/repository-gcs/licenses/jackson-NOTICE
deleted file mode 100644
index 4c976b7b4cc58..0000000000000
--- a/plugins/repository-gcs/licenses/jackson-NOTICE
+++ /dev/null
@@ -1,20 +0,0 @@
-# Jackson JSON processor
-
-Jackson is a high-performance, Free/Open Source JSON processing library.
-It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has
-been in development since 2007.
-It is currently developed by a community of developers, as well as supported
-commercially by FasterXML.com.
-
-## Licensing
-
-Jackson core and extension components may licensed under different licenses.
-To find the details that apply to this artifact see the accompanying LICENSE file.
-For more information, including possible other licensing options, contact
-FasterXML.com (http://fasterxml.com).
-
-## Credits
-
-A list of contributors may be found from CREDITS file, which is included
-in some artifacts (usually source distributions); but is always available
-from the source code management (SCM) system project uses.
diff --git a/plugins/repository-gcs/licenses/jackson-core-asl-1.9.11.jar.sha1 b/plugins/repository-gcs/licenses/jackson-core-asl-1.9.11.jar.sha1
deleted file mode 100644
index ed70030899aa0..0000000000000
--- a/plugins/repository-gcs/licenses/jackson-core-asl-1.9.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e32303ef8bd18a5c9272780d49b81c95e05ddf43
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/log4j-1.2-api-2.11.1.jar.sha1 b/plugins/repository-gcs/licenses/log4j-1.2-api-2.11.1.jar.sha1
new file mode 100644
index 0000000000000..575d75dbda8c5
--- /dev/null
+++ b/plugins/repository-gcs/licenses/log4j-1.2-api-2.11.1.jar.sha1
@@ -0,0 +1 @@
+3aba3398fe064a3eab4331f88161c7480e848418
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/log4j-LICENSE.txt b/plugins/repository-gcs/licenses/log4j-LICENSE.txt
new file mode 100644
index 0000000000000..6279e5206de13
--- /dev/null
+++ b/plugins/repository-gcs/licenses/log4j-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 1999-2005 The Apache Software Foundation
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-gcs/licenses/log4j-NOTICE.txt b/plugins/repository-gcs/licenses/log4j-NOTICE.txt
new file mode 100644
index 0000000000000..0375732360047
--- /dev/null
+++ b/plugins/repository-gcs/licenses/log4j-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache log4j
+Copyright 2007 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/opencensus-api-0.15.0.jar.sha1 b/plugins/repository-gcs/licenses/opencensus-api-0.15.0.jar.sha1
deleted file mode 100644
index e200e2e24a7df..0000000000000
--- a/plugins/repository-gcs/licenses/opencensus-api-0.15.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9a098392b287d7924660837f4eba0ce252013683
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/opencensus-api-0.18.0.jar.sha1 b/plugins/repository-gcs/licenses/opencensus-api-0.18.0.jar.sha1
new file mode 100644
index 0000000000000..8b95ab4e4c49c
--- /dev/null
+++ b/plugins/repository-gcs/licenses/opencensus-api-0.18.0.jar.sha1
@@ -0,0 +1 @@
+b89a8f8dfd1e1e0d68d83c82a855624814b19a6e
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.15.0.jar.sha1 b/plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.15.0.jar.sha1
deleted file mode 100644
index b642e1ebebd59..0000000000000
--- a/plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.15.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d88690591669d9b5ba6d91d9eac7736e58ccf3da
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.18.0.jar.sha1 b/plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.18.0.jar.sha1
new file mode 100644
index 0000000000000..1757e00591110
--- /dev/null
+++ b/plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.18.0.jar.sha1
@@ -0,0 +1 @@
+76a37e4a931d5801a9e25b0c0353e5f37c4d1e8e
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/proto-google-common-protos-1.12.0.jar.sha1 b/plugins/repository-gcs/licenses/proto-google-common-protos-1.12.0.jar.sha1
deleted file mode 100644
index 47f3c178a68c6..0000000000000
--- a/plugins/repository-gcs/licenses/proto-google-common-protos-1.12.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1140cc74df039deb044ed0e320035e674dc13062
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/proto-google-common-protos-1.16.0.jar.sha1 b/plugins/repository-gcs/licenses/proto-google-common-protos-1.16.0.jar.sha1
new file mode 100644
index 0000000000000..7762b7a3ebdc3
--- /dev/null
+++ b/plugins/repository-gcs/licenses/proto-google-common-protos-1.16.0.jar.sha1
@@ -0,0 +1 @@
+2c5f022ea3b8e8df6a619c4cd8faf9af86022daa
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/protobuf-java-3.6.0.jar.sha1 b/plugins/repository-gcs/licenses/protobuf-java-3.6.0.jar.sha1
deleted file mode 100644
index 050ebd44c9282..0000000000000
--- a/plugins/repository-gcs/licenses/protobuf-java-3.6.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5333f7e422744d76840c08a106e28e519fbe3acd
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/protobuf-java-3.7.1.jar.sha1 b/plugins/repository-gcs/licenses/protobuf-java-3.7.1.jar.sha1
new file mode 100644
index 0000000000000..51d30a4c18568
--- /dev/null
+++ b/plugins/repository-gcs/licenses/protobuf-java-3.7.1.jar.sha1
@@ -0,0 +1 @@
+0bce1b6dc9e4531169542ab37a1c8641bcaa8afb
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/protobuf-java-util-3.6.0.jar.sha1 b/plugins/repository-gcs/licenses/protobuf-java-util-3.6.0.jar.sha1
deleted file mode 100644
index cc85974499a65..0000000000000
--- a/plugins/repository-gcs/licenses/protobuf-java-util-3.6.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3680d0042d4fe0b95ada844ff24da0698a7f0773
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/protobuf-java-util-3.7.1.jar.sha1 b/plugins/repository-gcs/licenses/protobuf-java-util-3.7.1.jar.sha1
new file mode 100644
index 0000000000000..d08f6be735b84
--- /dev/null
+++ b/plugins/repository-gcs/licenses/protobuf-java-util-3.7.1.jar.sha1
@@ -0,0 +1 @@
+45dc95896cfad26397675fdabef7b032d6db4bb6
\ No newline at end of file
diff --git a/plugins/repository-gcs/qa/google-cloud-storage/build.gradle b/plugins/repository-gcs/qa/google-cloud-storage/build.gradle
index 4f201a812f2b2..d2411c7577692 100644
--- a/plugins/repository-gcs/qa/google-cloud-storage/build.gradle
+++ b/plugins/repository-gcs/qa/google-cloud-storage/build.gradle
@@ -17,12 +17,15 @@
  * under the License.
  */
 
+
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.test.AntFixture
 
 import java.security.KeyPair
 import java.security.KeyPairGenerator
 
+import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
+
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -90,13 +93,13 @@ integTest {
 testClusters.integTest {
     plugin file(project(':plugins:repository-gcs').bundlePlugin.archiveFile)
 
-    keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile
+    keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE
 
     if (useFixture) {
         tasks.integTest.dependsOn createServiceAccountFile, googleCloudStorageFixture
         /* Use a closure on the string to delay evaluation until tests are executed */
-        setting 'gcs.client.integration_test.endpoint', { "http://${googleCloudStorageFixture.addressAndPort}" }
-        setting 'gcs.client.integration_test.token_uri', { "http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/token" }
+        setting 'gcs.client.integration_test.endpoint', { "http://${googleCloudStorageFixture.addressAndPort}" }, IGNORE_VALUE
+        setting 'gcs.client.integration_test.token_uri', { "http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/token" }, IGNORE_VALUE
     } else {
         println "Using an external service to test the repository-gcs plugin"
     }
diff --git a/plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml b/plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml
index ac649229001db..553b6a3e14e50 100644
--- a/plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml
+++ b/plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml
@@ -28,6 +28,9 @@ setup:
 
 ---
 "Snapshot/Restore with repository-gcs":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   # Get repository
   - do:
@@ -134,9 +137,9 @@ setup:
         repository: repository
         snapshot: snapshot-one,snapshot-two
 
-  - is_true: snapshots
-  - match: { snapshots.0.state : SUCCESS }
-  - match: { snapshots.1.state : SUCCESS }
+  - is_true: responses.0.snapshots
+  - match: { responses.0.snapshots.0.state : SUCCESS }
+  - match: { responses.0.snapshots.1.state : SUCCESS }
 
   # Delete the index
   - do:
@@ -213,13 +216,18 @@ setup:
 
 ---
 "Get a non existing snapshot":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   - do:
-      catch: /snapshot_missing_exception/
       snapshot.get:
         repository: repository
         snapshot: missing
 
+  - is_true: responses.0.error
+  - match: { responses.0.error.type: snapshot_missing_exception }
+
 ---
 "Delete a non existing snapshot":
 
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java
index fb81a5c90039f..75d4ad92fbf8e 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.repositories.gcs;
 
+import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStoreException;
@@ -55,6 +56,11 @@ public Map<String, BlobMetaData> listBlobs() throws IOException {
         return blobStore.listBlobs(path);
     }
 
+    @Override
+    public Map<String, BlobContainer> children() throws IOException {
+        return blobStore.listChildren(path());
+    }
+
     @Override
     public Map<String, BlobMetaData> listBlobsByPrefix(String prefix) throws IOException {
         return blobStore.listBlobsByPrefix(path, prefix);
@@ -80,6 +86,11 @@ public void deleteBlob(String blobName) throws IOException {
         blobStore.deleteBlob(buildKey(blobName));
     }
 
+    @Override
+    public void delete() throws IOException {
+        blobStore.deleteDirectory(path().buildAsString());
+    }
+
     @Override
     public void deleteBlobsIgnoringIfNotExists(List<String> blobNames) throws IOException {
         blobStore.deleteBlobsIgnoringIfNotExists(blobNames.stream().map(this::buildKey).collect(Collectors.toList()));
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java
index 4214e5d408210..2477fd2962bde 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.repositories.gcs;
 
+import com.google.api.gax.paging.Page;
 import com.google.cloud.BatchResult;
 import com.google.cloud.ReadChannel;
 import com.google.cloud.WriteChannel;
@@ -30,7 +31,6 @@
 import com.google.cloud.storage.Storage.BlobListOption;
 import com.google.cloud.storage.StorageBatch;
 import com.google.cloud.storage.StorageException;
-
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobMetaData;
@@ -50,11 +50,11 @@
 import java.nio.channels.WritableByteChannel;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.NoSuchFileException;
-import java.util.Map;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
@@ -132,13 +132,34 @@ Map<String, BlobMetaData> listBlobs(String path) throws IOException {
     Map<String, BlobMetaData> listBlobsByPrefix(String path, String prefix) throws IOException {
         final String pathPrefix = buildKey(path, prefix);
         final MapBuilder<String, BlobMetaData> mapBuilder = MapBuilder.newMapBuilder();
-        SocketAccess.doPrivilegedVoidIOException(() -> {
-            client().get(bucketName).list(BlobListOption.prefix(pathPrefix)).iterateAll().forEach(blob -> {
-                assert blob.getName().startsWith(path);
-                final String suffixName = blob.getName().substring(path.length());
-                mapBuilder.put(suffixName, new PlainBlobMetaData(suffixName, blob.getSize()));
-            });
-        });
+        SocketAccess.doPrivilegedVoidIOException(
+            () -> client().get(bucketName).list(BlobListOption.currentDirectory(), BlobListOption.prefix(pathPrefix)).iterateAll().forEach(
+                blob -> {
+                    assert blob.getName().startsWith(path);
+                    if (blob.isDirectory() == false) {
+                        final String suffixName = blob.getName().substring(path.length());
+                        mapBuilder.put(suffixName, new PlainBlobMetaData(suffixName, blob.getSize()));
+                    }
+                }));
+        return mapBuilder.immutableMap();
+    }
+
+    Map<String, BlobContainer> listChildren(BlobPath path) throws IOException {
+        final String pathStr = path.buildAsString();
+        final MapBuilder<String, BlobContainer> mapBuilder = MapBuilder.newMapBuilder();
+        SocketAccess.doPrivilegedVoidIOException
+            (() -> client().get(bucketName).list(BlobListOption.currentDirectory(), BlobListOption.prefix(pathStr)).iterateAll().forEach(
+                blob -> {
+                    if (blob.isDirectory()) {
+                        assert blob.getName().startsWith(pathStr);
+                        assert blob.getName().endsWith("/");
+                        // Strip path prefix and trailing slash
+                        final String suffixName = blob.getName().substring(pathStr.length(), blob.getName().length() - 1);
+                        if (suffixName.isEmpty() == false) {
+                            mapBuilder.put(suffixName, new GoogleCloudStorageBlobContainer(path.add(suffixName), this));
+                        }
+                    }
+                }));
         return mapBuilder.immutableMap();
     }
 
@@ -286,6 +307,23 @@ void deleteBlob(String blobName) throws IOException {
         }
     }
 
+    /**
+     * Deletes the given path and all its children.
+     *
+     * @param pathStr Name of path to delete
+     */
+    void deleteDirectory(String pathStr) throws IOException {
+        SocketAccess.doPrivilegedVoidIOException(() -> {
+            Page<Blob> page = client().get(bucketName).list(BlobListOption.prefix(pathStr));
+            do {
+                final Collection<String> blobsToDelete = new ArrayList<>();
+                page.getValues().forEach(b -> blobsToDelete.add(b.getName()));
+                deleteBlobsIgnoringIfNotExists(blobsToDelete);
+                page = page.getNextPage();
+            } while (page != null);
+        });
+    }
+
     /**
      * Deletes multiple blobs from the specific bucket using a batch request
      *
@@ -295,11 +333,6 @@ void deleteBlobsIgnoringIfNotExists(Collection<String> blobNames) throws IOExcep
         if (blobNames.isEmpty()) {
             return;
         }
-        // for a single op submit a simple delete instead of a batch of size 1
-        if (blobNames.size() == 1) {
-            deleteBlob(blobNames.iterator().next());
-            return;
-        }
         final List<BlobId> blobIdsToDelete = blobNames.stream().map(blob -> BlobId.of(bucketName, blob)).collect(Collectors.toList());
         final List<BlobId> failedBlobs = Collections.synchronizedList(new ArrayList<>());
         final StorageException e = SocketAccess.doPrivilegedIOException(() -> {
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
index 804fafd5e855e..6382a537c4682 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
@@ -57,7 +57,6 @@ class GoogleCloudStorageRepository extends BlobStoreRepository {
     static final Setting<String> CLIENT_NAME = new Setting<>("client", "default", Function.identity());
 
     private final GoogleCloudStorageService storageService;
-    private final BlobPath basePath;
     private final ByteSizeValue chunkSize;
     private final String bucket;
     private final String clientName;
@@ -65,24 +64,27 @@ class GoogleCloudStorageRepository extends BlobStoreRepository {
     GoogleCloudStorageRepository(RepositoryMetaData metadata, Environment environment,
                                         NamedXContentRegistry namedXContentRegistry,
                                         GoogleCloudStorageService storageService, ThreadPool threadPool) {
-        super(metadata, environment.settings(), namedXContentRegistry, threadPool);
+        super(metadata, environment.settings(), namedXContentRegistry, threadPool, buildBasePath(metadata));
         this.storageService = storageService;
 
+        this.chunkSize = getSetting(CHUNK_SIZE, metadata);
+        this.bucket = getSetting(BUCKET, metadata);
+        this.clientName = CLIENT_NAME.get(metadata.settings());
+        logger.debug(
+            "using bucket [{}], base_path [{}], chunk_size [{}], compress [{}]", bucket, basePath(), chunkSize, isCompress());
+    }
+
+    private static BlobPath buildBasePath(RepositoryMetaData metadata) {
         String basePath = BASE_PATH.get(metadata.settings());
         if (Strings.hasLength(basePath)) {
             BlobPath path = new BlobPath();
             for (String elem : basePath.split("/")) {
                 path = path.add(elem);
             }
-            this.basePath = path;
+            return path;
         } else {
-            this.basePath = BlobPath.cleanPath();
+            return BlobPath.cleanPath();
         }
-
-        this.chunkSize = getSetting(CHUNK_SIZE, metadata);
-        this.bucket = getSetting(BUCKET, metadata);
-        this.clientName = CLIENT_NAME.get(metadata.settings());
-        logger.debug("using bucket [{}], base_path [{}], chunk_size [{}], compress [{}]", bucket, basePath, chunkSize, isCompress());
     }
 
     @Override
@@ -90,11 +92,6 @@ protected GoogleCloudStorageBlobStore createBlobStore() {
         return new GoogleCloudStorageBlobStore(bucket, clientName, storageService);
     }
 
-    @Override
-    protected BlobPath basePath() {
-        return basePath;
-    }
-
     @Override
     protected ByteSizeValue chunkSize() {
         return chunkSize;
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
index f30b8b47d82b8..970296eff95f4 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
@@ -21,7 +21,6 @@
 
 import com.google.api.client.googleapis.GoogleUtils;
 import com.google.api.client.http.HttpTransport;
-import com.google.api.client.http.javanet.DefaultConnectionFactory;
 import com.google.api.client.http.javanet.NetHttpTransport;
 import com.google.auth.oauth2.ServiceAccountCredentials;
 import com.google.cloud.http.HttpTransportOptions;
@@ -36,10 +35,7 @@
 import org.elasticsearch.common.util.LazyInitializable;
 
 import java.io.IOException;
-import java.net.HttpURLConnection;
 import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
@@ -106,10 +102,16 @@ public Storage client(final String clientName) throws IOException {
      * @return a new client storage instance that can be used to manage objects
      *         (blobs)
      */
-    private Storage createClient(final String clientName, final GoogleCloudStorageClientSettings clientSettings) throws IOException {
+    private static Storage createClient(String clientName, GoogleCloudStorageClientSettings clientSettings) throws IOException {
         logger.debug(() -> new ParameterizedMessage("creating GCS client with client_name [{}], endpoint [{}]", clientName,
                 clientSettings.getHost()));
-        final HttpTransport httpTransport = SocketAccess.doPrivilegedIOException(() -> createHttpTransport(clientSettings.getHost()));
+        final HttpTransport httpTransport = SocketAccess.doPrivilegedIOException(() -> {
+            final NetHttpTransport.Builder builder = new NetHttpTransport.Builder();
+            // requires java.lang.RuntimePermission "setFactory"
+            // Pin the TLS trust certificates.
+            builder.trustCertificates(GoogleUtils.getCertificateTrustStore());
+            return builder.build();
+        });
         final HttpTransportOptions httpTransportOptions = HttpTransportOptions.newBuilder()
                 .setConnectTimeout(toTimeout(clientSettings.getConnectTimeout()))
                 .setReadTimeout(toTimeout(clientSettings.getReadTimeout()))
@@ -147,54 +149,6 @@ private Storage createClient(final String clientName, final GoogleCloudStorageCl
         return storageOptionsBuilder.build().getService();
     }
 
-    /**
-     * Pins the TLS trust certificates and, more importantly, overrides connection
-     * URLs in the case of a custom endpoint setting because some connections don't
-     * fully honor this setting (bugs in the SDK). The default connection factory
-     * opens a new connection for each request. This is required for the storage
-     * instance to be thread-safe.
-     **/
-    private static HttpTransport createHttpTransport(final String endpoint) throws Exception {
-        final NetHttpTransport.Builder builder = new NetHttpTransport.Builder();
-        // requires java.lang.RuntimePermission "setFactory"
-        builder.trustCertificates(GoogleUtils.getCertificateTrustStore());
-        if (Strings.hasLength(endpoint)) {
-            final URL endpointUrl = URI.create(endpoint).toURL();
-            // it is crucial to open a connection for each URL (see {@code
-            // DefaultConnectionFactory#openConnection}) instead of reusing connections,
-            // because the storage instance has to be thread-safe as it is cached.
-            builder.setConnectionFactory(new DefaultConnectionFactory() {
-                @Override
-                public HttpURLConnection openConnection(final URL originalUrl) throws IOException {
-                    // test if the URL is built correctly, ie following the `host` setting
-                    if (originalUrl.getHost().equals(endpointUrl.getHost()) && originalUrl.getPort() == endpointUrl.getPort()
-                            && originalUrl.getProtocol().equals(endpointUrl.getProtocol())) {
-                        return super.openConnection(originalUrl);
-                    }
-                    // override connection URLs because some don't follow the config. See
-                    // https://github.com/GoogleCloudPlatform/google-cloud-java/issues/3254 and
-                    // https://github.com/GoogleCloudPlatform/google-cloud-java/issues/3255
-                    URI originalUri;
-                    try {
-                        originalUri = originalUrl.toURI();
-                    } catch (final URISyntaxException e) {
-                        throw new RuntimeException(e);
-                    }
-                    String overridePath = "/";
-                    if (originalUri.getRawPath() != null) {
-                        overridePath = originalUri.getRawPath();
-                    }
-                    if (originalUri.getRawQuery() != null) {
-                        overridePath += "?" + originalUri.getRawQuery();
-                    }
-                    return super.openConnection(
-                            new URL(endpointUrl.getProtocol(), endpointUrl.getHost(), endpointUrl.getPort(), overridePath));
-                }
-            });
-        }
-        return builder.build();
-    }
-
     /**
      * Converts timeout values from the settings to a timeout value for the Google
      * Cloud SDK
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java
new file mode 100644
index 0000000000000..06eb63ddd22f0
--- /dev/null
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.gcs;
+
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.SecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase;
+
+import java.util.Base64;
+import java.util.Collection;
+
+import static org.hamcrest.Matchers.blankOrNullString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
+public class GoogleCloudStorageThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(GoogleCloudStoragePlugin.class);
+    }
+
+    @Override
+    protected SecureSettings credentials() {
+        assertThat(System.getProperty("test.google.account"), not(blankOrNullString()));
+        assertThat(System.getProperty("test.google.bucket"), not(blankOrNullString()));
+
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setFile("gcs.client.default.credentials_file",
+            Base64.getDecoder().decode(System.getProperty("test.google.account")));
+        return secureSettings;
+    }
+
+    @Override
+    protected void createRepository(final String repoName) {
+        AcknowledgedResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo")
+            .setType("gcs")
+            .setSettings(Settings.builder()
+                .put("bucket", System.getProperty("test.google.bucket"))
+                .put("base_path", System.getProperty("test.google.base", "/"))
+            ).get();
+        assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
+    }
+}
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockStorage.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockStorage.java
index eddf2a9f78082..ca6ca60e41e95 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockStorage.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockStorage.java
@@ -281,6 +281,11 @@ public void close() {
         return null;
     }
 
+    @Override
+    public WriteChannel writer(URL signedURL) {
+        return null;
+    }
+
     // Everything below this line is not implemented.
 
     @Override
@@ -288,6 +293,11 @@ public CopyWriter copy(CopyRequest copyRequest) {
         return null;
     }
 
+    @Override
+    public Blob create(BlobInfo blobInfo, byte[] content, int offset, int length, BlobTargetOption... options) {
+        return null;
+    }
+
     @Override
     public Bucket create(BucketInfo bucketInfo, BucketTargetOption... options) {
         return null;
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index 3d6dcd29d6d19..3060867b01b08 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -24,6 +24,8 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
 
+import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
+
 apply plugin: 'elasticsearch.test.fixtures'
 
 esplugin {
@@ -52,6 +54,7 @@ dependencies {
   compile 'com.google.guava:guava:11.0.2'
   compile 'com.google.protobuf:protobuf-java:2.5.0'
   compile 'commons-logging:commons-logging:1.1.3'
+  compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   compile 'commons-cli:commons-cli:1.2'
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile 'commons-collections:commons-collections:3.2.2'
@@ -65,7 +68,14 @@ dependencies {
   hdfsFixture project(':test:fixtures:hdfs-fixture')
   // Set the keytab files in the classpath so that we can access them from test code without the security manager
   // freaking out.
-  testRuntime fileTree(dir: project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs","hdfs_hdfs.build.elastic.co.keytab").parent, include: ['*.keytab'])
+  testRuntime files(project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs","hdfs_hdfs.build.elastic.co.keytab").parent)
+}
+
+normalization {
+    runtimeClasspath {
+        // ignore generated keytab files for the purposes of build avoidance
+        ignore '*.keytab'
+    }
 }
 
 dependencyLicenses {
@@ -86,7 +96,7 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture',
       // the hdfs.MiniHDFS fixture writes the ports file when
       // it's ready, so we can just wait for the file to exist
       return fixture.portsFile.exists()
-    }    
+    }
 
     final List<String> miniHDFSArgs = []
 
@@ -115,20 +125,29 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture',
   }
 }
 
+Set disabledIntegTestTaskNames = ['integTestSecure', 'integTestSecureHa']
+
 for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSecureHa']) {
   task "${integTestTaskName}"(type: RestIntegTestTask) {
     description = "Runs rest tests against an elasticsearch cluster with HDFS."
     dependsOn(project.bundlePlugin)
+
+    if (disabledIntegTestTaskNames.contains(integTestTaskName)) {
+      enabled = false;
+    }
+
     runner {
       if (integTestTaskName.contains("Secure")) {
-        dependsOn secureHdfsFixture
-        systemProperty "test.krb5.principal.es", "elasticsearch@${realm}"
-        systemProperty "test.krb5.principal.hdfs", "hdfs/hdfs.build.elastic.co@${realm}"
-        jvmArgs "-Djava.security.krb5.conf=${krb5conf}"
-        systemProperty (
-                "test.krb5.keytab.hdfs",
-                project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs","hdfs_hdfs.build.elastic.co.keytab")
-        )
+        if (disabledIntegTestTaskNames.contains(integTestTaskName) == false) {
+            dependsOn secureHdfsFixture
+            nonInputProperties.systemProperty "test.krb5.principal.es", "elasticsearch@${realm}"
+            nonInputProperties.systemProperty "test.krb5.principal.hdfs", "hdfs/hdfs.build.elastic.co@${realm}"
+            jvmArgs "-Djava.security.krb5.conf=${krb5conf}"
+            nonInputProperties.systemProperty (
+                    "test.krb5.keytab.hdfs",
+                    project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs","hdfs_hdfs.build.elastic.co.keytab")
+            )
+        }
       }
     }
   }
@@ -139,7 +158,7 @@ for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSec
       systemProperty "java.security.krb5.conf", krb5conf
       extraConfigFile(
               "repository-hdfs/krb5.keytab",
-              file("${project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "elasticsearch.keytab")}")
+              file("${project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "elasticsearch.keytab")}"), IGNORE_VALUE
       )
     }
   }
diff --git a/plugins/repository-hdfs/licenses/log4j-1.2-api-2.11.1.jar.sha1 b/plugins/repository-hdfs/licenses/log4j-1.2-api-2.11.1.jar.sha1
new file mode 100644
index 0000000000000..575d75dbda8c5
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/log4j-1.2-api-2.11.1.jar.sha1
@@ -0,0 +1 @@
+3aba3398fe064a3eab4331f88161c7480e848418
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/log4j-LICENSE.txt b/plugins/repository-hdfs/licenses/log4j-LICENSE.txt
new file mode 100644
index 0000000000000..6279e5206de13
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/log4j-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 1999-2005 The Apache Software Foundation
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-hdfs/licenses/log4j-NOTICE.txt b/plugins/repository-hdfs/licenses/log4j-NOTICE.txt
new file mode 100644
index 0000000000000..0375732360047
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/log4j-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache log4j
+Copyright 2007 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
\ No newline at end of file
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
index e9b45a9b52e70..b050645f9952c 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
@@ -25,6 +25,7 @@
 import org.apache.hadoop.fs.Options.CreateOpts;
 import org.apache.hadoop.fs.Path;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.fs.FsBlobContainer;
@@ -77,6 +78,11 @@ public void deleteBlob(String blobName) throws IOException {
         }
     }
 
+    @Override
+    public void delete() throws IOException {
+        store.execute(fileContext -> fileContext.delete(path, true));
+    }
+
     @Override
     public InputStream readBlob(String blobName) throws IOException {
         // FSDataInputStream does buffering internally
@@ -137,11 +143,13 @@ public void writeBlobAtomic(String blobName, InputStream inputStream, long blobS
 
     @Override
     public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable final String prefix) throws IOException {
-        FileStatus[] files = store.execute(fileContext -> (fileContext.util().listStatus(path,
-            path -> prefix == null || path.getName().startsWith(prefix))));
-        Map<String, BlobMetaData> map = new LinkedHashMap<String, BlobMetaData>();
+        FileStatus[] files = store.execute(fileContext -> fileContext.util().listStatus(path,
+            path -> prefix == null || path.getName().startsWith(prefix)));
+        Map<String, BlobMetaData> map = new LinkedHashMap<>();
         for (FileStatus file : files) {
-            map.put(file.getPath().getName(), new PlainBlobMetaData(file.getPath().getName(), file.getLen()));
+            if (file.isFile()) {
+                map.put(file.getPath().getName(), new PlainBlobMetaData(file.getPath().getName(), file.getLen()));
+            }
         }
         return Collections.unmodifiableMap(map);
     }
@@ -151,6 +159,19 @@ public Map<String, BlobMetaData> listBlobs() throws IOException {
         return listBlobsByPrefix(null);
     }
 
+    @Override
+    public Map<String, BlobContainer> children() throws IOException {
+        FileStatus[] files = store.execute(fileContext -> fileContext.util().listStatus(path));
+        Map<String, BlobContainer> map = new LinkedHashMap<>();
+        for (FileStatus file : files) {
+            if (file.isDirectory()) {
+                final String name = file.getPath().getName();
+                map.put(name, new HdfsBlobContainer(path().add(name), store, new Path(path, name), bufferSize, securityContext));
+            }
+        }
+        return Collections.unmodifiableMap(map);
+    }
+
     /**
      * Exists to wrap underlying InputStream methods that might make socket connections in
      * doPrivileged blocks. This is due to the way that hdfs client libraries might open
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
index b614753d83883..b51f843162a74 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
@@ -59,7 +59,6 @@ public final class HdfsRepository extends BlobStoreRepository {
 
     private final Environment environment;
     private final ByteSizeValue chunkSize;
-    private final BlobPath basePath = BlobPath.cleanPath();
     private final URI uri;
     private final String pathSetting;
 
@@ -69,7 +68,7 @@ public final class HdfsRepository extends BlobStoreRepository {
 
     public HdfsRepository(RepositoryMetaData metadata, Environment environment,
                           NamedXContentRegistry namedXContentRegistry, ThreadPool threadPool) {
-        super(metadata, environment.settings(), namedXContentRegistry, threadPool);
+        super(metadata, environment.settings(), namedXContentRegistry, threadPool, BlobPath.cleanPath());
 
         this.environment = environment;
         this.chunkSize = metadata.settings().getAsBytesSize("chunk_size", null);
@@ -233,11 +232,6 @@ protected HdfsBlobStore createBlobStore() {
         return blobStore;
     }
 
-    @Override
-    protected BlobPath basePath() {
-        return basePath;
-    }
-
     @Override
     protected ByteSizeValue chunkSize() {
         return chunkSize;
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsRepositoryTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsRepositoryTests.java
new file mode 100644
index 0000000000000..e34f290a8e299
--- /dev/null
+++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsRepositoryTests.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.repositories.hdfs;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.bootstrap.JavaVersion;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.SecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase;
+
+import java.util.Collection;
+
+import static org.hamcrest.Matchers.equalTo;
+
+@ThreadLeakFilters(filters = HdfsClientThreadLeakFilter.class)
+public class HdfsRepositoryTests extends AbstractThirdPartyRepositoryTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(HdfsPlugin.class);
+    }
+
+    @Override
+    protected SecureSettings credentials() {
+        return new MockSecureSettings();
+    }
+
+    @Override
+    protected void createRepository(String repoName) {
+        assumeFalse("https://github.com/elastic/elasticsearch/issues/31498", JavaVersion.current().equals(JavaVersion.parse("11")));
+        AcknowledgedResponse putRepositoryResponse = client().admin().cluster().preparePutRepository(repoName)
+            .setType("hdfs")
+            .setSettings(Settings.builder()
+                .put("uri", "hdfs:///")
+                .put("conf.fs.AbstractFileSystem.hdfs.impl", TestingFs.class.getName())
+                .put("path", "foo")
+                .put("chunk_size", randomIntBetween(100, 1000) + "k")
+                .put("compress", randomBoolean())
+            ).get();
+        assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
+    }
+}
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java
index 88454188da588..41423d25b9f9f 100644
--- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java
+++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java
@@ -27,9 +27,13 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.RepositoryException;
+import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
+import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil;
 import org.elasticsearch.snapshots.SnapshotState;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Collection;
 
@@ -91,7 +95,7 @@ public void testSimpleWorkflow() {
             .prepareGetSnapshots("test-repo")
             .setSnapshots("test-snap")
             .get()
-            .getSnapshots()
+            .getSnapshots("test-repo")
             .get(0)
             .state(),
             equalTo(SnapshotState.SUCCESS));
@@ -145,6 +149,9 @@ public void testSimpleWorkflow() {
         ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
         assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true));
         assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false));
+        final BlobStoreRepository repo =
+            (BlobStoreRepository) getInstanceFromNode(RepositoriesService.class).repository("test-repo");
+        BlobStoreTestUtil.assertConsistency(repo, repo.threadPool().executor(ThreadPool.Names.GENERIC));
     }
 
     public void testMissingUri() {
diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_get.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_get.yml
index f38f4783b195b..6ed93e07160b2 100644
--- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_get.yml
+++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_get.yml
@@ -46,8 +46,8 @@
         repository: test_snapshot_get_repository
         snapshot: test_snapshot_get
 
-  - length: { snapshots: 1 }
-  - match: { snapshots.0.snapshot : test_snapshot_get }
+  - length: { responses.0.snapshots: 1 }
+  - match: { responses.0.snapshots.0.snapshot : test_snapshot_get }
 
   # List snapshot info
   - do:
@@ -55,8 +55,8 @@
         repository: test_snapshot_get_repository
         snapshot: "*"
 
-  - length: { snapshots: 1 }
-  - match: { snapshots.0.snapshot : test_snapshot_get }
+  - length: { responses.0.snapshots: 1 }
+  - match: { responses.0.snapshots.0.snapshot : test_snapshot_get }
 
   # Remove our snapshot
   - do: 
diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yml
index c2a37964e70a7..dda910ae36c26 100644
--- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yml
+++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yml
@@ -21,7 +21,7 @@
         repository: test_snapshot_repository_ro
         snapshot: "_all"
 
-  - length: { snapshots: 1 }
+  - length: { responses.0.snapshots: 1 }
 
   # Remove our repository
   - do:
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index 60a4e97cfa493..b6bf60c1c92e5 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -1,8 +1,9 @@
-import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.test.AntFixture
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
+import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -28,17 +29,17 @@ esplugin {
 }
 
 versions << [
-  'aws': '1.11.505'
+  'aws': '1.11.562'
 ]
 
 dependencies {
   compile "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
-  compile "com.amazonaws:aws-java-sdk-kms:${versions.aws}"
   compile "com.amazonaws:aws-java-sdk-core:${versions.aws}"
   compile "com.amazonaws:jmespath-java:${versions.aws}"
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
+  compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
   compile 'com.fasterxml.jackson.core:jackson-databind:2.8.11.3'
@@ -75,6 +76,7 @@ test {
   // these are tested explicitly in separate test tasks
   exclude '**/*CredentialsTests.class'
   exclude '**/S3BlobStoreRepositoryTests.class'
+  exclude '**/S3RepositoryThirdPartyTests.class'
 }
 
 boolean useFixture = false
@@ -134,6 +136,14 @@ if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
   throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
 }
 
+task thirdPartyTest(type: Test) {
+  include '**/S3RepositoryThirdPartyTests.class'
+  systemProperty 'test.s3.account', s3PermanentAccessKey
+  systemProperty 'test.s3.key', s3PermanentSecretKey
+  systemProperty 'test.s3.bucket', s3PermanentBucket
+  systemProperty 'test.s3.base', s3PermanentBasePath
+}
+
 if (useFixture) {
   apply plugin: 'elasticsearch.test.fixtures'
   task writeDockerFile {
@@ -151,6 +161,31 @@ if (useFixture) {
     dependsOn(writeDockerFile)
   }
 
+  def minioAddress = {
+    int minioPort = postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000"
+    assert minioPort > 0
+    'http://127.0.0.1:' + minioPort
+  }
+
+  File minioAddressFile = new File(project.buildDir, 'generated-resources/s3Fixture.address')
+
+  normalization {
+    runtimeClasspath {
+      // ignore generated address file for the purposes of build avoidance
+      ignore 's3Fixture.address'
+    }
+  }
+
+  thirdPartyTest {
+    dependsOn tasks.bundlePlugin, tasks.postProcessFixture
+    outputs.file(minioAddressFile)
+    doFirst {
+      file(minioAddressFile).text = "${ -> minioAddress.call() }"
+    }
+    // TODO: this could be a nonInputProperties.systemProperty so we don't need a file
+    systemProperty 'test.s3.endpoint', minioAddressFile.name
+  }
+
   task integTestMinio(type: RestIntegTestTask) {
     description = "Runs REST tests using the Minio repository."
     dependsOn tasks.bundlePlugin, tasks.postProcessFixture
@@ -164,16 +199,11 @@ if (useFixture) {
     }
   }
   check.dependsOn(integTestMinio)
-  BuildPlugin.requireDocker(tasks.integTestMinio)
 
   testClusters.integTestMinio {
     keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
     keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
-    setting 's3.client.integration_test_permanent.endpoint', {
-      int minioPort = postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000"
-      assert minioPort > 0
-      return 'http://127.0.0.1:' + minioPort
-    }
+    setting 's3.client.integration_test_permanent.endpoint', minioAddress, IGNORE_VALUE
     plugin file(tasks.bundlePlugin.archiveFile)
   }
 
@@ -191,6 +221,8 @@ if (useFixture) {
   }
 }
 
+check.dependsOn(thirdPartyTest)
+
 File parentFixtures = new File(project.buildDir, "fixtures")
 File s3FixtureFile = new File(parentFixtures, 's3Fixture.properties')
 
@@ -251,12 +283,12 @@ testClusters.integTest {
   keystore 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
 
   if (useFixture) {
-    setting 's3.client.integration_test_permanent.endpoint', { "http://${s3Fixture.addressAndPort}" }
-    setting 's3.client.integration_test_temporary.endpoint', { "http://${s3Fixture.addressAndPort}" }
-    setting 's3.client.integration_test_ec2.endpoint', { "http://${s3Fixture.addressAndPort}" }
+    setting 's3.client.integration_test_permanent.endpoint', { "http://${s3Fixture.addressAndPort}" }, IGNORE_VALUE
+    setting 's3.client.integration_test_temporary.endpoint', { "http://${s3Fixture.addressAndPort}" }, IGNORE_VALUE
+    setting 's3.client.integration_test_ec2.endpoint', { "http://${s3Fixture.addressAndPort}" }, IGNORE_VALUE
 
     // to redirect InstanceProfileCredentialsProvider to custom auth point
-    systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${s3Fixture.addressAndPort}" }
+    systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${s3Fixture.addressAndPort}" }, IGNORE_VALUE
   } else {
     println "Using an external service to test the repository-s3 plugin"
   }
@@ -278,9 +310,9 @@ if (useFixture) {
   check.dependsOn(integTestECS)
 
   testClusters.integTestECS {
-    setting 's3.client.integration_test_ecs.endpoint', { "http://${s3Fixture.addressAndPort}" }
+    setting 's3.client.integration_test_ecs.endpoint', { "http://${s3Fixture.addressAndPort}" }, IGNORE_VALUE
     plugin file(tasks.bundlePlugin.archiveFile)
-    environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "http://${s3Fixture.addressAndPort}/ecs_credentials_endpoint" }
+    environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "http://${s3Fixture.addressAndPort}/ecs_credentials_endpoint" }, IGNORE_VALUE
   }
 }
 
diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.505.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.505.jar.sha1
deleted file mode 100644
index add5db290e8a8..0000000000000
--- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.505.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d19328c227b2b5ad81d137361ebc9cbcd0396465
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.562.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.562.jar.sha1
new file mode 100644
index 0000000000000..ed8ded6a3608c
--- /dev/null
+++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.562.jar.sha1
@@ -0,0 +1 @@
+b5fc47ec1b5afe180f5ebb4eda755acdca7a20ae
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.505.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.505.jar.sha1
deleted file mode 100644
index ab2bfdbc1897b..0000000000000
--- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.505.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2a219919090a6cadd7e119c899c90343ad9c0077
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.505.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.505.jar.sha1
deleted file mode 100644
index 04c4cc213c690..0000000000000
--- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.505.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b4cf82765b04a579609314ab7f296a9a0ddae1cf
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.562.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.562.jar.sha1
new file mode 100644
index 0000000000000..8e852fe9b275f
--- /dev/null
+++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.562.jar.sha1
@@ -0,0 +1 @@
+1712c878f7e9483ceac1eb2356a9457a3c8df03e
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.505.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.505.jar.sha1
deleted file mode 100644
index 803d4fe85d77c..0000000000000
--- a/plugins/repository-s3/licenses/jmespath-java-1.11.505.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-067234d307b210097e247a49f08875e0cd3f2b95
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.562.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.562.jar.sha1
new file mode 100644
index 0000000000000..8e2d0e1935a3f
--- /dev/null
+++ b/plugins/repository-s3/licenses/jmespath-java-1.11.562.jar.sha1
@@ -0,0 +1 @@
+1147ed0ad1f2c5a16b8271e38e3cda5cd488c8ae
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/log4j-1.2-api-2.11.1.jar.sha1 b/plugins/repository-s3/licenses/log4j-1.2-api-2.11.1.jar.sha1
new file mode 100644
index 0000000000000..575d75dbda8c5
--- /dev/null
+++ b/plugins/repository-s3/licenses/log4j-1.2-api-2.11.1.jar.sha1
@@ -0,0 +1 @@
+3aba3398fe064a3eab4331f88161c7480e848418
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/log4j-LICENSE.txt b/plugins/repository-s3/licenses/log4j-LICENSE.txt
new file mode 100644
index 0000000000000..6279e5206de13
--- /dev/null
+++ b/plugins/repository-s3/licenses/log4j-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 1999-2005 The Apache Software Foundation
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-s3/licenses/log4j-NOTICE.txt b/plugins/repository-s3/licenses/log4j-NOTICE.txt
new file mode 100644
index 0000000000000..0375732360047
--- /dev/null
+++ b/plugins/repository-s3/licenses/log4j-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache log4j
+Copyright 2007 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
\ No newline at end of file
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
index aabdd8b52e8e1..c2e1f3de7f0dd 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java
@@ -25,6 +25,7 @@
 import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
 import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
 import com.amazonaws.services.s3.model.MultiObjectDeleteException;
 import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.ObjectMetadata;
@@ -38,6 +39,7 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStoreException;
@@ -50,6 +52,7 @@
 import java.io.InputStream;
 import java.nio.file.NoSuchFileException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -127,12 +130,53 @@ public void deleteBlob(String blobName) throws IOException {
         deleteBlobIgnoringIfNotExists(blobName);
     }
 
+    @Override
+    public void delete() throws IOException {
+        try (AmazonS3Reference clientReference = blobStore.clientReference()) {
+            ObjectListing prevListing = null;
+            while (true) {
+                ObjectListing list;
+                if (prevListing != null) {
+                    final ObjectListing finalPrevListing = prevListing;
+                    list = SocketAccess.doPrivileged(() -> clientReference.client().listNextBatchOfObjects(finalPrevListing));
+                } else {
+                    final ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
+                    listObjectsRequest.setBucketName(blobStore.bucket());
+                    listObjectsRequest.setPrefix(keyPath);
+                    list = SocketAccess.doPrivileged(() -> clientReference.client().listObjects(listObjectsRequest));
+                }
+                final List<String> blobsToDelete =
+                    list.getObjectSummaries().stream().map(S3ObjectSummary::getKey).collect(Collectors.toList());
+                if (list.isTruncated()) {
+                    doDeleteBlobs(blobsToDelete, false);
+                    prevListing = list;
+                } else {
+                    final List<String> lastBlobsToDelete = new ArrayList<>(blobsToDelete);
+                    lastBlobsToDelete.add(keyPath);
+                    doDeleteBlobs(lastBlobsToDelete, false);
+                    break;
+                }
+            }
+        } catch (final AmazonClientException e) {
+            throw new IOException("Exception when deleting blob container [" + keyPath + "]", e);
+        }
+    }
+
     @Override
     public void deleteBlobsIgnoringIfNotExists(List<String> blobNames) throws IOException {
+        doDeleteBlobs(blobNames, true);
+    }
+
+    private void doDeleteBlobs(List<String> blobNames, boolean relative) throws IOException {
         if (blobNames.isEmpty()) {
             return;
         }
-        final Set<String> outstanding = blobNames.stream().map(this::buildKey).collect(Collectors.toSet());
+        final Set<String> outstanding;
+        if (relative) {
+            outstanding = blobNames.stream().map(this::buildKey).collect(Collectors.toSet());
+        } else {
+            outstanding = new HashSet<>(blobNames);
+        }
         try (AmazonS3Reference clientReference = blobStore.clientReference()) {
             // S3 API only allows 1k blobs per delete so we split up the given blobs into requests of max. 1k deletes
             final List<DeleteObjectsRequest> deleteRequests = new ArrayList<>();
@@ -203,12 +247,15 @@ public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String blobNamePref
                     final ObjectListing finalPrevListing = prevListing;
                     list = SocketAccess.doPrivileged(() -> clientReference.client().listNextBatchOfObjects(finalPrevListing));
                 } else {
+                    final ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
+                    listObjectsRequest.setBucketName(blobStore.bucket());
+                    listObjectsRequest.setDelimiter("/");
                     if (blobNamePrefix != null) {
-                        list = SocketAccess.doPrivileged(() -> clientReference.client().listObjects(blobStore.bucket(),
-                                buildKey(blobNamePrefix)));
+                        listObjectsRequest.setPrefix(buildKey(blobNamePrefix));
                     } else {
-                        list = SocketAccess.doPrivileged(() -> clientReference.client().listObjects(blobStore.bucket(), keyPath));
+                        listObjectsRequest.setPrefix(keyPath);
                     }
+                    list = SocketAccess.doPrivileged(() -> clientReference.client().listObjects(listObjectsRequest));
                 }
                 for (final S3ObjectSummary summary : list.getObjectSummaries()) {
                     final String name = summary.getKey().substring(keyPath.length());
@@ -231,6 +278,52 @@ public Map<String, BlobMetaData> listBlobs() throws IOException {
         return listBlobsByPrefix(null);
     }
 
+    @Override
+    public Map<String, BlobContainer> children() throws IOException {
+        try (AmazonS3Reference clientReference = blobStore.clientReference()) {
+            ObjectListing prevListing = null;
+            final var entries = new ArrayList<Map.Entry<String, BlobContainer>>();
+            while (true) {
+                ObjectListing list;
+                if (prevListing != null) {
+                    final ObjectListing finalPrevListing = prevListing;
+                    list = SocketAccess.doPrivileged(() -> clientReference.client().listNextBatchOfObjects(finalPrevListing));
+                } else {
+                    final ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
+                    listObjectsRequest.setBucketName(blobStore.bucket());
+                    listObjectsRequest.setPrefix(keyPath);
+                    listObjectsRequest.setDelimiter("/");
+                    list = SocketAccess.doPrivileged(() -> clientReference.client().listObjects(listObjectsRequest));
+                }
+                for (final String summary : list.getCommonPrefixes()) {
+                    final String name = summary.substring(keyPath.length());
+                    if (name.isEmpty() == false) {
+                        // Stripping the trailing slash off of the common prefix
+                        final String last = name.substring(0, name.length() - 1);
+                        final BlobPath path = path().add(last);
+                        entries.add(entry(last, blobStore.blobContainer(path)));
+                    }
+                }
+                assert list.getObjectSummaries().stream().noneMatch(s -> {
+                    for (String commonPrefix : list.getCommonPrefixes()) {
+                        if (s.getKey().substring(keyPath.length()).startsWith(commonPrefix)) {
+                            return true;
+                        }
+                    }
+                    return false;
+                }) : "Response contained children for listed common prefixes.";
+                if (list.isTruncated()) {
+                    prevListing = list;
+                } else {
+                    break;
+                }
+            }
+            return Maps.ofEntries(entries);
+        } catch (final AmazonClientException e) {
+            throw new IOException("Exception when listing children of [" +  path().buildAsString() + ']', e);
+        }
+    }
+
     private String buildKey(String blobName) {
         return keyPath + blobName;
     }
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index a91e7bf663f92..50b6e3265f9d3 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -150,8 +150,6 @@ class S3Repository extends BlobStoreRepository {
 
     private final ByteSizeValue chunkSize;
 
-    private final BlobPath basePath;
-
     private final boolean serverSideEncryption;
 
     private final String storageClass;
@@ -165,7 +163,7 @@ class S3Repository extends BlobStoreRepository {
                  final Settings settings,
                  final NamedXContentRegistry namedXContentRegistry,
                  final S3Service service, final ThreadPool threadPool) {
-        super(metadata, settings, namedXContentRegistry, threadPool);
+        super(metadata, settings, namedXContentRegistry, threadPool, buildBasePath(metadata));
         this.service = service;
 
         // Parse and validate the user's S3 Storage Class setting
@@ -183,13 +181,6 @@ class S3Repository extends BlobStoreRepository {
                 ") can't be lower than " + BUFFER_SIZE_SETTING.getKey() + " (" + bufferSize + ").");
         }
 
-        final String basePath = BASE_PATH_SETTING.get(metadata.settings());
-        if (Strings.hasLength(basePath)) {
-            this.basePath = new BlobPath().add(basePath);
-        } else {
-            this.basePath = BlobPath.cleanPath();
-        }
-
         this.serverSideEncryption = SERVER_SIDE_ENCRYPTION_SETTING.get(metadata.settings());
 
         this.storageClass = STORAGE_CLASS_SETTING.get(metadata.settings());
@@ -211,15 +202,18 @@ class S3Repository extends BlobStoreRepository {
                 storageClass);
     }
 
-    @Override
-    protected S3BlobStore createBlobStore() {
-        return new S3BlobStore(service, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass, metadata);
+    private static BlobPath buildBasePath(RepositoryMetaData metadata) {
+        final String basePath = BASE_PATH_SETTING.get(metadata.settings());
+        if (Strings.hasLength(basePath)) {
+            return new BlobPath().add(basePath);
+        } else {
+            return BlobPath.cleanPath();
+        }
     }
 
-    // only use for testing
     @Override
-    protected BlobStore blobStore() {
-        return super.blobStore();
+    protected S3BlobStore createBlobStore() {
+        return new S3BlobStore(service, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass, metadata);
     }
 
     // only use for testing
@@ -228,11 +222,6 @@ protected BlobStore getBlobStore() {
         return super.getBlobStore();
     }
 
-    @Override
-    protected BlobPath basePath() {
-        return basePath;
-    }
-
     @Override
     protected ByteSizeValue chunkSize() {
         return chunkSize;
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java
new file mode 100644
index 0000000000000..7e7ac8d430062
--- /dev/null
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.repositories.s3;
+
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.blobstore.BlobMetaData;
+import org.elasticsearch.common.blobstore.BlobPath;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.SecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase;
+import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
+import org.elasticsearch.test.StreamsUtils;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.Executor;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.hamcrest.Matchers.blankOrNullString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
+public class S3RepositoryThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(S3RepositoryPlugin.class);
+    }
+
+    @Override
+    protected SecureSettings credentials() {
+        assertThat(System.getProperty("test.s3.account"), not(blankOrNullString()));
+        assertThat(System.getProperty("test.s3.key"), not(blankOrNullString()));
+        assertThat(System.getProperty("test.s3.bucket"), not(blankOrNullString()));
+
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("s3.client.default.access_key", System.getProperty("test.s3.account"));
+        secureSettings.setString("s3.client.default.secret_key", System.getProperty("test.s3.key"));
+        return secureSettings;
+    }
+
+    @Override
+    protected void createRepository(String repoName) {
+        Settings.Builder settings = Settings.builder()
+            .put("bucket", System.getProperty("test.s3.bucket"))
+            .put("base_path", System.getProperty("test.s3.base", "testpath"));
+        final String endpointPath = System.getProperty("test.s3.endpoint");
+        if (endpointPath != null) {
+            try {
+                settings = settings.put("endpoint", StreamsUtils.copyToStringFromClasspath("/" + endpointPath));
+            } catch (IOException e) {
+                throw new AssertionError(e);
+            }
+        }
+        AcknowledgedResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo")
+            .setType("s3")
+            .setSettings(settings).get();
+        assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
+    }
+
+    @Override
+    protected boolean assertCorruptionVisible(BlobStoreRepository repo, Executor genericExec) throws Exception {
+        // S3 is only eventually consistent for the list operations used by this assertions so we retry for 10 minutes assuming that
+        // listing operations will become consistent within these 10 minutes.
+        assertBusy(() -> assertTrue(super.assertCorruptionVisible(repo, genericExec)), 10L, TimeUnit.MINUTES);
+        return true;
+    }
+
+    @Override
+    protected void assertConsistentRepository(BlobStoreRepository repo, Executor executor) throws Exception {
+        // S3 is only eventually consistent for the list operations used by this assertions so we retry for 10 minutes assuming that
+        // listing operations will become consistent within these 10 minutes.
+        assertBusy(() -> super.assertConsistentRepository(repo, executor), 10L, TimeUnit.MINUTES);
+    }
+
+    protected void assertBlobsByPrefix(BlobPath path, String prefix, Map<String, BlobMetaData> blobs) throws Exception {
+        // AWS S3 is eventually consistent so we retry for 10 minutes assuming a list operation will never take longer than that
+        // to become consistent.
+        assertBusy(() -> super.assertBlobsByPrefix(path, prefix, blobs), 10L, TimeUnit.MINUTES);
+    }
+
+    @Override
+    protected void assertChildren(BlobPath path, Collection<String> children) throws Exception {
+        // AWS S3 is eventually consistent so we retry for 10 minutes assuming a list operation will never take longer than that
+        // to become consistent.
+        assertBusy(() -> super.assertChildren(path, children), 10L, TimeUnit.MINUTES);
+    }
+
+    @Override
+    protected void assertDeleted(BlobPath path, String name) throws Exception {
+        // AWS S3 is eventually consistent so we retry for 10 minutes assuming a list operation will never take longer than that
+        // to become consistent.
+        assertBusy(() -> super.assertDeleted(path, name), 10L, TimeUnit.MINUTES);
+    }
+}
diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml
index e6c94f8c408d9..d319bf8984a97 100644
--- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml
+++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml
@@ -31,7 +31,6 @@ setup:
 
 ---
 "Try to create repository with broken endpoint override and named client":
-
   # Register repository with broken endpoint setting
   - do:
       catch: /repository_verification_exception/
@@ -108,6 +107,9 @@ setup:
 
 ---
 "Snapshot and Restore with repository-s3 using permanent credentials":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   # Get repository
   - do:
@@ -215,9 +217,9 @@ setup:
         repository: repository_permanent
         snapshot: snapshot-one,snapshot-two
 
-  - is_true: snapshots
-  - match: { snapshots.0.state : SUCCESS }
-  - match: { snapshots.1.state : SUCCESS }
+  - is_true: responses.0.snapshots
+  - match: { responses.0.snapshots.0.state : SUCCESS }
+  - match: { responses.0.snapshots.1.state : SUCCESS }
 
   # Delete the index
   - do:
@@ -322,13 +324,18 @@ setup:
 
 ---
 "Get a non existing snapshot":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   - do:
-      catch: /snapshot_missing_exception/
       snapshot.get:
         repository: repository_permanent
         snapshot: missing
 
+  - is_true: responses.0.error
+  - match: { responses.0.error.type: snapshot_missing_exception }
+
 ---
 "Delete a non existing snapshot":
 
diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml
index d5bdcd9c4f203..3ad6c3959634b 100644
--- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml
+++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml
@@ -18,6 +18,9 @@ setup:
 
 ---
 "Snapshot and Restore with repository-s3 using temporary credentials":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   # Get repository
   - do:
@@ -122,9 +125,9 @@ setup:
         repository: repository_temporary
         snapshot: snapshot-one,snapshot-two
 
-  - is_true: snapshots
-  - match: { snapshots.0.state : SUCCESS }
-  - match: { snapshots.1.state : SUCCESS }
+  - is_true: responses.0.snapshots
+  - match: { responses.0.snapshots.0.state : SUCCESS }
+  - match: { responses.0.snapshots.1.state : SUCCESS }
 
   # Delete the index
   - do:
@@ -229,13 +232,18 @@ setup:
 
 ---
 "Get a non existing snapshot":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   - do:
-      catch: /snapshot_missing_exception/
       snapshot.get:
         repository: repository_temporary
         snapshot: missing
 
+  - is_true: responses.0.error
+  - match: { responses.0.error.type: snapshot_missing_exception }
+
 ---
 "Delete a non existing snapshot":
 
diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml
index 829ae6197659c..fa1d3fc10fb13 100644
--- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml
+++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml
@@ -18,6 +18,9 @@ setup:
 
 ---
 "Snapshot and Restore with repository-s3 using ec2 credentials":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   # Get repository
   - do:
@@ -122,9 +125,9 @@ setup:
         repository: repository_ec2
         snapshot: snapshot-one,snapshot-two
 
-  - is_true: snapshots
-  - match: { snapshots.0.state : SUCCESS }
-  - match: { snapshots.1.state : SUCCESS }
+  - is_true: responses.0.snapshots
+  - match: { responses.0.snapshots.0.state : SUCCESS }
+  - match: { responses.0.snapshots.1.state : SUCCESS }
 
   # Delete the index
   - do:
@@ -229,13 +232,18 @@ setup:
 
 ---
 "Get a non existing snapshot":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   - do:
-      catch: /snapshot_missing_exception/
       snapshot.get:
         repository: repository_ec2
         snapshot: missing
 
+  - is_true: responses.0.error
+  - match: { responses.0.error.type: snapshot_missing_exception }
+
 ---
 "Delete a non existing snapshot":
 
diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml
index c59d3a32badc7..99736fb25ff24 100644
--- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml
+++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml
@@ -18,6 +18,9 @@ setup:
 
 ---
 "Snapshot and Restore with repository-s3 using ecs credentials":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   # Get repository
   - do:
@@ -122,9 +125,9 @@ setup:
         repository: repository_ecs
         snapshot: snapshot-one,snapshot-two
 
-  - is_true: snapshots
-  - match: { snapshots.0.state : SUCCESS }
-  - match: { snapshots.1.state : SUCCESS }
+  - is_true: responses.0.snapshots
+  - match: { responses.0.snapshots.0.state : SUCCESS }
+  - match: { responses.0.snapshots.1.state : SUCCESS }
 
   # Delete the index
   - do:
@@ -229,13 +232,18 @@ setup:
 
 ---
 "Get a non existing snapshot":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   - do:
-      catch: /snapshot_missing_exception/
       snapshot.get:
         repository: repository_ecs
         snapshot: missing
 
+  - is_true: responses.0.error
+  - match: { responses.0.error.type: snapshot_missing_exception }
+
 ---
 "Delete a non existing snapshot":
 
diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbmmapfs/SmbMmapFsDirectoryFactory.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbmmapfs/SmbMmapFsDirectoryFactory.java
index 13b6f9401abc5..ea604c7957027 100644
--- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbmmapfs/SmbMmapFsDirectoryFactory.java
+++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/smbmmapfs/SmbMmapFsDirectoryFactory.java
@@ -22,17 +22,20 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.LockFactory;
 import org.apache.lucene.store.MMapDirectory;
+import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.store.FsDirectoryFactory;
 import org.elasticsearch.index.store.SmbDirectoryWrapper;
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.HashSet;
 
 public final class SmbMmapFsDirectoryFactory extends FsDirectoryFactory {
 
     @Override
     protected Directory newFSDirectory(Path location, LockFactory lockFactory, IndexSettings indexSettings) throws IOException {
-        return new SmbDirectoryWrapper(new MMapDirectory(location, lockFactory));
+        return new SmbDirectoryWrapper(setPreload(new MMapDirectory(location, lockFactory), lockFactory, new HashSet<>(
+            indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING))));
     }
 }
diff --git a/plugins/transport-nio/build.gradle b/plugins/transport-nio/build.gradle
index 7800ff6951a89..bf9b42d0558b2 100644
--- a/plugins/transport-nio/build.gradle
+++ b/plugins/transport-nio/build.gradle
@@ -21,11 +21,10 @@ apply plugin: "nebula.maven-scm"
 esplugin {
     description 'The nio transport.'
     classname 'org.elasticsearch.transport.nio.NioTransportPlugin'
-    hasClientJar = true
 }
 
 dependencies {
-    compile "org.elasticsearch:elasticsearch-nio:${version}"
+    compile project(':libs:elasticsearch-nio')
 
     // network stack
     compile "io.netty:netty-buffer:${versions.netty}"
@@ -53,6 +52,10 @@ thirdPartyAudit {
         'org.apache.commons.logging.Log',
         'org.apache.commons.logging.LogFactory',
 
+        // from Log4j (deliberate, Netty will fallback to Log4j 2)
+        'org.apache.log4j.Level',
+        'org.apache.log4j.Logger',
+
         // from io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (netty)
         'org.bouncycastle.cert.X509v3CertificateBuilder',
         'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter',
diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/HttpReadWriteHandler.java b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/HttpReadWriteHandler.java
index 7a4fbfe42aefa..3c3406b0eda45 100644
--- a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/HttpReadWriteHandler.java
+++ b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/HttpReadWriteHandler.java
@@ -31,14 +31,14 @@
 import io.netty.handler.codec.http.HttpResponseEncoder;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.http.CorsHandler;
 import org.elasticsearch.http.HttpHandlingSettings;
 import org.elasticsearch.http.HttpPipelinedRequest;
 import org.elasticsearch.http.HttpReadTimeoutException;
-import org.elasticsearch.http.nio.cors.NioCorsConfig;
 import org.elasticsearch.http.nio.cors.NioCorsHandler;
 import org.elasticsearch.nio.FlushOperation;
 import org.elasticsearch.nio.InboundChannelBuffer;
-import org.elasticsearch.nio.ReadWriteHandler;
+import org.elasticsearch.nio.NioChannelHandler;
 import org.elasticsearch.nio.SocketChannelContext;
 import org.elasticsearch.nio.TaskScheduler;
 import org.elasticsearch.nio.WriteOperation;
@@ -50,7 +50,7 @@
 import java.util.function.BiConsumer;
 import java.util.function.LongSupplier;
 
-public class HttpReadWriteHandler implements ReadWriteHandler {
+public class HttpReadWriteHandler implements NioChannelHandler {
 
     private final NettyAdaptor adaptor;
     private final NioHttpChannel nioHttpChannel;
@@ -63,7 +63,7 @@ public class HttpReadWriteHandler implements ReadWriteHandler {
     private int inFlightRequests = 0;
 
     public HttpReadWriteHandler(NioHttpChannel nioHttpChannel, NioHttpServerTransport transport, HttpHandlingSettings settings,
-                                NioCorsConfig corsConfig, TaskScheduler taskScheduler, LongSupplier nanoClock) {
+                                CorsHandler.Config corsConfig, TaskScheduler taskScheduler, LongSupplier nanoClock) {
         this.nioHttpChannel = nioHttpChannel;
         this.transport = transport;
         this.taskScheduler = taskScheduler;
@@ -140,6 +140,11 @@ public List<FlushOperation> pollFlushOperations() {
         return copiedOperations;
     }
 
+    @Override
+    public boolean closeNow() {
+        return false;
+    }
+
     @Override
     public void close() throws IOException {
         try {
diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java
index fa0f3e9572c98..99a0d87e19c91 100644
--- a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java
+++ b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java
@@ -19,14 +19,11 @@
 
 package org.elasticsearch.http.nio;
 
-import io.netty.handler.codec.http.HttpMethod;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.PageCacheRecycler;
@@ -34,8 +31,6 @@
 import org.elasticsearch.http.AbstractHttpServerTransport;
 import org.elasticsearch.http.HttpChannel;
 import org.elasticsearch.http.HttpServerChannel;
-import org.elasticsearch.http.nio.cors.NioCorsConfig;
-import org.elasticsearch.http.nio.cors.NioCorsConfigBuilder;
 import org.elasticsearch.nio.BytesChannelContext;
 import org.elasticsearch.nio.ChannelFactory;
 import org.elasticsearch.nio.InboundChannelBuffer;
@@ -44,7 +39,6 @@
 import org.elasticsearch.nio.NioSocketChannel;
 import org.elasticsearch.nio.ServerChannelContext;
 import org.elasticsearch.nio.SocketChannelContext;
-import org.elasticsearch.rest.RestUtils;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.nio.NioGroupFactory;
 import org.elasticsearch.transport.nio.PageAllocator;
@@ -53,17 +47,8 @@
 import java.net.InetSocketAddress;
 import java.nio.channels.ServerSocketChannel;
 import java.nio.channels.SocketChannel;
-import java.util.Arrays;
 import java.util.function.Consumer;
-import java.util.regex.Pattern;
-import java.util.regex.PatternSyntaxException;
-
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_MAX_AGE;
+
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CHUNK_SIZE;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH;
@@ -73,12 +58,10 @@
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_TCP_REUSE_ADDRESS;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_TCP_SEND_BUFFER_SIZE;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_PIPELINING_MAX_EVENTS;
-import static org.elasticsearch.http.nio.cors.NioCorsHandler.ANY_ORIGIN;
 
 public class NioHttpServerTransport extends AbstractHttpServerTransport {
     private static final Logger logger = LogManager.getLogger(NioHttpServerTransport.class);
 
-    protected final NioCorsConfig corsConfig;
     protected final PageAllocator pageAllocator;
     private final NioGroupFactory nioGroupFactory;
 
@@ -102,7 +85,6 @@ public NioHttpServerTransport(Settings settings, NetworkService networkService,
         ByteSizeValue maxHeaderSize = SETTING_HTTP_MAX_HEADER_SIZE.get(settings);
         ByteSizeValue maxInitialLineLength = SETTING_HTTP_MAX_INITIAL_LINE_LENGTH.get(settings);
         int pipeliningMaxEvents = SETTING_PIPELINING_MAX_EVENTS.get(settings);
-        this.corsConfig = buildCorsConfig(settings);
 
         this.tcpNoDelay = SETTING_HTTP_TCP_NO_DELAY.get(settings);
         this.tcpKeepAlive = SETTING_HTTP_TCP_KEEP_ALIVE.get(settings);
@@ -155,42 +137,6 @@ protected ChannelFactory<NioHttpServerChannel, NioHttpChannel> channelFactory()
         return new HttpChannelFactory();
     }
 
-    static NioCorsConfig buildCorsConfig(Settings settings) {
-        if (SETTING_CORS_ENABLED.get(settings) == false) {
-            return NioCorsConfigBuilder.forOrigins().disable().build();
-        }
-        String origin = SETTING_CORS_ALLOW_ORIGIN.get(settings);
-        final NioCorsConfigBuilder builder;
-        if (Strings.isNullOrEmpty(origin)) {
-            builder = NioCorsConfigBuilder.forOrigins();
-        } else if (origin.equals(ANY_ORIGIN)) {
-            builder = NioCorsConfigBuilder.forAnyOrigin();
-        } else {
-            try {
-                Pattern p = RestUtils.checkCorsSettingForRegex(origin);
-                if (p == null) {
-                    builder = NioCorsConfigBuilder.forOrigins(RestUtils.corsSettingAsArray(origin));
-                } else {
-                    builder = NioCorsConfigBuilder.forPattern(p);
-                }
-            } catch (PatternSyntaxException e) {
-                throw new SettingsException("Bad regex in [" + SETTING_CORS_ALLOW_ORIGIN.getKey() + "]: [" + origin + "]", e);
-            }
-        }
-        if (SETTING_CORS_ALLOW_CREDENTIALS.get(settings)) {
-            builder.allowCredentials();
-        }
-        String[] strMethods = Strings.tokenizeToStringArray(SETTING_CORS_ALLOW_METHODS.get(settings), ",");
-        HttpMethod[] methods = Arrays.stream(strMethods)
-            .map(HttpMethod::valueOf)
-            .toArray(HttpMethod[]::new);
-        return builder.allowedRequestMethods(methods)
-            .maxAge(SETTING_CORS_MAX_AGE.get(settings))
-            .allowedRequestHeaders(Strings.tokenizeToStringArray(SETTING_CORS_ALLOW_HEADERS.get(settings), ","))
-            .shortCircuit()
-            .build();
-    }
-
     protected void acceptChannel(NioSocketChannel socketChannel) {
         super.serverAcceptedChannel((HttpChannel) socketChannel);
     }
diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsConfig.java b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsConfig.java
deleted file mode 100644
index 1ffffdf0d315d..0000000000000
--- a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsConfig.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.http.nio.cors;
-
-import io.netty.handler.codec.http.DefaultHttpHeaders;
-import io.netty.handler.codec.http.EmptyHttpHeaders;
-import io.netty.handler.codec.http.HttpHeaders;
-import io.netty.handler.codec.http.HttpMethod;
-
-import java.util.Collections;
-import java.util.LinkedHashSet;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.regex.Pattern;
-
-/**
- * Configuration for Cross-Origin Resource Sharing (CORS).
- *
- * This class was lifted from the Netty project:
- *  https://github.com/netty/netty
- */
-public final class NioCorsConfig {
-
-    private final Optional<Set<String>> origins;
-    private final Optional<Pattern> pattern;
-    private final boolean anyOrigin;
-    private final boolean enabled;
-    private final boolean allowCredentials;
-    private final long maxAge;
-    private final Set<HttpMethod> allowedRequestMethods;
-    private final Set<String> allowedRequestHeaders;
-    private final Map<CharSequence, Callable<?>> preflightHeaders;
-    private final boolean shortCircuit;
-
-    NioCorsConfig(final NioCorsConfigBuilder builder) {
-        origins = builder.origins.map(s -> new LinkedHashSet<>(s));
-        pattern = builder.pattern;
-        anyOrigin = builder.anyOrigin;
-        enabled = builder.enabled;
-        allowCredentials = builder.allowCredentials;
-        maxAge = builder.maxAge;
-        allowedRequestMethods = builder.requestMethods;
-        allowedRequestHeaders = builder.requestHeaders;
-        preflightHeaders = builder.preflightHeaders;
-        shortCircuit = builder.shortCircuit;
-    }
-
-    /**
-     * Determines if support for CORS is enabled.
-     *
-     * @return {@code true} if support for CORS is enabled, false otherwise.
-     */
-    public boolean isCorsSupportEnabled() {
-        return enabled;
-    }
-
-    /**
-     * Determines whether a wildcard origin, '*', is supported. This also means that null origins are
-     * supported.
-     *
-     * @return {@code boolean} true if any origin is allowed.
-     */
-    public boolean isAnyOriginSupported() {
-        return anyOrigin;
-    }
-
-    /**
-     * Returns the set of allowed origins.
-     *
-     * @return {@code Set} the allowed origins.
-     */
-    public Optional<Set<String>> origins() {
-        return origins;
-    }
-
-    /**
-     * Returns whether the input origin is allowed by this configuration.
-     *
-     * @return {@code true} if the origin is allowed, otherwise {@code false}
-     */
-    public boolean isOriginAllowed(final String origin) {
-        if (origins.isPresent()) {
-            return origins.get().contains(origin);
-        } else if (pattern.isPresent()) {
-            return pattern.get().matcher(origin).matches();
-        }
-        return false;
-    }
-
-    /**
-     * Determines if credentials are supported for CORS requests.
-     *
-     * By default credentials are not included in CORS requests but if isCredentialsAllowed returns
-     * true credentials will be added to CORS requests. Setting this value to true will set the
-     * CORS 'Access-Control-Allow-Credentials' response header to true.
-     *
-     * Please note that credentials support needs to be enabled on the client side as well.
-     * The client needs to opt-in to send credentials by calling:
-     * <pre>
-     * xhr.withCredentials = true;
-     * </pre>
-     * The default value for 'withCredentials' is false in which case no credentials are sent.
-     * Setting this to true will included cookies in cross origin requests.
-     *
-     * @return {@code true} if credentials are supported.
-     */
-    public boolean isCredentialsAllowed() {
-        return allowCredentials;
-    }
-
-    /**
-     * Gets the maxAge setting.
-     *
-     * When making a preflight request the client has to perform two request with can be inefficient.
-     * This setting will set the CORS 'Access-Control-Max-Age' response header and enables the
-     * caching of the preflight response for the specified time. During this time no preflight
-     * request will be made.
-     *
-     * @return {@code long} the time in seconds that a preflight request may be cached.
-     */
-    public long maxAge() {
-        return maxAge;
-    }
-
-    /**
-     * Returns the allowed set of Request Methods. The Http methods that should be returned in the
-     * CORS 'Access-Control-Request-Method' response header.
-     *
-     * @return {@code Set} of {@link HttpMethod}s that represent the allowed Request Methods.
-     */
-    public Set<HttpMethod> allowedRequestMethods() {
-        return Collections.unmodifiableSet(allowedRequestMethods);
-    }
-
-    /**
-     * Returns the allowed set of Request Headers.
-     *
-     * The header names returned from this method will be used to set the CORS
-     * 'Access-Control-Allow-Headers' response header.
-     *
-     * @return {@code Set<String>} of strings that represent the allowed Request Headers.
-     */
-    public Set<String> allowedRequestHeaders() {
-        return Collections.unmodifiableSet(allowedRequestHeaders);
-    }
-
-    /**
-     * Returns HTTP response headers that should be added to a CORS preflight response.
-     *
-     * @return {@link HttpHeaders} the HTTP response headers to be added.
-     */
-    public HttpHeaders preflightResponseHeaders() {
-        if (preflightHeaders.isEmpty()) {
-            return EmptyHttpHeaders.INSTANCE;
-        }
-        final HttpHeaders preflightHeaders = new DefaultHttpHeaders();
-        for (Map.Entry<CharSequence, Callable<?>> entry : this.preflightHeaders.entrySet()) {
-            final Object value = getValue(entry.getValue());
-            if (value instanceof Iterable) {
-                preflightHeaders.add(entry.getKey().toString(), (Iterable<?>) value);
-            } else {
-                preflightHeaders.add(entry.getKey().toString(), value);
-            }
-        }
-        return preflightHeaders;
-    }
-
-    /**
-     * Determines whether a CORS request should be rejected if it's invalid before being
-     * further processing.
-     *
-     * CORS headers are set after a request is processed. This may not always be desired
-     * and this setting will check that the Origin is valid and if it is not valid no
-     * further processing will take place, and a error will be returned to the calling client.
-     *
-     * @return {@code true} if a CORS request should short-circuit upon receiving an invalid Origin header.
-     */
-    public boolean isShortCircuit() {
-        return shortCircuit;
-    }
-
-    private static <T> T getValue(final Callable<T> callable) {
-        try {
-            return callable.call();
-        } catch (final Exception e) {
-            throw new IllegalStateException("Could not generate value for callable [" + callable + ']', e);
-        }
-    }
-
-    @Override
-    public String toString() {
-        return  "CorsConfig[enabled=" + enabled +
-            ", origins=" + origins +
-            ", anyOrigin=" + anyOrigin +
-            ", isCredentialsAllowed=" + allowCredentials +
-            ", maxAge=" + maxAge +
-            ", allowedRequestMethods=" + allowedRequestMethods +
-            ", allowedRequestHeaders=" + allowedRequestHeaders +
-            ", preflightHeaders=" + preflightHeaders + ']';
-    }
-
-}
diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsConfigBuilder.java b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsConfigBuilder.java
deleted file mode 100644
index 62eda913b0ac7..0000000000000
--- a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsConfigBuilder.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.http.nio.cors;
-
-import io.netty.handler.codec.http.HttpMethod;
-
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.regex.Pattern;
-
-/**
- * Builder used to configure and build a {@link NioCorsConfig} instance.
- *
- * This class was lifted from the Netty project:
- *  https://github.com/netty/netty
- */
-public final class NioCorsConfigBuilder {
-
-    /**
-     * Creates a Builder instance with it's origin set to '*'.
-     *
-     * @return Builder to support method chaining.
-     */
-    public static NioCorsConfigBuilder forAnyOrigin() {
-        return new NioCorsConfigBuilder();
-    }
-
-    /**
-     * Create a {@link NioCorsConfigBuilder} instance with the specified pattern origin.
-     *
-     * @param pattern the regular expression pattern to match incoming origins on.
-     * @return {@link NioCorsConfigBuilder} with the configured origin pattern.
-     */
-    public static NioCorsConfigBuilder forPattern(final Pattern pattern) {
-        if (pattern == null) {
-            throw new IllegalArgumentException("CORS pattern cannot be null");
-        }
-        return new NioCorsConfigBuilder(pattern);
-    }
-
-    /**
-     * Creates a {@link NioCorsConfigBuilder} instance with the specified origins.
-     *
-     * @return {@link NioCorsConfigBuilder} to support method chaining.
-     */
-    public static NioCorsConfigBuilder forOrigins(final String... origins) {
-        return new NioCorsConfigBuilder(origins);
-    }
-
-    Optional<Set<String>> origins;
-    Optional<Pattern> pattern;
-    final boolean anyOrigin;
-    boolean enabled = true;
-    boolean allowCredentials;
-    long maxAge;
-    final Set<HttpMethod> requestMethods = new HashSet<>();
-    final Set<String> requestHeaders = new HashSet<>();
-    final Map<CharSequence, Callable<?>> preflightHeaders = new HashMap<>();
-    boolean shortCircuit;
-
-    /**
-     * Creates a new Builder instance with the origin passed in.
-     *
-     * @param origins the origin to be used for this builder.
-     */
-    NioCorsConfigBuilder(final String... origins) {
-        this.origins = Optional.of(new LinkedHashSet<>(Arrays.asList(origins)));
-        pattern = Optional.empty();
-        anyOrigin = false;
-    }
-
-    /**
-     * Creates a new Builder instance allowing any origin, "*" which is the
-     * wildcard origin.
-     *
-     */
-    NioCorsConfigBuilder() {
-        anyOrigin = true;
-        origins = Optional.empty();
-        pattern = Optional.empty();
-    }
-
-    /**
-     * Creates a new Builder instance allowing any origin that matches the pattern.
-     *
-     * @param pattern the pattern to match against for incoming origins.
-     */
-    NioCorsConfigBuilder(final Pattern pattern) {
-        this.pattern = Optional.of(pattern);
-        origins = Optional.empty();
-        anyOrigin = false;
-    }
-
-    /**
-     * Disables CORS support.
-     *
-     * @return {@link NioCorsConfigBuilder} to support method chaining.
-     */
-    public NioCorsConfigBuilder disable() {
-        enabled = false;
-        return this;
-    }
-
-    /**
-     * By default cookies are not included in CORS requests, but this method will enable cookies to
-     * be added to CORS requests. Calling this method will set the CORS 'Access-Control-Allow-Credentials'
-     * response header to true.
-     *
-     * Please note, that cookie support needs to be enabled on the client side as well.
-     * The client needs to opt-in to send cookies by calling:
-     * <pre>
-     * xhr.withCredentials = true;
-     * </pre>
-     * The default value for 'withCredentials' is false in which case no cookies are sent.
-     * Setting this to true will included cookies in cross origin requests.
-     *
-     * @return {@link NioCorsConfigBuilder} to support method chaining.
-     */
-    public NioCorsConfigBuilder allowCredentials() {
-        allowCredentials = true;
-        return this;
-    }
-
-    /**
-     * When making a preflight request the client has to perform two request with can be inefficient.
-     * This setting will set the CORS 'Access-Control-Max-Age' response header and enables the
-     * caching of the preflight response for the specified time. During this time no preflight
-     * request will be made.
-     *
-     * @param max the maximum time, in seconds, that the preflight response may be cached.
-     * @return {@link NioCorsConfigBuilder} to support method chaining.
-     */
-    public NioCorsConfigBuilder maxAge(final long max) {
-        maxAge = max;
-        return this;
-    }
-
-    /**
-     * Specifies the allowed set of HTTP Request Methods that should be returned in the
-     * CORS 'Access-Control-Request-Method' response header.
-     *
-     * @param methods the {@link HttpMethod}s that should be allowed.
-     * @return {@link NioCorsConfigBuilder} to support method chaining.
-     */
-    public NioCorsConfigBuilder allowedRequestMethods(final HttpMethod... methods) {
-        requestMethods.addAll(Arrays.asList(methods));
-        return this;
-    }
-
-    /**
-     * Specifies the if headers that should be returned in the CORS 'Access-Control-Allow-Headers'
-     * response header.
-     *
-     * If a client specifies headers on the request, for example by calling:
-     * <pre>
-     * xhr.setRequestHeader('My-Custom-Header', "SomeValue");
-     * </pre>
-     * the server will receive the above header name in the 'Access-Control-Request-Headers' of the
-     * preflight request. The server will then decide if it allows this header to be sent for the
-     * real request (remember that a preflight is not the real request but a request asking the server
-     * if it allow a request).
-     *
-     * @param headers the headers to be added to the preflight 'Access-Control-Allow-Headers' response header.
-     * @return {@link NioCorsConfigBuilder} to support method chaining.
-     */
-    public NioCorsConfigBuilder allowedRequestHeaders(final String... headers) {
-        requestHeaders.addAll(Arrays.asList(headers));
-        return this;
-    }
-
-    /**
-     * Specifies that a CORS request should be rejected if it's invalid before being
-     * further processing.
-     *
-     * CORS headers are set after a request is processed. This may not always be desired
-     * and this setting will check that the Origin is valid and if it is not valid no
-     * further processing will take place, and a error will be returned to the calling client.
-     *
-     * @return {@link NioCorsConfigBuilder} to support method chaining.
-     */
-    public NioCorsConfigBuilder shortCircuit() {
-        shortCircuit = true;
-        return this;
-    }
-
-    /**
-     * Builds a {@link NioCorsConfig} with settings specified by previous method calls.
-     *
-     * @return {@link NioCorsConfig} the configured CorsConfig instance.
-     */
-    public NioCorsConfig build() {
-        if (preflightHeaders.isEmpty()) {
-            preflightHeaders.put("date", DateValueGenerator.INSTANCE);
-            preflightHeaders.put("content-length", new ConstantValueGenerator("0"));
-        }
-        return new NioCorsConfig(this);
-    }
-
-    /**
-     * This class is used for preflight HTTP response values that do not need to be
-     * generated, but instead the value is "static" in that the same value will be returned
-     * for each call.
-     */
-    private static final class ConstantValueGenerator implements Callable<Object> {
-
-        private final Object value;
-
-        /**
-         * Sole constructor.
-         *
-         * @param value the value that will be returned when the call method is invoked.
-         */
-        private ConstantValueGenerator(final Object value) {
-            if (value == null) {
-                throw new IllegalArgumentException("value must not be null");
-            }
-            this.value = value;
-        }
-
-        @Override
-        public Object call() {
-            return value;
-        }
-    }
-
-    /**
-     * This callable is used for the DATE preflight HTTP response HTTP header.
-     * It's value must be generated when the response is generated, hence will be
-     * different for every call.
-     */
-    private static final class DateValueGenerator implements Callable<Date> {
-
-        static final DateValueGenerator INSTANCE = new DateValueGenerator();
-
-        @Override
-        public Date call() throws Exception {
-            return new Date();
-        }
-    }
-
-}
diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsHandler.java b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsHandler.java
index dfb531992f8ef..bdeffdfa1c715 100644
--- a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsHandler.java
+++ b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/cors/NioCorsHandler.java
@@ -32,15 +32,17 @@
 import io.netty.handler.codec.http.HttpResponse;
 import io.netty.handler.codec.http.HttpResponseStatus;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.http.CorsHandler;
 import org.elasticsearch.http.nio.NioHttpResponse;
 
+import java.util.Date;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 /**
  * Handles <a href="http://www.w3.org/TR/cors/">Cross Origin Resource Sharing</a> (CORS) requests.
  * <p>
- * This handler can be configured using a {@link NioCorsConfig}, please
+ * This handler can be configured using a {@link CorsHandler.Config}, please
  * refer to this class for details about the configuration options available.
  *
  * This code was borrowed from Netty 4 and refactored to work for Elasticsearch's Netty 3 setup.
@@ -50,13 +52,13 @@ public class NioCorsHandler extends ChannelDuplexHandler {
     public static final String ANY_ORIGIN = "*";
     private static Pattern SCHEME_PATTERN = Pattern.compile("^https?://");
 
-    private final NioCorsConfig config;
+    private final CorsHandler.Config config;
     private FullHttpRequest request;
 
     /**
-     * Creates a new instance with the specified {@link NioCorsConfig}.
+     * Creates a new instance with the specified {@link CorsHandler.Config}.
      */
-    public NioCorsHandler(final NioCorsConfig config) {
+    public NioCorsHandler(final CorsHandler.Config config) {
         if (config == null) {
             throw new NullPointerException();
         }
@@ -76,7 +78,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception
                     releaseRequest();
                 }
             }
-            if (config.isShortCircuit() && !validateOrigin()) {
+            if (!validateOrigin()) {
                 try {
                     forbidden(ctx, request);
                     return;
@@ -96,7 +98,7 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise)
         ctx.write(response, promise);
     }
 
-    public static void setCorsResponseHeaders(HttpRequest request, HttpResponse resp, NioCorsConfig config) {
+    public static void setCorsResponseHeaders(HttpRequest request, HttpResponse resp, CorsHandler.Config config) {
         if (!config.isCorsSupportEnabled()) {
             return;
         }
@@ -161,7 +163,8 @@ private static boolean isSameOrigin(final String origin, final String host) {
      * @param response the HttpResponse to which the preflight response headers should be added.
      */
     private void setPreflightHeaders(final HttpResponse response) {
-        response.headers().add(config.preflightResponseHeaders());
+        response.headers().add("date", new Date());
+        response.headers().add("content-length", "0");
     }
 
     private boolean setOrigin(final HttpResponse response) {
diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java
index 93a846ea90f81..9ee66341a487a 100644
--- a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java
+++ b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java
@@ -37,14 +37,13 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.http.CorsHandler;
 import org.elasticsearch.http.HttpChannel;
 import org.elasticsearch.http.HttpHandlingSettings;
 import org.elasticsearch.http.HttpReadTimeoutException;
 import org.elasticsearch.http.HttpRequest;
 import org.elasticsearch.http.HttpResponse;
 import org.elasticsearch.http.HttpTransportSettings;
-import org.elasticsearch.http.nio.cors.NioCorsConfig;
-import org.elasticsearch.http.nio.cors.NioCorsConfigBuilder;
 import org.elasticsearch.http.nio.cors.NioCorsHandler;
 import org.elasticsearch.nio.FlushOperation;
 import org.elasticsearch.nio.InboundChannelBuffer;
@@ -98,7 +97,7 @@ public void setMocks() {
         channel = mock(NioHttpChannel.class);
         taskScheduler = mock(TaskScheduler.class);
 
-        NioCorsConfig corsConfig = NioCorsConfigBuilder.forAnyOrigin().build();
+        CorsHandler.Config corsConfig = CorsHandler.disabled();
         handler = new HttpReadWriteHandler(channel, transport, httpHandlingSettings, corsConfig, taskScheduler, System::nanoTime);
         handler.channelRegistered();
     }
@@ -329,7 +328,7 @@ public void testReadTimeout() throws IOException {
         Settings settings = Settings.builder().put(SETTING_HTTP_READ_TIMEOUT.getKey(), timeValue).build();
         HttpHandlingSettings httpHandlingSettings = HttpHandlingSettings.fromSettings(settings);
 
-        NioCorsConfig corsConfig = NioCorsConfigBuilder.forAnyOrigin().build();
+        CorsHandler.Config corsConfig = CorsHandler.disabled();
         TaskScheduler taskScheduler = new TaskScheduler();
 
         Iterator<Integer> timeValues = Arrays.asList(0, 2, 4, 6, 8).iterator();
@@ -378,7 +377,7 @@ private static NioHttpResponse emptyGetResponse(int sequenceNumber) {
 
     private FullHttpResponse executeCorsRequest(final Settings settings, final String originValue, final String host) throws IOException {
         HttpHandlingSettings httpSettings = HttpHandlingSettings.fromSettings(settings);
-        NioCorsConfig corsConfig = NioHttpServerTransport.buildCorsConfig(settings);
+        CorsHandler.Config corsConfig = CorsHandler.fromSettings(settings);
         HttpReadWriteHandler handler = new HttpReadWriteHandler(channel, transport, httpSettings, corsConfig, taskScheduler,
             System::nanoTime);
         handler.channelRegistered();
diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpClient.java b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpClient.java
index 634ea7b44af74..ed55007f3ba61 100644
--- a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpClient.java
+++ b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpClient.java
@@ -49,7 +49,7 @@
 import org.elasticsearch.nio.NioSelector;
 import org.elasticsearch.nio.NioServerSocketChannel;
 import org.elasticsearch.nio.NioSocketChannel;
-import org.elasticsearch.nio.ReadWriteHandler;
+import org.elasticsearch.nio.NioChannelHandler;
 import org.elasticsearch.nio.SocketChannelContext;
 import org.elasticsearch.nio.WriteOperation;
 import org.elasticsearch.tasks.Task;
@@ -207,7 +207,7 @@ public NioServerSocketChannel createServerChannel(NioSelector selector, ServerSo
         }
     }
 
-    private static class HttpClientHandler implements ReadWriteHandler {
+    private static class HttpClientHandler implements NioChannelHandler {
 
         private final NettyAdaptor adaptor;
         private final CountDownLatch latch;
@@ -277,6 +277,11 @@ public int consumeReads(InboundChannelBuffer channelBuffer) throws IOException {
             return bytesConsumed;
         }
 
+        @Override
+        public boolean closeNow() {
+            return false;
+        }
+
         @Override
         public void close() throws IOException {
             try {
diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java
index 0b470fda00a6b..bc3208eb29809 100644
--- a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java
+++ b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java
@@ -32,12 +32,10 @@
 import io.netty.handler.codec.http.HttpUtil;
 import io.netty.handler.codec.http.HttpVersion;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
@@ -48,7 +46,6 @@
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.HttpTransportSettings;
 import org.elasticsearch.http.NullDispatcher;
-import org.elasticsearch.http.nio.cors.NioCorsConfig;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.nio.NioSocketChannel;
 import org.elasticsearch.rest.BytesRestResponse;
@@ -64,22 +61,11 @@
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.regex.PatternSyntaxException;
-import java.util.stream.Collectors;
-
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
-import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_MAX_AGE;
+
 import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
 import static org.elasticsearch.rest.RestStatus.OK;
 import static org.hamcrest.Matchers.containsString;
@@ -115,48 +101,6 @@ public void shutdown() throws Exception {
         bigArrays = null;
     }
 
-    public void testCorsConfig() {
-        final Set<String> methods = new HashSet<>(Arrays.asList("get", "options", "post"));
-        final Set<String> headers = new HashSet<>(Arrays.asList("Content-Type", "Content-Length"));
-        final String prefix = randomBoolean() ? " " : ""; // sometimes have a leading whitespace between comma delimited elements
-        final Settings settings = Settings.builder()
-            .put(SETTING_CORS_ENABLED.getKey(), true)
-            .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "*")
-            .put(SETTING_CORS_ALLOW_METHODS.getKey(), Strings.collectionToDelimitedString(methods, ",", prefix, ""))
-            .put(SETTING_CORS_ALLOW_HEADERS.getKey(), Strings.collectionToDelimitedString(headers, ",", prefix, ""))
-            .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
-            .build();
-        final NioCorsConfig corsConfig = NioHttpServerTransport.buildCorsConfig(settings);
-        assertTrue(corsConfig.isAnyOriginSupported());
-        assertEquals(headers, corsConfig.allowedRequestHeaders());
-        assertEquals(methods, corsConfig.allowedRequestMethods().stream().map(HttpMethod::name).collect(Collectors.toSet()));
-    }
-
-    public void testCorsConfigWithDefaults() {
-        final Set<String> methods = Strings.commaDelimitedListToSet(SETTING_CORS_ALLOW_METHODS.getDefault(Settings.EMPTY));
-        final Set<String> headers = Strings.commaDelimitedListToSet(SETTING_CORS_ALLOW_HEADERS.getDefault(Settings.EMPTY));
-        final long maxAge = SETTING_CORS_MAX_AGE.getDefault(Settings.EMPTY);
-        final Settings settings = Settings.builder().put(SETTING_CORS_ENABLED.getKey(), true).build();
-        final NioCorsConfig corsConfig = NioHttpServerTransport.buildCorsConfig(settings);
-        assertFalse(corsConfig.isAnyOriginSupported());
-        assertEquals(Collections.emptySet(), corsConfig.origins().get());
-        assertEquals(headers, corsConfig.allowedRequestHeaders());
-        assertEquals(methods, corsConfig.allowedRequestMethods().stream().map(HttpMethod::name).collect(Collectors.toSet()));
-        assertEquals(maxAge, corsConfig.maxAge());
-        assertFalse(corsConfig.isCredentialsAllowed());
-    }
-
-    public void testCorsConfigWithBadRegex() {
-        final Settings settings = Settings.builder()
-            .put(SETTING_CORS_ENABLED.getKey(), true)
-            .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "/[*/")
-            .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
-            .build();
-        SettingsException e = expectThrows(SettingsException.class, () -> NioHttpServerTransport.buildCorsConfig(settings));
-        assertThat(e.getMessage(), containsString("Bad regex in [http.cors.allow-origin]: [/[*/]"));
-        assertThat(e.getCause(), instanceOf(PatternSyntaxException.class));
-    }
-
     /**
      * Test that {@link NioHttpServerTransport} supports the "Expect: 100-continue" HTTP header
      * @throws InterruptedException if the client communication with the server is interrupted
diff --git a/qa/build.gradle b/qa/build.gradle
index f1727f115155d..9266a09b25735 100644
--- a/qa/build.gradle
+++ b/qa/build.gradle
@@ -1,25 +1,20 @@
 
 import org.elasticsearch.gradle.test.RestIntegTestTask
+import org.elasticsearch.gradle.testclusters.TestClustersPlugin
 
 subprojects { Project subproj ->
   subproj.tasks.withType(RestIntegTestTask) {
-    subproj.extensions.configure("${it.name}Cluster") { cluster ->
-      cluster.distribution = System.getProperty('tests.distribution', 'oss')
-      if (cluster.distribution == 'default') {
-        /*
-         * Add Elastic's repositories so we can resolve older versions of the
-         * default distribution. Those aren't in maven central.
-         */
-        repositories {
-          maven {
-            name "elastic"
-            url "https://artifacts.elastic.co/maven"
-          }
-          maven {
-            name "elastic-snapshots"
-            url "https://snapshots.elastic.co/maven"
-          }
-        }
+    if (subproj.extensions.findByName("${it.name}Cluster")) {
+      subproj.extensions.configure("${it.name}Cluster") { cluster ->
+        cluster.distribution = System.getProperty('tests.distribution', 'oss')
+      }
+    }
+  }
+  plugins.withType(TestClustersPlugin).whenPluginAdded {
+    afterEvaluate {
+      // We need to delay this so it's not overwritten in RestIntegTestTask 
+      testClusters.all {
+        distribution = System.getProperty('tests.distribution', 'oss').toUpperCase()
       }
     }
   }
diff --git a/qa/ccs-unavailable-clusters/build.gradle b/qa/ccs-unavailable-clusters/build.gradle
index c1f2bc9627108..749623b26c329 100644
--- a/qa/ccs-unavailable-clusters/build.gradle
+++ b/qa/ccs-unavailable-clusters/build.gradle
@@ -16,10 +16,11 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.test-with-dependencies'
 
 dependencies {
-    testCompile "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}"
+    testCompile project(":client:rest-high-level")
 }
diff --git a/qa/die-with-dignity/build.gradle b/qa/die-with-dignity/build.gradle
index 140df6e283ab8..a40f6366e6515 100644
--- a/qa/die-with-dignity/build.gradle
+++ b/qa/die-with-dignity/build.gradle
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.esplugin'
 
 esplugin {
@@ -24,14 +25,17 @@ esplugin {
     classname 'org.elasticsearch.DieWithDignityPlugin'
 }
 
-integTestRunner {
+integTest.runner {
     systemProperty 'tests.security.manager', 'false'
     systemProperty 'tests.system_call_filter', 'false'
-    nonInputProperties.systemProperty 'pidfile', "${-> integTest.getNodes().get(0).pidFile}"
-    nonInputProperties.systemProperty 'log', "${-> integTest.getNodes().get(0).homeDir}/logs/${-> integTest.getNodes().get(0).clusterName}_server.json"
+    nonInputProperties.systemProperty 'log', "${-> testClusters.integTest.singleNode().getServerLog()}"
     systemProperty 'runtime.java.home', "${project.runtimeJavaHome}"
 }
 
+testClusters.integTest {
+    systemProperty "die.with.dignity.test", "whatever"
+}
+
 test.enabled = false
 
 check.dependsOn integTest
diff --git a/qa/die-with-dignity/src/main/java/org/elasticsearch/DieWithDignityPlugin.java b/qa/die-with-dignity/src/main/java/org/elasticsearch/DieWithDignityPlugin.java
index ed1e3d3879a5d..8027eeb8948d0 100644
--- a/qa/die-with-dignity/src/main/java/org/elasticsearch/DieWithDignityPlugin.java
+++ b/qa/die-with-dignity/src/main/java/org/elasticsearch/DieWithDignityPlugin.java
@@ -36,6 +36,10 @@
 
 public class DieWithDignityPlugin extends Plugin implements ActionPlugin {
 
+    public DieWithDignityPlugin() {
+        assert System.getProperty("die.with.dignity.test") != null : "test should pass the `die.with.dignity.test` property";
+    }
+
     @Override
     public List<RestHandler> getRestHandlers(
             final Settings settings,
diff --git a/qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java b/qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java
index 16398b380cfe1..c6350f92ae78f 100644
--- a/qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java
+++ b/qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java
@@ -19,16 +19,10 @@
 
 package org.elasticsearch.qa.die_with_dignity;
 
-import org.apache.http.ConnectionClosedException;
-import org.apache.lucene.util.Constants;
-import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.common.logging.JsonLogLine;
-import org.elasticsearch.common.logging.JsonLogsStream;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.rest.ESRestTestCase;
-import org.hamcrest.Matcher;
-import org.hamcrest.Matchers;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -38,82 +32,55 @@
 import java.nio.file.Path;
 import java.util.Iterator;
 import java.util.List;
-import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.either;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.hasToString;
-import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.not;
 
 public class DieWithDignityIT extends ESRestTestCase {
 
     public void testDieWithDignity() throws Exception {
-        // deleting the PID file prevents stopping the cluster from failing since it occurs if and only if the PID file exists
-        final Path pidFile = PathUtils.get(System.getProperty("pidfile"));
-        final List<String> pidFileLines = Files.readAllLines(pidFile);
-        assertThat(pidFileLines, hasSize(1));
-        final int pid = Integer.parseInt(pidFileLines.get(0));
-        Files.delete(pidFile);
-        IOException e = expectThrows(IOException.class,
-            () -> client().performRequest(new Request("GET", "/_die_with_dignity")));
-        Matcher<IOException> failureMatcher = instanceOf(ConnectionClosedException.class);
-        if (Constants.WINDOWS) {
-            /*
-             * If the other side closes the connection while we're waiting to fill our buffer
-             * we can get IOException with the message below. It seems to only come up on
-             * Windows and it *feels* like it could be a ConnectionClosedException but
-             * upstream does not consider this a bug:
-             * https://issues.apache.org/jira/browse/HTTPASYNC-134
-             *
-             * So we catch it here and consider it "ok".
-             */
-            failureMatcher = either(failureMatcher)
-                .or(hasToString(containsString("An existing connection was forcibly closed by the remote host")));
-        }
-        assertThat(e, failureMatcher);
+        expectThrows(
+            IOException.class,
+            () -> client().performRequest(new Request("GET", "/_die_with_dignity"))
+        );
 
         // the Elasticsearch process should die and disappear from the output of jps
         assertBusy(() -> {
             final String jpsPath = PathUtils.get(System.getProperty("runtime.java.home"), "bin/jps").toString();
-            final Process process = new ProcessBuilder().command(jpsPath).start();
-            assertThat(process.waitFor(), equalTo(0));
+            final Process process = new ProcessBuilder().command(jpsPath, "-v").start();
+
             try (InputStream is = process.getInputStream();
                  BufferedReader in = new BufferedReader(new InputStreamReader(is, "UTF-8"))) {
                 String line;
                 while ((line = in.readLine()) != null) {
-                    final int currentPid = Integer.parseInt(line.split("\\s+")[0]);
-                    assertThat(line, pid, not(equalTo(currentPid)));
+                    assertThat(line, line, not(containsString("-Ddie.with.dignity.test")));
                 }
             }
         });
 
+        // parse the logs and ensure that Elasticsearch died with the expected cause
+        final List<String> lines = Files.readAllLines(PathUtils.get(System.getProperty("log")));
+
+        final Iterator<String> it = lines.iterator();
+
+        boolean fatalError = false;
+        boolean fatalErrorInThreadExiting = false;
         try {
-            // parse the logs and ensure that Elasticsearch died with the expected cause
-            Path path = PathUtils.get(System.getProperty("log"));
-            try (Stream<JsonLogLine> stream = JsonLogsStream.from(path)) {
-                final Iterator<JsonLogLine> it = stream.iterator();
-
-                boolean fatalError = false;
-                boolean fatalErrorInThreadExiting = false;
-
-                while (it.hasNext() && (fatalError == false || fatalErrorInThreadExiting == false)) {
-                    final JsonLogLine line = it.next();
-                    if (isFatalError(line)) {
-                        fatalError = true;
-                    } else if (isFatalErrorInThreadExiting(line) || isWarnExceptionReceived(line)) {
-                        fatalErrorInThreadExiting = true;
-                        assertThat(line.stacktrace(),
-                            hasItem(Matchers.containsString("java.lang.OutOfMemoryError: die with dignity")));
-                    }
+            while (it.hasNext() && (fatalError == false || fatalErrorInThreadExiting == false)) {
+                final String line = it.next();
+                if (line.matches(".*ERROR.*o\\.e\\.ExceptionsHelper.*integTest-0.*fatal error.*")) {
+                    fatalError = true;
+                } else if (line.matches(".*ERROR.*o\\.e\\.b\\.ElasticsearchUncaughtExceptionHandler.*integTest-0.*"
+                    + "fatal error in thread \\[Thread-\\d+\\], exiting.*")) {
+                    fatalErrorInThreadExiting = true;
+                    assertTrue(it.hasNext());
+                    assertThat(it.next(), containsString("java.lang.OutOfMemoryError: die with dignity"));
                 }
-
-                assertTrue(fatalError);
-                assertTrue(fatalErrorInThreadExiting);
             }
+
+            assertTrue(fatalError);
+            assertTrue(fatalErrorInThreadExiting);
+
         } catch (AssertionError ae) {
             Path path = PathUtils.get(System.getProperty("log"));
             debugLogs(path);
@@ -121,38 +88,26 @@ public void testDieWithDignity() throws Exception {
         }
     }
 
-    private boolean isWarnExceptionReceived(JsonLogLine line) {
-        return line.level().equals("WARN")
-            && line.component().equals("o.e.h.AbstractHttpServerTransport")
-            && line.nodeName().equals("node-0")
-            && line.message().contains("caught exception while handling client http traffic");
-    }
-
     private void debugLogs(Path path) throws IOException {
         try (BufferedReader reader = Files.newBufferedReader(path)) {
-            Terminal terminal = Terminal.DEFAULT;
-            reader.lines().forEach(line -> terminal.println(line));
+            reader.lines().forEach(line -> logger.info(line));
         }
     }
 
-    private boolean isFatalErrorInThreadExiting(JsonLogLine line) {
-        return line.level().equals("ERROR")
-            && line.component().equals("o.e.b.ElasticsearchUncaughtExceptionHandler")
-            && line.nodeName().equals("node-0")
-            && line.message().matches("fatal error in thread \\[Thread-\\d+\\], exiting$");
-    }
-
-    private boolean isFatalError(JsonLogLine line) {
-        return line.level().equals("ERROR")
-            && line.component().equals("o.e.ExceptionsHelper")
-            && line.nodeName().equals("node-0")
-            && line.message().contains("fatal error");
-    }
-
     @Override
     protected boolean preserveClusterUponCompletion() {
         // as the cluster is dead its state can not be wiped successfully so we have to bypass wiping the cluster
         return true;
     }
 
+    @Override
+    protected final Settings restClientSettings() {
+        return Settings.builder().put(super.restClientSettings())
+            // increase the timeout here to 90 seconds to handle long waits for a green
+            // cluster health. the waits for green need to be longer than a minute to
+            // account for delayed shards
+            .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "1s")
+            .build();
+    }
+
 }
diff --git a/qa/evil-tests/build.gradle b/qa/evil-tests/build.gradle
index 2f9239e5c2f22..38c1b3e1a9aa9 100644
--- a/qa/evil-tests/build.gradle
+++ b/qa/evil-tests/build.gradle
@@ -23,6 +23,7 @@
  * threads, etc.
  */
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java
index 44d3c2a88a55b..49e30ac4b5ed3 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java
@@ -51,10 +51,11 @@ public void testMissingWritePermission() throws IOException {
             Settings build = Settings.builder()
                     .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString())
                     .putList(Environment.PATH_DATA_SETTING.getKey(), tempPaths).build();
-            IOException exception = expectThrows(IOException.class, () -> {
+            IllegalStateException exception = expectThrows(IllegalStateException.class, () -> {
                 new NodeEnvironment(build, TestEnvironment.newEnvironment(build));
             });
-            assertTrue(exception.getMessage(), exception.getMessage().startsWith(path.toString()));
+            assertTrue(exception.getCause().getCause().getMessage(),
+                exception.getCause().getCause().getMessage().startsWith(path.toString()));
         }
     }
 
@@ -62,7 +63,7 @@ public void testMissingWritePermissionOnIndex() throws IOException {
         assumeTrue("posix filesystem", isPosix);
         final String[] tempPaths = tmpPaths();
         Path path = PathUtils.get(randomFrom(tempPaths));
-        Path fooIndex = path.resolve("nodes").resolve("0").resolve(NodeEnvironment.INDICES_FOLDER)
+        Path fooIndex = path.resolve(NodeEnvironment.INDICES_FOLDER)
             .resolve("foo");
         Files.createDirectories(fooIndex);
         try (PosixPermissionsResetter attr = new PosixPermissionsResetter(fooIndex)) {
@@ -82,7 +83,7 @@ public void testMissingWritePermissionOnShard() throws IOException {
         assumeTrue("posix filesystem", isPosix);
         final String[] tempPaths = tmpPaths();
         Path path = PathUtils.get(randomFrom(tempPaths));
-        Path fooIndex = path.resolve("nodes").resolve("0").resolve(NodeEnvironment.INDICES_FOLDER)
+        Path fooIndex = path.resolve(NodeEnvironment.INDICES_FOLDER)
             .resolve("foo");
         Path fooShard = fooIndex.resolve("0");
         Path fooShardIndex = fooShard.resolve("index");
diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
index 50eee32810adc..350ef19ccc6c5 100644
--- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
+++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
@@ -917,7 +917,7 @@ public void testSoftDeletes() throws Exception {
                 mappingsAndSettings.startObject("settings");
                 mappingsAndSettings.field("number_of_shards", 1);
                 mappingsAndSettings.field("number_of_replicas", 1);
-                if (getOldClusterVersion().onOrAfter(Version.V_6_5_0) && randomBoolean()) {
+                if (randomBoolean()) {
                     mappingsAndSettings.field("soft_deletes.enabled", true);
                 }
                 mappingsAndSettings.endObject();
@@ -961,7 +961,6 @@ public void testSoftDeletes() throws Exception {
      * This test creates an index in the old cluster and then closes it. When the cluster is fully restarted in a newer version,
      * it verifies that the index exists and is replicated if the old version supports replication.
      */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/39576")
     public void testClosedIndices() throws Exception {
         if (isRunningAgainstOldCluster()) {
             createIndex(index, Settings.builder()
@@ -1049,19 +1048,26 @@ private void assertClosedIndex(final String index, final boolean checkRoutingTab
         }
     }
 
+    @SuppressWarnings("unchecked")
     private void checkSnapshot(final String snapshotName, final int count, final Version tookOnVersion) throws IOException {
         // Check the snapshot metadata, especially the version
         Request listSnapshotRequest = new Request("GET", "/_snapshot/repo/" + snapshotName);
-        Map<String, Object> listSnapshotResponse = entityAsMap(client().performRequest(listSnapshotRequest));
-        assertEquals(singletonList(snapshotName), XContentMapValues.extractValue("snapshots.snapshot", listSnapshotResponse));
-        assertEquals(singletonList("SUCCESS"), XContentMapValues.extractValue("snapshots.state", listSnapshotResponse));
-        assertEquals(singletonList(tookOnVersion.toString()), XContentMapValues.extractValue("snapshots.version", listSnapshotResponse));
+        Map<String, Object> responseMap = entityAsMap(client().performRequest(listSnapshotRequest));
+        Map<String, Object> snapResponse;
+        if (responseMap.get("responses") != null) {
+            snapResponse = (Map<String, Object>) ((List<Object>) responseMap.get("responses")).get(0);
+        } else {
+            snapResponse = responseMap;
+        }
+
+        assertEquals(singletonList(snapshotName), XContentMapValues.extractValue("snapshots.snapshot", snapResponse));
+        assertEquals(singletonList("SUCCESS"), XContentMapValues.extractValue("snapshots.state", snapResponse));
+        assertEquals(singletonList(tookOnVersion.toString()), XContentMapValues.extractValue("snapshots.version", snapResponse));
 
         // Remove the routing setting and template so we can test restoring them.
         Request clearRoutingFromSettings = new Request("PUT", "/_cluster/settings");
         clearRoutingFromSettings.setJsonEntity("{\"persistent\":{\"cluster.routing.allocation.exclude.test_attr\": null}}");
         client().performRequest(clearRoutingFromSettings);
-
         client().performRequest(new Request("DELETE", "/_template/test_template"));
 
         // Restore
diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java
index 861a774ee17d2..2c62facbfc7cc 100644
--- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java
+++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java
@@ -188,7 +188,7 @@ public void testQueryBuilderBWC() throws Exception {
                 assertEquals(201, rsp.getStatusLine().getStatusCode());
             }
         } else {
-            NamedWriteableRegistry registry = new NamedWriteableRegistry(new SearchModule(Settings.EMPTY, false,
+            NamedWriteableRegistry registry = new NamedWriteableRegistry(new SearchModule(Settings.EMPTY,
                 Collections.emptyList()).getNamedWriteables());
 
             for (int i = 0; i < CANDIDATES.size(); i++) {
diff --git a/qa/logging-config/build.gradle b/qa/logging-config/build.gradle
index 544614ef26958..438079b9942fe 100644
--- a/qa/logging-config/build.gradle
+++ b/qa/logging-config/build.gradle
@@ -17,22 +17,22 @@
  * under the License.
  */
 
-
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.standalone-test'
 
-integTestCluster {
+testClusters.integTest  {
     /**
      * Provide a custom log4j configuration where layout is an old style pattern and confirm that Elasticsearch 
      * can successfully startup.
      */
-    extraConfigFile 'log4j2.properties', 'custom-log4j2.properties'
+    extraConfigFile 'log4j2.properties', file('custom-log4j2.properties')
 }
 
-integTestRunner {
+integTest.runner {
     nonInputProperties.systemProperty 'tests.logfile',
-            "${ -> integTest.nodes[0].homeDir}/logs/${ -> integTest.nodes[0].clusterName }_server.log"
+            "${ -> testClusters.integTest.singleNode().getServerLog().absolutePath.replaceAll(".json", ".log")}"
 }
 
 test {
diff --git a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java
index bbb20737c4708..2416eb02bfd4e 100644
--- a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java
+++ b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.common.logging;
 
-import org.apache.log4j.Level;
+import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.core.LoggerContext;
@@ -67,7 +67,6 @@ public void tearDown() throws Exception {
         super.tearDown();
     }
 
-    @SuppressWarnings("unchecked")
     public void testJsonLayout() throws IOException {
         final Logger testLogger = LogManager.getLogger("test");
 
@@ -90,7 +89,6 @@ public void testJsonLayout() throws IOException {
         }
     }
 
-    @SuppressWarnings("unchecked")
     public void testPrefixLoggerInJson() throws IOException {
         Logger shardIdLogger = Loggers.getLogger("shardIdLogger", ShardId.fromString("[indexName][123]"));
         shardIdLogger.info("This is an info message with a shardId");
diff --git a/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java b/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java
index 2e6b32d79fc41..6ab9a00c1e9f5 100644
--- a/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java
+++ b/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java
@@ -40,7 +40,7 @@
  * The intention is to confirm that users can still run their Elasticsearch instances with previous configurations.
  */
 public class CustomLoggingConfigIT extends ESRestTestCase {
-    private static final String NODE_STARTED = ".*node-0.*cluster.uuid.*node.id.*recovered.*cluster_state.*";
+    private static final String NODE_STARTED = ".*integTest-0.*cluster.uuid.*node.id.*recovered.*cluster_state.*";
 
     public void testSuccessfulStartupWithCustomConfig() throws Exception {
         assertBusy(() -> {
diff --git a/qa/multi-cluster-search/build.gradle b/qa/multi-cluster-search/build.gradle
index bca12be6754ab..1913c86fc9c92 100644
--- a/qa/multi-cluster-search/build.gradle
+++ b/qa/multi-cluster-search/build.gradle
@@ -19,42 +19,43 @@
 
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
-  testCompile "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}"
+  testCompile project(":client:rest-high-level")
 }
 
-task remoteClusterTest(type: RestIntegTestTask) {
+task 'remote-cluster'(type: RestIntegTestTask) {
   mustRunAfter(precommit)
+  runner {
+    systemProperty 'tests.rest.suite', 'remote_cluster'
+  }
 }
 
-remoteClusterTestCluster {
-  numNodes = 2
-  clusterName = 'remote-cluster'
-  setting 'cluster.remote.connect', false
+testClusters.'remote-cluster' {
+  numberOfNodes = 2
+  setting 'cluster.remote.connect', 'false'
 }
 
-remoteClusterTestRunner {
-  systemProperty 'tests.rest.suite', 'remote_cluster'
+task mixedClusterTest(type: RestIntegTestTask) {
+  useCluster testClusters.'remote-cluster'
+  runner {
+    dependsOn 'remote-cluster'
+    systemProperty 'tests.rest.suite', 'multi_cluster'
+  }
 }
 
-task mixedClusterTest(type: RestIntegTestTask) {}
-
-mixedClusterTestCluster {
-  dependsOn remoteClusterTestRunner
-  setting 'cluster.remote.my_remote_cluster.seeds', "\"${-> remoteClusterTest.nodes.get(0).transportUri()}\""
-  setting 'cluster.remote.connections_per_cluster', 1
-  setting 'cluster.remote.connect', true
+testClusters.mixedClusterTest {
+  setting 'cluster.remote.my_remote_cluster.seeds',
+          { "\"${testClusters.'remote-cluster'.getAllTransportPortURI().get(0)}\"" }
+  setting 'cluster.remote.connections_per_cluster', '1'
+  setting 'cluster.remote.connect', 'true'
 }
 
-mixedClusterTestRunner {
-  systemProperty 'tests.rest.suite', 'multi_cluster'
-  finalizedBy 'remoteClusterTestCluster#node0.stop','remoteClusterTestCluster#node1.stop'
-}
 
 task integTest {
-  dependsOn = [mixedClusterTest]
+  dependsOn mixedClusterTest
 }
 
 test.enabled = false // no unit tests for multi-cluster-search, only integration tests
diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
index 32ddc77113bc8..602a31288ef46 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
@@ -356,7 +356,7 @@ public void testRecoveryWithSoftDeletes() throws Exception {
                 // before timing out
                 .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
                 .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
-            if (getNodeId(v -> v.onOrAfter(Version.V_6_5_0)) != null && randomBoolean()) {
+            if (randomBoolean()) {
                 settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
             }
             createIndex(index, settings.build());
@@ -567,4 +567,22 @@ private void ensureGlobalCheckpointSynced(String index) throws Exception {
                 });
         }, 60, TimeUnit.SECONDS);
     }
+
+    /** Ensure that we can always execute update requests regardless of the version of cluster */
+    public void testUpdateDoc() throws Exception {
+        final String index = "test_update_doc";
+        if (CLUSTER_TYPE == ClusterType.OLD) {
+            Settings.Builder settings = Settings.builder()
+                .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+                .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2);
+            createIndex(index, settings.build());
+        }
+        ensureGreen(index);
+        indexDocs(index, 0, 10);
+        for (int i = 0; i < 10; i++) {
+            Request update = new Request("POST", index + "/_update/" + i);
+            update.setJsonEntity("{\"doc\": {\"f\": " + randomNonNegativeLong() + "}}");
+            client().performRequest(update);
+        }
+    }
 }
diff --git a/qa/smoke-test-client/build.gradle b/qa/smoke-test-client/build.gradle
deleted file mode 100644
index 637913083b48c..0000000000000
--- a/qa/smoke-test-client/build.gradle
+++ /dev/null
@@ -1,52 +0,0 @@
-import org.elasticsearch.gradle.test.RestIntegTestTask
-
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-apply plugin: 'elasticsearch.standalone-rest-test'
-apply plugin: 'elasticsearch.rest-test'
-
-// TODO: this test works, but it isn't really a rest test...should we have another plugin for "non rest test that just needs N clusters?"
-
-dependencies {
-    testCompile project(path: ':client:transport', configuration: 'runtime') // randomly swapped in as a transport
-}
-
-task singleNodeIntegTest(type: RestIntegTestTask) {
-    mustRunAfter(precommit)
-}
-
-singleNodeIntegTestCluster {
-    numNodes = 1
-    setting 'discovery.type', 'single-node'
-}
-
-integTestCluster.dependsOn(singleNodeIntegTestRunner, 'singleNodeIntegTestCluster#stop')
-
-check.dependsOn(integTest)
-
-
-testingConventions {
-    naming.clear()
-    naming {
-        IT {
-            baseClass 'org.elasticsearch.smoketest.ESSmokeClientTestCase'
-        }
-    }
-}
\ No newline at end of file
diff --git a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java b/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java
deleted file mode 100644
index 00460a8f004d0..0000000000000
--- a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.smoketest;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.apache.lucene.util.LuceneTestCase;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.network.NetworkModule;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.transport.client.PreBuiltTransportClient;
-import org.elasticsearch.transport.nio.MockNioTransportPlugin;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.URL;
-import java.nio.file.Path;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Locale;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiOfLength;
-import static org.hamcrest.Matchers.notNullValue;
-
-/**
- * An abstract base class to run integration tests against an Elasticsearch cluster running outside of the test process.
- * <p>
- * You can define a list of transport addresses from where you can reach your cluster by setting "tests.cluster" system
- * property. It defaults to "localhost:9300". If you run this from `gradle integTest` then it will start the clsuter for
- * you and set up the property.
- * <p>
- * If you want to debug this module from your IDE, then start an external cluster by yourself, maybe with `gradle run`,
- * then run JUnit. If you changed the default port, set "-Dtests.cluster=localhost:PORT" when running your test.
- */
-@LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose")
-public abstract class ESSmokeClientTestCase extends LuceneTestCase {
-
-    /**
-     * Key used to eventually switch to using an external cluster and provide its transport addresses
-     */
-    public static final String TESTS_CLUSTER = "tests.cluster";
-
-    protected static final Logger logger = LogManager.getLogger(ESSmokeClientTestCase.class);
-
-    private static final AtomicInteger counter = new AtomicInteger();
-    private static Client client;
-    private static String clusterAddresses;
-    protected String index;
-
-    private static Client startClient(Path tempDir, TransportAddress... transportAddresses) {
-        Settings.Builder builder = Settings.builder()
-            .put("node.name", "qa_smoke_client_" + counter.getAndIncrement())
-            .put("client.transport.ignore_cluster_name", true)
-            .put(Environment.PATH_HOME_SETTING.getKey(), tempDir);
-        final Collection<Class<? extends Plugin>> plugins;
-        String transportKey = MockNioTransportPlugin.MOCK_NIO_TRANSPORT_NAME;
-        Class<? extends Plugin> transportPlugin = MockNioTransportPlugin.class;
-        if (random().nextBoolean()) {
-            builder.put(NetworkModule.TRANSPORT_TYPE_KEY, transportKey);
-            plugins = Collections.singleton(transportPlugin);
-        } else {
-            plugins = Collections.emptyList();
-        }
-        TransportClient client = new PreBuiltTransportClient(builder.build(), plugins).addTransportAddresses(transportAddresses);
-
-        logger.info("--> Elasticsearch Java TransportClient started");
-
-        Exception clientException = null;
-        try {
-            ClusterHealthResponse health = client.admin().cluster().prepareHealth().get();
-            logger.info("--> connected to [{}] cluster which is running [{}] node(s).",
-                    health.getClusterName(), health.getNumberOfNodes());
-        } catch (Exception e) {
-            clientException = e;
-        }
-
-        assumeNoException("Sounds like your cluster is not running at " + clusterAddresses, clientException);
-
-        return client;
-    }
-
-    private static Client startClient() throws IOException {
-        String[] stringAddresses = clusterAddresses.split(",");
-        TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length];
-        int i = 0;
-        for (String stringAddress : stringAddresses) {
-            URL url = new URL("http://" + stringAddress);
-            InetAddress inetAddress = InetAddress.getByName(url.getHost());
-            transportAddresses[i++] = new TransportAddress(new InetSocketAddress(inetAddress, url.getPort()));
-        }
-        return startClient(createTempDir(), transportAddresses);
-    }
-
-    public static Client getClient() {
-        if (client == null) {
-            try {
-                client = startClient();
-            } catch (IOException e) {
-                logger.error("can not start the client", e);
-            }
-            assertThat(client, notNullValue());
-        }
-        return client;
-    }
-
-    @BeforeClass
-    public static void initializeSettings() {
-        clusterAddresses = System.getProperty(TESTS_CLUSTER);
-        if (clusterAddresses == null || clusterAddresses.isEmpty()) {
-            fail("Must specify " + TESTS_CLUSTER + " for smoke client test");
-        }
-    }
-
-    @AfterClass
-    public static void stopTransportClient() {
-        if (client != null) {
-            client.close();
-            client = null;
-        }
-    }
-
-    @Before
-    public void defineIndexName() {
-        doClean();
-        index = "qa-smoke-test-client-" + randomAsciiOfLength(10).toLowerCase(Locale.getDefault());
-    }
-
-    @After
-    public void cleanIndex() {
-        doClean();
-    }
-
-    private void doClean() {
-        if (client != null) {
-            try {
-                client.admin().indices().prepareDelete(index).get();
-            } catch (Exception e) {
-                // We ignore this cleanup exception
-            }
-        }
-    }
-}
diff --git a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/SmokeTestClientIT.java b/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/SmokeTestClientIT.java
deleted file mode 100644
index 53753b06b8aa9..0000000000000
--- a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/SmokeTestClientIT.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.smoketest;
-
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.Client;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.Matchers.greaterThan;
-
-public class SmokeTestClientIT extends ESSmokeClientTestCase {
-
-    /**
-     * Check that we are connected to a cluster named "elasticsearch".
-     */
-    public void testSimpleClient() {
-        final Client client = getClient();
-
-        // START SNIPPET: java-doc-admin-cluster-health
-        final ClusterHealthResponse health =
-                client.admin().cluster().prepareHealth().setWaitForYellowStatus().get();
-        final String clusterName = health.getClusterName();
-        final int numberOfNodes = health.getNumberOfNodes();
-        // END SNIPPET: java-doc-admin-cluster-health
-        assertThat(
-                "cluster [" + clusterName + "] should have at least 1 node",
-                numberOfNodes,
-                greaterThan(0));
-    }
-
-    /**
-     * Create an index and index some docs
-     */
-    public void testPutDocument() {
-        final Client client = getClient();
-
-        // START SNIPPET: java-doc-index-doc-simple
-        client.prepareIndex(index, "doc", "1")  // Index, Type, Id
-                .setSource("foo", "bar")        // Simple document: { "foo" : "bar" }
-                .get();                         // Execute and wait for the result
-        // END SNIPPET: java-doc-index-doc-simple
-
-        // START SNIPPET: java-doc-admin-indices-refresh
-        // Prepare a refresh action on a given index, execute and wait for the result
-        client.admin().indices().prepareRefresh(index).get();
-        // END SNIPPET: java-doc-admin-indices-refresh
-
-        // START SNIPPET: java-doc-search-simple
-        final SearchResponse searchResponse = client.prepareSearch(index).get();
-        assertThat(searchResponse.getHits().getTotalHits().value, is(1L));
-        // END SNIPPET: java-doc-search-simple
-    }
-
-}
-
diff --git a/qa/smoke-test-http/build.gradle b/qa/smoke-test-http/build.gradle
index ef1a97fc7abaf..90fc9e3950147 100644
--- a/qa/smoke-test-http/build.gradle
+++ b/qa/smoke-test-http/build.gradle
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.test-with-dependencies'
@@ -26,7 +27,7 @@ dependencies {
     testCompile project(path: ':plugins:transport-nio', configuration: 'runtime') // for http
 }
 
-integTestRunner {
+integTest.runner {
     /*
      * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
      * other if we allow them to set the number of available processors as it's set-once in Netty.
diff --git a/qa/smoke-test-ingest-disabled/build.gradle b/qa/smoke-test-ingest-disabled/build.gradle
index cf3ca9a713f96..a0abc8b129d1c 100644
--- a/qa/smoke-test-ingest-disabled/build.gradle
+++ b/qa/smoke-test-ingest-disabled/build.gradle
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -24,6 +25,6 @@ dependencies {
     testCompile project(path: ':modules:ingest-common', configuration: 'runtime')
 }
 
-integTestCluster {
+testClusters.integTest {
     setting 'node.ingest', 'false'
 }
diff --git a/qa/smoke-test-ingest-with-all-dependencies/build.gradle b/qa/smoke-test-ingest-with-all-dependencies/build.gradle
index 9267f90cd7e0a..9f5c40ac93798 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/build.gradle
+++ b/qa/smoke-test-ingest-with-all-dependencies/build.gradle
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle
index cd64262fcdce1..25964a871e052 100644
--- a/qa/smoke-test-multinode/build.gradle
+++ b/qa/smoke-test-multinode/build.gradle
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -24,12 +25,18 @@ integTest {
   includePackaged = true
 }
 
-integTestCluster {
-  numNodes = 2
+File repo = file("$buildDir/testclusters/repo")
+testClusters.integTest {
+  numberOfNodes = 2
+  setting 'path.repo', repo.absolutePath
 }
 
-integTestRunner {
-  if ('default'.equals(integTestCluster.distribution)) {
+integTest.runner {
+  doFirst {
+    project.delete(repo)
+    repo.mkdirs()
+  }
+  if ('default'.equalsIgnoreCase(System.getProperty('tests.distribution', 'oss'))) {
     systemProperty 'tests.rest.blacklist', [
         'cat.templates/10_basic/No templates',
         'cat.templates/10_basic/Sort templates',
diff --git a/qa/smoke-test-plugins/build.gradle b/qa/smoke-test-plugins/build.gradle
index 602dfa2d6ea4f..bdd5e74184e83 100644
--- a/qa/smoke-test-plugins/build.gradle
+++ b/qa/smoke-test-plugins/build.gradle
@@ -19,15 +19,18 @@
 
 import org.elasticsearch.gradle.MavenFilteringHack
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
-ext.pluginsCount = 0
-project(':plugins').getChildProjects().each { pluginName, pluginProject ->
-  integTestCluster {
-    plugin pluginProject.path
+int pluginsCount = 0
+
+testClusters.integTest {
+  project(':plugins').getChildProjects().each { pluginName, pluginProject ->
+    plugin file(pluginProject.tasks.bundlePlugin.archiveFile)
+    tasks.integTest.dependsOn pluginProject.tasks.bundlePlugin
+    pluginsCount += 1
   }
-  pluginsCount += 1
 }
 assert pluginsCount > 0
 
diff --git a/qa/unconfigured-node-name/build.gradle b/qa/unconfigured-node-name/build.gradle
index 3b0faa10a7e33..56805f37beae3 100644
--- a/qa/unconfigured-node-name/build.gradle
+++ b/qa/unconfigured-node-name/build.gradle
@@ -17,18 +17,15 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
-integTestCluster {
-  setting 'node.name', null
-  // Run with no discovery configuration at all, demonstrating that a node in its
-  // "out-of-the-box" configuration can automatically bootstrap a cluster
-  autoSetInitialMasterNodes = false
-  autoSetHostsProvider = false
+testClusters.integTest {
+  nameCustomization = { null }
 }
 
-integTestRunner {
+integTest.runner {
   nonInputProperties.systemProperty 'tests.logfile',
-    "${ -> integTest.nodes[0].homeDir}/logs/${ -> integTest.nodes[0].clusterName }_server.json"
+    "${ -> testClusters.integTest.singleNode().getServerLog() }"
 }
diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index 7c342436dd3ca..f5cfcdda03cc6 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -35,7 +35,7 @@ dependencies {
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
 
-  compile project(':libs:core')
+  compile project(':libs:elasticsearch-core')
 
   // pulls in the jar built by this project and its dependencies
   packagingTest project(path: project.path, configuration: 'runtime')
diff --git a/qa/wildfly/build.gradle b/qa/wildfly/build.gradle
index bcb55079b8269..ed2e4a098b642 100644
--- a/qa/wildfly/build.gradle
+++ b/qa/wildfly/build.gradle
@@ -26,6 +26,7 @@ import java.util.stream.Stream
  */
 
 apply plugin: 'war'
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -70,9 +71,9 @@ dependencies {
     compile "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:${versions.jackson}"
     compile "org.apache.logging.log4j:log4j-api:${versions.log4j}"
     compile "org.apache.logging.log4j:log4j-core:${versions.log4j}"
-    compile project(path: ':client:rest-high-level')
+    compile project(':client:rest-high-level')
     wildfly "org.jboss:wildfly:${wildflyVersion}@zip"
-    testCompile "org.elasticsearch.test:framework:${VersionProperties.elasticsearch}"
+    testCompile project(':test:framework')
 }
 
 task unzipWildfly(type: Sync) {
@@ -88,12 +89,13 @@ task deploy(type: Copy) {
 
 task writeElasticsearchProperties {
     onlyIf { !Os.isFamily(Os.FAMILY_WINDOWS) }
-    dependsOn 'integTestCluster#wait', deploy
+    useCluster testClusters.integTest
+    dependsOn deploy
     doLast {
         final File elasticsearchProperties = file("${wildflyInstall}/standalone/configuration/elasticsearch.properties")
         elasticsearchProperties.write(
                 [
-                        "http.uri=${-> integTest.getNodes().get(0).httpUri()}"
+                        "http.uri=${-> testClusters.integTest.getAllHttpSocketURI().get(0)}"
                 ].join("\n"))
     }
 }
@@ -210,6 +212,7 @@ if (!Os.isFamily(Os.FAMILY_WINDOWS)) {
     integTestRunner.finalizedBy(stopWildfly)
 } else {
     integTest.enabled = false
+    testingConventions.enabled = false
 }
 
 check.dependsOn(integTest)
diff --git a/qa/wildfly/src/test/java/org/elasticsearch/wildfly/WildflyIT.java b/qa/wildfly/src/test/java/org/elasticsearch/wildfly/WildflyIT.java
index 28e11f021a1c7..16e0227f270de 100644
--- a/qa/wildfly/src/test/java/org/elasticsearch/wildfly/WildflyIT.java
+++ b/qa/wildfly/src/test/java/org/elasticsearch/wildfly/WildflyIT.java
@@ -27,7 +27,8 @@
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.util.EntityUtils;
-import org.apache.log4j.Logger;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestRuleLimitSysouts;
 import org.elasticsearch.cluster.ClusterModule;
@@ -53,9 +54,9 @@
 @TestRuleLimitSysouts.Limit(bytes = 14000)
 public class WildflyIT extends LuceneTestCase {
 
-    private Logger logger = Logger.getLogger(WildflyIT.class);
+    private Logger logger = LogManager.getLogger(WildflyIT.class);
 
-    public void testTransportClient() throws URISyntaxException, IOException {
+    public void testRestClient() throws URISyntaxException, IOException {
         try (CloseableHttpClient client = HttpClientBuilder.create().build()) {
             final String str = String.format(
                 Locale.ROOT,
diff --git a/rest-api-spec/README.markdown b/rest-api-spec/README.markdown
index 56b5e9b553b83..241433d8c5cc4 100644
--- a/rest-api-spec/README.markdown
+++ b/rest-api-spec/README.markdown
@@ -10,9 +10,9 @@ Example for the ["Create Index"](http://www.elastic.co/guide/en/elasticsearch/re
 {
   "indices.create": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-index.html",
+    "stability": "stable",
     "methods": ["PUT", "POST"],
     "url": {
-      "path": "/{index}",
       "paths": ["/{index}"],
       "parts": {
         "index": {
@@ -39,13 +39,105 @@ The specification contains:
 
 * The _name_ of the API (`indices.create`), which usually corresponds to the client calls
 * Link to the documentation at <http://elastic.co>
+* `stability` indicating the state of the API, has to be declared explicitly or YAML tests will fail
+    * `experimental` highly likely to break in the near future (minor/path), no bwc guarantees. 
+    Possibly removed in the future.
+    * `beta` less likely to break or be removed but still reserve the right to do so
+    * `stable` No backwards breaking changes in a minor 
 * List of HTTP methods for the endpoint
 * URL specification: path, parts, parameters
 * Whether body is allowed for the endpoint or not and its description
 
+**NOTE** 
+If an API is stable but it response should be treated as an arbitrary map of key values please notate this as followed
+
+```json
+{
+  "api.name": {
+    "stability" : "stable",
+    "response": {
+      "treat_json_as_key_value" : true
+    }
+  }
+}
+```
+
+
 The `methods` and `url.paths` elements list all possible HTTP methods and URLs for the endpoint;
 it is the responsibility of the developer to use this information for a sensible API on the target platform.
 
+## Backwards compatibility 
+
+The specification follows the same backward compatibility guarantees as Elasticsearch. 
+
+- Within a Major, additions only. 
+  - If an item has been documented wrong it should be deprecated instead as removing these might break downstream clients.
+- Major version change, may deprecate pieces or simply remove them given enough deprecation time.
+
+## Deprecations
+
+The spec allows for deprecations of:
+
+#### Entire API:
+
+```json
+{
+  "api" : {
+    "documentation": "...",
+    "deprecated" : {
+      "version" : "7.0.0",
+      "description" : "Reason API is being deprecated"
+    },
+  }
+}
+```
+
+#### Specific paths:
+
+```json
+{
+  "api": {
+    "documentation": "",
+    "url": {
+      "paths": ["/_monitoring/bulk"],
+      "deprecated_paths" : [
+        {
+          "version" : "7.0.0",
+          "path" : "/_monitoring/{type}/bulk",
+          "description" : "Specifying types in urls has been deprecated"
+        }
+      ]
+    }
+  }
+}
+```
+
+Here `paths` describes the preferred paths and `deprecated_paths` indicates `paths` that will still work but are now 
+deprecated.
+
+#### Parameters 
+
+```json
+{
+  "api": {
+    "documentation": "",
+    "methods": ["GET"],
+    "url": {
+      "params": {
+        "stored_fields": {
+          "type": "list",
+          "description" : "",
+          "deprecated" : {
+            "version" : "7.0.0",
+            "description" : "Reason parameter is being deprecated"
+          }
+        }
+      }
+    }
+  }
+}
+```
+
 ## License
 
 This software is licensed under the Apache License, version 2 ("ALv2").
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json b/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json
index a07f362bef064..e02aa1b0d611a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json
@@ -1,9 +1,9 @@
 {
   "bulk": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-bulk.html",
+    "stability": "stable",
     "methods": ["POST", "PUT"],
     "url": {
-      "path": "/_bulk",
       "paths": ["/_bulk", "/{index}/_bulk", "/{index}/{type}/_bulk"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
index d30c6ace6f723..2c89c1dcdd15d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
@@ -1,9 +1,9 @@
 {
   "cat.aliases": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-alias.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/aliases",
       "paths": ["/_cat/aliases", "/_cat/aliases/{name}"],
       "parts": {
         "name": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.allocation.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.allocation.json
index 7c826890607e4..711a894094277 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.allocation.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.allocation.json
@@ -1,9 +1,9 @@
 {
   "cat.allocation": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-allocation.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/allocation",
       "paths": ["/_cat/allocation", "/_cat/allocation/{node_id}"],
       "parts": {
         "node_id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.count.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.count.json
index 4311d9a3be12d..7317e4866a255 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.count.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.count.json
@@ -1,9 +1,9 @@
 {
   "cat.count": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-count.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/count",
       "paths": ["/_cat/count", "/_cat/count/{index}"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.fielddata.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.fielddata.json
index 88c7eee126294..acba8e58668cb 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.fielddata.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.fielddata.json
@@ -1,9 +1,9 @@
 {
   "cat.fielddata": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-fielddata.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/fielddata",
       "paths": ["/_cat/fielddata", "/_cat/fielddata/{fields}"],
       "parts": {
         "fields": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.health.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.health.json
index e858e83b465ed..36be2f1afedd8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.health.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.health.json
@@ -1,9 +1,9 @@
 {
   "cat.health": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-health.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/health",
       "paths": ["/_cat/health"],
       "parts": {
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json
index e893fb1b0447e..16a935517aa23 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json
@@ -1,9 +1,9 @@
 {
   "cat.help": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat",
       "paths": ["/_cat"],
       "parts": {
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
index 24961a57cf331..3246b8f73ea21 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
@@ -1,9 +1,9 @@
 {
   "cat.indices": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-indices.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/indices",
       "paths": ["/_cat/indices", "/_cat/indices/{index}"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.master.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.master.json
index ab87b2adb7e72..a97d3a5ca2bdd 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.master.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.master.json
@@ -1,9 +1,9 @@
 {
   "cat.master": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-master.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/master",
       "paths": ["/_cat/master"],
       "parts": {
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodeattrs.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodeattrs.json
index 3d8a4a77a7704..6a9ee3c424a57 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodeattrs.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodeattrs.json
@@ -1,9 +1,9 @@
 {
   "cat.nodeattrs": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-nodeattrs.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/nodeattrs",
       "paths": ["/_cat/nodeattrs"],
       "parts": {
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodes.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodes.json
index 1b3c1266a63a0..9e17bf14a42dc 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodes.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodes.json
@@ -1,9 +1,9 @@
 {
   "cat.nodes": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-nodes.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/nodes",
       "paths": ["/_cat/nodes"],
       "parts": {
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.pending_tasks.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.pending_tasks.json
index 983b82482ae40..0fc951c424590 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.pending_tasks.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.pending_tasks.json
@@ -1,9 +1,9 @@
 {
   "cat.pending_tasks": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-pending-tasks.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/pending_tasks",
       "paths": ["/_cat/pending_tasks"],
       "parts": {
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.plugins.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.plugins.json
index 93c7feababa99..3217ec0ce62f6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.plugins.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.plugins.json
@@ -1,9 +1,9 @@
 {
   "cat.plugins": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-plugins.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/plugins",
       "paths": ["/_cat/plugins"],
       "params": {
         "format": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.recovery.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.recovery.json
index 42f91cedfdd6c..c592d8c23db40 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.recovery.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.recovery.json
@@ -1,9 +1,9 @@
 {
   "cat.recovery": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-recovery.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/recovery",
       "paths": ["/_cat/recovery", "/_cat/recovery/{index}"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.repositories.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.repositories.json
index c640a568fde0b..0b006fc023cf4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.repositories.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.repositories.json
@@ -1,9 +1,9 @@
 {
   "cat.repositories": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-repositories.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/repositories",
       "paths": ["/_cat/repositories"],
       "parts": {
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json
index 3306b2f753b2a..869b179a6cc7a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json
@@ -1,9 +1,9 @@
 {
   "cat.segments": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-segments.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/segments",
       "paths": ["/_cat/segments", "/_cat/segments/{index}"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json
index 2ad714e7225d7..abf96f666e474 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json
@@ -1,9 +1,9 @@
 {
   "cat.shards": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-shards.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/shards",
       "paths": ["/_cat/shards", "/_cat/shards/{index}"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json
index eec22e2e0412d..1087537114c14 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json
@@ -1,9 +1,9 @@
 {
   "cat.snapshots": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-snapshots.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/snapshots",
       "paths": [
         "/_cat/snapshots",
         "/_cat/snapshots/{repository}"],
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json
index 1958843e601c7..25b10a9147969 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json
@@ -1,9 +1,9 @@
 {
   "cat.tasks": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/tasks",
       "paths": ["/_cat/tasks"],
       "parts": {
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.templates.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.templates.json
index f0757c2d652dc..5bd6fd8b10273 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.templates.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.templates.json
@@ -1,9 +1,9 @@
 {
   "cat.templates": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-templates.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/templates",
       "paths": ["/_cat/templates", "/_cat/templates/{name}"],
       "parts": {
         "name": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.thread_pool.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.thread_pool.json
index 70a11000bbe41..056a1421d719f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.thread_pool.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.thread_pool.json
@@ -1,9 +1,9 @@
 {
   "cat.thread_pool": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-thread-pool.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cat/thread_pool",
       "paths": ["/_cat/thread_pool","/_cat/thread_pool/{thread_pool_patterns}"],
       "parts": {
         "thread_pool_patterns": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/clear_scroll.json b/rest-api-spec/src/main/resources/rest-api-spec/api/clear_scroll.json
index b2b0343230c66..eb01710fd8415 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/clear_scroll.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/clear_scroll.json
@@ -1,9 +1,9 @@
 {
   "clear_scroll": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-request-scroll.html",
+    "stability": "stable",
     "methods": ["DELETE"],
     "url": {
-      "path": "/_search/scroll",
       "paths": [ "/_search/scroll"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json
index 26b24cfb69760..7af79af09ea9b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json
@@ -1,9 +1,9 @@
 {
   "cluster.allocation_explain": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-allocation-explain.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/_cluster/allocation/explain",
       "paths": ["/_cluster/allocation/explain"],
       "parts": {},
       "params": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json
index baba4e3436abc..40c5a8d9f0eae 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json
@@ -1,9 +1,9 @@
 {
   "cluster.get_settings": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-update-settings.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cluster/settings",
       "paths": ["/_cluster/settings"],
       "parts": {},
       "params": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
index 49d0ef5c83aee..1ea7c37943931 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
@@ -1,9 +1,9 @@
 {
   "cluster.health": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-health.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cluster/health",
       "paths": ["/_cluster/health", "/_cluster/health/{index}"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.pending_tasks.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.pending_tasks.json
index fb5e1609ff39d..7b8b2dbcedf1d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.pending_tasks.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.pending_tasks.json
@@ -1,9 +1,9 @@
 {
   "cluster.pending_tasks": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-pending.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cluster/pending_tasks",
       "paths": ["/_cluster/pending_tasks"],
       "parts": {
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_settings.json
index 5fcf03102836a..4681640f4b12f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_settings.json
@@ -1,9 +1,9 @@
 {
   "cluster.put_settings": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-update-settings.html",
+    "stability": "stable",
     "methods": ["PUT"],
     "url": {
-      "path": "/_cluster/settings",
       "paths": ["/_cluster/settings"],
       "parts": {},
       "params": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.remote_info.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.remote_info.json
index 3237853154493..4fdb89b891056 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.remote_info.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.remote_info.json
@@ -1,9 +1,9 @@
 {
   "cluster.remote_info": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-remote-info.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_remote/info",
       "paths": ["/_remote/info"],
       "params": {}
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.reroute.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.reroute.json
index 8bb85ca087ae8..c7d8ece64d8eb 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.reroute.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.reroute.json
@@ -1,9 +1,9 @@
 {
   "cluster.reroute": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-reroute.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/_cluster/reroute",
       "paths": ["/_cluster/reroute"],
       "parts": {
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
index fec3f45000497..cd7f40cd86017 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
@@ -1,9 +1,9 @@
 {
   "cluster.state": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-state.html",
+    "stability" : "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cluster/state",
       "paths": [
         "/_cluster/state",
         "/_cluster/state/{metric}",
@@ -57,6 +57,9 @@
         }
       }
     },
-    "body": null
+    "body": null,
+    "response": {
+      "treat_json_as_key_value" : true
+    }
   }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json
index 36400ed2f7c4c..192cf7dcdf5ee 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json
@@ -1,9 +1,9 @@
 {
   "cluster.stats": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-stats.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_cluster/stats",
       "paths": ["/_cluster/stats", "/_cluster/stats/nodes/{node_id}"],
       "parts": {
         "node_id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
index b933091b9a416..e115a0ae06fb4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
@@ -1,17 +1,10 @@
 {
   "count": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-count.html",
+    "stability": "stable",
     "methods": ["POST", "GET"],
     "url": {
-      "path": "/_count",
       "paths": ["/_count", "/{index}/_count"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0.0",
-          "path" : "/{index}/{type}/_count",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
       "parts": {
         "index": {
           "type" : "list",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/create.json b/rest-api-spec/src/main/resources/rest-api-spec/api/create.json
index 65fcf02807ba1..d0e2375d0014b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/create.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/create.json
@@ -1,9 +1,9 @@
 {
   "create": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-index_.html",
+    "stability": "stable",
     "methods": ["PUT","POST"],
     "url": {
-      "path": "/{index}/_create/{id}",
       "paths": ["/{index}/_create/{id}"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json
index 0152374028832..27c7e62ddaad0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json
@@ -1,9 +1,9 @@
 {
   "delete": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-delete.html",
+    "stability": "stable",
     "methods": ["DELETE"],
     "url": {
-      "path": "/{index}/_doc/{id}",
       "paths": ["/{index}/_doc/{id}"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
index aa1d8b4be4f26..d4aadea8b3d80 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
@@ -1,9 +1,9 @@
 {
   "delete_by_query": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-delete-by-query.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/{index}/_delete_by_query",
       "paths": ["/{index}/_delete_by_query"],
       "comment": "most things below this are just copied from search.json",
       "parts": {
@@ -86,9 +86,9 @@
           "type" : "time",
           "description" : "Explicit timeout for each search request. Defaults to no timeout."
         },
-        "size": {
+        "max_docs": {
           "type" : "number",
-          "description" : "Number of hits to return (default: 10)"
+          "description" : "Maximum number of documents to process (default: all documents)"
         },
         "sort": {
           "type" : "list",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query_rethrottle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query_rethrottle.json
index f49af01cfc3b7..1104af560239d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query_rethrottle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query_rethrottle.json
@@ -1,9 +1,9 @@
 {
   "delete_by_query_rethrottle": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/_delete_by_query/{task_id}/_rethrottle",
       "paths": ["/_delete_by_query/{task_id}/_rethrottle"],
       "parts": {
         "task_id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json
index 83bb690cc0428..d407aa0bfe694 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json
@@ -1,9 +1,9 @@
 {
   "delete_script": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting.html",
+    "stability": "stable",
     "methods": ["DELETE"],
     "url": {
-      "path": "/_scripts/{id}",
       "paths": [ "/_scripts/{id}" ],
       "parts": {
         "id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/exists.json b/rest-api-spec/src/main/resources/rest-api-spec/api/exists.json
index 2a451344521e4..35112389aa392 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/exists.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/exists.json
@@ -1,9 +1,9 @@
 {
   "exists": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html",
+    "stability": "stable",
     "methods": ["HEAD"],
     "url": {
-      "path": "/{index}/_doc/{id}",
       "paths": ["/{index}/_doc/{id}"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json b/rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json
index 30e56141ec001..7a34094100d02 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json
@@ -1,9 +1,9 @@
 {
   "exists_source": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html",
+    "stability": "stable",
     "methods": ["HEAD"],
     "url": {
-      "path": "/{index}/_source/{id}",
       "paths": ["/{index}/_source/{id}"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/explain.json b/rest-api-spec/src/main/resources/rest-api-spec/api/explain.json
index 203ef23c9cc10..11f32801d4d19 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/explain.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/explain.json
@@ -1,9 +1,9 @@
 {
   "explain": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-explain.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/{index}/_explain/{id}",
       "paths": ["/{index}/_explain/{id}"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json b/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json
index ebb520fd433b9..d41785615090b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json
@@ -1,9 +1,9 @@
 {
   "field_caps": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-field-caps.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/_field_caps",
       "paths": [
         "/_field_caps",
         "/{index}/_field_caps"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get.json
index f4e0fdd5f90ef..f02ca3e4a4304 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get.json
@@ -1,9 +1,9 @@
 {
   "get": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/{index}/_doc/{id}",
       "paths": ["/{index}/_doc/{id}"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get_script.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get_script.json
index 0b2d6c5a5b9c2..6beaf8ae85a6c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/get_script.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get_script.json
@@ -1,9 +1,9 @@
 {
   "get_script": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_scripts/{id}",
       "paths": [ "/_scripts/{id}" ],
       "parts": {
         "id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json
index d6f6964aa7c36..0c7414f818411 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json
@@ -1,9 +1,9 @@
 {
   "get_source": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/{index}/_source/{id}",
       "paths": ["/{index}/_source/{id}"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/index.json b/rest-api-spec/src/main/resources/rest-api-spec/api/index.json
index 438032980a3c5..d85e0d709944c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/index.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/index.json
@@ -1,9 +1,9 @@
 {
   "index": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-index_.html",
+    "stability": "stable",
     "methods": ["POST", "PUT"],
     "url": {
-      "path": "/{index}/_doc",
       "paths": ["/{index}/_doc/{id}", "/{index}/_doc"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json
index c340ccd2f37c2..be119620d6cdf 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json
@@ -1,9 +1,9 @@
 {
   "indices.analyze": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-analyze.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/_analyze",
       "paths": ["/_analyze", "/{index}/_analyze"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
index 7e62371dd674d..9e591275927a8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
@@ -1,9 +1,9 @@
 {
   "indices.clear_cache": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-clearcache.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/_cache/clear",
       "paths": ["/_cache/clear", "/{index}/_cache/clear"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
index 55fd245f26c91..187cce0d07796 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
@@ -1,9 +1,9 @@
 {
   "indices.close": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-open-close.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/{index}/_close",
       "paths": ["/{index}/_close"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json
index d4a16e576e1b9..dfe3236c9c053 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json
@@ -1,9 +1,9 @@
 {
   "indices.create": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-index.html",
+    "stability": "stable",
     "methods": ["PUT"],
     "url": {
-      "path": "/{index}",
       "paths": ["/{index}"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
index c391242e9de15..4c2b7eaaa2fb2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
@@ -1,9 +1,9 @@
 {
   "indices.delete": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-delete-index.html",
+    "stability": "stable",
     "methods": ["DELETE"],
     "url": {
-      "path": "/{index}",
       "paths": ["/{index}"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_alias.json
index 30a32fbfef871..c6e4ec8295443 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_alias.json
@@ -1,9 +1,9 @@
 {
   "indices.delete_alias": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html",
+    "stability": "stable",
     "methods": ["DELETE"],
     "url": {
-      "path": "/{index}/_alias/{name}",
       "paths": ["/{index}/_alias/{name}", "/{index}/_aliases/{name}"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_template.json
index b311c9bbda82a..0b3758953b3fb 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_template.json
@@ -1,9 +1,9 @@
 {
   "indices.delete_template": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+    "stability": "stable",
     "methods": ["DELETE"],
     "url": {
-      "path": "/_template/{name}",
       "paths": ["/_template/{name}"],
       "parts": {
         "name": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
index 7f04f00b40def..b84f4d2cacbf3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
@@ -1,9 +1,9 @@
 {
   "indices.exists": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-exists.html",
+    "stability": "stable",
     "methods": [ "HEAD" ],
     "url": {
-      "path": "/{index}",
       "paths": [ "/{index}" ],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
index aea20b2b634d0..d95c423b66db5 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
@@ -1,9 +1,9 @@
 {
   "indices.exists_alias": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html",
+    "stability": "stable",
     "methods": ["HEAD"],
     "url": {
-      "path": "/_alias/{name}",
       "paths": ["/_alias/{name}", "/{index}/_alias/{name}"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json
index 3fb9d1e207e1e..8140c5748fb4d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json
@@ -1,9 +1,9 @@
 {
   "indices.exists_template": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+    "stability": "stable",
     "methods": ["HEAD"],
     "url": {
-      "path": "/_template/{name}",
       "paths": [ "/_template/{name}" ],
       "parts": {
         "name": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json
index a788202c93ae2..10b69b2647f82 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_type.json
@@ -1,13 +1,13 @@
 {
   "indices.exists_type": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-types-exists.html",
+    "stability": "stable",
     "methods": ["HEAD"],
     "deprecated" : {
       "version" : "7.0.0",
       "description" : "Types are being removed from elasticsearch and therefor this API is on the way out. Read more here: https://www.elastic.co/guide/en/elasticsearch/reference/master/removal-of-types.html"
     },
     "url": {
-      "path": "/{index}/_mapping/{type}",
       "paths": ["/{index}/_mapping/{type}"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json
index 77d9e037163fa..7c6c9cb32cd7d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush.json
@@ -1,9 +1,9 @@
 {
   "indices.flush": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-flush.html",
+    "stability": "stable",
     "methods": ["POST", "GET"],
     "url": {
-      "path": "/_flush",
       "paths": ["/_flush", "/{index}/_flush"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush_synced.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush_synced.json
index 08488eae8648b..1b9ab1bb5eff5 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush_synced.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.flush_synced.json
@@ -1,9 +1,9 @@
 {
   "indices.flush_synced": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-synced-flush.html",
+    "stability": "stable",
     "methods": ["POST", "GET"],
     "url": {
-      "path": "/_flush/synced",
       "paths": [
         "/_flush/synced",
         "/{index}/_flush/synced"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
index d87ce2a4451d6..c75bf0e47ef96 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
@@ -1,9 +1,9 @@
 {
   "indices.forcemerge": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-forcemerge.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/_forcemerge",
       "paths": ["/_forcemerge", "/{index}/_forcemerge"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
index 76e6ed00feb26..a0e947504b3a0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
@@ -1,9 +1,9 @@
 {
   "indices.get":{
     "documentation":"http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-index.html",
+    "stability": "stable",
     "methods":[ "GET" ],
     "url":{
-      "path":"/{index}",
       "paths":[ "/{index}" ],
       "parts":{
         "index":{
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
index b68d7c527fcdc..db819f446ad1d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
@@ -1,9 +1,9 @@
 {
   "indices.get_alias": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_alias/",
       "paths": [ "/_alias", "/_alias/{name}", "/{index}/_alias/{name}", "/{index}/_alias"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
index 305cb700ceae9..83a199cbdc045 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
@@ -1,9 +1,9 @@
 {
   "indices.get_field_mapping": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-field-mapping.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_mapping/field/{fields}",
       "paths": ["/_mapping/field/{fields}", "/{index}/_mapping/field/{fields}"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
index abfc3d3867e9a..ad55adc18933d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
@@ -1,9 +1,9 @@
 {
   "indices.get_mapping": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-mapping.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_mapping",
       "paths": ["/_mapping", "/{index}/_mapping"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
index ed22cc837d6a8..d86f2b0a3b810 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
@@ -1,9 +1,9 @@
 {
   "indices.get_settings": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-settings.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_settings",
       "paths": ["/_settings", "/{index}/_settings", "/{index}/_settings/{name}", "/_settings/{name}"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json
index e2aae3b7444aa..f81fc337dd4ac 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json
@@ -1,9 +1,9 @@
 {
   "indices.get_template": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_template/{name}",
       "paths": [
         "/_template",
         "/_template/{name}"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
index 3c114a450eef2..4770fba4e9c4a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
@@ -1,9 +1,9 @@
 {
   "indices.get_upgrade": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-upgrade.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_upgrade",
       "paths": ["/_upgrade", "/{index}/_upgrade"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
index 86c39988e181f..36cc75e264935 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
@@ -1,9 +1,9 @@
 {
   "indices.open": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-open-close.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/{index}/_open",
       "paths": ["/{index}/_open"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_alias.json
index da53d5ac441e1..8cbcfcdd25e53 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_alias.json
@@ -1,9 +1,9 @@
 {
   "indices.put_alias": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html",
+    "stability": "stable",
     "methods": ["PUT", "POST"],
     "url": {
-      "path": "/{index}/_alias/{name}",
       "paths": ["/{index}/_alias/{name}", "/{index}/_aliases/{name}"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
index c46e6b15549cc..064c532917c74 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
@@ -1,9 +1,9 @@
 {
   "indices.put_mapping": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-put-mapping.html",
+    "stability": "stable",
     "methods": ["PUT", "POST"],
     "url": {
-      "path": "{index}/_mapping",
       "paths": ["{index}/_mapping"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
index 3055cb8e32e2e..3ef98b8b0a54c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_settings.json
@@ -1,9 +1,9 @@
 {
   "indices.put_settings": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-update-settings.html",
+    "stability": "stable",
     "methods": ["PUT"],
     "url": {
-      "path": "/_settings",
       "paths": ["/_settings", "/{index}/_settings"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json
index 65aa9506ff9f1..58235afa281f2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json
@@ -1,9 +1,9 @@
 {
   "indices.put_template": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+    "stability": "stable",
     "methods": ["PUT", "POST"],
     "url": {
-      "path": "/_template/{name}",
       "paths": ["/_template/{name}"],
       "parts": {
         "name": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.recovery.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.recovery.json
index 80b7f1a000986..6c3b574d23b0d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.recovery.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.recovery.json
@@ -1,9 +1,9 @@
 {
     "indices.recovery" : {
         "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-recovery.html",
+        "stability": "stable",
         "methods": ["GET"],
         "url": {
-            "path": "/_recovery",
             "paths": ["/_recovery", "/{index}/_recovery"],
             "parts": {
                 "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json
index a32974d017f22..157aae6c9178e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.refresh.json
@@ -1,9 +1,9 @@
 {
   "indices.refresh": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-refresh.html",
+    "stability": "stable",
     "methods": ["POST", "GET"],
     "url": {
-      "path": "/_refresh",
       "paths": ["/_refresh", "/{index}/_refresh"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
index 7bf1513969fb3..3d5addef6b5cb 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
@@ -1,9 +1,9 @@
 {
   "indices.rollover": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-rollover-index.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/{alias}/_rollover",
       "paths": ["/{alias}/_rollover", "/{alias}/_rollover/{new_index}"],
       "parts": {
         "alias": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
index 3ecf1b23aaea2..a39d492b5a40c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
@@ -1,9 +1,9 @@
 {
   "indices.segments": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-segments.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_segments",
       "paths": ["/_segments", "/{index}/_segments"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json
index ecdaa75a3a9e4..ff64c40f6674e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shard_stores.json
@@ -1,9 +1,9 @@
 {
   "indices.shard_stores": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-shards-stores.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_shard_stores",
       "paths": ["/_shard_stores", "/{index}/_shard_stores"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json
index 5ef943eacba6c..111444b7ebca8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json
@@ -1,9 +1,9 @@
 {
   "indices.shrink": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-shrink-index.html",
+    "stability": "stable",
     "methods": ["PUT", "POST"],
     "url": {
-      "path": "/{index}/_shrink/{target}",
       "paths": ["/{index}/_shrink/{target}"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json
index a79fa7b708269..10830b4bb9504 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json
@@ -1,9 +1,9 @@
 {
   "indices.split": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-split-index.html",
+    "stability": "stable",
     "methods": ["PUT", "POST"],
     "url": {
-      "path": "/{index}/_split/{target}",
       "paths": ["/{index}/_split/{target}"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
index c86a2c1147a9b..3be4d9387245b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
@@ -1,9 +1,9 @@
 {
   "indices.stats": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-stats.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_stats",
       "paths": [
         "/_stats",
         "/_stats/{metric}",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.update_aliases.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.update_aliases.json
index 30c369e410a25..8c75adfe68bf0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.update_aliases.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.update_aliases.json
@@ -1,9 +1,9 @@
 {
   "indices.update_aliases": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/_aliases",
       "paths": ["/_aliases"],
       "parts": {
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json
index 1e2413ee723cc..ce408a4e1e509 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.upgrade.json
@@ -1,9 +1,9 @@
 {
   "indices.upgrade": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-upgrade.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/_upgrade",
       "paths": ["/_upgrade", "/{index}/_upgrade"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json
index 6f1395b1582a0..a4cf1d0e387c8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.validate_query.json
@@ -1,9 +1,9 @@
 {
   "indices.validate_query": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-validate.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/_validate/query",
       "paths": ["/_validate/query", "/{index}/_validate/query"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/info.json b/rest-api-spec/src/main/resources/rest-api-spec/api/info.json
index 63754eb7f7c3c..c1fec52fc8ca4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/info.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/info.json
@@ -1,9 +1,9 @@
 {
   "info": {
     "documentation": "http://www.elastic.co/guide/",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/",
       "paths": ["/"],
       "parts": {
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_pipeline.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_pipeline.json
index c3b51de862097..f1b3168941814 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_pipeline.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_pipeline.json
@@ -1,9 +1,9 @@
 {
   "ingest.delete_pipeline": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-pipeline-api.html",
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_ingest/pipeline/{id}",
       "paths": [ "/_ingest/pipeline/{id}" ],
       "parts": {
         "id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_pipeline.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_pipeline.json
index 16a07e072b771..e91226c1859f9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_pipeline.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_pipeline.json
@@ -1,9 +1,9 @@
 {
   "ingest.get_pipeline": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/get-pipeline-api.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_ingest/pipeline/{id}",
       "paths": [ "/_ingest/pipeline", "/_ingest/pipeline/{id}" ],
       "parts": {
         "id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.processor_grok.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.processor_grok.json
index bf40be853e2d1..cb98ef30df948 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.processor_grok.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.processor_grok.json
@@ -1,9 +1,9 @@
 {
   "ingest.processor_grok": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/grok-processor.html#grok-processor-rest-get",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_ingest/processor/grok",
       "paths": ["/_ingest/processor/grok"],
       "parts": {
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_pipeline.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_pipeline.json
index 1ea77901d8dbd..166b2b9d4190e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_pipeline.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_pipeline.json
@@ -1,9 +1,9 @@
 {
   "ingest.put_pipeline": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/put-pipeline-api.html",
+    "stability": "stable",
     "methods": [ "PUT" ],
     "url": {
-      "path": "/_ingest/pipeline/{id}",
       "paths": [ "/_ingest/pipeline/{id}" ],
       "parts": {
         "id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.simulate.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.simulate.json
index c16008ad6b659..5010c02424e5b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.simulate.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.simulate.json
@@ -1,9 +1,9 @@
 {
   "ingest.simulate": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/simulate-pipeline-api.html",
+    "stability": "stable",
     "methods": [ "GET", "POST" ],
     "url": {
-      "path": "/_ingest/pipeline/_simulate",
       "paths": [ "/_ingest/pipeline/_simulate", "/_ingest/pipeline/{id}/_simulate" ],
       "parts": {
         "id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/mget.json b/rest-api-spec/src/main/resources/rest-api-spec/api/mget.json
index c04fb34beac18..bd8e258c2e206 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/mget.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/mget.json
@@ -1,9 +1,9 @@
 {
   "mget": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-multi-get.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/_mget",
       "paths": ["/_mget", "/{index}/_mget"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
index 73780a1835893..6f3e0c5e9d3bb 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
@@ -1,9 +1,9 @@
 {
   "msearch": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-multi-search.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/_msearch",
       "paths": ["/_msearch", "/{index}/_msearch"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json
index 7bc957d98ff29..412f4a508f433 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json
@@ -1,9 +1,9 @@
 {
   "msearch_template": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/search-multi-search.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/_msearch/template",
       "paths": ["/_msearch/template", "/{index}/_msearch/template"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/mtermvectors.json b/rest-api-spec/src/main/resources/rest-api-spec/api/mtermvectors.json
index aaff8e73259cf..7b722a5bebfa5 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/mtermvectors.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/mtermvectors.json
@@ -1,9 +1,9 @@
 {
   "mtermvectors" : {
     "documentation" : "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-multi-termvectors.html",
+    "stability": "stable",
     "methods" : ["GET", "POST"],
     "url" : {
-      "path" : "/_mtermvectors",
       "paths" : ["/_mtermvectors", "/{index}/_mtermvectors"],
       "parts" : {
         "index" : {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.hot_threads.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.hot_threads.json
index 95df4f53aa579..b94fbaaaedf88 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.hot_threads.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.hot_threads.json
@@ -1,9 +1,9 @@
 {
   "nodes.hot_threads": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-hot-threads.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_nodes/hot_threads",
       "paths": ["/_nodes/hot_threads",  "/_nodes/{node_id}/hot_threads"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.info.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.info.json
index c86d35dc1a95a..ca6637d81dcb4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.info.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.info.json
@@ -1,9 +1,9 @@
 {
   "nodes.info": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-info.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_nodes",
       "paths": ["/_nodes", "/_nodes/{node_id}", "/_nodes/{metric}", "/_nodes/{node_id}/{metric}"],
       "parts": {
         "node_id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.reload_secure_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.reload_secure_settings.json
index 0a566df35bbc2..e40e66cb266bb 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.reload_secure_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.reload_secure_settings.json
@@ -1,9 +1,9 @@
 {
   "nodes.reload_secure_settings": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/secure-settings.html#reloadable-secure-settings",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/_nodes/reload_secure_settings",
       "paths": ["/_nodes/reload_secure_settings", "/_nodes/{node_id}/reload_secure_settings"],
       "parts": {
         "node_id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json
index 7b426d03918b2..50af7b6d478fc 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json
@@ -1,9 +1,9 @@
 {
   "nodes.stats": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-stats.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_nodes/stats",
       "paths": [
         "/_nodes/stats",
         "/_nodes/{node_id}/stats",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.usage.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.usage.json
index 97c3f201f6cdf..6b70af58c08ea 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.usage.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.usage.json
@@ -1,9 +1,9 @@
 {
   "nodes.usage": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-usage.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_nodes/usage",
       "paths": [
         "/_nodes/usage",
         "/_nodes/{node_id}/usage",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ping.json
index 41310bb30d494..d49dd78721659 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ping.json
@@ -1,9 +1,9 @@
 {
   "ping": {
     "documentation": "http://www.elastic.co/guide/",
+    "stability": "stable",
     "methods": ["HEAD"],
     "url": {
-      "path": "/",
       "paths": ["/"],
       "parts": {
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/put_script.json b/rest-api-spec/src/main/resources/rest-api-spec/api/put_script.json
index 34bd4f63c285e..5de2bf9bd306f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/put_script.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/put_script.json
@@ -1,9 +1,9 @@
 {
   "put_script": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting.html",
+    "stability": "stable",
     "methods": ["PUT", "POST"],
     "url": {
-      "path": "/_scripts/{id}",
       "paths": [ "/_scripts/{id}", "/_scripts/{id}/{context}" ],
       "parts": {
         "id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json
index 5c9cebf741135..8bb2212d7a68a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json
@@ -1,9 +1,9 @@
 {
   "rank_eval": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/search-rank-eval.html",
+    "stability" : "experimental",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/_rank_eval",
       "paths": ["/_rank_eval", "/{index}/_rank_eval"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json
index e85eadb5bc41e..e4305bdd7a484 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json
@@ -1,9 +1,9 @@
 {
   "reindex": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-reindex.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/_reindex",
       "paths": ["/_reindex"],
       "parts": {},
       "params": {
@@ -39,6 +39,10 @@
           "type": "number",
           "default": 1,
           "description": "The number of slices this task should be divided into. Defaults to 1 meaning the task isn't sliced into subtasks."
+        },
+        "max_docs": {
+          "type" : "number",
+          "description" : "Maximum number of documents to process (default: all documents)"
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex_rethrottle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex_rethrottle.json
index 2763eb8983fd5..a6957c5f7c71a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex_rethrottle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex_rethrottle.json
@@ -1,9 +1,9 @@
 {
   "reindex_rethrottle": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-reindex.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/_reindex/{task_id}/_rethrottle",
       "paths": ["/_reindex/{task_id}/_rethrottle"],
       "parts": {
         "task_id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/render_search_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/render_search_template.json
index 8f27c12b819da..fa3af422fee17 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/render_search_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/render_search_template.json
@@ -1,9 +1,9 @@
 {
   "render_search_template": {
     "documentation": "http://www.elasticsearch.org/guide/en/elasticsearch/reference/master/search-template.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/_render/template",
       "paths": [ "/_render/template", "/_render/template/{id}" ],
       "parts": {
         "id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/scripts_painless_execute.json b/rest-api-spec/src/main/resources/rest-api-spec/api/scripts_painless_execute.json
index c02627cfd874c..3cb9c18ef2103 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/scripts_painless_execute.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/scripts_painless_execute.json
@@ -1,9 +1,9 @@
 {
   "scripts_painless_execute": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/painless/master/painless-execute-api.html",
+    "stability" : "experimental",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/_scripts/painless/_execute",
       "paths": ["/_scripts/painless/_execute"],
       "parts": {
       },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json b/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json
index 31e02a172fd0e..88851b2db1e34 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json
@@ -1,9 +1,9 @@
 {
   "scroll": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-request-scroll.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/_search/scroll",
       "paths": ["/_search/scroll"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
index 75444eb66767b..30f5c6747f3a8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
@@ -1,9 +1,9 @@
 {
   "search": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-search.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/_search",
       "paths": ["/_search", "/{index}/_search"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
index b3de107b79787..082603209925e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_shards.json
@@ -1,9 +1,9 @@
 {
   "search_shards": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-shards.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/{index}/_search_shards",
       "paths": ["/_search_shards", "/{index}/_search_shards"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
index 81a293d4a9b34..a6777d0628126 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
@@ -1,9 +1,9 @@
 {
   "search_template": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/search-template.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/_search/template",
       "paths": ["/_search/template", "/{index}/_search/template"],
       "parts": {
         "index": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create.json
index 29ae2206c8568..66be1eeff675c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create.json
@@ -1,9 +1,9 @@
 {
   "snapshot.create": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+    "stability": "stable",
     "methods": ["PUT", "POST"],
     "url": {
-      "path": "/_snapshot/{repository}/{snapshot}",
       "paths": ["/_snapshot/{repository}/{snapshot}"],
       "parts": {
         "repository": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create_repository.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create_repository.json
index 7a73f6abc3c28..830eb2952886e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create_repository.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create_repository.json
@@ -1,9 +1,9 @@
 {
   "snapshot.create_repository": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+    "stability": "stable",
     "methods": ["PUT", "POST"],
     "url": {
-      "path": "/_snapshot/{repository}",
       "paths": ["/_snapshot/{repository}"],
       "parts": {
         "repository": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete.json
index 6668289f442a6..c04dc5410ea10 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete.json
@@ -1,9 +1,9 @@
 {
   "snapshot.delete": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+    "stability": "stable",
     "methods": ["DELETE"],
     "url": {
-      "path": "/_snapshot/{repository}/{snapshot}",
       "paths": ["/_snapshot/{repository}/{snapshot}"],
       "parts": {
         "repository": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete_repository.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete_repository.json
index 4f0c43b8f0317..1912423978173 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete_repository.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete_repository.json
@@ -1,9 +1,9 @@
 {
   "snapshot.delete_repository": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+    "stability": "stable",
     "methods": ["DELETE"],
     "url": {
-      "path": "/_snapshot/{repository}",
       "paths": ["/_snapshot/{repository}"],
       "parts": {
         "repository": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get.json
index 02f5259bc2ec8..b23a3fbc0281d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get.json
@@ -1,9 +1,9 @@
 {
   "snapshot.get": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_snapshot/{repository}/{snapshot}",
       "paths": ["/_snapshot/{repository}/{snapshot}"],
       "parts": {
         "repository": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_repository.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_repository.json
index bf6a660146173..9aa13758097c4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_repository.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_repository.json
@@ -1,9 +1,9 @@
 {
   "snapshot.get_repository": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_snapshot",
       "paths": ["/_snapshot", "/_snapshot/{repository}"],
       "parts": {
         "repository": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json
index bdd265799ce4b..6107dd1663a37 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json
@@ -1,9 +1,9 @@
 {
   "snapshot.restore": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/_snapshot/{repository}/{snapshot}/_restore",
       "paths": ["/_snapshot/{repository}/{snapshot}/_restore"],
       "parts": {
         "repository": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json
index cba488de793bc..e76ce7893951e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json
@@ -1,9 +1,9 @@
 {
   "snapshot.status": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_snapshot/_status",
       "paths": ["/_snapshot/_status", "/_snapshot/{repository}/_status", "/_snapshot/{repository}/{snapshot}/_status"],
       "parts": {
         "repository": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.verify_repository.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.verify_repository.json
index 0d9a97c10877b..c875991b21a56 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.verify_repository.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.verify_repository.json
@@ -1,9 +1,9 @@
 {
     "snapshot.verify_repository": {
         "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+        "stability": "stable",
         "methods": ["POST"],
         "url": {
-            "path": "/_snapshot/{repository}/_verify",
             "paths": ["/_snapshot/{repository}/_verify"],
             "parts": {
                 "repository": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.cancel.json b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.cancel.json
index 244ca986f9b7f..4ee287ee6454b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.cancel.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.cancel.json
@@ -1,9 +1,9 @@
 {
   "tasks.cancel": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/_tasks",
       "paths": ["/_tasks/_cancel", "/_tasks/{task_id}/_cancel"],
       "parts": {
         "task_id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.get.json
index 3fefdb8f965ff..63702f7c592c6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.get.json
@@ -1,9 +1,9 @@
 {
   "tasks.get": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_tasks/{task_id}",
       "paths": ["/_tasks/{task_id}"],
       "parts": {
         "task_id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
index f16967eafeb02..8bb31723e0be4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
@@ -1,9 +1,9 @@
 {
   "tasks.list": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_tasks",
       "paths": ["/_tasks"],
       "parts": {},
       "params": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/termvectors.json b/rest-api-spec/src/main/resources/rest-api-spec/api/termvectors.json
index bbbdc7c87ad0b..2bfacaf676c67 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/termvectors.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/termvectors.json
@@ -1,9 +1,9 @@
 {
   "termvectors" : {
     "documentation" : "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-termvectors.html",
+    "stability": "stable",
     "methods" : ["GET", "POST"],
     "url" : {
-      "path" : "/{index}/_termvectors/{id}",
       "paths" : ["/{index}/_termvectors/{id}", "/{index}/_termvectors"],
       "parts" : {
         "index" : {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update.json
index 02435190674cf..4b850002a3337 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/update.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update.json
@@ -1,9 +1,9 @@
 {
   "update": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/docs-update.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/{index}/_update/{id}",
       "paths": ["/{index}/_update/{id}"],
       "deprecated_paths" : [
         {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
index 10b6e6d87f277..4048b4a55d962 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
@@ -1,9 +1,9 @@
 {
   "update_by_query": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-update-by-query.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/{index}/_update_by_query",
       "paths": ["/{index}/_update_by_query"],
       "comment": "most things below this are just copied from search.json",
       "parts": {
@@ -90,9 +90,9 @@
           "type" : "time",
           "description" : "Explicit timeout for each search request. Defaults to no timeout."
         },
-        "size": {
+        "max_docs": {
           "type" : "number",
-          "description" : "Number of hits to return (default: 10)"
+          "description" : "Maximum number of documents to process (default: all documents)"
         },
         "sort": {
           "type" : "list",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query_rethrottle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query_rethrottle.json
index 9ec2540b43006..e79f2902329a6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query_rethrottle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query_rethrottle.json
@@ -1,9 +1,9 @@
 {
   "update_by_query_rethrottle": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/_update_by_query/{task_id}/_rethrottle",
       "paths": ["/_update_by_query/{task_id}/_rethrottle"],
       "parts": {
         "task_id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yml
index 6e03ceb98c716..4f425f4817638 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.snapshots/10_basic.yml
@@ -1,5 +1,9 @@
 ---
 "Help":
+  - skip:
+      version: " - 7.9.99"
+      reason: Repository field added in 8.0
+
   - do:
       cat.snapshots:
         help: true
@@ -7,6 +11,7 @@
   - match:
       $body: |
                /^   id                .+   \n
+                    repository        .+   \n
                     status            .+   \n
                     start_epoch       .+   \n
                     start_time        .+   \n
@@ -21,6 +26,9 @@
                $/
 ---
 "Test cat snapshots output":
+  - skip:
+      version: " - 7.9.99"
+      reason: Repository field added in 8.0
 
   - do:
       snapshot.create_repository:
@@ -74,6 +82,6 @@
 
   - match:
       $body: |
-               /^   snap1\s+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n
-                    snap2\s+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n
+               /^   snap1\s+ test_cat_snapshots_1\s+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n
+                    snap2\s+ test_cat_snapshots_1\s+ SUCCESS\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \d+\s+ \d\d\:\d\d\:\d\d\s+ \S+\s+ 2\s+ 2\s+ 0\s+ 2\s*\n
                $/
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/count/11_basic_with_types.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/count/11_basic_with_types.yml
deleted file mode 100644
index 48cfc610b435e..0000000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/count/11_basic_with_types.yml
+++ /dev/null
@@ -1,66 +0,0 @@
-setup:
-  - do:
-      indices.create:
-          index:  test
-  - do:
-      index:
-          index:  test
-          type:   test
-          id:     1
-          body:   { foo: bar }
-
-  - do:
-      indices.refresh:
-        index: [test]
-
----
-"count with body":
-  - do:
-      count:
-        index: test
-        type: test
-        body:
-          query:
-            match:
-              foo: bar
-
-  - match: {count : 1}
-
-  - do:
-      count:
-        index: test
-        body:
-          query:
-            match:
-              foo: test
-
-  - match: {count : 0}
-
----
-"count with empty body":
-# empty body should default to match_all query
-  - do:
-      count:
-        index: test
-        type: test
-        body: { }
-
-  - match: {count : 1}
-
-  - do:
-      count:
-        index: test
-        type: test
-
-  - match: {count : 1}
-
----
-"count body without query element":
-  - do:
-      catch: bad_request
-      count:
-        index: test
-        type: test
-        body:
-          match:
-            foo: bar
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yml
index 8062a96f3e6bc..a852e6d3beef1 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yml
@@ -72,5 +72,4 @@
     - match:  { detail.tokenizer.tokens.0.token: foo }
     - match:  { detail.tokenizer.tokens.1.token: bar }
     - match:  { detail.tokenizer.tokens.2.token: buzz }
-    - match:  { detail.tokenfilters.0.name: "_anonymous_tokenfilter" }
     - match:  { detail.tokenfilters.0.tokens.0.token: bar }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml
index f227e076aa9c6..4806601cec263 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.recovery/10_basic.yml
@@ -130,3 +130,28 @@
         index: [v*]
 
   - match: { $body: {} }
+---
+"Indices recovery test with detailed parameter":
+  - skip:
+      version: " - 7.2.99"
+      reason: bug with detailed parameter fixed in 7.3
+
+  - do:
+      indices.create:
+        index:  test_3
+        body:
+          settings:
+            index:
+              number_of_replicas: 0
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+
+  - do:
+      indices.recovery:
+        index: [test_3]
+        human: true
+        detailed: true
+
+  - match: { test_3.shards.0.index.files.details: [] }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml
index 83b9c429bbdbf..016bd5912d3a3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml
@@ -77,3 +77,47 @@ setup:
       indices.stats:
         metric: [ translog ]
   - gte: { indices.test.primaries.translog.earliest_last_modified_age: 0 }
+
+---
+"Translog stats on closed indices":
+  - skip:
+      version: " - 7.2.99"
+      reason:  "closed indices have translog stats starting version 7.3.0"
+
+  - do:
+      index:
+        index: test
+        id:    1
+        body:  { "foo": "bar" }
+
+  - do:
+      index:
+        index: test
+        id:    2
+        body:  { "foo": "bar" }
+
+  - do:
+      index:
+        index: test
+        id:    3
+        body:  { "foo": "bar" }
+
+  - do:
+      indices.stats:
+        metric: [ translog ]
+  - match: { indices.test.primaries.translog.operations: 3 }
+  - match: { indices.test.primaries.translog.uncommitted_operations: 3 }
+
+  - do:
+      indices.close:
+        index: test
+        wait_for_active_shards: 1
+  - is_true: acknowledged
+
+  - do:
+      indices.stats:
+        metric: [ translog ]
+        expand_wildcards: all
+        forbid_closed_indices: false
+  - match: { indices.test.primaries.translog.operations: 3 }
+  - match: { indices.test.primaries.translog.uncommitted_operations: 0 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/20_docs.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/20_docs.yml
index bb1b25a0dcb40..50eb344d99048 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/20_docs.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/20_docs.yml
@@ -41,12 +41,10 @@
                 like:
                   -
                     _index: test_1
-                    _type: _doc
                     doc:
                       foo: bar
                   -
                     _index: test_1
-                    _type: _doc
                     _id: 2
                   -
                     _id: 3
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/30_unlike.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/30_unlike.yml
index abea4c8fbe57a..a0f96eb6b2d1f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/30_unlike.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mlt/30_unlike.yml
@@ -40,11 +40,9 @@
               more_like_this:
                 like:
                     _index: test_1
-                    _type: _doc
                     _id: 1
                 unlike:
                     _index: test_1
-                    _type: _doc
                     _id: 3
                 include: true
                 min_doc_freq: 0
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/12_basic_with_types.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/12_basic_with_types.yml
deleted file mode 100644
index 64e88de404ab7..0000000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/12_basic_with_types.yml
+++ /dev/null
@@ -1,97 +0,0 @@
----
-setup:
-
-  - do:
-      index:
-          index:  index_1
-          type:   test
-          id:     1
-          body:   { foo: bar }
-
-  - do:
-      index:
-          index:  index_1
-          type:   test
-          id:     2
-          body:   { foo: baz }
-
-  - do:
-      index:
-          index:  index_1
-          type:   test
-          id:     3
-          body:   { foo: foo }
-
-  - do:
-      index:
-          index:  index_2
-          type:   test
-          id:     1
-          body:   { foo: foo }
-
-  - do:
-      indices.refresh: {}
-
----
-"Basic multi-search":
-
-  - do:
-      msearch:
-        rest_total_hits_as_int: true
-        body:
-          - index: index_*
-          - query:
-              match: {foo: foo}
-          - index: index_2
-          - query:
-              match_all: {}
-          - index: index_1
-          - query:
-              match: {foo: foo}
-          - index: index_3
-          - query:
-              match_all: {}
-          - type: test
-          - query:
-              match_all: {}
-
-  - match:  { responses.0.hits.total:     2  }
-  - match:  { responses.1.hits.total:     1  }
-  - match:  { responses.2.hits.total:     1  }
-  - match:  { responses.3.error.root_cause.0.type: index_not_found_exception }
-  - match:  { responses.3.error.root_cause.0.reason: "/no.such.index/" }
-  - match:  { responses.3.error.root_cause.0.index: index_3 }
-  - match:  { responses.4.hits.total:     4  }
-
----
-"Least impact smoke test":
-# only passing these parameters to make sure they are consumed
-  - do:
-      msearch:
-        rest_total_hits_as_int: true
-        max_concurrent_shard_requests: 1
-        max_concurrent_searches: 1
-        body:
-          - index: index_*
-          - query:
-              match: {foo: foo}
-          - index: index_2
-          - query:
-              match_all: {}
-          - index: index_1
-          - query:
-              match: {foo: foo}
-          - index: index_3
-          - query:
-              match_all: {}
-          - type: test
-          - query:
-              match_all: {}
-
-  - match:  { responses.0.hits.total:     2  }
-  - match:  { responses.1.hits.total:     1  }
-  - match:  { responses.2.hits.total:     1  }
-  - match:  { responses.3.error.root_cause.0.type: index_not_found_exception }
-  - match:  { responses.3.error.root_cause.0.reason: "/no.such.index/" }
-  - match:  { responses.3.error.root_cause.0.index: index_3 }
-  - match:  { responses.4.hits.total:     4  }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/20_deprecated.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/20_deprecated.yml
index db8e566f4b613..8c058deff841c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/20_deprecated.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mtermvectors/20_deprecated.yml
@@ -40,7 +40,6 @@
                "docs":
                    -
                        "_index" : "testidx"
-                       "_type" : "_doc"
                        "_id" : "testing_document"
                        "version" : 1
                        "_version_type" : "external"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml
index cde56fa41e3d9..76274e9034d62 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml
@@ -92,16 +92,19 @@ setup:
               aggs:
                 users:
                   top_hits:
-                    sort: "users.last.keyword"
+                    sort:
+                      users.last.keyword:
+                        nested:
+                          path: users
                     seq_no_primary_term: true
 
   - match: { hits.total: 2 }
   - length: { aggregations.groups.buckets.0.users.hits.hits: 2 }
-  - match: { aggregations.groups.buckets.0.users.hits.hits.0._id: "1" }
+  - match: { aggregations.groups.buckets.0.users.hits.hits.0._id: "2" }
   - match: { aggregations.groups.buckets.0.users.hits.hits.0._index: my-index }
   - gte: { aggregations.groups.buckets.0.users.hits.hits.0._seq_no: 0 }
   - gte: { aggregations.groups.buckets.0.users.hits.hits.0._primary_term: 1 }
-  - match: { aggregations.groups.buckets.0.users.hits.hits.1._id: "2" }
+  - match: { aggregations.groups.buckets.0.users.hits.hits.1._id: "1" }
   - match: { aggregations.groups.buckets.0.users.hits.hits.1._index: my-index }
   - gte: { aggregations.groups.buckets.0.users.hits.hits.1._seq_no: 0 }
   - gte: { aggregations.groups.buckets.0.users.hits.hits.1._primary_term: 1 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/280_rare_terms.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/280_rare_terms.yml
new file mode 100644
index 0000000000000..a82caddd9cfd4
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/280_rare_terms.yml
@@ -0,0 +1,316 @@
+setup:
+  - skip:
+      version: " - 7.2.99"
+      reason: RareTerms added in 7.3.0
+  - do:
+      indices.create:
+          index: test_1
+          body:
+            settings:
+              number_of_replicas: 0
+            mappings:
+              properties:
+                str:
+                   type: keyword
+                ip:
+                   type: ip
+                boolean:
+                   type: boolean
+                integer:
+                  type: long
+                number:
+                  type: long
+                date:
+                  type: date
+
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+
+---
+"Basic test":
+  - do:
+      index:
+        index: test_1
+        id: 1
+        body: { "str" : "abc" }
+
+  - do:
+      index:
+        index: test_1
+        id: 2
+        body: { "str": "abc" }
+
+  - do:
+      index:
+        index: test_1
+        id: 3
+        body: { "str": "bcd" }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "str_terms" : { "rare_terms" : { "field" : "str", "max_doc_count" : 1 } } } }
+
+  - match: { hits.total.value: 3 }
+  - length: { aggregations.str_terms.buckets: 1 }
+  - match: { aggregations.str_terms.buckets.0.key: "bcd" }
+  - is_false: aggregations.str_terms.buckets.0.key_as_string
+  - match: { aggregations.str_terms.buckets.0.doc_count: 1 }
+
+---
+"IP test":
+  - do:
+      index:
+        index: test_1
+        id: 1
+        body: { "ip": "::1" }
+
+  - do:
+      index:
+        index: test_1
+        id: 2
+        body: { "ip": "127.0.0.1" }
+
+  - do:
+      index:
+        index: test_1
+        id: 3
+        body: { "ip": "::1" }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "ip_terms" : { "rare_terms" : { "field" : "ip" } } } }
+
+  - match: { hits.total.value: 3 }
+  - length: { aggregations.ip_terms.buckets: 1 }
+  - match: { aggregations.ip_terms.buckets.0.key: "127.0.0.1" }
+  - is_false: aggregations.ip_terms.buckets.0.key_as_string
+  - match: { aggregations.ip_terms.buckets.0.doc_count: 1 }
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "ip_terms" : { "rare_terms" : { "field" : "ip", "include" : [ "127.0.0.1" ] } } } }
+
+  - match: { hits.total.value: 3 }
+  - length: { aggregations.ip_terms.buckets: 1 }
+  - match: { aggregations.ip_terms.buckets.0.key: "127.0.0.1" }
+  - is_false: aggregations.ip_terms.buckets.0.key_as_string
+  - match: { aggregations.ip_terms.buckets.0.doc_count: 1 }
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "ip_terms" : { "rare_terms" : { "field" : "ip", "exclude" : [ "127.0.0.1" ] } } } }
+
+  - match: { hits.total.value: 3 }
+  - length: { aggregations.ip_terms.buckets: 0 }
+
+  - do:
+      catch: request
+      search:
+        index: test_1
+        body: { "size" : 0, "aggs" : { "ip_terms" : { "rare_terms" : { "field" : "ip", "exclude" :  "127.*"  } } } }
+
+
+
+---
+"Boolean test":
+  - do:
+      index:
+        index: test_1
+        id: 1
+        body: { "boolean": true }
+
+  - do:
+      index:
+        index: test_1
+        id: 2
+        body: { "boolean": false }
+
+  - do:
+      index:
+        index: test_1
+        id: 3
+        body: { "boolean": true }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "boolean_terms" : { "rare_terms" : { "field" : "boolean" } } } }
+
+  - match: { hits.total.value: 3 }
+  - length: { aggregations.boolean_terms.buckets: 1 }
+  - match: { aggregations.boolean_terms.buckets.0.key: 0 }
+  - match: { aggregations.boolean_terms.buckets.0.key_as_string: "false" }
+  - match: { aggregations.boolean_terms.buckets.0.doc_count: 1 }
+
+---
+"Integer test":
+  - do:
+      index:
+        index: test_1
+        id: 1
+        body: { "integer": 1234 }
+
+  - do:
+      index:
+        index: test_1
+        id: 2
+        body: { "integer": 5678 }
+
+  - do:
+      index:
+        index: test_1
+        id: 3
+        body: { "integer": 1234 }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "integer_terms" : { "rare_terms" : { "field" : "integer" } } } }
+
+  - match: { hits.total.value: 3 }
+
+  - length: { aggregations.integer_terms.buckets: 1 }
+
+  - match: { aggregations.integer_terms.buckets.0.key: 5678 }
+  - is_false: aggregations.integer_terms.buckets.0.key_as_string
+  - match: { aggregations.integer_terms.buckets.0.doc_count: 1 }
+
+---
+"Date test":
+  - do:
+      index:
+        index: test_1
+        id: 1
+        body: { "date": "2016-05-03" }
+
+  - do:
+      index:
+        index: test_1
+        id: 2
+        body: { "date": "2014-09-01" }
+
+  - do:
+      index:
+        index: test_1
+        id: 3
+        body: { "date": "2016-05-03" }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "date_terms" : { "rare_terms" : { "field" : "date" } } } }
+
+  - match: { hits.total.value: 3 }
+
+  - length: { aggregations.date_terms.buckets: 1 }
+  - match: { aggregations.date_terms.buckets.0.key: 1409529600000 }
+  - match: { aggregations.date_terms.buckets.0.key_as_string: "2014-09-01T00:00:00.000Z" }
+  - match: { aggregations.date_terms.buckets.0.doc_count: 1 }
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "date_terms" : { "rare_terms" : { "field" : "date", "include" : [ "2014-09-01" ] } } } }
+
+  - match: { hits.total.value: 3 }
+  - length: { aggregations.date_terms.buckets: 1 }
+  - match: { aggregations.date_terms.buckets.0.key_as_string: "2014-09-01T00:00:00.000Z" }
+  - match: { aggregations.date_terms.buckets.0.doc_count: 1 }
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "date_terms" : { "rare_terms" : { "field" : "date", "exclude" : [ "2014-09-01" ] } } } }
+
+  - match: { hits.total.value: 3 }
+  - length: { aggregations.date_terms.buckets: 0 }
+
+---
+"Unmapped strings":
+
+  - do:
+      index:
+        index: test_1
+        id: 1
+        body: {}
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "string_terms" : { "rare_terms" : { "field" : "unmapped_string"} } } }
+
+  - match: { hits.total.value: 1 }
+  - length: { aggregations.string_terms.buckets: 0 }
+
+---
+"Unmapped booleans":
+
+  - do:
+      index:
+        index: test_1
+        id: 1
+        body: {}
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "boolean_terms" : { "rare_terms" : { "field" : "unmapped_boolean" } } } }
+
+  - match: { hits.total.value: 1 }
+  - length: { aggregations.boolean_terms.buckets: 0 }
+
+---
+"Unmapped dates":
+
+  - do:
+      index:
+        index: test_1
+        id: 1
+        body: {}
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "date_terms" : { "rare_terms" : { "field" : "unmapped_date"} } } }
+
+  - match: { hits.total.value: 1 }
+  - length: { aggregations.date_terms.buckets: 0 }
+
+---
+"Unmapped longs":
+
+  - do:
+      index:
+        index: test_1
+        id: 1
+        body: {}
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "long_terms" : { "rare_terms" : { "field" : "unmapped_long", "value_type" : "long" } } } }
+
+  - match: { hits.total.value: 1 }
+  - length: { aggregations.long_terms.buckets: 0 }
+
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/230_interval_query.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/230_interval_query.yml
index 46bf2cada8e4d..82aa0883008a8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/230_interval_query.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/230_interval_query.yml
@@ -384,3 +384,43 @@ setup:
   - match: { hits.total.value: 1 }
   - match: { hits.hits.0._id: "4" }
 
+---
+"Test prefix":
+  - skip:
+      version: " - 8.0.0"
+      reason: "TODO: change to 7.3 in backport"
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            intervals:
+              text:
+                all_of:
+                  intervals:
+                    - match:
+                        query: cold
+                    - prefix:
+                        prefix: out
+  - match: { hits.total.value: 3 }
+
+---
+"Test wildcard":
+  - skip:
+      version: " - 8.0.0"
+      reason: "TODO: change to 7.3 in backport"
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            intervals:
+              text:
+                all_of:
+                  intervals:
+                    - match:
+                        query: cold
+                    - wildcard:
+                        pattern: out?ide
+  - match: { hits.total.value: 3 }
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/310_match_bool_prefix.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/310_match_bool_prefix.yml
index aa6a5158b4795..f92b0ffda80e3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/310_match_bool_prefix.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/310_match_bool_prefix.yml
@@ -345,19 +345,3 @@ setup:
               type: bool_prefix
               fields: [ "my_field1", "my_field2" ]
               slop: 1
-
----
-"multi_match multiple fields with cutoff_frequency throws exception":
-
-  - do:
-      catch: /\[cutoff_frequency\] not allowed for type \[bool_prefix\]/
-      search:
-        rest_total_hits_as_int: true
-        index: test
-        body:
-          query:
-            multi_match:
-              query: "brown"
-              type: bool_prefix
-              fields: [ "my_field1", "my_field2" ]
-              cutoff_frequency: 0.001
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml
index e2b7279f1cd68..219c5a70f331e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml
@@ -11,6 +11,9 @@ setup:
 
 ---
 "Get snapshot info":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   - do:
       indices.create:
@@ -31,8 +34,8 @@ setup:
         repository: test_repo_get_1
         snapshot: test_snapshot
 
-  - is_true: snapshots
-  - is_true: snapshots.0.failures
+  - is_true: responses.0.snapshots
+  - is_true: responses.0.snapshots.0.failures
 
   - do:
       snapshot.delete:
@@ -41,15 +44,23 @@ setup:
 
 ---
 "Get missing snapshot info throws an exception":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   - do:
-      catch: /snapshot_missing_exception.+ is missing/
       snapshot.get:
         repository: test_repo_get_1
         snapshot: test_nonexistent_snapshot
 
+  - is_true: responses.0.error
+  - match: { responses.0.error.type: snapshot_missing_exception }
+
 ---
 "Get missing snapshot info succeeds when ignore_unavailable is true":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   - do:
       snapshot.get:
@@ -57,10 +68,14 @@ setup:
         snapshot: test_nonexistent_snapshot
         ignore_unavailable: true
 
-  - is_true: snapshots
+  - is_true: responses.0.snapshots
 
 ---
 "Get snapshot info when verbose is false":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
+
   - do:
       indices.create:
         index: test_index
@@ -81,12 +96,13 @@ setup:
         snapshot: test_snapshot
         verbose: false
 
-  - is_true: snapshots
-  - match: { snapshots.0.snapshot: test_snapshot }
-  - match: { snapshots.0.state: SUCCESS }
-  - is_false: snapshots.0.failures
-  - is_false: snapshots.0.shards
-  - is_false: snapshots.0.version
+  - is_true: responses.0.snapshots
+  - match: { responses.0.snapshots.0.snapshot: test_snapshot }
+  - match: { responses.0.snapshots.0.state: SUCCESS }
+  - is_false: responses.0.snapshots.0.failures
+  - is_false: responses.0.snapshots.0.shards
+  - is_false: responses.0.snapshots.0.version
+  - is_false: responses.0.snapshots.0._meta
 
   - do:
       snapshot.delete:
@@ -95,6 +111,9 @@ setup:
 
 ---
 "Get snapshot info contains include_global_state":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
 
   - do:
       indices.create:
@@ -117,10 +136,10 @@ setup:
         repository: test_repo_get_1
         snapshot: test_snapshot_with_include_global_state
 
-  - is_true: snapshots
-  - match: { snapshots.0.snapshot: test_snapshot_with_include_global_state }
-  - match: { snapshots.0.state: SUCCESS }
-  - match: { snapshots.0.include_global_state: true }
+  - is_true: responses.0.snapshots
+  - match: { responses.0.snapshots.0.snapshot: test_snapshot_with_include_global_state }
+  - match: { responses.0.snapshots.0.state: SUCCESS }
+  - match: { responses.0.snapshots.0.include_global_state: true }
 
   - do:
       snapshot.delete:
@@ -140,12 +159,50 @@ setup:
         repository: test_repo_get_1
         snapshot: test_snapshot_without_include_global_state
 
-  - is_true: snapshots
-  - match: { snapshots.0.snapshot: test_snapshot_without_include_global_state }
-  - match: { snapshots.0.state: SUCCESS }
-  - match: { snapshots.0.include_global_state: false }
+  - is_true: responses.0.snapshots
+  - match: { responses.0.snapshots.0.snapshot: test_snapshot_without_include_global_state }
+  - match: { responses.0.snapshots.0.state: SUCCESS }
+  - match: { responses.0.snapshots.0.include_global_state: false }
 
   - do:
       snapshot.delete:
         repository: test_repo_get_1
         snapshot: test_snapshot_without_include_global_state
+
+---
+"Get snapshot info with metadata":
+  - skip:
+      version: " - 7.9.99"
+      reason: "8.0 changes get snapshots response format"
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            number_of_shards:   1
+            number_of_replicas: 0
+
+  - do:
+      snapshot.create:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_with_metadata
+        wait_for_completion: true
+        body: |
+          { "metadata": {"taken_by": "test", "foo": {"bar": "baz"}} }
+
+  - do:
+      snapshot.get:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_with_metadata
+
+  - is_true: responses.0.snapshots
+  - match: { responses.0.snapshots.0.snapshot: test_snapshot_with_metadata }
+  - match: { responses.0.snapshots.0.state: SUCCESS }
+  - match: { responses.0.snapshots.0.metadata.taken_by: test }
+  - match: { responses.0.snapshots.0.metadata.foo.bar: baz }
+
+  - do:
+      snapshot.delete:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_with_metadata
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yml
index eb7d335c28c60..e3ba834de074f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yml
@@ -33,6 +33,5 @@
         realtime: false
 
   - match: { _index: "testidx" }
-  - match: { _type: "_doc" }
   - match: { _id: "1" }
   - is_false: found
diff --git a/server/build.gradle b/server/build.gradle
index 391fdf46469f0..33fab57bc0626 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -74,13 +74,13 @@ if (!isEclipse && !isIdea) {
 
 dependencies {
 
-  compile "org.elasticsearch:elasticsearch-core:${version}"
-  compile "org.elasticsearch:elasticsearch-secure-sm:${version}"
-  compile "org.elasticsearch:elasticsearch-x-content:${version}"
-  compile "org.elasticsearch:elasticsearch-geo:${version}"
+  compile project(':libs:elasticsearch-core')
+  compile project(':libs:elasticsearch-secure-sm')
+  compile project(':libs:elasticsearch-x-content')
+  compile project(":libs:elasticsearch-geo")
   
-  compileOnly project(':libs:plugin-classloader')
-  testRuntime project(':libs:plugin-classloader')
+  compileOnly project(':libs:elasticsearch-plugin-classloader')
+  testRuntime project(':libs:elasticsearch-plugin-classloader')
 
   // lucene
   compile "org.apache.lucene:lucene-core:${versions.lucene}"
@@ -100,8 +100,8 @@ dependencies {
   compile "org.apache.lucene:lucene-suggest:${versions.lucene}"
 
   // utilities
-  compile "org.elasticsearch:elasticsearch-cli:${version}"
-  compile 'com.carrotsearch:hppc:0.7.1'
+  compile project(":libs:elasticsearch-cli")
+  compile 'com.carrotsearch:hppc:0.8.1'
 
   // time handling, remove with java 8 time
   compile "joda-time:joda-time:${versions.joda}"
@@ -118,8 +118,6 @@ dependencies {
   // logging
   compile "org.apache.logging.log4j:log4j-api:${versions.log4j}"
   compile "org.apache.logging.log4j:log4j-core:${versions.log4j}", optional
-  // to bridge dependencies that are still on Log4j 1 to Log4j 2
-  compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}", optional
 
   // repackaged jna with native bits linked against all elastic supported platforms
   compile "org.elasticsearch:jna:${versions.jna}"
@@ -129,9 +127,9 @@ dependencies {
   }
 
   if (isEclipse == false || project.path == ":server-tests") {
-    testCompile("org.elasticsearch.test:framework:${version}") {
+    testCompile(project(":test:framework")) {
       // tests use the locally compiled version of server
-      exclude group: 'org.elasticsearch', module: 'elasticsearch'
+      exclude group: 'org.elasticsearch', module: 'server'
     }
   }
 }
diff --git a/server/licenses/hppc-0.7.1.jar.sha1 b/server/licenses/hppc-0.7.1.jar.sha1
deleted file mode 100644
index aa191a6c93b99..0000000000000
--- a/server/licenses/hppc-0.7.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8b5057f74ea378c0150a1860874a3ebdcb713767
\ No newline at end of file
diff --git a/server/licenses/hppc-0.8.1.jar.sha1 b/server/licenses/hppc-0.8.1.jar.sha1
new file mode 100644
index 0000000000000..47684ed023210
--- /dev/null
+++ b/server/licenses/hppc-0.8.1.jar.sha1
@@ -0,0 +1 @@
+ffc7ba8f289428b9508ab484b8001dea944ae603
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java b/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
index c696d476bbb43..5f00631ad6028 100644
--- a/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
+++ b/server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
@@ -22,11 +22,8 @@
 import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermStates;
 import org.apache.lucene.index.TermState;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.index.TermStates;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.Query;
@@ -278,45 +275,6 @@ public int hashCode() {
         return Objects.hash(classHash(), Arrays.hashCode(equalsTerms()));
     }
 
-    public static BlendedTermQuery commonTermsBlendedQuery(Term[] terms, final float[] boosts, final float maxTermFrequency) {
-        return new BlendedTermQuery(terms, boosts) {
-            @Override
-            protected Query topLevelQuery(Term[] terms, TermStates[] ctx, int[] docFreqs, int maxDoc) {
-                BooleanQuery.Builder highBuilder = new BooleanQuery.Builder();
-                BooleanQuery.Builder lowBuilder = new BooleanQuery.Builder();
-                for (int i = 0; i < terms.length; i++) {
-                    Query query = new TermQuery(terms[i], ctx[i]);
-                    if (boosts != null && boosts[i] != 1f) {
-                        query = new BoostQuery(query, boosts[i]);
-                    }
-                    if ((maxTermFrequency >= 1f && docFreqs[i] > maxTermFrequency)
-                            || (docFreqs[i] > (int) Math.ceil(maxTermFrequency
-                            * maxDoc))) {
-                        highBuilder.add(query, BooleanClause.Occur.SHOULD);
-                    } else {
-                        lowBuilder.add(query, BooleanClause.Occur.SHOULD);
-                    }
-                }
-                BooleanQuery high = highBuilder.build();
-                BooleanQuery low = lowBuilder.build();
-                if (low.clauses().isEmpty()) {
-                    BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
-                    for (BooleanClause booleanClause : high) {
-                        queryBuilder.add(booleanClause.getQuery(), Occur.MUST);
-                    }
-                    return queryBuilder.build();
-                } else if (high.clauses().isEmpty()) {
-                    return low;
-                } else {
-                    return new BooleanQuery.Builder()
-                        .add(high, BooleanClause.Occur.SHOULD)
-                        .add(low, BooleanClause.Occur.MUST)
-                        .build();
-                }
-            }
-        };
-    }
-
     public static BlendedTermQuery dismaxBlendedQuery(Term[] terms, final float tieBreakerMultiplier) {
         return dismaxBlendedQuery(terms, null, tieBreakerMultiplier);
     }
diff --git a/server/src/main/java/org/apache/lucene/queries/ExtendedCommonTermsQuery.java b/server/src/main/java/org/apache/lucene/queries/ExtendedCommonTermsQuery.java
deleted file mode 100644
index 249b7fa83b50b..0000000000000
--- a/server/src/main/java/org/apache/lucene/queries/ExtendedCommonTermsQuery.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.lucene.queries;
-
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.elasticsearch.common.lucene.search.Queries;
-
-/**
- * Extended version of {@link CommonTermsQuery} that allows to pass in a
- * {@code minimumNumberShouldMatch} specification that uses the actual num of high frequent terms
- * to calculate the minimum matching terms.
- */
-public class ExtendedCommonTermsQuery extends CommonTermsQuery {
-
-    public ExtendedCommonTermsQuery(Occur highFreqOccur, Occur lowFreqOccur, float maxTermFrequency) {
-        super(highFreqOccur, lowFreqOccur, maxTermFrequency);
-    }
-
-    private String lowFreqMinNumShouldMatchSpec;
-    private String highFreqMinNumShouldMatchSpec;
-
-    @Override
-    protected int calcLowFreqMinimumNumberShouldMatch(int numOptional) {
-        return calcMinimumNumberShouldMatch(lowFreqMinNumShouldMatchSpec, numOptional);
-    }
-
-    protected int calcMinimumNumberShouldMatch(String spec, int numOptional) {
-        if (spec == null) {
-            return 0;
-        }
-        return Queries.calculateMinShouldMatch(numOptional, spec);
-    }
-
-    @Override
-    protected int calcHighFreqMinimumNumberShouldMatch(int numOptional) {
-        return calcMinimumNumberShouldMatch(highFreqMinNumShouldMatchSpec, numOptional);
-    }
-
-    public void setHighFreqMinimumNumberShouldMatch(String spec) {
-        this.highFreqMinNumShouldMatchSpec = spec;
-    }
-
-    public String getHighFreqMinimumNumberShouldMatchSpec() {
-        return highFreqMinNumShouldMatchSpec;
-    }
-
-    public void setLowFreqMinimumNumberShouldMatch(String spec) {
-        this.lowFreqMinNumShouldMatchSpec = spec;
-    }
-
-    public String getLowFreqMinimumNumberShouldMatchSpec() {
-        return lowFreqMinNumShouldMatchSpec;
-    }
-
-    public float getMaxTermFrequency() {
-        return this.maxTermFrequency;
-    }
-
-}
diff --git a/server/src/main/java/org/apache/lucene/util/CombinedBitSet.java b/server/src/main/java/org/apache/lucene/util/CombinedBitSet.java
new file mode 100644
index 0000000000000..cb1bd819ab24a
--- /dev/null
+++ b/server/src/main/java/org/apache/lucene/util/CombinedBitSet.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.util;
+
+import org.apache.lucene.search.DocIdSetIterator;
+
+/**
+ * A {@link BitSet} implementation that combines two instances of {@link BitSet} and {@link Bits}
+ * to provide a single merged view.
+ */
+public final class CombinedBitSet extends BitSet implements Bits {
+    private final BitSet first;
+    private final Bits second;
+    private final int length;
+
+    public CombinedBitSet(BitSet first, Bits second) {
+        this.first = first;
+        this.second = second;
+        this.length = first.length();
+    }
+
+    public BitSet getFirst() {
+        return first;
+    }
+
+    /**
+     * This implementation is slow and requires to iterate over all bits to compute
+     * the intersection. Use {@link #approximateCardinality()} for
+     * a fast approximation.
+     */
+    @Override
+    public int cardinality() {
+        int card = 0;
+        for (int i = 0; i < length; i++) {
+            card += get(i) ? 1 : 0;
+        }
+        return card;
+    }
+
+    @Override
+    public int approximateCardinality() {
+        return first.cardinality();
+    }
+
+    @Override
+    public int prevSetBit(int index) {
+        assert index >= 0 && index < length : "index=" + index + ", numBits=" + length();
+        int prev = first.prevSetBit(index);
+        while (prev != -1 && second.get(prev) == false) {
+            if (prev == 0) {
+                return -1;
+            }
+            prev = first.prevSetBit(prev-1);
+        }
+        return prev;
+    }
+
+    @Override
+    public int nextSetBit(int index) {
+        assert index >= 0 && index < length : "index=" + index + " numBits=" + length();
+        int next = first.nextSetBit(index);
+        while (next != DocIdSetIterator.NO_MORE_DOCS && second.get(next) == false) {
+            if (next == length() - 1) {
+                return DocIdSetIterator.NO_MORE_DOCS;
+            }
+            next = first.nextSetBit(next+1);
+        }
+        return next;
+    }
+
+    @Override
+    public long ramBytesUsed() {
+        return first.ramBytesUsed();
+    }
+
+    @Override
+    public boolean get(int index) {
+        return first.get(index) && second.get(index);
+    }
+
+    @Override
+    public int length() {
+        return length;
+    }
+
+    @Override
+    public void set(int i) {
+        throw new UnsupportedOperationException("not implemented");
+    }
+
+    @Override
+    public void clear(int i) {
+        throw new UnsupportedOperationException("not implemented");
+    }
+
+    @Override
+    public void clear(int startIndex, int endIndex) {
+        throw new UnsupportedOperationException("not implemented");
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchException.java b/server/src/main/java/org/elasticsearch/ElasticsearchException.java
index 260b443a6a557..51662414e0d07 100644
--- a/server/src/main/java/org/elasticsearch/ElasticsearchException.java
+++ b/server/src/main/java/org/elasticsearch/ElasticsearchException.java
@@ -1027,7 +1027,7 @@ private enum ElasticsearchExceptionHandle {
                 org.elasticsearch.index.shard.ShardNotInPrimaryModeException.class,
                 org.elasticsearch.index.shard.ShardNotInPrimaryModeException::new,
                 155,
-                Version.V_6_8_1);
+                UNKNOWN_VERSION_ADDED);
 
         final Class<? extends ElasticsearchException> exceptionClass;
         final CheckedFunction<StreamInput, ? extends ElasticsearchException, IOException> constructor;
diff --git a/server/src/main/java/org/elasticsearch/ExceptionsHelper.java b/server/src/main/java/org/elasticsearch/ExceptionsHelper.java
index 48461ffe30d4b..56d2cdbc087eb 100644
--- a/server/src/main/java/org/elasticsearch/ExceptionsHelper.java
+++ b/server/src/main/java/org/elasticsearch/ExceptionsHelper.java
@@ -38,12 +38,14 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.IdentityHashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Queue;
 import java.util.Set;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 public final class ExceptionsHelper {
@@ -185,22 +187,14 @@ public static <T extends Throwable> T useOrSuppress(T first, T second) {
      * @return Corruption indicating exception if one is found, otherwise {@code null}
      */
     public static IOException unwrapCorruption(Throwable t) {
-        if (t != null) {
-            do {
-                for (Class<?> clazz : CORRUPTION_EXCEPTIONS) {
-                    if (clazz.isInstance(t)) {
-                        return (IOException) t;
-                    }
-                }
-                for (Throwable suppressed : t.getSuppressed()) {
-                    IOException corruptionException = unwrapCorruption(suppressed);
-                    if (corruptionException != null) {
-                        return corruptionException;
-                    }
+        return t == null ? null : ExceptionsHelper.<IOException>unwrapCausesAndSuppressed(t, cause -> {
+            for (Class<?> clazz : CORRUPTION_EXCEPTIONS) {
+                if (clazz.isInstance(cause)) {
+                    return true;
                 }
-            } while ((t = t.getCause()) != null);
-        }
-        return null;
+            }
+            return false;
+        }).orElse(null);
     }
 
     /**
@@ -213,7 +207,11 @@ public static IOException unwrapCorruption(Throwable t) {
      */
     public static Throwable unwrap(Throwable t, Class<?>... clazzes) {
         if (t != null) {
+            final Set<Throwable> seen = Collections.newSetFromMap(new IdentityHashMap<>());
             do {
+                if (seen.add(t) == false) {
+                    return null;
+                }
                 for (Class<?> clazz : clazzes) {
                     if (clazz.isInstance(t)) {
                         return t;
@@ -246,33 +244,22 @@ public static boolean reThrowIfNotNull(@Nullable Throwable e) {
         return true;
     }
 
-    static final int MAX_ITERATIONS = 1024;
-
-    /**
-     * Unwrap the specified throwable looking for any suppressed errors or errors as a root cause of the specified throwable.
-     *
-     * @param cause the root throwable
-     * @return an optional error if one is found suppressed or a root cause in the tree rooted at the specified throwable
-     */
-    public static Optional<Error> maybeError(final Throwable cause, final Logger logger) {
-        // early terminate if the cause is already an error
-        if (cause instanceof Error) {
-            return Optional.of((Error) cause);
+    @SuppressWarnings("unchecked")
+    private static <T extends Throwable> Optional<T> unwrapCausesAndSuppressed(Throwable cause, Predicate<Throwable> predicate) {
+        if (predicate.test(cause)) {
+            return Optional.of((T) cause);
         }
 
         final Queue<Throwable> queue = new LinkedList<>();
         queue.add(cause);
-        int iterations = 0;
+        final Set<Throwable> seen = Collections.newSetFromMap(new IdentityHashMap<>());
         while (queue.isEmpty() == false) {
-            iterations++;
-            // this is a guard against deeply nested or circular chains of exceptions
-            if (iterations > MAX_ITERATIONS) {
-                logger.warn("giving up looking for fatal errors", cause);
-                break;
-            }
             final Throwable current = queue.remove();
-            if (current instanceof Error) {
-                return Optional.of((Error) current);
+            if (seen.add(current) == false) {
+                continue;
+            }
+            if (predicate.test(current)) {
+                return Optional.of((T) current);
             }
             Collections.addAll(queue, current.getSuppressed());
             if (current.getCause() != null) {
@@ -283,21 +270,24 @@ public static Optional<Error> maybeError(final Throwable cause, final Logger log
     }
 
     /**
-     * See {@link #maybeError(Throwable, Logger)}. Uses the class-local logger.
+     * Unwrap the specified throwable looking for any suppressed errors or errors as a root cause of the specified throwable.
+     *
+     * @param cause the root throwable
+     * @return an optional error if one is found suppressed or a root cause in the tree rooted at the specified throwable
      */
     public static Optional<Error> maybeError(final Throwable cause) {
-        return maybeError(cause, logger);
+        return unwrapCausesAndSuppressed(cause, t -> t instanceof Error);
     }
 
     /**
      * If the specified cause is an unrecoverable error, this method will rethrow the cause on a separate thread so that it can not be
      * caught and bubbles up to the uncaught exception handler. Note that the cause tree is examined for any {@link Error}. See
-     * {@link #maybeError(Throwable, Logger)} for the semantics.
+     * {@link #maybeError(Throwable)} for the semantics.
      *
      * @param throwable the throwable to possibly throw on another thread
      */
     public static void maybeDieOnAnotherThread(final Throwable throwable) {
-        ExceptionsHelper.maybeError(throwable, logger).ifPresent(error -> {
+        ExceptionsHelper.maybeError(throwable).ifPresent(error -> {
             /*
              * Here be dragons. We want to rethrow this so that it bubbles up to the uncaught exception handler. Yet, sometimes the stack
              * contains statements that catch any throwable (e.g., Netty, and the JDK futures framework). This means that a rethrow here
diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java
index c685d39c7562f..152446cb700be 100644
--- a/server/src/main/java/org/elasticsearch/Version.java
+++ b/server/src/main/java/org/elasticsearch/Version.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.collect.ImmutableOpenIntMap;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -46,54 +47,53 @@ public class Version implements Comparable<Version>, ToXContentFragment {
      */
     public static final int V_EMPTY_ID = 0;
     public static final Version V_EMPTY = new Version(V_EMPTY_ID, org.apache.lucene.util.Version.LATEST);
-    public static final int V_6_4_0_ID = 6040099;
-    public static final Version V_6_4_0 = new Version(V_6_4_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
-    public static final int V_6_4_1_ID = 6040199;
-    public static final Version V_6_4_1 = new Version(V_6_4_1_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
-    public static final int V_6_4_2_ID = 6040299;
-    public static final Version V_6_4_2 = new Version(V_6_4_2_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
-    public static final int V_6_4_3_ID = 6040399;
-    public static final Version V_6_4_3 = new Version(V_6_4_3_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
-    public static final int V_6_5_0_ID = 6050099;
-    public static final Version V_6_5_0 = new Version(V_6_5_0_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);
-    public static final int V_6_5_1_ID = 6050199;
-    public static final Version V_6_5_1 = new Version(V_6_5_1_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);
-    public static final int V_6_5_2_ID = 6050299;
-    public static final Version V_6_5_2 = new Version(V_6_5_2_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);
-    public static final int V_6_5_3_ID = 6050399;
-    public static final Version V_6_5_3 = new Version(V_6_5_3_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);
-    public static final int V_6_5_4_ID = 6050499;
-    public static final Version V_6_5_4 = new Version(V_6_5_4_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);
-    public static final int V_6_6_0_ID = 6060099;
-    public static final Version V_6_6_0 = new Version(V_6_6_0_ID, org.apache.lucene.util.Version.LUCENE_7_6_0);
-    public static final int V_6_6_1_ID = 6060199;
-    public static final Version V_6_6_1 = new Version(V_6_6_1_ID, org.apache.lucene.util.Version.LUCENE_7_6_0);
-    public static final int V_6_6_2_ID = 6060299;
-    public static final Version V_6_6_2 = new Version(V_6_6_2_ID, org.apache.lucene.util.Version.LUCENE_7_6_0);
-    public static final int V_6_8_0_ID = 6080099;
-    public static final Version V_6_8_0 = new Version(V_6_8_0_ID, org.apache.lucene.util.Version.LUCENE_7_7_0);
-    public static final int V_6_8_1_ID = 6080199;
-    public static final Version V_6_8_1 = new Version(V_6_8_1_ID, org.apache.lucene.util.Version.LUCENE_7_7_0);
-    public static final int V_7_0_0_ID = 7000099;
-    public static final Version V_7_0_0 = new Version(V_7_0_0_ID, org.apache.lucene.util.Version.LUCENE_8_0_0);
-    public static final int V_7_0_1_ID = 7000199;
-    public static final Version V_7_0_1 = new Version(V_7_0_1_ID, org.apache.lucene.util.Version.LUCENE_8_0_0);
-    public static final int V_7_1_0_ID = 7010099;
-    public static final Version V_7_1_0 = new Version(V_7_1_0_ID, org.apache.lucene.util.Version.LUCENE_8_0_0);
-    public static final int V_7_1_1_ID = 7010199;
-    public static final Version V_7_1_1 = new Version(V_7_1_1_ID, org.apache.lucene.util.Version.LUCENE_8_0_0);
-    public static final int V_7_2_0_ID = 7020099;
-    public static final Version V_7_2_0 = new Version(V_7_2_0_ID, org.apache.lucene.util.Version.LUCENE_8_0_0);
-    public static final int V_7_3_0_ID = 7030099;
-    public static final Version V_7_3_0 = new Version(V_7_3_0_ID, org.apache.lucene.util.Version.LUCENE_8_1_0);
-    public static final int V_8_0_0_ID = 8000099;
-    public static final Version V_8_0_0 = new Version(V_8_0_0_ID, org.apache.lucene.util.Version.LUCENE_8_1_0);
+    public static final Version V_7_0_0 = new Version(7000099, org.apache.lucene.util.Version.LUCENE_8_0_0);
+    public static final Version V_7_0_1 = new Version(7000199, org.apache.lucene.util.Version.LUCENE_8_0_0);
+    public static final Version V_7_1_0 = new Version(7010099, org.apache.lucene.util.Version.LUCENE_8_0_0);
+    public static final Version V_7_1_1 = new Version(7010199, org.apache.lucene.util.Version.LUCENE_8_0_0);
+    public static final Version V_7_2_0 = new Version(7020099, org.apache.lucene.util.Version.LUCENE_8_0_0);
+    public static final Version V_7_2_1 = new Version(7020199, org.apache.lucene.util.Version.LUCENE_8_0_0);
+    public static final Version V_7_3_0 = new Version(7030099, org.apache.lucene.util.Version.LUCENE_8_1_0);
+    public static final Version V_7_4_0 = new Version(7040099, org.apache.lucene.util.Version.LUCENE_8_1_0);
+    public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_8_1_0);
     public static final Version CURRENT = V_8_0_0;
 
+    private static final ImmutableOpenIntMap<Version> idToVersion;
 
     static {
+        final ImmutableOpenIntMap.Builder<Version> builder = ImmutableOpenIntMap.builder();
+
+        for (final Field declaredField : Version.class.getFields()) {
+            if (declaredField.getType().equals(Version.class)) {
+                final String fieldName = declaredField.getName();
+                if (fieldName.equals("CURRENT") || fieldName.equals("V_EMPTY")) {
+                    continue;
+                }
+                assert fieldName.matches("V_\\d+_\\d+_\\d+")
+                        : "expected Version field [" + fieldName + "] to match V_\\d+_\\d+_\\d+";
+                try {
+                    final Version version = (Version) declaredField.get(null);
+                    if (Assertions.ENABLED) {
+                        final String[] fields = fieldName.split("_");
+                        final int major = Integer.valueOf(fields[1]) * 1000000;
+                        final int minor = Integer.valueOf(fields[2]) * 10000;
+                        final int revision = Integer.valueOf(fields[3]) * 100;
+                        final int expectedId = major + minor + revision + 99;
+                        assert version.id == expectedId :
+                                "expected version [" + fieldName + "] to have id [" + expectedId + "] but was [" + version.id + "]";
+                    }
+                    final Version maybePrevious = builder.put(version.id, version);
+                    assert maybePrevious == null :
+                            "expected [" + version.id + "] to be uniquely mapped but saw [" + maybePrevious + "] and [" + version + "]";
+                } catch (final IllegalAccessException e) {
+                    assert false : "Version field [" + fieldName + "] should be public";
+                }
+            }
+        }
         assert CURRENT.luceneVersion.equals(org.apache.lucene.util.Version.LATEST) : "Version must be upgraded to ["
                 + org.apache.lucene.util.Version.LATEST + "] is still set to [" + CURRENT.luceneVersion + "]";
+
+        idToVersion = builder.build();
     }
 
     public static Version readVersion(StreamInput in) throws IOException {
@@ -101,49 +101,10 @@ public static Version readVersion(StreamInput in) throws IOException {
     }
 
     public static Version fromId(int id) {
+        if (idToVersion.containsKey(id)) {
+            return idToVersion.get(id);
+        }
         switch (id) {
-            case V_8_0_0_ID:
-                return V_8_0_0;
-            case V_7_3_0_ID:
-                return V_7_3_0;
-            case V_7_2_0_ID:
-                return V_7_2_0;
-            case V_7_1_1_ID:
-                return V_7_1_1;
-            case V_7_1_0_ID:
-                return V_7_1_0;
-            case V_7_0_1_ID:
-                return V_7_0_1;
-            case V_7_0_0_ID:
-                return V_7_0_0;
-            case V_6_8_1_ID:
-                return V_6_8_1;
-            case V_6_8_0_ID:
-                return V_6_8_0;
-            case V_6_6_2_ID:
-                return V_6_6_2;
-            case V_6_6_1_ID:
-                return V_6_6_1;
-            case V_6_6_0_ID:
-                return V_6_6_0;
-            case V_6_5_4_ID:
-                return V_6_5_4;
-            case V_6_5_3_ID:
-                return V_6_5_3;
-            case V_6_5_2_ID:
-                return V_6_5_2;
-            case V_6_5_1_ID:
-                return V_6_5_1;
-            case V_6_5_0_ID:
-                return V_6_5_0;
-            case V_6_4_3_ID:
-                return V_6_4_3;
-            case V_6_4_2_ID:
-                return V_6_4_2;
-            case V_6_4_1_ID:
-                return V_6_4_1;
-            case V_6_4_0_ID:
-                return V_6_4_0;
             case V_EMPTY_ID:
                 return V_EMPTY;
             default:
@@ -221,7 +182,7 @@ public static Version fromString(String version) {
             version = version.substring(0, version.length() - 9);
         }
         String[] parts = version.split("[.-]");
-        if (parts.length < 3 || parts.length > 4) {
+        if (parts.length != 3) {
             throw new IllegalArgumentException(
                     "the version needs to contain major, minor, and revision, and optionally the build: " + version);
         }
@@ -234,31 +195,13 @@ public static Version fromString(String version) {
             if (rawMajor >=7 && parts.length == 4) { // we don't support qualifier as part of the version anymore
                 throw new IllegalArgumentException("illegal version format - qualifiers are only supported until version 6.x");
             }
-            final int betaOffset = rawMajor < 5 ? 0 : 25;
             //we reverse the version id calculation based on some assumption as we can't reliably reverse the modulo
             final int major = rawMajor * 1000000;
             final int minor = Integer.parseInt(parts[1]) * 10000;
             final int revision = Integer.parseInt(parts[2]) * 100;
 
-
-            int build = 99;
-            if (parts.length == 4) {
-                String buildStr = parts[3];
-                if (buildStr.startsWith("alpha")) {
-                    assert rawMajor >= 5 : "major must be >= 5 but was " + major;
-                    build = Integer.parseInt(buildStr.substring(5));
-                    assert build < 25 : "expected a alpha build but " + build + " >= 25";
-                } else if (buildStr.startsWith("Beta") || buildStr.startsWith("beta")) {
-                    build = betaOffset + Integer.parseInt(buildStr.substring(4));
-                    assert build < 50 : "expected a beta build but " + build + " >= 50";
-                } else if (buildStr.startsWith("RC") || buildStr.startsWith("rc")) {
-                    build = Integer.parseInt(buildStr.substring(2)) + 50;
-                } else {
-                    throw new IllegalArgumentException("unable to parse version " + version);
-                }
-            }
-
-            return fromId(major + minor + revision + build);
+            // TODO: 99 is leftover from alpha/beta/rc, it should be removed
+            return fromId(major + minor + revision + 99);
 
         } catch (NumberFormatException e) {
             throw new IllegalArgumentException("unable to parse version " + version, e);
@@ -327,13 +270,16 @@ public Version minimumCompatibilityVersion() {
         if (major == 6) {
             // force the minimum compatibility for version 6 to 5.6 since we don't reference version 5 anymore
             return Version.fromId(5060099);
-        } else if (major >= 7) {
-            // all major versions from 7 onwards are compatible with last minor series of the previous major
+        } else if (major == 7) {
+            // force the minimum compatibility for version 7 to 6.8 since we don't reference version 6 anymore
+            return Version.fromId(6080099);
+        } else if (major >= 8) {
+            // all major versions from 8 onwards are compatible with last minor series of the previous major
             Version bwcVersion = null;
 
             for (int i = DeclaredVersionsHolder.DECLARED_VERSIONS.size() - 1; i >= 0; i--) {
                 final Version candidateVersion = DeclaredVersionsHolder.DECLARED_VERSIONS.get(i);
-                if (candidateVersion.major == major - 1 && candidateVersion.isRelease() && after(candidateVersion)) {
+                if (candidateVersion.major == major - 1 && after(candidateVersion)) {
                     if (bwcVersion != null && candidateVersion.minor < bwcVersion.minor) {
                         break;
                     }
@@ -389,27 +335,7 @@ public static void main(String[] args) {
 
     @Override
     public String toString() {
-        StringBuilder sb = new StringBuilder();
-        sb.append(major).append('.').append(minor).append('.').append(revision);
-        if (isAlpha()) {
-            sb.append("-alpha");
-            sb.append(build);
-        } else if (isBeta()) {
-            if (major >= 2) {
-                sb.append("-beta");
-            } else {
-                sb.append(".Beta");
-            }
-            sb.append(major < 5 ? build : build-25);
-        } else if (build < 99) {
-            if (major >= 2) {
-                sb.append("-rc");
-            } else {
-                sb.append(".RC");
-            }
-            sb.append(build - 50);
-        }
-        return sb.toString();
+        return major + "." + minor + "." + revision;
     }
 
     @Override
@@ -435,27 +361,6 @@ public int hashCode() {
         return id;
     }
 
-    public boolean isBeta() {
-        return major < 5 ? build < 50 : build >= 25 && build < 50;
-    }
-
-    /**
-     * Returns true iff this version is an alpha version
-     * Note: This has been introduced in elasticsearch version 5. Previous versions will never
-     * have an alpha version.
-     */
-    public boolean isAlpha() {
-        return major < 5 ? false :  build < 25;
-    }
-
-    public boolean isRC() {
-        return build > 50 && build < 99;
-    }
-
-    public boolean isRelease() {
-        return build == 99;
-    }
-
     /**
      * Extracts a sorted list of declared version constants from a class.
      * The argument would normally be Version.class but is exposed for
@@ -477,7 +382,7 @@ public static List<Version> getDeclaredVersions(final Class<?> versionClass) {
                 case "V_EMPTY":
                     continue;
             }
-            assert field.getName().matches("V(_\\d+)+(_(alpha|beta|rc)\\d+)?") : field.getName();
+            assert field.getName().matches("V(_\\d+){3}?") : field.getName();
             try {
                 versions.add(((Version) field.get(null)));
             } catch (final IllegalAccessException e) {
diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java
index 2cfe66372115f..9ccd4c9fdfdb0 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -85,9 +85,8 @@
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksAction;
 import org.elasticsearch.action.admin.cluster.tasks.TransportPendingClusterTasksAction;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.TransportIndicesAliasesAction;
-import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistAction;
-import org.elasticsearch.action.admin.indices.alias.exists.TransportAliasesExistAction;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
 import org.elasticsearch.action.admin.indices.alias.get.TransportGetAliasesAction;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
@@ -100,10 +99,6 @@
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
 import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
-import org.elasticsearch.action.admin.indices.exists.indices.TransportIndicesExistsAction;
-import org.elasticsearch.action.admin.indices.exists.types.TransportTypesExistsAction;
-import org.elasticsearch.action.admin.indices.exists.types.TypesExistsAction;
 import org.elasticsearch.action.admin.indices.flush.FlushAction;
 import org.elasticsearch.action.admin.indices.flush.SyncedFlushAction;
 import org.elasticsearch.action.admin.indices.flush.TransportFlushAction;
@@ -118,6 +113,7 @@
 import org.elasticsearch.action.admin.indices.mapping.get.TransportGetFieldMappingsIndexAction;
 import org.elasticsearch.action.admin.indices.mapping.get.TransportGetMappingsAction;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction;
 import org.elasticsearch.action.admin.indices.open.OpenIndexAction;
 import org.elasticsearch.action.admin.indices.open.TransportOpenIndexAction;
@@ -204,12 +200,16 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.NamedRegistry;
 import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.inject.TypeLiteral;
 import org.elasticsearch.common.inject.multibindings.MapBinder;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction;
 import org.elasticsearch.index.seqno.RetentionLeaseActions;
+import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction;
+import org.elasticsearch.index.seqno.RetentionLeaseSyncAction;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.persistent.CompletionPersistentTaskAction;
 import org.elasticsearch.persistent.RemovePersistentTaskAction;
@@ -347,7 +347,6 @@ public class ActionModule extends AbstractModule {
 
     private static final Logger logger = LogManager.getLogger(ActionModule.class);
 
-    private final boolean transportClient;
     private final Settings settings;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
     private final IndexScopedSettings indexScopedSettings;
@@ -359,13 +358,13 @@ public class ActionModule extends AbstractModule {
     private final AutoCreateIndex autoCreateIndex;
     private final DestructiveOperations destructiveOperations;
     private final RestController restController;
-    private final TransportPutMappingAction.RequestValidators mappingRequestValidators;
+    private final RequestValidators<PutMappingRequest> mappingRequestValidators;
+    private final RequestValidators<IndicesAliasesRequest> indicesAliasesRequestRequestValidators;
 
-    public ActionModule(boolean transportClient, Settings settings, IndexNameExpressionResolver indexNameExpressionResolver,
+    public ActionModule(Settings settings, IndexNameExpressionResolver indexNameExpressionResolver,
                         IndexScopedSettings indexScopedSettings, ClusterSettings clusterSettings, SettingsFilter settingsFilter,
                         ThreadPool threadPool, List<ActionPlugin> actionPlugins, NodeClient nodeClient,
-            CircuitBreakerService circuitBreakerService, UsageService usageService) {
-        this.transportClient = transportClient;
+                        CircuitBreakerService circuitBreakerService, UsageService usageService) {
         this.settings = settings;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
         this.indexScopedSettings = indexScopedSettings;
@@ -374,7 +373,7 @@ public ActionModule(boolean transportClient, Settings settings, IndexNameExpress
         this.actionPlugins = actionPlugins;
         actions = setupActions(actionPlugins);
         actionFilters = setupActionFilters(actionPlugins);
-        autoCreateIndex = transportClient ? null : new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver);
+        autoCreateIndex = new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver);
         destructiveOperations = new DestructiveOperations(settings, clusterSettings);
         Set<String> headers = Stream.concat(
             actionPlugins.stream().flatMap(p -> p.getRestHeaders().stream()),
@@ -391,15 +390,12 @@ public ActionModule(boolean transportClient, Settings settings, IndexNameExpress
                 restWrapper = newRestWrapper;
             }
         }
-        mappingRequestValidators = new TransportPutMappingAction.RequestValidators(
-            actionPlugins.stream().flatMap(p -> p.mappingRequestValidators().stream()).collect(Collectors.toList())
-        );
-
-        if (transportClient) {
-            restController = null;
-        } else {
-            restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService);
-        }
+        mappingRequestValidators = new RequestValidators<>(
+            actionPlugins.stream().flatMap(p -> p.mappingRequestValidators().stream()).collect(Collectors.toList()));
+        indicesAliasesRequestRequestValidators = new RequestValidators<>(
+                actionPlugins.stream().flatMap(p -> p.indicesAliasesRequestValidators().stream()).collect(Collectors.toList()));
+
+        restController = new RestController(headers, restWrapper, nodeClient, circuitBreakerService, usageService);
     }
 
 
@@ -419,9 +415,8 @@ public void register(ActionHandler<?, ?> handler) {
             }
 
             public <Request extends ActionRequest, Response extends ActionResponse> void register(
-                Action<Response> action, Class<? extends TransportAction<Request, Response>> transportAction,
-                Class<?>... supportTransportActions) {
-                register(new ActionHandler<>(action, transportAction, supportTransportActions));
+                ActionType<Response> action, Class<? extends TransportAction<Request, Response>> transportAction) {
+                register(new ActionHandler<>(action, transportAction));
             }
         }
         ActionRegistry actions = new ActionRegistry();
@@ -466,11 +461,9 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
         actions.register(GetIndexAction.INSTANCE, TransportGetIndexAction.class);
         actions.register(OpenIndexAction.INSTANCE, TransportOpenIndexAction.class);
         actions.register(CloseIndexAction.INSTANCE, TransportCloseIndexAction.class);
-        actions.register(IndicesExistsAction.INSTANCE, TransportIndicesExistsAction.class);
-        actions.register(TypesExistsAction.INSTANCE, TransportTypesExistsAction.class);
         actions.register(GetMappingsAction.INSTANCE, TransportGetMappingsAction.class);
-        actions.register(GetFieldMappingsAction.INSTANCE, TransportGetFieldMappingsAction.class,
-                TransportGetFieldMappingsIndexAction.class);
+        actions.register(GetFieldMappingsAction.INSTANCE, TransportGetFieldMappingsAction.class);
+        actions.register(TransportGetFieldMappingsIndexAction.TYPE, TransportGetFieldMappingsIndexAction.class);
         actions.register(PutMappingAction.INSTANCE, TransportPutMappingAction.class);
         actions.register(IndicesAliasesAction.INSTANCE, TransportIndicesAliasesAction.class);
         actions.register(UpdateSettingsAction.INSTANCE, TransportUpdateSettingsAction.class);
@@ -488,20 +481,19 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
         actions.register(UpgradeSettingsAction.INSTANCE, TransportUpgradeSettingsAction.class);
         actions.register(ClearIndicesCacheAction.INSTANCE, TransportClearIndicesCacheAction.class);
         actions.register(GetAliasesAction.INSTANCE, TransportGetAliasesAction.class);
-        actions.register(AliasesExistAction.INSTANCE, TransportAliasesExistAction.class);
         actions.register(GetSettingsAction.INSTANCE, TransportGetSettingsAction.class);
 
         actions.register(IndexAction.INSTANCE, TransportIndexAction.class);
         actions.register(GetAction.INSTANCE, TransportGetAction.class);
         actions.register(TermVectorsAction.INSTANCE, TransportTermVectorsAction.class);
-        actions.register(MultiTermVectorsAction.INSTANCE, TransportMultiTermVectorsAction.class,
-                TransportShardMultiTermsVectorAction.class);
+        actions.register(MultiTermVectorsAction.INSTANCE, TransportMultiTermVectorsAction.class);
+        actions.register(TransportShardMultiTermsVectorAction.TYPE, TransportShardMultiTermsVectorAction.class);
         actions.register(DeleteAction.INSTANCE, TransportDeleteAction.class);
         actions.register(UpdateAction.INSTANCE, TransportUpdateAction.class);
-        actions.register(MultiGetAction.INSTANCE, TransportMultiGetAction.class,
-                TransportShardMultiGetAction.class);
-        actions.register(BulkAction.INSTANCE, TransportBulkAction.class,
-                TransportShardBulkAction.class);
+        actions.register(MultiGetAction.INSTANCE, TransportMultiGetAction.class);
+        actions.register(TransportShardMultiGetAction.TYPE, TransportShardMultiGetAction.class);
+        actions.register(BulkAction.INSTANCE, TransportBulkAction.class);
+        actions.register(TransportShardBulkAction.TYPE, TransportShardBulkAction.class);
         actions.register(SearchAction.INSTANCE, TransportSearchAction.class);
         actions.register(SearchScrollAction.INSTANCE, TransportSearchScrollAction.class);
         actions.register(MultiSearchAction.INSTANCE, TransportMultiSearchAction.class);
@@ -515,8 +507,8 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
         actions.register(GetStoredScriptAction.INSTANCE, TransportGetStoredScriptAction.class);
         actions.register(DeleteStoredScriptAction.INSTANCE, TransportDeleteStoredScriptAction.class);
 
-        actions.register(FieldCapabilitiesAction.INSTANCE, TransportFieldCapabilitiesAction.class,
-            TransportFieldCapabilitiesIndexAction.class);
+        actions.register(FieldCapabilitiesAction.INSTANCE, TransportFieldCapabilitiesAction.class);
+        actions.register(TransportFieldCapabilitiesIndexAction.TYPE, TransportFieldCapabilitiesIndexAction.class);
 
         actions.register(PutPipelineAction.INSTANCE, PutPipelineTransportAction.class);
         actions.register(GetPipelineAction.INSTANCE, GetPipelineTransportAction.class);
@@ -536,6 +528,11 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
         actions.register(RetentionLeaseActions.Renew.INSTANCE, RetentionLeaseActions.Renew.TransportAction.class);
         actions.register(RetentionLeaseActions.Remove.INSTANCE, RetentionLeaseActions.Remove.TransportAction.class);
 
+        // internal actions
+        actions.register(GlobalCheckpointSyncAction.TYPE, GlobalCheckpointSyncAction.class);
+        actions.register(RetentionLeaseBackgroundSyncAction.TYPE, RetentionLeaseBackgroundSyncAction.class);
+        actions.register(RetentionLeaseSyncAction.TYPE, RetentionLeaseSyncAction.class);
+
         return unmodifiableMap(actions.getRegistry());
     }
 
@@ -661,7 +658,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         registerHandler.accept(new RestMasterAction(settings, restController));
         registerHandler.accept(new RestNodesAction(settings, restController));
         registerHandler.accept(new RestTasksAction(settings, restController, nodesInCluster));
-        registerHandler.accept(new RestIndicesAction(settings, restController, indexNameExpressionResolver));
+        registerHandler.accept(new RestIndicesAction(settings, restController));
         registerHandler.accept(new RestSegmentsAction(settings, restController));
         // Fully qualified to prevent interference with rest.action.count.RestCountAction
         registerHandler.accept(new org.elasticsearch.rest.action.cat.RestCountAction(settings, restController));
@@ -690,26 +687,21 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
     protected void configure() {
         bind(ActionFilters.class).toInstance(actionFilters);
         bind(DestructiveOperations.class).toInstance(destructiveOperations);
-        bind(TransportPutMappingAction.RequestValidators.class).toInstance(mappingRequestValidators);
-
-        if (false == transportClient) {
-            // Supporting classes only used when not a transport client
-            bind(AutoCreateIndex.class).toInstance(autoCreateIndex);
-            bind(TransportLivenessAction.class).asEagerSingleton();
-
-            // register Action -> transportAction Map used by NodeClient
-            @SuppressWarnings("rawtypes")
-            MapBinder<Action, TransportAction> transportActionsBinder
-                    = MapBinder.newMapBinder(binder(), Action.class, TransportAction.class);
-            for (ActionHandler<?, ?> action : actions.values()) {
-                // bind the action as eager singleton, so the map binder one will reuse it
-                bind(action.getTransportAction()).asEagerSingleton();
-                transportActionsBinder.addBinding(action.getAction()).to(action.getTransportAction()).asEagerSingleton();
-                for (Class<?> supportAction : action.getSupportTransportActions()) {
-                    bind(supportAction).asEagerSingleton();
-                }
-            }
+        bind(new TypeLiteral<RequestValidators<PutMappingRequest>>() {}).toInstance(mappingRequestValidators);
+        bind(new TypeLiteral<RequestValidators<IndicesAliasesRequest>>() {}).toInstance(indicesAliasesRequestRequestValidators);
+        bind(AutoCreateIndex.class).toInstance(autoCreateIndex);
+        bind(TransportLivenessAction.class).asEagerSingleton();
+
+        // register ActionType -> transportAction Map used by NodeClient
+        @SuppressWarnings("rawtypes")
+        MapBinder<ActionType, TransportAction> transportActionsBinder
+                = MapBinder.newMapBinder(binder(), ActionType.class, TransportAction.class);
+        for (ActionHandler<?, ?> action : actions.values()) {
+            // bind the action as eager singleton, so the map binder one will reuse it
+            bind(action.getTransportAction()).asEagerSingleton();
+            transportActionsBinder.addBinding(action.getAction()).to(action.getTransportAction()).asEagerSingleton();
         }
+
     }
 
     public ActionFilters getActionFilters() {
diff --git a/server/src/main/java/org/elasticsearch/action/ActionRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/ActionRequestBuilder.java
index a3aa8ac2a5222..166bec9e065b0 100644
--- a/server/src/main/java/org/elasticsearch/action/ActionRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionRequestBuilder.java
@@ -26,11 +26,11 @@
 
 public abstract class ActionRequestBuilder<Request extends ActionRequest, Response extends ActionResponse> {
 
-    protected final Action<Response> action;
+    protected final ActionType<Response> action;
     protected final Request request;
     protected final ElasticsearchClient client;
 
-    protected ActionRequestBuilder(ElasticsearchClient client, Action<Response> action, Request request) {
+    protected ActionRequestBuilder(ElasticsearchClient client, ActionType<Response> action, Request request) {
         Objects.requireNonNull(action, "action must not be null");
         this.action = action;
         this.request = request;
diff --git a/server/src/main/java/org/elasticsearch/action/Action.java b/server/src/main/java/org/elasticsearch/action/ActionType.java
similarity index 75%
rename from server/src/main/java/org/elasticsearch/action/Action.java
rename to server/src/main/java/org/elasticsearch/action/ActionType.java
index f0df6202072a4..02f8f3c6fc22f 100644
--- a/server/src/main/java/org/elasticsearch/action/Action.java
+++ b/server/src/main/java/org/elasticsearch/action/ActionType.java
@@ -26,15 +26,27 @@
 /**
  * A generic action. Should strive to make it a singleton.
  */
-public abstract class Action<Response extends ActionResponse> {
+public class ActionType<Response extends ActionResponse> {
 
     private final String name;
+    private final Writeable.Reader<Response> responseReader;
 
     /**
      * @param name The name of the action, must be unique across actions.
+     * @deprecated Pass a {@link Writeable.Reader} with {@link }
      */
-    protected Action(String name) {
+    @Deprecated
+    protected ActionType(String name) {
+        this(name, null);
+    }
+
+    /**
+     * @param name The name of the action, must be unique across actions.
+     * @param responseReader A reader for the response type
+     */
+    public ActionType(String name, Writeable.Reader<Response> responseReader) {
         this.name = name;
+        this.responseReader = responseReader;
     }
 
     /**
@@ -44,23 +56,11 @@ public String name() {
         return this.name;
     }
 
-    /**
-     * Creates a new response instance.
-     * @deprecated Implement {@link #getResponseReader()} instead and make this method throw an
-     *             {@link UnsupportedOperationException}
-     */
-    @Deprecated
-    public abstract Response newResponse();
-
     /**
      * Get a reader that can create a new instance of the class from a {@link org.elasticsearch.common.io.stream.StreamInput}
      */
     public Writeable.Reader<Response> getResponseReader() {
-        return in -> {
-            Response response = newResponse();
-            response.readFrom(in);
-            return response;
-        };
+        return responseReader;
     }
 
     /**
@@ -72,7 +72,7 @@ public TransportRequestOptions transportOptions(Settings settings) {
 
     @Override
     public boolean equals(Object o) {
-        return o instanceof Action && name.equals(((Action<?>) o).name());
+        return o instanceof ActionType && name.equals(((ActionType<?>) o).name());
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/RequestValidators.java b/server/src/main/java/org/elasticsearch/action/RequestValidators.java
new file mode 100644
index 0000000000000..95bfcca51d218
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/RequestValidators.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.index.Index;
+
+import java.util.Collection;
+import java.util.Optional;
+
+public class RequestValidators<T extends ActionRequest> {
+
+        private final Collection<RequestValidator<T>> validators;
+
+        public RequestValidators(Collection<RequestValidator<T>> validators) {
+            this.validators = validators;
+        }
+
+        public Optional<Exception> validateRequest(final T request, final ClusterState state, final Index[] indices) {
+            Exception exception = null;
+            for (final var validator : validators) {
+                final Optional<Exception> maybeException = validator.validateRequest(request, state, indices);
+                if (maybeException.isEmpty()) continue;
+                if (exception == null) {
+                    exception = maybeException.get();
+                } else {
+                    exception.addSuppressed(maybeException.get());
+                }
+            }
+            return Optional.ofNullable(exception);
+        }
+
+    /**
+     * A validator that validates an request associated with indices before executing it.
+     */
+    public interface RequestValidator<T extends ActionRequest> {
+
+        /**
+         * Validates a given request with its associated concrete indices and the current state.
+         *
+         * @param request the request to validate
+         * @param state   the current cluster state
+         * @param indices the concrete indices that associated with the given request
+         * @return an optional exception indicates a reason that the given request should be aborted, otherwise empty
+         */
+        Optional<Exception> validateRequest(T request, ClusterState state, Index[] indices);
+
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/action/RoutingMissingException.java b/server/src/main/java/org/elasticsearch/action/RoutingMissingException.java
index 1ecd9593e1c36..e3ef791814ac8 100644
--- a/server/src/main/java/org/elasticsearch/action/RoutingMissingException.java
+++ b/server/src/main/java/org/elasticsearch/action/RoutingMissingException.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
@@ -33,6 +34,10 @@ public class RoutingMissingException extends ElasticsearchException {
 
     private final String id;
 
+    public RoutingMissingException(String index, String id) {
+        this(index, MapperService.SINGLE_MAPPING_NAME, id);
+    }
+
     public RoutingMissingException(String index, String type, String id) {
         super("routing is required for [" + index + "]/[" + type + "]/[" + id + "]");
         Objects.requireNonNull(index, "index must not be null");
diff --git a/server/src/main/java/org/elasticsearch/action/StreamableResponseActionType.java b/server/src/main/java/org/elasticsearch/action/StreamableResponseActionType.java
new file mode 100644
index 0000000000000..b8206bb03f89d
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/action/StreamableResponseActionType.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action;
+
+import org.elasticsearch.common.io.stream.Writeable;
+
+/**
+ * An action for with the response type implements {@link org.elasticsearch.common.io.stream.Streamable}.
+ * @deprecated Use {@link ActionType} directly and provide a {@link Writeable.Reader}
+ */
+@Deprecated
+public abstract class StreamableResponseActionType<Response extends ActionResponse> extends ActionType<Response> {
+
+    protected StreamableResponseActionType(String name) {
+        super(name);
+    }
+
+    /**
+     * Creates a new response instance.
+     * @deprecated Implement {@link #getResponseReader()} instead and make this method throw an
+     *             {@link UnsupportedOperationException}
+     */
+    @Deprecated
+    public abstract Response newResponse();
+
+    @Override
+    public final Writeable.Reader<Response> getResponseReader() {
+        return in -> {
+            Response response = newResponse();
+            response.readFrom(in);
+            return response;
+        };
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/TransportActionNodeProxy.java b/server/src/main/java/org/elasticsearch/action/TransportActionNodeProxy.java
index a4c3e17e80208..4319a745ba816 100644
--- a/server/src/main/java/org/elasticsearch/action/TransportActionNodeProxy.java
+++ b/server/src/main/java/org/elasticsearch/action/TransportActionNodeProxy.java
@@ -30,10 +30,10 @@
 public class TransportActionNodeProxy<Request extends ActionRequest, Response extends ActionResponse> {
 
     private final TransportService transportService;
-    private final Action<Response> action;
+    private final ActionType<Response> action;
     private final TransportRequestOptions transportOptions;
 
-    public TransportActionNodeProxy(Settings settings, Action<Response> action, TransportService transportService) {
+    public TransportActionNodeProxy(Settings settings, ActionType<Response> action, TransportService transportService) {
         this.action = action;
         this.transportService = transportService;
         this.transportOptions = action.transportOptions(settings);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainAction.java
index 19d5378b305ed..acaaed9eaa985 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainAction.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.action.admin.cluster.allocation;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for explaining shard allocation for a shard in the cluster
+ * ActionType for explaining shard allocation for a shard in the cluster
  */
-public class ClusterAllocationExplainAction extends Action<ClusterAllocationExplainResponse> {
+public class ClusterAllocationExplainAction extends StreamableResponseActionType<ClusterAllocationExplainResponse> {
 
     public static final ClusterAllocationExplainAction INSTANCE = new ClusterAllocationExplainAction();
     public static final String NAME = "cluster:monitor/allocation/explain";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
index e3ee0dd7b1524..2fe4982acc63e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
@@ -41,6 +41,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.gateway.GatewayAllocator;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -88,7 +89,7 @@ protected ClusterAllocationExplainResponse newResponse() {
     }
 
     @Override
-    protected void masterOperation(final ClusterAllocationExplainRequest request, final ClusterState state,
+    protected void masterOperation(Task task, final ClusterAllocationExplainRequest request, final ClusterState state,
                                    final ActionListener<ClusterAllocationExplainResponse> listener) {
         final RoutingNodes routingNodes = state.getRoutingNodes();
         final ClusterInfo clusterInfo = clusterInfoService.getClusterInfo();
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsAction.java
index 6fdd324fa47c6..a2f0c721b5da4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsAction.java
@@ -18,10 +18,10 @@
  */
 package org.elasticsearch.action.admin.cluster.configuration;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
 
-public class AddVotingConfigExclusionsAction extends Action<AddVotingConfigExclusionsResponse> {
+public class AddVotingConfigExclusionsAction extends ActionType<AddVotingConfigExclusionsResponse> {
     public static final AddVotingConfigExclusionsAction INSTANCE = new AddVotingConfigExclusionsAction();
     public static final String NAME = "cluster:admin/voting_config/add_exclusions";
 
@@ -29,11 +29,6 @@ private AddVotingConfigExclusionsAction() {
         super(NAME);
     }
 
-    @Override
-    public AddVotingConfigExclusionsResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Reader<AddVotingConfigExclusionsResponse> getResponseReader() {
         return AddVotingConfigExclusionsResponse::new;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/ClearVotingConfigExclusionsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/ClearVotingConfigExclusionsAction.java
index 49b578f48adf8..6091800693f49 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/ClearVotingConfigExclusionsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/ClearVotingConfigExclusionsAction.java
@@ -18,10 +18,10 @@
  */
 package org.elasticsearch.action.admin.cluster.configuration;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
 
-public class ClearVotingConfigExclusionsAction extends Action<ClearVotingConfigExclusionsResponse> {
+public class ClearVotingConfigExclusionsAction extends ActionType<ClearVotingConfigExclusionsResponse> {
     public static final ClearVotingConfigExclusionsAction INSTANCE = new ClearVotingConfigExclusionsAction();
     public static final String NAME = "cluster:admin/voting_config/clear_exclusions";
 
@@ -29,11 +29,6 @@ private ClearVotingConfigExclusionsAction() {
         super(NAME);
     }
 
-    @Override
-    public ClearVotingConfigExclusionsResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Reader<ClearVotingConfigExclusionsResponse> getResponseReader() {
         return ClearVotingConfigExclusionsResponse::new;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java
index 512321c7e4432..a6474510e88f3 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java
@@ -40,6 +40,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPool.Names;
 import org.elasticsearch.transport.TransportService;
@@ -78,7 +79,7 @@ protected AddVotingConfigExclusionsResponse read(StreamInput in) throws IOExcept
     }
 
     @Override
-    protected void masterOperation(AddVotingConfigExclusionsRequest request, ClusterState state,
+    protected void masterOperation(Task task, AddVotingConfigExclusionsRequest request, ClusterState state,
                                    ActionListener<AddVotingConfigExclusionsResponse> listener) throws Exception {
 
         resolveVotingConfigExclusionsAndCheckMaximum(request, state); // throws IAE if no nodes matched or maximum exceeded
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java
index 7cd19a824be05..3b441e7ea9db7 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java
@@ -38,6 +38,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPool.Names;
 import org.elasticsearch.transport.TransportService;
@@ -72,7 +73,7 @@ protected ClearVotingConfigExclusionsResponse read(StreamInput in) throws IOExce
     }
 
     @Override
-    protected void masterOperation(ClearVotingConfigExclusionsRequest request, ClusterState initialState,
+    protected void masterOperation(Task task, ClearVotingConfigExclusionsRequest request, ClusterState initialState,
                                    ActionListener<ClearVotingConfigExclusionsResponse> listener) throws Exception {
 
         final long startTimeMillis = threadPool.relativeTimeInMillis();
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthAction.java
index 0cd148ee231e0..e8f5ecfaf5b66 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.cluster.health;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class ClusterHealthAction extends Action<ClusterHealthResponse> {
+public class ClusterHealthAction extends StreamableResponseActionType<ClusterHealthResponse> {
 
     public static final ClusterHealthAction INSTANCE = new ClusterHealthAction();
     public static final String NAME = "cluster:monitor/health";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
index 0caae77d7dead..08b8730b02419 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
@@ -77,13 +77,6 @@ protected ClusterHealthResponse newResponse() {
         return new ClusterHealthResponse();
     }
 
-    @Override
-    protected final void masterOperation(ClusterHealthRequest request, ClusterState state,
-                                         ActionListener<ClusterHealthResponse> listener) throws Exception {
-        logger.warn("attempt to execute a cluster health operation without a task");
-        throw new UnsupportedOperationException("task parameter is required for this operation");
-    }
-
     @Override
     protected void masterOperation(Task task, final ClusterHealthRequest request, final ClusterState unusedState,
                                    final ActionListener<ClusterHealthResponse> listener) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsAction.java
index 4ea7ee5bc3bbe..4833625d29522 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.cluster.node.hotthreads;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class NodesHotThreadsAction extends Action<NodesHotThreadsResponse> {
+public class NodesHotThreadsAction extends StreamableResponseActionType<NodesHotThreadsResponse> {
 
     public static final NodesHotThreadsAction INSTANCE = new NodesHotThreadsAction();
     public static final String NAME = "cluster:monitor/nodes/hot_threads";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
index 6321813f189fb..4f85177b6e671 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.monitor.jvm.HotThreads;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -54,8 +55,8 @@ protected NodesHotThreadsResponse newResponse(NodesHotThreadsRequest request,
     }
 
     @Override
-    protected NodeRequest newNodeRequest(String nodeId, NodesHotThreadsRequest request) {
-        return new NodeRequest(nodeId, request);
+    protected NodeRequest newNodeRequest(NodesHotThreadsRequest request) {
+        return new NodeRequest(request);
     }
 
     @Override
@@ -64,7 +65,7 @@ protected NodeHotThreads newNodeResponse() {
     }
 
     @Override
-    protected NodeHotThreads nodeOperation(NodeRequest request) {
+    protected NodeHotThreads nodeOperation(NodeRequest request, Task task) {
         HotThreads hotThreads = new HotThreads()
                 .busiestThreads(request.request.threads)
                 .type(request.request.type)
@@ -85,8 +86,7 @@ public static class NodeRequest extends BaseNodeRequest {
         public NodeRequest() {
         }
 
-        NodeRequest(String nodeId, NodesHotThreadsRequest request) {
-            super(nodeId);
+        NodeRequest(NodesHotThreadsRequest request) {
             this.request = request;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoAction.java
index edc5ed7e83f0f..e94390d8f92d8 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.cluster.node.info;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class NodesInfoAction extends Action<NodesInfoResponse> {
+public class NodesInfoAction extends StreamableResponseActionType<NodesInfoResponse> {
 
     public static final NodesInfoAction INSTANCE = new NodesInfoAction();
     public static final String NAME = "cluster:monitor/nodes/info";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java
index 3bee722a472ec..4d70b97ca2cf6 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -74,8 +74,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             }
 
             builder.startArray("roles");
-            for (DiscoveryNode.Role role : nodeInfo.getNode().getRoles()) {
-                builder.value(role.getRoleName());
+            for (DiscoveryNodeRole role : nodeInfo.getNode().getRoles()) {
+                builder.value(role.roleName());
             }
             builder.endArray();
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
index a1f9790af9351..903f6adb7b931 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.node.NodeService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -56,8 +57,8 @@ protected NodesInfoResponse newResponse(NodesInfoRequest nodesInfoRequest,
     }
 
     @Override
-    protected NodeInfoRequest newNodeRequest(String nodeId, NodesInfoRequest request) {
-        return new NodeInfoRequest(nodeId, request);
+    protected NodeInfoRequest newNodeRequest(NodesInfoRequest request) {
+        return new NodeInfoRequest(request);
     }
 
     @Override
@@ -66,7 +67,7 @@ protected NodeInfo newNodeResponse() {
     }
 
     @Override
-    protected NodeInfo nodeOperation(NodeInfoRequest nodeRequest) {
+    protected NodeInfo nodeOperation(NodeInfoRequest nodeRequest, Task task) {
         NodesInfoRequest request = nodeRequest.request;
         return nodeService.info(request.settings(), request.os(), request.process(), request.jvm(), request.threadPool(),
                 request.transport(), request.http(), request.plugins(), request.ingest(), request.indices());
@@ -79,8 +80,7 @@ public static class NodeInfoRequest extends BaseNodeRequest {
         public NodeInfoRequest() {
         }
 
-        public NodeInfoRequest(String nodeId, NodesInfoRequest request) {
-            super(nodeId);
+        public NodeInfoRequest(NodesInfoRequest request) {
             this.request = request;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsAction.java
index 19e8fc1929c5d..e22595c187092 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsAction.java
@@ -19,10 +19,10 @@
 
 package org.elasticsearch.action.admin.cluster.node.reload;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 public class NodesReloadSecureSettingsAction
-        extends Action<NodesReloadSecureSettingsResponse> {
+        extends StreamableResponseActionType<NodesReloadSecureSettingsResponse> {
 
     public static final NodesReloadSecureSettingsAction INSTANCE = new NodesReloadSecureSettingsAction();
     public static final String NAME = "cluster:admin/nodes/reload_secure_settings";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java
index f2fef743a0d37..27860b52557e7 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.plugins.ReloadablePlugin;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -68,8 +69,8 @@ protected NodesReloadSecureSettingsResponse newResponse(NodesReloadSecureSetting
     }
 
     @Override
-    protected NodeRequest newNodeRequest(String nodeId, NodesReloadSecureSettingsRequest request) {
-        return new NodeRequest(nodeId, request);
+    protected NodeRequest newNodeRequest(NodesReloadSecureSettingsRequest request) {
+        return new NodeRequest(request);
     }
 
     @Override
@@ -78,7 +79,7 @@ protected NodesReloadSecureSettingsResponse.NodeResponse newNodeResponse() {
     }
 
     @Override
-    protected NodesReloadSecureSettingsResponse.NodeResponse nodeOperation(NodeRequest nodeReloadRequest) {
+    protected NodesReloadSecureSettingsResponse.NodeResponse nodeOperation(NodeRequest nodeReloadRequest, Task task) {
         try (KeyStoreWrapper keystore = KeyStoreWrapper.load(environment.configFile())) {
             // reread keystore from config file
             if (keystore == null) {
@@ -116,8 +117,7 @@ public static class NodeRequest extends BaseNodeRequest {
         public NodeRequest() {
         }
 
-        NodeRequest(String nodeId, NodesReloadSecureSettingsRequest request) {
-            super(nodeId);
+        NodeRequest(NodesReloadSecureSettingsRequest request) {
             this.request = request;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
index 7a69bc0e506bd..1d32fbef2c557 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -269,8 +270,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field("ip", getNode().getAddress());
 
         builder.startArray("roles");
-        for (DiscoveryNode.Role role : getNode().getRoles()) {
-            builder.value(role.getRoleName());
+        for (DiscoveryNodeRole role : getNode().getRoles()) {
+            builder.value(role.roleName());
         }
         builder.endArray();
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsAction.java
index bc8c81ef1e0f5..dbe7deed74a73 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.cluster.node.stats;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class NodesStatsAction extends Action<NodesStatsResponse> {
+public class NodesStatsAction extends StreamableResponseActionType<NodesStatsResponse> {
 
     public static final NodesStatsAction INSTANCE = new NodesStatsAction();
     public static final String NAME = "cluster:monitor/nodes/stats";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
index 1028da916a2c1..f399304a8a10e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.node.NodeService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -55,8 +56,8 @@ protected NodesStatsResponse newResponse(NodesStatsRequest request, List<NodeSta
     }
 
     @Override
-    protected NodeStatsRequest newNodeRequest(String nodeId, NodesStatsRequest request) {
-        return new NodeStatsRequest(nodeId, request);
+    protected NodeStatsRequest newNodeRequest(NodesStatsRequest request) {
+        return new NodeStatsRequest(request);
     }
 
     @Override
@@ -65,7 +66,7 @@ protected NodeStats newNodeResponse() {
     }
 
     @Override
-    protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) {
+    protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest, Task task) {
         NodesStatsRequest request = nodeStatsRequest.request;
         return nodeService.stats(request.indices(), request.os(), request.process(), request.jvm(), request.threadPool(),
                 request.fs(), request.transport(), request.http(), request.breaker(), request.script(), request.discovery(),
@@ -79,8 +80,7 @@ public static class NodeStatsRequest extends BaseNodeRequest {
         public NodeStatsRequest() {
         }
 
-        NodeStatsRequest(String nodeId, NodesStatsRequest request) {
-            super(nodeId);
+        NodeStatsRequest(NodesStatsRequest request) {
             this.request = request;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksAction.java
index 23cb69cf807f7..a0fa139dc7364 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksAction.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.action.admin.cluster.node.tasks.cancel;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
 /**
- * Action for cancelling running tasks
+ * ActionType for cancelling running tasks
  */
-public class CancelTasksAction extends Action<CancelTasksResponse> {
+public class CancelTasksAction extends ActionType<CancelTasksResponse> {
 
     public static final CancelTasksAction INSTANCE = new CancelTasksAction();
     public static final String NAME = "cluster:admin/tasks/cancel";
@@ -34,11 +34,6 @@ private CancelTasksAction() {
         super(NAME);
     }
 
-    @Override
-    public CancelTasksResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<CancelTasksResponse> getResponseReader() {
         return CancelTasksResponse::new;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskAction.java
index 4e88963de4c98..978e07555b517 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskAction.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.action.admin.cluster.node.tasks.get;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for retrieving a list of currently running tasks
+ * ActionType for retrieving a list of currently running tasks
  */
-public class GetTaskAction extends Action<GetTaskResponse> {
+public class GetTaskAction extends StreamableResponseActionType<GetTaskResponse> {
     public static final String TASKS_ORIGIN = "tasks";
 
     public static final GetTaskAction INSTANCE = new GetTaskAction();
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
index d1d72da544560..2b0ac0233be29 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
@@ -56,7 +56,7 @@
 import static org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction.waitForCompletionTimeout;
 
 /**
- * Action to get a single task. If the task isn't running then it'll try to request the status from request index.
+ * ActionType to get a single task. If the task isn't running then it'll try to request the status from request index.
  *
  * The general flow is:
  * <ul>
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksAction.java
index abba798c83cc1..7fb0b94cd8766 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksAction.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.action.admin.cluster.node.tasks.list;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
 /**
- * Action for retrieving a list of currently running tasks
+ * ActionType for retrieving a list of currently running tasks
  */
-public class ListTasksAction extends Action<ListTasksResponse> {
+public class ListTasksAction extends ActionType<ListTasksResponse> {
 
     public static final ListTasksAction INSTANCE = new ListTasksAction();
     public static final String NAME = "cluster:monitor/tasks/lists";
@@ -34,11 +34,6 @@ private ListTasksAction() {
         super(NAME);
     }
 
-    @Override
-    public ListTasksResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<ListTasksResponse> getResponseReader() {
         return ListTasksResponse::new;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
index 557338ed0f77e..d6fc541ba687a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.tasks.BaseTasksResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
@@ -175,8 +176,8 @@ public XContentBuilder toXContentGroupedByNode(XContentBuilder builder, Params p
                 builder.field("ip", node.getAddress());
 
                 builder.startArray("roles");
-                for (DiscoveryNode.Role role : node.getRoles()) {
-                    builder.value(role.getRoleName());
+                for (DiscoveryNodeRole role : node.getRoles()) {
+                    builder.value(role.roleName());
                 }
                 builder.endArray();
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageAction.java
index 6bc6dce54945a..aaf7616a19b1b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.cluster.node.usage;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class NodesUsageAction extends Action<NodesUsageResponse> {
+public class NodesUsageAction extends StreamableResponseActionType<NodesUsageResponse> {
 
     public static final NodesUsageAction INSTANCE = new NodesUsageAction();
     public static final String NAME = "cluster:monitor/nodes/usage";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageRequestBuilder.java
index d8af249be0251..3133a6a18947b 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageRequestBuilder.java
@@ -19,14 +19,14 @@
 
 package org.elasticsearch.action.admin.cluster.node.usage;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 
 public class NodesUsageRequestBuilder
         extends NodesOperationRequestBuilder<NodesUsageRequest, NodesUsageResponse, NodesUsageRequestBuilder> {
 
-    public NodesUsageRequestBuilder(ElasticsearchClient client, Action<NodesUsageResponse> action) {
+    public NodesUsageRequestBuilder(ElasticsearchClient client, ActionType<NodesUsageResponse> action) {
         super(client, action, new NodesUsageRequest());
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java
index c34ca9b614432..ed40aaddae24a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.usage.UsageService;
@@ -53,8 +54,8 @@ protected NodesUsageResponse newResponse(NodesUsageRequest request, List<NodeUsa
     }
 
     @Override
-    protected NodeUsageRequest newNodeRequest(String nodeId, NodesUsageRequest request) {
-        return new NodeUsageRequest(nodeId, request);
+    protected NodeUsageRequest newNodeRequest(NodesUsageRequest request) {
+        return new NodeUsageRequest(request);
     }
 
     @Override
@@ -63,7 +64,7 @@ protected NodeUsage newNodeResponse() {
     }
 
     @Override
-    protected NodeUsage nodeOperation(NodeUsageRequest nodeUsageRequest) {
+    protected NodeUsage nodeOperation(NodeUsageRequest nodeUsageRequest, Task task) {
         NodesUsageRequest request = nodeUsageRequest.request;
         return usageService.getUsageStats(clusterService.localNode(), request.restActions());
     }
@@ -75,8 +76,7 @@ public static class NodeUsageRequest extends BaseNodeRequest {
         public NodeUsageRequest() {
         }
 
-        NodeUsageRequest(String nodeId, NodesUsageRequest request) {
-            super(nodeId);
+        NodeUsageRequest(NodesUsageRequest request) {
             this.request = request;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoAction.java
index 3b998049daaaa..6ced57bde05c1 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/remote/RemoteInfoAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.cluster.remote;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public final class RemoteInfoAction extends Action<RemoteInfoResponse> {
+public final class RemoteInfoAction extends StreamableResponseActionType<RemoteInfoResponse> {
 
     public static final String NAME = "cluster:monitor/remote/info";
     public static final RemoteInfoAction INSTANCE = new RemoteInfoAction();
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/DeleteRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/DeleteRepositoryAction.java
index a31984e222793..791e4c6e57d70 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/DeleteRepositoryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/DeleteRepositoryAction.java
@@ -19,13 +19,14 @@
 
 package org.elasticsearch.action.admin.cluster.repositories.delete;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
 /**
  * Unregister repository action
  */
-public class DeleteRepositoryAction extends Action<AcknowledgedResponse> {
+public class DeleteRepositoryAction extends ActionType<AcknowledgedResponse> {
 
     public static final DeleteRepositoryAction INSTANCE = new DeleteRepositoryAction();
     public static final String NAME = "cluster:admin/repository/delete";
@@ -35,8 +36,8 @@ private DeleteRepositoryAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java
index 48b9bdaa4511b..5a93c9dd6f2e0 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java
@@ -29,10 +29,14 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 /**
  * Transport action for unregister repository operation
  */
@@ -54,9 +58,14 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
@@ -65,7 +74,7 @@ protected ClusterBlockException checkBlock(DeleteRepositoryRequest request, Clus
     }
 
     @Override
-    protected void masterOperation(final DeleteRepositoryRequest request, ClusterState state,
+    protected void masterOperation(Task task, final DeleteRepositoryRequest request, ClusterState state,
                                    final ActionListener<AcknowledgedResponse> listener) {
         repositoriesService.unregisterRepository(
             request, ActionListener.delegateFailure(listener,
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesAction.java
index d89e466461d9b..3b4408ea3dd3e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesAction.java
@@ -19,12 +19,13 @@
 
 package org.elasticsearch.action.admin.cluster.repositories.get;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.common.io.stream.Writeable;
 
 /**
  * Get repositories action
  */
-public class GetRepositoriesAction extends Action<GetRepositoriesResponse> {
+public class GetRepositoriesAction extends ActionType<GetRepositoriesResponse> {
 
     public static final GetRepositoriesAction INSTANCE = new GetRepositoriesAction();
     public static final String NAME = "cluster:admin/repository/get";
@@ -34,8 +35,8 @@ private GetRepositoriesAction() {
     }
 
     @Override
-    public GetRepositoriesResponse newResponse() {
-        return new GetRepositoriesResponse();
+    public Writeable.Reader<GetRepositoriesResponse> getResponseReader() {
+        return GetRepositoriesResponse::new;
     }
 }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java
index 24228aa565871..4fb9cbaebe941 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/GetRepositoriesResponse.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.List;
 
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
@@ -39,16 +38,16 @@
  */
 public class GetRepositoriesResponse extends ActionResponse implements ToXContentObject {
 
-    private RepositoriesMetaData repositories;
-
-    GetRepositoriesResponse() {
-        repositories = new RepositoriesMetaData(Collections.emptyList());
-    }
+    private final RepositoriesMetaData repositories;
 
     GetRepositoriesResponse(RepositoriesMetaData repositories) {
         this.repositories = repositories;
     }
 
+    public GetRepositoriesResponse(StreamInput in) throws IOException {
+        repositories = new RepositoriesMetaData(in);
+    }
+
     /**
      * List of repositories to return
      *
@@ -61,7 +60,7 @@ public List<RepositoryMetaData> repositories() {
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        repositories = new RepositoriesMetaData(in);
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java
index 4b3ee1cd9251a..3f0ee69d2a5aa 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java
@@ -31,11 +31,14 @@
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.repositories.RepositoryMissingException;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedHashSet;
@@ -62,7 +65,12 @@ protected String executor() {
 
     @Override
     protected GetRepositoriesResponse newResponse() {
-        return new GetRepositoriesResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected GetRepositoriesResponse read(StreamInput in) throws IOException {
+        return new GetRepositoriesResponse(in);
     }
 
     @Override
@@ -71,7 +79,7 @@ protected ClusterBlockException checkBlock(GetRepositoriesRequest request, Clust
     }
 
     @Override
-    protected void masterOperation(final GetRepositoriesRequest request, ClusterState state,
+    protected void masterOperation(Task task, final GetRepositoriesRequest request, ClusterState state,
                                    final ActionListener<GetRepositoriesResponse> listener) {
         MetaData metaData = state.metaData();
         RepositoriesMetaData repositories = metaData.custom(RepositoriesMetaData.TYPE);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryAction.java
index 7c98f591e0e1d..0aa5c6a5d51f9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryAction.java
@@ -19,13 +19,14 @@
 
 package org.elasticsearch.action.admin.cluster.repositories.put;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
 /**
  * Register repository action
  */
-public class PutRepositoryAction extends Action<AcknowledgedResponse> {
+public class PutRepositoryAction extends ActionType<AcknowledgedResponse> {
 
     public static final PutRepositoryAction INSTANCE = new PutRepositoryAction();
     public static final String NAME = "cluster:admin/repository/put";
@@ -35,8 +36,8 @@ private PutRepositoryAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java
index a10e5c878a7c9..21f4c7e6f384d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java
@@ -29,10 +29,14 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 /**
  * Transport action for register repository operation
  */
@@ -54,9 +58,14 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
@@ -65,7 +74,7 @@ protected ClusterBlockException checkBlock(PutRepositoryRequest request, Cluster
     }
 
     @Override
-    protected void masterOperation(final PutRepositoryRequest request, ClusterState state,
+    protected void masterOperation(Task task, final PutRepositoryRequest request, ClusterState state,
                                    final ActionListener<AcknowledgedResponse> listener) {
         repositoriesService.registerRepository(request, ActionListener.delegateFailure(listener,
             (delegatedListener, response) -> delegatedListener.onResponse(new AcknowledgedResponse(response.isAcknowledged()))));
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java
index d4ec1d3a8bcb4..3de71e346f3c4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -66,7 +67,7 @@ protected ClusterBlockException checkBlock(VerifyRepositoryRequest request, Clus
     }
 
     @Override
-    protected void masterOperation(final VerifyRepositoryRequest request, ClusterState state,
+    protected void masterOperation(Task task, final VerifyRepositoryRequest request, ClusterState state,
                                    final ActionListener<VerifyRepositoryResponse> listener) {
         repositoriesService.verifyRepository(request.name(), ActionListener.delegateFailure(listener,
             (delegatedListener, verifyResponse) ->
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryAction.java
index 743b0a57f3aa3..9601ceecf0dcd 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryAction.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.action.admin.cluster.repositories.verify;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
  * Unregister repository action
  */
-public class VerifyRepositoryAction extends Action<VerifyRepositoryResponse> {
+public class VerifyRepositoryAction extends StreamableResponseActionType<VerifyRepositoryResponse> {
 
     public static final VerifyRepositoryAction INSTANCE = new VerifyRepositoryAction();
     public static final String NAME = "cluster:admin/repository/verify";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteAction.java
index 06b083e3414b2..82577a58cace7 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteAction.java
@@ -19,9 +19,10 @@
 
 package org.elasticsearch.action.admin.cluster.reroute;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class ClusterRerouteAction extends Action<ClusterRerouteResponse> {
+public class ClusterRerouteAction extends ActionType<ClusterRerouteResponse> {
 
     public static final ClusterRerouteAction INSTANCE = new ClusterRerouteAction();
     public static final String NAME = "cluster:admin/reroute";
@@ -31,7 +32,7 @@ private ClusterRerouteAction() {
     }
 
     @Override
-    public ClusterRerouteResponse newResponse() {
-        return new ClusterRerouteResponse();
+    public Writeable.Reader<ClusterRerouteResponse> getResponseReader() {
+        return ClusterRerouteResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
index 652a794e0f1cb..95ec34acc6b69 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
@@ -35,11 +35,13 @@
  */
 public class ClusterRerouteResponse extends AcknowledgedResponse implements ToXContentObject {
 
-    private ClusterState state;
-    private RoutingExplanations explanations;
-
-    ClusterRerouteResponse() {
+    private final ClusterState state;
+    private final RoutingExplanations explanations;
 
+    ClusterRerouteResponse(StreamInput in) throws IOException {
+        super(in);
+        state = ClusterState.readFrom(in, null);
+        explanations = RoutingExplanations.readFrom(in);
     }
 
     ClusterRerouteResponse(boolean acknowledged, ClusterState state, RoutingExplanations explanations) {
@@ -59,13 +61,6 @@ public RoutingExplanations getExplanations() {
         return this.explanations;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        state = ClusterState.readFrom(in, null);
-        explanations = RoutingExplanations.readFrom(in);
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
index 4b51145204a61..eea3dafcb1eba 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
@@ -46,9 +46,12 @@
 import org.elasticsearch.common.collect.ImmutableOpenIntMap;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -80,11 +83,16 @@ protected ClusterBlockException checkBlock(ClusterRerouteRequest request, Cluste
 
     @Override
     protected ClusterRerouteResponse newResponse() {
-        return new ClusterRerouteResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(final ClusterRerouteRequest request, final ClusterState state,
+    protected ClusterRerouteResponse read(StreamInput in) throws IOException {
+        return new ClusterRerouteResponse(in);
+    }
+
+    @Override
+    protected void masterOperation(Task task, final ClusterRerouteRequest request, final ClusterState state,
                                    final ActionListener<ClusterRerouteResponse> listener) {
         Map<String, List<AbstractAllocateAllocationCommand>> stalePrimaryAllocations = new HashMap<>();
         for (AllocationCommand command : request.getCommands().commands()) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsAction.java
index af6a87f5a57e6..0bdb77d0c670a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsAction.java
@@ -19,9 +19,10 @@
 
 package org.elasticsearch.action.admin.cluster.settings;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class ClusterUpdateSettingsAction extends Action<ClusterUpdateSettingsResponse> {
+public class ClusterUpdateSettingsAction extends ActionType<ClusterUpdateSettingsResponse> {
 
     public static final ClusterUpdateSettingsAction INSTANCE = new ClusterUpdateSettingsAction();
     public static final String NAME = "cluster:admin/settings/update";
@@ -31,7 +32,7 @@ private ClusterUpdateSettingsAction() {
     }
 
     @Override
-    public ClusterUpdateSettingsResponse newResponse() {
-        return new ClusterUpdateSettingsResponse();
+    public Writeable.Reader<ClusterUpdateSettingsResponse> getResponseReader() {
+        return ClusterUpdateSettingsResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java
index 4e925b398b6d5..5023e16249a4c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java
@@ -31,6 +31,8 @@
 import java.io.IOException;
 import java.util.Objects;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
 /**
  * A response for a cluster update settings action.
  */
@@ -40,23 +42,22 @@ public class ClusterUpdateSettingsResponse extends AcknowledgedResponse {
     private static final ParseField TRANSIENT = new ParseField("transient");
 
     private static final ConstructingObjectParser<ClusterUpdateSettingsResponse, Void> PARSER = new ConstructingObjectParser<>(
-            "cluster_update_settings_response", true, a -> new ClusterUpdateSettingsResponse((boolean) a[0]));
+            "cluster_update_settings_response", true, args -> {
+                return new ClusterUpdateSettingsResponse((boolean) args[0], (Settings) args[1], (Settings) args[2]);
+            });
     static {
         declareAcknowledgedField(PARSER);
-        PARSER.declareObject((r, p) -> r.persistentSettings = p, (p, c) -> Settings.fromXContent(p), PERSISTENT);
-        PARSER.declareObject((r, t) -> r.transientSettings = t, (p, c) -> Settings.fromXContent(p), TRANSIENT);
+        PARSER.declareObject(constructorArg(), (p, c) -> Settings.fromXContent(p), TRANSIENT);
+        PARSER.declareObject(constructorArg(), (p, c) -> Settings.fromXContent(p), PERSISTENT);
     }
 
-    Settings transientSettings;
-    Settings persistentSettings;
-
-    ClusterUpdateSettingsResponse() {
-        this.persistentSettings = Settings.EMPTY;
-        this.transientSettings = Settings.EMPTY;
-    }
+    final Settings transientSettings;
+    final Settings persistentSettings;
 
-    ClusterUpdateSettingsResponse(boolean acknowledged) {
-        super(acknowledged);
+    ClusterUpdateSettingsResponse(StreamInput in) throws IOException {
+        super(in);
+        transientSettings = Settings.readSettingsFromStream(in);
+        persistentSettings = Settings.readSettingsFromStream(in);
     }
 
     ClusterUpdateSettingsResponse(boolean acknowledged, Settings transientSettings, Settings persistentSettings) {
@@ -65,13 +66,6 @@ public class ClusterUpdateSettingsResponse extends AcknowledgedResponse {
         this.transientSettings = transientSettings;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        transientSettings = Settings.readSettingsFromStream(in);
-        persistentSettings = Settings.readSettingsFromStream(in);
-    }
-
     public Settings getTransientSettings() {
         return transientSettings;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
index cf34094290a87..ec55a21bac4bf 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
@@ -36,10 +36,14 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 public class TransportClusterUpdateSettingsAction extends
     TransportMasterNodeAction<ClusterUpdateSettingsRequest, ClusterUpdateSettingsResponse> {
 
@@ -81,11 +85,16 @@ protected ClusterBlockException checkBlock(ClusterUpdateSettingsRequest request,
 
     @Override
     protected ClusterUpdateSettingsResponse newResponse() {
-        return new ClusterUpdateSettingsResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected ClusterUpdateSettingsResponse read(StreamInput in) throws IOException {
+        return new ClusterUpdateSettingsResponse(in);
     }
 
     @Override
-    protected void masterOperation(final ClusterUpdateSettingsRequest request, final ClusterState state,
+    protected void masterOperation(Task task, final ClusterUpdateSettingsRequest request, final ClusterState state,
                                    final ActionListener<ClusterUpdateSettingsResponse> listener) {
         final SettingsUpdater updater = new SettingsUpdater(clusterSettings);
         clusterService.submitStateUpdateTask("cluster_update_settings",
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsAction.java
index 869aecf095431..837fb9e39709f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsAction.java
@@ -19,10 +19,10 @@
 
 package org.elasticsearch.action.admin.cluster.shards;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
-public class ClusterSearchShardsAction extends Action<ClusterSearchShardsResponse> {
+public class ClusterSearchShardsAction extends ActionType<ClusterSearchShardsResponse> {
 
     public static final ClusterSearchShardsAction INSTANCE = new ClusterSearchShardsAction();
     public static final String NAME = "indices:admin/shards/search_shards";
@@ -31,11 +31,6 @@ private ClusterSearchShardsAction() {
         super(NAME);
     }
 
-    @Override
-    public ClusterSearchShardsResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<ClusterSearchShardsResponse> getResponseReader() {
         return ClusterSearchShardsResponse::new;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java
index 39006cd1e8407..23b9f34a241c3 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -82,7 +83,7 @@ protected ClusterSearchShardsResponse read(StreamInput in) throws IOException {
     }
 
     @Override
-    protected void masterOperation(final ClusterSearchShardsRequest request, final ClusterState state,
+    protected void masterOperation(Task task, final ClusterSearchShardsRequest request, final ClusterState state,
                                    final ActionListener<ClusterSearchShardsResponse> listener) {
         ClusterState clusterState = clusterService.state();
         String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotAction.java
index d37132a1d81e7..a7a8e5467c71e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotAction.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.create;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
  * Create snapshot action
  */
-public class CreateSnapshotAction extends Action<CreateSnapshotResponse> {
+public class CreateSnapshotAction extends StreamableResponseActionType<CreateSnapshotResponse> {
 
     public static final CreateSnapshotAction INSTANCE = new CreateSnapshotAction();
     public static final String NAME = "cluster:admin/snapshot/create";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
index 8f83da053b215..e0250da5ed1cb 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
@@ -19,12 +19,14 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.create;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -46,6 +48,7 @@
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
+import static org.elasticsearch.snapshots.SnapshotInfo.METADATA_FIELD_INTRODUCED;
 
 /**
  * Create snapshot request
@@ -63,6 +66,7 @@
  */
 public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotRequest>
         implements IndicesRequest.Replaceable, ToXContentObject {
+    public static int MAXIMUM_METADATA_BYTES = 1024; // chosen arbitrarily
 
     private String snapshot;
 
@@ -80,6 +84,8 @@ public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotReque
 
     private boolean waitForCompletion;
 
+    private Map<String, Object> userMetadata;
+
     public CreateSnapshotRequest() {
     }
 
@@ -104,6 +110,9 @@ public CreateSnapshotRequest(StreamInput in) throws IOException {
         includeGlobalState = in.readBoolean();
         waitForCompletion = in.readBoolean();
         partial = in.readBoolean();
+        if (in.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) {
+            userMetadata = in.readMap();
+        }
     }
 
     @Override
@@ -117,6 +126,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(includeGlobalState);
         out.writeBoolean(waitForCompletion);
         out.writeBoolean(partial);
+        if (out.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) {
+            out.writeMap(userMetadata);
+        }
     }
 
     @Override
@@ -144,9 +156,28 @@ public ActionRequestValidationException validate() {
         if (settings == null) {
             validationException = addValidationError("settings is null", validationException);
         }
+        final int metadataSize = metadataSize(userMetadata);
+        if (metadataSize > MAXIMUM_METADATA_BYTES) {
+            validationException = addValidationError("metadata must be smaller than 1024 bytes, but was [" + metadataSize + "]",
+                validationException);
+        }
         return validationException;
     }
 
+    private static int metadataSize(Map<String, Object> userMetadata) {
+        if (userMetadata == null) {
+            return 0;
+        }
+        try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
+            builder.value(userMetadata);
+            int size = BytesReference.bytes(builder).length();
+            return size;
+        } catch (IOException e) {
+            // This should not be possible as we are just rendering the xcontent in memory
+            throw new ElasticsearchException(e);
+        }
+    }
+
     /**
      * Sets the snapshot name
      *
@@ -378,6 +409,15 @@ public boolean includeGlobalState() {
         return includeGlobalState;
     }
 
+    public Map<String, Object> userMetadata() {
+        return userMetadata;
+    }
+
+    public CreateSnapshotRequest userMetadata(Map<String, Object> userMetadata) {
+        this.userMetadata = userMetadata;
+        return this;
+    }
+
     /**
      * Parses snapshot definition.
      *
@@ -405,6 +445,11 @@ public CreateSnapshotRequest source(Map<String, Object> source) {
                 settings((Map<String, Object>) entry.getValue());
             } else if (name.equals("include_global_state")) {
                 includeGlobalState = nodeBooleanValue(entry.getValue(), "include_global_state");
+            } else if (name.equals("metadata")) {
+                if (entry.getValue() != null && (entry.getValue() instanceof Map == false)) {
+                    throw new IllegalArgumentException("malformed metadata, should be an object");
+                }
+                userMetadata((Map<String, Object>) entry.getValue());
             }
         }
         indicesOptions(IndicesOptions.fromMap(source, indicesOptions));
@@ -433,6 +478,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (indicesOptions != null) {
             indicesOptions.toXContent(builder, params);
         }
+        builder.field("metadata", userMetadata);
         builder.endObject();
         return builder;
     }
@@ -460,12 +506,14 @@ public boolean equals(Object o) {
             Arrays.equals(indices, that.indices) &&
             Objects.equals(indicesOptions, that.indicesOptions) &&
             Objects.equals(settings, that.settings) &&
-            Objects.equals(masterNodeTimeout, that.masterNodeTimeout);
+            Objects.equals(masterNodeTimeout, that.masterNodeTimeout) &&
+            Objects.equals(userMetadata, that.userMetadata);
     }
 
     @Override
     public int hashCode() {
-        int result = Objects.hash(snapshot, repository, indicesOptions, partial, settings, includeGlobalState, waitForCompletion);
+        int result = Objects.hash(snapshot, repository, indicesOptions, partial, settings, includeGlobalState,
+            waitForCompletion, userMetadata);
         result = 31 * result + Arrays.hashCode(indices);
         return result;
     }
@@ -482,6 +530,7 @@ public String toString() {
             ", includeGlobalState=" + includeGlobalState +
             ", waitForCompletion=" + waitForCompletion +
             ", masterNodeTimeout=" + masterNodeTimeout +
+            ", metadata=" + userMetadata +
             '}';
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
index 73f9a0742a719..a3de0d4f0a975 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.snapshots.SnapshotsService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -70,8 +71,8 @@ protected ClusterBlockException checkBlock(CreateSnapshotRequest request, Cluste
     }
 
     @Override
-    protected void masterOperation(final CreateSnapshotRequest request, ClusterState state,
-        final ActionListener<CreateSnapshotResponse> listener) {
+    protected void masterOperation(Task task, final CreateSnapshotRequest request, ClusterState state,
+                                   final ActionListener<CreateSnapshotResponse> listener) {
         if (request.waitForCompletion()) {
             snapshotsService.executeSnapshot(request, ActionListener.map(listener, CreateSnapshotResponse::new));
         } else {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotAction.java
index 64f4e60fb2d0e..97a3405e9bdf6 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotAction.java
@@ -19,13 +19,14 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.delete;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
 /**
  * Delete snapshot action
  */
-public class DeleteSnapshotAction extends Action<AcknowledgedResponse> {
+public class DeleteSnapshotAction extends ActionType<AcknowledgedResponse> {
 
     public static final DeleteSnapshotAction INSTANCE = new DeleteSnapshotAction();
     public static final String NAME = "cluster:admin/snapshot/delete";
@@ -35,8 +36,8 @@ private DeleteSnapshotAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
index 140d480242dc0..3f980225f434c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
@@ -29,10 +29,14 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.snapshots.SnapshotsService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 /**
  * Transport action for delete snapshot operation
  */
@@ -53,9 +57,14 @@ protected String executor() {
         return ThreadPool.Names.GENERIC;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
@@ -65,7 +74,7 @@ protected ClusterBlockException checkBlock(DeleteSnapshotRequest request, Cluste
     }
 
     @Override
-    protected void masterOperation(final DeleteSnapshotRequest request, ClusterState state,
+    protected void masterOperation(Task task, final DeleteSnapshotRequest request, ClusterState state,
                                    final ActionListener<AcknowledgedResponse> listener) {
         snapshotsService.deleteSnapshot(request.repository(), request.snapshot(),
             ActionListener.map(listener, v -> new AcknowledgedResponse(true)), false);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsAction.java
index b5015ff5c23b0..ab0823b98f71c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsAction.java
@@ -19,12 +19,13 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.get;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.common.io.stream.Writeable;
 
 /**
  * Get snapshots action
  */
-public class GetSnapshotsAction extends Action<GetSnapshotsResponse> {
+public class GetSnapshotsAction extends ActionType<GetSnapshotsResponse> {
 
     public static final GetSnapshotsAction INSTANCE = new GetSnapshotsAction();
     public static final String NAME = "cluster:admin/snapshot/get";
@@ -34,8 +35,8 @@ private GetSnapshotsAction() {
     }
 
     @Override
-    public GetSnapshotsResponse newResponse() {
-        return new GetSnapshotsResponse();
+    public Writeable.Reader<GetSnapshotsResponse> getResponseReader() {
+        return GetSnapshotsResponse::new;
     }
 }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java
index 41ae57031d320..4bfd656700684 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.get;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.Strings;
@@ -37,8 +38,9 @@ public class GetSnapshotsRequest extends MasterNodeRequest<GetSnapshotsRequest>
     public static final String ALL_SNAPSHOTS = "_all";
     public static final String CURRENT_SNAPSHOT = "_current";
     public static final boolean DEFAULT_VERBOSE_MODE = true;
+    public static final Version MULTIPLE_REPOSITORIES_SUPPORT_ADDED = Version.V_8_0_0;
 
-    private String repository;
+    private String[] repositories;
 
     private String[] snapshots = Strings.EMPTY_ARRAY;
 
@@ -50,28 +52,32 @@ public GetSnapshotsRequest() {
     }
 
     /**
-     * Constructs a new get snapshots request with given repository name and list of snapshots
+     * Constructs a new get snapshots request with given repository names and list of snapshots
      *
-     * @param repository repository name
+     * @param repositories repository names
      * @param snapshots  list of snapshots
      */
-    public GetSnapshotsRequest(String repository, String[] snapshots) {
-        this.repository = repository;
+    public GetSnapshotsRequest(String[] repositories, String[] snapshots) {
+        this.repositories = repositories;
         this.snapshots = snapshots;
     }
 
     /**
-     * Constructs a new get snapshots request with given repository name
+     * Constructs a new get snapshots request with given repository names
      *
-     * @param repository repository name
+     * @param repositories repository names
      */
-    public GetSnapshotsRequest(String repository) {
-        this.repository = repository;
+    public GetSnapshotsRequest(String... repositories) {
+        this.repositories = repositories;
     }
 
     public GetSnapshotsRequest(StreamInput in) throws IOException {
         super(in);
-        repository = in.readString();
+        if (in.getVersion().onOrAfter(MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) {
+            repositories = in.readStringArray();
+        } else {
+            repositories = new String[]{in.readString()};
+        }
         snapshots = in.readStringArray();
         ignoreUnavailable = in.readBoolean();
         verbose = in.readBoolean();
@@ -80,7 +86,15 @@ public GetSnapshotsRequest(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeString(repository);
+        if (out.getVersion().onOrAfter(MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) {
+            out.writeStringArray(repositories);
+        } else {
+            if (repositories.length != 1) {
+                throw new IllegalArgumentException("Requesting snapshots from multiple repositories is not supported in versions prior " +
+                        "to " + MULTIPLE_REPOSITORIES_SUPPORT_ADDED.toString());
+            }
+            out.writeString(repositories[0]);
+        }
         out.writeStringArray(snapshots);
         out.writeBoolean(ignoreUnavailable);
         out.writeBoolean(verbose);
@@ -89,30 +103,30 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
-        if (repository == null) {
-            validationException = addValidationError("repository is missing", validationException);
+        if (repositories == null || repositories.length == 0) {
+            validationException = addValidationError("repositories are missing", validationException);
         }
         return validationException;
     }
 
     /**
-     * Sets repository name
+     * Sets repository names
      *
-     * @param repository repository name
+     * @param repositories repository names
      * @return this request
      */
-    public GetSnapshotsRequest repository(String repository) {
-        this.repository = repository;
+    public GetSnapshotsRequest repositories(String... repositories) {
+        this.repositories = repositories;
         return this;
     }
 
     /**
-     * Returns repository name
+     * Returns repository names
      *
-     * @return repository name
+     * @return repository names
      */
-    public String repository() {
-        return this.repository;
+    public String[] repositories() {
+        return this.repositories;
     }
 
     /**
@@ -176,4 +190,4 @@ public boolean verbose() {
     public void readFrom(StreamInput in) throws IOException {
         throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
-}
+}
\ No newline at end of file
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java
index 052f8da0c7508..e4219c858551a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java
@@ -30,27 +30,20 @@ public class GetSnapshotsRequestBuilder extends MasterNodeOperationRequestBuilde
         GetSnapshotsResponse, GetSnapshotsRequestBuilder> {
 
     /**
-     * Constructs the new get snapshot request
+     * Constructs the new get snapshot request with specified repositories
      */
-    public GetSnapshotsRequestBuilder(ElasticsearchClient client, GetSnapshotsAction action) {
-        super(client, action, new GetSnapshotsRequest());
+    public GetSnapshotsRequestBuilder(ElasticsearchClient client, GetSnapshotsAction action, String... repositories) {
+        super(client, action, new GetSnapshotsRequest(repositories));
     }
 
     /**
-     * Constructs the new get snapshot request with specified repository
-     */
-    public GetSnapshotsRequestBuilder(ElasticsearchClient client, GetSnapshotsAction action, String repository) {
-        super(client, action, new GetSnapshotsRequest(repository));
-    }
-
-    /**
-     * Sets the repository name
+     * Sets the repository names
      *
-     * @param repository repository name
+     * @param repositories repository names
      * @return this builder
      */
-    public GetSnapshotsRequestBuilder setRepository(String repository) {
-        request.repository(repository);
+    public GetSnapshotsRequestBuilder setRepositories(String... repositories) {
+        request.repositories(repositories);
         return this;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
index 6f757cb60ca86..1ecbd222c6d84 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
@@ -19,12 +19,14 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.get;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -32,89 +34,241 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
-import java.util.Objects;
+import java.util.Map;
+import java.util.Set;
 
 /**
  * Get snapshots response
  */
 public class GetSnapshotsResponse extends ActionResponse implements ToXContentObject {
 
-    @SuppressWarnings("unchecked")
-    private static final ConstructingObjectParser<GetSnapshotsResponse, Void> GET_SNAPSHOT_PARSER =
-        new ConstructingObjectParser<>(GetSnapshotsResponse.class.getName(), true,
-            (args) -> new GetSnapshotsResponse((List<SnapshotInfo>) args[0]));
+    private static final ConstructingObjectParser<GetSnapshotsResponse, Void> PARSER =
+            new ConstructingObjectParser<>(GetSnapshotsResponse.class.getName(), true,
+                    (args) -> new GetSnapshotsResponse((List<Response>) args[0]));
 
     static {
-        GET_SNAPSHOT_PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(),
-            (p, c) -> SnapshotInfo.SNAPSHOT_INFO_PARSER.apply(p, c).build(), new ParseField("snapshots"));
+        PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(),
+                (p, c) -> Response.fromXContent(p), new ParseField("responses"));
     }
 
-    private List<SnapshotInfo> snapshots = Collections.emptyList();
+    public GetSnapshotsResponse(StreamInput in) throws IOException {
+        if (in.getVersion().onOrAfter(GetSnapshotsRequest.MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) {
+            int successfulSize = in.readVInt();
+            Map<String, List<SnapshotInfo>> successfulResponses = new HashMap<>(successfulSize);
+            for (int i = 0; i < successfulSize; i++) {
+                String repository = in.readString();
+                int size = in.readVInt();
+                List<SnapshotInfo> snapshotInfos = new ArrayList<>(size);
+                for (int j = 0; j < size; j++) {
+                    snapshotInfos.add(new SnapshotInfo(in));
+                }
+                successfulResponses.put(repository, snapshotInfos);
+            }
 
-    GetSnapshotsResponse() {
+            int failedSize = in.readVInt();
+            Map<String, ElasticsearchException> failedResponses = new HashMap<>(failedSize);
+            for (int i = 0; i < failedSize; i++) {
+                String repository = in.readString();
+                ElasticsearchException error = in.readException();
+                failedResponses.put(repository, error);
+            }
+            this.successfulResponses = Collections.unmodifiableMap(successfulResponses);
+            this.failedResponses = Collections.unmodifiableMap(failedResponses);
+        } else {
+            int size = in.readVInt();
+            List<SnapshotInfo> snapshots = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                snapshots.add(new SnapshotInfo(in));
+            }
+            this.successfulResponses = Collections.singletonMap("unknown", snapshots);
+            this.failedResponses = Collections.emptyMap();
+        }
     }
 
-    GetSnapshotsResponse(List<SnapshotInfo> snapshots) {
-        this.snapshots = Collections.unmodifiableList(snapshots);
+
+    public static class Response {
+        private String repository;
+        private List<SnapshotInfo> snapshots;
+        private ElasticsearchException error;
+
+        private static final ConstructingObjectParser<Response, Void> RESPONSE_PARSER =
+                new ConstructingObjectParser<>(Response.class.getName(), true,
+                        (args) -> new Response((String) args[0],
+                                (List<SnapshotInfo>) args[1], (ElasticsearchException) args[2]));
+
+        static {
+            RESPONSE_PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("repository"));
+            RESPONSE_PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(),
+                    (p, c) -> SnapshotInfo.SNAPSHOT_INFO_PARSER.apply(p, c).build(), new ParseField("snapshots"));
+            RESPONSE_PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(),
+                    (p, c) -> ElasticsearchException.fromXContent(p), new ParseField("error"));
+        }
+
+        private Response(String repository, List<SnapshotInfo> snapshots, ElasticsearchException error) {
+            this.repository = repository;
+            this.snapshots = snapshots;
+            this.error = error;
+        }
+
+        public static Response snapshots(String repository, List<SnapshotInfo> snapshots) {
+            return new Response(repository, snapshots, null);
+        }
+
+        public static Response error(String repository, ElasticsearchException error) {
+            return new Response(repository, null, error);
+        }
+
+        private static Response fromXContent(XContentParser parser) throws IOException {
+            return RESPONSE_PARSER.parse(parser, null);
+        }
+    }
+
+    private final Map<String, List<SnapshotInfo>> successfulResponses;
+    private final Map<String, ElasticsearchException> failedResponses;
+
+    public GetSnapshotsResponse(Collection<Response> responses) {
+        Map<String, List<SnapshotInfo>> successfulResponses = new HashMap<>();
+        Map<String, ElasticsearchException> failedResponses = new HashMap<>();
+        for (Response response : responses) {
+            if (response.snapshots != null) {
+                assert response.error == null;
+                successfulResponses.put(response.repository, response.snapshots);
+            } else {
+                assert response.snapshots == null;
+                failedResponses.put(response.repository, response.error);
+            }
+        }
+        this.successfulResponses = Collections.unmodifiableMap(successfulResponses);
+        this.failedResponses = Collections.unmodifiableMap(failedResponses);
     }
 
     /**
-     * Returns the list of snapshots
-     *
-     * @return the list of snapshots
+     * Returns list of snapshots for the specified repository.
+     * @param repo - repository name.
+     * @return list of snapshots.
+     * @throws IllegalArgumentException if there is no such repository in the response.
+     * @throws ElasticsearchException if an exception occurred when retrieving snapshots from the repository.
      */
-    public List<SnapshotInfo> getSnapshots() {
-        return snapshots;
+    public List<SnapshotInfo> getSnapshots(String repo) {
+        List<SnapshotInfo> snapshots = successfulResponses.get(repo);
+        if (snapshots != null) {
+            return snapshots;
+        }
+        ElasticsearchException error = failedResponses.get(repo);
+        if (error == null) {
+            throw new IllegalArgumentException("No such repository");
+        }
+        throw error;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        int size = in.readVInt();
-        List<SnapshotInfo> builder = new ArrayList<>(size);
-        for (int i = 0; i < size; i++) {
-            builder.add(new SnapshotInfo(in));
-        }
-        snapshots = Collections.unmodifiableList(builder);
+    /**
+     * Returns list of repositories for both successful and unsuccessful responses.
+     */
+    public Set<String> getRepositories() {
+        return Sets.union(successfulResponses.keySet(), failedResponses.keySet());
     }
 
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeVInt(snapshots.size());
-        for (SnapshotInfo snapshotInfo : snapshots) {
-            snapshotInfo.writeTo(out);
-        }
+    /**
+     * Returns a map of repository name to the list of {@link SnapshotInfo} for each successful response.
+     */
+    public Map<String, List<SnapshotInfo>> getSuccessfulResponses() {
+        return successfulResponses;
+    }
+
+    /**
+     * Returns a map of repository name to {@link ElasticsearchException} for each unsuccessful response.
+     */
+    public Map<String, ElasticsearchException> getFailedResponses() {
+        return failedResponses;
+    }
+
+    /**
+     * Returns true if there is a least one failed response.
+     */
+    public boolean isFailed() {
+        return failedResponses.isEmpty() == false;
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.startArray("snapshots");
-        for (SnapshotInfo snapshotInfo : snapshots) {
-            snapshotInfo.toXContent(builder, params);
+        builder.startArray("responses");
+
+        for (Map.Entry<String, List<SnapshotInfo>> snapshots : successfulResponses.entrySet()) {
+            builder.startObject();
+            builder.field("repository", snapshots.getKey());
+            builder.startArray("snapshots");
+            for (SnapshotInfo snapshot : snapshots.getValue()) {
+                snapshot.toXContent(builder, params);
+            }
+            builder.endArray();
+            builder.endObject();
+        }
+
+        for (Map.Entry<String, ElasticsearchException> error : failedResponses.entrySet()) {
+            builder.startObject();
+            builder.field("repository", error.getKey());
+            ElasticsearchException.generateFailureXContent(builder, params, error.getValue(), true);
+            builder.endObject();
         }
+
         builder.endArray();
         builder.endObject();
         return builder;
     }
 
-    public static GetSnapshotsResponse fromXContent(XContentParser parser) throws IOException {
-        return GET_SNAPSHOT_PARSER.parse(parser, null);
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        if (out.getVersion().onOrAfter(GetSnapshotsRequest.MULTIPLE_REPOSITORIES_SUPPORT_ADDED)) {
+            out.writeVInt(successfulResponses.size());
+            for (Map.Entry<String, List<SnapshotInfo>> snapshots : successfulResponses.entrySet()) {
+                out.writeString(snapshots.getKey());
+                out.writeVInt(snapshots.getValue().size());
+                for (SnapshotInfo snapshotInfo : snapshots.getValue()) {
+                    snapshotInfo.writeTo(out);
+                }
+            }
+            out.writeVInt(failedResponses.size());
+            for (Map.Entry<String, ElasticsearchException> error : failedResponses.entrySet()) {
+                out.writeString(error.getKey());
+                out.writeException(error.getValue());
+            }
+        } else {
+            if (successfulResponses.size() + failedResponses.size() != 1) {
+                throw new IllegalArgumentException("Requesting snapshots from multiple repositories is not supported in versions prior " +
+                        "to " + GetSnapshotsRequest.MULTIPLE_REPOSITORIES_SUPPORT_ADDED.toString());
+            }
+
+            if (successfulResponses.size() == 1) {
+                List<SnapshotInfo> snapshotInfos = successfulResponses.values().iterator().next();
+                out.writeVInt(snapshotInfos.size());
+                for (SnapshotInfo snapshotInfo : snapshotInfos) {
+                    snapshotInfo.writeTo(out);
+                }
+            }
+
+            if (failedResponses.isEmpty() == false) {
+                throw failedResponses.values().iterator().next();
+            }
+        }
     }
 
     @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        GetSnapshotsResponse that = (GetSnapshotsResponse) o;
-        return Objects.equals(snapshots, that.snapshots);
+    public void readFrom(StreamInput in) throws IOException {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    public static GetSnapshotsResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null);
     }
 
     @Override
-    public int hashCode() {
-        return Objects.hash(snapshots);
+    public String toString() {
+        return Strings.toString(this);
     }
-}
+}
\ No newline at end of file
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
index 23ffbd0dd1e3c..96fe976ea0fbb 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
@@ -20,15 +20,24 @@
 package org.elasticsearch.action.admin.cluster.snapshots.get;
 
 import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionListenerResponseHandler;
+import org.elasticsearch.action.ActionRunnable;
+import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesAction;
+import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
+import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.GroupedActionListener;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoryData;
@@ -36,9 +45,12 @@
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotMissingException;
 import org.elasticsearch.snapshots.SnapshotsService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -70,7 +82,12 @@ protected String executor() {
 
     @Override
     protected GetSnapshotsResponse newResponse() {
-        return new GetSnapshotsResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected GetSnapshotsResponse read(StreamInput in) throws IOException {
+        return new GetSnapshotsResponse(in);
     }
 
     @Override
@@ -79,75 +96,116 @@ protected ClusterBlockException checkBlock(GetSnapshotsRequest request, ClusterS
     }
 
     @Override
-    protected void masterOperation(final GetSnapshotsRequest request, final ClusterState state,
+    protected void masterOperation(Task task, final GetSnapshotsRequest request, final ClusterState state,
                                    final ActionListener<GetSnapshotsResponse> listener) {
-        try {
-            final String repository = request.repository();
-            final Map<String, SnapshotId> allSnapshotIds = new HashMap<>();
-            final List<SnapshotInfo> currentSnapshots = new ArrayList<>();
-            for (SnapshotInfo snapshotInfo : snapshotsService.currentSnapshots(repository)) {
-                SnapshotId snapshotId = snapshotInfo.snapshotId();
-                allSnapshotIds.put(snapshotId.getName(), snapshotId);
-                currentSnapshots.add(snapshotInfo);
-            }
+        final String[] repositories = request.repositories();
+        transportService.sendChildRequest(transportService.getLocalNode(), GetRepositoriesAction.NAME,
+                new GetRepositoriesRequest(repositories), task, TransportRequestOptions.EMPTY,
+                new ActionListenerResponseHandler<>(
+                        ActionListener.wrap(
+                                response ->
+                                        // switch to GENERIC thread pool because it might be long running operation
+                                        threadPool.executor(ThreadPool.Names.GENERIC).execute(
+                                                () -> getMultipleReposSnapshotInfo(response.repositories(), request.snapshots(),
+                                                        request.ignoreUnavailable(), request.verbose(), listener)),
+                                listener::onFailure),
+                        GetRepositoriesResponse::new));
+    }
 
-            final RepositoryData repositoryData;
-            if (isCurrentSnapshotsOnly(request.snapshots()) == false) {
-                repositoryData = snapshotsService.getRepositoryData(repository);
-                for (SnapshotId snapshotId : repositoryData.getAllSnapshotIds()) {
-                    allSnapshotIds.put(snapshotId.getName(), snapshotId);
+    private void getMultipleReposSnapshotInfo(List<RepositoryMetaData> repos, String[] snapshots, boolean ignoreUnavailable,
+                                              boolean verbose, ActionListener<GetSnapshotsResponse> listener) {
+        // short-circuit if there are no repos, because we can not create GroupedActionListener of size 0
+        if (repos.isEmpty()) {
+            listener.onResponse(new GetSnapshotsResponse(Collections.emptyList()));
+            return;
+        }
+        final GroupedActionListener<GetSnapshotsResponse.Response> groupedActionListener =
+                new GroupedActionListener<>(
+                        ActionListener.map(listener, responses -> {
+                            assert repos.size() == responses.size();
+                            return new GetSnapshotsResponse(responses);
+                        }), repos.size());
+
+        // run concurrently for all repos on GENERIC thread pool
+        for (final RepositoryMetaData repo : repos) {
+            threadPool.executor(ThreadPool.Names.GENERIC).execute(new ActionRunnable<>(groupedActionListener) {
+                @Override
+                protected void doRun() {
+                    try {
+                        groupedActionListener.onResponse(GetSnapshotsResponse.Response.snapshots(
+                                repo.name(), getSingleRepoSnapshotInfo(repo.name(), snapshots, ignoreUnavailable, verbose)));
+                    } catch (ElasticsearchException e) {
+                        groupedActionListener.onResponse(GetSnapshotsResponse.Response.error(repo.name(), e));
+                    }
                 }
-            } else {
-                repositoryData = null;
+            });
+        }
+    }
+
+    private List<SnapshotInfo> getSingleRepoSnapshotInfo(String repo, String[] snapshots, boolean ignoreUnavailable, boolean verbose) {
+        final Map<String, SnapshotId> allSnapshotIds = new HashMap<>();
+        final List<SnapshotInfo> currentSnapshots = new ArrayList<>();
+        for (SnapshotInfo snapshotInfo : snapshotsService.currentSnapshots(repo)) {
+            SnapshotId snapshotId = snapshotInfo.snapshotId();
+            allSnapshotIds.put(snapshotId.getName(), snapshotId);
+            currentSnapshots.add(snapshotInfo);
+        }
+
+        final RepositoryData repositoryData;
+        if (isCurrentSnapshotsOnly(snapshots) == false) {
+            repositoryData = snapshotsService.getRepositoryData(repo);
+            for (SnapshotId snapshotId : repositoryData.getAllSnapshotIds()) {
+                allSnapshotIds.put(snapshotId.getName(), snapshotId);
             }
+        } else {
+            repositoryData = null;
+        }
 
-            final Set<SnapshotId> toResolve = new HashSet<>();
-            if (isAllSnapshots(request.snapshots())) {
-                toResolve.addAll(allSnapshotIds.values());
-            } else {
-                for (String snapshotOrPattern : request.snapshots()) {
-                    if (GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshotOrPattern)) {
-                        toResolve.addAll(currentSnapshots.stream().map(SnapshotInfo::snapshotId).collect(Collectors.toList()));
-                    } else if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) {
-                        if (allSnapshotIds.containsKey(snapshotOrPattern)) {
-                            toResolve.add(allSnapshotIds.get(snapshotOrPattern));
-                        } else if (request.ignoreUnavailable() == false) {
-                            throw new SnapshotMissingException(repository, snapshotOrPattern);
-                        }
-                    } else {
-                        for (Map.Entry<String, SnapshotId> entry : allSnapshotIds.entrySet()) {
-                            if (Regex.simpleMatch(snapshotOrPattern, entry.getKey())) {
-                                toResolve.add(entry.getValue());
-                            }
+        final Set<SnapshotId> toResolve = new HashSet<>();
+        if (isAllSnapshots(snapshots)) {
+            toResolve.addAll(allSnapshotIds.values());
+        } else {
+            for (String snapshotOrPattern : snapshots) {
+                if (GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshotOrPattern)) {
+                    toResolve.addAll(currentSnapshots.stream().map(SnapshotInfo::snapshotId).collect(Collectors.toList()));
+                } else if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) {
+                    if (allSnapshotIds.containsKey(snapshotOrPattern)) {
+                        toResolve.add(allSnapshotIds.get(snapshotOrPattern));
+                    } else if (ignoreUnavailable == false) {
+                        throw new SnapshotMissingException(repo, snapshotOrPattern);
+                    }
+                } else {
+                    for (Map.Entry<String, SnapshotId> entry : allSnapshotIds.entrySet()) {
+                        if (Regex.simpleMatch(snapshotOrPattern, entry.getKey())) {
+                            toResolve.add(entry.getValue());
                         }
                     }
                 }
+            }
 
-                if (toResolve.isEmpty() && request.ignoreUnavailable() == false && isCurrentSnapshotsOnly(request.snapshots()) == false) {
-                    throw new SnapshotMissingException(repository, request.snapshots()[0]);
-                }
+            if (toResolve.isEmpty() && ignoreUnavailable == false && isCurrentSnapshotsOnly(snapshots) == false) {
+                throw new SnapshotMissingException(repo, snapshots[0]);
             }
+        }
 
-            final List<SnapshotInfo> snapshotInfos;
-            if (request.verbose()) {
-                final Set<SnapshotId> incompatibleSnapshots = repositoryData != null ?
+        final List<SnapshotInfo> snapshotInfos;
+        if (verbose) {
+            final Set<SnapshotId> incompatibleSnapshots = repositoryData != null ?
                     new HashSet<>(repositoryData.getIncompatibleSnapshotIds()) : Collections.emptySet();
-                snapshotInfos = snapshotsService.snapshots(repository, new ArrayList<>(toResolve),
-                    incompatibleSnapshots, request.ignoreUnavailable());
+            snapshotInfos = snapshotsService.snapshots(repo, new ArrayList<>(toResolve),
+                    incompatibleSnapshots, ignoreUnavailable);
+        } else {
+            if (repositoryData != null) {
+                // want non-current snapshots as well, which are found in the repository data
+                snapshotInfos = buildSimpleSnapshotInfos(toResolve, repositoryData, currentSnapshots);
             } else {
-                if (repositoryData != null) {
-                    // want non-current snapshots as well, which are found in the repository data
-                    snapshotInfos = buildSimpleSnapshotInfos(toResolve, repositoryData, currentSnapshots);
-                } else {
-                    // only want current snapshots
-                    snapshotInfos = currentSnapshots.stream().map(SnapshotInfo::basic).collect(Collectors.toList());
-                    CollectionUtil.timSort(snapshotInfos);
-                }
+                // only want current snapshots
+                snapshotInfos = currentSnapshots.stream().map(SnapshotInfo::basic).collect(Collectors.toList());
+                CollectionUtil.timSort(snapshotInfos);
             }
-            listener.onResponse(new GetSnapshotsResponse(snapshotInfos));
-        } catch (Exception e) {
-            listener.onFailure(e);
         }
+
+        return snapshotInfos;
     }
 
     private boolean isAllSnapshots(String[] snapshots) {
@@ -158,7 +216,7 @@ private boolean isCurrentSnapshotsOnly(String[] snapshots) {
         return (snapshots.length == 1 && GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshots[0]));
     }
 
-    private List<SnapshotInfo> buildSimpleSnapshotInfos(final Set<SnapshotId> toResolve,
+    private static List<SnapshotInfo> buildSimpleSnapshotInfos(final Set<SnapshotId> toResolve,
                                                         final RepositoryData repositoryData,
                                                         final List<SnapshotInfo> currentSnapshots) {
         List<SnapshotInfo> snapshotInfos = new ArrayList<>();
@@ -172,7 +230,7 @@ private List<SnapshotInfo> buildSimpleSnapshotInfos(final Set<SnapshotId> toReso
             for (SnapshotId snapshotId : repositoryData.getSnapshots(indexId)) {
                 if (toResolve.contains(snapshotId)) {
                     snapshotsToIndices.computeIfAbsent(snapshotId, (k) -> new ArrayList<>())
-                                      .add(indexId.getName());
+                            .add(indexId.getName());
                 }
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotAction.java
index e633ce43e66cd..7f473d10a28ab 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotAction.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.restore;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
  * Restore snapshot action
  */
-public class RestoreSnapshotAction extends Action<RestoreSnapshotResponse> {
+public class RestoreSnapshotAction extends StreamableResponseActionType<RestoreSnapshotResponse> {
 
     public static final RestoreSnapshotAction INSTANCE = new RestoreSnapshotAction();
     public static final String NAME = "cluster:admin/snapshot/restore";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
index 935902432d90f..5178a5224a058 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.snapshots.RestoreService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -72,7 +73,7 @@ protected ClusterBlockException checkBlock(RestoreSnapshotRequest request, Clust
     }
 
     @Override
-    protected void masterOperation(final RestoreSnapshotRequest request, final ClusterState state,
+    protected void masterOperation(Task task, final RestoreSnapshotRequest request, final ClusterState state,
                                    final ActionListener<RestoreSnapshotResponse> listener) {
         restoreService.restoreSnapshot(request, ActionListener.delegateFailure(listener,
             (delegatedListener, restoreCompletionResponse) -> {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusAction.java
index ea28d26b40ffa..ec95e0e14f13f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusAction.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.status;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
  * Snapshots status action
  */
-public class SnapshotsStatusAction extends Action<SnapshotsStatusResponse> {
+public class SnapshotsStatusAction extends StreamableResponseActionType<SnapshotsStatusResponse> {
 
     public static final SnapshotsStatusAction INSTANCE = new SnapshotsStatusAction();
     public static final String NAME = "cluster:admin/snapshot/status";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
index 1f55c1e00cef6..74b9e8cb3c592 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotShardsService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -48,7 +49,7 @@
 import static java.util.Collections.unmodifiableMap;
 
 /**
- * Transport client that collects snapshot shard statuses from data nodes
+ * Transport action that collects snapshot shard statuses from data nodes
  */
 public class TransportNodesSnapshotsStatus extends TransportNodesAction<TransportNodesSnapshotsStatus.Request,
                                                                         TransportNodesSnapshotsStatus.NodesSnapshotStatus,
@@ -68,8 +69,8 @@ public TransportNodesSnapshotsStatus(ThreadPool threadPool, ClusterService clust
     }
 
     @Override
-    protected NodeRequest newNodeRequest(String nodeId, Request request) {
-        return new NodeRequest(nodeId, request);
+    protected NodeRequest newNodeRequest(Request request) {
+        return new NodeRequest(request);
     }
 
     @Override
@@ -83,7 +84,7 @@ protected NodesSnapshotStatus newResponse(Request request, List<NodeSnapshotStat
     }
 
     @Override
-    protected NodeSnapshotStatus nodeOperation(NodeRequest request) {
+    protected NodeSnapshotStatus nodeOperation(NodeRequest request, Task task) {
         Map<Snapshot, Map<ShardId, SnapshotIndexShardStatus>> snapshotMapBuilder = new HashMap<>();
         try {
             final String nodeId = clusterService.localNode().getId();
@@ -168,8 +169,7 @@ public static class NodeRequest extends BaseNodeRequest {
         public NodeRequest() {
         }
 
-        NodeRequest(String nodeId, TransportNodesSnapshotsStatus.Request request) {
-            super(nodeId);
+        NodeRequest(TransportNodesSnapshotsStatus.Request request) {
             snapshots = Arrays.asList(request.snapshots);
         }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
index 8430d1868c88d..234bc51b95b25 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
@@ -42,6 +42,7 @@
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotMissingException;
 import org.elasticsearch.snapshots.SnapshotsService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -90,7 +91,7 @@ protected SnapshotsStatusResponse newResponse() {
     }
 
     @Override
-    protected void masterOperation(final SnapshotsStatusRequest request,
+    protected void masterOperation(Task task, final SnapshotsStatusRequest request,
                                    final ClusterState state,
                                    final ActionListener<SnapshotsStatusResponse> listener) throws Exception {
         List<SnapshotsInProgress.Entry> currentSnapshots =
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateAction.java
index f48df06d53c6f..b8a08444efccd 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.cluster.state;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class ClusterStateAction extends Action<ClusterStateResponse> {
+public class ClusterStateAction extends StreamableResponseActionType<ClusterStateResponse> {
 
     public static final ClusterStateAction INSTANCE = new ClusterStateAction();
     public static final String NAME = "cluster:monitor/state";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
index cedca2d77e192..9669abf8e931a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
@@ -38,6 +38,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.node.NodeClosedException;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -77,7 +78,7 @@ protected ClusterStateResponse newResponse() {
     }
 
     @Override
-    protected void masterOperation(final ClusterStateRequest request, final ClusterState state,
+    protected void masterOperation(Task task, final ClusterStateRequest request, final ClusterState state,
                                    final ActionListener<ClusterStateResponse> listener) throws IOException {
 
         final Predicate<ClusterState> acceptableClusterStatePredicate
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsAction.java
index 049ce62d9df40..0652ab35e266a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.cluster.stats;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class ClusterStatsAction extends Action<ClusterStatsResponse> {
+public class ClusterStatsAction extends StreamableResponseActionType<ClusterStatsResponse> {
 
     public static final ClusterStatsAction INSTANCE = new ClusterStatsAction();
     public static final String NAME = "cluster:monitor/stats";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java
index 49d82e574fb6b..bcadbc8e3292a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java
@@ -21,11 +21,11 @@
 
 import com.carrotsearch.hppc.ObjectIntHashMap;
 import com.carrotsearch.hppc.cursors.ObjectIntCursor;
-
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.network.NetworkModule;
@@ -49,6 +49,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeMap;
 import java.util.concurrent.atomic.AtomicInteger;
 
 public class ClusterStatsNodes implements ToXContentFragment {
@@ -186,27 +187,27 @@ public static class Counts implements ToXContentFragment {
         private final int total;
         private final Map<String, Integer> roles;
 
-        private Counts(List<NodeInfo> nodeInfos) {
-            this.roles = new HashMap<>();
-            for (DiscoveryNode.Role role : DiscoveryNode.Role.values()) {
-                this.roles.put(role.getRoleName(), 0);
+        private Counts(final List<NodeInfo> nodeInfos) {
+            // TODO: do we need to report zeros?
+            final Map<String, Integer> roles = new HashMap<>(DiscoveryNode.getPossibleRoleNames().size());
+            roles.put(COORDINATING_ONLY, 0);
+            for (final String possibleRoleName : DiscoveryNode.getPossibleRoleNames()) {
+                roles.put(possibleRoleName, 0);
             }
-            this.roles.put(COORDINATING_ONLY, 0);
 
             int total = 0;
-            for (NodeInfo nodeInfo : nodeInfos) {
+            for (final NodeInfo nodeInfo : nodeInfos) {
                 total++;
                 if (nodeInfo.getNode().getRoles().isEmpty()) {
-                    Integer count = roles.get(COORDINATING_ONLY);
-                    roles.put(COORDINATING_ONLY, ++count);
+                    roles.merge(COORDINATING_ONLY, 1, Integer::sum);
                 } else {
-                    for (DiscoveryNode.Role role : nodeInfo.getNode().getRoles()) {
-                        Integer count = roles.get(role.getRoleName());
-                        roles.put(role.getRoleName(), ++count);
+                    for (DiscoveryNodeRole role : nodeInfo.getNode().getRoles()) {
+                        roles.merge(role.roleName(), 1, Integer::sum);
                     }
                 }
             }
             this.total = total;
+            this.roles = Map.copyOf(roles);
         }
 
         public int getTotal() {
@@ -225,7 +226,7 @@ static final class Fields {
         public XContentBuilder toXContent(XContentBuilder builder, Params params)
                 throws IOException {
             builder.field(Fields.TOTAL, total);
-            for (Map.Entry<String, Integer> entry : roles.entrySet()) {
+            for (Map.Entry<String, Integer> entry : new TreeMap<>(roles).entrySet()) {
                 builder.field(entry.getKey(), entry.getValue());
             }
             return builder;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index 4cf81c24fbf1a..ef3bb6dfab24e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -42,6 +42,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.node.NodeService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -81,8 +82,8 @@ protected ClusterStatsResponse newResponse(ClusterStatsRequest request,
     }
 
     @Override
-    protected ClusterStatsNodeRequest newNodeRequest(String nodeId, ClusterStatsRequest request) {
-        return new ClusterStatsNodeRequest(nodeId, request);
+    protected ClusterStatsNodeRequest newNodeRequest(ClusterStatsRequest request) {
+        return new ClusterStatsNodeRequest(request);
     }
 
     @Override
@@ -91,7 +92,7 @@ protected ClusterStatsNodeResponse newNodeResponse() {
     }
 
     @Override
-    protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest) {
+    protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest, Task task) {
         NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, true, false, false);
         NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE,
                 true, true, true, false, true, false, false, false, false, false, false, false);
@@ -142,8 +143,7 @@ public static class ClusterStatsNodeRequest extends BaseNodeRequest {
         public ClusterStatsNodeRequest() {
         }
 
-        ClusterStatsNodeRequest(String nodeId, ClusterStatsRequest request) {
-            super(nodeId);
+        ClusterStatsNodeRequest(ClusterStatsRequest request) {
             this.request = request;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptAction.java
index 7ba8ee0754fed..35814f2221f76 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptAction.java
@@ -19,10 +19,11 @@
 
 package org.elasticsearch.action.admin.cluster.storedscripts;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class DeleteStoredScriptAction extends Action<AcknowledgedResponse> {
+public class DeleteStoredScriptAction extends ActionType<AcknowledgedResponse> {
 
     public static final DeleteStoredScriptAction INSTANCE = new DeleteStoredScriptAction();
     public static final String NAME = "cluster:admin/script/delete";
@@ -32,7 +33,7 @@ private DeleteStoredScriptAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptAction.java
index e8015a4487496..ed74c5b928e0e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.cluster.storedscripts;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class GetStoredScriptAction extends Action<GetStoredScriptResponse> {
+public class GetStoredScriptAction extends StreamableResponseActionType<GetStoredScriptResponse> {
 
     public static final GetStoredScriptAction INSTANCE = new GetStoredScriptAction();
     public static final String NAME = "cluster:admin/script/get";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptAction.java
index 8e5475283c2bc..6cbf57f3bd891 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptAction.java
@@ -19,23 +19,22 @@
 
 package org.elasticsearch.action.admin.cluster.storedscripts;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
 
-public class PutStoredScriptAction extends Action<AcknowledgedResponse> {
+public class PutStoredScriptAction extends ActionType<AcknowledgedResponse> {
 
     public static final PutStoredScriptAction INSTANCE = new PutStoredScriptAction();
     public static final String NAME = "cluster:admin/script/put";
 
-
     private PutStoredScriptAction() {
         super(NAME);
     }
 
-
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java
index 085850f118777..86fb16c63d4d6 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java
@@ -29,10 +29,14 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 public class TransportDeleteStoredScriptAction extends TransportMasterNodeAction<DeleteStoredScriptRequest, AcknowledgedResponse> {
 
     private final ScriptService scriptService;
@@ -51,13 +55,18 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(DeleteStoredScriptRequest request, ClusterState state,
+    protected void masterOperation(Task task, DeleteStoredScriptRequest request, ClusterState state,
                                    ActionListener<AcknowledgedResponse> listener) throws Exception {
         scriptService.deleteStoredScript(clusterService, request, listener);
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java
index 19e86d1b6722d..1e67e025b61b9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -57,7 +58,7 @@ protected GetStoredScriptResponse newResponse() {
     }
 
     @Override
-    protected void masterOperation(GetStoredScriptRequest request, ClusterState state,
+    protected void masterOperation(Task task, GetStoredScriptRequest request, ClusterState state,
                                    ActionListener<GetStoredScriptResponse> listener) throws Exception {
         listener.onResponse(new GetStoredScriptResponse(request.id(), scriptService.getStoredScript(state, request)));
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java
index 99fa3979d87d0..ceabd31c45604 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java
@@ -29,10 +29,14 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 public class TransportPutStoredScriptAction extends TransportMasterNodeAction<PutStoredScriptRequest, AcknowledgedResponse> {
 
     private final ScriptService scriptService;
@@ -51,13 +55,18 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(PutStoredScriptRequest request, ClusterState state,
+    protected void masterOperation(Task task, PutStoredScriptRequest request, ClusterState state,
                                    ActionListener<AcknowledgedResponse> listener) throws Exception {
         scriptService.putStoredScript(clusterService, request, listener);
     }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksAction.java
index 296c65146a03c..240575bb1da45 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.cluster.tasks;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class PendingClusterTasksAction extends Action<PendingClusterTasksResponse> {
+public class PendingClusterTasksAction extends StreamableResponseActionType<PendingClusterTasksResponse> {
 
     public static final PendingClusterTasksAction INSTANCE = new PendingClusterTasksAction();
     public static final String NAME = "cluster:monitor/task";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java
index cb061a25363cc..d62e7c13247bc 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.PendingClusterTask;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -65,7 +66,7 @@ protected PendingClusterTasksResponse newResponse() {
     }
 
     @Override
-    protected void masterOperation(PendingClusterTasksRequest request, ClusterState state,
+    protected void masterOperation(Task task, PendingClusterTasksRequest request, ClusterState state,
                                    ActionListener<PendingClusterTasksResponse> listener) {
         logger.trace("fetching pending tasks from cluster service");
         final List<PendingClusterTask> pendingTasks = clusterService.getMasterService().pendingTasks();
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesAction.java
index 7f7743f81730a..fe18bd9315459 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesAction.java
@@ -19,10 +19,11 @@
 
 package org.elasticsearch.action.admin.indices.alias;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class IndicesAliasesAction extends Action<AcknowledgedResponse> {
+public class IndicesAliasesAction extends ActionType<AcknowledgedResponse> {
 
     public static final IndicesAliasesAction INSTANCE = new IndicesAliasesAction();
     public static final String NAME = "indices:admin/aliases";
@@ -32,7 +33,7 @@ private IndicesAliasesAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
index 20b01989285ad..4fe3094f33cef 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.admin.indices.alias;
 
 import org.elasticsearch.ElasticsearchGenerationException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.AliasesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -62,6 +63,7 @@
 public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesRequest> implements ToXContentObject {
 
     private List<AliasActions> allAliasActions = new ArrayList<>();
+    private String origin = "";
 
     // indices options that require every specified index to exist, expand wildcards only to open
     // indices, don't allow that no indices are resolved from wildcard expressions and resolve the
@@ -526,6 +528,15 @@ public int hashCode() {
         }
     }
 
+    public String origin() {
+        return origin;
+    }
+
+    public IndicesAliasesRequest origin(final String origin) {
+        this.origin = Objects.requireNonNull(origin);
+        return this;
+    }
+
     /**
      * Add the action to this request and validate it.
      */
@@ -556,12 +567,23 @@ public ActionRequestValidationException validate() {
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         allAliasActions = in.readList(AliasActions::new);
+        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+            origin = in.readOptionalString();
+        } else {
+            origin = null;
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeList(allAliasActions);
+        // noinspection StatementWithEmptyBody
+        if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+            out.writeOptionalString(origin);
+        } else {
+            // nothing to do here, here for symmetry with IndicesAliasesRequest#readFrom
+        }
     }
 
     public IndicesOptions indicesOptions() {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
index 252a91a0d5b37..e4f460d4aa7c3 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
@@ -21,6 +21,7 @@
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.RequestValidators;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -37,14 +38,20 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 
 import static java.util.Collections.unmodifiableList;
@@ -55,14 +62,21 @@
 public class TransportIndicesAliasesAction extends TransportMasterNodeAction<IndicesAliasesRequest, AcknowledgedResponse> {
 
     private final MetaDataIndexAliasesService indexAliasesService;
+    private final RequestValidators<IndicesAliasesRequest> requestValidators;
 
     @Inject
-    public TransportIndicesAliasesAction(TransportService transportService, ClusterService clusterService,
-                                         ThreadPool threadPool, MetaDataIndexAliasesService indexAliasesService,
-                                         ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+    public TransportIndicesAliasesAction(
+            final TransportService transportService,
+            final ClusterService clusterService,
+            final ThreadPool threadPool,
+            final MetaDataIndexAliasesService indexAliasesService,
+            final ActionFilters actionFilters,
+            final IndexNameExpressionResolver indexNameExpressionResolver,
+            final RequestValidators<IndicesAliasesRequest> requestValidators) {
         super(IndicesAliasesAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,
             IndicesAliasesRequest::new);
         this.indexAliasesService = indexAliasesService;
+        this.requestValidators = Objects.requireNonNull(requestValidators);
     }
 
     @Override
@@ -71,9 +85,14 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
@@ -86,7 +105,7 @@ protected ClusterBlockException checkBlock(IndicesAliasesRequest request, Cluste
     }
 
     @Override
-    protected void masterOperation(final IndicesAliasesRequest request, final ClusterState state,
+    protected void masterOperation(Task task, final IndicesAliasesRequest request, final ClusterState state,
                                    final ActionListener<AcknowledgedResponse> listener) {
 
         //Expand the indices names
@@ -96,23 +115,28 @@ protected void masterOperation(final IndicesAliasesRequest request, final Cluste
         // Resolve all the AliasActions into AliasAction instances and gather all the aliases
         Set<String> aliases = new HashSet<>();
         for (AliasActions action : actions) {
-            String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request.indicesOptions(), action.indices());
+            final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request.indicesOptions(), action.indices());
+            final Optional<Exception> maybeException = requestValidators.validateRequest(request, state, concreteIndices);
+            if (maybeException.isPresent()) {
+                listener.onFailure(maybeException.get());
+                return;
+            }
             Collections.addAll(aliases, action.getOriginalAliases());
-            for (String index : concreteIndices) {
+            for (final Index index : concreteIndices) {
                 switch (action.actionType()) {
                 case ADD:
-                    for (String alias : concreteAliases(action, state.metaData(), index)) {
-                        finalActions.add(new AliasAction.Add(index, alias, action.filter(), action.indexRouting(),
+                    for (String alias : concreteAliases(action, state.metaData(), index.getName())) {
+                        finalActions.add(new AliasAction.Add(index.getName(), alias, action.filter(), action.indexRouting(),
                             action.searchRouting(), action.writeIndex()));
                     }
                     break;
                 case REMOVE:
-                    for (String alias : concreteAliases(action, state.metaData(), index)) {
-                        finalActions.add(new AliasAction.Remove(index, alias));
+                    for (String alias : concreteAliases(action, state.metaData(), index.getName())) {
+                        finalActions.add(new AliasAction.Remove(index.getName(), alias));
                     }
                     break;
                 case REMOVE_INDEX:
-                    finalActions.add(new AliasAction.RemoveIndex(index));
+                    finalActions.add(new AliasAction.RemoveIndex(index.getName()));
                     break;
                 default:
                     throw new IllegalArgumentException("Unsupported action [" + action.actionType() + "]");
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/TransportAliasesExistAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/TransportAliasesExistAction.java
deleted file mode 100644
index 858c4184bfcba..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/exists/TransportAliasesExistAction.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.action.admin.indices.alias.exists;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.block.ClusterBlockException;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-
-public class TransportAliasesExistAction extends TransportMasterNodeReadAction<GetAliasesRequest, AliasesExistResponse> {
-
-    @Inject
-    public TransportAliasesExistAction(TransportService transportService, ClusterService clusterService,
-                                       ThreadPool threadPool, ActionFilters actionFilters,
-                                       IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(AliasesExistAction.NAME, transportService, clusterService, threadPool, actionFilters, GetAliasesRequest::new,
-            indexNameExpressionResolver);
-    }
-
-    @Override
-    protected String executor() {
-        // very lightweight operation, no need to fork
-        return ThreadPool.Names.SAME;
-    }
-
-    @Override
-    protected ClusterBlockException checkBlock(GetAliasesRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ,
-            indexNameExpressionResolver.concreteIndexNames(state, request));
-    }
-
-    @Override
-    protected AliasesExistResponse newResponse() {
-        return new AliasesExistResponse();
-    }
-
-    @Override
-    protected void masterOperation(GetAliasesRequest request, ClusterState state, ActionListener<AliasesExistResponse> listener) {
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request);
-        boolean result = state.metaData().hasAliases(request.aliases(), concreteIndices);
-        listener.onResponse(new AliasesExistResponse(result));
-    }
-
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/BaseAliasesRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/BaseAliasesRequestBuilder.java
index ba8b45582ea5f..57aa9cb468e5e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/BaseAliasesRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/BaseAliasesRequestBuilder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.action.admin.indices.alias.get;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
@@ -30,7 +30,7 @@ public abstract class BaseAliasesRequestBuilder<Response extends ActionResponse,
                                                 Builder extends BaseAliasesRequestBuilder<Response, Builder>>
     extends MasterNodeReadOperationRequestBuilder<GetAliasesRequest, Response, Builder> {
 
-    public BaseAliasesRequestBuilder(ElasticsearchClient client, Action<Response> action, String... aliases) {
+    public BaseAliasesRequestBuilder(ElasticsearchClient client, ActionType<Response> action, String... aliases) {
         super(client, action, new GetAliasesRequest(aliases));
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesAction.java
index db423c2aaaa07..2798f21e820a4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.indices.alias.get;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class GetAliasesAction extends Action<GetAliasesResponse> {
+public class GetAliasesAction extends StreamableResponseActionType<GetAliasesResponse> {
 
     public static final GetAliasesAction INSTANCE = new GetAliasesAction();
     public static final String NAME = "indices:admin/aliases/get";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java
index d1a573e6da8d7..30de49031e101 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -63,7 +64,7 @@ protected GetAliasesResponse newResponse() {
     }
 
     @Override
-    protected void masterOperation(GetAliasesRequest request, ClusterState state, ActionListener<GetAliasesResponse> listener) {
+    protected void masterOperation(Task task, GetAliasesRequest request, ClusterState state, ActionListener<GetAliasesResponse> listener) {
         String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request);
         ImmutableOpenMap<String, List<AliasMetaData>> aliases = state.metaData().findAliases(request, concreteIndices);
         listener.onResponse(new GetAliasesResponse(postProcess(request, concreteIndices, aliases)));
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeAction.java
index 3677cd6cb4e43..59d882fe8e18f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeAction.java
@@ -19,10 +19,33 @@
 
 package org.elasticsearch.action.admin.indices.analyze;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.support.single.shard.SingleShardRequest;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.analysis.NameOrDefinition;
 
-public class AnalyzeAction extends Action<AnalyzeResponse> {
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+public class AnalyzeAction extends ActionType<AnalyzeAction.Response> {
 
     public static final AnalyzeAction INSTANCE = new AnalyzeAction();
     public static final String NAME = "indices:admin/analyze";
@@ -32,12 +55,754 @@ private AnalyzeAction() {
     }
 
     @Override
-    public Writeable.Reader<AnalyzeResponse> getResponseReader() {
-        return AnalyzeResponse::new;
+    public Writeable.Reader<Response> getResponseReader() {
+        return Response::new;
     }
 
-    @Override
-    public AnalyzeResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    /**
+     * A request to analyze a text associated with a specific index. Allow to provide
+     * the actual analyzer name to perform the analysis with.
+     */
+    public static class Request extends SingleShardRequest<Request> {
+
+        private String[] text;
+        private String analyzer;
+        private NameOrDefinition tokenizer;
+        private final List<NameOrDefinition> tokenFilters = new ArrayList<>();
+        private final List<NameOrDefinition> charFilters = new ArrayList<>();
+        private String field;
+        private boolean explain = false;
+        private String[] attributes = Strings.EMPTY_ARRAY;
+        private String normalizer;
+
+        public Request() {
+        }
+
+        Request(StreamInput in) throws IOException {
+            super(in);
+            text = in.readStringArray();
+            analyzer = in.readOptionalString();
+            tokenizer = in.readOptionalWriteable(NameOrDefinition::new);
+            tokenFilters.addAll(in.readList(NameOrDefinition::new));
+            charFilters.addAll(in.readList(NameOrDefinition::new));
+            field = in.readOptionalString();
+            explain = in.readBoolean();
+            attributes = in.readStringArray();
+            normalizer = in.readOptionalString();
+        }
+
+        /**
+         * Constructs a new analyzer request for the provided index.
+         *
+         * @param index The text to analyze
+         */
+        public Request(String index) {
+            this.index(index);
+        }
+
+        public String[] text() {
+            return this.text;
+        }
+
+        public Request text(String... text) {
+            this.text = text;
+            return this;
+        }
+
+        public Request text(List<String> text) {
+            this.text = text.toArray(new String[]{});
+            return this;
+        }
+
+        public Request analyzer(String analyzer) {
+            this.analyzer = analyzer;
+            return this;
+        }
+
+        public String analyzer() {
+            return this.analyzer;
+        }
+
+        public Request tokenizer(String tokenizer) {
+            this.tokenizer = new NameOrDefinition(tokenizer);
+            return this;
+        }
+
+        public Request tokenizer(Map<String, ?> tokenizer) {
+            this.tokenizer = new NameOrDefinition(tokenizer);
+            return this;
+        }
+
+        public void tokenizer(NameOrDefinition tokenizer) {
+            this.tokenizer = tokenizer;
+        }
+
+        public NameOrDefinition tokenizer() {
+            return this.tokenizer;
+        }
+
+        public Request addTokenFilter(String tokenFilter) {
+            this.tokenFilters.add(new NameOrDefinition(tokenFilter));
+            return this;
+        }
+
+        public Request addTokenFilter(Map<String, ?> tokenFilter) {
+            this.tokenFilters.add(new NameOrDefinition(tokenFilter));
+            return this;
+        }
+
+        public void setTokenFilters(List<NameOrDefinition> tokenFilters) {
+            this.tokenFilters.addAll(tokenFilters);
+        }
+
+        public List<NameOrDefinition> tokenFilters() {
+            return this.tokenFilters;
+        }
+
+        public Request addCharFilter(Map<String, ?> charFilter) {
+            this.charFilters.add(new NameOrDefinition(charFilter));
+            return this;
+        }
+
+        public Request addCharFilter(String charFilter) {
+            this.charFilters.add(new NameOrDefinition(charFilter));
+            return this;
+        }
+
+        public void setCharFilters(List<NameOrDefinition> charFilters) {
+            this.charFilters.addAll(charFilters);
+        }
+
+        public List<NameOrDefinition> charFilters() {
+            return this.charFilters;
+        }
+
+        public Request field(String field) {
+            this.field = field;
+            return this;
+        }
+
+        public String field() {
+            return this.field;
+        }
+
+        public Request explain(boolean explain) {
+            this.explain = explain;
+            return this;
+        }
+
+        public boolean explain() {
+            return this.explain;
+        }
+
+        public Request attributes(String... attributes) {
+            if (attributes == null) {
+                throw new IllegalArgumentException("attributes must not be null");
+            }
+            this.attributes = attributes;
+            return this;
+        }
+
+        public void attributes(List<String> attributes) {
+            this.attributes = attributes.toArray(new String[]{});
+        }
+
+        public String[] attributes() {
+            return this.attributes;
+        }
+
+        public String normalizer() {
+            return this.normalizer;
+        }
+
+        public Request normalizer(String normalizer) {
+            this.normalizer = normalizer;
+            return this;
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            ActionRequestValidationException validationException = null;
+            if (text == null || text.length == 0) {
+                validationException = addValidationError("text is missing", validationException);
+            }
+            if ((index == null || index.length() == 0) && normalizer != null) {
+                validationException = addValidationError("index is required if normalizer is specified", validationException);
+            }
+            if (normalizer != null && (tokenizer != null || analyzer != null)) {
+                validationException
+                    = addValidationError("tokenizer/analyze should be null if normalizer is specified", validationException);
+            }
+            if (analyzer != null && (tokenizer != null || charFilters.isEmpty() == false || tokenFilters.isEmpty() == false)) {
+                validationException
+                    = addValidationError("cannot define extra components on a named analyzer", validationException);
+            }
+            if (normalizer != null && (tokenizer != null || charFilters.isEmpty() == false || tokenFilters.isEmpty() == false)) {
+                validationException
+                    = addValidationError("cannot define extra components on a named normalizer", validationException);
+            }
+            if (field != null && (tokenizer != null || charFilters.isEmpty() == false || tokenFilters.isEmpty() == false)) {
+                validationException
+                    = addValidationError("cannot define extra components on a field-specific analyzer", validationException);
+            }
+            return validationException;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeStringArray(text);
+            out.writeOptionalString(analyzer);
+            out.writeOptionalWriteable(tokenizer);
+            out.writeList(tokenFilters);
+            out.writeList(charFilters);
+            out.writeOptionalString(field);
+            out.writeBoolean(explain);
+            out.writeStringArray(attributes);
+            out.writeOptionalString(normalizer);
+        }
+
+        public static Request fromXContent(XContentParser parser, String index) throws IOException {
+            Request request = new Request(index);
+            PARSER.parse(parser, request, null);
+            return request;
+        }
+
+        private static final ObjectParser<Request, Void> PARSER = new ObjectParser<>("analyze_request", null);
+        static {
+            PARSER.declareStringArray(Request::text, new ParseField("text"));
+            PARSER.declareString(Request::analyzer, new ParseField("analyzer"));
+            PARSER.declareField(Request::tokenizer, (p, c) -> NameOrDefinition.fromXContent(p),
+                new ParseField("tokenizer"), ObjectParser.ValueType.OBJECT_OR_STRING);
+            PARSER.declareObjectArray(Request::setTokenFilters, (p, c) -> NameOrDefinition.fromXContent(p),
+                new ParseField("filter"));
+            PARSER.declareObjectArray(Request::setCharFilters, (p, c) -> NameOrDefinition.fromXContent(p),
+                new ParseField("char_filter"));
+            PARSER.declareString(Request::field, new ParseField("field"));
+            PARSER.declareBoolean(Request::explain, new ParseField("explain"));
+            PARSER.declareStringArray(Request::attributes, new ParseField("attributes"));
+            PARSER.declareString(Request::normalizer, new ParseField("normalizer"));
+        }
+
+    }
+
+    public static class Response extends ActionResponse implements ToXContentObject {
+
+        private final DetailAnalyzeResponse detail;
+        private final List<AnalyzeToken> tokens;
+
+        public Response(List<AnalyzeToken> tokens, DetailAnalyzeResponse detail) {
+            this.tokens = tokens;
+            this.detail = detail;
+        }
+
+        public Response(StreamInput in) throws IOException {
+            super.readFrom(in);
+            int size = in.readVInt();
+            if (size > 0) {
+                tokens = new ArrayList<>(size);
+                for (int i = 0; i < size; i++) {
+                    tokens.add(new AnalyzeToken(in));
+                }
+            }
+            else {
+                tokens = null;
+            }
+            detail = in.readOptionalWriteable(DetailAnalyzeResponse::new);
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+        }
+
+        public List<AnalyzeToken> getTokens() {
+            return this.tokens;
+        }
+
+        public DetailAnalyzeResponse detail() {
+            return this.detail;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            if (tokens != null) {
+                builder.startArray(Fields.TOKENS);
+                for (AnalyzeToken token : tokens) {
+                    token.toXContent(builder, params);
+                }
+                builder.endArray();
+            }
+
+            if (detail != null) {
+                builder.startObject(Fields.DETAIL);
+                detail.toXContent(builder, params);
+                builder.endObject();
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            if (tokens != null) {
+                out.writeVInt(tokens.size());
+                for (AnalyzeToken token : tokens) {
+                    token.writeTo(out);
+                }
+            } else {
+                out.writeVInt(0);
+            }
+            out.writeOptionalWriteable(detail);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Response that = (Response) o;
+            return Objects.equals(detail, that.detail) &&
+                Objects.equals(tokens, that.tokens);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(detail, tokens);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this, true, true);
+        }
+
+        static final class Fields {
+            static final String TOKENS = "tokens";
+
+            static final String DETAIL = "detail";
+        }
     }
+
+    public static class AnalyzeToken implements Writeable, ToXContentObject {
+        private final String term;
+        private final int startOffset;
+        private final int endOffset;
+        private final int position;
+        private final int positionLength;
+        private final Map<String, Object> attributes;
+        private final String type;
+
+        static final String TOKEN = "token";
+        static final String START_OFFSET = "start_offset";
+        static final String END_OFFSET = "end_offset";
+        static final String TYPE = "type";
+        static final String POSITION = "position";
+        static final String POSITION_LENGTH = "positionLength";
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            AnalyzeToken that = (AnalyzeToken) o;
+            return startOffset == that.startOffset &&
+                endOffset == that.endOffset &&
+                position == that.position &&
+                positionLength == that.positionLength &&
+                Objects.equals(term, that.term) &&
+                Objects.equals(attributes, that.attributes) &&
+                Objects.equals(type, that.type);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(term, startOffset, endOffset, position, positionLength, attributes, type);
+        }
+
+        public AnalyzeToken(String term, int position, int startOffset, int endOffset, int positionLength,
+                            String type, Map<String, Object> attributes) {
+            this.term = term;
+            this.position = position;
+            this.startOffset = startOffset;
+            this.endOffset = endOffset;
+            this.positionLength = positionLength;
+            this.type = type;
+            this.attributes = attributes;
+        }
+
+        AnalyzeToken(StreamInput in) throws IOException {
+            term = in.readString();
+            startOffset = in.readInt();
+            endOffset = in.readInt();
+            position = in.readVInt();
+            Integer len = in.readOptionalVInt();
+            if (len != null) {
+                positionLength = len;
+            } else {
+                positionLength = 1;
+            }
+            type = in.readOptionalString();
+            attributes = in.readMap();
+        }
+
+        public String getTerm() {
+            return this.term;
+        }
+
+        public int getStartOffset() {
+            return this.startOffset;
+        }
+
+        public int getEndOffset() {
+            return this.endOffset;
+        }
+
+        public int getPosition() {
+            return this.position;
+        }
+
+        public int getPositionLength() {
+            return this.positionLength;
+        }
+
+        public String getType() {
+            return this.type;
+        }
+
+        public Map<String, Object> getAttributes(){
+            return this.attributes;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(TOKEN, term);
+            builder.field(START_OFFSET, startOffset);
+            builder.field(END_OFFSET, endOffset);
+            builder.field(TYPE, type);
+            builder.field(POSITION, position);
+            if (positionLength > 1) {
+                builder.field(POSITION_LENGTH, positionLength);
+            }
+            if (attributes != null && !attributes.isEmpty()) {
+                Map<String, Object> sortedAttributes = new TreeMap<>(attributes);
+                for (Map.Entry<String, Object> entity : sortedAttributes.entrySet()) {
+                    builder.field(entity.getKey(), entity.getValue());
+                }
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(term);
+            out.writeInt(startOffset);
+            out.writeInt(endOffset);
+            out.writeVInt(position);
+            out.writeOptionalVInt(positionLength > 1 ? positionLength : null);
+            out.writeOptionalString(type);
+            out.writeMapWithConsistentOrder(attributes);
+        }
+    }
+
+    public static class DetailAnalyzeResponse implements Writeable, ToXContentFragment {
+
+        private final boolean customAnalyzer;
+        private final AnalyzeTokenList analyzer;
+        private final CharFilteredText[] charfilters;
+        private final AnalyzeTokenList tokenizer;
+        private final AnalyzeTokenList[] tokenfilters;
+
+        public DetailAnalyzeResponse(AnalyzeTokenList analyzer) {
+            this(false, analyzer, null, null, null);
+        }
+
+        public DetailAnalyzeResponse(CharFilteredText[] charfilters, AnalyzeTokenList tokenizer, AnalyzeTokenList[] tokenfilters) {
+            this(true, null, charfilters, tokenizer, tokenfilters);
+        }
+
+        DetailAnalyzeResponse(boolean customAnalyzer,
+                                     AnalyzeTokenList analyzer,
+                                     CharFilteredText[] charfilters,
+                                     AnalyzeTokenList tokenizer,
+                                     AnalyzeTokenList[] tokenfilters) {
+            this.customAnalyzer = customAnalyzer;
+            this.analyzer = analyzer;
+            this.charfilters = charfilters;
+            this.tokenizer = tokenizer;
+            this.tokenfilters = tokenfilters;
+        }
+
+        DetailAnalyzeResponse(StreamInput in) throws IOException {
+            this.customAnalyzer = in.readBoolean();
+            if (customAnalyzer) {
+                tokenizer = new AnalyzeTokenList(in);
+                int size = in.readVInt();
+                if (size > 0) {
+                    charfilters = new CharFilteredText[size];
+                    for (int i = 0; i < size; i++) {
+                        charfilters[i] = new CharFilteredText(in);
+                    }
+                } else {
+                    charfilters = null;
+                }
+                size = in.readVInt();
+                if (size > 0) {
+                    tokenfilters = new AnalyzeTokenList[size];
+                    for (int i = 0; i < size; i++) {
+                        tokenfilters[i] = new AnalyzeTokenList(in);
+                    }
+                } else {
+                    tokenfilters = null;
+                }
+                analyzer = null;
+            } else {
+                analyzer = new AnalyzeTokenList(in);
+                tokenfilters = null;
+                tokenizer = null;
+                charfilters = null;
+            }
+        }
+
+        public AnalyzeTokenList analyzer() {
+            return this.analyzer;
+        }
+
+        public CharFilteredText[] charfilters() {
+            return this.charfilters;
+        }
+
+        public AnalyzeTokenList tokenizer() {
+            return tokenizer;
+        }
+
+        public AnalyzeTokenList[] tokenfilters() {
+            return tokenfilters;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            DetailAnalyzeResponse that = (DetailAnalyzeResponse) o;
+            return customAnalyzer == that.customAnalyzer &&
+                Objects.equals(analyzer, that.analyzer) &&
+                Arrays.equals(charfilters, that.charfilters) &&
+                Objects.equals(tokenizer, that.tokenizer) &&
+                Arrays.equals(tokenfilters, that.tokenfilters);
+        }
+
+        @Override
+        public int hashCode() {
+            int result = Objects.hash(customAnalyzer, analyzer, tokenizer);
+            result = 31 * result + Arrays.hashCode(charfilters);
+            result = 31 * result + Arrays.hashCode(tokenfilters);
+            return result;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field("custom_analyzer", customAnalyzer);
+
+            if (analyzer != null) {
+                builder.startObject("analyzer");
+                analyzer.toXContentWithoutObject(builder, params);
+                builder.endObject();
+            }
+
+            if (charfilters != null) {
+                builder.startArray("charfilters");
+                for (CharFilteredText charfilter : charfilters) {
+                    charfilter.toXContent(builder, params);
+                }
+                builder.endArray();
+            }
+
+            if (tokenizer != null) {
+                builder.startObject("tokenizer");
+                tokenizer.toXContentWithoutObject(builder, params);
+                builder.endObject();
+            }
+
+            if (tokenfilters != null) {
+                builder.startArray("tokenfilters");
+                for (AnalyzeTokenList tokenfilter : tokenfilters) {
+                    tokenfilter.toXContent(builder, params);
+                }
+                builder.endArray();
+            }
+            return builder;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeBoolean(customAnalyzer);
+            if (customAnalyzer) {
+                tokenizer.writeTo(out);
+                if (charfilters != null) {
+                    out.writeVInt(charfilters.length);
+                    for (CharFilteredText charfilter : charfilters) {
+                        charfilter.writeTo(out);
+                    }
+                } else {
+                    out.writeVInt(0);
+                }
+                if (tokenfilters != null) {
+                    out.writeVInt(tokenfilters.length);
+                    for (AnalyzeTokenList tokenfilter : tokenfilters) {
+                        tokenfilter.writeTo(out);
+                    }
+                } else {
+                    out.writeVInt(0);
+                }
+            } else {
+                analyzer.writeTo(out);
+            }
+        }
+    }
+
+    public static class AnalyzeTokenList implements Writeable, ToXContentObject {
+        private final String name;
+        private final AnalyzeToken[] tokens;
+
+        static final String NAME = "name";
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            AnalyzeTokenList that = (AnalyzeTokenList) o;
+            return Objects.equals(name, that.name) &&
+                Arrays.equals(tokens, that.tokens);
+        }
+
+        @Override
+        public int hashCode() {
+            int result = Objects.hash(name);
+            result = 31 * result + Arrays.hashCode(tokens);
+            return result;
+        }
+
+        public AnalyzeTokenList(String name, AnalyzeToken[] tokens) {
+            this.name = name;
+            this.tokens = tokens;
+        }
+
+        AnalyzeTokenList(StreamInput in) throws IOException {
+            name = in.readString();
+            int size = in.readVInt();
+            if (size > 0) {
+                tokens = new AnalyzeToken[size];
+                for (int i = 0; i < size; i++) {
+                    tokens[i] = new AnalyzeToken(in);
+                }
+            }
+            else {
+                tokens = null;
+            }
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public AnalyzeToken[] getTokens() {
+            return tokens;
+        }
+
+        void toXContentWithoutObject(XContentBuilder builder, Params params) throws IOException {
+            builder.field(NAME, this.name);
+            builder.startArray(Response.Fields.TOKENS);
+            if (tokens != null) {
+                for (AnalyzeToken token : tokens) {
+                    token.toXContent(builder, params);
+                }
+            }
+            builder.endArray();
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            toXContentWithoutObject(builder, params);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(name);
+            if (tokens != null) {
+                out.writeVInt(tokens.length);
+                for (AnalyzeToken token : tokens) {
+                    token.writeTo(out);
+                }
+            } else {
+                out.writeVInt(0);
+            }
+        }
+    }
+
+    public static class CharFilteredText implements Writeable, ToXContentObject {
+        private final String name;
+        private final String[] texts;
+
+        static final String NAME = "name";
+        static final String FILTERED_TEXT = "filtered_text";
+
+        public CharFilteredText(String name, String[] texts) {
+            this.name = name;
+            if (texts != null) {
+                this.texts = texts;
+            } else {
+                this.texts = Strings.EMPTY_ARRAY;
+            }
+        }
+
+        CharFilteredText(StreamInput in) throws IOException {
+            name = in.readString();
+            texts = in.readStringArray();
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public String[] getTexts() {
+            return texts;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(NAME, name);
+            builder.array(FILTERED_TEXT, texts);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(name);
+            out.writeStringArray(texts);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            CharFilteredText that = (CharFilteredText) o;
+            return Objects.equals(name, that.name) &&
+                Arrays.equals(texts, that.texts);
+        }
+
+        @Override
+        public int hashCode() {
+            int result = Objects.hash(name);
+            result = 31 * result + Arrays.hashCode(texts);
+            return result;
+        }
+    }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java
deleted file mode 100644
index a2712c2d4c107..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.action.admin.indices.analyze;
-
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.support.single.shard.SingleShardRequest;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContentFragment;
-import org.elasticsearch.common.xcontent.ToXContentObject;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentType;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-import static org.elasticsearch.action.ValidateActions.addValidationError;
-
-/**
- * A request to analyze a text associated with a specific index. Allow to provide
- * the actual analyzer name to perform the analysis with.
- */
-public class AnalyzeRequest extends SingleShardRequest<AnalyzeRequest> implements ToXContentObject {
-
-    private String[] text;
-
-    private String analyzer;
-
-    private NameOrDefinition tokenizer;
-
-    private final List<NameOrDefinition> tokenFilters = new ArrayList<>();
-
-    private final List<NameOrDefinition> charFilters = new ArrayList<>();
-
-    private String field;
-
-    private boolean explain = false;
-
-    private String[] attributes = Strings.EMPTY_ARRAY;
-
-    private String normalizer;
-
-    public static class NameOrDefinition implements Writeable, ToXContentFragment {
-        // exactly one of these two members is not null
-        public final String name;
-        public final Settings definition;
-
-        NameOrDefinition(String name) {
-            this.name = Objects.requireNonNull(name);
-            this.definition = null;
-        }
-
-        NameOrDefinition(Map<String, ?> definition) {
-            this.name = null;
-            Objects.requireNonNull(definition);
-            try {
-                XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-                builder.map(definition);
-                this.definition = Settings.builder().loadFromSource(Strings.toString(builder), builder.contentType()).build();
-            } catch (IOException e) {
-                throw new IllegalArgumentException("Failed to parse [" + definition + "]", e);
-            }
-        }
-
-        NameOrDefinition(StreamInput in) throws IOException {
-            name = in.readOptionalString();
-            if (in.readBoolean()) {
-                definition = Settings.readSettingsFromStream(in);
-            } else {
-                definition = null;
-            }
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeOptionalString(name);
-            boolean isNotNullDefinition = this.definition != null;
-            out.writeBoolean(isNotNullDefinition);
-            if (isNotNullDefinition) {
-                Settings.writeSettingsToStream(definition, out);
-            }
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            if (definition == null) {
-                return builder.value(name);
-            }
-            return definition.toXContent(builder, params);
-        }
-
-    }
-
-    public AnalyzeRequest() {
-    }
-
-    /**
-     * Constructs a new analyzer request for the provided index.
-     *
-     * @param index The text to analyze
-     */
-    public AnalyzeRequest(String index) {
-        this.index(index);
-    }
-
-    public String[] text() {
-        return this.text;
-    }
-
-    public AnalyzeRequest text(String... text) {
-        this.text = text;
-        return this;
-    }
-
-    public AnalyzeRequest analyzer(String analyzer) {
-        this.analyzer = analyzer;
-        return this;
-    }
-
-    public String analyzer() {
-        return this.analyzer;
-    }
-
-    public AnalyzeRequest tokenizer(String tokenizer) {
-        this.tokenizer = new NameOrDefinition(tokenizer);
-        return this;
-    }
-
-    public AnalyzeRequest tokenizer(Map<String, ?> tokenizer) {
-        this.tokenizer = new NameOrDefinition(tokenizer);
-        return this;
-    }
-
-    public NameOrDefinition tokenizer() {
-        return this.tokenizer;
-    }
-
-    public AnalyzeRequest addTokenFilter(String tokenFilter) {
-        this.tokenFilters.add(new NameOrDefinition(tokenFilter));
-        return this;
-    }
-
-    public AnalyzeRequest addTokenFilter(Map<String, ?> tokenFilter) {
-        this.tokenFilters.add(new NameOrDefinition(tokenFilter));
-        return this;
-    }
-
-    public List<NameOrDefinition> tokenFilters() {
-        return this.tokenFilters;
-    }
-
-    public AnalyzeRequest addCharFilter(Map<String, ?> charFilter) {
-        this.charFilters.add(new NameOrDefinition(charFilter));
-        return this;
-    }
-
-    public AnalyzeRequest addCharFilter(String charFilter) {
-        this.charFilters.add(new NameOrDefinition(charFilter));
-        return this;
-    }
-
-    public List<NameOrDefinition> charFilters() {
-        return this.charFilters;
-    }
-
-    public AnalyzeRequest field(String field) {
-        this.field = field;
-        return this;
-    }
-
-    public String field() {
-        return this.field;
-    }
-
-    public AnalyzeRequest explain(boolean explain) {
-        this.explain = explain;
-        return this;
-    }
-
-    public boolean explain() {
-        return this.explain;
-    }
-
-    public AnalyzeRequest attributes(String... attributes) {
-        if (attributes == null) {
-            throw new IllegalArgumentException("attributes must not be null");
-        }
-        this.attributes = attributes;
-        return this;
-    }
-
-    public String[] attributes() {
-        return this.attributes;
-    }
-
-    public String normalizer() {
-        return this.normalizer;
-    }
-
-    public AnalyzeRequest normalizer(String normalizer) {
-        this.normalizer = normalizer;
-        return this;
-    }
-
-    @Override
-    public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = null;
-        if (text == null || text.length == 0) {
-            validationException = addValidationError("text is missing", validationException);
-        }
-        if ((index == null || index.length() == 0) && normalizer != null) {
-            validationException = addValidationError("index is required if normalizer is specified", validationException);
-        }
-        if (normalizer != null && (tokenizer != null || analyzer != null)) {
-            validationException = addValidationError("tokenizer/analyze should be null if normalizer is specified", validationException);
-        }
-        return validationException;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        text = in.readStringArray();
-        analyzer = in.readOptionalString();
-        tokenizer = in.readOptionalWriteable(NameOrDefinition::new);
-        tokenFilters.addAll(in.readList(NameOrDefinition::new));
-        charFilters.addAll(in.readList(NameOrDefinition::new));
-        field = in.readOptionalString();
-        explain = in.readBoolean();
-        attributes = in.readStringArray();
-        normalizer = in.readOptionalString();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeStringArray(text);
-        out.writeOptionalString(analyzer);
-        out.writeOptionalWriteable(tokenizer);
-        out.writeList(tokenFilters);
-        out.writeList(charFilters);
-        out.writeOptionalString(field);
-        out.writeBoolean(explain);
-        out.writeStringArray(attributes);
-        out.writeOptionalString(normalizer);
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        builder.field("text", text);
-        if (Strings.isNullOrEmpty(analyzer) == false) {
-            builder.field("analyzer", analyzer);
-        }
-        if (tokenizer != null) {
-            tokenizer.toXContent(builder, params);
-        }
-        if (tokenFilters.size() > 0) {
-            builder.field("filter", tokenFilters);
-        }
-        if (charFilters.size() > 0) {
-            builder.field("char_filter", charFilters);
-        }
-        if (Strings.isNullOrEmpty(field) == false) {
-            builder.field("field", field);
-        }
-        if (explain) {
-            builder.field("explain", true);
-        }
-        if (attributes.length > 0) {
-            builder.field("attributes", attributes);
-        }
-        if (Strings.isNullOrEmpty(normalizer) == false) {
-            builder.field("normalizer", normalizer);
-        }
-        return builder.endObject();
-    }
-
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java
index 3893cb25d9dbb..2bd1724c5e69f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java
@@ -23,14 +23,15 @@
 
 import java.util.Map;
 
-public class AnalyzeRequestBuilder extends SingleShardOperationRequestBuilder<AnalyzeRequest, AnalyzeResponse, AnalyzeRequestBuilder> {
+public class AnalyzeRequestBuilder
+    extends SingleShardOperationRequestBuilder<AnalyzeAction.Request, AnalyzeAction.Response, AnalyzeRequestBuilder> {
 
     public AnalyzeRequestBuilder(ElasticsearchClient client, AnalyzeAction action) {
-        super(client, action, new AnalyzeRequest());
+        super(client, action, new AnalyzeAction.Request());
     }
 
     public AnalyzeRequestBuilder(ElasticsearchClient client, AnalyzeAction action, String index, String... text) {
-        super(client, action, new AnalyzeRequest(index).text(text));
+        super(client, action, new AnalyzeAction.Request(index).text(text));
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java
deleted file mode 100644
index 7e6d525cefb93..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java
+++ /dev/null
@@ -1,320 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.action.admin.indices.analyze;
-
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ConstructingObjectParser;
-import org.elasticsearch.common.xcontent.ToXContentObject;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.TreeMap;
-
-import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
-import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
-
-public class AnalyzeResponse extends ActionResponse implements Iterable<AnalyzeResponse.AnalyzeToken>, ToXContentObject {
-
-    public static class AnalyzeToken implements Writeable, ToXContentObject {
-        private final String term;
-        private final int startOffset;
-        private final int endOffset;
-        private final int position;
-        private final int positionLength;
-        private final Map<String, Object> attributes;
-        private final String type;
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            AnalyzeToken that = (AnalyzeToken) o;
-            return startOffset == that.startOffset &&
-                endOffset == that.endOffset &&
-                position == that.position &&
-                positionLength == that.positionLength &&
-                Objects.equals(term, that.term) &&
-                Objects.equals(attributes, that.attributes) &&
-                Objects.equals(type, that.type);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(term, startOffset, endOffset, position, positionLength, attributes, type);
-        }
-
-        AnalyzeToken(String term, int position, int startOffset, int endOffset, int positionLength,
-                            String type, Map<String, Object> attributes) {
-            this.term = term;
-            this.position = position;
-            this.startOffset = startOffset;
-            this.endOffset = endOffset;
-            this.positionLength = positionLength;
-            this.type = type;
-            this.attributes = attributes;
-        }
-
-        AnalyzeToken(StreamInput in) throws IOException {
-            term = in.readString();
-            startOffset = in.readInt();
-            endOffset = in.readInt();
-            position = in.readVInt();
-            Integer len = in.readOptionalVInt();
-            if (len != null) {
-                positionLength = len;
-            } else {
-                positionLength = 1;
-            }
-            type = in.readOptionalString();
-            attributes = in.readMap();
-        }
-
-        public String getTerm() {
-            return this.term;
-        }
-
-        public int getStartOffset() {
-            return this.startOffset;
-        }
-
-        public int getEndOffset() {
-            return this.endOffset;
-        }
-
-        public int getPosition() {
-            return this.position;
-        }
-
-        public int getPositionLength() {
-            return this.positionLength;
-        }
-
-        public String getType() {
-            return this.type;
-        }
-
-        public Map<String, Object> getAttributes(){
-            return this.attributes;
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            builder.field(Fields.TOKEN, term);
-            builder.field(Fields.START_OFFSET, startOffset);
-            builder.field(Fields.END_OFFSET, endOffset);
-            builder.field(Fields.TYPE, type);
-            builder.field(Fields.POSITION, position);
-            if (positionLength > 1) {
-                builder.field(Fields.POSITION_LENGTH, positionLength);
-            }
-            if (attributes != null && !attributes.isEmpty()) {
-                Map<String, Object> sortedAttributes = new TreeMap<>(attributes);
-                for (Map.Entry<String, Object> entity : sortedAttributes.entrySet()) {
-                    builder.field(entity.getKey(), entity.getValue());
-                }
-            }
-            builder.endObject();
-            return builder;
-        }
-
-        public static AnalyzeToken fromXContent(XContentParser parser) throws IOException {
-            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
-            String field = null;
-            String term = "";
-            int position = -1;
-            int startOffset = -1;
-            int endOffset = -1;
-            int positionLength = 1;
-            String type = "";
-            Map<String, Object> attributes = new HashMap<>();
-            for (XContentParser.Token t = parser.nextToken(); t != XContentParser.Token.END_OBJECT; t = parser.nextToken()) {
-                if (t == XContentParser.Token.FIELD_NAME) {
-                    field = parser.currentName();
-                    continue;
-                }
-                if (Fields.TOKEN.equals(field)) {
-                    term = parser.text();
-                } else if (Fields.POSITION.equals(field)) {
-                    position = parser.intValue();
-                } else if (Fields.START_OFFSET.equals(field)) {
-                    startOffset = parser.intValue();
-                } else if (Fields.END_OFFSET.equals(field)) {
-                    endOffset = parser.intValue();
-                } else if (Fields.POSITION_LENGTH.equals(field)) {
-                    positionLength = parser.intValue();
-                } else if (Fields.TYPE.equals(field)) {
-                    type = parser.text();
-                } else {
-                    if (t == XContentParser.Token.VALUE_STRING) {
-                        attributes.put(field, parser.text());
-                    } else if (t == XContentParser.Token.VALUE_NUMBER) {
-                        attributes.put(field, parser.numberValue());
-                    } else if (t == XContentParser.Token.VALUE_BOOLEAN) {
-                        attributes.put(field, parser.booleanValue());
-                    } else if (t == XContentParser.Token.START_OBJECT) {
-                        attributes.put(field, parser.map());
-                    } else if (t == XContentParser.Token.START_ARRAY) {
-                        attributes.put(field, parser.list());
-                    }
-                }
-            }
-            return new AnalyzeToken(term, position, startOffset, endOffset, positionLength, type, attributes);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeString(term);
-            out.writeInt(startOffset);
-            out.writeInt(endOffset);
-            out.writeVInt(position);
-            out.writeOptionalVInt(positionLength > 1 ? positionLength : null);
-            out.writeOptionalString(type);
-            out.writeMapWithConsistentOrder(attributes);
-        }
-    }
-
-    private final DetailAnalyzeResponse detail;
-    private final List<AnalyzeToken> tokens;
-
-    public AnalyzeResponse(List<AnalyzeToken> tokens, DetailAnalyzeResponse detail) {
-        this.tokens = tokens;
-        this.detail = detail;
-    }
-
-    public AnalyzeResponse(StreamInput in) throws IOException {
-        super.readFrom(in);
-        int size = in.readVInt();
-        if (size > 0) {
-            tokens = new ArrayList<>(size);
-            for (int i = 0; i < size; i++) {
-                tokens.add(new AnalyzeToken(in));
-            }
-        }
-        else {
-            tokens = null;
-        }
-        detail = in.readOptionalWriteable(DetailAnalyzeResponse::new);
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
-    public List<AnalyzeToken> getTokens() {
-        return this.tokens;
-    }
-
-    public DetailAnalyzeResponse detail() {
-        return this.detail;
-    }
-
-    @Override
-    public Iterator<AnalyzeToken> iterator() {
-        return tokens.iterator();
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        if (tokens != null) {
-            builder.startArray(Fields.TOKENS);
-            for (AnalyzeToken token : tokens) {
-                token.toXContent(builder, params);
-            }
-            builder.endArray();
-        }
-
-        if (detail != null) {
-            builder.startObject(Fields.DETAIL);
-            detail.toXContent(builder, params);
-            builder.endObject();
-        }
-        builder.endObject();
-        return builder;
-    }
-
-    private static final ConstructingObjectParser<AnalyzeResponse, Void> PARSER = new ConstructingObjectParser<>("analyze_response",
-        true, args -> new AnalyzeResponse((List<AnalyzeToken>) args[0], (DetailAnalyzeResponse) args[1]));
-    static {
-        PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> AnalyzeToken.fromXContent(p), new ParseField(Fields.TOKENS));
-        PARSER.declareObject(optionalConstructorArg(), DetailAnalyzeResponse.PARSER, new ParseField(Fields.DETAIL));
-    }
-
-    public static AnalyzeResponse fromXContent(XContentParser parser) throws IOException {
-        return PARSER.parse(parser, null);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        if (tokens != null) {
-            out.writeVInt(tokens.size());
-            for (AnalyzeToken token : tokens) {
-                token.writeTo(out);
-            }
-        } else {
-            out.writeVInt(0);
-        }
-        out.writeOptionalWriteable(detail);
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        AnalyzeResponse that = (AnalyzeResponse) o;
-        return Objects.equals(detail, that.detail) &&
-            Objects.equals(tokens, that.tokens);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(detail, tokens);
-    }
-
-    @Override
-    public String toString() {
-        return Strings.toString(this, true, true);
-    }
-
-    static final class Fields {
-        static final String TOKENS = "tokens";
-        static final String TOKEN = "token";
-        static final String START_OFFSET = "start_offset";
-        static final String END_OFFSET = "end_offset";
-        static final String TYPE = "type";
-        static final String POSITION = "position";
-        static final String POSITION_LENGTH = "positionLength";
-        static final String DETAIL = "detail";
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java
deleted file mode 100644
index 1e84d9e0a2e1a..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java
+++ /dev/null
@@ -1,400 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.admin.indices.analyze;
-
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ConstructingObjectParser;
-import org.elasticsearch.common.xcontent.ToXContentFragment;
-import org.elasticsearch.common.xcontent.ToXContentObject;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.lang.reflect.Array;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Objects;
-
-import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
-import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
-
-public class DetailAnalyzeResponse implements Writeable, ToXContentFragment {
-
-    private final boolean customAnalyzer;
-    private final AnalyzeTokenList analyzer;
-    private final CharFilteredText[] charfilters;
-    private final AnalyzeTokenList tokenizer;
-    private final AnalyzeTokenList[] tokenfilters;
-
-    public DetailAnalyzeResponse(AnalyzeTokenList analyzer) {
-        this(false, analyzer, null, null, null);
-    }
-
-    public DetailAnalyzeResponse(CharFilteredText[] charfilters, AnalyzeTokenList tokenizer, AnalyzeTokenList[] tokenfilters) {
-        this(true, null, charfilters, tokenizer, tokenfilters);
-    }
-
-    public DetailAnalyzeResponse(boolean customAnalyzer,
-                                 AnalyzeTokenList analyzer,
-                                 CharFilteredText[] charfilters,
-                                 AnalyzeTokenList tokenizer,
-                                 AnalyzeTokenList[] tokenfilters) {
-        this.customAnalyzer = customAnalyzer;
-        this.analyzer = analyzer;
-        this.charfilters = charfilters;
-        this.tokenizer = tokenizer;
-        this.tokenfilters = tokenfilters;
-    }
-
-    public DetailAnalyzeResponse(StreamInput in) throws IOException {
-        this.customAnalyzer = in.readBoolean();
-        if (customAnalyzer) {
-            tokenizer = new AnalyzeTokenList(in);
-            int size = in.readVInt();
-            if (size > 0) {
-                charfilters = new CharFilteredText[size];
-                for (int i = 0; i < size; i++) {
-                    charfilters[i] = new CharFilteredText(in);
-                }
-            }
-            else {
-                charfilters = null;
-            }
-            size = in.readVInt();
-            if (size > 0) {
-                tokenfilters = new AnalyzeTokenList[size];
-                for (int i = 0; i < size; i++) {
-                    tokenfilters[i] = new AnalyzeTokenList(in);
-                }
-            }
-            else {
-                tokenfilters = null;
-            }
-            analyzer = null;
-        } else {
-            analyzer = new AnalyzeTokenList(in);
-            tokenfilters = null;
-            tokenizer = null;
-            charfilters = null;
-        }
-    }
-
-    public AnalyzeTokenList analyzer() {
-        return this.analyzer;
-    }
-
-    public CharFilteredText[] charfilters() {
-        return this.charfilters;
-    }
-
-    public AnalyzeTokenList tokenizer() {
-        return tokenizer;
-    }
-
-    public AnalyzeTokenList[] tokenfilters() {
-        return tokenfilters;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        DetailAnalyzeResponse that = (DetailAnalyzeResponse) o;
-        return customAnalyzer == that.customAnalyzer &&
-            Objects.equals(analyzer, that.analyzer) &&
-            Arrays.equals(charfilters, that.charfilters) &&
-            Objects.equals(tokenizer, that.tokenizer) &&
-            Arrays.equals(tokenfilters, that.tokenfilters);
-    }
-
-    @Override
-    public int hashCode() {
-        int result = Objects.hash(customAnalyzer, analyzer, tokenizer);
-        result = 31 * result + Arrays.hashCode(charfilters);
-        result = 31 * result + Arrays.hashCode(tokenfilters);
-        return result;
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(Fields.CUSTOM_ANALYZER, customAnalyzer);
-
-        if (analyzer != null) {
-            builder.startObject(Fields.ANALYZER);
-            analyzer.toXContentWithoutObject(builder, params);
-            builder.endObject();
-        }
-
-        if (charfilters != null) {
-            builder.startArray(Fields.CHARFILTERS);
-            for (CharFilteredText charfilter : charfilters) {
-                charfilter.toXContent(builder, params);
-            }
-            builder.endArray();
-        }
-
-        if (tokenizer != null) {
-            builder.startObject(Fields.TOKENIZER);
-            tokenizer.toXContentWithoutObject(builder, params);
-            builder.endObject();
-        }
-
-        if (tokenfilters != null) {
-            builder.startArray(Fields.TOKENFILTERS);
-            for (AnalyzeTokenList tokenfilter : tokenfilters) {
-                tokenfilter.toXContent(builder, params);
-            }
-            builder.endArray();
-        }
-        return builder;
-    }
-
-    @SuppressWarnings("unchecked")
-    private static <T> T[] fromList(Class<T> clazz, List<T> list) {
-        if (list == null) {
-            return null;
-        }
-        return list.toArray((T[])Array.newInstance(clazz, 0));
-    }
-
-    static final ConstructingObjectParser<DetailAnalyzeResponse, Void> PARSER = new ConstructingObjectParser<>("detail",
-        true, args -> new DetailAnalyzeResponse((boolean) args[0], (AnalyzeTokenList) args[1],
-        fromList(CharFilteredText.class, (List<CharFilteredText>)args[2]),
-        (AnalyzeTokenList) args[3],
-        fromList(AnalyzeTokenList.class, (List<AnalyzeTokenList>)args[4])));
-
-    static {
-        PARSER.declareBoolean(constructorArg(), new ParseField(Fields.CUSTOM_ANALYZER));
-        PARSER.declareObject(optionalConstructorArg(), AnalyzeTokenList.PARSER, new ParseField(Fields.ANALYZER));
-        PARSER.declareObjectArray(optionalConstructorArg(), CharFilteredText.PARSER, new ParseField(Fields.CHARFILTERS));
-        PARSER.declareObject(optionalConstructorArg(), AnalyzeTokenList.PARSER, new ParseField(Fields.TOKENIZER));
-        PARSER.declareObjectArray(optionalConstructorArg(), AnalyzeTokenList.PARSER, new ParseField(Fields.TOKENFILTERS));
-    }
-
-    public static DetailAnalyzeResponse fromXContent(XContentParser parser) throws IOException {
-        return PARSER.parse(parser, null);
-    }
-
-    static final class Fields {
-        static final String NAME = "name";
-        static final String FILTERED_TEXT = "filtered_text";
-        static final String CUSTOM_ANALYZER = "custom_analyzer";
-        static final String ANALYZER = "analyzer";
-        static final String CHARFILTERS = "charfilters";
-        static final String TOKENIZER = "tokenizer";
-        static final String TOKENFILTERS = "tokenfilters";
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeBoolean(customAnalyzer);
-        if (customAnalyzer) {
-            tokenizer.writeTo(out);
-            if (charfilters != null) {
-                out.writeVInt(charfilters.length);
-                for (CharFilteredText charfilter : charfilters) {
-                    charfilter.writeTo(out);
-                }
-            } else {
-                out.writeVInt(0);
-            }
-            if (tokenfilters != null) {
-                out.writeVInt(tokenfilters.length);
-                for (AnalyzeTokenList tokenfilter : tokenfilters) {
-                    tokenfilter.writeTo(out);
-                }
-            } else {
-                out.writeVInt(0);
-            }
-        } else {
-            analyzer.writeTo(out);
-        }
-    }
-
-    public static class AnalyzeTokenList implements Writeable, ToXContentObject {
-        private final String name;
-        private final AnalyzeResponse.AnalyzeToken[] tokens;
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            AnalyzeTokenList that = (AnalyzeTokenList) o;
-            return Objects.equals(name, that.name) &&
-                Arrays.equals(tokens, that.tokens);
-        }
-
-        @Override
-        public int hashCode() {
-            int result = Objects.hash(name);
-            result = 31 * result + Arrays.hashCode(tokens);
-            return result;
-        }
-
-        public AnalyzeTokenList(String name, AnalyzeResponse.AnalyzeToken[] tokens) {
-            this.name = name;
-            this.tokens = tokens;
-        }
-
-        public AnalyzeTokenList(StreamInput in) throws IOException {
-            name = in.readString();
-            int size = in.readVInt();
-            if (size > 0) {
-                tokens = new AnalyzeResponse.AnalyzeToken[size];
-                for (int i = 0; i < size; i++) {
-                    tokens[i] = new AnalyzeResponse.AnalyzeToken(in);
-                }
-            }
-            else {
-                tokens = null;
-            }
-        }
-
-        public String getName() {
-            return name;
-        }
-
-        public AnalyzeResponse.AnalyzeToken[] getTokens() {
-            return tokens;
-        }
-
-        XContentBuilder toXContentWithoutObject(XContentBuilder builder, Params params) throws IOException {
-            builder.field(Fields.NAME, this.name);
-            builder.startArray(AnalyzeResponse.Fields.TOKENS);
-            if (tokens != null) {
-                for (AnalyzeResponse.AnalyzeToken token : tokens) {
-                    token.toXContent(builder, params);
-                }
-            }
-            builder.endArray();
-            return builder;
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            toXContentWithoutObject(builder, params);
-            builder.endObject();
-            return builder;
-        }
-
-        private static final ConstructingObjectParser<AnalyzeTokenList, Void> PARSER = new ConstructingObjectParser<>("token_list",
-            true, args -> new AnalyzeTokenList((String) args[0],
-            fromList(AnalyzeResponse.AnalyzeToken.class, (List<AnalyzeResponse.AnalyzeToken>)args[1])));
-
-        static {
-            PARSER.declareString(constructorArg(), new ParseField(Fields.NAME));
-            PARSER.declareObjectArray(constructorArg(), (p, c) -> AnalyzeResponse.AnalyzeToken.fromXContent(p),
-                new ParseField(AnalyzeResponse.Fields.TOKENS));
-        }
-
-        public static AnalyzeTokenList fromXContent(XContentParser parser) throws IOException {
-            return PARSER.parse(parser, null);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeString(name);
-            if (tokens != null) {
-                out.writeVInt(tokens.length);
-                for (AnalyzeResponse.AnalyzeToken token : tokens) {
-                    token.writeTo(out);
-                }
-            } else {
-                out.writeVInt(0);
-            }
-        }
-    }
-
-    public static class CharFilteredText implements Writeable, ToXContentObject {
-        private final String name;
-        private final String[] texts;
-
-        public CharFilteredText(String name, String[] texts) {
-            this.name = name;
-            if (texts != null) {
-                this.texts = texts;
-            } else {
-                this.texts = Strings.EMPTY_ARRAY;
-            }
-        }
-
-        public CharFilteredText(StreamInput in) throws IOException {
-            name = in.readString();
-            texts = in.readStringArray();
-        }
-
-        public String getName() {
-            return name;
-        }
-
-        public String[] getTexts() {
-            return texts;
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            builder.field(Fields.NAME, name);
-            builder.array(Fields.FILTERED_TEXT, texts);
-            builder.endObject();
-            return builder;
-        }
-
-        private static final ConstructingObjectParser<CharFilteredText, Void> PARSER = new ConstructingObjectParser<>("char_filtered_text",
-            true, args -> new CharFilteredText((String) args[0], ((List<String>) args[1]).toArray(new String[0])));
-
-        static {
-            PARSER.declareString(constructorArg(), new ParseField(Fields.NAME));
-            PARSER.declareStringArray(constructorArg(), new ParseField(Fields.FILTERED_TEXT));
-        }
-
-        public static CharFilteredText fromXContent(XContentParser parser) throws IOException {
-            return PARSER.parse(parser, null);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeString(name);
-            out.writeStringArray(texts);
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            CharFilteredText that = (CharFilteredText) o;
-            return Objects.equals(name, that.name) &&
-                Arrays.equals(texts, that.texts);
-        }
-
-        @Override
-        public int hashCode() {
-            int result = Objects.hash(name);
-            result = 31 * result + Arrays.hashCode(texts);
-            return result;
-        }
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
index 55bd593742667..773852860c60e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
@@ -28,45 +28,37 @@
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.internal.io.IOUtils;
-import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.index.analysis.AnalyzerComponents;
+import org.elasticsearch.index.analysis.AnalyzerComponentsProvider;
 import org.elasticsearch.index.analysis.CharFilterFactory;
-import org.elasticsearch.index.analysis.CustomAnalyzer;
-import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.analysis.NameOrDefinition;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.analysis.NormalizingCharFilterFactory;
-import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
-import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -74,35 +66,32 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.function.Function;
 
 /**
  * Transport action used to execute analyze requests
  */
-public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeRequest, AnalyzeResponse> {
+public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeAction.Request, AnalyzeAction.Response> {
 
     private final Settings settings;
     private final IndicesService indicesService;
-    private final Environment environment;
 
     @Inject
     public TransportAnalyzeAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
                                   TransportService transportService, IndicesService indicesService, ActionFilters actionFilters,
-                                  IndexNameExpressionResolver indexNameExpressionResolver, Environment environment) {
+                                  IndexNameExpressionResolver indexNameExpressionResolver) {
         super(AnalyzeAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
-            AnalyzeRequest::new, ThreadPool.Names.ANALYZE);
+            AnalyzeAction.Request::new, ThreadPool.Names.ANALYZE);
         this.settings = settings;
         this.indicesService = indicesService;
-        this.environment = environment;
     }
 
     @Override
-    protected Writeable.Reader<AnalyzeResponse> getResponseReader() {
-        return AnalyzeResponse::new;
+    protected Writeable.Reader<AnalyzeAction.Response> getResponseReader() {
+        return AnalyzeAction.Response::new;
     }
 
     @Override
-    protected boolean resolveIndex(AnalyzeRequest request) {
+    protected boolean resolveIndex(AnalyzeAction.Request request) {
         return request.index() != null;
     }
 
@@ -124,143 +113,116 @@ protected ShardsIterator shards(ClusterState state, InternalRequest request) {
     }
 
     @Override
-    protected AnalyzeResponse shardOperation(AnalyzeRequest request, ShardId shardId) {
-        try {
-            final IndexService indexService;
-            if (shardId != null) {
-                indexService = indicesService.indexServiceSafe(shardId.getIndex());
-            } else {
-                indexService = null;
-            }
-            String field = null;
-            Analyzer analyzer = null;
-            if (request.field() != null) {
-                if (indexService == null) {
-                    throw new IllegalArgumentException(
-                        "No index provided, and trying to analyzer based on a specific field which requires the index parameter");
-                }
-                MappedFieldType fieldType = indexService.mapperService().fullName(request.field());
-                if (fieldType != null) {
-                    if (fieldType.tokenized() || fieldType instanceof KeywordFieldMapper.KeywordFieldType) {
-                        analyzer = fieldType.indexAnalyzer();
-                    } else {
-                        throw new IllegalArgumentException("Can't process field [" + request.field() +
-                            "], Analysis requests are only supported on tokenized fields");
-                    }
-                    field = fieldType.name();
-                }
-            }
-            if (field == null) {
-                /**
-                 * TODO: _all is disabled by default and index.query.default_field can define multiple fields or patterns so we should
-                 * probably makes the field name mandatory in analyze query.
-                 **/
-                if (indexService != null) {
-                    field = indexService.getIndexSettings().getDefaultFields().get(0);
-                }
+    protected AnalyzeAction.Response shardOperation(AnalyzeAction.Request request, ShardId shardId) throws IOException {
+        final IndexService indexService = getIndexService(shardId);
+        final int maxTokenCount = indexService == null ?
+            IndexSettings.MAX_TOKEN_COUNT_SETTING.get(settings) : indexService.getIndexSettings().getMaxTokenCount();
+
+        return analyze(request, indicesService.getAnalysis(), indexService, maxTokenCount);
+    }
+
+    public static AnalyzeAction.Response analyze(AnalyzeAction.Request request, AnalysisRegistry analysisRegistry,
+                                          IndexService indexService, int maxTokenCount) throws IOException {
+
+        IndexSettings settings = indexService == null ? null : indexService.getIndexSettings();
+
+        // First, we check to see if the request requires a custom analyzer.  If so, then we
+        // need to build it and then close it after use.
+        try (Analyzer analyzer = buildCustomAnalyzer(request, analysisRegistry, settings)) {
+            if (analyzer != null) {
+                return analyze(request, analyzer, maxTokenCount);
             }
-            final AnalysisRegistry analysisRegistry = indicesService.getAnalysis();
-            final int maxTokenCount = indexService == null ?
-                IndexSettings.MAX_TOKEN_COUNT_SETTING.get(settings) : indexService.getIndexSettings().getMaxTokenCount();
-            return analyze(request, field, analyzer, indexService != null ? indexService.getIndexAnalyzers() : null,
-                analysisRegistry, environment, maxTokenCount);
-        } catch (IOException e) {
-            throw new ElasticsearchException("analysis failed", e);
         }
 
+        // Otherwise we use a built-in analyzer, which should not be closed
+        return analyze(request, getAnalyzer(request, analysisRegistry, indexService), maxTokenCount);
+    }
+
+    private IndexService getIndexService(ShardId shardId) {
+        if (shardId != null) {
+            return indicesService.indexServiceSafe(shardId.getIndex());
+        }
+        return null;
     }
 
-    public static AnalyzeResponse analyze(AnalyzeRequest request, String field, Analyzer analyzer, IndexAnalyzers indexAnalyzers,
-            AnalysisRegistry analysisRegistry, Environment environment, int maxTokenCount) throws IOException {
-        boolean closeAnalyzer = false;
-        if (analyzer == null && request.analyzer() != null) {
-            if (indexAnalyzers == null) {
-                analyzer = analysisRegistry.getAnalyzer(request.analyzer());
+    private static Analyzer getAnalyzer(AnalyzeAction.Request request, AnalysisRegistry analysisRegistry,
+                                        IndexService indexService) throws IOException {
+        if (request.analyzer() != null) {
+            if (indexService == null) {
+                Analyzer analyzer = analysisRegistry.getAnalyzer(request.analyzer());
                 if (analyzer == null) {
                     throw new IllegalArgumentException("failed to find global analyzer [" + request.analyzer() + "]");
                 }
+                return analyzer;
             } else {
-                analyzer = indexAnalyzers.get(request.analyzer());
+                Analyzer analyzer = indexService.getIndexAnalyzers().get(request.analyzer());
                 if (analyzer == null) {
                     throw new IllegalArgumentException("failed to find analyzer [" + request.analyzer() + "]");
                 }
+                return analyzer;
             }
-        } else if (request.tokenizer() != null) {
-            final IndexSettings indexSettings = indexAnalyzers == null ? null : indexAnalyzers.getIndexSettings();
-            Tuple<String, TokenizerFactory> tokenizerFactory = parseTokenizerFactory(request, indexAnalyzers,
-                        analysisRegistry, environment);
-
-            List<CharFilterFactory> charFilterFactoryList =
-                parseCharFilterFactories(request, indexSettings, analysisRegistry, environment, false);
-
-            List<TokenFilterFactory> tokenFilterFactoryList = parseTokenFilterFactories(request, indexSettings, analysisRegistry,
-                environment, tokenizerFactory, charFilterFactoryList, false);
-
-            analyzer = new CustomAnalyzer(tokenizerFactory.v1(), tokenizerFactory.v2(),
-                charFilterFactoryList.toArray(new CharFilterFactory[charFilterFactoryList.size()]),
-                tokenFilterFactoryList.toArray(new TokenFilterFactory[tokenFilterFactoryList.size()]));
-            closeAnalyzer = true;
-        } else if (request.normalizer() != null) {
+        }
+        if (request.normalizer() != null) {
             // Get normalizer from indexAnalyzers
-            analyzer = indexAnalyzers.getNormalizer(request.normalizer());
+            if (indexService == null) {
+                throw new IllegalArgumentException("analysis based on a normalizer requires an index");
+            }
+            Analyzer analyzer = indexService.getIndexAnalyzers().getNormalizer(request.normalizer());
             if (analyzer == null) {
                 throw new IllegalArgumentException("failed to find normalizer under [" + request.normalizer() + "]");
             }
-        } else if (((request.tokenFilters() != null && request.tokenFilters().size() > 0)
-                || (request.charFilters() != null && request.charFilters().size() > 0))) {
-            final IndexSettings indexSettings = indexAnalyzers == null ? null : indexAnalyzers.getIndexSettings();
-            // custom normalizer = if normalizer == null but filter or char_filter is not null and tokenizer/analyzer is null
-            // get charfilter and filter from request
-            List<CharFilterFactory> charFilterFactoryList =
-                parseCharFilterFactories(request, indexSettings, analysisRegistry, environment, true);
-
-            final String keywordTokenizerName = "keyword";
-            TokenizerFactory keywordTokenizerFactory = getTokenizerFactory(analysisRegistry, environment, keywordTokenizerName);
-
-            List<TokenFilterFactory> tokenFilterFactoryList =
-                parseTokenFilterFactories(request, indexSettings, analysisRegistry, environment,
-                    new Tuple<>(keywordTokenizerName, keywordTokenizerFactory), charFilterFactoryList, true);
-
-            analyzer = new CustomAnalyzer("keyword_for_normalizer",
-                keywordTokenizerFactory,
-                charFilterFactoryList.toArray(new CharFilterFactory[charFilterFactoryList.size()]),
-                tokenFilterFactoryList.toArray(new TokenFilterFactory[tokenFilterFactoryList.size()]));
-            closeAnalyzer = true;
-        } else if (analyzer == null) {
-            if (indexAnalyzers == null) {
-                analyzer = analysisRegistry.getAnalyzer("standard");
-            } else {
-                analyzer = indexAnalyzers.getDefaultIndexAnalyzer();
-            }
         }
-        if (analyzer == null) {
-            throw new IllegalArgumentException("failed to find analyzer");
+        if (request.field() != null) {
+            if (indexService == null) {
+                throw new IllegalArgumentException("analysis based on a specific field requires an index");
+            }
+            MappedFieldType fieldType = indexService.mapperService().fullName(request.field());
+            if (fieldType != null) {
+                if (fieldType.tokenized() || fieldType instanceof KeywordFieldMapper.KeywordFieldType) {
+                    return fieldType.indexAnalyzer();
+                } else {
+                    throw new IllegalArgumentException("Can't process field [" + request.field() +
+                        "], Analysis requests are only supported on tokenized fields");
+                }
+            }
         }
-
-        List<AnalyzeResponse.AnalyzeToken> tokens = null;
-        DetailAnalyzeResponse detail = null;
-
-        if (request.explain()) {
-            detail = detailAnalyze(request, analyzer, field, maxTokenCount);
+        if (indexService == null) {
+            return analysisRegistry.getAnalyzer("standard");
         } else {
-            tokens = simpleAnalyze(request, analyzer, field, maxTokenCount);
+            return indexService.getIndexAnalyzers().getDefaultIndexAnalyzer();
         }
+    }
 
-        if (closeAnalyzer) {
-            analyzer.close();
+    private static Analyzer buildCustomAnalyzer(AnalyzeAction.Request request, AnalysisRegistry analysisRegistry,
+                                                IndexSettings indexSettings) throws IOException {
+        if (request.tokenizer() != null) {
+            return analysisRegistry.buildCustomAnalyzer(indexSettings, false,
+                request.tokenizer(), request.charFilters(), request.tokenFilters());
+        } else if (((request.tokenFilters() != null && request.tokenFilters().size() > 0)
+            || (request.charFilters() != null && request.charFilters().size() > 0))) {
+            return analysisRegistry.buildCustomAnalyzer(indexSettings, true, new NameOrDefinition("keyword"),
+                request.charFilters(), request.tokenFilters());
         }
+        return null;
+    }
 
-        return new AnalyzeResponse(tokens, detail);
+    private static AnalyzeAction.Response analyze(AnalyzeAction.Request request, Analyzer analyzer, int maxTokenCount) {
+        if (request.explain()) {
+            return new AnalyzeAction.Response(null, detailAnalyze(request, analyzer, maxTokenCount));
+        }
+        return new AnalyzeAction.Response(simpleAnalyze(request, analyzer, maxTokenCount), null);
     }
 
-    private static List<AnalyzeResponse.AnalyzeToken> simpleAnalyze(AnalyzeRequest request,
-            Analyzer analyzer, String field, int maxTokenCount) {
+    private static List<AnalyzeAction.AnalyzeToken> simpleAnalyze(AnalyzeAction.Request request,
+                                                                  Analyzer analyzer, int maxTokenCount) {
         TokenCounter tc = new TokenCounter(maxTokenCount);
-        List<AnalyzeResponse.AnalyzeToken> tokens = new ArrayList<>();
+        List<AnalyzeAction.AnalyzeToken> tokens = new ArrayList<>();
         int lastPosition = -1;
         int lastOffset = 0;
+        // Note that we always pass "" as the field to the various Analyzer methods, because
+        // the analyzers we use here are all field-specific and so ignore this parameter
         for (String text : request.text()) {
-            try (TokenStream stream = analyzer.tokenStream(field, text)) {
+            try (TokenStream stream = analyzer.tokenStream("", text)) {
                 stream.reset();
                 CharTermAttribute term = stream.addAttribute(CharTermAttribute.class);
                 PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class);
@@ -273,7 +235,7 @@ private static List<AnalyzeResponse.AnalyzeToken> simpleAnalyze(AnalyzeRequest r
                     if (increment > 0) {
                         lastPosition = lastPosition + increment;
                     }
-                    tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(),
+                    tokens.add(new AnalyzeAction.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(),
                         lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), null));
                     tc.increment();
                 }
@@ -281,8 +243,8 @@ private static List<AnalyzeResponse.AnalyzeToken> simpleAnalyze(AnalyzeRequest r
                 lastOffset += offset.endOffset();
                 lastPosition += posIncr.getPositionIncrement();
 
-                lastPosition += analyzer.getPositionIncrementGap(field);
-                lastOffset += analyzer.getOffsetGap(field);
+                lastPosition += analyzer.getPositionIncrementGap("");
+                lastOffset += analyzer.getOffsetGap("");
             } catch (IOException e) {
                 throw new ElasticsearchException("failed to analyze", e);
             }
@@ -290,8 +252,9 @@ private static List<AnalyzeResponse.AnalyzeToken> simpleAnalyze(AnalyzeRequest r
         return tokens;
     }
 
-    private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analyzer analyzer, String field, int maxTokenCount) {
-        DetailAnalyzeResponse detailResponse;
+    private static AnalyzeAction.DetailAnalyzeResponse detailAnalyze(AnalyzeAction.Request request, Analyzer analyzer,
+                                                                     int maxTokenCount) {
+        AnalyzeAction.DetailAnalyzeResponse detailResponse;
         final Set<String> includeAttributes = new HashSet<>();
         if (request.attributes() != null) {
             for (String attribute : request.attributes()) {
@@ -299,18 +262,23 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy
             }
         }
 
-        CustomAnalyzer customAnalyzer = null;
-        if (analyzer instanceof CustomAnalyzer) {
-            customAnalyzer = (CustomAnalyzer) analyzer;
-        } else if (analyzer instanceof NamedAnalyzer && ((NamedAnalyzer) analyzer).analyzer() instanceof CustomAnalyzer) {
-            customAnalyzer = (CustomAnalyzer) ((NamedAnalyzer) analyzer).analyzer();
+        // maybe unwrap analyzer from NamedAnalyzer
+        Analyzer potentialCustomAnalyzer = analyzer;
+        if (analyzer instanceof NamedAnalyzer) {
+            potentialCustomAnalyzer = ((NamedAnalyzer) analyzer).analyzer();
         }
 
-        if (customAnalyzer != null) {
-            // customAnalyzer = divide charfilter, tokenizer tokenfilters
-            CharFilterFactory[] charFilterFactories = customAnalyzer.charFilters();
-            TokenizerFactory tokenizerFactory = customAnalyzer.tokenizerFactory();
-            TokenFilterFactory[] tokenFilterFactories = customAnalyzer.tokenFilters();
+        if (potentialCustomAnalyzer instanceof AnalyzerComponentsProvider) {
+            AnalyzerComponentsProvider customAnalyzer = (AnalyzerComponentsProvider) potentialCustomAnalyzer;
+            // note: this is not field-name dependent in our cases so we can leave out the argument
+            int positionIncrementGap = potentialCustomAnalyzer.getPositionIncrementGap("");
+            int offsetGap = potentialCustomAnalyzer.getOffsetGap("");
+            AnalyzerComponents components = customAnalyzer.getComponents();
+            // divide charfilter, tokenizer tokenfilters
+            CharFilterFactory[] charFilterFactories = components.getCharFilters();
+            TokenizerFactory tokenizerFactory = components.getTokenizerFactory();
+            TokenFilterFactory[] tokenFilterFactories = components.getTokenFilters();
+            String tokenizerName = components.getTokenizerName();
 
             String[][] charFiltersTexts = new String[charFilterFactories != null ? charFilterFactories.length : 0][request.text().length];
             TokenListCreator[] tokenFiltersTokenListCreator = new TokenListCreator[tokenFilterFactories != null ?
@@ -336,7 +304,7 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy
                 // analyzing only tokenizer
                 Tokenizer tokenizer = tokenizerFactory.create();
                 tokenizer.setReader(reader);
-                tokenizerTokenListCreator.analyze(tokenizer, customAnalyzer, field, includeAttributes);
+                tokenizerTokenListCreator.analyze(tokenizer, includeAttributes, positionIncrementGap, offsetGap);
 
                 // analyzing each tokenfilter
                 if (tokenFilterFactories != null) {
@@ -346,31 +314,31 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy
                         }
                         TokenStream stream = createStackedTokenStream(request.text()[textIndex],
                             charFilterFactories, tokenizerFactory, tokenFilterFactories, tokenFilterIndex + 1);
-                        tokenFiltersTokenListCreator[tokenFilterIndex].analyze(stream, customAnalyzer, field, includeAttributes);
+                        tokenFiltersTokenListCreator[tokenFilterIndex].analyze(stream, includeAttributes, positionIncrementGap, offsetGap);
                     }
                 }
             }
 
-            DetailAnalyzeResponse.CharFilteredText[] charFilteredLists =
-                new DetailAnalyzeResponse.CharFilteredText[charFiltersTexts.length];
+            AnalyzeAction.CharFilteredText[] charFilteredLists =
+                new AnalyzeAction.CharFilteredText[charFiltersTexts.length];
 
             if (charFilterFactories != null) {
                 for (int charFilterIndex = 0; charFilterIndex < charFiltersTexts.length; charFilterIndex++) {
-                    charFilteredLists[charFilterIndex] = new DetailAnalyzeResponse.CharFilteredText(
+                    charFilteredLists[charFilterIndex] = new AnalyzeAction.CharFilteredText(
                         charFilterFactories[charFilterIndex].name(), charFiltersTexts[charFilterIndex]);
                 }
             }
-            DetailAnalyzeResponse.AnalyzeTokenList[] tokenFilterLists =
-                new DetailAnalyzeResponse.AnalyzeTokenList[tokenFiltersTokenListCreator.length];
+            AnalyzeAction.AnalyzeTokenList[] tokenFilterLists =
+                new AnalyzeAction.AnalyzeTokenList[tokenFiltersTokenListCreator.length];
 
             if (tokenFilterFactories != null) {
                 for (int tokenFilterIndex = 0; tokenFilterIndex < tokenFiltersTokenListCreator.length; tokenFilterIndex++) {
-                    tokenFilterLists[tokenFilterIndex] = new DetailAnalyzeResponse.AnalyzeTokenList(
+                    tokenFilterLists[tokenFilterIndex] = new AnalyzeAction.AnalyzeTokenList(
                         tokenFilterFactories[tokenFilterIndex].name(), tokenFiltersTokenListCreator[tokenFilterIndex].getArrayTokens());
                 }
             }
-            detailResponse = new DetailAnalyzeResponse(charFilteredLists, new DetailAnalyzeResponse.AnalyzeTokenList(
-                    customAnalyzer.getTokenizerName(), tokenizerTokenListCreator.getArrayTokens()), tokenFilterLists);
+            detailResponse = new AnalyzeAction.DetailAnalyzeResponse(charFilteredLists,
+                    new AnalyzeAction.AnalyzeTokenList(tokenizerName, tokenizerTokenListCreator.getArrayTokens()), tokenFilterLists);
         } else {
             String name;
             if (analyzer instanceof NamedAnalyzer) {
@@ -381,10 +349,11 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy
 
             TokenListCreator tokenListCreator = new TokenListCreator(maxTokenCount);
             for (String text : request.text()) {
-                tokenListCreator.analyze(analyzer.tokenStream(field, text), analyzer, field,
-                        includeAttributes);
+                tokenListCreator.analyze(analyzer.tokenStream("", text), includeAttributes, analyzer.getPositionIncrementGap(""),
+                        analyzer.getOffsetGap(""));
             }
-            detailResponse = new DetailAnalyzeResponse(new DetailAnalyzeResponse.AnalyzeTokenList(name, tokenListCreator.getArrayTokens()));
+            detailResponse
+                = new AnalyzeAction.DetailAnalyzeResponse(new AnalyzeAction.AnalyzeTokenList(name, tokenListCreator.getArrayTokens()));
         }
         return detailResponse;
     }
@@ -443,7 +412,7 @@ private void increment(){
     private static class TokenListCreator {
         int lastPosition = -1;
         int lastOffset = 0;
-        List<AnalyzeResponse.AnalyzeToken> tokens;
+        List<AnalyzeAction.AnalyzeToken> tokens;
         private TokenCounter tc;
 
         TokenListCreator(int maxTokenCount) {
@@ -451,7 +420,7 @@ private static class TokenListCreator {
             tc = new TokenCounter(maxTokenCount);
         }
 
-        private void analyze(TokenStream stream, Analyzer analyzer, String field, Set<String> includeAttributes) {
+        private void analyze(TokenStream stream, Set<String> includeAttributes, int positionIncrementGap, int offsetGap) {
             try {
                 stream.reset();
                 CharTermAttribute term = stream.addAttribute(CharTermAttribute.class);
@@ -465,7 +434,7 @@ private void analyze(TokenStream stream, Analyzer analyzer, String field, Set<St
                     if (increment > 0) {
                         lastPosition = lastPosition + increment;
                     }
-                    tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(),
+                    tokens.add(new AnalyzeAction.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(),
                         lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(),
                         extractExtendedAttributes(stream, includeAttributes)));
                     tc.increment();
@@ -474,8 +443,8 @@ private void analyze(TokenStream stream, Analyzer analyzer, String field, Set<St
                 lastOffset += offset.endOffset();
                 lastPosition += posIncr.getPositionIncrement();
 
-                lastPosition += analyzer.getPositionIncrementGap(field);
-                lastOffset += analyzer.getOffsetGap(field);
+                lastPosition += positionIncrementGap;
+                lastOffset += offsetGap;
 
             } catch (IOException e) {
                 throw new ElasticsearchException("failed to analyze", e);
@@ -484,8 +453,8 @@ private void analyze(TokenStream stream, Analyzer analyzer, String field, Set<St
             }
         }
 
-        private AnalyzeResponse.AnalyzeToken[] getArrayTokens() {
-            return tokens.toArray(new AnalyzeResponse.AnalyzeToken[tokens.size()]);
+        private AnalyzeAction.AnalyzeToken[] getArrayTokens() {
+            return tokens.toArray(new AnalyzeAction.AnalyzeToken[0]);
         }
 
     }
@@ -526,227 +495,4 @@ private static Map<String, Object> extractExtendedAttributes(TokenStream stream,
         return extendedAttributes;
     }
 
-    private static List<CharFilterFactory> parseCharFilterFactories(AnalyzeRequest request, IndexSettings indexSettings,
-                                                                    AnalysisRegistry analysisRegistry, Environment environment,
-                                                                    boolean normalizer) throws IOException {
-        List<CharFilterFactory> charFilterFactoryList = new ArrayList<>();
-        if (request.charFilters() != null && request.charFilters().size() > 0) {
-            List<AnalyzeRequest.NameOrDefinition> charFilters = request.charFilters();
-            for (AnalyzeRequest.NameOrDefinition charFilter : charFilters) {
-                CharFilterFactory charFilterFactory;
-                // parse anonymous settings
-                if (charFilter.definition != null) {
-                    Settings settings = getAnonymousSettings(charFilter.definition);
-                    String charFilterTypeName = settings.get("type");
-                    if (charFilterTypeName == null) {
-                        throw new IllegalArgumentException("Missing [type] setting for anonymous char filter: " + charFilter.definition);
-                    }
-                    AnalysisModule.AnalysisProvider<CharFilterFactory> charFilterFactoryFactory =
-                        analysisRegistry.getCharFilterProvider(charFilterTypeName);
-                    if (charFilterFactoryFactory == null) {
-                        throw new IllegalArgumentException("failed to find global char filter under [" + charFilterTypeName + "]");
-                    }
-                    // Need to set anonymous "name" of char_filter
-                    charFilterFactory = charFilterFactoryFactory.get(getNaIndexSettings(settings), environment, "_anonymous_charfilter",
-                        settings);
-                } else {
-                    AnalysisModule.AnalysisProvider<CharFilterFactory> charFilterFactoryFactory;
-                    if (indexSettings == null) {
-                        charFilterFactoryFactory = analysisRegistry.getCharFilterProvider(charFilter.name);
-                        if (charFilterFactoryFactory == null) {
-                            throw new IllegalArgumentException("failed to find global char filter under [" + charFilter.name + "]");
-                        }
-                        charFilterFactory = charFilterFactoryFactory.get(environment, charFilter.name);
-                    } else {
-                        charFilterFactoryFactory = analysisRegistry.getCharFilterProvider(charFilter.name, indexSettings);
-                        if (charFilterFactoryFactory == null) {
-                            throw new IllegalArgumentException("failed to find char filter under [" + charFilter.name + "]");
-                        }
-                        charFilterFactory = charFilterFactoryFactory.get(indexSettings, environment, charFilter.name,
-                            AnalysisRegistry.getSettingsFromIndexSettings(indexSettings,
-                                AnalysisRegistry.INDEX_ANALYSIS_CHAR_FILTER + "." + charFilter.name));
-                    }
-                }
-                if (charFilterFactory == null) {
-                    throw new IllegalArgumentException("failed to find char filter under [" + charFilter.name + "]");
-                }
-                if (normalizer) {
-                    if (charFilterFactory instanceof NormalizingCharFilterFactory == false) {
-                        throw new IllegalArgumentException("Custom normalizer may not use char filter ["
-                            + charFilterFactory.name() + "]");
-                    }
-                }
-                charFilterFactoryList.add(charFilterFactory);
-            }
-        }
-        return charFilterFactoryList;
-    }
-
-    public static class DeferredTokenFilterRegistry implements Function<String, TokenFilterFactory> {
-
-        private final AnalysisRegistry analysisRegistry;
-        private final IndexSettings indexSettings;
-        Map<String, TokenFilterFactory> prebuiltFilters;
-
-        public DeferredTokenFilterRegistry(AnalysisRegistry analysisRegistry, IndexSettings indexSettings) {
-            this.analysisRegistry = analysisRegistry;
-            if (indexSettings == null) {
-                // Settings are null when _analyze is called with no index name, so
-                // we create dummy settings which will make prebuilt analysis components
-                // available
-                Settings settings = Settings.builder()
-                    .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                    .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
-                    .build();
-                IndexMetaData metaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(settings).build();
-                indexSettings = new IndexSettings(metaData, Settings.EMPTY);
-            }
-            this.indexSettings = indexSettings;
-        }
-
-        @Override
-        public TokenFilterFactory apply(String s) {
-            if (prebuiltFilters == null) {
-                try {
-                    prebuiltFilters = analysisRegistry.buildTokenFilterFactories(indexSettings);
-                } catch (IOException e) {
-                    throw new UncheckedIOException(e);
-                }
-            }
-            return prebuiltFilters.get(s);
-        }
-    }
-
-    private static List<TokenFilterFactory> parseTokenFilterFactories(AnalyzeRequest request, IndexSettings indexSettings,
-                                                                      AnalysisRegistry analysisRegistry, Environment environment,
-                                                                      Tuple<String, TokenizerFactory> tokenizerFactory,
-                                                                      List<CharFilterFactory> charFilterFactoryList,
-                                                                      boolean normalizer) throws IOException {
-        List<TokenFilterFactory> tokenFilterFactoryList = new ArrayList<>();
-        DeferredTokenFilterRegistry deferredRegistry = new DeferredTokenFilterRegistry(analysisRegistry, indexSettings);
-        if (request.tokenFilters() != null && request.tokenFilters().size() > 0) {
-            List<AnalyzeRequest.NameOrDefinition> tokenFilters = request.tokenFilters();
-            for (AnalyzeRequest.NameOrDefinition tokenFilter : tokenFilters) {
-                TokenFilterFactory tokenFilterFactory;
-                // parse anonymous settings
-                if (tokenFilter.definition != null) {
-                    Settings settings = getAnonymousSettings(tokenFilter.definition);
-                    String filterTypeName = settings.get("type");
-                    if (filterTypeName == null) {
-                        throw new IllegalArgumentException("Missing [type] setting for anonymous token filter: " + tokenFilter.definition);
-                    }
-                    AnalysisModule.AnalysisProvider<TokenFilterFactory> tokenFilterFactoryFactory =
-                        analysisRegistry.getTokenFilterProvider(filterTypeName);
-                    if (tokenFilterFactoryFactory == null) {
-                        throw new IllegalArgumentException("failed to find global token filter under [" + filterTypeName + "]");
-                    }
-                    // Need to set anonymous "name" of tokenfilter
-                    tokenFilterFactory = tokenFilterFactoryFactory.get(getNaIndexSettings(settings), environment, "_anonymous_tokenfilter",
-                        settings);
-                    tokenFilterFactory = tokenFilterFactory.getChainAwareTokenFilterFactory(tokenizerFactory.v2(), charFilterFactoryList,
-                        tokenFilterFactoryList, deferredRegistry);
-
-                } else {
-                    AnalysisModule.AnalysisProvider<TokenFilterFactory> tokenFilterFactoryFactory;
-                    if (indexSettings == null) {
-                        tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider(tokenFilter.name);
-                        if (tokenFilterFactoryFactory == null) {
-                            throw new IllegalArgumentException("failed to find global token filter under [" + tokenFilter.name + "]");
-                        }
-                        tokenFilterFactory = tokenFilterFactoryFactory.get(environment, tokenFilter.name);
-                    } else {
-                        tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider(tokenFilter.name, indexSettings);
-                        if (tokenFilterFactoryFactory == null) {
-                            throw new IllegalArgumentException("failed to find token filter under [" + tokenFilter.name + "]");
-                        }
-                        Settings settings = AnalysisRegistry.getSettingsFromIndexSettings(indexSettings,
-                            AnalysisRegistry.INDEX_ANALYSIS_FILTER + "." + tokenFilter.name);
-                        tokenFilterFactory = tokenFilterFactoryFactory.get(indexSettings, environment, tokenFilter.name, settings);
-                        tokenFilterFactory = tokenFilterFactory.getChainAwareTokenFilterFactory(tokenizerFactory.v2(),
-                            charFilterFactoryList, tokenFilterFactoryList, deferredRegistry);
-                    }
-                }
-                if (tokenFilterFactory == null) {
-                    throw new IllegalArgumentException("failed to find or create token filter under [" + tokenFilter.name + "]");
-                }
-                if (normalizer) {
-                    if (tokenFilterFactory instanceof NormalizingTokenFilterFactory == false) {
-                        throw new IllegalArgumentException("Custom normalizer may not use filter ["
-                            + tokenFilterFactory.name() + "]");
-                    }
-                }
-                tokenFilterFactoryList.add(tokenFilterFactory);
-            }
-        }
-        return tokenFilterFactoryList;
-    }
-
-    private static Tuple<String, TokenizerFactory> parseTokenizerFactory(AnalyzeRequest request, IndexAnalyzers indexAnalzyers,
-                                                          AnalysisRegistry analysisRegistry, Environment environment) throws IOException {
-        String name;
-        TokenizerFactory tokenizerFactory;
-        final AnalyzeRequest.NameOrDefinition tokenizer = request.tokenizer();
-        // parse anonymous settings
-        if (tokenizer.definition != null) {
-            Settings settings = getAnonymousSettings(tokenizer.definition);
-            String tokenizerTypeName = settings.get("type");
-            if (tokenizerTypeName == null) {
-                throw new IllegalArgumentException("Missing [type] setting for anonymous tokenizer: " + tokenizer.definition);
-            }
-            AnalysisModule.AnalysisProvider<TokenizerFactory> tokenizerFactoryFactory =
-                analysisRegistry.getTokenizerProvider(tokenizerTypeName);
-            if (tokenizerFactoryFactory == null) {
-                throw new IllegalArgumentException("failed to find global tokenizer under [" + tokenizerTypeName + "]");
-            }
-            // Need to set anonymous "name" of tokenizer
-            name = "_anonymous_tokenizer";
-            tokenizerFactory = tokenizerFactoryFactory.get(getNaIndexSettings(settings), environment, "_anonymous_tokenizer", settings);
-        } else {
-            AnalysisModule.AnalysisProvider<TokenizerFactory> tokenizerFactoryFactory;
-            if (indexAnalzyers == null) {
-                tokenizerFactory = getTokenizerFactory(analysisRegistry, environment, tokenizer.name);
-                name = tokenizer.name;
-            } else {
-                tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider(tokenizer.name, indexAnalzyers.getIndexSettings());
-                if (tokenizerFactoryFactory == null) {
-                    throw new IllegalArgumentException("failed to find tokenizer under [" + tokenizer.name + "]");
-                }
-                name = tokenizer.name;
-                tokenizerFactory = tokenizerFactoryFactory.get(indexAnalzyers.getIndexSettings(), environment, tokenizer.name,
-                    AnalysisRegistry.getSettingsFromIndexSettings(indexAnalzyers.getIndexSettings(),
-                        AnalysisRegistry.INDEX_ANALYSIS_TOKENIZER + "." + tokenizer.name));
-            }
-        }
-        return new Tuple<>(name, tokenizerFactory);
-    }
-
-    private static TokenizerFactory getTokenizerFactory(AnalysisRegistry analysisRegistry, Environment environment,
-                                                        String name) throws IOException {
-        AnalysisModule.AnalysisProvider<TokenizerFactory> tokenizerFactoryFactory;
-        TokenizerFactory tokenizerFactory;
-        tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider(name);
-        if (tokenizerFactoryFactory == null) {
-            throw new IllegalArgumentException("failed to find global tokenizer under [" + name + "]");
-        }
-        tokenizerFactory = tokenizerFactoryFactory.get(environment, name);
-        return tokenizerFactory;
-    }
-
-    private static IndexSettings getNaIndexSettings(Settings settings) {
-        IndexMetaData metaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(settings).build();
-        return new IndexSettings(metaData, Settings.EMPTY);
-    }
-
-    private static Settings getAnonymousSettings(Settings providerSetting) {
-        return Settings.builder().put(providerSetting)
-            // for _na_
-            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
-            .build();
-    }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheAction.java
index e5bdd53bdc7a7..8f020df00edda 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.indices.cache.clear;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class ClearIndicesCacheAction extends Action<ClearIndicesCacheResponse> {
+public class ClearIndicesCacheAction extends StreamableResponseActionType<ClearIndicesCacheResponse> {
 
     public static final ClearIndicesCacheAction INSTANCE = new ClearIndicesCacheAction();
     public static final String NAME = "indices:admin/cache/clear";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexAction.java
index 5c3d60dd44013..f2a6cff2d3949 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexAction.java
@@ -19,9 +19,10 @@
 
 package org.elasticsearch.action.admin.indices.close;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class CloseIndexAction extends Action<CloseIndexResponse> {
+public class CloseIndexAction extends ActionType<CloseIndexResponse> {
 
     public static final CloseIndexAction INSTANCE = new CloseIndexAction();
     public static final String NAME = "indices:admin/close";
@@ -31,7 +32,7 @@ private CloseIndexAction() {
     }
 
     @Override
-    public CloseIndexResponse newResponse() {
-        return new CloseIndexResponse();
+    public Writeable.Reader<CloseIndexResponse> getResponseReader() {
+        return CloseIndexResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponse.java
index c653c264e95db..925b9dd4d5231 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponse.java
@@ -40,9 +40,15 @@
 
 public class CloseIndexResponse extends ShardsAcknowledgedResponse {
 
-    private List<IndexResult> indices;
+    private final List<IndexResult> indices;
 
-    CloseIndexResponse() {
+    CloseIndexResponse(StreamInput in) throws IOException {
+        super(in, in.getVersion().onOrAfter(Version.V_7_2_0));
+        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+            indices = unmodifiableList(in.readList(IndexResult::new));
+        } else {
+            indices = unmodifiableList(emptyList());
+        }
     }
 
     public CloseIndexResponse(final boolean acknowledged, final boolean shardsAcknowledged, final List<IndexResult> indices) {
@@ -54,19 +60,6 @@ public List<IndexResult> getIndices() {
         return indices;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        if (in.getVersion().onOrAfter(Version.V_7_2_0)) {
-            readShardsAcknowledged(in);
-        }
-        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
-            indices = unmodifiableList(in.readList(IndexResult::new));
-        } else {
-            indices = unmodifiableList(emptyList());
-        }
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
@@ -78,6 +71,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
+    @Override
     protected void addCustomFields(final XContentBuilder builder, final Params params) throws IOException {
         super.addCustomFields(builder, params);
         builder.startObject("indices");
@@ -190,7 +184,7 @@ public String toString() {
     public static class ShardResult implements Writeable, ToXContentFragment {
 
         private final int id;
-        private final ShardResult.Failure[] failures;
+        private final Failure[] failures;
 
         public ShardResult(final int id, final Failure[] failures) {
             this.id = id;
@@ -199,7 +193,7 @@ public ShardResult(final int id, final Failure[] failures) {
 
         ShardResult(final StreamInput in) throws IOException {
             this.id = in.readVInt();
-            this.failures = in.readOptionalArray(Failure::readFailure, ShardResult.Failure[]::new);
+            this.failures = in.readOptionalArray(Failure::readFailure, Failure[]::new);
         }
 
         @Override
@@ -227,9 +221,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
                 builder.startArray("failures");
                 if (failures != null) {
                     for (Failure failure : failures) {
-                        builder.startObject();
                         failure.toXContent(builder, params);
-                        builder.endObject();
                     }
                 }
                 builder.endArray();
@@ -242,7 +234,7 @@ public String toString() {
             return Strings.toString(this);
         }
 
-        public static class Failure extends DefaultShardOperationFailedException implements Writeable {
+        public static class Failure extends DefaultShardOperationFailedException {
 
             private @Nullable String nodeId;
 
@@ -275,11 +267,11 @@ public void writeTo(final StreamOutput out) throws IOException {
             }
 
             @Override
-            public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+            public XContentBuilder innerToXContent(final XContentBuilder builder, final Params params) throws IOException {
                 if (nodeId != null) {
                     builder.field("node", nodeId);
                 }
-                return super.toXContent(builder, params);
+                return super.innerToXContent(builder, params);
             }
 
             @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java
index 3c231d13845b2..d653f000d0bc5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -40,6 +41,7 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.Collections;
 
 /**
@@ -78,7 +80,12 @@ protected String executor() {
 
     @Override
     protected CloseIndexResponse newResponse() {
-        return new CloseIndexResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected CloseIndexResponse read(StreamInput in) throws IOException {
+        return new CloseIndexResponse(in);
     }
 
     @Override
@@ -97,13 +104,6 @@ protected ClusterBlockException checkBlock(CloseIndexRequest request, ClusterSta
             indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
-    @Override
-    protected void masterOperation(final CloseIndexRequest request,
-                                   final ClusterState state,
-                                   final ActionListener<CloseIndexResponse> listener) {
-        throw new UnsupportedOperationException("The task parameter is required");
-    }
-
     @Override
     protected void masterOperation(final Task task,
                                    final CloseIndexRequest request,
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java
index 22a0777f7bffb..79cbab4781947 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseAction.java
@@ -20,6 +20,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.support.ActionFilters;
@@ -94,12 +95,12 @@ protected void shardOperationOnPrimary(final ShardRequest shardRequest, final In
     }
 
     @Override
-    protected ReplicaResult shardOperationOnReplica(final ShardRequest shardRequest, final IndexShard replica) {
+    protected ReplicaResult shardOperationOnReplica(final ShardRequest shardRequest, final IndexShard replica) throws IOException {
         executeShardOperation(shardRequest, replica);
         return new ReplicaResult();
     }
 
-    private void executeShardOperation(final ShardRequest request, final IndexShard indexShard) {
+    private void executeShardOperation(final ShardRequest request, final IndexShard indexShard) throws IOException {
         final ShardId shardId = indexShard.shardId();
         if (indexShard.getActiveOperationsCount() != IndexShard.OPERATIONS_BLOCKED) {
             throw new IllegalStateException("Index shard " + shardId + " is not blocking all operations during closing");
@@ -109,9 +110,19 @@ private void executeShardOperation(final ShardRequest request, final IndexShard
         if (clusterBlocks.hasIndexBlock(shardId.getIndexName(), request.clusterBlock()) == false) {
             throw new IllegalStateException("Index shard " + shardId + " must be blocked by " + request.clusterBlock() + " before closing");
         }
-        indexShard.verifyShardBeforeIndexClosing();
-        indexShard.flush(new FlushRequest().force(true).waitIfOngoing(true));
-        logger.trace("{} shard is ready for closing", shardId);
+        if (request.isPhase1()) {
+            // in order to advance the global checkpoint to the maximum sequence number, the (persisted) local checkpoint needs to be
+            // advanced first, which, when using async translog syncing, does not automatically hold at the time where we have acquired
+            // all operation permits. Instead, this requires and explicit sync, which communicates the updated (persisted) local checkpoint
+            // to the primary (we call this phase1), and phase2 can then use the fact that the global checkpoint has moved to the maximum
+            // sequence number to pass the verifyShardBeforeIndexClosing check and create a safe commit where the maximum sequence number
+            // is equal to the global checkpoint.
+            indexShard.sync();
+        } else {
+            indexShard.verifyShardBeforeIndexClosing();
+            indexShard.flush(new FlushRequest().force(true).waitIfOngoing(true));
+            logger.trace("{} shard is ready for closing", shardId);
+        }
     }
 
     @Override
@@ -136,14 +147,22 @@ public static class ShardRequest extends ReplicationRequest<ShardRequest> {
 
         private final ClusterBlock clusterBlock;
 
+        private final boolean phase1;
+
         ShardRequest(StreamInput in) throws IOException {
             super(in);
             clusterBlock = new ClusterBlock(in);
+            if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+                phase1 = in.readBoolean();
+            } else {
+                phase1 = false;
+            }
         }
 
-        public ShardRequest(final ShardId shardId, final ClusterBlock clusterBlock, final TaskId parentTaskId) {
+        public ShardRequest(final ShardId shardId, final ClusterBlock clusterBlock, final boolean phase1, final TaskId parentTaskId) {
             super(shardId);
             this.clusterBlock = Objects.requireNonNull(clusterBlock);
+            this.phase1 = phase1;
             setParentTask(parentTaskId);
         }
 
@@ -161,10 +180,17 @@ public void readFrom(final StreamInput in) {
         public void writeTo(final StreamOutput out) throws IOException {
             super.writeTo(out);
             clusterBlock.writeTo(out);
+            if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+                out.writeBoolean(phase1);
+            }
         }
 
         public ClusterBlock clusterBlock() {
             return clusterBlock;
         }
+
+        public boolean isPhase1() {
+            return phase1;
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexAction.java
index 3993ea7256f61..cf7c7d08e0f72 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexAction.java
@@ -19,9 +19,10 @@
 
 package org.elasticsearch.action.admin.indices.create;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class CreateIndexAction extends Action<CreateIndexResponse> {
+public class CreateIndexAction extends ActionType<CreateIndexResponse> {
 
     public static final CreateIndexAction INSTANCE = new CreateIndexAction();
     public static final String NAME = "indices:admin/create";
@@ -31,7 +32,7 @@ private CreateIndexAction() {
     }
 
     @Override
-    public CreateIndexResponse newResponse() {
-        return new CreateIndexResponse();
+    public Writeable.Reader<CreateIndexResponse> getResponseReader() {
+        return CreateIndexResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java
index 79192693620dd..54142c2a90465 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java
@@ -52,22 +52,18 @@ protected static <T extends CreateIndexResponse> void declareFields(Constructing
         objectParser.declareField(constructorArg(), (parser, context) -> parser.textOrNull(), INDEX, ObjectParser.ValueType.STRING_OR_NULL);
     }
 
-    private String index;
+    private final String index;
 
-    public CreateIndexResponse() {}
+    protected CreateIndexResponse(StreamInput in) throws IOException {
+        super(in, true);
+        index = in.readString();
+    }
 
-    protected CreateIndexResponse(boolean acknowledged, boolean shardsAcknowledged, String index) {
+    public CreateIndexResponse(boolean acknowledged, boolean shardsAcknowledged, String index) {
         super(acknowledged, shardsAcknowledged);
         this.index = index;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        readShardsAcknowledged(in);
-        index = in.readString();
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java
index 03bbd0903a6d5..fdf8cc0c00c25 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java
@@ -29,9 +29,13 @@
 import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 /**
  * Create index action.
  */
@@ -56,7 +60,12 @@ protected String executor() {
 
     @Override
     protected CreateIndexResponse newResponse() {
-        return new CreateIndexResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected CreateIndexResponse read(StreamInput in) throws IOException {
+        return new CreateIndexResponse(in);
     }
 
     @Override
@@ -65,7 +74,7 @@ protected ClusterBlockException checkBlock(CreateIndexRequest request, ClusterSt
     }
 
     @Override
-    protected void masterOperation(final CreateIndexRequest request, final ClusterState state,
+    protected void masterOperation(Task task, final CreateIndexRequest request, final ClusterState state,
                                    final ActionListener<CreateIndexResponse> listener) {
         String cause = request.cause();
         if (cause.length() == 0) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexAction.java
index 6fde9089cef62..7e3a812223e78 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexAction.java
@@ -19,10 +19,11 @@
 
 package org.elasticsearch.action.admin.indices.delete;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class DeleteIndexAction extends Action<AcknowledgedResponse> {
+public class DeleteIndexAction extends ActionType<AcknowledgedResponse> {
 
     public static final DeleteIndexAction INSTANCE = new DeleteIndexAction();
     public static final String NAME = "indices:admin/delete";
@@ -32,7 +33,7 @@ private DeleteIndexAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
index 50f65312afbf0..384452ab8c945 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
@@ -32,11 +32,13 @@
 import org.elasticsearch.cluster.metadata.MetaDataDeleteIndexService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
@@ -65,9 +67,14 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
@@ -82,7 +89,7 @@ protected ClusterBlockException checkBlock(DeleteIndexRequest request, ClusterSt
     }
 
     @Override
-    protected void masterOperation(final DeleteIndexRequest request, final ClusterState state,
+    protected void masterOperation(Task task, final DeleteIndexRequest request, final ClusterState state,
                                    final ActionListener<AcknowledgedResponse> listener) {
         final Set<Index> concreteIndices = new HashSet<>(Arrays.asList(indexNameExpressionResolver.concreteIndices(state, request)));
         if (concreteIndices.isEmpty()) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/IndicesExistsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/IndicesExistsRequest.java
deleted file mode 100644
index 0c99175387db0..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/IndicesExistsRequest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.admin.indices.exists.indices;
-
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.IndicesRequest;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.action.support.master.MasterNodeReadRequest;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-
-import java.io.IOException;
-
-import static org.elasticsearch.action.ValidateActions.addValidationError;
-
-public class IndicesExistsRequest extends MasterNodeReadRequest<IndicesExistsRequest> implements IndicesRequest.Replaceable {
-
-    private String[] indices = Strings.EMPTY_ARRAY;
-    private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, true);
-
-    // for serialization
-    public IndicesExistsRequest() {
-
-    }
-
-    public IndicesExistsRequest(String... indices) {
-        this.indices = indices;
-    }
-
-    public IndicesExistsRequest(StreamInput in) throws IOException {
-        super(in);
-        indices = in.readStringArray();
-        indicesOptions = IndicesOptions.readIndicesOptions(in);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeStringArray(indices);
-        indicesOptions.writeIndicesOptions(out);
-    }
-
-    @Override
-    public String[] indices() {
-        return indices;
-    }
-
-    @Override
-    public IndicesExistsRequest indices(String... indices) {
-        this.indices = indices;
-        return this;
-    }
-
-    @Override
-    public IndicesOptions indicesOptions() {
-        return indicesOptions;
-    }
-
-    public IndicesExistsRequest expandWilcardsOpen(boolean expandWildcardsOpen) {
-        this.indicesOptions = IndicesOptions.fromOptions(indicesOptions.ignoreUnavailable(), indicesOptions.allowNoIndices(),
-                expandWildcardsOpen, indicesOptions.expandWildcardsClosed());
-        return this;
-    }
-
-    public IndicesExistsRequest expandWilcardsClosed(boolean expandWildcardsClosed) {
-        this.indicesOptions = IndicesOptions.fromOptions(indicesOptions.ignoreUnavailable(), indicesOptions.allowNoIndices(),
-                indicesOptions.expandWildcardsOpen(), expandWildcardsClosed);
-        return this;
-    }
-
-    @Override
-    public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = null;
-        if (indices == null || indices.length == 0) {
-            validationException = addValidationError("index/indices is missing", validationException);
-        }
-        return validationException;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/IndicesExistsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/IndicesExistsRequestBuilder.java
deleted file mode 100644
index 39a2ca7ef0a13..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/IndicesExistsRequestBuilder.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.admin.indices.exists.indices;
-
-import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
-import org.elasticsearch.client.ElasticsearchClient;
-
-public class IndicesExistsRequestBuilder
-        extends MasterNodeReadOperationRequestBuilder<IndicesExistsRequest, IndicesExistsResponse, IndicesExistsRequestBuilder> {
-
-    public IndicesExistsRequestBuilder(ElasticsearchClient client, IndicesExistsAction action, String... indices) {
-        super(client, action, new IndicesExistsRequest(indices));
-    }
-
-    public IndicesExistsRequestBuilder setIndices(String... indices) {
-        request.indices(indices);
-        return this;
-    }
-
-    /**
-     * Controls whether wildcard expressions will be expanded to existing open indices
-     */
-    public IndicesExistsRequestBuilder setExpandWildcardsOpen(boolean expandWildcardsOpen) {
-        request.expandWilcardsOpen(expandWildcardsOpen);
-        return this;
-    }
-
-    /**
-     * Controls whether wildcard expressions will be expanded to existing closed indices
-     */
-    public IndicesExistsRequestBuilder setExpandWildcardsClosed(boolean expandWildcardsClosed) {
-        request.expandWilcardsClosed(expandWildcardsClosed);
-        return this;
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java
deleted file mode 100644
index ba4d5eeea69d6..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.admin.indices.exists.indices;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.block.ClusterBlockException;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-
-/**
- * Indices exists action.
- */
-public class TransportIndicesExistsAction extends TransportMasterNodeReadAction<IndicesExistsRequest, IndicesExistsResponse> {
-
-    @Inject
-    public TransportIndicesExistsAction(TransportService transportService, ClusterService clusterService,
-                                        ThreadPool threadPool, ActionFilters actionFilters,
-                                        IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(IndicesExistsAction.NAME, transportService, clusterService, threadPool, actionFilters, IndicesExistsRequest::new,
-            indexNameExpressionResolver);
-    }
-
-    @Override
-    protected String executor() {
-        // lightweight in memory check
-        return ThreadPool.Names.SAME;
-    }
-
-    @Override
-    protected IndicesExistsResponse newResponse() {
-        return new IndicesExistsResponse();
-    }
-
-    @Override
-    protected ClusterBlockException checkBlock(IndicesExistsRequest request, ClusterState state) {
-        //make sure through indices options that the concrete indices call never throws IndexMissingException
-        IndicesOptions indicesOptions = IndicesOptions.fromOptions(true, true, request.indicesOptions().expandWildcardsOpen(),
-            request.indicesOptions().expandWildcardsClosed());
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ,
-            indexNameExpressionResolver.concreteIndexNames(state, indicesOptions, request.indices()));
-    }
-
-    @Override
-    protected void masterOperation(final IndicesExistsRequest request, final ClusterState state,
-                                   final ActionListener<IndicesExistsResponse> listener) {
-        boolean exists;
-        try {
-            // Similar as the previous behaviour, but now also aliases and wildcards are supported.
-            indexNameExpressionResolver.concreteIndexNames(state, request);
-            exists = true;
-        } catch (IndexNotFoundException e) {
-            exists = false;
-        }
-        listener.onResponse(new IndicesExistsResponse(exists));
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TransportTypesExistsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TransportTypesExistsAction.java
deleted file mode 100644
index 9b9c0ca7b1640..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TransportTypesExistsAction.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.action.admin.indices.exists.types;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.block.ClusterBlockException;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-
-/**
- * Types exists transport action.
- */
-public class TransportTypesExistsAction extends TransportMasterNodeReadAction<TypesExistsRequest, TypesExistsResponse> {
-
-    @Inject
-    public TransportTypesExistsAction(TransportService transportService, ClusterService clusterService,
-                                      ThreadPool threadPool, ActionFilters actionFilters,
-                                      IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(TypesExistsAction.NAME, transportService, clusterService, threadPool, actionFilters, TypesExistsRequest::new,
-            indexNameExpressionResolver);
-    }
-
-    @Override
-    protected String executor() {
-        // lightweight check
-        return ThreadPool.Names.SAME;
-    }
-
-    @Override
-    protected TypesExistsResponse newResponse() {
-        return new TypesExistsResponse();
-    }
-
-    @Override
-    protected ClusterBlockException checkBlock(TypesExistsRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ,
-            indexNameExpressionResolver.concreteIndexNames(state, request));
-    }
-
-    @Override
-    protected void masterOperation(final TypesExistsRequest request, final ClusterState state,
-                                   final ActionListener<TypesExistsResponse> listener) {
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request.indicesOptions(), request.indices());
-        if (concreteIndices.length == 0) {
-            listener.onResponse(new TypesExistsResponse(false));
-            return;
-        }
-
-        for (String concreteIndex : concreteIndices) {
-            if (!state.metaData().hasConcreteIndex(concreteIndex)) {
-                listener.onResponse(new TypesExistsResponse(false));
-                return;
-            }
-
-            MappingMetaData mapping = state.metaData().getIndices().get(concreteIndex).mapping();
-            if (mapping == null) {
-                listener.onResponse(new TypesExistsResponse(false));
-                return;
-            }
-
-            for (String type : request.types()) {
-                if (mapping.type().equals(type) == false) {
-                    listener.onResponse(new TypesExistsResponse(false));
-                    return;
-                }
-            }
-        }
-
-        listener.onResponse(new TypesExistsResponse(true));
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TypesExistsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TypesExistsRequest.java
deleted file mode 100644
index 851f175f4a4b7..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TypesExistsRequest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.action.admin.indices.exists.types;
-
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.IndicesRequest;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.action.support.master.MasterNodeReadRequest;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-
-import java.io.IOException;
-
-import static org.elasticsearch.action.ValidateActions.addValidationError;
-
-public class TypesExistsRequest extends MasterNodeReadRequest<TypesExistsRequest> implements IndicesRequest.Replaceable {
-
-    private String[] indices;
-    private String[] types;
-
-    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
-
-    public TypesExistsRequest() {
-    }
-
-    public TypesExistsRequest(String[] indices, String... types) {
-        this.indices = indices;
-        this.types = types;
-    }
-
-    public TypesExistsRequest(StreamInput in) throws IOException {
-        super(in);
-        indices = in.readStringArray();
-        types = in.readStringArray();
-        indicesOptions = IndicesOptions.readIndicesOptions(in);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeStringArray(indices);
-        out.writeStringArray(types);
-        indicesOptions.writeIndicesOptions(out);
-    }
-
-    @Override
-    public String[] indices() {
-        return indices;
-    }
-
-    @Override
-    public TypesExistsRequest indices(String... indices) {
-        this.indices = indices;
-        return this;
-    }
-
-    public String[] types() {
-        return types;
-    }
-
-    public void types(String[] types) {
-        this.types = types;
-    }
-
-    @Override
-    public IndicesOptions indicesOptions() {
-        return indicesOptions;
-    }
-
-    public TypesExistsRequest indicesOptions(IndicesOptions indicesOptions) {
-        this.indicesOptions = indicesOptions;
-        return this;
-    }
-
-    @Override
-    public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = null;
-        if (indices == null) { // Specifying '*' via rest api results in an empty array
-            validationException = addValidationError("index/indices is missing", validationException);
-        }
-        if (types == null || types.length == 0) {
-            validationException = addValidationError("type/types is missing", validationException);
-        }
-
-        return validationException;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TypesExistsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TypesExistsRequestBuilder.java
deleted file mode 100644
index 607a423605bfc..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/exists/types/TypesExistsRequestBuilder.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.action.admin.indices.exists.types;
-
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.Strings;
-
-/**
- * A builder for {@link TypesExistsRequest}.
- */
-@Deprecated
-public class TypesExistsRequestBuilder
-        extends MasterNodeReadOperationRequestBuilder<TypesExistsRequest, TypesExistsResponse, TypesExistsRequestBuilder> {
-
-    /**
-     * @param indices What indices to check for types
-     */
-    public TypesExistsRequestBuilder(ElasticsearchClient client, TypesExistsAction action, String... indices) {
-        super(client, action, new TypesExistsRequest(indices, Strings.EMPTY_ARRAY));
-    }
-
-    TypesExistsRequestBuilder(ElasticsearchClient client, TypesExistsAction action) {
-        super(client, action, new TypesExistsRequest());
-    }
-
-    /**
-     * @param indices What indices to check for types
-     */
-    public TypesExistsRequestBuilder setIndices(String[] indices) {
-        request.indices(indices);
-        return this;
-    }
-
-    /**
-     * @param types The types to check if they exist
-     */
-    public TypesExistsRequestBuilder setTypes(String... types) {
-        request.types(types);
-        return this;
-    }
-
-    /**
-     * @param indicesOptions Specifies how to resolve indices that aren't active / ready and indices wildcard expressions
-     */
-    public TypesExistsRequestBuilder setIndicesOptions(IndicesOptions indicesOptions) {
-        request.indicesOptions(indicesOptions);
-        return this;
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushAction.java
index 60d5b43a6c189..aab1dc775b99e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.indices.flush;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class FlushAction extends Action<FlushResponse> {
+public class FlushAction extends StreamableResponseActionType<FlushResponse> {
 
     public static final FlushAction INSTANCE = new FlushAction();
     public static final String NAME = "indices:admin/flush";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushAction.java
index 5005cd2ec0878..70153b767eef1 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushAction.java
@@ -19,10 +19,10 @@
 
 package org.elasticsearch.action.admin.indices.flush;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 
-public class SyncedFlushAction extends Action<SyncedFlushResponse> {
+public class SyncedFlushAction extends StreamableResponseActionType<SyncedFlushResponse> {
 
     public static final SyncedFlushAction INSTANCE = new SyncedFlushAction();
     public static final String NAME = "indices:admin/synced_flush";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeAction.java
index 51095435343cb..98d8c61dad788 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.indices.forcemerge;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class ForceMergeAction extends Action<ForceMergeResponse> {
+public class ForceMergeAction extends StreamableResponseActionType<ForceMergeResponse> {
 
     public static final ForceMergeAction INSTANCE = new ForceMergeAction();
     public static final String NAME = "indices:admin/forcemerge";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexAction.java
index 86396f246a414..c8a9eb85068d0 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.indices.get;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class GetIndexAction extends Action<GetIndexResponse> {
+public class GetIndexAction extends StreamableResponseActionType<GetIndexResponse> {
 
     public static final GetIndexAction INSTANCE = new GetIndexAction();
     public static final String NAME = "indices:admin/get";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsAction.java
index d372d8cf93f30..a16c6d4c532f4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsAction.java
@@ -19,10 +19,10 @@
 
 package org.elasticsearch.action.admin.indices.mapping.get;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
-public class GetFieldMappingsAction extends Action<GetFieldMappingsResponse> {
+public class GetFieldMappingsAction extends ActionType<GetFieldMappingsResponse> {
 
     public static final GetFieldMappingsAction INSTANCE = new GetFieldMappingsAction();
     public static final String NAME = "indices:admin/mappings/fields/get";
@@ -31,11 +31,6 @@ private GetFieldMappingsAction() {
         super(NAME);
     }
 
-    @Override
-    public GetFieldMappingsResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<GetFieldMappingsResponse> getResponseReader() {
         return GetFieldMappingsResponse::new;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java
index 149cba9a33e17..576d3812c0cf4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.single.shard.SingleShardRequest;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
@@ -31,14 +30,20 @@
 
 public class GetFieldMappingsIndexRequest extends SingleShardRequest<GetFieldMappingsIndexRequest> {
 
-    private boolean probablySingleFieldRequest;
-    private boolean includeDefaults;
-    private String[] fields = Strings.EMPTY_ARRAY;
-    private String[] types = Strings.EMPTY_ARRAY;
+    private final boolean probablySingleFieldRequest;
+    private final boolean includeDefaults;
+    private final String[] fields;
+    private final String[] types;
 
     private OriginalIndices originalIndices;
 
-    public GetFieldMappingsIndexRequest() {
+    GetFieldMappingsIndexRequest(StreamInput in) throws IOException {
+        super(in);
+        types = in.readStringArray();
+        fields = in.readStringArray();
+        includeDefaults = in.readBoolean();
+        probablySingleFieldRequest = in.readBoolean();
+        originalIndices = OriginalIndices.readOriginalIndices(in);
     }
 
     GetFieldMappingsIndexRequest(GetFieldMappingsRequest other, String index, boolean probablySingleFieldRequest) {
@@ -92,14 +97,4 @@ public void writeTo(StreamOutput out) throws IOException {
         OriginalIndices.writeOriginalIndices(originalIndices, out);
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        types = in.readStringArray();
-        fields = in.readStringArray();
-        includeDefaults = in.readBoolean();
-        probablySingleFieldRequest = in.readBoolean();
-        originalIndices = OriginalIndices.readOriginalIndices(in);
-    }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsAction.java
index 8bae685fff5dd..332fe97bca6e2 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.indices.mapping.get;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class GetMappingsAction extends Action<GetMappingsResponse> {
+public class GetMappingsAction extends StreamableResponseActionType<GetMappingsResponse> {
 
     public static final GetMappingsAction INSTANCE = new GetMappingsAction();
     public static final String NAME = "indices:admin/mappings/get";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java
index 2ebbf81f693ca..97c6493dfd950 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -39,17 +40,17 @@
 public class TransportGetFieldMappingsAction extends HandledTransportAction<GetFieldMappingsRequest, GetFieldMappingsResponse> {
 
     private final ClusterService clusterService;
-    private final TransportGetFieldMappingsIndexAction shardAction;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
+    private final NodeClient client;
 
     @Inject
     public TransportGetFieldMappingsAction(TransportService transportService, ClusterService clusterService,
-                                           TransportGetFieldMappingsIndexAction shardAction,
-                                           ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+                                           ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                           NodeClient client) {
         super(GetFieldMappingsAction.NAME, transportService, actionFilters, GetFieldMappingsRequest::new);
         this.clusterService = clusterService;
-        this.shardAction = shardAction;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
+        this.client = client;
     }
 
     @Override
@@ -66,7 +67,8 @@ protected void doExecute(Task task, GetFieldMappingsRequest request, final Actio
             boolean probablySingleFieldRequest = concreteIndices.length == 1 && request.types().length == 1 && request.fields().length == 1;
             for (final String index : concreteIndices) {
                 GetFieldMappingsIndexRequest shardRequest = new GetFieldMappingsIndexRequest(request, index, probablySingleFieldRequest);
-                shardAction.execute(shardRequest, new ActionListener<GetFieldMappingsResponse>() {
+
+                client.executeLocally(TransportGetFieldMappingsIndexAction.TYPE, shardRequest, new ActionListener<>() {
                     @Override
                     public void onResponse(GetFieldMappingsResponse result) {
                         indexResponses.set(indexCounter.getAndIncrement(), result);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
index 61a598c361cc9..7e8845773bab8 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
@@ -63,6 +64,7 @@ public class TransportGetFieldMappingsIndexAction
         extends TransportSingleShardAction<GetFieldMappingsIndexRequest, GetFieldMappingsResponse> {
 
     private static final String ACTION_NAME = GetFieldMappingsAction.NAME + "[index]";
+    public static final ActionType<GetFieldMappingsResponse> TYPE = new ActionType<>(ACTION_NAME, GetFieldMappingsResponse::new);
 
     protected final ClusterService clusterService;
     private final IndicesService indicesService;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/MappingRequestValidator.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/MappingRequestValidator.java
deleted file mode 100644
index 8d6608c575874..0000000000000
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/MappingRequestValidator.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.admin.indices.mapping.put;
-
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.index.Index;
-
-/**
- * A validator that validates a {@link PutMappingRequest} before executing it.
- * @see TransportPutMappingAction.RequestValidators
- */
-public interface MappingRequestValidator {
-
-    /**
-     * Validates a given put mapping request with its associated concrete indices and the current state.
-     *
-     * @param request the request to validate
-     * @param state   the current cluster state
-     * @param indices the concrete indices that associated with the given put mapping request
-     * @return a non-null exception indicates a reason that the given request should be aborted; otherwise returns null.
-     */
-    Exception validateRequest(PutMappingRequest request, ClusterState state, Index[] indices);
-}
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingAction.java
index ac845e6644c7b..97d908f01eec1 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingAction.java
@@ -19,10 +19,11 @@
 
 package org.elasticsearch.action.admin.indices.mapping.put;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class PutMappingAction extends Action<AcknowledgedResponse> {
+public class PutMappingAction extends ActionType<AcknowledgedResponse> {
 
     public static final PutMappingAction INSTANCE = new PutMappingAction();
     public static final String NAME = "indices:admin/mapping/put";
@@ -32,7 +33,7 @@ private PutMappingAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
index 2dd67299e7512..cd88947482747 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.action.admin.indices.mapping.put;
 
 import com.carrotsearch.hppc.ObjectHashSet;
-
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
index acd0d10281463..f2a21858f5330 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
@@ -21,6 +21,7 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.RequestValidators;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
@@ -32,12 +33,16 @@
 import org.elasticsearch.cluster.metadata.MetaDataMappingService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.Collection;
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Put mapping action.
@@ -45,17 +50,21 @@
 public class TransportPutMappingAction extends TransportMasterNodeAction<PutMappingRequest, AcknowledgedResponse> {
 
     private final MetaDataMappingService metaDataMappingService;
-    private final RequestValidators requestValidators;
+    private final RequestValidators<PutMappingRequest> requestValidators;
 
     @Inject
-    public TransportPutMappingAction(TransportService transportService, ClusterService clusterService,
-                                     ThreadPool threadPool, MetaDataMappingService metaDataMappingService,
-                                     ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-                                     RequestValidators requestValidators) {
+    public TransportPutMappingAction(
+            final TransportService transportService,
+            final ClusterService clusterService,
+            final ThreadPool threadPool,
+            final MetaDataMappingService metaDataMappingService,
+            final ActionFilters actionFilters,
+            final IndexNameExpressionResolver indexNameExpressionResolver,
+            final RequestValidators<PutMappingRequest> requestValidators) {
         super(PutMappingAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,
             PutMappingRequest::new);
         this.metaDataMappingService = metaDataMappingService;
-        this.requestValidators = requestValidators;
+        this.requestValidators = Objects.requireNonNull(requestValidators);
     }
 
     @Override
@@ -64,9 +73,14 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
@@ -81,15 +95,15 @@ protected ClusterBlockException checkBlock(PutMappingRequest request, ClusterSta
     }
 
     @Override
-    protected void masterOperation(final PutMappingRequest request, final ClusterState state,
+    protected void masterOperation(Task task, final PutMappingRequest request, final ClusterState state,
                                    final ActionListener<AcknowledgedResponse> listener) {
         try {
             final Index[] concreteIndices = request.getConcreteIndex() == null ?
                 indexNameExpressionResolver.concreteIndices(state, request)
                 : new Index[] {request.getConcreteIndex()};
-            final Exception validationException = requestValidators.validateRequest(request, state, concreteIndices);
-            if (validationException != null) {
-                listener.onFailure(validationException);
+            final Optional<Exception> maybeValidationException = requestValidators.validateRequest(request, state, concreteIndices);
+            if (maybeValidationException.isPresent()) {
+                listener.onFailure(maybeValidationException.get());
                 return;
             }
             PutMappingClusterStateUpdateRequest updateRequest = new PutMappingClusterStateUpdateRequest()
@@ -118,25 +132,4 @@ public void onFailure(Exception t) {
         }
     }
 
-
-    public static class RequestValidators {
-        private final Collection<MappingRequestValidator> validators;
-
-        public RequestValidators(Collection<MappingRequestValidator> validators) {
-            this.validators = validators;
-        }
-
-        private Exception validateRequest(PutMappingRequest request, ClusterState state, Index[] indices) {
-            Exception firstException = null;
-            for (MappingRequestValidator validator : validators) {
-                final Exception e = validator.validateRequest(request, state, indices);
-                if (firstException == null) {
-                    firstException = e;
-                } else {
-                    firstException.addSuppressed(e);
-                }
-            }
-            return firstException;
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexAction.java
index ea6f1eb6afd95..3dd871e97604c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexAction.java
@@ -19,9 +19,10 @@
 
 package org.elasticsearch.action.admin.indices.open;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class OpenIndexAction extends Action<OpenIndexResponse> {
+public class OpenIndexAction extends ActionType<OpenIndexResponse> {
 
     public static final OpenIndexAction INSTANCE = new OpenIndexAction();
     public static final String NAME = "indices:admin/open";
@@ -31,7 +32,7 @@ private OpenIndexAction() {
     }
 
     @Override
-    public OpenIndexResponse newResponse() {
-        return new OpenIndexResponse();
+    public Writeable.Reader<OpenIndexResponse> getResponseReader() {
+        return OpenIndexResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponse.java
index 91f27e8c0c3f7..c7abd0367b319 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponse.java
@@ -39,19 +39,14 @@ public class OpenIndexResponse extends ShardsAcknowledgedResponse {
         declareAcknowledgedAndShardsAcknowledgedFields(PARSER);
     }
 
-    public OpenIndexResponse() {
+    public OpenIndexResponse(StreamInput in) throws IOException {
+        super(in, true);
     }
 
     public OpenIndexResponse(boolean acknowledged, boolean shardsAcknowledged) {
         super(acknowledged, shardsAcknowledged);
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        readShardsAcknowledged(in);
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java
index b9323933f7601..c815a6b1132d7 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java
@@ -32,11 +32,14 @@
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 /**
  * Open index action
  */
@@ -64,7 +67,12 @@ protected String executor() {
 
     @Override
     protected OpenIndexResponse newResponse() {
-        return new OpenIndexResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected OpenIndexResponse read(StreamInput in) throws IOException {
+        return new OpenIndexResponse(in);
     }
 
     @Override
@@ -80,7 +88,7 @@ protected ClusterBlockException checkBlock(OpenIndexRequest request, ClusterStat
     }
 
     @Override
-    protected void masterOperation(final OpenIndexRequest request, final ClusterState state,
+    protected void masterOperation(Task task, final OpenIndexRequest request, final ClusterState state,
                                    final ActionListener<OpenIndexResponse> listener) {
         final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
         if (concreteIndices == null || concreteIndices.length == 0) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryAction.java
index bfe261b58843a..e41a766aa4b08 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/recovery/RecoveryAction.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.action.admin.indices.recovery;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
  * Recovery information action
  */
-public class RecoveryAction extends Action<RecoveryResponse> {
+public class RecoveryAction extends StreamableResponseActionType<RecoveryResponse> {
 
     public static final RecoveryAction INSTANCE = new RecoveryAction();
     public static final String NAME = "indices:monitor/recovery";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshAction.java
index b0dac076b2f49..f26d899371ecb 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.indices.refresh;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class RefreshAction extends Action<RefreshResponse> {
+public class RefreshAction extends StreamableResponseActionType<RefreshResponse> {
 
     public static final RefreshAction INSTANCE = new RefreshAction();
     public static final String NAME = "indices:admin/refresh";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverAction.java
index 0bcd4eefe0762..d0893d35a6754 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverAction.java
@@ -19,9 +19,10 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class RolloverAction extends Action<RolloverResponse> {
+public class RolloverAction extends ActionType<RolloverResponse> {
 
     public static final RolloverAction INSTANCE = new RolloverAction();
     public static final String NAME = "indices:admin/rollover";
@@ -31,7 +32,7 @@ private RolloverAction() {
     }
 
     @Override
-    public RolloverResponse newResponse() {
-        return new RolloverResponse();
+    public Writeable.Reader<RolloverResponse> getResponseReader() {
+        return RolloverResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
index cd58253d5e7c2..3c0431c7ea6c9 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
@@ -65,13 +65,27 @@ public final class RolloverResponse extends ShardsAcknowledgedResponse implement
         declareAcknowledgedAndShardsAcknowledgedFields(PARSER);
     }
 
-    private String oldIndex;
-    private String newIndex;
-    private Map<String, Boolean> conditionStatus;
-    private boolean dryRun;
-    private boolean rolledOver;
-
-    RolloverResponse() {
+    private final String oldIndex;
+    private final String newIndex;
+    private final Map<String, Boolean> conditionStatus;
+    private final boolean dryRun;
+    private final boolean rolledOver;
+    // Needs to be duplicated, because shardsAcknowledged gets (de)serailized as last field whereas
+    // in other subclasses of ShardsAcknowledgedResponse this field (de)serailized as first field.
+    private final boolean shardsAcknowledged;
+
+    RolloverResponse(StreamInput in) throws IOException {
+        super(in, false);
+        oldIndex = in.readString();
+        newIndex = in.readString();
+        int conditionSize = in.readVInt();
+        conditionStatus = new HashMap<>(conditionSize);
+        for (int i = 0; i < conditionSize; i++) {
+            conditionStatus.put(in.readString(), in.readBoolean());
+        }
+        dryRun = in.readBoolean();
+        rolledOver = in.readBoolean();
+        shardsAcknowledged = in.readBoolean();
     }
 
     public RolloverResponse(String oldIndex, String newIndex, Map<String, Boolean> conditionResults,
@@ -82,6 +96,7 @@ public RolloverResponse(String oldIndex, String newIndex, Map<String, Boolean> c
         this.dryRun = dryRun;
         this.rolledOver = rolledOver;
         this.conditionStatus = conditionResults;
+        this.shardsAcknowledged = shardsAcknowledged;
     }
 
     /**
@@ -120,18 +135,8 @@ public boolean isRolledOver() {
     }
 
     @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        oldIndex = in.readString();
-        newIndex = in.readString();
-        int conditionSize = in.readVInt();
-        conditionStatus = new HashMap<>(conditionSize);
-        for (int i = 0; i < conditionSize; i++) {
-            conditionStatus.put(in.readString(), in.readBoolean());
-        }
-        dryRun = in.readBoolean();
-        rolledOver = in.readBoolean();
-        readShardsAcknowledged(in);
+    public boolean isShardsAcknowledged() {
+        return shardsAcknowledged;
     }
 
     @Override
@@ -146,7 +151,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         out.writeBoolean(dryRun);
         out.writeBoolean(rolledOver);
-        writeShardsAcknowledged(out);
+        out.writeBoolean(shardsAcknowledged);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index c36d6092a7e49..60f099cb1d034 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -23,6 +23,8 @@
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.ActiveShardCount;
@@ -44,16 +46,21 @@
 import org.elasticsearch.cluster.metadata.MetaDataIndexAliasesService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.shard.DocsStats;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
@@ -89,7 +96,12 @@ protected String executor() {
 
     @Override
     protected RolloverResponse newResponse() {
-        return new RolloverResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected RolloverResponse read(StreamInput in) throws IOException {
+        return new RolloverResponse(in);
     }
 
     @Override
@@ -101,7 +113,7 @@ protected ClusterBlockException checkBlock(RolloverRequest request, ClusterState
     }
 
     @Override
-    protected void masterOperation(final RolloverRequest rolloverRequest, final ClusterState state,
+    protected void masterOperation(Task task, final RolloverRequest rolloverRequest, final ClusterState state,
                                    final ActionListener<RolloverResponse> listener) {
         final MetaData metaData = state.metaData();
         validate(metaData, rolloverRequest);
@@ -117,7 +129,9 @@ protected void masterOperation(final RolloverRequest rolloverRequest, final Clus
         final String rolloverIndexName = indexNameExpressionResolver.resolveDateMathExpression(unresolvedName);
         MetaDataCreateIndexService.validateIndexName(rolloverIndexName, state); // will fail if the index already exists
         checkNoDuplicatedAliasInIndexTemplate(metaData, rolloverIndexName, rolloverRequest.getAlias());
-        client.admin().indices().prepareStats(rolloverRequest.getAlias()).clear().setDocs(true).execute(
+        IndicesStatsRequest statsRequest = new IndicesStatsRequest().indices(rolloverRequest.getAlias()).clear().docs(true);
+        statsRequest.setParentTask(clusterService.localNode().getId(), task.getId());
+        client.execute(IndicesStatsAction.INSTANCE, statsRequest,
             new ActionListener<IndicesStatsResponse>() {
                 @Override
                 public void onResponse(IndicesStatsResponse statsResponse) {
@@ -233,7 +247,11 @@ static String generateRolloverIndexName(String sourceIndexName, IndexNameExpress
     }
 
     static Map<String, Boolean> evaluateConditions(final Collection<Condition<?>> conditions,
-                                                   final DocsStats docsStats, final IndexMetaData metaData) {
+                                                   @Nullable final DocsStats docsStats,
+                                                   @Nullable final IndexMetaData metaData) {
+        if (metaData == null) {
+            return conditions.stream().collect(Collectors.toMap(Condition::toString, cond -> false));
+        }
         final long numDocs = docsStats == null ? 0 : docsStats.getCount();
         final long indexSize = docsStats == null ? 0 : docsStats.getTotalSizeInBytes();
         final Condition.Stats stats = new Condition.Stats(numDocs, metaData.getCreationDate(), new ByteSizeValue(indexSize));
@@ -242,9 +260,18 @@ static Map<String, Boolean> evaluateConditions(final Collection<Condition<?>> co
             .collect(Collectors.toMap(result -> result.condition.toString(), result -> result.matched));
     }
 
-    static Map<String, Boolean> evaluateConditions(final Collection<Condition<?>> conditions, final IndexMetaData metaData,
-                                                    final IndicesStatsResponse statsResponse) {
-        return evaluateConditions(conditions, statsResponse.getIndex(metaData.getIndex().getName()).getPrimaries().getDocs(), metaData);
+    static Map<String, Boolean> evaluateConditions(final Collection<Condition<?>> conditions,
+                                                   @Nullable final IndexMetaData metaData,
+                                                   @Nullable final IndicesStatsResponse statsResponse) {
+        if (metaData == null) {
+            return conditions.stream().collect(Collectors.toMap(Condition::toString, cond -> false));
+        } else {
+            final DocsStats docsStats = Optional.ofNullable(statsResponse)
+                .map(stats -> stats.getIndex(metaData.getIndex().getName()))
+                .map(indexStats -> indexStats.getPrimaries().getDocs())
+                .orElse(null);
+            return evaluateConditions(conditions, docsStats, metaData);
+        }
     }
 
     static void validate(MetaData metaData, RolloverRequest request) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsAction.java
index 669c31d6b087d..edc9df6e91207 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentsAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.indices.segments;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class IndicesSegmentsAction extends Action<IndicesSegmentResponse> {
+public class IndicesSegmentsAction extends StreamableResponseActionType<IndicesSegmentResponse> {
 
     public static final IndicesSegmentsAction INSTANCE = new IndicesSegmentsAction();
     public static final String NAME = "indices:monitor/segments";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsAction.java
index e4149aaf8f15f..6e17be7241745 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.indices.settings.get;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class GetSettingsAction extends Action<GetSettingsResponse> {
+public class GetSettingsAction extends StreamableResponseActionType<GetSettingsResponse> {
 
     public static final GetSettingsAction INSTANCE = new GetSettingsAction();
     public static final String NAME = "indices:monitor/settings/get";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java
index 7f76439793365..fd0004beba292 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -79,7 +80,8 @@ private static boolean isFilteredRequest(GetSettingsRequest request) {
     }
 
     @Override
-    protected void masterOperation(GetSettingsRequest request, ClusterState state, ActionListener<GetSettingsResponse> listener) {
+    protected void masterOperation(Task task, GetSettingsRequest request, ClusterState state,
+                                   ActionListener<GetSettingsResponse> listener) {
         Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
         ImmutableOpenMap.Builder<String, Settings> indexToSettingsBuilder = ImmutableOpenMap.builder();
         ImmutableOpenMap.Builder<String, Settings> indexToDefaultSettingsBuilder = ImmutableOpenMap.builder();
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
index 9de0343365ea5..c3192631e2cf5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
@@ -33,10 +33,14 @@
 import org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 public class TransportUpdateSettingsAction extends TransportMasterNodeAction<UpdateSettingsRequest, AcknowledgedResponse> {
 
     private final MetaDataUpdateSettingsService updateSettingsService;
@@ -73,13 +77,18 @@ protected ClusterBlockException checkBlock(UpdateSettingsRequest request, Cluste
             indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(final UpdateSettingsRequest request, final ClusterState state,
+    protected void masterOperation(Task task, final UpdateSettingsRequest request, final ClusterState state,
                                    final ActionListener<AcknowledgedResponse> listener) {
         final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
         UpdateSettingsClusterStateUpdateRequest clusterStateUpdateRequest = new UpdateSettingsClusterStateUpdateRequest()
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsAction.java
index eb6574fb53094..df9b5137bb3d1 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsAction.java
@@ -19,10 +19,11 @@
 
 package org.elasticsearch.action.admin.indices.settings.put;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class UpdateSettingsAction extends Action<AcknowledgedResponse> {
+public class UpdateSettingsAction extends ActionType<AcknowledgedResponse> {
 
     public static final UpdateSettingsAction INSTANCE = new UpdateSettingsAction();
     public static final String NAME = "indices:admin/settings/update";
@@ -32,7 +33,7 @@ private UpdateSettingsAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestBuilder.java
index f44c24144030b..6146dcf87d691 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestBuilder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.action.admin.indices.shards;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -33,7 +33,7 @@ public class IndicesShardStoreRequestBuilder extends MasterNodeReadOperationRequ
         IndicesShardStoresResponse,
         IndicesShardStoreRequestBuilder> {
 
-    public IndicesShardStoreRequestBuilder(ElasticsearchClient client, Action<IndicesShardStoresResponse> action, String... indices) {
+    public IndicesShardStoreRequestBuilder(ElasticsearchClient client, ActionType<IndicesShardStoresResponse> action, String... indices) {
         super(client, action, new IndicesShardStoresRequest(indices));
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresAction.java
index d3ce0077d5e94..b06145bff0931 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresAction.java
@@ -19,16 +19,16 @@
 
 package org.elasticsearch.action.admin.indices.shards;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for {@link TransportIndicesShardStoresAction}
+ * ActionType for {@link TransportIndicesShardStoresAction}
  *
  * Exposes shard store information for requested indices.
  * Shard store information reports which nodes hold shard copies, how recent they are
  * and any exceptions on opening the shard index or from previous engine failures
  */
-public class IndicesShardStoresAction extends Action<IndicesShardStoresResponse> {
+public class IndicesShardStoresAction extends StreamableResponseActionType<IndicesShardStoresResponse> {
 
     public static final IndicesShardStoresAction INSTANCE = new IndicesShardStoresAction();
     public static final String NAME = "indices:monitor/shard_stores";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
index 86198cb74391d..87433ce044e07 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
@@ -258,12 +258,9 @@ public void writeTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
+        public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
             builder.field("node", nodeId());
-            super.innerToXContent(builder, params);
-            builder.endObject();
-            return builder;
+            return super.innerToXContent(builder, params);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
index 38b11530c2d93..3967e5e31f930 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
@@ -46,6 +46,7 @@
 import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards;
 import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -87,7 +88,7 @@ protected IndicesShardStoresResponse newResponse() {
     }
 
     @Override
-    protected void masterOperation(IndicesShardStoresRequest request, ClusterState state,
+    protected void masterOperation(Task task, IndicesShardStoresRequest request, ClusterState state,
                                    ActionListener<IndicesShardStoresResponse> listener) {
         final RoutingTable routingTables = state.routingTable();
         final RoutingNodes routingNodes = state.getRoutingNodes();
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java
index 42226570c0930..28b650cf0b747 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java
@@ -19,9 +19,10 @@
 
 package org.elasticsearch.action.admin.indices.shrink;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class ResizeAction extends Action<ResizeResponse> {
+public class ResizeAction extends ActionType<ResizeResponse> {
 
     public static final ResizeAction INSTANCE = new ResizeAction();
     public static final String NAME = "indices:admin/resize";
@@ -31,7 +32,7 @@ private ResizeAction() {
     }
 
     @Override
-    public ResizeResponse newResponse() {
-        return new ResizeResponse();
+    public Writeable.Reader<ResizeResponse> getResponseReader() {
+        return ResizeResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java
index e4b9a34b00415..eddd3a0da203a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.action.admin.indices.shrink;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
@@ -27,7 +27,7 @@
 
 public class ResizeRequestBuilder extends AcknowledgedRequestBuilder<ResizeRequest, ResizeResponse,
     ResizeRequestBuilder> {
-    public ResizeRequestBuilder(ElasticsearchClient client, Action<ResizeResponse> action) {
+    public ResizeRequestBuilder(ElasticsearchClient client, ActionType<ResizeResponse> action) {
         super(client, action, new ResizeRequest());
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponse.java
index ffe59f1e3ae18..c017c8dc9fa90 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponse.java
@@ -20,9 +20,12 @@
 package org.elasticsearch.action.admin.indices.shrink;
 
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
 
+import java.io.IOException;
+
 /**
  * A response for a resize index action, either shrink or split index.
  */
@@ -35,7 +38,8 @@ public final class ResizeResponse extends CreateIndexResponse {
         declareFields(PARSER);
     }
 
-    ResizeResponse() {
+    ResizeResponse(StreamInput in) throws IOException {
+        super(in);
     }
 
     public ResizeResponse(boolean acknowledged, boolean shardsAcknowledged, String index) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkAction.java
index a2c689ba360ee..57de49307018c 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkAction.java
@@ -19,9 +19,10 @@
 
 package org.elasticsearch.action.admin.indices.shrink;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class ShrinkAction extends Action<ResizeResponse> {
+public class ShrinkAction extends ActionType<ResizeResponse> {
 
     public static final ShrinkAction INSTANCE = new ShrinkAction();
     public static final String NAME = "indices:admin/shrink";
@@ -31,7 +32,7 @@ private ShrinkAction() {
     }
 
     @Override
-    public ResizeResponse newResponse() {
-        return new ResizeResponse();
+    public Writeable.Reader<ResizeResponse> getResponseReader() {
+        return ResizeResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java
index ff4e643a08227..178c15e473bd4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java
@@ -24,6 +24,9 @@
 import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.client.Client;
@@ -35,14 +38,17 @@
 import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.Locale;
 import java.util.Objects;
 import java.util.Set;
@@ -81,7 +87,12 @@ protected String executor() {
 
     @Override
     protected ResizeResponse newResponse() {
-        return new ResizeResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected ResizeResponse read(StreamInput in) throws IOException {
+        return new ResizeResponse(in);
     }
 
     @Override
@@ -90,13 +101,16 @@ protected ClusterBlockException checkBlock(ResizeRequest request, ClusterState s
     }
 
     @Override
-    protected void masterOperation(final ResizeRequest resizeRequest, final ClusterState state,
+    protected void masterOperation(Task task, final ResizeRequest resizeRequest, final ClusterState state,
                                    final ActionListener<ResizeResponse> listener) {
 
         // there is no need to fetch docs stats for split but we keep it simple and do it anyway for simplicity of the code
         final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getSourceIndex());
         final String targetIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getTargetIndexRequest().index());
-        client.admin().indices().prepareStats(sourceIndex).clear().setDocs(true).execute(
+        IndicesStatsRequestBuilder statsRequestBuilder = client.admin().indices().prepareStats(sourceIndex).clear().setDocs(true);
+        IndicesStatsRequest statsRequest = statsRequestBuilder.request();
+        statsRequest.setParentTask(clusterService.localNode().getId(), task.getId());
+        client.execute(IndicesStatsAction.INSTANCE, statsRequest,
             ActionListener.delegateFailure(listener, (delegatedListener, indicesStatsResponse) -> {
                 CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(resizeRequest, state,
                     i -> {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsAction.java
index 6765279eb6b87..4d2717d6a006f 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.indices.stats;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class IndicesStatsAction extends Action<IndicesStatsResponse> {
+public class IndicesStatsAction extends StreamableResponseActionType<IndicesStatsResponse> {
 
     public static final IndicesStatsAction INSTANCE = new IndicesStatsAction();
     public static final String NAME = "indices:monitor/stats";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateAction.java
index a8614277b409d..b08d2886b495e 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteIndexTemplateAction.java
@@ -19,10 +19,11 @@
 
 package org.elasticsearch.action.admin.indices.template.delete;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class DeleteIndexTemplateAction extends Action<AcknowledgedResponse> {
+public class DeleteIndexTemplateAction extends ActionType<AcknowledgedResponse> {
 
     public static final DeleteIndexTemplateAction INSTANCE = new DeleteIndexTemplateAction();
     public static final String NAME = "indices:admin/template/delete";
@@ -32,7 +33,7 @@ private DeleteIndexTemplateAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java
index b2421ed5174e5..c59aac00f109d 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java
@@ -30,9 +30,13 @@
 import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 /**
  * Delete index action.
  */
@@ -56,9 +60,14 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
@@ -67,7 +76,7 @@ protected ClusterBlockException checkBlock(DeleteIndexTemplateRequest request, C
     }
 
     @Override
-    protected void masterOperation(final DeleteIndexTemplateRequest request, final ClusterState state,
+    protected void masterOperation(Task task, final DeleteIndexTemplateRequest request, final ClusterState state,
                                    final ActionListener<AcknowledgedResponse> listener) {
         indexTemplateService.removeTemplates(
             new MetaDataIndexTemplateService
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesAction.java
index ac00b80079ca1..743c8e4f1a937 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesAction.java
@@ -18,9 +18,9 @@
  */
 package org.elasticsearch.action.admin.indices.template.get;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class GetIndexTemplatesAction extends Action<GetIndexTemplatesResponse> {
+public class GetIndexTemplatesAction extends StreamableResponseActionType<GetIndexTemplatesResponse> {
 
     public static final GetIndexTemplatesAction INSTANCE = new GetIndexTemplatesAction();
     public static final String NAME = "indices:admin/template/get";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java
index cd4cb6494e92d..1d733667fc3d5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -63,7 +64,7 @@ protected GetIndexTemplatesResponse newResponse() {
     }
 
     @Override
-    protected void masterOperation(GetIndexTemplatesRequest request, ClusterState state,
+    protected void masterOperation(Task task, GetIndexTemplatesRequest request, ClusterState state,
                                    ActionListener<GetIndexTemplatesResponse> listener) {
         List<IndexTemplateMetaData> results;
 
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateAction.java
index 1edc49e991453..2e57d252deab4 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateAction.java
@@ -19,10 +19,11 @@
 
 package org.elasticsearch.action.admin.indices.template.put;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class PutIndexTemplateAction extends Action<AcknowledgedResponse> {
+public class PutIndexTemplateAction extends ActionType<AcknowledgedResponse> {
 
     public static final PutIndexTemplateAction INSTANCE = new PutIndexTemplateAction();
     public static final String NAME = "indices:admin/template/put";
@@ -32,7 +33,7 @@ private PutIndexTemplateAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
index 0202d67814888..95e6cf9a7d158 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
@@ -31,11 +31,15 @@
 import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 /**
  * Put index template action.
  */
@@ -61,9 +65,14 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
@@ -72,7 +81,7 @@ protected ClusterBlockException checkBlock(PutIndexTemplateRequest request, Clus
     }
 
     @Override
-    protected void masterOperation(final PutIndexTemplateRequest request, final ClusterState state,
+    protected void masterOperation(Task task, final PutIndexTemplateRequest request, final ClusterState state,
                                    final ActionListener<AcknowledgedResponse> listener) {
         String cause = request.cause();
         if (cause.length() == 0) {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/UpgradeStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/UpgradeStatusAction.java
index 57506b615d692..f111e342aaff3 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/UpgradeStatusAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/UpgradeStatusAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.indices.upgrade.get;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class UpgradeStatusAction extends Action<UpgradeStatusResponse> {
+public class UpgradeStatusAction extends StreamableResponseActionType<UpgradeStatusResponse> {
 
     public static final UpgradeStatusAction INSTANCE = new UpgradeStatusAction();
     public static final String NAME = "indices:monitor/upgrade";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java
index 645c67b3fdc05..a747b6f058bc6 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java
@@ -32,9 +32,13 @@
 import org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 public class TransportUpgradeSettingsAction extends TransportMasterNodeAction<UpgradeSettingsRequest, AcknowledgedResponse> {
 
     private final MetaDataUpdateSettingsService updateSettingsService;
@@ -59,13 +63,18 @@ protected ClusterBlockException checkBlock(UpgradeSettingsRequest request, Clust
         return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(final UpgradeSettingsRequest request, final ClusterState state,
+    protected void masterOperation(Task task, final UpgradeSettingsRequest request, final ClusterState state,
                                    final ActionListener<AcknowledgedResponse> listener) {
         UpgradeSettingsClusterStateUpdateRequest clusterStateUpdateRequest = new UpgradeSettingsClusterStateUpdateRequest()
                 .ackTimeout(request.timeout())
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/UpgradeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/UpgradeAction.java
index 7ec83930e44bf..3a3344d1a2edb 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/UpgradeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/UpgradeAction.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.action.admin.indices.upgrade.post;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
  * Upgrade index/indices action.
  */
-public class UpgradeAction extends Action<UpgradeResponse> {
+public class UpgradeAction extends StreamableResponseActionType<UpgradeResponse> {
 
     public static final UpgradeAction INSTANCE = new UpgradeAction();
     public static final String NAME = "indices:admin/upgrade";
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/UpgradeSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/UpgradeSettingsAction.java
index 9b1ff0f616a54..3876e7be88225 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/UpgradeSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/UpgradeSettingsAction.java
@@ -19,10 +19,11 @@
 
 package org.elasticsearch.action.admin.indices.upgrade.post;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class UpgradeSettingsAction extends Action<AcknowledgedResponse> {
+public class UpgradeSettingsAction extends ActionType<AcknowledgedResponse> {
 
     public static final UpgradeSettingsAction INSTANCE = new UpgradeSettingsAction();
     public static final String NAME = "internal:indices/admin/upgrade";
@@ -32,7 +33,7 @@ private UpgradeSettingsAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java
index 6e10d3d42187f..8f85e91d29ee2 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java
@@ -193,7 +193,7 @@ protected ShardValidateQueryResponse shardOperation(ShardValidateQueryRequest re
         boolean valid;
         String explanation = null;
         String error = null;
-        ShardSearchLocalRequest shardSearchLocalRequest = new ShardSearchLocalRequest(request.shardId(), request.types(),
+        ShardSearchLocalRequest shardSearchLocalRequest = new ShardSearchLocalRequest(request.shardId(),
             request.nowInMillis(), request.filteringAliases());
         SearchContext searchContext = searchService.createSearchContext(shardSearchLocalRequest, SearchService.NO_TIMEOUT);
         try {
diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryAction.java
index 93151dd8a2ba9..713f94003642a 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.indices.validate.query;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class ValidateQueryAction extends Action<ValidateQueryResponse> {
+public class ValidateQueryAction extends StreamableResponseActionType<ValidateQueryResponse> {
 
     public static final ValidateQueryAction INSTANCE = new ValidateQueryAction();
     public static final String NAME = "indices:admin/validate/query";
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkAction.java
index f835b57a38b4c..bd4ac7ddf4788 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/BulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkAction.java
@@ -19,11 +19,11 @@
 
 package org.elasticsearch.action.bulk;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.transport.TransportRequestOptions;
 
-public class BulkAction extends Action<BulkResponse> {
+public class BulkAction extends StreamableResponseActionType<BulkResponse> {
 
     public static final BulkAction INSTANCE = new BulkAction();
     public static final String NAME = "indices:data/write/bulk";
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index eae849407666e..b0efa35819acb 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -96,7 +96,6 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
     private final AutoCreateIndex autoCreateIndex;
     private final ClusterService clusterService;
     private final IngestService ingestService;
-    private final TransportShardBulkAction shardBulkAction;
     private final LongSupplier relativeTimeProvider;
     private final IngestActionForwarder ingestForwarder;
     private final NodeClient client;
@@ -105,24 +104,21 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
     @Inject
     public TransportBulkAction(ThreadPool threadPool, TransportService transportService,
                                ClusterService clusterService, IngestService ingestService,
-                               TransportShardBulkAction shardBulkAction, NodeClient client,
-                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                               NodeClient client, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
                                AutoCreateIndex autoCreateIndex) {
-        this(threadPool, transportService, clusterService, ingestService, shardBulkAction, client, actionFilters,
+        this(threadPool, transportService, clusterService, ingestService, client, actionFilters,
             indexNameExpressionResolver, autoCreateIndex, System::nanoTime);
     }
 
     public TransportBulkAction(ThreadPool threadPool, TransportService transportService,
                                ClusterService clusterService, IngestService ingestService,
-                               TransportShardBulkAction shardBulkAction, NodeClient client,
-                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                               NodeClient client, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
                                AutoCreateIndex autoCreateIndex, LongSupplier relativeTimeProvider) {
         super(BulkAction.NAME, transportService, actionFilters, (Supplier<BulkRequest>) BulkRequest::new, ThreadPool.Names.WRITE);
         Objects.requireNonNull(relativeTimeProvider);
         this.threadPool = threadPool;
         this.clusterService = clusterService;
         this.ingestService = ingestService;
-        this.shardBulkAction = shardBulkAction;
         this.autoCreateIndex = autoCreateIndex;
         this.relativeTimeProvider = relativeTimeProvider;
         this.ingestForwarder = new IngestActionForwarder(transportService);
@@ -165,16 +161,24 @@ protected void doExecute(Task task, BulkRequest bulkRequest, ActionListener<Bulk
                 if (pipeline == null) {
                     // start to look for default pipeline via settings found in the index meta data
                     IndexMetaData indexMetaData = indicesMetaData.get(actionRequest.index());
+                    // check the alias for the index request (this is how normal index requests are modeled)
                     if (indexMetaData == null && indexRequest.index() != null) {
-                        // if the write request if through an alias use the write index's meta data
                         AliasOrIndex indexOrAlias = metaData.getAliasAndIndexLookup().get(indexRequest.index());
                         if (indexOrAlias != null && indexOrAlias.isAlias()) {
                             AliasOrIndex.Alias alias = (AliasOrIndex.Alias) indexOrAlias;
                             indexMetaData = alias.getWriteIndex();
                         }
                     }
+                    // check the alias for the action request (this is how upserts are modeled)
+                    if (indexMetaData == null && actionRequest.index() != null) {
+                        AliasOrIndex indexOrAlias = metaData.getAliasAndIndexLookup().get(actionRequest.index());
+                        if (indexOrAlias != null && indexOrAlias.isAlias()) {
+                            AliasOrIndex.Alias alias = (AliasOrIndex.Alias) indexOrAlias;
+                            indexMetaData = alias.getWriteIndex();
+                        }
+                    }
                     if (indexMetaData != null) {
-                        // Find the the default pipeline if one is defined from and existing index.
+                        // Find the default pipeline if one is defined from and existing index.
                         String defaultPipeline = IndexSettings.DEFAULT_PIPELINE.get(indexMetaData.getSettings());
                         indexRequest.setPipeline(defaultPipeline);
                         if (IngestService.NOOP_PIPELINE_NAME.equals(defaultPipeline) == false) {
@@ -432,7 +436,7 @@ protected void doRun() throws Exception {
                 if (task != null) {
                     bulkShardRequest.setParentTask(nodeId, task.getId());
                 }
-                shardBulkAction.execute(bulkShardRequest, new ActionListener<BulkShardResponse>() {
+                client.executeLocally(TransportShardBulkAction.TYPE, bulkShardRequest, new ActionListener<>() {
                     @Override
                     public void onResponse(BulkShardResponse bulkShardResponse) {
                         for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) {
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
index f49521a37463e..6479af1c7afc5 100644
--- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
+++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
@@ -24,10 +24,12 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.MessageSupplier;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.index.IndexRequest;
@@ -77,6 +79,12 @@
 public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequest, BulkShardRequest, BulkShardResponse> {
 
     public static final String ACTION_NAME = BulkAction.NAME + "[s]";
+    public static final ActionType<BulkShardResponse> TYPE = new StreamableResponseActionType<>(ACTION_NAME) {
+        @Override
+        public BulkShardResponse newResponse() {
+            return new BulkShardResponse();
+        }
+    };
 
     private static final Logger logger = LogManager.getLogger(TransportShardBulkAction.class);
 
@@ -341,11 +349,10 @@ private static boolean isConflictException(final Exception e) {
     private static BulkItemResponse processUpdateResponse(final UpdateRequest updateRequest, final String concreteIndex,
                                                           BulkItemResponse operationResponse, final UpdateHelper.Result translate) {
         final BulkItemResponse response;
-        DocWriteResponse.Result translatedResult = translate.getResponseResult();
         if (operationResponse.isFailed()) {
             response = new BulkItemResponse(operationResponse.getItemId(), DocWriteRequest.OpType.UPDATE, operationResponse.getFailure());
         } else {
-
+            final DocWriteResponse.Result translatedResult = translate.getResponseResult();
             final UpdateResponse updateResponse;
             if (translatedResult == DocWriteResponse.Result.CREATED || translatedResult == DocWriteResponse.Result.UPDATED) {
                 final IndexRequest updateIndexRequest = translate.action();
diff --git a/server/src/main/java/org/elasticsearch/action/delete/DeleteAction.java b/server/src/main/java/org/elasticsearch/action/delete/DeleteAction.java
index d78b6f60bffc3..be299715095dc 100644
--- a/server/src/main/java/org/elasticsearch/action/delete/DeleteAction.java
+++ b/server/src/main/java/org/elasticsearch/action/delete/DeleteAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.delete;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class DeleteAction extends Action<DeleteResponse> {
+public class DeleteAction extends StreamableResponseActionType<DeleteResponse> {
 
     public static final DeleteAction INSTANCE = new DeleteAction();
     public static final String NAME = "indices:data/write/delete";
diff --git a/server/src/main/java/org/elasticsearch/action/explain/ExplainAction.java b/server/src/main/java/org/elasticsearch/action/explain/ExplainAction.java
index ba5618ce7de21..2eab7215fce46 100644
--- a/server/src/main/java/org/elasticsearch/action/explain/ExplainAction.java
+++ b/server/src/main/java/org/elasticsearch/action/explain/ExplainAction.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.action.explain;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
 /**
  * Entry point for the explain feature.
  */
-public class ExplainAction extends Action<ExplainResponse> {
+public class ExplainAction extends ActionType<ExplainResponse> {
 
     public static final ExplainAction INSTANCE = new ExplainAction();
     public static final String NAME = "indices:data/read/explain";
@@ -34,11 +34,6 @@ private ExplainAction() {
         super(NAME);
     }
 
-    @Override
-    public ExplainResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<ExplainResponse> getResponseReader() {
         return ExplainResponse::new;
diff --git a/server/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java b/server/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java
index 260c63d340a6f..bbb9d24469c9e 100644
--- a/server/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java
@@ -74,6 +74,19 @@ public ExplainRequest(String index, String id) {
         this.id = id;
     }
 
+    ExplainRequest(StreamInput in) throws IOException {
+        super(in);
+        type = in.readString();
+        id = in.readString();
+        routing = in.readOptionalString();
+        preference = in.readOptionalString();
+        query = in.readNamedWriteable(QueryBuilder.class);
+        filteringAlias = new AliasFilter(in);
+        storedFields = in.readOptionalStringArray();
+        fetchSourceContext = in.readOptionalWriteable(FetchSourceContext::new);
+        nowInMillis = in.readVLong();
+    }
+
     /**
      * @deprecated Types are in the process of being removed.
      */
@@ -184,20 +197,6 @@ public ActionRequestValidationException validate() {
         return validationException;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        type = in.readString();
-        id = in.readString();
-        routing = in.readOptionalString();
-        preference = in.readOptionalString();
-        query = in.readNamedWriteable(QueryBuilder.class);
-        filteringAlias = new AliasFilter(in);
-        storedFields = in.readOptionalStringArray();
-        fetchSourceContext = in.readOptionalWriteable(FetchSourceContext::new);
-        nowInMillis = in.readVLong();
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java b/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
index c29da21fe4afe..e6a2ef5be9d04 100644
--- a/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
+++ b/server/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.lease.Releasables;
@@ -38,7 +37,6 @@
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.mapper.IdFieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
@@ -110,14 +108,8 @@ protected void asyncShardOperation(ExplainRequest request, ShardId shardId,
 
     @Override
     protected ExplainResponse shardOperation(ExplainRequest request, ShardId shardId) throws IOException {
-        String[] types;
-        if (MapperService.SINGLE_MAPPING_NAME.equals(request.type())) { // typeless explain call
-            types = Strings.EMPTY_ARRAY;
-        } else {
-            types = new String[] { request.type() };
-        }
-        ShardSearchLocalRequest shardSearchLocalRequest = new ShardSearchLocalRequest(shardId,
-                types, request.nowInMillis, request.filteringAlias());
+        ShardSearchLocalRequest shardSearchLocalRequest = new ShardSearchLocalRequest(shardId, request.nowInMillis,
+                request.filteringAlias());
         SearchContext context = searchService.createSearchContext(shardSearchLocalRequest, SearchService.NO_TIMEOUT);
         Engine.GetResult result = null;
         try {
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesAction.java
index 39c6ecce308e0..e64a3b67c102b 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.fieldcaps;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class FieldCapabilitiesAction extends Action<FieldCapabilitiesResponse> {
+public class FieldCapabilitiesAction extends StreamableResponseActionType<FieldCapabilitiesResponse> {
 
     public static final FieldCapabilitiesAction INSTANCE = new FieldCapabilitiesAction();
     public static final String NAME = "indices:data/read/field_caps";
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java
index 81266f12e37b3..6be8daacd51e8 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java
@@ -30,11 +30,15 @@
 
 public class FieldCapabilitiesIndexRequest extends SingleShardRequest<FieldCapabilitiesIndexRequest> {
 
-    private String[] fields;
-    private OriginalIndices originalIndices;
+    private final String[] fields;
+    private final OriginalIndices originalIndices;
 
     // For serialization
-    FieldCapabilitiesIndexRequest() {}
+    FieldCapabilitiesIndexRequest(StreamInput in) throws IOException {
+        super(in);
+        fields = in.readStringArray();
+        originalIndices = OriginalIndices.readOriginalIndices(in);
+    }
 
     FieldCapabilitiesIndexRequest(String[] fields, String index, OriginalIndices originalIndices) {
         super(index);
@@ -61,13 +65,6 @@ public IndicesOptions indicesOptions() {
         return originalIndices.indicesOptions();
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        fields = in.readStringArray();
-        originalIndices = OriginalIndices.readOriginalIndices(in);
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java
index ffb98dd5224ba..3176d0d31390b 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -47,21 +48,20 @@
 
 public class TransportFieldCapabilitiesAction extends HandledTransportAction<FieldCapabilitiesRequest, FieldCapabilitiesResponse> {
     private final ThreadPool threadPool;
+    private final NodeClient client;
     private final ClusterService clusterService;
-    private final TransportFieldCapabilitiesIndexAction shardAction;
     private final RemoteClusterService remoteClusterService;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
 
     @Inject
-    public TransportFieldCapabilitiesAction(TransportService transportService,
-                                            ClusterService clusterService, ThreadPool threadPool,
-                                            TransportFieldCapabilitiesIndexAction shardAction,
-                                            ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+    public TransportFieldCapabilitiesAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                            NodeClient client, ActionFilters actionFilters,
+                                            IndexNameExpressionResolver indexNameExpressionResolver) {
         super(FieldCapabilitiesAction.NAME, transportService, actionFilters, FieldCapabilitiesRequest::new);
         this.threadPool = threadPool;
+        this.client = client;
         this.clusterService = clusterService;
         this.remoteClusterService = transportService.getRemoteClusterService();
-        this.shardAction = shardAction;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
     }
 
@@ -108,7 +108,8 @@ public void onFailure(Exception e) {
                 }
             };
             for (String index : concreteIndices) {
-                shardAction.execute(new FieldCapabilitiesIndexRequest(request.fields(), index, localIndices), innerListener);
+                client.executeLocally(TransportFieldCapabilitiesIndexAction.TYPE,
+                    new FieldCapabilitiesIndexRequest(request.fields(), index, localIndices), innerListener);
             }
 
             // this is the cross cluster part of this API - we force the other cluster to not merge the results but instead
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java
index 274633b12a613..f391bf82eb944 100644
--- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.fieldcaps;
 
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
 import org.elasticsearch.cluster.ClusterState;
@@ -47,6 +48,8 @@ public class TransportFieldCapabilitiesIndexAction extends TransportSingleShardA
     FieldCapabilitiesIndexResponse> {
 
     private static final String ACTION_NAME = FieldCapabilitiesAction.NAME + "[index]";
+    public static final ActionType<FieldCapabilitiesIndexResponse> TYPE =
+        new ActionType<>(ACTION_NAME, FieldCapabilitiesIndexResponse::new);
 
     private final IndicesService indicesService;
 
diff --git a/server/src/main/java/org/elasticsearch/action/get/GetAction.java b/server/src/main/java/org/elasticsearch/action/get/GetAction.java
index 05d1b6c5a4c02..a36ec9d2fdc79 100644
--- a/server/src/main/java/org/elasticsearch/action/get/GetAction.java
+++ b/server/src/main/java/org/elasticsearch/action/get/GetAction.java
@@ -19,10 +19,10 @@
 
 package org.elasticsearch.action.get;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
-public class GetAction extends Action<GetResponse> {
+public class GetAction extends ActionType<GetResponse> {
 
     public static final GetAction INSTANCE = new GetAction();
     public static final String NAME = "indices:data/read/get";
@@ -31,11 +31,6 @@ private GetAction() {
         super(NAME);
     }
 
-    @Override
-    public GetResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<GetResponse> getResponseReader() {
         return GetResponse::new;
diff --git a/server/src/main/java/org/elasticsearch/action/get/GetRequest.java b/server/src/main/java/org/elasticsearch/action/get/GetRequest.java
index 390553fd8b55b..932602b3a2d22 100644
--- a/server/src/main/java/org/elasticsearch/action/get/GetRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/get/GetRequest.java
@@ -69,6 +69,21 @@ public GetRequest() {
         type = MapperService.SINGLE_MAPPING_NAME;
     }
 
+    GetRequest(StreamInput in) throws IOException {
+        super(in);
+        type = in.readString();
+        id = in.readString();
+        routing = in.readOptionalString();
+        preference = in.readOptionalString();
+        refresh = in.readBoolean();
+        storedFields = in.readOptionalStringArray();
+        realtime = in.readBoolean();
+
+        this.versionType = VersionType.fromValue(in.readByte());
+        this.version = in.readLong();
+        fetchSourceContext = in.readOptionalWriteable(FetchSourceContext::new);
+    }
+
     /**
      * Constructs a new get request against the specified index. The {@link #id(String)} must also be set.
      */
@@ -261,22 +276,6 @@ public VersionType versionType() {
         return this.versionType;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        type = in.readString();
-        id = in.readString();
-        routing = in.readOptionalString();
-        preference = in.readOptionalString();
-        refresh = in.readBoolean();
-        storedFields = in.readOptionalStringArray();
-        realtime = in.readBoolean();
-
-        this.versionType = VersionType.fromValue(in.readByte());
-        this.version = in.readLong();
-        fetchSourceContext = in.readOptionalWriteable(FetchSourceContext::new);
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetAction.java
index 9b69e33239b82..9d989c5644d00 100644
--- a/server/src/main/java/org/elasticsearch/action/get/MultiGetAction.java
+++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.get;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class MultiGetAction extends Action<MultiGetResponse> {
+public class MultiGetAction extends StreamableResponseActionType<MultiGetResponse> {
 
     public static final MultiGetAction INSTANCE = new MultiGetAction();
     public static final String NAME = "indices:data/read/mget";
diff --git a/server/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java b/server/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java
index 55f1fc6f5b9dd..1e66766aaabfa 100644
--- a/server/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java
@@ -33,14 +33,26 @@ public class MultiGetShardRequest extends SingleShardRequest<MultiGetShardReques
 
     private int shardId;
     private String preference;
-    boolean realtime = true;
-    boolean refresh;
+    private boolean realtime;
+    private boolean refresh;
 
     IntArrayList locations;
     List<MultiGetRequest.Item> items;
 
-    public MultiGetShardRequest() {
+    MultiGetShardRequest(StreamInput in) throws IOException {
+        super(in);
+        int size = in.readVInt();
+        locations = new IntArrayList(size);
+        items = new ArrayList<>(size);
 
+        for (int i = 0; i < size; i++) {
+            locations.add(in.readVInt());
+            items.add(MultiGetRequest.Item.readItem(in));
+        }
+
+        preference = in.readOptionalString();
+        refresh = in.readBoolean();
+        realtime = in.readBoolean();
     }
 
     MultiGetShardRequest(MultiGetRequest multiGetRequest, String index, int shardId) {
@@ -108,23 +120,6 @@ public String[] indices() {
         return indices;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        int size = in.readVInt();
-        locations = new IntArrayList(size);
-        items = new ArrayList<>(size);
-
-        for (int i = 0; i < size; i++) {
-            locations.add(in.readVInt());
-            items.add(MultiGetRequest.Item.readItem(in));
-        }
-
-        preference = in.readOptionalString();
-        refresh = in.readBoolean();
-        realtime = in.readBoolean();
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
index 375bdc18b605a..d98fd30b00d14 100644
--- a/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
+++ b/server/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -40,16 +41,16 @@
 public class TransportMultiGetAction extends HandledTransportAction<MultiGetRequest, MultiGetResponse> {
 
     private final ClusterService clusterService;
-    private final TransportShardMultiGetAction shardAction;
+    private final NodeClient client;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
 
     @Inject
     public TransportMultiGetAction(TransportService transportService, ClusterService clusterService,
-                                   TransportShardMultiGetAction shardAction, ActionFilters actionFilters,
+                                   NodeClient client, ActionFilters actionFilters,
                                    IndexNameExpressionResolver resolver) {
         super(MultiGetAction.NAME, transportService, actionFilters, MultiGetRequest::new);
         this.clusterService = clusterService;
-        this.shardAction = shardAction;
+        this.client = client;
         this.indexNameExpressionResolver = resolver;
     }
 
@@ -105,7 +106,7 @@ protected void executeShardAction(ActionListener<MultiGetResponse> listener,
         final AtomicInteger counter = new AtomicInteger(shardRequests.size());
 
         for (final MultiGetShardRequest shardRequest : shardRequests.values()) {
-            shardAction.execute(shardRequest, new ActionListener<MultiGetShardResponse>() {
+            client.executeLocally(TransportShardMultiGetAction.TYPE, shardRequest, new ActionListener<>() {
                 @Override
                 public void onResponse(MultiGetShardResponse response) {
                     for (int i = 0; i < response.locations.size(); i++) {
diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
index 9b8ea6bd6cac5..e7921ca489ce8 100644
--- a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
+++ b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.get;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
@@ -40,6 +41,7 @@
 public class TransportShardMultiGetAction extends TransportSingleShardAction<MultiGetShardRequest, MultiGetShardResponse> {
 
     private static final String ACTION_NAME = MultiGetAction.NAME + "[shard]";
+    public static final ActionType<MultiGetShardResponse> TYPE = new ActionType<>(ACTION_NAME, MultiGetShardResponse::new);
 
     private final IndicesService indicesService;
 
diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexAction.java b/server/src/main/java/org/elasticsearch/action/index/IndexAction.java
index 4f3e6068a2a23..b6afe88b770dd 100644
--- a/server/src/main/java/org/elasticsearch/action/index/IndexAction.java
+++ b/server/src/main/java/org/elasticsearch/action/index/IndexAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.index;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class IndexAction extends Action<IndexResponse> {
+public class IndexAction extends StreamableResponseActionType<IndexResponse> {
 
     public static final IndexAction INSTANCE = new IndexAction();
     public static final String NAME = "indices:data/write/index";
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineAction.java b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineAction.java
index 902614a3e83b3..55565960c658c 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineAction.java
@@ -19,10 +19,11 @@
 
 package org.elasticsearch.action.ingest;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class DeletePipelineAction extends Action<AcknowledgedResponse> {
+public class DeletePipelineAction extends ActionType<AcknowledgedResponse> {
 
     public static final DeletePipelineAction INSTANCE = new DeletePipelineAction();
     public static final String NAME = "cluster:admin/ingest/pipeline/delete";
@@ -32,7 +33,7 @@ public DeletePipelineAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
index 0b5145ff18cb9..8f69d20968014 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
@@ -28,10 +28,14 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 public class DeletePipelineTransportAction extends TransportMasterNodeAction<DeletePipelineRequest, AcknowledgedResponse> {
 
     private final IngestService ingestService;
@@ -49,14 +53,19 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(DeletePipelineRequest request, ClusterState state,
-        ActionListener<AcknowledgedResponse> listener) throws Exception {
+    protected void masterOperation(Task task, DeletePipelineRequest request, ClusterState state,
+                                   ActionListener<AcknowledgedResponse> listener) throws Exception {
         ingestService.delete(request, listener);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineAction.java b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineAction.java
index b2305227ac67f..48340af4039c3 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.ingest;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class GetPipelineAction extends Action<GetPipelineResponse> {
+public class GetPipelineAction extends StreamableResponseActionType<GetPipelineResponse> {
 
     public static final GetPipelineAction INSTANCE = new GetPipelineAction();
     public static final String NAME = "cluster:admin/ingest/pipeline/get";
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java
index e24e0a833b4c9..4501629284e63 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -53,7 +54,7 @@ protected GetPipelineResponse newResponse() {
     }
 
     @Override
-    protected void masterOperation(GetPipelineRequest request, ClusterState state, ActionListener<GetPipelineResponse> listener)
+    protected void masterOperation(Task task, GetPipelineRequest request, ClusterState state, ActionListener<GetPipelineResponse> listener)
             throws Exception {
         listener.onResponse(new GetPipelineResponse(IngestService.getPipelines(state, request.getIds())));
     }
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/IngestActionForwarder.java b/server/src/main/java/org/elasticsearch/action/ingest/IngestActionForwarder.java
index ae5a736bde66d..db4ecf119421f 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/IngestActionForwarder.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/IngestActionForwarder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.action.ingest;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.ActionRequest;
@@ -47,7 +47,7 @@ public IngestActionForwarder(TransportService transportService) {
         ingestNodes = new DiscoveryNode[0];
     }
 
-    public void forwardIngestRequest(Action<?> action, ActionRequest request, ActionListener<?> listener) {
+    public void forwardIngestRequest(ActionType<?> action, ActionRequest request, ActionListener<?> listener) {
         transportService.sendRequest(randomIngestNode(), action.name(), request,
             new ActionListenerResponseHandler(listener, action.getResponseReader()));
     }
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineAction.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineAction.java
index 4ebcff127cccd..1a2aa7725129a 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineAction.java
@@ -19,10 +19,11 @@
 
 package org.elasticsearch.action.ingest;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class PutPipelineAction extends Action<AcknowledgedResponse> {
+public class PutPipelineAction extends ActionType<AcknowledgedResponse> {
 
     public static final PutPipelineAction INSTANCE = new PutPipelineAction();
     public static final String NAME = "cluster:admin/ingest/pipeline/put";
@@ -32,7 +33,7 @@ public PutPipelineAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
index be1528a354bc3..95a9dbc7254b3 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
@@ -32,11 +32,14 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.ingest.IngestInfo;
 import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -62,13 +65,18 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(PutPipelineRequest request, ClusterState state, ActionListener<AcknowledgedResponse> listener)
+    protected void masterOperation(Task task, PutPipelineRequest request, ClusterState state, ActionListener<AcknowledgedResponse> listener)
             throws Exception {
         NodesInfoRequest nodesInfoRequest = new NodesInfoRequest();
         nodesInfoRequest.clear();
diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineAction.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineAction.java
index afeb4e01fb0ff..befa729282e7e 100644
--- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineAction.java
+++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.ingest;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class SimulatePipelineAction extends Action<SimulatePipelineResponse> {
+public class SimulatePipelineAction extends StreamableResponseActionType<SimulatePipelineResponse> {
 
     public static final SimulatePipelineAction INSTANCE = new SimulatePipelineAction();
     public static final String NAME = "cluster:admin/ingest/pipeline/simulate";
diff --git a/server/src/main/java/org/elasticsearch/action/main/MainAction.java b/server/src/main/java/org/elasticsearch/action/main/MainAction.java
index 831ddd0983fad..535c63d2a8800 100644
--- a/server/src/main/java/org/elasticsearch/action/main/MainAction.java
+++ b/server/src/main/java/org/elasticsearch/action/main/MainAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.main;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class MainAction extends Action<MainResponse> {
+public class MainAction extends StreamableResponseActionType<MainResponse> {
 
     public static final String NAME = "cluster:monitor/main";
     public static final MainAction INSTANCE = new MainAction();
diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index fdccfad7b47f5..8475272a5e2cd 100644
--- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -140,24 +140,29 @@ public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPha
         } else {
             Boolean allowPartialResults = request.allowPartialSearchResults();
             assert allowPartialResults != null : "SearchRequest missing setting for allowPartialSearchResults";
-            if (allowPartialResults == false && shardFailures.get() != null ){
-                if (logger.isDebugEnabled()) {
-                    final ShardOperationFailedException[] shardSearchFailures = ExceptionsHelper.groupBy(buildShardFailures());
-                    Throwable cause = shardSearchFailures.length == 0 ? null :
-                        ElasticsearchException.guessRootCauses(shardSearchFailures[0].getCause())[0];
-                    logger.debug(() -> new ParameterizedMessage("{} shards failed for phase: [{}]",
-                            shardSearchFailures.length, getName()), cause);
-                }
-                onPhaseFailure(currentPhase, "Partial shards failure", null);
-            } else {
-                if (logger.isTraceEnabled()) {
-                    final String resultsFrom = results.getSuccessfulResults()
-                        .map(r -> r.getSearchShardTarget().toString()).collect(Collectors.joining(","));
-                    logger.trace("[{}] Moving to next phase: [{}], based on results from: {} (cluster state version: {})",
-                        currentPhase.getName(), nextPhase.getName(), resultsFrom, clusterStateVersion);
+            if (allowPartialResults == false && shardFailures.get() != null) {
+                // check if there are actual failures in the atomic array since
+                // successful retries can reset the failures to null
+                ShardOperationFailedException[] shardSearchFailures = buildShardFailures();
+                if (shardSearchFailures.length > 0) {
+                    if (logger.isDebugEnabled()) {
+                        int numShardFailures = shardSearchFailures.length;
+                        shardSearchFailures = ExceptionsHelper.groupBy(shardSearchFailures);
+                        Throwable cause = ElasticsearchException.guessRootCauses(shardSearchFailures[0].getCause())[0];
+                        logger.debug(() -> new ParameterizedMessage("{} shards failed for phase: [{}]",
+                            numShardFailures, getName()), cause);
+                    }
+                    onPhaseFailure(currentPhase, "Partial shards failure", null);
+                    return;
                 }
-                executePhase(nextPhase);
             }
+            if (logger.isTraceEnabled()) {
+                final String resultsFrom = results.getSuccessfulResults()
+                    .map(r -> r.getSearchShardTarget().toString()).collect(Collectors.joining(","));
+                logger.trace("[{}] Moving to next phase: [{}], based on results from: {} (cluster state version: {})",
+                    currentPhase.getName(), nextPhase.getName(), resultsFrom, clusterStateVersion);
+            }
+            executePhase(nextPhase);
         }
     }
 
@@ -279,8 +284,7 @@ public final SearchRequest getRequest() {
         return request;
     }
 
-    @Override
-    public final SearchResponse buildSearchResponse(InternalSearchResponse internalSearchResponse, String scrollId) {
+    protected final SearchResponse buildSearchResponse(InternalSearchResponse internalSearchResponse, String scrollId) {
         ShardSearchFailure[] failures = buildShardFailures();
         Boolean allowPartialResults = request.allowPartialSearchResults();
         assert allowPartialResults != null : "SearchRequest missing setting for allowPartialSearchResults";
@@ -291,6 +295,11 @@ public final SearchResponse buildSearchResponse(InternalSearchResponse internalS
             skippedOps.get(), buildTookInMillis(), failures, clusters);
     }
 
+    @Override
+    public void sendSearchResponse(InternalSearchResponse internalSearchResponse, String scrollId) {
+        listener.onResponse(buildSearchResponse(internalSearchResponse, scrollId));
+    }
+
     @Override
     public final void onPhaseFailure(SearchPhase phase, String msg, Throwable cause) {
         raisePhaseFailure(new SearchPhaseExecutionException(phase.getName(), msg, cause, buildShardFailures()));
@@ -311,11 +320,6 @@ public final void execute(Runnable command) {
         executor.execute(command);
     }
 
-    @Override
-    public final void onResponse(SearchResponse response) {
-        listener.onResponse(response);
-    }
-
     @Override
     public final void onFailure(Exception e) {
         listener.onFailure(e);
diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearScrollAction.java b/server/src/main/java/org/elasticsearch/action/search/ClearScrollAction.java
index 660ed1ee17860..c7b959ed0bd44 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ClearScrollAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ClearScrollAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.search;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class ClearScrollAction extends Action<ClearScrollResponse> {
+public class ClearScrollAction extends StreamableResponseActionType<ClearScrollResponse> {
 
     public static final ClearScrollAction INSTANCE = new ClearScrollAction();
     public static final String NAME = "indices:data/read/scroll/clear";
diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
index afc81b21da4d5..301cb600452b2 100644
--- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.search.collapse.CollapseBuilder;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 
-import java.io.IOException;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -65,7 +64,7 @@ private boolean isCollapseRequest() {
     }
 
     @Override
-    public void run() throws IOException {
+    public void run() {
         if (isCollapseRequest() && searchResponse.hits().getHits().length > 0) {
             SearchRequest searchRequest = context.getRequest();
             CollapseBuilder collapseBuilder = searchRequest.source().collapse();
diff --git a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
index 1bbca35cb9a54..2115b4fa99849 100644
--- a/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java
@@ -22,8 +22,8 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.search.ScoreDoc;
-import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.OriginalIndices;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
@@ -76,10 +76,10 @@ final class FetchSearchPhase extends SearchPhase {
     }
 
     @Override
-    public void run() throws IOException {
-        context.execute(new ActionRunnable<SearchResponse>(context) {
+    public void run() {
+        context.execute(new AbstractRunnable() {
             @Override
-            public void doRun() throws IOException {
+            protected void doRun() throws Exception {
                 // we do the heavy lifting in this inner run method where we reduce aggs etc. that's why we fork this phase
                 // off immediately instead of forking when we send back the response to the user since there we only need
                 // to merge together the fetched results which is a linear operation.
@@ -209,8 +209,8 @@ private void moveToNextPhase(SearchPhaseController searchPhaseController,
     private static SearchPhase sendResponsePhase(InternalSearchResponse response, String scrollId, SearchPhaseContext context) {
         return new SearchPhase("response") {
             @Override
-            public void run() throws IOException {
-                context.onResponse(context.buildSearchResponse(response, scrollId));
+            public void run() {
+                context.sendSearchResponse(response, scrollId);
             }
         };
     }
diff --git a/server/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java
index 65f613521c0a8..6b9efe30fb791 100644
--- a/server/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java
+++ b/server/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java
@@ -139,13 +139,13 @@ public final void run() {
                 for (int index = 0; index < shardsIts.size(); index++) {
                     final SearchShardIterator shardRoutings = shardsIts.get(index);
                     if (shardRoutings.size() == 0) {
-                        if(missingShards.length() >0 ){
+                        if(missingShards.length() > 0){
                             missingShards.append(", ");
                         }
                         missingShards.append(shardRoutings.shardId());
                     }
                 }
-                if (missingShards.length() >0) {
+                if (missingShards.length() > 0) {
                     //Status red - shard is missing all copies and would produce partial results for an index search
                     final String msg = "Search rejected due to missing shards ["+ missingShards +
                             "]. Consider using `allow_partial_search_results` setting to bypass this error.";
diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchAction.java
index 9017a7b94ecb4..dd34dd94e69f7 100644
--- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchAction.java
@@ -19,10 +19,10 @@
 
 package org.elasticsearch.action.search;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
-public class MultiSearchAction extends Action<MultiSearchResponse> {
+public class MultiSearchAction extends ActionType<MultiSearchResponse> {
 
     public static final MultiSearchAction INSTANCE = new MultiSearchAction();
     public static final String NAME = "indices:data/read/msearch";
@@ -31,11 +31,6 @@ private MultiSearchAction() {
         super(NAME);
     }
 
-    @Override
-    public MultiSearchResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<MultiSearchResponse> getResponseReader() {
         return MultiSearchResponse::new;
diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
index 528be0369166e..25500efd5ed5b 100644
--- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
@@ -170,7 +170,6 @@ public static void readMultiLineFormat(BytesReference data,
                                            CheckedBiConsumer<SearchRequest, XContentParser, IOException> consumer,
                                            String[] indices,
                                            IndicesOptions indicesOptions,
-                                           String[] types,
                                            String routing,
                                            String searchType,
                                            Boolean ccsMinimizeRoundtrips,
@@ -190,9 +189,6 @@ public static void readMultiLineFormat(BytesReference data,
             if (indicesOptions != null) {
                 searchRequest.indicesOptions(indicesOptions);
             }
-            if (types != null && types.length > 0) {
-                searchRequest.types(types);
-            }
             if (routing != null) {
                 searchRequest.routing(routing);
             }
@@ -219,8 +215,6 @@ public static void readMultiLineFormat(BytesReference data,
                                 throw new IllegalArgumentException("explicit index in multi search is not allowed");
                             }
                             searchRequest.indices(nodeStringArrayValue(value));
-                        } else if ("type".equals(entry.getKey()) || "types".equals(entry.getKey())) {
-                            searchRequest.types(nodeStringArrayValue(value));
                         } else if ("search_type".equals(entry.getKey()) || "searchType".equals(entry.getKey())) {
                             searchRequest.searchType(nodeStringValue(value, null));
                         } else if ("ccs_minimize_roundtrips".equals(entry.getKey()) || "ccsMinimizeRoundtrips".equals(entry.getKey())) {
@@ -320,9 +314,6 @@ public static void writeSearchRequestParams(SearchRequest request, XContentBuild
             xContentBuilder.field("ignore_unavailable", request.indicesOptions().ignoreUnavailable());
             xContentBuilder.field("allow_no_indices", request.indicesOptions().allowNoIndices());
         }
-        if (request.types() != null) {
-            xContentBuilder.field("types", request.types());
-        }
         if (request.searchType() != null) {
             xContentBuilder.field("search_type", request.searchType().name().toLowerCase(Locale.ROOT));
         }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchAction.java
index d665595e8d34d..402219001a28d 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchAction.java
@@ -19,10 +19,10 @@
 
 package org.elasticsearch.action.search;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
-public class SearchAction extends Action<SearchResponse> {
+public class SearchAction extends ActionType<SearchResponse> {
 
     public static final SearchAction INSTANCE = new SearchAction();
     public static final String NAME = "indices:data/read/search";
@@ -31,11 +31,6 @@ private SearchAction() {
         super(NAME);
     }
 
-    @Override
-    public SearchResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<SearchResponse> getResponseReader() {
         return SearchResponse::new;
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
index 9829ff6a98337..28838defa3ec4 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseContext.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.action.search;
 
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.search.SearchShardTarget;
@@ -32,7 +31,7 @@
 /**
  * This class provide contextual state and access to resources across multiple search phases.
  */
-interface SearchPhaseContext extends ActionListener<SearchResponse>, Executor {
+interface SearchPhaseContext extends Executor {
     // TODO maybe we can make this concrete later - for now we just implement this in the base class for all initial phases
 
     /**
@@ -56,11 +55,16 @@ interface SearchPhaseContext extends ActionListener<SearchResponse>, Executor {
     SearchRequest getRequest();
 
     /**
-     * Builds the final search response that should be send back to the user.
+     * Builds and sends the final search response back to the user.
      * @param internalSearchResponse the internal search response
      * @param scrollId an optional scroll ID if this search is a scroll search
      */
-    SearchResponse buildSearchResponse(InternalSearchResponse internalSearchResponse, String scrollId);
+    void sendSearchResponse(InternalSearchResponse internalSearchResponse, String scrollId);
+
+    /**
+     * Notifies the top-level listener of the provided exception
+     */
+    void onFailure(Exception e);
 
     /**
      * This method will communicate a fatal phase failure back to the user. In contrast to a shard failure
@@ -113,5 +117,4 @@ default void sendReleaseSearchContext(long contextId, Transport.Connection conne
      * a response is returned to the user indicating that all shards have failed.
      */
     void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPhase);
-
 }
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
index 0125084c37099..ca6cba2f43f91 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -21,6 +21,7 @@
 
 import com.carrotsearch.hppc.IntArrayList;
 import com.carrotsearch.hppc.ObjectObjectHashMap;
+
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.CollectionStatistics;
 import org.apache.lucene.search.FieldDoc;
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index 53dafc153fc4b..602853e10b292 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.search;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
@@ -91,8 +92,6 @@ public final class SearchRequest extends ActionRequest implements IndicesRequest
 
     private int preFilterShardSize = DEFAULT_PRE_FILTER_SHARD_SIZE;
 
-    private String[] types = Strings.EMPTY_ARRAY;
-
     private boolean ccsMinimizeRoundtrips = true;
 
     public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled();
@@ -172,7 +171,6 @@ private SearchRequest(SearchRequest searchRequest, String[] indices, String loca
         this.scroll = searchRequest.scroll;
         this.searchType = searchRequest.searchType;
         this.source = searchRequest.source;
-        this.types = searchRequest.types;
         this.localClusterAlias = localClusterAlias;
         this.absoluteStartMillis = absoluteStartMillis;
         this.finalReduce = finalReduce;
@@ -192,7 +190,14 @@ public SearchRequest(StreamInput in) throws IOException {
         preference = in.readOptionalString();
         scroll = in.readOptionalWriteable(Scroll::new);
         source = in.readOptionalWriteable(SearchSourceBuilder::new);
-        types = in.readStringArray();
+        if (in.getVersion().before(Version.V_8_0_0)) {
+            // types no longer relevant so ignore
+            String[] types = in.readStringArray();
+            if (types.length > 0) {
+                throw new IllegalStateException(
+                        "types are no longer supported in search requests but found [" + Arrays.toString(types) + "]");
+            }
+        }
         indicesOptions = IndicesOptions.readIndicesOptions(in);
         requestCache = in.readOptionalBoolean();
         batchedReduceSize = in.readVInt();
@@ -219,7 +224,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(preference);
         out.writeOptionalWriteable(scroll);
         out.writeOptionalWriteable(source);
-        out.writeStringArray(types);
+        if (out.getVersion().before(Version.V_8_0_0)) {
+            // types not supported so send an empty array to previous versions
+            out.writeStringArray(Strings.EMPTY_ARRAY);
+        }
         indicesOptions.writeIndicesOptions(out);
         out.writeOptionalBoolean(requestCache);
         out.writeVInt(batchedReduceSize);
@@ -342,35 +350,6 @@ public void setCcsMinimizeRoundtrips(boolean ccsMinimizeRoundtrips) {
         this.ccsMinimizeRoundtrips = ccsMinimizeRoundtrips;
     }
 
-    /**
-     * The document types to execute the search against. Defaults to be executed against
-     * all types.
-     *
-     * @deprecated Types are in the process of being removed. Instead of using a type, prefer to
-     * filter on a field on the document.
-     */
-    @Deprecated
-    public String[] types() {
-        return types;
-    }
-
-    /**
-     * The document types to execute the search against. Defaults to be executed against
-     * all types.
-     *
-     * @deprecated Types are in the process of being removed. Instead of using a type, prefer to
-     * filter on a field on the document.
-     */
-    @Deprecated
-    public SearchRequest types(String... types) {
-        Objects.requireNonNull(types, "types must not be null");
-        for (String type : types) {
-            Objects.requireNonNull(type, "type must not be null");
-        }
-        this.types = types;
-        return this;
-    }
-
     /**
      * A comma separated list of routing values to control the shards the search will be executed on.
      */
@@ -590,9 +569,6 @@ public String getDescription() {
                 sb.append("indices[");
                 Strings.arrayToDelimitedString(indices, ",", sb);
                 sb.append("], ");
-                sb.append("types[");
-                Strings.arrayToDelimitedString(types, ",", sb);
-                sb.append("], ");
                 sb.append("search_type[").append(searchType).append("], ");
                 if (source != null) {
 
@@ -626,7 +602,6 @@ public boolean equals(Object o) {
                 Objects.equals(source, that.source) &&
                 Objects.equals(requestCache, that.requestCache)  &&
                 Objects.equals(scroll, that.scroll) &&
-                Arrays.equals(types, that.types) &&
                 Objects.equals(batchedReduceSize, that.batchedReduceSize) &&
                 Objects.equals(maxConcurrentShardRequests, that.maxConcurrentShardRequests) &&
                 Objects.equals(preFilterShardSize, that.preFilterShardSize) &&
@@ -640,7 +615,7 @@ public boolean equals(Object o) {
     @Override
     public int hashCode() {
         return Objects.hash(searchType, Arrays.hashCode(indices), routing, preference, source, requestCache,
-                scroll, Arrays.hashCode(types), indicesOptions, batchedReduceSize, maxConcurrentShardRequests, preFilterShardSize,
+                scroll, indicesOptions, batchedReduceSize, maxConcurrentShardRequests, preFilterShardSize,
                 allowPartialSearchResults, localClusterAlias, absoluteStartMillis, ccsMinimizeRoundtrips);
     }
 
@@ -650,7 +625,6 @@ public String toString() {
                 "searchType=" + searchType +
                 ", indices=" + Arrays.toString(indices) +
                 ", indicesOptions=" + indicesOptions +
-                ", types=" + Arrays.toString(types) +
                 ", routing='" + routing + '\'' +
                 ", preference='" + preference + '\'' +
                 ", requestCache=" + requestCache +
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index 96c93c974cabb..ceaee96f5c131 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -58,17 +58,6 @@ public SearchRequestBuilder setIndices(String... indices) {
         return this;
     }
 
-    /**
-     * The document types to execute the search against. Defaults to be executed against
-     * all types.
-     * @deprecated Types are going away, prefer filtering on a field.
-     */
-    @Deprecated
-    public SearchRequestBuilder setTypes(String... types) {
-        request.types(types);
-        return this;
-    }
-
     /**
      * The search type to execute, defaults to {@link org.elasticsearch.action.search.SearchType#DEFAULT}.
      */
@@ -378,7 +367,9 @@ public SearchRequestBuilder setTrackScores(boolean trackScores) {
     }
 
     /**
-     * Indicates if the total hit count for the query should be tracked. Defaults to {@code true}
+     * Indicates if the total hit count for the query should be tracked. Requests will count total hit count accurately
+     * up to 10,000 by default, see {@link #setTrackTotalHitsUpTo(int)} to change this value or set to true/false to always/never
+     * count accurately.
      */
     public SearchRequestBuilder setTrackTotalHits(boolean trackTotalHits) {
         sourceBuilder().trackTotalHits(trackTotalHits);
@@ -386,7 +377,7 @@ public SearchRequestBuilder setTrackTotalHits(boolean trackTotalHits) {
     }
 
     /**
-     * Indicates if the total hit count for the query should be tracked. Defaults to {@code true}
+     * Indicates the total hit count that should be tracked accurately or null if the value is unset. Defaults to 10,000.
      */
     public SearchRequestBuilder setTrackTotalHitsUpTo(int trackTotalHitsUpTo) {
         sourceBuilder().trackTotalHitsUpTo(trackTotalHitsUpTo);
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAction.java b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAction.java
index 0b4adfc1ba55c..32c84f86ea038 100644
--- a/server/src/main/java/org/elasticsearch/action/search/SearchScrollAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/SearchScrollAction.java
@@ -19,10 +19,10 @@
 
 package org.elasticsearch.action.search;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
-public class SearchScrollAction extends Action<SearchResponse> {
+public class SearchScrollAction extends ActionType<SearchResponse> {
 
     public static final SearchScrollAction INSTANCE = new SearchScrollAction();
     public static final String NAME = "indices:data/read/scroll";
@@ -31,11 +31,6 @@ private SearchScrollAction() {
         super(NAME);
     }
 
-    @Override
-    public SearchResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<SearchResponse> getResponseReader() {
         return SearchResponse::new;
diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index a7c0a785c7fce..d4832fb0d7a10 100644
--- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -472,10 +473,89 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
         Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, searchRequest.routing(),
             searchRequest.indices());
         routingMap = routingMap == null ? Collections.emptyMap() : Collections.unmodifiableMap(routingMap);
-        String[] concreteIndices = new String[indices.length];
-        for (int i = 0; i < indices.length; i++) {
-            concreteIndices[i] = indices[i].getName();
+        Map<String, Float> concreteIndexBoosts = resolveIndexBoosts(searchRequest, clusterState);
+
+        if (shouldSplitIndices(searchRequest)) {
+            //Execute two separate searches when we can, so that indices that are being written to are searched as quickly as possible.
+            //Otherwise their search context would need to stay open for too long between the query and the fetch phase, due to other
+            //indices (possibly slower) being searched at the same time.
+            List<String> writeIndicesList = new ArrayList<>();
+            List<String> readOnlyIndicesList = new ArrayList<>();
+            splitIndices(indices, clusterState, writeIndicesList, readOnlyIndicesList);
+            String[] writeIndices = writeIndicesList.toArray(new String[0]);
+            String[] readOnlyIndices = readOnlyIndicesList.toArray(new String[0]);
+
+            if (readOnlyIndices.length == 0) {
+                executeSearch(task, timeProvider, searchRequest, localIndices, writeIndices, routingMap,
+                    aliasFilter, concreteIndexBoosts, remoteShardIterators, remoteConnections, clusterState, listener, clusters);
+            } else if (writeIndices.length == 0 && remoteShardIterators.isEmpty()) {
+                executeSearch(task, timeProvider, searchRequest, localIndices, readOnlyIndices, routingMap,
+                    aliasFilter, concreteIndexBoosts, remoteShardIterators, remoteConnections, clusterState, listener, clusters);
+            } else {
+                //Split the search in two whenever throttled indices are searched together with ordinary indices (local or remote), so
+                //that we don't keep the search context open for too long between query and fetch for ordinary indices due to slow indices.
+                CountDown countDown = new CountDown(2);
+                AtomicReference<Exception> exceptions = new AtomicReference<>();
+                SearchResponseMerger searchResponseMerger = createSearchResponseMerger(searchRequest.source(), timeProvider,
+                    searchService::createReduceContext);
+                CountDownActionListener<SearchResponse, SearchResponse> countDownActionListener =
+                    new CountDownActionListener<>(countDown, exceptions, listener) {
+                        @Override
+                        void innerOnResponse(SearchResponse searchResponse) {
+                            searchResponseMerger.add(searchResponse);
+                        }
+
+                        @Override
+                        SearchResponse createFinalResponse() {
+                            return searchResponseMerger.getMergedResponse(clusters);
+                        }
+                    };
+
+                //Note that the indices set to the new SearchRequest won't be retrieved from it, as they have been already resolved and
+                //will be provided separately to executeSearch.
+                SearchRequest writeIndicesRequest = SearchRequest.subSearchRequest(searchRequest, writeIndices,
+                    RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY, timeProvider.getAbsoluteStartMillis(), false);
+                executeSearch(task, timeProvider, writeIndicesRequest, localIndices, writeIndices, routingMap,
+                    aliasFilter, concreteIndexBoosts, remoteShardIterators, remoteConnections, clusterState, countDownActionListener,
+                    SearchResponse.Clusters.EMPTY);
+
+                //Note that the indices set to the new SearchRequest won't be retrieved from it, as they have been already resolved and
+                //will be provided separately to executeSearch.
+                SearchRequest readOnlyIndicesRequest = SearchRequest.subSearchRequest(searchRequest, readOnlyIndices,
+                    RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY, timeProvider.getAbsoluteStartMillis(), false);
+                executeSearch(task, timeProvider, readOnlyIndicesRequest, localIndices, readOnlyIndices, routingMap,
+                    aliasFilter, concreteIndexBoosts, Collections.emptyList(), (alias, id) -> null, clusterState, countDownActionListener,
+                    SearchResponse.Clusters.EMPTY);
+            }
+        } else {
+            String[] concreteIndices = Arrays.stream(indices).map(Index::getName).toArray(String[]::new);
+            executeSearch(task, timeProvider, searchRequest, localIndices, concreteIndices, routingMap,
+                aliasFilter, concreteIndexBoosts, remoteShardIterators, remoteConnections, clusterState, listener, clusters);
+        }
+    }
+
+    static boolean shouldSplitIndices(SearchRequest searchRequest) {
+        return searchRequest.scroll() == null && searchRequest.searchType() != DFS_QUERY_THEN_FETCH
+            && (searchRequest.source() == null || searchRequest.source().size() != 0);
+    }
+
+    static void splitIndices(Index[] indices, ClusterState clusterState, List<String> writeIndices, List<String> readOnlyIndices) {
+        for (Index index : indices) {
+            ClusterBlockException writeBlock = clusterState.blocks().indexBlockedException(ClusterBlockLevel.WRITE, index.getName());
+            if (writeBlock == null) {
+                writeIndices.add(index.getName());
+            } else {
+                readOnlyIndices.add(index.getName());
+            }
         }
+    }
+
+    private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, SearchRequest searchRequest,
+                               OriginalIndices localIndices, String[] concreteIndices, Map<String, Set<String>> routingMap,
+                               Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
+                               List<SearchShardIterator> remoteShardIterators, BiFunction<String, String, DiscoveryNode> remoteConnections,
+                               ClusterState clusterState, ActionListener<SearchResponse> listener, SearchResponse.Clusters clusters) {
+
         Map<String, Long> nodeSearchCounts = searchTransportService.getPendingSearchRequests();
         GroupShardsIterator<ShardIterator> localShardsIterator = clusterService.operationRouting().searchShards(clusterState,
                 concreteIndices, routingMap, searchRequest.preference(), searchService.getResponseCollectorService(), nodeSearchCounts);
@@ -484,8 +564,6 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
 
         failIfOverShardCountLimit(clusterService, shardIterators.size());
 
-        Map<String, Float> concreteIndexBoosts = resolveIndexBoosts(searchRequest, clusterState);
-
         // optimize search type for cases where there is only one shard group to search on
         if (shardIterators.size() == 1) {
             // if we only have one group, then we always want Q_T_F, no need for DFS, and no need to do THEN since we hit one shard
@@ -498,11 +576,9 @@ private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, Sea
         if (searchRequest.isSuggestOnly()) {
             // disable request cache if we have only suggest
             searchRequest.requestCache(false);
-            switch (searchRequest.searchType()) {
-                case DFS_QUERY_THEN_FETCH:
-                    // convert to Q_T_F if we have only suggest
-                    searchRequest.searchType(QUERY_THEN_FETCH);
-                    break;
+            if (searchRequest.searchType() == DFS_QUERY_THEN_FETCH) {
+                // convert to Q_T_F if we have only suggest
+                searchRequest.searchType(QUERY_THEN_FETCH);
             }
         }
 
@@ -611,22 +687,16 @@ private static void failIfOverShardCountLimit(ClusterService clusterService, int
         }
     }
 
-    abstract static class CCSActionListener<Response, FinalResponse> implements ActionListener<Response> {
-        private final String clusterAlias;
-        private final boolean skipUnavailable;
+    abstract static class CountDownActionListener<Response, FinalResponse> implements ActionListener<Response> {
         private final CountDown countDown;
-        private final AtomicInteger skippedClusters;
         private final AtomicReference<Exception> exceptions;
-        private final ActionListener<FinalResponse> originalListener;
+        private final ActionListener<FinalResponse> delegateListener;
 
-        CCSActionListener(String clusterAlias, boolean skipUnavailable, CountDown countDown, AtomicInteger skippedClusters,
-                          AtomicReference<Exception> exceptions, ActionListener<FinalResponse> originalListener) {
-            this.clusterAlias = clusterAlias;
-            this.skipUnavailable = skipUnavailable;
+        CountDownActionListener(CountDown countDown, AtomicReference<Exception> exceptions,
+                                ActionListener<FinalResponse> delegateListener) {
             this.countDown = countDown;
-            this.skippedClusters = skippedClusters;
             this.exceptions = exceptions;
-            this.originalListener = originalListener;
+            this.delegateListener = delegateListener;
         }
 
         @Override
@@ -637,26 +707,7 @@ public final void onResponse(Response response) {
 
         abstract void innerOnResponse(Response response);
 
-        @Override
-        public final void onFailure(Exception e) {
-            if (skipUnavailable) {
-                skippedClusters.incrementAndGet();
-            } else {
-                Exception exception = e;
-                if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(clusterAlias) == false) {
-                    exception = wrapRemoteClusterFailure(clusterAlias, e);
-                }
-                if (exceptions.compareAndSet(null, exception) == false) {
-                    exceptions.accumulateAndGet(exception, (previous, current) -> {
-                        current.addSuppressed(previous);
-                        return current;
-                    });
-                }
-            }
-            maybeFinish();
-        }
-
-        private void maybeFinish() {
+        final void maybeFinish() {
             if (countDown.countDown()) {
                 Exception exception = exceptions.get();
                 if (exception == null) {
@@ -664,17 +715,56 @@ private void maybeFinish() {
                     try {
                         response = createFinalResponse();
                     } catch(Exception e) {
-                        originalListener.onFailure(e);
+                        delegateListener.onFailure(e);
                         return;
                     }
-                    originalListener.onResponse(response);
+                    delegateListener.onResponse(response);
                 } else {
-                    originalListener.onFailure(exceptions.get());
+                    delegateListener.onFailure(exceptions.get());
                 }
             }
         }
 
         abstract FinalResponse createFinalResponse();
+
+        @Override
+        public void onFailure(Exception e) {
+            if (exceptions.compareAndSet(null, e) == false) {
+                exceptions.accumulateAndGet(e, (previous, current) -> {
+                    current.addSuppressed(previous);
+                    return current;
+                });
+            }
+            maybeFinish();
+        }
+    }
+
+    abstract static class CCSActionListener<Response, FinalResponse> extends CountDownActionListener<Response, FinalResponse> {
+        private final String clusterAlias;
+        private final boolean skipUnavailable;
+        private final AtomicInteger skippedClusters;
+
+        CCSActionListener(String clusterAlias, boolean skipUnavailable, CountDown countDown, AtomicInteger skippedClusters,
+                          AtomicReference<Exception> exceptions, ActionListener<FinalResponse> originalListener) {
+            super(countDown, exceptions, originalListener);
+            this.clusterAlias = clusterAlias;
+            this.skipUnavailable = skipUnavailable;
+            this.skippedClusters = skippedClusters;
+        }
+
+        @Override
+        public final void onFailure(Exception e) {
+            if (skipUnavailable) {
+                skippedClusters.incrementAndGet();
+                maybeFinish();
+            } else {
+                Exception exception = e;
+                if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(clusterAlias) == false) {
+                    exception = wrapRemoteClusterFailure(clusterAlias, e);
+                }
+                super.onFailure(exception);
+            }
+        }
     }
 
     private static RemoteTransportException wrapRemoteClusterFailure(String clusterAlias, Exception e) {
diff --git a/server/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java b/server/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
index 7aa7dfb62a63c..aa3e91c634a80 100644
--- a/server/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
+++ b/server/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -35,7 +36,7 @@
 import static org.elasticsearch.ExceptionsHelper.detailedMessage;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 
-public class DefaultShardOperationFailedException extends ShardOperationFailedException {
+public class DefaultShardOperationFailedException extends ShardOperationFailedException implements Writeable {
 
     private static final String INDEX = "index";
     private static final String SHARD_ID = "shard";
@@ -90,13 +91,13 @@ public String toString() {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+    public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         innerToXContent(builder, params);
         builder.endObject();
         return builder;
     }
-    
+
     protected XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("shard", shardId());
         builder.field("index", index());
diff --git a/server/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java b/server/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java
index ad72ef10139ba..00a89c982e02b 100644
--- a/server/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java
+++ b/server/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java
@@ -23,53 +23,14 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.threadpool.ThreadPool;
 
-import java.util.concurrent.Future;
-
 /**
  * An action listener that wraps another action listener and threading its execution.
  */
 public final class ThreadedActionListener<Response> implements ActionListener<Response> {
 
-    /**
-     * Wrapper that can be used to automatically wrap a listener in a threaded listener if needed.
-     */
-    public static class Wrapper {
-
-        private final Logger logger;
-        private final ThreadPool threadPool;
-
-        private final boolean threadedListener;
-
-        public Wrapper(Logger logger, Settings settings, ThreadPool threadPool) {
-            this.logger = logger;
-            this.threadPool = threadPool;
-             // Should the action listener be threaded or not by default. Action listeners are automatically threaded for
-            // the transport client in order to make sure client side code is not executed on IO threads.
-            this.threadedListener = TransportClient.CLIENT_TYPE.equals(Client.CLIENT_TYPE_SETTING_S.get(settings));
-        }
-
-        public <Response> ActionListener<Response> wrap(ActionListener<Response> listener) {
-            if (threadedListener == false) {
-                return listener;
-            }
-            // if its a future, the callback is very lightweight (flipping a bit) so no need to wrap it
-            if (listener instanceof Future) {
-                return listener;
-            }
-            // already threaded...
-            if (listener instanceof ThreadedActionListener) {
-                return listener;
-            }
-            return new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.LISTENER, listener, false);
-        }
-    }
-
     private final Logger logger;
     private final ThreadPool threadPool;
     private final String executor;
diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastOperationRequestBuilder.java
index fb6fdd3fa2421..dfcca19883e50 100644
--- a/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastOperationRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastOperationRequestBuilder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.action.support.broadcast;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -30,7 +30,7 @@ public abstract class BroadcastOperationRequestBuilder<
             RequestBuilder extends BroadcastOperationRequestBuilder<Request, Response, RequestBuilder>
         > extends ActionRequestBuilder<Request, Response> {
 
-    protected BroadcastOperationRequestBuilder(ElasticsearchClient client, Action<Response> action, Request request) {
+    protected BroadcastOperationRequestBuilder(ElasticsearchClient client, ActionType<Response> action, Request request) {
         super(client, action, request);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequestBuilder.java
index fd4432f27d690..d37d2d45917fe 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequestBuilder.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.action.support.master;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.unit.TimeValue;
 
@@ -29,7 +29,7 @@ public abstract class AcknowledgedRequestBuilder<Request extends AcknowledgedReq
         RequestBuilder extends AcknowledgedRequestBuilder<Request, Response, RequestBuilder>>
         extends MasterNodeOperationRequestBuilder<Request, Response, RequestBuilder> {
 
-    protected AcknowledgedRequestBuilder(ElasticsearchClient client, Action<Response> action, Request request) {
+    protected AcknowledgedRequestBuilder(ElasticsearchClient client, ActionType<Response> action, Request request) {
         super(client, action, request);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
index 21a0d3f21a804..4e1540fd995e1 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/AcknowledgedResponse.java
@@ -45,9 +45,11 @@ protected static <T extends AcknowledgedResponse> void declareAcknowledgedField(
             ObjectParser.ValueType.BOOLEAN);
     }
 
-    protected boolean acknowledged;
+    protected final boolean acknowledged;
 
-    public AcknowledgedResponse() {
+    public AcknowledgedResponse(StreamInput in) throws IOException {
+        super(in);
+        acknowledged = in.readBoolean();
     }
 
     public AcknowledgedResponse(boolean acknowledged) {
@@ -64,8 +66,7 @@ public final boolean isAcknowledged() {
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        acknowledged = in.readBoolean();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequestBuilder.java
index c99cfa0f13b2f..08c59b72e0919 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeOperationRequestBuilder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.action.support.master;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -32,7 +32,7 @@ public abstract class MasterNodeOperationRequestBuilder<Request extends MasterNo
         RequestBuilder extends MasterNodeOperationRequestBuilder<Request, Response, RequestBuilder>>
         extends ActionRequestBuilder<Request, Response> {
 
-    protected MasterNodeOperationRequestBuilder(ElasticsearchClient client, Action<Response> action, Request request) {
+    protected MasterNodeOperationRequestBuilder(ElasticsearchClient client, ActionType<Response> action, Request request) {
         super(client, action, request);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeReadOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeReadOperationRequestBuilder.java
index e17987264a28d..f385746365e9a 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeReadOperationRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/MasterNodeReadOperationRequestBuilder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.action.support.master;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.client.ElasticsearchClient;
 
@@ -30,7 +30,7 @@ public abstract class MasterNodeReadOperationRequestBuilder<Request extends Mast
         RequestBuilder extends MasterNodeReadOperationRequestBuilder<Request, Response, RequestBuilder>>
         extends MasterNodeOperationRequestBuilder<Request, Response, RequestBuilder> {
 
-    protected MasterNodeReadOperationRequestBuilder(ElasticsearchClient client, Action<Response> action, Request request) {
+    protected MasterNodeReadOperationRequestBuilder(ElasticsearchClient client, ActionType<Response> action, Request request) {
         super(client, action, request);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponse.java b/server/src/main/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponse.java
index 21ff8fe644997..e088f4414a217 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponse.java
@@ -42,10 +42,15 @@ protected static <T extends ShardsAcknowledgedResponse> void declareAcknowledged
                 ObjectParser.ValueType.BOOLEAN);
     }
 
-    private boolean shardsAcknowledged;
-
-
-    protected ShardsAcknowledgedResponse() {
+    private final boolean shardsAcknowledged;
+
+    protected ShardsAcknowledgedResponse(StreamInput in, boolean readShardsAcknowledged) throws IOException {
+        super(in);
+        if (readShardsAcknowledged) {
+            this.shardsAcknowledged = in.readBoolean();
+        } else {
+            this.shardsAcknowledged = false;
+        }
     }
 
     protected ShardsAcknowledgedResponse(boolean acknowledged, boolean shardsAcknowledged) {
@@ -63,10 +68,6 @@ public boolean isShardsAcknowledged() {
         return shardsAcknowledged;
     }
 
-    protected void readShardsAcknowledged(StreamInput in) throws IOException {
-        shardsAcknowledged = in.readBoolean();
-    }
-
     protected void writeShardsAcknowledged(StreamOutput out) throws IOException {
         out.writeBoolean(shardsAcknowledged);
     }
@@ -80,14 +81,14 @@ protected void addCustomFields(XContentBuilder builder, Params params) throws IO
     public boolean equals(Object o) {
         if (super.equals(o)) {
             ShardsAcknowledgedResponse that = (ShardsAcknowledgedResponse) o;
-            return shardsAcknowledged == that.shardsAcknowledged;
+            return isShardsAcknowledged() == that.isShardsAcknowledged();
         }
         return false;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), shardsAcknowledged);
+        return Objects.hash(super.hashCode(), isShardsAcknowledged());
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
index 595b72f8da803..03c7346df5aab 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
@@ -118,14 +118,8 @@ protected Response read(StreamInput in) throws IOException {
         return response;
     }
 
-    protected abstract void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) throws Exception;
-
-    /**
-     * Override this operation if access to the task parameter is needed
-     */
-    protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<Response> listener) throws Exception {
-        masterOperation(request, state, listener);
-    }
+    protected abstract void masterOperation(Task task, Request request, ClusterState state,
+                                            ActionListener<Response> listener) throws Exception;
 
     protected boolean localExecute(Request request) {
         return false;
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequestBuilder.java
index d6193b8d726cf..41833a563c4e4 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequestBuilder.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.action.support.master.info;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
@@ -30,7 +30,7 @@ public abstract class ClusterInfoRequestBuilder<Request extends ClusterInfoReque
         extends MasterNodeReadOperationRequestBuilder<Request, Response, Builder> {
 
 
-    protected ClusterInfoRequestBuilder(ElasticsearchClient client, Action<Response> action, Request request) {
+    protected ClusterInfoRequestBuilder(ElasticsearchClient client, ActionType<Response> action, Request request) {
         super(client, action, request);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java b/server/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java
index dc46bfc6d1d31..0c3c70c9d02dd 100644
--- a/server/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -46,7 +47,8 @@ protected String executor() {
     }
 
     @Override
-    protected final void masterOperation(final Request request, final ClusterState state, final ActionListener<Response> listener) {
+    protected final void masterOperation(Task task, final Request request, final ClusterState state,
+                                         final ActionListener<Response> listener) {
         String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request);
         doMasterOperation(request, concreteIndices, state, listener);
     }
diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodeRequest.java b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodeRequest.java
index 1ce32247eab18..21b09bdaaae0a 100644
--- a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodeRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodeRequest.java
@@ -19,33 +19,31 @@
 
 package org.elasticsearch.action.support.nodes;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
 
+// TODO: this class can be removed in master once 7.x is bumped to 7.4.0
 public abstract class BaseNodeRequest extends TransportRequest {
 
-    private String nodeId;
-
-    public BaseNodeRequest() {
-
-    }
-
-    protected BaseNodeRequest(String nodeId) {
-        this.nodeId = nodeId;
-    }
+    public BaseNodeRequest() {}
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        nodeId = in.readString();
+        if (in.getVersion().before(Version.V_7_3_0)) {
+            in.readString(); // previously nodeId
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeString(nodeId);
+        if (out.getVersion().before(Version.V_7_3_0)) {
+            out.writeString(""); // previously nodeId
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java
index dac584a614429..ca2932291fd3a 100644
--- a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java
@@ -38,8 +38,7 @@ public abstract class BaseNodesRequest<Request extends BaseNodesRequest<Request>
      *
      * See {@link DiscoveryNodes#resolveNodes} for a full description of the options.
      *
-     * TODO: once we stop using the transport client as a gateway to the cluster, we can get rid of this and resolve it to concrete nodes
-     * in the rest layer
+     * TODO: we can get rid of this and resolve it to concrete nodes in the rest layer
      **/
     private String[] nodesIds;
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/NodesOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/nodes/NodesOperationRequestBuilder.java
index 626481cef80c6..ca31a956224f3 100644
--- a/server/src/main/java/org/elasticsearch/action/support/nodes/NodesOperationRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/nodes/NodesOperationRequestBuilder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.action.support.nodes;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.unit.TimeValue;
@@ -28,7 +28,7 @@ public abstract class NodesOperationRequestBuilder<Request extends BaseNodesRequ
         RequestBuilder extends NodesOperationRequestBuilder<Request, Response, RequestBuilder>>
         extends ActionRequestBuilder<Request, Response> {
 
-    protected NodesOperationRequestBuilder(ElasticsearchClient client, Action<Response> action, Request request) {
+    protected NodesOperationRequestBuilder(ElasticsearchClient client, ActionType<Response> action, Request request) {
         super(client, action, request);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
index bc4fb5d38c328..43f661084a89d 100644
--- a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
@@ -119,15 +119,11 @@ protected NodesResponse newResponse(NodesRequest request, AtomicReferenceArray n
      */
     protected abstract NodesResponse newResponse(NodesRequest request, List<NodeResponse> responses, List<FailedNodeException> failures);
 
-    protected abstract NodeRequest newNodeRequest(String nodeId, NodesRequest request);
+    protected abstract NodeRequest newNodeRequest(NodesRequest request);
 
     protected abstract NodeResponse newNodeResponse();
 
-    protected abstract NodeResponse nodeOperation(NodeRequest request);
-
-    protected NodeResponse nodeOperation(NodeRequest request, Task task) {
-        return nodeOperation(request);
-    }
+    protected abstract NodeResponse nodeOperation(NodeRequest request, Task task);
 
     /**
      * resolve node ids to concrete nodes of the incoming request
@@ -174,7 +170,7 @@ void start() {
                 final DiscoveryNode node = nodes[i];
                 final String nodeId = node.getId();
                 try {
-                    TransportRequest nodeRequest = newNodeRequest(nodeId, request);
+                    TransportRequest nodeRequest = newNodeRequest(request);
                     if (task != null) {
                         nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId());
                     }
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
index 7917d9c05078b..d328f06eb6895 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
@@ -111,6 +111,7 @@ public void execute() throws Exception {
     private void handlePrimaryResult(final PrimaryResultT primaryResult) {
         this.primaryResult = primaryResult;
         primary.updateLocalCheckpointForShard(primary.routingEntry().allocationId().getId(), primary.localCheckpoint());
+        primary.updateGlobalCheckpointForShard(primary.routingEntry().allocationId().getId(), primary.globalCheckpoint());
         final ReplicaRequest replicaRequest = primaryResult.replicaRequest();
         if (replicaRequest != null) {
             if (logger.isTraceEnabled()) {
@@ -123,7 +124,7 @@ private void handlePrimaryResult(final PrimaryResultT primaryResult) {
             // is valid for this replication group. If we would sample in the reverse, the global checkpoint might be based on a subset
             // of the sampled replication group, and advanced further than what the given replication group would allow it to.
             // This would entail that some shards could learn about a global checkpoint that would be higher than its local checkpoint.
-            final long globalCheckpoint = primary.globalCheckpoint();
+            final long globalCheckpoint = primary.computedGlobalCheckpoint();
             // we have to capture the max_seq_no_of_updates after this request was completed on the primary to make sure the value of
             // max_seq_no_of_updates on replica when this request is executed is at least the value on the primary when it was executed
             // on.
@@ -341,16 +342,23 @@ public interface Primary<
         void updateGlobalCheckpointForShard(String allocationId, long globalCheckpoint);
 
         /**
-         * Returns the local checkpoint on the primary shard.
+         * Returns the persisted local checkpoint on the primary shard.
          *
          * @return the local checkpoint
          */
         long localCheckpoint();
 
         /**
-         * Returns the global checkpoint on the primary shard.
+         * Returns the global checkpoint computed on the primary shard.
          *
-         * @return the global checkpoint
+         * @return the computed global checkpoint
+         */
+        long computedGlobalCheckpoint();
+
+        /**
+         * Returns the persisted global checkpoint on the primary shard.
+         *
+         * @return the persisted global checkpoint
          */
         long globalCheckpoint();
 
@@ -419,16 +427,16 @@ void performOn(ShardRouting replica, RequestT replicaRequest,
     public interface ReplicaResponse {
 
         /**
-         * The local checkpoint for the shard.
+         * The persisted local checkpoint for the shard.
          *
-         * @return the local checkpoint
+         * @return the persisted local checkpoint
          **/
         long localCheckpoint();
 
         /**
-         * The global checkpoint for the shard.
+         * The persisted global checkpoint for the shard.
          *
-         * @return the global checkpoint
+         * @return the persisted global checkpoint
          **/
         long globalCheckpoint();
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java
index fed4c9a5f101f..d83319f62f0de 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequestBuilder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.action.support.replication;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
@@ -30,7 +30,7 @@ public abstract class ReplicationRequestBuilder<Request extends ReplicationReque
         RequestBuilder extends ReplicationRequestBuilder<Request, Response, RequestBuilder>>
         extends ActionRequestBuilder<Request, Response> {
 
-    protected ReplicationRequestBuilder(ElasticsearchClient client, Action<Response> action, Request request) {
+    protected ReplicationRequestBuilder(ElasticsearchClient client, ActionType<Response> action, Request request) {
         super(client, action, request);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
index d19009433deb5..e338c6e5c329e 100644
--- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
@@ -526,7 +526,7 @@ public void onResponse(Releasable releasable) {
                 final ReplicaResult replicaResult = shardOperationOnReplica(replicaRequest.getRequest(), replica);
                 releasable.close(); // release shard operation lock before responding to caller
                 final TransportReplicationAction.ReplicaResponse response =
-                        new ReplicaResponse(replica.getLocalCheckpoint(), replica.getGlobalCheckpoint());
+                        new ReplicaResponse(replica.getLocalCheckpoint(), replica.getLastSyncedGlobalCheckpoint());
                 replicaResult.respond(new ResponseListener(response));
             } catch (final Exception e) {
                 Releasables.closeWhileHandlingException(releasable); // release shard operation lock before responding to caller
@@ -892,10 +892,6 @@ public void close() {
             operationLock.close();
         }
 
-        public long getLocalCheckpoint() {
-            return indexShard.getLocalCheckpoint();
-        }
-
         public ShardRouting routingEntry() {
             return indexShard.routingEntry();
         }
@@ -943,7 +939,12 @@ public long localCheckpoint() {
 
         @Override
         public long globalCheckpoint() {
-            return indexShard.getGlobalCheckpoint();
+            return indexShard.getLastSyncedGlobalCheckpoint();
+        }
+
+        @Override
+        public long computedGlobalCheckpoint() {
+            return indexShard.getLastKnownGlobalCheckpoint();
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequestBuilder.java
index 924815d75b85a..722826615ae21 100644
--- a/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequestBuilder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.action.support.single.instance;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -29,7 +29,7 @@ public abstract class InstanceShardOperationRequestBuilder<Request extends Insta
         Response extends ActionResponse, RequestBuilder extends InstanceShardOperationRequestBuilder<Request, Response, RequestBuilder>>
         extends ActionRequestBuilder<Request, Response> {
 
-    protected InstanceShardOperationRequestBuilder(ElasticsearchClient client, Action<Response> action, Request request) {
+    protected InstanceShardOperationRequestBuilder(ElasticsearchClient client, ActionType<Response> action, Request request) {
         super(client, action, request);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardOperationRequestBuilder.java
index 4f1da43fd11bb..197b7e57bfeea 100644
--- a/server/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardOperationRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardOperationRequestBuilder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.action.support.single.shard;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -28,7 +28,7 @@ public abstract class SingleShardOperationRequestBuilder<Request extends SingleS
         RequestBuilder extends SingleShardOperationRequestBuilder<Request, Response, RequestBuilder>>
         extends ActionRequestBuilder<Request, Response> {
 
-    protected SingleShardOperationRequestBuilder(ElasticsearchClient client, Action<Response> action, Request request) {
+    protected SingleShardOperationRequestBuilder(ElasticsearchClient client, ActionType<Response> action, Request request) {
         super(client, action, request);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardRequest.java b/server/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardRequest.java
index 15b10832b4eff..747b6c54ee8a2 100644
--- a/server/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/support/single/shard/SingleShardRequest.java
@@ -48,6 +48,15 @@ public abstract class SingleShardRequest<Request extends SingleShardRequest<Requ
     public SingleShardRequest() {
     }
 
+    public SingleShardRequest(StreamInput in) throws IOException {
+        super(in);
+        if (in.readBoolean()) {
+            internalShardId = new ShardId(in);
+        }
+        index = in.readOptionalString();
+        // no need to pass threading over the network, they are always false when coming throw a thread pool
+    }
+
     protected SingleShardRequest(String index) {
         this.index = index;
     }
@@ -93,16 +102,6 @@ public IndicesOptions indicesOptions() {
         return INDICES_OPTIONS;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        if (in.readBoolean()) {
-            internalShardId = new ShardId(in);
-        }
-        index = in.readOptionalString();
-        // no need to pass threading over the network, they are always false when coming throw a thread pool
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
index 123ed11769aa7..260fbe2baca7f 100644
--- a/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
+++ b/server/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
@@ -51,7 +51,6 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.function.Supplier;
 
 import static org.elasticsearch.action.support.TransportActions.isShardNotAvailableException;
 
@@ -73,7 +72,7 @@ public abstract class TransportSingleShardAction<Request extends SingleShardRequ
 
     protected TransportSingleShardAction(String actionName, ThreadPool threadPool, ClusterService clusterService,
                                          TransportService transportService, ActionFilters actionFilters,
-                                         IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request,
+                                         IndexNameExpressionResolver indexNameExpressionResolver, Writeable.Reader<Request> request,
                                          String executor) {
         super(actionName, actionFilters, transportService.getTaskManager());
         this.threadPool = threadPool;
@@ -85,9 +84,9 @@ protected TransportSingleShardAction(String actionName, ThreadPool threadPool, C
         this.executor = executor;
 
         if (!isSubAction()) {
-            transportService.registerRequestHandler(actionName, request, ThreadPool.Names.SAME, new TransportHandler());
+            transportService.registerRequestHandler(actionName, ThreadPool.Names.SAME, request, new TransportHandler());
         }
-        transportService.registerRequestHandler(transportShardAction, request, ThreadPool.Names.SAME, new ShardTransportHandler());
+        transportService.registerRequestHandler(transportShardAction, ThreadPool.Names.SAME, request, new ShardTransportHandler());
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java
index 52e3c2cf44e33..9fba027e751a0 100644
--- a/server/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.action.support.tasks;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.unit.TimeValue;
@@ -33,7 +33,7 @@ public class TasksRequestBuilder<
             RequestBuilder extends TasksRequestBuilder<Request, Response, RequestBuilder>
         > extends ActionRequestBuilder<Request, Response> {
 
-    protected TasksRequestBuilder(ElasticsearchClient client, Action<Response> action, Request request) {
+    protected TasksRequestBuilder(ElasticsearchClient client, ActionType<Response> action, Request request) {
         super(client, action, request);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsAction.java
index a894b3480f10f..e02969856e9b0 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.termvectors;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class MultiTermVectorsAction extends Action<MultiTermVectorsResponse> {
+public class MultiTermVectorsAction extends StreamableResponseActionType<MultiTermVectorsResponse> {
 
     public static final MultiTermVectorsAction INSTANCE = new MultiTermVectorsAction();
     public static final String NAME = "indices:data/read/mtv";
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsItemResponse.java b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsItemResponse.java
index 14ac59cb132bd..9cacf166952fb 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsItemResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsItemResponse.java
@@ -53,16 +53,6 @@ public String getIndex() {
         return response.getIndex();
     }
 
-    /**
-     * The type of the document.
-     */
-    public String getType() {
-        if (failure != null) {
-            return failure.getType();
-        }
-        return response.getType();
-    }
-
     /**
      * The id of the document.
      */
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequest.java b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequest.java
index 5f869e2ac7235..96a207ca1a595 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequest.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.mapper.MapperService;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -52,8 +51,8 @@ public MultiTermVectorsRequest add(TermVectorsRequest termVectorsRequest) {
         return this;
     }
 
-    public MultiTermVectorsRequest add(String index, @Nullable String type, String id) {
-        requests.add(new TermVectorsRequest(index, type, id));
+    public MultiTermVectorsRequest add(String index, String id) {
+        requests.add(new TermVectorsRequest(index, id));
         return this;
     }
 
@@ -102,9 +101,6 @@ public void add(TermVectorsRequest template, @Nullable XContentParser parser) th
                                 throw new IllegalArgumentException("docs array element should include an object");
                             }
                             TermVectorsRequest termVectorsRequest = new TermVectorsRequest(template);
-                            if (termVectorsRequest.type() == null) {
-                                termVectorsRequest.type(MapperService.SINGLE_MAPPING_NAME);
-                            }
                             TermVectorsRequest.parseRequest(termVectorsRequest, parser);
                             add(termVectorsRequest);
                         }
@@ -143,7 +139,7 @@ public void readFrom(StreamInput in) throws IOException {
         int size = in.readVInt();
         requests = new ArrayList<>(size);
         for (int i = 0; i < size; i++) {
-            requests.add(TermVectorsRequest.readTermVectorsRequest(in));
+            requests.add(new TermVectorsRequest(in));
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequestBuilder.java
index b982c4f720930..7d0d1c781ec75 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsRequestBuilder.java
@@ -21,7 +21,6 @@
 
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.Nullable;
 
 public class MultiTermVectorsRequestBuilder extends ActionRequestBuilder<MultiTermVectorsRequest, MultiTermVectorsResponse> {
 
@@ -29,16 +28,16 @@ public MultiTermVectorsRequestBuilder(ElasticsearchClient client, MultiTermVecto
         super(client, action, new MultiTermVectorsRequest());
     }
 
-    public MultiTermVectorsRequestBuilder add(String index, @Nullable String type, Iterable<String> ids) {
+    public MultiTermVectorsRequestBuilder add(String index, Iterable<String> ids) {
         for (String id : ids) {
-            request.add(index, type, id);
+            request.add(index, id);
         }
         return this;
     }
 
-    public MultiTermVectorsRequestBuilder add(String index, @Nullable String type, String... ids) {
+    public MultiTermVectorsRequestBuilder add(String index, String... ids) {
         for (String id : ids) {
-            request.add(index, type, id);
+            request.add(index, id);
         }
         return this;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
index 97a2007410edd..2264958422ad1 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.termvectors;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -38,7 +39,6 @@ public class MultiTermVectorsResponse extends ActionResponse implements Iterable
      */
     public static class Failure implements Streamable {
         private String index;
-        private String type;
         private String id;
         private Exception cause;
 
@@ -46,9 +46,8 @@ public static class Failure implements Streamable {
 
         }
 
-        public Failure(String index, String type, String id, Exception cause) {
+        public Failure(String index, String id, Exception cause) {
             this.index = index;
-            this.type = type;
             this.id = id;
             this.cause = cause;
         }
@@ -60,16 +59,6 @@ public String getIndex() {
             return this.index;
         }
 
-        /**
-         * The type of the action.
-         *
-         * @deprecated Types are in the process of being removed.
-         */
-        @Deprecated
-        public String getType() {
-            return type;
-        }
-
         /**
          * The id of the action.
          */
@@ -93,7 +82,13 @@ public static Failure readFailure(StreamInput in) throws IOException {
         @Override
         public void readFrom(StreamInput in) throws IOException {
             index = in.readString();
-            type = in.readOptionalString();
+            if (in.getVersion().before(Version.V_8_0_0)) {
+                // types no longer relevant so ignore
+                String type = in.readOptionalString();
+                if (type != null) {
+                    throw new IllegalStateException("types are no longer supported but found [" + type + "]");
+                }
+            }
             id = in.readString();
             cause = in.readException();
         }
@@ -101,7 +96,10 @@ public void readFrom(StreamInput in) throws IOException {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(index);
-            out.writeOptionalString(type);
+            if (out.getVersion().before(Version.V_8_0_0)) {
+                // types not supported so send an empty array to previous versions
+                out.writeOptionalString(null);
+            }
             out.writeString(id);
             out.writeException(cause);
         }
@@ -134,7 +132,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 builder.startObject();
                 Failure failure = response.getFailure();
                 builder.field(Fields._INDEX, failure.getIndex());
-                builder.field(Fields._TYPE, failure.getType());
                 builder.field(Fields._ID, failure.getId());
                 ElasticsearchException.generateFailureXContent(builder, params, failure.getCause(), true);
                 builder.endObject();
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardRequest.java b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardRequest.java
index 81f1b023578a7..2255b3243f123 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardRequest.java
@@ -37,8 +37,17 @@ public class MultiTermVectorsShardRequest extends SingleShardRequest<MultiTermVe
     IntArrayList locations;
     List<TermVectorsRequest> requests;
 
-    public MultiTermVectorsShardRequest() {
+    MultiTermVectorsShardRequest(StreamInput in) throws IOException {
+        super(in);
+        int size = in.readVInt();
+        locations = new IntArrayList(size);
+        requests = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            locations.add(in.readVInt());
+            requests.add(new TermVectorsRequest(in));
+        }
 
+        preference = in.readOptionalString();
     }
 
     MultiTermVectorsShardRequest(String index, int shardId) {
@@ -86,20 +95,6 @@ public String[] indices() {
         return indices;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        int size = in.readVInt();
-        locations = new IntArrayList(size);
-        requests = new ArrayList<>(size);
-        for (int i = 0; i < size; i++) {
-            locations.add(in.readVInt());
-            requests.add(TermVectorsRequest.readTermVectorsRequest(in));
-        }
-
-        preference = in.readOptionalString();
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsAction.java
index 9b223eed3a3c8..a6696a28608d6 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsAction.java
@@ -19,10 +19,10 @@
 
 package org.elasticsearch.action.termvectors;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
-public class TermVectorsAction extends Action<TermVectorsResponse> {
+public class TermVectorsAction extends ActionType<TermVectorsResponse> {
 
     public static final TermVectorsAction INSTANCE = new TermVectorsAction();
     public static final String NAME = "indices:data/read/tv";
@@ -31,11 +31,6 @@ private TermVectorsAction() {
         super(NAME);
     }
 
-    @Override
-    public TermVectorsResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<TermVectorsResponse> getResponseReader() {
         return TermVectorsResponse::new;
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
index 0206ef3def02d..09f7a15dff795 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.action.termvectors;
 
-import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.RealtimeRequest;
 import org.elasticsearch.action.ValidateActions;
@@ -31,7 +31,6 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -39,7 +38,6 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
-import org.elasticsearch.rest.action.document.RestTermVectorsAction;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -54,18 +52,13 @@
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 
 /**
- * Request returning the term vector (doc frequency, positions, offsets) for a
- * document.
+ * Request returning the term vector (doc frequency, positions, offsets) for a document.
  * <p>
- * Note, the {@link #index()}, {@link #type(String)} and {@link #id(String)} are
- * required.
+ * Note, the {@link #index()} and {@link #id(String)} are required.
  */
 public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> implements RealtimeRequest {
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(
-        LogManager.getLogger(TermVectorsRequest.class));
 
     private static final ParseField INDEX = new ParseField("_index");
-    private static final ParseField TYPE = new ParseField("_type");
     private static final ParseField ID = new ParseField("_id");
     private static final ParseField ROUTING = new ParseField("routing");
     private static final ParseField VERSION = new ParseField("version");
@@ -79,8 +72,6 @@ public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> i
     private static final ParseField DOC = new ParseField("doc");
 
 
-    private String type;
-
     private String id;
 
     private BytesReference doc;
@@ -142,26 +133,65 @@ public void writeTo(StreamOutput out) throws IOException {
     public TermVectorsRequest() {
     }
 
+    TermVectorsRequest(StreamInput in) throws IOException {
+        super(in);
+        if (in.getVersion().before(Version.V_8_0_0)) {
+            // types no longer relevant so ignore
+            in.readString();
+        }
+        id = in.readString();
+
+        if (in.readBoolean()) {
+            doc = in.readBytesReference();
+            xContentType = in.readEnum(XContentType.class);
+        }
+        routing = in.readOptionalString();
+        preference = in.readOptionalString();
+        long flags = in.readVLong();
+
+        flagsEnum.clear();
+        for (Flag flag : Flag.values()) {
+            if ((flags & (1 << flag.ordinal())) != 0) {
+                flagsEnum.add(flag);
+            }
+        }
+        int numSelectedFields = in.readVInt();
+        if (numSelectedFields > 0) {
+            selectedFields = new HashSet<>();
+            for (int i = 0; i < numSelectedFields; i++) {
+                selectedFields.add(in.readString());
+            }
+        }
+        if (in.readBoolean()) {
+            perFieldAnalyzer = readPerFieldAnalyzer(in.readMap());
+        }
+        if (in.readBoolean()) {
+            filterSettings = new FilterSettings();
+            filterSettings.readFrom(in);
+        }
+        realtime = in.readBoolean();
+        versionType = VersionType.fromValue(in.readByte());
+        version = in.readLong();
+    }
+
     /**
      * Constructs a new term vector request for a document that will be fetch
-     * from the provided index. Use {@link #type(String)} and
-     * {@link #id(String)} to specify the document to load.
+     * from the provided index. Use and {@link #id(String)} to specify the
+     * document to load.
      */
-    public TermVectorsRequest(String index, String type, String id) {
+    public TermVectorsRequest(String index, String id) {
         super(index);
         this.id = id;
-        this.type = type;
     }
 
     /**
      * Constructs a new term vector request for a document that will be fetch
-     * from the provided index. Use {@link #type(String)} and
-     * {@link #id(String)} to specify the document to load.
+     * from the provided index. Use {@link #id(String)} to specify the
+     * document to load.
      */
     public TermVectorsRequest(TermVectorsRequest other) {
         super(other.index());
         this.id = other.id();
-        this.type = other.type();
         if (other.doc != null) {
             this.doc = new BytesArray(other.doc().toBytesRef(), true);
             this.xContentType = other.xContentType;
@@ -184,7 +214,6 @@ public TermVectorsRequest(TermVectorsRequest other) {
     public TermVectorsRequest(MultiGetRequest.Item item) {
         super(item.index());
         this.id = item.id();
-        this.type = item.type();
         this.selectedFields(item.storedFields());
         this.routing(item.routing());
     }
@@ -193,21 +222,6 @@ public EnumSet<Flag> getFlags() {
         return flagsEnum;
     }
 
-    /**
-     * Sets the type of document to get the term vector for.
-     */
-    public TermVectorsRequest type(String type) {
-        this.type = type;
-        return this;
-    }
-
-    /**
-     * Returns the type of document to get the term vector for.
-     */
-    public String type() {
-        return type;
-    }
-
     /**
      * Returns the id of document the term vector is requested for.
      */
@@ -461,65 +475,19 @@ private void setFlag(Flag flag, boolean set) {
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = super.validateNonNullIndex();
-        if (type == null) {
-            validationException = ValidateActions.addValidationError("type is missing", validationException);
-        }
         if (id == null && doc == null) {
             validationException = ValidateActions.addValidationError("id or doc is missing", validationException);
         }
         return validationException;
     }
 
-    public static TermVectorsRequest readTermVectorsRequest(StreamInput in) throws IOException {
-        TermVectorsRequest termVectorsRequest = new TermVectorsRequest();
-        termVectorsRequest.readFrom(in);
-        return termVectorsRequest;
-    }
-
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        type = in.readString();
-        id = in.readString();
-
-        if (in.readBoolean()) {
-            doc = in.readBytesReference();
-            xContentType = in.readEnum(XContentType.class);
-        }
-        routing = in.readOptionalString();
-        preference = in.readOptionalString();
-        long flags = in.readVLong();
-
-        flagsEnum.clear();
-        for (Flag flag : Flag.values()) {
-            if ((flags & (1 << flag.ordinal())) != 0) {
-                flagsEnum.add(flag);
-            }
-        }
-        int numSelectedFields = in.readVInt();
-        if (numSelectedFields > 0) {
-            selectedFields = new HashSet<>();
-            for (int i = 0; i < numSelectedFields; i++) {
-                selectedFields.add(in.readString());
-            }
-        }
-        if (in.readBoolean()) {
-            perFieldAnalyzer = readPerFieldAnalyzer(in.readMap());
-        }
-        if (in.readBoolean()) {
-            filterSettings = new FilterSettings();
-            filterSettings.readFrom(in);
-        }
-        realtime = in.readBoolean();
-        versionType = VersionType.fromValue(in.readByte());
-        version = in.readLong();
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeString(type);
+        if (out.getVersion().before(Version.V_8_0_0)) {
+            // types not supported so send an empty array to previous versions
+            out.writeString("_doc");
+        }
         out.writeString(id);
 
         out.writeBoolean(doc != null);
@@ -599,10 +567,6 @@ public static void parseRequest(TermVectorsRequest termVectorsRequest, XContentP
                 } else if (INDEX.match(currentFieldName, parser.getDeprecationHandler())) {
                     // the following is important for multi request parsing.
                     termVectorsRequest.index = parser.text();
-                } else if (TYPE.match(currentFieldName, parser.getDeprecationHandler())) {
-                    termVectorsRequest.type = parser.text();
-                    deprecationLogger.deprecatedAndMaybeLog("termvectors_with_types",
-                        RestTermVectorsAction.TYPES_DEPRECATION_MESSAGE);
                 } else if (ID.match(currentFieldName, parser.getDeprecationHandler())) {
                     if (termVectorsRequest.doc != null) {
                         throw new ElasticsearchParseException("failed to parse term vectors request. " +
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequestBuilder.java
index 80bf769b9714e..7bdf46bf08cba 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequestBuilder.java
@@ -44,8 +44,8 @@ public TermVectorsRequestBuilder(ElasticsearchClient client, TermVectorsAction a
      * from the provided index. Use {@code index}, {@code type} and
      * {@code id} to specify the document to load.
      */
-    public TermVectorsRequestBuilder(ElasticsearchClient client, TermVectorsAction action, String index, String type, String id) {
-        super(client, action, new TermVectorsRequest(index, type, id));
+    public TermVectorsRequestBuilder(ElasticsearchClient client, TermVectorsAction action, String index, String id) {
+        super(client, action, new TermVectorsRequest(index, id));
     }
 
     /**
@@ -56,14 +56,6 @@ public TermVectorsRequestBuilder setIndex(String index) {
         return this;
     }
 
-    /**
-     * Sets the type of the document.
-     */
-    public TermVectorsRequestBuilder setType(String type) {
-        request.type(type);
-        return this;
-    }
-
     /**
      * Sets the id of the document.
      */
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
index 3d0fb75f8d3eb..2cd1e3cccd7e5 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
@@ -27,6 +27,7 @@
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CharsRefBuilder;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.termvectors.TermVectorsRequest.Flag;
 import org.elasticsearch.common.Nullable;
@@ -38,6 +39,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.search.dfs.AggregatedDfs;
 
 import java.io.IOException;
@@ -79,7 +81,6 @@ private static class FieldStrings {
     private BytesReference termVectors;
     private BytesReference headerRef;
     private String index;
-    private String type;
     private String id;
     private long docVersion;
     private boolean exists = false;
@@ -94,9 +95,8 @@ private static class FieldStrings {
     int[] currentEndOffset = new int[0];
     BytesReference[] currentPayloads = new BytesReference[0];
 
-    public TermVectorsResponse(String index, String type, String id) {
+    public TermVectorsResponse(String index, String id) {
         this.index = index;
-        this.type = type;
         this.id = id;
     }
 
@@ -105,7 +105,10 @@ public TermVectorsResponse(String index, String type, String id) {
 
     TermVectorsResponse(StreamInput in) throws IOException {
         index = in.readString();
-        type = in.readString();
+        if (in.getVersion().before(Version.V_8_0_0)) {
+            // types no longer relevant so ignore
+            in.readString();
+        }
         id = in.readString();
         docVersion = in.readVLong();
         exists = in.readBoolean();
@@ -120,7 +123,10 @@ public TermVectorsResponse(String index, String type, String id) {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(index);
-        out.writeString(type);
+        if (out.getVersion().before(Version.V_8_0_0)) {
+            // types not supported so send an empty array to previous versions
+            out.writeString(MapperService.SINGLE_MAPPING_NAME);
+        }
         out.writeString(id);
         out.writeVLong(docVersion);
         final boolean docExists = isExists();
@@ -176,11 +182,9 @@ public int size() {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         assert index != null;
-        assert type != null;
         assert id != null;
         builder.startObject();
         builder.field(FieldStrings._INDEX, index);
-        builder.field(FieldStrings._TYPE, type);
         if (!isArtificial()) {
             builder.field(FieldStrings._ID, id);
         }
@@ -391,10 +395,6 @@ public String getIndex() {
         return index;
     }
 
-    public String getType() {
-        return type;
-    }
-
     public String getId() {
         return id;
     }
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java
index 20514733aff31..1edd589a08a55 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -41,16 +42,16 @@
 public class TransportMultiTermVectorsAction extends HandledTransportAction<MultiTermVectorsRequest, MultiTermVectorsResponse> {
 
     private final ClusterService clusterService;
-    private final TransportShardMultiTermsVectorAction shardAction;
+    private final NodeClient client;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
 
     @Inject
     public TransportMultiTermVectorsAction(TransportService transportService, ClusterService clusterService,
-                                           TransportShardMultiTermsVectorAction shardAction, ActionFilters actionFilters,
+                                           NodeClient client, ActionFilters actionFilters,
                                            IndexNameExpressionResolver indexNameExpressionResolver) {
         super(MultiTermVectorsAction.NAME, transportService, actionFilters, MultiTermVectorsRequest::new);
         this.clusterService = clusterService;
-        this.shardAction = shardAction;
+        this.client = client;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
     }
 
@@ -69,7 +70,7 @@ protected void doExecute(Task task, final MultiTermVectorsRequest request, final
                 termVectorsRequest.index()));
             if (!clusterState.metaData().hasConcreteIndex(termVectorsRequest.index())) {
                 responses.set(i, new MultiTermVectorsItemResponse(null,
-                    new MultiTermVectorsResponse.Failure(termVectorsRequest.index(), termVectorsRequest.type(), termVectorsRequest.id(),
+                        new MultiTermVectorsResponse.Failure(termVectorsRequest.index(), termVectorsRequest.id(),
                         new IndexNotFoundException(termVectorsRequest.index()))));
                 continue;
             }
@@ -77,8 +78,8 @@ protected void doExecute(Task task, final MultiTermVectorsRequest request, final
             if (termVectorsRequest.routing() == null &&
                 clusterState.getMetaData().routingRequired(concreteSingleIndex)) {
                 responses.set(i, new MultiTermVectorsItemResponse(null,
-                    new MultiTermVectorsResponse.Failure(concreteSingleIndex, termVectorsRequest.type(), termVectorsRequest.id(),
-                        new RoutingMissingException(concreteSingleIndex, termVectorsRequest.type(), termVectorsRequest.id()))));
+                        new MultiTermVectorsResponse.Failure(concreteSingleIndex, termVectorsRequest.id(),
+                                new RoutingMissingException(concreteSingleIndex, termVectorsRequest.id()))));
                 continue;
             }
             ShardId shardId = clusterService.operationRouting().shardId(clusterState, concreteSingleIndex,
@@ -106,7 +107,7 @@ protected void executeShardAction(ActionListener<MultiTermVectorsResponse> liste
         final AtomicInteger counter = new AtomicInteger(shardRequests.size());
 
         for (final MultiTermVectorsShardRequest shardRequest : shardRequests.values()) {
-            shardAction.execute(shardRequest, new ActionListener<MultiTermVectorsShardResponse>() {
+            client.executeLocally(TransportShardMultiTermsVectorAction.TYPE, shardRequest, new ActionListener<>() {
                 @Override
                 public void onResponse(MultiTermVectorsShardResponse response) {
                     for (int i = 0; i < response.locations.size(); i++) {
@@ -124,8 +125,7 @@ public void onFailure(Exception e) {
                     for (int i = 0; i < shardRequest.locations.size(); i++) {
                         TermVectorsRequest termVectorsRequest = shardRequest.requests.get(i);
                         responses.set(shardRequest.locations.get(i), new MultiTermVectorsItemResponse(null,
-                                new MultiTermVectorsResponse.Failure(shardRequest.index(), termVectorsRequest.type(),
-                                        termVectorsRequest.id(), e)));
+                                new MultiTermVectorsResponse.Failure(shardRequest.index(), termVectorsRequest.id(), e)));
                     }
                     if (counter.decrementAndGet() == 0) {
                         finishHim();
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java
index 0292757121eff..e49ea292c61c8 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java
@@ -20,6 +20,8 @@
 package org.elasticsearch.action.termvectors;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
@@ -43,6 +45,12 @@ public class TransportShardMultiTermsVectorAction extends
     private final IndicesService indicesService;
 
     private static final String ACTION_NAME = MultiTermVectorsAction.NAME + "[shard]";
+    public static final ActionType<MultiTermVectorsShardResponse> TYPE = new StreamableResponseActionType<>(ACTION_NAME) {
+        @Override
+        public MultiTermVectorsShardResponse newResponse() {
+            return new MultiTermVectorsShardResponse();
+        }
+    };
 
     @Inject
     public TransportShardMultiTermsVectorAction(ClusterService clusterService, TransportService transportService,
@@ -88,10 +96,10 @@ protected MultiTermVectorsShardResponse shardOperation(MultiTermVectorsShardRequ
                 if (TransportActions.isShardNotAvailableException(e)) {
                     throw e;
                 } else {
-                    logger.debug(() -> new ParameterizedMessage("{} failed to execute multi term vectors for [{}]/[{}]",
-                        shardId, termVectorsRequest.type(), termVectorsRequest.id()), e);
+                    logger.debug(() -> new ParameterizedMessage("{} failed to execute multi term vectors for [{}]", shardId,
+                            termVectorsRequest.id()), e);
                     response.add(request.locations.get(i),
-                            new MultiTermVectorsResponse.Failure(request.index(), termVectorsRequest.type(), termVectorsRequest.id(), e));
+                            new MultiTermVectorsResponse.Failure(request.index(), termVectorsRequest.id(), e));
                 }
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java b/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
index 0e212ab7cce2f..48ff18cb49b18 100644
--- a/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
@@ -81,7 +81,7 @@ protected void resolveRequest(ClusterState state, InternalRequest request) {
         request.request().routing(state.metaData().resolveIndexRouting(request.request().routing(), request.request().index()));
         // Fail fast on the node that received the request.
         if (request.request().routing() == null && state.getMetaData().routingRequired(request.concreteIndex())) {
-            throw new RoutingMissingException(request.concreteIndex(), request.request().type(), request.request().id());
+            throw new RoutingMissingException(request.concreteIndex(), request.request().id());
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateAction.java b/server/src/main/java/org/elasticsearch/action/update/UpdateAction.java
index 1c8c80b61cef1..f24f5436f5e4e 100644
--- a/server/src/main/java/org/elasticsearch/action/update/UpdateAction.java
+++ b/server/src/main/java/org/elasticsearch/action/update/UpdateAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.update;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class UpdateAction extends Action<UpdateResponse> {
+public class UpdateAction extends StreamableResponseActionType<UpdateResponse> {
 
     public static final UpdateAction INSTANCE = new UpdateAction();
     public static final String NAME = "indices:data/write/update";
diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Security.java b/server/src/main/java/org/elasticsearch/bootstrap/Security.java
index 2a537186f6af8..b198dee979442 100644
--- a/server/src/main/java/org/elasticsearch/bootstrap/Security.java
+++ b/server/src/main/java/org/elasticsearch/bootstrap/Security.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.secure_sm.SecureSM;
 import org.elasticsearch.transport.TcpTransport;
-import org.elasticsearch.transport.TransportSettings;
 
 import java.io.IOException;
 import java.net.SocketPermission;
@@ -61,7 +60,7 @@
 /**
  * Initializes SecurityManager with necessary permissions.
  * <br>
- * <h1>Initialization</h1>
+ * <h2>Initialization</h2>
  * The JVM is not initially started with security manager enabled,
  * instead we turn it on early in the startup process. This is a tradeoff
  * between security and ease of use:
@@ -72,7 +71,7 @@
  *       otherwise be permitted.</li>
  * </ul>
  * <br>
- * <h1>Permissions</h1>
+ * <h2>Permissions</h2>
  * Permissions use a policy file packaged as a resource, this file is
  * also used in tests. File permissions are generated dynamically and
  * combined with this policy file.
@@ -92,7 +91,7 @@
  * cleanups to the scripting apis). But still it can provide some defense for users
  * that enable dynamic scripting without being fully aware of the consequences.
  * <br>
- * <h1>Debugging Security</h1>
+ * <h2>Debugging Security</h2>
  * A good place to start when there is a problem is to turn on security debugging:
  * <pre>
  * ES_JAVA_OPTS="-Djava.security.debug=access,failure" bin/elasticsearch
@@ -362,17 +361,6 @@ private static void addSocketPermissionForTransportProfiles(final Permissions po
         }
     }
 
-    /**
-     * Add dynamic {@link SocketPermission} based on transport settings.
-     *
-     * @param policy          the {@link Permissions} instance to apply the dynamic {@link SocketPermission}s to
-     * @param settings        the {@link Settings} instance to read the transport settings from
-     */
-    private static void addSocketPermissionForTransport(final Permissions policy, final Settings settings) {
-        final String transportRange = TransportSettings.PORT.get(settings);
-        addSocketPermissionForPortRange(policy, transportRange);
-    }
-
     /**
      * Add dynamic {@link SocketPermission} for the specified port range.
      *
diff --git a/server/src/main/java/org/elasticsearch/client/Client.java b/server/src/main/java/org/elasticsearch/client/Client.java
index 07871709f5726..3ea28ad074be0 100644
--- a/server/src/main/java/org/elasticsearch/client/Client.java
+++ b/server/src/main/java/org/elasticsearch/client/Client.java
@@ -77,11 +77,9 @@
  * simply returns an {@link org.elasticsearch.action.ActionFuture}, while the second accepts an
  * {@link org.elasticsearch.action.ActionListener}.
  * <p>
- * A client can either be retrieved from a {@link org.elasticsearch.node.Node} started, or connected remotely
- * to one or more nodes using {@link org.elasticsearch.client.transport.TransportClient}.
+ * A client can be retrieved from a started {@link org.elasticsearch.node.Node}.
  *
  * @see org.elasticsearch.node.Node#client()
- * @see org.elasticsearch.client.transport.TransportClient
  */
 public interface Client extends ElasticsearchClient, Releasable {
 
@@ -365,10 +363,9 @@ public interface Client extends ElasticsearchClient, Releasable {
      * Builder for the term vector request.
      *
      * @param index The index to load the document from
-     * @param type  The type of the document
      * @param id    The id of the document
      */
-    TermVectorsRequestBuilder prepareTermVectors(String index, String type, String id);
+    TermVectorsRequestBuilder prepareTermVectors(String index, String id);
 
     /**
      * Multi get term vectors.
diff --git a/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java b/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
index 42aaed10d6172..cd874b62a40b4 100644
--- a/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
+++ b/server/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
@@ -484,19 +484,19 @@ public interface ClusterAdminClient extends ElasticsearchClient {
     CreateSnapshotRequestBuilder prepareCreateSnapshot(String repository, String name);
 
     /**
-     * Get snapshot.
+     * Get snapshots.
      */
     ActionFuture<GetSnapshotsResponse> getSnapshots(GetSnapshotsRequest request);
 
     /**
-     * Get snapshot.
+     * Get snapshots.
      */
     void getSnapshots(GetSnapshotsRequest request, ActionListener<GetSnapshotsResponse> listener);
 
     /**
-     * Get snapshot.
+     * Get snapshots.
      */
-    GetSnapshotsRequestBuilder prepareGetSnapshots(String repository);
+    GetSnapshotsRequestBuilder prepareGetSnapshots(String... repository);
 
     /**
      * Delete snapshot.
diff --git a/server/src/main/java/org/elasticsearch/client/ElasticsearchClient.java b/server/src/main/java/org/elasticsearch/client/ElasticsearchClient.java
index 284c2b8d51420..f59da77d88408 100644
--- a/server/src/main/java/org/elasticsearch/client/ElasticsearchClient.java
+++ b/server/src/main/java/org/elasticsearch/client/ElasticsearchClient.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.client;
 
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
@@ -30,7 +30,7 @@
 public interface ElasticsearchClient {
 
     /**
-     * Executes a generic action, denoted by an {@link org.elasticsearch.action.Action}.
+     * Executes a generic action, denoted by an {@link ActionType}.
      *
      * @param action           The action type to execute.
      * @param request          The action request.
@@ -39,10 +39,10 @@ public interface ElasticsearchClient {
      * @return A future allowing to get back the response.
      */
     <Request extends ActionRequest, Response extends ActionResponse> ActionFuture<Response> execute(
-        Action<Response> action, Request request);
+        ActionType<Response> action, Request request);
 
     /**
-     * Executes a generic action, denoted by an {@link Action}.
+     * Executes a generic action, denoted by an {@link ActionType}.
      *
      * @param action           The action type to execute.
      * @param request          The action request.
@@ -51,7 +51,7 @@ <Request extends ActionRequest, Response extends ActionResponse> ActionFuture<Re
      * @param <Response>       The response type.
      */
     <Request extends ActionRequest, Response extends ActionResponse> void execute(
-        Action<Response> action, Request request, ActionListener<Response> listener);
+        ActionType<Response> action, Request request, ActionListener<Response> listener);
 
     /**
      * Returns the threadpool used to execute requests on this client
diff --git a/server/src/main/java/org/elasticsearch/client/FilterClient.java b/server/src/main/java/org/elasticsearch/client/FilterClient.java
index b4230710414be..245a4d5986ebb 100644
--- a/server/src/main/java/org/elasticsearch/client/FilterClient.java
+++ b/server/src/main/java/org/elasticsearch/client/FilterClient.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.client;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -61,7 +61,7 @@ public void close() {
 
     @Override
     protected <Request extends ActionRequest, Response extends ActionResponse>
-    void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+    void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
         in().execute(action, request, listener);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java b/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
index d5a73981f29f1..40c4c1046577b 100644
--- a/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
+++ b/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
@@ -23,14 +23,11 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
-import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistRequestBuilder;
-import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistResponse;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse;
@@ -42,12 +39,6 @@
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
-import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequestBuilder;
-import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
@@ -123,58 +114,6 @@
  */
 public interface IndicesAdminClient extends ElasticsearchClient {
 
-    /**
-     * Indices Exists.
-     *
-     * @param request The indices exists request
-     * @return The result future
-     * @see Requests#indicesExistsRequest(String...)
-     */
-    ActionFuture<IndicesExistsResponse> exists(IndicesExistsRequest request);
-
-    /**
-     * The status of one or more indices.
-     *
-     * @param request  The indices status request
-     * @param listener A listener to be notified with a result
-     * @see Requests#indicesExistsRequest(String...)
-     */
-    void exists(IndicesExistsRequest request, ActionListener<IndicesExistsResponse> listener);
-
-    /**
-     * Indices exists.
-     */
-    IndicesExistsRequestBuilder prepareExists(String... indices);
-
-
-    /**
-     * Types exists.
-     *
-     * @deprecated Types are deprecated and are in the process of being removed.
-     * @param request The types exists request
-     * @return The result future
-     */
-    @Deprecated
-    ActionFuture<TypesExistsResponse> typesExists(TypesExistsRequest request);
-
-    /**
-     * Types exists.
-     *
-     * @deprecated Types are deprecated and are in the process of being removed.
-     * @param request  The types exists
-     * @param listener A listener to be notified with a result
-     */
-    @Deprecated
-    void typesExists(TypesExistsRequest request, ActionListener<TypesExistsResponse> listener);
-
-    /**
-     * Types exists.
-     *
-     * @deprecated Types are deprecated and are in the process of being removed.
-     */
-    @Deprecated
-    TypesExistsRequestBuilder prepareTypesExists(String... index);
-
     /**
      * Indices stats.
      */
@@ -585,26 +524,6 @@ public interface IndicesAdminClient extends ElasticsearchClient {
      */
     GetAliasesRequestBuilder prepareGetAliases(String... aliases);
 
-    /**
-     * Allows to check to existence of aliases from indices.
-     */
-    AliasesExistRequestBuilder prepareAliasesExist(String... aliases);
-
-    /**
-     * Check to existence of index aliases.
-     *
-     * @param request The result future
-     */
-    ActionFuture<AliasesExistResponse> aliasesExist(GetAliasesRequest request);
-
-    /**
-     * Check the existence of specified index aliases.
-     *
-     * @param request  The index aliases request
-     * @param listener A listener to be notified with a result
-     */
-    void aliasesExist(GetAliasesRequest request, ActionListener<AliasesExistResponse> listener);
-
     /**
      * Get index metadata for particular indices.
      *
@@ -672,12 +591,12 @@ public interface IndicesAdminClient extends ElasticsearchClient {
     /**
      * Analyze text under the provided index.
      */
-    ActionFuture<AnalyzeResponse> analyze(AnalyzeRequest request);
+    ActionFuture<AnalyzeAction.Response> analyze(AnalyzeAction.Request request);
 
     /**
      * Analyze text under the provided index.
      */
-    void analyze(AnalyzeRequest request, ActionListener<AnalyzeResponse> listener);
+    void analyze(AnalyzeAction.Request request, ActionListener<AnalyzeAction.Response> listener);
 
     /**
      * Analyze text under the provided index.
@@ -819,4 +738,5 @@ public interface IndicesAdminClient extends ElasticsearchClient {
      * Swaps the index pointed to by an alias given all provided conditions are satisfied
      */
     void rolloverIndex(RolloverRequest request, ActionListener<RolloverResponse> listener);
+
 }
diff --git a/server/src/main/java/org/elasticsearch/client/OriginSettingClient.java b/server/src/main/java/org/elasticsearch/client/OriginSettingClient.java
index ad5a602fbae72..b2a8978f1584a 100644
--- a/server/src/main/java/org/elasticsearch/client/OriginSettingClient.java
+++ b/server/src/main/java/org/elasticsearch/client/OriginSettingClient.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.client;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -45,7 +45,7 @@ public OriginSettingClient(Client in, String origin) {
 
     @Override
     protected <Request extends ActionRequest, Response extends ActionResponse>
-            void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+            void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
         final Supplier<ThreadContext.StoredContext> supplier = in().threadPool().getThreadContext().newRestorableContext(false);
         try (ThreadContext.StoredContext ignore = in().threadPool().getThreadContext().stashWithOrigin(origin)) {
             super.doExecute(action, request, new ContextPreservingActionListener<>(supplier, listener));
diff --git a/server/src/main/java/org/elasticsearch/client/ParentTaskAssigningClient.java b/server/src/main/java/org/elasticsearch/client/ParentTaskAssigningClient.java
index 27de6619053b3..f33afdf51ec09 100644
--- a/server/src/main/java/org/elasticsearch/client/ParentTaskAssigningClient.java
+++ b/server/src/main/java/org/elasticsearch/client/ParentTaskAssigningClient.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.client;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -58,7 +58,7 @@ public Client unwrap() {
 
     @Override
     protected <Request extends ActionRequest, Response extends ActionResponse>
-    void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+    void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
         request.setParentTask(parentTask);
         super.doExecute(action, request, listener);
     }
diff --git a/server/src/main/java/org/elasticsearch/client/Requests.java b/server/src/main/java/org/elasticsearch/client/Requests.java
index 19ad2fb397edc..a3eb23eebfe20 100644
--- a/server/src/main/java/org/elasticsearch/client/Requests.java
+++ b/server/src/main/java/org/elasticsearch/client/Requests.java
@@ -45,7 +45,6 @@
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
@@ -162,16 +161,6 @@ public static IndicesSegmentsRequest indicesSegmentsRequest(String... indices) {
     public static IndicesShardStoresRequest indicesShardStoresRequest(String... indices) {
         return new IndicesShardStoresRequest(indices);
     }
-    /**
-     * Creates an indices exists request.
-     *
-     * @param indices The indices to check if they exists or not.
-     * @return The indices exists request
-     * @see org.elasticsearch.client.IndicesAdminClient#exists(org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest)
-     */
-    public static IndicesExistsRequest indicesExistsRequest(String... indices) {
-        return new IndicesExistsRequest(indices);
-    }
 
     /**
      * Creates a create index request.
@@ -494,13 +483,13 @@ public static CreateSnapshotRequest createSnapshotRequest(String repository, Str
     }
 
     /**
-     * Gets snapshots from repository
+     * Gets snapshots from repositories
      *
-     * @param repository repository name
+     * @param repositories repository names
      * @return get snapshot  request
      */
-    public static GetSnapshotsRequest getSnapshotsRequest(String repository) {
-        return new GetSnapshotsRequest(repository);
+    public static GetSnapshotsRequest getSnapshotsRequest(String... repositories) {
+        return new GetSnapshotsRequest(repositories);
     }
 
     /**
@@ -534,5 +523,4 @@ public static DeleteSnapshotRequest deleteSnapshotRequest(String repository, Str
     public static SnapshotsStatusRequest snapshotsStatusRequest(String repository) {
         return new SnapshotsStatusRequest(repository);
     }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/client/node/NodeClient.java b/server/src/main/java/org/elasticsearch/client/node/NodeClient.java
index 0ad863c936741..96b2c593a5ded 100644
--- a/server/src/main/java/org/elasticsearch/client/node/NodeClient.java
+++ b/server/src/main/java/org/elasticsearch/client/node/NodeClient.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.client.node;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -41,10 +41,10 @@
  */
 public class NodeClient extends AbstractClient {
 
-    private Map<Action, TransportAction> actions;
+    private Map<ActionType, TransportAction> actions;
     /**
      * The id of the local {@link DiscoveryNode}. Useful for generating task ids from tasks returned by
-     * {@link #executeLocally(Action, ActionRequest, TaskListener)}.
+     * {@link #executeLocally(ActionType, ActionRequest, TaskListener)}.
      */
     private Supplier<String> localNodeId;
     private RemoteClusterService remoteClusterService;
@@ -53,7 +53,7 @@ public NodeClient(Settings settings, ThreadPool threadPool) {
         super(settings, threadPool);
     }
 
-    public void initialize(Map<Action, TransportAction> actions, Supplier<String> localNodeId,
+    public void initialize(Map<ActionType, TransportAction> actions, Supplier<String> localNodeId,
                            RemoteClusterService remoteClusterService) {
         this.actions = actions;
         this.localNodeId = localNodeId;
@@ -67,47 +67,47 @@ public void close() {
 
     @Override
     public <Request extends ActionRequest, Response extends ActionResponse>
-    void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+    void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
         // Discard the task because the Client interface doesn't use it.
         executeLocally(action, request, listener);
     }
 
     /**
-     * Execute an {@link Action} locally, returning that {@link Task} used to track it, and linking an {@link ActionListener}. Prefer this
-     * method if you don't need access to the task when listening for the response. This is the method used to implement the {@link Client}
-     * interface.
+     * Execute an {@link ActionType} locally, returning that {@link Task} used to track it, and linking an {@link ActionListener}.
+     * Prefer this method if you don't need access to the task when listening for the response. This is the method used to
+     * implement the {@link Client} interface.
      */
     public <    Request extends ActionRequest,
                 Response extends ActionResponse
-            > Task executeLocally(Action<Response> action, Request request, ActionListener<Response> listener) {
+            > Task executeLocally(ActionType<Response> action, Request request, ActionListener<Response> listener) {
         return transportAction(action).execute(request, listener);
     }
 
     /**
-     * Execute an {@link Action} locally, returning that {@link Task} used to track it, and linking an {@link TaskListener}. Prefer this
-     * method if you need access to the task when listening for the response.
+     * Execute an {@link ActionType} locally, returning that {@link Task} used to track it, and linking an {@link TaskListener}.
+     * Prefer this method if you need access to the task when listening for the response.
      */
     public <    Request extends ActionRequest,
                 Response extends ActionResponse
-            > Task executeLocally(Action<Response> action, Request request, TaskListener<Response> listener) {
+            > Task executeLocally(ActionType<Response> action, Request request, TaskListener<Response> listener) {
         return transportAction(action).execute(request, listener);
     }
 
     /**
      * The id of the local {@link DiscoveryNode}. Useful for generating task ids from tasks returned by
-     * {@link #executeLocally(Action, ActionRequest, TaskListener)}.
+     * {@link #executeLocally(ActionType, ActionRequest, TaskListener)}.
      */
     public String getLocalNodeId() {
         return localNodeId.get();
     }
 
     /**
-     * Get the {@link TransportAction} for an {@link Action}, throwing exceptions if the action isn't available.
+     * Get the {@link TransportAction} for an {@link ActionType}, throwing exceptions if the action isn't available.
      */
     @SuppressWarnings("unchecked")
     private <    Request extends ActionRequest,
                 Response extends ActionResponse
-            > TransportAction<Request, Response> transportAction(Action<Response> action) {
+            > TransportAction<Request, Response> transportAction(ActionType<Response> action) {
         if (actions == null) {
             throw new IllegalStateException("NodeClient has not been initialized");
         }
diff --git a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java
index e79f0567babe6..c3119256fc7cf 100644
--- a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java
+++ b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java
@@ -21,7 +21,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
@@ -134,17 +134,12 @@
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
-import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistAction;
-import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistRequestBuilder;
-import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistResponse;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheAction;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder;
@@ -160,14 +155,6 @@
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
-import org.elasticsearch.action.admin.indices.exists.types.TypesExistsAction;
-import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequestBuilder;
-import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushAction;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder;
@@ -314,7 +301,6 @@
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.action.search.SearchScrollRequestBuilder;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.action.support.ThreadedActionListener;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.termvectors.MultiTermVectorsAction;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
@@ -351,14 +337,12 @@ public abstract class AbstractClient implements Client {
     protected final Settings settings;
     private final ThreadPool threadPool;
     private final Admin admin;
-    private final ThreadedActionListener.Wrapper threadedWrapper;
 
     public AbstractClient(Settings settings, ThreadPool threadPool) {
         this.settings = settings;
         this.threadPool = threadPool;
         this.admin = new Admin(this);
         this.logger =LogManager.getLogger(this.getClass());
-        this.threadedWrapper = new ThreadedActionListener.Wrapper(logger, settings, threadPool);
     }
 
     @Override
@@ -378,7 +362,7 @@ public final AdminClient admin() {
 
     @Override
     public final <Request extends ActionRequest, Response extends ActionResponse> ActionFuture<Response> execute(
-        Action<Response> action, Request request) {
+        ActionType<Response> action, Request request) {
         PlainActionFuture<Response> actionFuture = PlainActionFuture.newFuture();
         execute(action, request, actionFuture);
         return actionFuture;
@@ -389,13 +373,12 @@ public final <Request extends ActionRequest, Response extends ActionResponse> Ac
      */
     @Override
     public final <Request extends ActionRequest, Response extends ActionResponse> void execute(
-        Action<Response> action, Request request, ActionListener<Response> listener) {
-        listener = threadedWrapper.wrap(listener);
+        ActionType<Response> action, Request request, ActionListener<Response> listener) {
         doExecute(action, request, listener);
     }
 
     protected abstract <Request extends ActionRequest, Response extends ActionResponse>
-    void doExecute(Action<Response> action, Request request, ActionListener<Response> listener);
+    void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener);
 
     @Override
     public ActionFuture<IndexResponse> index(final IndexRequest request) {
@@ -578,8 +561,8 @@ public TermVectorsRequestBuilder prepareTermVectors() {
     }
 
     @Override
-    public TermVectorsRequestBuilder prepareTermVectors(String index, String type, String id) {
-        return new TermVectorsRequestBuilder(this, TermVectorsAction.INSTANCE, index, type, id);
+    public TermVectorsRequestBuilder prepareTermVectors(String index, String id) {
+        return new TermVectorsRequestBuilder(this, TermVectorsAction.INSTANCE, index, id);
     }
 
     @Override
@@ -673,13 +656,13 @@ static class ClusterAdmin implements ClusterAdminClient {
 
         @Override
         public <Request extends ActionRequest, Response extends ActionResponse> ActionFuture<Response> execute(
-            Action<Response> action, Request request) {
+            ActionType<Response> action, Request request) {
             return client.execute(action, request);
         }
 
         @Override
         public <Request extends ActionRequest, Response extends ActionResponse> void execute(
-            Action<Response> action, Request request, ActionListener<Response> listener) {
+            ActionType<Response> action, Request request, ActionListener<Response> listener) {
             client.execute(action, request, listener);
         }
 
@@ -955,8 +938,8 @@ public void getSnapshots(GetSnapshotsRequest request, ActionListener<GetSnapshot
         }
 
         @Override
-        public GetSnapshotsRequestBuilder prepareGetSnapshots(String repository) {
-            return new GetSnapshotsRequestBuilder(this, GetSnapshotsAction.INSTANCE, repository);
+        public GetSnapshotsRequestBuilder prepareGetSnapshots(String... repositories) {
+            return new GetSnapshotsRequestBuilder(this, GetSnapshotsAction.INSTANCE, repositories);
         }
 
 
@@ -1204,13 +1187,13 @@ static class IndicesAdmin implements IndicesAdminClient {
 
         @Override
         public <Request extends ActionRequest, Response extends ActionResponse> ActionFuture<Response> execute(
-            Action<Response> action, Request request) {
+            ActionType<Response> action, Request request) {
             return client.execute(action, request);
         }
 
         @Override
         public <Request extends ActionRequest, Response extends ActionResponse> void execute(
-            Action<Response> action, Request request, ActionListener<Response> listener) {
+            ActionType<Response> action, Request request, ActionListener<Response> listener) {
             client.execute(action, request, listener);
         }
 
@@ -1219,37 +1202,6 @@ public ThreadPool threadPool() {
             return client.threadPool();
         }
 
-        @Override
-        public ActionFuture<IndicesExistsResponse> exists(final IndicesExistsRequest request) {
-            return execute(IndicesExistsAction.INSTANCE, request);
-        }
-
-        @Override
-        public void exists(final IndicesExistsRequest request, final ActionListener<IndicesExistsResponse> listener) {
-            execute(IndicesExistsAction.INSTANCE, request, listener);
-        }
-
-        @Override
-        public IndicesExistsRequestBuilder prepareExists(String... indices) {
-            return new IndicesExistsRequestBuilder(this, IndicesExistsAction.INSTANCE, indices);
-        }
-
-        @Deprecated
-        @Override
-        public ActionFuture<TypesExistsResponse> typesExists(TypesExistsRequest request) {
-            return execute(TypesExistsAction.INSTANCE, request);
-        }
-
-        @Override
-        public void typesExists(TypesExistsRequest request, ActionListener<TypesExistsResponse> listener) {
-            execute(TypesExistsAction.INSTANCE, request, listener);
-        }
-
-        @Override
-        public TypesExistsRequestBuilder prepareTypesExists(String... index) {
-            return new TypesExistsRequestBuilder(this, TypesExistsAction.INSTANCE, index);
-        }
-
         @Override
         public ActionFuture<AcknowledgedResponse> aliases(final IndicesAliasesRequest request) {
             return execute(IndicesAliasesAction.INSTANCE, request);
@@ -1285,21 +1237,6 @@ public ActionFuture<ClearIndicesCacheResponse> clearCache(final ClearIndicesCach
             return execute(ClearIndicesCacheAction.INSTANCE, request);
         }
 
-        @Override
-        public void aliasesExist(GetAliasesRequest request, ActionListener<AliasesExistResponse> listener) {
-            execute(AliasesExistAction.INSTANCE, request, listener);
-        }
-
-        @Override
-        public ActionFuture<AliasesExistResponse> aliasesExist(GetAliasesRequest request) {
-            return execute(AliasesExistAction.INSTANCE, request);
-        }
-
-        @Override
-        public AliasesExistRequestBuilder prepareAliasesExist(String... aliases) {
-            return new AliasesExistRequestBuilder(this, AliasesExistAction.INSTANCE, aliases);
-        }
-
         @Override
         public ActionFuture<GetIndexResponse> getIndex(GetIndexRequest request) {
             return execute(GetIndexAction.INSTANCE, request);
@@ -1596,12 +1533,12 @@ public UpdateSettingsRequestBuilder prepareUpdateSettings(String... indices) {
         }
 
         @Override
-        public ActionFuture<AnalyzeResponse> analyze(final AnalyzeRequest request) {
+        public ActionFuture<AnalyzeAction.Response> analyze(final AnalyzeAction.Request request) {
             return execute(AnalyzeAction.INSTANCE, request);
         }
 
         @Override
-        public void analyze(final AnalyzeRequest request, final ActionListener<AnalyzeResponse> listener) {
+        public void analyze(final AnalyzeAction.Request request, final ActionListener<AnalyzeAction.Response> listener) {
             execute(AnalyzeAction.INSTANCE, request, listener);
         }
 
@@ -1732,7 +1669,7 @@ public Client filterWithHeader(Map<String, String> headers) {
         return new FilterClient(this) {
             @Override
             protected <Request extends ActionRequest, Response extends ActionResponse>
-            void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+            void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
                 ThreadContext threadContext = threadPool().getThreadContext();
                 try (ThreadContext.StoredContext ctx = threadContext.stashAndMergeHeaders(headers)) {
                     super.doExecute(action, request, listener);
diff --git a/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java
deleted file mode 100644
index 4c2f4932de2f2..0000000000000
--- a/server/src/main/java/org/elasticsearch/client/transport/TransportClient.java
+++ /dev/null
@@ -1,404 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client.transport;
-
-import org.elasticsearch.action.Action;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionModule;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.client.support.AbstractClient;
-import org.elasticsearch.cluster.ClusterModule;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.breaker.CircuitBreaker;
-import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.inject.ModulesBuilder;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.network.NetworkModule;
-import org.elasticsearch.common.network.NetworkService;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.PageCacheRecycler;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.core.internal.io.IOUtils;
-import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.node.InternalSettingsPreparer;
-import org.elasticsearch.node.Node;
-import org.elasticsearch.plugins.ActionPlugin;
-import org.elasticsearch.plugins.NetworkPlugin;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.PluginsService;
-import org.elasticsearch.plugins.SearchPlugin;
-import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.threadpool.ExecutorBuilder;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.Transport;
-import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.transport.TransportSettings;
-
-import java.io.Closeable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static java.util.stream.Collectors.toList;
-import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
-
-/**
- * The transport client allows to create a client that is not part of the cluster, but simply connects to one
- * or more nodes directly by adding their respective addresses using
- * {@link #addTransportAddress(org.elasticsearch.common.transport.TransportAddress)}.
- * <p>
- * The transport client important modules used is the {@link org.elasticsearch.common.network.NetworkModule} which is
- * started in client mode (only connects, no bind).
- *
- * @deprecated {@link TransportClient} is deprecated in favour of the High Level REST client and will
- * be removed in Elasticsearch 8.0.
- */
-@Deprecated
-public abstract class TransportClient extends AbstractClient {
-
-    public static final Setting<TimeValue> CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL =
-        Setting.positiveTimeSetting("client.transport.nodes_sampler_interval", timeValueSeconds(5), Setting.Property.NodeScope);
-    public static final Setting<TimeValue> CLIENT_TRANSPORT_PING_TIMEOUT =
-        Setting.positiveTimeSetting("client.transport.ping_timeout", timeValueSeconds(5), Setting.Property.NodeScope);
-    public static final Setting<Boolean> CLIENT_TRANSPORT_IGNORE_CLUSTER_NAME =
-        Setting.boolSetting("client.transport.ignore_cluster_name", false, Setting.Property.NodeScope);
-    public static final Setting<Boolean> CLIENT_TRANSPORT_SNIFF =
-        Setting.boolSetting("client.transport.sniff", false, Setting.Property.NodeScope);
-
-    public static final String TRANSPORT_CLIENT_FEATURE = "transport_client";
-
-    private static PluginsService newPluginService(final Settings settings, Collection<Class<? extends Plugin>> plugins) {
-        final Settings.Builder settingsBuilder = Settings.builder()
-                .put(TransportSettings.PING_SCHEDULE.getKey(), "5s") // enable by default the transport schedule ping interval
-                .put(InternalSettingsPreparer.prepareSettings(settings))
-                .put(NetworkService.NETWORK_SERVER.getKey(), false)
-                .put(CLIENT_TYPE_SETTING_S.getKey(), CLIENT_TYPE);
-        return new PluginsService(settingsBuilder.build(), null, null, null, plugins);
-    }
-
-    protected static Collection<Class<? extends Plugin>> addPlugins(Collection<Class<? extends Plugin>> collection,
-                                                                    Class<? extends Plugin>... plugins) {
-        return addPlugins(collection, Arrays.asList(plugins));
-    }
-
-    protected static Collection<Class<? extends Plugin>> addPlugins(Collection<Class<? extends Plugin>> collection,
-            Collection<Class<? extends Plugin>> plugins) {
-        ArrayList<Class<? extends Plugin>> list = new ArrayList<>(collection);
-        for (Class<? extends Plugin> p : plugins) {
-            if (list.contains(p)) {
-                throw new IllegalArgumentException("plugin already exists: " + p);
-            }
-            list.add(p);
-        }
-        return list;
-    }
-
-    private static ClientTemplate buildTemplate(Settings providedSettings, Settings defaultSettings,
-                                                Collection<Class<? extends Plugin>> plugins, HostFailureListener failureListner) {
-        if (Node.NODE_NAME_SETTING.exists(providedSettings) == false) {
-            providedSettings = Settings.builder().put(providedSettings).put(Node.NODE_NAME_SETTING.getKey(), "_client_").build();
-        }
-        final PluginsService pluginsService = newPluginService(providedSettings, plugins);
-        final Settings settings =
-                Settings.builder()
-                        .put(defaultSettings)
-                        .put(pluginsService.updatedSettings())
-                        .put(TransportSettings.FEATURE_PREFIX + "." + TRANSPORT_CLIENT_FEATURE, true)
-                        .build();
-        final List<Closeable> resourcesToClose = new ArrayList<>();
-        final ThreadPool threadPool = new ThreadPool(settings);
-        resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS));
-        final NetworkService networkService = new NetworkService(Collections.emptyList());
-        try {
-            final List<Setting<?>> additionalSettings = new ArrayList<>(pluginsService.getPluginSettings());
-            final List<String> additionalSettingsFilter = new ArrayList<>(pluginsService.getPluginSettingsFilter());
-            for (final ExecutorBuilder<?> builder : threadPool.builders()) {
-                additionalSettings.addAll(builder.getRegisteredSettings());
-            }
-            SettingsModule settingsModule =
-                    new SettingsModule(settings, additionalSettings, additionalSettingsFilter, Collections.emptySet());
-
-            SearchModule searchModule = new SearchModule(settings, true, pluginsService.filterPlugins(SearchPlugin.class));
-            IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
-            List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
-            entries.addAll(NetworkModule.getNamedWriteables());
-            entries.addAll(searchModule.getNamedWriteables());
-            entries.addAll(indicesModule.getNamedWriteables());
-            entries.addAll(ClusterModule.getNamedWriteables());
-            entries.addAll(pluginsService.filterPlugins(Plugin.class).stream()
-                                         .flatMap(p -> p.getNamedWriteables().stream())
-                                         .collect(Collectors.toList()));
-            NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries);
-            NamedXContentRegistry xContentRegistry = new NamedXContentRegistry(Stream.of(
-                    searchModule.getNamedXContents().stream(),
-                    pluginsService.filterPlugins(Plugin.class).stream()
-                            .flatMap(p -> p.getNamedXContent().stream())
-                    ).flatMap(Function.identity()).collect(toList()));
-
-            ModulesBuilder modules = new ModulesBuilder();
-            // plugin modules must be added here, before others or we can get crazy injection errors...
-            for (Module pluginModule : pluginsService.createGuiceModules()) {
-                modules.add(pluginModule);
-            }
-            modules.add(b -> b.bind(ThreadPool.class).toInstance(threadPool));
-            ActionModule actionModule = new ActionModule(true, settings, null, settingsModule.getIndexScopedSettings(),
-                    settingsModule.getClusterSettings(), settingsModule.getSettingsFilter(), threadPool,
-                    pluginsService.filterPlugins(ActionPlugin.class), null, null, null);
-            modules.add(actionModule);
-
-            CircuitBreakerService circuitBreakerService = Node.createCircuitBreakerService(settingsModule.getSettings(),
-                settingsModule.getClusterSettings());
-            resourcesToClose.add(circuitBreakerService);
-            PageCacheRecycler pageCacheRecycler = new PageCacheRecycler(settings);
-            BigArrays bigArrays = new BigArrays(pageCacheRecycler, circuitBreakerService, CircuitBreaker.REQUEST);
-            modules.add(settingsModule);
-            NetworkModule networkModule = new NetworkModule(settings, true, pluginsService.filterPlugins(NetworkPlugin.class), threadPool,
-                bigArrays, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, null);
-            final Transport transport = networkModule.getTransportSupplier().get();
-            final TransportService transportService = new TransportService(settings, transport, threadPool,
-                networkModule.getTransportInterceptor(),
-                boundTransportAddress -> DiscoveryNode.createLocal(settings, new TransportAddress(TransportAddress.META_ADDRESS, 0),
-                    UUIDs.randomBase64UUID()), null, Collections.emptySet());
-            modules.add((b -> {
-                b.bind(BigArrays.class).toInstance(bigArrays);
-                b.bind(PageCacheRecycler.class).toInstance(pageCacheRecycler);
-                b.bind(PluginsService.class).toInstance(pluginsService);
-                b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService);
-                b.bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
-                b.bind(Transport.class).toInstance(transport);
-                b.bind(TransportService.class).toInstance(transportService);
-                b.bind(NetworkService.class).toInstance(networkService);
-            }));
-
-            Injector injector = modules.createInjector();
-            final TransportClientNodesService nodesService =
-                new TransportClientNodesService(settings, transportService, threadPool, failureListner == null
-                    ? (t, e) -> {} : failureListner);
-
-            // construct the list of client actions
-            final List<ActionPlugin> actionPlugins = pluginsService.filterPlugins(ActionPlugin.class);
-            final List<Action> clientActions =
-                    actionPlugins.stream().flatMap(p -> p.getClientActions().stream()).collect(Collectors.toList());
-            // add all the base actions
-            final List<? extends Action<?>> baseActions =
-                    actionModule.getActions().values().stream().map(ActionPlugin.ActionHandler::getAction).collect(Collectors.toList());
-            clientActions.addAll(baseActions);
-            final TransportProxyClient proxy = new TransportProxyClient(settings, transportService, nodesService, clientActions);
-
-            List<LifecycleComponent> pluginLifecycleComponents = new ArrayList<>(pluginsService.getGuiceServiceClasses().stream()
-                .map(injector::getInstance).collect(Collectors.toList()));
-            resourcesToClose.addAll(pluginLifecycleComponents);
-
-            transportService.start();
-            transportService.acceptIncomingRequests();
-
-            ClientTemplate transportClient = new ClientTemplate(injector, pluginLifecycleComponents, nodesService, proxy,
-                namedWriteableRegistry);
-            resourcesToClose.clear();
-            return transportClient;
-        } finally {
-            IOUtils.closeWhileHandlingException(resourcesToClose);
-        }
-    }
-
-    private static final class ClientTemplate {
-        final Injector injector;
-        private final List<LifecycleComponent> pluginLifecycleComponents;
-        private final TransportClientNodesService nodesService;
-        private final TransportProxyClient proxy;
-        private final NamedWriteableRegistry namedWriteableRegistry;
-
-        private ClientTemplate(Injector injector, List<LifecycleComponent> pluginLifecycleComponents,
-                TransportClientNodesService nodesService, TransportProxyClient proxy, NamedWriteableRegistry namedWriteableRegistry) {
-            this.injector = injector;
-            this.pluginLifecycleComponents = pluginLifecycleComponents;
-            this.nodesService = nodesService;
-            this.proxy = proxy;
-            this.namedWriteableRegistry = namedWriteableRegistry;
-        }
-
-        Settings getSettings() {
-            return injector.getInstance(Settings.class);
-        }
-
-        ThreadPool getThreadPool() {
-            return injector.getInstance(ThreadPool.class);
-        }
-    }
-
-    public static final String CLIENT_TYPE = "transport";
-
-    final Injector injector;
-    protected final NamedWriteableRegistry namedWriteableRegistry;
-
-    private final List<LifecycleComponent> pluginLifecycleComponents;
-    private final TransportClientNodesService nodesService;
-    private final TransportProxyClient proxy;
-
-    /**
-     * Creates a new TransportClient with the given settings and plugins
-     */
-    public TransportClient(Settings settings, Collection<Class<? extends Plugin>> plugins) {
-        this(buildTemplate(settings, Settings.EMPTY, plugins, null));
-    }
-
-    /**
-     * Creates a new TransportClient with the given settings, defaults and plugins.
-     * @param settings the client settings
-     * @param defaultSettings default settings that are merged after the plugins have added it's additional settings.
-     * @param plugins the client plugins
-     */
-    protected TransportClient(Settings settings, Settings defaultSettings, Collection<Class<? extends Plugin>> plugins,
-                              HostFailureListener hostFailureListener) {
-        this(buildTemplate(settings, defaultSettings, plugins, hostFailureListener));
-    }
-
-    private TransportClient(ClientTemplate template) {
-        super(template.getSettings(), template.getThreadPool());
-        this.injector = template.injector;
-        this.pluginLifecycleComponents = Collections.unmodifiableList(template.pluginLifecycleComponents);
-        this.nodesService = template.nodesService;
-        this.proxy = template.proxy;
-        this.namedWriteableRegistry = template.namedWriteableRegistry;
-    }
-
-    /**
-     * Returns the current registered transport addresses to use (added using
-     * {@link #addTransportAddress(org.elasticsearch.common.transport.TransportAddress)}.
-     */
-    public List<TransportAddress> transportAddresses() {
-        return nodesService.transportAddresses();
-    }
-
-    /**
-     * Returns the current connected transport nodes that this client will use.
-     * <p>
-     * The nodes include all the nodes that are currently alive based on the transport
-     * addresses provided.
-     */
-    public List<DiscoveryNode> connectedNodes() {
-        return nodesService.connectedNodes();
-    }
-
-    /**
-     * The list of filtered nodes that were not connected to, for example, due to
-     * mismatch in cluster name.
-     */
-    public List<DiscoveryNode> filteredNodes() {
-        return nodesService.filteredNodes();
-    }
-
-    /**
-     * Returns the listed nodes in the transport client (ones added to it).
-     */
-    public List<DiscoveryNode> listedNodes() {
-        return nodesService.listedNodes();
-    }
-
-    /**
-     * Adds a transport address that will be used to connect to.
-     * <p>
-     * The Node this transport address represents will be used if its possible to connect to it.
-     * If it is unavailable, it will be automatically connected to once it is up.
-     * <p>
-     * In order to get the list of all the current connected nodes, please see {@link #connectedNodes()}.
-     */
-    public TransportClient addTransportAddress(TransportAddress transportAddress) {
-        nodesService.addTransportAddresses(transportAddress);
-        return this;
-    }
-
-    /**
-     * Adds a list of transport addresses that will be used to connect to.
-     * <p>
-     * The Node this transport address represents will be used if its possible to connect to it.
-     * If it is unavailable, it will be automatically connected to once it is up.
-     * <p>
-     * In order to get the list of all the current connected nodes, please see {@link #connectedNodes()}.
-     */
-    public TransportClient addTransportAddresses(TransportAddress... transportAddress) {
-        nodesService.addTransportAddresses(transportAddress);
-        return this;
-    }
-
-    /**
-     * Removes a transport address from the list of transport addresses that are used to connect to.
-     */
-    public TransportClient removeTransportAddress(TransportAddress transportAddress) {
-        nodesService.removeTransportAddress(transportAddress);
-        return this;
-    }
-
-    /**
-     * Closes the client.
-     */
-    @Override
-    public void close() {
-        List<Closeable> closeables = new ArrayList<>();
-        closeables.add(nodesService);
-        closeables.add(injector.getInstance(TransportService.class));
-
-        for (LifecycleComponent plugin : pluginLifecycleComponents) {
-            closeables.add(plugin);
-        }
-        closeables.add(() -> ThreadPool.terminate(injector.getInstance(ThreadPool.class), 10, TimeUnit.SECONDS));
-        IOUtils.closeWhileHandlingException(closeables);
-    }
-
-    @Override
-    protected <Request extends ActionRequest, Response extends ActionResponse>
-    void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
-        proxy.execute(action, request, listener);
-    }
-
-    /**
-     * Listener that allows to be notified whenever a node failure / disconnect happens
-     */
-    @FunctionalInterface
-    public interface HostFailureListener {
-        /**
-         * Called once a node disconnect is detected.
-         * @param node the node that has been disconnected
-         * @param ex the exception causing the disconnection
-         */
-        void onNodeDisconnected(DiscoveryNode node, Exception ex);
-    }
-
-    // pkg private for testing
-    TransportClientNodesService getNodesService() {
-        return nodesService;
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/server/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
deleted file mode 100644
index 9d28ea0e5bc16..0000000000000
--- a/server/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
+++ /dev/null
@@ -1,582 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client.transport;
-
-import com.carrotsearch.hppc.cursors.ObjectCursor;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.cluster.node.liveness.LivenessRequest;
-import org.elasticsearch.action.admin.cluster.node.liveness.LivenessResponse;
-import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction;
-import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
-import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.Randomness;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.core.internal.io.IOUtils;
-import org.elasticsearch.threadpool.Scheduler;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.ConnectTransportException;
-import org.elasticsearch.transport.ConnectionProfile;
-import org.elasticsearch.transport.FutureTransportResponseHandler;
-import org.elasticsearch.transport.NodeDisconnectedException;
-import org.elasticsearch.transport.NodeNotConnectedException;
-import org.elasticsearch.transport.PlainTransportFuture;
-import org.elasticsearch.transport.Transport;
-import org.elasticsearch.transport.TransportException;
-import org.elasticsearch.transport.TransportRequestOptions;
-import org.elasticsearch.transport.TransportResponseHandler;
-import org.elasticsearch.transport.TransportService;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicInteger;
-
-final class TransportClientNodesService implements Closeable {
-
-    private static final Logger logger = LogManager.getLogger(TransportClientNodesService.class);
-
-    private final TimeValue nodesSamplerInterval;
-
-    private final long pingTimeout;
-
-    private final ClusterName clusterName;
-
-    private final TransportService transportService;
-
-    private final ThreadPool threadPool;
-
-    private final Version minCompatibilityVersion;
-
-    // nodes that are added to be discovered
-    private volatile List<DiscoveryNode> listedNodes = Collections.emptyList();
-
-    private final Object mutex = new Object();
-
-    private volatile List<DiscoveryNode> nodes = Collections.emptyList();
-    // Filtered nodes are nodes whose cluster name does not match the configured cluster name
-    private volatile List<DiscoveryNode> filteredNodes = Collections.emptyList();
-
-    private final AtomicInteger tempNodeIdGenerator = new AtomicInteger();
-
-    private final NodeSampler nodesSampler;
-
-    private volatile Scheduler.Cancellable nodesSamplerCancellable;
-
-    private final AtomicInteger randomNodeGenerator = new AtomicInteger(Randomness.get().nextInt());
-
-    private final boolean ignoreClusterName;
-
-    private volatile boolean closed;
-
-    private final TransportClient.HostFailureListener hostFailureListener;
-
-    // TODO: migrate this to use low level connections and single type channels
-    /** {@link ConnectionProfile} to use when to connecting to the listed nodes and doing a liveness check */
-    private static final ConnectionProfile LISTED_NODES_PROFILE;
-
-    static {
-        ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
-        builder.addConnections(1,
-            TransportRequestOptions.Type.BULK,
-            TransportRequestOptions.Type.PING,
-            TransportRequestOptions.Type.RECOVERY,
-            TransportRequestOptions.Type.REG,
-            TransportRequestOptions.Type.STATE);
-        LISTED_NODES_PROFILE = builder.build();
-    }
-
-    TransportClientNodesService(Settings settings, TransportService transportService,
-                                       ThreadPool threadPool, TransportClient.HostFailureListener hostFailureListener) {
-        this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
-        this.transportService = transportService;
-        this.threadPool = threadPool;
-        this.minCompatibilityVersion = Version.CURRENT.minimumCompatibilityVersion();
-
-        this.nodesSamplerInterval = TransportClient.CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL.get(settings);
-        this.pingTimeout = TransportClient.CLIENT_TRANSPORT_PING_TIMEOUT.get(settings).millis();
-        this.ignoreClusterName = TransportClient.CLIENT_TRANSPORT_IGNORE_CLUSTER_NAME.get(settings);
-
-        if (logger.isDebugEnabled()) {
-            logger.debug("node_sampler_interval[{}]", nodesSamplerInterval);
-        }
-
-        if (TransportClient.CLIENT_TRANSPORT_SNIFF.get(settings)) {
-            this.nodesSampler = new SniffNodesSampler();
-        } else {
-            this.nodesSampler = new SimpleNodeSampler();
-        }
-        this.hostFailureListener = hostFailureListener;
-        this.nodesSamplerCancellable = threadPool.schedule(new ScheduledNodeSampler(), nodesSamplerInterval, ThreadPool.Names.GENERIC);
-    }
-
-    public List<TransportAddress> transportAddresses() {
-        List<TransportAddress> lstBuilder = new ArrayList<>();
-        for (DiscoveryNode listedNode : listedNodes) {
-            lstBuilder.add(listedNode.getAddress());
-        }
-        return Collections.unmodifiableList(lstBuilder);
-    }
-
-    public List<DiscoveryNode> connectedNodes() {
-        return this.nodes;
-    }
-
-    public List<DiscoveryNode> filteredNodes() {
-        return this.filteredNodes;
-    }
-
-    public List<DiscoveryNode> listedNodes() {
-        return this.listedNodes;
-    }
-
-    public TransportClientNodesService addTransportAddresses(TransportAddress... transportAddresses) {
-        synchronized (mutex) {
-            if (closed) {
-                throw new IllegalStateException("transport client is closed, can't add an address");
-            }
-            List<TransportAddress> filtered = new ArrayList<>(transportAddresses.length);
-            for (TransportAddress transportAddress : transportAddresses) {
-                boolean found = false;
-                for (DiscoveryNode otherNode : listedNodes) {
-                    if (otherNode.getAddress().equals(transportAddress)) {
-                        found = true;
-                        logger.debug("address [{}] already exists with [{}], ignoring...", transportAddress, otherNode);
-                        break;
-                    }
-                }
-                if (!found) {
-                    filtered.add(transportAddress);
-                }
-            }
-            if (filtered.isEmpty()) {
-                return this;
-            }
-            List<DiscoveryNode> builder = new ArrayList<>(listedNodes);
-            for (TransportAddress transportAddress : filtered) {
-                DiscoveryNode node = new DiscoveryNode("#transport#-" + tempNodeIdGenerator.incrementAndGet(),
-                        transportAddress, Collections.emptyMap(), Collections.emptySet(), minCompatibilityVersion);
-                logger.debug("adding address [{}]", node);
-                builder.add(node);
-            }
-            listedNodes = Collections.unmodifiableList(builder);
-            nodesSampler.sample();
-        }
-        return this;
-    }
-
-    public TransportClientNodesService removeTransportAddress(TransportAddress transportAddress) {
-        synchronized (mutex) {
-            if (closed) {
-                throw new IllegalStateException("transport client is closed, can't remove an address");
-            }
-            List<DiscoveryNode> listNodesBuilder = new ArrayList<>();
-            for (DiscoveryNode otherNode : listedNodes) {
-                if (!otherNode.getAddress().equals(transportAddress)) {
-                    listNodesBuilder.add(otherNode);
-                } else {
-                    logger.debug("removing address [{}] from listed nodes", otherNode);
-                }
-            }
-            listedNodes = Collections.unmodifiableList(listNodesBuilder);
-            List<DiscoveryNode> nodesBuilder = new ArrayList<>();
-            for (DiscoveryNode otherNode : nodes) {
-                if (!otherNode.getAddress().equals(transportAddress)) {
-                    nodesBuilder.add(otherNode);
-                } else {
-                    logger.debug("disconnecting from node with address [{}]", otherNode);
-                    transportService.disconnectFromNode(otherNode);
-                }
-            }
-            nodes = Collections.unmodifiableList(nodesBuilder);
-            nodesSampler.sample();
-        }
-        return this;
-    }
-
-    public <Response> void execute(NodeListenerCallback<Response> callback, ActionListener<Response> listener) {
-        // we first read nodes before checking the closed state; this
-        // is because otherwise we could be subject to a race where we
-        // read the state as not being closed, and then the client is
-        // closed and the nodes list is cleared, and then a
-        // NoNodeAvailableException is thrown
-        // it is important that the order of first setting the state of
-        // closed and then clearing the list of nodes is maintained in
-        // the close method
-        final List<DiscoveryNode> nodes = this.nodes;
-        if (closed) {
-            throw new IllegalStateException("transport client is closed");
-        }
-        ensureNodesAreAvailable(nodes);
-        int index = getNodeNumber();
-        RetryListener<Response> retryListener = new RetryListener<>(callback, listener, nodes, index, hostFailureListener);
-        DiscoveryNode node = retryListener.getNode(0);
-        try {
-            callback.doWithNode(node, retryListener);
-        } catch (Exception e) {
-            try {
-                //this exception can't come from the TransportService as it doesn't throw exception at all
-                listener.onFailure(e);
-            } finally {
-                retryListener.maybeNodeFailed(node, e);
-            }
-        }
-    }
-
-    public static class RetryListener<Response> implements ActionListener<Response> {
-        private final NodeListenerCallback<Response> callback;
-        private final ActionListener<Response> listener;
-        private final List<DiscoveryNode> nodes;
-        private final int index;
-        private final TransportClient.HostFailureListener hostFailureListener;
-
-        private volatile int i;
-
-        RetryListener(NodeListenerCallback<Response> callback, ActionListener<Response> listener,
-                      List<DiscoveryNode> nodes, int index, TransportClient.HostFailureListener hostFailureListener) {
-            this.callback = callback;
-            this.listener = listener;
-            this.nodes = nodes;
-            this.index = index;
-            this.hostFailureListener = hostFailureListener;
-        }
-
-        @Override
-        public void onResponse(Response response) {
-            listener.onResponse(response);
-        }
-
-        @Override
-        public void onFailure(Exception e) {
-            Throwable throwable = ExceptionsHelper.unwrapCause(e);
-            if (throwable instanceof ConnectTransportException) {
-                maybeNodeFailed(getNode(this.i), (ConnectTransportException) throwable);
-                int i = ++this.i;
-                if (i >= nodes.size()) {
-                    listener.onFailure(new NoNodeAvailableException("None of the configured nodes were available: " + nodes, e));
-                } else {
-                    try {
-                        callback.doWithNode(getNode(i), this);
-                    } catch(final Exception inner) {
-                        inner.addSuppressed(e);
-                        // this exception can't come from the TransportService as it doesn't throw exceptions at all
-                        listener.onFailure(inner);
-                    }
-                }
-            } else {
-                listener.onFailure(e);
-            }
-        }
-
-        final DiscoveryNode getNode(int i) {
-            return nodes.get((index + i) % nodes.size());
-        }
-
-        final void maybeNodeFailed(DiscoveryNode node, Exception ex) {
-            if (ex instanceof NodeDisconnectedException || ex instanceof NodeNotConnectedException) {
-                hostFailureListener.onNodeDisconnected(node, ex);
-            }
-        }
-    }
-
-    @Override
-    public void close() {
-        synchronized (mutex) {
-            if (closed) {
-                return;
-            }
-            closed = true;
-            if (nodesSamplerCancellable != null) {
-                nodesSamplerCancellable.cancel();
-            }
-            for (DiscoveryNode node : nodes) {
-                transportService.disconnectFromNode(node);
-            }
-            for (DiscoveryNode listedNode : listedNodes) {
-                transportService.disconnectFromNode(listedNode);
-            }
-            nodes = Collections.emptyList();
-        }
-    }
-
-    private int getNodeNumber() {
-        int index = randomNodeGenerator.incrementAndGet();
-        if (index < 0) {
-            index = 0;
-            randomNodeGenerator.set(0);
-        }
-        return index;
-    }
-
-    private void ensureNodesAreAvailable(List<DiscoveryNode> nodes) {
-        if (nodes.isEmpty()) {
-            String message = String.format(Locale.ROOT, "None of the configured nodes are available: %s", this.listedNodes);
-            throw new NoNodeAvailableException(message);
-        }
-    }
-
-    abstract class NodeSampler {
-        public void sample() {
-            synchronized (mutex) {
-                if (closed) {
-                    return;
-                }
-                doSample();
-            }
-        }
-
-        protected abstract void doSample();
-
-        /**
-         * Establishes the node connections. If validateInHandshake is set to true, the connection will fail if
-         * node returned in the handshake response is different than the discovery node.
-         */
-        List<DiscoveryNode> establishNodeConnections(Set<DiscoveryNode> nodes) {
-            for (Iterator<DiscoveryNode> it = nodes.iterator(); it.hasNext(); ) {
-                DiscoveryNode node = it.next();
-                if (!transportService.nodeConnected(node)) {
-                    try {
-                        logger.trace("connecting to node [{}]", node);
-                        transportService.connectToNode(node);
-                    } catch (Exception e) {
-                        it.remove();
-                        logger.debug(() -> new ParameterizedMessage("failed to connect to discovered node [{}]", node), e);
-                    }
-                }
-            }
-
-            return List.copyOf(nodes);
-        }
-    }
-
-    class ScheduledNodeSampler implements Runnable {
-        @Override
-        public void run() {
-            try {
-                nodesSampler.sample();
-                if (!closed) {
-                    nodesSamplerCancellable = threadPool.schedule(this, nodesSamplerInterval, ThreadPool.Names.GENERIC);
-                }
-            } catch (Exception e) {
-                logger.warn("failed to sample", e);
-            }
-        }
-    }
-
-    class SimpleNodeSampler extends NodeSampler {
-
-        @Override
-        protected void doSample() {
-            HashSet<DiscoveryNode> newNodes = new HashSet<>();
-            ArrayList<DiscoveryNode> newFilteredNodes = new ArrayList<>();
-            for (DiscoveryNode listedNode : listedNodes) {
-                try (Transport.Connection connection = transportService.openConnection(listedNode, LISTED_NODES_PROFILE)){
-                    final PlainTransportFuture<LivenessResponse> handler = new PlainTransportFuture<>(
-                        new FutureTransportResponseHandler<LivenessResponse>() {
-                            @Override
-                            public LivenessResponse read(StreamInput in) throws IOException {
-                                LivenessResponse response = new LivenessResponse();
-                                response.readFrom(in);
-                                return response;
-                            }
-                        });
-                    transportService.sendRequest(connection, TransportLivenessAction.NAME, new LivenessRequest(),
-                        TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE).withTimeout(pingTimeout).build(),
-                        handler);
-                    final LivenessResponse livenessResponse = handler.txGet();
-                    if (!ignoreClusterName && !clusterName.equals(livenessResponse.getClusterName())) {
-                        logger.warn("node {} not part of the cluster {}, ignoring...", listedNode, clusterName);
-                        newFilteredNodes.add(listedNode);
-                    } else {
-                        // use discovered information but do keep the original transport address,
-                        // so people can control which address is exactly used.
-                        DiscoveryNode nodeWithInfo = livenessResponse.getDiscoveryNode();
-                        newNodes.add(new DiscoveryNode(nodeWithInfo.getName(), nodeWithInfo.getId(), nodeWithInfo.getEphemeralId(),
-                            nodeWithInfo.getHostName(), nodeWithInfo.getHostAddress(), listedNode.getAddress(),
-                            nodeWithInfo.getAttributes(), nodeWithInfo.getRoles(), nodeWithInfo.getVersion()));
-                    }
-                } catch (ConnectTransportException e) {
-                    logger.debug(() -> new ParameterizedMessage("failed to connect to node [{}], ignoring...", listedNode), e);
-                    hostFailureListener.onNodeDisconnected(listedNode, e);
-                } catch (Exception e) {
-                    logger.info(() -> new ParameterizedMessage("failed to get node info for {}, disconnecting...", listedNode), e);
-                }
-            }
-
-            nodes = establishNodeConnections(newNodes);
-            filteredNodes = Collections.unmodifiableList(newFilteredNodes);
-        }
-    }
-
-    class SniffNodesSampler extends NodeSampler {
-
-        @Override
-        protected void doSample() {
-            // the nodes we are going to ping include the core listed nodes that were added
-            // and the last round of discovered nodes
-            Set<DiscoveryNode> nodesToPing = new HashSet<>();
-            for (DiscoveryNode node : listedNodes) {
-                nodesToPing.add(node);
-            }
-            for (DiscoveryNode node : nodes) {
-                nodesToPing.add(node);
-            }
-
-            final CountDownLatch latch = new CountDownLatch(nodesToPing.size());
-            final ConcurrentMap<DiscoveryNode, ClusterStateResponse> clusterStateResponses = ConcurrentCollections.newConcurrentMap();
-            try {
-                for (final DiscoveryNode nodeToPing : nodesToPing) {
-                    threadPool.executor(ThreadPool.Names.MANAGEMENT).execute(new AbstractRunnable() {
-
-                        /**
-                         * we try to reuse existing connections but if needed we will open a temporary connection
-                         * that will be closed at the end of the execution.
-                         */
-                        Transport.Connection connectionToClose = null;
-
-                        void onDone() {
-                            try {
-                                IOUtils.closeWhileHandlingException(connectionToClose);
-                            } finally {
-                                latch.countDown();
-                            }
-                        }
-
-                        @Override
-                        public void onFailure(Exception e) {
-                            onDone();
-                            if (e instanceof ConnectTransportException) {
-                                logger.debug(() -> new ParameterizedMessage("failed to connect to node [{}], ignoring...", nodeToPing), e);
-                                hostFailureListener.onNodeDisconnected(nodeToPing, e);
-                            } else {
-                                logger.info(() -> new ParameterizedMessage(
-                                        "failed to get local cluster state info for {}, disconnecting...", nodeToPing), e);
-                            }
-                        }
-
-                        @Override
-                        protected void doRun() throws Exception {
-                            Transport.Connection pingConnection = null;
-                            if (nodes.contains(nodeToPing)) {
-                                try {
-                                    pingConnection = transportService.getConnection(nodeToPing);
-                                } catch (NodeNotConnectedException e) {
-                                    // will use a temp connection
-                                }
-                            }
-                            if (pingConnection == null) {
-                                logger.trace("connecting to cluster node [{}]", nodeToPing);
-                                connectionToClose = transportService.openConnection(nodeToPing, LISTED_NODES_PROFILE);
-                                pingConnection = connectionToClose;
-                            }
-                            transportService.sendRequest(pingConnection, ClusterStateAction.NAME,
-                                Requests.clusterStateRequest().clear().nodes(true).local(true),
-                                TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE)
-                                    .withTimeout(pingTimeout).build(),
-                                new TransportResponseHandler<ClusterStateResponse>() {
-
-                                    @Override
-                                    public ClusterStateResponse read(StreamInput in) throws IOException {
-                                        final ClusterStateResponse clusterStateResponse = new ClusterStateResponse();
-                                        clusterStateResponse.readFrom(in);
-                                        return clusterStateResponse;
-                                    }
-
-                                    @Override
-                                    public String executor() {
-                                        return ThreadPool.Names.SAME;
-                                    }
-
-                                    @Override
-                                    public void handleResponse(ClusterStateResponse response) {
-                                        clusterStateResponses.put(nodeToPing, response);
-                                        onDone();
-                                    }
-
-                                    @Override
-                                    public void handleException(TransportException e) {
-                                        logger.info(() -> new ParameterizedMessage(
-                                                "failed to get local cluster state for {}, disconnecting...", nodeToPing), e);
-                                        try {
-                                            hostFailureListener.onNodeDisconnected(nodeToPing, e);
-                                        } finally {
-                                            onDone();
-                                        }
-                                    }
-                                });
-                        }
-                    });
-                }
-                latch.await();
-            } catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-                return;
-            }
-
-            HashSet<DiscoveryNode> newNodes = new HashSet<>();
-            HashSet<DiscoveryNode> newFilteredNodes = new HashSet<>();
-            for (Map.Entry<DiscoveryNode, ClusterStateResponse> entry : clusterStateResponses.entrySet()) {
-                if (!ignoreClusterName && !clusterName.equals(entry.getValue().getClusterName())) {
-                    logger.warn("node {} not part of the cluster {}, ignoring...",
-                            entry.getValue().getState().nodes().getLocalNode(), clusterName);
-                    newFilteredNodes.add(entry.getKey());
-                    continue;
-                }
-                for (ObjectCursor<DiscoveryNode> cursor : entry.getValue().getState().nodes().getDataNodes().values()) {
-                    newNodes.add(cursor.value);
-                }
-            }
-
-            nodes = establishNodeConnections(newNodes);
-            filteredNodes = List.copyOf(newFilteredNodes);
-        }
-    }
-
-    public interface NodeListenerCallback<Response> {
-
-        void doWithNode(DiscoveryNode node, ActionListener<Response> listener);
-    }
-
-    // pkg private for testing
-    void doSample() {
-        nodesSampler.doSample();
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java b/server/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java
deleted file mode 100644
index d79e2a9119e6a..0000000000000
--- a/server/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client.transport;
-
-import org.elasticsearch.action.Action;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.TransportActionNodeProxy;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.transport.TransportService;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static java.util.Collections.unmodifiableMap;
-
-final class TransportProxyClient {
-
-    private final TransportClientNodesService nodesService;
-    private final Map<Action, TransportActionNodeProxy> proxies;
-
-    TransportProxyClient(Settings settings, TransportService transportService,
-                                TransportClientNodesService nodesService, List<Action> actions) {
-        this.nodesService = nodesService;
-        Map<Action, TransportActionNodeProxy> proxies = new HashMap<>();
-        for (Action action : actions) {
-            proxies.put(action, new TransportActionNodeProxy(settings, action, transportService));
-        }
-        this.proxies = unmodifiableMap(proxies);
-    }
-
-    public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends
-        ActionRequestBuilder<Request, Response>> void execute(final Action<Response> action,
-                                                                              final Request request, ActionListener<Response> listener) {
-        final TransportActionNodeProxy<Request, Response> proxy = proxies.get(action);
-        assert proxy != null : "no proxy found for action: " + action;
-        nodesService.execute((n, l) -> proxy.execute(n, request, l), listener);
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index d3739fd6cdcba..d0448e2be22c9 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService;
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
 import org.elasticsearch.cluster.routing.DelayedAllocationService;
-import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
@@ -65,7 +64,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.ingest.IngestMetadata;
@@ -78,13 +76,11 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 import java.util.function.Function;
 import java.util.function.Supplier;
 
@@ -134,35 +130,6 @@ public static List<Entry> getNamedWriteables() {
         return entries;
     }
 
-    static final Set<String> PRE_6_3_METADATA_CUSTOMS_WHITE_LIST = Collections.unmodifiableSet(Sets.newHashSet(
-        IndexGraveyard.TYPE, IngestMetadata.TYPE, RepositoriesMetaData.TYPE, ScriptMetaData.TYPE));
-
-    static final Set<String> PRE_6_3_CLUSTER_CUSTOMS_WHITE_LIST = Collections.unmodifiableSet(Sets.newHashSet(
-        RestoreInProgress.TYPE, SnapshotDeletionsInProgress.TYPE, SnapshotsInProgress.TYPE));
-
-    /**
-     * For interoperability with transport clients older than 6.3, we need to strip customs
-     * from the cluster state that the client might not be able to deserialize
-     *
-     * @param clusterState the cluster state to filter the customs from
-     * @return the adapted cluster state
-     */
-    public static ClusterState filterCustomsForPre63Clients(ClusterState clusterState) {
-        final ClusterState.Builder builder = ClusterState.builder(clusterState);
-        clusterState.customs().keysIt().forEachRemaining(name -> {
-            if (PRE_6_3_CLUSTER_CUSTOMS_WHITE_LIST.contains(name) == false) {
-                builder.removeCustom(name);
-            }
-        });
-        final MetaData.Builder metaBuilder = MetaData.builder(clusterState.metaData());
-        clusterState.metaData().customs().keysIt().forEachRemaining(name -> {
-            if (PRE_6_3_METADATA_CUSTOMS_WHITE_LIST.contains(name) == false) {
-                metaBuilder.removeCustom(name);
-            }
-        });
-        return builder.metaData(metaBuilder).build();
-    }
-
     public static List<NamedXContentRegistry.Entry> getNamedXWriteables() {
         List<NamedXContentRegistry.Entry> entries = new ArrayList<>();
         // Metadata
@@ -274,7 +241,6 @@ protected void configure() {
         bind(MetaDataUpdateSettingsService.class).asEagerSingleton();
         bind(MetaDataIndexTemplateService.class).asEagerSingleton();
         bind(IndexNameExpressionResolver.class).toInstance(indexNameExpressionResolver);
-        bind(RoutingService.class).asEagerSingleton();
         bind(DelayedAllocationService.class).asEagerSingleton();
         bind(ShardStateAction.class).asEagerSingleton();
         bind(NodeMappingRefreshAction.class).asEagerSingleton();
diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
index 6a5e2a324965f..51a1dc3835e9c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
+++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
@@ -23,7 +23,6 @@
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.Version;
-import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.coordination.CoordinationMetaData;
@@ -114,12 +113,9 @@ default Optional<String> getRequiredFeature() {
          * Tests whether or not the custom should be serialized. The criteria are:
          * <ul>
          * <li>the output stream must be at least the minimum supported version of the custom</li>
-         * <li>the output stream must have the feature required by the custom (if any) or not be a transport client</li>
          * </ul>
          * <p>
-         * That is, we only serialize customs to clients than can understand the custom based on the version of the client and the features
-         * that the client has. For transport clients we can be lenient in requiring a feature in which case we do not send the custom but
-         * for connected nodes we always require that the node has the required feature.
+         * That is, we only serialize customs to clients than can understand the custom based on the version of the client.
          *
          * @param out    the output stream
          * @param custom the custom to serialize
@@ -127,15 +123,7 @@ default Optional<String> getRequiredFeature() {
          * @return true if the custom should be serialized and false otherwise
          */
         static <T extends VersionedNamedWriteable & FeatureAware> boolean shouldSerialize(final StreamOutput out, final T custom) {
-            if (out.getVersion().before(custom.getMinimalSupportedVersion())) {
-                return false;
-            }
-            if (custom.getRequiredFeature().isPresent()) {
-                final String requiredFeature = custom.getRequiredFeature().get();
-                // if it is a transport client we are lenient yet for a connected node it must have the required feature
-                return out.hasFeature(requiredFeature) || out.hasFeature(TransportClient.TRANSPORT_CLIENT_FEATURE) == false;
-            }
-            return true;
+            return out.getVersion().onOrAfter(custom.getMinimalSupportedVersion());
         }
 
     }
@@ -297,7 +285,8 @@ public RoutingNodes getRoutingNodes() {
     public String toString() {
         StringBuilder sb = new StringBuilder();
         final String TAB = "   ";
-        sb.append("cluster uuid: ").append(metaData.clusterUUID()).append("\n");
+        sb.append("cluster uuid: ").append(metaData.clusterUUID())
+            .append(" [committed: ").append(metaData.clusterUUIDCommitted()).append("]").append("\n");
         sb.append("version: ").append(version).append("\n");
         sb.append("state uuid: ").append(stateUUID).append("\n");
         sb.append("from_diff: ").append(wasReadFromDiff).append("\n");
diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
index 8d78f9c838e38..4b8936198911e 100644
--- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
@@ -131,13 +131,13 @@ public void onMaster() {
             logger.trace("I have been elected master, scheduling a ClusterInfoUpdateJob");
         }
 
-        // Submit a job that will start after DEFAULT_STARTING_INTERVAL, and reschedule itself after running
+        // Submit a job that will reschedule itself after running
         threadPool.scheduleUnlessShuttingDown(updateFrequency, executorName(), new SubmitReschedulingClusterInfoUpdatedJob());
 
         try {
             if (clusterService.state().getNodes().getDataNodes().size() > 1) {
                 // Submit an info update job to be run immediately
-                threadPool.executor(executorName()).execute(() -> maybeRefresh());
+                threadPool.executor(executorName()).execute(this::maybeRefresh);
             }
         } catch (EsRejectedExecutionException ex) {
             logger.debug("Couldn't schedule cluster info update task - node might be shutting down", ex);
@@ -173,7 +173,7 @@ public void clusterChanged(ClusterChangedEvent event) {
             if (logger.isDebugEnabled()) {
                 logger.debug("data node was added, retrieving new cluster info");
             }
-            threadPool.executor(executorName()).execute(() -> maybeRefresh());
+            threadPool.executor(executorName()).execute(this::maybeRefresh);
         }
 
         if (this.isMaster && event.nodesRemoved()) {
@@ -316,7 +316,7 @@ public void onResponse(IndicesStatsResponse indicesStatsResponse) {
                 ShardStats[] stats = indicesStatsResponse.getShards();
                 ImmutableOpenMap.Builder<String, Long> newShardSizes = ImmutableOpenMap.builder();
                 ImmutableOpenMap.Builder<ShardRouting, String> newShardRoutingToDataPath = ImmutableOpenMap.builder();
-                buildShardLevelInfo(logger, stats, newShardSizes, newShardRoutingToDataPath, clusterService.state());
+                buildShardLevelInfo(logger, stats, newShardSizes, newShardRoutingToDataPath);
                 shardSizes = newShardSizes.build();
                 shardRoutingToDataPath = newShardRoutingToDataPath.build();
             }
@@ -365,7 +365,7 @@ public void onFailure(Exception e) {
     }
 
     static void buildShardLevelInfo(Logger logger, ShardStats[] stats, ImmutableOpenMap.Builder<String, Long> newShardSizes,
-                                    ImmutableOpenMap.Builder<ShardRouting, String> newShardRoutingToDataPath, ClusterState state) {
+                                    ImmutableOpenMap.Builder<ShardRouting, String> newShardRoutingToDataPath) {
         for (ShardStats s : stats) {
             newShardRoutingToDataPath.put(s.getShardRouting(), s.getDataPath());
             long size = s.getStats().getStore().sizeInBytes();
diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
index ae9506706e36a..039c7ec447767 100644
--- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
+++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
@@ -44,6 +44,8 @@
 import java.util.Map;
 import java.util.Objects;
 
+import static org.elasticsearch.snapshots.SnapshotInfo.METADATA_FIELD_INTRODUCED;
+
 /**
  * Meta data about snapshots that are currently executing
  */
@@ -84,11 +86,12 @@ public static class Entry {
         private final ImmutableOpenMap<String, List<ShardId>> waitingIndices;
         private final long startTime;
         private final long repositoryStateId;
+        @Nullable private final Map<String, Object> userMetadata;
         @Nullable private final String failure;
 
         public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List<IndexId> indices,
                      long startTime, long repositoryStateId, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards,
-                     String failure) {
+                     String failure, Map<String, Object> userMetadata) {
             this.state = state;
             this.snapshot = snapshot;
             this.includeGlobalState = includeGlobalState;
@@ -104,21 +107,23 @@ public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, Sta
             }
             this.repositoryStateId = repositoryStateId;
             this.failure = failure;
+            this.userMetadata = userMetadata;
         }
 
         public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List<IndexId> indices,
-            long startTime, long repositoryStateId, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
-            this(snapshot, includeGlobalState, partial, state, indices, startTime, repositoryStateId, shards, null);
+                     long startTime, long repositoryStateId, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards,
+                     Map<String, Object> userMetadata) {
+            this(snapshot, includeGlobalState, partial, state, indices, startTime, repositoryStateId, shards, null, userMetadata);
         }
 
         public Entry(Entry entry, State state, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
             this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime,
-                entry.repositoryStateId, shards, entry.failure);
+                entry.repositoryStateId, shards, entry.failure, entry.userMetadata);
         }
 
         public Entry(Entry entry, State state, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards, String failure) {
             this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime,
-                 entry.repositoryStateId, shards, failure);
+                 entry.repositoryStateId, shards, failure, entry.userMetadata);
         }
 
         public Entry(Entry entry, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
@@ -149,6 +154,10 @@ public boolean includeGlobalState() {
             return includeGlobalState;
         }
 
+        public Map<String, Object> userMetadata() {
+            return userMetadata;
+        }
+
         public boolean partial() {
             return partial;
         }
@@ -419,6 +428,10 @@ public SnapshotsInProgress(StreamInput in) throws IOException {
             }
             long repositoryStateId = in.readLong();
             final String failure = in.readOptionalString();
+            Map<String, Object> userMetadata = null;
+            if (in.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) {
+                userMetadata = in.readMap();
+            }
             entries[i] = new Entry(snapshot,
                                    includeGlobalState,
                                    partial,
@@ -427,7 +440,9 @@ public SnapshotsInProgress(StreamInput in) throws IOException {
                                    startTime,
                                    repositoryStateId,
                                    builder.build(),
-                                   failure);
+                                   failure,
+                                   userMetadata
+                );
         }
         this.entries = Arrays.asList(entries);
     }
@@ -452,6 +467,9 @@ public void writeTo(StreamOutput out) throws IOException {
             }
             out.writeLong(entry.repositoryStateId);
             out.writeOptionalString(entry.failure);
+            if (out.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) {
+                out.writeMap(entry.userMetadata);
+            }
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
index 1f9a45c4713f6..d28cf40026d6c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
+++ b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
@@ -36,7 +36,7 @@
 import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RoutingService;
+import org.elasticsearch.cluster.routing.RerouteService;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedShard;
@@ -89,7 +89,7 @@ public class ShardStateAction {
 
     @Inject
     public ShardStateAction(ClusterService clusterService, TransportService transportService,
-                            AllocationService allocationService, RoutingService routingService, ThreadPool threadPool) {
+                            AllocationService allocationService, RerouteService rerouteService, ThreadPool threadPool) {
         this.transportService = transportService;
         this.clusterService = clusterService;
         this.threadPool = threadPool;
@@ -98,7 +98,7 @@ public ShardStateAction(ClusterService clusterService, TransportService transpor
             new ShardStartedTransportHandler(clusterService, new ShardStartedClusterStateTaskExecutor(allocationService, logger), logger));
         transportService.registerRequestHandler(SHARD_FAILED_ACTION_NAME, ThreadPool.Names.SAME, FailedShardEntry::new,
             new ShardFailedTransportHandler(clusterService,
-                new ShardFailedClusterStateTaskExecutor(allocationService, routingService, logger), logger));
+                new ShardFailedClusterStateTaskExecutor(allocationService, rerouteService, logger), logger));
     }
 
     private void sendShardAction(final String actionName, final ClusterState currentState,
@@ -280,12 +280,12 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
 
     public static class ShardFailedClusterStateTaskExecutor implements ClusterStateTaskExecutor<FailedShardEntry> {
         private final AllocationService allocationService;
-        private final RoutingService routingService;
+        private final RerouteService rerouteService;
         private final Logger logger;
 
-        public ShardFailedClusterStateTaskExecutor(AllocationService allocationService, RoutingService routingService, Logger logger) {
+        public ShardFailedClusterStateTaskExecutor(AllocationService allocationService, RerouteService rerouteService, Logger logger) {
             this.allocationService = allocationService;
-            this.routingService = routingService;
+            this.rerouteService = rerouteService;
             this.logger = logger;
         }
 
@@ -379,7 +379,9 @@ public void clusterStatePublished(ClusterChangedEvent clusterChangedEvent) {
                 if (logger.isTraceEnabled()) {
                     logger.trace("{}, scheduling a reroute", reason);
                 }
-                routingService.reroute(reason);
+                rerouteService.reroute(reason, ActionListener.wrap(
+                    r -> logger.trace("{}, reroute completed", reason),
+                    e -> logger.debug(new ParameterizedMessage("{}, reroute failed", reason), e)));
             }
         }
     }
@@ -580,7 +582,11 @@ public ClusterTasksResult<StartedShardEntry> execute(ClusterState currentState,
 
         @Override
         public void onFailure(String source, Exception e) {
-            logger.error(() -> new ParameterizedMessage("unexpected failure during [{}]", source), e);
+            if (e instanceof FailedToCommitClusterStateException || e instanceof NotMasterException) {
+                logger.debug(() -> new ParameterizedMessage("failure during [{}]", source), e);
+            } else {
+                logger.error(() -> new ParameterizedMessage("unexpected failure during [{}]", source), e);
+            }
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java
index a707a9ae980b7..c249458dd23cf 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java
@@ -20,7 +20,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
 import org.elasticsearch.cluster.coordination.CoordinationState.VoteCollection;
@@ -122,14 +121,16 @@ static class ClusterFormationState {
         private final List<TransportAddress> resolvedAddresses;
         private final List<DiscoveryNode> foundPeers;
         private final long currentTerm;
+        private final ElectionStrategy electionStrategy;
 
         ClusterFormationState(Settings settings, ClusterState clusterState, List<TransportAddress> resolvedAddresses,
-                              List<DiscoveryNode> foundPeers, long currentTerm) {
+                              List<DiscoveryNode> foundPeers, long currentTerm, ElectionStrategy electionStrategy) {
             this.settings = settings;
             this.clusterState = clusterState;
             this.resolvedAddresses = resolvedAddresses;
             this.foundPeers = foundPeers;
             this.currentTerm = currentTerm;
+            this.electionStrategy = electionStrategy;
         }
 
         String getDescription() {
@@ -150,8 +151,6 @@ String getDescription() {
 
             if (clusterState.getLastAcceptedConfiguration().isEmpty()) {
 
-                // TODO handle the case that there is a 6.x node around here, when rolling upgrades are supported
-
                 final String bootstrappingDescription;
 
                 if (INITIAL_MASTER_NODES_SETTING.get(Settings.EMPTY).equals(INITIAL_MASTER_NODES_SETTING.get(settings))) {
@@ -164,8 +163,8 @@ String getDescription() {
                 }
 
                 return String.format(Locale.ROOT,
-                    "master not discovered yet, this node has not previously joined a bootstrapped (v%d+) cluster, and %s: %s",
-                    Version.V_6_6_0.major + 1, bootstrappingDescription, discoveryStateIgnoringQuorum);
+                    "master not discovered yet, this node has not previously joined a bootstrapped cluster, and %s: %s",
+                    bootstrappingDescription, discoveryStateIgnoringQuorum);
             }
 
             assert clusterState.getLastCommittedConfiguration().isEmpty() == false;
@@ -188,7 +187,9 @@ String getDescription() {
             final VoteCollection voteCollection = new VoteCollection();
             foundPeers.forEach(voteCollection::addVote);
             final String isQuorumOrNot
-                = CoordinationState.isElectionQuorum(voteCollection, clusterState) ? "is a quorum" : "is not a quorum";
+                = electionStrategy.isElectionQuorum(clusterState.nodes().getLocalNode(), currentTerm, clusterState.term(),
+                    clusterState.version(), clusterState.getLastCommittedConfiguration(), clusterState.getLastAcceptedConfiguration(),
+                    voteCollection) ? "is a quorum" : "is not a quorum";
 
             return String.format(Locale.ROOT,
                 "master not discovered or elected yet, an election requires %s, have discovered %s which %s; %s",
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java
index ac75c83c19a26..7aad43aaab288 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationState.java
@@ -24,13 +24,14 @@
 import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.settings.Settings;
 
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 
 /**
  * The core class of the cluster state coordination algorithm, directly implementing the
@@ -42,6 +43,8 @@ public class CoordinationState {
 
     private final DiscoveryNode localNode;
 
+    private final ElectionStrategy electionStrategy;
+
     // persisted state
     private final PersistedState persistedState;
 
@@ -53,11 +56,12 @@ public class CoordinationState {
     private VotingConfiguration lastPublishedConfiguration;
     private VoteCollection publishVotes;
 
-    public CoordinationState(Settings settings, DiscoveryNode localNode, PersistedState persistedState) {
+    public CoordinationState(DiscoveryNode localNode, PersistedState persistedState, ElectionStrategy electionStrategy) {
         this.localNode = localNode;
 
         // persisted state
         this.persistedState = persistedState;
+        this.electionStrategy = electionStrategy;
 
         // transient state
         this.joinVotes = new VoteCollection();
@@ -100,13 +104,9 @@ public boolean electionWon() {
         return electionWon;
     }
 
-    public boolean isElectionQuorum(VoteCollection votes) {
-        return isElectionQuorum(votes, getLastAcceptedState());
-    }
-
-    static boolean isElectionQuorum(VoteCollection votes, ClusterState lastAcceptedState) {
-        return votes.isQuorum(lastAcceptedState.getLastCommittedConfiguration())
-            && votes.isQuorum(lastAcceptedState.getLastAcceptedConfiguration());
+    public boolean isElectionQuorum(VoteCollection joinVotes) {
+        return electionStrategy.isElectionQuorum(localNode, getCurrentTerm(), getLastAcceptedTerm(), getLastAcceptedVersion(),
+            getLastCommittedConfiguration(), getLastAcceptedConfiguration(), joinVotes);
     }
 
     public boolean isPublishQuorum(VoteCollection votes) {
@@ -117,6 +117,11 @@ public boolean containsJoinVoteFor(DiscoveryNode node) {
         return joinVotes.containsVoteFor(node);
     }
 
+    // used for tests
+    boolean containsJoin(Join join) {
+        return joinVotes.getJoins().contains(join);
+    }
+
     public boolean joinVotesHaveQuorumFor(VotingConfiguration votingConfiguration) {
         return joinVotes.isQuorum(votingConfiguration);
     }
@@ -243,10 +248,11 @@ public boolean handleJoin(Join join) {
             throw new CoordinationStateRejectedException("rejecting join since this node has not received its initial configuration yet");
         }
 
-        boolean added = joinVotes.addVote(join.getSourceNode());
+        boolean added = joinVotes.addJoinVote(join);
         boolean prevElectionWon = electionWon;
         electionWon = isElectionQuorum(joinVotes);
-        assert !prevElectionWon || electionWon; // we cannot go from won to not won
+        assert !prevElectionWon || electionWon : // we cannot go from won to not won
+            "locaNode= " + localNode + ", join=" + join + ", joinVotes=" + joinVotes;
         logger.debug("handleJoin: added join {} from [{}] for election, electionWon={} lastAcceptedTerm={} lastAcceptedVersion={}", join,
             join.getSourceNode(), electionWon, lastAcceptedTerm, getLastAcceptedVersion());
 
@@ -489,18 +495,28 @@ default void markLastAcceptedStateAsCommitted() {
     }
 
     /**
-     * A collection of votes, used to calculate quorums.
+     * A collection of votes, used to calculate quorums. Optionally records the Joins as well.
      */
     public static class VoteCollection {
 
         private final Map<String, DiscoveryNode> nodes;
+        private final Set<Join> joins;
 
         public boolean addVote(DiscoveryNode sourceNode) {
             return nodes.put(sourceNode.getId(), sourceNode) == null;
         }
 
+        public boolean addJoinVote(Join join) {
+            final boolean added = addVote(join.getSourceNode());
+            if (added) {
+                joins.add(join);
+            }
+            return added;
+        }
+
         public VoteCollection() {
             nodes = new HashMap<>();
+            joins = new HashSet<>();
         }
 
         public boolean isQuorum(VotingConfiguration configuration) {
@@ -519,24 +535,31 @@ public Collection<DiscoveryNode> nodes() {
             return Collections.unmodifiableCollection(nodes.values());
         }
 
+        public Set<Join> getJoins() {
+            return Collections.unmodifiableSet(joins);
+        }
+
         @Override
         public String toString() {
-            return "VoteCollection{" + String.join(",", nodes.keySet()) + "}";
+            return "VoteCollection{votes=" + nodes.keySet() + ", joins=" + joins + "}";
         }
 
         @Override
         public boolean equals(Object o) {
             if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
+            if (!(o instanceof VoteCollection)) return false;
 
             VoteCollection that = (VoteCollection) o;
 
-            return nodes.equals(that.nodes);
+            if (!nodes.equals(that.nodes)) return false;
+            return joins.equals(that.joins);
         }
 
         @Override
         public int hashCode() {
-            return nodes.hashCode();
+            int result = nodes.hashCode();
+            result = 31 * result + joins.hashCode();
+            return result;
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java
index 1e7b38e50d1e9..8de27337f1cba 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java
@@ -33,11 +33,13 @@
 import org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper.ClusterFormationState;
 import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfigExclusion;
 import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
+import org.elasticsearch.cluster.coordination.CoordinationState.VoteCollection;
 import org.elasticsearch.cluster.coordination.FollowersChecker.FollowerCheckRequest;
 import org.elasticsearch.cluster.coordination.JoinHelper.InitialJoinAccumulator;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RerouteService;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterApplier;
 import org.elasticsearch.cluster.service.ClusterApplier.ClusterApplyListener;
@@ -81,6 +83,7 @@
 import java.util.function.BiConsumer;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
 import static org.elasticsearch.cluster.coordination.NoMasterBlockService.NO_MASTER_BLOCK_ID;
@@ -98,6 +101,7 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery
 
     private final Settings settings;
     private final boolean singleNodeDiscovery;
+    private final ElectionStrategy electionStrategy;
     private final TransportService transportService;
     private final MasterService masterService;
     private final AllocationService allocationService;
@@ -138,18 +142,25 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery
     private JoinHelper.JoinAccumulator joinAccumulator;
     private Optional<CoordinatorPublication> currentPublication = Optional.empty();
 
+    /**
+     * @param nodeName The name of the node, used to name the {@link java.util.concurrent.ExecutorService} of the {@link SeedHostsResolver}.
+     * @param onJoinValidators A collection of join validators to restrict which nodes may join the cluster.
+     */
     public Coordinator(String nodeName, Settings settings, ClusterSettings clusterSettings, TransportService transportService,
                        NamedWriteableRegistry namedWriteableRegistry, AllocationService allocationService, MasterService masterService,
                        Supplier<CoordinationState.PersistedState> persistedStateSupplier, SeedHostsProvider seedHostsProvider,
-                       ClusterApplier clusterApplier, Collection<BiConsumer<DiscoveryNode, ClusterState>> onJoinValidators, Random random) {
+                       ClusterApplier clusterApplier, Collection<BiConsumer<DiscoveryNode, ClusterState>> onJoinValidators, Random random,
+                       RerouteService rerouteService, ElectionStrategy electionStrategy) {
         this.settings = settings;
         this.transportService = transportService;
         this.masterService = masterService;
         this.allocationService = allocationService;
         this.onJoinValidators = JoinTaskExecutor.addBuiltInJoinValidators(onJoinValidators);
         this.singleNodeDiscovery = DiscoveryModule.SINGLE_NODE_DISCOVERY_TYPE.equals(DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings));
+        this.electionStrategy = electionStrategy;
         this.joinHelper = new JoinHelper(settings, allocationService, masterService, transportService,
-            this::getCurrentTerm, this::getStateForMasterService, this::handleJoinRequest, this::joinLeaderInTerm, this.onJoinValidators);
+            this::getCurrentTerm, this::getStateForMasterService, this::handleJoinRequest, this::joinLeaderInTerm, this.onJoinValidators,
+            rerouteService);
         this.persistedStateSupplier = persistedStateSupplier;
         this.noMasterBlockService = new NoMasterBlockService(settings, clusterSettings);
         this.lastKnownLeader = Optional.empty();
@@ -158,13 +169,13 @@ public Coordinator(String nodeName, Settings settings, ClusterSettings clusterSe
         this.publishTimeout = PUBLISH_TIMEOUT_SETTING.get(settings);
         this.random = random;
         this.electionSchedulerFactory = new ElectionSchedulerFactory(settings, random, transportService.getThreadPool());
-        this.preVoteCollector = new PreVoteCollector(transportService, this::startElection, this::updateMaxTermSeen);
+        this.preVoteCollector = new PreVoteCollector(transportService, this::startElection, this::updateMaxTermSeen, electionStrategy);
         configuredHostsResolver = new SeedHostsResolver(nodeName, settings, transportService, seedHostsProvider);
         this.peerFinder = new CoordinatorPeerFinder(settings, transportService,
             new HandshakingTransportAddressConnector(settings, transportService), configuredHostsResolver);
         this.publicationHandler = new PublicationTransportHandler(transportService, namedWriteableRegistry,
             this::handlePublishRequest, this::handleApplyCommit);
-        this.leaderChecker = new LeaderChecker(settings, transportService, getOnLeaderFailure());
+        this.leaderChecker = new LeaderChecker(settings, transportService, this::onLeaderFailure);
         this.followersChecker = new FollowersChecker(settings, transportService, this::onFollowerCheckRequest, this::removeNode);
         this.nodeRemovalExecutor = new NodeRemovalClusterStateTaskExecutor(allocationService, logger);
         this.clusterApplier = clusterApplier;
@@ -180,23 +191,18 @@ public Coordinator(String nodeName, Settings settings, ClusterSettings clusterSe
 
     private ClusterFormationState getClusterFormationState() {
         return new ClusterFormationState(settings, getStateForMasterService(), peerFinder.getLastResolvedAddresses(),
-            StreamSupport.stream(peerFinder.getFoundPeers().spliterator(), false).collect(Collectors.toList()), getCurrentTerm());
+            Stream.concat(Stream.of(getLocalNode()), StreamSupport.stream(peerFinder.getFoundPeers().spliterator(), false))
+                    .collect(Collectors.toList()), getCurrentTerm(), electionStrategy);
     }
 
-    private Runnable getOnLeaderFailure() {
-        return new Runnable() {
-            @Override
-            public void run() {
-                synchronized (mutex) {
-                    becomeCandidate("onLeaderFailure");
-                }
-            }
-
-            @Override
-            public String toString() {
-                return "notification of leader failure";
+    private void onLeaderFailure(Exception e) {
+        synchronized (mutex) {
+            if (mode != Mode.CANDIDATE) {
+                assert lastKnownLeader.isPresent();
+                logger.info(new ParameterizedMessage("master node [{}] failed, restarting discovery", lastKnownLeader.get()), e);
             }
-        };
+            becomeCandidate("onLeaderFailure");
+        }
     }
 
     private void removeNode(DiscoveryNode discoveryNode, String reason) {
@@ -361,9 +367,8 @@ private void startElection() {
             // The preVoteCollector is only active while we are candidate, but it does not call this method with synchronisation, so we have
             // to check our mode again here.
             if (mode == Mode.CANDIDATE) {
-                if (electionQuorumContainsLocalNode(getLastAcceptedState()) == false) {
-                    logger.trace("skip election as local node is not part of election quorum: {}",
-                        getLastAcceptedState().coordinationMetaData());
+                if (localNodeMayWinElection(getLastAcceptedState()) == false) {
+                    logger.trace("skip election as local node may not win it: {}", getLastAcceptedState().coordinationMetaData());
                     return;
                 }
 
@@ -388,16 +393,17 @@ private void abdicateTo(DiscoveryNode newMaster) {
         becomeCandidate("after abdicating to " + newMaster);
     }
 
-    private static boolean electionQuorumContainsLocalNode(ClusterState lastAcceptedState) {
+    private static boolean localNodeMayWinElection(ClusterState lastAcceptedState) {
         final DiscoveryNode localNode = lastAcceptedState.nodes().getLocalNode();
         assert localNode != null;
-        return electionQuorumContains(lastAcceptedState, localNode);
+        return nodeMayWinElection(lastAcceptedState, localNode);
     }
 
-    private static boolean electionQuorumContains(ClusterState lastAcceptedState, DiscoveryNode node) {
+    private static boolean nodeMayWinElection(ClusterState lastAcceptedState, DiscoveryNode node) {
         final String nodeId = node.getId();
         return lastAcceptedState.getLastCommittedConfiguration().getNodeIds().contains(nodeId)
-            || lastAcceptedState.getLastAcceptedConfiguration().getNodeIds().contains(nodeId);
+            || lastAcceptedState.getLastAcceptedConfiguration().getNodeIds().contains(nodeId)
+            || lastAcceptedState.getVotingConfigExclusions().stream().noneMatch(vce -> vce.getNodeId().equals(nodeId));
     }
 
     private Optional<Join> ensureTermAtLeast(DiscoveryNode sourceNode, long targetTerm) {
@@ -644,7 +650,7 @@ boolean publicationInProgress() {
     protected void doStart() {
         synchronized (mutex) {
             CoordinationState.PersistedState persistedState = persistedStateSupplier.get();
-            coordinationState.set(new CoordinationState(settings, getLocalNode(), persistedState));
+            coordinationState.set(new CoordinationState(getLocalNode(), persistedState, electionStrategy));
             peerFinder.setCurrentTerm(getCurrentTerm());
             configuredHostsResolver.start();
             final ClusterState lastAcceptedState = coordinationState.get().getLastAcceptedState();
@@ -699,7 +705,6 @@ public void invariant() {
             assert followersChecker.getFastResponseState().term == getCurrentTerm() : followersChecker.getFastResponseState();
             assert followersChecker.getFastResponseState().mode == getMode() : followersChecker.getFastResponseState();
             assert (applierState.nodes().getMasterNodeId() == null) == applierState.blocks().hasGlobalBlockWithId(NO_MASTER_BLOCK_ID);
-            assert applierState.nodes().getMasterNodeId() == null || applierState.metaData().clusterUUIDCommitted();
             assert preVoteCollector.getPreVoteResponse().equals(getPreVoteResponse())
                 : preVoteCollector + " vs " + getPreVoteResponse();
 
@@ -835,8 +840,8 @@ public boolean setInitialConfiguration(final VotingConfiguration votingConfigura
             metaDataBuilder.coordinationMetaData(coordinationMetaData);
 
             coordinationState.get().setInitialState(ClusterState.builder(currentState).metaData(metaDataBuilder).build());
-            assert electionQuorumContainsLocalNode(getLastAcceptedState()) :
-                "initial state does not have local node in its election quorum: " + getLastAcceptedState().coordinationMetaData();
+            assert localNodeMayWinElection(getLastAcceptedState()) :
+                "initial state does not allow local node to win election: " + getLastAcceptedState().coordinationMetaData();
             preVoteCollector.update(getPreVoteResponse(), null); // pick up the change to last-accepted version
             startElectionScheduler();
             return true;
@@ -1097,11 +1102,10 @@ protected void onFoundPeersUpdated() {
             synchronized (mutex) {
                 final Iterable<DiscoveryNode> foundPeers = getFoundPeers();
                 if (mode == Mode.CANDIDATE) {
-                    final CoordinationState.VoteCollection expectedVotes = new CoordinationState.VoteCollection();
+                    final VoteCollection expectedVotes = new VoteCollection();
                     foundPeers.forEach(expectedVotes::addVote);
                     expectedVotes.addVote(Coordinator.this.getLocalNode());
-                    final ClusterState lastAcceptedState = coordinationState.get().getLastAcceptedState();
-                    final boolean foundQuorum = CoordinationState.isElectionQuorum(expectedVotes, lastAcceptedState);
+                    final boolean foundQuorum = coordinationState.get().isElectionQuorum(expectedVotes);
 
                     if (foundQuorum) {
                         if (electionScheduler == null) {
@@ -1132,8 +1136,8 @@ public void run() {
                     if (mode == Mode.CANDIDATE) {
                         final ClusterState lastAcceptedState = coordinationState.get().getLastAcceptedState();
 
-                        if (electionQuorumContainsLocalNode(lastAcceptedState) == false) {
-                            logger.trace("skip prevoting as local node is not part of election quorum: {}",
+                        if (localNodeMayWinElection(lastAcceptedState) == false) {
+                            logger.trace("skip prevoting as local node may not win election: {}",
                                 lastAcceptedState.coordinationMetaData());
                             return;
                         }
@@ -1165,9 +1169,13 @@ public Iterable<DiscoveryNode> getFoundPeers() {
      */
     boolean cancelCommittedPublication() {
         synchronized (mutex) {
-            if (currentPublication.isPresent() && currentPublication.get().isCommitted()) {
-                currentPublication.get().cancel("cancelCommittedPublication");
-                return true;
+            if (currentPublication.isPresent()) {
+                final CoordinatorPublication publication = currentPublication.get();
+                if (publication.isCommitted()) {
+                    publication.cancel("cancelCommittedPublication");
+                    logger.debug("Cancelled publication of [{}].", publication);
+                    return true;
+                }
             }
             return false;
         }
@@ -1290,16 +1298,32 @@ public void onSuccess(String source) {
                                     updateMaxTermSeen(getCurrentTerm());
 
                                     if (mode == Mode.LEADER) {
+                                        // if necessary, abdicate to another node or improve the voting configuration
+                                        boolean attemptReconfiguration = true;
                                         final ClusterState state = getLastAcceptedState(); // committed state
-                                        if (electionQuorumContainsLocalNode(state) == false) {
+                                        if (localNodeMayWinElection(state) == false) {
                                             final List<DiscoveryNode> masterCandidates = completedNodes().stream()
                                                 .filter(DiscoveryNode::isMasterNode)
-                                                .filter(node -> electionQuorumContains(state, node))
+                                                .filter(node -> nodeMayWinElection(state, node))
+                                                .filter(node -> {
+                                                    // check if master candidate would be able to get an election quorum if we were to
+                                                    // abdicate to it. Assume that every node that completed the publication can provide
+                                                    // a vote in that next election and has the latest state.
+                                                    final long futureElectionTerm = state.term() + 1;
+                                                    final VoteCollection futureVoteCollection = new VoteCollection();
+                                                    completedNodes().forEach(completedNode -> futureVoteCollection.addJoinVote(
+                                                        new Join(completedNode, node, futureElectionTerm, state.term(), state.version())));
+                                                    return electionStrategy.isElectionQuorum(node, futureElectionTerm,
+                                                        state.term(), state.version(), state.getLastCommittedConfiguration(),
+                                                        state.getLastAcceptedConfiguration(), futureVoteCollection);
+                                                })
                                                 .collect(Collectors.toList());
                                             if (masterCandidates.isEmpty() == false) {
                                                 abdicateTo(masterCandidates.get(random.nextInt(masterCandidates.size())));
+                                                attemptReconfiguration = false;
                                             }
-                                        } else {
+                                        }
+                                        if (attemptReconfiguration) {
                                             scheduleReconfigurationIfNeeded();
                                         }
                                     }
@@ -1333,7 +1357,7 @@ private void handleAssociatedJoin(Join join) {
         }
 
         @Override
-        protected boolean isPublishQuorum(CoordinationState.VoteCollection votes) {
+        protected boolean isPublishQuorum(VoteCollection votes) {
             assert Thread.holdsLock(mutex) : "Coordinator mutex not held";
             return coordinationState.get().isPublishQuorum(votes);
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ElectionStrategy.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ElectionStrategy.java
new file mode 100644
index 0000000000000..b9e00cd52fc4e
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ElectionStrategy.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
+import org.elasticsearch.cluster.coordination.CoordinationState.VoteCollection;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+
+/**
+ * Allows plugging in a custom election strategy, restricting the notion of an election quorum.
+ * Custom additional quorum restrictions can be defined by implementing the {@link #satisfiesAdditionalQuorumConstraints} method.
+ */
+public abstract class ElectionStrategy {
+
+    public static final ElectionStrategy DEFAULT_INSTANCE = new ElectionStrategy() {
+        @Override
+        protected boolean satisfiesAdditionalQuorumConstraints(DiscoveryNode localNode, long localCurrentTerm, long localAcceptedTerm,
+                                                               long localAcceptedVersion, VotingConfiguration lastCommittedConfiguration,
+                                                               VotingConfiguration lastAcceptedConfiguration, VoteCollection joinVotes) {
+            return true;
+        }
+    };
+
+    protected ElectionStrategy() {
+
+    }
+
+    /**
+     * Whether there is an election quorum from the point of view of the given local node under the provided voting configurations
+     */
+    public final boolean isElectionQuorum(DiscoveryNode localNode, long localCurrentTerm, long localAcceptedTerm, long localAcceptedVersion,
+                                          VotingConfiguration lastCommittedConfiguration, VotingConfiguration lastAcceptedConfiguration,
+                                          VoteCollection joinVotes) {
+        return joinVotes.isQuorum(lastCommittedConfiguration) &&
+            joinVotes.isQuorum(lastAcceptedConfiguration) &&
+            satisfiesAdditionalQuorumConstraints(localNode, localCurrentTerm, localAcceptedTerm, localAcceptedVersion,
+                lastCommittedConfiguration, lastAcceptedConfiguration, joinVotes);
+    }
+
+    /**
+     * The extension point to be overridden by plugins. Defines additional constraints on the election quorum.
+     * @param localNode                  the local node for the election quorum
+     * @param localCurrentTerm           the current term of the local node
+     * @param localAcceptedTerm          the last accepted term of the local node
+     * @param localAcceptedVersion       the last accepted version of the local node
+     * @param lastCommittedConfiguration the last committed configuration for the election quorum
+     * @param lastAcceptedConfiguration  the last accepted configuration for the election quorum
+     * @param joinVotes                  the votes that were provided so far
+     * @return true iff the additional quorum constraints are satisfied
+     */
+    protected abstract boolean satisfiesAdditionalQuorumConstraints(DiscoveryNode localNode,
+                                                                    long localCurrentTerm,
+                                                                    long localAcceptedTerm,
+                                                                    long localAcceptedVersion,
+                                                                    VotingConfiguration lastCommittedConfiguration,
+                                                                    VotingConfiguration lastAcceptedConfiguration,
+                                                                    VoteCollection joinVotes);
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java
index 3a52324661fc0..5d4f847bc7c26 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.cluster.coordination.Coordinator.Mode;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RerouteService;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.MasterService;
 import org.elasticsearch.common.Priority;
@@ -91,11 +92,11 @@ public class JoinHelper {
     JoinHelper(Settings settings, AllocationService allocationService, MasterService masterService,
                TransportService transportService, LongSupplier currentTermSupplier, Supplier<ClusterState> currentStateSupplier,
                BiConsumer<JoinRequest, JoinCallback> joinHandler, Function<StartJoinRequest, Join> joinLeaderInTerm,
-               Collection<BiConsumer<DiscoveryNode, ClusterState>> joinValidators) {
+               Collection<BiConsumer<DiscoveryNode, ClusterState>> joinValidators, RerouteService rerouteService) {
         this.masterService = masterService;
         this.transportService = transportService;
         this.joinTimeout = JOIN_TIMEOUT_SETTING.get(settings);
-        this.joinTaskExecutor = new JoinTaskExecutor(allocationService, logger) {
+        this.joinTaskExecutor = new JoinTaskExecutor(allocationService, logger, rerouteService) {
 
             @Override
             public ClusterTasksResult<JoinTaskExecutor.Task> execute(ClusterState currentState, List<JoinTaskExecutor.Task> joiningTasks)
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java
index 7434a246eed8a..c5ed468993ff7 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java
@@ -20,6 +20,7 @@
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.NotMasterException;
@@ -28,6 +29,7 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RerouteService;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 
@@ -44,6 +46,7 @@ public class JoinTaskExecutor implements ClusterStateTaskExecutor<JoinTaskExecut
     private final AllocationService allocationService;
 
     private final Logger logger;
+    private final RerouteService rerouteService;
 
     public static class Task {
 
@@ -80,9 +83,10 @@ public boolean isFinishElectionTask() {
         private static final String FINISH_ELECTION_TASK_REASON = "_FINISH_ELECTION_";
     }
 
-    public JoinTaskExecutor(AllocationService allocationService, Logger logger) {
+    public JoinTaskExecutor(AllocationService allocationService, Logger logger, RerouteService rerouteService) {
         this.allocationService = allocationService;
         this.logger = logger;
+        this.rerouteService = rerouteService;
     }
 
     @Override
@@ -146,8 +150,11 @@ public ClusterTasksResult<Task> execute(ClusterState currentState, List<Task> jo
             results.success(joinTask);
         }
         if (nodesChanged) {
-            newState.nodes(nodesBuilder);
-            return results.build(allocationService.reroute(newState.build(), "node_join"));
+            rerouteService.reroute("post-join reroute", ActionListener.wrap(
+                r -> logger.trace("post-join reroute completed"),
+                e -> logger.debug("post-join reroute failed", e)));
+
+            return results.build(allocationService.adaptAutoExpandReplicas(newState.nodes(nodesBuilder).build()));
         } else {
             // we must return a new cluster state instance to force publishing. This is important
             // for the joining node to finalize its join and set us as a master
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java b/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java
index 5bc5ea866ee83..703c08bf260c7 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Nullable;
@@ -33,6 +34,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.threadpool.ThreadPool.Names;
 import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.NodeDisconnectedException;
 import org.elasticsearch.transport.TransportConnectionListener;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
@@ -48,6 +50,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 
 /**
  * The LeaderChecker is responsible for allowing followers to check that the currently elected leader is still connected and healthy. We are
@@ -75,20 +78,17 @@ public class LeaderChecker {
     public static final Setting<Integer> LEADER_CHECK_RETRY_COUNT_SETTING =
         Setting.intSetting("cluster.fault_detection.leader_check.retry_count", 3, 1, Setting.Property.NodeScope);
 
-    private final Settings settings;
-
     private final TimeValue leaderCheckInterval;
     private final TimeValue leaderCheckTimeout;
     private final int leaderCheckRetryCount;
     private final TransportService transportService;
-    private final Runnable onLeaderFailure;
+    private final Consumer<Exception> onLeaderFailure;
 
     private AtomicReference<CheckScheduler> currentChecker = new AtomicReference<>();
 
     private volatile DiscoveryNodes discoveryNodes;
 
-    public LeaderChecker(final Settings settings, final TransportService transportService, final Runnable onLeaderFailure) {
-        this.settings = settings;
+    public LeaderChecker(final Settings settings, final TransportService transportService, final Consumer<Exception> onLeaderFailure) {
         leaderCheckInterval = LEADER_CHECK_INTERVAL_SETTING.get(settings);
         leaderCheckTimeout = LEADER_CHECK_TIMEOUT_SETTING.get(settings);
         leaderCheckRetryCount = LEADER_CHECK_RETRY_COUNT_SETTING.get(settings);
@@ -234,16 +234,19 @@ public void handleException(TransportException exp) {
                         }
 
                         if (exp instanceof ConnectTransportException || exp.getCause() instanceof ConnectTransportException) {
-                            logger.debug(new ParameterizedMessage("leader [{}] disconnected, failing immediately", leader), exp);
-                            leaderFailed();
+                            logger.debug(new ParameterizedMessage(
+                                "leader [{}] disconnected during check", leader), exp);
+                            leaderFailed(new ConnectTransportException(leader, "disconnected during check", exp));
                             return;
                         }
 
                         long failureCount = failureCountSinceLastSuccess.incrementAndGet();
                         if (failureCount >= leaderCheckRetryCount) {
-                            logger.debug(new ParameterizedMessage("{} consecutive failures (limit [{}] is {}) so leader [{}] has failed",
-                                failureCount, LEADER_CHECK_RETRY_COUNT_SETTING.getKey(), leaderCheckRetryCount, leader), exp);
-                            leaderFailed();
+                            logger.debug(new ParameterizedMessage(
+                                "leader [{}] has failed {} consecutive checks (limit [{}] is {}); last failure was:",
+                                leader, failureCount, LEADER_CHECK_RETRY_COUNT_SETTING.getKey(), leaderCheckRetryCount), exp);
+                            leaderFailed(new ElasticsearchException(
+                                "node [" + leader + "] failed [" + failureCount + "] consecutive checks", exp));
                             return;
                         }
 
@@ -259,9 +262,19 @@ public String executor() {
                 });
         }
 
-        void leaderFailed() {
+        void leaderFailed(Exception e) {
             if (isClosed.compareAndSet(false, true)) {
-                transportService.getThreadPool().generic().execute(onLeaderFailure);
+                transportService.getThreadPool().generic().execute(new Runnable() {
+                    @Override
+                    public void run() {
+                        onLeaderFailure.accept(e);
+                    }
+
+                    @Override
+                    public String toString() {
+                        return "notification of leader failure: " + e.getMessage();
+                    }
+                });
             } else {
                 logger.trace("already closed, not failing leader");
             }
@@ -269,7 +282,8 @@ void leaderFailed() {
 
         void handleDisconnectedNode(DiscoveryNode discoveryNode) {
             if (discoveryNode.equals(leader)) {
-                leaderFailed();
+                logger.debug("leader [{}] disconnected", leader);
+                leaderFailed(new NodeDisconnectedException(discoveryNode, "disconnected"));
             }
         }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/PreVoteCollector.java b/server/src/main/java/org/elasticsearch/cluster/coordination/PreVoteCollector.java
index 5001e3be18183..f683057b1fcd1 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/PreVoteCollector.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/PreVoteCollector.java
@@ -35,12 +35,11 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.Set;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.LongConsumer;
 
-import static org.elasticsearch.cluster.coordination.CoordinationState.isElectionQuorum;
-import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentSet;
+import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap;
 
 public class PreVoteCollector {
 
@@ -51,14 +50,17 @@ public class PreVoteCollector {
     private final TransportService transportService;
     private final Runnable startElection;
     private final LongConsumer updateMaxTermSeen;
+    private final ElectionStrategy electionStrategy;
 
     // Tuple for simple atomic updates. null until the first call to `update()`.
     private volatile Tuple<DiscoveryNode, PreVoteResponse> state; // DiscoveryNode component is null if there is currently no known leader.
 
-    PreVoteCollector(final TransportService transportService, final Runnable startElection, final LongConsumer updateMaxTermSeen) {
+    PreVoteCollector(final TransportService transportService, final Runnable startElection, final LongConsumer updateMaxTermSeen,
+                     final ElectionStrategy electionStrategy) {
         this.transportService = transportService;
         this.startElection = startElection;
         this.updateMaxTermSeen = updateMaxTermSeen;
+        this.electionStrategy = electionStrategy;
 
         // TODO does this need to be on the generic threadpool or can it use SAME?
         transportService.registerRequestHandler(REQUEST_PRE_VOTE_ACTION_NAME, Names.GENERIC, false, false,
@@ -128,7 +130,7 @@ public String toString() {
     }
 
     private class PreVotingRound implements Releasable {
-        private final Set<DiscoveryNode> preVotesReceived = newConcurrentSet();
+        private final Map<DiscoveryNode, PreVoteResponse> preVotesReceived = newConcurrentMap();
         private final AtomicBoolean electionStarted = new AtomicBoolean();
         private final PreVoteRequest preVoteRequest;
         private final ClusterState clusterState;
@@ -185,11 +187,20 @@ private void handlePreVoteResponse(final PreVoteResponse response, final Discove
                 return;
             }
 
-            preVotesReceived.add(sender);
+            preVotesReceived.put(sender, response);
+
+            // create a fake VoteCollection based on the pre-votes and check if there is an election quorum
             final VoteCollection voteCollection = new VoteCollection();
-            preVotesReceived.forEach(voteCollection::addVote);
+            final DiscoveryNode localNode = clusterState.nodes().getLocalNode();
+            final PreVoteResponse localPreVoteResponse = getPreVoteResponse();
+
+            preVotesReceived.forEach((node, preVoteResponse) -> voteCollection.addJoinVote(
+                new Join(node, localNode, preVoteResponse.getCurrentTerm(),
+                preVoteResponse.getLastAcceptedTerm(), preVoteResponse.getLastAcceptedVersion())));
 
-            if (isElectionQuorum(voteCollection, clusterState) == false) {
+            if (electionStrategy.isElectionQuorum(clusterState.nodes().getLocalNode(), localPreVoteResponse.getCurrentTerm(),
+                localPreVoteResponse.getLastAcceptedTerm(), localPreVoteResponse.getLastAcceptedVersion(),
+                clusterState.getLastCommittedConfiguration(), clusterState.getLastAcceptedConfiguration(), voteCollection) == false) {
                 logger.debug("{} added {} from {}, no quorum yet", this, response, sender);
                 return;
             }
diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Reconfigurator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Reconfigurator.java
index 7a3a54d73b2fe..fd9bab3af11ec 100644
--- a/server/src/main/java/org/elasticsearch/cluster/coordination/Reconfigurator.java
+++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Reconfigurator.java
@@ -150,6 +150,11 @@ static class VotingConfigNode implements Comparable<VotingConfigNode> {
 
         @Override
         public int compareTo(VotingConfigNode other) {
+            // prefer current master
+            final int currentMasterComp = Boolean.compare(other.currentMaster, currentMaster);
+            if (currentMasterComp != 0) {
+                return currentMasterComp;
+            }
             // prefer nodes that are live
             final int liveComp = Boolean.compare(other.live, live);
             if (liveComp != 0) {
@@ -160,11 +165,6 @@ public int compareTo(VotingConfigNode other) {
             if (inCurrentConfigComp != 0) {
                 return inCurrentConfigComp;
             }
-            // prefer current master
-            final int currentMasterComp = Boolean.compare(other.currentMaster, currentMaster);
-            if (currentMasterComp != 0) {
-                return currentMasterComp;
-            }
             // tiebreak by node id to have stable ordering
             return id.compareTo(other.id);
         }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
index 0d41d5f86b3ac..478e76b525610 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
@@ -140,15 +140,13 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
 
-        AliasMetaData that = (AliasMetaData) o;
+        final AliasMetaData that = (AliasMetaData) o;
 
         if (alias != null ? !alias.equals(that.alias) : that.alias != null) return false;
         if (filter != null ? !filter.equals(that.filter) : that.filter != null) return false;
         if (indexRouting != null ? !indexRouting.equals(that.indexRouting) : that.indexRouting != null) return false;
-        if (searchRouting != null ? !searchRouting.equals(that.searchRouting) : that.searchRouting != null)
-            return false;
-        if (writeIndex != null ? writeIndex != that.writeIndex : that.writeIndex != null)
-            return false;
+        if (searchRouting != null ? !searchRouting.equals(that.searchRouting) : that.searchRouting != null) return false;
+        if (writeIndex != null ? writeIndex != that.writeIndex : that.writeIndex != null) return false;
 
         return true;
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DiffableStringMap.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DiffableStringMap.java
index 46433eed8a657..b6e31e92698e6 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DiffableStringMap.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DiffableStringMap.java
@@ -39,6 +39,8 @@
  */
 public class DiffableStringMap extends AbstractMap<String, String> implements Diffable<DiffableStringMap> {
 
+    public static final DiffableStringMap EMPTY = new DiffableStringMap(Collections.emptyMap());
+
     private final Map<String, String> innerMap;
 
     DiffableStringMap(final Map<String, String> map) {
@@ -75,6 +77,8 @@ public static Diff<DiffableStringMap> readDiffFrom(StreamInput in) throws IOExce
      */
     public static class DiffableStringMapDiff implements Diff<DiffableStringMap> {
 
+        public static final DiffableStringMapDiff EMPTY = new DiffableStringMapDiff(DiffableStringMap.EMPTY, DiffableStringMap.EMPTY);
+
         private final List<String> deletes;
         private final Map<String, String> upserts; // diffs also become upserts
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
index 412e16e14a3a5..3598753f80d9c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
@@ -1408,10 +1408,10 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti
                     throw new IllegalArgumentException("Unexpected token " + token);
                 }
             }
-            if (Assertions.ENABLED && Version.indexCreated(builder.settings).onOrAfter(Version.V_6_5_0)) {
+            if (Assertions.ENABLED) {
                 assert mappingVersion : "mapping version should be present for indices created on or after 6.5.0";
             }
-            if (Assertions.ENABLED && Version.indexCreated(builder.settings).onOrAfter(Version.V_6_5_0)) {
+            if (Assertions.ENABLED) {
                 assert settingsVersion : "settings version should be present for indices created on or after 6.5.0";
             }
             if (Assertions.ENABLED && Version.indexCreated(builder.settings).onOrAfter(Version.V_7_2_0)) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
index 1c84e8ce9fc6a..93832e08cd048 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
@@ -20,7 +20,6 @@
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
-import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.cluster.Diff;
@@ -32,7 +31,6 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -52,8 +50,6 @@
 
 public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaData> {
 
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(IndexTemplateMetaData.class));
-
     private final String name;
 
     private final int order;
@@ -97,7 +93,7 @@ public IndexTemplateMetaData(String name, int order, Integer version,
         this.name = name;
         this.order = order;
         this.version = version;
-        this.patterns= patterns;
+        this.patterns = patterns;
         this.settings = settings;
         this.mappings = mappings;
         this.aliases = aliases;
@@ -227,7 +223,7 @@ public void writeTo(StreamOutput out) throws IOException {
     public static class Builder {
 
         private static final Set<String> VALID_FIELDS = Sets.newHashSet(
-            "template", "order", "mappings", "settings", "index_patterns", "aliases", "version");
+            "order", "mappings", "settings", "index_patterns", "aliases", "version");
 
         private String name;
 
@@ -487,11 +483,7 @@ public static IndexTemplateMetaData fromXContent(XContentParser parser, String t
                         builder.patterns(index_patterns);
                     }
                 } else if (token.isValue()) {
-                    // Prior to 5.1.0, elasticsearch only supported a single index pattern called `template` (#21009)
-                    if("template".equals(currentFieldName)) {
-                        deprecationLogger.deprecated("Deprecated field [template] used, replaced by [index_patterns]");
-                        builder.patterns(Collections.singletonList(parser.text()));
-                    } else if ("order".equals(currentFieldName)) {
+                    if ("order".equals(currentFieldName)) {
                         builder.order(parser.intValue());
                     } else if ("version".equals(currentFieldName)) {
                         builder.version(parser.intValue());
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index a9f751453f6fa..4c2eeec72a970 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -25,6 +25,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.AliasesRequest;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterState.FeatureAware;
@@ -168,6 +169,7 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, Clust
     private final Settings transientSettings;
     private final Settings persistentSettings;
     private final Settings settings;
+    private final DiffableStringMap hashesOfConsistentSettings;
     private final ImmutableOpenMap<String, IndexMetaData> indices;
     private final ImmutableOpenMap<String, IndexTemplateMetaData> templates;
     private final ImmutableOpenMap<String, Custom> customs;
@@ -182,7 +184,7 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, Clust
     private final SortedMap<String, AliasOrIndex> aliasAndIndexLookup;
 
     MetaData(String clusterUUID, boolean clusterUUIDCommitted, long version, CoordinationMetaData coordinationMetaData,
-             Settings transientSettings, Settings persistentSettings,
+             Settings transientSettings, Settings persistentSettings, DiffableStringMap hashesOfConsistentSettings,
              ImmutableOpenMap<String, IndexMetaData> indices, ImmutableOpenMap<String, IndexTemplateMetaData> templates,
              ImmutableOpenMap<String, Custom> customs, String[] allIndices, String[] allOpenIndices, String[] allClosedIndices,
              SortedMap<String, AliasOrIndex> aliasAndIndexLookup) {
@@ -193,6 +195,7 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, Clust
         this.transientSettings = transientSettings;
         this.persistentSettings = persistentSettings;
         this.settings = Settings.builder().put(persistentSettings).put(transientSettings).build();
+        this.hashesOfConsistentSettings = hashesOfConsistentSettings;
         this.indices = indices;
         this.customs = customs;
         this.templates = templates;
@@ -244,6 +247,10 @@ public Settings persistentSettings() {
         return this.persistentSettings;
     }
 
+    public Map<String, String> hashesOfConsistentSettings() {
+        return this.hashesOfConsistentSettings;
+    }
+
     public CoordinationMetaData coordinationMetaData() {
         return this.coordinationMetaData;
     }
@@ -360,38 +367,6 @@ private ImmutableOpenMap<String, List<AliasMetaData>> findAliases(final String[]
         return mapBuilder.build();
     }
 
-    /**
-     * Checks if at least one of the specified aliases exists in the specified concrete indices. Wildcards are supported in the
-     * alias names for partial matches.
-     *
-     * @param aliases         The names of the index aliases to find
-     * @param concreteIndices The concrete indexes the index aliases must point to order to be returned.
-     * @return whether at least one of the specified aliases exists in one of the specified concrete indices.
-     */
-    public boolean hasAliases(final String[] aliases, String[] concreteIndices) {
-        assert aliases != null;
-        assert concreteIndices != null;
-        if (concreteIndices.length == 0) {
-            return false;
-        }
-
-        Iterable<String> intersection = HppcMaps.intersection(ObjectHashSet.from(concreteIndices), indices.keys());
-        for (String index : intersection) {
-            IndexMetaData indexMetaData = indices.get(index);
-            List<AliasMetaData> filteredValues = new ArrayList<>();
-            for (ObjectCursor<AliasMetaData> cursor : indexMetaData.getAliases().values()) {
-                AliasMetaData value = cursor.value;
-                if (Regex.simpleMatch(aliases, value.alias())) {
-                    filteredValues.add(value);
-                }
-            }
-            if (!filteredValues.isEmpty()) {
-                return true;
-            }
-        }
-        return false;
-    }
-
     /**
      * Finds all mappings for types and concrete indices. Types are expanded to include all types that match the glob
      * patterns in the types array. Empty types array, null or {"_all"} will be expanded to all types available for
@@ -765,6 +740,9 @@ public static boolean isGlobalStateEquals(MetaData metaData1, MetaData metaData2
         if (!metaData1.persistentSettings.equals(metaData2.persistentSettings)) {
             return false;
         }
+        if (!metaData1.hashesOfConsistentSettings.equals(metaData2.hashesOfConsistentSettings)) {
+            return false;
+        }
         if (!metaData1.templates.equals(metaData2.templates())) {
             return false;
         }
@@ -819,6 +797,7 @@ private static class MetaDataDiff implements Diff<MetaData> {
         private CoordinationMetaData coordinationMetaData;
         private Settings transientSettings;
         private Settings persistentSettings;
+        private Diff<DiffableStringMap> hashesOfConsistentSettings;
         private Diff<ImmutableOpenMap<String, IndexMetaData>> indices;
         private Diff<ImmutableOpenMap<String, IndexTemplateMetaData>> templates;
         private Diff<ImmutableOpenMap<String, Custom>> customs;
@@ -830,6 +809,7 @@ private static class MetaDataDiff implements Diff<MetaData> {
             coordinationMetaData = after.coordinationMetaData;
             transientSettings = after.transientSettings;
             persistentSettings = after.persistentSettings;
+            hashesOfConsistentSettings = after.hashesOfConsistentSettings.diff(before.hashesOfConsistentSettings);
             indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer());
             templates = DiffableUtils.diff(before.templates, after.templates, DiffableUtils.getStringKeySerializer());
             customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
@@ -842,6 +822,11 @@ private static class MetaDataDiff implements Diff<MetaData> {
             coordinationMetaData = new CoordinationMetaData(in);
             transientSettings = Settings.readSettingsFromStream(in);
             persistentSettings = Settings.readSettingsFromStream(in);
+            if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+                hashesOfConsistentSettings = DiffableStringMap.readDiffFrom(in);
+            } else {
+                hashesOfConsistentSettings = DiffableStringMap.DiffableStringMapDiff.EMPTY;
+            }
             indices = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexMetaData::readFrom,
                 IndexMetaData::readDiffFrom);
             templates = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexTemplateMetaData::readFrom,
@@ -857,6 +842,9 @@ public void writeTo(StreamOutput out) throws IOException {
             coordinationMetaData.writeTo(out);
             Settings.writeSettingsToStream(transientSettings, out);
             Settings.writeSettingsToStream(persistentSettings, out);
+            if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+                hashesOfConsistentSettings.writeTo(out);
+            }
             indices.writeTo(out);
             templates.writeTo(out);
             customs.writeTo(out);
@@ -871,6 +859,7 @@ public MetaData apply(MetaData part) {
             builder.coordinationMetaData(coordinationMetaData);
             builder.transientSettings(transientSettings);
             builder.persistentSettings(persistentSettings);
+            builder.hashesOfConsistentSettings(hashesOfConsistentSettings.apply(part.hashesOfConsistentSettings));
             builder.indices(indices.apply(part.indices));
             builder.templates(templates.apply(part.templates));
             builder.customs(customs.apply(part.customs));
@@ -886,6 +875,9 @@ public static MetaData readFrom(StreamInput in) throws IOException {
         builder.coordinationMetaData(new CoordinationMetaData(in));
         builder.transientSettings(readSettingsFromStream(in));
         builder.persistentSettings(readSettingsFromStream(in));
+        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+            builder.hashesOfConsistentSettings(new DiffableStringMap(in));
+        }
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
             builder.put(IndexMetaData.readFrom(in), false);
@@ -910,6 +902,9 @@ public void writeTo(StreamOutput out) throws IOException {
         coordinationMetaData.writeTo(out);
         writeSettingsToStream(transientSettings, out);
         writeSettingsToStream(persistentSettings, out);
+        if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+            hashesOfConsistentSettings.writeTo(out);
+        }
         out.writeVInt(indices.size());
         for (IndexMetaData indexMetaData : this) {
             indexMetaData.writeTo(out);
@@ -950,6 +945,7 @@ public static class Builder {
         private CoordinationMetaData coordinationMetaData = CoordinationMetaData.EMPTY_META_DATA;
         private Settings transientSettings = Settings.Builder.EMPTY_SETTINGS;
         private Settings persistentSettings = Settings.Builder.EMPTY_SETTINGS;
+        private DiffableStringMap hashesOfConsistentSettings = new DiffableStringMap(Collections.emptyMap());
 
         private final ImmutableOpenMap.Builder<String, IndexMetaData> indices;
         private final ImmutableOpenMap.Builder<String, IndexTemplateMetaData> templates;
@@ -969,6 +965,7 @@ public Builder(MetaData metaData) {
             this.coordinationMetaData = metaData.coordinationMetaData;
             this.transientSettings = metaData.transientSettings;
             this.persistentSettings = metaData.persistentSettings;
+            this.hashesOfConsistentSettings = metaData.hashesOfConsistentSettings;
             this.version = metaData.version;
             this.indices = ImmutableOpenMap.builder(metaData.indices);
             this.templates = ImmutableOpenMap.builder(metaData.templates);
@@ -1132,6 +1129,20 @@ public Builder persistentSettings(Settings settings) {
             return this;
         }
 
+        public DiffableStringMap hashesOfConsistentSettings() {
+            return this.hashesOfConsistentSettings;
+        }
+
+        public Builder hashesOfConsistentSettings(DiffableStringMap hashesOfConsistentSettings) {
+            this.hashesOfConsistentSettings = hashesOfConsistentSettings;
+            return this;
+        }
+
+        public Builder hashesOfConsistentSettings(Map<String, String> hashesOfConsistentSettings) {
+            this.hashesOfConsistentSettings = new DiffableStringMap(hashesOfConsistentSettings);
+            return this;
+        }
+
         public Builder version(long version) {
             this.version = version;
             return this;
@@ -1205,8 +1216,8 @@ public MetaData build() {
             String[] allClosedIndicesArray = allClosedIndices.toArray(new String[allClosedIndices.size()]);
 
             return new MetaData(clusterUUID, clusterUUIDCommitted, version, coordinationMetaData, transientSettings, persistentSettings,
-                    indices.build(), templates.build(), customs.build(), allIndicesArray, allOpenIndicesArray, allClosedIndicesArray,
-                    aliasAndIndexLookup);
+                    hashesOfConsistentSettings, indices.build(), templates.build(), customs.build(), allIndicesArray, allOpenIndicesArray,
+                    allClosedIndicesArray, aliasAndIndexLookup);
         }
 
         private SortedMap<String, AliasOrIndex> buildAliasAndIndexLookup() {
@@ -1330,6 +1341,8 @@ public static MetaData fromXContent(XContentParser parser) throws IOException {
                         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                             builder.put(IndexMetaData.Builder.fromXContent(parser), false);
                         }
+                    } else if ("hashes_of_consistent_settings".equals(currentFieldName)) {
+                        builder.hashesOfConsistentSettings(parser.mapStrings());
                     } else if ("templates".equals(currentFieldName)) {
                         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                             builder.put(IndexTemplateMetaData.Builder.fromXContent(parser, parser.currentName()));
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java
index ef4583e98e544..d37a134c0a77c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java
@@ -389,11 +389,26 @@ private void sendVerifyShardBeforeCloseRequest(final IndexShardRoutingTable shar
             }
             final TaskId parentTaskId = new TaskId(clusterService.localNode().getId(), request.taskId());
             final TransportVerifyShardBeforeCloseAction.ShardRequest shardRequest =
-                new TransportVerifyShardBeforeCloseAction.ShardRequest(shardId, closingBlock, parentTaskId);
+                new TransportVerifyShardBeforeCloseAction.ShardRequest(shardId, closingBlock, true, parentTaskId);
             if (request.ackTimeout() != null) {
                 shardRequest.timeout(request.ackTimeout());
             }
-            transportVerifyShardBeforeCloseAction.execute(shardRequest, listener);
+            transportVerifyShardBeforeCloseAction.execute(shardRequest, new ActionListener<>() {
+                @Override
+                public void onResponse(ReplicationResponse replicationResponse) {
+                    final TransportVerifyShardBeforeCloseAction.ShardRequest shardRequest =
+                        new TransportVerifyShardBeforeCloseAction.ShardRequest(shardId, closingBlock, false, parentTaskId);
+                    if (request.ackTimeout() != null) {
+                        shardRequest.timeout(request.ackTimeout());
+                    }
+                    transportVerifyShardBeforeCloseAction.execute(shardRequest, listener);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    listener.onFailure(e);
+                }
+            });
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
index 483835f633e7e..72fdea8e6092d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
@@ -195,7 +195,7 @@ public Set<Entry<String, NamedAnalyzer>> entrySet() {
                 }
             };
             try (IndexAnalyzers fakeIndexAnalzyers =
-                     new IndexAnalyzers(indexSettings, analyzerMap, analyzerMap, analyzerMap)) {
+                     new IndexAnalyzers(analyzerMap, analyzerMap, analyzerMap)) {
                 MapperService mapperService = new MapperService(indexSettings, fakeIndexAnalzyers, xContentRegistry, similarityService,
                         mapperRegistry, () -> null);
                 mapperService.merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY);
diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
index afab4af320416..6e9ad85aec56c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
+++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
@@ -32,11 +32,14 @@
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 
 /**
@@ -78,8 +81,7 @@ public static boolean isIngestNode(Settings settings) {
     private final TransportAddress address;
     private final Map<String, String> attributes;
     private final Version version;
-    private final Set<Role> roles;
-
+    private final Set<DiscoveryNodeRole> roles;
 
     /**
      * Creates a new {@link DiscoveryNode}
@@ -95,7 +97,7 @@ public static boolean isIngestNode(Settings settings) {
      * @param version          the version of the node
      */
     public DiscoveryNode(final String id, TransportAddress address, Version version) {
-        this(id, address, Collections.emptyMap(), EnumSet.allOf(Role.class), version);
+        this(id, address, Collections.emptyMap(), DiscoveryNodeRole.BUILT_IN_ROLES, version);
     }
 
     /**
@@ -113,7 +115,7 @@ public DiscoveryNode(final String id, TransportAddress address, Version version)
      * @param roles            node roles
      * @param version          the version of the node
      */
-    public DiscoveryNode(String id, TransportAddress address, Map<String, String> attributes, Set<Role> roles,
+    public DiscoveryNode(String id, TransportAddress address, Map<String, String> attributes, Set<DiscoveryNodeRole> roles,
                          Version version) {
         this("", id, address, attributes, roles, version);
     }
@@ -135,7 +137,7 @@ public DiscoveryNode(String id, TransportAddress address, Map<String, String> at
      * @param version          the version of the node
      */
     public DiscoveryNode(String nodeName, String nodeId, TransportAddress address,
-                         Map<String, String> attributes, Set<Role> roles, Version version) {
+                         Map<String, String> attributes, Set<DiscoveryNodeRole> roles, Version version) {
         this(nodeName, nodeId, UUIDs.randomBase64UUID(), address.address().getHostString(), address.getAddress(), address, attributes,
             roles, version);
     }
@@ -159,7 +161,7 @@ public DiscoveryNode(String nodeName, String nodeId, TransportAddress address,
      * @param version          the version of the node
      */
     public DiscoveryNode(String nodeName, String nodeId, String ephemeralId, String hostName, String hostAddress,
-                         TransportAddress address, Map<String, String> attributes, Set<Role> roles, Version version) {
+                         TransportAddress address, Map<String, String> attributes, Set<DiscoveryNodeRole> roles, Version version) {
         if (nodeName != null) {
             this.nodeName = nodeName.intern();
         } else {
@@ -178,37 +180,27 @@ public DiscoveryNode(String nodeName, String nodeId, String ephemeralId, String
         this.attributes = Collections.unmodifiableMap(attributes);
         //verify that no node roles are being provided as attributes
         Predicate<Map<String, String>> predicate =  (attrs) -> {
-            for (Role role : Role.values()) {
-                assert attrs.containsKey(role.getRoleName()) == false;
+            boolean success = true;
+            for (final DiscoveryNodeRole role : DiscoveryNode.roleNameToPossibleRoles.values()) {
+                success &= attrs.containsKey(role.roleName()) == false;
+                assert success : role.roleName();
             }
-            return true;
+            return success;
         };
-        assert predicate.test(attributes);
-        Set<Role> rolesSet = EnumSet.noneOf(Role.class);
-        rolesSet.addAll(roles);
-        this.roles = Collections.unmodifiableSet(rolesSet);
+        assert predicate.test(attributes) : attributes;
+        this.roles = Set.copyOf(roles);
     }
 
     /** Creates a DiscoveryNode representing the local node. */
     public static DiscoveryNode createLocal(Settings settings, TransportAddress publishAddress, String nodeId) {
         Map<String, String> attributes = Node.NODE_ATTRIBUTES.getAsMap(settings);
-        Set<Role> roles = getRolesFromSettings(settings);
+        Set<DiscoveryNodeRole> roles = getRolesFromSettings(settings);
         return new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), nodeId, publishAddress, attributes, roles, Version.CURRENT);
     }
 
     /** extract node roles from the given settings */
-    public static Set<Role> getRolesFromSettings(Settings settings) {
-        Set<Role> roles = EnumSet.noneOf(Role.class);
-        if (Node.NODE_INGEST_SETTING.get(settings)) {
-            roles.add(Role.INGEST);
-        }
-        if (Node.NODE_MASTER_SETTING.get(settings)) {
-            roles.add(Role.MASTER);
-        }
-        if (Node.NODE_DATA_SETTING.get(settings)) {
-            roles.add(Role.DATA);
-        }
-        return roles;
+    public static Set<DiscoveryNodeRole> getRolesFromSettings(final Settings settings) {
+        return roleNameToPossibleRoles.values().stream().filter(s -> s.roleSetting().get(settings)).collect(Collectors.toUnmodifiableSet());
     }
 
     /**
@@ -229,10 +221,41 @@ public DiscoveryNode(StreamInput in) throws IOException {
             this.attributes.put(in.readString(), in.readString());
         }
         int rolesSize = in.readVInt();
-        this.roles = EnumSet.noneOf(Role.class);
-        for (int i = 0; i < rolesSize; i++) {
-            this.roles.add(in.readEnum(Role.class));
+        final Set<DiscoveryNodeRole> roles = new HashSet<>(rolesSize);
+        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+            for (int i = 0; i < rolesSize; i++) {
+                final String roleName = in.readString();
+                final String roleNameAbbreviation = in.readString();
+                final DiscoveryNodeRole role = roleNameToPossibleRoles.get(roleName);
+                if (role == null) {
+                    roles.add(new DiscoveryNodeRole.UnknownRole(roleName, roleNameAbbreviation));
+                } else {
+                    assert roleName.equals(role.roleName()) : "role name [" + roleName + "] does not match role [" + role.roleName() + "]";
+                    assert roleNameAbbreviation.equals(role.roleNameAbbreviation())
+                            : "role name abbreviation [" + roleName + "] does not match role [" + role.roleNameAbbreviation() + "]";
+                    roles.add(role);
+                }
+            }
+        } else {
+            // an old node will only send us legacy roles since pluggable roles is a new concept
+            for (int i = 0; i < rolesSize; i++) {
+                final LegacyRole legacyRole = in.readEnum(LegacyRole.class);
+                switch (legacyRole) {
+                    case MASTER:
+                        roles.add(DiscoveryNodeRole.MASTER_ROLE);
+                        break;
+                    case DATA:
+                        roles.add(DiscoveryNodeRole.DATA_ROLE);
+                        break;
+                    case INGEST:
+                        roles.add(DiscoveryNodeRole.INGEST_ROLE);
+                        break;
+                    default:
+                        throw new AssertionError(legacyRole.roleName());
+                }
+            }
         }
+        this.roles = Set.copyOf(roles);
         this.version = Version.readVersion(in);
     }
 
@@ -249,9 +272,26 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(entry.getKey());
             out.writeString(entry.getValue());
         }
-        out.writeVInt(roles.size());
-        for (Role role : roles) {
-            out.writeEnum(role);
+        if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+            out.writeVInt(roles.size());
+            for (final DiscoveryNodeRole role : roles) {
+                out.writeString(role.roleName());
+                out.writeString(role.roleNameAbbreviation());
+            }
+        } else {
+            // an old node will only understand legacy roles since pluggable roles is a new concept
+            final List<DiscoveryNodeRole> rolesToWrite =
+                    roles.stream().filter(DiscoveryNodeRole.BUILT_IN_ROLES::contains).collect(Collectors.toUnmodifiableList());
+            out.writeVInt(rolesToWrite.size());
+            for (final DiscoveryNodeRole role : rolesToWrite) {
+                if (role == DiscoveryNodeRole.MASTER_ROLE) {
+                    out.writeEnum(LegacyRole.MASTER);
+                } else if (role == DiscoveryNodeRole.DATA_ROLE) {
+                    out.writeEnum(LegacyRole.DATA);
+                } else if (role == DiscoveryNodeRole.INGEST_ROLE) {
+                    out.writeEnum(LegacyRole.INGEST);
+                }
+            }
         }
         Version.writeVersion(version, out);
     }
@@ -299,28 +339,28 @@ public Map<String, String> getAttributes() {
      * Should this node hold data (shards) or not.
      */
     public boolean isDataNode() {
-        return roles.contains(Role.DATA);
+        return roles.contains(DiscoveryNodeRole.DATA_ROLE);
     }
 
     /**
      * Can this node become master or not.
      */
     public boolean isMasterNode() {
-        return roles.contains(Role.MASTER);
+        return roles.contains(DiscoveryNodeRole.MASTER_ROLE);
     }
 
     /**
      * Returns a boolean that tells whether this an ingest node or not
      */
     public boolean isIngestNode() {
-        return roles.contains(Role.INGEST);
+        return roles.contains(DiscoveryNodeRole.INGEST_ROLE);
     }
 
     /**
      * Returns a set of all the roles that the node fulfills.
      * If the node doesn't have any specific role, the set is returned empty, which means that the node is a coordinating only node.
      */
-    public Set<Role> getRoles() {
+    public Set<DiscoveryNodeRole> getRoles() {
         return roles;
     }
 
@@ -368,6 +408,11 @@ public String toString() {
         sb.append('{').append(ephemeralId).append('}');
         sb.append('{').append(hostName).append('}');
         sb.append('{').append(address).append('}');
+        if (roles.isEmpty() == false) {
+            sb.append('{');
+            roles.stream().map(DiscoveryNodeRole::roleNameAbbreviation).sorted().forEach(sb::append);
+            sb.append('}');
+        }
         if (!attributes.isEmpty()) {
             sb.append(attributes);
         }
@@ -391,29 +436,43 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
+    private static Map<String, DiscoveryNodeRole> roleNameToPossibleRoles;
+
+    public static void setPossibleRoles(final Set<DiscoveryNodeRole> possibleRoles) {
+        final Map<String, DiscoveryNodeRole> roleNameToPossibleRoles =
+                possibleRoles.stream().collect(Collectors.toUnmodifiableMap(DiscoveryNodeRole::roleName, Function.identity()));
+        // collect the abbreviation names into a map to ensure that there are not any duplicate abbreviations
+        final Map<String, DiscoveryNodeRole> roleNameAbbreviationToPossibleRoles = roleNameToPossibleRoles.values()
+                .stream()
+                .collect(Collectors.toUnmodifiableMap(DiscoveryNodeRole::roleNameAbbreviation, Function.identity()));
+        assert roleNameToPossibleRoles.size() == roleNameAbbreviationToPossibleRoles.size() :
+                "roles by name [" + roleNameToPossibleRoles + "], roles by name abbreviation [" + roleNameAbbreviationToPossibleRoles + "]";
+        DiscoveryNode.roleNameToPossibleRoles = roleNameToPossibleRoles;
+    }
+
+    public static Set<String> getPossibleRoleNames() {
+        return roleNameToPossibleRoles.keySet();
+    }
+
     /**
      * Enum that holds all the possible roles that that a node can fulfill in a cluster.
      * Each role has its name and a corresponding abbreviation used by cat apis.
      */
-    public enum Role {
-        MASTER("master", "m"),
-        DATA("data", "d"),
-        INGEST("ingest", "i");
+    private enum LegacyRole {
+        MASTER("master"),
+        DATA("data"),
+        INGEST("ingest");
 
         private final String roleName;
-        private final String abbreviation;
 
-        Role(String roleName, String abbreviation) {
+        LegacyRole(final String roleName) {
             this.roleName = roleName;
-            this.abbreviation = abbreviation;
         }
 
-        public String getRoleName() {
+        public String roleName() {
             return roleName;
         }
 
-        public String getAbbreviation() {
-            return abbreviation;
-        }
     }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java
new file mode 100644
index 0000000000000..4bdf27793f66d
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.node;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.node.Node;
+
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Represents a node role.
+ */
+public abstract class DiscoveryNodeRole {
+
+    private final String roleName;
+
+    /**
+     * The name of the role.
+     *
+     * @return the role name
+     */
+    public final String roleName() {
+        return roleName;
+    }
+
+    private final String roleNameAbbreviation;
+
+    /**
+     * The abbreviation of the name of the role. This is used in the cat nodes API to display an abbreviated version of the name of the
+     * role.
+     *
+     * @return the role name abbreviation
+     */
+    public final String roleNameAbbreviation() {
+        return roleNameAbbreviation;
+    }
+
+    private final boolean isKnownRole;
+
+    /**
+     * Whether this role is known by this node, or is an {@link DiscoveryNodeRole.UnknownRole}.
+     */
+    public final boolean isKnownRole() {
+        return isKnownRole;
+    }
+
+    protected DiscoveryNodeRole(final String roleName, final String roleNameAbbreviation) {
+        this(true, roleName, roleNameAbbreviation);
+    }
+
+    private DiscoveryNodeRole(final boolean isKnownRole, final String roleName, final String roleNameAbbreviation) {
+        this.isKnownRole = isKnownRole;
+        this.roleName = Objects.requireNonNull(roleName);
+        this.roleNameAbbreviation = Objects.requireNonNull(roleNameAbbreviation);
+    }
+
+    protected abstract Setting<Boolean> roleSetting();
+
+    @Override
+    public final boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        DiscoveryNodeRole that = (DiscoveryNodeRole) o;
+        return roleName.equals(that.roleName) &&
+            roleNameAbbreviation.equals(that.roleNameAbbreviation) &&
+            isKnownRole == that.isKnownRole;
+    }
+
+    @Override
+    public final int hashCode() {
+        return Objects.hash(isKnownRole, roleName(), roleNameAbbreviation());
+    }
+
+    @Override
+    public final String toString() {
+        return "DiscoveryNodeRole{" +
+                "roleName='" + roleName + '\'' +
+                ", roleNameAbbreviation='" + roleNameAbbreviation + '\'' +
+                (isKnownRole ? "" : ", isKnownRole=false") +
+                '}';
+    }
+
+    /**
+     * Represents the role for a data node.
+     */
+    public static final DiscoveryNodeRole DATA_ROLE = new DiscoveryNodeRole("data", "d") {
+
+        @Override
+        protected Setting<Boolean> roleSetting() {
+            return Node.NODE_DATA_SETTING;
+        }
+
+    };
+
+    /**
+     * Represents the role for an ingest node.
+     */
+    public static final DiscoveryNodeRole INGEST_ROLE = new DiscoveryNodeRole("ingest", "i") {
+
+        @Override
+        protected Setting<Boolean> roleSetting() {
+            return Node.NODE_INGEST_SETTING;
+        }
+
+    };
+
+    /**
+     * Represents the role for a master-eligible node.
+     */
+    public static final DiscoveryNodeRole MASTER_ROLE = new DiscoveryNodeRole("master", "m") {
+
+        @Override
+        protected Setting<Boolean> roleSetting() {
+            return Node.NODE_MASTER_SETTING;
+        }
+
+    };
+
+    /**
+     * The built-in node roles.
+     */
+    public static Set<DiscoveryNodeRole> BUILT_IN_ROLES = Set.of(DATA_ROLE, INGEST_ROLE, MASTER_ROLE);
+
+    /**
+     * Represents an unknown role. This can occur if a newer version adds a role that an older version does not know about, or a newer
+     * version removes a role that an older version knows about.
+     */
+    static class UnknownRole extends DiscoveryNodeRole {
+
+        /**
+         * Construct an unknown role with the specified role name and role name abbreviation.
+         *
+         * @param roleName             the role name
+         * @param roleNameAbbreviation the role name abbreviation
+         */
+        UnknownRole(final String roleName, final String roleNameAbbreviation) {
+            super(false, roleName, roleNameAbbreviation);
+        }
+
+        @Override
+        protected Setting<Boolean> roleSetting() {
+            // since this setting is not registered, it will always return false when testing if the local node has the role
+            assert false;
+            return Setting.boolSetting("node. " + roleName(), false, Setting.Property.NodeScope);
+        }
+
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
index 02f4d5d93bf4e..8fb325d594707 100644
--- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
+++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.util.set.Sets;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -353,19 +354,19 @@ public String[] resolveNodes(String... nodes) {
                     if (index != -1) {
                         String matchAttrName = nodeId.substring(0, index);
                         String matchAttrValue = nodeId.substring(index + 1);
-                        if (DiscoveryNode.Role.DATA.getRoleName().equals(matchAttrName)) {
+                        if (DiscoveryNodeRole.DATA_ROLE.roleName().equals(matchAttrName)) {
                             if (Booleans.parseBoolean(matchAttrValue, true)) {
                                 resolvedNodesIds.addAll(dataNodes.keys());
                             } else {
                                 resolvedNodesIds.removeAll(dataNodes.keys());
                             }
-                        } else if (DiscoveryNode.Role.MASTER.getRoleName().equals(matchAttrName)) {
+                        } else if (DiscoveryNodeRole.MASTER_ROLE.roleName().equals(matchAttrName)) {
                             if (Booleans.parseBoolean(matchAttrValue, true)) {
                                 resolvedNodesIds.addAll(masterNodes.keys());
                             } else {
                                 resolvedNodesIds.removeAll(masterNodes.keys());
                             }
-                        } else if (DiscoveryNode.Role.INGEST.getRoleName().equals(matchAttrName)) {
+                        } else if (DiscoveryNodeRole.INGEST_ROLE.roleName().equals(matchAttrName)) {
                             if (Booleans.parseBoolean(matchAttrValue, true)) {
                                 resolvedNodesIds.addAll(ingestNodes.keys());
                             } else {
@@ -378,6 +379,17 @@ public String[] resolveNodes(String... nodes) {
                                 resolvedNodesIds.removeAll(getCoordinatingOnlyNodes().keys());
                             }
                         } else {
+                            for (DiscoveryNode node : this) {
+                                for (DiscoveryNodeRole role : Sets.difference(node.getRoles(), DiscoveryNodeRole.BUILT_IN_ROLES)) {
+                                    if (role.roleName().equals(matchAttrName)) {
+                                        if (Booleans.parseBoolean(matchAttrValue, true)) {
+                                            resolvedNodesIds.add(node.getId());
+                                        } else {
+                                            resolvedNodesIds.remove(node.getId());
+                                        }
+                                    }
+                                }
+                            }
                             for (DiscoveryNode node : this) {
                                 for (Map.Entry<String, String> entry : node.getAttributes().entrySet()) {
                                     String attrName = entry.getKey();
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/BatchedRerouteService.java b/server/src/main/java/org/elasticsearch/cluster/routing/BatchedRerouteService.java
new file mode 100644
index 0000000000000..4ed4caadabe49
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/BatchedRerouteService.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainListenableActionFuture;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.NotMasterException;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Priority;
+
+import java.util.function.BiFunction;
+
+/**
+ * A {@link BatchedRerouteService} is a {@link RerouteService} that batches together reroute requests to avoid unnecessary extra reroutes.
+ * This component only does meaningful work on the elected master node. Reroute requests will fail with a {@link NotMasterException} on
+ * other nodes.
+ */
+public class BatchedRerouteService implements RerouteService {
+    private static final Logger logger = LogManager.getLogger(BatchedRerouteService.class);
+
+    private static final String CLUSTER_UPDATE_TASK_SOURCE = "cluster_reroute";
+
+    private final ClusterService clusterService;
+    private final BiFunction<ClusterState, String, ClusterState> reroute;
+
+    private final Object mutex = new Object();
+    @Nullable // null if no reroute is currently pending
+    private PlainListenableActionFuture<Void> pendingRerouteListeners;
+
+    /**
+     * @param reroute Function that computes the updated cluster state after it has been rerouted.
+     */
+    public BatchedRerouteService(ClusterService clusterService, BiFunction<ClusterState, String, ClusterState> reroute) {
+        this.clusterService = clusterService;
+        this.reroute = reroute;
+    }
+
+    /**
+     * Initiates a reroute.
+     */
+    @Override
+    public final void reroute(String reason, ActionListener<Void> listener) {
+        final PlainListenableActionFuture<Void> currentListeners;
+        synchronized (mutex) {
+            if (pendingRerouteListeners != null) {
+                logger.trace("already has pending reroute, adding [{}] to batch", reason);
+                pendingRerouteListeners.addListener(listener);
+                return;
+            }
+            currentListeners = PlainListenableActionFuture.newListenableFuture();
+            currentListeners.addListener(listener);
+            pendingRerouteListeners = currentListeners;
+        }
+        logger.trace("rerouting [{}]", reason);
+        try {
+            clusterService.submitStateUpdateTask(CLUSTER_UPDATE_TASK_SOURCE + "(" + reason + ")",
+                new ClusterStateUpdateTask(Priority.HIGH) {
+                    @Override
+                    public ClusterState execute(ClusterState currentState) {
+                        synchronized (mutex) {
+                            assert pendingRerouteListeners == currentListeners;
+                            pendingRerouteListeners = null;
+                        }
+                        return reroute.apply(currentState, reason);
+                    }
+
+                    @Override
+                    public void onNoLongerMaster(String source) {
+                        synchronized (mutex) {
+                            if (pendingRerouteListeners == currentListeners) {
+                                pendingRerouteListeners = null;
+                            }
+                        }
+                        currentListeners.onFailure(new NotMasterException("delayed reroute [" + reason + "] cancelled"));
+                        // no big deal, the new master will reroute again
+                    }
+
+                    @Override
+                    public void onFailure(String source, Exception e) {
+                        synchronized (mutex) {
+                            if (pendingRerouteListeners == currentListeners) {
+                                pendingRerouteListeners = null;
+                            }
+                        }
+                        final ClusterState state = clusterService.state();
+                        if (logger.isTraceEnabled()) {
+                            logger.error(() -> new ParameterizedMessage("unexpected failure during [{}], current state:\n{}",
+                                source, state), e);
+                        } else {
+                            logger.error(() -> new ParameterizedMessage("unexpected failure during [{}], current state version [{}]",
+                                source, state.version()), e);
+                        }
+                        currentListeners.onFailure(new ElasticsearchException("delayed reroute [" + reason + "] failed", e));
+                    }
+
+                    @Override
+                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                        currentListeners.onResponse(null);
+                    }
+                });
+        } catch (Exception e) {
+            synchronized (mutex) {
+                assert pendingRerouteListeners == currentListeners;
+                pendingRerouteListeners = null;
+            }
+            ClusterState state = clusterService.state();
+            logger.warn(() -> new ParameterizedMessage("failed to reroute routing table, current state:\n{}", state), e);
+            currentListeners.onFailure(new ElasticsearchException("delayed reroute [" + reason + "] could not be submitted", e));
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
index ca3661f3e6f9c..5a98e9456f43d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
@@ -105,6 +105,10 @@ public class IndexShardRoutingTable implements Iterable<ShardRouting> {
                 // create the target initializing shard routing on the node the shard is relocating to
                 allInitializingShards.add(shard.getTargetRelocatingShard());
                 allAllocationIds.add(shard.getTargetRelocatingShard().allocationId().getId());
+
+                assert shard.assignedToNode() : "relocating from unassigned " + shard;
+                assert shard.getTargetRelocatingShard().assignedToNode() : "relocating to unassigned " + shard.getTargetRelocatingShard();
+                assignedShards.add(shard.getTargetRelocatingShard());
             }
             if (shard.assignedToNode()) {
                 assignedShards.add(shard);
@@ -211,7 +215,7 @@ public List<ShardRouting> getActiveShards() {
     }
 
     /**
-     * Returns a {@link List} of assigned shards
+     * Returns a {@link List} of assigned shards, including relocation targets
      *
      * @return a {@link List} of shards
      */
@@ -518,11 +522,6 @@ public ShardRouting getByAllocationId(String allocationId) {
             if (shardRouting.allocationId().getId().equals(allocationId)) {
                 return shardRouting;
             }
-            if (shardRouting.relocating()) {
-                if (shardRouting.getTargetRelocatingShard().allocationId().getId().equals(allocationId)) {
-                    return shardRouting.getTargetRelocatingShard();
-                }
-            }
         }
         return null;
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/LazilyInitializedRerouteService.java b/server/src/main/java/org/elasticsearch/cluster/routing/LazilyInitializedRerouteService.java
new file mode 100644
index 0000000000000..672d6dbc3a08c
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/LazilyInitializedRerouteService.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.routing;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.action.ActionListener;
+
+/**
+ * A {@link RerouteService} that can be initialized lazily. The real reroute service, {@link BatchedRerouteService}, depends on components
+ * constructed quite late in the construction of the node, but other components constructed earlier eventually need access to the reroute
+ * service too.
+ */
+public class LazilyInitializedRerouteService implements RerouteService {
+
+    private final SetOnce<RerouteService> delegate = new SetOnce<>();
+
+    @Override
+    public void reroute(String reason, ActionListener<Void> listener) {
+        assert delegate.get() != null;
+        delegate.get().reroute(reason, listener);
+    }
+
+    public void setRerouteService(RerouteService rerouteService) {
+        delegate.set(rerouteService);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RerouteService.java b/server/src/main/java/org/elasticsearch/cluster/routing/RerouteService.java
new file mode 100644
index 0000000000000..11a49322e10fa
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/RerouteService.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.routing;
+
+import org.elasticsearch.action.ActionListener;
+
+/**
+ * Asynchronously performs a cluster reroute, updating any shard states and rebalancing the cluster if appropriate.
+ */
+@FunctionalInterface
+public interface RerouteService {
+    void reroute(String reason, ActionListener<Void> listener);
+}
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
deleted file mode 100644
index 89e19e02b30ed..0000000000000
--- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.routing;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
-import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * A {@link RoutingService} listens to clusters state. When this service
- * receives a {@link ClusterChangedEvent} the cluster state will be verified and
- * the routing tables might be updated.
- * <p>
- * Note: The {@link RoutingService} is responsible for cluster wide operations
- * that include modifications to the cluster state. Such an operation can only
- * be performed on the clusters master node. Unless the local node this service
- * is running on is the clusters master node this service will not perform any
- * actions.
- * </p>
- */
-public class RoutingService extends AbstractLifecycleComponent {
-    private static final Logger logger = LogManager.getLogger(RoutingService.class);
-
-    private static final String CLUSTER_UPDATE_TASK_SOURCE = "cluster_reroute";
-
-    private final ClusterService clusterService;
-    private final AllocationService allocationService;
-
-    private AtomicBoolean rerouting = new AtomicBoolean();
-
-    @Inject
-    public RoutingService(ClusterService clusterService, AllocationService allocationService) {
-        this.clusterService = clusterService;
-        this.allocationService = allocationService;
-    }
-
-    @Override
-    protected void doStart() {
-    }
-
-    @Override
-    protected void doStop() {
-    }
-
-    @Override
-    protected void doClose() {
-    }
-
-    /**
-     * Initiates a reroute.
-     */
-    public final void reroute(String reason) {
-        try {
-            if (lifecycle.stopped()) {
-                return;
-            }
-            if (rerouting.compareAndSet(false, true) == false) {
-                logger.trace("already has pending reroute, ignoring {}", reason);
-                return;
-            }
-            logger.trace("rerouting {}", reason);
-            clusterService.submitStateUpdateTask(CLUSTER_UPDATE_TASK_SOURCE + "(" + reason + ")",
-                new ClusterStateUpdateTask(Priority.HIGH) {
-                    @Override
-                    public ClusterState execute(ClusterState currentState) {
-                        rerouting.set(false);
-                        return allocationService.reroute(currentState, reason);
-                    }
-
-                    @Override
-                    public void onNoLongerMaster(String source) {
-                        rerouting.set(false);
-                        // no biggie
-                    }
-
-                    @Override
-                    public void onFailure(String source, Exception e) {
-                        rerouting.set(false);
-                        ClusterState state = clusterService.state();
-                        if (logger.isTraceEnabled()) {
-                            logger.error(() -> new ParameterizedMessage("unexpected failure during [{}], current state:\n{}",
-                                source, state), e);
-                        } else {
-                            logger.error(() -> new ParameterizedMessage("unexpected failure during [{}], current state version [{}]",
-                                source, state.version()), e);
-                        }
-                    }
-                });
-        } catch (Exception e) {
-            rerouting.set(false);
-            ClusterState state = clusterService.state();
-            logger.warn(() -> new ParameterizedMessage("failed to reroute routing table, current state:\n{}", state), e);
-        }
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
index c688a120a8b6a..9b22e63cf9ae5 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
@@ -240,7 +240,7 @@ public ClusterState disassociateDeadNodes(ClusterState clusterState, boolean rer
      * Checks if the are replicas with the auto-expand feature that need to be adapted.
      * Returns an updated cluster state if changes were necessary, or the identical cluster if no changes were required.
      */
-    private ClusterState adaptAutoExpandReplicas(ClusterState clusterState) {
+    public ClusterState adaptAutoExpandReplicas(ClusterState clusterState) {
         final Map<Integer, List<String>> autoExpandReplicaChanges =
             AutoExpandReplicas.getAutoExpandReplicaChanges(clusterState.metaData(), clusterState.nodes());
         if (autoExpandReplicaChanges.isEmpty()) {
@@ -337,16 +337,16 @@ public CommandsResult reroute(final ClusterState clusterState, AllocationCommand
         allocation.debugDecision(true);
         // we ignore disable allocation, because commands are explicit
         allocation.ignoreDisable(true);
-        RoutingExplanations explanations = commands.execute(allocation, explain);
-        // we revert the ignore disable flag, since when rerouting, we want the original setting to take place
-        allocation.ignoreDisable(false);
-        // the assumption is that commands will move / act on shards (or fail through exceptions)
-        // so, there will always be shard "movements", so no need to check on reroute
 
         if (retryFailed) {
             resetFailedAllocationCounter(allocation);
         }
 
+        RoutingExplanations explanations = commands.execute(allocation, explain);
+        // we revert the ignore disable flag, since when rerouting, we want the original setting to take place
+        allocation.ignoreDisable(false);
+        // the assumption is that commands will move / act on shards (or fail through exceptions)
+        // so, there will always be shard "movements", so no need to check on reroute
         reroute(allocation);
         return new CommandsResult(explanations, buildResultAndLogHealthChange(clusterState, allocation, "reroute commands"));
     }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetaDataUpdater.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetaDataUpdater.java
index 54625a15e8d80..18d3b45fd9c7b 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetaDataUpdater.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetaDataUpdater.java
@@ -194,9 +194,13 @@ private IndexMetaData.Builder updateInSyncAllocations(RoutingTable newRoutingTab
             // of replicas was decreased while shards were unassigned.
             int maxActiveShards = oldIndexMetaData.getNumberOfReplicas() + 1; // +1 for the primary
             IndexShardRoutingTable newShardRoutingTable = newRoutingTable.shardRoutingTable(shardId);
+            assert newShardRoutingTable.assignedShards().stream()
+                .filter(ShardRouting::isRelocationTarget).map(s -> s.allocationId().getId()).noneMatch(inSyncAllocationIds::contains)
+                : newShardRoutingTable.assignedShards() + " vs " + inSyncAllocationIds;
             if (inSyncAllocationIds.size() > oldInSyncAllocationIds.size() && inSyncAllocationIds.size() > maxActiveShards) {
                 // trim entries that have no corresponding shard routing in the cluster state (i.e. trim unavailable copies)
-                List<ShardRouting> assignedShards = newShardRoutingTable.assignedShards();
+                List<ShardRouting> assignedShards = newShardRoutingTable.assignedShards()
+                    .stream().filter(s -> s.isRelocationTarget() == false).collect(Collectors.toList());
                 assert assignedShards.size() <= maxActiveShards :
                     "cannot have more assigned shards " + assignedShards + " than maximum possible active shards " + maxActiveShards;
                 Set<String> assignedAllocations = assignedShards.stream().map(s -> s.allocationId().getId()).collect(Collectors.toSet());
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
index 4d037570dd266..2e3219e67c7ae 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
@@ -110,13 +110,20 @@ public RerouteExplanation execute(RoutingAllocation allocation, boolean explain)
             return explainOrThrowMissingRoutingNode(allocation, explain, discoNode);
         }
 
-        final ShardRouting shardRouting;
         try {
-            shardRouting = allocation.routingTable().shardRoutingTable(index, shardId).primaryShard();
+            allocation.routingTable().shardRoutingTable(index, shardId).primaryShard();
         } catch (IndexNotFoundException | ShardNotFoundException e) {
             return explainOrThrowRejectedCommand(explain, allocation, e);
         }
-        if (shardRouting.unassigned() == false) {
+
+        ShardRouting shardRouting = null;
+        for (ShardRouting shard : allocation.routingNodes().unassigned()) {
+            if (shard.getIndexName().equals(index) && shard.getId() == shardId && shard.primary()) {
+                shardRouting = shard;
+                break;
+            }
+        }
+        if (shardRouting == null) {
             return explainOrThrowRejectedCommand(explain, allocation, "primary [" + index + "][" + shardId + "] is already assigned");
         }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java
index 709681f2b2008..5e1bcd81bb5fa 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.allocation.RerouteExplanation;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
@@ -35,6 +34,7 @@
 import org.elasticsearch.index.shard.ShardNotFoundException;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -101,20 +101,34 @@ public RerouteExplanation execute(RoutingAllocation allocation, boolean explain)
             return explainOrThrowMissingRoutingNode(allocation, explain, discoNode);
         }
 
-        final ShardRouting primaryShardRouting;
         try {
-            primaryShardRouting = allocation.routingTable().shardRoutingTable(index, shardId).primaryShard();
+            allocation.routingTable().shardRoutingTable(index, shardId).primaryShard();
         } catch (IndexNotFoundException | ShardNotFoundException e) {
             return explainOrThrowRejectedCommand(explain, allocation, e);
         }
-        if (primaryShardRouting.unassigned()) {
+
+        ShardRouting primaryShardRouting = null;
+        for (RoutingNode node : allocation.routingNodes()) {
+            for (ShardRouting shard : node) {
+                if (shard.getIndexName().equals(index) && shard.getId() == shardId && shard.primary()) {
+                    primaryShardRouting = shard;
+                    break;
+                }
+            }
+        }
+        if (primaryShardRouting == null) {
             return explainOrThrowRejectedCommand(explain, allocation,
                 "trying to allocate a replica shard [" + index + "][" + shardId +
                     "], while corresponding primary shard is still unassigned");
         }
 
-        List<ShardRouting> replicaShardRoutings =
-            allocation.routingTable().shardRoutingTable(index, shardId).replicaShardsWithState(ShardRoutingState.UNASSIGNED);
+        List<ShardRouting> replicaShardRoutings = new ArrayList<>();
+        for (ShardRouting shard : allocation.routingNodes().unassigned()) {
+            if (shard.getIndexName().equals(index) && shard.getId() == shardId && shard.primary() == false) {
+                replicaShardRoutings.add(shard);
+            }
+        }
+
         ShardRouting shardRouting;
         if (replicaShardRoutings.isEmpty()) {
             return explainOrThrowRejectedCommand(explain, allocation,
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java
index f4c9aba17d71e..7e645c2cfcb6f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java
@@ -108,13 +108,20 @@ public RerouteExplanation execute(RoutingAllocation allocation, boolean explain)
             return explainOrThrowMissingRoutingNode(allocation, explain, discoNode);
         }
 
-        final ShardRouting shardRouting;
         try {
-            shardRouting = allocation.routingTable().shardRoutingTable(index, shardId).primaryShard();
+            allocation.routingTable().shardRoutingTable(index, shardId).primaryShard();
         } catch (IndexNotFoundException | ShardNotFoundException e) {
             return explainOrThrowRejectedCommand(explain, allocation, e);
         }
-        if (shardRouting.unassigned() == false) {
+
+        ShardRouting shardRouting = null;
+        for (ShardRouting shard : allocation.routingNodes().unassigned()) {
+            if (shard.getIndexName().equals(index) && shard.getId() == shardId && shard.primary()) {
+                shardRouting = shard;
+                break;
+            }
+        }
+        if (shardRouting == null) {
             return explainOrThrowRejectedCommand(explain, allocation, "primary [" + index + "][" + shardId + "] is already assigned");
         }
 
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
index 1f048fca76c09..0838999c4f367 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
@@ -131,12 +131,12 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
 
         // flag that determines whether the low threshold checks below can be skipped. We use this for a primary shard that is freshly
         // allocated and empty.
-        boolean skipLowTresholdChecks = shardRouting.primary() &&
+        boolean skipLowThresholdChecks = shardRouting.primary() &&
             shardRouting.active() == false && shardRouting.recoverySource().getType() == RecoverySource.Type.EMPTY_STORE;
 
         // checks for exact byte comparisons
         if (freeBytes < diskThresholdSettings.getFreeBytesThresholdLow().getBytes()) {
-            if (skipLowTresholdChecks == false) {
+            if (skipLowThresholdChecks == false) {
                 if (logger.isDebugEnabled()) {
                     logger.debug("less than the required {} free bytes threshold ({} free) on node {}, preventing allocation",
                             diskThresholdSettings.getFreeBytesThresholdLow(), freeBytesValue, node.nodeId());
@@ -178,7 +178,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
         // checks for percentage comparisons
         if (freeDiskPercentage < diskThresholdSettings.getFreeDiskThresholdLow()) {
             // If the shard is a replica or is a non-empty primary, check the low threshold
-            if (skipLowTresholdChecks == false) {
+            if (skipLowThresholdChecks == false) {
                 if (logger.isDebugEnabled()) {
                     logger.debug("more than the allowed {} used disk threshold ({} used) on node [{}], preventing allocation",
                             Strings.format1Decimals(usedDiskThresholdLow, "%"),
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
index f83f2606b14b6..fded43a4bdd19 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
@@ -73,7 +73,7 @@ public ClusterService(Settings settings, ClusterSettings clusterSettings, Thread
     }
 
     public ClusterService(Settings settings, ClusterSettings clusterSettings, MasterService masterService,
-        ClusterApplierService clusterApplierService) {
+                          ClusterApplierService clusterApplierService) {
         this.settings = settings;
         this.nodeName = Node.NODE_NAME_SETTING.get(settings);
         this.masterService = masterService;
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/EmptyDirTask.groovy b/server/src/main/java/org/elasticsearch/common/TriConsumer.java
similarity index 59%
rename from buildSrc/src/main/groovy/org/elasticsearch/gradle/EmptyDirTask.groovy
rename to server/src/main/java/org/elasticsearch/common/TriConsumer.java
index 12971b7d701ae..94d3a000f326b 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/EmptyDirTask.groovy
+++ b/server/src/main/java/org/elasticsearch/common/TriConsumer.java
@@ -15,35 +15,26 @@
  * KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations
  * under the License.
+ *
  */
-package org.elasticsearch.gradle
-
-import org.gradle.api.DefaultTask
-import org.gradle.api.tasks.Input
-import org.gradle.api.tasks.TaskAction
-import org.gradle.internal.nativeintegration.filesystem.Chmod
 
-import javax.inject.Inject
+package org.elasticsearch.common;
 
 /**
- * Creates an empty directory.
+ * Represents an operation that accepts three arguments and returns no result.
+ *
+ * @param <S> the type of the first argument
+ * @param <T> the type of the second argument
+ * @param <U> the type of the third argument
  */
-class EmptyDirTask extends DefaultTask {
-  @Input
-  Object dir
-
-  @Input
-  int dirMode = 0755
-
-  @TaskAction
-  void create() {
-    dir = dir as File
-    dir.mkdirs()
-    getChmod().chmod(dir, dirMode)
-  }
-
-  @Inject
-  Chmod getChmod() {
-    throw new UnsupportedOperationException()
-  }
+@FunctionalInterface
+public interface TriConsumer<S, T, U> {
+    /**
+     * Applies this function to the given arguments.
+     *
+     * @param s the first function argument
+     * @param t the second function argument
+     * @param u the third function argument
+     */
+    void apply(S s, T t, U u);
 }
diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java b/server/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java
index 9c1bacb51bac7..a44d1fb05308a 100644
--- a/server/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java
+++ b/server/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java
@@ -109,6 +109,12 @@ public interface BlobContainer {
      */
     void deleteBlob(String blobName) throws IOException;
 
+    /**
+     * Deletes this container and all its contents from the repository.
+     * @throws IOException on failure
+     */
+    void delete() throws IOException;
+
     /**
      * Deletes the blobs with given names. Unlike {@link #deleteBlob(String)} this method will not throw an exception
      * when one or multiple of the given blobs don't exist and simply ignore this case.
@@ -158,6 +164,16 @@ default void deleteBlobIgnoringIfNotExists(String blobName) throws IOException {
      */
     Map<String, BlobMetaData> listBlobs() throws IOException;
 
+    /**
+     * Lists all child containers under this container. A child container is defined as a container whose {@link #path()} method returns
+     * a path that has this containers {@link #path()} return as its prefix and has one more path element than the current
+     * container's path.
+     *
+     * @return Map of name of the child container to child container
+     * @throws IOException on failure to list child containers
+     */
+    Map<String, BlobContainer> children() throws IOException;
+
     /**
      * Lists all blobs in the container that match the specified prefix.
      *
diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java b/server/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java
index ea02aebb0aaad..d3acd02a06d1f 100644
--- a/server/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java
+++ b/server/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.common.blobstore;
 
+import org.elasticsearch.common.Nullable;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
@@ -68,6 +70,20 @@ public String buildAsString() {
         return p + SEPARATOR;
     }
 
+    /**
+     * Returns this path's parent path.
+     *
+     * @return Parent path or {@code null} if there is none
+     */
+    @Nullable
+    public BlobPath parent() {
+        if (paths.isEmpty()) {
+            return null;
+        } else {
+            return new BlobPath(List.copyOf(paths.subList(0, paths.size() - 1)));
+        }
+    }
+
     @Override
     public String toString() {
         StringBuilder sb = new StringBuilder();
diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
index bab984bd85c74..b51115b246673 100644
--- a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
+++ b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.common.blobstore.fs;
 
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
@@ -73,9 +74,22 @@ public Map<String, BlobMetaData> listBlobs() throws IOException {
         return listBlobsByPrefix(null);
     }
 
+    @Override
+    public Map<String, BlobContainer> children() throws IOException {
+        Map<String, BlobContainer> builder = new HashMap<>();
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) {
+            for (Path file : stream) {
+                if (Files.isDirectory(file)) {
+                    final String name = file.getFileName().toString();
+                    builder.put(name, new FsBlobContainer(blobStore, path().add(name), file));
+                }
+            }
+        }
+        return unmodifiableMap(builder);
+    }
+
     @Override
     public Map<String, BlobMetaData> listBlobsByPrefix(String blobNamePrefix) throws IOException {
-        // If we get duplicate files we should just take the last entry
         Map<String, BlobMetaData> builder = new HashMap<>();
 
         blobNamePrefix = blobNamePrefix == null ? "" : blobNamePrefix;
@@ -108,6 +122,11 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx
         }
     }
 
+    @Override
+    public void delete() throws IOException {
+        IOUtils.rm(path);
+    }
+
     @Override
     public boolean blobExists(String blobName) {
         return Files.exists(path.resolve(blobName));
diff --git a/server/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java b/server/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java
index 3b1202fe66f42..2ff2f4e95dfdd 100644
--- a/server/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java
+++ b/server/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java
@@ -21,11 +21,9 @@
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.core.internal.io.Streams;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -93,10 +91,6 @@ public static BytesReference uncompress(BytesReference bytes) throws IOException
     }
 
     private static BytesReference uncompress(BytesReference bytes, Compressor compressor) throws IOException {
-        StreamInput compressed = compressor.streamInput(bytes.streamInput());
-        BytesStreamOutput bStream = new BytesStreamOutput();
-        Streams.copy(compressed, bStream);
-        compressed.close();
-        return bStream.bytes();
+        return Streams.readFully(compressor.streamInput(bytes.streamInput()));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoJson.java b/server/src/main/java/org/elasticsearch/common/geo/GeoJson.java
index 3489eca8b58e4..45db221a65dbf 100644
--- a/server/src/main/java/org/elasticsearch/common/geo/GeoJson.java
+++ b/server/src/main/java/org/elasticsearch/common/geo/GeoJson.java
@@ -44,6 +44,7 @@
 import org.elasticsearch.geo.geometry.Polygon;
 import org.elasticsearch.geo.geometry.Rectangle;
 import org.elasticsearch.geo.geometry.ShapeType;
+import org.elasticsearch.geo.utils.GeometryValidator;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -64,14 +65,22 @@ public final class GeoJson {
     private static final ParseField FIELD_ORIENTATION = new ParseField("orientation");
     private static final ParseField FIELD_RADIUS = new ParseField("radius");
 
-    private GeoJson() {
+    private final boolean rightOrientation;
+    private final boolean coerce;
+    private final GeometryValidator validator;
 
+    public GeoJson(boolean rightOrientation, boolean coerce, GeometryValidator validator) {
+        this.rightOrientation = rightOrientation;
+        this.coerce = coerce;
+        this.validator = validator;
     }
 
-    public static Geometry fromXContent(XContentParser parser, boolean rightOrientation, boolean coerce, boolean ignoreZValue)
+    public Geometry fromXContent(XContentParser parser)
         throws IOException {
         try (XContentSubParser subParser = new XContentSubParser(parser)) {
-            return PARSER.apply(subParser, new ParserContext(rightOrientation, coerce, ignoreZValue));
+            Geometry geometry = PARSER.apply(subParser, this);
+            validator.validate(geometry);
+            return geometry;
         }
     }
 
@@ -197,31 +206,19 @@ private XContentBuilder coordinatesToXContent(Polygon polygon) throws IOExceptio
         return builder.endObject();
     }
 
-    private static class ParserContext {
-        public final boolean defaultOrientation;
-        public final boolean coerce;
-        public final boolean ignoreZValue;
-
-        ParserContext(boolean defaultOrientation, boolean coerce, boolean ignoreZValue) {
-            this.defaultOrientation = defaultOrientation;
-            this.coerce = coerce;
-            this.ignoreZValue = ignoreZValue;
-        }
-    }
-
-    private static ConstructingObjectParser<Geometry, ParserContext> PARSER =
+    private static ConstructingObjectParser<Geometry, GeoJson> PARSER =
         new ConstructingObjectParser<>("geojson", true, (a, c) -> {
             String type = (String) a[0];
             CoordinateNode coordinates = (CoordinateNode) a[1];
             @SuppressWarnings("unchecked") List<Geometry> geometries = (List<Geometry>) a[2];
             Boolean orientation = orientationFromString((String) a[3]);
             DistanceUnit.Distance radius = (DistanceUnit.Distance) a[4];
-            return createGeometry(type, geometries, coordinates, orientation, c.defaultOrientation, c.coerce, radius);
+            return createGeometry(type, geometries, coordinates, orientation, c.rightOrientation, c.coerce, radius);
         });
 
     static {
         PARSER.declareString(constructorArg(), FIELD_TYPE);
-        PARSER.declareField(optionalConstructorArg(), (p, c) -> parseCoordinates(p, c.ignoreZValue), FIELD_COORDINATES,
+        PARSER.declareField(optionalConstructorArg(), (p, c) -> parseCoordinates(p), FIELD_COORDINATES,
             ObjectParser.ValueType.VALUE_ARRAY);
         PARSER.declareObjectArray(optionalConstructorArg(), PARSER, FIELD_GEOMETRIES);
         PARSER.declareString(optionalConstructorArg(), FIELD_ORIENTATION);
@@ -304,20 +301,20 @@ private static void verifyNulls(String type, List<Geometry> geometries, Boolean
      * Recursive method which parses the arrays of coordinates used to define
      * Shapes
      */
-    private static CoordinateNode parseCoordinates(XContentParser parser, boolean ignoreZValue) throws IOException {
+    private static CoordinateNode parseCoordinates(XContentParser parser) throws IOException {
         XContentParser.Token token = parser.nextToken();
         // Base cases
         if (token != XContentParser.Token.START_ARRAY &&
             token != XContentParser.Token.END_ARRAY &&
             token != XContentParser.Token.VALUE_NULL) {
-            return new CoordinateNode(parseCoordinate(parser, ignoreZValue));
+            return new CoordinateNode(parseCoordinate(parser));
         } else if (token == XContentParser.Token.VALUE_NULL) {
             throw new IllegalArgumentException("coordinates cannot contain NULL values)");
         }
 
         List<CoordinateNode> nodes = new ArrayList<>();
         while (token != XContentParser.Token.END_ARRAY) {
-            CoordinateNode node = parseCoordinates(parser, ignoreZValue);
+            CoordinateNode node = parseCoordinates(parser);
             if (nodes.isEmpty() == false && nodes.get(0).numDimensions() != node.numDimensions()) {
                 throw new ElasticsearchParseException("Exception parsing coordinates: number of dimensions do not match");
             }
@@ -331,7 +328,7 @@ private static CoordinateNode parseCoordinates(XContentParser parser, boolean ig
     /**
      * Parser a singe set of 2 or 3 coordinates
      */
-    private static Point parseCoordinate(XContentParser parser, boolean ignoreZValue) throws IOException {
+    private static Point parseCoordinate(XContentParser parser) throws IOException {
         // Add support for coerce here
         if (parser.currentToken() != XContentParser.Token.VALUE_NUMBER) {
             throw new ElasticsearchParseException("geo coordinates must be numbers");
@@ -345,7 +342,7 @@ private static Point parseCoordinate(XContentParser parser, boolean ignoreZValue
         // alt (for storing purposes only - future use includes 3d shapes)
         double alt = Double.NaN;
         if (token == XContentParser.Token.VALUE_NUMBER) {
-            alt = GeoPoint.assertZValue(ignoreZValue, parser.doubleValue());
+            alt = parser.doubleValue();
             parser.nextToken();
         }
         // do not support > 3 dimensions
diff --git a/server/src/main/java/org/apache/lucene/search/XIndexSearcher.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryFormat.java
similarity index 52%
rename from server/src/main/java/org/apache/lucene/search/XIndexSearcher.java
rename to server/src/main/java/org/elasticsearch/common/geo/GeometryFormat.java
index 100c5f4944afe..5e14e20050bb3 100644
--- a/server/src/main/java/org/apache/lucene/search/XIndexSearcher.java
+++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryFormat.java
@@ -17,30 +17,29 @@
  * under the License.
  */
 
-package org.apache.lucene.search;
+package org.elasticsearch.common.geo;
 
-import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.geo.geometry.Geometry;
 
 import java.io.IOException;
-import java.util.List;
+import java.text.ParseException;
 
 /**
- * A wrapper for {@link IndexSearcher} that makes {@link IndexSearcher#search(List, Weight, Collector)}
- * visible by sub-classes.
+ * Geometry serializer/deserializer
  */
-public class XIndexSearcher extends IndexSearcher {
-    private final IndexSearcher in;
+public interface GeometryFormat {
 
-    public XIndexSearcher(IndexSearcher in) {
-        super(in.getIndexReader());
-        this.in = in;
-        setSimilarity(in.getSimilarity());
-        setQueryCache(in.getQueryCache());
-        setQueryCachingPolicy(in.getQueryCachingPolicy());
-    }
+    /**
+     * Parser JSON representation of a geometry
+     */
+    Geometry fromXContent(XContentParser parser) throws IOException, ParseException;
+
+    /**
+     * Serializes the geometry into its JSON representation
+     */
+    XContentBuilder toXContent(Geometry geometry, XContentBuilder builder, ToXContent.Params params) throws IOException;
 
-    @Override
-    public void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
-        in.search(leaves, weight, collector);
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryParser.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryParser.java
index 8e1db18ccdd97..b96e41df5e472 100644
--- a/server/src/main/java/org/elasticsearch/common/geo/GeometryParser.java
+++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryParser.java
@@ -20,8 +20,12 @@
 package org.elasticsearch.common.geo;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.geo.geometry.Geometry;
+import org.elasticsearch.geo.utils.GeographyValidator;
+import org.elasticsearch.geo.utils.GeometryValidator;
 import org.elasticsearch.geo.utils.WellKnownText;
 
 import java.io.IOException;
@@ -32,22 +36,77 @@
  */
 public final class GeometryParser {
 
-    private GeometryParser() {
+    private final GeoJson geoJsonParser;
+    private final WellKnownText wellKnownTextParser;
+    private final GeometryValidator validator;
 
+    public GeometryParser(boolean rightOrientation, boolean coerce, boolean ignoreZValue) {
+        validator = new GeographyValidator(ignoreZValue);
+        geoJsonParser = new GeoJson(rightOrientation, coerce, validator);
+        wellKnownTextParser = new WellKnownText(coerce, validator);
     }
 
     /**
      * Parses supplied XContent into Geometry
      */
-    public static Geometry parse(XContentParser parser, boolean orientation, boolean coerce, boolean ignoreZValue) throws IOException,
-        ParseException {
+    public Geometry parse(XContentParser parser) throws IOException, ParseException {
+        return geometryFormat(parser).fromXContent(parser);
+    }
+
+    /**
+     * Returns a geometry format object that can parse and then serialize the object back to the same format.
+     */
+    public GeometryFormat geometryFormat(XContentParser parser) {
         if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
-            return null;
+            return new GeometryFormat() {
+                @Override
+                public Geometry fromXContent(XContentParser parser) throws IOException {
+                    return null;
+                }
+
+                @Override
+                public XContentBuilder toXContent(Geometry geometry, XContentBuilder builder, ToXContent.Params params) throws IOException {
+                    if (geometry != null) {
+                        // We don't know the format of the original geometry - so going with default
+                        return GeoJson.toXContent(geometry, builder, params);
+                    } else {
+                        return builder.nullValue();
+                    }
+                }
+            };
         } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
-            return GeoJson.fromXContent(parser, orientation, coerce, ignoreZValue);
+            return new GeometryFormat() {
+                @Override
+                public Geometry fromXContent(XContentParser parser) throws IOException {
+                    return geoJsonParser.fromXContent(parser);
+                }
+
+                @Override
+                public XContentBuilder toXContent(Geometry geometry, XContentBuilder builder, ToXContent.Params params) throws IOException {
+                    if (geometry != null) {
+                        return GeoJson.toXContent(geometry, builder, params);
+                    } else {
+                        return builder.nullValue();
+                    }
+                }
+            };
         } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
-            // TODO: Add support for ignoreZValue and coerce to WKT
-            return WellKnownText.fromWKT(parser.text());
+            return new GeometryFormat() {
+                @Override
+                public Geometry fromXContent(XContentParser parser) throws IOException, ParseException {
+                    return wellKnownTextParser.fromWKT(parser.text());
+                }
+
+                @Override
+                public XContentBuilder toXContent(Geometry geometry, XContentBuilder builder, ToXContent.Params params) throws IOException {
+                    if (geometry != null) {
+                        return builder.value(wellKnownTextParser.toWKT(geometry));
+                    } else {
+                        return builder.nullValue();
+                    }
+                }
+            };
+
         }
         throw new ElasticsearchParseException("shape must be an object consisting of type and coordinates");
     }
diff --git a/server/src/main/java/org/elasticsearch/common/hash/MessageDigests.java b/server/src/main/java/org/elasticsearch/common/hash/MessageDigests.java
index 8bcef7b8ff4cb..df8f3e2fa7f43 100644
--- a/server/src/main/java/org/elasticsearch/common/hash/MessageDigests.java
+++ b/server/src/main/java/org/elasticsearch/common/hash/MessageDigests.java
@@ -95,15 +95,24 @@ private static MessageDigest get(ThreadLocal<MessageDigest> messageDigest) {
      * @return a hex representation of the input as a String.
      */
     public static String toHexString(byte[] bytes) {
-        Objects.requireNonNull(bytes);
-        StringBuilder sb = new StringBuilder(2 * bytes.length);
+        return new String(toHexCharArray(bytes));
+    }
 
+    /**
+     * Encodes the byte array into a newly created hex char array, without allocating any other temporary variables.
+     *
+     * @param bytes the input to be encoded as hex.
+     * @return the hex encoding of the input as a char array.
+     */
+    public static char[] toHexCharArray(byte[] bytes) {
+        Objects.requireNonNull(bytes);
+        final char[] result = new char[2 * bytes.length];
         for (int i = 0; i < bytes.length; i++) {
             byte b = bytes[i];
-            sb.append(HEX_DIGITS[b >> 4 & 0xf]).append(HEX_DIGITS[b & 0xf]);
+            result[2 * i] = HEX_DIGITS[b >> 4 & 0xf];
+            result[2 * i + 1] = HEX_DIGITS[b & 0xf];
         }
-
-        return sb.toString();
+        return result;
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/common/hash/MurmurHash3.java b/server/src/main/java/org/elasticsearch/common/hash/MurmurHash3.java
index a52f0e8acc4ae..a9232e06657ad 100644
--- a/server/src/main/java/org/elasticsearch/common/hash/MurmurHash3.java
+++ b/server/src/main/java/org/elasticsearch/common/hash/MurmurHash3.java
@@ -21,6 +21,8 @@
 
 import org.elasticsearch.common.util.ByteUtils;
 
+import java.util.Objects;
+
 
 /**
  * MurmurHash3 hashing functions.
@@ -36,6 +38,24 @@ public static class Hash128 {
         public long h1;
         /** higher 64 bits part **/
         public long h2;
+
+        @Override
+        public boolean equals(Object other) {
+            if (this == other) {
+                return true;
+            }
+            if (other == null || getClass() != other.getClass()) {
+                return false;
+            }
+            Hash128 that = (Hash128) other;
+            return Objects.equals(this.h1, that.h1)
+                && Objects.equals(this.h2, that.h2);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(h1, h2);
+        }
     }
 
     private static long C1 = 0x87c37b91114253d5L;
@@ -160,4 +180,22 @@ public static Hash128 hash128(byte[] key, int offset, int length, long seed, Has
         return hash;
     }
 
+    /**
+     * A 64-bit variant which accepts a long to hash, and returns the 64bit long hash.
+     * This is useful if the input is already in long (or smaller) format and you don't
+     * need the full 128b width and flexibility of
+     * {@link MurmurHash3#hash128(byte[], int, int, long, Hash128)}
+     *
+     * Given the limited nature of this variant, it should be faster than the 128b version
+     * when you only need 128b (many fewer instructions)
+     */
+    public static long murmur64(long h) {
+        h ^= h >>> 33;
+        h *= 0xff51afd7ed558ccdL;
+        h ^= h >>> 33;
+        h *= 0xc4ceb9fe1a85ec53L;
+        h ^= h >>> 33;
+        return h;
+    }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/common/inject/Binder.java b/server/src/main/java/org/elasticsearch/common/inject/Binder.java
index 03d164bcbaa52..55e219047c2c1 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/Binder.java
+++ b/server/src/main/java/org/elasticsearch/common/inject/Binder.java
@@ -31,7 +31,7 @@
  * used to create an {@link Injector}. Guice provides this object to your
  * application's {@link Module} implementors so they may each contribute
  * their own bindings and other registrations.
- * <h3>The Guice Binding EDSL</h3>
+ * <h2>The Guice Binding EDSL</h2>
  * <p>
  * Guice uses an <i>embedded domain-specific language</i>, or EDSL, to help you
  * create bindings simply and readably.  This approach is great for overall
diff --git a/server/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider.java b/server/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider.java
index 9e2d0e379867c..2ef0b89fb1570 100644
--- a/server/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider.java
+++ b/server/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider.java
@@ -39,7 +39,7 @@
 /**
  * Provides a factory that combines the caller's arguments with injector-supplied values to
  * construct objects.
- * <h3>Defining a factory</h3>
+ * <h2>Defining a factory</h2>
  * Create an interface whose methods return the constructed type, or any of its supertypes. The
  * method's parameters are the arguments required to build the constructed type.
  * <pre>public interface PaymentFactory {
diff --git a/server/src/main/java/org/elasticsearch/common/io/Streams.java b/server/src/main/java/org/elasticsearch/common/io/Streams.java
index 46a6956914f90..4a8f2f5de5b74 100644
--- a/server/src/main/java/org/elasticsearch/common/io/Streams.java
+++ b/server/src/main/java/org/elasticsearch/common/io/Streams.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStream;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.BufferedReader;
@@ -226,6 +227,17 @@ public static BytesStream flushOnCloseStream(BytesStream os) {
         return new FlushOnCloseOutputStream(os);
     }
 
+    /**
+     * Reads all bytes from the given {@link InputStream} and closes it afterwards.
+     */
+    public static BytesReference readFully(InputStream in) throws IOException {
+        try (InputStream inputStream = in) {
+            BytesStreamOutput out = new BytesStreamOutput();
+            copy(inputStream, out);
+            return out.bytes();
+        }
+    }
+
     /**
      * A wrapper around a {@link BytesStream} that makes the close operation a flush. This is
      * needed as sometimes a stream will be closed but the bytes that the stream holds still need
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index 7a763c5a04995..2e213cccf1306 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -30,11 +30,13 @@
 import org.apache.lucene.util.CharsRef;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
+import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
@@ -59,6 +61,7 @@
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
@@ -358,6 +361,21 @@ public String readOptionalString() throws IOException {
         return null;
     }
 
+    @Nullable
+    public SecureString readOptionalSecureString() throws IOException {
+        SecureString value = null;
+        BytesReference bytesRef = readOptionalBytesReference();
+        if (bytesRef != null) {
+            byte[] bytes = BytesReference.toBytes(bytesRef);
+            try {
+                value = new SecureString(CharArrays.utf8BytesToChars(bytes));
+            } finally {
+                Arrays.fill(bytes, (byte) 0);
+            }
+        }
+        return value;
+    }
+
     @Nullable
     public Float readOptionalFloat() throws IOException {
         if (readBoolean()) {
@@ -415,6 +433,16 @@ public String readString() throws IOException {
         return spare.toString();
     }
 
+    public SecureString readSecureString() throws IOException {
+        BytesReference bytesRef = readBytesReference();
+        byte[] bytes = BytesReference.toBytes(bytesRef);
+        try {
+            return new SecureString(CharArrays.utf8BytesToChars(bytes));
+        } finally {
+            Arrays.fill(bytes, (byte) 0);
+        }
+    }
+
     public final float readFloat() throws IOException {
         return Float.intBitsToFloat(readInt());
     }
diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index 431e00fcf063a..8d944e6f8a17c 100644
--- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -32,10 +32,13 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.stream.Writeable.Writer;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
@@ -58,6 +61,7 @@
 import java.time.Instant;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
@@ -324,6 +328,19 @@ public void writeOptionalString(@Nullable String str) throws IOException {
         }
     }
 
+    public void writeOptionalSecureString(@Nullable SecureString secureStr) throws IOException {
+        if (secureStr == null) {
+            writeOptionalBytesReference(null);
+        } else {
+            final byte[] secureStrBytes = CharArrays.toUtf8Bytes(secureStr.getChars());
+            try {
+                writeOptionalBytesReference(new BytesArray(secureStrBytes));
+            } finally {
+                Arrays.fill(secureStrBytes, (byte) 0);
+            }
+        }
+    }
+
     /**
      * Writes an optional {@link Integer}.
      */
@@ -414,6 +431,15 @@ public void writeString(String str) throws IOException {
         writeBytes(buffer, offset);
     }
 
+    public void writeSecureString(SecureString secureStr) throws IOException {
+        final byte[] secureStrBytes = CharArrays.toUtf8Bytes(secureStr.getChars());
+        try {
+            writeBytesReference(new BytesArray(secureStrBytes));
+        } finally {
+            Arrays.fill(secureStrBytes, (byte) 0);
+        }
+    }
+
     public void writeFloat(float v) throws IOException {
         writeInt(Float.floatToIntBits(v));
     }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
index 88d7c57f5e424..4b0ec9ea08773 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -27,7 +27,6 @@
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.document.LatLonDocValuesField;
-import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.CorruptIndexException;
@@ -95,7 +94,6 @@
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 
 import java.io.IOException;
 import java.text.ParseException;
@@ -105,7 +103,6 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.function.LongConsumer;
 
 public class Lucene {
     public static final String LATEST_DOC_VALUES_FORMAT = "Lucene70";
@@ -1050,39 +1047,4 @@ public CacheHelper getReaderCacheHelper() {
             }
         };
     }
-
-    /**
-     * Scans sequence numbers (i.e., {@link SeqNoFieldMapper#NAME}) between {@code fromSeqNo}(inclusive) and {@code toSeqNo}(inclusive)
-     * in the provided directory reader. This method invokes the callback {@code onNewSeqNo} whenever a sequence number value is found.
-     *
-     * @param directoryReader the directory reader to scan
-     * @param fromSeqNo       the lower bound of a range of seq_no to scan (inclusive)
-     * @param toSeqNo         the upper bound of a range of seq_no to scan (inclusive)
-     * @param onNewSeqNo      the callback to be called whenever a new valid sequence number is found
-     */
-    public static void scanSeqNosInReader(DirectoryReader directoryReader, long fromSeqNo, long toSeqNo,
-                                          LongConsumer onNewSeqNo) throws IOException {
-        final DirectoryReader reader = Lucene.wrapAllDocsLive(directoryReader);
-        final IndexSearcher searcher = new IndexSearcher(reader);
-        searcher.setQueryCache(null);
-        final Query query = LongPoint.newRangeQuery(SeqNoFieldMapper.NAME, fromSeqNo, toSeqNo);
-        final Weight weight = searcher.createWeight(query, ScoreMode.COMPLETE_NO_SCORES, 1.0f);
-        for (LeafReaderContext leaf : reader.leaves()) {
-            final Scorer scorer = weight.scorer(leaf);
-            if (scorer == null) {
-                continue;
-            }
-            final DocIdSetIterator docIdSetIterator = scorer.iterator();
-            final NumericDocValues seqNoDocValues = leaf.reader().getNumericDocValues(SeqNoFieldMapper.NAME);
-            int docId;
-            while ((docId = docIdSetIterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
-                if (seqNoDocValues == null || seqNoDocValues.advanceExact(docId) == false) {
-                    throw new IllegalStateException("seq_no doc_values not found for doc_id=" + docId);
-                }
-                final long seqNo = seqNoDocValues.longValue();
-                assert fromSeqNo <= seqNo && seqNo <= toSeqNo : "from_seq_no=" + fromSeqNo + " seq_no=" + seqNo + " to_seq_no=" + toSeqNo;
-                onNewSeqNo.accept(seqNo);
-            }
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/MinimumScoreCollector.java b/server/src/main/java/org/elasticsearch/common/lucene/MinimumScoreCollector.java
index f99d68952e557..4e558f163ebca 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/MinimumScoreCollector.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/MinimumScoreCollector.java
@@ -65,6 +65,6 @@ public void doSetNextReader(LeafReaderContext context) throws IOException {
 
     @Override
     public ScoreMode scoreMode() {
-        return ScoreMode.COMPLETE;
+        return collector.scoreMode() == ScoreMode.TOP_SCORES ? ScoreMode.TOP_SCORES :  ScoreMode.COMPLETE;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/Queries.java b/server/src/main/java/org/elasticsearch/common/lucene/search/Queries.java
index 96a0cafc35b11..2e004dfad2e55 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/Queries.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/Queries.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.common.lucene.search;
 
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.ExtendedCommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
@@ -148,8 +147,6 @@ public static Query applyMinimumShouldMatch(BooleanQuery query, @Nullable String
     public static Query maybeApplyMinimumShouldMatch(Query query, @Nullable String minimumShouldMatch) {
         if (query instanceof BooleanQuery) {
             return applyMinimumShouldMatch((BooleanQuery) query, minimumShouldMatch);
-        } else if (query instanceof ExtendedCommonTermsQuery) {
-            ((ExtendedCommonTermsQuery)query).setLowFreqMinimumNumberShouldMatch(minimumShouldMatch);
         }
         return query;
     }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java b/server/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java
index 1010c917eca82..b958b293183d2 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/search/XMoreLikeThis.java
@@ -97,7 +97,7 @@
  * above.  The frequency and length thresholds could be parameters, etc.
  * Doug
  * </pre>
- * <h3>Initial Usage</h3>
+ * <h2>Initial Usage</h2>
  * <p>
  * This class has lots of options to try to make it efficient and flexible.
  * The simplest possible usage is as follows. The bold
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java b/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java
index 889721a49e07a..94ba9abf58fde 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java
@@ -74,7 +74,9 @@ final class PerThreadIDVersionAndSeqNoLookup {
             // If a segment contains only no-ops, it does not have _uid but has both _soft_deletes and _tombstone fields.
             final NumericDocValues softDeletesDV = reader.getNumericDocValues(Lucene.SOFT_DELETES_FIELD);
             final NumericDocValues tombstoneDV = reader.getNumericDocValues(SeqNoFieldMapper.TOMBSTONE_NAME);
-            if (softDeletesDV == null || tombstoneDV == null) {
+            // this is a special case when we pruned away all IDs in a segment since all docs are deleted.
+            final boolean allDocsDeleted = (softDeletesDV != null && reader.numDocs() == 0);
+            if ((softDeletesDV == null || tombstoneDV == null) && allDocsDeleted == false) {
                 throw new IllegalArgumentException("reader does not have _uid terms but not a no-op segment; " +
                     "_soft_deletes [" + softDeletesDV + "], _tombstone [" + tombstoneDV + "]");
             }
@@ -100,38 +102,20 @@ public DocIdAndVersion lookupVersion(BytesRef id, boolean loadSeqNo, LeafReaderC
         throws IOException {
         assert context.reader().getCoreCacheHelper().getKey().equals(readerKey) :
             "context's reader is not the same as the reader class was initialized on.";
-        int docID = getDocID(id, context.reader().getLiveDocs());
+        int docID = getDocID(id, context);
 
         if (docID != DocIdSetIterator.NO_MORE_DOCS) {
-            final NumericDocValues versions = context.reader().getNumericDocValues(VersionFieldMapper.NAME);
-            if (versions == null) {
-                throw new IllegalArgumentException("reader misses the [" + VersionFieldMapper.NAME + "] field");
-            }
-            if (versions.advanceExact(docID) == false) {
-                throw new IllegalArgumentException("Document [" + docID + "] misses the [" + VersionFieldMapper.NAME + "] field");
-            }
             final long seqNo;
             final long term;
             if (loadSeqNo) {
-                NumericDocValues seqNos = context.reader().getNumericDocValues(SeqNoFieldMapper.NAME);
-                // remove the null check in 7.0 once we can't read indices with no seq#
-                if (seqNos != null && seqNos.advanceExact(docID)) {
-                    seqNo = seqNos.longValue();
-                } else {
-                    seqNo =  UNASSIGNED_SEQ_NO;
-                }
-                NumericDocValues terms = context.reader().getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME);
-                if (terms != null && terms.advanceExact(docID)) {
-                    term = terms.longValue();
-                } else {
-                    term = UNASSIGNED_PRIMARY_TERM;
-                }
-
+                seqNo = readNumericDocValues(context.reader(), SeqNoFieldMapper.NAME, docID);
+                term = readNumericDocValues(context.reader(), SeqNoFieldMapper.PRIMARY_TERM_NAME, docID);
             } else {
                 seqNo = UNASSIGNED_SEQ_NO;
                 term = UNASSIGNED_PRIMARY_TERM;
             }
-            return new DocIdAndVersion(docID, versions.longValue(), seqNo, term, context.reader(), context.docBase);
+            final long version = readNumericDocValues(context.reader(), VersionFieldMapper.NAME, docID);
+            return new DocIdAndVersion(docID, version, seqNo, term, context.reader(), context.docBase);
         } else {
             return null;
         }
@@ -141,9 +125,10 @@ public DocIdAndVersion lookupVersion(BytesRef id, boolean loadSeqNo, LeafReaderC
      * returns the internal lucene doc id for the given id bytes.
      * {@link DocIdSetIterator#NO_MORE_DOCS} is returned if not found
      * */
-    private int getDocID(BytesRef id, Bits liveDocs) throws IOException {
+    private int getDocID(BytesRef id, LeafReaderContext context) throws IOException {
         // termsEnum can possibly be null here if this leaf contains only no-ops.
         if (termsEnum != null && termsEnum.seekExact(id)) {
+            final Bits liveDocs = context.reader().getLiveDocs();
             int docID = DocIdSetIterator.NO_MORE_DOCS;
             // there may be more than one matching docID, in the case of nested docs, so we want the last one:
             docsEnum = termsEnum.postings(docsEnum, 0);
@@ -159,41 +144,23 @@ private int getDocID(BytesRef id, Bits liveDocs) throws IOException {
         }
     }
 
+    private static long readNumericDocValues(LeafReader reader, String field, int docId) throws IOException {
+        final NumericDocValues dv = reader.getNumericDocValues(field);
+        if (dv == null || dv.advanceExact(docId) == false) {
+            assert false : "document [" + docId + "] does not have docValues for [" + field + "]";
+            throw new IllegalStateException("document [" + docId + "] does not have docValues for [" + field + "]");
+        }
+        return dv.longValue();
+    }
+
     /** Return null if id is not found. */
     DocIdAndSeqNo lookupSeqNo(BytesRef id, LeafReaderContext context) throws IOException {
         assert context.reader().getCoreCacheHelper().getKey().equals(readerKey) :
             "context's reader is not the same as the reader class was initialized on.";
-        // termsEnum can possibly be null here if this leaf contains only no-ops.
-        if (termsEnum != null && termsEnum.seekExact(id)) {
-            docsEnum = termsEnum.postings(docsEnum, 0);
-            final Bits liveDocs = context.reader().getLiveDocs();
-            DocIdAndSeqNo result = null;
-            int docID = docsEnum.nextDoc();
-            if (docID != DocIdSetIterator.NO_MORE_DOCS) {
-                final NumericDocValues seqNoDV = context.reader().getNumericDocValues(SeqNoFieldMapper.NAME);
-                for (; docID != DocIdSetIterator.NO_MORE_DOCS; docID = docsEnum.nextDoc()) {
-                    final long seqNo;
-                    // remove the null check in 7.0 once we can't read indices with no seq#
-                    if (seqNoDV != null && seqNoDV.advanceExact(docID)) {
-                        seqNo = seqNoDV.longValue();
-                    } else {
-                        seqNo = UNASSIGNED_SEQ_NO;
-                    }
-                    final boolean isLive = (liveDocs == null || liveDocs.get(docID));
-                    if (isLive) {
-                        // The live document must always be the latest copy, thus we can early terminate here.
-                        // If a nested docs is live, we return the first doc which doesn't have term (only the last doc has term).
-                        // This should not be an issue since we no longer use primary term as tier breaker when comparing operations.
-                        assert result == null || result.seqNo <= seqNo :
-                            "the live doc does not have the highest seq_no; live_seq_no=" + seqNo + " < deleted_seq_no=" + result.seqNo;
-                        return new DocIdAndSeqNo(docID, seqNo, context, isLive);
-                    }
-                    if (result == null || result.seqNo < seqNo) {
-                        result = new DocIdAndSeqNo(docID, seqNo, context, isLive);
-                    }
-                }
-            }
-            return result;
+        final int docID = getDocID(id, context);
+        if (docID != DocIdSetIterator.NO_MORE_DOCS) {
+            final long seqNo = readNumericDocValues(context.reader(), SeqNoFieldMapper.NAME, docID);
+            return new DocIdAndSeqNo(docID, seqNo, context);
         } else {
             return null;
         }
diff --git a/server/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java b/server/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java
index 611887342adca..bfd859c115fe2 100644
--- a/server/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java
+++ b/server/src/main/java/org/elasticsearch/common/lucene/uid/VersionsAndSeqNoResolver.java
@@ -114,13 +114,11 @@ public static class DocIdAndSeqNo {
         public final int docId;
         public final long seqNo;
         public final LeafReaderContext context;
-        public final boolean isLive;
 
-        DocIdAndSeqNo(int docId, long seqNo, LeafReaderContext context, boolean isLive) {
+        DocIdAndSeqNo(int docId, long seqNo, LeafReaderContext context) {
             this.docId = docId;
             this.seqNo = seqNo;
             this.context = context;
-            this.isLive = isLive;
         }
     }
 
@@ -149,32 +147,21 @@ public static DocIdAndVersion loadDocIdAndVersion(IndexReader reader, Term term,
 
     /**
      * Loads the internal docId and sequence number of the latest copy for a given uid from the provided reader.
-     * The flag {@link DocIdAndSeqNo#isLive} indicates whether the returned document is live or (soft)deleted.
-     * This returns {@code null} if no such document matching the given term uid.
+     * The result is either null or the live and latest version of the given uid.
      */
     public static DocIdAndSeqNo loadDocIdAndSeqNo(IndexReader reader, Term term) throws IOException {
         final PerThreadIDVersionAndSeqNoLookup[] lookups = getLookupState(reader, term.field());
         final List<LeafReaderContext> leaves = reader.leaves();
-        DocIdAndSeqNo latest = null;
         // iterate backwards to optimize for the frequently updated documents
         // which are likely to be in the last segments
         for (int i = leaves.size() - 1; i >= 0; i--) {
             final LeafReaderContext leaf = leaves.get(i);
             final PerThreadIDVersionAndSeqNoLookup lookup = lookups[leaf.ord];
             final DocIdAndSeqNo result = lookup.lookupSeqNo(term.bytes(), leaf);
-            if (result == null) {
-                continue;
-            }
-            if (result.isLive) {
-                // The live document must always be the latest copy, thus we can early terminate here.
-                assert latest == null || latest.seqNo <= result.seqNo :
-                    "the live doc does not have the highest seq_no; live_seq_no=" + result.seqNo + " < deleted_seq_no=" + latest.seqNo;
+            if (result != null) {
                 return result;
             }
-            if (latest == null || latest.seqNo < result.seqNo) {
-                latest = result;
-            }
         }
-        return latest;
+        return null;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/server/src/main/java/org/elasticsearch/common/network/NetworkModule.java
index bbfae10991e84..8b363ebcc0b2f 100644
--- a/server/src/main/java/org/elasticsearch/common/network/NetworkModule.java
+++ b/server/src/main/java/org/elasticsearch/common/network/NetworkModule.java
@@ -75,7 +75,6 @@ public final class NetworkModule {
     public static final Setting<String> TRANSPORT_TYPE_SETTING = Setting.simpleString(TRANSPORT_TYPE_KEY, Property.NodeScope);
 
     private final Settings settings;
-    private final boolean transportClient;
 
     private static final List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
     private static final List<NamedXContentRegistry.Entry> namedXContents = new ArrayList<>();
@@ -106,9 +105,8 @@ public final class NetworkModule {
     /**
      * Creates a network module that custom networking classes can be plugged into.
      * @param settings The settings for the node
-     * @param transportClient True if only transport classes should be allowed to be registered, false otherwise.
      */
-    public NetworkModule(Settings settings, boolean transportClient, List<NetworkPlugin> plugins, ThreadPool threadPool,
+    public NetworkModule(Settings settings, List<NetworkPlugin> plugins, ThreadPool threadPool,
                          BigArrays bigArrays,
                          PageCacheRecycler pageCacheRecycler,
                          CircuitBreakerService circuitBreakerService,
@@ -116,14 +114,11 @@ public NetworkModule(Settings settings, boolean transportClient, List<NetworkPlu
                          NamedXContentRegistry xContentRegistry,
                          NetworkService networkService, HttpServerTransport.Dispatcher dispatcher) {
         this.settings = settings;
-        this.transportClient = transportClient;
         for (NetworkPlugin plugin : plugins) {
             Map<String, Supplier<HttpServerTransport>> httpTransportFactory = plugin.getHttpTransports(settings, threadPool, bigArrays,
                 pageCacheRecycler, circuitBreakerService, xContentRegistry, networkService, dispatcher);
-            if (transportClient == false) {
-                for (Map.Entry<String, Supplier<HttpServerTransport>> entry : httpTransportFactory.entrySet()) {
-                    registerHttpTransport(entry.getKey(), entry.getValue());
-                }
+            for (Map.Entry<String, Supplier<HttpServerTransport>> entry : httpTransportFactory.entrySet()) {
+                registerHttpTransport(entry.getKey(), entry.getValue());
             }
             Map<String, Supplier<Transport>> transportFactory = plugin.getTransports(settings, threadPool, pageCacheRecycler,
                 circuitBreakerService, namedWriteableRegistry, networkService);
@@ -138,10 +133,6 @@ public NetworkModule(Settings settings, boolean transportClient, List<NetworkPlu
         }
     }
 
-    public boolean isTransportClient() {
-        return transportClient;
-    }
-
     /** Adds a transport implementation that can be selected by setting {@link #TRANSPORT_TYPE_KEY}. */
     private void registerTransport(String key, Supplier<Transport> factory) {
         if (transportFactories.putIfAbsent(key, factory) != null) {
@@ -150,21 +141,17 @@ private void registerTransport(String key, Supplier<Transport> factory) {
     }
 
     /** Adds an http transport implementation that can be selected by setting {@link #HTTP_TYPE_KEY}. */
-    // TODO: we need another name than "http transport"....so confusing with transportClient...
     private void registerHttpTransport(String key, Supplier<HttpServerTransport> factory) {
-        if (transportClient) {
-            throw new IllegalArgumentException("Cannot register http transport " + key + " for transport client");
-        }
         if (transportHttpFactories.putIfAbsent(key, factory) != null) {
             throw new IllegalArgumentException("transport for name: " + key + " is already registered");
         }
     }
 
+    // TODO: consider moving this to the ClusterModule
+    // this lives here instead of the more aptly named ClusterModule because it used to be used by the Transport client
     /**
      * Register an allocation command.
      * <p>
-     * This lives here instead of the more aptly named ClusterModule because the Transport client needs these to be registered.
-     * </p>
      * @param reader the reader to read it from a stream
      * @param parser the parser to read it from XContent
      * @param commandName the names under which the command should be parsed. The {@link ParseField#getPreferredName()} is special because
diff --git a/server/src/main/java/org/elasticsearch/common/network/NetworkUtils.java b/server/src/main/java/org/elasticsearch/common/network/NetworkUtils.java
index 4e8f871ed3044..6a49f0d7658d4 100644
--- a/server/src/main/java/org/elasticsearch/common/network/NetworkUtils.java
+++ b/server/src/main/java/org/elasticsearch/common/network/NetworkUtils.java
@@ -200,10 +200,14 @@ static InetAddress[] getGlobalAddresses() throws IOException {
     static InetAddress[] getAllAddresses() throws IOException {
         return filterAllAddresses(address -> true, "no up-and-running addresses found");
     }
-    
+
+    static Optional<NetworkInterface> maybeGetInterfaceByName(List<NetworkInterface> networkInterfaces, String name) {
+        return networkInterfaces.stream().filter(netIf -> name.equals(netIf.getName())).findFirst();
+    }
+
     /** Returns addresses for the given interface (it must be marked up) */
     static InetAddress[] getAddressesForInterface(String name) throws SocketException {
-        Optional<NetworkInterface> networkInterface = getInterfaces().stream().filter((netIf) -> name.equals(netIf.getName())).findFirst();
+        Optional<NetworkInterface> networkInterface = maybeGetInterfaceByName(getInterfaces(), name);
 
         if (networkInterface.isPresent() == false) {
             throw new IllegalArgumentException("No interface named '" + name + "' found, got " + getInterfaces());
diff --git a/server/src/main/java/org/elasticsearch/common/regex/Regex.java b/server/src/main/java/org/elasticsearch/common/regex/Regex.java
index 1f4e465141222..8734ab1febe8b 100644
--- a/server/src/main/java/org/elasticsearch/common/regex/Regex.java
+++ b/server/src/main/java/org/elasticsearch/common/regex/Regex.java
@@ -88,7 +88,7 @@ public static boolean simpleMatch(String pattern, String str) {
         if (pattern == null || str == null) {
             return false;
         }
-        int firstIndex = pattern.indexOf('*');
+        final int firstIndex = pattern.indexOf('*');
         if (firstIndex == -1) {
             return pattern.equals(str);
         }
@@ -96,14 +96,15 @@ public static boolean simpleMatch(String pattern, String str) {
             if (pattern.length() == 1) {
                 return true;
             }
-            int nextIndex = pattern.indexOf('*', firstIndex + 1);
+            final int nextIndex = pattern.indexOf('*', firstIndex + 1);
             if (nextIndex == -1) {
-                return str.endsWith(pattern.substring(1));
+                // str.endsWith(pattern.substring(1)), but avoiding the construction of pattern.substring(1):
+                return str.regionMatches(str.length() - pattern.length() + 1, pattern, 1, pattern.length() - 1);
             } else if (nextIndex == 1) {
                 // Double wildcard "**" - skipping the first "*"
                 return simpleMatch(pattern.substring(1), str);
             }
-            String part = pattern.substring(1, nextIndex);
+            final String part = pattern.substring(1, nextIndex);
             int partIndex = str.indexOf(part);
             while (partIndex != -1) {
                 if (simpleMatch(pattern.substring(nextIndex), str.substring(partIndex + part.length()))) {
@@ -113,9 +114,9 @@ public static boolean simpleMatch(String pattern, String str) {
             }
             return false;
         }
-        return (str.length() >= firstIndex &&
-                pattern.substring(0, firstIndex).equals(str.substring(0, firstIndex)) &&
-                simpleMatch(pattern.substring(firstIndex), str.substring(firstIndex)));
+        return str.regionMatches(0, pattern, 0, firstIndex)
+            && (firstIndex == pattern.length() - 1 // only wildcard in pattern is at the end, so no need to look at the rest of the string
+                || simpleMatch(pattern.substring(firstIndex), str.substring(firstIndex)));
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index 026dfa4633991..867b628a5f97c 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.bootstrap.BootstrapSettings;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.InternalClusterInfoService;
@@ -113,12 +112,12 @@
  * Encapsulates all valid cluster level settings.
  */
 public final class ClusterSettings extends AbstractScopedSettings {
+
     public ClusterSettings(final Settings nodeSettings, final Set<Setting<?>> settingsSet) {
         this(nodeSettings, settingsSet, Collections.emptySet());
     }
 
-    public ClusterSettings(
-            final Settings nodeSettings, final Set<Setting<?>> settingsSet, final Set<SettingUpgrader<?>> settingUpgraders) {
+    public ClusterSettings(final Settings nodeSettings, final Set<Setting<?>> settingsSet, final Set<SettingUpgrader<?>> settingUpgraders) {
         super(nodeSettings, settingsSet, settingUpgraders, Property.NodeScope);
         addSettingsUpdater(new LoggingSettingUpdater(nodeSettings));
     }
@@ -176,11 +175,6 @@ public void apply(Settings value, Settings current, Settings previous) {
 
     public static Set<Setting<?>> BUILT_IN_CLUSTER_SETTINGS = Set.of(
             AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING,
-            // TODO these transport client settings are kind of odd here and should only be valid if we are a transport client
-            TransportClient.CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL,
-            TransportClient.CLIENT_TRANSPORT_PING_TIMEOUT,
-            TransportClient.CLIENT_TRANSPORT_IGNORE_CLUSTER_NAME,
-            TransportClient.CLIENT_TRANSPORT_SNIFF,
             AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING,
             BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING,
             BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING,
@@ -358,6 +352,7 @@ public void apply(Settings value, Settings current, Settings previous) {
             Node.INITIAL_STATE_TIMEOUT_SETTING,
             DiscoveryModule.DISCOVERY_TYPE_SETTING,
             DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING,
+            DiscoveryModule.ELECTION_STRATEGY_SETTING,
             SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING,
             SeedHostsResolver.DISCOVERY_SEED_RESOLVER_MAX_CONCURRENT_RESOLVERS_SETTING,
             SeedHostsResolver.DISCOVERY_SEED_RESOLVER_TIMEOUT_SETTING,
diff --git a/server/src/main/java/org/elasticsearch/common/settings/ConsistentSettingsService.java b/server/src/main/java/org/elasticsearch/common/settings/ConsistentSettingsService.java
new file mode 100644
index 0000000000000..411a470238638
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/settings/ConsistentSettingsService.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalNodeMasterListener;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.nio.charset.StandardCharsets;
+import java.security.NoSuchAlgorithmException;
+import java.security.spec.InvalidKeySpecException;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
+
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.PBEKeySpec;
+
+/**
+ * Used to publish secure setting hashes in the cluster state and to validate those hashes against the local values of those same settings.
+ * This is colloquially referred to as the secure setting consistency check. It will publish and verify hashes only for the collection
+ * of settings passed in the constructor. The settings have to have the {@link Setting.Property#Consistent} property. 
+ */
+public final class ConsistentSettingsService {
+    private static final Logger logger = LogManager.getLogger(ConsistentSettingsService.class);
+
+    private final Settings settings;
+    private final ClusterService clusterService;
+    private final Collection<Setting<?>> secureSettingsCollection;
+    private final SecretKeyFactory pbkdf2KeyFactory;
+
+    public ConsistentSettingsService(Settings settings, ClusterService clusterService,
+                                     Collection<Setting<?>> secureSettingsCollection) {
+        this.settings = settings;
+        this.clusterService = clusterService;
+        this.secureSettingsCollection = secureSettingsCollection;
+        // this is used to compute the PBKDF2 hash (the published one)
+        try {
+            this.pbkdf2KeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA512");
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException("The \"PBKDF2WithHmacSHA512\" algorithm is required for consistent secure settings' hashes", e);
+        }
+    }
+
+    /**
+     * Returns a {@link LocalNodeMasterListener} that will publish hashes of all the settings passed in the constructor. These hashes are
+     * published by the master node only. Note that this is not designed for {@link SecureSettings} implementations that are mutable.
+     */
+    public LocalNodeMasterListener newHashPublisher() {
+        // eagerly compute hashes to be published
+        final Map<String, String> computedHashesOfConsistentSettings = computeHashesOfConsistentSecureSettings();
+        return new HashesPublisher(computedHashesOfConsistentSettings, clusterService);
+    }
+
+    /**
+     * Verifies that the hashes of consistent secure settings in the latest {@code ClusterState} verify for the values of those same
+     * settings on the local node. The settings to be checked are passed in the constructor. Also, validates that a missing local
+     * value is also missing in the published set, and vice-versa.  
+     */
+    public boolean areAllConsistent() {
+        final ClusterState state = clusterService.state();
+        final Map<String, String> publishedHashesOfConsistentSettings = state.metaData().hashesOfConsistentSettings();
+        final Set<String> publishedSettingKeysToVerify = new HashSet<>();
+        publishedSettingKeysToVerify.addAll(publishedHashesOfConsistentSettings.keySet());
+        final AtomicBoolean allConsistent = new AtomicBoolean(true);
+        forEachConcreteSecureSettingDo(concreteSecureSetting -> {
+            final String publishedSaltAndHash = publishedHashesOfConsistentSettings.get(concreteSecureSetting.getKey());
+            final byte[] localHash = concreteSecureSetting.getSecretDigest(settings);
+            if (publishedSaltAndHash == null && localHash == null) {
+                // consistency of missing
+                logger.debug("no published hash for the consistent secure setting [{}] but it also does NOT exist on the local node",
+                        concreteSecureSetting.getKey());
+            } else if (publishedSaltAndHash == null && localHash != null) {
+                // setting missing on master but present locally
+                logger.warn("no published hash for the consistent secure setting [{}] but it exists on the local node",
+                        concreteSecureSetting.getKey());
+                if (state.nodes().isLocalNodeElectedMaster()) {
+                    throw new IllegalStateException("Master node cannot validate consistent setting. No published hash for ["
+                            + concreteSecureSetting.getKey() + "] but setting exists.");
+                }
+                allConsistent.set(false);
+            } else if (publishedSaltAndHash != null && localHash == null) {
+                // setting missing locally but present on master
+                logger.warn("the consistent secure setting [{}] does not exist on the local node but there is a published hash for it",
+                        concreteSecureSetting.getKey());
+                allConsistent.set(false);
+            } else {
+                assert publishedSaltAndHash != null;
+                assert localHash != null;
+                final String[] parts = publishedSaltAndHash.split(":");
+                if (parts == null || parts.length != 2) {
+                    throw new IllegalArgumentException("published hash [" + publishedSaltAndHash + " ] for secure setting ["
+                            + concreteSecureSetting.getKey() + "] is invalid");
+                }
+                final String publishedSalt = parts[0];
+                final String publishedHash = parts[1];
+                final byte[] computedSaltedHashBytes = computeSaltedPBKDF2Hash(localHash, publishedSalt.getBytes(StandardCharsets.UTF_8));
+                final String computedSaltedHash = new String(Base64.getEncoder().encode(computedSaltedHashBytes), StandardCharsets.UTF_8);
+                if (false == publishedHash.equals(computedSaltedHash)) {
+                    logger.warn("the published hash [{}] of the consistent secure setting [{}] differs from the locally computed one [{}]",
+                            publishedHash, concreteSecureSetting.getKey(), computedSaltedHash);
+                    if (state.nodes().isLocalNodeElectedMaster()) {
+                        throw new IllegalStateException("Master node cannot validate consistent setting. The published hash ["
+                                + publishedHash + "] of the consistent secure setting [" + concreteSecureSetting.getKey()
+                                + "] differs from the locally computed one [" + computedSaltedHash + "].");
+                    }
+                    allConsistent.set(false);
+                }
+            }
+            publishedSettingKeysToVerify.remove(concreteSecureSetting.getKey());
+        });
+        // another case of settings missing locally, when group settings have not expanded to all the keys published
+        for (String publishedSettingKey : publishedSettingKeysToVerify) {
+            for (Setting<?> setting : secureSettingsCollection) {
+                if (setting.match(publishedSettingKey)) {
+                    // setting missing locally but present on master
+                    logger.warn("the consistent secure setting [{}] does not exist on the local node but there is a published hash for it",
+                            publishedSettingKey);
+                    allConsistent.set(false);
+                }
+            }
+        }
+        return allConsistent.get();
+    }
+
+    /**
+     * Iterate over the passed in secure settings, expanding {@link Setting.AffixSetting} to concrete settings, in the scope of the local
+     * settings.
+     */
+    private void forEachConcreteSecureSettingDo(Consumer<SecureSetting<?>> secureSettingConsumer) {
+        for (Setting<?> setting : secureSettingsCollection) {
+            assert setting.isConsistent() : "[" + setting.getKey() + "] is not a consistent setting";
+            if (setting instanceof Setting.AffixSetting<?>) {
+                ((Setting.AffixSetting<?>)setting).getAllConcreteSettings(settings).forEach(concreteSetting -> {
+                    assert concreteSetting instanceof SecureSetting<?> : "[" + concreteSetting.getKey() + "] is not a secure setting";
+                    secureSettingConsumer.accept((SecureSetting<?>)concreteSetting);
+                });
+            } else if (setting instanceof SecureSetting<?>) {
+                secureSettingConsumer.accept((SecureSetting<?>) setting);
+            } else {
+                assert false : "Unrecognized consistent secure setting [" + setting.getKey() + "]";
+            }
+        }
+    }
+
+    private Map<String, String> computeHashesOfConsistentSecureSettings() {
+        final Map<String, String> hashesBySettingKey = new HashMap<>();
+        forEachConcreteSecureSettingDo(concreteSecureSetting -> {
+            final byte[] localHash = concreteSecureSetting.getSecretDigest(settings);
+            if (localHash != null) {
+                final String salt = UUIDs.randomBase64UUID();
+                final byte[] publicHash = computeSaltedPBKDF2Hash(localHash, salt.getBytes(StandardCharsets.UTF_8));
+                final String encodedPublicHash = new String(Base64.getEncoder().encode(publicHash), StandardCharsets.UTF_8);
+                hashesBySettingKey.put(concreteSecureSetting.getKey(), salt + ":" + encodedPublicHash);
+            }
+        });
+        return hashesBySettingKey;
+    }
+
+    private byte[] computeSaltedPBKDF2Hash(byte[] bytes, byte[] salt) {
+        final int iterations = 5000;
+        final int keyLength = 512;
+        char[] value = null;
+        try {
+            value = MessageDigests.toHexCharArray(bytes);
+            final PBEKeySpec spec = new PBEKeySpec(value, salt, iterations, keyLength);
+            final SecretKey key = pbkdf2KeyFactory.generateSecret(spec);
+            return key.getEncoded();
+        } catch (InvalidKeySpecException e) {
+            throw new RuntimeException("Unexpected exception when computing PBKDF2 hash", e);
+        } finally {
+            if (value != null) {
+                Arrays.fill(value, '0');
+            }
+        }
+    }
+
+    static final class HashesPublisher implements LocalNodeMasterListener {
+
+        // eagerly compute hashes to be published
+        final Map<String, String> computedHashesOfConsistentSettings;
+        final ClusterService clusterService;
+
+        HashesPublisher(Map<String, String> computedHashesOfConsistentSettings, ClusterService clusterService) {
+            this.computedHashesOfConsistentSettings = Map.copyOf(computedHashesOfConsistentSettings);
+            this.clusterService = clusterService;
+        }
+
+        @Override
+        public void onMaster() {
+            clusterService.submitStateUpdateTask("publish-secure-settings-hashes", new ClusterStateUpdateTask(Priority.URGENT) {
+                @Override
+                public ClusterState execute(ClusterState currentState) {
+                    final Map<String, String> publishedHashesOfConsistentSettings = currentState.metaData()
+                            .hashesOfConsistentSettings();
+                    if (computedHashesOfConsistentSettings.equals(publishedHashesOfConsistentSettings)) {
+                        logger.debug("Nothing to publish. What is already published matches this node's view.");
+                        return currentState;
+                    } else {
+                        return ClusterState.builder(currentState).metaData(MetaData.builder(currentState.metaData())
+                                .hashesOfConsistentSettings(computedHashesOfConsistentSettings)).build();
+                    }
+                }
+
+                @Override
+                public void onFailure(String source, Exception e) {
+                    logger.error("unable to publish secure settings hashes", e);
+                }
+
+            });
+        }
+
+        @Override
+        public void offMaster() {
+            logger.trace("I am no longer master, nothing to do");
+        }
+
+        @Override
+        public String executorName() {
+            return ThreadPool.Names.SAME;
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
index 64cdd7165f2a3..7ad69c1eebe0c 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.hash.MessageDigests;
 
 import javax.crypto.Cipher;
 import javax.crypto.CipherInputStream;
@@ -85,6 +86,17 @@ private enum EntryType {
         FILE
     }
 
+    /** An entry in the keystore. The bytes are opaque and interpreted based on the entry type. */
+    private static class Entry {
+        final byte[] bytes;
+        final byte[] sha256Digest;
+
+        Entry(byte[] bytes) {
+            this.bytes = bytes;
+            this.sha256Digest = MessageDigests.sha256().digest(bytes);
+        }
+    }
+
     /**
      * A regex for the valid characters that a setting name in the keystore may use.
      */
@@ -148,7 +160,7 @@ private enum EntryType {
     private final byte[] dataBytes;
 
     /** The decrypted secret data. See {@link #decrypt(char[])}. */
-    private final SetOnce<Map<String, byte[]>> entries = new SetOnce<>();
+    private final SetOnce<Map<String, Entry>> entries = new SetOnce<>();
     private volatile boolean closed;
 
     private KeyStoreWrapper(int formatVersion, boolean hasPassword, byte[] dataBytes) {
@@ -350,7 +362,7 @@ public void decrypt(char[] password) throws GeneralSecurityException, IOExceptio
                 int entrySize = input.readInt();
                 byte[] entryBytes = new byte[entrySize];
                 input.readFully(entryBytes);
-                entries.get().put(setting, entryBytes);
+                entries.get().put(setting, new Entry(entryBytes));
             }
             if (input.read() != -1) {
                 throw new SecurityException("Keystore has been corrupted or tampered with");
@@ -369,11 +381,11 @@ private byte[] encrypt(char[] password, byte[] salt, byte[] iv) throws GeneralSe
         try (CipherOutputStream cipherStream = new CipherOutputStream(bytes, cipher);
              DataOutputStream output = new DataOutputStream(cipherStream)) {
             output.writeInt(entries.get().size());
-            for (Map.Entry<String, byte[]> mapEntry : entries.get().entrySet()) {
+            for (Map.Entry<String, Entry> mapEntry : entries.get().entrySet()) {
                 output.writeUTF(mapEntry.getKey());
-                byte[] entry = mapEntry.getValue();
-                output.writeInt(entry.length);
-                output.write(entry);
+                byte[] entryBytes = mapEntry.getValue().bytes;
+                output.writeInt(entryBytes.length);
+                output.write(entryBytes);
             }
         }
         return bytes.toByteArray();
@@ -448,7 +460,7 @@ private void decryptLegacyEntries() throws GeneralSecurityException, IOException
             }
             Arrays.fill(chars, '\0');
 
-            entries.get().put(setting, bytes);
+            entries.get().put(setting, new Entry(bytes));
         }
     }
 
@@ -521,8 +533,8 @@ public Set<String> getSettingNames() {
     @Override
     public synchronized SecureString getString(String setting) {
         ensureOpen();
-        byte[] entry = entries.get().get(setting);
-        ByteBuffer byteBuffer = ByteBuffer.wrap(entry);
+        Entry entry = entries.get().get(setting);
+        ByteBuffer byteBuffer = ByteBuffer.wrap(entry.bytes);
         CharBuffer charBuffer = StandardCharsets.UTF_8.decode(byteBuffer);
         return new SecureString(Arrays.copyOfRange(charBuffer.array(), charBuffer.position(), charBuffer.limit()));
     }
@@ -530,8 +542,19 @@ public synchronized SecureString getString(String setting) {
     @Override
     public synchronized InputStream getFile(String setting) {
         ensureOpen();
-        byte[] entry = entries.get().get(setting);
-        return new ByteArrayInputStream(entry);
+        Entry entry = entries.get().get(setting);
+        return new ByteArrayInputStream(entry.bytes);
+    }
+
+    /**
+     * Returns the SHA256 digest for the setting's value, even after {@code #close()} has been called. The setting must exist. The digest is
+     * used to check for value changes without actually storing the value.
+     */
+    @Override
+    public byte[] getSHA256Digest(String setting) {
+        assert entries.get() != null : "Keystore is not loaded";
+        Entry entry = entries.get().get(setting);
+        return entry.sha256Digest;
     }
 
     /**
@@ -553,9 +576,9 @@ synchronized void setString(String setting, char[] value) {
 
         ByteBuffer byteBuffer = StandardCharsets.UTF_8.encode(CharBuffer.wrap(value));
         byte[] bytes = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.position(), byteBuffer.limit());
-        byte[] oldEntry = entries.get().put(setting, bytes);
+        Entry oldEntry = entries.get().put(setting, new Entry(bytes));
         if (oldEntry != null) {
-            Arrays.fill(oldEntry, (byte)0);
+            Arrays.fill(oldEntry.bytes, (byte)0);
         }
     }
 
@@ -564,18 +587,18 @@ synchronized void setFile(String setting, byte[] bytes) {
         ensureOpen();
         validateSettingName(setting);
 
-        byte[] oldEntry = entries.get().put(setting, Arrays.copyOf(bytes, bytes.length));
+        Entry oldEntry = entries.get().put(setting, new Entry(Arrays.copyOf(bytes, bytes.length)));
         if (oldEntry != null) {
-            Arrays.fill(oldEntry, (byte)0);
+            Arrays.fill(oldEntry.bytes, (byte)0);
         }
     }
 
     /** Remove the given setting from the keystore. */
     void remove(String setting) {
         ensureOpen();
-        byte[] oldEntry = entries.get().remove(setting);
+        Entry oldEntry = entries.get().remove(setting);
         if (oldEntry != null) {
-            Arrays.fill(oldEntry, (byte)0);
+            Arrays.fill(oldEntry.bytes, (byte)0);
         }
     }
 
@@ -590,8 +613,8 @@ private void ensureOpen() {
     public synchronized void close() {
         this.closed = true;
         if (null != entries.get() && entries.get().isEmpty() == false) {
-            for (byte[] entry : entries.get().values()) {
-                Arrays.fill(entry, (byte) 0);
+            for (Entry entry : entries.get().values()) {
+                Arrays.fill(entry.bytes, (byte) 0);
             }
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java b/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java
index 33f4718aa45e4..e022e4e3760a5 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java
@@ -37,7 +37,7 @@ public abstract class SecureSetting<T> extends Setting<T> {
     /** Determines whether legacy settings with sensitive values should be allowed. */
     private static final boolean ALLOW_INSECURE_SETTINGS = Booleans.parseBoolean(System.getProperty("es.allow_insecure_settings", "false"));
 
-    private static final Set<Property> ALLOWED_PROPERTIES = EnumSet.of(Property.Deprecated);
+    private static final Set<Property> ALLOWED_PROPERTIES = EnumSet.of(Property.Deprecated, Property.Consistent);
 
     private static final Property[] FIXED_PROPERTIES = {
         Property.NodeScope
@@ -97,6 +97,23 @@ public T get(Settings settings) {
         }
     }
 
+    /**
+     * Returns the digest of this secure setting's value or {@code null} if the setting is missing (inside the keystore). This method can be
+     * called even after the {@code SecureSettings} have been closed, unlike {@code #get(Settings)}. The digest is used to check for changes
+     * of the value (by re-reading the {@code SecureSettings}), without actually transmitting the value to compare with.
+     */
+    public byte[] getSecretDigest(Settings settings) {
+        final SecureSettings secureSettings = settings.getSecureSettings();
+        if (secureSettings == null || false == secureSettings.getSettingNames().contains(getKey())) {
+            return null;
+        }
+        try {
+            return secureSettings.getSHA256Digest(getKey());
+        } catch (GeneralSecurityException e) {
+            throw new RuntimeException("failed to read secure setting " + getKey(), e);
+        }
+    }
+
     /** Returns the secret setting from the keyStoreReader store. */
     abstract T getSecret(SecureSettings secureSettings) throws GeneralSecurityException;
 
diff --git a/server/src/main/java/org/elasticsearch/common/settings/SecureSettings.java b/server/src/main/java/org/elasticsearch/common/settings/SecureSettings.java
index 98f980c1ec6c8..7f92b382dd7b1 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/SecureSettings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/SecureSettings.java
@@ -42,6 +42,8 @@ public interface SecureSettings extends Closeable {
     /** Return a file setting. The {@link InputStream} should be closed once it is used. */
     InputStream getFile(String setting) throws GeneralSecurityException;
 
+    byte[] getSHA256Digest(String setting) throws GeneralSecurityException;
+
     @Override
     void close() throws IOException;
 }
diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java
index 1e5079124c345..d4164b474de0b 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java
@@ -112,6 +112,11 @@ public enum Property {
          */
         NodeScope,
 
+        /**
+         * Secure setting values equal on all nodes
+         */
+        Consistent,
+
         /**
          * Index scope
          */
@@ -167,6 +172,7 @@ private Setting(Key key, @Nullable Setting<T> fallbackSetting, Function<Settings
             checkPropertyRequiresIndexScope(propertiesAsSet, Property.NotCopyableOnResize);
             checkPropertyRequiresIndexScope(propertiesAsSet, Property.InternalIndex);
             checkPropertyRequiresIndexScope(propertiesAsSet, Property.PrivateIndex);
+            checkPropertyRequiresNodeScope(propertiesAsSet, Property.Consistent);
             this.properties = propertiesAsSet;
         }
     }
@@ -177,6 +183,12 @@ private void checkPropertyRequiresIndexScope(final EnumSet<Property> properties,
         }
     }
 
+    private void checkPropertyRequiresNodeScope(final EnumSet<Property> properties, final Property property) {
+        if (properties.contains(property) && properties.contains(Property.NodeScope) == false) {
+            throw new IllegalArgumentException("non-node-scoped setting [" + key + "] can not have property [" + property + "]");
+        }
+    }
+
     /**
      * Creates a new Setting instance
      * @param key the settings key for this setting.
@@ -321,6 +333,14 @@ public boolean hasNodeScope() {
         return properties.contains(Property.NodeScope);
     }
 
+    /**
+     * Returns <code>true</code> if this setting's value can be checked for equality across all nodes. Only {@link SecureSetting} instances
+     * may have this qualifier.
+     */
+    public boolean isConsistent() {
+        return properties.contains(Property.Consistent);
+    }
+
     /**
      * Returns <code>true</code> if this setting has an index scope, otherwise <code>false</code>
      */
diff --git a/server/src/main/java/org/elasticsearch/common/settings/Settings.java b/server/src/main/java/org/elasticsearch/common/settings/Settings.java
index 5789abf76d8b0..e92d53f801183 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/Settings.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/Settings.java
@@ -1324,15 +1324,20 @@ public Set<String> getSettingNames() {
         }
 
         @Override
-        public SecureString getString(String setting) throws GeneralSecurityException{
+        public SecureString getString(String setting) throws GeneralSecurityException {
             return delegate.getString(addPrefix.apply(setting));
         }
 
         @Override
-        public InputStream getFile(String setting) throws GeneralSecurityException{
+        public InputStream getFile(String setting) throws GeneralSecurityException {
             return delegate.getFile(addPrefix.apply(setting));
         }
 
+        @Override
+        public byte[] getSHA256Digest(String setting) throws GeneralSecurityException {
+            return delegate.getSHA256Digest(addPrefix.apply(setting));
+        }
+
         @Override
         public void close() throws IOException {
             delegate.close();
diff --git a/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java b/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
index 6a78e81d7f3f4..58c9cbc520456 100644
--- a/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
+++ b/server/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
@@ -49,6 +49,7 @@ public class SettingsModule implements Module {
     private final Set<String> settingsFilterPattern = new HashSet<>();
     private final Map<String, Setting<?>> nodeSettings = new HashMap<>();
     private final Map<String, Setting<?>> indexSettings = new HashMap<>();
+    private final Set<Setting<?>> consistentSettings = new HashSet<>();
     private final IndexScopedSettings indexScopedSettings;
     private final ClusterSettings clusterSettings;
     private final SettingsFilter settingsFilter;
@@ -157,7 +158,6 @@ public void configure(Binder binder) {
         binder.bind(IndexScopedSettings.class).toInstance(indexScopedSettings);
     }
 
-
     /**
      * Registers a new setting. This method should be used by plugins in order to expose any custom settings the plugin defines.
      * Unless a setting is registered the setting is unusable. If a setting is never the less specified the node will reject
@@ -175,6 +175,19 @@ private void registerSetting(Setting<?> setting) {
                 if (existingSetting != null) {
                     throw new IllegalArgumentException("Cannot register setting [" + setting.getKey() + "] twice");
                 }
+                if (setting.isConsistent()) {
+                    if (setting instanceof Setting.AffixSetting<?>) {
+                        if (((Setting.AffixSetting<?>)setting).getConcreteSettingForNamespace("_na_") instanceof SecureSetting<?>) {
+                            consistentSettings.add(setting);
+                        } else {
+                            throw new IllegalArgumentException("Invalid consistent secure setting [" + setting.getKey() + "]");
+                        }
+                    } else if (setting instanceof SecureSetting<?>) {
+                        consistentSettings.add(setting);
+                    } else {
+                        throw new IllegalArgumentException("Invalid consistent secure setting [" + setting.getKey() + "]");
+                    }
+                }
                 nodeSettings.put(setting.getKey(), setting);
             }
             if (setting.hasIndexScope()) {
@@ -182,6 +195,9 @@ private void registerSetting(Setting<?> setting) {
                 if (existingSetting != null) {
                     throw new IllegalArgumentException("Cannot register setting [" + setting.getKey() + "] twice");
                 }
+                if (setting.isConsistent()) {
+                    throw new IllegalStateException("Consistent setting [" + setting.getKey() + "] cannot be index scoped");
+                }
                 indexSettings.put(setting.getKey(), setting);
             }
         } else {
@@ -215,6 +231,10 @@ public ClusterSettings getClusterSettings() {
         return clusterSettings;
     }
 
+    public Set<Setting<?>> getConsistentSettings() {
+        return consistentSettings;
+    }
+
     public SettingsFilter getSettingsFilter() {
         return settingsFilter;
     }
diff --git a/server/src/main/java/org/elasticsearch/common/time/DateMathParser.java b/server/src/main/java/org/elasticsearch/common/time/DateMathParser.java
index 3ba392822ca0c..fc2d231bb2f6c 100644
--- a/server/src/main/java/org/elasticsearch/common/time/DateMathParser.java
+++ b/server/src/main/java/org/elasticsearch/common/time/DateMathParser.java
@@ -31,7 +31,7 @@
 public interface DateMathParser {
 
     /**
-     * Parse a date math expression without timzeone info and rounding down.
+     * Parse a date math expression without timezone info and rounding down.
      */
     default Instant parse(String text, LongSupplier now) {
         return parse(text, now, false, (ZoneId) null);
@@ -43,8 +43,8 @@ default Instant parse(String text, LongSupplier now) {
 
     // exists for backcompat, do not use!
     @Deprecated
-    default Instant parse(String text, LongSupplier now, boolean roundUp, DateTimeZone tz) {
-        return parse(text, now, roundUp, tz == null ? null : ZoneId.of(tz.getID()));
+    default Instant parse(String text, LongSupplier now, boolean roundUpProperty, DateTimeZone tz) {
+        return parse(text, now, roundUpProperty, tz == null ? null : ZoneId.of(tz.getID()));
     }
 
     /**
@@ -65,11 +65,11 @@ default Instant parse(String text, LongSupplier now, boolean roundUp, DateTimeZo
      * s    second
      *
      *
-     * @param text      the input
-     * @param now       a supplier to retrieve the current date in milliseconds, if needed for additions
-     * @param roundUp   should the result be rounded up
-     * @param tz        an optional timezone that should be applied before returning the milliseconds since the epoch
-     * @return          the parsed date as an Instant since the epoch
+     * @param text              the input
+     * @param now               a supplier to retrieve the current date in milliseconds, if needed for additions
+     * @param roundUpProperty   should the result be rounded up with the granularity of the rounding (e.g. <code>now/M</code>)
+     * @param tz                an optional timezone that should be applied before returning the milliseconds since the epoch
+     * @return                  the parsed date as an Instant since the epoch
      */
-    Instant parse(String text, LongSupplier now, boolean roundUp, ZoneId tz);
+    Instant parse(String text, LongSupplier now, boolean roundUpProperty, ZoneId tz);
 }
diff --git a/server/src/main/java/org/elasticsearch/common/time/JavaDateFormatter.java b/server/src/main/java/org/elasticsearch/common/time/JavaDateFormatter.java
index d0f4200b3bafe..3513600582278 100644
--- a/server/src/main/java/org/elasticsearch/common/time/JavaDateFormatter.java
+++ b/server/src/main/java/org/elasticsearch/common/time/JavaDateFormatter.java
@@ -38,6 +38,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.Consumer;
+import java.util.stream.Collectors;
 
 class JavaDateFormatter implements DateFormatter {
 
@@ -58,6 +59,12 @@ class JavaDateFormatter implements DateFormatter {
     private final List<DateTimeFormatter> parsers;
     private final DateTimeFormatter roundupParser;
 
+    private JavaDateFormatter(String format, DateTimeFormatter printer, DateTimeFormatter roundupParser, List<DateTimeFormatter> parsers) {
+        this.format = format;
+        this.printer = printer;
+        this.roundupParser = roundupParser;
+        this.parsers = parsers;
+    }
     JavaDateFormatter(String format, DateTimeFormatter printer, DateTimeFormatter... parsers) {
         this(format, printer, builder -> ROUND_UP_BASE_FIELDS.forEach(builder::parseDefaulting), parsers);
     }
@@ -155,9 +162,8 @@ public DateFormatter withZone(ZoneId zoneId) {
         if (zoneId.equals(zone())) {
             return this;
         }
-
-        return new JavaDateFormatter(format, printer.withZone(zoneId),
-            parsers.stream().map(p -> p.withZone(zoneId)).toArray(size -> new DateTimeFormatter[size]));
+        return new JavaDateFormatter(format, printer.withZone(zoneId), getRoundupParser().withZone(zoneId),
+            parsers.stream().map(p -> p.withZone(zoneId)).collect(Collectors.toList()));
     }
 
     @Override
@@ -166,9 +172,8 @@ public DateFormatter withLocale(Locale locale) {
         if (locale.equals(locale())) {
             return this;
         }
-
-        return new JavaDateFormatter(format, printer.withLocale(locale),
-            parsers.stream().map(p -> p.withLocale(locale)).toArray(size -> new DateTimeFormatter[size]));
+        return new JavaDateFormatter(format, printer.withLocale(locale), getRoundupParser().withLocale(locale),
+            parsers.stream().map(p -> p.withLocale(locale)).collect(Collectors.toList()));
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/common/time/JavaDateMathParser.java b/server/src/main/java/org/elasticsearch/common/time/JavaDateMathParser.java
index dc7c195e2fd6c..78d4f10d87cbf 100644
--- a/server/src/main/java/org/elasticsearch/common/time/JavaDateMathParser.java
+++ b/server/src/main/java/org/elasticsearch/common/time/JavaDateMathParser.java
@@ -59,7 +59,7 @@ public class JavaDateMathParser implements DateMathParser {
     }
 
     @Override
-    public Instant parse(String text, LongSupplier now, boolean roundUp, ZoneId timeZone) {
+    public Instant parse(String text, LongSupplier now, boolean roundUpProperty, ZoneId timeZone) {
         Instant time;
         String mathString;
         if (text.startsWith("now")) {
@@ -73,16 +73,16 @@ public Instant parse(String text, LongSupplier now, boolean roundUp, ZoneId time
         } else {
             int index = text.indexOf("||");
             if (index == -1) {
-                return parseDateTime(text, timeZone, roundUp);
+                return parseDateTime(text, timeZone, roundUpProperty);
             }
             time = parseDateTime(text.substring(0, index), timeZone, false);
             mathString = text.substring(index + 2);
         }
 
-        return parseMath(mathString, time, roundUp, timeZone);
+        return parseMath(mathString, time, roundUpProperty, timeZone);
     }
 
-    private Instant parseMath(final String mathString, final Instant time, final boolean roundUp,
+    private Instant parseMath(final String mathString, final Instant time, final boolean roundUpProperty,
                            ZoneId timeZone) throws ElasticsearchParseException {
         if (timeZone == null) {
             timeZone = ZoneOffset.UTC;
@@ -133,78 +133,79 @@ private Instant parseMath(final String mathString, final Instant time, final boo
                 case 'y':
                     if (round) {
                         dateTime = dateTime.withDayOfYear(1).with(LocalTime.MIN);
+                        if (roundUpProperty) {
+                            dateTime = dateTime.plusYears(1);
+                        }
                     } else {
                         dateTime = dateTime.plusYears(sign * num);
                     }
-                    if (roundUp) {
-                        dateTime = dateTime.plusYears(1);
-                    }
                     break;
                 case 'M':
                     if (round) {
                         dateTime = dateTime.withDayOfMonth(1).with(LocalTime.MIN);
+                        if (roundUpProperty) {
+                            dateTime = dateTime.plusMonths(1);
+                        }
                     } else {
                         dateTime = dateTime.plusMonths(sign * num);
                     }
-                    if (roundUp) {
-                        dateTime = dateTime.plusMonths(1);
-                    }
                     break;
                 case 'w':
                     if (round) {
                         dateTime = dateTime.with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)).with(LocalTime.MIN);
+                        if (roundUpProperty) {
+                            dateTime = dateTime.plusWeeks(1);
+                        }
                     } else {
                         dateTime = dateTime.plusWeeks(sign * num);
                     }
-                    if (roundUp) {
-                        dateTime = dateTime.plusWeeks(1);
-                    }
                     break;
                 case 'd':
                     if (round) {
                         dateTime = dateTime.with(LocalTime.MIN);
+                        if (roundUpProperty) {
+                            dateTime = dateTime.plusDays(1);
+                        }
                     } else {
                         dateTime = dateTime.plusDays(sign * num);
                     }
-                    if (roundUp) {
-                        dateTime = dateTime.plusDays(1);
-                    }
                     break;
                 case 'h':
                 case 'H':
                     if (round) {
                         dateTime = dateTime.withMinute(0).withSecond(0).withNano(0);
+                        if (roundUpProperty) {
+                            dateTime = dateTime.plusHours(1);
+                        }
                     } else {
                         dateTime = dateTime.plusHours(sign * num);
                     }
-                    if (roundUp) {
-                        dateTime = dateTime.plusHours(1);
-                    }
                     break;
                 case 'm':
                     if (round) {
                         dateTime = dateTime.withSecond(0).withNano(0);
+                        if (roundUpProperty) {
+                            dateTime = dateTime.plusMinutes(1);
+                        }
                     } else {
                         dateTime = dateTime.plusMinutes(sign * num);
                     }
-                    if (roundUp) {
-                        dateTime = dateTime.plusMinutes(1);
-                    }
                     break;
                 case 's':
                     if (round) {
                         dateTime = dateTime.withNano(0);
+                        if (roundUpProperty) {
+                            dateTime = dateTime.plusSeconds(1);
+                        }
                     } else {
                         dateTime = dateTime.plusSeconds(sign * num);
                     }
-                    if (roundUp) {
-                        dateTime = dateTime.plusSeconds(1);
-                    }
                     break;
                 default:
                     throw new ElasticsearchParseException("unit [{}] not supported for date math [{}]", unit, mathString);
             }
-            if (roundUp) {
+            if (round && roundUpProperty) {
+                // subtract 1 millisecond to get the largest inclusive value
                 dateTime = dateTime.minus(1, ChronoField.MILLI_OF_SECOND.getBaseUnit());
             }
         }
diff --git a/server/src/main/java/org/elasticsearch/common/util/CuckooFilter.java b/server/src/main/java/org/elasticsearch/common/util/CuckooFilter.java
new file mode 100644
index 0000000000000..54099735fba47
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/util/CuckooFilter.java
@@ -0,0 +1,521 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.util;
+
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.packed.PackedInts;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Objects;
+import java.util.Random;
+
+/**
+ * An approximate set membership datastructure
+ *
+ * CuckooFilters are similar to Bloom Filters in usage; values are inserted, and the Cuckoo
+ * can be asked if it has seen a particular value before.  Because the structure is approximate,
+ * it can return false positives (says it has seen an item when it has not).  False negatives
+ * are not possible though; if the structure says it _has not_ seen an item, that can be
+ * trusted.
+ *
+ * The filter can "saturate" at which point the map has hit it's configured load factor (or near enough
+ * that a large number of evictions are not able to find a free slot) and will refuse to accept
+ * any new insertions.
+ *
+ * NOTE: this version does not support deletions, and as such does not save duplicate
+ * fingerprints (e.g. when inserting, if the fingerprint is already present in the
+ * candidate buckets, it is not inserted).  By not saving duplicates, the CuckooFilter
+ * loses the ability to delete values.  But not by allowing deletions, we can save space
+ * (do not need to waste slots on duplicate fingerprints), and we do not need to worry
+ * about inserts "overflowing" a bucket because the same item has been repeated repeatedly
+ *
+ * NOTE: this CuckooFilter exposes a number of Expert APIs which assume the caller has
+ * intimate knowledge about how the algorithm works.  It is recommended to use
+ * {@link SetBackedScalingCuckooFilter} instead.
+ *
+ * Based on the paper:
+ *
+ * Fan, Bin, et al. "Cuckoo filter: Practically better than bloom."
+ * Proceedings of the 10th ACM International on Conference on emerging Networking Experiments and Technologies. ACM, 2014.
+ *
+ * https://www.cs.cmu.edu/~dga/papers/cuckoo-conext2014.pdf
+ */
+public class CuckooFilter implements Writeable {
+
+    private static final double LN_2 = Math.log(2);
+    private static final int MAX_EVICTIONS = 500;
+    static final int EMPTY = 0;
+
+    private final PackedInts.Mutable data;
+    private final int numBuckets;
+    private final int bitsPerEntry;
+    private final int fingerprintMask;
+    private final int entriesPerBucket;
+    private final Random rng;
+    private int count;
+    private int evictedFingerprint = EMPTY;
+
+    /**
+     * @param capacity The number of expected inserts.  The filter can hold more than this value, it is just an estimate
+     * @param fpp The desired false positive rate.  Smaller values will reduce the
+     *            false positives at expense of larger size
+     * @param rng A random number generator, used with the cuckoo hashing process
+     */
+    CuckooFilter(long capacity, double fpp, Random rng) {
+        this.rng = rng;
+        this.entriesPerBucket = entriesPerBucket(fpp);
+        double loadFactor = getLoadFactor(entriesPerBucket);
+        this.bitsPerEntry = bitsPerEntry(fpp, entriesPerBucket);
+        this.numBuckets = getNumBuckets(capacity, loadFactor, entriesPerBucket);
+
+        if ((long) numBuckets * (long) entriesPerBucket > Integer.MAX_VALUE) {
+            throw new IllegalArgumentException("Attempted to create [" + numBuckets * entriesPerBucket
+                + "] entries which is > Integer.MAX_VALUE");
+        }
+        this.data = PackedInts.getMutable(numBuckets * entriesPerBucket, bitsPerEntry, PackedInts.COMPACT);
+
+        // puts the bits at the right side of the mask, e.g. `0000000000001111` for bitsPerEntry = 4
+        this.fingerprintMask = (0x80000000 >> (bitsPerEntry - 1)) >>> (Integer.SIZE - bitsPerEntry);
+    }
+
+    /**
+     * This ctor is likely slow and should only be used for testing
+     */
+    CuckooFilter(CuckooFilter other) {
+        this.numBuckets = other.numBuckets;
+        this.bitsPerEntry = other.bitsPerEntry;
+        this.entriesPerBucket = other.entriesPerBucket;
+        this.count = other.count;
+        this.evictedFingerprint = other.evictedFingerprint;
+        this.rng = other.rng;
+        this.fingerprintMask = other.fingerprintMask;
+
+        // This shouldn't happen, but as a sanity check
+        if ((long) numBuckets * (long) entriesPerBucket > Integer.MAX_VALUE) {
+            throw new IllegalArgumentException("Attempted to create [" + numBuckets * entriesPerBucket
+                + "] entries which is > Integer.MAX_VALUE");
+        }
+        // TODO this is probably super slow, but just used for testing atm
+        this.data = PackedInts.getMutable(numBuckets * entriesPerBucket, bitsPerEntry, PackedInts.COMPACT);
+        for (int i = 0; i < other.data.size(); i++) {
+            data.set(i, other.data.get(i));
+        }
+    }
+
+    CuckooFilter(StreamInput in, Random rng) throws IOException {
+        this.numBuckets = in.readVInt();
+        this.bitsPerEntry = in.readVInt();
+        this.entriesPerBucket = in.readVInt();
+        this.count = in.readVInt();
+        this.evictedFingerprint = in.readVInt();
+        this.rng = rng;
+
+        this.fingerprintMask = (0x80000000 >> (bitsPerEntry - 1)) >>> (Integer.SIZE - bitsPerEntry);
+
+        data = (PackedInts.Mutable) PackedInts.getReader(new DataInput() {
+            @Override
+            public byte readByte() throws IOException {
+                return in.readByte();
+            }
+
+            @Override
+            public void readBytes(byte[] b, int offset, int len) throws IOException {
+                in.readBytes(b, offset, len);
+            }
+        });
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(numBuckets);
+        out.writeVInt(bitsPerEntry);
+        out.writeVInt(entriesPerBucket);
+        out.writeVInt(count);
+        out.writeVInt(evictedFingerprint);
+
+        data.save(new DataOutput() {
+            @Override
+            public void writeByte(byte b) throws IOException {
+                out.writeByte(b);
+            }
+
+            @Override
+            public void writeBytes(byte[] b, int offset, int length) throws IOException {
+                out.writeBytes(b, offset, length);
+            }
+        });
+    }
+
+    /**
+     * Get the number of unique items that are being tracked
+     */
+    public int getCount() {
+        return count;
+    }
+
+    /**
+     * Returns the number of buckets that has been chosen based
+     * on the initial configuration
+     *
+     * Expert-level API
+     */
+    int getNumBuckets() {
+        return numBuckets;
+    }
+
+    /**
+     * Returns the number of bits used per entry
+     *
+     * Expert-level API
+     */
+    int getBitsPerEntry() {
+        return bitsPerEntry;
+    }
+
+    /**
+     * Returns the cached fingerprint mask.  This is simply a mask for the
+     * first bitsPerEntry bits, used by {@link CuckooFilter#fingerprint(int, int, int)}
+     * to generate the fingerprint of a hash
+     *
+     * Expert-level API
+     */
+    int getFingerprintMask() {
+        return fingerprintMask;
+    }
+
+    /**
+     * Returns an iterator that returns the long[] representation of each bucket.  The value
+     * inside each long will be a fingerprint (or 0L, representing empty).
+     *
+     * Expert-level API
+     */
+    Iterator<long[]> getBuckets() {
+        return new Iterator<>() {
+            int current = 0;
+
+            @Override
+            public boolean hasNext() {
+                return current < numBuckets;
+            }
+
+            @Override
+            public long[] next() {
+                long[] values = new long[entriesPerBucket];
+                int offset = getOffset(current, 0);
+                data.get(offset, values, 0, entriesPerBucket);
+                current += 1;
+                return values;
+            }
+        };
+    }
+
+    /**
+     * Returns true if the set might contain the provided value, false otherwise.  False values are
+     * 100% accurate, while true values may be a false-positive.
+     */
+    boolean mightContain(long hash) {
+        int bucket = hashToIndex((int) hash, numBuckets);
+        int fingerprint = fingerprint((int) (hash  >>> 32), bitsPerEntry, fingerprintMask);
+        int alternateIndex = alternateIndex(bucket, fingerprint, numBuckets);
+
+        return mightContainFingerprint(bucket, fingerprint, alternateIndex);
+    }
+
+    /**
+     * Returns true if the bucket or it's alternate bucket contains the fingerprint.
+     *
+     * Expert-level API, use {@link CuckooFilter#mightContain(long)} to check if
+     * a value is in the filter.
+     */
+    boolean mightContainFingerprint(int bucket, int fingerprint, int alternateBucket) {
+
+        // check all entries for both buckets and the evicted slot
+        return hasFingerprint(bucket, fingerprint) || hasFingerprint(alternateBucket, fingerprint) || evictedFingerprint == fingerprint;
+    }
+
+    /**
+     * Return's true if any of the entries in the bucket contain the fingerprint
+     */
+    private boolean hasFingerprint(int bucket, long fingerprint) {
+        long[] values = new long[entriesPerBucket];
+        int offset = getOffset(bucket, 0);
+        data.get(offset, values, 0, entriesPerBucket);
+
+        for (int i = 0; i < entriesPerBucket; i++) {
+            if (values[i] == fingerprint) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Add's the hash to the bucket or alternate bucket.  Returns true if the insertion was
+     * successful, false if the filter is saturated.
+     */
+    boolean add(long hash) {
+        // Each bucket needs 32 bits, so we truncate for the first bucket and shift/truncate for second
+        int bucket = hashToIndex((int) hash, numBuckets);
+        int fingerprint = fingerprint((int) (hash  >>> 32), bitsPerEntry, fingerprintMask);
+        return mergeFingerprint(bucket, fingerprint);
+    }
+
+    /**
+     * Attempts to merge the fingerprint into the specified bucket or it's alternate bucket.
+     * Returns true if the insertion was successful, false if the filter is saturated.
+     *
+     * Expert-level API, use {@link CuckooFilter#add(long)} to insert
+     * values into the filter
+     */
+    boolean mergeFingerprint(int bucket, int fingerprint) {
+        // If we already have an evicted fingerprint we are full, no need to try
+        if (evictedFingerprint != EMPTY) {
+            return false;
+        }
+
+        int alternateBucket = alternateIndex(bucket, fingerprint, numBuckets);
+        if (tryInsert(bucket, fingerprint) || tryInsert(alternateBucket, fingerprint)) {
+            count += 1;
+            return true;
+        }
+
+        for (int i = 0; i < MAX_EVICTIONS; i++) {
+            // overwrite our alternate bucket, and a random entry
+            int offset = getOffset(alternateBucket, rng.nextInt(entriesPerBucket - 1));
+            int oldFingerprint = (int) data.get(offset);
+            data.set(offset, fingerprint);
+
+            // replace details and start again
+            fingerprint = oldFingerprint;
+            bucket = alternateBucket;
+            alternateBucket = alternateIndex(bucket, fingerprint, numBuckets);
+
+            // Only try to insert into alternate bucket
+            if (tryInsert(alternateBucket, fingerprint)) {
+                count += 1;
+                return true;
+            }
+        }
+
+        // If we get this far, we failed to insert the value after MAX_EVICTION rounds,
+        // so cache the last evicted value (so we don't lose it) and signal we failed
+        evictedFingerprint = fingerprint;
+        return false;
+    }
+
+    /**
+     * Low-level insert method. Attempts to write the fingerprint into an empty entry
+     * at this bucket's position.  Returns true if that was sucessful, false if all entries
+     * were occupied.
+     *
+     * If the fingerprint already exists in one of the entries, it will not duplicate the
+     * fingerprint like the original paper.  This means the filter _cannot_ support deletes,
+     * but is not sensitive to "overflowing" buckets with repeated inserts
+     */
+    private boolean tryInsert(int bucket, int fingerprint) {
+        long[] values = new long[entriesPerBucket];
+        int offset = getOffset(bucket, 0);
+        data.get(offset, values, 0, entriesPerBucket);
+
+        // TODO implement semi-sorting
+        for (int i = 0; i < values.length; i++) {
+            if (values[i] == EMPTY) {
+                data.set(offset + i, fingerprint);
+                return true;
+            } else if (values[i] == fingerprint) {
+                // Already have the fingerprint, no need to save
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Converts a hash into a bucket index (primary or alternate).
+     *
+     * If the hash is negative, this flips the bits.  The hash is then modulo numBuckets
+     * to get the final index.
+     *
+     * Expert-level API
+     */
+    static int hashToIndex(int hash, int numBuckets) {
+        return hash & (numBuckets - 1);
+    }
+
+    /**
+     * Calculates the alternate bucket for a given bucket:fingerprint tuple
+     *
+     * The alternate bucket is the fingerprint multiplied by a mixing constant,
+     * then xor'd against the bucket.  This new value is modulo'd against
+     * the buckets via {@link CuckooFilter#hashToIndex(int, int)} to get the final
+     * index.
+     *
+     * Note that the xor makes this operation reversible as long as we have the
+     * fingerprint and current bucket (regardless of if that bucket was the primary
+     * or alternate).
+     *
+     * Expert-level API
+     */
+    static int alternateIndex(int bucket, int fingerprint, int numBuckets) {
+        /*
+            Reference impl uses murmur2 mixing constant:
+            https://github.com/efficient/cuckoofilter/blob/master/src/cuckoofilter.h#L78
+                // NOTE(binfan): originally we use:
+                // index ^ HashUtil::BobHash((const void*) (&tag), 4)) & table_->INDEXMASK;
+                // now doing a quick-n-dirty way:
+                // 0x5bd1e995 is the hash constant from MurmurHash2
+                return IndexHash((uint32_t)(index ^ (tag * 0x5bd1e995)));
+         */
+        int index = bucket ^ (fingerprint * 0x5bd1e995);
+        return hashToIndex(index, numBuckets);
+    }
+
+    /**
+     * Given the bucket and entry position, returns the absolute offset
+     * inside the PackedInts datastructure
+     */
+    private int getOffset(int bucket, int position) {
+        return (bucket * entriesPerBucket) + position;
+    }
+
+    /**
+     * Calculates the fingerprint for a given hash.
+     *
+     * The fingerprint is simply the first `bitsPerEntry` number of bits that are non-zero.
+     * If the entire hash is zero, `(int) 1` is used
+     *
+     * Expert-level API
+     */
+    static int fingerprint(int hash, int bitsPerEntry, int fingerprintMask) {
+        if (hash == 0) {
+            // we use 0 as "empty" so if the hash actually hashes to zero... return 1
+            // Some other impls will re-hash with a salt but this seems simpler
+            return 1;
+        }
+
+        for (int i = 0; i + bitsPerEntry <= Long.SIZE; i += bitsPerEntry) {
+            int v = (hash >> i) & fingerprintMask;
+            if (v != 0) {
+                return v;
+            }
+        }
+        return 1;
+    }
+
+    /**
+     * Calculate the optimal number of bits per entry
+     */
+    private int bitsPerEntry(double fpp, int numEntriesPerBucket) {
+        return (int) Math.round(log2((2 * numEntriesPerBucket) / fpp));
+    }
+
+    /**
+     * Calculate the optimal number of entries per bucket.  Will return 2, 4 or 8
+     * depending on the false positive rate
+     */
+    private int entriesPerBucket(double fpp) {
+        /*
+          Empirical constants from paper:
+            "the space-optimal bucket size depends on the target false positive rate ε:
+             when ε > 0.002, having two entries per bucket yields slightly better results
+             than using four entries per bucket; when ε decreases to 0.00001 < ε <= 0.002,
+             four entries per bucket minimzes space"
+         */
+
+        if (fpp > 0.002) {
+            return 2;
+        } else if (fpp > 0.00001 && fpp <= 0.002) {
+            return 4;
+        }
+        return 8;
+    }
+
+    /**
+     * Calculates the optimal load factor for the filter, given the number of entries
+     * per bucket.  Will return 0.84, 0.955 or 0.98 depending on b
+     */
+    private double getLoadFactor(int b) {
+        if ((b == 2 || b == 4 || b == 8) == false) {
+            throw new IllegalArgumentException("b must be one of [2,4,8]");
+        }
+        /*
+          Empirical constants from the paper:
+            "With k = 2 hash functions, the load factor α is 50% when bucket size b = 1 (i.e
+            the hash table is directly mapped), but increases to 84%, 95%, 98% respectively
+            using bucket size b = 2, 4, 8"
+         */
+        if (b == 2) {
+            return 0.84D;
+        } else if (b == 4) {
+            return 0.955D;
+        } else {
+            return 0.98D;
+        }
+    }
+
+    /**
+     * Calculates the optimal number of buckets for this filter.  The xor used in the bucketing
+     * algorithm requires this to be a power of two, so the optimal number of buckets will
+     * be rounded to the next largest power of two where applicable.
+     *
+     * TODO: there are schemes to avoid powers of two, might want to investigate those
+     */
+    private int getNumBuckets(long capacity, double loadFactor, int b) {
+        long buckets = Math.round((((double) capacity / loadFactor)) / (double) b);
+
+        // Rounds up to nearest power of 2
+        return 1 << -Integer.numberOfLeadingZeros((int)buckets - 1);
+    }
+
+    private double log2(double x) {
+        return Math.log(x) / LN_2;
+    }
+
+    public long getSizeInBytes() {
+        // (numBuckets, bitsPerEntry, fingerprintMask, entriesPerBucket, count, evictedFingerprint) * 4b == 24b
+        return data.ramBytesUsed() + 24;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(numBuckets, bitsPerEntry, entriesPerBucket, count, evictedFingerprint);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        final CuckooFilter that = (CuckooFilter) other;
+        return Objects.equals(this.numBuckets, that.numBuckets)
+            && Objects.equals(this.bitsPerEntry, that.bitsPerEntry)
+            && Objects.equals(this.entriesPerBucket, that.entriesPerBucket)
+            && Objects.equals(this.count, that.count)
+            && Objects.equals(this.evictedFingerprint, that.evictedFingerprint);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/util/SetBackedScalingCuckooFilter.java b/server/src/main/java/org/elasticsearch/common/util/SetBackedScalingCuckooFilter.java
new file mode 100644
index 0000000000000..0c754aa6e895c
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/common/util/SetBackedScalingCuckooFilter.java
@@ -0,0 +1,408 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.util;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.hash.MurmurHash3;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.Random;
+import java.util.Set;
+import java.util.function.Consumer;
+
+/**
+ * An approximate set membership datastructure that scales as more unique values are inserted.
+ * Can definitively say if a member does not exist (no false negatives), but may say an item exists
+ * when it does not (has false positives).  Similar in usage to a Bloom Filter.
+ *
+ * Internally, the datastructure maintains a Set of hashes up to a specified threshold.  This provides
+ * 100% accurate membership queries.
+ *
+ * When the threshold is breached, a list of CuckooFilters are created and used to track membership.
+ * These filters are approximate similar to Bloom Filters.
+ *
+ * This datastructure scales as more values are inserted by growing the list of CuckooFilters.
+ * Final size is dependent on the cardinality of data inserted, and the precision specified.
+ */
+public class SetBackedScalingCuckooFilter implements Writeable {
+
+    /**
+     * This is the estimated insertion capacity for each individual internal CuckooFilter.
+     */
+    private static final int FILTER_CAPACITY = 1000000;
+
+    /**
+     * This set is used to track the insertions before we convert over to an approximate
+     * filter. This gives us 100% accuracy for small cardinalities.  This will be null
+     * if isSetMode = false;
+     *
+     * package-private for testing
+     */
+    Set<Long> hashes;
+
+    /**
+     * This list holds our approximate filters, after we have migrated out of a set.
+     * This will be null if isSetMode = true;
+     */
+    List<CuckooFilter> filters;
+
+    private final int threshold;
+    private final Random rng;
+    private final int capacity;
+    private final double fpp;
+    private Consumer<Long> breaker = aLong -> {
+        //noop
+    };
+
+    // cached here for performance reasons
+    private int numBuckets = 0;
+    private int bitsPerEntry = 0;
+    private int fingerprintMask = 0;
+    private MurmurHash3.Hash128 scratchHash = new MurmurHash3.Hash128();
+
+    // True if we are tracking inserts with a set, false otherwise
+    private boolean isSetMode = true;
+
+    /**
+     * @param threshold The number of distinct values that should be tracked
+     *                  before converting to an approximate representation
+     * @param rng A random number generator needed for the cuckoo hashing process
+     * @param fpp the false-positive rate that should be used for the cuckoo filters.
+     */
+    public SetBackedScalingCuckooFilter(int threshold, Random rng, double fpp) {
+        if (threshold <= 0) {
+            throw new IllegalArgumentException("[threshold] must be a positive integer");
+        }
+
+        // We have to ensure that, in the worst case, two full sets can be converted into
+        // one cuckoo filter without overflowing.  This keeps merging logic simpler
+        if (threshold * 2 > FILTER_CAPACITY) {
+            throw new IllegalArgumentException("[threshold] must be smaller than [" + (FILTER_CAPACITY / 2) + "]");
+        }
+        if (fpp < 0) {
+            throw new IllegalArgumentException("[fpp] must be a positive double");
+        }
+        this.hashes = new HashSet<>(threshold);
+        this.threshold = threshold;
+        this.rng = rng;
+        this.capacity = FILTER_CAPACITY;
+        this.fpp = fpp;
+    }
+
+    public SetBackedScalingCuckooFilter(SetBackedScalingCuckooFilter other) {
+        this.threshold = other.threshold;
+        this.isSetMode = other.isSetMode;
+        this.rng = other.rng;
+        this.breaker = other.breaker;
+        this.capacity = other.capacity;
+        this.fpp = other.fpp;
+        if (isSetMode) {
+            this.hashes = new HashSet<>(other.hashes);
+        } else {
+            this.filters = new ArrayList<>(other.filters);
+            this.numBuckets = filters.get(0).getNumBuckets();
+            this.fingerprintMask = filters.get(0).getFingerprintMask();
+            this.bitsPerEntry = filters.get(0).getBitsPerEntry();
+        }
+    }
+
+    public SetBackedScalingCuckooFilter(StreamInput in, Random rng) throws IOException {
+        this.threshold = in.readVInt();
+        this.isSetMode = in.readBoolean();
+        this.rng = rng;
+        this.capacity = in.readVInt();
+        this.fpp = in.readDouble();
+
+        if (isSetMode) {
+            this.hashes = in.readSet(StreamInput::readZLong);
+        } else {
+            this.filters = in.readList(in12 -> new CuckooFilter(in12, rng));
+            this.numBuckets = filters.get(0).getNumBuckets();
+            this.fingerprintMask = filters.get(0).getFingerprintMask();
+            this.bitsPerEntry = filters.get(0).getBitsPerEntry();
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(threshold);
+        out.writeBoolean(isSetMode);
+        out.writeVInt(capacity);
+        out.writeDouble(fpp);
+        if (isSetMode) {
+            out.writeCollection(hashes, StreamOutput::writeZLong);
+        } else {
+            out.writeList(filters);
+        }
+    }
+
+    /**
+     * Registers a circuit breaker with the datastructure.
+     *
+     * CuckooFilter's can "saturate" and refuse to accept any new values.  When this happens,
+     * the datastructure scales by adding a new filter.  This new filter's bytes will be tracked
+     * in the registered breaker when configured.
+     */
+    public void registerBreaker(Consumer<Long> breaker) {
+        this.breaker = Objects.requireNonNull(breaker, "Circuit Breaker Consumer cannot be null");
+        breaker.accept(getSizeInBytes());
+    }
+
+    /**
+     * Returns true if the set might contain the provided value, false otherwise.  False values are
+     * 100% accurate, while true values may be a false-positive.
+     */
+    public boolean mightContain(BytesRef value) {
+        MurmurHash3.Hash128 hash = MurmurHash3.hash128(value.bytes, value.offset, value.length, 0, scratchHash);
+        return mightContainHash(hash.h1);
+    }
+
+    /**
+     * Returns true if the set might contain the provided value, false otherwise.  False values are
+     * 100% accurate, while true values may be a false-positive.
+     */
+    public boolean mightContain(long value) {
+        long hash = MurmurHash3.murmur64(value);
+        return mightContainHash(hash);
+    }
+
+    /**
+     * Returns true if the set might contain the provided value, false otherwise.  False values are
+     * 100% accurate, while true values may be a false-positive.
+     */
+    private boolean mightContainHash(long hash) {
+        if (isSetMode) {
+            return hashes.contains(hash);
+        }
+
+        // We calculate these once up front for all the filters and use the expert API
+        int bucket = CuckooFilter.hashToIndex((int) hash, numBuckets);
+        int fingerprint = CuckooFilter.fingerprint((int) (hash >> 32), bitsPerEntry, fingerprintMask);
+        int alternateIndex = CuckooFilter.alternateIndex(bucket, fingerprint, numBuckets);
+
+        for (CuckooFilter filter : filters) {
+            if (filter.mightContainFingerprint(bucket, fingerprint, alternateIndex)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Returns true if any of the filters contain this fingerprint at the specified bucket.
+     * This is an expert-level API since it is dealing with buckets and fingerprints, not raw values
+     * being hashed.
+     */
+    private boolean mightContainFingerprint(int bucket, int fingerprint) {
+        int alternateIndex = CuckooFilter.alternateIndex(bucket, fingerprint, numBuckets);
+        for (CuckooFilter filter : filters) {
+            if (filter.mightContainFingerprint(bucket, fingerprint, alternateIndex)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Add's the provided value to the set for tracking
+     */
+    public void add(BytesRef value) {
+        MurmurHash3.Hash128 hash = MurmurHash3.hash128(value.bytes, value.offset, value.length, 0, scratchHash);
+        addHash(hash.h1);
+    }
+
+    /**
+     * Add's the provided value to the set for tracking
+     */
+    public void add(long value) {
+        addHash(MurmurHash3.murmur64(value));
+    }
+
+    private void addHash(long hash) {
+        if (isSetMode) {
+            hashes.add(hash);
+            maybeConvert();
+            return;
+        }
+
+        boolean success = filters.get(filters.size() - 1).add(hash);
+        if (success == false) {
+            // filter is full, create a new one and insert there
+            CuckooFilter t = new CuckooFilter(capacity, fpp, rng);
+            t.add(hash);
+            filters.add(t);
+            breaker.accept(t.getSizeInBytes()); // make sure we account for the new filter
+        }
+    }
+
+    private void maybeConvert() {
+        if (isSetMode && hashes.size() > threshold) {
+            convert();
+        }
+    }
+
+    /**
+     * If we still holding values in a set, convert this filter into an approximate, cuckoo-backed filter.
+     * This will create a list of CuckooFilters, and null out the set of hashes
+     */
+    void convert() {
+        if (isSetMode == false) {
+            throw new IllegalStateException("Cannot convert SetBackedScalingCuckooFilter to approximate " +
+                "when it has already been converted.");
+        }
+        long oldSize = getSizeInBytes();
+
+        filters = new ArrayList<>();
+        CuckooFilter t = new CuckooFilter(capacity, fpp, rng);
+        // Cache the chosen numBuckets for later use
+        numBuckets = t.getNumBuckets();
+        fingerprintMask = t.getFingerprintMask();
+        bitsPerEntry = t.getBitsPerEntry();
+
+        hashes.forEach(t::add);
+        filters.add(t);
+
+        hashes = null;
+        isSetMode = false;
+
+        breaker.accept(-oldSize); // this zeros out the overhead of the set
+        breaker.accept(getSizeInBytes()); // this adds back in the new overhead of the cuckoo filters
+
+    }
+
+    /**
+     * Get the approximate size of this datastructure.  Approximate because only the Set occupants
+     * are tracked, not the overhead of the Set itself.
+     */
+    public long getSizeInBytes() {
+        long bytes = 13; // fpp (double), threshold (int), isSetMode (boolean)
+        if (hashes != null) {
+            bytes = (hashes.size() * 16);
+        }
+        if (filters != null) {
+            bytes += filters.stream().mapToLong(CuckooFilter::getSizeInBytes).sum();
+        }
+        return bytes;
+    }
+
+
+    /**
+     * Merge `other` cuckoo filter into this cuckoo.  After merging, this filter's state will
+     * be the union of the two.  During the merging process, the internal Set may be upgraded
+     * to a cuckoo if it goes over threshold
+     */
+    public void merge(SetBackedScalingCuckooFilter other) {
+        // Some basic sanity checks to make sure we can merge
+        if (this.threshold != other.threshold) {
+            throw new IllegalStateException("Cannot merge other CuckooFilter because thresholds do not match: ["
+                + this.threshold + "] vs [" + other.threshold + "]");
+        }
+        if (this.capacity != other.capacity) {
+            throw new IllegalStateException("Cannot merge other CuckooFilter because capacities do not match: ["
+                + this.capacity + "] vs [" + other.capacity + "]");
+        }
+        if (this.fpp != other.fpp) {
+            throw new IllegalStateException("Cannot merge other CuckooFilter because precisions do not match: ["
+                + this.fpp + "] vs [" + other.fpp + "]");
+        }
+
+        if (isSetMode && other.isSetMode) {
+            // Both in sets, merge collections then see if we need to convert to cuckoo
+            hashes.addAll(other.hashes);
+            maybeConvert();
+        } else if (isSetMode && other.isSetMode == false) {
+            // Other is in cuckoo mode, so we convert our set to a cuckoo, then
+            // call the merge function again.  Since both are now in set-mode
+            // this will fall through to the last conditional and do a cuckoo-cuckoo merge
+            convert();
+            merge(other);
+        } else if (isSetMode == false && other.isSetMode) {
+            // Rather than converting the other to a cuckoo first, we can just
+            // replay the values directly into our filter.
+            other.hashes.forEach(this::add);
+        } else {
+            // Both are in cuckoo mode, merge raw fingerprints
+
+            CuckooFilter currentFilter = filters.get(filters.size() - 1);
+
+            for (CuckooFilter otherFilter : other.filters) {
+
+                // The iterator returns an array of longs corresponding to the
+                // fingerprints for buckets at the current position
+                Iterator<long[]> iter = otherFilter.getBuckets();
+                int bucket = 0;
+                while (iter.hasNext()) {
+                    long[] fingerprints = iter.next();
+
+                    // We check to see if the fingerprint is present in any of the existing filters
+                    // (in the same bucket/alternate bucket), or if the fingerprint is empty.  In these cases
+                    // we can skip the fingerprint
+                    for (long fingerprint : fingerprints) {
+                        if (fingerprint == CuckooFilter.EMPTY || mightContainFingerprint(bucket, (int) fingerprint)) {
+                            continue;
+                        }
+                        // Try to insert into the last filter in our list
+                        if (currentFilter.mergeFingerprint(bucket, (int) fingerprint) == false) {
+                            // if we failed, the filter is now saturated and we need to create a new one
+                            CuckooFilter t = new CuckooFilter(capacity, fpp, rng);
+                            filters.add(t);
+                            breaker.accept(t.getSizeInBytes()); // make sure we account for the new filter
+
+                            currentFilter = filters.get(filters.size() - 1);
+                        }
+                    }
+                    bucket += 1;
+                }
+            }
+        }
+    }
+
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(hashes, filters, threshold, isSetMode, capacity, fpp);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        final SetBackedScalingCuckooFilter that = (SetBackedScalingCuckooFilter) other;
+        return Objects.equals(this.hashes, that.hashes)
+            && Objects.equals(this.filters, that.filters)
+            && Objects.equals(this.threshold, that.threshold)
+            && Objects.equals(this.isSetMode, that.isSetMode)
+            && Objects.equals(this.capacity, that.capacity)
+            && Objects.equals(this.fpp, that.fpp);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/AsyncIOProcessor.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/AsyncIOProcessor.java
index ad68471041b8d..ad5fd09edc0f5 100644
--- a/server/src/main/java/org/elasticsearch/common/util/concurrent/AsyncIOProcessor.java
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/AsyncIOProcessor.java
@@ -28,6 +28,7 @@
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.Semaphore;
 import java.util.function.Consumer;
+import java.util.function.Supplier;
 
 /**
  * This async IO processor allows to batch IO operations and have a single writer processing the write operations.
@@ -39,11 +40,13 @@
 public abstract class AsyncIOProcessor<Item> {
     private final Logger logger;
     private final ArrayBlockingQueue<Tuple<Item, Consumer<Exception>>> queue;
+    private final ThreadContext threadContext;
     private final Semaphore promiseSemaphore = new Semaphore(1);
 
-    protected AsyncIOProcessor(Logger logger, int queueSize) {
+    protected AsyncIOProcessor(Logger logger, int queueSize, ThreadContext threadContext) {
         this.logger = logger;
         this.queue = new ArrayBlockingQueue<>(queueSize);
+        this.threadContext = threadContext;
     }
 
     /**
@@ -58,11 +61,10 @@ public final void put(Item item, Consumer<Exception> listener) {
 
         // we first try make a promise that we are responsible for the processing
         final boolean promised = promiseSemaphore.tryAcquire();
-        final Tuple<Item, Consumer<Exception>> itemTuple = new Tuple<>(item, listener);
         if (promised == false) {
             // in this case we are not responsible and can just block until there is space
             try {
-                queue.put(new Tuple<>(item, listener));
+                queue.put(new Tuple<>(item, preserveContext(listener)));
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
                 listener.accept(e);
@@ -73,34 +75,33 @@ public final void put(Item item, Consumer<Exception> listener) {
         // while we are draining that mean we might exit below too early in the while loop if the drainAndSync call is fast.
         if (promised || promiseSemaphore.tryAcquire()) {
             final List<Tuple<Item, Consumer<Exception>>> candidates = new ArrayList<>();
-            try {
-                if (promised) {
-                    // we are responsible for processing we don't need to add the tuple to the queue we can just add it to the candidates
-                    candidates.add(itemTuple);
-                }
-                // since we made the promise to process we gotta do it here at least once
-                drainAndProcess(candidates);
-            } finally {
-                promiseSemaphore.release(); // now to ensure we are passing it on we release the promise so another thread can take over
+            if (promised) {
+                // we are responsible for processing we don't need to add the tuple to the queue we can just add it to the candidates
+                // no need to preserve context for listener since it runs in current thread.
+                candidates.add(new Tuple<>(item, listener));
             }
+            // since we made the promise to process we gotta do it here at least once
+            drainAndProcessAndRelease(candidates);
             while (queue.isEmpty() == false && promiseSemaphore.tryAcquire()) {
                 // yet if the queue is not empty AND nobody else has yet made the promise to take over we continue processing
-                try {
-                    drainAndProcess(candidates);
-                } finally {
-                    promiseSemaphore.release();
-                }
+                drainAndProcessAndRelease(candidates);
             }
         }
     }
 
-    private void drainAndProcess(List<Tuple<Item, Consumer<Exception>>> candidates) {
-        queue.drainTo(candidates);
-        processList(candidates);
+    private void drainAndProcessAndRelease(List<Tuple<Item, Consumer<Exception>>> candidates) {
+        Exception exception;
+        try {
+            queue.drainTo(candidates);
+            exception = processList(candidates);
+        } finally {
+            promiseSemaphore.release();
+        }
+        notifyList(candidates, exception);
         candidates.clear();
     }
 
-    private void processList(List<Tuple<Item, Consumer<Exception>>> candidates) {
+    private Exception processList(List<Tuple<Item, Consumer<Exception>>> candidates) {
         Exception exception = null;
         if (candidates.isEmpty() == false) {
             try {
@@ -111,6 +112,10 @@ private void processList(List<Tuple<Item, Consumer<Exception>>> candidates) {
                 exception = ex;
             }
         }
+        return exception;
+    }
+
+    private void notifyList(List<Tuple<Item, Consumer<Exception>>> candidates, Exception exception) {
         for (Tuple<Item, Consumer<Exception>> tuple : candidates) {
             Consumer<Exception> consumer = tuple.v2();
             try {
@@ -121,6 +126,15 @@ private void processList(List<Tuple<Item, Consumer<Exception>>> candidates) {
         }
     }
 
+    private Consumer<Exception> preserveContext(Consumer<Exception> consumer) {
+        Supplier<ThreadContext.StoredContext> restorableContext = threadContext.newRestorableContext(false);
+        return e -> {
+            try (ThreadContext.StoredContext ignore = restorableContext.get()) {
+                consumer.accept(e);
+            }
+        };
+    }
+
     /**
      * Writes or processes the items out or to disk.
      */
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
index 29cd7f6682a64..7b304cd092a26 100644
--- a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
+++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
@@ -124,6 +124,7 @@ public static EsThreadPoolExecutor newAutoQueueFixed(String name, int size, int
      */
     public static Throwable rethrowErrors(Runnable runnable) {
         if (runnable instanceof RunnableFuture) {
+            assert ((RunnableFuture) runnable).isDone();
             try {
                 ((RunnableFuture) runnable).get();
             } catch (final Exception e) {
diff --git a/server/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java b/server/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
index a14def8fa86b5..892dfe9eaa16e 100644
--- a/server/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
+++ b/server/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
@@ -23,7 +23,9 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.coordination.Coordinator;
+import org.elasticsearch.cluster.coordination.ElectionStrategy;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RerouteService;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterApplier;
 import org.elasticsearch.cluster.service.ClusterApplierService;
@@ -74,16 +76,24 @@ public class DiscoveryModule {
         Setting.listSetting("discovery.seed_providers", Collections.emptyList(), Function.identity(),
             Property.NodeScope);
 
+    public static final String DEFAULT_ELECTION_STRATEGY = "default";
+
+    public static final Setting<String> ELECTION_STRATEGY_SETTING =
+        new Setting<>("cluster.election.strategy", DEFAULT_ELECTION_STRATEGY, Function.identity(), Property.NodeScope);
+
     private final Discovery discovery;
 
     public DiscoveryModule(Settings settings, ThreadPool threadPool, TransportService transportService,
                            NamedWriteableRegistry namedWriteableRegistry, NetworkService networkService, MasterService masterService,
                            ClusterApplier clusterApplier, ClusterSettings clusterSettings, List<DiscoveryPlugin> plugins,
-                           AllocationService allocationService, Path configFile, GatewayMetaState gatewayMetaState) {
+                           AllocationService allocationService, Path configFile, GatewayMetaState gatewayMetaState,
+                           RerouteService rerouteService) {
         final Collection<BiConsumer<DiscoveryNode, ClusterState>> joinValidators = new ArrayList<>();
         final Map<String, Supplier<SeedHostsProvider>> hostProviders = new HashMap<>();
         hostProviders.put("settings", () -> new SettingsBasedSeedHostsProvider(settings, transportService));
         hostProviders.put("file", () -> new FileBasedSeedHostsProvider(configFile));
+        final Map<String, ElectionStrategy> electionStrategies = new HashMap<>();
+        electionStrategies.put(DEFAULT_ELECTION_STRATEGY, ElectionStrategy.DEFAULT_INSTANCE);
         for (DiscoveryPlugin plugin : plugins) {
             plugin.getSeedHostProviders(transportService, networkService).forEach((key, value) -> {
                 if (hostProviders.put(key, value) != null) {
@@ -94,6 +104,11 @@ public DiscoveryModule(Settings settings, ThreadPool threadPool, TransportServic
             if (joinValidator != null) {
                 joinValidators.add(joinValidator);
             }
+            plugin.getElectionStrategies().forEach((key, value) -> {
+                if (electionStrategies.put(key, value) != null) {
+                    throw new IllegalArgumentException("Cannot register election strategy [" + key + "] twice");
+                }
+            });
         }
 
         List<String> seedProviderNames = DISCOVERY_SEED_PROVIDERS_SETTING.get(settings);
@@ -124,12 +139,17 @@ public DiscoveryModule(Settings settings, ThreadPool threadPool, TransportServic
             return Collections.unmodifiableList(addresses);
         };
 
+        final ElectionStrategy electionStrategy = electionStrategies.get(ELECTION_STRATEGY_SETTING.get(settings));
+        if (electionStrategy == null) {
+            throw new IllegalArgumentException("Unknown election strategy " + ELECTION_STRATEGY_SETTING.get(settings));
+        }
+
         if (ZEN2_DISCOVERY_TYPE.equals(discoveryType) || SINGLE_NODE_DISCOVERY_TYPE.equals(discoveryType)) {
             discovery = new Coordinator(NODE_NAME_SETTING.get(settings),
                 settings, clusterSettings,
                 transportService, namedWriteableRegistry, allocationService, masterService,
                 () -> gatewayMetaState.getPersistedState(settings, (ClusterApplierService) clusterApplier), seedHostsProvider,
-                clusterApplier, joinValidators, new Random(Randomness.get().nextLong()));
+                clusterApplier, joinValidators, new Random(Randomness.get().nextLong()), rerouteService, electionStrategy);
         } else {
             throw new IllegalArgumentException("Unknown discovery type [" + discoveryType + "]");
         }
diff --git a/server/src/main/java/org/elasticsearch/discovery/HandshakingTransportAddressConnector.java b/server/src/main/java/org/elasticsearch/discovery/HandshakingTransportAddressConnector.java
index 7f2512f97f87b..4e90ae02e12ac 100644
--- a/server/src/main/java/org/elasticsearch/discovery/HandshakingTransportAddressConnector.java
+++ b/server/src/main/java/org/elasticsearch/discovery/HandshakingTransportAddressConnector.java
@@ -21,6 +21,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -89,6 +90,13 @@ protected void doRun() throws Exception {
                     remoteNode = transportService.handshake(connection, probeHandshakeTimeout.millis());
                     // success means (amongst other things) that the cluster names match
                     logger.trace("[{}] handshake successful: {}", this, remoteNode);
+                } catch (Exception e) {
+                    // we opened a connection and successfully performed a low-level handshake, so we were definitely talking to an
+                    // Elasticsearch node, but the high-level handshake failed indicating some kind of mismatched configurations
+                    // (e.g. cluster name) that the user should address
+                    logger.warn(new ParameterizedMessage("handshake failed for [{}]", this), e);
+                    listener.onFailure(e);
+                    return;
                 } finally {
                     IOUtils.closeWhileHandlingException(connection);
                 }
diff --git a/server/src/main/java/org/elasticsearch/discovery/SeedHostsResolver.java b/server/src/main/java/org/elasticsearch/discovery/SeedHostsResolver.java
index aa4c39ad8276d..3efbff52d822c 100644
--- a/server/src/main/java/org/elasticsearch/discovery/SeedHostsResolver.java
+++ b/server/src/main/java/org/elasticsearch/discovery/SeedHostsResolver.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.CancellableThreads;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.discovery.PeerFinder.ConfiguredHostsResolver;
@@ -67,6 +68,7 @@ public class SeedHostsResolver extends AbstractLifecycleComponent implements Con
     private final TimeValue resolveTimeout;
     private final String nodeName;
     private final int concurrentConnects;
+    private final CancellableThreads cancellableThreads = new CancellableThreads();
 
     public SeedHostsResolver(String nodeName, Settings settings, TransportService transportService,
                              SeedHostsProvider seedProvider) {
@@ -98,11 +100,11 @@ public List<TransportAddress> resolveHosts(final List<String> hosts) {
                 .stream()
                 .map(hn -> (Callable<TransportAddress[]>) () -> transportService.addressesFromString(hn))
                 .collect(Collectors.toList());
-        final List<Future<TransportAddress[]>> futures;
+        final SetOnce<List<Future<TransportAddress[]>>> futures = new SetOnce<>();
         try {
-            futures = executorService.get().invokeAll(callables, resolveTimeout.nanos(), TimeUnit.NANOSECONDS);
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
+            cancellableThreads.execute(() ->
+                futures.set(executorService.get().invokeAll(callables, resolveTimeout.nanos(), TimeUnit.NANOSECONDS)));
+        } catch (CancellableThreads.ExecutionCancelledException e) {
             return Collections.emptyList();
         }
         final List<TransportAddress> transportAddresses = new ArrayList<>();
@@ -112,10 +114,10 @@ public List<TransportAddress> resolveHosts(final List<String> hosts) {
         // ExecutorService#invokeAll guarantees that the futures are returned in the iteration order of the tasks so we can associate the
         // hostname with the corresponding task by iterating together
         final Iterator<String> it = hosts.iterator();
-        for (final Future<TransportAddress[]> future : futures) {
+        for (final Future<TransportAddress[]> future : futures.get()) {
+            assert future.isDone();
             final String hostname = it.next();
             if (!future.isCancelled()) {
-                assert future.isDone();
                 try {
                     final TransportAddress[] addresses = future.get();
                     logger.trace("resolved host [{}] to {}", hostname, addresses);
@@ -151,6 +153,7 @@ protected void doStart() {
 
     @Override
     protected void doStop() {
+        cancellableThreads.cancel("stopping SeedHostsResolver");
         ThreadPool.terminate(executorService.get(), 10, TimeUnit.SECONDS);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java
index 4d19dd66732fc..160662a63e5b3 100644
--- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java
+++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.UUIDs;
@@ -45,6 +46,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.gateway.MetaDataStateFormat;
@@ -70,6 +72,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -81,6 +84,7 @@
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -90,9 +94,9 @@
  */
 public final class NodeEnvironment  implements Closeable {
     public static class NodePath {
-        /* ${data.paths}/nodes/0 */
+        /* ${data.paths} */
         public final Path path;
-        /* ${data.paths}/nodes/0/indices */
+        /* ${data.paths}/indices */
         public final Path indicesPath;
         /** Cached FileStore from path */
         public final FileStore fileStore;
@@ -115,7 +119,7 @@ public NodePath(Path path) throws IOException {
 
         /**
          * Resolves the given shards directory against this NodePath
-         * ${data.paths}/nodes/{node.id}/indices/{index.uuid}/{shard.id}
+         * ${data.paths}/indices/{index.uuid}/{shard.id}
          */
         public Path resolve(ShardId shardId) {
             return resolve(shardId.getIndex()).resolve(Integer.toString(shardId.id()));
@@ -123,7 +127,7 @@ public Path resolve(ShardId shardId) {
 
         /**
          * Resolves index directory against this NodePath
-         * ${data.paths}/nodes/{node.id}/indices/{index.uuid}
+         * ${data.paths}/indices/{index.uuid}
          */
         public Path resolve(Index index) {
             return resolve(index.getUUID());
@@ -170,7 +174,6 @@ public String toString() {
     public static final Setting<Boolean> ENABLE_LUCENE_SEGMENT_INFOS_TRACE_SETTING =
         Setting.boolSetting("node.enable_lucene_segment_infos_trace", false, Property.NodeScope);
 
-    public static final String NODES_FOLDER = "nodes";
     public static final String INDICES_FOLDER = "indices";
     public static final String NODE_LOCK_FILENAME = "node.lock";
 
@@ -179,20 +182,28 @@ public static class NodeLock implements Releasable {
         private final Lock[] locks;
         private final NodePath[] nodePaths;
 
+
+        public NodeLock(final Logger logger,
+                        final Environment environment,
+                        final CheckedFunction<Path, Boolean, IOException> pathFunction) throws IOException {
+            this(logger, environment, pathFunction, Function.identity());
+        }
+
         /**
          * Tries to acquire a node lock for a node id, throws {@code IOException} if it is unable to acquire it
          * @param pathFunction function to check node path before attempt of acquiring a node lock
          */
         public NodeLock(final Logger logger,
                         final Environment environment,
-                        final CheckedFunction<Path, Boolean, IOException> pathFunction) throws IOException {
+                        final CheckedFunction<Path, Boolean, IOException> pathFunction,
+                        final Function<Path, Path> subPathMapping) throws IOException {
             nodePaths = new NodePath[environment.dataFiles().length];
             locks = new Lock[nodePaths.length];
             try {
                 final Path[] dataPaths = environment.dataFiles();
                 for (int dirIndex = 0; dirIndex < dataPaths.length; dirIndex++) {
                     Path dataDir = dataPaths[dirIndex];
-                    Path dir = resolveNodePath(dataDir);
+                    Path dir = subPathMapping.apply(dataDir);
                     if (pathFunction.apply(dir) == false) {
                         continue;
                     }
@@ -247,7 +258,7 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
             sharedDataPath = environment.sharedDataFile();
 
             for (Path path : environment.dataFiles()) {
-                Files.createDirectories(resolveNodePath(path));
+                Files.createDirectories(path);
             }
 
             final NodeLock nodeLock;
@@ -264,7 +275,6 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
 
             this.locks = nodeLock.locks;
             this.nodePaths = nodeLock.nodePaths;
-            this.nodeMetaData = loadOrCreateNodeMetaData(settings, logger, nodePaths);
 
             logger.debug("using node location {}", Arrays.toString(nodePaths));
 
@@ -278,6 +288,10 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
                 ensureAtomicMoveSupported(nodePaths);
             }
 
+            if (upgradeLegacyNodeFolders(logger, settings, environment, nodeLock)) {
+                assertCanWrite();
+            }
+
             if (DiscoveryNode.isDataNode(settings) == false) {
                 if (DiscoveryNode.isMasterNode(settings) == false) {
                     ensureNoIndexMetaData(nodePaths);
@@ -286,6 +300,8 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
                 ensureNoShardData(nodePaths);
             }
 
+            this.nodeMetaData = loadOrCreateNodeMetaData(settings, logger, nodePaths);
+
             success = true;
         } finally {
             if (success == false) {
@@ -295,13 +311,128 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
     }
 
     /**
-     * Resolve a specific nodes/{node.id} path for the specified path and node lock id.
-     *
-     * @param path       the path
-     * @return the resolved path
+     * Upgrades all data paths that have been written to by an older ES version to the 8.0+ compatible folder layout,
+     * removing the "nodes/${lockId}" folder prefix
      */
-    public static Path resolveNodePath(final Path path) {
-        return path.resolve(NODES_FOLDER).resolve("0");
+    private static boolean upgradeLegacyNodeFolders(Logger logger, Settings settings, Environment environment,
+                                                    NodeLock nodeLock) throws IOException {
+        boolean upgradeNeeded = false;
+
+        // check if we can do an auto-upgrade
+        for (Path path : environment.dataFiles()) {
+            final Path nodesFolderPath = path.resolve("nodes");
+            if (Files.isDirectory(nodesFolderPath)) {
+                final List<Integer> nodeLockIds = new ArrayList<>();
+
+                try (DirectoryStream<Path> stream = Files.newDirectoryStream(nodesFolderPath)) {
+                    for (Path nodeLockIdPath : stream) {
+                        String fileName = nodeLockIdPath.getFileName().toString();
+                        if (Files.isDirectory(nodeLockIdPath) && fileName.chars().allMatch(Character::isDigit)) {
+                            int nodeLockId = Integer.parseInt(fileName);
+                            nodeLockIds.add(nodeLockId);
+                        } else if (FileSystemUtils.isDesktopServicesStore(nodeLockIdPath) == false) {
+                            throw new IllegalStateException("unexpected file/folder encountered during data folder upgrade: " +
+                                nodeLockIdPath);
+                        }
+                    }
+                }
+
+                if (nodeLockIds.isEmpty() == false) {
+                    upgradeNeeded = true;
+
+                    if (nodeLockIds.equals(Arrays.asList(0)) == false) {
+                        throw new IllegalStateException("data path " + nodesFolderPath + " cannot be upgraded automatically because it " +
+                            "contains data from nodes with ordinals " + nodeLockIds + ", due to previous use of the now obsolete " +
+                            "[node.max_local_storage_nodes] setting. Please check the breaking changes docs for the current version of " +
+                            "Elasticsearch to find an upgrade path");
+                    }
+                }
+            }
+        }
+
+        if (upgradeNeeded == false) {
+            logger.trace("data folder upgrade not required");
+            return false;
+        }
+
+        logger.info("upgrading legacy data folders: {}", Arrays.toString(environment.dataFiles()));
+
+        // acquire locks on legacy path for duration of upgrade (to ensure there is no older ES version running on this path)
+        final NodeLock legacyNodeLock;
+        try {
+            legacyNodeLock = new NodeLock(logger, environment, dir -> true, path -> path.resolve("nodes").resolve("0"));
+        } catch (IOException e) {
+            final String message = String.format(
+                Locale.ROOT,
+                "failed to obtain legacy node locks, tried %s;" +
+                    " maybe these locations are not writable or multiple nodes were started on the same data path?",
+                Arrays.toString(environment.dataFiles()));
+            throw new IllegalStateException(message, e);
+        }
+
+        // move contents from legacy path to new path
+        assert nodeLock.getNodePaths().length == legacyNodeLock.getNodePaths().length;
+        try {
+            final List<CheckedRunnable<IOException>> upgradeActions = new ArrayList<>();
+            for (int i = 0; i < legacyNodeLock.getNodePaths().length; i++) {
+                final NodePath legacyNodePath = legacyNodeLock.getNodePaths()[i];
+                final NodePath nodePath = nodeLock.getNodePaths()[i];
+
+                // determine folders to move and check that there are no extra files/folders
+                final Set<String> folderNames = new HashSet<>();
+
+                try (DirectoryStream<Path> stream = Files.newDirectoryStream(legacyNodePath.path)) {
+                    for (Path subFolderPath : stream) {
+                        final String fileName = subFolderPath.getFileName().toString();
+                        if (FileSystemUtils.isDesktopServicesStore(subFolderPath)) {
+                            // ignore
+                        } else if (FileSystemUtils.isAccessibleDirectory(subFolderPath, logger)) {
+                            if (fileName.equals(INDICES_FOLDER) == false && // indices folder
+                                fileName.equals(MetaDataStateFormat.STATE_DIR_NAME) == false) { // global metadata & node state folder
+                                throw new IllegalStateException("unexpected folder encountered during data folder upgrade: " +
+                                    subFolderPath);
+                            }
+                            final Path targetSubFolderPath = nodePath.path.resolve(fileName);
+                            if (Files.exists(targetSubFolderPath)) {
+                                throw new IllegalStateException("target folder already exists during data folder upgrade: " +
+                                    targetSubFolderPath);
+                            }
+                            folderNames.add(fileName);
+                        } else if (fileName.equals(NODE_LOCK_FILENAME) == false &&
+                                   fileName.equals(TEMP_FILE_NAME) == false) {
+                            throw new IllegalStateException("unexpected file/folder encountered during data folder upgrade: " +
+                                subFolderPath);
+                        }
+                    }
+                }
+
+                assert Sets.difference(folderNames, Sets.newHashSet(INDICES_FOLDER, MetaDataStateFormat.STATE_DIR_NAME)).isEmpty() :
+                    "expected indices and/or state dir folder but was " + folderNames;
+
+                upgradeActions.add(() -> {
+                    for (String folderName : folderNames) {
+                        final Path sourceSubFolderPath = legacyNodePath.path.resolve(folderName);
+                        final Path targetSubFolderPath = nodePath.path.resolve(folderName);
+                        Files.move(sourceSubFolderPath, targetSubFolderPath, StandardCopyOption.ATOMIC_MOVE);
+                        logger.info("data folder upgrade: moved from [{}] to [{}]", sourceSubFolderPath, targetSubFolderPath);
+                    }
+                    IOUtils.fsync(nodePath.path, true);
+                });
+            }
+            // now do the actual upgrade. start by upgrading the node metadata file before moving anything, since a downgrade in an
+            // intermediate state would be pretty disastrous
+            loadOrCreateNodeMetaData(settings, logger, legacyNodeLock.getNodePaths());
+            for (CheckedRunnable<IOException> upgradeAction : upgradeActions) {
+                upgradeAction.run();
+            }
+        } finally {
+            legacyNodeLock.close();
+        }
+
+        // upgrade successfully completed, remove legacy nodes folders
+        IOUtils.rm(Stream.of(environment.dataFiles()).map(path -> path.resolve("nodes")).toArray(Path[]::new));
+
+        return true;
     }
 
     private void maybeLogPathDetails() throws IOException {
@@ -364,10 +495,25 @@ private void maybeLogHeapDetails() {
     private static NodeMetaData loadOrCreateNodeMetaData(Settings settings, Logger logger,
                                                          NodePath... nodePaths) throws IOException {
         final Path[] paths = Arrays.stream(nodePaths).map(np -> np.path).toArray(Path[]::new);
+
+        final Set<String> nodeIds = new HashSet<>();
+        for (final Path path : paths) {
+            final NodeMetaData metaData = NodeMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, path);
+            if (metaData != null) {
+                nodeIds.add(metaData.nodeId());
+            }
+        }
+        if (nodeIds.size() > 1) {
+            throw new IllegalStateException(
+                "data paths " + Arrays.toString(paths) + " belong to multiple nodes with IDs " + nodeIds);
+        }
+
         NodeMetaData metaData = NodeMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, paths);
         if (metaData == null) {
+            assert nodeIds.isEmpty() : nodeIds;
             metaData = new NodeMetaData(generateNodeId(settings), Version.CURRENT);
         } else {
+            assert nodeIds.equals(Collections.singleton(metaData.nodeId())) : nodeIds + " doesn't match " + metaData;
             metaData = metaData.upgradeToCurrentVersion();
         }
 
@@ -801,14 +947,14 @@ public Path[] availableShardPaths(ShardId shardId) {
     }
 
     /**
-     * Returns all folder names in ${data.paths}/nodes/{node.id}/indices folder
+     * Returns all folder names in ${data.paths}/indices folder
      */
     public Set<String> availableIndexFolders() throws IOException {
         return availableIndexFolders(p -> false);
     }
 
     /**
-     * Returns folder names in ${data.paths}/nodes/{node.id}/indices folder that don't match the given predicate.
+     * Returns folder names in ${data.paths}/indices folder that don't match the given predicate.
      * @param excludeIndexPathIdsPredicate folder names to exclude
      */
     public Set<String> availableIndexFolders(Predicate<String> excludeIndexPathIdsPredicate) throws IOException {
@@ -825,7 +971,7 @@ public Set<String> availableIndexFolders(Predicate<String> excludeIndexPathIdsPr
     }
 
     /**
-     * Return all directory names in the nodes/{node.id}/indices directory for the given node path.
+     * Return all directory names in the indices directory for the given node path.
      *
      * @param nodePath the path
      * @return all directories that could be indices for the given node path.
@@ -836,7 +982,7 @@ public Set<String> availableIndexFoldersForPath(final NodePath nodePath) throws
     }
 
     /**
-     * Return directory names in the nodes/{node.id}/indices directory for the given node path that don't match the given predicate.
+     * Return directory names in the indices directory for the given node path that don't match the given predicate.
      *
      * @param nodePath the path
      * @param excludeIndexPathIdsPredicate folder names to exclude
@@ -865,7 +1011,7 @@ public Set<String> availableIndexFoldersForPath(final NodePath nodePath, Predica
     }
 
     /**
-     * Resolves all existing paths to <code>indexFolderName</code> in ${data.paths}/nodes/{node.id}/indices
+     * Resolves all existing paths to <code>indexFolderName</code> in ${data.paths}/indices
      */
     public Path[] resolveIndexFolder(String indexFolderName) {
         if (nodePaths == null || locks == null) {
diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java b/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
index 82627cfdc0b82..7451f25460e0c 100644
--- a/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
+++ b/server/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
@@ -21,10 +21,12 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
+import org.elasticsearch.cluster.routing.RerouteService;
 import org.elasticsearch.cluster.routing.RoutingNodes;
-import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
 import org.elasticsearch.cluster.routing.allocation.FailedShard;
@@ -42,7 +44,7 @@ public class GatewayAllocator {
 
     private static final Logger logger = LogManager.getLogger(GatewayAllocator.class);
 
-    private final RoutingService routingService;
+    private final RerouteService rerouteService;
 
     private final PrimaryShardAllocator primaryShardAllocator;
     private final ReplicaShardAllocator replicaShardAllocator;
@@ -53,10 +55,10 @@ public class GatewayAllocator {
         asyncFetchStore = ConcurrentCollections.newConcurrentMap();
 
     @Inject
-    public GatewayAllocator(RoutingService routingService,
+    public GatewayAllocator(RerouteService rerouteService,
                             TransportNodesListGatewayStartedShards startedAction,
                             TransportNodesListShardStoreMetaData storeAction) {
-        this.routingService = routingService;
+        this.rerouteService = rerouteService;
         this.primaryShardAllocator = new InternalPrimaryShardAllocator(startedAction);
         this.replicaShardAllocator = new InternalReplicaShardAllocator(storeAction);
     }
@@ -70,7 +72,7 @@ public void cleanCaches() {
 
     // for tests
     protected GatewayAllocator() {
-        this.routingService = null;
+        this.rerouteService = null;
         this.primaryShardAllocator = null;
         this.replicaShardAllocator = null;
     }
@@ -137,7 +139,9 @@ class InternalAsyncFetch<T extends BaseNodeResponse> extends AsyncShardFetch<T>
         @Override
         protected void reroute(ShardId shardId, String reason) {
             logger.trace("{} scheduling reroute for {}", shardId, reason);
-            routingService.reroute("async_shard_fetch");
+            rerouteService.reroute("async_shard_fetch", ActionListener.wrap(
+                r -> logger.trace("{} scheduled reroute completed for {}", shardId, reason),
+                e -> logger.debug(new ParameterizedMessage("{} scheduled reroute failed for {}", shardId, reason), e)));
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayService.java b/server/src/main/java/org/elasticsearch/gateway/GatewayService.java
index b7b7d0759980e..3e9c25847f6a7 100644
--- a/server/src/main/java/org/elasticsearch/gateway/GatewayService.java
+++ b/server/src/main/java/org/elasticsearch/gateway/GatewayService.java
@@ -85,7 +85,7 @@ public class GatewayService extends AbstractLifecycleComponent implements Cluste
 
     private final Runnable recoveryRunnable;
 
-    private final AtomicBoolean recovered = new AtomicBoolean();
+    private final AtomicBoolean recoveryInProgress = new AtomicBoolean();
     private final AtomicBoolean scheduledRecovery = new AtomicBoolean();
 
     @Inject
@@ -211,7 +211,7 @@ public void onFailure(Exception e) {
 
                     @Override
                     protected void doRun() {
-                        if (recovered.compareAndSet(false, true)) {
+                        if (recoveryInProgress.compareAndSet(false, true)) {
                             logger.info("recover_after_time [{}] elapsed. performing state recovery...", recoverAfterTime);
                             recoveryRunnable.run();
                         }
@@ -219,7 +219,7 @@ protected void doRun() {
                 }, recoverAfterTime, ThreadPool.Names.GENERIC);
             }
         } else {
-            if (recovered.compareAndSet(false, true)) {
+            if (recoveryInProgress.compareAndSet(false, true)) {
                 threadPool.generic().execute(new AbstractRunnable() {
                     @Override
                     public void onFailure(final Exception e) {
@@ -237,7 +237,7 @@ protected void doRun() {
     }
 
     private void resetRecoveredFlags() {
-        recovered.set(false);
+        recoveryInProgress.set(false);
         scheduledRecovery.set(false);
     }
 
@@ -256,6 +256,9 @@ public ClusterState execute(final ClusterState currentState) {
         @Override
         public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) {
             logger.info("recovered [{}] indices into cluster_state", newState.metaData().indices().size());
+            // reset flag even though state recovery completed, to ensure that if we subsequently become leader again based on a
+            // not-recovered state, that we again do another state recovery.
+            resetRecoveredFlags();
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
index ab0fad88ecdfa..d06ad91dbbcf2 100644
--- a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
+++ b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -67,8 +68,8 @@ public ActionFuture<NodesGatewayMetaState> list(String[] nodesIds, @Nullable Tim
     }
 
     @Override
-    protected NodeRequest newNodeRequest(String nodeId, Request request) {
-        return new NodeRequest(nodeId);
+    protected NodeRequest newNodeRequest(Request request) {
+        return new NodeRequest();
     }
 
     @Override
@@ -82,7 +83,7 @@ protected NodesGatewayMetaState newResponse(Request request, List<NodeGatewayMet
     }
 
     @Override
-    protected NodeGatewayMetaState nodeOperation(NodeRequest request) {
+    protected NodeGatewayMetaState nodeOperation(NodeRequest request, Task task) {
         return new NodeGatewayMetaState(clusterService.localNode(), metaState.getMetaData());
     }
 
@@ -114,14 +115,6 @@ protected void writeNodesTo(StreamOutput out, List<NodeGatewayMetaState> nodes)
     }
 
     public static class NodeRequest extends BaseNodeRequest {
-
-        public NodeRequest() {
-        }
-
-        NodeRequest(String nodeId) {
-            super(nodeId);
-        }
-
     }
 
     public static class NodeGatewayMetaState extends BaseNodeResponse {
diff --git a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
index 1893be3acd518..be33bb031c32c 100644
--- a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
+++ b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
@@ -45,6 +45,7 @@
 import org.elasticsearch.index.shard.ShardStateMetaData;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -92,8 +93,8 @@ public void list(ShardId shardId, DiscoveryNode[] nodes,
     }
 
     @Override
-    protected NodeRequest newNodeRequest(String nodeId, Request request) {
-        return new NodeRequest(nodeId, request);
+    protected NodeRequest newNodeRequest(Request request) {
+        return new NodeRequest(request);
     }
 
     @Override
@@ -108,7 +109,7 @@ protected NodesGatewayStartedShards newResponse(Request request,
     }
 
     @Override
-    protected NodeGatewayStartedShards nodeOperation(NodeRequest request) {
+    protected NodeGatewayStartedShards nodeOperation(NodeRequest request, Task task) {
         try {
             final ShardId shardId = request.getShardId();
             logger.trace("{} loading local shard state info", shardId);
@@ -222,8 +223,7 @@ public static class NodeRequest extends BaseNodeRequest {
         public NodeRequest() {
         }
 
-        public NodeRequest(String nodeId, Request request) {
-            super(nodeId);
+        public NodeRequest(Request request) {
             this.shardId = request.shardId();
         }
 
diff --git a/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java
index ab6168cad522e..6b901958218a6 100644
--- a/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java
+++ b/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java
@@ -73,6 +73,7 @@ public abstract class AbstractHttpServerTransport extends AbstractLifecycleCompo
     protected final BigArrays bigArrays;
     protected final ThreadPool threadPool;
     protected final Dispatcher dispatcher;
+    protected final CorsHandler.Config corsConfig;
     private final NamedXContentRegistry xContentRegistry;
 
     protected final PortsRange port;
@@ -94,6 +95,7 @@ protected AbstractHttpServerTransport(Settings settings, NetworkService networkS
         this.xContentRegistry = xContentRegistry;
         this.dispatcher = dispatcher;
         this.handlingSettings = HttpHandlingSettings.fromSettings(settings);
+        this.corsConfig = CorsHandler.fromSettings(settings);
 
         // we can't make the network.bind_host a fallback since we already fall back to http.host hence the extra conditional here
         List<String> httpBindHost = SETTING_HTTP_BIND_HOST.get(settings);
diff --git a/server/src/main/java/org/elasticsearch/http/CorsHandler.java b/server/src/main/java/org/elasticsearch/http/CorsHandler.java
new file mode 100644
index 0000000000000..6bd0b09265dfc
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/http/CorsHandler.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Copyright 2013 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License, version
+ * 2.0 (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.elasticsearch.http;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestUtils;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_MAX_AGE;
+
+/**
+ * This file is forked from the https://netty.io project. In particular it combines the following three
+ * files: io.netty.handler.codec.http.cors.CorsHandler, io.netty.handler.codec.http.cors.CorsConfig, and
+ * io.netty.handler.codec.http.cors.CorsConfigBuilder.
+ *
+ * It modifies the original netty code to operation on Elasticsearch http request/response abstractions.
+ * Additionally, it removes CORS features that are not used by Elasticsearch.
+ */
+public class CorsHandler {
+
+    public static final String ANY_ORIGIN = "*";
+
+    private CorsHandler() {
+    }
+
+    public static class Config {
+
+        private final boolean enabled;
+        private final Optional<Set<String>> origins;
+        private final Optional<Pattern> pattern;
+        private final boolean anyOrigin;
+        private final boolean credentialsAllowed;
+        private final Set<RestRequest.Method> allowedRequestMethods;
+        private final Set<String> allowedRequestHeaders;
+        private final long maxAge;
+
+        public Config(Builder builder) {
+            this.enabled = builder.enabled;
+            origins = builder.origins.map(HashSet::new);
+            pattern = builder.pattern;
+            anyOrigin = builder.anyOrigin;
+            this.credentialsAllowed = builder.allowCredentials;
+            this.allowedRequestMethods = Collections.unmodifiableSet(builder.requestMethods);
+            this.allowedRequestHeaders = Collections.unmodifiableSet(builder.requestHeaders);
+            this.maxAge = builder.maxAge;
+        }
+
+        public boolean isCorsSupportEnabled() {
+            return enabled;
+        }
+
+        public boolean isAnyOriginSupported() {
+            return anyOrigin;
+        }
+
+        public boolean isOriginAllowed(String origin) {
+            if (origins.isPresent()) {
+                return origins.get().contains(origin);
+            } else if (pattern.isPresent()) {
+                return pattern.get().matcher(origin).matches();
+            }
+            return false;
+        }
+
+        public boolean isCredentialsAllowed() {
+            return credentialsAllowed;
+        }
+
+        public Set<RestRequest.Method> allowedRequestMethods() {
+            return allowedRequestMethods;
+        }
+
+        public Set<String> allowedRequestHeaders() {
+            return allowedRequestHeaders;
+        }
+
+        public long maxAge() {
+            return maxAge;
+        }
+
+        public Optional<Set<String>> origins() {
+            return origins;
+        }
+
+        @Override
+        public String toString() {
+            return "Config{" +
+                "enabled=" + enabled +
+                ", origins=" + origins +
+                ", pattern=" + pattern +
+                ", anyOrigin=" + anyOrigin +
+                ", credentialsAllowed=" + credentialsAllowed +
+                ", allowedRequestMethods=" + allowedRequestMethods +
+                ", allowedRequestHeaders=" + allowedRequestHeaders +
+                ", maxAge=" + maxAge +
+                '}';
+        }
+
+        private static class Builder {
+
+            private boolean enabled = true;
+            private Optional<Set<String>> origins;
+            private Optional<Pattern> pattern;
+            private final boolean anyOrigin;
+            private boolean allowCredentials = false;
+            long maxAge;
+            private final Set<RestRequest.Method> requestMethods = new HashSet<>();
+            private final Set<String> requestHeaders = new HashSet<>();
+
+            private Builder() {
+                anyOrigin = true;
+                origins = Optional.empty();
+                pattern = Optional.empty();
+            }
+
+            private Builder(final String... origins) {
+                this.origins = Optional.of(new LinkedHashSet<>(Arrays.asList(origins)));
+                pattern = Optional.empty();
+                anyOrigin = false;
+            }
+
+            private Builder(final Pattern pattern) {
+                this.pattern = Optional.of(pattern);
+                origins = Optional.empty();
+                anyOrigin = false;
+            }
+
+            static Builder forOrigins(final String... origins) {
+                return new Builder(origins);
+            }
+
+            static Builder forAnyOrigin() {
+                return new Builder();
+            }
+
+            static Builder forPattern(Pattern pattern) {
+                return new Builder(pattern);
+            }
+
+            Builder allowCredentials() {
+                this.allowCredentials = true;
+                return this;
+            }
+
+
+            public Builder allowedRequestMethods(RestRequest.Method[] methods) {
+                requestMethods.addAll(Arrays.asList(methods));
+                return this;
+            }
+
+            public Builder maxAge(int maxAge) {
+                this.maxAge = maxAge;
+                return this;
+            }
+
+            public Builder allowedRequestHeaders(String[] headers) {
+                requestHeaders.addAll(Arrays.asList(headers));
+                return this;
+            }
+
+            public Config build() {
+                return new Config(this);
+            }
+        }
+    }
+
+    public static Config disabled() {
+        Config.Builder builder = new Config.Builder();
+        builder.enabled = false;
+        return new Config(builder);
+    }
+
+    public static Config fromSettings(Settings settings) {
+        if (SETTING_CORS_ENABLED.get(settings) == false) {
+            return disabled();
+        }
+        String origin = SETTING_CORS_ALLOW_ORIGIN.get(settings);
+        final CorsHandler.Config.Builder builder;
+        if (Strings.isNullOrEmpty(origin)) {
+            builder = CorsHandler.Config.Builder.forOrigins();
+        } else if (origin.equals(CorsHandler.ANY_ORIGIN)) {
+            builder = CorsHandler.Config.Builder.forAnyOrigin();
+        } else {
+            try {
+                Pattern p = RestUtils.checkCorsSettingForRegex(origin);
+                if (p == null) {
+                    builder = CorsHandler.Config.Builder.forOrigins(RestUtils.corsSettingAsArray(origin));
+                } else {
+                    builder = CorsHandler.Config.Builder.forPattern(p);
+                }
+            } catch (PatternSyntaxException e) {
+                throw new SettingsException("Bad regex in [" + SETTING_CORS_ALLOW_ORIGIN.getKey() + "]: [" + origin + "]", e);
+            }
+        }
+        if (SETTING_CORS_ALLOW_CREDENTIALS.get(settings)) {
+            builder.allowCredentials();
+        }
+        String[] strMethods = Strings.tokenizeToStringArray(SETTING_CORS_ALLOW_METHODS.get(settings), ",");
+        RestRequest.Method[] methods = Arrays.stream(strMethods)
+            .map(s -> s.toUpperCase(Locale.ENGLISH))
+            .map(RestRequest.Method::valueOf)
+            .toArray(RestRequest.Method[]::new);
+        Config config = builder.allowedRequestMethods(methods)
+            .maxAge(SETTING_CORS_MAX_AGE.get(settings))
+            .allowedRequestHeaders(Strings.tokenizeToStringArray(SETTING_CORS_ALLOW_HEADERS.get(settings), ","))
+            .build();
+        return config;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/http/DefaultRestChannel.java b/server/src/main/java/org/elasticsearch/http/DefaultRestChannel.java
index 9d21896182c67..5191a8d85cef6 100644
--- a/server/src/main/java/org/elasticsearch/http/DefaultRestChannel.java
+++ b/server/src/main/java/org/elasticsearch/http/DefaultRestChannel.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -76,29 +77,42 @@ protected BytesStreamOutput newBytesOutput() {
 
     @Override
     public void sendResponse(RestResponse restResponse) {
-        HttpResponse httpResponse;
-        if (RestRequest.Method.HEAD == request.method()) {
-            httpResponse = httpRequest.createResponse(restResponse.status(), BytesArray.EMPTY);
-        } else {
-            httpResponse = httpRequest.createResponse(restResponse.status(), restResponse.content());
+        final ArrayList<Releasable> toClose = new ArrayList<>(3);
+        if (isCloseConnection()) {
+            toClose.add(() -> CloseableChannel.closeChannel(httpChannel));
         }
 
-        // TODO: Ideally we should move the setting of Cors headers into :server
-        // NioCorsHandler.setCorsResponseHeaders(nettyRequest, resp, corsConfig);
+        boolean success = false;
+        try {
+            final BytesReference content = restResponse.content();
+            if (content instanceof Releasable) {
+                toClose.add((Releasable) content);
+            }
 
-        String opaque = request.header(X_OPAQUE_ID);
-        if (opaque != null) {
-            setHeaderField(httpResponse, X_OPAQUE_ID, opaque);
-        }
+            BytesReference finalContent = content;
+            try {
+                if (request.method() == RestRequest.Method.HEAD) {
+                    finalContent = BytesArray.EMPTY;
+                }
+            } catch (IllegalArgumentException ignored) {
+                assert restResponse.status() == RestStatus.METHOD_NOT_ALLOWED :
+                    "request HTTP method is unsupported but HTTP status is not METHOD_NOT_ALLOWED(405)";
+            }
 
-        // Add all custom headers
-        addCustomHeaders(httpResponse, restResponse.getHeaders());
-        addCustomHeaders(httpResponse, threadContext.getResponseHeaders());
+            final HttpResponse httpResponse = httpRequest.createResponse(restResponse.status(), finalContent);
 
-        ArrayList<Releasable> toClose = new ArrayList<>(3);
+            // TODO: Ideally we should move the setting of Cors headers into :server
+            // NioCorsHandler.setCorsResponseHeaders(nettyRequest, resp, corsConfig);
+
+            String opaque = request.header(X_OPAQUE_ID);
+            if (opaque != null) {
+                setHeaderField(httpResponse, X_OPAQUE_ID, opaque);
+            }
+
+            // Add all custom headers
+            addCustomHeaders(httpResponse, restResponse.getHeaders());
+            addCustomHeaders(httpResponse, threadContext.getResponseHeaders());
 
-        boolean success = false;
-        try {
             // If our response doesn't specify a content-type header, set one
             setHeaderField(httpResponse, CONTENT_TYPE, restResponse.contentType(), false);
             // If our response has no content-length, calculate and set one
@@ -106,19 +120,11 @@ public void sendResponse(RestResponse restResponse) {
 
             addCookies(httpResponse);
 
-            BytesReference content = restResponse.content();
-            if (content instanceof Releasable) {
-                toClose.add((Releasable) content);
-            }
             BytesStreamOutput bytesStreamOutput = bytesOutputOrNull();
             if (bytesStreamOutput instanceof ReleasableBytesStreamOutput) {
                 toClose.add((Releasable) bytesStreamOutput);
             }
 
-            if (isCloseConnection()) {
-                toClose.add(() -> CloseableChannel.closeChannel(httpChannel));
-            }
-
             ActionListener<Void> listener = ActionListener.wrap(() -> Releasables.close(toClose));
             httpChannel.sendResponse(httpResponse, listener);
             success = true;
@@ -127,7 +133,6 @@ public void sendResponse(RestResponse restResponse) {
                 Releasables.close(toClose);
             }
         }
-
     }
 
     private void setHeaderField(HttpResponse response, String headerField, String value) {
diff --git a/server/src/main/java/org/elasticsearch/http/HttpRequest.java b/server/src/main/java/org/elasticsearch/http/HttpRequest.java
index 496fec23312b0..02a3a58d1702d 100644
--- a/server/src/main/java/org/elasticsearch/http/HttpRequest.java
+++ b/server/src/main/java/org/elasticsearch/http/HttpRequest.java
@@ -37,6 +37,12 @@ enum HttpVersion {
         HTTP_1_1
     }
 
+    /**
+     * Returns the HTTP method used in the HTTP request.
+     *
+     * @return the {@link RestRequest.Method} used in the REST request
+     * @throws IllegalArgumentException if the HTTP method is invalid
+     */
     RestRequest.Method method();
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java
index ca0f34803cc0c..6ef335144eb52 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexModule.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java
@@ -19,6 +19,10 @@
 
 package org.elasticsearch.index;
 
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FilterDirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.search.similarities.BM25Similarity;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.store.MMapDirectory;
@@ -26,6 +30,7 @@
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.TriFunction;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
@@ -38,10 +43,10 @@
 import org.elasticsearch.index.cache.query.DisabledQueryCache;
 import org.elasticsearch.index.cache.query.IndexQueryCache;
 import org.elasticsearch.index.cache.query.QueryCache;
+import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineFactory;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.shard.IndexEventListener;
-import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.IndexingOperationListener;
 import org.elasticsearch.index.shard.SearchOperationListener;
 import org.elasticsearch.index.similarity.SimilarityService;
@@ -112,7 +117,8 @@ public final class IndexModule {
     private final IndexSettings indexSettings;
     private final AnalysisRegistry analysisRegistry;
     private final EngineFactory engineFactory;
-    private SetOnce<IndexSearcherWrapperFactory> indexSearcherWrapper = new SetOnce<>();
+    private SetOnce<Function<IndexService,
+        CheckedFunction<DirectoryReader, DirectoryReader, IOException>>> indexReaderWrapper = new SetOnce<>();
     private final Set<IndexEventListener> indexEventListeners = new HashSet<>();
     private final Map<String, TriFunction<Settings, Version, ScriptService, Similarity>> similarities = new HashMap<>();
     private final Map<String, IndexStorePlugin.DirectoryFactory> directoryFactories;
@@ -277,13 +283,26 @@ public void addSimilarity(String name, TriFunction<Settings, Version, ScriptServ
     }
 
     /**
-     * Sets a {@link org.elasticsearch.index.IndexModule.IndexSearcherWrapperFactory} that is called once the IndexService
-     * is fully constructed.
-     * Note: this method can only be called once per index. Multiple wrappers are not supported.
+     * Sets the factory for creating new {@link DirectoryReader} wrapper instances.
+     * The factory ({@link Function}) is called once the IndexService is fully constructed.
+     * NOTE: this method can only be called once per index. Multiple wrappers are not supported.
+     * <p>
+     * The {@link CheckedFunction} is invoked each time a {@link Engine.Searcher} is requested to do an operation,
+     * for example search, and must return a new directory reader wrapping the provided directory reader or if no
+     * wrapping was performed the provided directory reader.
+     * The wrapped reader can filter out document just like delete documents etc. but must not change any term or
+     * document content.
+     * NOTE: The index reader wrapper ({@link CheckedFunction}) has a per-request lifecycle,
+     * must delegate {@link IndexReader#getReaderCacheHelper()}, {@link LeafReader#getCoreCacheHelper()}
+     * and must be an instance of {@link FilterDirectoryReader} that eventually exposes the original reader
+     * via {@link FilterDirectoryReader#getDelegate()}.
+     * The returned reader is closed once it goes out of scope.
+     * </p>
      */
-    public void setSearcherWrapper(IndexSearcherWrapperFactory indexSearcherWrapperFactory) {
+    public void setReaderWrapper(Function<IndexService,
+                                    CheckedFunction<DirectoryReader, DirectoryReader, IOException>> indexReaderWrapperFactory) {
         ensureNotFrozen();
-        this.indexSearcherWrapper.set(indexSearcherWrapperFactory);
+        this.indexReaderWrapper.set(indexReaderWrapperFactory);
     }
 
     IndexEventListener freeze() { // pkg private for testing
@@ -348,16 +367,6 @@ public boolean match(String setting) {
 
     }
 
-    /**
-     * Factory for creating new {@link IndexSearcherWrapper} instances
-     */
-    public interface IndexSearcherWrapperFactory {
-        /**
-         * Returns a new IndexSearcherWrapper. This method is called once per index per node
-         */
-        IndexSearcherWrapper newWrapper(IndexService indexService);
-    }
-
     public static Type defaultStoreType(final boolean allowMmap) {
         if (allowMmap && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
             return Type.HYBRIDFS;
@@ -384,8 +393,8 @@ public IndexService newIndexService(
             NamedWriteableRegistry namedWriteableRegistry)
         throws IOException {
         final IndexEventListener eventListener = freeze();
-        IndexSearcherWrapperFactory searcherWrapperFactory = indexSearcherWrapper.get() == null
-            ? (shard) -> null : indexSearcherWrapper.get();
+        Function<IndexService, CheckedFunction<DirectoryReader, DirectoryReader, IOException>> readerWrapperFactory =
+            indexReaderWrapper.get() == null ? (shard) -> null : indexReaderWrapper.get();
         eventListener.beforeIndexCreated(indexSettings.getIndex(), indexSettings.getSettings());
         final IndexStorePlugin.DirectoryFactory directoryFactory = getDirectoryFactory(indexSettings, directoryFactories);
         final QueryCache queryCache;
@@ -402,7 +411,7 @@ public IndexService newIndexService(
         return new IndexService(indexSettings, indexCreationContext, environment, xContentRegistry,
                 new SimilarityService(indexSettings, scriptService, similarities),
                 shardStoreDeleter, analysisRegistry, engineFactory, circuitBreakerService, bigArrays, threadPool, scriptService,
-                client, queryCache, directoryFactory, eventListener, searcherWrapperFactory, mapperRegistry,
+                client, queryCache, directoryFactory, eventListener, readerWrapperFactory, mapperRegistry,
                 indicesFieldDataCache, searchOperationListeners, indexOperationListeners, namedWriteableRegistry);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java
index 2d86a2b436d9d..c6c5a7b90106a 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexService.java
@@ -20,16 +20,19 @@
 package org.elasticsearch.index;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReaderContext;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Accountable;
 import org.elasticsearch.Assertions;
-import org.elasticsearch.Version;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
@@ -58,7 +61,6 @@
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.seqno.RetentionLeaseSyncer;
 import org.elasticsearch.index.shard.IndexEventListener;
-import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardClosedException;
 import org.elasticsearch.index.shard.IndexingOperationListener;
@@ -90,6 +92,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
+import java.util.function.Function;
 import java.util.function.LongSupplier;
 import java.util.function.Supplier;
 
@@ -104,7 +107,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     private final NodeEnvironment nodeEnv;
     private final ShardStoreDeleter shardStoreDeleter;
     private final IndexStorePlugin.DirectoryFactory directoryFactory;
-    private final IndexSearcherWrapper searcherWrapper;
+    private final CheckedFunction<DirectoryReader, DirectoryReader, IOException> readerWrapper;
     private final IndexCache indexCache;
     private final MapperService mapperService;
     private final NamedXContentRegistry xContentRegistry;
@@ -151,7 +154,7 @@ public IndexService(
             QueryCache queryCache,
             IndexStorePlugin.DirectoryFactory directoryFactory,
             IndexEventListener eventListener,
-            IndexModule.IndexSearcherWrapperFactory wrapperFactory,
+            Function<IndexService, CheckedFunction<DirectoryReader, DirectoryReader, IOException>> wrapperFactory,
             MapperRegistry mapperRegistry,
             IndicesFieldDataCache indicesFieldDataCache,
             List<SearchOperationListener> searchOperationListeners,
@@ -203,7 +206,7 @@ public IndexService(
         this.directoryFactory = directoryFactory;
         this.engineFactory = Objects.requireNonNull(engineFactory);
         // initialize this last -- otherwise if the wrapper requires any other member to be non-null we fail with an NPE
-        this.searcherWrapper = wrapperFactory.newWrapper(this);
+        this.readerWrapper = wrapperFactory.apply(this);
         this.searchOperationListeners = Collections.unmodifiableList(searchOperationListeners);
         this.indexingOperationListeners = Collections.unmodifiableList(indexingOperationListeners);
         // kick off async ops for the first shard in this index
@@ -416,7 +419,7 @@ public synchronized IndexShard createShard(
                     similarityService,
                     engineFactory,
                     eventListener,
-                    searcherWrapper,
+                    readerWrapper,
                     threadPool,
                     bigArrays,
                     engineWarmer,
@@ -518,19 +521,24 @@ public IndexSettings getIndexSettings() {
         return indexSettings;
     }
 
+    private IndexSearcher newCachedSearcher(int shardId, IndexReaderContext context) {
+        IndexSearcher searcher = new IndexSearcher(context);
+        searcher.setQueryCache(cache().query());
+        searcher.setQueryCachingPolicy(getShard(shardId).getQueryCachingPolicy());
+        return searcher;
+    }
+
     /**
-     * Creates a new QueryShardContext. The context has not types set yet, if types are required set them via
-     * {@link QueryShardContext#setTypes(String...)}.
+     * Creates a new QueryShardContext.
      *
-     * Passing a {@code null} {@link IndexReader} will return a valid context, however it won't be able to make
-     * {@link IndexReader}-specific optimizations, such as rewriting containing range queries.
+     * Passing a {@code null} {@link IndexReader} will return a valid context, however it won't be able to make {@link IndexReader}-specific
+     * optimizations, such as rewriting containing range queries.
      */
     public QueryShardContext newQueryShardContext(int shardId, IndexReader indexReader, LongSupplier nowInMillis, String clusterAlias) {
         return new QueryShardContext(
-            shardId, indexSettings, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(),
-                similarityService(), scriptService, xContentRegistry,
-               namedWriteableRegistry, client, indexReader,
-            nowInMillis, clusterAlias);
+            shardId, indexSettings, indexCache.bitsetFilterCache(), context -> newCachedSearcher(shardId, context),
+            indexFieldData::getForField, mapperService(), similarityService(), scriptService, xContentRegistry, namedWriteableRegistry,
+            client, indexReader, nowInMillis, clusterAlias);
     }
 
     /**
@@ -661,9 +669,7 @@ public IndexMetaData getMetaData() {
     public synchronized void updateMetaData(final IndexMetaData currentIndexMetaData, final IndexMetaData newIndexMetaData) {
         final boolean updateIndexMetaData = indexSettings.updateIndexMetaData(newIndexMetaData);
 
-        if (Assertions.ENABLED
-                && currentIndexMetaData != null
-                && currentIndexMetaData.getCreationVersion().onOrAfter(Version.V_6_5_0)) {
+        if (Assertions.ENABLED && currentIndexMetaData != null) {
             final long currentSettingsVersion = currentIndexMetaData.getSettingsVersion();
             final long newSettingsVersion = newIndexMetaData.getSettingsVersion();
             if (currentSettingsVersion == newSettingsVersion) {
@@ -748,8 +754,8 @@ public final EngineFactory getEngineFactory() {
         return engineFactory;
     }
 
-    final IndexSearcherWrapper getSearcherWrapper() {
-        return searcherWrapper;
+    final CheckedFunction<DirectoryReader, DirectoryReader, IOException> getReaderWrapper() {
+        return readerWrapper;
     } // pkg private for testing
 
     final IndexStorePlugin.DirectoryFactory getDirectoryFactory() {
@@ -934,6 +940,11 @@ final class AsyncTrimTranslogTask extends BaseAsyncTask {
                 .getSettings().getAsTime(INDEX_TRANSLOG_RETENTION_CHECK_INTERVAL_SETTING, TimeValue.timeValueMinutes(10)));
         }
 
+        @Override
+        protected boolean mustReschedule() {
+            return indexService.closed.get() == false;
+        }
+
         @Override
         protected void runInternal() {
             indexService.maybeTrimTranslog();
@@ -1025,8 +1036,8 @@ AsyncTranslogFSync getFsyncTask() { // for tests
         return fsyncTask;
     }
 
-    AsyncGlobalCheckpointTask getGlobalCheckpointTask() {
-        return globalCheckpointTask;
+    AsyncTrimTranslogTask getTrimTranslogTask() { // for tests
+        return trimTranslogTask;
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
index d4cc38f0b959c..ebcf362b1c9e4 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -458,7 +458,7 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti
         generationThresholdSize = scopedSettings.get(INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING);
         mergeSchedulerConfig = new MergeSchedulerConfig(this);
         gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis();
-        softDeleteEnabled = version.onOrAfter(Version.V_6_5_0) && scopedSettings.get(INDEX_SOFT_DELETES_SETTING);
+        softDeleteEnabled = scopedSettings.get(INDEX_SOFT_DELETES_SETTING);
         softDeleteRetentionOperations = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING);
         retentionLeaseMillis = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING).millis();
         warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING);
diff --git a/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java b/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java
index abd67a47049d3..a2193a1bdf73d 100644
--- a/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java
+++ b/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.index;
 
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
@@ -170,13 +170,6 @@ public String toString() {
                 sb.append("-1");
             }
             sb.append("], ");
-            if (context.getQueryShardContext().getTypes() == null) {
-                sb.append("types[], ");
-            } else {
-                sb.append("types[");
-                Strings.arrayToDelimitedString(context.getQueryShardContext().getTypes(), ",", sb);
-                sb.append("], ");
-            }
             if (context.groupStats() == null) {
                 sb.append("stats[], ");
             } else {
diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
index 684d36c311f8b..755266604add5 100644
--- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
+++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
@@ -21,6 +21,7 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.internal.io.IOUtils;
@@ -33,11 +34,15 @@
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -86,13 +91,7 @@ public AnalysisRegistry(Environment environment,
             new PrebuiltAnalysis(preConfiguredCharFilters, preConfiguredTokenFilters, preConfiguredTokenizers, preConfiguredAnalyzers);
     }
 
-    /**
-     * Returns a {@link Settings} by groupName from {@link IndexSettings} or a default {@link Settings}
-     * @param indexSettings an index settings
-     * @param groupName tokenizer/token filter/char filter name
-     * @return {@link Settings}
-     */
-    public static Settings getSettingsFromIndexSettings(IndexSettings indexSettings, String groupName) {
+    private static Settings getSettingsFromIndexSettings(IndexSettings indexSettings, String groupName) {
         Settings settings = indexSettings.getSettings().getAsSettings(groupName);
         if (settings.isEmpty()) {
             settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, indexSettings.getIndexVersionCreated()).build();
@@ -100,24 +99,75 @@ public static Settings getSettingsFromIndexSettings(IndexSettings indexSettings,
         return settings;
     }
 
+    private static final IndexSettings NO_INDEX_SETTINGS = new IndexSettings(
+        IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE)
+            .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+            .numberOfReplicas(0)
+            .numberOfShards(1)
+            .build(),
+        Settings.EMPTY
+    );
+
+    private <T> T getComponentFactory(IndexSettings settings, NameOrDefinition nod,
+                                      String componentType,
+                                      Function<String, AnalysisProvider<T>> globalComponentProvider,
+                                      Function<String, AnalysisProvider<T>> prebuiltComponentProvider,
+                                      BiFunction<String, IndexSettings, AnalysisProvider<T>> indexComponentProvider) throws IOException {
+        if (nod.definition != null) {
+            // custom component, so we build it from scratch
+            String type = nod.definition.get("type");
+            if (type == null) {
+                throw new IllegalArgumentException("Missing [type] setting for anonymous " + componentType + ": " + nod.definition);
+            }
+            AnalysisProvider<T> factory = globalComponentProvider.apply(type);
+            if (factory == null) {
+                throw new IllegalArgumentException("failed to find global " + componentType + " under [" + type + "]");
+            }
+            if (settings == null) {
+                settings = NO_INDEX_SETTINGS;
+            }
+            return factory.get(settings, environment, "__anonymous__" + type, nod.definition);
+        }
+        if (settings == null) {
+            // no index provided, so we use prebuilt analysis components
+            AnalysisProvider<T> factory = prebuiltComponentProvider.apply(nod.name);
+            if (factory == null) {
+                // if there's no prebuilt component, try loading a global one to build with no settings
+                factory = globalComponentProvider.apply(nod.name);
+                if (factory == null) {
+                    throw new IllegalArgumentException("failed to find global " + componentType + " under [" + nod.name + "]");
+                }
+            }
+            return factory.get(environment, nod.name);
+        } else {
+            // get the component from index settings
+            AnalysisProvider<T> factory = indexComponentProvider.apply(nod.name, settings);
+            if (factory == null) {
+                throw new IllegalArgumentException("failed to find " + componentType + " under [" + nod.name + "]");
+            }
+            Settings s = getSettingsFromIndexSettings(settings, "index.analysis." + componentType + "." + nod.name);
+            return factory.get(settings, environment, nod.name, s);
+        }
+    }
+
     /**
      * Returns a registered {@link TokenizerFactory} provider by name or <code>null</code> if the tokenizer was not registered
      */
-    public AnalysisModule.AnalysisProvider<TokenizerFactory> getTokenizerProvider(String tokenizer) {
+    private AnalysisModule.AnalysisProvider<TokenizerFactory> getTokenizerProvider(String tokenizer) {
         return tokenizers.getOrDefault(tokenizer, this.prebuiltAnalysis.getTokenizerFactory(tokenizer));
     }
 
     /**
      * Returns a registered {@link TokenFilterFactory} provider by name or <code>null</code> if the token filter was not registered
      */
-    public AnalysisModule.AnalysisProvider<TokenFilterFactory> getTokenFilterProvider(String tokenFilter) {
+    private AnalysisModule.AnalysisProvider<TokenFilterFactory> getTokenFilterProvider(String tokenFilter) {
         return tokenFilters.getOrDefault(tokenFilter, this.prebuiltAnalysis.getTokenFilterFactory(tokenFilter));
     }
 
     /**
      * Returns a registered {@link CharFilterFactory} provider by name or <code>null</code> if the char filter was not registered
      */
-    public AnalysisModule.AnalysisProvider<CharFilterFactory> getCharFilterProvider(String charFilter) {
+    private AnalysisModule.AnalysisProvider<CharFilterFactory> getCharFilterProvider(String charFilter) {
         return charFilters.getOrDefault(charFilter, this.prebuiltAnalysis.getCharFilterFactory(charFilter));
     }
 
@@ -135,11 +185,7 @@ public Analyzer getAnalyzer(String analyzer) throws IOException {
                             throw new ElasticsearchException("failed to load analyzer for name " + key, ex);
                         }}
             );
-        } else if ("standard_html_strip".equals(analyzer)) {
-            throw new IllegalArgumentException("[standard_html_strip] analyzer is not supported for new indices, " +
-                "use a custom analyzer using [standard] tokenizer and [html_strip] char_filter, plus [lowercase] filter");
         }
-
         return analyzerProvider.get(environment, analyzer).get();
     }
 
@@ -164,6 +210,67 @@ public IndexAnalyzers build(IndexSettings indexSettings) throws IOException {
         return build(indexSettings, analyzerFactories, normalizerFactories, tokenizerFactories, charFilterFactories, tokenFilterFactories);
     }
 
+    /**
+     * Creates a custom analyzer from a collection of {@link NameOrDefinition} specifications for each component
+     *
+     * Callers are responsible for closing the returned Analyzer
+     */
+    public NamedAnalyzer buildCustomAnalyzer(IndexSettings indexSettings, boolean normalizer, NameOrDefinition tokenizer,
+                                             List<NameOrDefinition> charFilters, List<NameOrDefinition> tokenFilters) throws IOException {
+        TokenizerFactory tokenizerFactory
+            = getComponentFactory(indexSettings, tokenizer, "tokenizer",
+            this::getTokenizerProvider, prebuiltAnalysis::getTokenizerFactory, this::getTokenizerProvider);
+
+        List<CharFilterFactory> charFilterFactories = new ArrayList<>();
+        for (NameOrDefinition nod : charFilters) {
+            charFilterFactories.add(getComponentFactory(indexSettings, nod, "char_filter",
+                this::getCharFilterProvider, prebuiltAnalysis::getCharFilterFactory, this::getCharFilterProvider));
+        }
+
+        List<TokenFilterFactory> tokenFilterFactories = new ArrayList<>();
+        for (NameOrDefinition nod : tokenFilters) {
+            TokenFilterFactory tff = getComponentFactory(indexSettings, nod, "filter",
+                this::getTokenFilterProvider, prebuiltAnalysis::getTokenFilterFactory, this::getTokenFilterProvider);
+            if (normalizer && tff instanceof NormalizingTokenFilterFactory == false) {
+                throw new IllegalArgumentException("Custom normalizer may not use filter [" + tff.name() + "]");
+            }
+            tff = tff.getChainAwareTokenFilterFactory(tokenizerFactory, charFilterFactories, tokenFilterFactories, name -> {
+                try {
+                    return getComponentFactory(indexSettings, new NameOrDefinition(name), "filter",
+                        this::getTokenFilterProvider, prebuiltAnalysis::getTokenFilterFactory, this::getTokenFilterProvider);
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            });
+            tokenFilterFactories.add(tff);
+        }
+
+        String tokenizerName = tokenizer.name == null ? "_anonymous_tokenizer" : tokenizer.name;
+        if (normalizer) {
+            tokenizerName = "keyword_for_normalizer";
+        }
+        Analyzer analyzer = new CustomAnalyzer(tokenizerName, tokenizerFactory,
+            charFilterFactories.toArray(new CharFilterFactory[]{}),
+            tokenFilterFactories.toArray(new TokenFilterFactory[]{}));
+        return produceAnalyzer("__custom__", new AnalyzerProvider<>() {
+            @Override
+            public String name() {
+                return "__custom__";
+            }
+
+            @Override
+            public AnalyzerScope scope() {
+                return AnalyzerScope.GLOBAL;
+            }
+
+            @Override
+            public Analyzer get() {
+                return analyzer;
+            }
+        }, null, null, null);
+
+    }
+
     public Map<String, TokenFilterFactory> buildTokenFilterFactories(IndexSettings indexSettings) throws IOException {
         final Map<String, Settings> tokenFiltersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_FILTER);
         return buildMapping(Component.FILTER, indexSettings, tokenFiltersSettings, this.tokenFilters,
@@ -181,12 +288,12 @@ public Map<String, CharFilterFactory> buildCharFilterFactories(IndexSettings ind
             prebuiltAnalysis.preConfiguredCharFilterFactories);
     }
 
-    public Map<String, AnalyzerProvider<?>> buildAnalyzerFactories(IndexSettings indexSettings) throws IOException {
+    private Map<String, AnalyzerProvider<?>> buildAnalyzerFactories(IndexSettings indexSettings) throws IOException {
         final Map<String, Settings> analyzersSettings = indexSettings.getSettings().getGroups("index.analysis.analyzer");
         return buildMapping(Component.ANALYZER, indexSettings, analyzersSettings, analyzers, prebuiltAnalysis.analyzerProviderFactories);
     }
 
-    public Map<String, AnalyzerProvider<?>> buildNormalizerFactories(IndexSettings indexSettings) throws IOException {
+    private Map<String, AnalyzerProvider<?>> buildNormalizerFactories(IndexSettings indexSettings) throws IOException {
         final Map<String, Settings> normalizersSettings = indexSettings.getSettings().getGroups("index.analysis.normalizer");
         // TODO: Have pre-built normalizers
         return buildMapping(Component.NORMALIZER, indexSettings, normalizersSettings, normalizers, Collections.emptyMap());
@@ -200,7 +307,7 @@ public Map<String, AnalyzerProvider<?>> buildNormalizerFactories(IndexSettings i
      * @param indexSettings an index settings
      * @return {@link TokenizerFactory} provider or <code>null</code>
      */
-    public AnalysisProvider<TokenizerFactory> getTokenizerProvider(String tokenizer, IndexSettings indexSettings) {
+    private AnalysisProvider<TokenizerFactory> getTokenizerProvider(String tokenizer, IndexSettings indexSettings) {
         return getProvider(Component.TOKENIZER, tokenizer, indexSettings, "index.analysis.tokenizer", tokenizers,
                 this::getTokenizerProvider);
     }
@@ -213,7 +320,7 @@ public AnalysisProvider<TokenizerFactory> getTokenizerProvider(String tokenizer,
      * @param indexSettings an index settings
      * @return {@link TokenFilterFactory} provider or <code>null</code>
      */
-    public AnalysisProvider<TokenFilterFactory> getTokenFilterProvider(String tokenFilter, IndexSettings indexSettings) {
+    private AnalysisProvider<TokenFilterFactory> getTokenFilterProvider(String tokenFilter, IndexSettings indexSettings) {
         return getProvider(Component.FILTER, tokenFilter, indexSettings, "index.analysis.filter", tokenFilters,
                 this::getTokenFilterProvider);
     }
@@ -226,7 +333,7 @@ public AnalysisProvider<TokenFilterFactory> getTokenFilterProvider(String tokenF
      * @param indexSettings an index settings
      * @return {@link CharFilterFactory} provider or <code>null</code>
      */
-    public AnalysisProvider<CharFilterFactory> getCharFilterProvider(String charFilter, IndexSettings indexSettings) {
+    private AnalysisProvider<CharFilterFactory> getCharFilterProvider(String charFilter, IndexSettings indexSettings) {
         return getProvider(Component.CHAR_FILTER, charFilter, indexSettings, "index.analysis.char_filter", charFilters,
                 this::getCharFilterProvider);
     }
@@ -385,19 +492,19 @@ private PrebuiltAnalysis(
             this.preConfiguredTokenizers = preConfiguredTokenizers;
         }
 
-        public AnalysisProvider<CharFilterFactory> getCharFilterFactory(String name) {
+        AnalysisProvider<CharFilterFactory> getCharFilterFactory(String name) {
             return preConfiguredCharFilterFactories.get(name);
         }
 
-        public AnalysisProvider<TokenFilterFactory> getTokenFilterFactory(String name) {
+        AnalysisProvider<TokenFilterFactory> getTokenFilterFactory(String name) {
             return preConfiguredTokenFilters.get(name);
         }
 
-        public AnalysisProvider<TokenizerFactory> getTokenizerFactory(String name) {
+        AnalysisProvider<TokenizerFactory> getTokenizerFactory(String name) {
             return preConfiguredTokenizers.get(name);
         }
 
-        public AnalysisProvider<AnalyzerProvider<?>> getAnalyzerProvider(String name) {
+        AnalysisProvider<AnalyzerProvider<?>> getAnalyzerProvider(String name) {
             return analyzerProviderFactories.get(name);
         }
 
@@ -414,7 +521,6 @@ public IndexAnalyzers build(IndexSettings indexSettings,
                                 Map<String, TokenizerFactory> tokenizerFactoryFactories,
                                 Map<String, CharFilterFactory> charFilterFactoryFactories,
                                 Map<String, TokenFilterFactory> tokenFilterFactoryFactories) {
-
         Map<String, NamedAnalyzer> analyzers = new HashMap<>();
         Map<String, NamedAnalyzer> normalizers = new HashMap<>();
         Map<String, NamedAnalyzer> whitespaceNormalizers = new HashMap<>();
@@ -453,12 +559,14 @@ public IndexAnalyzers build(IndexSettings indexSettings,
                 throw new IllegalArgumentException("analyzer name must not start with '_'. got \"" + analyzer.getKey() + "\"");
             }
         }
-        return new IndexAnalyzers(indexSettings, analyzers, normalizers, whitespaceNormalizers);
+        return new IndexAnalyzers(analyzers, normalizers, whitespaceNormalizers);
     }
 
-    private static NamedAnalyzer produceAnalyzer(String name, AnalyzerProvider<?> analyzerFactory,
-            Map<String, TokenFilterFactory> tokenFilters, Map<String, CharFilterFactory> charFilters,
-            Map<String, TokenizerFactory> tokenizers) {
+    private static NamedAnalyzer produceAnalyzer(String name,
+                                        AnalyzerProvider<?> analyzerFactory,
+                                        Map<String, TokenFilterFactory> tokenFilters,
+                                        Map<String, CharFilterFactory> charFilters,
+                                        Map<String, TokenizerFactory> tokenizers) {
         /*
          * Lucene defaults positionIncrementGap to 0 in all analyzers but
          * Elasticsearch defaults them to 0 only before version 2.0
diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalyzerComponents.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalyzerComponents.java
new file mode 100644
index 0000000000000..f150ac54558e0
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalyzerComponents.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A class that groups analysis components necessary to produce a custom analyzer.
+ * See {@link ReloadableCustomAnalyzer} for an example usage.
+ */
+public final class AnalyzerComponents {
+    private final String tokenizerName;
+    private final TokenizerFactory tokenizerFactory;
+    private final CharFilterFactory[] charFilters;
+    private final TokenFilterFactory[] tokenFilters;
+    private final AnalysisMode analysisMode;
+
+    AnalyzerComponents(String tokenizerName, TokenizerFactory tokenizerFactory, CharFilterFactory[] charFilters,
+                       TokenFilterFactory[] tokenFilters) {
+        this.tokenizerName = tokenizerName;
+        this.tokenizerFactory = tokenizerFactory;
+        this.charFilters = charFilters;
+        this.tokenFilters = tokenFilters;
+        AnalysisMode mode = AnalysisMode.ALL;
+        for (TokenFilterFactory f : tokenFilters) {
+            mode = mode.merge(f.getAnalysisMode());
+        }
+        this.analysisMode = mode;
+    }
+
+    static AnalyzerComponents createComponents(String name, Settings analyzerSettings, final Map<String, TokenizerFactory> tokenizers,
+            final Map<String, CharFilterFactory> charFilters, final Map<String, TokenFilterFactory> tokenFilters) {
+        String tokenizerName = analyzerSettings.get("tokenizer");
+        if (tokenizerName == null) {
+            throw new IllegalArgumentException("Custom Analyzer [" + name + "] must be configured with a tokenizer");
+        }
+
+        TokenizerFactory tokenizer = tokenizers.get(tokenizerName);
+        if (tokenizer == null) {
+            throw new IllegalArgumentException(
+                    "Custom Analyzer [" + name + "] failed to find tokenizer under name " + "[" + tokenizerName + "]");
+        }
+
+        List<String> charFilterNames = analyzerSettings.getAsList("char_filter");
+        List<CharFilterFactory> charFiltersList = new ArrayList<>(charFilterNames.size());
+        for (String charFilterName : charFilterNames) {
+            CharFilterFactory charFilter = charFilters.get(charFilterName);
+            if (charFilter == null) {
+                throw new IllegalArgumentException(
+                        "Custom Analyzer [" + name + "] failed to find char_filter under name " + "[" + charFilterName + "]");
+            }
+            charFiltersList.add(charFilter);
+        }
+
+        List<String> tokenFilterNames = analyzerSettings.getAsList("filter");
+        List<TokenFilterFactory> tokenFilterList = new ArrayList<>(tokenFilterNames.size());
+        for (String tokenFilterName : tokenFilterNames) {
+            TokenFilterFactory tokenFilter = tokenFilters.get(tokenFilterName);
+            if (tokenFilter == null) {
+                throw new IllegalArgumentException(
+                        "Custom Analyzer [" + name + "] failed to find filter under name " + "[" + tokenFilterName + "]");
+            }
+            tokenFilter = tokenFilter.getChainAwareTokenFilterFactory(tokenizer, charFiltersList, tokenFilterList, tokenFilters::get);
+            tokenFilterList.add(tokenFilter);
+        }
+
+        return new AnalyzerComponents(tokenizerName, tokenizer, charFiltersList.toArray(new CharFilterFactory[charFiltersList.size()]),
+                tokenFilterList.toArray(new TokenFilterFactory[tokenFilterList.size()]));
+    }
+
+    public String getTokenizerName() {
+        return tokenizerName;
+    }
+
+    public TokenizerFactory getTokenizerFactory() {
+        return tokenizerFactory;
+    }
+
+    public TokenFilterFactory[] getTokenFilters() {
+        return tokenFilters;
+    }
+
+    public CharFilterFactory[] getCharFilters() {
+        return charFilters;
+    }
+
+    public AnalysisMode analysisMode() {
+        return this.analysisMode;
+    }
+}
\ No newline at end of file
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/DenseVectorFieldTypeTests.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalyzerComponentsProvider.java
similarity index 76%
rename from modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/DenseVectorFieldTypeTests.java
rename to server/src/main/java/org/elasticsearch/index/analysis/AnalyzerComponentsProvider.java
index 7bbf862e0fc2b..84a3a14038f92 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/DenseVectorFieldTypeTests.java
+++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalyzerComponentsProvider.java
@@ -17,12 +17,13 @@
  * under the License.
  */
 
-package org.elasticsearch.index.mapper;
+package org.elasticsearch.index.analysis;
 
-public class DenseVectorFieldTypeTests extends FieldTypeTestCase {
+/**
+ * Analyzers that provide access to their token filters should implement this
+ */
+public interface AnalyzerComponentsProvider {
+
+    AnalyzerComponents getComponents();
 
-    @Override
-    protected MappedFieldType createDefaultFieldType() {
-        return new DenseVectorFieldMapper.DenseVectorFieldType();
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java
index a41ee33564400..685dd2a7de036 100644
--- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java
+++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java
@@ -25,15 +25,9 @@
 
 import java.io.Reader;
 
-public final class CustomAnalyzer extends Analyzer {
-
-    private final String tokenizerName;
-    private final TokenizerFactory tokenizerFactory;
-
-    private final CharFilterFactory[] charFilters;
-
-    private final TokenFilterFactory[] tokenFilters;
+public final class CustomAnalyzer extends Analyzer implements AnalyzerComponentsProvider {
 
+    private final AnalyzerComponents components;
     private final int positionIncrementGap;
     private final int offsetGap;
     private final AnalysisMode analysisMode;
@@ -45,10 +39,7 @@ public CustomAnalyzer(String tokenizerName, TokenizerFactory tokenizerFactory, C
 
     public CustomAnalyzer(String tokenizerName, TokenizerFactory tokenizerFactory, CharFilterFactory[] charFilters,
             TokenFilterFactory[] tokenFilters, int positionIncrementGap, int offsetGap) {
-        this.tokenizerName = tokenizerName;
-        this.tokenizerFactory = tokenizerFactory;
-        this.charFilters = charFilters;
-        this.tokenFilters = tokenFilters;
+        this.components = new AnalyzerComponents(tokenizerName, tokenizerFactory, charFilters, tokenFilters);
         this.positionIncrementGap = positionIncrementGap;
         this.offsetGap = offsetGap;
         // merge and transfer token filter analysis modes with analyzer
@@ -63,19 +54,19 @@ public CustomAnalyzer(String tokenizerName, TokenizerFactory tokenizerFactory, C
      * The name of the tokenizer as configured by the user.
      */
     public String getTokenizerName() {
-        return tokenizerName;
+        return this.components.getTokenizerName();
     }
 
     public TokenizerFactory tokenizerFactory() {
-        return tokenizerFactory;
+        return this.components.getTokenizerFactory();
     }
 
     public TokenFilterFactory[] tokenFilters() {
-        return tokenFilters;
+        return this.components.getTokenFilters();
     }
 
     public CharFilterFactory[] charFilters() {
-        return charFilters;
+        return this.components.getCharFilters();
     }
 
     @Override
@@ -95,11 +86,16 @@ public AnalysisMode getAnalysisMode() {
         return this.analysisMode;
     }
 
+    @Override
+    public AnalyzerComponents getComponents() {
+        return this.components;
+    }
+
     @Override
     protected TokenStreamComponents createComponents(String fieldName) {
-        Tokenizer tokenizer = tokenizerFactory.create();
+        Tokenizer tokenizer = this.tokenizerFactory().create();
         TokenStream tokenStream = tokenizer;
-        for (TokenFilterFactory tokenFilter : tokenFilters) {
+        for (TokenFilterFactory tokenFilter : tokenFilters()) {
             tokenStream = tokenFilter.create(tokenStream);
         }
         return new TokenStreamComponents(tokenizer, tokenStream);
@@ -107,6 +103,7 @@ protected TokenStreamComponents createComponents(String fieldName) {
 
     @Override
     protected Reader initReader(String fieldName, Reader reader) {
+        CharFilterFactory[] charFilters = charFilters();
         if (charFilters != null && charFilters.length > 0) {
             for (CharFilterFactory charFilter : charFilters) {
                 reader = charFilter.create(reader);
@@ -117,18 +114,18 @@ protected Reader initReader(String fieldName, Reader reader) {
 
     @Override
     protected Reader initReaderForNormalization(String fieldName, Reader reader) {
-      for (CharFilterFactory charFilter : charFilters) {
-          reader = charFilter.normalize(reader);
-      }
-      return reader;
+        for (CharFilterFactory charFilter : charFilters()) {
+            reader = charFilter.normalize(reader);
+        }
+        return reader;
     }
 
     @Override
     protected TokenStream normalize(String fieldName, TokenStream in) {
-      TokenStream result = in;
-      for (TokenFilterFactory filter : tokenFilters) {
-          result = filter.normalize(result);
-      }
-      return result;
+        TokenStream result = in;
+        for (TokenFilterFactory filter : tokenFilters()) {
+            result = filter.normalize(result);
+        }
+        return result;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java
index 8080a6af876a4..d8a50838e9df4 100644
--- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java
+++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzerProvider.java
@@ -19,23 +19,24 @@
 
 package org.elasticsearch.index.analysis;
 
+import org.apache.lucene.analysis.Analyzer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.index.analysis.AnalyzerComponents.createComponents;
+
 /**
  * A custom analyzer that is built out of a single {@link org.apache.lucene.analysis.Tokenizer} and a list
  * of {@link org.apache.lucene.analysis.TokenFilter}s.
  */
-public class CustomAnalyzerProvider extends AbstractIndexAnalyzerProvider<CustomAnalyzer> {
+public class CustomAnalyzerProvider extends AbstractIndexAnalyzerProvider<Analyzer> {
 
     private final Settings analyzerSettings;
 
-    private CustomAnalyzer customAnalyzer;
+    private Analyzer customAnalyzer;
 
     public CustomAnalyzerProvider(IndexSettings indexSettings,
                                   String name, Settings settings) {
@@ -43,58 +44,33 @@ public CustomAnalyzerProvider(IndexSettings indexSettings,
         this.analyzerSettings = settings;
     }
 
-    void build(final Map<String, TokenizerFactory> tokenizers, final Map<String, CharFilterFactory> charFilters,
-                      final Map<String, TokenFilterFactory> tokenFilters) {
-        String tokenizerName = analyzerSettings.get("tokenizer");
-        if (tokenizerName == null) {
-            throw new IllegalArgumentException("Custom Analyzer [" + name() + "] must be configured with a tokenizer");
-        }
-
-        TokenizerFactory tokenizer = tokenizers.get(tokenizerName);
-        if (tokenizer == null) {
-            throw new IllegalArgumentException("Custom Analyzer [" + name() + "] failed to find tokenizer under name " +
-                "[" + tokenizerName + "]");
-        }
-
-        List<String> charFilterNames = analyzerSettings.getAsList("char_filter");
-        List<CharFilterFactory> charFiltersList = new ArrayList<>(charFilterNames.size());
-        for (String charFilterName : charFilterNames) {
-            CharFilterFactory charFilter = charFilters.get(charFilterName);
-            if (charFilter == null) {
-                throw new IllegalArgumentException("Custom Analyzer [" + name() + "] failed to find char_filter under name " +
-                    "[" + charFilterName + "]");
-            }
-            charFiltersList.add(charFilter);
-        }
+    void build(final Map<String, TokenizerFactory> tokenizers,
+               final Map<String, CharFilterFactory> charFilters,
+               final Map<String, TokenFilterFactory> tokenFilters) {
+        customAnalyzer = create(name(), analyzerSettings, tokenizers, charFilters, tokenFilters);
+    }
 
+    /**
+     * Factory method that either returns a plain {@link ReloadableCustomAnalyzer} if the components used for creation are supporting index
+     * and search time use, or a {@link ReloadableCustomAnalyzer} if the components are intended for search time use only.
+     */
+    private static Analyzer create(String name, Settings analyzerSettings, Map<String, TokenizerFactory> tokenizers,
+            Map<String, CharFilterFactory> charFilters,
+            Map<String, TokenFilterFactory> tokenFilters) {
         int positionIncrementGap = TextFieldMapper.Defaults.POSITION_INCREMENT_GAP;
-
         positionIncrementGap = analyzerSettings.getAsInt("position_increment_gap", positionIncrementGap);
-
         int offsetGap = analyzerSettings.getAsInt("offset_gap", -1);
-
-        List<String> tokenFilterNames = analyzerSettings.getAsList("filter");
-        List<TokenFilterFactory> tokenFilterList = new ArrayList<>(tokenFilterNames.size());
-        for (String tokenFilterName : tokenFilterNames) {
-            TokenFilterFactory tokenFilter = tokenFilters.get(tokenFilterName);
-            if (tokenFilter == null) {
-                throw new IllegalArgumentException("Custom Analyzer [" + name() + "] failed to find filter under name " +
-                    "[" + tokenFilterName + "]");
-            }
-            tokenFilter = tokenFilter.getChainAwareTokenFilterFactory(tokenizer, charFiltersList, tokenFilterList, tokenFilters::get);
-            tokenFilterList.add(tokenFilter);
+        AnalyzerComponents components = createComponents(name, analyzerSettings, tokenizers, charFilters, tokenFilters);
+        if (components.analysisMode().equals(AnalysisMode.SEARCH_TIME)) {
+            return new ReloadableCustomAnalyzer(components, positionIncrementGap, offsetGap);
+        } else {
+            return new CustomAnalyzer(components.getTokenizerName(), components.getTokenizerFactory(), components.getCharFilters(),
+                    components.getTokenFilters(), positionIncrementGap, offsetGap);
         }
-
-        this.customAnalyzer = new CustomAnalyzer(tokenizerName, tokenizer,
-                charFiltersList.toArray(new CharFilterFactory[charFiltersList.size()]),
-                tokenFilterList.toArray(new TokenFilterFactory[tokenFilterList.size()]),
-                positionIncrementGap,
-                offsetGap
-        );
     }
 
     @Override
-    public CustomAnalyzer get() {
+    public Analyzer get() {
         return this.customAnalyzer;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java
index 4f1cbeb4022ac..a44ade8d15391 100644
--- a/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java
+++ b/server/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java
@@ -19,13 +19,12 @@
 package org.elasticsearch.index.analysis;
 
 import org.elasticsearch.core.internal.io.IOUtils;
-import org.elasticsearch.index.AbstractIndexComponent;
-import org.elasticsearch.index.IndexSettings;
 
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static java.util.Collections.unmodifiableMap;
@@ -40,14 +39,13 @@
  *
  * @see AnalysisRegistry
  */
-public final class IndexAnalyzers extends AbstractIndexComponent implements Closeable {
+public final class IndexAnalyzers implements Closeable {
     private final Map<String, NamedAnalyzer> analyzers;
     private final Map<String, NamedAnalyzer> normalizers;
     private final Map<String, NamedAnalyzer> whitespaceNormalizers;
 
-    public IndexAnalyzers(IndexSettings indexSettings, Map<String, NamedAnalyzer> analyzers, Map<String, NamedAnalyzer> normalizers,
+    public IndexAnalyzers(Map<String, NamedAnalyzer> analyzers, Map<String, NamedAnalyzer> normalizers,
             Map<String, NamedAnalyzer> whitespaceNormalizers) {
-        super(indexSettings);
         Objects.requireNonNull(analyzers.get(DEFAULT_ANALYZER_NAME), "the default analyzer must be set");
         if (analyzers.get(DEFAULT_ANALYZER_NAME).name().equals(DEFAULT_ANALYZER_NAME) == false) {
             throw new IllegalStateException(
@@ -65,6 +63,13 @@ public NamedAnalyzer get(String name) {
         return analyzers.get(name);
     }
 
+    /**
+     * Returns an (unmodifiable) map of containing the index analyzers
+     */
+    public Map<String, NamedAnalyzer> getAnalyzers() {
+        return analyzers;
+    }
+
     /**
      * Returns a normalizer mapped to the given name or <code>null</code> if not present
      */
@@ -102,8 +107,9 @@ public NamedAnalyzer getDefaultSearchQuoteAnalyzer() {
 
     @Override
     public void close() throws IOException {
-       IOUtils.close(() -> Stream.concat(analyzers.values().stream(), normalizers.values().stream())
-           .filter(a -> a.scope() == AnalyzerScope.INDEX)
-           .iterator());
+        IOUtils.close(Stream.of(analyzers.values().stream(), normalizers.values().stream(), whitespaceNormalizers.values().stream())
+            .flatMap(s -> s)
+            .filter(a -> a.scope() == AnalyzerScope.INDEX)
+            .collect(Collectors.toList()));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/analysis/NameOrDefinition.java b/server/src/main/java/org/elasticsearch/index/analysis/NameOrDefinition.java
new file mode 100644
index 0000000000000..1831904de6b6f
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/analysis/NameOrDefinition.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParseException;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+public class NameOrDefinition implements Writeable, ToXContentFragment {
+    // exactly one of these two members is not null
+    public final String name;
+    public final Settings definition;
+
+    public NameOrDefinition(String name) {
+        this.name = Objects.requireNonNull(name);
+        this.definition = null;
+    }
+
+    public NameOrDefinition(Map<String, ?> definition) {
+        this.name = null;
+        Objects.requireNonNull(definition);
+        try {
+            XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+            builder.map(definition);
+            this.definition = Settings.builder().loadFromSource(Strings.toString(builder), builder.contentType()).build();
+        } catch (IOException e) {
+            throw new IllegalArgumentException("Failed to parse [" + definition + "]", e);
+        }
+    }
+
+    public NameOrDefinition(StreamInput in) throws IOException {
+        name = in.readOptionalString();
+        if (in.readBoolean()) {
+            definition = Settings.readSettingsFromStream(in);
+        } else {
+            definition = null;
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalString(name);
+        boolean isNotNullDefinition = this.definition != null;
+        out.writeBoolean(isNotNullDefinition);
+        if (isNotNullDefinition) {
+            Settings.writeSettingsToStream(definition, out);
+        }
+    }
+
+    public static NameOrDefinition fromXContent(XContentParser parser) throws IOException {
+        if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
+            return new NameOrDefinition(parser.text());
+        }
+        if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+            return new NameOrDefinition(parser.map());
+        }
+        throw new XContentParseException(parser.getTokenLocation(),
+            "Expected [VALUE_STRING] or [START_OBJECT], got " + parser.currentToken());
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        if (definition == null) {
+            builder.value(name);
+        } else {
+            builder.startObject();
+            definition.toXContent(builder, params);
+            builder.endObject();
+        }
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        NameOrDefinition that = (NameOrDefinition) o;
+        return Objects.equals(name, that.name) &&
+            Objects.equals(definition, that.definition);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, definition);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java
index 4831d88f3aa1f..0c53cc323d678 100644
--- a/server/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java
+++ b/server/src/main/java/org/elasticsearch/index/analysis/NamedAnalyzer.java
@@ -112,8 +112,8 @@ public void checkAllowedInMode(AnalysisMode mode) {
             return; // everything allowed if this analyzer is in ALL mode
         }
         if (this.getAnalysisMode() != mode) {
-            if (analyzer instanceof CustomAnalyzer) {
-                TokenFilterFactory[] tokenFilters = ((CustomAnalyzer) analyzer).tokenFilters();
+            if (analyzer instanceof AnalyzerComponentsProvider) {
+                TokenFilterFactory[] tokenFilters = ((AnalyzerComponentsProvider) analyzer).getComponents().getTokenFilters();
                 List<String> offendingFilters = new ArrayList<>();
                 for (TokenFilterFactory tokenFilter : tokenFilters) {
                     if (tokenFilter.getAnalysisMode() != mode) {
diff --git a/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java
new file mode 100644
index 0000000000000..7d3b8532caeb0
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzer.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.util.CloseableThreadLocal;
+import org.elasticsearch.common.settings.Settings;
+
+import java.io.Reader;
+import java.util.Map;
+
+public final class ReloadableCustomAnalyzer extends Analyzer implements AnalyzerComponentsProvider {
+
+    private volatile AnalyzerComponents components;
+
+    private CloseableThreadLocal<AnalyzerComponents> storedComponents = new CloseableThreadLocal<>();
+
+    private final int positionIncrementGap;
+
+    private final int offsetGap;
+
+    /**
+     * An alternative {@link ReuseStrategy} that allows swapping the stored analyzer components when they change.
+     * This is used to change e.g. token filters in search time analyzers.
+     */
+    private static final ReuseStrategy UPDATE_STRATEGY = new ReuseStrategy() {
+        @Override
+        public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) {
+            ReloadableCustomAnalyzer custom = (ReloadableCustomAnalyzer) analyzer;
+            AnalyzerComponents components = custom.getComponents();
+            AnalyzerComponents storedComponents = custom.getStoredComponents();
+            if (storedComponents == null || components != storedComponents) {
+                custom.setStoredComponents(components);
+                return null;
+            }
+            TokenStreamComponents tokenStream = (TokenStreamComponents) getStoredValue(analyzer);
+            assert tokenStream != null;
+            return tokenStream;
+        }
+
+        @Override
+        public void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents tokenStream) {
+            setStoredValue(analyzer, tokenStream);
+        }
+    };
+
+    ReloadableCustomAnalyzer(AnalyzerComponents components, int positionIncrementGap, int offsetGap) {
+        super(UPDATE_STRATEGY);
+        if (components.analysisMode().equals(AnalysisMode.SEARCH_TIME) == false) {
+            throw new IllegalArgumentException(
+                    "ReloadableCustomAnalyzer must only be initialized with analysis components in AnalysisMode.SEARCH_TIME mode");
+        }
+        this.components = components;
+        this.positionIncrementGap = positionIncrementGap;
+        this.offsetGap = offsetGap;
+    }
+
+    @Override
+    public AnalyzerComponents getComponents() {
+        return this.components;
+    }
+
+    @Override
+    public int getPositionIncrementGap(String fieldName) {
+        return this.positionIncrementGap;
+    }
+
+    @Override
+    public int getOffsetGap(String field) {
+        if (this.offsetGap < 0) {
+            return super.getOffsetGap(field);
+        }
+        return this.offsetGap;
+    }
+
+    public AnalysisMode getAnalysisMode() {
+        return this.components.analysisMode();
+    }
+
+    @Override
+    protected Reader initReaderForNormalization(String fieldName, Reader reader) {
+        final AnalyzerComponents components = getComponents();
+        for (CharFilterFactory charFilter : components.getCharFilters()) {
+            reader = charFilter.normalize(reader);
+        }
+        return reader;
+    }
+
+    @Override
+    protected TokenStream normalize(String fieldName, TokenStream in) {
+        final AnalyzerComponents components = getComponents();
+        TokenStream result = in;
+        for (TokenFilterFactory filter : components.getTokenFilters()) {
+            result = filter.normalize(result);
+        }
+        return result;
+    }
+
+    public synchronized void reload(String name,
+                                    Settings settings,
+                                    final Map<String, TokenizerFactory> tokenizers,
+                                    final Map<String, CharFilterFactory> charFilters,
+                                    final Map<String, TokenFilterFactory> tokenFilters) {
+        AnalyzerComponents components = AnalyzerComponents.createComponents(name, settings, tokenizers, charFilters, tokenFilters);
+        this.components = components;
+    }
+
+    @Override
+    public void close() {
+        super.close();
+        storedComponents.close();
+    }
+
+    private void setStoredComponents(AnalyzerComponents components) {
+        storedComponents.set(components);
+    }
+
+    private AnalyzerComponents getStoredComponents() {
+        return storedComponents.get();
+    }
+
+    @Override
+    protected TokenStreamComponents createComponents(String fieldName) {
+        final AnalyzerComponents components = getStoredComponents();
+        Tokenizer tokenizer = components.getTokenizerFactory().create();
+        TokenStream tokenStream = tokenizer;
+        for (TokenFilterFactory tokenFilter : components.getTokenFilters()) {
+            tokenStream = tokenFilter.create(tokenStream);
+        }
+        return new TokenStreamComponents(tokenizer, tokenStream);
+    }
+
+    @Override
+    protected Reader initReader(String fieldName, Reader reader) {
+        final AnalyzerComponents components = getStoredComponents();
+        if (components.getCharFilters() != null && components.getCharFilters().length > 0) {
+            for (CharFilterFactory charFilter : components.getCharFilters()) {
+                reader = charFilter.create(reader);
+            }
+        }
+        return reader;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java b/server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java
index addb16d58d031..313598e1d8ec7 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java
@@ -144,7 +144,8 @@ synchronized boolean releaseCommit(final IndexCommit snapshotCommit) {
     /**
      * Find a safe commit point from a list of existing commits based on the supplied global checkpoint.
      * The max sequence number of a safe commit point should be at most the global checkpoint.
-     * If an index was created before v6.2, and we haven't retained a safe commit yet, this method will return the oldest commit.
+     * If an index was created before 6.2 or recovered from remote, we might not have a safe commit.
+     * In this case, this method will return the oldest index commit.
      *
      * @param commits          a list of existing commit points
      * @param globalCheckpoint the persisted global checkpoint from the translog, see {@link Translog#readGlobalCheckpoint(Path, String)}
@@ -172,22 +173,13 @@ private static int indexOfKeptCommits(List<? extends IndexCommit> commits, long
             if (expectedTranslogUUID.equals(commitUserData.get(Translog.TRANSLOG_UUID_KEY)) == false) {
                 return i + 1;
             }
-            // 5.x commits do not contain MAX_SEQ_NO, we should not keep it and the older commits.
-            if (commitUserData.containsKey(SequenceNumbers.MAX_SEQ_NO) == false) {
-                return Math.min(commits.size() - 1, i + 1);
-            }
             final long maxSeqNoFromCommit = Long.parseLong(commitUserData.get(SequenceNumbers.MAX_SEQ_NO));
             if (maxSeqNoFromCommit <= globalCheckpoint) {
                 return i;
             }
         }
-        /*
-         * We may reach to this point in these cases:
-         * 1. In the previous 6.x, we keep only the last commit - which is likely not a safe commit if writes are in progress.
-         * Thus, after upgrading, we may not find a safe commit until we can reserve one.
-         * 2. In peer-recovery, if the file-based happens, a replica will be received the latest commit from a primary.
-         * However, that commit may not be a safe commit if writes are in progress in the primary.
-         */
+        // If an index was created before 6.2 or recovered from remote, we might not have a safe commit.
+        // In this case, we return the oldest index commit instead.
         return 0;
     }
 
@@ -204,11 +196,9 @@ synchronized boolean hasSnapshottedCommits() {
     boolean hasUnreferencedCommits() throws IOException {
         final IndexCommit lastCommit = this.lastCommit;
         if (safeCommit != lastCommit) { // Race condition can happen but harmless
-            if (lastCommit.getUserData().containsKey(SequenceNumbers.MAX_SEQ_NO)) {
-                final long maxSeqNoFromLastCommit = Long.parseLong(lastCommit.getUserData().get(SequenceNumbers.MAX_SEQ_NO));
-                // We can clean up the current safe commit if the last commit is safe
-                return globalCheckpointSupplier.getAsLong() >= maxSeqNoFromLastCommit;
-            }
+            final long maxSeqNoFromLastCommit = Long.parseLong(lastCommit.getUserData().get(SequenceNumbers.MAX_SEQ_NO));
+            // We can clean up the current safe commit if the last commit is safe
+            return globalCheckpointSupplier.getAsLong() >= maxSeqNoFromLastCommit;
         }
         return false;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/engine/CombinedDocValues.java b/server/src/main/java/org/elasticsearch/index/engine/CombinedDocValues.java
new file mode 100644
index 0000000000000..009dc9799d52a
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/engine/CombinedDocValues.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.engine;
+
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.elasticsearch.index.mapper.SeqNoFieldMapper;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.mapper.VersionFieldMapper;
+
+import java.io.IOException;
+import java.util.Objects;
+
+final class CombinedDocValues {
+    private final NumericDocValues versionDV;
+    private final NumericDocValues seqNoDV;
+    private final NumericDocValues primaryTermDV;
+    private final NumericDocValues tombstoneDV;
+    private final NumericDocValues recoverySource;
+
+    CombinedDocValues(LeafReader leafReader) throws IOException {
+        this.versionDV = Objects.requireNonNull(leafReader.getNumericDocValues(VersionFieldMapper.NAME), "VersionDV is missing");
+        this.seqNoDV = Objects.requireNonNull(leafReader.getNumericDocValues(SeqNoFieldMapper.NAME), "SeqNoDV is missing");
+        this.primaryTermDV = Objects.requireNonNull(
+            leafReader.getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME), "PrimaryTermDV is missing");
+        this.tombstoneDV = leafReader.getNumericDocValues(SeqNoFieldMapper.TOMBSTONE_NAME);
+        this.recoverySource = leafReader.getNumericDocValues(SourceFieldMapper.RECOVERY_SOURCE_NAME);
+    }
+
+    long docVersion(int segmentDocId) throws IOException {
+        assert versionDV.docID() < segmentDocId;
+        if (versionDV.advanceExact(segmentDocId) == false) {
+            throw new IllegalStateException("DocValues for field [" + VersionFieldMapper.NAME + "] is not found");
+        }
+        return versionDV.longValue();
+    }
+
+    long docSeqNo(int segmentDocId) throws IOException {
+        assert seqNoDV.docID() < segmentDocId;
+        if (seqNoDV.advanceExact(segmentDocId) == false) {
+            throw new IllegalStateException("DocValues for field [" + SeqNoFieldMapper.NAME + "] is not found");
+        }
+        return seqNoDV.longValue();
+    }
+
+    long docPrimaryTerm(int segmentDocId) throws IOException {
+        if (primaryTermDV == null) {
+            return -1L;
+        }
+        assert primaryTermDV.docID() < segmentDocId;
+        // Use -1 for docs which don't have primary term. The caller considers those docs as nested docs.
+        if (primaryTermDV.advanceExact(segmentDocId) == false) {
+            return -1;
+        }
+        return primaryTermDV.longValue();
+    }
+
+    boolean isTombstone(int segmentDocId) throws IOException {
+        if (tombstoneDV == null) {
+            return false;
+        }
+        assert tombstoneDV.docID() < segmentDocId;
+        return tombstoneDV.advanceExact(segmentDocId) && tombstoneDV.longValue() > 0;
+    }
+
+    boolean hasRecoverySource(int segmentDocId) throws IOException {
+        if (recoverySource == null) {
+            return false;
+        }
+        assert recoverySource.docID() < segmentDocId;
+        return recoverySource.advanceExact(segmentDocId);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
index 2d210b716d4b7..e21b816aefd80 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -793,9 +793,9 @@ public final CommitStats commitStats() {
     }
 
     /**
-     * @return the local checkpoint for this Engine
+     * @return the persisted local checkpoint for this Engine
      */
-    public abstract long getLocalCheckpoint();
+    public abstract long getPersistedLocalCheckpoint();
 
     /**
      * @return a {@link SeqNoStats} object, using local state and the supplied global checkpoint
@@ -933,7 +933,7 @@ protected void writerSegmentStats(SegmentsStats stats) {
     /** How much heap is used that would be freed by a refresh.  Note that this may throw {@link AlreadyClosedException}. */
     public abstract long getIndexBufferRAMBytesUsed();
 
-    protected Segment[] getSegmentInfo(SegmentInfos lastCommittedSegmentInfos, boolean verbose) {
+    final Segment[] getSegmentInfo(SegmentInfos lastCommittedSegmentInfos, boolean verbose) {
         ensureOpen();
         Map<String, Segment> segments = new HashMap<>();
         // first, go over and compute the search ones...
@@ -960,8 +960,8 @@ protected Segment[] getSegmentInfo(SegmentInfos lastCommittedSegmentInfos, boole
                     segment = new Segment(info.info.name);
                     segment.search = false;
                     segment.committed = true;
-                    segment.docCount = info.info.maxDoc();
-                    segment.delDocCount = info.getDelCount();
+                    segment.delDocCount = info.getDelCount() + info.getSoftDelCount();
+                    segment.docCount = info.info.maxDoc() - segment.delDocCount;
                     segment.version = info.info.getVersion();
                     segment.compound = info.info.getUseCompoundFile();
                     try {
@@ -1142,7 +1142,7 @@ public abstract void forceMerge(boolean flush, int maxNumSegments, boolean onlyE
      */
     @SuppressWarnings("finally")
     private void maybeDie(final String maybeMessage, final Throwable maybeFatal) {
-        ExceptionsHelper.maybeError(maybeFatal, logger).ifPresent(error -> {
+        ExceptionsHelper.maybeError(maybeFatal).ifPresent(error -> {
             try {
                 logger.error(maybeMessage, error);
             } finally {
diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 9fb63d0de019d..d96220983c971 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -20,7 +20,10 @@
 package org.elasticsearch.index.engine;
 
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.codecs.blocktree.BlockTreeTermsReader;
+import org.apache.lucene.codecs.blocktree.BlockTreeTermsReader.FSTLoadMode;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexCommit;
@@ -29,20 +32,28 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.LiveIndexWriterConfig;
 import org.apache.lucene.index.MergePolicy;
 import org.apache.lucene.index.SegmentCommitInfo;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.index.SoftDeletesRetentionMergePolicy;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ReferenceManager;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.SearcherFactory;
 import org.apache.lucene.search.SearcherManager;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.Weight;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
 import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.InfoStream;
 import org.elasticsearch.Assertions;
@@ -64,12 +75,14 @@
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.VersionType;
+import org.elasticsearch.index.fieldvisitor.IdOnlyFieldVisitor;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.merge.OnGoingMerge;
 import org.elasticsearch.index.seqno.LocalCheckpointTracker;
@@ -77,6 +90,7 @@
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.ElasticsearchMergePolicy;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.store.FsDirectoryFactory;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogConfig;
@@ -89,7 +103,6 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -103,8 +116,9 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.BiFunction;
+import java.util.function.LongConsumer;
 import java.util.function.LongSupplier;
-import java.util.function.Supplier;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 public class InternalEngine extends Engine {
@@ -191,13 +205,21 @@ public InternalEngine(EngineConfig engineConfig) {
             throttle = new IndexThrottle();
             try {
                 trimUnsafeCommits(engineConfig);
-                translog = openTranslog(engineConfig, translogDeletionPolicy, engineConfig.getGlobalCheckpointSupplier());
+                translog = openTranslog(engineConfig, translogDeletionPolicy, engineConfig.getGlobalCheckpointSupplier(),
+                    seqNo -> {
+                        final LocalCheckpointTracker tracker = getLocalCheckpointTracker();
+                        assert tracker != null || getTranslog().isOpen() == false;
+                        if (tracker != null) {
+                            tracker.markSeqNoAsPersisted(seqNo);
+                        }
+                    });
                 assert translog.getGeneration() != null;
                 this.translog = translog;
                 this.softDeleteEnabled = engineConfig.getIndexSettings().isSoftDeleteEnabled();
                 this.softDeletesPolicy = newSoftDeletesPolicy();
                 this.combinedDeletionPolicy =
                     new CombinedDeletionPolicy(logger, translogDeletionPolicy, softDeletesPolicy, translog::getLastSyncedGlobalCheckpoint);
+                this.localCheckpointTracker = createLocalCheckpointTracker(localCheckpointTrackerSupplier);
                 writer = createWriter();
                 bootstrapAppendOnlyInfoFromWriter(writer);
                 historyUUID = loadHistoryUUID(writer);
@@ -227,11 +249,17 @@ public InternalEngine(EngineConfig engineConfig) {
             for (ReferenceManager.RefreshListener listener: engineConfig.getInternalRefreshListener()) {
                 this.internalSearcherManager.addListener(listener);
             }
-            this.localCheckpointTracker = createLocalCheckpointTracker(engineConfig, lastCommittedSegmentInfos, logger,
-                () -> acquireSearcher("create_local_checkpoint_tracker", SearcherScope.INTERNAL), localCheckpointTrackerSupplier);
-            this.lastRefreshedCheckpointListener = new LastRefreshedCheckpointListener(localCheckpointTracker.getCheckpoint());
+            this.lastRefreshedCheckpointListener = new LastRefreshedCheckpointListener(localCheckpointTracker.getProcessedCheckpoint());
             this.internalSearcherManager.addListener(lastRefreshedCheckpointListener);
             maxSeqNoOfUpdatesOrDeletes = new AtomicLong(SequenceNumbers.max(localCheckpointTracker.getMaxSeqNo(), translog.getMaxSeqNo()));
+            if (softDeleteEnabled && localCheckpointTracker.getPersistedCheckpoint() < localCheckpointTracker.getMaxSeqNo()) {
+                try (Searcher searcher = acquireSearcher("restore_version_map_and_checkpoint_tracker", SearcherScope.INTERNAL)) {
+                    restoreVersionMapAndCheckpointTracker(Lucene.wrapAllDocsLive(searcher.getDirectoryReader()));
+                } catch (IOException e) {
+                    throw new EngineCreationFailureException(config().getShardId(),
+                        "failed to restore version map and local checkpoint tracker", e);
+                }
+            }
             success = true;
         } finally {
             if (success == false) {
@@ -245,30 +273,16 @@ public InternalEngine(EngineConfig engineConfig) {
         logger.trace("created new InternalEngine");
     }
 
-    private static LocalCheckpointTracker createLocalCheckpointTracker(EngineConfig engineConfig, SegmentInfos lastCommittedSegmentInfos,
-        Logger logger, Supplier<Searcher> searcherSupplier, BiFunction<Long, Long, LocalCheckpointTracker> localCheckpointTrackerSupplier) {
-        try {
-            final SequenceNumbers.CommitInfo seqNoStats =
-                SequenceNumbers.loadSeqNoInfoFromLuceneCommit(lastCommittedSegmentInfos.userData.entrySet());
-            final long maxSeqNo = seqNoStats.maxSeqNo;
-            final long localCheckpoint = seqNoStats.localCheckpoint;
-            logger.trace("recovered maximum sequence number [{}] and local checkpoint [{}]", maxSeqNo, localCheckpoint);
-            final LocalCheckpointTracker tracker = localCheckpointTrackerSupplier.apply(maxSeqNo, localCheckpoint);
-            // Operations that are optimized using max_seq_no_of_updates optimization must not be processed twice; otherwise, they will
-            // create duplicates in Lucene. To avoid this we check the LocalCheckpointTracker to see if an operation was already processed.
-            // Thus, we need to restore the LocalCheckpointTracker bit by bit to ensure the consistency between LocalCheckpointTracker and
-            // Lucene index. This is not the only solution since we can bootstrap max_seq_no_of_updates with max_seq_no of the commit to
-            // disable the MSU optimization during recovery. Here we prefer to maintain the consistency of LocalCheckpointTracker.
-            if (localCheckpoint < maxSeqNo && engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
-                try (Searcher searcher = searcherSupplier.get()) {
-                    Lucene.scanSeqNosInReader(searcher.getDirectoryReader(), localCheckpoint + 1, maxSeqNo,
-                        tracker::markSeqNoAsCompleted);
-                }
-            }
-            return tracker;
-        } catch (IOException ex) {
-            throw new EngineCreationFailureException(engineConfig.getShardId(), "failed to create local checkpoint tracker", ex);
-        }
+    private LocalCheckpointTracker createLocalCheckpointTracker(
+        BiFunction<Long, Long, LocalCheckpointTracker> localCheckpointTrackerSupplier) throws IOException {
+        final long maxSeqNo;
+        final long localCheckpoint;
+        final SequenceNumbers.CommitInfo seqNoStats =
+            SequenceNumbers.loadSeqNoInfoFromLuceneCommit(store.readLastCommittedSegmentsInfo().userData.entrySet());
+        maxSeqNo = seqNoStats.maxSeqNo;
+        localCheckpoint = seqNoStats.localCheckpoint;
+        logger.trace("recovered maximum sequence number [{}] and local checkpoint [{}]", maxSeqNo, localCheckpoint);
+        return localCheckpointTrackerSupplier.apply(maxSeqNo, localCheckpoint);
     }
 
     private SoftDeletesPolicy newSoftDeletesPolicy() throws IOException {
@@ -360,7 +374,7 @@ protected int getRefCount(IndexSearcher reference) {
     public int restoreLocalHistoryFromTranslog(TranslogRecoveryRunner translogRecoveryRunner) throws IOException {
         try (ReleasableLock ignored = readLock.acquire()) {
             ensureOpen();
-            final long localCheckpoint = localCheckpointTracker.getCheckpoint();
+            final long localCheckpoint = localCheckpointTracker.getProcessedCheckpoint();
             try (Translog.Snapshot snapshot = getTranslog().newSnapshotFromMinSeqNo(localCheckpoint + 1)) {
                 return translogRecoveryRunner.run(this, snapshot);
             }
@@ -371,19 +385,23 @@ public int restoreLocalHistoryFromTranslog(TranslogRecoveryRunner translogRecove
     public int fillSeqNoGaps(long primaryTerm) throws IOException {
         try (ReleasableLock ignored = writeLock.acquire()) {
             ensureOpen();
-            final long localCheckpoint = localCheckpointTracker.getCheckpoint();
+            final long localCheckpoint = localCheckpointTracker.getProcessedCheckpoint();
             final long maxSeqNo = localCheckpointTracker.getMaxSeqNo();
             int numNoOpsAdded = 0;
             for (
                     long seqNo = localCheckpoint + 1;
                     seqNo <= maxSeqNo;
-                    seqNo = localCheckpointTracker.getCheckpoint() + 1 /* the local checkpoint might have advanced so we leap-frog */) {
+                    seqNo = localCheckpointTracker.getProcessedCheckpoint() + 1 /* leap-frog the local checkpoint */) {
                 innerNoOp(new NoOp(seqNo, primaryTerm, Operation.Origin.PRIMARY, System.nanoTime(), "filling gaps"));
                 numNoOpsAdded++;
-                assert seqNo <= localCheckpointTracker.getCheckpoint()
-                        : "local checkpoint did not advance; was [" + seqNo + "], now [" + localCheckpointTracker.getCheckpoint() + "]";
+                assert seqNo <= localCheckpointTracker.getProcessedCheckpoint() :
+                    "local checkpoint did not advance; was [" + seqNo + "], now [" + localCheckpointTracker.getProcessedCheckpoint() + "]";
 
             }
+            syncTranslog(); // to persist noops associated with the advancement of the local checkpoint
+            assert localCheckpointTracker.getPersistedCheckpoint() == maxSeqNo
+                : "persisted local checkpoint did not advance to max seq no; is [" + localCheckpointTracker.getPersistedCheckpoint() +
+                "], max seq no [" + maxSeqNo + "]";
             return numNoOpsAdded;
         }
     }
@@ -461,13 +479,13 @@ private void recoverFromTranslogInternal(TranslogRecoveryRunner translogRecovery
     }
 
     private Translog openTranslog(EngineConfig engineConfig, TranslogDeletionPolicy translogDeletionPolicy,
-                                        LongSupplier globalCheckpointSupplier) throws IOException {
+                                  LongSupplier globalCheckpointSupplier, LongConsumer persistedSequenceNumberConsumer) throws IOException {
 
         final TranslogConfig translogConfig = engineConfig.getTranslogConfig();
         final String translogUUID = loadTranslogUUIDFromLastCommit();
         // We expect that this shard already exists, so it must already have an existing translog else something is badly wrong!
         return new Translog(translogConfig, translogUUID, translogDeletionPolicy, globalCheckpointSupplier,
-            engineConfig.getPrimaryTermSupplier());
+            engineConfig.getPrimaryTermSupplier(), persistedSequenceNumberConsumer);
     }
 
     // Package private for testing purposes only
@@ -671,21 +689,26 @@ enum OpVsLuceneDocStatus {
         LUCENE_DOC_NOT_FOUND
     }
 
+    private static OpVsLuceneDocStatus compareOpToVersionMapOnSeqNo(String id, long seqNo, long primaryTerm, VersionValue versionValue) {
+        Objects.requireNonNull(versionValue);
+        if (seqNo > versionValue.seqNo) {
+            return OpVsLuceneDocStatus.OP_NEWER;
+        } else if (seqNo == versionValue.seqNo) {
+            assert versionValue.term == primaryTerm : "primary term not matched; id=" + id + " seq_no=" + seqNo
+                + " op_term=" + primaryTerm + " existing_term=" + versionValue.term;
+            return OpVsLuceneDocStatus.OP_STALE_OR_EQUAL;
+        } else {
+            return OpVsLuceneDocStatus.OP_STALE_OR_EQUAL;
+        }
+    }
+
     private OpVsLuceneDocStatus compareOpToLuceneDocBasedOnSeqNo(final Operation op) throws IOException {
         assert op.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO : "resolving ops based on seq# but no seqNo is found";
         final OpVsLuceneDocStatus status;
         VersionValue versionValue = getVersionFromMap(op.uid().bytes());
         assert incrementVersionLookup();
         if (versionValue != null) {
-            if (op.seqNo() > versionValue.seqNo) {
-                status = OpVsLuceneDocStatus.OP_NEWER;
-            } else if (op.seqNo() == versionValue.seqNo) {
-                assert versionValue.term == op.primaryTerm() : "primary term not matched; id=" + op.id() + " seq_no=" + op.seqNo()
-                    + " op_term=" + op.primaryTerm() + " existing_term=" + versionValue.term;
-                status = OpVsLuceneDocStatus.OP_STALE_OR_EQUAL;
-            } else {
-                status = OpVsLuceneDocStatus.OP_STALE_OR_EQUAL;
-            }
+            status = compareOpToVersionMapOnSeqNo(op.id(), op.seqNo(), op.primaryTerm(), versionValue);
         } else {
             // load from index
             assert incrementIndexVersionLookup();
@@ -694,13 +717,9 @@ private OpVsLuceneDocStatus compareOpToLuceneDocBasedOnSeqNo(final Operation op)
                 if (docAndSeqNo == null) {
                     status = OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND;
                 } else if (op.seqNo() > docAndSeqNo.seqNo) {
-                    if (docAndSeqNo.isLive) {
-                        status = OpVsLuceneDocStatus.OP_NEWER;
-                    } else {
-                        status = OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND;
-                    }
+                    status = OpVsLuceneDocStatus.OP_NEWER;
                 } else if (op.seqNo() == docAndSeqNo.seqNo) {
-                    assert localCheckpointTracker.contains(op.seqNo()) || softDeleteEnabled == false :
+                    assert localCheckpointTracker.hasProcessed(op.seqNo()) || softDeleteEnabled == false :
                         "local checkpoint tracker is not updated seq_no=" + op.seqNo() + " id=" + op.id();
                     status = OpVsLuceneDocStatus.OP_STALE_OR_EQUAL;
                 } else {
@@ -903,7 +922,12 @@ public IndexResult index(Index index) throws IOException {
                     versionMap.maybePutIndexUnderLock(index.uid().bytes(),
                         new IndexVersionValue(translogLocation, plan.versionForIndexing, index.seqNo(), index.primaryTerm()));
                 }
-                localCheckpointTracker.markSeqNoAsCompleted(indexResult.getSeqNo());
+                localCheckpointTracker.markSeqNoAsProcessed(indexResult.getSeqNo());
+                if (indexResult.getTranslogLocation() == null) {
+                    // the op is coming from the translog (and is hence persisted already) or it does not have a sequence number
+                    assert index.origin().isFromTranslog() || indexResult.getSeqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO;
+                    localCheckpointTracker.markSeqNoAsPersisted(indexResult.getSeqNo());
+                }
                 indexResult.setTook(System.nanoTime() - index.startTime());
                 indexResult.freeze();
                 return indexResult;
@@ -943,7 +967,7 @@ protected final IndexingStrategy planIndexingAsNonPrimary(Index index) throws IO
             // unlike the primary, replicas don't really care to about creation status of documents
             // this allows to ignore the case where a document was found in the live version maps in
             // a delete state and return false for the created flag in favor of code simplicity
-            if (index.seqNo() <= localCheckpointTracker.getCheckpoint()){
+            if (index.seqNo() <= localCheckpointTracker.getProcessedCheckpoint()){
                 // the operation seq# is lower then the current local checkpoint and thus was already put into lucene
                 // this can happen during recovery where older operations are sent from the translog that are already
                 // part of the lucene commit (either from a peer recovery or a local translog)
@@ -1256,7 +1280,12 @@ public DeleteResult delete(Delete delete) throws IOException {
                 final Translog.Location location = translog.add(new Translog.Delete(delete, deleteResult));
                 deleteResult.setTranslogLocation(location);
             }
-            localCheckpointTracker.markSeqNoAsCompleted(deleteResult.getSeqNo());
+            localCheckpointTracker.markSeqNoAsProcessed(deleteResult.getSeqNo());
+            if (deleteResult.getTranslogLocation() == null) {
+                // the op is coming from the translog (and is hence persisted already) or does not have a sequence number (version conflict)
+                assert delete.origin().isFromTranslog() || deleteResult.getSeqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO;
+                localCheckpointTracker.markSeqNoAsPersisted(deleteResult.getSeqNo());
+            }
             deleteResult.setTook(System.nanoTime() - delete.startTime());
             deleteResult.freeze();
         } catch (RuntimeException | IOException e) {
@@ -1289,7 +1318,7 @@ protected final DeletionStrategy planDeletionAsNonPrimary(Delete delete) throws
         // this allows to ignore the case where a document was found in the live version maps in
         // a delete state and return true for the found flag in favor of code simplicity
         final DeletionStrategy plan;
-        if (delete.seqNo() <= localCheckpointTracker.getCheckpoint()) {
+        if (delete.seqNo() <= localCheckpointTracker.getProcessedCheckpoint()) {
             // the operation seq# is lower then the current local checkpoint and thus was already put into lucene
             // this can happen during recovery where older operations are sent from the translog that are already
             // part of the lucene commit (either from a peer recovery or a local translog)
@@ -1379,7 +1408,7 @@ private DeleteResult deleteInLucene(Delete delete, DeletionStrategy plan) throws
             return new DeleteResult(
                 plan.versionOfDeletion, getPrimaryTerm(), delete.seqNo(), plan.currentlyDeleted == false);
         } catch (Exception ex) {
-            if (indexWriter.getTragicException() == null) {
+            if (ex instanceof AlreadyClosedException == false && indexWriter.getTragicException() == null) {
                 throw new AssertionError("delete operation should never fail at document level", ex);
             }
             throw ex;
@@ -1464,9 +1493,8 @@ private NoOpResult innerNoOp(final NoOp noOp) throws IOException {
             final NoOpResult noOpResult;
             final Optional<Exception> preFlightError = preFlightCheckForNoOp(noOp);
             if (preFlightError.isPresent()) {
-                noOpResult = new NoOpResult(getPrimaryTerm(), noOp.seqNo(), preFlightError.get());
+                noOpResult = new NoOpResult(getPrimaryTerm(), SequenceNumbers.UNASSIGNED_SEQ_NO, preFlightError.get());
             } else {
-                Exception failure = null;
                 markSeqNoAsSeen(noOp.seqNo());
                 if (softDeleteEnabled) {
                     try {
@@ -1483,23 +1511,24 @@ private NoOpResult innerNoOp(final NoOp noOp) throws IOException {
                         doc.add(softDeletesField);
                         indexWriter.addDocument(doc);
                     } catch (Exception ex) {
-                        if (maybeFailEngine("noop", ex)) {
-                            throw ex;
+                        if (ex instanceof AlreadyClosedException == false && indexWriter.getTragicException() == null) {
+                            throw new AssertionError("noop operation should never fail at document level", ex);
                         }
-                        failure = ex;
+                        throw ex;
                     }
                 }
-                if (failure == null) {
-                    noOpResult = new NoOpResult(getPrimaryTerm(), noOp.seqNo());
-                } else {
-                    noOpResult = new NoOpResult(getPrimaryTerm(), noOp.seqNo(), failure);
-                }
+                noOpResult = new NoOpResult(getPrimaryTerm(), noOp.seqNo());
                 if (noOp.origin().isFromTranslog() == false && noOpResult.getResultType() == Result.Type.SUCCESS) {
                     final Translog.Location location = translog.add(new Translog.NoOp(noOp.seqNo(), noOp.primaryTerm(), noOp.reason()));
                     noOpResult.setTranslogLocation(location);
                 }
             }
-            localCheckpointTracker.markSeqNoAsCompleted(seqNo);
+            localCheckpointTracker.markSeqNoAsProcessed(noOpResult.getSeqNo());
+            if (noOpResult.getTranslogLocation() == null) {
+                // the op is coming from the translog (and is hence persisted already) or it does not have a sequence number
+                assert noOp.origin().isFromTranslog() || noOpResult.getSeqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO;
+                localCheckpointTracker.markSeqNoAsPersisted(noOpResult.getSeqNo());
+            }
             noOpResult.setTook(System.nanoTime() - noOp.startTime());
             noOpResult.freeze();
             return noOpResult;
@@ -1529,7 +1558,7 @@ final boolean refresh(String source, SearcherScope scope, boolean block) throws
         // since it flushes the index as well (though, in terms of concurrency, we are allowed to do it)
         // both refresh types will result in an internal refresh but only the external will also
         // pass the new reader reference to the external reader manager.
-        final long localCheckpointBeforeRefresh = getLocalCheckpoint();
+        final long localCheckpointBeforeRefresh = localCheckpointTracker.getProcessedCheckpoint();
         boolean refreshed;
         try (ReleasableLock lock = readLock.acquire()) {
             ensureOpen();
@@ -1671,9 +1700,9 @@ public boolean shouldPeriodicallyFlush() {
          * This method is to maintain translog only, thus IndexWriter#hasUncommittedChanges condition is not considered.
          */
         final long translogGenerationOfNewCommit =
-            translog.getMinGenerationForSeqNo(localCheckpointTracker.getCheckpoint() + 1).translogFileGeneration;
+            translog.getMinGenerationForSeqNo(localCheckpointTracker.getProcessedCheckpoint() + 1).translogFileGeneration;
         return translogGenerationOfLastCommit < translogGenerationOfNewCommit
-            || localCheckpointTracker.getCheckpoint() == localCheckpointTracker.getMaxSeqNo();
+            || localCheckpointTracker.getProcessedCheckpoint() == localCheckpointTracker.getMaxSeqNo();
     }
 
     @Override
@@ -1860,7 +1889,7 @@ private void pruneDeletedTombstones() {
          */
         final long timeMSec = engineConfig.getThreadPool().relativeTimeInMillis();
         final long maxTimestampToPrune = timeMSec - engineConfig.getIndexSettings().getGcDeletesInMillis();
-        versionMap.pruneTombstones(maxTimestampToPrune, localCheckpointTracker.getCheckpoint());
+        versionMap.pruneTombstones(maxTimestampToPrune, localCheckpointTracker.getProcessedCheckpoint());
         lastDeleteVersionPruneTimeMSec = timeMSec;
     }
 
@@ -1870,8 +1899,9 @@ void clearDeletedTombstones() {
     }
 
     // for testing
-    final Collection<DeleteVersionValue> getDeletedTombstones() {
-        return versionMap.getAllTombstones().values();
+    final Map<BytesRef, VersionValue> getVersionMap() {
+        return Stream.concat(versionMap.getAllCurrent().entrySet().stream(), versionMap.getAllTombstones().entrySet().stream())
+            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
     }
 
     @Override
@@ -2140,10 +2170,21 @@ IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOEx
         }
     }
 
+    static Map<String, String> getReaderAttributes(Directory directory) {
+        Directory unwrap = FilterDirectory.unwrap(directory);
+        boolean defaultOffHeap = FsDirectoryFactory.isHybridFs(unwrap) || unwrap instanceof MMapDirectory;
+        return Map.of(
+            BlockTreeTermsReader.FST_MODE_KEY, // if we are using MMAP for term dics we force all off heap unless it's the ID field
+            defaultOffHeap ? FSTLoadMode.OFF_HEAP.name() : FSTLoadMode.ON_HEAP.name()
+            , BlockTreeTermsReader.FST_MODE_KEY + "." + IdFieldMapper.NAME, // always force ID field on-heap for fast updates
+            FSTLoadMode.ON_HEAP.name());
+    }
+
     private IndexWriterConfig getIndexWriterConfig() {
         final IndexWriterConfig iwc = new IndexWriterConfig(engineConfig.getAnalyzer());
         iwc.setCommitOnClose(false); // we by default don't commit on close
         iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
+        iwc.setReaderAttributes(getReaderAttributes(store.directory()));
         iwc.setIndexDeletionPolicy(combinedDeletionPolicy);
         // with tests.verbose, lucene sets this up: plumb to align with filesystem stream
         boolean verbose = false;
@@ -2160,7 +2201,8 @@ private IndexWriterConfig getIndexWriterConfig() {
         iwc.setSoftDeletesField(Lucene.SOFT_DELETES_FIELD);
         if (softDeleteEnabled) {
             mergePolicy = new RecoverySourcePruneMergePolicy(SourceFieldMapper.RECOVERY_SOURCE_NAME, softDeletesPolicy::getRetentionQuery,
-                new SoftDeletesRetentionMergePolicy(Lucene.SOFT_DELETES_FIELD, softDeletesPolicy::getRetentionQuery, mergePolicy));
+                new SoftDeletesRetentionMergePolicy(Lucene.SOFT_DELETES_FIELD, softDeletesPolicy::getRetentionQuery,
+                    new PrunePostingsMergePolicy(mergePolicy, IdFieldMapper.NAME)));
         }
         iwc.setMergePolicy(new ElasticsearchMergePolicy(mergePolicy));
         iwc.setSimilarity(engineConfig.getSimilarity());
@@ -2336,7 +2378,7 @@ protected void doRun() throws Exception {
     protected void commitIndexWriter(final IndexWriter writer, final Translog translog, @Nullable final String syncId) throws IOException {
         ensureCanFlush();
         try {
-            final long localCheckpoint = localCheckpointTracker.getCheckpoint();
+            final long localCheckpoint = localCheckpointTracker.getProcessedCheckpoint();
             final Translog.TranslogGeneration translogGeneration = translog.getMinGenerationForSeqNo(localCheckpoint + 1);
             final String translogFileGeneration = Long.toString(translogGeneration.translogFileGeneration);
             final String translogUUID = translogGeneration.translogUUID;
@@ -2421,7 +2463,6 @@ public MergeStats getMergeStats() {
         return mergeScheduler.stats();
     }
 
-    // Used only for testing! Package private to prevent anyone else from using it
     LocalCheckpointTracker getLocalCheckpointTracker() {
         return localCheckpointTracker;
     }
@@ -2431,9 +2472,13 @@ public long getLastSyncedGlobalCheckpoint() {
         return getTranslog().getLastSyncedGlobalCheckpoint();
     }
 
+    public long getProcessedLocalCheckpoint() {
+        return localCheckpointTracker.getProcessedCheckpoint();
+    }
+
     @Override
-    public long getLocalCheckpoint() {
-        return localCheckpointTracker.getCheckpoint();
+    public long getPersistedLocalCheckpoint() {
+        return localCheckpointTracker.getPersistedCheckpoint();
     }
 
     /**
@@ -2456,7 +2501,7 @@ protected final boolean hasBeenProcessedBefore(Operation op) {
                 assert versionMap.assertKeyedLockHeldByCurrentThread(op.uid().bytes());
             }
         }
-        return localCheckpointTracker.contains(op.seqNo());
+        return localCheckpointTracker.hasProcessed(op.seqNo());
     }
 
     @Override
@@ -2490,10 +2535,6 @@ private boolean incrementIndexVersionLookup() {
         return true;
     }
 
-    int getVersionMapSize() {
-        return versionMap.getAllCurrent().size();
-    }
-
     boolean isSafeAccessRequired() {
         return versionMap.isSafeAccessRequired();
     }
@@ -2550,7 +2591,7 @@ public Translog.Snapshot newChangesSnapshot(String source, MapperService mapperS
 
     @Override
     public boolean hasCompleteOperationHistory(String source, MapperService mapperService, long startingSeqNo) throws IOException {
-        final long currentLocalCheckpoint = getLocalCheckpointTracker().getCheckpoint();
+        final long currentLocalCheckpoint = localCheckpointTracker.getProcessedCheckpoint();
         // avoid scanning translog if not necessary
         if (startingSeqNo > currentLocalCheckpoint) {
             return true;
@@ -2560,11 +2601,11 @@ public boolean hasCompleteOperationHistory(String source, MapperService mapperSe
             Translog.Operation operation;
             while ((operation = snapshot.next()) != null) {
                 if (operation.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
-                    tracker.markSeqNoAsCompleted(operation.seqNo());
+                    tracker.markSeqNoAsProcessed(operation.seqNo());
                 }
             }
         }
-        return tracker.getCheckpoint() >= currentLocalCheckpoint;
+        return tracker.getProcessedCheckpoint() >= currentLocalCheckpoint;
     }
 
     /**
@@ -2680,7 +2721,7 @@ private final class LastRefreshedCheckpointListener implements ReferenceManager.
         @Override
         public void beforeRefresh() {
             // all changes until this point should be visible after refresh
-            pendingCheckpoint = localCheckpointTracker.getCheckpoint();
+            pendingCheckpoint = localCheckpointTracker.getProcessedCheckpoint();
         }
 
         @Override
@@ -2741,7 +2782,7 @@ private boolean assertMaxSeqNoOfUpdatesIsAdvanced(Term id, long seqNo, boolean a
         // Operations can be processed on a replica in a different order than on the primary. If the order on the primary is index-1,
         // delete-2, index-3, and the order on a replica is index-1, index-3, delete-2, then the msu of index-3 on the replica is 2
         // even though it is an update (overwrites index-1). We should relax this assertion if there is a pending gap in the seq_no.
-        if (relaxIfGapInSeqNo && getLocalCheckpoint() < maxSeqNoOfUpdates) {
+        if (relaxIfGapInSeqNo && localCheckpointTracker.getProcessedCheckpoint() < maxSeqNoOfUpdates) {
             return true;
         }
         assert seqNo <= maxSeqNoOfUpdates : "id=" + id + " seq_no=" + seqNo + " msu=" + maxSeqNoOfUpdates;
@@ -2756,4 +2797,58 @@ private static void trimUnsafeCommits(EngineConfig engineConfig) throws IOExcept
         final long minRetainedTranslogGen = Translog.readMinTranslogGeneration(translogPath, translogUUID);
         store.trimUnsafeCommits(globalCheckpoint, minRetainedTranslogGen, engineConfig.getIndexSettings().getIndexVersionCreated());
     }
+
+    /**
+     * Restores the live version map and local checkpoint of this engine using documents (including soft-deleted)
+     * after the local checkpoint in the safe commit. This step ensures the live version map and checkpoint tracker
+     * are in sync with the Lucene commit.
+     */
+    private void restoreVersionMapAndCheckpointTracker(DirectoryReader directoryReader) throws IOException {
+        final IndexSearcher searcher = new IndexSearcher(directoryReader);
+        searcher.setQueryCache(null);
+        final Query query = LongPoint.newRangeQuery(SeqNoFieldMapper.NAME, getPersistedLocalCheckpoint() + 1, Long.MAX_VALUE);
+        final Weight weight = searcher.createWeight(query, ScoreMode.COMPLETE_NO_SCORES, 1.0f);
+        for (LeafReaderContext leaf : directoryReader.leaves()) {
+            final Scorer scorer = weight.scorer(leaf);
+            if (scorer == null) {
+                continue;
+            }
+            final CombinedDocValues dv = new CombinedDocValues(leaf.reader());
+            final IdOnlyFieldVisitor idFieldVisitor = new IdOnlyFieldVisitor();
+            final DocIdSetIterator iterator = scorer.iterator();
+            int docId;
+            while ((docId = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+                final long primaryTerm = dv.docPrimaryTerm(docId);
+                if (primaryTerm == -1L) {
+                    continue; // skip children docs which do not have primary term
+                }
+                final long seqNo = dv.docSeqNo(docId);
+                localCheckpointTracker.markSeqNoAsProcessed(seqNo);
+                localCheckpointTracker.markSeqNoAsPersisted(seqNo);
+                idFieldVisitor.reset();
+                leaf.reader().document(docId, idFieldVisitor);
+                if (idFieldVisitor.getId() == null) {
+                    assert dv.isTombstone(docId);
+                    continue;
+                }
+                final BytesRef uid = new Term(IdFieldMapper.NAME, Uid.encodeId(idFieldVisitor.getId())).bytes();
+                try (Releasable ignored = versionMap.acquireLock(uid)) {
+                    final VersionValue curr = versionMap.getUnderLock(uid);
+                    if (curr == null ||
+                        compareOpToVersionMapOnSeqNo(idFieldVisitor.getId(), seqNo, primaryTerm, curr) == OpVsLuceneDocStatus.OP_NEWER) {
+                        if (dv.isTombstone(docId)) {
+                            // use 0L for the start time so we can prune this delete tombstone quickly
+                            // when the local checkpoint advances (i.e., after a recovery completed).
+                            final long startTime = 0L;
+                            versionMap.putDeleteUnderLock(uid, new DeleteVersionValue(dv.docVersion(docId), seqNo, primaryTerm, startTime));
+                        } else {
+                            versionMap.putIndexUnderLock(uid, new IndexVersionValue(null, dv.docVersion(docId), seqNo, primaryTerm));
+                        }
+                    }
+                }
+            }
+        }
+        // remove live entries in the version map
+        refresh("restore_version_map_and_checkpoint_tracker", SearcherScope.INTERNAL, true);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
index a3e86ab1606df..bf5ecc3350a5b 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java
@@ -40,14 +40,12 @@
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.Uid;
-import org.elasticsearch.index.mapper.VersionFieldMapper;
 import org.elasticsearch.index.translog.Translog;
 
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.Comparator;
 import java.util.List;
-import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -213,11 +211,8 @@ private void fillParallelArray(ScoreDoc[] scoreDocs, ParallelArray parallelArray
 
     private TopDocs searchOperations(ScoreDoc after) throws IOException {
         final Query rangeQuery = LongPoint.newRangeQuery(SeqNoFieldMapper.NAME, Math.max(fromSeqNo, lastSeenSeqNo), toSeqNo);
-        final Sort sortedBySeqNoThenByTerm = new Sort(
-            new SortField(SeqNoFieldMapper.NAME, SortField.Type.LONG),
-            new SortField(SeqNoFieldMapper.PRIMARY_TERM_NAME, SortField.Type.LONG, true)
-        );
-        return indexSearcher.searchAfter(after, rangeQuery, searchBatchSize, sortedBySeqNoThenByTerm);
+        final Sort sortedBySeqNo = new Sort(new SortField(SeqNoFieldMapper.NAME, SortField.Type.LONG));
+        return indexSearcher.searchAfter(after, rangeQuery, searchBatchSize, sortedBySeqNo);
     }
 
     private Translog.Operation readDocAsOp(int docIndex) throws IOException {
@@ -305,64 +300,4 @@ private static final class ParallelArray {
         }
     }
 
-    private static final class CombinedDocValues {
-        private final NumericDocValues versionDV;
-        private final NumericDocValues seqNoDV;
-        private final NumericDocValues primaryTermDV;
-        private final NumericDocValues tombstoneDV;
-        private final NumericDocValues recoverySource;
-
-        CombinedDocValues(LeafReader leafReader) throws IOException {
-            this.versionDV = Objects.requireNonNull(leafReader.getNumericDocValues(VersionFieldMapper.NAME), "VersionDV is missing");
-            this.seqNoDV = Objects.requireNonNull(leafReader.getNumericDocValues(SeqNoFieldMapper.NAME), "SeqNoDV is missing");
-            this.primaryTermDV = Objects.requireNonNull(
-                leafReader.getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME), "PrimaryTermDV is missing");
-            this.tombstoneDV = leafReader.getNumericDocValues(SeqNoFieldMapper.TOMBSTONE_NAME);
-            this.recoverySource = leafReader.getNumericDocValues(SourceFieldMapper.RECOVERY_SOURCE_NAME);
-        }
-
-        long docVersion(int segmentDocId) throws IOException {
-            assert versionDV.docID() < segmentDocId;
-            if (versionDV.advanceExact(segmentDocId) == false) {
-                throw new IllegalStateException("DocValues for field [" + VersionFieldMapper.NAME + "] is not found");
-            }
-            return versionDV.longValue();
-        }
-
-        long docSeqNo(int segmentDocId) throws IOException {
-            assert seqNoDV.docID() < segmentDocId;
-            if (seqNoDV.advanceExact(segmentDocId) == false) {
-                throw new IllegalStateException("DocValues for field [" + SeqNoFieldMapper.NAME + "] is not found");
-            }
-            return seqNoDV.longValue();
-        }
-
-        long docPrimaryTerm(int segmentDocId) throws IOException {
-            if (primaryTermDV == null) {
-                return -1L;
-            }
-            assert primaryTermDV.docID() < segmentDocId;
-            // Use -1 for docs which don't have primary term. The caller considers those docs as nested docs.
-            if (primaryTermDV.advanceExact(segmentDocId) == false) {
-                return -1;
-            }
-            return primaryTermDV.longValue();
-        }
-
-        boolean isTombstone(int segmentDocId) throws IOException {
-            if (tombstoneDV == null) {
-                return false;
-            }
-            assert tombstoneDV.docID() < segmentDocId;
-            return tombstoneDV.advanceExact(segmentDocId) && tombstoneDV.longValue() > 0;
-        }
-
-        boolean hasRecoverySource(int segmentDocId) throws IOException {
-            if (recoverySource == null) {
-                return false;
-            }
-            assert recoverySource.docID() < segmentDocId;
-            return recoverySource.advanceExact(segmentDocId);
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
index a41f07e994b93..2244dd57037cb 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
@@ -27,16 +27,24 @@
 import org.apache.lucene.index.SegmentReader;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.util.concurrent.ReleasableLock;
+import org.elasticsearch.index.store.Store;
+import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.index.translog.TranslogConfig;
+import org.elasticsearch.index.translog.TranslogDeletionPolicy;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.List;
+import java.util.Map;
 import java.util.function.Function;
 
 /**
  * NoOpEngine is an engine implementation that does nothing but the bare minimum
  * required in order to have an engine. All attempts to do something (search,
- * index, get), throw {@link UnsupportedOperationException}.
+ * index, get), throw {@link UnsupportedOperationException}. However, NoOpEngine
+ * allows to trim any existing translog files through the usage of the
+ * {{@link #trimUnreferencedTranslogFiles()}} method.
  */
 public final class NoOpEngine extends ReadOnlyEngine {
 
@@ -46,7 +54,7 @@ public NoOpEngine(EngineConfig config) {
         super(config, null, null, true, Function.identity());
         this.stats = new SegmentsStats();
         Directory directory = store.directory();
-        try (DirectoryReader reader = DirectoryReader.open(directory)) {
+        try (DirectoryReader reader = DirectoryReader.open(directory, OFF_HEAP_READER_ATTRIBUTES)) {
             for (LeafReaderContext ctx : reader.getContext().leaves()) {
                 SegmentReader segmentReader = Lucene.segmentReader(ctx.reader());
                 fillSegmentStats(segmentReader, true, stats);
@@ -116,4 +124,54 @@ public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean incl
             return super.segmentsStats(includeSegmentFileSizes, includeUnloadedSegments);
         }
     }
+
+    /**
+     * This implementation will trim existing translog files using a {@link TranslogDeletionPolicy}
+     * that retains nothing but the last translog generation from safe commit.
+     */
+    @Override
+    public void trimUnreferencedTranslogFiles() {
+        final Store store = this.engineConfig.getStore();
+        store.incRef();
+        try (ReleasableLock lock = readLock.acquire()) {
+            ensureOpen();
+            final List<IndexCommit> commits = DirectoryReader.listCommits(store.directory());
+            if (commits.size() == 1) {
+                final Map<String, String> commitUserData = getLastCommittedSegmentInfos().getUserData();
+                final String translogUuid = commitUserData.get(Translog.TRANSLOG_UUID_KEY);
+                if (translogUuid == null) {
+                    throw new IllegalStateException("commit doesn't contain translog unique id");
+                }
+                if (commitUserData.containsKey(Translog.TRANSLOG_GENERATION_KEY) == false) {
+                    throw new IllegalStateException("commit doesn't contain translog generation id");
+                }
+                final long lastCommitGeneration = Long.parseLong(commitUserData.get(Translog.TRANSLOG_GENERATION_KEY));
+                final TranslogConfig translogConfig = engineConfig.getTranslogConfig();
+                final long minTranslogGeneration = Translog.readMinTranslogGeneration(translogConfig.getTranslogPath(), translogUuid);
+
+                if (minTranslogGeneration < lastCommitGeneration) {
+                    // a translog deletion policy that retains nothing but the last translog generation from safe commit
+                    final TranslogDeletionPolicy translogDeletionPolicy = new TranslogDeletionPolicy(-1, -1);
+                    translogDeletionPolicy.setTranslogGenerationOfLastCommit(lastCommitGeneration);
+                    translogDeletionPolicy.setMinTranslogGenerationForRecovery(lastCommitGeneration);
+
+                    try (Translog translog = new Translog(translogConfig, translogUuid, translogDeletionPolicy,
+                        engineConfig.getGlobalCheckpointSupplier(), engineConfig.getPrimaryTermSupplier(), seqNo -> {})) {
+                        translog.trimUnreferencedReaders();
+                        // refresh the translog stats
+                        this.translogStats = translog.stats();
+                    }
+                }
+            }
+        } catch (final Exception e) {
+            try {
+                failEngine("translog trimming failed", e);
+            } catch (Exception inner) {
+                e.addSuppressed(inner);
+            }
+            throw new EngineException(shardId, "failed to trim translog", e);
+        } finally {
+            store.decRef();
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/engine/PrunePostingsMergePolicy.java b/server/src/main/java/org/elasticsearch/index/engine/PrunePostingsMergePolicy.java
new file mode 100644
index 0000000000000..0c973ba93f65d
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/engine/PrunePostingsMergePolicy.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.engine;
+
+import org.apache.lucene.codecs.FieldsProducer;
+import org.apache.lucene.index.CodecReader;
+import org.apache.lucene.index.FilterCodecReader;
+import org.apache.lucene.index.FilterLeafReader;
+import org.apache.lucene.index.FilteredTermsEnum;
+import org.apache.lucene.index.ImpactsEnum;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.OneMergeWrappingMergePolicy;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ * This merge policy drops id field postings for all delete documents this can be
+ * useful to guarantee consistent update performance even if a large number of deleted / updated documents
+ * are retained. Merging postings away is efficient since lucene visits postings term by term and
+ * with the original live-docs being available we are adding a negotiable overhead such that we can
+ * prune soft-deletes by default. Yet, using this merge policy will cause loosing all search capabilities on top of
+ * soft deleted documents independent of the retention policy. Note, in order for this merge policy to be effective it needs to be added
+ * before the {@link org.apache.lucene.index.SoftDeletesRetentionMergePolicy} because otherwise only documents that are deleted / removed
+ * anyways will be pruned.
+ */
+final class PrunePostingsMergePolicy extends OneMergeWrappingMergePolicy {
+
+    PrunePostingsMergePolicy(MergePolicy in, String idField) {
+        super(in, toWrap -> new OneMerge(toWrap.segments) {
+            @Override
+            public CodecReader wrapForMerge(CodecReader reader) throws IOException {
+                CodecReader wrapped = toWrap.wrapForMerge(reader);
+                return wrapReader(wrapped, idField);
+            }
+        });
+    }
+
+    private static CodecReader wrapReader(CodecReader reader, String idField) {
+        Bits liveDocs = reader.getLiveDocs();
+        if (liveDocs == null) {
+            return reader; // no deleted docs - we are good!
+        }
+        final boolean fullyDeletedSegment = reader.numDocs() == 0;
+        return new FilterCodecReader(reader) {
+
+            @Override
+            public FieldsProducer getPostingsReader() {
+                FieldsProducer postingsReader = super.getPostingsReader();
+                if (postingsReader == null) {
+                    return null;
+                }
+                return new FieldsProducer() {
+                    @Override
+                    public void close() throws IOException {
+                        postingsReader.close();
+                    }
+
+                    @Override
+                    public void checkIntegrity() throws IOException {
+                        postingsReader.checkIntegrity();
+                    }
+
+                    @Override
+                    public Iterator<String> iterator() {
+                        return postingsReader.iterator();
+                    }
+
+                    @Override
+                    public Terms terms(String field) throws IOException {
+                        Terms in = postingsReader.terms(field);
+                        if (idField.equals(field) && in != null) {
+                            return new FilterLeafReader.FilterTerms(in) {
+                                @Override
+                                public TermsEnum iterator() throws IOException {
+                                    TermsEnum iterator = super.iterator();
+                                    return new FilteredTermsEnum(iterator, false) {
+                                        private PostingsEnum internal;
+
+                                        @Override
+                                        protected AcceptStatus accept(BytesRef term) throws IOException {
+                                            if (fullyDeletedSegment) {
+                                                return AcceptStatus.END; // short-cut this if we don't match anything
+                                            }
+                                            internal = postings(internal, PostingsEnum.NONE);
+                                            if (internal.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+                                                return AcceptStatus.YES;
+                                            }
+                                            return AcceptStatus.NO;
+                                        }
+
+                                        @Override
+                                        public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {
+                                            if (reuse instanceof OnlyLiveDocsPostingsEnum) {
+                                                OnlyLiveDocsPostingsEnum reuseInstance = (OnlyLiveDocsPostingsEnum) reuse;
+                                                reuseInstance.reset(super.postings(reuseInstance.in, flags));
+                                                return reuseInstance;
+                                            }
+                                            return new OnlyLiveDocsPostingsEnum(super.postings(null, flags), liveDocs);
+                                        }
+
+                                        @Override
+                                        public ImpactsEnum impacts(int flags) throws IOException {
+                                            throw new UnsupportedOperationException();
+                                        }
+                                    };
+                                }
+                            };
+                        } else {
+                            return in;
+                        }
+                    }
+
+                    @Override
+                    public int size() {
+                        return postingsReader.size();
+                    }
+
+                    @Override
+                    public long ramBytesUsed() {
+                        return postingsReader.ramBytesUsed();
+                    }
+                };
+            }
+
+            @Override
+            public CacheHelper getCoreCacheHelper() {
+                return null;
+            }
+
+            @Override
+            public CacheHelper getReaderCacheHelper() {
+                return null;
+            }
+        };
+    }
+
+    private static final class OnlyLiveDocsPostingsEnum extends PostingsEnum {
+
+        private final Bits liveDocs;
+        private PostingsEnum in;
+
+        OnlyLiveDocsPostingsEnum(PostingsEnum in, Bits liveDocs) {
+            this.liveDocs = liveDocs;
+            reset(in);
+        }
+
+        void reset(PostingsEnum in) {
+            this.in = in;
+        }
+
+        @Override
+        public int docID() {
+            return in.docID();
+        }
+
+        @Override
+        public int nextDoc() throws IOException {
+            int docId;
+            do {
+                docId = in.nextDoc();
+            } while (docId != DocIdSetIterator.NO_MORE_DOCS && liveDocs.get(docId) == false);
+            return docId;
+        }
+
+        @Override
+        public int advance(int target) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public long cost() {
+            return in.cost();
+        }
+
+        @Override
+        public int freq() throws IOException {
+            return in.freq();
+        }
+
+        @Override
+        public int nextPosition() throws IOException {
+            return in.nextPosition();
+        }
+
+        @Override
+        public int startOffset() throws IOException {
+            return in.startOffset();
+        }
+
+        @Override
+        public int endOffset() throws IOException {
+            return in.endOffset();
+        }
+
+        @Override
+        public BytesRef getPayload() throws IOException {
+            return in.getPayload();
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
index 9d5f6054243e4..ecd611859b578 100644
--- a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
+++ b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.index.engine;
 
+import org.apache.lucene.codecs.blocktree.BlockTreeTermsReader;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexReader;
@@ -41,13 +42,17 @@
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.index.translog.TranslogConfig;
+import org.elasticsearch.index.translog.TranslogDeletionPolicy;
 import org.elasticsearch.index.translog.TranslogStats;
 
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.function.BiFunction;
 import java.util.function.Function;
@@ -62,15 +67,22 @@
  */
 public class ReadOnlyEngine extends Engine {
 
+    /**
+     * Reader attributes used for read only engines. These attributes prevent loading term dictionaries on-heap even if the field is an
+     * ID field if we are reading form memory maps.
+     */
+    public static final Map<String, String> OFF_HEAP_READER_ATTRIBUTES = Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY,
+        BlockTreeTermsReader.FSTLoadMode.AUTO.name());
     private final SegmentInfos lastCommittedSegmentInfos;
     private final SeqNoStats seqNoStats;
-    private final TranslogStats translogStats;
     private final SearcherManager searcherManager;
     private final IndexCommit indexCommit;
     private final Lock indexWriterLock;
     private final DocsStats docsStats;
     private final RamAccountingSearcherFactory searcherFactory;
 
+    protected volatile TranslogStats translogStats;
+
     /**
      * Creates a new ReadOnlyEngine. This ctor can also be used to open a read-only engine on top of an already opened
      * read-write engine. It allows to optionally obtain the writer locks for the shard which would time-out if another
@@ -99,7 +111,6 @@ public ReadOnlyEngine(EngineConfig config, SeqNoStats seqNoStats, TranslogStats
                 // yet this makes sure nobody else does. including some testing tools that try to be messy
                 indexWriterLock = obtainLock ? directory.obtainLock(IndexWriter.WRITE_LOCK_NAME) : null;
                 this.lastCommittedSegmentInfos = Lucene.readSegmentInfos(directory);
-                this.translogStats = translogStats == null ? new TranslogStats(0, 0, 0, 0, 0) : translogStats;
                 if (seqNoStats == null) {
                     seqNoStats = buildSeqNoStats(config, lastCommittedSegmentInfos);
                     ensureMaxSeqNoEqualsToGlobalCheckpoint(seqNoStats);
@@ -110,6 +121,8 @@ public ReadOnlyEngine(EngineConfig config, SeqNoStats seqNoStats, TranslogStats
                 reader = wrapReader(reader, readerWrapperFunction);
                 searcherManager = new SearcherManager(reader, searcherFactory);
                 this.docsStats = docsStats(lastCommittedSegmentInfos);
+                assert translogStats != null || obtainLock : "mutiple translogs instances should not be opened at the same time";
+                this.translogStats = translogStats != null ? translogStats : translogStats(config, lastCommittedSegmentInfos);
                 this.indexWriterLock = indexWriterLock;
                 success = true;
             } finally {
@@ -132,12 +145,12 @@ protected void ensureMaxSeqNoEqualsToGlobalCheckpoint(final SeqNoStats seqNoStat
         final Version indexVersionCreated = engineConfig.getIndexSettings().getIndexVersionCreated();
         if (indexVersionCreated.onOrAfter(Version.V_7_2_0) ||
             (seqNoStats.getGlobalCheckpoint() != SequenceNumbers.UNASSIGNED_SEQ_NO)) {
+            assert assertMaxSeqNoEqualsToGlobalCheckpoint(seqNoStats.getMaxSeqNo(), seqNoStats.getGlobalCheckpoint());
             if (seqNoStats.getMaxSeqNo() != seqNoStats.getGlobalCheckpoint()) {
                 throw new IllegalStateException("Maximum sequence number [" + seqNoStats.getMaxSeqNo()
                     + "] from last commit does not match global checkpoint [" + seqNoStats.getGlobalCheckpoint() + "]");
             }
         }
-        assert assertMaxSeqNoEqualsToGlobalCheckpoint(seqNoStats.getMaxSeqNo(), seqNoStats.getGlobalCheckpoint());
     }
 
     protected boolean assertMaxSeqNoEqualsToGlobalCheckpoint(final long maxSeqNo, final long globalCheckpoint) {
@@ -165,7 +178,7 @@ protected final DirectoryReader wrapReader(DirectoryReader reader,
     }
 
     protected DirectoryReader open(IndexCommit commit) throws IOException {
-        return DirectoryReader.open(commit);
+        return DirectoryReader.open(commit, OFF_HEAP_READER_ATTRIBUTES);
     }
 
     private DocsStats docsStats(final SegmentInfos lastCommittedSegmentInfos) {
@@ -207,6 +220,26 @@ private static SeqNoStats buildSeqNoStats(EngineConfig config, SegmentInfos info
         return new SeqNoStats(maxSeqNo, localCheckpoint, config.getGlobalCheckpointSupplier().getAsLong());
     }
 
+    private static TranslogStats translogStats(final EngineConfig config, final SegmentInfos infos) throws IOException {
+        final String translogUuid = infos.getUserData().get(Translog.TRANSLOG_UUID_KEY);
+        if (translogUuid == null) {
+            throw new IllegalStateException("commit doesn't contain translog unique id");
+        }
+        final long translogGenOfLastCommit = Long.parseLong(infos.getUserData().get(Translog.TRANSLOG_GENERATION_KEY));
+        final TranslogConfig translogConfig = config.getTranslogConfig();
+        final TranslogDeletionPolicy translogDeletionPolicy = new TranslogDeletionPolicy(
+            config.getIndexSettings().getTranslogRetentionSize().getBytes(),
+            config.getIndexSettings().getTranslogRetentionAge().getMillis()
+        );
+        translogDeletionPolicy.setTranslogGenerationOfLastCommit(translogGenOfLastCommit);
+
+        try (Translog translog = new Translog(translogConfig, translogUuid, translogDeletionPolicy, config.getGlobalCheckpointSupplier(),
+                config.getPrimaryTermSupplier(), seqNo -> {})
+        ) {
+            return translog.stats();
+        }
+    }
+
     @Override
     public GetResult get(Get get, BiFunction<String, SearcherScope, Searcher> searcherFactory) throws EngineException {
         return getFromSearcher(get, searcherFactory, SearcherScope.EXTERNAL);
@@ -320,7 +353,7 @@ public Translog.Location getTranslogLastWriteLocation() {
     }
 
     @Override
-    public long getLocalCheckpoint() {
+    public long getPersistedLocalCheckpoint() {
         return seqNoStats.getLocalCheckpoint();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
index bb3388bc943c9..11388190c3092 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
@@ -47,6 +47,7 @@
 import org.elasticsearch.search.sort.NestedSortBuilder;
 
 import java.io.IOException;
+import java.util.function.Function;
 
 /**
  * Thread-safe utility class that allows to get per-segment values via the
@@ -114,11 +115,14 @@ public static class Nested {
             private final BitSetProducer rootFilter;
             private final Query innerQuery;
             private final NestedSortBuilder nestedSort;
+            private final Function<IndexReaderContext, IndexSearcher> searcherFactory;
 
-            public Nested(BitSetProducer rootFilter, Query innerQuery, NestedSortBuilder nestedSort) {
+            public Nested(BitSetProducer rootFilter, Query innerQuery, NestedSortBuilder nestedSort,
+                          Function<IndexReaderContext, IndexSearcher> searcherFactory) {
                 this.rootFilter = rootFilter;
                 this.innerQuery = innerQuery;
                 this.nestedSort = nestedSort;
+                this.searcherFactory = searcherFactory;
             }
 
             public Query getInnerQuery() {
@@ -143,7 +147,7 @@ public BitSet rootDocs(LeafReaderContext ctx) throws IOException {
              */
             public DocIdSetIterator innerDocs(LeafReaderContext ctx) throws IOException {
                 final IndexReaderContext topLevelCtx = ReaderUtil.getTopLevelContext(ctx);
-                IndexSearcher indexSearcher = new IndexSearcher(topLevelCtx);
+                IndexSearcher indexSearcher = searcherFactory.apply(topLevelCtx);
                 Weight weight = indexSearcher.createWeight(indexSearcher.rewrite(innerQuery), ScoreMode.COMPLETE_NO_SCORES, 1f);
                 Scorer s = weight.scorer(ctx);
                 return s == null ? null : s.iterator();
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java
index 8a9fabc9e1354..9d4dbaef21125 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java
@@ -47,4 +47,11 @@ public interface IndexOrdinalsFieldData extends IndexFieldData.Global<AtomicOrdi
      * or null if global ordinals are not needed (constant value or single segment).
      */
     OrdinalMap getOrdinalMap();
+
+    /**
+     * Whether this field data is able to provide a mapping between global and segment ordinals,
+     * by returning the underlying {@link OrdinalMap}. If this method returns false, then calling
+     * {@link #getOrdinalMap} will result in an {@link UnsupportedOperationException}.
+     */
+    boolean supportsGlobalOrdinalsMapping();
 }
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java
index 2bf2abac957de..33073dff2515b 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.index.OrdinalMap;
 import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LongValues;
 
@@ -34,12 +35,12 @@ final class GlobalOrdinalMapping extends SortedSetDocValues {
     private final SortedSetDocValues values;
     private final OrdinalMap ordinalMap;
     private final LongValues mapping;
-    private final SortedSetDocValues[] bytesValues;
+    private final TermsEnum[] lookups;
 
-    GlobalOrdinalMapping(OrdinalMap ordinalMap, SortedSetDocValues[] bytesValues, int segmentIndex) {
+    GlobalOrdinalMapping(OrdinalMap ordinalMap, SortedSetDocValues values, TermsEnum[] lookups, int segmentIndex) {
         super();
-        this.values = bytesValues[segmentIndex];
-        this.bytesValues = bytesValues;
+        this.values = values;
+        this.lookups = lookups;
         this.ordinalMap = ordinalMap;
         this.mapping = ordinalMap.getGlobalOrds(segmentIndex);
     }
@@ -72,7 +73,8 @@ public long nextOrd() throws IOException {
     public BytesRef lookupOrd(long globalOrd) throws IOException {
         final long segmentOrd = ordinalMap.getFirstSegmentOrd(globalOrd);
         int readerIndex = ordinalMap.getFirstSegmentNumber(globalOrd);
-        return bytesValues[readerIndex].lookupOrd(segmentOrd);
+        lookups[readerIndex].seekExact(segmentOrd);
+        return lookups[readerIndex].term();
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java
index a869e5a40d4ed..04fa72cfdd8a1 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java
@@ -22,53 +22,64 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.OrdinalMap;
 import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.Accountable;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
 import org.elasticsearch.search.MultiValueMode;
 
+import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.function.Function;
 
 /**
- * {@link IndexFieldData} base class for concrete global ordinals implementations.
+ * Concrete implementation of {@link IndexOrdinalsFieldData} for global ordinals.
+ * A single instance of this class should be used to cache global ordinals per {@link DirectoryReader}.
+ * However {@link #loadGlobal(DirectoryReader)} always creates a new instance of {@link Consumer} from the cached
+ * value in order to reuse the segment's {@link TermsEnum} that are needed to retrieve terms from global ordinals.
+ * Each instance of {@link Consumer} uses a new set of {@link TermsEnum} that can be reused during the collection,
+ * this is done to avoid creating all segment's {@link TermsEnum} each time we want to access the values of a single
+ * segment.
  */
-public class GlobalOrdinalsIndexFieldData extends AbstractIndexComponent implements IndexOrdinalsFieldData, Accountable {
+public final class GlobalOrdinalsIndexFieldData extends AbstractIndexComponent implements IndexOrdinalsFieldData, Accountable {
 
     private final String fieldName;
     private final long memorySizeInBytes;
 
     private final OrdinalMap ordinalMap;
-    private final Atomic[] atomicReaders;
+    private final AtomicOrdinalsFieldData[] segmentAfd;
     private final Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction;
 
-
-    protected GlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, AtomicOrdinalsFieldData[] segmentAfd,
-                                           OrdinalMap ordinalMap, long memorySizeInBytes, Function<SortedSetDocValues,
-                                           ScriptDocValues<?>> scriptFunction) {
+    protected GlobalOrdinalsIndexFieldData(IndexSettings indexSettings,
+                                           String fieldName,
+                                           AtomicOrdinalsFieldData[] segmentAfd,
+                                           OrdinalMap ordinalMap,
+                                           long memorySizeInBytes,
+                                           Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction) {
         super(indexSettings);
         this.fieldName = fieldName;
         this.memorySizeInBytes = memorySizeInBytes;
         this.ordinalMap = ordinalMap;
-        this.atomicReaders = new Atomic[segmentAfd.length];
-        for (int i = 0; i < segmentAfd.length; i++) {
-            atomicReaders[i] = new Atomic(segmentAfd[i], ordinalMap, i);
-        }
+        this.segmentAfd = segmentAfd;
         this.scriptFunction = scriptFunction;
     }
 
+    public IndexOrdinalsFieldData newConsumer(DirectoryReader source) {
+        return new Consumer(source, indexSettings);
+    }
+
     @Override
     public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
-        return load(context);
+        throw new IllegalStateException("loadDirect(LeafReaderContext) should not be called in this context");
     }
 
     @Override
@@ -92,9 +103,7 @@ public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMod
     }
 
     @Override
-    public void clear() {
-        // no need to clear, because this is cached and cleared in AbstractBytesIndexFieldData
-    }
+    public void clear() {}
 
     @Override
     public long ramBytesUsed() {
@@ -109,7 +118,7 @@ public Collection<Accountable> getChildResources() {
 
     @Override
     public AtomicOrdinalsFieldData load(LeafReaderContext context) {
-        return atomicReaders[context.ord];
+        throw new IllegalStateException("load(LeafReaderContext) should not be called in this context");
     }
 
     @Override
@@ -117,45 +126,118 @@ public OrdinalMap getOrdinalMap() {
         return ordinalMap;
     }
 
-    private final class Atomic extends AbstractAtomicOrdinalsFieldData {
+    @Override
+    public boolean supportsGlobalOrdinalsMapping() {
+        return true;
+    }
+
+    /**
+     * A non-thread safe {@link IndexOrdinalsFieldData} for global ordinals that creates the {@link TermsEnum} of each
+     * segment once and use them to provide a single lookup per segment.
+     */
+    public class Consumer extends AbstractIndexComponent implements IndexOrdinalsFieldData, Accountable {
+        private final DirectoryReader source;
+        private TermsEnum[] lookups;
+
+        Consumer(DirectoryReader source, IndexSettings settings) {
+            super(settings);
+            this.source = source;
+        }
 
-        private final AtomicOrdinalsFieldData afd;
-        private final OrdinalMap ordinalMap;
-        private final int segmentIndex;
+        /**
+         * Lazy creation of the {@link TermsEnum} for each segment present in this reader
+         */
+        private TermsEnum[] getOrLoadTermsEnums() {
+            if (lookups == null) {
+                lookups = new TermsEnum[segmentAfd.length];
+                for (int i = 0; i < lookups.length; i++) {
+                    try {
+                        lookups[i] = segmentAfd[i].getOrdinalsValues().termsEnum();
+                    } catch (IOException e) {
+                        throw new UncheckedIOException("Failed to load terms enum", e);
+                    }
+                }
+            }
+            return lookups;
+        }
 
-        private Atomic(AtomicOrdinalsFieldData afd, OrdinalMap ordinalMap, int segmentIndex) {
-            super(scriptFunction);
-            this.afd = afd;
-            this.ordinalMap = ordinalMap;
-            this.segmentIndex = segmentIndex;
+        @Override
+        public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
+            return load(context);
         }
 
         @Override
-        public SortedSetDocValues getOrdinalsValues() {
-            final SortedSetDocValues values = afd.getOrdinalsValues();
-            if (values.getValueCount() == ordinalMap.getValueCount()) {
-                // segment ordinals match global ordinals
-                return values;
-            }
-            final SortedSetDocValues[] bytesValues = new SortedSetDocValues[atomicReaders.length];
-            for (int i = 0; i < bytesValues.length; i++) {
-                bytesValues[i] = atomicReaders[i].afd.getOrdinalsValues();
-            }
-            return new GlobalOrdinalMapping(ordinalMap, bytesValues, segmentIndex);
+        public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
+            return this;
         }
 
+        @Override
+        public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
+            return this;
+        }
+
+        @Override
+        public String getFieldName() {
+            return fieldName;
+        }
+
+        @Override
+        public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+            throw new UnsupportedOperationException("no global ordinals sorting yet");
+        }
+
+        @Override
+        public void clear() {}
+
         @Override
         public long ramBytesUsed() {
-            return afd.ramBytesUsed();
+            return memorySizeInBytes;
         }
 
         @Override
         public Collection<Accountable> getChildResources() {
-            return afd.getChildResources();
+            return Collections.emptyList();
+        }
+
+        @Override
+        public AtomicOrdinalsFieldData load(LeafReaderContext context) {
+            assert source.getReaderCacheHelper().getKey() == context.parent.reader().getReaderCacheHelper().getKey();
+            return new AbstractAtomicOrdinalsFieldData(scriptFunction) {
+                @Override
+                public SortedSetDocValues getOrdinalsValues() {
+                    final SortedSetDocValues values = segmentAfd[context.ord].getOrdinalsValues();
+                    if (values.getValueCount() == ordinalMap.getValueCount()) {
+                        // segment ordinals match global ordinals
+                        return values;
+                    }
+                    final TermsEnum[] atomicLookups = getOrLoadTermsEnums();
+                    return new GlobalOrdinalMapping(ordinalMap, values, atomicLookups, context.ord);
+                }
+
+                @Override
+                public long ramBytesUsed() {
+                    return segmentAfd[context.ord].ramBytesUsed();
+                }
+
+
+                @Override
+                public Collection<Accountable> getChildResources() {
+                    return segmentAfd[context.ord].getChildResources();
+                }
+
+                @Override
+                public void close() {}
+            };
+        }
+
+        @Override
+        public boolean supportsGlobalOrdinalsMapping() {
+            return true;
         }
 
         @Override
-        public void close() {
+        public OrdinalMap getOrdinalMap() {
+            return ordinalMap;
         }
 
     }
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
index 0dc0de838a3e5..081875ae1e233 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
+import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsIndexFieldData;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 
 import java.io.IOException;
@@ -60,6 +61,17 @@ public OrdinalMap getOrdinalMap() {
 
     @Override
     public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
+        IndexOrdinalsFieldData fieldData = loadGlobalInternal(indexReader);
+        if (fieldData instanceof GlobalOrdinalsIndexFieldData) {
+            // we create a new instance of the cached value for each consumer in order
+            // to avoid creating new TermsEnums for each segment in the cached instance
+            return ((GlobalOrdinalsIndexFieldData) fieldData).newConsumer(indexReader);
+        } else {
+            return fieldData;
+        }
+    }
+
+    private IndexOrdinalsFieldData loadGlobalInternal(DirectoryReader indexReader) {
         if (indexReader.leaves().size() <= 1) {
             // ordinals are already global
             return this;
@@ -126,6 +138,11 @@ protected TermsEnum filter(Terms terms, TermsEnum iterator, LeafReader reader) t
         return iterator;
     }
 
+    @Override
+    public boolean supportsGlobalOrdinalsMapping() {
+        return false;
+    }
+
     private static final class FrequencyFilter extends FilteredTermsEnum {
 
         private int minFreq;
diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
index b71dcc7593417..4aaddc7b8dca6 100644
--- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
+++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsIndexFieldData;
 import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.search.MultiValueMode;
@@ -92,6 +93,17 @@ public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exce
 
     @Override
     public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
+        IndexOrdinalsFieldData fieldData = loadGlobalInternal(indexReader);
+        if (fieldData instanceof GlobalOrdinalsIndexFieldData) {
+            // we create a new instance of the cached value for each consumer in order
+            // to avoid creating new TermsEnums for each segment in the cached instance
+            return ((GlobalOrdinalsIndexFieldData) fieldData).newConsumer(indexReader);
+        } else {
+            return fieldData;
+        }
+    }
+
+    private IndexOrdinalsFieldData loadGlobalInternal(DirectoryReader indexReader) {
         if (indexReader.leaves().size() <= 1) {
             // ordinals are already global
             return this;
@@ -134,4 +146,9 @@ public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) thr
     public OrdinalMap getOrdinalMap() {
         return null;
     }
+
+    @Override
+    public boolean supportsGlobalOrdinalsMapping() {
+        return true;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/IdOnlyFieldVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/IdOnlyFieldVisitor.java
new file mode 100644
index 0000000000000..13566b841c832
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/IdOnlyFieldVisitor.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fieldvisitor;
+
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.StoredFieldVisitor;
+import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
+
+public final class IdOnlyFieldVisitor extends StoredFieldVisitor {
+    private String id = null;
+    private boolean visited = false;
+
+    @Override
+    public Status needsField(FieldInfo fieldInfo) {
+        if (visited) {
+            return Status.STOP;
+        }
+        if (IdFieldMapper.NAME.equals(fieldInfo.name)) {
+            visited = true;
+            return Status.YES;
+        } else {
+            return Status.NO;
+        }
+    }
+
+    @Override
+    public void binaryField(FieldInfo fieldInfo, byte[] value) {
+        assert IdFieldMapper.NAME.equals(fieldInfo.name) : fieldInfo;
+        if (IdFieldMapper.NAME.equals(fieldInfo.name)) {
+            id = Uid.decodeId(value);
+        }
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public void reset() {
+        id = null;
+        visited = false;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java
index 74892bf7d516c..20151f301d791 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/BaseGeoShapeFieldMapper.java
@@ -190,7 +190,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
                 }
             }
             final Builder builder;
-            if (parsedDeprecatedParams || parserContext.indexVersionCreated().before(Version.V_6_6_0)) {
+            if (parsedDeprecatedParams) {
                 // Legacy index-based shape
                 builder = new LegacyGeoShapeFieldMapper.Builder(name, deprecatedParameters);
             } else {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ContentPath.java b/server/src/main/java/org/elasticsearch/index/mapper/ContentPath.java
index 3c67d3ee7f31c..7f3e26312ad87 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/ContentPath.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/ContentPath.java
@@ -66,4 +66,8 @@ public String pathAsText(String name) {
         sb.append(name);
         return sb.toString();
     }
+
+    public int length() {
+        return index;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index 9f5293e7da97a..07231759113e9 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -764,13 +764,17 @@ private static Mapper.Builder<?,?> createBuilderFromDynamicValue(final ParseCont
             return builder;
         } else if (token == XContentParser.Token.VALUE_NUMBER) {
             XContentParser.NumberType numberType = context.parser().numberType();
-            if (numberType == XContentParser.NumberType.INT || numberType == XContentParser.NumberType.LONG) {
+            if (numberType == XContentParser.NumberType.INT
+                    || numberType == XContentParser.NumberType.LONG
+                    || numberType == XContentParser.NumberType.BIG_INTEGER) {
                 Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG);
                 if (builder == null) {
                     builder = newLongBuilder(currentFieldName);
                 }
                 return builder;
-            } else if (numberType == XContentParser.NumberType.FLOAT || numberType == XContentParser.NumberType.DOUBLE) {
+            } else if (numberType == XContentParser.NumberType.FLOAT
+                    || numberType == XContentParser.NumberType.DOUBLE
+                    || numberType == XContentParser.NumberType.BIG_DECIMAL) {
                 Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE);
                 if (builder == null) {
                     // no templates are defined, we use float by default instead of double
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java
new file mode 100644
index 0000000000000..0bbd383079c76
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.common.settings.Settings;
+
+/**
+ * A field mapper that supports lookup of dynamic sub-keys. If the field mapper is named 'my_field',
+ * then a user is able to search on the field in both of the following ways:
+ * - Using the field name 'my_field', which will delegate to the field type
+ *   {@link DynamicKeyFieldMapper#fieldType()} as usual.
+ * - Using any sub-key, for example 'my_field.some_key'. In this case, the search is delegated
+ *   to {@link DynamicKeyFieldMapper#keyedFieldType(String)}, with 'some_key' passed as the
+ *   argument. The field mapper is allowed to create a new field type dynamically in order
+ *   to handle the search.
+ *
+ * To prevent conflicts between these dynamic sub-keys and multi-fields, any field mappers
+ * implementing this interface should explicitly disallow multi-fields. The constructor makes
+ * sure to passes an empty multi-fields list to help prevent conflicting sub-keys from being
+ * registered.
+ *
+ * Note: we anticipate that 'flattened' fields will be the only implementation of this
+ * interface. Flattened object fields live in the 'mapper-flattened' module.
+ */
+public abstract class DynamicKeyFieldMapper extends FieldMapper {
+
+    public DynamicKeyFieldMapper(String simpleName,
+                                 MappedFieldType fieldType,
+                                 MappedFieldType defaultFieldType,
+                                 Settings indexSettings,
+                                 CopyTo copyTo) {
+        super(simpleName, fieldType, defaultFieldType, indexSettings, MultiFields.empty(), copyTo);
+    }
+
+    public abstract MappedFieldType keyedFieldType(String key);
+
+}
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
index 29f22d8dc2cf0..23753b881f20c 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
@@ -33,6 +33,8 @@
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.support.AbstractXContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper.FieldNamesFieldType;
 import org.elasticsearch.index.similarity.SimilarityProvider;
@@ -46,6 +48,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.HashMap;
 import java.util.Objects;
 import java.util.stream.StreamSupport;
 
@@ -190,7 +193,7 @@ public Builder nullValue(Object nullValue) {
             return this;
         }
 
-        public T addMultiField(Mapper.Builder mapperBuilder) {
+        public T addMultiField(Mapper.Builder<?, ?> mapperBuilder) {
             multiFieldsBuilder.add(mapperBuilder);
             return builder;
         }
@@ -276,14 +279,33 @@ public void parse(ParseContext context) throws IOException {
                 context.doc().add(field);
             }
         } catch (Exception e) {
-            throw new MapperParsingException("failed to parse field [{}] of type [{}] in document with id '{}'", e, fieldType().name(),
-                    fieldType().typeName(), context.sourceToParse().id());
+            String valuePreview = "";
+            try {
+                XContentParser parser = context.parser();
+                Object complexValue = AbstractXContentParser.readValue(parser, ()-> new HashMap<String, Object>());
+                if (complexValue == null) {
+                    valuePreview = "null";
+                } else {
+                    valuePreview = complexValue.toString();
+                }
+            } catch (Exception innerException) {
+                throw new MapperParsingException("failed to parse field [{}] of type [{}] in document with id '{}'. " +
+                    "Could not parse field value preview,",
+                    e, fieldType().name(), fieldType().typeName(), context.sourceToParse().id());
+            }
+
+            throw new MapperParsingException("failed to parse field [{}] of type [{}] in document with id '{}'. " +
+                "Preview of field's value: '{}'", e, fieldType().name(), fieldType().typeName(),
+                context.sourceToParse().id(), valuePreview);
         }
         multiFields.parse(this, context);
     }
 
     /**
      * Parse the field value and populate <code>fields</code>.
+     *
+     * Implementations of this method should ensure that on failing to parse parser.currentToken() must be the
+     * current failing token
      */
     protected abstract void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException;
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
index 27d061d8c2788..7d8d5ef943de7 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
@@ -20,11 +20,13 @@
 package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.common.collect.CopyOnWriteHashMap;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.regex.Regex;
 
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
@@ -36,15 +38,30 @@ class FieldTypeLookup implements Iterable<MappedFieldType> {
     final CopyOnWriteHashMap<String, MappedFieldType> fullNameToFieldType;
     private final CopyOnWriteHashMap<String, String> aliasToConcreteName;
 
+    private final CopyOnWriteHashMap<String, DynamicKeyFieldMapper> dynamicKeyMappers;
+
+    /**
+     * The maximum field depth of any mapper that implements {@link DynamicKeyFieldMapper}.
+     * Allows us stop searching for a 'dynamic key' mapper as soon as we've passed the maximum
+     * possible field depth.
+     */
+    private final int maxDynamicKeyDepth;
+
     FieldTypeLookup() {
         fullNameToFieldType = new CopyOnWriteHashMap<>();
         aliasToConcreteName = new CopyOnWriteHashMap<>();
+        dynamicKeyMappers = new CopyOnWriteHashMap<>();
+        maxDynamicKeyDepth = 0;
     }
 
     private FieldTypeLookup(CopyOnWriteHashMap<String, MappedFieldType> fullNameToFieldType,
-                            CopyOnWriteHashMap<String, String> aliasToConcreteName) {
+                            CopyOnWriteHashMap<String, String> aliasToConcreteName,
+                            CopyOnWriteHashMap<String, DynamicKeyFieldMapper> dynamicKeyMappers,
+                            int maxDynamicKeyDepth) {
         this.fullNameToFieldType = fullNameToFieldType;
         this.aliasToConcreteName = aliasToConcreteName;
+        this.dynamicKeyMappers = dynamicKeyMappers;
+        this.maxDynamicKeyDepth = maxDynamicKeyDepth;
     }
 
     /**
@@ -63,14 +80,21 @@ public FieldTypeLookup copyAndAddAll(String type,
 
         CopyOnWriteHashMap<String, MappedFieldType> fullName = this.fullNameToFieldType;
         CopyOnWriteHashMap<String, String> aliases = this.aliasToConcreteName;
+        CopyOnWriteHashMap<String, DynamicKeyFieldMapper> dynamicKeyMappers = this.dynamicKeyMappers;
 
         for (FieldMapper fieldMapper : fieldMappers) {
+            String fieldName = fieldMapper.name();
             MappedFieldType fieldType = fieldMapper.fieldType();
             MappedFieldType fullNameFieldType = fullName.get(fieldType.name());
 
             if (Objects.equals(fieldType, fullNameFieldType) == false) {
                 fullName = fullName.copyAndPut(fieldType.name(), fieldType);
             }
+
+            if (fieldMapper instanceof DynamicKeyFieldMapper) {
+                DynamicKeyFieldMapper dynamicKeyMapper = (DynamicKeyFieldMapper) fieldMapper;
+                dynamicKeyMappers = dynamicKeyMappers.copyAndPut(fieldName, dynamicKeyMapper);
+            }
         }
 
         for (FieldAliasMapper fieldAliasMapper : fieldAliasMappers) {
@@ -83,20 +107,97 @@ public FieldTypeLookup copyAndAddAll(String type,
             }
         }
 
-        return new FieldTypeLookup(fullName, aliases);
+        int maxDynamicKeyDepth = getMaxDynamicKeyDepth(aliases, dynamicKeyMappers);
+
+        return new FieldTypeLookup(fullName, aliases, dynamicKeyMappers, maxDynamicKeyDepth);
+    }
+
+    private static int getMaxDynamicKeyDepth(CopyOnWriteHashMap<String, String> aliases,
+                                             CopyOnWriteHashMap<String, DynamicKeyFieldMapper> dynamicKeyMappers) {
+        int maxFieldDepth = 0;
+        for (Map.Entry<String, String> entry : aliases.entrySet()) {
+            String aliasName = entry.getKey();
+            String path = entry.getValue();
+            if (dynamicKeyMappers.containsKey(path)) {
+                maxFieldDepth = Math.max(maxFieldDepth, fieldDepth(aliasName));
+            }
+        }
+
+        for (String fieldName : dynamicKeyMappers.keySet()) {
+            if (dynamicKeyMappers.containsKey(fieldName)) {
+                maxFieldDepth = Math.max(maxFieldDepth, fieldDepth(fieldName));
+            }
+        }
+
+        return maxFieldDepth;
     }
 
+    /**
+     * Computes the total depth of this field by counting the number of parent fields
+     * in its path. As an example, the field 'parent1.parent2.field' has depth 3.
+     */
+    private static int fieldDepth(String field) {
+        int numDots = 0;
+        int dotIndex = -1;
+        while (true) {
+            dotIndex = field.indexOf('.', dotIndex + 1);
+            if (dotIndex < 0) {
+                break;
+            }
+            numDots++;
+        }
+        return numDots + 1;
+    }
 
-    /** Returns the field for the given field */
+    /**
+     * Returns the mapped field type for the given field name.
+     */
     public MappedFieldType get(String field) {
         String concreteField = aliasToConcreteName.getOrDefault(field, field);
-        return fullNameToFieldType.get(concreteField);
+        MappedFieldType fieldType = fullNameToFieldType.get(concreteField);
+        if (fieldType != null) {
+            return fieldType;
+        }
+
+        // If the mapping contains fields that support dynamic sub-key lookup, check
+        // if this could correspond to a keyed field of the form 'path_to_field.path_to_key'.
+        return !dynamicKeyMappers.isEmpty() ? getKeyedFieldType(field) : null;
+    }
+
+    /**
+     * Check if the given field corresponds to a dynamic lookup mapper of the
+     * form 'path_to_field.path_to_key'. If so, returns a field type that
+     * can be used to perform searches on this field.
+     */
+    private MappedFieldType getKeyedFieldType(String field) {
+        int dotIndex = -1;
+        int fieldDepth = 0;
+
+        while (true) {
+            if (++fieldDepth > maxDynamicKeyDepth) {
+                return null;
+            }
+
+            dotIndex = field.indexOf('.', dotIndex + 1);
+            if (dotIndex < 0) {
+                return null;
+            }
+
+            String parentField = field.substring(0, dotIndex);
+            String concreteField = aliasToConcreteName.getOrDefault(parentField, parentField);
+            DynamicKeyFieldMapper mapper = dynamicKeyMappers.get(concreteField);
+
+            if (mapper != null) {
+                String key = field.substring(dotIndex + 1);
+                return mapper.keyedFieldType(key);
+            }
+        }
     }
 
     /**
      * Returns a list of the full names of a simple match regex like pattern against full name and index name.
      */
-    public Collection<String> simpleMatchToFullName(String pattern) {
+    public Set<String> simpleMatchToFullName(String pattern) {
         Set<String> fields = new HashSet<>();
         for (MappedFieldType fieldType : this) {
             if (Regex.simpleMatch(pattern, fieldType.name())) {
@@ -113,6 +214,20 @@ public Collection<String> simpleMatchToFullName(String pattern) {
 
     @Override
     public Iterator<MappedFieldType> iterator() {
-        return fullNameToFieldType.values().iterator();
+        Iterator<MappedFieldType> concreteFieldTypes = fullNameToFieldType.values().iterator();
+
+        if (dynamicKeyMappers.isEmpty()) {
+            return concreteFieldTypes;
+        } else {
+            Iterator<MappedFieldType> keyedFieldTypes = dynamicKeyMappers.values().stream()
+                .<MappedFieldType>map(mapper -> mapper.keyedFieldType(""))
+                .iterator();
+            return Iterators.concat(concreteFieldTypes, keyedFieldTypes);
+        }
+    }
+
+    // Visible for testing.
+    int maxKeyedLookupDepth() {
+        return maxDynamicKeyDepth;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java
index 9c8726a498a3c..c4996eab901dd 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java
@@ -208,8 +208,6 @@ private void setupFieldTypeDeprecatedParameters(BuilderContext context) {
             }
             if (deprecatedParameters.tree != null) {
                 ft.setTree(deprecatedParameters.tree);
-            } else if (context.indexCreatedVersion().before(Version.V_6_6_0)) {
-                ft.setTree(DeprecatedParameters.PrefixTrees.GEOHASH);
             }
             if (deprecatedParameters.treeLevels != null) {
                 ft.setTreeLevels(deprecatedParameters.treeLevels);
@@ -527,8 +525,7 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
         super.doXContentBody(builder, includeDefaults, params);
 
         if (includeDefaults
-            || (fieldType().tree().equals(indexCreatedVersion.onOrAfter(Version.V_6_6_0) ?
-                    DeprecatedParameters.Defaults.TREE : DeprecatedParameters.PrefixTrees.GEOHASH)) == false) {
+            || (fieldType().tree().equals(DeprecatedParameters.Defaults.TREE)) == false) {
             builder.field(DeprecatedParameters.Names.TREE.getPreferredName(), fieldType().tree());
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
index 5ef689709400d..411045abaf796 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
@@ -390,7 +390,8 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew
     /**
      * Create an {@link IntervalsSource} to be used for proximity queries
      */
-    public IntervalsSource intervals(String query, int max_gaps, boolean ordered, NamedAnalyzer analyzer) throws IOException {
+    public IntervalsSource intervals(String query, int max_gaps, boolean ordered,
+                                     NamedAnalyzer analyzer, boolean prefix) throws IOException {
         throw new IllegalArgumentException("Can only use interval queries on text fields - not on [" + name
             + "] which is of type [" + typeName() + "]");
     }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index 487a6ac4789e3..6002348050700 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -46,8 +47,13 @@
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexSortConfig;
+import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.index.analysis.CharFilterFactory;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.analysis.ReloadableCustomAnalyzer;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.index.mapper.Mapper.BuilderContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.similarity.SimilarityService;
@@ -260,9 +266,7 @@ private void assertMappingVersion(
             final IndexMetaData currentIndexMetaData,
             final IndexMetaData newIndexMetaData,
             final Map<String, DocumentMapper> updatedEntries) {
-        if (Assertions.ENABLED
-                && currentIndexMetaData != null
-                && currentIndexMetaData.getCreationVersion().onOrAfter(Version.V_6_5_0)) {
+        if (Assertions.ENABLED && currentIndexMetaData != null) {
             if (currentIndexMetaData.getMappingVersion() == newIndexMetaData.getMappingVersion()) {
                 // if the mapping version is unchanged, then there should not be any updates and all mappings should be the same
                 assert updatedEntries.isEmpty() : updatedEntries;
@@ -744,10 +748,10 @@ public MappedFieldType fullName(String fullName) {
      * Returns all the fields that match the given pattern. If the pattern is prefixed with a type
      * then the fields will be returned with a type prefix.
      */
-    public Collection<String> simpleMatchToFullName(String pattern) {
+    public Set<String> simpleMatchToFullName(String pattern) {
         if (Regex.isSimpleMatchPattern(pattern) == false) {
             // no wildcards
-            return Collections.singletonList(pattern);
+            return Collections.singleton(pattern);
         }
         return fieldTypes.simpleMatchToFullName(pattern);
     }
@@ -843,4 +847,24 @@ protected Analyzer getWrappedAnalyzer(String fieldName) {
             return defaultAnalyzer;
         }
     }
+
+    public synchronized List<String> reloadSearchAnalyzers(AnalysisRegistry registry) throws IOException {
+        logger.info("reloading search analyzers");
+        // refresh indexAnalyzers and search analyzers
+        final Map<String, TokenizerFactory> tokenizerFactories = registry.buildTokenizerFactories(indexSettings);
+        final Map<String, CharFilterFactory> charFilterFactories = registry.buildCharFilterFactories(indexSettings);
+        final Map<String, TokenFilterFactory> tokenFilterFactories = registry.buildTokenFilterFactories(indexSettings);
+        final Map<String, Settings> settings = indexSettings.getSettings().getGroups("index.analysis.analyzer");
+        final List<String> reloadedAnalyzers = new ArrayList<>();
+        for (NamedAnalyzer namedAnalyzer : indexAnalyzers.getAnalyzers().values()) {
+            if (namedAnalyzer.analyzer() instanceof ReloadableCustomAnalyzer) {
+                ReloadableCustomAnalyzer analyzer = (ReloadableCustomAnalyzer) namedAnalyzer.analyzer();
+                String analyzerName = namedAnalyzer.name();
+                Settings analyzerSettings = settings.get(analyzerName);
+                analyzer.reload(analyzerName, analyzerSettings, tokenizerFactories, charFilterFactories, tokenFilterFactories);
+                reloadedAnalyzers.add(analyzerName);
+            }
+        }
+        return reloadedAnalyzers;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
index 4cfd5be2afed3..b2c536a900e33 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
@@ -24,7 +24,6 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.IndexSettings;
 
@@ -458,18 +457,10 @@ public Iterable<Document> nonRootDocuments() {
         void postParse() {
             if (documents.size() > 1) {
                 docsReversed = true;
-                if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_6_5_0)) {
-                    /**
-                     * For indices created on or after {@link Version#V_6_5_0} we preserve the order
-                     * of the children while ensuring that parents appear after them.
-                     */
-                    List<Document> newDocs = reorderParent(documents);
-                    documents.clear();
-                    documents.addAll(newDocs);
-                } else {
-                    // reverse the order of docs for nested docs support, parent should be last
-                    Collections.reverse(documents);
-                }
+                // We preserve the order of the children while ensuring that parents appear after them.
+                List<Document> newDocs = reorderParent(documents);
+                documents.clear();
+                documents.addAll(newDocs);
             }
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
index 8a3203ad8e7e0..d1ae07a7f0b9d 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
@@ -44,6 +44,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SynonymQuery;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.intervals.Intervals;
 import org.apache.lucene.search.intervals.IntervalsSource;
 import org.apache.lucene.search.spans.FieldMaskingSpanQuery;
 import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
@@ -51,6 +52,7 @@
 import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.Operations;
@@ -109,7 +111,8 @@ public static class Defaults {
     public static class Builder extends FieldMapper.Builder<Builder, TextFieldMapper> {
 
         private int positionIncrementGap = POSITION_INCREMENT_GAP_USE_ANALYZER;
-        private PrefixFieldType prefixFieldType;
+        private int minPrefixChars = -1;
+        private int maxPrefixChars = -1;
 
         public Builder(String name) {
             super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
@@ -160,6 +163,7 @@ public Builder fielddataFrequencyFilter(double minFreq, double maxFreq, int minS
         }
 
         public Builder indexPrefixes(int minChars, int maxChars) {
+
             if (minChars > maxChars) {
                 throw new IllegalArgumentException("min_chars [" + minChars + "] must be less than max_chars [" + maxChars + "]");
             }
@@ -169,8 +173,8 @@ public Builder indexPrefixes(int minChars, int maxChars) {
             if (maxChars >= 20) {
                 throw new IllegalArgumentException("max_chars [" + maxChars + "] must be less than 20");
             }
-            this.prefixFieldType = new PrefixFieldType(name(), name() + "._index_prefix", minChars, maxChars);
-            fieldType().setPrefixFieldType(this.prefixFieldType);
+            this.minPrefixChars = minChars;
+            this.maxPrefixChars = maxChars;
             return this;
         }
 
@@ -187,21 +191,28 @@ public TextFieldMapper build(BuilderContext context) {
             }
             setupFieldType(context);
             PrefixFieldMapper prefixMapper = null;
-            if (prefixFieldType != null) {
+            if (minPrefixChars != -1) {
+                /**
+                 * Mappings before v7.2.1 use {@link Builder#name} instead of {@link Builder#fullName}
+                 * to build prefix field names so we preserve the name that was used at creation time
+                 * even if it is different from the expected one (in case the field is nested under an object
+                 * or a multi-field). This way search will continue to work on old indices and new indices
+                 * will use the expected full name.
+                 **/
+                String fullName = context.indexCreatedVersion().before(Version.V_7_2_1) ? name() : buildFullName(context);
+                PrefixFieldType prefixFieldType =
+                    new PrefixFieldType(fullName, fullName + "._index_prefix", minPrefixChars, maxPrefixChars);
+                fieldType().setPrefixFieldType(prefixFieldType);
                 if (fieldType().isSearchable() == false) {
                     throw new IllegalArgumentException("Cannot set index_prefixes on unindexed field [" + name() + "]");
                 }
                 // Copy the index options of the main field to allow phrase queries on
                 // the prefix field.
-                if (context.indexCreatedVersion().onOrAfter(Version.V_6_4_0)) {
-                    if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS) {
-                        // frequencies are not needed because prefix queries always use a constant score
-                        prefixFieldType.setIndexOptions(IndexOptions.DOCS);
-                    } else {
-                        prefixFieldType.setIndexOptions(fieldType.indexOptions());
-                    }
-                } else if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {
-                    prefixFieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+                if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS) {
+                    // frequencies are not needed because prefix queries always use a constant score
+                    prefixFieldType.setIndexOptions(IndexOptions.DOCS);
+                } else {
+                    prefixFieldType.setIndexOptions(fieldType.indexOptions());
                 }
                 if (fieldType.storeTermVectorOffsets()) {
                     prefixFieldType.setStoreTermVectorOffsets(true);
@@ -408,6 +419,17 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, Quer
                 .build();
         }
 
+        public IntervalsSource intervals(BytesRef term) {
+            if (term.length > maxChars) {
+                return Intervals.prefix(term.utf8ToString());
+            }
+            if (term.length >= minChars) {
+                return Intervals.fixField(name(), Intervals.term(term));
+            }
+            String wildcardTerm = term.utf8ToString() + "?".repeat(Math.max(0, minChars - term.length));
+            return Intervals.or(Intervals.fixField(name(), Intervals.wildcard(wildcardTerm)), Intervals.term(term));
+        }
+
         @Override
         public PrefixFieldType clone() {
             return new PrefixFieldType(parentField, name(), minChars, maxChars);
@@ -636,10 +658,21 @@ public Query existsQuery(QueryShardContext context) {
         }
 
         @Override
-        public IntervalsSource intervals(String text, int maxGaps, boolean ordered, NamedAnalyzer analyzer) throws IOException {
+        public IntervalsSource intervals(String text, int maxGaps, boolean ordered,
+                                         NamedAnalyzer analyzer, boolean prefix) throws IOException {
             if (indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
                 throw new IllegalArgumentException("Cannot create intervals over field [" + name() + "] with no positions indexed");
             }
+            if (analyzer == null) {
+                analyzer = searchAnalyzer();
+            }
+            if (prefix) {
+                BytesRef normalizedTerm = analyzer.normalize(name(), text);
+                if (prefixFieldType != null) {
+                    return prefixFieldType.intervals(normalizedTerm);
+                }
+                return Intervals.prefix(normalizedTerm.utf8ToString()); // TODO make Intervals.prefix() take a BytesRef
+            }
             IntervalBuilder builder = new IntervalBuilder(name(), analyzer == null ? searchAnalyzer() : analyzer);
             return builder.analyzeText(text, maxGaps, ordered);
         }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
index 9848a23cac11b..12c80361a855c 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
@@ -22,6 +22,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.index.IndexOptions;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -219,11 +220,18 @@ public static boolean parseMultiField(FieldMapper.Builder builder, String name,
                                           String propName, Object propNode) {
         if (propName.equals("fields")) {
             if (parserContext.isWithinMultiField()) {
-                deprecationLogger.deprecatedAndMaybeLog("multifield_within_multifield", "At least one multi-field, [" + name + "], was " +
-                    "encountered that itself contains a multi-field. Defining multi-fields within a multi-field is deprecated and will " +
-                    "no longer be supported in 8.0. To resolve the issue, all instances of [fields] that occur within a [fields] block " +
-                    "should be removed from the mappings, either by flattening the chained [fields] blocks into a single level, or " +
-                    "switching to [copy_to] if appropriate.");
+                // For indices created prior to 8.0, we only emit a deprecation warning and do not fail type parsing. This is to
+                // maintain the backwards-compatibility guarantee that we can always load indexes from the previous major version.
+                if (parserContext.indexVersionCreated().before(Version.V_8_0_0)) {
+                    deprecationLogger.deprecatedAndMaybeLog("multifield_within_multifield", "At least one multi-field, [" + name + "], " +
+                        "was encountered that itself contains a multi-field. Defining multi-fields within a multi-field is deprecated " +
+                        "and is not supported for indices created in 8.0 and later. To migrate the mappings, all instances of [fields] " +
+                        "that occur within a [fields] block should be removed from the mappings, either by flattening the chained " +
+                        "[fields] blocks into a single level, or switching to [copy_to] if appropriate.");
+                } else {
+                    throw new IllegalArgumentException("Encountered a multi-field [" + name + "] which itself contains a multi-field. " +
+                        "Defining chained multi-fields is not supported.");
+                }
             }
 
             parserContext = parserContext.createMultiFieldContext(parserContext);
diff --git a/server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
deleted file mode 100644
index d646dc4bb4b07..0000000000000
--- a/server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
+++ /dev/null
@@ -1,406 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.query;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.ExtendedCommonTermsQuery;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.mapper.MappedFieldType;
-
-import java.io.IOException;
-import java.util.Objects;
-
-/**
- * CommonTermsQuery query is a query that executes high-frequency terms in a
- * optional sub-query to prevent slow queries due to "common" terms like
- * stopwords. This query basically builds 2 queries off the {@code #add(Term)
- * added} terms where low-frequency terms are added to a required boolean clause
- * and high-frequency terms are added to an optional boolean clause. The
- * optional clause is only executed if the required "low-frequency' clause
- * matches.
- */
-public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQueryBuilder> {
-
-    public static final String NAME = "common";
-
-    public static final float DEFAULT_CUTOFF_FREQ = 0.01f;
-    public static final Operator DEFAULT_HIGH_FREQ_OCCUR = Operator.OR;
-    public static final Operator DEFAULT_LOW_FREQ_OCCUR = Operator.OR;
-
-    private static final ParseField CUTOFF_FREQUENCY_FIELD = new ParseField("cutoff_frequency");
-    private static final ParseField MINIMUM_SHOULD_MATCH_FIELD = new ParseField("minimum_should_match");
-    private static final ParseField LOW_FREQ_OPERATOR_FIELD = new ParseField("low_freq_operator");
-    private static final ParseField HIGH_FREQ_OPERATOR_FIELD = new ParseField("high_freq_operator");
-    private static final ParseField DISABLE_COORD_FIELD = new ParseField("disable_coord")
-            .withAllDeprecated("disable_coord has been removed");
-    private static final ParseField ANALYZER_FIELD = new ParseField("analyzer");
-    private static final ParseField QUERY_FIELD = new ParseField("query");
-    private static final ParseField HIGH_FREQ_FIELD = new ParseField("high_freq");
-    private static final ParseField LOW_FREQ_FIELD = new ParseField("low_freq");
-
-    private final String fieldName;
-
-    private final Object text;
-
-    private Operator highFreqOperator = DEFAULT_HIGH_FREQ_OCCUR;
-
-    private Operator lowFreqOperator = DEFAULT_LOW_FREQ_OCCUR;
-
-    private String analyzer = null;
-
-    private String lowFreqMinimumShouldMatch = null;
-
-    private String highFreqMinimumShouldMatch = null;
-
-    private float cutoffFrequency = DEFAULT_CUTOFF_FREQ;
-
-    /**
-     * Constructs a new common terms query.
-     */
-    public CommonTermsQueryBuilder(String fieldName, Object text) {
-        if (Strings.isEmpty(fieldName)) {
-            throw new IllegalArgumentException("field name is null or empty");
-        }
-        if (text == null) {
-            throw new IllegalArgumentException("text cannot be null");
-        }
-        this.fieldName = fieldName;
-        this.text = text;
-    }
-
-    /**
-     * Read from a stream.
-     */
-    public CommonTermsQueryBuilder(StreamInput in) throws IOException {
-        super(in);
-        fieldName = in.readString();
-        text = in.readGenericValue();
-        highFreqOperator = Operator.readFromStream(in);
-        lowFreqOperator = Operator.readFromStream(in);
-        analyzer = in.readOptionalString();
-        lowFreqMinimumShouldMatch = in.readOptionalString();
-        highFreqMinimumShouldMatch = in.readOptionalString();
-        cutoffFrequency = in.readFloat();
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeString(this.fieldName);
-        out.writeGenericValue(this.text);
-        highFreqOperator.writeTo(out);
-        lowFreqOperator.writeTo(out);
-        out.writeOptionalString(analyzer);
-        out.writeOptionalString(lowFreqMinimumShouldMatch);
-        out.writeOptionalString(highFreqMinimumShouldMatch);
-        out.writeFloat(cutoffFrequency);
-    }
-
-    public String fieldName() {
-        return this.fieldName;
-    }
-
-    public Object value() {
-        return this.text;
-    }
-
-    /**
-     * Sets the operator to use for terms with a high document frequency
-     * (greater than or equal to {@link #cutoffFrequency(float)}. Defaults to
-     * {@code AND}.
-     */
-    public CommonTermsQueryBuilder highFreqOperator(Operator operator) {
-        this.highFreqOperator = (operator == null) ? DEFAULT_HIGH_FREQ_OCCUR : operator;
-        return this;
-    }
-
-    public Operator highFreqOperator() {
-        return highFreqOperator;
-    }
-
-    /**
-     * Sets the operator to use for terms with a low document frequency (less
-     * than {@link #cutoffFrequency(float)}. Defaults to {@code AND}.
-     */
-    public CommonTermsQueryBuilder lowFreqOperator(Operator operator) {
-        this.lowFreqOperator = (operator == null) ? DEFAULT_LOW_FREQ_OCCUR : operator;
-        return this;
-    }
-
-    public Operator lowFreqOperator() {
-        return lowFreqOperator;
-    }
-
-    /**
-     * Explicitly set the analyzer to use. Defaults to use explicit mapping
-     * config for the field, or, if not set, the default search analyzer.
-     */
-    public CommonTermsQueryBuilder analyzer(String analyzer) {
-        this.analyzer = analyzer;
-        return this;
-    }
-
-    public String analyzer() {
-        return this.analyzer;
-    }
-
-    /**
-     * Sets the cutoff document frequency for high / low frequent terms. A value
-     * in [0..1] (or absolute number &gt;=1) representing the maximum threshold of
-     * a terms document frequency to be considered a low frequency term.
-     * Defaults to
-     * {@code {@value #DEFAULT_CUTOFF_FREQ}}
-     */
-    public CommonTermsQueryBuilder cutoffFrequency(float cutoffFrequency) {
-        this.cutoffFrequency = cutoffFrequency;
-        return this;
-    }
-
-    public float cutoffFrequency() {
-        return this.cutoffFrequency;
-    }
-
-    /**
-     * Sets the minimum number of high frequent query terms that need to match in order to
-     * produce a hit when there are no low frequent terms.
-     */
-    public CommonTermsQueryBuilder highFreqMinimumShouldMatch(String highFreqMinimumShouldMatch) {
-        this.highFreqMinimumShouldMatch = highFreqMinimumShouldMatch;
-        return this;
-    }
-
-    public String highFreqMinimumShouldMatch() {
-        return this.highFreqMinimumShouldMatch;
-    }
-
-    /**
-     * Sets the minimum number of low frequent query terms that need to match in order to
-     * produce a hit.
-     */
-    public CommonTermsQueryBuilder lowFreqMinimumShouldMatch(String lowFreqMinimumShouldMatch) {
-        this.lowFreqMinimumShouldMatch = lowFreqMinimumShouldMatch;
-        return this;
-    }
-
-    public String lowFreqMinimumShouldMatch() {
-        return this.lowFreqMinimumShouldMatch;
-    }
-
-    @Override
-    protected void doXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(NAME);
-        builder.startObject(fieldName);
-        builder.field(QUERY_FIELD.getPreferredName(), text);
-        builder.field(HIGH_FREQ_OPERATOR_FIELD.getPreferredName(), highFreqOperator.toString());
-        builder.field(LOW_FREQ_OPERATOR_FIELD.getPreferredName(), lowFreqOperator.toString());
-        if (analyzer != null) {
-            builder.field(ANALYZER_FIELD.getPreferredName(), analyzer);
-        }
-        builder.field(CUTOFF_FREQUENCY_FIELD.getPreferredName(), cutoffFrequency);
-        if (lowFreqMinimumShouldMatch != null || highFreqMinimumShouldMatch != null) {
-            builder.startObject(MINIMUM_SHOULD_MATCH_FIELD.getPreferredName());
-            if (lowFreqMinimumShouldMatch != null) {
-                builder.field(LOW_FREQ_FIELD.getPreferredName(), lowFreqMinimumShouldMatch);
-            }
-            if (highFreqMinimumShouldMatch != null) {
-                builder.field(HIGH_FREQ_FIELD.getPreferredName(), highFreqMinimumShouldMatch);
-            }
-            builder.endObject();
-        }
-        printBoostAndQueryName(builder);
-        builder.endObject();
-        builder.endObject();
-    }
-
-    public static CommonTermsQueryBuilder fromXContent(XContentParser parser) throws IOException {
-        String fieldName = null;
-        Object text = null;
-        float boost = AbstractQueryBuilder.DEFAULT_BOOST;
-        String analyzer = null;
-        String lowFreqMinimumShouldMatch = null;
-        String highFreqMinimumShouldMatch = null;
-        Operator highFreqOperator = CommonTermsQueryBuilder.DEFAULT_HIGH_FREQ_OCCUR;
-        Operator lowFreqOperator = CommonTermsQueryBuilder.DEFAULT_LOW_FREQ_OCCUR;
-        float cutoffFrequency = CommonTermsQueryBuilder.DEFAULT_CUTOFF_FREQ;
-        String queryName = null;
-        XContentParser.Token token;
-        String currentFieldName = null;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token == XContentParser.Token.START_OBJECT) {
-                throwParsingExceptionOnMultipleFields(NAME, parser.getTokenLocation(), fieldName, currentFieldName);
-                fieldName = currentFieldName;
-                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                    if (token == XContentParser.Token.FIELD_NAME) {
-                        currentFieldName = parser.currentName();
-                    } else if (token == XContentParser.Token.START_OBJECT) {
-                        if (MINIMUM_SHOULD_MATCH_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            String innerFieldName = null;
-                            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                                if (token == XContentParser.Token.FIELD_NAME) {
-                                    innerFieldName = parser.currentName();
-                                } else if (token.isValue()) {
-                                    if (LOW_FREQ_FIELD.match(innerFieldName, parser.getDeprecationHandler())) {
-                                        lowFreqMinimumShouldMatch = parser.text();
-                                    } else if (HIGH_FREQ_FIELD.match(innerFieldName, parser.getDeprecationHandler())) {
-                                        highFreqMinimumShouldMatch = parser.text();
-                                    } else {
-                                        throw new ParsingException(parser.getTokenLocation(), "[" + CommonTermsQueryBuilder.NAME +
-                                                "] query does not support [" + innerFieldName
-                                                + "] for [" + currentFieldName + "]");
-                                    }
-                                } else {
-                                    throw new ParsingException(parser.getTokenLocation(), "[" + CommonTermsQueryBuilder.NAME +
-                                            "] unexpected token type [" + token
-                                            + "] after [" + innerFieldName + "]");
-                                }
-                            }
-                        } else {
-                            throw new ParsingException(parser.getTokenLocation(), "[" + CommonTermsQueryBuilder.NAME +
-                                    "] query does not support [" + currentFieldName + "]");
-                        }
-                    } else if (token.isValue()) {
-                        if (QUERY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            text = parser.objectText();
-                        } else if (ANALYZER_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            analyzer = parser.text();
-                        } else if (DISABLE_COORD_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            // ignore
-                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            boost = parser.floatValue();
-                        } else if (HIGH_FREQ_OPERATOR_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            highFreqOperator = Operator.fromString(parser.text());
-                        } else if (LOW_FREQ_OPERATOR_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            lowFreqOperator = Operator.fromString(parser.text());
-                        } else if (MINIMUM_SHOULD_MATCH_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            lowFreqMinimumShouldMatch = parser.text();
-                        } else if (CUTOFF_FREQUENCY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            cutoffFrequency = parser.floatValue();
-                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            queryName = parser.text();
-                        } else {
-                            throw new ParsingException(parser.getTokenLocation(), "[" + CommonTermsQueryBuilder.NAME +
-                                    "] query does not support [" + currentFieldName + "]");
-                        }
-                    }
-                }
-            } else {
-                throwParsingExceptionOnMultipleFields(NAME, parser.getTokenLocation(), fieldName, parser.currentName());
-                fieldName = parser.currentName();
-                text = parser.objectText();
-            }
-        }
-
-        return new CommonTermsQueryBuilder(fieldName, text)
-                .lowFreqMinimumShouldMatch(lowFreqMinimumShouldMatch)
-                .highFreqMinimumShouldMatch(highFreqMinimumShouldMatch)
-                .analyzer(analyzer)
-                .highFreqOperator(highFreqOperator)
-                .lowFreqOperator(lowFreqOperator)
-                .cutoffFrequency(cutoffFrequency)
-                .boost(boost)
-                .queryName(queryName);
-    }
-
-    @Override
-    public String getWriteableName() {
-        return NAME;
-    }
-
-    @Override
-    protected Query doToQuery(QueryShardContext context) throws IOException {
-        String field;
-        MappedFieldType fieldType = context.fieldMapper(fieldName);
-        if (fieldType != null) {
-            field = fieldType.name();
-        } else {
-            field = fieldName;
-        }
-
-        Analyzer analyzerObj;
-        if (analyzer == null) {
-            if (fieldType != null) {
-                analyzerObj = context.getSearchAnalyzer(fieldType);
-            } else {
-                analyzerObj = context.getMapperService().searchAnalyzer();
-            }
-        } else {
-            analyzerObj = context.getMapperService().getIndexAnalyzers().get(analyzer);
-            if (analyzerObj == null) {
-                throw new QueryShardException(context, "[common] analyzer [" + analyzer + "] not found");
-            }
-        }
-
-        Occur highFreqOccur = highFreqOperator.toBooleanClauseOccur();
-        Occur lowFreqOccur = lowFreqOperator.toBooleanClauseOccur();
-
-        ExtendedCommonTermsQuery commonsQuery = new ExtendedCommonTermsQuery(highFreqOccur, lowFreqOccur, cutoffFrequency);
-        return parseQueryString(commonsQuery, text, field, analyzerObj, lowFreqMinimumShouldMatch, highFreqMinimumShouldMatch);
-    }
-
-    private static Query parseQueryString(ExtendedCommonTermsQuery query, Object queryString, String field, Analyzer analyzer,
-                                         String lowFreqMinimumShouldMatch, String highFreqMinimumShouldMatch) throws IOException {
-        // Logic similar to QueryParser#getFieldQuery
-        try (TokenStream source = analyzer.tokenStream(field, queryString.toString())) {
-            source.reset();
-            CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
-            BytesRefBuilder builder = new BytesRefBuilder();
-            while (source.incrementToken()) {
-                // UTF-8
-                builder.copyChars(termAtt);
-                query.add(new Term(field, builder.toBytesRef()));
-            }
-        }
-
-        query.setLowFreqMinimumNumberShouldMatch(lowFreqMinimumShouldMatch);
-        query.setHighFreqMinimumNumberShouldMatch(highFreqMinimumShouldMatch);
-        return query;
-    }
-
-    @Override
-    protected int doHashCode() {
-        return Objects.hash(fieldName, text, highFreqOperator, lowFreqOperator, analyzer,
-                lowFreqMinimumShouldMatch, highFreqMinimumShouldMatch, cutoffFrequency);
-    }
-
-    @Override
-    protected boolean doEquals(CommonTermsQueryBuilder other) {
-        return Objects.equals(fieldName, other.fieldName) &&
-                Objects.equals(text, other.text) &&
-                Objects.equals(highFreqOperator, other.highFreqOperator) &&
-                Objects.equals(lowFreqOperator, other.lowFreqOperator) &&
-                Objects.equals(analyzer, other.analyzer) &&
-                Objects.equals(lowFreqMinimumShouldMatch, other.lowFreqMinimumShouldMatch) &&
-                Objects.equals(highFreqMinimumShouldMatch, other.highFreqMinimumShouldMatch) &&
-                Objects.equals(cutoffFrequency, other.cutoffFrequency);
-    }
-}
diff --git a/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java
index 358a2fccff108..e09d71938add4 100644
--- a/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java
@@ -22,7 +22,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
@@ -33,14 +33,12 @@
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Objects;
@@ -62,8 +60,6 @@ public class IdsQueryBuilder extends AbstractQueryBuilder<IdsQueryBuilder> {
 
     private final Set<String> ids = new HashSet<>();
 
-    private String[] types = Strings.EMPTY_ARRAY;
-
     /**
      * Creates a new IdsQueryBuilder with no types specified upfront
      */
@@ -76,38 +72,23 @@ public IdsQueryBuilder() {
      */
     public IdsQueryBuilder(StreamInput in) throws IOException {
         super(in);
-        types = in.readStringArray();
+        if (in.getVersion().before(Version.V_8_0_0)) {
+            // types no longer relevant so ignore
+            String[] types = in.readStringArray();
+            if (types.length > 0) {
+                throw new IllegalStateException("types are no longer supported in ids query but found [" + Arrays.toString(types) + "]");
+            }
+        }
         Collections.addAll(ids, in.readStringArray());
     }
 
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeStringArray(types);
-        out.writeStringArray(ids.toArray(new String[ids.size()]));
-    }
-
-    /**
-     * Add types to query
-     *
-     * @deprecated Types are in the process of being removed, prefer to filter on a field instead.
-     */
-    @Deprecated
-    public IdsQueryBuilder types(String... types) {
-        if (types == null) {
-            throw new IllegalArgumentException("[" + NAME + "] types cannot be null");
+        if (out.getVersion().before(Version.V_8_0_0)) {
+            // types not supported so send an empty array to previous versions
+            out.writeStringArray(Strings.EMPTY_ARRAY);
         }
-        this.types = types;
-        return this;
-    }
-
-    /**
-     * Returns the types used in this query
-     *
-     * @deprecated Types are in the process of being removed, prefer to filter on a field instead.
-     */
-    @Deprecated
-    public String[] types() {
-        return this.types;
+        out.writeStringArray(ids.toArray(new String[ids.size()]));
     }
 
     /**
@@ -131,9 +112,6 @@ public Set<String> ids() {
     @Override
     protected void doXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(NAME);
-        if (types.length > 0) {
-            builder.array(TYPE_FIELD.getPreferredName(), types);
-        }
         builder.startArray(VALUES_FIELD.getPreferredName());
         for (String value : ids) {
             builder.value(value);
@@ -147,18 +125,13 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
             () -> new IdsQueryBuilder());
 
     static {
-        PARSER.declareStringArray(fromList(String.class, IdsQueryBuilder::types), IdsQueryBuilder.TYPE_FIELD);
         PARSER.declareStringArray(fromList(String.class, IdsQueryBuilder::addIds), IdsQueryBuilder.VALUES_FIELD);
         declareStandardFields(PARSER);
     }
 
     public static IdsQueryBuilder fromXContent(XContentParser parser) {
         try {
-            IdsQueryBuilder builder = PARSER.apply(parser, null);
-            if (builder.types().length > 0) {
-                deprecationLogger.deprecatedAndMaybeLog("ids_query_with_types", TYPES_DEPRECATION_MESSAGE);
-            }
-            return builder;
+            return PARSER.apply(parser, null);
         } catch (IllegalArgumentException e) {
             throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e);
         }
@@ -179,33 +152,17 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
         if (this.ids.isEmpty()) {
              return Queries.newMatchNoDocsQuery("Missing ids in \"" + this.getName() + "\" query.");
         } else {
-            final DocumentMapper mapper = context.getMapperService().documentMapper();
-            Collection<String> typesForQuery;
-            if (types.length == 0) {
-                typesForQuery = context.queryTypes();
-            } else if (types.length == 1 && MetaData.ALL.equals(types[0])) {
-                typesForQuery = Collections.singleton(mapper.type());
-            } else {
-                typesForQuery = new HashSet<>(Arrays.asList(types));
-            }
-
-            if (typesForQuery.contains(mapper.type())) {
-                return idField.termsQuery(new ArrayList<>(ids), context);
-            } else {
-                return new MatchNoDocsQuery("Type mismatch");
-            }
-            
+            return idField.termsQuery(new ArrayList<>(ids), context);
         }
     }
 
     @Override
     protected int doHashCode() {
-        return Objects.hash(ids, Arrays.hashCode(types));
+        return Objects.hash(ids);
     }
 
     @Override
     protected boolean doEquals(IdsQueryBuilder other) {
-        return Objects.equals(ids, other.ids) &&
-               Arrays.equals(types, other.types);
+        return Objects.equals(ids, other.ids);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/IntervalBuilder.java b/server/src/main/java/org/elasticsearch/index/query/IntervalBuilder.java
index 92b4fa664193c..5e1047684840f 100644
--- a/server/src/main/java/org/elasticsearch/index/query/IntervalBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/IntervalBuilder.java
@@ -96,7 +96,7 @@ protected IntervalsSource analyzeText(CachingTokenFilter stream, int maxGaps, bo
         // formulate a single term, boolean, or phrase.
 
         if (numTokens == 0) {
-            return null;
+            return NO_INTERVALS;
         } else if (numTokens == 1) {
             // single term
             return analyzeTerm(stream);
@@ -231,7 +231,7 @@ protected List<IntervalsSource> analyzeGraph(TokenStream source) throws IOExcept
         return clauses;
     }
 
-    private static final IntervalsSource NO_INTERVALS = new IntervalsSource() {
+    static final IntervalsSource NO_INTERVALS = new IntervalsSource() {
 
         @Override
         public IntervalIterator intervals(String field, LeafReaderContext ctx) {
diff --git a/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java b/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java
index e551654af9a76..d4d28057c12d0 100644
--- a/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java
+++ b/server/src/main/java/org/elasticsearch/index/query/IntervalsSourceProvider.java
@@ -19,10 +19,12 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.search.intervals.FilteredIntervalsSource;
 import org.apache.lucene.search.intervals.IntervalIterator;
 import org.apache.lucene.search.intervals.Intervals;
 import org.apache.lucene.search.intervals.IntervalsSource;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
@@ -78,9 +80,13 @@ public static IntervalsSourceProvider fromXContent(XContentParser parser) throws
                 return Disjunction.fromXContent(parser);
             case "all_of":
                 return Combine.fromXContent(parser);
+            case "prefix":
+                return Prefix.fromXContent(parser);
+            case "wildcard":
+                return Wildcard.fromXContent(parser);
         }
         throw new ParsingException(parser.getTokenLocation(),
-            "Unknown interval type [" + parser.currentName() + "], expecting one of [match, any_of, all_of]");
+            "Unknown interval type [" + parser.currentName() + "], expecting one of [match, any_of, all_of, prefix]");
     }
 
     private static IntervalsSourceProvider parseInnerIntervals(XContentParser parser) throws IOException {
@@ -138,10 +144,10 @@ public IntervalsSource getSource(QueryShardContext context, MappedFieldType fiel
             if (useField != null) {
                 fieldType = context.fieldMapper(useField);
                 assert fieldType != null;
-                source = Intervals.fixField(useField, fieldType.intervals(query, maxGaps, ordered, analyzer));
+                source = Intervals.fixField(useField, fieldType.intervals(query, maxGaps, ordered, analyzer, false));
             }
             else {
-                source = fieldType.intervals(query, maxGaps, ordered, analyzer);
+                source = fieldType.intervals(query, maxGaps, ordered, analyzer, false);
             }
             if (filter != null) {
                 return filter.filter(source, context, fieldType);
@@ -440,6 +446,226 @@ public static Combine fromXContent(XContentParser parser) {
         }
     }
 
+    public static class Prefix extends IntervalsSourceProvider {
+
+        public static final String NAME = "prefix";
+
+        private final String prefix;
+        private final String analyzer;
+        private final String useField;
+
+        public Prefix(String prefix, String analyzer, String useField) {
+            this.prefix = prefix;
+            this.analyzer = analyzer;
+            this.useField = useField;
+        }
+
+        public Prefix(StreamInput in) throws IOException {
+            this.prefix = in.readString();
+            this.analyzer = in.readOptionalString();
+            this.useField = in.readOptionalString();
+        }
+
+        @Override
+        public IntervalsSource getSource(QueryShardContext context, MappedFieldType fieldType) throws IOException {
+            NamedAnalyzer analyzer = null;
+            if (this.analyzer != null) {
+                analyzer = context.getMapperService().getIndexAnalyzers().get(this.analyzer);
+            }
+            IntervalsSource source;
+            if (useField != null) {
+                fieldType = context.fieldMapper(useField);
+                assert fieldType != null;
+                source = Intervals.fixField(useField, fieldType.intervals(prefix, 0, false, analyzer, true));
+            }
+            else {
+                source = fieldType.intervals(prefix, 0, false, analyzer, true);
+            }
+            return source;
+        }
+
+        @Override
+        public void extractFields(Set<String> fields) {
+            if (useField != null) {
+                fields.add(useField);
+            }
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Prefix prefix = (Prefix) o;
+            return Objects.equals(this.prefix, prefix.prefix) &&
+                Objects.equals(analyzer, prefix.analyzer) &&
+                Objects.equals(useField, prefix.useField);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(prefix, analyzer, useField);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return NAME;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(prefix);
+            out.writeOptionalString(analyzer);
+            out.writeOptionalString(useField);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject(NAME);
+            builder.field("prefix", prefix);
+            if (analyzer != null) {
+                builder.field("analyzer", analyzer);
+            }
+            if (useField != null) {
+                builder.field("use_field", useField);
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        private static final ConstructingObjectParser<Prefix, Void> PARSER = new ConstructingObjectParser<>(NAME, args -> {
+            String term = (String) args[0];
+            String analyzer = (String) args[1];
+            String useField = (String) args[2];
+            return new Prefix(term, analyzer, useField);
+        });
+        static {
+            PARSER.declareString(constructorArg(), new ParseField("prefix"));
+            PARSER.declareString(optionalConstructorArg(), new ParseField("analyzer"));
+            PARSER.declareString(optionalConstructorArg(), new ParseField("use_field"));
+        }
+
+        public static Prefix fromXContent(XContentParser parser) throws IOException {
+            return PARSER.parse(parser, null);
+        }
+    }
+
+    public static class Wildcard extends IntervalsSourceProvider {
+
+        public static final String NAME = "wildcard";
+
+        private final String pattern;
+        private final String analyzer;
+        private final String useField;
+
+        public Wildcard(String pattern, String analyzer, String useField) {
+            this.pattern = pattern;
+            this.analyzer = analyzer;
+            this.useField = useField;
+        }
+
+        public Wildcard(StreamInput in) throws IOException {
+            this.pattern = in.readString();
+            this.analyzer = in.readOptionalString();
+            this.useField = in.readOptionalString();
+        }
+
+        @Override
+        public IntervalsSource getSource(QueryShardContext context, MappedFieldType fieldType) {
+            NamedAnalyzer analyzer = fieldType.searchAnalyzer();
+            if (this.analyzer != null) {
+                analyzer = context.getMapperService().getIndexAnalyzers().get(this.analyzer);
+            }
+            IntervalsSource source;
+            if (useField != null) {
+                fieldType = context.fieldMapper(useField);
+                assert fieldType != null;
+                checkPositions(fieldType);
+                if (this.analyzer == null) {
+                    analyzer = fieldType.searchAnalyzer();
+                }
+                BytesRef normalizedTerm = analyzer.normalize(useField, pattern);
+                // TODO Intervals.wildcard() should take BytesRef
+                source = Intervals.fixField(useField, Intervals.wildcard(normalizedTerm.utf8ToString()));
+            }
+            else {
+                checkPositions(fieldType);
+                BytesRef normalizedTerm = analyzer.normalize(fieldType.name(), pattern);
+                source = Intervals.wildcard(normalizedTerm.utf8ToString());
+            }
+            return source;
+        }
+
+        private void checkPositions(MappedFieldType type) {
+            if (type.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
+                throw new IllegalArgumentException("Cannot create intervals over field [" + type.name() + "] with no positions indexed");
+            }
+        }
+
+        @Override
+        public void extractFields(Set<String> fields) {
+            if (useField != null) {
+                fields.add(useField);
+            }
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Prefix prefix = (Prefix) o;
+            return Objects.equals(pattern, prefix.prefix) &&
+                Objects.equals(analyzer, prefix.analyzer) &&
+                Objects.equals(useField, prefix.useField);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(pattern, analyzer, useField);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return NAME;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(pattern);
+            out.writeOptionalString(analyzer);
+            out.writeOptionalString(useField);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject(NAME);
+            builder.field("pattern", pattern);
+            if (analyzer != null) {
+                builder.field("analyzer", analyzer);
+            }
+            if (useField != null) {
+                builder.field("use_field", useField);
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        private static final ConstructingObjectParser<Wildcard, Void> PARSER = new ConstructingObjectParser<>(NAME, args -> {
+            String term = (String) args[0];
+            String analyzer = (String) args[1];
+            String useField = (String) args[2];
+            return new Wildcard(term, analyzer, useField);
+        });
+        static {
+            PARSER.declareString(constructorArg(), new ParseField("pattern"));
+            PARSER.declareString(optionalConstructorArg(), new ParseField("analyzer"));
+            PARSER.declareString(optionalConstructorArg(), new ParseField("use_field"));
+        }
+
+        public static Wildcard fromXContent(XContentParser parser) throws IOException {
+            return PARSER.parse(parser, null);
+        }
+    }
+
     static class ScriptFilterSource extends FilteredIntervalsSource {
 
         final IntervalFilterScript script;
diff --git a/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
index 0d451bd86f264..8d5b6e71927e2 100644
--- a/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -42,8 +43,8 @@
  * result of the analysis.
  */
 public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
+
     public static final ParseField ZERO_TERMS_QUERY_FIELD = new ParseField("zero_terms_query");
-    public static final ParseField CUTOFF_FREQUENCY_FIELD = new ParseField("cutoff_frequency");
     public static final ParseField LENIENT_FIELD = new ParseField("lenient");
     public static final ParseField FUZZY_TRANSPOSITIONS_FIELD = new ParseField("fuzzy_transpositions");
     public static final ParseField FUZZY_REWRITE_FIELD = new ParseField("fuzzy_rewrite");
@@ -85,8 +86,6 @@ public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
 
     private MatchQuery.ZeroTermsQuery zeroTermsQuery = MatchQuery.DEFAULT_ZERO_TERMS_QUERY;
 
-    private Float cutoffFrequency = null;
-
     private boolean autoGenerateSynonymsPhraseQuery = true;
 
     /**
@@ -121,7 +120,10 @@ public MatchQueryBuilder(StreamInput in) throws IOException {
         minimumShouldMatch = in.readOptionalString();
         fuzzyRewrite = in.readOptionalString();
         fuzziness = in.readOptionalWriteable(Fuzziness::new);
-        cutoffFrequency = in.readOptionalFloat();
+        // cutoff_frequency has been removed
+        if (in.getVersion().before(Version.V_8_0_0)) {
+            in.readOptionalFloat();
+        }
         autoGenerateSynonymsPhraseQuery = in.readBoolean();
     }
 
@@ -140,7 +142,10 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeOptionalString(minimumShouldMatch);
         out.writeOptionalString(fuzzyRewrite);
         out.writeOptionalWriteable(fuzziness);
-        out.writeOptionalFloat(cutoffFrequency);
+        // cutoff_frequency has been removed
+        if (out.getVersion().before(Version.V_8_0_0)) {
+            out.writeOptionalFloat(null);
+        }
         out.writeBoolean(autoGenerateSynonymsPhraseQuery);
     }
 
@@ -231,21 +236,6 @@ public int maxExpansions() {
         return this.maxExpansions;
     }
 
-    /**
-     * Set a cutoff value in [0..1] (or absolute number &gt;=1) representing the
-     * maximum threshold of a terms document frequency to be considered a low
-     * frequency term.
-     */
-    public MatchQueryBuilder cutoffFrequency(float cutoff) {
-        this.cutoffFrequency = cutoff;
-        return this;
-    }
-
-    /** Gets the optional cutoff value, can be {@code null} if not set previously */
-    public Float cutoffFrequency() {
-        return this.cutoffFrequency;
-    }
-
     /** Sets optional minimumShouldMatch value to apply to the query */
     public MatchQueryBuilder minimumShouldMatch(String minimumShouldMatch) {
         this.minimumShouldMatch = minimumShouldMatch;
@@ -362,9 +352,6 @@ public void doXContent(XContentBuilder builder, Params params) throws IOExceptio
         builder.field(FUZZY_TRANSPOSITIONS_FIELD.getPreferredName(), fuzzyTranspositions);
         builder.field(LENIENT_FIELD.getPreferredName(), lenient);
         builder.field(ZERO_TERMS_QUERY_FIELD.getPreferredName(), zeroTermsQuery.toString());
-        if (cutoffFrequency != null) {
-            builder.field(CUTOFF_FREQUENCY_FIELD.getPreferredName(), cutoffFrequency);
-        }
         builder.field(GENERATE_SYNONYMS_PHRASE_QUERY.getPreferredName(), autoGenerateSynonymsPhraseQuery);
         printBoostAndQueryName(builder);
         builder.endObject();
@@ -389,7 +376,6 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
         matchQuery.setTranspositions(fuzzyTranspositions);
         matchQuery.setFuzzyRewriteMethod(QueryParsers.parseRewriteMethod(fuzzyRewrite, null, LoggingDeprecationHandler.INSTANCE));
         matchQuery.setLenient(lenient);
-        matchQuery.setCommonTermsCutoff(cutoffFrequency);
         matchQuery.setZeroTermsQuery(zeroTermsQuery);
         matchQuery.setAutoGenerateSynonymsPhraseQuery(autoGenerateSynonymsPhraseQuery);
 
@@ -411,7 +397,6 @@ protected boolean doEquals(MatchQueryBuilder other) {
                Objects.equals(lenient, other.lenient) &&
                Objects.equals(fuzzyTranspositions, other.fuzzyTranspositions) &&
                Objects.equals(zeroTermsQuery, other.zeroTermsQuery) &&
-               Objects.equals(cutoffFrequency, other.cutoffFrequency) &&
                Objects.equals(autoGenerateSynonymsPhraseQuery, other.autoGenerateSynonymsPhraseQuery);
     }
 
@@ -419,7 +404,7 @@ protected boolean doEquals(MatchQueryBuilder other) {
     protected int doHashCode() {
         return Objects.hash(fieldName, value, operator, analyzer,
                 fuzziness, prefixLength, maxExpansions, minimumShouldMatch,
-                fuzzyRewrite, lenient, fuzzyTranspositions, zeroTermsQuery, cutoffFrequency, autoGenerateSynonymsPhraseQuery);
+                fuzzyRewrite, lenient, fuzzyTranspositions, zeroTermsQuery, autoGenerateSynonymsPhraseQuery);
     }
 
     @Override
@@ -440,7 +425,6 @@ public static MatchQueryBuilder fromXContent(XContentParser parser) throws IOExc
         boolean fuzzyTranspositions = FuzzyQuery.defaultTranspositions;
         String fuzzyRewrite = null;
         boolean lenient = MatchQuery.DEFAULT_LENIENCY;
-        Float cutOffFrequency = null;
         ZeroTermsQuery zeroTermsQuery = MatchQuery.DEFAULT_ZERO_TERMS_QUERY;
         boolean autoGenerateSynonymsPhraseQuery = true;
         String queryName = null;
@@ -478,8 +462,6 @@ public static MatchQueryBuilder fromXContent(XContentParser parser) throws IOExc
                             fuzzyTranspositions = parser.booleanValue();
                         } else if (LENIENT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                             lenient = parser.booleanValue();
-                        } else if (CUTOFF_FREQUENCY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                            cutOffFrequency = parser.floatValue();
                         } else if (ZERO_TERMS_QUERY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                             String zeroTermsValue = parser.text();
                             if ("none".equalsIgnoreCase(zeroTermsValue)) {
@@ -526,14 +508,10 @@ public static MatchQueryBuilder fromXContent(XContentParser parser) throws IOExc
         matchQuery.fuzzyTranspositions(fuzzyTranspositions);
         matchQuery.maxExpansions(maxExpansion);
         matchQuery.lenient(lenient);
-        if (cutOffFrequency != null) {
-            matchQuery.cutoffFrequency(cutOffFrequency);
-        }
         matchQuery.zeroTermsQuery(zeroTermsQuery);
         matchQuery.autoGenerateSynonymsPhraseQuery(autoGenerateSynonymsPhraseQuery);
         matchQuery.queryName(queryName);
         matchQuery.boost(boost);
         return matchQuery;
     }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
index 11530ce5f30b3..0d71829830b85 100644
--- a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
@@ -27,6 +27,7 @@
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.termvectors.MultiTermVectorsItemResponse;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
@@ -55,7 +56,6 @@
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType;
 
 import java.io.IOException;
@@ -69,7 +69,6 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
-import java.util.stream.Stream;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 
@@ -117,7 +116,6 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
     private static final ParseField FAIL_ON_UNSUPPORTED_FIELD = new ParseField("fail_on_unsupported_field");
 
     private static final ParseField INDEX = new ParseField("_index");
-    private static final ParseField TYPE = new ParseField("_type");
     private static final ParseField ID = new ParseField("_id");
     public static final ParseField DOC = new ParseField("doc");
     private static final ParseField PER_FIELD_ANALYZER = new ParseField("per_field_analyzer");
@@ -158,7 +156,6 @@ public static final class Item implements ToXContentObject, Writeable {
         public static final Item[] EMPTY_ARRAY = new Item[0];
 
         private String index;
-        private String type;
         private String id;
         private BytesReference doc;
         private XContentType xContentType;
@@ -176,7 +173,6 @@ public Item() {
                 throw new IllegalArgumentException("Item requires either id or doc to be non-null");
             }
             this.index = copy.index;
-            this.type = copy.type;
             this.id = copy.id;
             this.routing = copy.routing;
             this.doc = copy.doc;
@@ -216,51 +212,18 @@ public Item(@Nullable String index, XContentBuilder doc) {
             this.xContentType = doc.contentType();
         }
 
-        /**
-         * Constructor for a given item / document request
-         *
-         * @param index the index where the document is located
-         * @param type the type of the document
-         * @param id and its id
-         *
-         * @deprecated Types are in the process of being removed, use {@link Item(String, String)} instead.
-         */
-        @Deprecated
-        public Item(@Nullable String index, @Nullable String type, String id) {
-            if (id == null) {
-                throw new IllegalArgumentException("Item requires id to be non-null");
-            }
-            this.index = index;
-            this.type = type;
-            this.id = id;
-        }
-
-        /**
-         * Constructor for an artificial document request, that is not present in the index.
-         *
-         * @param index the index to be used for parsing the doc
-         * @param type the type to be used for parsing the doc
-         * @param doc the document specification
-         *
-         * @deprecated Types are in the process of being removed, use {@link Item(String, XContentBuilder)} instead.
-         */
-        @Deprecated
-        public Item(@Nullable String index, @Nullable String type, XContentBuilder doc) {
-            if (doc == null) {
-                throw new IllegalArgumentException("Item requires doc to be non-null");
-            }
-            this.index = index;
-            this.type = type;
-            this.doc = BytesReference.bytes(doc);
-            this.xContentType = doc.contentType();
-        }
-
         /**
          * Read from a stream.
          */
         Item(StreamInput in) throws IOException {
             index = in.readOptionalString();
-            type = in.readOptionalString();
+            if (in.getVersion().before(Version.V_8_0_0)) {
+                // types no longer relevant so ignore
+                String type = in.readOptionalString();
+                if (type != null) {
+                    throw new IllegalStateException("types are no longer supported but found [" + type + "]");
+                }
+            }
             if (in.readBoolean()) {
                 doc = (BytesReference) in.readGenericValue();
                 xContentType = in.readEnum(XContentType.class);
@@ -277,7 +240,10 @@ public Item(@Nullable String index, @Nullable String type, XContentBuilder doc)
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalString(index);
-            out.writeOptionalString(type);
+            if (out.getVersion().before(Version.V_8_0_0)) {
+                // types not supported so send an empty array to previous versions
+                out.writeOptionalString(null);
+            }
             out.writeBoolean(doc != null);
             if (doc != null) {
                 out.writeGenericValue(doc);
@@ -301,23 +267,6 @@ public Item index(String index) {
             return this;
         }
 
-        /**
-         * @deprecated Types are in the process of being removed.
-         */
-        @Deprecated
-        public String type() {
-            return type;
-        }
-
-        /**
-         * @deprecated Types are in the process of being removed.
-         */
-        @Deprecated
-        public Item type(String type) {
-            this.type = type;
-            return this;
-        }
-
         public String id() {
             return id;
         }
@@ -382,7 +331,7 @@ XContentType xContentType() {
          * Convert this to a {@link TermVectorsRequest} for fetching the terms of the document.
          */
         TermVectorsRequest toTermVectorsRequest() {
-            TermVectorsRequest termVectorsRequest = new TermVectorsRequest(index, type, id)
+            TermVectorsRequest termVectorsRequest = new TermVectorsRequest(index, id)
                     .selectedFields(fields)
                     .routing(routing)
                     .version(version)
@@ -413,8 +362,6 @@ public static Item parse(XContentParser parser, Item item) throws IOException {
                 } else if (currentFieldName != null) {
                     if (INDEX.match(currentFieldName, parser.getDeprecationHandler())) {
                         item.index = parser.text();
-                    } else if (TYPE.match(currentFieldName, parser.getDeprecationHandler())) {
-                        item.type = parser.text();
                     } else if (ID.match(currentFieldName, parser.getDeprecationHandler())) {
                         item.id = parser.text();
                     } else if (DOC.match(currentFieldName, parser.getDeprecationHandler())) {
@@ -462,9 +409,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             if (this.index != null) {
                 builder.field(INDEX.getPreferredName(), this.index);
             }
-            if (this.type != null) {
-                builder.field(TYPE.getPreferredName(), this.type);
-            }
             if (this.id != null) {
                 builder.field(ID.getPreferredName(), this.id);
             }
@@ -505,7 +449,7 @@ public String toString() {
 
         @Override
         public int hashCode() {
-            return Objects.hash(index, type, id, doc, Arrays.hashCode(fields), perFieldAnalyzer, routing,
+            return Objects.hash(index, id, doc, Arrays.hashCode(fields), perFieldAnalyzer, routing,
                     version, versionType);
         }
 
@@ -515,7 +459,6 @@ public boolean equals(Object o) {
             if (!(o instanceof Item)) return false;
             Item other = (Item) o;
             return Objects.equals(index, other.index) &&
-                    Objects.equals(type, other.type) &&
                     Objects.equals(id, other.id) &&
                     Objects.equals(doc, other.doc) &&
                     Arrays.equals(fields, other.fields) &&  // otherwise we are comparing pointers
@@ -964,18 +907,9 @@ public static MoreLikeThisQueryBuilder fromXContent(XContentParser parser) throw
         if (stopWords != null) {
             moreLikeThisQueryBuilder.stopWords(stopWords);
         }
-
-        if (moreLikeThisQueryBuilder.isTypeless() == false) {
-            deprecationLogger.deprecatedAndMaybeLog("more_like_this_query_with_types", TYPES_DEPRECATION_MESSAGE);
-        }
         return moreLikeThisQueryBuilder;
     }
 
-    public boolean isTypeless() {
-        return Stream.concat(Arrays.stream(likeItems), Arrays.stream(unlikeItems))
-            .allMatch(item -> item.type == null);
-    }
-
     private static void parseLikeField(XContentParser parser, List<String> texts, List<Item> items) throws IOException {
         if (parser.currentToken().isValue()) {
             texts.add(parser.text());
@@ -1132,9 +1066,6 @@ private static void setDefaultIndexTypeFields(QueryShardContext context, Item it
         if (item.index() == null) {
             item.index(context.index().getName());
         }
-        if (item.type() == null) {
-            item.type(MapperService.SINGLE_MAPPING_NAME);
-        }
         // default fields if not present but don't override for artificial docs
         if ((item.fields() == null || item.fields().length == 0) && item.doc() == null) {
             if (useDefaultField) {
diff --git a/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
index 7827c032ea0d7..07f7ae4b79398 100644
--- a/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
@@ -50,6 +51,7 @@
  * Same as {@link MatchQueryBuilder} but supports multiple fields.
  */
 public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQueryBuilder> {
+
     public static final String NAME = "multi_match";
 
     public static final MultiMatchQueryBuilder.Type DEFAULT_TYPE = MultiMatchQueryBuilder.Type.BEST_FIELDS;
@@ -63,7 +65,6 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQuery
     private static final ParseField SLOP_FIELD = new ParseField("slop");
     private static final ParseField ZERO_TERMS_QUERY_FIELD = new ParseField("zero_terms_query");
     private static final ParseField LENIENT_FIELD = new ParseField("lenient");
-    private static final ParseField CUTOFF_FREQUENCY_FIELD = new ParseField("cutoff_frequency");
     private static final ParseField TIE_BREAKER_FIELD = new ParseField("tie_breaker");
     private static final ParseField FUZZY_REWRITE_FIELD = new ParseField("fuzzy_rewrite");
     private static final ParseField MINIMUM_SHOULD_MATCH_FIELD = new ParseField("minimum_should_match");
@@ -91,7 +92,6 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQuery
     private String fuzzyRewrite = null;
     private Float tieBreaker;
     private Boolean lenient;
-    private Float cutoffFrequency = null;
     private MatchQuery.ZeroTermsQuery zeroTermsQuery = DEFAULT_ZERO_TERMS_QUERY;
     private boolean autoGenerateSynonymsPhraseQuery = true;
     private boolean fuzzyTranspositions = DEFAULT_FUZZY_TRANSPOSITIONS;
@@ -230,7 +230,9 @@ public MultiMatchQueryBuilder(StreamInput in) throws IOException {
         fuzzyRewrite = in.readOptionalString();
         tieBreaker = in.readOptionalFloat();
         lenient = in.readOptionalBoolean();
-        cutoffFrequency = in.readOptionalFloat();
+        if (in.getVersion().before(Version.V_8_0_0)) {
+            in.readOptionalFloat();
+        }
         zeroTermsQuery = MatchQuery.ZeroTermsQuery.readFromStream(in);
         autoGenerateSynonymsPhraseQuery = in.readBoolean();
         fuzzyTranspositions = in.readBoolean();
@@ -255,7 +257,9 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         out.writeOptionalString(fuzzyRewrite);
         out.writeOptionalFloat(tieBreaker);
         out.writeOptionalBoolean(lenient);
-        out.writeOptionalFloat(cutoffFrequency);
+        if (out.getVersion().before(Version.V_8_0_0)) {
+            out.writeOptionalFloat(null);
+        }
         zeroTermsQuery.writeTo(out);
         out.writeBoolean(autoGenerateSynonymsPhraseQuery);
         out.writeBoolean(fuzzyTranspositions);
@@ -480,30 +484,6 @@ public boolean lenient() {
         return lenient == null ? MatchQuery.DEFAULT_LENIENCY : lenient;
     }
 
-    /**
-     * Set a cutoff value in [0..1] (or absolute number &gt;=1) representing the
-     * maximum threshold of a terms document frequency to be considered a low
-     * frequency term.
-     */
-    public MultiMatchQueryBuilder cutoffFrequency(float cutoff) {
-        this.cutoffFrequency = cutoff;
-        return this;
-    }
-
-    /**
-     * Set a cutoff value in [0..1] (or absolute number &gt;=1) representing the
-     * maximum threshold of a terms document frequency to be considered a low
-     * frequency term.
-     */
-    public MultiMatchQueryBuilder cutoffFrequency(Float cutoff) {
-        this.cutoffFrequency = cutoff;
-        return this;
-    }
-
-    public Float cutoffFrequency() {
-        return cutoffFrequency;
-    }
-
     public MultiMatchQueryBuilder zeroTermsQuery(MatchQuery.ZeroTermsQuery zeroTermsQuery) {
         if (zeroTermsQuery == null) {
             throw new IllegalArgumentException("[" + NAME + "] requires zero terms query to be non-null");
@@ -577,9 +557,6 @@ public void doXContent(XContentBuilder builder, Params params) throws IOExceptio
         if (lenient != null) {
             builder.field(LENIENT_FIELD.getPreferredName(), lenient);
         }
-        if (cutoffFrequency != null) {
-            builder.field(CUTOFF_FREQUENCY_FIELD.getPreferredName(), cutoffFrequency);
-        }
         builder.field(ZERO_TERMS_QUERY_FIELD.getPreferredName(), zeroTermsQuery.toString());
         builder.field(GENERATE_SYNONYMS_PHRASE_QUERY.getPreferredName(), autoGenerateSynonymsPhraseQuery);
         builder.field(FUZZY_TRANSPOSITIONS_FIELD.getPreferredName(), fuzzyTranspositions);
@@ -600,7 +577,6 @@ public static MultiMatchQueryBuilder fromXContent(XContentParser parser) throws
         String minimumShouldMatch = null;
         String fuzzyRewrite = null;
         Float tieBreaker = null;
-        Float cutoffFrequency = null;
         Boolean lenient = null;
         MatchQuery.ZeroTermsQuery zeroTermsQuery = DEFAULT_ZERO_TERMS_QUERY;
         boolean autoGenerateSynonymsPhraseQuery = true;
@@ -650,8 +626,6 @@ public static MultiMatchQueryBuilder fromXContent(XContentParser parser) throws
                     fuzzyRewrite = parser.textOrNull();
                 } else if (TIE_BREAKER_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     tieBreaker = parser.floatValue();
-                }  else if (CUTOFF_FREQUENCY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                    cutoffFrequency = parser.floatValue();
                 } else if (LENIENT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     lenient = parser.booleanValue();
                 } else if (ZERO_TERMS_QUERY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
@@ -694,16 +668,10 @@ public static MultiMatchQueryBuilder fromXContent(XContentParser parser) throws
                 "[" + SLOP_FIELD.getPreferredName() + "] not allowed for type [" + type.parseField.getPreferredName() + "]");
         }
 
-        if (cutoffFrequency != null && type == Type.BOOL_PREFIX) {
-            throw new ParsingException(parser.getTokenLocation(),
-                "[" + CUTOFF_FREQUENCY_FIELD.getPreferredName() + "] not allowed for type [" + type.parseField.getPreferredName() + "]");
-        }
-
         MultiMatchQueryBuilder builder = new MultiMatchQueryBuilder(value)
                 .fields(fieldsBoosts)
                 .type(type)
                 .analyzer(analyzer)
-                .cutoffFrequency(cutoffFrequency)
                 .fuzzyRewrite(fuzzyRewrite)
                 .maxExpansions(maxExpansions)
                 .minimumShouldMatch(minimumShouldMatch)
@@ -771,9 +739,6 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
         if (tieBreaker != null) {
             multiMatchQuery.setTieBreaker(tieBreaker);
         }
-        if (cutoffFrequency != null) {
-            multiMatchQuery.setCommonTermsCutoff(cutoffFrequency);
-        }
         if (lenient != null) {
             multiMatchQuery.setLenient(lenient);
         }
@@ -804,7 +769,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
     protected int doHashCode() {
         return Objects.hash(value, fieldsBoosts, type, operator, analyzer, slop, fuzziness,
                 prefixLength, maxExpansions, minimumShouldMatch, fuzzyRewrite, tieBreaker, lenient,
-                cutoffFrequency, zeroTermsQuery, autoGenerateSynonymsPhraseQuery, fuzzyTranspositions);
+                zeroTermsQuery, autoGenerateSynonymsPhraseQuery, fuzzyTranspositions);
     }
 
     @Override
@@ -822,7 +787,6 @@ protected boolean doEquals(MultiMatchQueryBuilder other) {
                 Objects.equals(fuzzyRewrite, other.fuzzyRewrite) &&
                 Objects.equals(tieBreaker, other.tieBreaker) &&
                 Objects.equals(lenient, other.lenient) &&
-                Objects.equals(cutoffFrequency, other.cutoffFrequency) &&
                 Objects.equals(zeroTermsQuery, other.zeroTermsQuery) &&
                 Objects.equals(autoGenerateSynonymsPhraseQuery, other.autoGenerateSynonymsPhraseQuery) &&
                 Objects.equals(fuzzyTranspositions, other.fuzzyTranspositions);
diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java b/server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
index 30284703e8d6b..dfa842741dff2 100644
--- a/server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
+++ b/server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
@@ -61,16 +61,6 @@ public static MatchQueryBuilder matchQuery(String name, Object text) {
         return new MatchQueryBuilder(name, text);
     }
 
-    /**
-     * Creates a common query for the provided field name and text.
-     *
-     * @param fieldName The field name.
-     * @param text The query text (to be analyzed).
-     */
-    public static CommonTermsQueryBuilder commonTermsQuery(String fieldName, Object text) {
-        return new CommonTermsQueryBuilder(fieldName, text);
-    }
-
     /**
      * Creates a match query with type "BOOLEAN" for the provided field name and text.
      *
diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
index 0c33ee7102346..634f9d06edc11 100644
--- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
+++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
@@ -22,6 +22,8 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReaderContext;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.join.BitSetProducer;
 import org.apache.lucene.search.similarities.Similarity;
@@ -31,7 +33,6 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.lucene.search.Queries;
@@ -57,14 +58,13 @@
 import org.elasticsearch.transport.RemoteClusterAware;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
+import java.util.function.Function;
 import java.util.function.LongSupplier;
 
 /**
@@ -81,54 +81,49 @@ public class QueryShardContext extends QueryRewriteContext {
     private final MapperService mapperService;
     private final SimilarityService similarityService;
     private final BitsetFilterCache bitsetFilterCache;
+    private final Function<IndexReaderContext, IndexSearcher> searcherFactory;
     private final BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataService;
     private final int shardId;
     private final IndexReader reader;
-    private String[] types = Strings.EMPTY_ARRAY;
     private boolean cacheable = true;
     private final SetOnce<Boolean> frozen = new SetOnce<>();
     private final Index fullyQualifiedIndex;
 
-    public void setTypes(String... types) {
-        this.types = types;
-    }
-
-    public String[] getTypes() {
-        return types;
-    }
-
     private final Map<String, Query> namedQueries = new HashMap<>();
     private boolean allowUnmappedFields;
     private boolean mapUnmappedFieldAsString;
     private NestedScope nestedScope;
 
     public QueryShardContext(int shardId, IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache,
+                             Function<IndexReaderContext, IndexSearcher> searcherFactory,
                              BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup, MapperService mapperService,
                              SimilarityService similarityService, ScriptService scriptService, NamedXContentRegistry xContentRegistry,
                              NamedWriteableRegistry namedWriteableRegistry, Client client, IndexReader reader, LongSupplier nowInMillis,
                              String clusterAlias) {
-        this(shardId, indexSettings, bitsetFilterCache, indexFieldDataLookup, mapperService, similarityService, scriptService,
-            xContentRegistry, namedWriteableRegistry, client, reader, nowInMillis, new Index(RemoteClusterAware.buildRemoteIndexName(
-                clusterAlias, indexSettings.getIndex().getName()), indexSettings.getIndex().getUUID()));
+        this(shardId, indexSettings, bitsetFilterCache, searcherFactory, indexFieldDataLookup, mapperService, similarityService,
+            scriptService, xContentRegistry, namedWriteableRegistry, client, reader, nowInMillis,
+            new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()),
+                indexSettings.getIndex().getUUID()));
     }
 
     public QueryShardContext(QueryShardContext source) {
-        this(source.shardId, source.indexSettings, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService,
-                source.similarityService, source.scriptService, source.getXContentRegistry(), source.getWriteableRegistry(),
-                source.client, source.reader, source.nowInMillis, source.fullyQualifiedIndex);
-        this.types = source.getTypes();
+        this(source.shardId, source.indexSettings, source.bitsetFilterCache, source.searcherFactory, source.indexFieldDataService,
+                source.mapperService, source.similarityService, source.scriptService, source.getXContentRegistry(),
+                source.getWriteableRegistry(), source.client, source.reader, source.nowInMillis, source.fullyQualifiedIndex);
     }
 
     private QueryShardContext(int shardId, IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache,
+                             Function<IndexReaderContext, IndexSearcher> searcherFactory,
                              BiFunction<MappedFieldType, String, IndexFieldData<?>> indexFieldDataLookup, MapperService mapperService,
                              SimilarityService similarityService, ScriptService scriptService, NamedXContentRegistry xContentRegistry,
                              NamedWriteableRegistry namedWriteableRegistry, Client client, IndexReader reader, LongSupplier nowInMillis,
                              Index fullyQualifiedIndex) {
-        super(xContentRegistry, namedWriteableRegistry,client, nowInMillis);
+        super(xContentRegistry, namedWriteableRegistry, client, nowInMillis);
         this.shardId = shardId;
         this.similarityService = similarityService;
         this.mapperService = mapperService;
         this.bitsetFilterCache = bitsetFilterCache;
+        this.searcherFactory = searcherFactory;
         this.indexFieldDataService = indexFieldDataLookup;
         this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields();
         this.nestedScope = new NestedScope();
@@ -173,6 +168,10 @@ public BitSetProducer bitsetFilter(Query filter) {
         return bitsetFilterCache.getBitSetProducer(filter);
     }
 
+    public IndexSearcher newCachedSearcher(IndexReaderContext context) {
+        return searcherFactory.apply(context);
+    }
+
     public <IFD extends IndexFieldData<?>> IFD getForField(MappedFieldType fieldType) {
         return (IFD) indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName());
     }
@@ -192,7 +191,7 @@ public Map<String, Query> copyNamedQueries() {
      * Returns all the fields that match a given pattern. If prefixed with a
      * type then the fields will be returned with a type prefix.
      */
-    public Collection<String> simpleMatchToIndexNames(String pattern) {
+    public Set<String> simpleMatchToIndexNames(String pattern) {
         return mapperService.simpleMatchToFullName(pattern);
     }
 
@@ -256,24 +255,12 @@ MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMap
         }
     }
 
-    /**
-     * Returns the narrowed down explicit types, or, if not set, all types.
-     */
-    public Collection<String> queryTypes() {
-        String[] types = getTypes();
-        if (types == null || types.length == 0 || (types.length == 1 && types[0].equals("_all"))) {
-            DocumentMapper mapper = getMapperService().documentMapper();
-            return mapper == null ? Collections.emptyList() : Collections.singleton(mapper.type());
-        }
-        return Arrays.asList(types);
-    }
-
     private SearchLookup lookup = null;
 
     public SearchLookup lookup() {
         if (lookup == null) {
             lookup = new SearchLookup(getMapperService(),
-                mappedFieldType -> indexFieldDataService.apply(mappedFieldType, fullyQualifiedIndex.getName()), types);
+                    mappedFieldType -> indexFieldDataService.apply(mappedFieldType, fullyQualifiedIndex.getName()));
         }
         return lookup;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreQueryBuilder.java
index fb53f1c9560cc..b5cb15b3d00aa 100644
--- a/server/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreQueryBuilder.java
@@ -23,17 +23,17 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.search.function.ScriptScoreFunction;
+import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.common.lucene.search.function.ScriptScoreFunction;
-import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery;
 import org.elasticsearch.index.query.InnerHitContextBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.script.Script;
 
 import java.io.IOException;
 import java.util.Map;
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
index add8eb6b668ae..0f27fb4753e9b 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
@@ -42,7 +42,7 @@
 
 public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScrollRequest<Self>> extends ActionRequest {
 
-    public static final int SIZE_ALL_MATCHES = -1;
+    public static final int MAX_DOCS_ALL_MATCHES = -1;
     public static final TimeValue DEFAULT_SCROLL_TIMEOUT = timeValueMinutes(5);
     public static final int DEFAULT_SCROLL_SIZE = 1000;
 
@@ -59,7 +59,7 @@ public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScr
      * Maximum number of processed documents. Defaults to -1 meaning process all
      * documents.
      */
-    private int size = SIZE_ALL_MATCHES;
+    private int maxDocs = MAX_DOCS_ALL_MATCHES;
 
     /**
      * Should version conflicts cause aborts? Defaults to true.
@@ -151,10 +151,10 @@ public ActionRequestValidationException validate() {
         if (maxRetries < 0) {
             e = addValidationError("retries cannot be negative", e);
         }
-        if (false == (size == -1 || size > 0)) {
+        if (false == (maxDocs == -1 || maxDocs > 0)) {
             e = addValidationError(
-                    "size should be greater than 0 if the request is limited to some number of documents or -1 if it isn't but it was ["
-                            + size + "]",
+                    "maxDocs should be greater than 0 if the request is limited to some number of documents or -1 if it isn't but it was ["
+                            + maxDocs + "]",
                     e);
         }
         if (searchRequest.source().slice() != null && slices != DEFAULT_SLICES) {
@@ -167,19 +167,19 @@ public ActionRequestValidationException validate() {
      * Maximum number of processed documents. Defaults to -1 meaning process all
      * documents.
      */
-    public int getSize() {
-        return size;
+    public int getMaxDocs() {
+        return maxDocs;
     }
 
     /**
      * Maximum number of processed documents. Defaults to -1 meaning process all
      * documents.
      */
-    public Self setSize(int size) {
-        if (size < 0) {
-            throw new IllegalArgumentException("[size] parameter cannot be negative, found [" + size + "]");
+    public Self setMaxDocs(int maxDocs) {
+        if (maxDocs < 0) {
+            throw new IllegalArgumentException("[max_docs] parameter cannot be negative, found [" + maxDocs + "]");
         }
-        this.size = size;
+        this.maxDocs = maxDocs;
         return self();
     }
 
@@ -404,10 +404,10 @@ protected Self doForSlice(Self request, TaskId slicingTask, int totalSlices) {
                 .setRequestsPerSecond(requestsPerSecond / totalSlices)
                 // Sub requests don't have workers
                 .setSlices(1);
-        if (size != -1) {
-            // Size is split between workers. This means the size might round
+        if (maxDocs != MAX_DOCS_ALL_MATCHES) {
+            // maxDocs is split between workers. This means the maxDocs might round
             // down!
-            request.setSize(size == SIZE_ALL_MATCHES ? SIZE_ALL_MATCHES : size / totalSlices);
+            request.setMaxDocs(maxDocs / totalSlices);
         }
         // Set the parent task so this task is cancelled if we cancel the parent
         request.setParentTask(slicingTask);
@@ -425,7 +425,7 @@ public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         searchRequest = new SearchRequest(in);
         abortOnVersionConflict = in.readBoolean();
-        size = in.readVInt();
+        maxDocs = in.readVInt();
         refresh = in.readBoolean();
         timeout = in.readTimeValue();
         activeShardCount = ActiveShardCount.readFrom(in);
@@ -440,7 +440,7 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         searchRequest.writeTo(out);
         out.writeBoolean(abortOnVersionConflict);
-        out.writeVInt(size);
+        out.writeVInt(maxDocs);
         out.writeBoolean(refresh);
         out.writeTimeValue(timeout);
         activeShardCount.writeTo(out);
@@ -460,9 +460,6 @@ protected void searchToString(StringBuilder b) {
         } else {
             b.append("[all indices]");
         }
-        if (searchRequest.types() != null && searchRequest.types().length != 0) {
-            b.append(Arrays.toString(searchRequest.types()));
-        }
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java b/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
index a14ef850c5079..f6b025c8265a5 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.support.ActiveShardCount;
@@ -35,7 +35,7 @@ public abstract class AbstractBulkByScrollRequestBuilder<
     private final SearchRequestBuilder source;
 
     protected AbstractBulkByScrollRequestBuilder(ElasticsearchClient client,
-                                                 Action<BulkByScrollResponse> action, SearchRequestBuilder source, Request request) {
+                                                 ActionType<BulkByScrollResponse> action, SearchRequestBuilder source, Request request) {
         super(client, action, request);
         this.source = source;
     }
@@ -67,10 +67,22 @@ public Self filter(QueryBuilder filter) {
     }
 
     /**
-     * The maximum number of documents to attempt.
+     * Maximum number of processed documents. Defaults to processing all
+     * documents.
+     * @deprecated please use maxDocs(int) instead.
      */
+    @Deprecated
     public Self size(int size) {
-        request.setSize(size);
+        return maxDocs(size);
+    }
+
+
+    /**
+     * Maximum number of processed documents. Defaults to processing all
+     * documents.
+     */
+    public Self maxDocs(int maxDocs) {
+        request.setMaxDocs(maxDocs);
         return self();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java b/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
index 519c6e062d565..0e0382e146da4 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.script.Script;
@@ -29,8 +29,8 @@ public abstract class AbstractBulkIndexByScrollRequestBuilder<
                 Self extends AbstractBulkIndexByScrollRequestBuilder<Request, Self>>
         extends AbstractBulkByScrollRequestBuilder<Request, Self> {
 
-    protected AbstractBulkIndexByScrollRequestBuilder(ElasticsearchClient client,
-                                                      Action<BulkByScrollResponse> action, SearchRequestBuilder search, Request request) {
+    protected AbstractBulkIndexByScrollRequestBuilder(ElasticsearchClient client, ActionType<BulkByScrollResponse> action,
+                                                      SearchRequestBuilder search, Request request) {
         super(client, action, search, request);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java b/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
index 454983ba7942a..bbc12fb2c2aea 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
@@ -70,9 +70,8 @@ public ClientScrollableHitSource(Logger logger, BackoffPolicy backoffPolicy, Thr
     @Override
     public void doStart(Consumer<? super Response> onResponse) {
         if (logger.isDebugEnabled()) {
-            logger.debug("executing initial scroll against {}{}",
-                    isEmpty(firstSearchRequest.indices()) ? "all indices" : firstSearchRequest.indices(),
-                    isEmpty(firstSearchRequest.types()) ? "" : firstSearchRequest.types());
+            logger.debug("executing initial scroll against {}",
+                    isEmpty(firstSearchRequest.indices()) ? "all indices" : firstSearchRequest.indices());
         }
         searchWithRetry(listener -> client.search(firstSearchRequest, listener), r -> consume(r, onResponse));
     }
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java
index c7cfe28e2c0be..092f93ed2e54e 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class DeleteByQueryAction extends Action<BulkByScrollResponse> {
+public class DeleteByQueryAction extends StreamableResponseActionType<BulkByScrollResponse> {
 
     public static final DeleteByQueryAction INSTANCE = new DeleteByQueryAction();
     public static final String NAME = "indices:data/write/delete/byquery";
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
index 227ddd489779c..c1e2f011a99de 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
@@ -87,19 +87,6 @@ public DeleteByQueryRequest setQuery(QueryBuilder query) {
         return this;
     }
 
-    /**
-     * Set the document types for the delete
-     * @deprecated Types are in the process of being removed. Instead of
-     * using a type, prefer to filter on a field of the document.
-     */
-    @Deprecated
-    public DeleteByQueryRequest setDocTypes(String... types) {
-        if (types != null) {
-            getSearchRequest().types(types);
-        }
-        return this;
-    }
-
     /**
      * Set routing limiting the process to the shards that match that routing value
      */
@@ -140,21 +127,6 @@ public String getRouting() {
         return getSearchRequest().routing();
     }
 
-    /**
-     * Gets the document types on which this request would be executed. Returns an empty array if all
-     * types are to be processed.
-     * @deprecated Types are in the process of being removed. Instead of
-     * using a type, prefer to filter on a field of the document.
-     */
-    @Deprecated
-    public String[] getDocTypes() {
-        if (getSearchRequest().types() != null) {
-            return getSearchRequest().types();
-        } else {
-            return new String[0];
-        }
-    }
-
     @Override
     protected DeleteByQueryRequest self() {
         return this;
@@ -208,29 +180,6 @@ public IndicesOptions indicesOptions() {
         return getSearchRequest().indicesOptions();
     }
 
-    /**
-     * Gets the document types on which this request would be executed.
-     * @deprecated Types are in the process of being removed. Instead of
-     * using a type, prefer to filter on a field of the document.
-     */
-    @Deprecated
-    public String[] types() {
-        assert getSearchRequest() != null;
-        return getSearchRequest().types();
-    }
-
-    /**
-     * Set the document types for the delete
-     * @deprecated Types are in the process of being removed. Instead of
-     * using a type, prefer to filter on a field of the document.
-     */
-    @Deprecated
-    public DeleteByQueryRequest types(String... types) {
-        assert getSearchRequest() != null;
-        getSearchRequest().types(types);
-        return this;
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
index 202aa6e93f19d..77d304f4c17af 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -27,12 +27,12 @@
 public class DeleteByQueryRequestBuilder extends
         AbstractBulkByScrollRequestBuilder<DeleteByQueryRequest, DeleteByQueryRequestBuilder> {
 
-    public DeleteByQueryRequestBuilder(ElasticsearchClient client, Action<BulkByScrollResponse> action) {
+    public DeleteByQueryRequestBuilder(ElasticsearchClient client, ActionType<BulkByScrollResponse> action) {
         this(client, action, new SearchRequestBuilder(client, SearchAction.INSTANCE));
     }
 
     private DeleteByQueryRequestBuilder(ElasticsearchClient client,
-                                        Action<BulkByScrollResponse> action,
+                                        ActionType<BulkByScrollResponse> action,
                                         SearchRequestBuilder search) {
         super(client, action, search, new DeleteByQueryRequest(search.request()));
     }
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java b/server/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java
index 86d0c96602a3c..9ad9baa5abb90 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class ReindexAction extends Action<BulkByScrollResponse> {
+public class ReindexAction extends StreamableResponseActionType<BulkByScrollResponse> {
     public static final ReindexAction INSTANCE = new ReindexAction();
     public static final String NAME = "indices:data/write/reindex";
 
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
index de171e88fbca1..bce848998159c 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
@@ -19,24 +19,42 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.script.Script;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
-
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Objects.requireNonNull;
 import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
 import static org.elasticsearch.index.VersionType.INTERNAL;
 
 /**
@@ -138,16 +156,6 @@ public ReindexRequest setSourceIndices(String... sourceIndices) {
         return this;
     }
 
-    /**
-     * Set the document types which need to be copied from the source indices
-     */
-    public ReindexRequest setSourceDocTypes(String... docTypes) {
-        if (docTypes != null) {
-            this.getSearchRequest().types(docTypes);
-        }
-        return this;
-    }
-
     /**
      * Sets the scroll size for setting how many documents are to be processed in one batch during reindex
      */
@@ -292,13 +300,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.startObject("source");
             if (remoteInfo != null) {
                 builder.field("remote", remoteInfo);
-                builder.rawField("query", remoteInfo.getQuery().streamInput(), builder.contentType());
+                builder.rawField("query", remoteInfo.getQuery().streamInput(), RemoteInfo.QUERY_CONTENT_TYPE.type());
             }
             builder.array("index", getSearchRequest().indices());
-            String[] types = getSearchRequest().types();
-            if (types.length > 0) {
-                builder.array("type", types);
-            }
             getSearchRequest().source().innerToXContent(builder, params);
             builder.endObject();
         }
@@ -322,8 +326,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
         {
             // Other fields
-            if (getSize() != -1 || getSize() > 0) {
-                builder.field("size", getSize());
+            if (getMaxDocs() != -1) {
+                builder.field("max_docs", getMaxDocs());
             }
             if (getScript() != null) {
                 builder.field("script", getScript());
@@ -335,4 +339,163 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.endObject();
         return builder;
     }
+
+    static final ObjectParser<ReindexRequest, Void> PARSER = new ObjectParser<>("reindex");
+
+    static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in reindex requests is deprecated.";
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(ReindexRequest.class));
+
+    static {
+        ObjectParser.Parser<ReindexRequest, Void> sourceParser = (parser, request, context) -> {
+            // Funky hack to work around Search not having a proper ObjectParser and us wanting to extract query if using remote.
+            Map<String, Object> source = parser.map();
+            String[] indices = extractStringArray(source, "index");
+            if (indices != null) {
+                request.getSearchRequest().indices(indices);
+            }
+            request.setRemoteInfo(buildRemoteInfo(source));
+            XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType());
+            builder.map(source);
+            try (InputStream stream = BytesReference.bytes(builder).streamInput();
+                 XContentParser innerParser = parser.contentType().xContent()
+                     .createParser(parser.getXContentRegistry(), parser.getDeprecationHandler(), stream)) {
+                request.getSearchRequest().source().parseXContent(innerParser, false);
+            }
+        };
+
+        ObjectParser<IndexRequest, Void> destParser = new ObjectParser<>("dest");
+        destParser.declareString(IndexRequest::index, new ParseField("index"));
+        destParser.declareString((request, type) -> {
+            deprecationLogger.deprecatedAndMaybeLog("reindex_with_types", TYPES_DEPRECATION_MESSAGE);
+            request.type(type);
+        }, new ParseField("type"));
+        destParser.declareString(IndexRequest::routing, new ParseField("routing"));
+        destParser.declareString(IndexRequest::opType, new ParseField("op_type"));
+        destParser.declareString(IndexRequest::setPipeline, new ParseField("pipeline"));
+        destParser.declareString((s, i) -> s.versionType(VersionType.fromString(i)), new ParseField("version_type"));
+
+        PARSER.declareField(sourceParser::parse, new ParseField("source"), ObjectParser.ValueType.OBJECT);
+        PARSER.declareField((p, v, c) -> destParser.parse(p, v.getDestination(), c), new ParseField("dest"), ObjectParser.ValueType.OBJECT);
+        PARSER.declareInt(ReindexRequest::setMaxDocsValidateIdentical, new ParseField("max_docs"));
+        // avoid silently accepting an ignored size.
+        PARSER.declareInt((r,s) -> failOnSizeSpecified(), new ParseField("size"));
+        PARSER.declareField((p, v, c) -> v.setScript(Script.parse(p)), new ParseField("script"),
+            ObjectParser.ValueType.OBJECT);
+        PARSER.declareString(ReindexRequest::setConflicts, new ParseField("conflicts"));
+    }
+
+    public static ReindexRequest fromXContent(XContentParser parser) throws IOException {
+        ReindexRequest reindexRequest = new ReindexRequest();
+        PARSER.parse(parser, reindexRequest, null);
+        return reindexRequest;
+    }
+
+    /**
+     * Yank a string array from a map. Emulates XContent's permissive String to
+     * String array conversions.
+     */
+    private static String[] extractStringArray(Map<String, Object> source, String name) {
+        Object value = source.remove(name);
+        if (value == null) {
+            return null;
+        }
+        if (value instanceof List) {
+            @SuppressWarnings("unchecked")
+            List<String> list = (List<String>) value;
+            return list.toArray(new String[list.size()]);
+        } else if (value instanceof String) {
+            return new String[] {(String) value};
+        } else {
+            throw new IllegalArgumentException("Expected [" + name + "] to be a list of a string but was [" + value + ']');
+        }
+    }
+
+    static RemoteInfo buildRemoteInfo(Map<String, Object> source) throws IOException {
+        @SuppressWarnings("unchecked")
+        Map<String, Object> remote = (Map<String, Object>) source.remove("remote");
+        if (remote == null) {
+            return null;
+        }
+        String username = extractString(remote, "username");
+        String password = extractString(remote, "password");
+        String hostInRequest = requireNonNull(extractString(remote, "host"), "[host] must be specified to reindex from a remote cluster");
+        URI uri;
+        try {
+            uri = new URI(hostInRequest);
+            // URI has less stringent URL parsing than our code. We want to fail if all values are not provided.
+            if (uri.getPort() == -1) {
+                throw new URISyntaxException(hostInRequest, "The port was not defined in the [host]");
+            }
+        } catch (URISyntaxException ex) {
+            throw new IllegalArgumentException("[host] must be of the form [scheme]://[host]:[port](/[pathPrefix])? but was ["
+                + hostInRequest + "]", ex);
+        }
+
+        String scheme = uri.getScheme();
+        String host = uri.getHost();
+        int port = uri.getPort();
+
+        String pathPrefix = null;
+        if (uri.getPath().isEmpty() == false) {
+            pathPrefix = uri.getPath();
+        }
+
+        Map<String, String> headers = extractStringStringMap(remote, "headers");
+        TimeValue socketTimeout = extractTimeValue(remote, "socket_timeout", RemoteInfo.DEFAULT_SOCKET_TIMEOUT);
+        TimeValue connectTimeout = extractTimeValue(remote, "connect_timeout", RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
+        if (false == remote.isEmpty()) {
+            throw new IllegalArgumentException(
+                "Unsupported fields in [remote]: [" + Strings.collectionToCommaDelimitedString(remote.keySet()) + "]");
+        }
+        return new RemoteInfo(scheme, host, port, pathPrefix, RemoteInfo.queryForRemote(source),
+            username, password, headers, socketTimeout, connectTimeout);
+    }
+
+    private static String extractString(Map<String, Object> source, String name) {
+        Object value = source.remove(name);
+        if (value == null) {
+            return null;
+        }
+        if (value instanceof String) {
+            return (String) value;
+        }
+        throw new IllegalArgumentException("Expected [" + name + "] to be a string but was [" + value + "]");
+    }
+
+    private static Map<String, String> extractStringStringMap(Map<String, Object> source, String name) {
+        Object value = source.remove(name);
+        if (value == null) {
+            return emptyMap();
+        }
+        if (false == value instanceof Map) {
+            throw new IllegalArgumentException("Expected [" + name + "] to be an object containing strings but was [" + value + "]");
+        }
+        Map<?, ?> map = (Map<?, ?>) value;
+        for (Map.Entry<?, ?> entry : map.entrySet()) {
+            if (false == entry.getKey() instanceof String || false == entry.getValue() instanceof String) {
+                throw new IllegalArgumentException("Expected [" + name + "] to be an object containing strings but has [" + entry + "]");
+            }
+        }
+        @SuppressWarnings("unchecked") // We just checked....
+            Map<String, String> safe = (Map<String, String>) map;
+        return safe;
+    }
+
+    private static TimeValue extractTimeValue(Map<String, Object> source, String name, TimeValue defaultValue) {
+        String string = extractString(source, name);
+        return string == null ? defaultValue : parseTimeValue(string, name);
+    }
+
+    static void setMaxDocsValidateIdentical(AbstractBulkByScrollRequest<?> request, int maxDocs) {
+        if (request.getMaxDocs() != AbstractBulkByScrollRequest.MAX_DOCS_ALL_MATCHES && request.getMaxDocs() != maxDocs) {
+            throw new IllegalArgumentException("[max_docs] set to two different values [" + request.getMaxDocs() + "]" +
+                " and [" + maxDocs + "]");
+        } else {
+            request.setMaxDocs(maxDocs);
+        }
+    }
+
+    private static void failOnSizeSpecified() {
+        throw new IllegalArgumentException("invalid parameter [size], use [max_docs] instead");
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
index c775ae197db93..a934584bc9724 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.index.IndexAction;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchAction;
@@ -31,13 +31,13 @@ public class ReindexRequestBuilder extends
     private final IndexRequestBuilder destination;
 
     public ReindexRequestBuilder(ElasticsearchClient client,
-            Action<BulkByScrollResponse> action) {
+            ActionType<BulkByScrollResponse> action) {
         this(client, action, new SearchRequestBuilder(client, SearchAction.INSTANCE),
                 new IndexRequestBuilder(client, IndexAction.INSTANCE));
     }
 
     private ReindexRequestBuilder(ElasticsearchClient client,
-            Action<BulkByScrollResponse> action,
+            ActionType<BulkByScrollResponse> action,
             SearchRequestBuilder search, IndexRequestBuilder destination) {
         super(client, action, search, new ReindexRequest(search.request(), destination.request()));
         this.destination = destination;
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java b/server/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java
index 2bca878df8873..10d6138a5338c 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/RemoteInfo.java
@@ -25,16 +25,24 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Objects;
 
 import static java.util.Collections.unmodifiableMap;
 import static java.util.Objects.requireNonNull;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 
 public class RemoteInfo implements Writeable, ToXContentObject {
     /**
@@ -46,6 +54,8 @@ public class RemoteInfo implements Writeable, ToXContentObject {
      */
     public static final TimeValue DEFAULT_CONNECT_TIMEOUT = timeValueSeconds(30);
 
+    public static final XContent QUERY_CONTENT_TYPE = JsonXContent.jsonXContent;
+
     private final String scheme;
     private final String host;
     private final int port;
@@ -65,6 +75,7 @@ public class RemoteInfo implements Writeable, ToXContentObject {
 
     public RemoteInfo(String scheme, String host, int port, String pathPrefix, BytesReference query, String username, String password,
                       Map<String, String> headers, TimeValue socketTimeout, TimeValue connectTimeout) {
+        assert isQueryJson(query) : "Query does not appear to be JSON";
         this.scheme = requireNonNull(scheme, "[scheme] must be specified to reindex from a remote cluster");
         this.host = requireNonNull(host, "[host] must be specified to reindex from a remote cluster");
         this.port = port;
@@ -205,4 +216,50 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.endObject();
         return builder;
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        RemoteInfo that = (RemoteInfo) o;
+        return port == that.port &&
+            Objects.equals(scheme, that.scheme) &&
+            Objects.equals(host, that.host) &&
+            Objects.equals(pathPrefix, that.pathPrefix) &&
+            Objects.equals(query, that.query) &&
+            Objects.equals(username, that.username) &&
+            Objects.equals(password, that.password) &&
+            Objects.equals(headers, that.headers) &&
+            Objects.equals(socketTimeout, that.socketTimeout) &&
+            Objects.equals(connectTimeout, that.connectTimeout);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(scheme, host, port, pathPrefix, query, username, password, headers, socketTimeout, connectTimeout);
+    }
+
+    static BytesReference queryForRemote(Map<String, Object> source) throws IOException {
+        XContentBuilder builder = XContentBuilder.builder(QUERY_CONTENT_TYPE).prettyPrint();
+        Object query = source.remove("query");
+        if (query == null) {
+            return BytesReference.bytes(matchAllQuery().toXContent(builder, ToXContent.EMPTY_PARAMS));
+        }
+        if (!(query instanceof Map)) {
+            throw new IllegalArgumentException("Expected [query] to be an object but was [" + query + "]");
+        }
+        @SuppressWarnings("unchecked")
+        Map<String, Object> map = (Map<String, Object>) query;
+        return BytesReference.bytes(builder.map(map));
+    }
+
+    private static boolean isQueryJson(BytesReference bytesReference) {
+        try (XContentParser parser = QUERY_CONTENT_TYPE.createParser(NamedXContentRegistry.EMPTY,
+            DeprecationHandler.THROW_UNSUPPORTED_OPERATION, bytesReference.streamInput())) {
+            Map<String, Object> query = parser.map();
+            return true;
+        } catch (IOException e) {
+            throw new AssertionError("Could not parse JSON", e);
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryAction.java b/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryAction.java
index 250a267ea255d..3c0523d34ee56 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryAction.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class UpdateByQueryAction extends Action<BulkByScrollResponse> {
+public class UpdateByQueryAction extends StreamableResponseActionType<BulkByScrollResponse> {
     public static final UpdateByQueryAction INSTANCE = new UpdateByQueryAction();
     public static final String NAME = "indices:data/write/update/byquery";
 
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java b/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
index 03922ccc79bd4..138af524bf9c9 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
@@ -83,19 +83,6 @@ public UpdateByQueryRequest setQuery(QueryBuilder query) {
         return this;
     }
 
-    /**
-     * Set the document types for the update
-     * @deprecated Types are in the process of being removed. Instead of
-     * using a type, prefer to filter on a field of the document.
-     */
-    @Deprecated
-    public UpdateByQueryRequest setDocTypes(String... types) {
-        if (types != null) {
-            getSearchRequest().types(types);
-        }
-        return this;
-    }
-
     /**
      * Set routing limiting the process to the shards that match that routing value
      */
@@ -136,21 +123,6 @@ public String getRouting() {
         return getSearchRequest().routing();
     }
 
-    /**
-     * Gets the document types on which this request would be executed. Returns an empty array if all
-     * types are to be processed.
-     * @deprecated Types are in the process of being removed. Instead of
-     * using a type, prefer to filter on a field of the document.
-     */
-    @Deprecated
-    public String[] getDocTypes() {
-        if (getSearchRequest().types() != null) {
-            return getSearchRequest().types();
-        } else {
-            return new String[0];
-        }
-    }
-
     /**
      * Ingest pipeline to set on index requests made by this action.
      */
diff --git a/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java b/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
index 6b9600dea5eae..91a77e988a881 100644
--- a/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -27,12 +27,12 @@
 public class UpdateByQueryRequestBuilder extends
         AbstractBulkIndexByScrollRequestBuilder<UpdateByQueryRequest, UpdateByQueryRequestBuilder> {
 
-    public UpdateByQueryRequestBuilder(ElasticsearchClient client, Action<BulkByScrollResponse> action) {
+    public UpdateByQueryRequestBuilder(ElasticsearchClient client, ActionType<BulkByScrollResponse> action) {
         this(client, action, new SearchRequestBuilder(client, SearchAction.INSTANCE));
     }
 
     private UpdateByQueryRequestBuilder(ElasticsearchClient client,
-            Action<BulkByScrollResponse> action,
+            ActionType<BulkByScrollResponse> action,
             SearchRequestBuilder search) {
         super(client, action, search, new UpdateByQueryRequest(search.request()));
     }
diff --git a/server/src/main/java/org/elasticsearch/index/search/MatchQuery.java b/server/src/main/java/org/elasticsearch/index/search/MatchQuery.java
index da7273aa66303..8a43dfbdb3f96 100644
--- a/server/src/main/java/org/elasticsearch/index/search/MatchQuery.java
+++ b/server/src/main/java/org/elasticsearch/index/search/MatchQuery.java
@@ -29,14 +29,11 @@
 import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.ExtendedCommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
 import org.apache.lucene.search.spans.SpanNearQuery;
 import org.apache.lucene.search.spans.SpanOrQuery;
@@ -171,8 +168,6 @@ public void writeTo(StreamOutput out) throws IOException {
 
     protected ZeroTermsQuery zeroTermsQuery = DEFAULT_ZERO_TERMS_QUERY;
 
-    protected Float commonTermsCutoff = null;
-
     protected boolean autoGenerateSynonymsPhraseQuery = true;
 
     public MatchQuery(QueryShardContext context) {
@@ -194,10 +189,6 @@ public void setOccur(BooleanClause.Occur occur) {
         this.occur = occur;
     }
 
-    public void setCommonTermsCutoff(Float cutoff) {
-        this.commonTermsCutoff = cutoff;
-    }
-
     public void setEnablePositionIncrements(boolean enablePositionIncrements) {
         this.enablePositionIncrements = enablePositionIncrements;
     }
@@ -247,7 +238,7 @@ public Query parse(Type type, String fieldName, Object value) throws IOException
         Analyzer analyzer = getAnalyzer(fieldType, type == Type.PHRASE || type == Type.PHRASE_PREFIX);
         assert analyzer != null;
 
-        MatchQueryBuilder builder = new MatchQueryBuilder(analyzer, fieldType);
+        MatchQueryBuilder builder = new MatchQueryBuilder(analyzer, fieldType, enablePositionIncrements, autoGenerateSynonymsPhraseQuery);
 
         /*
          * If a keyword analyzer is used, we know that further analysis isn't
@@ -272,11 +263,7 @@ protected final Query parseInternal(Type type, String fieldName, MatchQueryBuild
         final Query query;
         switch (type) {
             case BOOLEAN:
-                if (commonTermsCutoff == null) {
-                    query = builder.createBooleanQuery(fieldName, value.toString(), occur);
-                } else {
-                    query = createCommonTermsQuery(builder, fieldName, value.toString(), occur, occur, commonTermsCutoff);
-                }
+                query = builder.createBooleanQuery(fieldName, value.toString(), occur);
                 break;
 
             case BOOLEAN_PREFIX:
@@ -298,30 +285,6 @@ protected final Query parseInternal(Type type, String fieldName, MatchQueryBuild
         return query == null ? zeroTermsQuery() : query;
     }
 
-    private Query createCommonTermsQuery(MatchQueryBuilder builder, String field, String queryText,
-                                         Occur highFreqOccur, Occur lowFreqOccur, float maxTermFrequency) {
-        Query booleanQuery = builder.createBooleanQuery(field, queryText, lowFreqOccur);
-        if (booleanQuery != null && booleanQuery instanceof BooleanQuery) {
-            BooleanQuery bq = (BooleanQuery) booleanQuery;
-            return boolToExtendedCommonTermsQuery(bq, highFreqOccur, lowFreqOccur, maxTermFrequency);
-        }
-        return booleanQuery;
-    }
-
-    private Query boolToExtendedCommonTermsQuery(BooleanQuery bq,
-                                                 Occur highFreqOccur,
-                                                 Occur lowFreqOccur,
-                                                 float maxTermFrequency) {
-        ExtendedCommonTermsQuery query = new ExtendedCommonTermsQuery(highFreqOccur, lowFreqOccur, maxTermFrequency);
-        for (BooleanClause clause : bq.clauses()) {
-            if ((clause.getQuery() instanceof TermQuery) == false) {
-                return bq;
-            }
-            query.add(((TermQuery) clause.getQuery()).getTerm());
-        }
-        return query;
-    }
-
     protected Analyzer getAnalyzer(MappedFieldType fieldType, boolean quoted) {
         if (analyzer == null) {
             return quoted ? context.getSearchQuoteAnalyzer(fieldType) : context.getSearchAnalyzer(fieldType);
@@ -353,15 +316,16 @@ class MatchQueryBuilder extends QueryBuilder {
         /**
          * Creates a new QueryBuilder using the given analyzer.
          */
-        MatchQueryBuilder(Analyzer analyzer, MappedFieldType fieldType) {
+        MatchQueryBuilder(Analyzer analyzer, MappedFieldType fieldType,
+                            boolean enablePositionIncrements, boolean autoGenerateSynonymsPhraseQuery) {
             super(analyzer);
             this.fieldType = fieldType;
+            setEnablePositionIncrements(enablePositionIncrements);
             if (hasPositions(fieldType)) {
                 setAutoGenerateMultiTermSynonymsPhraseQuery(autoGenerateSynonymsPhraseQuery);
             } else {
                 setAutoGenerateMultiTermSynonymsPhraseQuery(false);
             }
-            setEnablePositionIncrements(enablePositionIncrements);
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java b/server/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java
index b9943870df727..cbc06a6ff081d 100644
--- a/server/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java
+++ b/server/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java
@@ -138,9 +138,11 @@ private List<Query> buildCrossFieldQuery(MultiMatchQueryBuilder.Type type, Map<S
         for (Map.Entry<Analyzer, List<FieldAndBoost>> group : groups.entrySet()) {
             final MatchQueryBuilder builder;
             if (group.getValue().size() == 1) {
-                builder = new MatchQueryBuilder(group.getKey(), group.getValue().get(0).fieldType);
+                builder = new MatchQueryBuilder(group.getKey(), group.getValue().get(0).fieldType,
+                    enablePositionIncrements, autoGenerateSynonymsPhraseQuery);
             } else {
-                builder = new BlendedQueryBuilder(group.getKey(), group.getValue(), tieBreaker);
+                builder = new BlendedQueryBuilder(group.getKey(), group.getValue(), tieBreaker,
+                    enablePositionIncrements, autoGenerateSynonymsPhraseQuery);
             }
 
             /*
@@ -170,8 +172,9 @@ private class BlendedQueryBuilder extends MatchQueryBuilder {
         private final List<FieldAndBoost> blendedFields;
         private final float tieBreaker;
 
-        BlendedQueryBuilder(Analyzer analyzer, List<FieldAndBoost> blendedFields, float tieBreaker) {
-            super(analyzer, blendedFields.get(0).fieldType);
+        BlendedQueryBuilder(Analyzer analyzer, List<FieldAndBoost> blendedFields, float tieBreaker,
+                                boolean enablePositionIncrements, boolean autoGenerateSynonymsPhraseQuery) {
+            super(analyzer, blendedFields.get(0).fieldType, enablePositionIncrements, autoGenerateSynonymsPhraseQuery);
             this.blendedFields = blendedFields;
             this.tieBreaker = tieBreaker;
         }
@@ -182,12 +185,12 @@ protected Query newSynonymQuery(Term[] terms) {
             for (int i = 0; i < terms.length; i++) {
                 values[i] = terms[i].bytes();
             }
-            return blendTerms(context, values, commonTermsCutoff, tieBreaker, lenient, blendedFields);
+            return blendTerms(context, values, tieBreaker, lenient, blendedFields);
         }
 
         @Override
         protected Query newTermQuery(Term term) {
-            return blendTerm(context, term.bytes(), commonTermsCutoff, tieBreaker, lenient, blendedFields);
+            return blendTerm(context, term.bytes(), tieBreaker, lenient, blendedFields);
         }
 
         @Override
@@ -230,13 +233,13 @@ protected Query analyzeMultiPhrase(String field, TokenStream stream, int slop) t
         }
     }
 
-    static Query blendTerm(QueryShardContext context, BytesRef value, Float commonTermsCutoff, float tieBreaker,
+    static Query blendTerm(QueryShardContext context, BytesRef value, float tieBreaker,
                            boolean lenient, List<FieldAndBoost> blendedFields) {
 
-        return blendTerms(context, new BytesRef[] {value}, commonTermsCutoff, tieBreaker, lenient, blendedFields);
+        return blendTerms(context, new BytesRef[] {value}, tieBreaker, lenient, blendedFields);
     }
 
-    static Query blendTerms(QueryShardContext context, BytesRef[] values, Float commonTermsCutoff, float tieBreaker,
+    static Query blendTerms(QueryShardContext context, BytesRef[] values, float tieBreaker,
                             boolean lenient, List<FieldAndBoost> blendedFields) {
 
         List<Query> queries = new ArrayList<>();
@@ -276,11 +279,7 @@ static Query blendTerms(QueryShardContext context, BytesRef[] values, Float comm
         if (i > 0) {
             terms = Arrays.copyOf(terms, i);
             blendedBoost = Arrays.copyOf(blendedBoost, i);
-            if (commonTermsCutoff != null) {
-                queries.add(BlendedTermQuery.commonTermsBlendedQuery(terms, blendedBoost, commonTermsCutoff));
-            } else {
-                queries.add(BlendedTermQuery.dismaxBlendedQuery(terms, blendedBoost, tieBreaker));
-            }
+            queries.add(BlendedTermQuery.dismaxBlendedQuery(terms, blendedBoost, tieBreaker));
         }
         if (queries.size() == 1) {
             return queries.get(0);
diff --git a/server/src/main/java/org/elasticsearch/index/search/NestedHelper.java b/server/src/main/java/org/elasticsearch/index/search/NestedHelper.java
index 1c17fa0cb935f..ab2ae044bdbaf 100644
--- a/server/src/main/java/org/elasticsearch/index/search/NestedHelper.java
+++ b/server/src/main/java/org/elasticsearch/index/search/NestedHelper.java
@@ -192,7 +192,7 @@ boolean mightMatchNonNestedDocs(String field, String nestedPath) {
         return true; // the field is not a sub field of the nested path
     }
 
-    private static String parentObject(String field) {
+    public static String parentObject(String field) {
         int lastDot = field.lastIndexOf('.');
         if (lastDot == -1) {
             return null;
diff --git a/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java b/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java
index 3acf2929687c5..8d6198e17e2cc 100644
--- a/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java
+++ b/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java
@@ -30,6 +30,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * Helpers to extract and expand field names and boosts
@@ -55,6 +56,10 @@ public static Map<String, Float> parseFieldsAndWeights(List<String> fields) {
             } else {
                 fieldName = field;
             }
+            // handle duplicates
+            if (fieldsAndWeights.containsKey(field)) {
+                boost *= fieldsAndWeights.get(field);
+            }
             fieldsAndWeights.put(fieldName, boost);
         }
         return fieldsAndWeights;
@@ -84,7 +89,13 @@ public static Map<String, Float> resolveMappingFields(QueryShardContext context,
             float weight = fieldEntry.getValue() == null ? 1.0f : fieldEntry.getValue();
             Map<String, Float> fieldMap = resolveMappingField(context, fieldEntry.getKey(), weight,
                 !multiField, !allField, fieldSuffix);
-            resolvedFields.putAll(fieldMap);
+            for (Map.Entry<String, Float> field : fieldMap.entrySet()) {
+                float boost = field.getValue();
+                if (resolvedFields.containsKey(field.getKey())) {
+                    boost *= resolvedFields.get(field.getKey());
+                }
+                resolvedFields.put(field.getKey(), boost);
+            }
         }
         checkForTooManyFields(resolvedFields, context);
         return resolvedFields;
@@ -120,8 +131,9 @@ public static Map<String, Float> resolveMappingField(QueryShardContext context,
      */
     public static Map<String, Float> resolveMappingField(QueryShardContext context, String fieldOrPattern, float weight,
                                                          boolean acceptAllTypes, boolean acceptMetadataField, String fieldSuffix) {
-        Collection<String> allFields = context.simpleMatchToIndexNames(fieldOrPattern);
+        Set<String> allFields = context.simpleMatchToIndexNames(fieldOrPattern);
         Map<String, Float> fields = new HashMap<>();
+
         for (String fieldName : allFields) {
             if (fieldSuffix != null && context.fieldMapper(fieldName + fieldSuffix) != null) {
                 fieldName = fieldName + fieldSuffix;
@@ -149,8 +161,17 @@ public static Map<String, Float> resolveMappingField(QueryShardContext context,
                     // other exceptions are parsing errors or not indexed fields: keep
                 }
             }
-            fields.put(fieldName, weight);
+
+            // Deduplicate aliases and their concrete fields.
+            String resolvedFieldName = fieldType.name();
+            if (allFields.contains(resolvedFieldName)) {
+                fieldName = resolvedFieldName;
+            }
+
+            float w = fields.getOrDefault(fieldName, 1.0F);
+            fields.put(fieldName, w * weight);
         }
+
         checkForTooManyFields(fields, context);
         return fields;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java
index 7bec4fc87f1a9..22be2131e3347 100644
--- a/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java
+++ b/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java
@@ -153,6 +153,12 @@ private QueryStringQueryParser(QueryShardContext context, String defaultField,
         this.lenient = lenient;
     }
 
+    @Override
+    public void setEnablePositionIncrements(boolean enable) {
+        super.setEnablePositionIncrements(enable);
+        queryBuilder.setEnablePositionIncrements(enable);
+    }
+
     @Override
     public void setDefaultOperator(Operator op) {
         super.setDefaultOperator(op);
@@ -423,7 +429,8 @@ protected Query handleBareFuzzy(String field, Token fuzzySlop, String termImage)
         if (fuzzySlop.image.length() == 1) {
             return getFuzzyQuery(field, termImage, fuzziness.asDistance(termImage));
         }
-        return getFuzzyQuery(field, termImage, Fuzziness.fromString(fuzzySlop.image.substring(1)).asFloat());
+        float distance = Fuzziness.fromString(fuzzySlop.image.substring(1)).asDistance(termImage);
+        return getFuzzyQuery(field, termImage, distance);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
index d67cbc833d666..a7f29d14b49b8 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
@@ -19,10 +19,9 @@
 
 package org.elasticsearch.index.seqno;
 
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.lucene.store.AlreadyClosedException;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.replication.ReplicationRequest;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
@@ -33,9 +32,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.IndexShardClosedException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesService;
@@ -55,6 +52,12 @@ public class GlobalCheckpointSyncAction extends TransportReplicationAction<
         ReplicationResponse> {
 
     public static String ACTION_NAME = "indices:admin/seq_no/global_checkpoint_sync";
+    public static ActionType<ReplicationResponse> TYPE = new StreamableResponseActionType<>(ACTION_NAME) {
+        @Override
+        public ReplicationResponse newResponse() {
+            return new ReplicationResponse();
+        }
+    };
 
     @Inject
     public GlobalCheckpointSyncAction(
@@ -81,21 +84,6 @@ public GlobalCheckpointSyncAction(
                 ThreadPool.Names.MANAGEMENT);
     }
 
-    public void updateGlobalCheckpointForShard(final ShardId shardId) {
-        final ThreadContext threadContext = threadPool.getThreadContext();
-        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
-            threadContext.markAsSystemContext();
-            execute(
-                    new Request(shardId),
-                    ActionListener.wrap(r -> {
-                    }, e -> {
-                        if (ExceptionsHelper.unwrap(e, AlreadyClosedException.class, IndexShardClosedException.class) == null) {
-                            logger.info(new ParameterizedMessage("{} global checkpoint sync failed", shardId), e);
-                        }
-                    }));
-        }
-    }
-
     @Override
     protected ReplicationResponse newResponseInstance() {
         return new ReplicationResponse();
@@ -118,7 +106,7 @@ protected ReplicaResult shardOperationOnReplica(final Request request, final Ind
 
     private void maybeSyncTranslog(final IndexShard indexShard) throws IOException {
         if (indexShard.getTranslogDurability() == Translog.Durability.REQUEST &&
-            indexShard.getLastSyncedGlobalCheckpoint() < indexShard.getGlobalCheckpoint()) {
+            indexShard.getLastSyncedGlobalCheckpoint() < indexShard.getLastKnownGlobalCheckpoint()) {
             indexShard.sync();
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java b/server/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java
index a19d9ac4abb94..185d3b2ad258c 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java
@@ -22,6 +22,8 @@
 import com.carrotsearch.hppc.LongObjectHashMap;
 import org.elasticsearch.common.SuppressForbidden;
 
+import java.util.concurrent.atomic.AtomicLong;
+
 /**
  * This class generates sequences numbers and keeps track of the so-called "local checkpoint" which is the highest number for which all
  * previous sequence numbers have been processed (inclusive).
@@ -35,20 +37,31 @@ public class LocalCheckpointTracker {
     static final short BIT_SET_SIZE = 1024;
 
     /**
-     * A collection of bit sets representing pending sequence numbers. Each sequence number is mapped to a bit set by dividing by the
+     * A collection of bit sets representing processed sequence numbers. Each sequence number is mapped to a bit set by dividing by the
      * bit set size.
      */
     final LongObjectHashMap<CountedBitSet> processedSeqNo = new LongObjectHashMap<>();
 
     /**
-     * The current local checkpoint, i.e., all sequence numbers no more than this number have been completed.
+     * A collection of bit sets representing durably persisted sequence numbers. Each sequence number is mapped to a bit set by dividing by
+     * the bit set size.
+     */
+    final LongObjectHashMap<CountedBitSet> persistedSeqNo = new LongObjectHashMap<>();
+
+    /**
+     * The current local checkpoint, i.e., all sequence numbers no more than this number have been processed.
+     */
+    final AtomicLong processedCheckpoint = new AtomicLong();
+
+    /**
+     * The current persisted local checkpoint, i.e., all sequence numbers no more than this number have been durably persisted.
      */
-    volatile long checkpoint;
+    final AtomicLong persistedCheckpoint = new AtomicLong();
 
     /**
      * The next available sequence number.
      */
-    private volatile long nextSeqNo;
+    final AtomicLong nextSeqNo = new AtomicLong();
 
     /**
      * Initialize the local checkpoint service. The {@code maxSeqNo} should be set to the last sequence number assigned, or
@@ -68,8 +81,9 @@ public LocalCheckpointTracker(final long maxSeqNo, final long localCheckpoint) {
             throw new IllegalArgumentException(
                 "max seq. no. must be non-negative or [" + SequenceNumbers.NO_OPS_PERFORMED + "] but was [" + maxSeqNo + "]");
         }
-        nextSeqNo = maxSeqNo == SequenceNumbers.NO_OPS_PERFORMED ? 0 : maxSeqNo + 1;
-        checkpoint = localCheckpoint;
+        nextSeqNo.set(maxSeqNo + 1);
+        processedCheckpoint.set(localCheckpoint);
+        persistedCheckpoint.set(localCheckpoint);
     }
 
     /**
@@ -77,48 +91,67 @@ public LocalCheckpointTracker(final long maxSeqNo, final long localCheckpoint) {
      *
      * @return the next assigned sequence number
      */
-    public synchronized long generateSeqNo() {
-        return nextSeqNo++;
+    public long generateSeqNo() {
+        return nextSeqNo.getAndIncrement();
     }
 
     /**
      * Marks the provided sequence number as seen and updates the max_seq_no if needed.
      */
-    public synchronized void advanceMaxSeqNo(long seqNo) {
-        if (seqNo >= nextSeqNo) {
-            nextSeqNo = seqNo + 1;
-        }
+    public void advanceMaxSeqNo(final long seqNo) {
+        nextSeqNo.accumulateAndGet(seqNo + 1, Math::max);
     }
 
     /**
-     * Marks the processing of the provided sequence number as completed as updates the checkpoint if possible.
+     * Marks the provided sequence number as processed and updates the processed checkpoint if possible.
      *
-     * @param seqNo the sequence number to mark as completed
+     * @param seqNo the sequence number to mark as processed
      */
-    public synchronized void markSeqNoAsCompleted(final long seqNo) {
+    public synchronized void markSeqNoAsProcessed(final long seqNo) {
+        markSeqNo(seqNo, processedCheckpoint, processedSeqNo);
+    }
+
+    /**
+     * Marks the provided sequence number as persisted and updates the checkpoint if possible.
+     *
+     * @param seqNo the sequence number to mark as persisted
+     */
+    public synchronized void markSeqNoAsPersisted(final long seqNo) {
+        markSeqNo(seqNo, persistedCheckpoint, persistedSeqNo);
+    }
+
+    private void markSeqNo(final long seqNo, final AtomicLong checkPoint, final LongObjectHashMap<CountedBitSet> bitSetMap) {
+        assert Thread.holdsLock(this);
         // make sure we track highest seen sequence number
-        if (seqNo >= nextSeqNo) {
-            nextSeqNo = seqNo + 1;
-        }
-        if (seqNo <= checkpoint) {
+        advanceMaxSeqNo(seqNo);
+        if (seqNo <= checkPoint.get()) {
             // this is possible during recovery where we might replay an operation that was also replicated
             return;
         }
-        final CountedBitSet bitSet = getBitSetForSeqNo(seqNo);
+        final CountedBitSet bitSet = getBitSetForSeqNo(bitSetMap, seqNo);
         final int offset = seqNoToBitSetOffset(seqNo);
         bitSet.set(offset);
-        if (seqNo == checkpoint + 1) {
-            updateCheckpoint();
+        if (seqNo == checkPoint.get() + 1) {
+            updateCheckpoint(checkPoint, bitSetMap);
         }
     }
 
     /**
-     * The current checkpoint which can be advanced by {@link #markSeqNoAsCompleted(long)}.
+     * The current checkpoint which can be advanced by {@link #markSeqNoAsProcessed(long)}.
      *
      * @return the current checkpoint
      */
-    public long getCheckpoint() {
-        return checkpoint;
+    public long getProcessedCheckpoint() {
+        return processedCheckpoint.get();
+    }
+
+    /**
+     * The current persisted checkpoint which can be advanced by {@link #markSeqNoAsPersisted(long)}.
+     *
+     * @return the current persisted checkpoint
+     */
+    public long getPersistedCheckpoint() {
+        return persistedCheckpoint.get();
     }
 
     /**
@@ -127,17 +160,17 @@ public long getCheckpoint() {
      * @return the maximum sequence number
      */
     public long getMaxSeqNo() {
-        return nextSeqNo - 1;
+        return nextSeqNo.get() - 1;
     }
 
 
     /**
      * constructs a {@link SeqNoStats} object, using local state and the supplied global checkpoint
      *
-     * This is needed to make sure the local checkpoint and max seq no are consistent
+     * This is needed to make sure the persisted local checkpoint and max seq no are consistent
      */
     public synchronized SeqNoStats getStats(final long globalCheckpoint) {
-        return new SeqNoStats(getMaxSeqNo(), getCheckpoint(), globalCheckpoint);
+        return new SeqNoStats(getMaxSeqNo(), getPersistedCheckpoint(), globalCheckpoint);
     }
 
     /**
@@ -147,70 +180,74 @@ public synchronized SeqNoStats getStats(final long globalCheckpoint) {
      * @throws InterruptedException if the thread was interrupted while blocking on the condition
      */
     @SuppressForbidden(reason = "Object#wait")
-    public synchronized void waitForOpsToComplete(final long seqNo) throws InterruptedException {
-        while (checkpoint < seqNo) {
+    public synchronized void waitForProcessedOpsToComplete(final long seqNo) throws InterruptedException {
+        while (processedCheckpoint.get() < seqNo) {
             // notified by updateCheckpoint
             this.wait();
         }
     }
 
     /**
-     * Checks if the given sequence number was marked as completed in this tracker.
+     * Checks if the given sequence number was marked as processed in this tracker.
      */
-    public boolean contains(final long seqNo) {
+    public boolean hasProcessed(final long seqNo) {
         assert seqNo >= 0 : "invalid seq_no=" + seqNo;
-        if (seqNo >= nextSeqNo) {
+        if (seqNo >= nextSeqNo.get()) {
             return false;
         }
-        if (seqNo <= checkpoint) {
+        if (seqNo <= processedCheckpoint.get()) {
             return true;
         }
         final long bitSetKey = getBitSetKey(seqNo);
         final int bitSetOffset = seqNoToBitSetOffset(seqNo);
         synchronized (this) {
+            // check again under lock
+            if (seqNo <= processedCheckpoint.get()) {
+                return true;
+            }
             final CountedBitSet bitSet = processedSeqNo.get(bitSetKey);
             return bitSet != null && bitSet.get(bitSetOffset);
         }
     }
 
     /**
-     * Moves the checkpoint to the last consecutively processed sequence number. This method assumes that the sequence number following the
-     * current checkpoint is processed.
+     * Moves the checkpoint to the last consecutively processed sequence number. This method assumes that the sequence number
+     * following the current checkpoint is processed.
      */
     @SuppressForbidden(reason = "Object#notifyAll")
-    private void updateCheckpoint() {
+    private void updateCheckpoint(AtomicLong checkPoint, LongObjectHashMap<CountedBitSet> bitSetMap) {
         assert Thread.holdsLock(this);
-        assert getBitSetForSeqNo(checkpoint + 1).get(seqNoToBitSetOffset(checkpoint + 1)) :
+        assert getBitSetForSeqNo(bitSetMap, checkPoint.get() + 1).get(seqNoToBitSetOffset(checkPoint.get() + 1)) :
             "updateCheckpoint is called but the bit following the checkpoint is not set";
         try {
             // keep it simple for now, get the checkpoint one by one; in the future we can optimize and read words
-            long bitSetKey = getBitSetKey(checkpoint);
-            CountedBitSet current = processedSeqNo.get(bitSetKey);
+            long bitSetKey = getBitSetKey(checkPoint.get());
+            CountedBitSet current = bitSetMap.get(bitSetKey);
             if (current == null) {
                 // the bit set corresponding to the checkpoint has already been removed, set ourselves up for the next bit set
-                assert checkpoint % BIT_SET_SIZE == BIT_SET_SIZE - 1;
-                current = processedSeqNo.get(++bitSetKey);
+                assert checkPoint.get() % BIT_SET_SIZE == BIT_SET_SIZE - 1;
+                current = bitSetMap.get(++bitSetKey);
             }
             do {
-                checkpoint++;
+                checkPoint.incrementAndGet();
                 /*
                  * The checkpoint always falls in the current bit set or we have already cleaned it; if it falls on the last bit of the
                  * current bit set, we can clean it.
                  */
-                if (checkpoint == lastSeqNoInBitSet(bitSetKey)) {
+                if (checkPoint.get() == lastSeqNoInBitSet(bitSetKey)) {
                     assert current != null;
-                    final CountedBitSet removed = processedSeqNo.remove(bitSetKey);
+                    final CountedBitSet removed = bitSetMap.remove(bitSetKey);
                     assert removed == current;
-                    current = processedSeqNo.get(++bitSetKey);
+                    current = bitSetMap.get(++bitSetKey);
                 }
-            } while (current != null && current.get(seqNoToBitSetOffset(checkpoint + 1)));
+            } while (current != null && current.get(seqNoToBitSetOffset(checkPoint.get() + 1)));
         } finally {
-            // notifies waiters in waitForOpsToComplete
+            // notifies waiters in waitForProcessedOpsToComplete
             this.notifyAll();
         }
     }
 
-    private long lastSeqNoInBitSet(final long bitSetKey) {
+    private static long lastSeqNoInBitSet(final long bitSetKey) {
         return (1 + bitSetKey) * BIT_SET_SIZE - 1;
     }
 
@@ -220,32 +257,32 @@ private long lastSeqNoInBitSet(final long bitSetKey) {
      * @param seqNo the sequence number to obtain the bit set for
      * @return the bit set corresponding to the provided sequence number
      */
-    private long getBitSetKey(final long seqNo) {
+    private static long getBitSetKey(final long seqNo) {
         return seqNo / BIT_SET_SIZE;
     }
 
-    private CountedBitSet getBitSetForSeqNo(final long seqNo) {
+    private CountedBitSet getBitSetForSeqNo(final LongObjectHashMap<CountedBitSet> bitSetMap, final long seqNo) {
         assert Thread.holdsLock(this);
         final long bitSetKey = getBitSetKey(seqNo);
-        final int index = processedSeqNo.indexOf(bitSetKey);
+        final int index = bitSetMap.indexOf(bitSetKey);
         final CountedBitSet bitSet;
-        if (processedSeqNo.indexExists(index)) {
-            bitSet = processedSeqNo.indexGet(index);
+        if (bitSetMap.indexExists(index)) {
+            bitSet = bitSetMap.indexGet(index);
         } else {
             bitSet = new CountedBitSet(BIT_SET_SIZE);
-            processedSeqNo.indexInsert(index, bitSetKey, bitSet);
+            bitSetMap.indexInsert(index, bitSetKey, bitSet);
         }
         return bitSet;
     }
 
     /**
      * Obtain the position in the bit set corresponding to the provided sequence number. The bit set corresponding to the sequence number
-     * can be obtained via {@link #getBitSetForSeqNo(long)}.
+     * can be obtained via {@link #getBitSetForSeqNo(LongObjectHashMap, long)}.
      *
      * @param seqNo the sequence number to obtain the position for
      * @return the position in the bit set corresponding to the provided sequence number
      */
-    private int seqNoToBitSetOffset(final long seqNo) {
+    private static int seqNoToBitSetOffset(final long seqNo) {
         return Math.toIntExact(seqNo % BIT_SET_SIZE);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
index 892056674019f..20e998b1e12a6 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
@@ -105,12 +105,13 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L
     private volatile long operationPrimaryTerm;
 
     /**
-     * Boolean flag that indicates if a relocation handoff is in progress. A handoff is started by calling {@link #startRelocationHandoff}
-     * and is finished by either calling {@link #completeRelocationHandoff} or {@link #abortRelocationHandoff}, depending on whether the
-     * handoff was successful or not. During the handoff, which has as main objective to transfer the internal state of the global
-     * checkpoint tracker from the relocation source to the target, the list of in-sync shard copies cannot grow, otherwise the relocation
-     * target might miss this information and increase the global checkpoint to eagerly. As consequence, some of the methods in this class
-     * are not allowed to be called while a handoff is in progress, in particular {@link #markAllocationIdAsInSync}.
+     * Boolean flag that indicates if a relocation handoff is in progress. A handoff is started by calling
+     * {@link #startRelocationHandoff(String)} and is finished by either calling {@link #completeRelocationHandoff} or
+     * {@link #abortRelocationHandoff}, depending on whether the handoff was successful or not. During the handoff, which has as main
+     * objective to transfer the internal state of the global checkpoint tracker from the relocation source to the target, the list of
+     * in-sync shard copies cannot grow, otherwise the relocation target might miss this information and increase the global checkpoint
+     * to eagerly. As consequence, some of the methods in this class are not allowed to be called while a handoff is in progress,
+     * in particular {@link #markAllocationIdAsInSync}.
      *
      * A notable exception to this is the method {@link #updateFromMaster}, which is still allowed to be called during a relocation handoff.
      * The reason for this is that the handoff might fail and can be aborted (using {@link #abortRelocationHandoff}), in which case
@@ -146,9 +147,15 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L
     final Map<String, CheckpointState> checkpoints;
 
     /**
-     * A callback invoked when the global checkpoint is updated. For primary mode this occurs if the computed global checkpoint advances on
-     * the basis of state changes tracked here. For non-primary mode this occurs if the local knowledge of the global checkpoint advances
-     * due to an update from the primary.
+     * The current in-memory global checkpoint. In primary mode, this is a cached version of the checkpoint computed from the local
+     * checkpoints. In replica mode, this is the in-memory global checkpoint that's communicated by the primary.
+     */
+    volatile long globalCheckpoint;
+
+    /**
+     * A callback invoked when the in-memory global checkpoint is updated. For primary mode this occurs if the computed global checkpoint
+     * advances on the basis of state changes tracked here. For non-primary mode this occurs if the local knowledge of the global checkpoint
+     * advances due to an update from the primary.
      */
     private final LongConsumer onGlobalCheckpointUpdated;
 
@@ -221,12 +228,15 @@ public synchronized Tuple<Boolean, RetentionLeases> getRetentionLeases(final boo
                 .leases()
                 .stream()
                 .collect(Collectors.groupingBy(lease -> currentTimeMillis - lease.timestamp() > retentionLeaseMillis));
-        if (partitionByExpiration.get(true) == null) {
+        final Collection<RetentionLease> expiredLeases = partitionByExpiration.get(true);
+        if (expiredLeases == null) {
             // early out as no retention leases have expired
+            logger.debug("no retention leases are expired from current retention leases [{}]", retentionLeases);
             return Tuple.tuple(false, retentionLeases);
         }
         final Collection<RetentionLease> nonExpiredLeases =
                 partitionByExpiration.get(false) != null ? partitionByExpiration.get(false) : Collections.emptyList();
+        logger.debug("expiring retention leases [{}] from current retention leases [{}]", expiredLeases, retentionLeases);
         retentionLeases = new RetentionLeases(operationPrimaryTerm, retentionLeases.version() + 1, nonExpiredLeases);
         return Tuple.tuple(true, retentionLeases);
     }
@@ -255,6 +265,7 @@ public RetentionLease addRetentionLease(
                 throw new RetentionLeaseAlreadyExistsException(id);
             }
             retentionLease = new RetentionLease(id, retainingSequenceNumber, currentTimeMillisSupplier.getAsLong(), source);
+            logger.debug("adding new retention lease [{}] to current retention leases [{}]", retentionLease, retentionLeases);
             retentionLeases = new RetentionLeases(
                     operationPrimaryTerm,
                     retentionLeases.version() + 1,
@@ -312,6 +323,7 @@ public void removeRetentionLease(final String id, final ActionListener<Replicati
             if (retentionLeases.contains(id) == false) {
                 throw new RetentionLeaseNotFoundException(id);
             }
+            logger.debug("removing retention lease [{}] from current retention leases [{}]", id, retentionLeases);
             retentionLeases = new RetentionLeases(
                     operationPrimaryTerm,
                     retentionLeases.version() + 1,
@@ -341,7 +353,10 @@ public synchronized void updateRetentionLeasesOnReplica(final RetentionLeases re
      * @throws IOException if an I/O exception occurs reading the retention leases
      */
     public RetentionLeases loadRetentionLeases(final Path path) throws IOException {
-        final RetentionLeases retentionLeases = RetentionLeases.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, path);
+        final RetentionLeases retentionLeases;
+        synchronized (retentionLeasePersistenceLock) {
+            retentionLeases = RetentionLeases.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, path);
+        }
 
         // TODO after backporting we expect this never to happen in 8.x, so adjust this to throw an exception instead.
         assert Version.CURRENT.major <= 8 : "throw an exception instead of returning EMPTY on null";
@@ -385,13 +400,13 @@ public boolean assertRetentionLeasesPersisted(final Path path) throws IOExceptio
     public static class CheckpointState implements Writeable {
 
         /**
-         * the last local checkpoint information that we have for this shard
+         * the last local checkpoint information that we have for this shard. All operations up to this point are properly fsynced to disk.
          */
         long localCheckpoint;
 
         /**
-         * the last global checkpoint information that we have for this shard. This information is computed for the primary if
-         * the tracker is in primary mode and received from the primary if in replica mode.
+         * the last global checkpoint information that we have for this shard. This is the global checkpoint that's fsynced to disk on the
+         * respective shard, and all operations up to this point are properly fsynced to disk as well.
          */
         long globalCheckpoint;
         /**
@@ -475,9 +490,9 @@ public int hashCode() {
     }
 
     /**
-     * Get the local knowledge of the global checkpoints for all in-sync allocation IDs.
+     * Get the local knowledge of the persisted global checkpoints for all in-sync allocation IDs.
      *
-     * @return a map from allocation ID to the local knowledge of the global checkpoint for that allocation ID
+     * @return a map from allocation ID to the local knowledge of the persisted global checkpoint for that allocation ID
      */
     public synchronized ObjectLongMap<String> getInSyncGlobalCheckpoints() {
         assert primaryMode;
@@ -530,24 +545,11 @@ public boolean isRelocated() {
      * as a logical operator, many of the invariants are written under the form (!A || B), they should be read as (A implies B) however.
      */
     private boolean invariant() {
-        assert checkpoints.get(shardAllocationId) != null :
-            "checkpoints map should always have an entry for the current shard";
-
         // local checkpoints only set during primary mode
-        assert primaryMode || checkpoints.values().stream()
-            .allMatch(lcps -> lcps.localCheckpoint == SequenceNumbers.UNASSIGNED_SEQ_NO ||
-                lcps.localCheckpoint == SequenceNumbers.PRE_60_NODE_CHECKPOINT);
+        assert primaryMode || checkpoints.values().stream().allMatch(lcps -> lcps.localCheckpoint == SequenceNumbers.UNASSIGNED_SEQ_NO);
 
-        // global checkpoints for other shards only set during primary mode
-        assert primaryMode
-                || checkpoints
-                .entrySet()
-                .stream()
-                .filter(e -> e.getKey().equals(shardAllocationId) == false)
-                .map(Map.Entry::getValue)
-                .allMatch(cps ->
-                        (cps.globalCheckpoint == SequenceNumbers.UNASSIGNED_SEQ_NO
-                                || cps.globalCheckpoint == SequenceNumbers.PRE_60_NODE_CHECKPOINT));
+        // global checkpoints only set during primary mode
+        assert primaryMode || checkpoints.values().stream().allMatch(cps -> cps.globalCheckpoint == SequenceNumbers.UNASSIGNED_SEQ_NO);
 
         // relocation handoff can only occur in primary mode
         assert !handoffInProgress || primaryMode;
@@ -576,14 +578,14 @@ private boolean invariant() {
 
         // the computed global checkpoint is always up-to-date
         assert !primaryMode
-                || getGlobalCheckpoint() == computeGlobalCheckpoint(pendingInSync, checkpoints.values(), getGlobalCheckpoint())
+                || globalCheckpoint == computeGlobalCheckpoint(pendingInSync, checkpoints.values(), globalCheckpoint)
                 : "global checkpoint is not up-to-date, expected: " +
-                computeGlobalCheckpoint(pendingInSync, checkpoints.values(), getGlobalCheckpoint()) + " but was: " + getGlobalCheckpoint();
+                computeGlobalCheckpoint(pendingInSync, checkpoints.values(), globalCheckpoint) + " but was: " + globalCheckpoint;
 
         // when in primary mode, the global checkpoint is at most the minimum local checkpoint on all in-sync shard copies
         assert !primaryMode
-                || getGlobalCheckpoint() <= inSyncCheckpointStates(checkpoints, CheckpointState::getLocalCheckpoint, LongStream::min)
-                : "global checkpoint [" + getGlobalCheckpoint() + "] "
+                || globalCheckpoint <= inSyncCheckpointStates(checkpoints, CheckpointState::getLocalCheckpoint, LongStream::min)
+                : "global checkpoint [" + globalCheckpoint + "] "
                 + "for primary mode allocation ID [" + shardAllocationId + "] "
                 + "more than in-sync local checkpoints [" + checkpoints + "]";
 
@@ -626,7 +628,7 @@ private static long inSyncCheckpointStates(
                                 .stream()
                                 .filter(cps -> cps.inSync)
                                 .mapToLong(function)
-                                .filter(v -> v != SequenceNumbers.PRE_60_NODE_CHECKPOINT && v != SequenceNumbers.UNASSIGNED_SEQ_NO));
+                                .filter(v -> v != SequenceNumbers.UNASSIGNED_SEQ_NO));
         return value.isPresent() ? value.getAsLong() : SequenceNumbers.UNASSIGNED_SEQ_NO;
     }
 
@@ -657,8 +659,8 @@ public ReplicationTracker(
         this.operationPrimaryTerm = operationPrimaryTerm;
         this.handoffInProgress = false;
         this.appliedClusterStateVersion = -1L;
+        this.globalCheckpoint = globalCheckpoint;
         this.checkpoints = new HashMap<>(1 + indexSettings.getNumberOfReplicas());
-        checkpoints.put(allocationId, new CheckpointState(SequenceNumbers.UNASSIGNED_SEQ_NO, globalCheckpoint, false, false));
         this.onGlobalCheckpointUpdated = Objects.requireNonNull(onGlobalCheckpointUpdated);
         this.currentTimeMillisSupplier = Objects.requireNonNull(currentTimeMillisSupplier);
         this.onSyncRetentionLeases = Objects.requireNonNull(onSyncRetentionLeases);
@@ -685,28 +687,26 @@ private ReplicationGroup calculateReplicationGroup() {
     }
 
     /**
-     * Returns the global checkpoint for the shard.
+     * Returns the in-memory global checkpoint for the shard.
      *
      * @return the global checkpoint
      */
-    public synchronized long getGlobalCheckpoint() {
-        final CheckpointState cps = checkpoints.get(shardAllocationId);
-        assert cps != null;
-        return cps.globalCheckpoint;
+    public long getGlobalCheckpoint() {
+        return globalCheckpoint;
     }
 
     @Override
     public long getAsLong() {
-        return getGlobalCheckpoint();
+        return globalCheckpoint;
     }
 
     /**
      * Updates the global checkpoint on a replica shard after it has been updated by the primary.
      *
-     * @param globalCheckpoint the global checkpoint
-     * @param reason           the reason the global checkpoint was updated
+     * @param newGlobalCheckpoint the new global checkpoint
+     * @param reason              the reason the global checkpoint was updated
      */
-    public synchronized void updateGlobalCheckpointOnReplica(final long globalCheckpoint, final String reason) {
+    public synchronized void updateGlobalCheckpointOnReplica(final long newGlobalCheckpoint, final String reason) {
         assert invariant();
         assert primaryMode == false;
         /*
@@ -715,18 +715,17 @@ public synchronized void updateGlobalCheckpointOnReplica(final long globalCheckp
          * replica shards). In these cases, the local knowledge of the global checkpoint could be higher than the sync from the lagging
          * primary.
          */
-        updateGlobalCheckpoint(
-                shardAllocationId,
-                globalCheckpoint,
-                current -> {
-                    logger.trace("updated global checkpoint from [{}] to [{}] due to [{}]", current, globalCheckpoint, reason);
-                    onGlobalCheckpointUpdated.accept(globalCheckpoint);
-                });
+        final long previousGlobalCheckpoint = globalCheckpoint;
+        if (newGlobalCheckpoint > previousGlobalCheckpoint) {
+            globalCheckpoint = newGlobalCheckpoint;
+            logger.trace("updated global checkpoint from [{}] to [{}] due to [{}]", previousGlobalCheckpoint, globalCheckpoint, reason);
+            onGlobalCheckpointUpdated.accept(globalCheckpoint);
+        }
         assert invariant();
     }
 
     /**
-     * Update the local knowledge of the global checkpoint for the specified allocation ID.
+     * Update the local knowledge of the persisted global checkpoint for the specified allocation ID.
      *
      * @param allocationId     the allocation ID to update the global checkpoint for
      * @param globalCheckpoint the global checkpoint
@@ -735,24 +734,15 @@ public synchronized void updateGlobalCheckpointForShard(final String allocationI
         assert primaryMode;
         assert handoffInProgress == false;
         assert invariant();
-        updateGlobalCheckpoint(
-                allocationId,
-                globalCheckpoint,
-                current -> logger.trace(
-                        "updated local knowledge for [{}] on the primary of the global checkpoint from [{}] to [{}]",
-                        allocationId,
-                        current,
-                        globalCheckpoint));
-        assert invariant();
-    }
-
-    private void updateGlobalCheckpoint(final String allocationId, final long globalCheckpoint, LongConsumer ifUpdated) {
         final CheckpointState cps = checkpoints.get(allocationId);
         assert !this.shardAllocationId.equals(allocationId) || cps != null;
         if (cps != null && globalCheckpoint > cps.globalCheckpoint) {
+            final long previousGlobalCheckpoint = cps.globalCheckpoint;
             cps.globalCheckpoint = globalCheckpoint;
-            ifUpdated.accept(cps.globalCheckpoint);
+            logger.trace("updated local knowledge for [{}] on the primary of the global checkpoint from [{}] to [{}]",
+                allocationId, previousGlobalCheckpoint, globalCheckpoint);
         }
+        assert invariant();
     }
 
     /**
@@ -810,23 +800,14 @@ public synchronized void updateFromMaster(final long applyingClusterStateVersion
                 }
             } else {
                 for (String initializingId : initializingAllocationIds) {
-                    if (shardAllocationId.equals(initializingId) == false) {
-                        final long localCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO;
-                        final long globalCheckpoint = localCheckpoint;
-                        checkpoints.put(initializingId, new CheckpointState(localCheckpoint, globalCheckpoint, false, false));
-                    }
+                    final long localCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO;
+                    final long globalCheckpoint = localCheckpoint;
+                    checkpoints.put(initializingId, new CheckpointState(localCheckpoint, globalCheckpoint, false, false));
                 }
                 for (String inSyncId : inSyncAllocationIds) {
-                    if (shardAllocationId.equals(inSyncId)) {
-                        // current shard is initially marked as not in-sync because we don't know better at that point
-                        CheckpointState checkpointState = checkpoints.get(shardAllocationId);
-                        checkpointState.inSync = true;
-                        checkpointState.tracked = true;
-                    } else {
-                        final long localCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO;
-                        final long globalCheckpoint = localCheckpoint;
-                        checkpoints.put(inSyncId, new CheckpointState(localCheckpoint, globalCheckpoint, true, true));
-                    }
+                    final long localCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO;
+                    final long globalCheckpoint = localCheckpoint;
+                    checkpoints.put(inSyncId, new CheckpointState(localCheckpoint, globalCheckpoint, true, true));
                 }
             }
             appliedClusterStateVersion = applyingClusterStateVersion;
@@ -911,13 +892,9 @@ public synchronized void markAllocationIdAsInSync(final String allocationId, fin
     }
 
     private boolean updateLocalCheckpoint(String allocationId, CheckpointState cps, long localCheckpoint) {
-        // a local checkpoint of PRE_60_NODE_CHECKPOINT cannot be overridden
-        assert cps.localCheckpoint != SequenceNumbers.PRE_60_NODE_CHECKPOINT ||
-            localCheckpoint == SequenceNumbers.PRE_60_NODE_CHECKPOINT :
-            "pre-6.0 shard copy " + allocationId + " unexpected to send valid local checkpoint " + localCheckpoint;
-        // a local checkpoint for a shard copy should be a valid sequence number or the pre-6.0 sequence number indicator
-        assert localCheckpoint != SequenceNumbers.UNASSIGNED_SEQ_NO :
-                "invalid local checkpoint for shard copy [" + allocationId + "]";
+        // a local checkpoint for a shard copy should be a valid sequence number
+        assert localCheckpoint >= SequenceNumbers.NO_OPS_PERFORMED :
+            "invalid local checkpoint [" + localCheckpoint + "] for shard copy [" + allocationId + "]";
         if (localCheckpoint > cps.localCheckpoint) {
             logger.trace("updated local checkpoint of [{}] from [{}] to [{}]", allocationId, cps.localCheckpoint, localCheckpoint);
             cps.localCheckpoint = localCheckpoint;
@@ -976,8 +953,6 @@ private static long computeGlobalCheckpoint(final Set<String> pendingInSync, fin
                 if (cps.localCheckpoint == SequenceNumbers.UNASSIGNED_SEQ_NO) {
                     // unassigned in-sync replica
                     return fallback;
-                } else if (cps.localCheckpoint == SequenceNumbers.PRE_60_NODE_CHECKPOINT) {
-                    // 5.x replica, ignore for global checkpoint calculation
                 } else {
                     minLocalCheckpoint = Math.min(cps.localCheckpoint, minLocalCheckpoint);
                 }
@@ -992,13 +967,11 @@ private static long computeGlobalCheckpoint(final Set<String> pendingInSync, fin
      */
     private synchronized void updateGlobalCheckpointOnPrimary() {
         assert primaryMode;
-        final CheckpointState cps = checkpoints.get(shardAllocationId);
-        final long globalCheckpoint = cps.globalCheckpoint;
         final long computedGlobalCheckpoint = computeGlobalCheckpoint(pendingInSync, checkpoints.values(), getGlobalCheckpoint());
         assert computedGlobalCheckpoint >= globalCheckpoint : "new global checkpoint [" + computedGlobalCheckpoint +
             "] is lower than previous one [" + globalCheckpoint + "]";
         if (globalCheckpoint != computedGlobalCheckpoint) {
-            cps.globalCheckpoint = computedGlobalCheckpoint;
+            globalCheckpoint = computedGlobalCheckpoint;
             logger.trace("updated global checkpoint to [{}]", computedGlobalCheckpoint);
             onGlobalCheckpointUpdated.accept(computedGlobalCheckpoint);
         }
@@ -1007,11 +980,15 @@ private synchronized void updateGlobalCheckpointOnPrimary() {
     /**
      * Initiates a relocation handoff and returns the corresponding primary context.
      */
-    public synchronized PrimaryContext startRelocationHandoff() {
+    public synchronized PrimaryContext startRelocationHandoff(String targetAllocationId) {
         assert invariant();
         assert primaryMode;
         assert handoffInProgress == false;
         assert pendingInSync.isEmpty() : "relocation handoff started while there are still shard copies pending in-sync: " + pendingInSync;
+        if (checkpoints.containsKey(targetAllocationId) == false) {
+            // can happen if the relocation target was removed from cluster but the recovery process isn't aware of that.
+            throw new IllegalStateException("relocation target [" + targetAllocationId + "] is no longer part of the replication group");
+        }
         handoffInProgress = true;
         // copy clusterStateVersion and checkpoints and return
         // all the entries from checkpoints that are inSync: the reason we don't need to care about initializing non-insync entries
@@ -1048,20 +1025,10 @@ public synchronized void completeRelocationHandoff() {
         primaryMode = false;
         handoffInProgress = false;
         relocated = true;
-        // forget all checkpoint information except for global checkpoint of current shard
-        checkpoints.entrySet().stream().forEach(e -> {
-            final CheckpointState cps = e.getValue();
-            if (cps.localCheckpoint != SequenceNumbers.UNASSIGNED_SEQ_NO &&
-                cps.localCheckpoint != SequenceNumbers.PRE_60_NODE_CHECKPOINT) {
-                cps.localCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO;
-            }
-            if (e.getKey().equals(shardAllocationId) == false) {
-                // don't throw global checkpoint information of current shard away
-                if (cps.globalCheckpoint != SequenceNumbers.UNASSIGNED_SEQ_NO &&
-                    cps.globalCheckpoint != SequenceNumbers.PRE_60_NODE_CHECKPOINT) {
-                    cps.globalCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO;
-                }
-            }
+        // forget all checkpoint information
+        checkpoints.forEach((key, cps) -> {
+            cps.localCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO;
+            cps.globalCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO;
         });
         assert invariant();
     }
@@ -1107,7 +1074,7 @@ private Runnable getMasterUpdateOperationFromCurrentState() {
     }
 
     /**
-     * Whether the are shards blocking global checkpoint advancement. Used by tests.
+     * Whether the are shards blocking global checkpoint advancement.
      */
     public synchronized boolean pendingInSync() {
         assert primaryMode;
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseActions.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseActions.java
index 74c98bf3dca19..bcb1eeeed1e73 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseActions.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseActions.java
@@ -19,10 +19,10 @@
 
 package org.elasticsearch.index.seqno;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.single.shard.SingleShardRequest;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
@@ -44,7 +44,6 @@
 
 import java.io.IOException;
 import java.util.Objects;
-import java.util.function.Supplier;
 
 /**
  * This class holds all actions related to retention leases. Note carefully that these actions are executed under a primary permit. Care is
@@ -70,7 +69,7 @@ abstract static class TransportRetentionLeaseAction<T extends Request<T>> extend
                 final ActionFilters actionFilters,
                 final IndexNameExpressionResolver indexNameExpressionResolver,
                 final IndicesService indicesService,
-                final Supplier<T> requestSupplier) {
+                final Writeable.Reader<T> requestSupplier) {
             super(
                     name,
                     threadPool,
@@ -124,7 +123,7 @@ protected boolean resolveIndex(final T request) {
 
     }
 
-    public static class Add extends Action<Response> {
+    public static class Add extends StreamableResponseActionType<Response> {
 
         public static final Add INSTANCE = new Add();
         public static final String ACTION_NAME = "indices:admin/seq_no/add_retention_lease";
@@ -177,7 +176,7 @@ public Response newResponse() {
 
     }
 
-    public static class Renew extends Action<Response> {
+    public static class Renew extends StreamableResponseActionType<Response> {
 
         public static final Renew INSTANCE = new Renew();
         public static final String ACTION_NAME = "indices:admin/seq_no/renew_retention_lease";
@@ -223,7 +222,7 @@ public Response newResponse() {
 
     }
 
-    public static class Remove extends Action<Response> {
+    public static class Remove extends StreamableResponseActionType<Response> {
 
         public static final Remove INSTANCE = new Remove();
         public static final String ACTION_NAME = "indices:admin/seq_no/remove_retention_lease";
@@ -272,19 +271,22 @@ public Response newResponse() {
 
     private abstract static class Request<T extends SingleShardRequest<T>> extends SingleShardRequest<T> {
 
-        private ShardId shardId;
+        private final ShardId shardId;
 
         public ShardId getShardId() {
             return shardId;
         }
 
-        private String id;
+        private final String id;
 
         public String getId() {
             return id;
         }
 
-        Request() {
+        Request(StreamInput in) throws IOException {
+            super(in);
+            shardId = new ShardId(in);
+            id = in.readString();
         }
 
         Request(final ShardId shardId, final String id) {
@@ -298,13 +300,6 @@ public ActionRequestValidationException validate() {
             return null;
         }
 
-        @Override
-        public void readFrom(final StreamInput in) throws IOException {
-            super.readFrom(in);
-            shardId = new ShardId(in);
-            id = in.readString();
-        }
-
         @Override
         public void writeTo(final StreamOutput out) throws IOException {
             super.writeTo(out);
@@ -316,19 +311,22 @@ public void writeTo(final StreamOutput out) throws IOException {
 
     private abstract static class AddOrRenewRequest<T extends SingleShardRequest<T>> extends Request<T> {
 
-        private long retainingSequenceNumber;
+        private final long retainingSequenceNumber;
 
         public long getRetainingSequenceNumber() {
             return retainingSequenceNumber;
         }
 
-        private String source;
+        private final String source;
 
         public String getSource() {
             return source;
         }
 
-        AddOrRenewRequest() {
+        AddOrRenewRequest(StreamInput in) throws IOException {
+            super(in);
+            retainingSequenceNumber = in.readZLong();
+            source = in.readString();
         }
 
         AddOrRenewRequest(final ShardId shardId, final String id, final long retainingSequenceNumber, final String source) {
@@ -340,13 +338,6 @@ public String getSource() {
             this.source = Objects.requireNonNull(source);
         }
 
-        @Override
-        public void readFrom(final StreamInput in) throws IOException {
-            super.readFrom(in);
-            retainingSequenceNumber = in.readZLong();
-            source = in.readString();
-        }
-
         @Override
         public void writeTo(final StreamOutput out) throws IOException {
             super.writeTo(out);
@@ -358,7 +349,8 @@ public void writeTo(final StreamOutput out) throws IOException {
 
     public static class AddRequest extends AddOrRenewRequest<AddRequest> {
 
-        public AddRequest() {
+        AddRequest(StreamInput in) throws IOException {
+            super(in);
         }
 
         public AddRequest(final ShardId shardId, final String id, final long retainingSequenceNumber, final String source) {
@@ -369,7 +361,8 @@ public AddRequest(final ShardId shardId, final String id, final long retainingSe
 
     public static class RenewRequest extends AddOrRenewRequest<RenewRequest> {
 
-        public RenewRequest() {
+        RenewRequest(StreamInput in) throws IOException {
+            super(in);
         }
 
         public RenewRequest(final ShardId shardId, final String id, final long retainingSequenceNumber, final String source) {
@@ -380,7 +373,8 @@ public RenewRequest(final ShardId shardId, final String id, final long retaining
 
     public static class RemoveRequest extends Request<RemoveRequest> {
 
-        public RemoveRequest() {
+        RemoveRequest(StreamInput in) throws IOException {
+            super(in);
         }
 
         public RemoveRequest(final ShardId shardId, final String id) {
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java
index 9be2e55057331..184b936784c0c 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncAction.java
@@ -21,10 +21,9 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.lucene.store.AlreadyClosedException;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.replication.ReplicationRequest;
@@ -37,10 +36,8 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.gateway.WriteStateException;
 import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.IndexShardClosedException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -61,6 +58,12 @@ public class RetentionLeaseBackgroundSyncAction extends TransportReplicationActi
         ReplicationResponse> {
 
     public static String ACTION_NAME = "indices:admin/seq_no/retention_lease_background_sync";
+    public static ActionType<ReplicationResponse> TYPE = new StreamableResponseActionType<>(ACTION_NAME) {
+        @Override
+        public ReplicationResponse newResponse() {
+            return new ReplicationResponse();
+        }
+    };
 
     private static final Logger LOGGER = LogManager.getLogger(RetentionLeaseSyncAction.class);
 
@@ -93,39 +96,6 @@ public RetentionLeaseBackgroundSyncAction(
                 ThreadPool.Names.MANAGEMENT);
     }
 
-    /**
-     * Background sync the specified retention leases for the specified shard.
-     *
-     * @param shardId         the shard to sync
-     * @param retentionLeases the retention leases to sync
-     */
-    public void backgroundSync(
-            final ShardId shardId,
-            final RetentionLeases retentionLeases) {
-        Objects.requireNonNull(shardId);
-        Objects.requireNonNull(retentionLeases);
-        final ThreadContext threadContext = threadPool.getThreadContext();
-        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
-            // we have to execute under the system context so that if security is enabled the sync is authorized
-            threadContext.markAsSystemContext();
-            execute(
-                    new Request(shardId, retentionLeases),
-                    ActionListener.wrap(
-                            r -> {},
-                            e -> {
-                                if (ExceptionsHelper.isTransportStoppedForAction(e, ACTION_NAME + "[p]")) {
-                                    // we are likely shutting down
-                                    return;
-                                }
-                                if (ExceptionsHelper.unwrap(e, AlreadyClosedException.class, IndexShardClosedException.class) != null) {
-                                    // the shard is closed
-                                    return;
-                                }
-                                getLogger().warn(new ParameterizedMessage("{} retention lease background sync failed", shardId), e);
-                            }));
-        }
-    }
-
     @Override
     protected void shardOperationOnPrimary(
             final Request request,
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java
index 6129affe493ae..1c4f1b74cbccf 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java
@@ -21,10 +21,9 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.lucene.store.AlreadyClosedException;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.WriteResponse;
@@ -39,10 +38,8 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.gateway.WriteStateException;
 import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.IndexShardClosedException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -59,6 +56,12 @@ public class RetentionLeaseSyncAction extends
         TransportWriteAction<RetentionLeaseSyncAction.Request, RetentionLeaseSyncAction.Request, RetentionLeaseSyncAction.Response> {
 
     public static String ACTION_NAME = "indices:admin/seq_no/retention_lease_sync";
+    public static ActionType<Response> TYPE = new StreamableResponseActionType<>(ACTION_NAME) {
+        @Override
+        public Response newResponse() {
+            return new Response();
+        }
+    };
 
     private static final Logger LOGGER = LogManager.getLogger(RetentionLeaseSyncAction.class);
 
@@ -91,37 +94,6 @@ public RetentionLeaseSyncAction(
                 ThreadPool.Names.MANAGEMENT, false);
     }
 
-    /**
-     * Sync the specified retention leases for the specified shard. The callback is invoked when the sync succeeds or fails.
-     *
-     * @param shardId         the shard to sync
-     * @param retentionLeases the retention leases to sync
-     * @param listener        the callback to invoke when the sync completes normally or abnormally
-     */
-    public void sync(
-            final ShardId shardId,
-            final RetentionLeases retentionLeases,
-            final ActionListener<ReplicationResponse> listener) {
-        Objects.requireNonNull(shardId);
-        Objects.requireNonNull(retentionLeases);
-        Objects.requireNonNull(listener);
-        final ThreadContext threadContext = threadPool.getThreadContext();
-        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
-            // we have to execute under the system context so that if security is enabled the sync is authorized
-            threadContext.markAsSystemContext();
-            execute(
-                    new RetentionLeaseSyncAction.Request(shardId, retentionLeases),
-                    ActionListener.wrap(
-                            listener::onResponse,
-                            e -> {
-                                if (ExceptionsHelper.unwrap(e, AlreadyClosedException.class, IndexShardClosedException.class) == null) {
-                                    getLogger().warn(new ParameterizedMessage("{} retention lease sync failed", shardId), e);
-                                }
-                                listener.onFailure(e);
-                            }));
-        }
-    }
-
     @Override
     protected void shardOperationOnPrimary(Request request, IndexShard primary,
             ActionListener<PrimaryResult<Request, Response>> listener) {
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/SeqNoStats.java b/server/src/main/java/org/elasticsearch/index/seqno/SeqNoStats.java
index a56f8670c23b1..e1b992643fac8 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/SeqNoStats.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/SeqNoStats.java
@@ -58,7 +58,7 @@ public long getMaxSeqNo() {
         return maxSeqNo;
     }
 
-    /** the maximum sequence number for which all previous operations (including) have been completed */
+    /** the maximum sequence number for which all previous operations (including) have been persisted */
     public long getLocalCheckpoint() {
         return localCheckpoint;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/seqno/SequenceNumbers.java b/server/src/main/java/org/elasticsearch/index/seqno/SequenceNumbers.java
index 6336e83338f8c..87257a97076da 100644
--- a/server/src/main/java/org/elasticsearch/index/seqno/SequenceNumbers.java
+++ b/server/src/main/java/org/elasticsearch/index/seqno/SequenceNumbers.java
@@ -28,10 +28,6 @@ public class SequenceNumbers {
 
     public static final String LOCAL_CHECKPOINT_KEY = "local_checkpoint";
     public static final String MAX_SEQ_NO = "max_seq_no";
-    /**
-     * Represents a checkpoint coming from a pre-6.0 node
-     */
-    public static final long PRE_60_NODE_CHECKPOINT = -3L;
     /**
      * Represents an unassigned sequence number (e.g., can be used on primary operations before they are executed).
      */
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java b/server/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java
deleted file mode 100644
index 3a6df72a740e9..0000000000000
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexSearcherWrapper.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.shard;
-
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.FilterDirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.search.IndexSearcher;
-import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
-import org.elasticsearch.core.internal.io.IOUtils;
-import org.elasticsearch.index.engine.Engine;
-
-import java.io.IOException;
-
-/**
- * Extension point to add custom functionality at request time to the {@link DirectoryReader}
- * and {@link IndexSearcher} managed by the {@link IndexShard}.
- */
-public class IndexSearcherWrapper {
-
-    /**
-     * Wraps the given {@link DirectoryReader}. The wrapped reader can filter out document just like delete documents etc. but
-     * must not change any term or document content.
-     * <p>
-     * NOTE: The wrapper has a per-request lifecycle, must delegate {@link IndexReader#getReaderCacheHelper()},
-     * {@link LeafReader#getCoreCacheHelper()} and must be an instance of {@link FilterDirectoryReader} that
-     * eventually exposes the original reader via  {@link FilterDirectoryReader#getDelegate()}.
-     * The returned reader is closed once it goes out of scope.
-     * </p>
-     * @param reader The provided directory reader to be wrapped to add custom functionality
-     * @return a new directory reader wrapping the provided directory reader or if no wrapping was performed
-     *         the provided directory reader
-     */
-    protected DirectoryReader wrap(DirectoryReader reader) throws IOException {
-        return reader;
-    }
-
-    /**
-     * @param searcher      The provided index searcher to be wrapped to add custom functionality
-     * @return a new index searcher wrapping the provided index searcher or if no wrapping was performed
-     *         the provided index searcher
-     */
-    protected IndexSearcher wrap(IndexSearcher searcher) throws IOException {
-        return searcher;
-    }
-    /**
-     * If there are configured {@link IndexSearcherWrapper} instances, the {@link IndexSearcher} of the provided engine searcher
-     * gets wrapped and a new {@link Engine.Searcher} instances is returned, otherwise the provided {@link Engine.Searcher} is returned.
-     *
-     * This is invoked each time a {@link Engine.Searcher} is requested to do an operation. (for example search)
-     */
-    public final Engine.Searcher wrap(Engine.Searcher engineSearcher) throws IOException {
-        final ElasticsearchDirectoryReader elasticsearchDirectoryReader =
-            ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(engineSearcher.getDirectoryReader());
-        if (elasticsearchDirectoryReader == null) {
-            throw new IllegalStateException("Can't wrap non elasticsearch directory reader");
-        }
-        NonClosingReaderWrapper nonClosingReaderWrapper = new NonClosingReaderWrapper(engineSearcher.getDirectoryReader());
-        DirectoryReader reader = wrap(nonClosingReaderWrapper);
-        if (reader != nonClosingReaderWrapper) {
-            if (reader.getReaderCacheHelper() != elasticsearchDirectoryReader.getReaderCacheHelper()) {
-                throw new IllegalStateException("wrapped directory reader doesn't delegate IndexReader#getCoreCacheKey," +
-                    " wrappers must override this method and delegate to the original readers core cache key. Wrapped readers can't be " +
-                    "used as cache keys since their are used only per request which would lead to subtle bugs");
-            }
-            if (ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(reader) != elasticsearchDirectoryReader) {
-                // prevent that somebody wraps with a non-filter reader
-                throw new IllegalStateException("wrapped directory reader hides actual ElasticsearchDirectoryReader but shouldn't");
-            }
-        }
-
-        final IndexSearcher origIndexSearcher = engineSearcher.searcher();
-        final IndexSearcher innerIndexSearcher = new IndexSearcher(reader);
-        innerIndexSearcher.setQueryCache(origIndexSearcher.getQueryCache());
-        innerIndexSearcher.setQueryCachingPolicy(origIndexSearcher.getQueryCachingPolicy());
-        innerIndexSearcher.setSimilarity(origIndexSearcher.getSimilarity());
-        // TODO: Right now IndexSearcher isn't wrapper friendly, when it becomes wrapper friendly we should revise this extension point
-        // For example if IndexSearcher#rewrite() is overwritten than also IndexSearcher#createNormalizedWeight needs to be overwritten
-        // This needs to be fixed before we can allow the IndexSearcher from Engine to be wrapped multiple times
-        final IndexSearcher indexSearcher = wrap(innerIndexSearcher);
-        if (reader == nonClosingReaderWrapper && indexSearcher == innerIndexSearcher) {
-            return engineSearcher;
-        } else {
-            // we close the reader to make sure wrappers can release resources if needed....
-            // our NonClosingReaderWrapper makes sure that our reader is not closed
-            return new Engine.Searcher(engineSearcher.source(), indexSearcher, () ->
-                IOUtils.close(indexSearcher.getIndexReader(), // this will close the wrappers excluding the NonClosingReaderWrapper
-                engineSearcher)); // this will run the closeable on the wrapped engine searcher
-        }
-    }
-
-    private static final class NonClosingReaderWrapper extends FilterDirectoryReader {
-
-        private NonClosingReaderWrapper(DirectoryReader in) throws IOException {
-            super(in, new SubReaderWrapper() {
-                @Override
-                public LeafReader wrap(LeafReader reader) {
-                    return reader;
-                }
-            });
-        }
-
-        @Override
-        protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
-            return new NonClosingReaderWrapper(in);
-        }
-
-        @Override
-        protected void doClose() throws IOException {
-            // don't close here - mimic the MultiReader#doClose = false behavior that FilterDirectoryReader doesn't have
-        }
-
-        @Override
-        public CacheHelper getReaderCacheHelper() {
-            return in.getReaderCacheHelper();
-        }
-
-    }
-
-}
diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index fdd95614756b7..d869ae934becd 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -23,9 +23,13 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.index.CheckIndex;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FilterDirectoryReader;
 import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryCachingPolicy;
 import org.apache.lucene.search.ReferenceManager;
@@ -50,6 +54,7 @@
 import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Tuple;
@@ -66,6 +71,7 @@
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.AsyncIOProcessor;
 import org.elasticsearch.common.util.concurrent.RunOnce;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.gateway.WriteStateException;
@@ -243,7 +249,7 @@ Runnable getGlobalCheckpointSyncer() {
     private static final EnumSet<IndexShardState> writeAllowedStates = EnumSet.of(IndexShardState.RECOVERING,
         IndexShardState.POST_RECOVERY, IndexShardState.STARTED);
 
-    private final IndexSearcherWrapper searcherWrapper;
+    private final CheckedFunction<DirectoryReader, DirectoryReader, IOException> readerWrapper;
 
     /**
      * True if this shard is still indexing (recently) and false if we've been idle for long enough (as periodically checked by {@link
@@ -269,7 +275,7 @@ public IndexShard(
             final SimilarityService similarityService,
             final @Nullable EngineFactory engineFactory,
             final IndexEventListener indexEventListener,
-            final IndexSearcherWrapper indexSearcherWrapper,
+            final CheckedFunction<DirectoryReader, DirectoryReader, IOException> indexReaderWrapper,
             final ThreadPool threadPool,
             final BigArrays bigArrays,
             final Engine.Warmer warmer,
@@ -291,6 +297,7 @@ public IndexShard(
         this.indexSortSupplier = indexSortSupplier;
         this.indexEventListener = indexEventListener;
         this.threadPool = threadPool;
+        this.translogSyncProcessor = createTranslogSyncProcessor(logger, threadPool.getThreadContext(), this::getEngine);
         this.mapperService = mapperService;
         this.indexCache = indexCache;
         this.internalIndexingStats = new InternalIndexingStats();
@@ -349,7 +356,7 @@ public boolean shouldCache(Query query) {
             cachingPolicy = new UsageTrackingQueryCachingPolicy();
         }
         indexShardOperationPermits = new IndexShardOperationPermits(shardId, threadPool);
-        searcherWrapper = indexSearcherWrapper;
+        readerWrapper = indexReaderWrapper;
         refreshListeners = buildRefreshListeners();
         lastSearcherAccess.set(threadPool.relativeTimeInMillis());
         persistMetadata(path, indexSettings, shardRouting, null, logger);
@@ -540,7 +547,8 @@ public void updateShardState(final ShardRouting newRouting,
                                  */
                                 engine.rollTranslogGeneration();
                                 engine.fillSeqNoGaps(newPrimaryTerm);
-                                replicationTracker.updateLocalCheckpoint(currentRouting.allocationId().getId(), getLocalCheckpoint());
+                                replicationTracker.updateLocalCheckpoint(currentRouting.allocationId().getId(),
+                                    getLocalCheckpoint());
                                 primaryReplicaSyncer.accept(this, new ActionListener<ResyncTask>() {
                                     @Override
                                     public void onResponse(ResyncTask resyncTask) {
@@ -618,10 +626,11 @@ public IndexShardState markAsRecovering(String reason, RecoveryState recoverySta
      *
      * @param consumer a {@link Runnable} that is executed after operations are blocked
      * @throws IllegalIndexShardStateException if the shard is not relocating due to concurrent cancellation
+     * @throws IllegalStateException           if the relocation target is no longer part of the replication group
      * @throws InterruptedException            if blocking operations is interrupted
      */
-    public void relocated(final Consumer<ReplicationTracker.PrimaryContext> consumer)
-                                            throws IllegalIndexShardStateException, InterruptedException {
+    public void relocated(final String targetAllocationId, final Consumer<ReplicationTracker.PrimaryContext> consumer)
+        throws IllegalIndexShardStateException, IllegalStateException, InterruptedException {
         assert shardRouting.primary() : "only primaries can be marked as relocated: " + shardRouting;
         final Releasable forceRefreshes = refreshListeners.forceRefreshes();
         try {
@@ -635,7 +644,7 @@ public void relocated(final Consumer<ReplicationTracker.PrimaryContext> consumer
                  * network operation. Doing this under the mutex can implicitly block the cluster state update thread on network operations.
                  */
                 verifyRelocatingState();
-                final ReplicationTracker.PrimaryContext primaryContext = replicationTracker.startRelocationHandoff();
+                final ReplicationTracker.PrimaryContext primaryContext = replicationTracker.startRelocationHandoff(targetAllocationId);
                 try {
                     consumer.accept(primaryContext);
                     synchronized (mutex) {
@@ -1228,7 +1237,7 @@ private Engine.Searcher acquireSearcher(String source, Engine.SearcherScope scop
             != null : "DirectoryReader must be an instance or ElasticsearchDirectoryReader";
         boolean success = false;
         try {
-            final Engine.Searcher wrappedSearcher = searcherWrapper == null ? searcher : searcherWrapper.wrap(searcher);
+            final Engine.Searcher wrappedSearcher = readerWrapper == null ? searcher : wrapSearcher(searcher, readerWrapper);
             assert wrappedSearcher != null;
             success = true;
             return wrappedSearcher;
@@ -1241,6 +1250,72 @@ private Engine.Searcher acquireSearcher(String source, Engine.SearcherScope scop
         }
     }
 
+    static Engine.Searcher wrapSearcher(Engine.Searcher engineSearcher,
+                                        CheckedFunction<DirectoryReader, DirectoryReader, IOException> readerWrapper) throws IOException {
+        assert readerWrapper != null;
+        final ElasticsearchDirectoryReader elasticsearchDirectoryReader =
+            ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(engineSearcher.getDirectoryReader());
+        if (elasticsearchDirectoryReader == null) {
+            throw new IllegalStateException("Can't wrap non elasticsearch directory reader");
+        }
+        NonClosingReaderWrapper nonClosingReaderWrapper = new NonClosingReaderWrapper(engineSearcher.getDirectoryReader());
+        DirectoryReader reader = readerWrapper.apply(nonClosingReaderWrapper);
+        if (reader != nonClosingReaderWrapper) {
+            if (reader.getReaderCacheHelper() != elasticsearchDirectoryReader.getReaderCacheHelper()) {
+                throw new IllegalStateException("wrapped directory reader doesn't delegate IndexReader#getCoreCacheKey," +
+                    " wrappers must override this method and delegate to the original readers core cache key. Wrapped readers can't be " +
+                    "used as cache keys since their are used only per request which would lead to subtle bugs");
+            }
+            if (ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(reader) != elasticsearchDirectoryReader) {
+                // prevent that somebody wraps with a non-filter reader
+                throw new IllegalStateException("wrapped directory reader hides actual ElasticsearchDirectoryReader but shouldn't");
+            }
+        }
+
+        if (reader == nonClosingReaderWrapper) {
+            return engineSearcher;
+        } else {
+            final IndexSearcher origIndexSearcher = engineSearcher.searcher();
+            final IndexSearcher newIndexSearcher = new IndexSearcher(reader);
+            newIndexSearcher.setQueryCache(origIndexSearcher.getQueryCache());
+            newIndexSearcher.setQueryCachingPolicy(origIndexSearcher.getQueryCachingPolicy());
+            newIndexSearcher.setSimilarity(origIndexSearcher.getSimilarity());
+            // we close the reader to make sure wrappers can release resources if needed....
+            // our NonClosingReaderWrapper makes sure that our reader is not closed
+            return new Engine.Searcher(engineSearcher.source(), newIndexSearcher, () ->
+                IOUtils.close(newIndexSearcher.getIndexReader(), // this will close the wrappers excluding the NonClosingReaderWrapper
+                    engineSearcher)); // this will run the closeable on the wrapped engine searcher
+        }
+    }
+
+    private static final class NonClosingReaderWrapper extends FilterDirectoryReader {
+
+        private NonClosingReaderWrapper(DirectoryReader in) throws IOException {
+            super(in, new SubReaderWrapper() {
+                @Override
+                public LeafReader wrap(LeafReader reader) {
+                    return reader;
+                }
+            });
+        }
+
+        @Override
+        protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
+            return new NonClosingReaderWrapper(in);
+        }
+
+        @Override
+        protected void doClose() throws IOException {
+            // don't close here - mimic the MultiReader#doClose = false behavior that FilterDirectoryReader doesn't have
+        }
+
+        @Override
+        public CacheHelper getReaderCacheHelper() {
+            return in.getReaderCacheHelper();
+        }
+
+    }
+
     public void close(String reason, boolean flushEngine) throws IOException {
         synchronized (mutex) {
             try {
@@ -1865,7 +1940,7 @@ public void updateLocalCheckpointForShard(final String allocationId, final long
     }
 
     /**
-     * Update the local knowledge of the global checkpoint for the specified allocation ID.
+     * Update the local knowledge of the persisted global checkpoint for the specified allocation ID.
      *
      * @param allocationId     the allocation ID to update the global checkpoint for
      * @param globalCheckpoint the global checkpoint
@@ -2079,12 +2154,12 @@ public void markAllocationIdAsInSync(final String allocationId, final long local
     }
 
     /**
-     * Returns the local checkpoint for the shard.
+     * Returns the persisted local checkpoint for the shard.
      *
      * @return the local checkpoint
      */
     public long getLocalCheckpoint() {
-        return getEngine().getLocalCheckpoint();
+        return getEngine().getPersistedLocalCheckpoint();
     }
 
     /**
@@ -2092,7 +2167,7 @@ public long getLocalCheckpoint() {
      *
      * @return the global checkpoint
      */
-    public long getGlobalCheckpoint() {
+    public long getLastKnownGlobalCheckpoint() {
         return replicationTracker.getGlobalCheckpoint();
     }
 
@@ -2125,15 +2200,21 @@ public void maybeSyncGlobalCheckpoint(final String reason) {
             return;
         }
         assert assertPrimaryMode();
-        // only sync if there are not operations in flight
+        // only sync if there are no operations in flight, or when using async durability
         final SeqNoStats stats = getEngine().getSeqNoStats(replicationTracker.getGlobalCheckpoint());
-        if (stats.getMaxSeqNo() == stats.getGlobalCheckpoint()) {
+        final boolean asyncDurability = indexSettings().getTranslogDurability() == Translog.Durability.ASYNC;
+        if (stats.getMaxSeqNo() == stats.getGlobalCheckpoint() || asyncDurability) {
             final ObjectLongMap<String> globalCheckpoints = getInSyncGlobalCheckpoints();
-            final String allocationId = routingEntry().allocationId().getId();
-            assert globalCheckpoints.containsKey(allocationId);
-            final long globalCheckpoint = globalCheckpoints.get(allocationId);
+            final long globalCheckpoint = replicationTracker.getGlobalCheckpoint();
+            // async durability means that the local checkpoint might lag (as it is only advanced on fsync)
+            // periodically ask for the newest local checkpoint by syncing the global checkpoint, so that ultimately the global
+            // checkpoint can be synced. Also take into account that a shard might be pending sync, which means that it isn't
+            // in the in-sync set just yet but might be blocked on waiting for its persisted local checkpoint to catch up to
+            // the global checkpoint.
             final boolean syncNeeded =
-                    StreamSupport
+                (asyncDurability && (stats.getGlobalCheckpoint() < stats.getMaxSeqNo() || replicationTracker.pendingInSync()))
+                    // check if the persisted global checkpoint
+                    || StreamSupport
                             .stream(globalCheckpoints.values().spliterator(), false)
                             .anyMatch(v -> v.value < globalCheckpoint);
             // only sync if index is not closed and there is a shard lagging the primary
@@ -2192,7 +2273,8 @@ public void activateWithPrimaryContext(final ReplicationTracker.PrimaryContext p
         assert shardRouting.primary() && shardRouting.isRelocationTarget() :
             "only primary relocation target can update allocation IDs from primary context: " + shardRouting;
         assert primaryContext.getCheckpointStates().containsKey(routingEntry().allocationId().getId()) &&
-            getLocalCheckpoint() == primaryContext.getCheckpointStates().get(routingEntry().allocationId().getId()).getLocalCheckpoint();
+            getLocalCheckpoint() == primaryContext.getCheckpointStates().get(routingEntry().allocationId().getId())
+                .getLocalCheckpoint() || indexSettings().getTranslogDurability() == Translog.Durability.ASYNC;
         synchronized (mutex) {
             replicationTracker.activateWithPrimaryContext(primaryContext); // make changes to primaryMode flag only under mutex
         }
@@ -2733,7 +2815,7 @@ private void innerAcquireReplicaOperationPermit(final long opPrimaryTerm,
 
                     bumpPrimaryTerm(opPrimaryTerm, () -> {
                         updateGlobalCheckpointOnReplica(globalCheckpoint, "primary term transition");
-                        final long currentGlobalCheckpoint = getGlobalCheckpoint();
+                        final long currentGlobalCheckpoint = getLastKnownGlobalCheckpoint();
                         final long maxSeqNo = seqNoStats().getMaxSeqNo();
                         logger.info("detected new primary with primary term [{}], global checkpoint [{}], max_seq_no [{}]",
                             opPrimaryTerm, currentGlobalCheckpoint, maxSeqNo);
@@ -2780,19 +2862,24 @@ public List<String> getActiveOperations() {
         return indexShardOperationPermits.getActiveOperations();
     }
 
-    private final AsyncIOProcessor<Translog.Location> translogSyncProcessor = new AsyncIOProcessor<Translog.Location>(logger, 1024) {
-        @Override
-        protected void write(List<Tuple<Translog.Location, Consumer<Exception>>> candidates) throws IOException {
-            try {
-                getEngine().ensureTranslogSynced(candidates.stream().map(Tuple::v1));
-            } catch (AlreadyClosedException ex) {
-                // that's fine since we already synced everything on engine close - this also is conform with the methods
-                // documentation
-            } catch (IOException ex) { // if this fails we are in deep shit - fail the request
-                logger.debug("failed to sync translog", ex);
-                throw ex;
+    private final AsyncIOProcessor<Translog.Location> translogSyncProcessor;
+
+    private static AsyncIOProcessor<Translog.Location> createTranslogSyncProcessor(Logger logger, ThreadContext threadContext,
+                                                                                   Supplier<Engine> engineSupplier) {
+        return new AsyncIOProcessor<>(logger, 1024, threadContext) {
+            @Override
+            protected void write(List<Tuple<Translog.Location, Consumer<Exception>>> candidates) throws IOException {
+                try {
+                    engineSupplier.get().ensureTranslogSynced(candidates.stream().map(Tuple::v1));
+                } catch (AlreadyClosedException ex) {
+                    // that's fine since we already synced everything on engine close - this also is conform with the methods
+                    // documentation
+                } catch (IOException ex) { // if this fails we are in deep shit - fail the request
+                    logger.debug("failed to sync translog", ex);
+                    throw ex;
+                }
             }
-        }
+        };
     };
 
     /**
@@ -3103,7 +3190,8 @@ assert getActiveOperationsCount() == OPERATIONS_BLOCKED
         flush(new FlushRequest().waitIfOngoing(true));
 
         SetOnce<Engine> newEngineReference = new SetOnce<>();
-        final long globalCheckpoint = getGlobalCheckpoint();
+        final long globalCheckpoint = getLastKnownGlobalCheckpoint();
+        assert globalCheckpoint == getLastSyncedGlobalCheckpoint();
         synchronized (mutex) {
             verifyNotClosed();
             // we must create both new read-only engine and new read-write engine under mutex to ensure snapshotStoreMetadata,
diff --git a/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java b/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java
index 07aade952923b..17ef424185d1f 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/PrimaryReplicaSyncer.java
@@ -84,7 +84,7 @@ void setChunkSize(ByteSizeValue chunkSize) { // only settable for tests
     public void resync(final IndexShard indexShard, final ActionListener<ResyncTask> listener) {
         Translog.Snapshot snapshot = null;
         try {
-            final long startingSeqNo = indexShard.getGlobalCheckpoint() + 1;
+            final long startingSeqNo = indexShard.getLastKnownGlobalCheckpoint() + 1;
             final long maxSeqNo = indexShard.seqNoStats().getMaxSeqNo();
             final ShardId shardId = indexShard.shardId();
             // Wrap translog snapshot to make it synchronized as it is accessed by different threads through SnapshotSender.
diff --git a/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java b/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java
index 16db596515b4c..5fc3ba57980bf 100644
--- a/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java
+++ b/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java
@@ -140,17 +140,14 @@ protected void findAndProcessShardPath(OptionSet options, Environment environmen
                 IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, shardParent);
 
             final String shardIdFileName = path.getFileName().toString();
-            final String nodeIdFileName = shardParentParent.getParent().getFileName().toString();
             if (Files.isDirectory(path) && shardIdFileName.chars().allMatch(Character::isDigit) // SHARD-ID path element check
                 && NodeEnvironment.INDICES_FOLDER.equals(shardParentParent.getFileName().toString()) // `indices` check
-                && nodeIdFileName.chars().allMatch(Character::isDigit) // NODE-ID check
-                && NodeEnvironment.NODES_FOLDER.equals(shardParentParent.getParent().getParent().getFileName().toString()) // `nodes` check
             ) {
                 shardId = Integer.parseInt(shardIdFileName);
                 indexName = indexMetaData.getIndex().getName();
             } else {
                 throw new ElasticsearchException("Unable to resolve shard id. Wrong folder structure at [ " + path.toString()
-                    + " ], expected .../nodes/[NODE-ID]/indices/[INDEX-UUID]/[SHARD-ID]");
+                    + " ], expected .../indices/[INDEX-UUID]/[SHARD-ID]");
             }
         } else {
             // otherwise resolve shardPath based on the index name and shard id
diff --git a/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java b/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
index 57cbc961aacc0..3fe20a1f3b26e 100644
--- a/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
+++ b/server/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
@@ -212,9 +212,8 @@ private static void validateScoresArePositive(Version indexCreatedVersion, Simil
         for (int freq = 1; freq <= 10; ++freq) {
             float score = scorer.score(freq, norm);
             if (score < 0) {
-                fail(indexCreatedVersion, "Similarities should not return negative scores:\n" +
-                        scorer.explain(Explanation.match(freq, "term freq"), norm));
-                break;
+                throw new IllegalArgumentException("Similarities should not return negative scores:\n" +
+                    scorer.explain(Explanation.match(freq, "term freq"), norm));
             }
         }
     }
@@ -230,10 +229,9 @@ private static void validateScoresDoNotDecreaseWithFreq(Version indexCreatedVers
         for (int freq = 1; freq <= 10; ++freq) {
             float score = scorer.score(freq, norm);
             if (score < previousScore) {
-                fail(indexCreatedVersion, "Similarity scores should not decrease when term frequency increases:\n" +
-                        scorer.explain(Explanation.match(freq - 1, "term freq"), norm) + "\n" +
-                        scorer.explain(Explanation.match(freq, "term freq"), norm));
-                break;
+                throw new IllegalArgumentException("Similarity scores should not decrease when term frequency increases:\n" +
+                    scorer.explain(Explanation.match(freq - 1, "term freq"), norm) + "\n" +
+                    scorer.explain(Explanation.match(freq, "term freq"), norm));
             }
             previousScore = score;
         }
@@ -256,22 +254,12 @@ private static void validateScoresDoNotIncreaseWithNorm(Version indexCreatedVers
             }
             float score = scorer.score(1, norm);
             if (score > previousScore) {
-                fail(indexCreatedVersion, "Similarity scores should not increase when norm increases:\n" +
-                        scorer.explain(Explanation.match(1, "term freq"), norm - 1) + "\n" +
-                        scorer.explain(Explanation.match(1, "term freq"), norm));
-                break;
+                throw new IllegalArgumentException("Similarity scores should not increase when norm increases:\n" +
+                    scorer.explain(Explanation.match(1, "term freq"), norm - 1) + "\n" +
+                    scorer.explain(Explanation.match(1, "term freq"), norm));
             }
             previousScore = score;
             previousNorm = norm;
         }
     }
-
-    private static void fail(Version indexCreatedVersion, String message) {
-        if (indexCreatedVersion.onOrAfter(Version.V_7_0_0)) {
-            throw new IllegalArgumentException(message);
-        } else if (indexCreatedVersion.onOrAfter(Version.V_6_5_0)) {
-            deprecationLogger.deprecated(message);
-        }
-    }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java
index 84bb4c49b27d4..310f3de949527 100644
--- a/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java
+++ b/server/src/main/java/org/elasticsearch/index/store/FsDirectoryFactory.java
@@ -22,6 +22,7 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.FileSwitchDirectory;
+import org.apache.lucene.store.FilterDirectory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.LockFactory;
@@ -63,11 +64,7 @@ public Directory newDirectory(IndexSettings indexSettings, ShardPath path) throw
         final Path location = path.resolveIndex();
         final LockFactory lockFactory = indexSettings.getValue(INDEX_LOCK_FACTOR_SETTING);
         Files.createDirectories(location);
-        Directory wrapped = newFSDirectory(location, lockFactory, indexSettings);
-        Set<String> preLoadExtensions = new HashSet<>(
-                indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING));
-        wrapped = setPreload(wrapped, location, lockFactory, preLoadExtensions);
-        return wrapped;
+        return newFSDirectory(location, lockFactory, indexSettings);
     }
 
     protected Directory newFSDirectory(Path location, LockFactory lockFactory, IndexSettings indexSettings) throws IOException {
@@ -79,17 +76,20 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index
         } else {
             type = IndexModule.Type.fromSettingsKey(storeType);
         }
+        Set<String> preLoadExtensions = new HashSet<>(
+            indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING));
         switch (type) {
             case HYBRIDFS:
                 // Use Lucene defaults
                 final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory);
                 if (primaryDirectory instanceof MMapDirectory) {
-                    return new HybridDirectory(location, lockFactory, primaryDirectory);
+                    MMapDirectory mMapDirectory = (MMapDirectory) primaryDirectory;
+                    return new HybridDirectory(lockFactory, setPreload(mMapDirectory, lockFactory, preLoadExtensions));
                 } else {
                     return primaryDirectory;
                 }
             case MMAPFS:
-                return new MMapDirectory(location, lockFactory);
+                return setPreload(new MMapDirectory(location, lockFactory), lockFactory, preLoadExtensions);
             case SIMPLEFS:
                 return new SimpleFSDirectory(location, lockFactory);
             case NIOFS:
@@ -99,38 +99,57 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index
         }
     }
 
-    private static Directory setPreload(Directory directory, Path location, LockFactory lockFactory,
+    public static MMapDirectory setPreload(MMapDirectory mMapDirectory, LockFactory lockFactory,
             Set<String> preLoadExtensions) throws IOException {
-        if (preLoadExtensions.isEmpty() == false
-                && directory instanceof MMapDirectory
-                && ((MMapDirectory) directory).getPreload() == false) {
+        assert mMapDirectory.getPreload() == false;
+        if (preLoadExtensions.isEmpty() == false) {
             if (preLoadExtensions.contains("*")) {
-                ((MMapDirectory) directory).setPreload(true);
-                return directory;
+                mMapDirectory.setPreload(true);
+            } else {
+                return new PreLoadMMapDirectory(mMapDirectory, lockFactory, preLoadExtensions);
             }
-            MMapDirectory primary = new MMapDirectory(location, lockFactory);
-            primary.setPreload(true);
-            return new FileSwitchDirectory(preLoadExtensions, primary, directory, true) {
-                @Override
-                public String[] listAll() throws IOException {
-                    // avoid listing twice
-                    return primary.listAll();
-                }
-            };
         }
-        return directory;
+        return mMapDirectory;
+    }
+
+    /**
+     * Returns true iff the directory is a hybrid fs directory
+     */
+    public static boolean isHybridFs(Directory directory) {
+        Directory unwrap = FilterDirectory.unwrap(directory);
+        return unwrap instanceof HybridDirectory;
     }
 
     static final class HybridDirectory extends NIOFSDirectory {
-        private final FSDirectory randomAccessDirectory;
+        private final MMapDirectory delegate;
 
-        HybridDirectory(Path location, LockFactory lockFactory, FSDirectory randomAccessDirectory) throws IOException {
-            super(location, lockFactory);
-            this.randomAccessDirectory = randomAccessDirectory;
+        HybridDirectory(LockFactory lockFactory, MMapDirectory delegate) throws IOException {
+            super(delegate.getDirectory(), lockFactory);
+            this.delegate = delegate;
         }
 
         @Override
         public IndexInput openInput(String name, IOContext context) throws IOException {
+            if (useDelegate(name)) {
+                // we need to do these checks on the outer directory since the inner doesn't know about pending deletes
+                ensureOpen();
+                ensureCanRead(name);
+                // we only use the mmap to open inputs. Everything else is managed by the NIOFSDirectory otherwise
+                // we might run into trouble with files that are pendingDelete in one directory but still
+                // listed in listAll() from the other. We on the other hand don't want to list files from both dirs
+                // and intersect for perf reasons.
+                return delegate.openInput(name, context);
+            } else {
+                return super.openInput(name, context);
+            }
+        }
+
+        @Override
+        public void close() throws IOException {
+            IOUtils.close(super::close, delegate);
+        }
+
+        boolean useDelegate(String name) {
             String extension = FileSwitchDirectory.getExtension(name);
             switch(extension) {
                 // We are mmapping norms, docvalues as well as term dictionaries, all other files are served through NIOFS
@@ -138,27 +157,61 @@ public IndexInput openInput(String name, IOContext context) throws IOException {
                 case "nvd":
                 case "dvd":
                 case "tim":
+                case "tip":
                 case "cfs":
-                    // we need to do these checks on the outer directory since the inner doesn't know about pending deletes
-                    ensureOpen();
-                    ensureCanRead(name);
-                    // we only use the mmap to open inputs. Everything else is managed by the NIOFSDirectory otherwise
-                    // we might run into trouble with files that are pendingDelete in one directory but still
-                    // listed in listAll() from the other. We on the other hand don't want to list files from both dirs
-                    // and intersect for perf reasons.
-                    return randomAccessDirectory.openInput(name, context);
+                    return true;
                 default:
-                    return super.openInput(name, context);
+                    return false;
+            }
+        }
+
+        MMapDirectory getDelegate() {
+            return delegate;
+        }
+    }
+    // TODO it would be nice to share code between PreLoadMMapDirectory and HybridDirectory but due to the nesting aspect of
+    // directories here makes it tricky. It would be nice to allow MMAPDirectory to pre-load on a per IndexInput basis.
+    static final class PreLoadMMapDirectory extends MMapDirectory {
+        private final MMapDirectory delegate;
+        private final Set<String> preloadExtensions;
+
+        PreLoadMMapDirectory(MMapDirectory delegate, LockFactory lockFactory, Set<String> preload) throws IOException {
+            super(delegate.getDirectory(), lockFactory);
+            super.setPreload(false);
+            this.delegate = delegate;
+            this.delegate.setPreload(true);
+            this.preloadExtensions = preload;
+            assert getPreload() == false;
+        }
+
+        @Override
+        public void setPreload(boolean preload) {
+            throw new IllegalArgumentException("can't set preload on a preload-wrapper");
+        }
+
+        @Override
+        public IndexInput openInput(String name, IOContext context) throws IOException {
+            if (useDelegate(name)) {
+                // we need to do these checks on the outer directory since the inner doesn't know about pending deletes
+                ensureOpen();
+                ensureCanRead(name);
+                return delegate.openInput(name, context);
             }
+            return super.openInput(name, context);
         }
 
         @Override
-        public void close() throws IOException {
-            IOUtils.close(super::close, randomAccessDirectory);
+        public synchronized void close() throws IOException {
+            IOUtils.close(super::close, delegate);
+        }
+
+        boolean useDelegate(String name) {
+            final String extension = FileSwitchDirectory.getExtension(name);
+            return preloadExtensions.contains(extension);
         }
 
-        Directory getRandomAccessDirectory() {
-            return randomAccessDirectory;
+        MMapDirectory getDelegate() {
+            return delegate;
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/store/Store.java b/server/src/main/java/org/elasticsearch/index/store/Store.java
index 5f1f7d23a8c6a..410774114bd78 100644
--- a/server/src/main/java/org/elasticsearch/index/store/Store.java
+++ b/server/src/main/java/org/elasticsearch/index/store/Store.java
@@ -38,7 +38,6 @@
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.BufferedChecksum;
 import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.ByteBufferIndexInput;
 import org.apache.lucene.store.ChecksumIndexInput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FilterDirectory;
@@ -46,7 +45,6 @@
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.Lock;
-import org.apache.lucene.store.RandomAccessInput;
 import org.apache.lucene.store.SimpleFSDirectory;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
@@ -98,7 +96,6 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -137,7 +134,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
      * this by exploiting lucene internals and wrapping the IndexInput in a simple delegate.
      */
     public static final Setting<Boolean> FORCE_RAM_TERM_DICT = Setting.boolSetting("index.force_memory_term_dictionary", false,
-        Property.IndexScope);
+        Property.IndexScope, Property.Deprecated);
     static final String CODEC = "store";
     static final int VERSION_WRITE_THROWABLE= 2; // we write throwable since 2.0
     static final int VERSION_STACK_TRACE = 1; // we write the stack trace too since 1.4.0
@@ -172,8 +169,7 @@ public Store(ShardId shardId, IndexSettings indexSettings, Directory directory,
         final TimeValue refreshInterval = indexSettings.getValue(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING);
         logger.debug("store stats are refreshed with refresh_interval [{}]", refreshInterval);
         ByteSizeCachingDirectory sizeCachingDir = new ByteSizeCachingDirectory(directory, refreshInterval);
-        this.directory = new StoreDirectory(sizeCachingDir, Loggers.getLogger("index.store.deletes", shardId),
-            indexSettings.getValue(FORCE_RAM_TERM_DICT));
+        this.directory = new StoreDirectory(sizeCachingDir, Loggers.getLogger("index.store.deletes", shardId));
         this.shardLock = shardLock;
         this.onClose = onClose;
 
@@ -712,12 +708,10 @@ public int refCount() {
     static final class StoreDirectory extends FilterDirectory {
 
         private final Logger deletesLogger;
-        private final boolean forceRamTermDict;
 
-        StoreDirectory(ByteSizeCachingDirectory delegateDirectory, Logger deletesLogger, boolean forceRamTermDict) {
+        StoreDirectory(ByteSizeCachingDirectory delegateDirectory, Logger deletesLogger) {
             super(delegateDirectory);
             this.deletesLogger = deletesLogger;
-            this.forceRamTermDict = forceRamTermDict;
         }
 
         /** Estimate the cumulative size of all files in this directory in bytes. */
@@ -744,18 +738,6 @@ private void innerClose() throws IOException {
             super.close();
         }
 
-        @Override
-        public IndexInput openInput(String name, IOContext context) throws IOException {
-            IndexInput input = super.openInput(name, context);
-            if (name.endsWith(".tip") || name.endsWith(".cfs")) {
-                // only do this if we are reading cfs or tip file - all other files don't need this.
-                if (forceRamTermDict && input instanceof ByteBufferIndexInput) {
-                    return new DeoptimizingIndexInput(input.toString(), input);
-                }
-            }
-            return input;
-        }
-
         @Override
         public String toString() {
             return "store(" + in.toString() + ")";
@@ -1614,127 +1596,4 @@ private static IndexWriterConfig newIndexWriterConfig() {
                 // we also don't specify a codec here and merges should use the engines for this index
                 .setMergePolicy(NoMergePolicy.INSTANCE);
     }
-
-    /**
-     * see {@link #FORCE_RAM_TERM_DICT} for details
-     */
-    private static final class DeoptimizingIndexInput extends IndexInput {
-
-        private final IndexInput in;
-
-        private DeoptimizingIndexInput(String resourceDescription, IndexInput in) {
-            super(resourceDescription);
-            this.in = in;
-        }
-
-        @Override
-        public IndexInput clone() {
-            return new DeoptimizingIndexInput(toString(), in.clone());
-        }
-
-        @Override
-        public void close() throws IOException {
-            in.close();
-        }
-
-        @Override
-        public long getFilePointer() {
-            return in.getFilePointer();
-        }
-
-        @Override
-        public void seek(long pos) throws IOException {
-            in.seek(pos);
-        }
-
-        @Override
-        public long length() {
-            return in.length();
-        }
-
-        @Override
-        public String toString() {
-            return in.toString();
-        }
-
-        @Override
-        public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
-            return new DeoptimizingIndexInput(sliceDescription, in.slice(sliceDescription, offset, length));
-        }
-
-        @Override
-        public RandomAccessInput randomAccessSlice(long offset, long length) throws IOException {
-            return in.randomAccessSlice(offset, length);
-        }
-
-        @Override
-        public byte readByte() throws IOException {
-            return in.readByte();
-        }
-
-        @Override
-        public void readBytes(byte[] b, int offset, int len) throws IOException {
-            in.readBytes(b, offset, len);
-        }
-
-        @Override
-        public void readBytes(byte[] b, int offset, int len, boolean useBuffer) throws IOException {
-            in.readBytes(b, offset, len, useBuffer);
-        }
-
-        @Override
-        public short readShort() throws IOException {
-            return in.readShort();
-        }
-
-        @Override
-        public int readInt() throws IOException {
-            return in.readInt();
-        }
-
-        @Override
-        public int readVInt() throws IOException {
-            return in.readVInt();
-        }
-
-        @Override
-        public int readZInt() throws IOException {
-            return in.readZInt();
-        }
-
-        @Override
-        public long readLong() throws IOException {
-            return in.readLong();
-        }
-
-        @Override
-        public long readVLong() throws IOException {
-            return in.readVLong();
-        }
-
-        @Override
-        public long readZLong() throws IOException {
-            return in.readZLong();
-        }
-
-        @Override
-        public String readString() throws IOException {
-            return in.readString();
-        }
-
-        @Override
-        public Map<String, String> readMapOfStrings() throws IOException {
-            return in.readMapOfStrings();
-        }
-
-        @Override
-        public Set<String> readSetOfStrings() throws IOException {
-            return in.readSetOfStrings();
-        }
-
-        @Override
-        public void skipBytes(long numBytes) throws IOException {
-            in.skipBytes(numBytes);
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
index b53c3d8da427c..e5809460c0400 100644
--- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
+++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
@@ -80,8 +80,7 @@ public static TermVectorsResponse getTermVectors(IndexShard indexShard, TermVect
 
     static TermVectorsResponse getTermVectors(IndexShard indexShard, TermVectorsRequest request, LongSupplier nanoTimeSupplier) {
         final long startTime = nanoTimeSupplier.getAsLong();
-        final TermVectorsResponse termVectorsResponse = new TermVectorsResponse(indexShard.shardId().getIndex().getName(),
-            request.type(), request.id());
+        final TermVectorsResponse termVectorsResponse = new TermVectorsResponse(indexShard.shardId().getIndex().getName(), request.id());
         final Term uidTerm = new Term(IdFieldMapper.NAME, Uid.encodeId(request.id()));
 
         Fields termVectorsByField = null;
@@ -93,7 +92,7 @@ static TermVectorsResponse getTermVectors(IndexShard indexShard, TermVectorsRequ
             handleFieldWildcards(indexShard, request);
         }
 
-        try (Engine.GetResult get = indexShard.get(new Engine.Get(request.realtime(), false, request.type(),
+        try (Engine.GetResult get = indexShard.get(new Engine.Get(request.realtime(), false, MapperService.SINGLE_MAPPING_NAME,
             request.id(), uidTerm)
                 .version(request.version()).versionType(request.versionType()));
                 Engine.Searcher searcher = indexShard.acquireSearcher("term_vector")) {
@@ -215,7 +214,7 @@ private static Fields addGeneratedTermVectors(IndexShard indexShard, Engine.GetR
         String[] getFields = validFields.toArray(new String[validFields.size() + 1]);
         getFields[getFields.length - 1] = SourceFieldMapper.NAME;
         GetResult getResult = indexShard.getService().get(
-                get, request.id(), request.type(), getFields, null);
+                get, request.id(), MapperService.SINGLE_MAPPING_NAME, getFields, null);
         Fields generatedTermVectors = generateTermVectors(indexShard, getResult.sourceAsMap(), getResult.getFields().values(),
             request.offsets(), request.perFieldAnalyzer(), validFields);
 
@@ -295,7 +294,7 @@ private static Fields generateTermVectors(IndexShard indexShard,
 
     private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request) throws IOException {
         // parse the document, at the moment we do update the mapping, just like percolate
-        ParsedDocument parsedDocument = parseDocument(indexShard, indexShard.shardId().getIndexName(), request.type(), request.doc(),
+        ParsedDocument parsedDocument = parseDocument(indexShard, indexShard.shardId().getIndexName(), request.doc(),
             request.xContentType(), request.routing());
 
         // select the right fields and generate term vectors
@@ -324,12 +323,12 @@ private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVect
                 request.offsets(), request.perFieldAnalyzer(), seenFields);
     }
 
-    private static ParsedDocument parseDocument(IndexShard indexShard, String index, String type, BytesReference doc,
+    private static ParsedDocument parseDocument(IndexShard indexShard, String index, BytesReference doc,
                                                 XContentType xContentType, String routing) {
         MapperService mapperService = indexShard.mapperService();
-        DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(type);
+        DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(MapperService.SINGLE_MAPPING_NAME);
         ParsedDocument parsedDocument = docMapper.getDocumentMapper().parse(
-            new SourceToParse(index, type, "_id_for_tv_api", doc, xContentType, routing));
+                new SourceToParse(index, MapperService.SINGLE_MAPPING_NAME, "_id_for_tv_api", doc, xContentType, routing));
         if (docMapper.getMapping() != null) {
             parsedDocument.addDynamicMappingsUpdate(docMapper.getMapping());
         }
diff --git a/server/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java b/server/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java
index bfd9e31abcd47..8f5884d02758e 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java
@@ -148,7 +148,7 @@ public long getLastModifiedTime() throws IOException {
     }
 
     /**
-     * Reads a single opertation from the given location.
+     * Reads a single operation from the given location.
      */
     Translog.Operation read(Translog.Location location) throws IOException {
         assert location.generation == this.generation : "generation mismatch expected: " + generation + " got: " + location.generation;
diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java
index 7626270b6cdc5..82ad2046c510b 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -63,6 +63,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.LongConsumer;
 import java.util.function.LongSupplier;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -129,6 +130,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
     private final LongSupplier primaryTermSupplier;
     private final String translogUUID;
     private final TranslogDeletionPolicy deletionPolicy;
+    private final LongConsumer persistedSequenceNumberConsumer;
 
     /**
      * Creates a new Translog instance. This method will create a new transaction log unless the given {@link TranslogGeneration} is
@@ -146,14 +148,18 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
      *                                 examined and stored in the header whenever a new generation is rolled. It's guaranteed from outside
      *                                 that a new generation is rolled when the term is increased. This guarantee allows to us to validate
      *                                 and reject operation whose term is higher than the primary term stored in the translog header.
+     * @param persistedSequenceNumberConsumer a callback that's called whenever an operation with a given sequence number is successfully
+     *                                        persisted.
      */
     public Translog(
         final TranslogConfig config, final String translogUUID, TranslogDeletionPolicy deletionPolicy,
-        final LongSupplier globalCheckpointSupplier, final LongSupplier primaryTermSupplier) throws IOException {
+        final LongSupplier globalCheckpointSupplier, final LongSupplier primaryTermSupplier,
+        final LongConsumer persistedSequenceNumberConsumer) throws IOException {
         super(config.getShardId(), config.getIndexSettings());
         this.config = config;
         this.globalCheckpointSupplier = globalCheckpointSupplier;
         this.primaryTermSupplier = primaryTermSupplier;
+        this.persistedSequenceNumberConsumer = persistedSequenceNumberConsumer;
         this.deletionPolicy = deletionPolicy;
         this.translogUUID = translogUUID;
         bigArrays = config.getBigArrays();
@@ -190,7 +196,8 @@ public Translog(
             boolean success = false;
             current = null;
             try {
-                current = createWriter(checkpoint.generation + 1, getMinFileGeneration(), checkpoint.globalCheckpoint);
+                current = createWriter(checkpoint.generation + 1, getMinFileGeneration(), checkpoint.globalCheckpoint,
+                    persistedSequenceNumberConsumer);
                 success = true;
             } finally {
                 // we have to close all the recovered ones otherwise we leak file handles here
@@ -471,7 +478,8 @@ public long sizeInBytesByMinGen(long minGeneration) {
      * @throws IOException if creating the translog failed
      */
     TranslogWriter createWriter(long fileGeneration) throws IOException {
-        final TranslogWriter writer = createWriter(fileGeneration, getMinFileGeneration(), globalCheckpointSupplier.getAsLong());
+        final TranslogWriter writer = createWriter(fileGeneration, getMinFileGeneration(), globalCheckpointSupplier.getAsLong(),
+            persistedSequenceNumberConsumer);
         assert writer.sizeInBytes() == DEFAULT_HEADER_SIZE_IN_BYTES : "Mismatch translog header size; " +
             "empty translog size [" + writer.sizeInBytes() + ", header size [" + DEFAULT_HEADER_SIZE_IN_BYTES + "]";
         return writer;
@@ -486,7 +494,8 @@ TranslogWriter createWriter(long fileGeneration) throws IOException {
      *                                With no readers and no current, a call to  {@link #getMinFileGeneration()} would not work.
      * @param initialGlobalCheckpoint the global checkpoint to be written in the first checkpoint.
      */
-    TranslogWriter createWriter(long fileGeneration, long initialMinTranslogGen, long initialGlobalCheckpoint) throws IOException {
+    TranslogWriter createWriter(long fileGeneration, long initialMinTranslogGen, long initialGlobalCheckpoint,
+                                LongConsumer persistedSequenceNumberConsumer) throws IOException {
         final TranslogWriter newFile;
         try {
             newFile = TranslogWriter.create(
@@ -497,7 +506,8 @@ TranslogWriter createWriter(long fileGeneration, long initialMinTranslogGen, lon
                 getChannelFactory(),
                 config.getBufferSize(),
                 initialMinTranslogGen, initialGlobalCheckpoint,
-                globalCheckpointSupplier, this::getMinFileGeneration, primaryTermSupplier.getAsLong(), tragedy);
+                globalCheckpointSupplier, this::getMinFileGeneration, primaryTermSupplier.getAsLong(), tragedy,
+                persistedSequenceNumberConsumer);
         } catch (final IOException e) {
             throw new TranslogException(shardId, "failed to create new translog file", e);
         }
@@ -1868,7 +1878,7 @@ static String createEmptyTranslog(Path location, long initialGlobalCheckpoint, S
             location.resolve(getFilename(1)), channelFactory,
             new ByteSizeValue(10), 1, initialGlobalCheckpoint,
             () -> { throw new UnsupportedOperationException(); }, () -> { throw new UnsupportedOperationException(); }, primaryTerm,
-                new TragicExceptionHolder());
+                new TragicExceptionHolder(), seqNo -> { throw new UnsupportedOperationException(); });
         writer.close();
         return translogUUID;
     }
diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java
index d47f0d9d7fdfd..7f95b0ad9871a 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogHeader.java
@@ -30,12 +30,11 @@
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
 
+import java.io.EOFException;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
 import java.nio.file.Path;
 
-import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM;
-
 /**
  * Each translog file is started with a translog header then followed by translog operations.
  */
@@ -108,56 +107,57 @@ private static int headerSizeInBytes(int version, int uuidLength) {
      * Read a translog header from the given path and file channel
      */
     static TranslogHeader read(final String translogUUID, final Path path, final FileChannel channel) throws IOException {
-        // This input is intentionally not closed because closing it will close the FileChannel.
-        final BufferedChecksumStreamInput in =
-            new BufferedChecksumStreamInput(
+        try {
+            // This input is intentionally not closed because closing it will close the FileChannel.
+            final BufferedChecksumStreamInput in =
+                new BufferedChecksumStreamInput(
                     new InputStreamStreamInput(java.nio.channels.Channels.newInputStream(channel), channel.size()),
                     path.toString());
-        final int version;
-        try {
-            version = CodecUtil.checkHeader(new InputStreamDataInput(in), TRANSLOG_CODEC, VERSION_CHECKSUMS, VERSION_PRIMARY_TERM);
-        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) {
-            tryReportOldVersionError(path, channel);
-            throw new TranslogCorruptedException(path.toString(), "translog header corrupted", e);
-        }
-        if (version == VERSION_CHECKSUMS) {
-            throw new IllegalStateException("pre-2.0 translog found [" + path + "]");
-        }
-        // Read the translogUUID
-        final int uuidLen = in.readInt();
-        if (uuidLen > channel.size()) {
-            throw new TranslogCorruptedException(
-                    path.toString(),
-                    "UUID length can't be larger than the translog");
-        }
-        final BytesRef uuid = new BytesRef(uuidLen);
-        uuid.length = uuidLen;
-        in.read(uuid.bytes, uuid.offset, uuid.length);
-        final BytesRef expectedUUID = new BytesRef(translogUUID);
-        if (uuid.bytesEquals(expectedUUID) == false) {
-            throw new TranslogCorruptedException(
+            final int version;
+            try {
+                version = CodecUtil.checkHeader(new InputStreamDataInput(in), TRANSLOG_CODEC, VERSION_CHECKSUMS, VERSION_PRIMARY_TERM);
+            } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) {
+                tryReportOldVersionError(path, channel);
+                throw new TranslogCorruptedException(path.toString(), "translog header corrupted", e);
+            }
+            if (version == VERSION_CHECKSUMS) {
+                throw new IllegalStateException("pre-2.0 translog found [" + path + "]");
+            }
+            if (version == VERSION_CHECKPOINTS) {
+                throw new IllegalStateException("pre-6.3 translog found [" + path + "]");
+            }
+            // Read the translogUUID
+            final int uuidLen = in.readInt();
+            if (uuidLen > channel.size()) {
+                throw new TranslogCorruptedException(path.toString(), "UUID length can't be larger than the translog");
+            }
+            if (uuidLen <= 0) {
+                throw new TranslogCorruptedException(path.toString(), "UUID length must be positive");
+            }
+            final BytesRef uuid = new BytesRef(uuidLen);
+            uuid.length = uuidLen;
+            in.read(uuid.bytes, uuid.offset, uuid.length);
+            final BytesRef expectedUUID = new BytesRef(translogUUID);
+            if (uuid.bytesEquals(expectedUUID) == false) {
+                throw new TranslogCorruptedException(
                     path.toString(),
                     "expected shard UUID " + expectedUUID + " but got: " + uuid +
-                            " this translog file belongs to a different translog");
-        }
-        // Read the primary term
-        final long primaryTerm;
-        if (version == VERSION_PRIMARY_TERM) {
-            primaryTerm = in.readLong();
-        } else {
-            assert version == VERSION_CHECKPOINTS : "Unknown header version [" + version + "]";
-            primaryTerm = UNASSIGNED_PRIMARY_TERM;
-        }
-        // Verify the checksum
-        if (version >= VERSION_PRIMARY_TERM) {
+                        " this translog file belongs to a different translog");
+            }
+            // Read the primary term
+            assert version == VERSION_PRIMARY_TERM;
+            final long primaryTerm = in.readLong();
+            // Verify the checksum
             Translog.verifyChecksum(in);
+            assert primaryTerm >= 0 : "Primary term must be non-negative [" + primaryTerm + "]; translog path [" + path + "]";
+
+            final int headerSizeInBytes = headerSizeInBytes(version, uuid.length);
+            assert channel.position() == headerSizeInBytes :
+                "Header is not fully read; header size [" + headerSizeInBytes + "], position [" + channel.position() + "]";
+            return new TranslogHeader(translogUUID, primaryTerm, headerSizeInBytes);
+        } catch (EOFException e) {
+            throw new TranslogCorruptedException(path.toString(), "translog header truncated", e);
         }
-        assert primaryTerm >= 0 : "Primary term must be non-negative [" + primaryTerm + "]; translog path [" + path + "]";
-
-        final int headerSizeInBytes = headerSizeInBytes(version, uuid.length);
-        assert channel.position() == headerSizeInBytes :
-            "Header is not fully read; header size [" + headerSizeInBytes + "], position [" + channel.position() + "]";
-        return new TranslogHeader(translogUUID, primaryTerm, headerSizeInBytes);
     }
 
     private static void tryReportOldVersionError(final Path path, final FileChannel channel) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java
index bff3e4eb2f540..4cfc886219f60 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java
@@ -76,7 +76,7 @@ public Translog.Operation next() throws IOException {
         return null;
     }
 
-    protected Translog.Operation readOperation() throws IOException {
+    private Translog.Operation readOperation() throws IOException {
         final int opSize = readSize(reusableBuffer, position);
         reuse = checksummedStream(reusableBuffer, position, opSize, reuse);
         Translog.Operation op = read(reuse);
@@ -93,15 +93,19 @@ public long sizeInBytes() {
      * reads an operation at the given position into the given buffer.
      */
     protected void readBytes(ByteBuffer buffer, long position) throws IOException {
-        if (position >= length) {
-            throw new EOFException("read requested past EOF. pos [" + position + "] end: [" + length + "], generation: [" +
-                getGeneration() + "], path: [" + path + "]");
-        }
-        if (position < getFirstOperationOffset()) {
-            throw new IOException("read requested before position of first ops. pos [" + position + "] first op on: [" +
-                getFirstOperationOffset() + "], generation: [" + getGeneration() + "], path: [" + path + "]");
+        try {
+            if (position >= length) {
+                throw new EOFException("read requested past EOF. pos [" + position + "] end: [" + length + "], generation: [" +
+                    getGeneration() + "], path: [" + path + "]");
+            }
+            if (position < getFirstOperationOffset()) {
+                throw new IOException("read requested before position of first ops. pos [" + position + "] first op on: [" +
+                    getFirstOperationOffset() + "], generation: [" + getGeneration() + "], path: [" + path + "]");
+            }
+            Channels.readFromFileChannelWithEofException(channel, position, buffer);
+        } catch (EOFException e) {
+            throw new TranslogCorruptedException(path.toString(), "translog truncated", e);
         }
-        Channels.readFromFileChannelWithEofException(channel, position, buffer);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
index 6b00b0c5db3ff..0695a2bf65010 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.index.translog;
 
+import com.carrotsearch.hppc.LongArrayList;
+import com.carrotsearch.hppc.procedures.LongProcedure;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.Assertions;
@@ -42,6 +44,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.LongConsumer;
 import java.util.function.LongSupplier;
 
 public class TranslogWriter extends BaseTranslogReader implements Closeable {
@@ -64,10 +67,15 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
     private final LongSupplier globalCheckpointSupplier;
     private final LongSupplier minTranslogGenerationSupplier;
 
+    // callback that's called whenever an operation with a given sequence number is successfully persisted.
+    private final LongConsumer persistedSequenceNumberConsumer;
+
     protected final AtomicBoolean closed = new AtomicBoolean(false);
     // lock order synchronized(syncLock) -> synchronized(this)
     private final Object syncLock = new Object();
 
+    private LongArrayList nonFsyncedSequenceNumbers;
+
     private final Map<Long, Tuple<BytesReference, Exception>> seenSequenceNumbers;
 
     private TranslogWriter(
@@ -78,7 +86,8 @@ private TranslogWriter(
         final Path path,
         final ByteSizeValue bufferSize,
         final LongSupplier globalCheckpointSupplier, LongSupplier minTranslogGenerationSupplier, TranslogHeader header,
-        TragicExceptionHolder tragedy)
+        TragicExceptionHolder tragedy,
+        final LongConsumer persistedSequenceNumberConsumer)
             throws
             IOException {
         super(initialCheckpoint.generation, channel, path, header);
@@ -97,6 +106,8 @@ private TranslogWriter(
         this.maxSeqNo = initialCheckpoint.maxSeqNo;
         assert initialCheckpoint.trimmedAboveSeqNo == SequenceNumbers.UNASSIGNED_SEQ_NO : initialCheckpoint.trimmedAboveSeqNo;
         this.globalCheckpointSupplier = globalCheckpointSupplier;
+        this.nonFsyncedSequenceNumbers = new LongArrayList(64);
+        this.persistedSequenceNumberConsumer = persistedSequenceNumberConsumer;
         this.seenSequenceNumbers = Assertions.ENABLED ? new HashMap<>() : null;
         this.tragedy = tragedy;
     }
@@ -104,7 +115,7 @@ private TranslogWriter(
     public static TranslogWriter create(ShardId shardId, String translogUUID, long fileGeneration, Path file, ChannelFactory channelFactory,
                                         ByteSizeValue bufferSize, final long initialMinTranslogGen, long initialGlobalCheckpoint,
                                         final LongSupplier globalCheckpointSupplier, final LongSupplier minTranslogGenerationSupplier,
-                                        final long primaryTerm, TragicExceptionHolder tragedy)
+                                        final long primaryTerm, TragicExceptionHolder tragedy, LongConsumer persistedSequenceNumberConsumer)
         throws IOException {
         final FileChannel channel = channelFactory.open(file);
         try {
@@ -125,7 +136,7 @@ public static TranslogWriter create(ShardId shardId, String translogUUID, long f
                 writerGlobalCheckpointSupplier = globalCheckpointSupplier;
             }
             return new TranslogWriter(channelFactory, shardId, checkpoint, channel, file, bufferSize,
-                writerGlobalCheckpointSupplier, minTranslogGenerationSupplier, header, tragedy);
+                writerGlobalCheckpointSupplier, minTranslogGenerationSupplier, header, tragedy, persistedSequenceNumberConsumer);
         } catch (Exception exception) {
             // if we fail to bake the file-generation into the checkpoint we stick with the file and once we recover and that
             // file exists we remove it. We only apply this logic to the checkpoint.generation+1 any other file with a higher generation
@@ -177,6 +188,8 @@ public synchronized Translog.Location add(final BytesReference data, final long
         minSeqNo = SequenceNumbers.min(minSeqNo, seqNo);
         maxSeqNo = SequenceNumbers.max(maxSeqNo, seqNo);
 
+        nonFsyncedSequenceNumbers.add(seqNo);
+
         operationCounter++;
 
         assert assertNoSeqNumberConflict(seqNo, data);
@@ -338,7 +351,9 @@ private long getWrittenOffset() throws IOException {
      * @return <code>true</code> if this call caused an actual sync operation
      */
     public boolean syncUpTo(long offset) throws IOException {
+        boolean synced = false;
         if (lastSyncedCheckpoint.offset < offset && syncNeeded()) {
+            LongArrayList flushedSequenceNumbers = null;
             synchronized (syncLock) { // only one sync/checkpoint should happen concurrently but we wait
                 if (lastSyncedCheckpoint.offset < offset && syncNeeded()) {
                     // double checked locking - we don't want to fsync unless we have to and now that we have
@@ -349,6 +364,8 @@ public boolean syncUpTo(long offset) throws IOException {
                         try {
                             outputStream.flush();
                             checkpointToSync = getCheckpoint();
+                            flushedSequenceNumbers = nonFsyncedSequenceNumbers;
+                            nonFsyncedSequenceNumbers = new LongArrayList(64);
                         } catch (final Exception ex) {
                             closeWithTragicEvent(ex);
                             throw ex;
@@ -366,11 +383,14 @@ public boolean syncUpTo(long offset) throws IOException {
                     assert lastSyncedCheckpoint.offset <= checkpointToSync.offset :
                         "illegal state: " + lastSyncedCheckpoint.offset + " <= " + checkpointToSync.offset;
                     lastSyncedCheckpoint = checkpointToSync; // write protected by syncLock
-                    return true;
+                    synced = true;
                 }
             }
+            if (flushedSequenceNumbers != null) {
+                flushedSequenceNumbers.forEach((LongProcedure) persistedSequenceNumberConsumer::accept);
+            }
         }
-        return false;
+        return synced;
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java b/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java
index e99128fd3e0a2..7cf165a5b112d 100644
--- a/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java
+++ b/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java
@@ -168,7 +168,6 @@ public void execute(Terminal terminal, ShardPath shardPath, Directory indexDirec
 
     private boolean isTranslogClean(ShardPath shardPath, String translogUUID) throws IOException {
         // perform clean check of translog instead of corrupted marker file
-        boolean clean = true;
         try {
             final Path translogPath = shardPath.resolveTranslog();
             final long translogGlobalCheckpoint = Translog.readGlobalCheckpoint(translogPath, translogUUID);
@@ -182,20 +181,21 @@ private boolean isTranslogClean(ShardPath shardPath, String translogUUID) throws
                 new TranslogDeletionPolicy(indexSettings.getTranslogRetentionSize().getBytes(),
                     indexSettings.getTranslogRetentionAge().getMillis());
             try (Translog translog = new Translog(translogConfig, translogUUID,
-                translogDeletionPolicy, () -> translogGlobalCheckpoint, () -> primaryTerm);
+                translogDeletionPolicy, () -> translogGlobalCheckpoint, () -> primaryTerm, seqNo -> {});
                  Translog.Snapshot snapshot = translog.newSnapshot()) {
+                //noinspection StatementWithEmptyBody we are just checking that we can iterate through the whole snapshot
                 while (snapshot.next() != null) {
-                    // just iterate over snapshot
                 }
             }
+            return true;
         } catch (TranslogCorruptedException e) {
-            clean = false;
+            return false;
         }
-        return clean;
     }
 
     /** Write a checkpoint file to the given location with the given generation */
-    static void writeEmptyCheckpoint(Path filename, int translogLength, long translogGeneration, long globalCheckpoint) throws IOException {
+    private static void writeEmptyCheckpoint(Path filename, int translogLength, long translogGeneration, long globalCheckpoint)
+            throws IOException {
         Checkpoint emptyCheckpoint = Checkpoint.emptyTranslogCheckpoint(translogLength, translogGeneration,
             globalCheckpoint, translogGeneration);
         Checkpoint.write(FileChannel::open, filename, emptyCheckpoint,
@@ -234,7 +234,7 @@ private String deletingFilesDetails(Path translogPath, Set<Path> files) {
     }
 
     /** Return a Set of all files in a given directory */
-    public static Set<Path> filesInDirectory(Path directory) throws IOException {
+    private static Set<Path> filesInDirectory(Path directory) throws IOException {
         Set<Path> files = new TreeSet<>();
         try (DirectoryStream<Path> stream = Files.newDirectoryStream(directory)) {
             for (Path file : stream) {
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
index bef05ecda9fd8..fa1fa35588ec0 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java
@@ -55,7 +55,6 @@
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.mapper.TypeFieldMapper;
 import org.elasticsearch.index.mapper.VersionFieldMapper;
-import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction;
 import org.elasticsearch.index.shard.PrimaryReplicaSyncer;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.flush.SyncedFlushService;
@@ -244,7 +243,6 @@ protected void configure() {
         bind(IndicesClusterStateService.class).asEagerSingleton();
         bind(SyncedFlushService.class).asEagerSingleton();
         bind(TransportNodesListShardStoreMetaData.class).asEagerSingleton();
-        bind(GlobalCheckpointSyncAction.class).asEagerSingleton();
         bind(TransportResyncReplicationAction.class).asEagerSingleton();
         bind(PrimaryReplicaSyncer.class).asEagerSingleton();
     }
diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java
index 16382d15cd325..79f4de40aeafc 100644
--- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -66,7 +66,6 @@
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -1220,13 +1219,7 @@ public void run() {
             }
             // Reschedule itself to run again if not closed
             if (closed.get() == false) {
-                try {
-                    threadPool.schedule(this, interval, ThreadPool.Names.SAME);
-                } catch (EsRejectedExecutionException e) {
-                    if (closed.get() == false) {
-                        throw e;
-                    }
-                }
+                threadPool.scheduleUnlessShuttingDown(interval, ThreadPool.Names.SAME, this);
             }
         }
 
diff --git a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index 1ce39e283b1da..25a74d8e7773a 100644
--- a/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/server/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -22,10 +22,13 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.LockObtainFailedException;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateApplier;
@@ -47,6 +50,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.env.ShardLockObtainFailedException;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.Index;
@@ -61,6 +65,7 @@
 import org.elasticsearch.index.seqno.RetentionLeases;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.IndexShardClosedException;
 import org.elasticsearch.index.shard.IndexShardRelocatedException;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.PrimaryReplicaSyncer;
@@ -99,7 +104,7 @@
 import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;
 import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.REOPENED;
 
-public class IndicesClusterStateService extends AbstractLifecycleComponent implements ClusterStateApplier {
+public class IndicesClusterStateService extends AbstractLifecycleComponent implements ClusterStateApplier, RetentionLeaseSyncer {
     private static final Logger logger = LogManager.getLogger(IndicesClusterStateService.class);
 
     final AllocatedIndices<? extends Shard, ? extends AllocatedIndex<? extends Shard>> indicesService;
@@ -122,8 +127,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
     private final boolean sendRefreshMapping;
     private final List<IndexEventListener> buildInIndexListener;
     private final PrimaryReplicaSyncer primaryReplicaSyncer;
-    private final Consumer<ShardId> globalCheckpointSyncer;
-    private final RetentionLeaseSyncer retentionLeaseSyncer;
+    private final NodeClient client;
 
     @Inject
     public IndicesClusterStateService(
@@ -140,12 +144,10 @@ public IndicesClusterStateService(
             final PeerRecoverySourceService peerRecoverySourceService,
             final SnapshotShardsService snapshotShardsService,
             final PrimaryReplicaSyncer primaryReplicaSyncer,
-            final GlobalCheckpointSyncAction globalCheckpointSyncAction,
-            final RetentionLeaseSyncAction retentionLeaseSyncAction,
-            final RetentionLeaseBackgroundSyncAction retentionLeaseBackgroundSyncAction) {
+            final NodeClient client) {
         this(
                 settings,
-                indicesService,
+                (AllocatedIndices<? extends Shard, ? extends AllocatedIndex<? extends Shard>>) indicesService,
                 clusterService,
                 threadPool,
                 recoveryTargetService,
@@ -157,21 +159,7 @@ public IndicesClusterStateService(
                 peerRecoverySourceService,
                 snapshotShardsService,
                 primaryReplicaSyncer,
-                globalCheckpointSyncAction::updateGlobalCheckpointForShard,
-                new RetentionLeaseSyncer() {
-                    @Override
-                    public void sync(
-                            final ShardId shardId,
-                            final RetentionLeases retentionLeases,
-                            final ActionListener<ReplicationResponse> listener) {
-                        Objects.requireNonNull(retentionLeaseSyncAction).sync(shardId, retentionLeases, listener);
-                    }
-
-                    @Override
-                    public void backgroundSync(final ShardId shardId, final RetentionLeases retentionLeases) {
-                        Objects.requireNonNull(retentionLeaseBackgroundSyncAction).backgroundSync(shardId, retentionLeases);
-                    }
-                });
+                client);
     }
 
     // for tests
@@ -189,8 +177,7 @@ public void backgroundSync(final ShardId shardId, final RetentionLeases retentio
             final PeerRecoverySourceService peerRecoverySourceService,
             final SnapshotShardsService snapshotShardsService,
             final PrimaryReplicaSyncer primaryReplicaSyncer,
-            final Consumer<ShardId> globalCheckpointSyncer,
-            final RetentionLeaseSyncer retentionLeaseSyncer) {
+            final NodeClient client) {
         this.settings = settings;
         this.buildInIndexListener =
                 Arrays.asList(
@@ -207,9 +194,8 @@ public void backgroundSync(final ShardId shardId, final RetentionLeases retentio
         this.nodeMappingRefreshAction = nodeMappingRefreshAction;
         this.repositoriesService = repositoriesService;
         this.primaryReplicaSyncer = primaryReplicaSyncer;
-        this.globalCheckpointSyncer = globalCheckpointSyncer;
-        this.retentionLeaseSyncer = Objects.requireNonNull(retentionLeaseSyncer);
         this.sendRefreshMapping = settings.getAsBoolean("indices.cluster.send_refresh_mapping", true);
+        this.client = client;
     }
 
     @Override
@@ -299,6 +285,74 @@ private void updateFailedShardsCache(final ClusterState state) {
         }
     }
 
+    // overrideable by tests
+    protected void updateGlobalCheckpointForShard(final ShardId shardId) {
+        final ThreadContext threadContext = threadPool.getThreadContext();
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+            threadContext.markAsSystemContext();
+            client.executeLocally(GlobalCheckpointSyncAction.TYPE, new GlobalCheckpointSyncAction.Request(shardId),
+                ActionListener.wrap(r -> {
+                }, e -> {
+                    if (ExceptionsHelper.unwrap(e, AlreadyClosedException.class, IndexShardClosedException.class) == null) {
+                        getLogger().info(new ParameterizedMessage("{} global checkpoint sync failed", shardId), e);
+                    }
+                }));
+        }
+    }
+
+    @Override
+    public void sync(ShardId shardId, RetentionLeases retentionLeases, ActionListener<ReplicationResponse> listener) {
+        Objects.requireNonNull(shardId);
+        Objects.requireNonNull(retentionLeases);
+        Objects.requireNonNull(listener);
+        final ThreadContext threadContext = threadPool.getThreadContext();
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+            // we have to execute under the system context so that if security is enabled the sync is authorized
+            threadContext.markAsSystemContext();
+            client.executeLocally(RetentionLeaseSyncAction.TYPE,
+                new RetentionLeaseSyncAction.Request(shardId, retentionLeases),
+                ActionListener.wrap(
+                    listener::onResponse,
+                    e -> {
+                        if (ExceptionsHelper.unwrap(e, AlreadyClosedException.class, IndexShardClosedException.class) == null) {
+                            getLogger().warn(new ParameterizedMessage("{} retention lease sync failed", shardId), e);
+                        }
+                        listener.onFailure(e);
+                    }));
+        }
+    }
+
+    @Override
+    public void backgroundSync(ShardId shardId, RetentionLeases retentionLeases) {
+        Objects.requireNonNull(shardId);
+        Objects.requireNonNull(retentionLeases);
+        final ThreadContext threadContext = threadPool.getThreadContext();
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+            // we have to execute under the system context so that if security is enabled the sync is authorized
+            threadContext.markAsSystemContext();
+            client.executeLocally(RetentionLeaseBackgroundSyncAction.TYPE,
+                new RetentionLeaseBackgroundSyncAction.Request(shardId, retentionLeases),
+                ActionListener.wrap(
+                    r -> {},
+                    e -> {
+                        if (ExceptionsHelper.isTransportStoppedForAction(e, RetentionLeaseBackgroundSyncAction.ACTION_NAME + "[p]")) {
+                            // we are likely shutting down
+                            return;
+                        }
+                        if (ExceptionsHelper.unwrap(e, AlreadyClosedException.class, IndexShardClosedException.class) != null) {
+                            // the shard is closed
+                            return;
+                        }
+                        getLogger().warn(new ParameterizedMessage("{} retention lease background sync failed", shardId), e);
+                    }));
+        }
+    }
+
+    // overrideable by tests
+    Logger getLogger() {
+        return logger;
+    }
+
     /**
      * Deletes indices (with shard data).
      *
@@ -606,8 +660,8 @@ private void createShard(DiscoveryNodes nodes, RoutingTable routingTable, ShardR
                     new RecoveryListener(shardRouting, primaryTerm),
                     repositoriesService,
                     failedShardHandler,
-                    globalCheckpointSyncer,
-                    retentionLeaseSyncer);
+                    this::updateGlobalCheckpointForShard,
+                    this);
         } catch (Exception e) {
             failAndRemoveShard(shardRouting, true, "failed to create shard", e, state);
         }
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
index 6b1a893667f2c..92f1558d71eca 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
@@ -531,11 +531,11 @@ class FilesInfoRequestHandler implements TransportRequestHandler<RecoveryFilesIn
 
         @Override
         public void messageReceived(RecoveryFilesInfoRequest request, TransportChannel channel, Task task) throws Exception {
-            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
-            )) {
-                recoveryRef.target().receiveFileInfo(request.phase1FileNames, request.phase1FileSizes, request.phase1ExistingFileNames,
-                        request.phase1ExistingFileSizes, request.totalTranslogOps);
-                channel.sendResponse(TransportResponse.Empty.INSTANCE);
+            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId())) {
+                final ActionListener<TransportResponse> listener = new ChannelActionListener<>(channel, Actions.FILES_INFO, request);
+                recoveryRef.target().receiveFileInfo(
+                    request.phase1FileNames, request.phase1FileSizes, request.phase1ExistingFileNames, request.phase1ExistingFileSizes,
+                    request.totalTranslogOps, ActionListener.map(listener, nullVal -> TransportResponse.Empty.INSTANCE));
             }
         }
     }
@@ -544,10 +544,10 @@ class CleanFilesRequestHandler implements TransportRequestHandler<RecoveryCleanF
 
         @Override
         public void messageReceived(RecoveryCleanFilesRequest request, TransportChannel channel, Task task) throws Exception {
-            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
-            )) {
-                recoveryRef.target().cleanFiles(request.totalTranslogOps(), request.getGlobalCheckpoint(), request.sourceMetaSnapshot());
-                channel.sendResponse(TransportResponse.Empty.INSTANCE);
+            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId())) {
+                final ActionListener<TransportResponse> listener = new ChannelActionListener<>(channel, Actions.CLEAN_FILES, request);
+                recoveryRef.target().cleanFiles(request.totalTranslogOps(), request.getGlobalCheckpoint(), request.sourceMetaSnapshot(),
+                    ActionListener.map(listener, nullVal -> TransportResponse.Empty.INSTANCE));
             }
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
index 4b89e75691a76..3ae9598124b05 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -74,7 +75,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
-import java.util.function.Supplier;
+import java.util.function.IntSupplier;
 import java.util.stream.StreamSupport;
 
 import static org.elasticsearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED;
@@ -158,15 +159,21 @@ public void recoverToTarget(ActionListener<RecoveryResponse> listener) {
             final long startingSeqNo;
             final boolean isSequenceNumberBasedRecovery = request.startingSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO &&
                 isTargetSameHistory() && shard.hasCompleteHistoryOperations("peer-recovery", request.startingSeqNo());
-            final SendFileResult sendFileResult;
+
+            final StepListener<SendFileResult> sendFileStep = new StepListener<>();
+            final StepListener<TimeValue> prepareEngineStep = new StepListener<>();
+            final StepListener<SendSnapshotResult> sendSnapshotStep = new StepListener<>();
+            final StepListener<Void> finalizeStep = new StepListener<>();
+
             if (isSequenceNumberBasedRecovery) {
                 logger.trace("performing sequence numbers based recovery. starting at [{}]", request.startingSeqNo());
                 startingSeqNo = request.startingSeqNo();
-                sendFileResult = SendFileResult.EMPTY;
+                sendFileStep.onResponse(SendFileResult.EMPTY);
             } else {
-                final Engine.IndexCommitRef phase1Snapshot;
+                final Engine.IndexCommitRef safeCommitRef;
                 try {
-                    phase1Snapshot = shard.acquireSafeIndexCommit();
+                    safeCommitRef = shard.acquireSafeIndexCommit();
+                    resources.add(safeCommitRef);
                 } catch (final Exception e) {
                     throw new RecoveryEngineException(shard.shardId(), 1, "snapshot failed", e);
                 }
@@ -175,24 +182,29 @@ public void recoverToTarget(ActionListener<RecoveryResponse> listener) {
                 startingSeqNo = 0;
                 try {
                     final int estimateNumOps = shard.estimateNumberOfHistoryOperations("peer-recovery", startingSeqNo);
-                    sendFileResult = phase1(phase1Snapshot.getIndexCommit(), shard.getGlobalCheckpoint(), () -> estimateNumOps);
+                    shard.store().incRef();
+                    final Releasable releaseStore = Releasables.releaseOnce(shard.store()::decRef);
+                    resources.add(releaseStore);
+                    sendFileStep.whenComplete(r -> IOUtils.close(safeCommitRef, releaseStore), e -> {
+                        try {
+                            IOUtils.close(safeCommitRef, releaseStore);
+                        } catch (final IOException ex) {
+                            logger.warn("releasing snapshot caused exception", ex);
+                        }
+                    });
+                    phase1(safeCommitRef.getIndexCommit(), shard.getLastKnownGlobalCheckpoint(), () -> estimateNumOps, sendFileStep);
                 } catch (final Exception e) {
-                    throw new RecoveryEngineException(shard.shardId(), 1, "phase1 failed", e);
-                } finally {
-                    try {
-                        IOUtils.close(phase1Snapshot);
-                    } catch (final IOException ex) {
-                        logger.warn("releasing snapshot caused exception", ex);
-                    }
+                    throw new RecoveryEngineException(shard.shardId(), 1, "sendFileStep failed", e);
                 }
             }
             assert startingSeqNo >= 0 : "startingSeqNo must be non negative. got: " + startingSeqNo;
 
-            final StepListener<TimeValue> prepareEngineStep = new StepListener<>();
-            // For a sequence based recovery, the target can keep its local translog
-            prepareTargetForTranslog(isSequenceNumberBasedRecovery == false,
-                shard.estimateNumberOfHistoryOperations("peer-recovery", startingSeqNo), prepareEngineStep);
-            final StepListener<SendSnapshotResult> sendSnapshotStep = new StepListener<>();
+            sendFileStep.whenComplete(r -> {
+                // For a sequence based recovery, the target can keep its local translog
+                prepareTargetForTranslog(isSequenceNumberBasedRecovery == false,
+                    shard.estimateNumberOfHistoryOperations("peer-recovery", startingSeqNo), prepareEngineStep);
+            }, onFailure);
+
             prepareEngineStep.whenComplete(prepareEngineTime -> {
                 /*
                  * add shard to replication group (shard will receive replication requests from this point on) now that engine is open.
@@ -229,12 +241,12 @@ public void recoverToTarget(ActionListener<RecoveryResponse> listener) {
 
             }, onFailure);
 
-            final StepListener<Void> finalizeStep = new StepListener<>();
             sendSnapshotStep.whenComplete(r -> finalizeRecovery(r.targetLocalCheckpoint, finalizeStep), onFailure);
 
             finalizeStep.whenComplete(r -> {
                 final long phase1ThrottlingWaitTime = 0L; // TODO: return the actual throttle time
                 final SendSnapshotResult sendSnapshotResult = sendSnapshotStep.result();
+                final SendFileResult sendFileResult = sendFileStep.result();
                 final RecoveryResponse response = new RecoveryResponse(sendFileResult.phase1FileNames, sendFileResult.phase1FileSizes,
                     sendFileResult.phase1ExistingFileNames, sendFileResult.phase1ExistingFileSizes, sendFileResult.totalSize,
                     sendFileResult.existingTotalSize, sendFileResult.took.millis(), phase1ThrottlingWaitTime,
@@ -330,18 +342,17 @@ static final class SendFileResult {
      * segments that are missing. Only segments that have the same size and
      * checksum can be reused
      */
-    public SendFileResult phase1(final IndexCommit snapshot, final long globalCheckpoint, final Supplier<Integer> translogOps) {
+    void phase1(IndexCommit snapshot, long globalCheckpoint, IntSupplier translogOps, ActionListener<SendFileResult> listener) {
         cancellableThreads.checkForCancel();
         // Total size of segment files that are recovered
-        long totalSize = 0;
+        long totalSizeInBytes = 0;
         // Total size of segment files that were able to be re-used
-        long existingTotalSize = 0;
+        long existingTotalSizeInBytes = 0;
         final List<String> phase1FileNames = new ArrayList<>();
         final List<Long> phase1FileSizes = new ArrayList<>();
         final List<String> phase1ExistingFileNames = new ArrayList<>();
         final List<Long> phase1ExistingFileSizes = new ArrayList<>();
         final Store store = shard.store();
-        store.incRef();
         try {
             StopWatch stopWatch = new StopWatch().start();
             final Store.MetadataSnapshot recoverySourceMetadata;
@@ -367,12 +378,12 @@ public SendFileResult phase1(final IndexCommit snapshot, final long globalCheckp
                 for (StoreFileMetaData md : diff.identical) {
                     phase1ExistingFileNames.add(md.name());
                     phase1ExistingFileSizes.add(md.length());
-                    existingTotalSize += md.length();
+                    existingTotalSizeInBytes += md.length();
                     if (logger.isTraceEnabled()) {
                         logger.trace("recovery [phase1]: not recovering [{}], exist in local store and has checksum [{}]," +
                                         " size [{}]", md.name(), md.checksum(), md.length());
                     }
-                    totalSize += md.length();
+                    totalSizeInBytes += md.length();
                 }
                 List<StoreFileMetaData> phase1Files = new ArrayList<>(diff.different.size() + diff.missing.size());
                 phase1Files.addAll(diff.different);
@@ -386,75 +397,41 @@ public SendFileResult phase1(final IndexCommit snapshot, final long globalCheckp
                     }
                     phase1FileNames.add(md.name());
                     phase1FileSizes.add(md.length());
-                    totalSize += md.length();
+                    totalSizeInBytes += md.length();
                 }
 
                 logger.trace("recovery [phase1]: recovering_files [{}] with total_size [{}], reusing_files [{}] with total_size [{}]",
-                    phase1FileNames.size(), new ByteSizeValue(totalSize),
-                    phase1ExistingFileNames.size(), new ByteSizeValue(existingTotalSize));
-                cancellableThreads.execute(() -> recoveryTarget.receiveFileInfo(
-                    phase1FileNames, phase1FileSizes, phase1ExistingFileNames, phase1ExistingFileSizes, translogOps.get()));
-                sendFiles(store, phase1Files.toArray(new StoreFileMetaData[0]), translogOps);
-                // Send the CLEAN_FILES request, which takes all of the files that
-                // were transferred and renames them from their temporary file
-                // names to the actual file names. It also writes checksums for
-                // the files after they have been renamed.
-                //
-                // Once the files have been renamed, any other files that are not
-                // related to this recovery (out of date segments, for example)
-                // are deleted
-                try {
-                    cancellableThreads.executeIO(() ->
-                        recoveryTarget.cleanFiles(translogOps.get(), globalCheckpoint, recoverySourceMetadata));
-                } catch (RemoteTransportException | IOException targetException) {
-                    final IOException corruptIndexException;
-                    // we realized that after the index was copied and we wanted to finalize the recovery
-                    // the index was corrupted:
-                    //   - maybe due to a broken segments file on an empty index (transferred with no checksum)
-                    //   - maybe due to old segments without checksums or length only checks
-                    if ((corruptIndexException = ExceptionsHelper.unwrapCorruption(targetException)) != null) {
-                        try {
-                            final Store.MetadataSnapshot recoverySourceMetadata1 = store.getMetadata(snapshot);
-                            StoreFileMetaData[] metadata =
-                                    StreamSupport.stream(recoverySourceMetadata1.spliterator(), false).toArray(StoreFileMetaData[]::new);
-                            ArrayUtil.timSort(metadata, Comparator.comparingLong(StoreFileMetaData::length)); // check small files first
-                            for (StoreFileMetaData md : metadata) {
-                                cancellableThreads.checkForCancel();
-                                logger.debug("checking integrity for file {} after remove corruption exception", md);
-                                if (store.checkIntegrityNoException(md) == false) { // we are corrupted on the primary -- fail!
-                                    shard.failShard("recovery", corruptIndexException);
-                                    logger.warn("Corrupted file detected {} checksum mismatch", md);
-                                    throw corruptIndexException;
-                                }
-                            }
-                        } catch (IOException ex) {
-                            targetException.addSuppressed(ex);
-                            throw targetException;
-                        }
-                        // corruption has happened on the way to replica
-                        RemoteTransportException exception = new RemoteTransportException("File corruption occurred on recovery but " +
-                                "checksums are ok", null);
-                        exception.addSuppressed(targetException);
-                        logger.warn(() -> new ParameterizedMessage(
-                                "{} Remote file corruption during finalization of recovery on node {}. local checksum OK",
-                                shard.shardId(), request.targetNode()), corruptIndexException);
-                        throw exception;
-                    } else {
-                        throw targetException;
-                    }
-                }
+                    phase1FileNames.size(), new ByteSizeValue(totalSizeInBytes),
+                    phase1ExistingFileNames.size(), new ByteSizeValue(existingTotalSizeInBytes));
+                final StepListener<Void> sendFileInfoStep = new StepListener<>();
+                final StepListener<Void> cleanFilesStep = new StepListener<>();
+                cancellableThreads.execute(() ->
+                    recoveryTarget.receiveFileInfo(phase1FileNames, phase1FileSizes, phase1ExistingFileNames,
+                        phase1ExistingFileSizes, translogOps.getAsInt(), sendFileInfoStep));
+
+                sendFileInfoStep.whenComplete(r -> {
+                    sendFiles(store, phase1Files.toArray(new StoreFileMetaData[0]), translogOps);
+                    cleanFiles(store, recoverySourceMetadata, translogOps, globalCheckpoint, cleanFilesStep);
+                }, listener::onFailure);
+
+                final long totalSize = totalSizeInBytes;
+                final long existingTotalSize = existingTotalSizeInBytes;
+                cleanFilesStep.whenComplete(r -> {
+                    final TimeValue took = stopWatch.totalTime();
+                    logger.trace("recovery [phase1]: took [{}]", took);
+                    listener.onResponse(new SendFileResult(phase1FileNames, phase1FileSizes, totalSize, phase1ExistingFileNames,
+                        phase1ExistingFileSizes, existingTotalSize, took));
+                }, listener::onFailure);
             } else {
                 logger.trace("skipping [phase1]- identical sync id [{}] found on both source and target",
                     recoverySourceMetadata.getSyncId());
+                final TimeValue took = stopWatch.totalTime();
+                logger.trace("recovery [phase1]: took [{}]", took);
+                listener.onResponse(new SendFileResult(phase1FileNames, phase1FileSizes, totalSizeInBytes, phase1ExistingFileNames,
+                    phase1ExistingFileSizes, existingTotalSizeInBytes, took));
             }
-            final TimeValue took = stopWatch.totalTime();
-            logger.trace("recovery [phase1]: took [{}]", took);
-            return new SendFileResult(phase1FileNames, phase1FileSizes, totalSize, phase1ExistingFileNames,
-                phase1ExistingFileSizes, existingTotalSize, took);
         } catch (Exception e) {
-            throw new RecoverFilesRecoveryException(request.shardId(), phase1FileNames.size(), new ByteSizeValue(totalSize), e);
-        } finally {
-            store.decRef();
+            throw new RecoverFilesRecoveryException(request.shardId(), phase1FileNames.size(), new ByteSizeValue(totalSizeInBytes), e);
         }
     }
 
@@ -641,7 +618,7 @@ void finalizeRecovery(final long targetLocalCheckpoint, final ActionListener<Voi
          */
         runUnderPrimaryPermit(() -> shard.markAllocationIdAsInSync(request.targetAllocationId(), targetLocalCheckpoint),
             shardId + " marking " + request.targetAllocationId() + " as in sync", shard, cancellableThreads, logger);
-        final long globalCheckpoint = shard.getGlobalCheckpoint();
+        final long globalCheckpoint = shard.getLastKnownGlobalCheckpoint(); // this global checkpoint is persisted in finalizeRecovery
         final StepListener<Void> finalizeListener = new StepListener<>();
         cancellableThreads.executeIO(() -> recoveryTarget.finalizeRecovery(globalCheckpoint, finalizeListener));
         finalizeListener.whenComplete(r -> {
@@ -652,7 +629,7 @@ void finalizeRecovery(final long targetLocalCheckpoint, final ActionListener<Voi
                 logger.trace("performing relocation hand-off");
                 // TODO: make relocated async
                 // this acquires all IndexShard operation permits and will thus delay new recoveries until it is done
-                cancellableThreads.execute(() -> shard.relocated(recoveryTarget::handoffPrimaryContext));
+                cancellableThreads.execute(() -> shard.relocated(request.targetAllocationId(), recoveryTarget::handoffPrimaryContext));
                 /*
                  * if the recovery process fails after disabling primary mode on the source shard, both relocation source and
                  * target are failed (see {@link IndexShard#updateRoutingEntry}).
@@ -692,7 +669,7 @@ public String toString() {
                 '}';
     }
 
-    void sendFiles(Store store, StoreFileMetaData[] files, Supplier<Integer> translogOps) throws Exception {
+    void sendFiles(Store store, StoreFileMetaData[] files, IntSupplier translogOps) throws Exception {
         ArrayUtil.timSort(files, Comparator.comparingLong(StoreFileMetaData::length)); // send smallest first
         final LocalCheckpointTracker requestSeqIdTracker = new LocalCheckpointTracker(NO_OPS_PERFORMED, NO_OPS_PERFORMED);
         final AtomicReference<Tuple<StoreFileMetaData, Exception>> error = new AtomicReference<>();
@@ -709,19 +686,20 @@ void sendFiles(Store store, StoreFileMetaData[] files, Supplier<Integer> translo
                     final BytesArray content = new BytesArray(buffer, 0, bytesRead);
                     final boolean lastChunk = position + content.length() == md.length();
                     final long requestSeqId = requestSeqIdTracker.generateSeqNo();
-                    cancellableThreads.execute(() -> requestSeqIdTracker.waitForOpsToComplete(requestSeqId - maxConcurrentFileChunks));
+                    cancellableThreads.execute(
+                        () -> requestSeqIdTracker.waitForProcessedOpsToComplete(requestSeqId - maxConcurrentFileChunks));
                     cancellableThreads.checkForCancel();
                     if (error.get() != null) {
                         break;
                     }
                     final long requestFilePosition = position;
                     cancellableThreads.executeIO(() ->
-                        recoveryTarget.writeFileChunk(md, requestFilePosition, content, lastChunk, translogOps.get(),
+                        recoveryTarget.writeFileChunk(md, requestFilePosition, content, lastChunk, translogOps.getAsInt(),
                             ActionListener.wrap(
-                                r -> requestSeqIdTracker.markSeqNoAsCompleted(requestSeqId),
+                                r -> requestSeqIdTracker.markSeqNoAsProcessed(requestSeqId),
                                 e -> {
                                     error.compareAndSet(null, Tuple.tuple(md, e));
-                                    requestSeqIdTracker.markSeqNoAsCompleted(requestSeqId);
+                                    requestSeqIdTracker.markSeqNoAsProcessed(requestSeqId);
                                 }
                             )));
                     position += content.length();
@@ -734,27 +712,56 @@ void sendFiles(Store store, StoreFileMetaData[] files, Supplier<Integer> translo
         // When we terminate exceptionally, we don't wait for the outstanding requests as we don't use their results anyway.
         // This allows us to end quickly and eliminate the complexity of handling requestSeqIds in case of error.
         if (error.get() == null) {
-            cancellableThreads.execute(() -> requestSeqIdTracker.waitForOpsToComplete(requestSeqIdTracker.getMaxSeqNo()));
+            cancellableThreads.execute(() -> requestSeqIdTracker.waitForProcessedOpsToComplete(requestSeqIdTracker.getMaxSeqNo()));
         }
         if (error.get() != null) {
-            handleErrorOnSendFiles(store, error.get().v1(), error.get().v2());
+            handleErrorOnSendFiles(store, error.get().v2(), new StoreFileMetaData[]{error.get().v1()});
         }
     }
 
-    private void handleErrorOnSendFiles(Store store, StoreFileMetaData md, Exception e) throws Exception {
-        final IOException corruptIndexException;
-        if ((corruptIndexException = ExceptionsHelper.unwrapCorruption(e)) != null) {
-            if (store.checkIntegrityNoException(md) == false) { // we are corrupted on the primary -- fail!
-                logger.warn("{} Corrupted file detected {} checksum mismatch", shardId, md);
-                failEngine(corruptIndexException);
-                throw corruptIndexException;
+    private void cleanFiles(Store store, Store.MetadataSnapshot sourceMetadata, IntSupplier translogOps,
+                            long globalCheckpoint, ActionListener<Void> listener) {
+        // Send the CLEAN_FILES request, which takes all of the files that
+        // were transferred and renames them from their temporary file
+        // names to the actual file names. It also writes checksums for
+        // the files after they have been renamed.
+        //
+        // Once the files have been renamed, any other files that are not
+        // related to this recovery (out of date segments, for example)
+        // are deleted
+        cancellableThreads.execute(() -> recoveryTarget.cleanFiles(translogOps.getAsInt(), globalCheckpoint, sourceMetadata,
+            ActionListener.delegateResponse(listener, (l, e) -> ActionListener.completeWith(l, () -> {
+                StoreFileMetaData[] mds = StreamSupport.stream(sourceMetadata.spliterator(), false).toArray(StoreFileMetaData[]::new);
+                ArrayUtil.timSort(mds, Comparator.comparingLong(StoreFileMetaData::length)); // check small files first
+                handleErrorOnSendFiles(store, e, mds);
+                throw e;
+            }))));
+    }
+
+    private void handleErrorOnSendFiles(Store store, Exception e, StoreFileMetaData[] mds) throws Exception {
+        final IOException corruptIndexException = ExceptionsHelper.unwrapCorruption(e);
+        if (corruptIndexException != null) {
+            Exception localException = null;
+            for (StoreFileMetaData md : mds) {
+                cancellableThreads.checkForCancel();
+                logger.debug("checking integrity for file {} after remove corruption exception", md);
+                if (store.checkIntegrityNoException(md) == false) { // we are corrupted on the primary -- fail!
+                    logger.warn("{} Corrupted file detected {} checksum mismatch", shardId, md);
+                    if (localException == null) {
+                        localException = corruptIndexException;
+                    }
+                    failEngine(corruptIndexException);
+                }
+            }
+            if (localException != null) {
+                throw localException;
             } else { // corruption has happened on the way to replica
-                RemoteTransportException exception = new RemoteTransportException(
+                RemoteTransportException remoteException = new RemoteTransportException(
                     "File corruption occurred on recovery but checksums are ok", null);
-                exception.addSuppressed(e);
+                remoteException.addSuppressed(e);
                 logger.warn(() -> new ParameterizedMessage("{} Remote file corruption on node {}, recovering {}. local checksum OK",
-                    shardId, request.targetNode(), md), corruptIndexException);
-                throw exception;
+                    shardId, request.targetNode(), mds), corruptIndexException);
+                throw remoteException;
             }
         } else {
             throw e;
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
index 18018fc7db054..d29456ec42bed 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
@@ -927,7 +927,7 @@ public synchronized XContentBuilder toXContent(XContentBuilder builder, Params p
             builder.field(Fields.REUSED, reusedFileCount());
             builder.field(Fields.RECOVERED, recoveredFileCount());
             builder.field(Fields.PERCENT, String.format(Locale.ROOT, "%1.1f%%", recoveredFilesPercent()));
-            if (params.paramAsBoolean("details", false)) {
+            if (params.paramAsBoolean("detailed", false)) {
                 builder.startArray(Fields.DETAILS);
                 for (File file : fileDetails.values()) {
                     file.toXContent(builder, params);
@@ -954,7 +954,7 @@ public synchronized String toString() {
             }
         }
 
-        public File getFileDetails(String dest) {
+        public synchronized File getFileDetails(String dest) {
             return fileDetails.get(dest);
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
index b3c6d12ab96e3..0b3f95b2ec4a2 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
@@ -377,68 +377,75 @@ public void receiveFileInfo(List<String> phase1FileNames,
                                 List<Long> phase1FileSizes,
                                 List<String> phase1ExistingFileNames,
                                 List<Long> phase1ExistingFileSizes,
-                                int totalTranslogOps) {
-        final RecoveryState.Index index = state().getIndex();
-        for (int i = 0; i < phase1ExistingFileNames.size(); i++) {
-            index.addFileDetail(phase1ExistingFileNames.get(i), phase1ExistingFileSizes.get(i), true);
-        }
-        for (int i = 0; i < phase1FileNames.size(); i++) {
-            index.addFileDetail(phase1FileNames.get(i), phase1FileSizes.get(i), false);
-        }
-        state().getTranslog().totalOperations(totalTranslogOps);
-        state().getTranslog().totalOperationsOnStart(totalTranslogOps);
-
+                                int totalTranslogOps,
+                                ActionListener<Void> listener) {
+        ActionListener.completeWith(listener, () -> {
+            final RecoveryState.Index index = state().getIndex();
+            for (int i = 0; i < phase1ExistingFileNames.size(); i++) {
+                index.addFileDetail(phase1ExistingFileNames.get(i), phase1ExistingFileSizes.get(i), true);
+            }
+            for (int i = 0; i < phase1FileNames.size(); i++) {
+                index.addFileDetail(phase1FileNames.get(i), phase1FileSizes.get(i), false);
+            }
+            state().getTranslog().totalOperations(totalTranslogOps);
+            state().getTranslog().totalOperationsOnStart(totalTranslogOps);
+            return null;
+        });
     }
 
     @Override
-    public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetaData) throws IOException {
-        state().getTranslog().totalOperations(totalTranslogOps);
-        // first, we go and move files that were created with the recovery id suffix to
-        // the actual names, its ok if we have a corrupted index here, since we have replicas
-        // to recover from in case of a full cluster shutdown just when this code executes...
-        multiFileWriter.renameAllTempFiles();
-        final Store store = store();
-        store.incRef();
-        try {
-            store.cleanupAndVerify("recovery CleanFilesRequestHandler", sourceMetaData);
-            final String translogUUID = Translog.createEmptyTranslog(
-                indexShard.shardPath().resolveTranslog(), globalCheckpoint, shardId, indexShard.getPendingPrimaryTerm());
-            store.associateIndexWithNewTranslog(translogUUID);
-
-            if (indexShard.getRetentionLeases().leases().isEmpty()) {
-                // if empty, may be a fresh IndexShard, so write an empty leases file to disk
-                indexShard.persistRetentionLeases();
-                assert indexShard.loadRetentionLeases().leases().isEmpty();
-            } else {
-                assert indexShard.assertRetentionLeasesPersisted();
-            }
-
-        } catch (CorruptIndexException | IndexFormatTooNewException | IndexFormatTooOldException ex) {
-            // this is a fatal exception at this stage.
-            // this means we transferred files from the remote that have not be checksummed and they are
-            // broken. We have to clean up this shard entirely, remove all files and bubble it up to the
-            // source shard since this index might be broken there as well? The Source can handle this and checks
-            // its content on disk if possible.
+    public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetaData,
+                           ActionListener<Void> listener) {
+        ActionListener.completeWith(listener, () -> {
+            state().getTranslog().totalOperations(totalTranslogOps);
+            // first, we go and move files that were created with the recovery id suffix to
+            // the actual names, its ok if we have a corrupted index here, since we have replicas
+            // to recover from in case of a full cluster shutdown just when this code executes...
+            multiFileWriter.renameAllTempFiles();
+            final Store store = store();
+            store.incRef();
             try {
+                store.cleanupAndVerify("recovery CleanFilesRequestHandler", sourceMetaData);
+                final String translogUUID = Translog.createEmptyTranslog(
+                    indexShard.shardPath().resolveTranslog(), globalCheckpoint, shardId, indexShard.getPendingPrimaryTerm());
+                store.associateIndexWithNewTranslog(translogUUID);
+
+                if (indexShard.getRetentionLeases().leases().isEmpty()) {
+                    // if empty, may be a fresh IndexShard, so write an empty leases file to disk
+                    indexShard.persistRetentionLeases();
+                    assert indexShard.loadRetentionLeases().leases().isEmpty();
+                } else {
+                    assert indexShard.assertRetentionLeasesPersisted();
+                }
+
+            } catch (CorruptIndexException | IndexFormatTooNewException | IndexFormatTooOldException ex) {
+                // this is a fatal exception at this stage.
+                // this means we transferred files from the remote that have not be checksummed and they are
+                // broken. We have to clean up this shard entirely, remove all files and bubble it up to the
+                // source shard since this index might be broken there as well? The Source can handle this and checks
+                // its content on disk if possible.
                 try {
-                    store.removeCorruptionMarker();
-                } finally {
-                    Lucene.cleanLuceneIndex(store.directory()); // clean up and delete all files
+                    try {
+                        store.removeCorruptionMarker();
+                    } finally {
+                        Lucene.cleanLuceneIndex(store.directory()); // clean up and delete all files
+                    }
+                } catch (Exception e) {
+                    logger.debug("Failed to clean lucene index", e);
+                    ex.addSuppressed(e);
                 }
-            } catch (Exception e) {
-                logger.debug("Failed to clean lucene index", e);
-                ex.addSuppressed(e);
+                RecoveryFailedException rfe = new RecoveryFailedException(state(), "failed to clean after recovery", ex);
+                fail(rfe, true);
+                throw rfe;
+            } catch (Exception ex) {
+                RecoveryFailedException rfe = new RecoveryFailedException(state(), "failed to clean after recovery", ex);
+                fail(rfe, true);
+                throw rfe;
+            } finally {
+                store.decRef();
             }
-            RecoveryFailedException rfe = new RecoveryFailedException(state(), "failed to clean after recovery", ex);
-            fail(rfe, true);
-            throw rfe;
-        } catch (Exception ex) {
-            RecoveryFailedException rfe = new RecoveryFailedException(state(), "failed to clean after recovery", ex);
-            fail(rfe, true);
-            throw rfe;
-        } finally {
-            store.decRef();
-        }
+            return null;
+        });
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetHandler.java
index d03fe42d90146..9be076bc73215 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetHandler.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetHandler.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.index.store.StoreFileMetaData;
 import org.elasticsearch.index.translog.Translog;
 
-import java.io.IOException;
 import java.util.List;
 
 public interface RecoveryTargetHandler {
@@ -89,7 +88,8 @@ void receiveFileInfo(List<String> phase1FileNames,
                          List<Long> phase1FileSizes,
                          List<String> phase1ExistingFileNames,
                          List<Long> phase1ExistingFileSizes,
-                         int totalTranslogOps);
+                         int totalTranslogOps,
+                         ActionListener<Void> listener);
 
     /**
      * After all source files has been sent over, this command is sent to the target so it can clean any local
@@ -99,7 +99,7 @@ void receiveFileInfo(List<String> phase1FileNames,
      * @param globalCheckpoint the global checkpoint on the primary
      * @param sourceMetaData   meta data of the source store
      */
-    void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetaData) throws IOException;
+    void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetaData, ActionListener<Void> listener);
 
     /** writes a partial file chunk to the target store */
     void writeFileChunk(StoreFileMetaData fileMetaData, long position, BytesReference content,
diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java
index 230f53515754e..bb5457c1a3dca 100644
--- a/server/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java
+++ b/server/src/main/java/org/elasticsearch/indices/recovery/RemoteRecoveryTargetHandler.java
@@ -70,7 +70,6 @@ public RemoteRecoveryTargetHandler(long recoveryId, ShardId shardId, TransportSe
                 .withTimeout(recoverySettings.internalActionLongTimeout())
                 .build();
         this.fileChunkRequestOptions = TransportRequestOptions.builder()
-                // we are saving the cpu for other things
                 .withType(TransportRequestOptions.Type.RECOVERY)
                 .withTimeout(recoverySettings.internalActionTimeout())
                 .build();
@@ -130,22 +129,23 @@ public void indexTranslogOperations(
 
     @Override
     public void receiveFileInfo(List<String> phase1FileNames, List<Long> phase1FileSizes, List<String> phase1ExistingFileNames,
-                                List<Long> phase1ExistingFileSizes, int totalTranslogOps) {
-
+                                List<Long> phase1ExistingFileSizes, int totalTranslogOps, ActionListener<Void> listener) {
         RecoveryFilesInfoRequest recoveryInfoFilesRequest = new RecoveryFilesInfoRequest(recoveryId, shardId,
-                phase1FileNames, phase1FileSizes, phase1ExistingFileNames, phase1ExistingFileSizes, totalTranslogOps);
+            phase1FileNames, phase1FileSizes, phase1ExistingFileNames, phase1ExistingFileSizes, totalTranslogOps);
         transportService.submitRequest(targetNode, PeerRecoveryTargetService.Actions.FILES_INFO, recoveryInfoFilesRequest,
-                TransportRequestOptions.builder().withTimeout(recoverySettings.internalActionTimeout()).build(),
-                EmptyTransportResponseHandler.INSTANCE_SAME).txGet();
-
+            TransportRequestOptions.builder().withTimeout(recoverySettings.internalActionTimeout()).build(),
+            new ActionListenerResponseHandler<>(ActionListener.map(listener, r -> null),
+                in -> TransportResponse.Empty.INSTANCE, ThreadPool.Names.GENERIC));
     }
 
     @Override
-    public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetaData) throws IOException {
+    public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetaData,
+                           ActionListener<Void> listener) {
         transportService.submitRequest(targetNode, PeerRecoveryTargetService.Actions.CLEAN_FILES,
                 new RecoveryCleanFilesRequest(recoveryId, shardId, sourceMetaData, totalTranslogOps, globalCheckpoint),
                 TransportRequestOptions.builder().withTimeout(recoverySettings.internalActionTimeout()).build(),
-                EmptyTransportResponseHandler.INSTANCE_SAME).txGet();
+                new ActionListenerResponseHandler<>(ActionListener.map(listener, r -> null),
+                    in -> TransportResponse.Empty.INSTANCE, ThreadPool.Names.GENERIC));
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java b/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
index 20307af32f4ed..20633d24ba7eb 100644
--- a/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
+++ b/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
@@ -50,6 +50,7 @@
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.store.StoreFileMetaData;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -91,8 +92,8 @@ public void list(ShardId shardId, DiscoveryNode[] nodes, ActionListener<NodesSto
     }
 
     @Override
-    protected NodeRequest newNodeRequest(String nodeId, Request request) {
-        return new NodeRequest(nodeId, request);
+    protected NodeRequest newNodeRequest(Request request) {
+        return new NodeRequest(request);
     }
 
     @Override
@@ -107,7 +108,7 @@ protected NodesStoreFilesMetaData newResponse(Request request,
     }
 
     @Override
-    protected NodeStoreFilesMetaData nodeOperation(NodeRequest request) {
+    protected NodeStoreFilesMetaData nodeOperation(NodeRequest request, Task task) {
         try {
             return new NodeStoreFilesMetaData(clusterService.localNode(), listStoreMetaData(request.shardId));
         } catch (IOException e) {
@@ -290,8 +291,7 @@ public static class NodeRequest extends BaseNodeRequest {
         public NodeRequest() {
         }
 
-        NodeRequest(String nodeId, TransportNodesListShardStoreMetaData.Request request) {
-            super(nodeId);
+        NodeRequest(TransportNodesListShardStoreMetaData.Request request) {
             this.shardId = request.shardId;
         }
 
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java
index b2143d72ae65f..9e7d1b7b5bdbd 100644
--- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java
+++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java
@@ -80,7 +80,7 @@ public class IngestService implements ClusterStateApplier {
     // We know of all the processor factories when a node with all its plugin have been initialized. Also some
     // processor factories rely on other node services. Custom metadata is statically registered when classes
     // are loaded, so in the cluster state we just save the pipeline config and here we keep the actual pipelines around.
-    private volatile Map<String, Pipeline> pipelines = new HashMap<>();
+    private volatile Map<String, PipelineHolder> pipelines = Map.of();
     private final ThreadPool threadPool;
     private final IngestMetric totalMetrics = new IngestMetric();
 
@@ -236,7 +236,12 @@ public ClusterState execute(ClusterState currentState) {
      * Returns the pipeline by the specified id
      */
     public Pipeline getPipeline(String id) {
-        return pipelines.get(id);
+        PipelineHolder holder = pipelines.get(id);
+        if (holder != null) {
+            return holder.pipeline;
+        } else {
+            return null;
+        }
     }
 
     public Map<String, Processor.Factory> getProcessorFactories() {
@@ -252,52 +257,10 @@ public IngestInfo info() {
         return new IngestInfo(processorInfoList);
     }
 
-    Map<String, Pipeline> pipelines() {
+    Map<String, PipelineHolder> pipelines() {
         return pipelines;
     }
 
-    @Override
-    public void applyClusterState(final ClusterChangedEvent event) {
-        ClusterState state = event.state();
-        Map<String, Pipeline> originalPipelines = pipelines;
-        try {
-            innerUpdatePipelines(event.previousState(), state);
-        } catch (ElasticsearchParseException e) {
-            logger.warn("failed to update ingest pipelines", e);
-        }
-        //pipelines changed, so add the old metrics to the new metrics
-        if (originalPipelines != pipelines) {
-            pipelines.forEach((id, pipeline) -> {
-                Pipeline originalPipeline = originalPipelines.get(id);
-                if (originalPipeline != null) {
-                    pipeline.getMetrics().add(originalPipeline.getMetrics());
-                    List<Tuple<Processor, IngestMetric>> oldPerProcessMetrics = new ArrayList<>();
-                    List<Tuple<Processor, IngestMetric>> newPerProcessMetrics = new ArrayList<>();
-                    getProcessorMetrics(originalPipeline.getCompoundProcessor(), oldPerProcessMetrics);
-                    getProcessorMetrics(pipeline.getCompoundProcessor(), newPerProcessMetrics);
-                    //Best attempt to populate new processor metrics using a parallel array of the old metrics. This is not ideal since
-                    //the per processor metrics may get reset when the arrays don't match. However, to get to an ideal model, unique and
-                    //consistent id's per processor and/or semantic equals for each processor will be needed.
-                    if (newPerProcessMetrics.size() == oldPerProcessMetrics.size()) {
-                        Iterator<Tuple<Processor, IngestMetric>> oldMetricsIterator = oldPerProcessMetrics.iterator();
-                        for (Tuple<Processor, IngestMetric> compositeMetric : newPerProcessMetrics) {
-                            String type = compositeMetric.v1().getType();
-                            IngestMetric metric = compositeMetric.v2();
-                            if (oldMetricsIterator.hasNext()) {
-                                Tuple<Processor, IngestMetric> oldCompositeMetric = oldMetricsIterator.next();
-                                String oldType = oldCompositeMetric.v1().getType();
-                                IngestMetric oldMetric = oldCompositeMetric.v2();
-                                if (type.equals(oldType)) {
-                                    metric.add(oldMetric);
-                                }
-                            }
-                        }
-                    }
-                }
-            });
-        }
-    }
-
     /**
      * Recursive method to obtain all of the non-failure processors for given compoundProcessor. Since conditionals are implemented as
      * wrappers to the actual processor, always prefer the actual processor's metric over the conditional processor's metric.
@@ -324,25 +287,6 @@ private static List<Tuple<Processor, IngestMetric>> getProcessorMetrics(Compound
         return processorMetrics;
     }
 
-    private static Pipeline substitutePipeline(String id, ElasticsearchParseException e) {
-        String tag = e.getHeaderKeys().contains("processor_tag") ? e.getHeader("processor_tag").get(0) : null;
-        String type = e.getHeaderKeys().contains("processor_type") ? e.getHeader("processor_type").get(0) : "unknown";
-        String errorMessage = "pipeline with id [" + id + "] could not be loaded, caused by [" + e.getDetailedMessage() + "]";
-        Processor failureProcessor = new AbstractProcessor(tag) {
-            @Override
-            public IngestDocument execute(IngestDocument ingestDocument) {
-                throw new IllegalStateException(errorMessage);
-            }
-
-            @Override
-            public String getType() {
-                return type;
-            }
-        };
-        String description = "this is a place holder pipeline, because pipeline with id [" +  id + "] could not be loaded";
-        return new Pipeline(id, description, null, new CompoundProcessor(failureProcessor));
-    }
-
     static ClusterState innerPut(PutPipelineRequest request, ClusterState currentState) {
         IngestMetadata currentIngestMetadata = currentState.metaData().custom(IngestMetadata.TYPE);
         Map<String, PipelineConfiguration> pipelines;
@@ -403,10 +347,11 @@ protected void doRun() {
                     String pipelineId = indexRequest.getPipeline();
                     if (NOOP_PIPELINE_NAME.equals(pipelineId) == false) {
                         try {
-                            Pipeline pipeline = pipelines.get(pipelineId);
-                            if (pipeline == null) {
+                            PipelineHolder holder = pipelines.get(pipelineId);
+                            if (holder == null) {
                                 throw new IllegalArgumentException("pipeline with id [" + pipelineId + "] does not exist");
                             }
+                            Pipeline pipeline = holder.pipeline;
                             innerExecute(indexRequest, pipeline, itemDroppedHandler);
                             //this shouldn't be needed here but we do it for consistency with index api
                             // which requires it to prevent double execution
@@ -424,7 +369,8 @@ protected void doRun() {
     public IngestStats stats() {
         IngestStats.Builder statsBuilder = new IngestStats.Builder();
         statsBuilder.addTotalMetrics(totalMetrics);
-        pipelines.forEach((id, pipeline) -> {
+        pipelines.forEach((id, holder) -> {
+            Pipeline pipeline = holder.pipeline;
             CompoundProcessor rootProcessor = pipeline.getCompoundProcessor();
             statsBuilder.addPipelineMetrics(id, pipeline.getMetrics());
             List<Tuple<Processor, IngestMetric>> processorMetrics = new ArrayList<>();
@@ -503,37 +449,146 @@ private void innerExecute(IndexRequest indexRequest, Pipeline pipeline, Consumer
         }
     }
 
-    private void innerUpdatePipelines(ClusterState previousState, ClusterState state) {
+    @Override
+    public void applyClusterState(final ClusterChangedEvent event) {
+        ClusterState state = event.state();
         if (state.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
             return;
         }
 
-        IngestMetadata ingestMetadata = state.getMetaData().custom(IngestMetadata.TYPE);
-        IngestMetadata previousIngestMetadata = previousState.getMetaData().custom(IngestMetadata.TYPE);
-        if (Objects.equals(ingestMetadata, previousIngestMetadata)) {
+        IngestMetadata newIngestMetadata = state.getMetaData().custom(IngestMetadata.TYPE);
+        if (newIngestMetadata == null) {
             return;
         }
 
-        Map<String, Pipeline> pipelines = new HashMap<>();
-        List<ElasticsearchParseException> exceptions = new ArrayList<>();
-        for (PipelineConfiguration pipeline : ingestMetadata.getPipelines().values()) {
+        try {
+            innerUpdatePipelines(newIngestMetadata);
+        } catch (ElasticsearchParseException e) {
+            logger.warn("failed to update ingest pipelines", e);
+        }
+    }
+
+    void innerUpdatePipelines(IngestMetadata newIngestMetadata) {
+        Map<String, PipelineHolder> existingPipelines = this.pipelines;
+
+        // Lazy initialize these variables in order to favour the most like scenario that there are no pipeline changes:
+        Map<String, PipelineHolder> newPipelines = null;
+        List<ElasticsearchParseException> exceptions = null;
+        // Iterate over pipeline configurations in ingest metadata and constructs a new pipeline if there is no pipeline
+        // or the pipeline configuration has been modified
+        for (PipelineConfiguration newConfiguration : newIngestMetadata.getPipelines().values()) {
+            PipelineHolder previous = existingPipelines.get(newConfiguration.getId());
+            if (previous != null && previous.configuration.equals(newConfiguration)) {
+                continue;
+            }
+
+            if (newPipelines == null) {
+                newPipelines = new HashMap<>(existingPipelines);
+            }
             try {
-                pipelines.put(
-                    pipeline.getId(),
-                    Pipeline.create(pipeline.getId(), pipeline.getConfigAsMap(), processorFactories, scriptService)
+                Pipeline newPipeline =
+                    Pipeline.create(newConfiguration.getId(), newConfiguration.getConfigAsMap(), processorFactories, scriptService);
+                newPipelines.put(
+                    newConfiguration.getId(),
+                    new PipelineHolder(newConfiguration, newPipeline)
                 );
+
+                if (previous == null) {
+                    continue;
+                }
+                Pipeline oldPipeline = previous.pipeline;
+                newPipeline.getMetrics().add(oldPipeline.getMetrics());
+                List<Tuple<Processor, IngestMetric>> oldPerProcessMetrics = new ArrayList<>();
+                List<Tuple<Processor, IngestMetric>> newPerProcessMetrics = new ArrayList<>();
+                getProcessorMetrics(oldPipeline.getCompoundProcessor(), oldPerProcessMetrics);
+                getProcessorMetrics(newPipeline.getCompoundProcessor(), newPerProcessMetrics);
+                //Best attempt to populate new processor metrics using a parallel array of the old metrics. This is not ideal since
+                //the per processor metrics may get reset when the arrays don't match. However, to get to an ideal model, unique and
+                //consistent id's per processor and/or semantic equals for each processor will be needed.
+                if (newPerProcessMetrics.size() == oldPerProcessMetrics.size()) {
+                    Iterator<Tuple<Processor, IngestMetric>> oldMetricsIterator = oldPerProcessMetrics.iterator();
+                    for (Tuple<Processor, IngestMetric> compositeMetric : newPerProcessMetrics) {
+                        String type = compositeMetric.v1().getType();
+                        IngestMetric metric = compositeMetric.v2();
+                        if (oldMetricsIterator.hasNext()) {
+                            Tuple<Processor, IngestMetric> oldCompositeMetric = oldMetricsIterator.next();
+                            String oldType = oldCompositeMetric.v1().getType();
+                            IngestMetric oldMetric = oldCompositeMetric.v2();
+                            if (type.equals(oldType)) {
+                                metric.add(oldMetric);
+                            }
+                        }
+                    }
+                }
             } catch (ElasticsearchParseException e) {
-                pipelines.put(pipeline.getId(), substitutePipeline(pipeline.getId(), e));
+                Pipeline pipeline = substitutePipeline(newConfiguration.getId(), e);
+                newPipelines.put(newConfiguration.getId(), new PipelineHolder(newConfiguration, pipeline));
+                if (exceptions == null) {
+                    exceptions = new ArrayList<>();
+                }
                 exceptions.add(e);
             } catch (Exception e) {
                 ElasticsearchParseException parseException = new ElasticsearchParseException(
-                    "Error updating pipeline with id [" + pipeline.getId() + "]", e);
-                pipelines.put(pipeline.getId(), substitutePipeline(pipeline.getId(), parseException));
+                    "Error updating pipeline with id [" + newConfiguration.getId() + "]", e);
+                Pipeline pipeline = substitutePipeline(newConfiguration.getId(), parseException);
+                newPipelines.put(newConfiguration.getId(), new PipelineHolder(newConfiguration, pipeline));
+                if (exceptions == null) {
+                    exceptions = new ArrayList<>();
+                }
                 exceptions.add(parseException);
             }
         }
-        this.pipelines = Collections.unmodifiableMap(pipelines);
-        ExceptionsHelper.rethrowAndSuppress(exceptions);
+
+        // Iterate over the current active pipelines and check whether they are missing in the pipeline configuration and
+        // if so delete the pipeline from new Pipelines map:
+        for (Map.Entry<String, PipelineHolder> entry : existingPipelines.entrySet()) {
+            if (newIngestMetadata.getPipelines().get(entry.getKey()) == null) {
+                if (newPipelines == null) {
+                    newPipelines = new HashMap<>(existingPipelines);
+                }
+                newPipelines.remove(entry.getKey());
+            }
+        }
+
+        if (newPipelines != null) {
+            // Update the pipelines:
+            this.pipelines = Map.copyOf(newPipelines);
+
+            // Rethrow errors that may have occurred during creating new pipeline instances:
+            if (exceptions != null) {
+                ExceptionsHelper.rethrowAndSuppress(exceptions);
+            }
+        }
+    }
+
+    private static Pipeline substitutePipeline(String id, ElasticsearchParseException e) {
+        String tag = e.getHeaderKeys().contains("processor_tag") ? e.getHeader("processor_tag").get(0) : null;
+        String type = e.getHeaderKeys().contains("processor_type") ? e.getHeader("processor_type").get(0) : "unknown";
+        String errorMessage = "pipeline with id [" + id + "] could not be loaded, caused by [" + e.getDetailedMessage() + "]";
+        Processor failureProcessor = new AbstractProcessor(tag) {
+            @Override
+            public IngestDocument execute(IngestDocument ingestDocument) {
+                throw new IllegalStateException(errorMessage);
+            }
+
+            @Override
+            public String getType() {
+                return type;
+            }
+        };
+        String description = "this is a place holder pipeline, because pipeline with id [" +  id + "] could not be loaded";
+        return new Pipeline(id, description, null, new CompoundProcessor(failureProcessor));
+    }
+
+    static class PipelineHolder {
+
+        final PipelineConfiguration configuration;
+        final Pipeline pipeline;
+
+        PipelineHolder(PipelineConfiguration configuration, Pipeline pipeline) {
+            this.configuration = Objects.requireNonNull(configuration);
+            this.pipeline = Objects.requireNonNull(pipeline);
+        }
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java b/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
index 18173dd275a46..320bc15fda1f4 100644
--- a/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
+++ b/server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
@@ -42,6 +42,24 @@
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+/**
+ * The {@link OsProbe} class retrieves information about the physical and swap size of the machine
+ * memory, as well as the system load average and cpu load.
+ *
+ * In some exceptional cases, it's possible the underlying native method used by
+ * {@link #getFreePhysicalMemorySize()} and {@link #getTotalPhysicalMemorySize()} can return a
+ * negative value. Because of this, we prevent those methods from returning negative values,
+ * returning 0 instead.
+ *
+ * The OS can report a negative number in a number of cases:
+ * - Non-supported OSes (HP-UX, or AIX)
+ * - A failure of macOS to initialize host statistics
+ * - An OS that does not support the {@code _SC_PHYS_PAGES} or {@code _SC_PAGE_SIZE} flags for the {@code sysconf()} linux kernel call
+ * - An overflow of the product of {@code _SC_PHYS_PAGES} and {@code _SC_PAGE_SIZE}
+ * - An error case retrieving these values from a linux kernel
+ * - A non-standard libc implementation not implementing the required values
+ * For a more exhaustive explanation, see https://github.com/elastic/elasticsearch/pull/42725
+ */
 public class OsProbe {
 
     private static final OperatingSystemMXBean osMxBean = ManagementFactory.getOperatingSystemMXBean();
@@ -67,12 +85,19 @@ public class OsProbe {
      */
     public long getFreePhysicalMemorySize() {
         if (getFreePhysicalMemorySize == null) {
-            return -1;
+            logger.warn("getFreePhysicalMemorySize is not available");
+            return 0;
         }
         try {
-            return (long) getFreePhysicalMemorySize.invoke(osMxBean);
+            final long freeMem = (long) getFreePhysicalMemorySize.invoke(osMxBean);
+            if (freeMem < 0) {
+                logger.warn("OS reported a negative free memory value [{}]", freeMem);
+                return 0;
+            }
+            return freeMem;
         } catch (Exception e) {
-            return -1;
+            logger.warn("exception retrieving free physical memory", e);
+            return 0;
         }
     }
 
@@ -81,12 +106,19 @@ public long getFreePhysicalMemorySize() {
      */
     public long getTotalPhysicalMemorySize() {
         if (getTotalPhysicalMemorySize == null) {
-            return -1;
+            logger.warn("getTotalPhysicalMemorySize is not available");
+            return 0;
         }
         try {
-            return (long) getTotalPhysicalMemorySize.invoke(osMxBean);
+            final long totalMem = (long) getTotalPhysicalMemorySize.invoke(osMxBean);
+            if (totalMem < 0) {
+                logger.warn("OS reported a negative total memory value [{}]", totalMem);
+                return 0;
+            }
+            return totalMem;
         } catch (Exception e) {
-            return -1;
+            logger.warn("exception retrieving total physical memory", e);
+            return 0;
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/monitor/os/OsStats.java b/server/src/main/java/org/elasticsearch/monitor/os/OsStats.java
index 86047281a22fb..9dbd9e4365a53 100644
--- a/server/src/main/java/org/elasticsearch/monitor/os/OsStats.java
+++ b/server/src/main/java/org/elasticsearch/monitor/os/OsStats.java
@@ -228,13 +228,17 @@ public static class Mem implements Writeable, ToXContentFragment {
         private final long free;
 
         public Mem(long total, long free) {
+            assert total >= 0 : "expected total memory to be positive, got: " + total;
+            assert free >= 0 : "expected free memory to be positive, got: " + total;
             this.total = total;
             this.free = free;
         }
 
         public Mem(StreamInput in) throws IOException {
             this.total = in.readLong();
+            assert total >= 0 : "expected total memory to be positive, got: " + total;
             this.free = in.readLong();
+            assert free >= 0 : "expected free memory to be positive, got: " + total;
         }
 
         @Override
diff --git a/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java b/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java
index b73268d9985b5..2aee13a5b4a40 100644
--- a/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java
+++ b/server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java
@@ -36,10 +36,7 @@
 
 public class InternalSettingsPreparer {
 
-    /**
-     * Prepares settings for the transport client by gathering all
-     * elasticsearch system properties and setting defaults.
-     */
+    // TODO: refactor this method out, it used to exist for the transport client
     public static Settings prepareSettings(Settings input) {
         Settings.Builder output = Settings.builder();
         initializeSettings(output, input, Collections.emptyMap());
diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java
index 6e592ba324fb4..53990ad0c4c1e 100644
--- a/server/src/main/java/org/elasticsearch/node/Node.java
+++ b/server/src/main/java/org/elasticsearch/node/Node.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.Build;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchTimeoutException;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus;
 import org.elasticsearch.action.search.SearchExecutionStatsCollector;
@@ -55,7 +55,10 @@
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.cluster.metadata.TemplateUpgradeService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RoutingService;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.cluster.routing.BatchedRerouteService;
+import org.elasticsearch.cluster.routing.LazilyInitializedRerouteService;
+import org.elasticsearch.cluster.routing.RerouteService;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdMonitor;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.StopWatch;
@@ -64,7 +67,6 @@
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lease.Releasables;
@@ -74,6 +76,7 @@
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.ConsistentSettingsService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.SettingUpgrader;
@@ -311,6 +314,14 @@ protected Node(
             this.pluginsService = new PluginsService(tmpSettings, environment.configFile(), environment.modulesFile(),
                 environment.pluginsFile(), classpathPlugins);
             final Settings settings = pluginsService.updatedSettings();
+            final Set<DiscoveryNodeRole> possibleRoles = Stream.concat(
+                    DiscoveryNodeRole.BUILT_IN_ROLES.stream(),
+                    pluginsService.filterPlugins(Plugin.class)
+                            .stream()
+                            .map(Plugin::getRoles)
+                            .flatMap(Set::stream))
+                    .collect(Collectors.toSet());
+            DiscoveryNode.setPossibleRoles(possibleRoles);
             localNodeFactory = new LocalNodeFactory(settings, nodeEnvironment.nodeId());
 
             // create the environment based on the finalized (processed) view of the settings
@@ -355,26 +366,26 @@ protected Node(
             final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool);
             clusterService.addStateApplier(scriptModule.getScriptService());
             resourcesToClose.add(clusterService);
+            clusterService.addLocalNodeMasterListener(
+                    new ConsistentSettingsService(settings, clusterService, settingsModule.getConsistentSettings())
+                            .newHashPublisher());
             final IngestService ingestService = new IngestService(clusterService, threadPool, this.environment,
                 scriptModule.getScriptService(), analysisModule.getAnalysisRegistry(), pluginsService.filterPlugins(IngestPlugin.class));
-            final DiskThresholdMonitor listener = new DiskThresholdMonitor(settings, clusterService::state,
-                clusterService.getClusterSettings(), client);
+            final LazilyInitializedRerouteService lazilyInitializedRerouteService = new LazilyInitializedRerouteService();
+            final DiskThresholdMonitor diskThresholdMonitor = new DiskThresholdMonitor(settings, clusterService::state,
+                clusterService.getClusterSettings(), client, threadPool::relativeTimeInMillis, lazilyInitializedRerouteService);
             final ClusterInfoService clusterInfoService = newClusterInfoService(settings, clusterService, threadPool, client,
-                listener::onNewInfo);
+                diskThresholdMonitor::onNewInfo);
             final UsageService usageService = new UsageService();
 
             ModulesBuilder modules = new ModulesBuilder();
-            // plugin modules must be added here, before others or we can get crazy injection errors...
-            for (Module pluginModule : pluginsService.createGuiceModules()) {
-                modules.add(pluginModule);
-            }
             final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool, clusterInfoService);
             ClusterModule clusterModule = new ClusterModule(settings, clusterService, clusterPlugins, clusterInfoService);
             modules.add(clusterModule);
             IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class));
             modules.add(indicesModule);
 
-            SearchModule searchModule = new SearchModule(settings, false, pluginsService.filterPlugins(SearchPlugin.class));
+            SearchModule searchModule = new SearchModule(settings, pluginsService.filterPlugins(SearchPlugin.class));
             CircuitBreakerService circuitBreakerService = createCircuitBreakerService(settingsModule.getSettings(),
                 settingsModule.getClusterSettings());
             resourcesToClose.add(circuitBreakerService);
@@ -445,13 +456,13 @@ protected Node(
                                                  namedWriteableRegistry).stream())
                 .collect(Collectors.toList());
 
-            ActionModule actionModule = new ActionModule(false, settings, clusterModule.getIndexNameExpressionResolver(),
+            ActionModule actionModule = new ActionModule(settings, clusterModule.getIndexNameExpressionResolver(),
                 settingsModule.getIndexScopedSettings(), settingsModule.getClusterSettings(), settingsModule.getSettingsFilter(),
                 threadPool, pluginsService.filterPlugins(ActionPlugin.class), client, circuitBreakerService, usageService);
             modules.add(actionModule);
 
             final RestController restController = actionModule.getRestController();
-            final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class),
+            final NetworkModule networkModule = new NetworkModule(settings, pluginsService.filterPlugins(NetworkPlugin.class),
                 threadPool, bigArrays, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, xContentRegistry,
                 networkService, restController);
             Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders =
@@ -495,10 +506,13 @@ protected Node(
             RestoreService restoreService = new RestoreService(clusterService, repositoryService, clusterModule.getAllocationService(),
                 metaDataCreateIndexService, metaDataIndexUpgradeService, clusterService.getClusterSettings());
 
+            final RerouteService rerouteService
+                = new BatchedRerouteService(clusterService, clusterModule.getAllocationService()::reroute);
+            lazilyInitializedRerouteService.setRerouteService(rerouteService);
             final DiscoveryModule discoveryModule = new DiscoveryModule(settings, threadPool, transportService, namedWriteableRegistry,
                 networkService, clusterService.getMasterService(), clusterService.getClusterApplierService(),
                 clusterService.getClusterSettings(), pluginsService.filterPlugins(DiscoveryPlugin.class),
-                clusterModule.getAllocationService(), environment.configFile(), gatewayMetaState);
+                clusterModule.getAllocationService(), environment.configFile(), gatewayMetaState, rerouteService);
             this.nodeService = new NodeService(settings, threadPool, monitorService, discoveryModule.getDiscovery(),
                 transportService, indicesService, pluginsService, circuitBreakerService, scriptModule.getScriptService(),
                 httpServerTransport, ingestService, clusterService, settingsModule.getSettingsFilter(), responseCollectorService,
@@ -573,6 +587,7 @@ protected Node(
                     b.bind(SnapshotShardsService.class).toInstance(snapshotShardsService);
                     b.bind(TransportNodesSnapshotsStatus.class).toInstance(nodesSnapshotsStatus);
                     b.bind(RestoreService.class).toInstance(restoreService);
+                    b.bind(RerouteService.class).toInstance(rerouteService);
                 }
             );
             injector = modules.createInjector();
@@ -583,11 +598,9 @@ protected Node(
             List<LifecycleComponent> pluginLifecycleComponents = pluginComponents.stream()
                 .filter(p -> p instanceof LifecycleComponent)
                 .map(p -> (LifecycleComponent) p).collect(Collectors.toList());
-            pluginLifecycleComponents.addAll(pluginsService.getGuiceServiceClasses().stream()
-                .map(injector::getInstance).collect(Collectors.toList()));
             resourcesToClose.addAll(pluginLifecycleComponents);
             this.pluginLifecycleComponents = Collections.unmodifiableList(pluginLifecycleComponents);
-            client.initialize(injector.getInstance(new Key<Map<Action, TransportAction>>() {}),
+            client.initialize(injector.getInstance(new Key<Map<ActionType, TransportAction>>() {}),
                     () -> clusterService.localNode().getId(), transportService.getRemoteClusterService());
             this.namedWriteableRegistry = namedWriteableRegistry;
 
@@ -661,7 +674,6 @@ public Node start() throws NodeValidationException {
         injector.getInstance(IndicesClusterStateService.class).start();
         injector.getInstance(SnapshotsService.class).start();
         injector.getInstance(SnapshotShardsService.class).start();
-        injector.getInstance(RoutingService.class).start();
         injector.getInstance(SearchService.class).start();
         nodeService.getMonitorService().start();
 
@@ -779,7 +791,6 @@ private Node stop() {
         // This can confuse other nodes and delay things - mostly if we're the master and we're running tests.
         injector.getInstance(Discovery.class).stop();
         // we close indices first, so operations won't be allowed on it
-        injector.getInstance(RoutingService.class).stop();
         injector.getInstance(ClusterService.class).stop();
         injector.getInstance(NodeConnectionsService.class).stop();
         nodeService.getMonitorService().stop();
@@ -829,8 +840,6 @@ public synchronized void close() throws IOException {
         toClose.add(injector.getInstance(IndicesService.class));
         // close filter/fielddata caches after indices
         toClose.add(injector.getInstance(IndicesStore.class));
-        toClose.add(() -> stopWatch.stop().start("routing"));
-        toClose.add(injector.getInstance(RoutingService.class));
         toClose.add(() -> stopWatch.stop().start("cluster"));
         toClose.add(injector.getInstance(ClusterService.class));
         toClose.add(() -> stopWatch.stop().start("node_connections_service"));
diff --git a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java
index b09c33b59d2f5..ef216961489f9 100644
--- a/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java
+++ b/server/src/main/java/org/elasticsearch/persistent/CompletionPersistentTaskAction.java
@@ -18,9 +18,9 @@
  */
 package org.elasticsearch.persistent;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
@@ -34,6 +34,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -43,10 +44,10 @@
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
 /**
- * Action that is used by executor node to indicate that the persistent action finished or failed on the node and needs to be
+ * ActionType that is used by executor node to indicate that the persistent action finished or failed on the node and needs to be
  * removed from the cluster state in case of successful completion or restarted on some other node in case of failure.
  */
-public class CompletionPersistentTaskAction extends Action<PersistentTaskResponse> {
+public class CompletionPersistentTaskAction extends StreamableResponseActionType<PersistentTaskResponse> {
 
     public static final CompletionPersistentTaskAction INSTANCE = new CompletionPersistentTaskAction();
     public static final String NAME = "cluster:admin/persistent/completion";
@@ -161,7 +162,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state)
         }
 
         @Override
-        protected final void masterOperation(final Request request, ClusterState state,
+        protected final void masterOperation(Task ignoredTask, final Request request, ClusterState state,
                                              final ActionListener<PersistentTaskResponse> listener) {
             persistentTasksClusterService.completePersistentTask(request.taskId, request.allocationId, request.exception,
                 ActionListener.delegateFailure(listener,
diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java
index 260fabc67cdca..9a9a39cd32e04 100644
--- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java
+++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java
@@ -47,7 +47,7 @@
 
 /**
  * This component is responsible for coordination of execution of persistent tasks on individual nodes. It runs on all
- * non-transport client nodes in the cluster and monitors cluster state changes to detect started commands.
+ * nodes in the cluster and monitors cluster state changes to detect started commands.
  */
 public class PersistentTasksNodeService implements ClusterStateListener {
 
diff --git a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java
index 9664be255da17..f10560a807de1 100644
--- a/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java
+++ b/server/src/main/java/org/elasticsearch/persistent/PersistentTasksService.java
@@ -20,7 +20,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
@@ -129,7 +129,7 @@ public void sendRemoveRequest(final String taskId, final ActionListener<Persiste
      * The origin is set in the context and the listener is wrapped to ensure the proper context is restored
      */
     private <Req extends ActionRequest, Resp extends PersistentTaskResponse>
-        void execute(final Req request, final Action<Resp> action, final ActionListener<PersistentTask<?>> listener) {
+        void execute(final Req request, final ActionType<Resp> action, final ActionListener<PersistentTask<?>> listener) {
             try {
                 client.execute(action, request, ActionListener.map(listener, PersistentTaskResponse::getTask));
             } catch (Exception e) {
diff --git a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java
index 877033fe4f33a..8002be6626991 100644
--- a/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java
+++ b/server/src/main/java/org/elasticsearch/persistent/RemovePersistentTaskAction.java
@@ -18,9 +18,9 @@
  */
 package org.elasticsearch.persistent;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
@@ -34,13 +34,14 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
 import java.util.Objects;
 
-public class RemovePersistentTaskAction extends Action<PersistentTaskResponse> {
+public class RemovePersistentTaskAction extends StreamableResponseActionType<PersistentTaskResponse> {
 
     public static final RemovePersistentTaskAction INSTANCE = new RemovePersistentTaskAction();
     public static final String NAME = "cluster:admin/persistent/remove";
@@ -146,7 +147,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state)
         }
 
         @Override
-        protected final void masterOperation(final Request request, ClusterState state,
+        protected final void masterOperation(Task ignoredTask, final Request request, ClusterState state,
                                              final ActionListener<PersistentTaskResponse> listener) {
             persistentTasksClusterService.removePersistentTask(
                 request.taskId, ActionListener.delegateFailure(listener,
diff --git a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java
index 977239ee2b990..01fa8aa5edb1b 100644
--- a/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java
+++ b/server/src/main/java/org/elasticsearch/persistent/StartPersistentTaskAction.java
@@ -18,9 +18,9 @@
  */
 package org.elasticsearch.persistent;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
@@ -35,6 +35,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -46,7 +47,7 @@
 /**
  *  This action can be used to add the record for the persistent action to the cluster state.
  */
-public class StartPersistentTaskAction extends Action<PersistentTaskResponse> {
+public class StartPersistentTaskAction extends StreamableResponseActionType<PersistentTaskResponse> {
 
     public static final StartPersistentTaskAction INSTANCE = new StartPersistentTaskAction();
     public static final String NAME = "cluster:admin/persistent/start";
@@ -213,7 +214,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state)
         }
 
         @Override
-        protected final void masterOperation(final Request request, ClusterState state,
+        protected final void masterOperation(Task ignoredTask, final Request request, ClusterState state,
                                              final ActionListener<PersistentTaskResponse> listener) {
             persistentTasksClusterService.createPersistentTask(request.taskId, request.taskName, request.params,
                 ActionListener.delegateFailure(listener,
diff --git a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java
index 218154d37c9b7..e3514406d7080 100644
--- a/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java
+++ b/server/src/main/java/org/elasticsearch/persistent/UpdatePersistentTaskStatusAction.java
@@ -18,9 +18,9 @@
  */
 package org.elasticsearch.persistent;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
@@ -34,6 +34,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -42,7 +43,7 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-public class UpdatePersistentTaskStatusAction extends Action<PersistentTaskResponse> {
+public class UpdatePersistentTaskStatusAction extends StreamableResponseActionType<PersistentTaskResponse> {
 
     public static final UpdatePersistentTaskStatusAction INSTANCE = new UpdatePersistentTaskStatusAction();
     public static final String NAME = "cluster:admin/persistent/update_status";
@@ -176,7 +177,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state)
         }
 
         @Override
-        protected final void masterOperation(final Request request,
+        protected final void masterOperation(Task ignoredTask, final Request request,
                                              final ClusterState state,
                                              final ActionListener<PersistentTaskResponse> listener) {
             persistentTasksClusterService.updatePersistentTaskState(request.taskId, request.allocationId, request.state,
diff --git a/server/src/main/java/org/elasticsearch/plugins/ActionPlugin.java b/server/src/main/java/org/elasticsearch/plugins/ActionPlugin.java
index adc2fa8f0b282..ee6e026d5a535 100644
--- a/server/src/main/java/org/elasticsearch/plugins/ActionPlugin.java
+++ b/server/src/main/java/org/elasticsearch/plugins/ActionPlugin.java
@@ -19,17 +19,16 @@
 
 package org.elasticsearch.plugins;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.admin.indices.mapping.put.MappingRequestValidator;
-import org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction;
+import org.elasticsearch.action.RequestValidators;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -67,15 +66,15 @@ public interface ActionPlugin {
     }
 
     /**
-     * Client actions added by this plugin. This defaults to all of the {@linkplain Action} in
+     * Client actions added by this plugin. This defaults to all of the {@linkplain ActionType} in
      * {@linkplain ActionPlugin#getActions()}.
      */
-    default List<Action<? extends ActionResponse>> getClientActions() {
+    default List<ActionType<? extends ActionResponse>> getClientActions() {
         return getActions().stream().map(a -> a.action).collect(Collectors.toList());
     }
 
     /**
-     * Action filters added by this plugin.
+     * ActionType filters added by this plugin.
      */
     default List<ActionFilter> getActionFilters() {
         return Collections.emptyList();
@@ -129,22 +128,18 @@ default UnaryOperator<RestHandler> getRestHandlerWrapper(ThreadContext threadCon
     }
 
     final class ActionHandler<Request extends ActionRequest, Response extends ActionResponse> {
-        private final Action<Response> action;
+        private final ActionType<Response> action;
         private final Class<? extends TransportAction<Request, Response>> transportAction;
-        private final Class<?>[] supportTransportActions;
 
         /**
-         * Create a record of an action, the {@linkplain TransportAction} that handles it, and any supporting {@linkplain TransportActions}
-         * that are needed by that {@linkplain TransportAction}.
+         * Create a record of an action, the {@linkplain TransportAction} that handles it.
          */
-        public ActionHandler(Action<Response> action, Class<? extends TransportAction<Request, Response>> transportAction,
-                             Class<?>... supportTransportActions) {
+        public ActionHandler(ActionType<Response> action, Class<? extends TransportAction<Request, Response>> transportAction) {
             this.action = action;
             this.transportAction = transportAction;
-            this.supportTransportActions = supportTransportActions;
         }
 
-        public Action<Response> getAction() {
+        public ActionType<Response> getAction() {
             return action;
         }
 
@@ -152,17 +147,9 @@ public Class<? extends TransportAction<Request, Response>> getTransportAction()
             return transportAction;
         }
 
-        public Class<?>[] getSupportTransportActions() {
-            return supportTransportActions;
-        }
-
         @Override
         public String toString() {
-            StringBuilder b = new StringBuilder().append(action.name()).append(" is handled by ").append(transportAction.getName());
-            if (supportTransportActions.length > 0) {
-                b.append('[').append(Strings.arrayToCommaDelimitedString(supportTransportActions)).append(']');
-            }
-            return b.toString();
+            return action.name() + " is handled by " + transportAction.getName();
         }
 
         @Override
@@ -172,21 +159,25 @@ public boolean equals(Object obj) {
             }
             ActionHandler<?, ?> other = (ActionHandler<?, ?>) obj;
             return Objects.equals(action, other.action)
-                    && Objects.equals(transportAction, other.transportAction)
-                    && Objects.deepEquals(supportTransportActions, other.supportTransportActions);
+                    && Objects.equals(transportAction, other.transportAction);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(action, transportAction, supportTransportActions);
+            return Objects.hash(action, transportAction);
         }
     }
 
     /**
-     * Returns a collection of validators that are used by {@link TransportPutMappingAction.RequestValidators} to
-     * validate a {@link org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest} before the executing it.
+     * Returns a collection of validators that are used by {@link RequestValidators} to validate a
+     * {@link org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest} before the executing it.
      */
-    default Collection<MappingRequestValidator> mappingRequestValidators() {
+    default Collection<RequestValidators.RequestValidator<PutMappingRequest>> mappingRequestValidators() {
         return Collections.emptyList();
     }
+
+    default Collection<RequestValidators.RequestValidator<IndicesAliasesRequest>> indicesAliasesRequestValidators() {
+        return Collections.emptyList();
+    }
+
 }
diff --git a/server/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java b/server/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java
index 21cfbd0e99e8d..4757386efc5e8 100644
--- a/server/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java
+++ b/server/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.plugins;
 
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.coordination.ElectionStrategy;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -85,4 +86,11 @@ default Map<String, Supplier<SeedHostsProvider>> getSeedHostProviders(TransportS
      * {@link IllegalStateException} if the node and the cluster-state are incompatible.
      */
     default BiConsumer<DiscoveryNode,ClusterState> getJoinValidator() { return null; }
+
+    /**
+     * Allows plugging in election strategies (see {@link ElectionStrategy}) that define a customized notion of an election quorum.
+     */
+    default Map<String, ElectionStrategy> getElectionStrategies() {
+        return Collections.emptyMap();
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/plugins/Plugin.java b/server/src/main/java/org/elasticsearch/plugins/Plugin.java
index dcba40b58c21a..5bc8e9267a515 100644
--- a/server/src/main/java/org/elasticsearch/plugins/Plugin.java
+++ b/server/src/main/java/org/elasticsearch/plugins/Plugin.java
@@ -25,9 +25,9 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
@@ -50,6 +50,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.function.UnaryOperator;
 
 /**
@@ -82,21 +83,6 @@ protected Optional<String> getFeature() {
         return Optional.empty();
     }
 
-    /**
-     * Node level guice modules.
-     */
-    public Collection<Module> createGuiceModules() {
-        return Collections.emptyList();
-    }
-
-    /**
-     * Node level services that will be automatically started/stopped/closed. This classes must be constructed
-     * by injection with guice.
-     */
-    public Collection<Class<? extends LifecycleComponent>> getGuiceServiceClasses() {
-        return Collections.emptyList();
-    }
-
     /**
      * Returns components added by this plugin.
      *
@@ -236,6 +222,10 @@ public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
      */
     public List<BootstrapCheck> getBootstrapChecks() { return Collections.emptyList(); }
 
+    public Set<DiscoveryNodeRole> getRoles() {
+        return Set.of();
+    }
+
     /**
      * Close the resources opened by this plugin.
      *
diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginLoaderIndirection.java b/server/src/main/java/org/elasticsearch/plugins/PluginLoaderIndirection.java
index 7a07e4f08e849..f4c8f7794ea91 100644
--- a/server/src/main/java/org/elasticsearch/plugins/PluginLoaderIndirection.java
+++ b/server/src/main/java/org/elasticsearch/plugins/PluginLoaderIndirection.java
@@ -21,6 +21,7 @@
 
 import java.util.List;
 
+// TODO: remove this indirection now that transport client is gone
 /**
  * This class exists solely as an intermediate layer to avoid causing PluginsService
  * to load ExtendedPluginsClassLoader when used in the transport client.
diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java
index 240837077e1c3..feb18c61e3365 100644
--- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java
+++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java
@@ -33,8 +33,6 @@
 import org.elasticsearch.bootstrap.JarHell;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -114,7 +112,7 @@ public PluginsService(
         List<PluginInfo> pluginsList = new ArrayList<>();
         // we need to build a List of plugins for checking mandatory plugins
         final List<String> pluginsNames = new ArrayList<>();
-        // first we load plugins that are on the classpath. this is for tests and transport clients
+        // first we load plugins that are on the classpath. this is for tests
         for (Class<? extends Plugin> pluginClass : classpathPlugins) {
             Plugin plugin = loadPlugin(pluginClass, settings, configPath);
             PluginInfo pluginInfo = new PluginInfo(pluginClass.getName(), "classpath plugin", "NA", Version.CURRENT, "1.8",
@@ -237,14 +235,6 @@ public Settings updatedSettings() {
         return builder.put(this.settings).build();
     }
 
-    public Collection<Module> createGuiceModules() {
-        List<Module> modules = new ArrayList<>();
-        for (Tuple<PluginInfo, Plugin> plugin : plugins) {
-            modules.addAll(plugin.v2().createGuiceModules());
-        }
-        return modules;
-    }
-
     public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
         final ArrayList<ExecutorBuilder<?>> builders = new ArrayList<>();
         for (final Tuple<PluginInfo, Plugin> plugin : plugins) {
@@ -253,15 +243,6 @@ public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
         return builders;
     }
 
-    /** Returns all classes injected into guice by plugins which extend {@link LifecycleComponent}. */
-    public Collection<Class<? extends LifecycleComponent>> getGuiceServiceClasses() {
-        List<Class<? extends LifecycleComponent>> services = new ArrayList<>();
-        for (Tuple<PluginInfo, Plugin> plugin : plugins) {
-            services.addAll(plugin.v2().getGuiceServiceClasses());
-        }
-        return services;
-    }
-
     public void onIndexModule(IndexModule indexModule) {
         for (Tuple<PluginInfo, Plugin> plugin : plugins) {
             plugin.v2().onIndexModule(indexModule);
diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java
index 1fa42579617e1..8c9eff0698835 100644
--- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java
@@ -38,6 +38,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 public class FilterRepository implements Repository {
 
@@ -79,9 +80,10 @@ public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, Met
 
     @Override
     public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<IndexId> indices, long startTime, String failure, int totalShards,
-                                         List<SnapshotShardFailure> shardFailures, long repositoryStateId, boolean includeGlobalState) {
+                                         List<SnapshotShardFailure> shardFailures, long repositoryStateId, boolean includeGlobalState,
+                                         Map<String, Object> userMetadata) {
         return in.finalizeSnapshot(snapshotId, indices, startTime, failure, totalShards, shardFailures, repositoryStateId,
-            includeGlobalState);
+            includeGlobalState, userMetadata);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java
index 0eca92039fbf8..788459b16c540 100644
--- a/server/src/main/java/org/elasticsearch/repositories/Repository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java
@@ -37,6 +37,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 import java.util.function.Function;
 
 /**
@@ -134,7 +135,8 @@ default Repository create(RepositoryMetaData metaData, Function<String, Reposito
      * @return snapshot description
      */
     SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<IndexId> indices, long startTime, String failure, int totalShards,
-                                  List<SnapshotShardFailure> shardFailures, long repositoryStateId, boolean includeGlobalState);
+                                  List<SnapshotShardFailure> shardFailures, long repositoryStateId, boolean includeGlobalState,
+                                  Map<String, Object> userMetadata);
 
     /**
      * Deletes snapshot
diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
index 589e0432c032d..3d0daff98936e 100644
--- a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
+++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
@@ -442,8 +442,7 @@ public static RepositoryData snapshotsFromXContent(final XContentParser parser,
     /**
      * Writes the incompatible snapshot ids to x-content.
      */
-    public void incompatibleSnapshotsToXContent(final XContentBuilder builder) throws IOException {
-
+    public XContentBuilder incompatibleSnapshotsToXContent(XContentBuilder builder) throws IOException {
         builder.startObject();
         // write the incompatible snapshots list
         builder.startArray(INCOMPATIBLE_SNAPSHOTS);
@@ -452,6 +451,7 @@ public void incompatibleSnapshotsToXContent(final XContentBuilder builder) throw
         }
         builder.endArray();
         builder.endObject();
+        return builder;
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index 49b551b26b796..25a3bf0974f33 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -50,9 +50,8 @@
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.compress.NotXContentException;
+import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
-import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
 import org.elasticsearch.common.metrics.CounterMetric;
@@ -60,15 +59,11 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.core.internal.io.Streams;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException;
@@ -128,7 +123,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     protected final RepositoryMetaData metadata;
 
-    private final ThreadPool threadPool;
+    protected final ThreadPool threadPool;
 
     private static final int BUFFER_SIZE = 4096;
 
@@ -197,6 +192,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private final SetOnce<BlobStore> blobStore = new SetOnce<>();
 
+    private final BlobPath basePath;
+
     /**
      * Constructs new BlobStoreRepository
      * @param metadata   The metadata for this repository including name and settings
@@ -204,7 +201,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
      * @param threadPool Threadpool to run long running repository manipulations on asynchronously
      */
     protected BlobStoreRepository(RepositoryMetaData metadata, Settings settings, NamedXContentRegistry namedXContentRegistry,
-                                  ThreadPool threadPool) {
+                                  ThreadPool threadPool, BlobPath basePath) {
         this.settings = settings;
         this.metadata = metadata;
         this.threadPool = threadPool;
@@ -212,6 +209,7 @@ protected BlobStoreRepository(RepositoryMetaData metadata, Settings settings, Na
         snapshotRateLimiter = getRateLimiter(metadata.settings(), "max_snapshot_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
         restoreRateLimiter = getRateLimiter(metadata.settings(), "max_restore_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
         readOnly = metadata.settings().getAsBoolean("readonly", false);
+        this.basePath = basePath;
 
         indexShardSnapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT,
             BlobStoreIndexShardSnapshot::fromXContent, namedXContentRegistry, compress);
@@ -252,6 +250,10 @@ protected void doClose() {
         }
     }
 
+    public ThreadPool threadPool() {
+        return threadPool;
+    }
+
     // package private, only use for testing
     BlobContainer getBlobContainer() {
         return blobContainer.get();
@@ -283,9 +285,10 @@ protected BlobContainer blobContainer() {
     }
 
     /**
-     * maintains single lazy instance of {@link BlobStore}
+     * Maintains single lazy instance of {@link BlobStore}.
+     * Public for testing.
      */
-    protected BlobStore blobStore() {
+    public BlobStore blobStore() {
         assertSnapshotOrGenericThread();
 
         BlobStore store = blobStore.get();
@@ -317,8 +320,11 @@ protected BlobStore blobStore() {
 
     /**
      * Returns base path of the repository
+     * Public for testing.
      */
-    protected abstract BlobPath basePath();
+    public BlobPath basePath() {
+        return basePath;
+    }
 
     /**
      * Returns true if metadata and snapshot files should be compressed
@@ -363,10 +369,7 @@ public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, Met
 
             // write the index metadata for each index in the snapshot
             for (IndexId index : indices) {
-                final IndexMetaData indexMetaData = clusterMetaData.index(index.getName());
-                final BlobPath indexPath = basePath().add("indices").add(index.getId());
-                final BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
-                indexMetaDataFormat.write(indexMetaData, indexMetaDataBlobContainer, snapshotId.getUUID());
+                indexMetaDataFormat.write(clusterMetaData.index(index.getName()), indexContainer(index), snapshotId.getUUID());
             }
         } catch (IOException ex) {
             throw new SnapshotCreationException(metadata.name(), snapshotId, ex);
@@ -388,46 +391,68 @@ public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId, Action
                 logger.warn(() -> new ParameterizedMessage("cannot read snapshot file [{}]", snapshotId), ex);
             }
             // Delete snapshot from the index file, since it is the maintainer of truth of active snapshots
-            final RepositoryData repositoryData;
             final RepositoryData updatedRepositoryData;
+            final Map<String, BlobContainer> foundIndices;
             try {
-                repositoryData = getRepositoryData();
+                final RepositoryData repositoryData = getRepositoryData();
                 updatedRepositoryData = repositoryData.removeSnapshot(snapshotId);
+                // Cache the indices that were found before writing out the new index-N blob so that a stuck master will never
+                // delete an index that was created by another master node after writing this index-N blob.
+                foundIndices = blobStore().blobContainer(basePath().add("indices")).children();
                 writeIndexGen(updatedRepositoryData, repositoryStateId);
             } catch (Exception ex) {
                 listener.onFailure(new RepositoryException(metadata.name(), "failed to delete snapshot [" + snapshotId + "]", ex));
                 return;
             }
             final SnapshotInfo finalSnapshotInfo = snapshot;
-            final Collection<IndexId> unreferencedIndices = Sets.newHashSet(repositoryData.getIndices().values());
-            unreferencedIndices.removeAll(updatedRepositoryData.getIndices().values());
             try {
                 blobContainer().deleteBlobsIgnoringIfNotExists(
                     Arrays.asList(snapshotFormat.blobName(snapshotId.getUUID()), globalMetaDataFormat.blobName(snapshotId.getUUID())));
             } catch (IOException e) {
                 logger.warn(() -> new ParameterizedMessage("[{}] Unable to delete global metadata files", snapshotId), e);
             }
+            final var survivingIndices = updatedRepositoryData.getIndices();
             deleteIndices(
                 Optional.ofNullable(finalSnapshotInfo)
-                    .map(info -> info.indices().stream().map(repositoryData::resolveIndexId).collect(Collectors.toList()))
+                    .map(info -> info.indices().stream().filter(survivingIndices::containsKey)
+                        .map(updatedRepositoryData::resolveIndexId).collect(Collectors.toList()))
                     .orElse(Collections.emptyList()),
                 snapshotId,
                 ActionListener.map(listener, v -> {
-                    try {
-                        blobStore().blobContainer(basePath().add("indices")).deleteBlobsIgnoringIfNotExists(
-                            unreferencedIndices.stream().map(IndexId::getId).collect(Collectors.toList()));
-                    } catch (IOException e) {
-                        logger.warn(() ->
-                            new ParameterizedMessage(
-                                "[{}] indices {} are no longer part of any snapshots in the repository, " +
-                                    "but failed to clean up their index folders.", metadata.name(), unreferencedIndices), e);
-                    }
+                    cleanupStaleIndices(foundIndices, survivingIndices);
                     return null;
                 })
             );
         }
     }
 
+    private void cleanupStaleIndices(Map<String, BlobContainer> foundIndices, Map<String, IndexId> survivingIndices) {
+        try {
+            final Set<String> survivingIndexIds = survivingIndices.values().stream()
+                .map(IndexId::getId).collect(Collectors.toSet());
+            for (Map.Entry<String, BlobContainer> indexEntry : foundIndices.entrySet()) {
+                final String indexSnId = indexEntry.getKey();
+                try {
+                    if (survivingIndexIds.contains(indexSnId) == false) {
+                        logger.debug("[{}] Found stale index [{}]. Cleaning it up", metadata.name(), indexSnId);
+                        indexEntry.getValue().delete();
+                        logger.debug("[{}] Cleaned up stale index [{}]", metadata.name(), indexSnId);
+                    }
+                } catch (IOException e) {
+                    logger.warn(() -> new ParameterizedMessage(
+                        "[{}] index {} is no longer part of any snapshots in the repository, " +
+                            "but failed to clean up their index folders", metadata.name(), indexSnId), e);
+                }
+            }
+        } catch (Exception e) {
+            // TODO: We shouldn't be blanket catching and suppressing all exceptions here and instead handle them safely upstream.
+            //       Currently this catch exists as a stop gap solution to tackle unexpected runtime exceptions from implementations
+            //       bubbling up and breaking the snapshot functionality.
+            assert false : e;
+            logger.warn(new ParameterizedMessage("[{}] Exception during cleanup of stale indices", metadata.name()), e);
+        }
+    }
+
     private void deleteIndices(List<IndexId> indices, SnapshotId snapshotId, ActionListener<Void> listener) {
         if (indices.isEmpty()) {
             listener.onResponse(null);
@@ -450,8 +475,7 @@ protected void doRun() {
                     if (indexMetaData != null) {
                         for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) {
                             try {
-                                final ShardId sid = new ShardId(indexMetaData.getIndex(), shardId);
-                                new Context(snapshotId, indexId, sid, sid).delete();
+                                deleteShardSnapshot(indexId, new ShardId(indexMetaData.getIndex(), shardId), snapshotId);
                             } catch (SnapshotException ex) {
                                 final int finalShardId = shardId;
                                 logger.warn(() -> new ParameterizedMessage("[{}] failed to delete shard data for shard [{}][{}]",
@@ -466,9 +490,8 @@ protected void doRun() {
     }
 
     private void deleteIndexMetaDataBlobIgnoringErrors(SnapshotId snapshotId, IndexId indexId) {
-        BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(basePath().add("indices").add(indexId.getId()));
         try {
-            indexMetaDataFormat.delete(indexMetaDataBlobContainer, snapshotId.getUUID());
+            indexMetaDataFormat.delete(indexContainer(indexId), snapshotId.getUUID());
         } catch (IOException ex) {
             logger.warn(() -> new ParameterizedMessage("[{}] failed to delete metadata for index [{}]",
                 snapshotId, indexId.getName()), ex);
@@ -483,15 +506,16 @@ public SnapshotInfo finalizeSnapshot(final SnapshotId snapshotId,
                                          final int totalShards,
                                          final List<SnapshotShardFailure> shardFailures,
                                          final long repositoryStateId,
-                                         final boolean includeGlobalState) {
+                                         final boolean includeGlobalState,
+                                         final Map<String, Object> userMetadata) {
         SnapshotInfo blobStoreSnapshot = new SnapshotInfo(snapshotId,
             indices.stream().map(IndexId::getName).collect(Collectors.toList()),
             startTime, failure, System.currentTimeMillis(), totalShards, shardFailures,
-            includeGlobalState);
+            includeGlobalState, userMetadata);
         try {
+            final RepositoryData updatedRepositoryData = getRepositoryData().addSnapshot(snapshotId, blobStoreSnapshot.state(), indices);
             snapshotFormat.write(blobStoreSnapshot, blobContainer(), snapshotId.getUUID());
-            final RepositoryData repositoryData = getRepositoryData();
-            writeIndexGen(repositoryData.addSnapshot(snapshotId, blobStoreSnapshot.state(), indices), repositoryStateId);
+            writeIndexGen(updatedRepositoryData, repositoryStateId);
         } catch (FileAlreadyExistsException ex) {
             // if another master was elected and took over finalizing the snapshot, it is possible
             // that both nodes try to finalize the snapshot and write to the same blobs, so we just
@@ -528,8 +552,19 @@ public MetaData getSnapshotGlobalMetaData(final SnapshotId snapshotId) {
 
     @Override
     public IndexMetaData getSnapshotIndexMetaData(final SnapshotId snapshotId, final IndexId index) throws IOException {
-        final BlobPath indexPath = basePath().add("indices").add(index.getId());
-        return indexMetaDataFormat.read(blobStore().blobContainer(indexPath), snapshotId.getUUID());
+        return indexMetaDataFormat.read(indexContainer(index), snapshotId.getUUID());
+    }
+
+    private BlobPath indicesPath() {
+        return basePath().add("indices");
+    }
+
+    private BlobContainer indexContainer(IndexId indexId) {
+        return blobStore().blobContainer(indicesPath().add(indexId.getId()));
+    }
+
+    private BlobContainer shardContainer(IndexId indexId, ShardId shardId) {
+        return blobStore().blobContainer(indicesPath().add(indexId.getId()).add(Integer.toString(shardId.getId())));
     }
 
     /**
@@ -577,10 +612,9 @@ public String startVerification() {
                 String seed = UUIDs.randomBase64UUID();
                 byte[] testBytes = Strings.toUTF8Bytes(seed);
                 BlobContainer testContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed)));
-                String blobName = "master.dat";
                 BytesArray bytes = new BytesArray(testBytes);
                 try (InputStream stream = bytes.streamInput()) {
-                    testContainer.writeBlobAtomic(blobName, stream, bytes.length(), true);
+                    testContainer.writeBlobAtomic("master.dat", stream, bytes.length(), true);
                 }
                 return seed;
             }
@@ -610,32 +644,23 @@ public RepositoryData getRepositoryData() {
             final String snapshotsIndexBlobName = INDEX_FILE_PREFIX + Long.toString(indexGen);
 
             RepositoryData repositoryData;
-            try (InputStream blob = blobContainer().readBlob(snapshotsIndexBlobName)) {
-                BytesStreamOutput out = new BytesStreamOutput();
-                Streams.copy(blob, out);
-                // EMPTY is safe here because RepositoryData#fromXContent calls namedObject
-                try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
-                    LoggingDeprecationHandler.INSTANCE, out.bytes(), XContentType.JSON)) {
-                    repositoryData = RepositoryData.snapshotsFromXContent(parser, indexGen);
-                } catch (NotXContentException e) {
-                    logger.warn("[{}] index blob is not valid x-content [{} bytes]", snapshotsIndexBlobName, out.bytes().length());
-                    throw e;
-                }
+            // EMPTY is safe here because RepositoryData#fromXContent calls namedObject
+            try (InputStream blob = blobContainer().readBlob(snapshotsIndexBlobName);
+                 XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY,
+                     LoggingDeprecationHandler.INSTANCE, blob)) {
+                repositoryData = RepositoryData.snapshotsFromXContent(parser, indexGen);
             }
 
             // now load the incompatible snapshot ids, if they exist
-            try (InputStream blob = blobContainer().readBlob(INCOMPATIBLE_SNAPSHOTS_BLOB)) {
-                BytesStreamOutput out = new BytesStreamOutput();
-                Streams.copy(blob, out);
-                try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
-                    LoggingDeprecationHandler.INSTANCE, out.bytes(), XContentType.JSON)) {
-                    repositoryData = repositoryData.incompatibleSnapshotsFromXContent(parser);
-                }
+            try (InputStream blob = blobContainer().readBlob(INCOMPATIBLE_SNAPSHOTS_BLOB);
+                 XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY,
+                     LoggingDeprecationHandler.INSTANCE, blob)) {
+                repositoryData = repositoryData.incompatibleSnapshotsFromXContent(parser);
             } catch (NoSuchFileException e) {
                 if (isReadOnly()) {
                     logger.debug("[{}] Incompatible snapshots blob [{}] does not exist, the likely " +
-                                 "reason is that there are no incompatible snapshots in the repository",
-                                 metadata.name(), INCOMPATIBLE_SNAPSHOTS_BLOB);
+                            "reason is that there are no incompatible snapshots in the repository",
+                        metadata.name(), INCOMPATIBLE_SNAPSHOTS_BLOB);
                 } else {
                     // write an empty incompatible-snapshots blob - we do this so that there
                     // is a blob present, which helps speed up some cloud-based repositories
@@ -653,7 +678,7 @@ public RepositoryData getRepositoryData() {
         }
     }
 
-    public static String testBlobPrefix(String seed) {
+    private static String testBlobPrefix(String seed) {
         return TESTS_FILE + seed;
     }
 
@@ -673,25 +698,10 @@ protected void writeIndexGen(final RepositoryData repositoryData, final long rep
                                               "] - possibly due to simultaneous snapshot deletion requests");
         }
         final long newGen = currentGen + 1;
-        final BytesReference snapshotsBytes;
-        try (BytesStreamOutput bStream = new BytesStreamOutput()) {
-            try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) {
-                XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
-                repositoryData.snapshotsToXContent(builder);
-                builder.close();
-            }
-            snapshotsBytes = bStream.bytes();
-        }
         // write the index file
         final String indexBlob = INDEX_FILE_PREFIX + Long.toString(newGen);
         logger.debug("Repository [{}] writing new index generational blob [{}]", metadata.name(), indexBlob);
-        writeAtomic(indexBlob, snapshotsBytes, true);
-        // delete the N-2 index file if it exists, keep the previous one around as a backup
-        if (isReadOnly() == false && newGen - 2 >= 0) {
-            final String oldSnapshotIndexFile = INDEX_FILE_PREFIX + Long.toString(newGen - 2);
-            blobContainer().deleteBlobIgnoringIfNotExists(oldSnapshotIndexFile);
-        }
-
+        writeAtomic(indexBlob, BytesReference.bytes(repositoryData.snapshotsToXContent(XContentFactory.jsonBuilder())), true);
         // write the current generation to the index-latest file
         final BytesReference genBytes;
         try (BytesStreamOutput bStream = new BytesStreamOutput()) {
@@ -700,6 +710,15 @@ protected void writeIndexGen(final RepositoryData repositoryData, final long rep
         }
         logger.debug("Repository [{}] updating index.latest with generation [{}]", metadata.name(), newGen);
         writeAtomic(INDEX_LATEST_BLOB, genBytes, false);
+        // delete the N-2 index file if it exists, keep the previous one around as a backup
+        if (newGen - 2 >= 0) {
+            final String oldSnapshotIndexFile = INDEX_FILE_PREFIX + Long.toString(newGen - 2);
+            try {
+                blobContainer().deleteBlobIgnoringIfNotExists(oldSnapshotIndexFile);
+            } catch (IOException e) {
+                logger.warn("Failed to clean up old index blob [{}]", oldSnapshotIndexFile);
+            }
+        }
     }
 
     /**
@@ -709,16 +728,9 @@ protected void writeIndexGen(final RepositoryData repositoryData, final long rep
      */
     void writeIncompatibleSnapshots(RepositoryData repositoryData) throws IOException {
         assert isReadOnly() == false; // can not write to a read only repository
-        final BytesReference bytes;
-        try (BytesStreamOutput bStream = new BytesStreamOutput()) {
-            try (StreamOutput stream = new OutputStreamStreamOutput(bStream);
-                 XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream)) {
-                repositoryData.incompatibleSnapshotsToXContent(builder);
-            }
-            bytes = bStream.bytes();
-        }
         // write the incompatible snapshots blob
-        writeAtomic(INCOMPATIBLE_SNAPSHOTS_BLOB, bytes, false);
+        writeAtomic(INCOMPATIBLE_SNAPSHOTS_BLOB,
+            BytesReference.bytes(repositoryData.incompatibleSnapshotsToXContent(XContentFactory.jsonBuilder())), false);
     }
 
     /**
@@ -756,11 +768,7 @@ long latestIndexBlobId() throws IOException {
 
     // package private for testing
     long readSnapshotIndexLatestBlob() throws IOException {
-        try (InputStream blob = blobContainer().readBlob(INDEX_LATEST_BLOB)) {
-            BytesStreamOutput out = new BytesStreamOutput();
-            Streams.copy(blob, out);
-            return Numbers.bytesToLong(out.bytes().toBytesRef());
-        }
+        return Numbers.bytesToLong(Streams.readFully(blobContainer().readBlob(INDEX_LATEST_BLOB)).toBytesRef());
     }
 
     private long listBlobsToGetLatestIndexId() throws IOException {
@@ -793,306 +801,20 @@ private void writeAtomic(final String blobName, final BytesReference bytesRef, b
     @Override
     public void snapshotShard(Store store, MapperService mapperService, SnapshotId snapshotId, IndexId indexId,
                               IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus) {
-        SnapshotContext snapshotContext = new SnapshotContext(store, snapshotId, indexId, snapshotStatus, System.currentTimeMillis());
+        final ShardId shardId = store.shardId();
+        final long startTime = threadPool.relativeTimeInMillis();
         try {
-            snapshotContext.snapshot(snapshotIndexCommit);
-        } catch (Exception e) {
-            snapshotStatus.moveToFailed(System.currentTimeMillis(), ExceptionsHelper.detailedMessage(e));
-            if (e instanceof IndexShardSnapshotFailedException) {
-                throw (IndexShardSnapshotFailedException) e;
-            } else {
-                throw new IndexShardSnapshotFailedException(store.shardId(), e);
-            }
-        }
-    }
-
-    @Override
-    public void restoreShard(Store store, SnapshotId snapshotId,
-                             Version version, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState) {
-        ShardId shardId = store.shardId();
-        final Context context = new Context(snapshotId, indexId, shardId, snapshotShardId);
-        BlobPath path = basePath().add("indices").add(indexId.getId()).add(Integer.toString(snapshotShardId.getId()));
-        BlobContainer blobContainer = blobStore().blobContainer(path);
-        final RestoreContext snapshotContext = new RestoreContext(shardId, snapshotId, recoveryState, blobContainer);
-        try {
-            BlobStoreIndexShardSnapshot snapshot = context.loadSnapshot();
-            SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles());
-            snapshotContext.restore(snapshotFiles, store);
-        } catch (Exception e) {
-            throw new IndexShardRestoreFailedException(shardId, "failed to restore snapshot [" + snapshotId + "]", e);
-        }
-    }
-
-    @Override
-    public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, Version version, IndexId indexId, ShardId shardId) {
-        Context context = new Context(snapshotId, indexId, shardId);
-        BlobStoreIndexShardSnapshot snapshot = context.loadSnapshot();
-        return IndexShardSnapshotStatus.newDone(snapshot.startTime(), snapshot.time(),
-            snapshot.incrementalFileCount(), snapshot.totalFileCount(),
-            snapshot.incrementalSize(), snapshot.totalSize());
-    }
-
-    @Override
-    public void verify(String seed, DiscoveryNode localNode) {
-        assertSnapshotOrGenericThread();
-        if (isReadOnly()) {
-            try {
-                latestIndexBlobId();
-            } catch (IOException e) {
-                throw new RepositoryVerificationException(metadata.name(), "path " + basePath() +
-                    " is not accessible on node " + localNode, e);
-            }
-        } else {
-            BlobContainer testBlobContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed)));
-            if (testBlobContainer.blobExists("master.dat")) {
-                try {
-                    BytesArray bytes = new BytesArray(seed);
-                    try (InputStream stream = bytes.streamInput()) {
-                        testBlobContainer.writeBlob("data-" + localNode.getId() + ".dat", stream, bytes.length(), true);
-                    }
-                } catch (IOException exp) {
-                    throw new RepositoryVerificationException(metadata.name(), "store location [" + blobStore() +
-                        "] is not accessible on the node [" + localNode + "]", exp);
-                }
-            } else {
-                throw new RepositoryVerificationException(metadata.name(), "a file written by master to the store [" + blobStore() +
-                    "] cannot be accessed on the node [" + localNode + "]. " +
-                    "This might indicate that the store [" + blobStore() + "] is not shared between this node and the master node or " +
-                    "that permissions on the store don't allow reading files written by the master node");
-            }
-        }
-    }
-
-    @Override
-    public String toString() {
-        return "BlobStoreRepository[" +
-            "[" + metadata.name() +
-            "], [" + blobStore() + ']' +
-            ']';
-    }
-
-    /**
-     * Context for snapshot/restore operations
-     */
-    private class Context {
-
-        protected final SnapshotId snapshotId;
-
-        protected final ShardId shardId;
-
-        protected final BlobContainer blobContainer;
-
-        Context(SnapshotId snapshotId, IndexId indexId, ShardId shardId) {
-            this(snapshotId, indexId, shardId, shardId);
-        }
-
-        Context(SnapshotId snapshotId, IndexId indexId, ShardId shardId, ShardId snapshotShardId) {
-            this.snapshotId = snapshotId;
-            this.shardId = shardId;
-            blobContainer = blobStore().blobContainer(basePath().add("indices").add(indexId.getId())
-                .add(Integer.toString(snapshotShardId.getId())));
-        }
-
-        /**
-         * Delete shard snapshot
-         */
-        public void delete() {
-            final Map<String, BlobMetaData> blobs;
-            try {
-                blobs = blobContainer.listBlobs();
-            } catch (IOException e) {
-                throw new IndexShardSnapshotException(shardId, "Failed to list content of gateway", e);
-            }
-
-            Tuple<BlobStoreIndexShardSnapshots, Integer> tuple = buildBlobStoreIndexShardSnapshots(blobs);
-            BlobStoreIndexShardSnapshots snapshots = tuple.v1();
-            int fileListGeneration = tuple.v2();
-
-            try {
-                indexShardSnapshotFormat.delete(blobContainer, snapshotId.getUUID());
-            } catch (IOException e) {
-                logger.warn(new ParameterizedMessage("[{}] [{}] failed to delete shard snapshot file", shardId, snapshotId), e);
-            }
-
-            // Build a list of snapshots that should be preserved
-            List<SnapshotFiles> newSnapshotsList = new ArrayList<>();
-            for (SnapshotFiles point : snapshots) {
-                if (!point.snapshot().equals(snapshotId.getName())) {
-                    newSnapshotsList.add(point);
-                }
-            }
-            // finalize the snapshot and rewrite the snapshot index with the next sequential snapshot index
-            finalize(newSnapshotsList, fileListGeneration + 1, blobs, "snapshot deletion [" + snapshotId + "]");
-        }
-
-        /**
-         * Loads information about shard snapshot
-         */
-        BlobStoreIndexShardSnapshot loadSnapshot() {
-            try {
-                return indexShardSnapshotFormat.read(blobContainer, snapshotId.getUUID());
-            } catch (IOException ex) {
-                throw new SnapshotException(metadata.name(), snapshotId, "failed to read shard snapshot file for " + shardId, ex);
-            }
-        }
-
-        /**
-         * Writes a new index file for the shard and removes all unreferenced files from the repository.
-         *
-         * We need to be really careful in handling index files in case of failures to make sure we don't
-         * have index file that points to files that were deleted.
-         *
-         * @param snapshots          list of active snapshots in the container
-         * @param fileListGeneration the generation number of the snapshot index file
-         * @param blobs              list of blobs in the container
-         * @param reason             a reason explaining why the shard index file is written
-         */
-        protected void finalize(final List<SnapshotFiles> snapshots,
-                                final int fileListGeneration,
-                                final Map<String, BlobMetaData> blobs,
-                                final String reason) {
-            final String indexGeneration = Integer.toString(fileListGeneration);
-            final BlobStoreIndexShardSnapshots updatedSnapshots = new BlobStoreIndexShardSnapshots(snapshots);
-            try {
-                // Delete temporary index files first, as we might otherwise fail in the next step creating the new index file if an earlier
-                // attempt to write an index file with this generation failed mid-way after creating the temporary file.
-                final List<String> blobNames =
-                    blobs.keySet().stream().filter(FsBlobContainer::isTempBlobName).collect(Collectors.toList());
-                try {
-                    blobContainer.deleteBlobsIgnoringIfNotExists(blobNames);
-                } catch (IOException e) {
-                    logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to delete index blobs during finalization",
-                        snapshotId, shardId), e);
-                    throw e;
-                }
-
-                // If we deleted all snapshots, we don't need to create a new index file
-                if (snapshots.size() > 0) {
-                    indexShardSnapshotsFormat.writeAtomic(updatedSnapshots, blobContainer, indexGeneration);
-                }
-
-                // Delete old index files
-                final List<String> indexBlobs =
-                    blobs.keySet().stream().filter(blob -> blob.startsWith(SNAPSHOT_INDEX_PREFIX)).collect(Collectors.toList());
-                try {
-                    blobContainer.deleteBlobsIgnoringIfNotExists(indexBlobs);
-                } catch (IOException e) {
-                    logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to delete index blobs during finalization",
-                        snapshotId, shardId), e);
-                    throw e;
-                }
-
-                // Delete all blobs that don't exist in a snapshot
-                final List<String> orphanedBlobs = blobs.keySet().stream()
-                    .filter(blobName ->
-                        blobName.startsWith(DATA_BLOB_PREFIX) && updatedSnapshots.findNameFile(canonicalName(blobName)) == null)
-                    .collect(Collectors.toList());
-                try {
-                    blobContainer.deleteBlobsIgnoringIfNotExists(orphanedBlobs);
-                } catch (IOException e) {
-                    logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to delete data blobs during finalization",
-                        snapshotId, shardId), e);
-                }
-            } catch (IOException e) {
-                String message =
-                    "Failed to finalize " + reason + " with shard index [" + indexShardSnapshotsFormat.blobName(indexGeneration) + "]";
-                throw new IndexShardSnapshotFailedException(shardId, message, e);
-            }
-        }
-
-        /**
-         * Loads all available snapshots in the repository
-         *
-         * @param blobs list of blobs in repository
-         * @return tuple of BlobStoreIndexShardSnapshots and the last snapshot index generation
-         */
-        protected Tuple<BlobStoreIndexShardSnapshots, Integer> buildBlobStoreIndexShardSnapshots(Map<String, BlobMetaData> blobs) {
-            int latest = -1;
-            Set<String> blobKeys = blobs.keySet();
-            for (String name : blobKeys) {
-                if (name.startsWith(SNAPSHOT_INDEX_PREFIX)) {
-                    try {
-                        int gen = Integer.parseInt(name.substring(SNAPSHOT_INDEX_PREFIX.length()));
-                        if (gen > latest) {
-                            latest = gen;
-                        }
-                    } catch (NumberFormatException ex) {
-                        logger.warn("failed to parse index file name [{}]", name);
-                    }
-                }
-            }
-            if (latest >= 0) {
-                try {
-                    final BlobStoreIndexShardSnapshots shardSnapshots =
-                        indexShardSnapshotsFormat.read(blobContainer, Integer.toString(latest));
-                    return new Tuple<>(shardSnapshots, latest);
-                } catch (IOException e) {
-                    final String file = SNAPSHOT_INDEX_PREFIX + latest;
-                    logger.warn(() -> new ParameterizedMessage("failed to read index file [{}]", file), e);
-                }
-            } else if (blobKeys.isEmpty() == false) {
-                logger.warn("Could not find a readable index-N file in a non-empty shard snapshot directory [{}]", blobContainer.path());
-            }
-
-            // We couldn't load the index file - falling back to loading individual snapshots
-            List<SnapshotFiles> snapshots = new ArrayList<>();
-            for (String name : blobKeys) {
-                try {
-                    BlobStoreIndexShardSnapshot snapshot = null;
-                    if (name.startsWith(SNAPSHOT_PREFIX)) {
-                        snapshot = indexShardSnapshotFormat.readBlob(blobContainer, name);
-                    }
-                    if (snapshot != null) {
-                        snapshots.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles()));
-                    }
-                } catch (IOException e) {
-                    logger.warn(() -> new ParameterizedMessage("failed to read commit point [{}]", name), e);
-                }
-            }
-            return new Tuple<>(new BlobStoreIndexShardSnapshots(snapshots), latest);
-        }
-    }
-
-    /**
-     * Context for snapshot operations
-     */
-    private class SnapshotContext extends Context {
-
-        private final Store store;
-        private final IndexShardSnapshotStatus snapshotStatus;
-        private final long startTime;
-
-        /**
-         * Constructs new context
-         *
-         * @param store          store to be snapshotted
-         * @param snapshotId     snapshot id
-         * @param indexId        the id of the index being snapshotted
-         * @param snapshotStatus snapshot status to report progress
-         */
-        SnapshotContext(Store store, SnapshotId snapshotId, IndexId indexId, IndexShardSnapshotStatus snapshotStatus, long startTime) {
-            super(snapshotId, indexId, store.shardId());
-            this.snapshotStatus = snapshotStatus;
-            this.store = store;
-            this.startTime = startTime;
-        }
-
-        /**
-         * Create snapshot from index commit point
-         *
-         * @param snapshotIndexCommit snapshot commit point
-         */
-        public void snapshot(final IndexCommit snapshotIndexCommit) {
             logger.debug("[{}] [{}] snapshot to [{}] ...", shardId, snapshotId, metadata.name());
 
+            final BlobContainer shardContainer = shardContainer(indexId, shardId);
             final Map<String, BlobMetaData> blobs;
             try {
-                blobs = blobContainer.listBlobs();
+                blobs = shardContainer.listBlobs();
             } catch (IOException e) {
                 throw new IndexShardSnapshotFailedException(shardId, "failed to list blobs", e);
             }
 
-            Tuple<BlobStoreIndexShardSnapshots, Integer> tuple = buildBlobStoreIndexShardSnapshots(blobs);
+            Tuple<BlobStoreIndexShardSnapshots, Integer> tuple = buildBlobStoreIndexShardSnapshots(blobs, shardContainer);
             BlobStoreIndexShardSnapshots snapshots = tuple.v1();
             int fileListGeneration = tuple.v2();
 
@@ -1102,24 +824,24 @@ public void snapshot(final IndexCommit snapshotIndexCommit) {
             }
 
             final List<BlobStoreIndexShardSnapshot.FileInfo> indexCommitPointFiles = new ArrayList<>();
-
             store.incRef();
-            int indexIncrementalFileCount = 0;
-            int indexTotalNumberOfFiles = 0;
-            long indexIncrementalSize = 0;
-            long indexTotalFileCount = 0;
             try {
                 ArrayList<BlobStoreIndexShardSnapshot.FileInfo> filesToSnapshot = new ArrayList<>();
                 final Store.MetadataSnapshot metadata;
                 // TODO apparently we don't use the MetadataSnapshot#.recoveryDiff(...) here but we should
                 final Collection<String> fileNames;
                 try {
-                    logger.trace("[{}] [{}] Loading store metadata using index commit [{}]", shardId, snapshotId, snapshotIndexCommit);
+                    logger.trace(
+                        "[{}] [{}] Loading store metadata using index commit [{}]", shardId, snapshotId, snapshotIndexCommit);
                     metadata = store.getMetadata(snapshotIndexCommit);
                     fileNames = snapshotIndexCommit.getFileNames();
                 } catch (IOException e) {
                     throw new IndexShardSnapshotFailedException(shardId, "Failed to get store file metadata", e);
                 }
+                int indexIncrementalFileCount = 0;
+                int indexTotalNumberOfFiles = 0;
+                long indexIncrementalSize = 0;
+                long indexTotalFileCount = 0;
                 for (String fileName : fileNames) {
                     if (snapshotStatus.isAborted()) {
                         logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName);
@@ -1162,7 +884,7 @@ public void snapshot(final IndexCommit snapshotIndexCommit) {
 
                 for (BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo : filesToSnapshot) {
                     try {
-                        snapshotFile(snapshotFileInfo);
+                        snapshotFile(snapshotFileInfo, indexId, shardId, snapshotId, snapshotStatus, store);
                     } catch (IOException e) {
                         throw new IndexShardSnapshotFailedException(shardId, "Failed to perform snapshot (index files)", e);
                     }
@@ -1175,20 +897,20 @@ public void snapshot(final IndexCommit snapshotIndexCommit) {
 
             // now create and write the commit point
             final BlobStoreIndexShardSnapshot snapshot = new BlobStoreIndexShardSnapshot(snapshotId.getName(),
-                                                                        lastSnapshotStatus.getIndexVersion(),
-                                                                        indexCommitPointFiles,
-                                                                        lastSnapshotStatus.getStartTime(),
-                                                                        // snapshotStatus.startTime() is assigned on the same machine,
-                                                                        // so it's safe to use with VLong
-                                                                        System.currentTimeMillis() - lastSnapshotStatus.getStartTime(),
-                                                                        lastSnapshotStatus.getIncrementalFileCount(),
-                                                                        lastSnapshotStatus.getIncrementalSize()
+                lastSnapshotStatus.getIndexVersion(),
+                indexCommitPointFiles,
+                lastSnapshotStatus.getStartTime(),
+                // snapshotStatus.startTime() is assigned on the same machine,
+                // so it's safe to use the relative time in millis
+                threadPool.relativeTimeInMillis() - lastSnapshotStatus.getStartTime(),
+                lastSnapshotStatus.getIncrementalFileCount(),
+                lastSnapshotStatus.getIncrementalSize()
             );
 
             //TODO: The time stored in snapshot doesn't include cleanup time.
             logger.trace("[{}] [{}] writing shard snapshot file", shardId, snapshotId);
             try {
-                indexShardSnapshotFormat.write(snapshot, blobContainer, snapshotId.getUUID());
+                indexShardSnapshotFormat.write(snapshot, shardContainer, snapshotId.getUUID());
             } catch (IOException e) {
                 throw new IndexShardSnapshotFailedException(shardId, "Failed to write commit point", e);
             }
@@ -1201,151 +923,317 @@ public void snapshot(final IndexCommit snapshotIndexCommit) {
                 newSnapshotsList.add(point);
             }
             // finalize the snapshot and rewrite the snapshot index with the next sequential snapshot index
-            finalize(newSnapshotsList, fileListGeneration + 1, blobs, "snapshot creation [" + snapshotId + "]");
-            snapshotStatus.moveToDone(System.currentTimeMillis());
+            finalizeShard(newSnapshotsList, fileListGeneration, blobs, "snapshot creation [" + snapshotId + "]", shardContainer,
+                shardId, snapshotId);
+            snapshotStatus.moveToDone(threadPool.relativeTimeInMillis());
+        } catch (Exception e) {
+            snapshotStatus.moveToFailed(threadPool.relativeTimeInMillis(), ExceptionsHelper.detailedMessage(e));
+            if (e instanceof IndexShardSnapshotFailedException) {
+                throw (IndexShardSnapshotFailedException) e;
+            } else {
+                throw new IndexShardSnapshotFailedException(store.shardId(), e);
+            }
         }
+    }
 
-        /**
-         * Snapshot individual file
-         *
-         * @param fileInfo file to be snapshotted
-         */
-        private void snapshotFile(final BlobStoreIndexShardSnapshot.FileInfo fileInfo) throws IOException {
-            final String file = fileInfo.physicalName();
-            try (IndexInput indexInput = store.openVerifyingInput(file, IOContext.READONCE, fileInfo.metadata())) {
-                for (int i = 0; i < fileInfo.numberOfParts(); i++) {
-                    final long partBytes = fileInfo.partBytes(i);
-
-                    final InputStreamIndexInput inputStreamIndexInput = new InputStreamIndexInput(indexInput, partBytes);
-                    InputStream inputStream = inputStreamIndexInput;
-                    if (snapshotRateLimiter != null) {
-                        inputStream = new RateLimitingInputStream(inputStreamIndexInput, snapshotRateLimiter,
-                                                                  snapshotRateLimitingTimeInNanos::inc);
-                    }
-                    inputStream = new AbortableInputStream(inputStream, fileInfo.physicalName());
-                    blobContainer.writeBlob(fileInfo.partName(i), inputStream, partBytes, true);
+    @Override
+    public void restoreShard(Store store, SnapshotId snapshotId, Version version, IndexId indexId, ShardId snapshotShardId,
+                             RecoveryState recoveryState) {
+        ShardId shardId = store.shardId();
+        try {
+            final BlobContainer container = shardContainer(indexId, snapshotShardId);
+            BlobStoreIndexShardSnapshot snapshot = loadShardSnapshot(container, snapshotId);
+            SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles());
+            new FileRestoreContext(metadata.name(), shardId, snapshotId, recoveryState, BUFFER_SIZE) {
+                @Override
+                protected InputStream fileInputStream(BlobStoreIndexShardSnapshot.FileInfo fileInfo) {
+                    final InputStream dataBlobCompositeStream = new SlicedInputStream(fileInfo.numberOfParts()) {
+                        @Override
+                        protected InputStream openSlice(long slice) throws IOException {
+                            return container.readBlob(fileInfo.partName(slice));
+                        }
+                    };
+                    return restoreRateLimiter == null ? dataBlobCompositeStream
+                        : new RateLimitingInputStream(dataBlobCompositeStream, restoreRateLimiter, restoreRateLimitingTimeInNanos::inc);
                 }
-                Store.verify(indexInput);
-                snapshotStatus.addProcessedFile(fileInfo.length());
-            } catch (Exception t) {
-                failStoreIfCorrupted(t);
-                snapshotStatus.addProcessedFile(0);
-                throw t;
-            }
+            }.restore(snapshotFiles, store);
+        } catch (Exception e) {
+            throw new IndexShardRestoreFailedException(shardId, "failed to restore snapshot [" + snapshotId + "]", e);
         }
+    }
+
+    @Override
+    public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, Version version, IndexId indexId, ShardId shardId) {
+        BlobStoreIndexShardSnapshot snapshot = loadShardSnapshot(shardContainer(indexId, shardId), snapshotId);
+        return IndexShardSnapshotStatus.newDone(snapshot.startTime(), snapshot.time(),
+            snapshot.incrementalFileCount(), snapshot.totalFileCount(),
+            snapshot.incrementalSize(), snapshot.totalSize());
+    }
 
-        private void failStoreIfCorrupted(Exception e) {
-            if (Lucene.isCorruptionException(e)) {
+    @Override
+    public void verify(String seed, DiscoveryNode localNode) {
+        assertSnapshotOrGenericThread();
+        if (isReadOnly()) {
+            try {
+                latestIndexBlobId();
+            } catch (IOException e) {
+                throw new RepositoryVerificationException(metadata.name(), "path " + basePath() +
+                    " is not accessible on node " + localNode, e);
+            }
+        } else {
+            BlobContainer testBlobContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed)));
+            if (testBlobContainer.blobExists("master.dat")) {
                 try {
-                    store.markStoreCorrupted((IOException) e);
-                } catch (IOException inner) {
-                    inner.addSuppressed(e);
-                    logger.warn("store cannot be marked as corrupted", inner);
+                    BytesArray bytes = new BytesArray(seed);
+                    try (InputStream stream = bytes.streamInput()) {
+                        testBlobContainer.writeBlob("data-" + localNode.getId() + ".dat", stream, bytes.length(), true);
+                    }
+                } catch (IOException exp) {
+                    throw new RepositoryVerificationException(metadata.name(), "store location [" + blobStore() +
+                        "] is not accessible on the node [" + localNode + "]", exp);
                 }
+            } else {
+                throw new RepositoryVerificationException(metadata.name(), "a file written by master to the store [" + blobStore() +
+                    "] cannot be accessed on the node [" + localNode + "]. " +
+                    "This might indicate that the store [" + blobStore() + "] is not shared between this node and the master node or " +
+                    "that permissions on the store don't allow reading files written by the master node");
             }
         }
+    }
 
-        /**
-         * Checks if snapshot file already exists in the list of blobs
-         *
-         * @param fileInfo file to check
-         * @param blobs    list of blobs
-         * @return true if file exists in the list of blobs
-         */
-        private boolean snapshotFileExistsInBlobs(BlobStoreIndexShardSnapshot.FileInfo fileInfo, Map<String, BlobMetaData> blobs) {
-            BlobMetaData blobMetaData = blobs.get(fileInfo.name());
-            if (blobMetaData != null) {
-                return blobMetaData.length() == fileInfo.length();
-            } else if (blobs.containsKey(fileInfo.partName(0))) {
-                // multi part file sum up the size and check
-                int part = 0;
-                long totalSize = 0;
-                while (true) {
-                    blobMetaData = blobs.get(fileInfo.partName(part++));
-                    if (blobMetaData == null) {
-                        break;
-                    }
-                    totalSize += blobMetaData.length();
-                }
-                return totalSize == fileInfo.length();
-            }
-            // no file, not exact and not multipart
-            return false;
+    @Override
+    public String toString() {
+        return "BlobStoreRepository[" +
+            "[" + metadata.name() +
+            "], [" + blobStore() + ']' +
+            ']';
+    }
+
+    /**
+     * Delete shard snapshot
+     */
+    private void deleteShardSnapshot(IndexId indexId, ShardId snapshotShardId, SnapshotId snapshotId) {
+        final BlobContainer shardContainer = shardContainer(indexId, snapshotShardId);
+        final Map<String, BlobMetaData> blobs;
+        try {
+            blobs = shardContainer.listBlobs();
+        } catch (IOException e) {
+            throw new IndexShardSnapshotException(snapshotShardId, "Failed to list content of shard directory", e);
         }
 
-        private class AbortableInputStream extends FilterInputStream {
-            private final String fileName;
+        Tuple<BlobStoreIndexShardSnapshots, Integer> tuple = buildBlobStoreIndexShardSnapshots(blobs, shardContainer);
+        BlobStoreIndexShardSnapshots snapshots = tuple.v1();
+        int fileListGeneration = tuple.v2();
+
+        try {
+            indexShardSnapshotFormat.delete(shardContainer, snapshotId.getUUID());
+        } catch (IOException e) {
+            logger.warn(new ParameterizedMessage("[{}] [{}] failed to delete shard snapshot file", snapshotShardId, snapshotId), e);
+        }
 
-            AbortableInputStream(InputStream delegate, String fileName) {
-                super(delegate);
-                this.fileName = fileName;
+        // Build a list of snapshots that should be preserved
+        List<SnapshotFiles> newSnapshotsList = new ArrayList<>();
+        for (SnapshotFiles point : snapshots) {
+            if (!point.snapshot().equals(snapshotId.getName())) {
+                newSnapshotsList.add(point);
             }
+        }
+        // finalize the snapshot and rewrite the snapshot index with the next sequential snapshot index
+        finalizeShard(newSnapshotsList, fileListGeneration, blobs, "snapshot deletion [" + snapshotId + "]", shardContainer,
+            snapshotShardId, snapshotId);
+    }
 
-            @Override
-            public int read() throws IOException {
-                checkAborted();
-                return in.read();
+    /**
+     * Loads information about shard snapshot
+     */
+    private BlobStoreIndexShardSnapshot loadShardSnapshot(BlobContainer shardContainer, SnapshotId snapshotId) {
+        try {
+            return indexShardSnapshotFormat.read(shardContainer, snapshotId.getUUID());
+        } catch (IOException ex) {
+            throw new SnapshotException(metadata.name(), snapshotId,
+                "failed to read shard snapshot file for [" + shardContainer.path() + ']', ex);
+        }
+    }
+
+    /**
+     * Writes a new index file for the shard and removes all unreferenced files from the repository.
+     *
+     * We need to be really careful in handling index files in case of failures to make sure we don't
+     * have index file that points to files that were deleted.
+     *
+     * @param snapshots          list of active snapshots in the container
+     * @param fileListGeneration the generation number of the current snapshot index file
+     * @param blobs              list of blobs in the container
+     * @param reason             a reason explaining why the shard index file is written
+     */
+    private void finalizeShard(List<SnapshotFiles> snapshots, int fileListGeneration, Map<String, BlobMetaData> blobs,
+                               String reason, BlobContainer shardContainer, ShardId shardId, SnapshotId snapshotId) {
+        final String indexGeneration = Integer.toString(fileListGeneration + 1);
+        try {
+            final List<String> blobsToDelete;
+            if (snapshots.isEmpty()) {
+                // If we deleted all snapshots, we don't need to create a new index file and simply delete all the blobs we found
+                blobsToDelete = List.copyOf(blobs.keySet());
+            } else {
+                final BlobStoreIndexShardSnapshots updatedSnapshots = new BlobStoreIndexShardSnapshots(snapshots);
+                indexShardSnapshotsFormat.writeAtomic(updatedSnapshots, shardContainer, indexGeneration);
+                // Delete all previous index-N, data-blobs that are not referenced by the new index-N and temporary blobs
+                blobsToDelete = blobs.keySet().stream().filter(blob ->
+                    blob.startsWith(SNAPSHOT_INDEX_PREFIX)
+                        || blob.startsWith(DATA_BLOB_PREFIX) && updatedSnapshots.findNameFile(canonicalName(blob)) == null
+                        || FsBlobContainer.isTempBlobName(blob)).collect(Collectors.toList());
+            }
+            try {
+                shardContainer.deleteBlobsIgnoringIfNotExists(blobsToDelete);
+            } catch (IOException e) {
+                logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to delete blobs during finalization",
+                    snapshotId, shardId), e);
             }
+        } catch (IOException e) {
+            String message =
+                "Failed to finalize " + reason + " with shard index [" + indexShardSnapshotsFormat.blobName(indexGeneration) + "]";
+            throw new IndexShardSnapshotFailedException(shardId, message, e);
+        }
+    }
 
-            @Override
-            public int read(byte[] b, int off, int len) throws IOException {
-                checkAborted();
-                return in.read(b, off, len);
+    /**
+     * Loads all available snapshots in the repository
+     *
+     * @param blobs list of blobs in repository
+     * @return tuple of BlobStoreIndexShardSnapshots and the last snapshot index generation
+     */
+    private Tuple<BlobStoreIndexShardSnapshots, Integer> buildBlobStoreIndexShardSnapshots(Map<String, BlobMetaData> blobs,
+                                                                                           BlobContainer shardContainer) {
+        int latest = -1;
+        Set<String> blobKeys = blobs.keySet();
+        for (String name : blobKeys) {
+            if (name.startsWith(SNAPSHOT_INDEX_PREFIX)) {
+                try {
+                    int gen = Integer.parseInt(name.substring(SNAPSHOT_INDEX_PREFIX.length()));
+                    if (gen > latest) {
+                        latest = gen;
+                    }
+                } catch (NumberFormatException ex) {
+                    logger.warn("failed to parse index file name [{}]", name);
+                }
             }
+        }
+        if (latest >= 0) {
+            try {
+                final BlobStoreIndexShardSnapshots shardSnapshots =
+                    indexShardSnapshotsFormat.read(shardContainer, Integer.toString(latest));
+                return new Tuple<>(shardSnapshots, latest);
+            } catch (IOException e) {
+                final String file = SNAPSHOT_INDEX_PREFIX + latest;
+                logger.warn(() -> new ParameterizedMessage("failed to read index file [{}]", file), e);
+            }
+        } else if (blobKeys.isEmpty() == false) {
+            logger.warn("Could not find a readable index-N file in a non-empty shard snapshot directory [{}]", shardContainer.path());
+        }
 
-            private void checkAborted() {
-                if (snapshotStatus.isAborted()) {
-                    logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName);
-                    throw new IndexShardSnapshotFailedException(shardId, "Aborted");
+        // We couldn't load the index file - falling back to loading individual snapshots
+        List<SnapshotFiles> snapshots = new ArrayList<>();
+        for (String name : blobKeys) {
+            try {
+                BlobStoreIndexShardSnapshot snapshot = null;
+                if (name.startsWith(SNAPSHOT_PREFIX)) {
+                    snapshot = indexShardSnapshotFormat.readBlob(shardContainer, name);
+                }
+                if (snapshot != null) {
+                    snapshots.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles()));
                 }
+            } catch (IOException e) {
+                logger.warn(() -> new ParameterizedMessage("Failed to read blob [{}]", name), e);
             }
         }
+        return new Tuple<>(new BlobStoreIndexShardSnapshots(snapshots), latest);
     }
 
-    private static final class PartSliceStream extends SlicedInputStream {
+    /**
+     * Snapshot individual file
+     * @param fileInfo file to be snapshotted
+     */
+    private void snapshotFile(BlobStoreIndexShardSnapshot.FileInfo fileInfo, IndexId indexId, ShardId shardId, SnapshotId snapshotId,
+                              IndexShardSnapshotStatus snapshotStatus, Store store) throws IOException {
+        final BlobContainer shardContainer = shardContainer(indexId, shardId);
+        final String file = fileInfo.physicalName();
+        try (IndexInput indexInput = store.openVerifyingInput(file, IOContext.READONCE, fileInfo.metadata())) {
+            for (int i = 0; i < fileInfo.numberOfParts(); i++) {
+                final long partBytes = fileInfo.partBytes(i);
+
+                InputStream inputStream = new InputStreamIndexInput(indexInput, partBytes);
+                if (snapshotRateLimiter != null) {
+                    inputStream = new RateLimitingInputStream(inputStream, snapshotRateLimiter,
+                        snapshotRateLimitingTimeInNanos::inc);
+                }
+                // Make reads abortable by mutating the snapshotStatus object
+                inputStream = new FilterInputStream(inputStream) {
+                    @Override
+                    public int read() throws IOException {
+                        checkAborted();
+                        return super.read();
+                    }
 
-        private final BlobContainer container;
-        private final BlobStoreIndexShardSnapshot.FileInfo info;
+                    @Override
+                    public int read(byte[] b, int off, int len) throws IOException {
+                        checkAborted();
+                        return super.read(b, off, len);
+                    }
 
-        PartSliceStream(BlobContainer container, BlobStoreIndexShardSnapshot.FileInfo info) {
-            super(info.numberOfParts());
-            this.info = info;
-            this.container = container;
+                    private void checkAborted() {
+                        if (snapshotStatus.isAborted()) {
+                            logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId,
+                                snapshotId, fileInfo.physicalName());
+                            throw new IndexShardSnapshotFailedException(shardId, "Aborted");
+                        }
+                    }
+                };
+                shardContainer.writeBlob(fileInfo.partName(i), inputStream, partBytes, true);
+            }
+            Store.verify(indexInput);
+            snapshotStatus.addProcessedFile(fileInfo.length());
+        } catch (Exception t) {
+            failStoreIfCorrupted(store, t);
+            snapshotStatus.addProcessedFile(0);
+            throw t;
         }
+    }
 
-        @Override
-        protected InputStream openSlice(long slice) throws IOException {
-            return container.readBlob(info.partName(slice));
+    private static void failStoreIfCorrupted(Store store, Exception e) {
+        if (Lucene.isCorruptionException(e)) {
+            try {
+                store.markStoreCorrupted((IOException) e);
+            } catch (IOException inner) {
+                inner.addSuppressed(e);
+                logger.warn("store cannot be marked as corrupted", inner);
+            }
         }
     }
 
     /**
-     * Context for restore operations
+     * Checks if snapshot file already exists in the list of blobs
+     * @param fileInfo file to check
+     * @param blobs list of blobs
+     * @return true if file exists in the list of blobs
      */
-    private class RestoreContext extends FileRestoreContext {
-
-        private final BlobContainer blobContainer;
-
-        /**
-         * Constructs new restore context
-         * @param shardId    shard id to restore into
-         * @param snapshotId    snapshot id
-         * @param recoveryState recovery state to report progress
-         * @param blobContainer the blob container to read the files from
-         */
-        RestoreContext(ShardId shardId, SnapshotId snapshotId, RecoveryState recoveryState, BlobContainer blobContainer) {
-            super(metadata.name(), shardId, snapshotId, recoveryState, BUFFER_SIZE);
-            this.blobContainer = blobContainer;
-        }
-
-        @Override
-        protected InputStream fileInputStream(BlobStoreIndexShardSnapshot.FileInfo fileInfo) {
-            if (restoreRateLimiter == null) {
-                return new PartSliceStream(blobContainer, fileInfo);
-            } else {
-                RateLimitingInputStream.Listener listener = restoreRateLimitingTimeInNanos::inc;
-                return new RateLimitingInputStream(new PartSliceStream(blobContainer, fileInfo), restoreRateLimiter, listener);
+    private static boolean snapshotFileExistsInBlobs(BlobStoreIndexShardSnapshot.FileInfo fileInfo, Map<String, BlobMetaData> blobs) {
+        BlobMetaData blobMetaData = blobs.get(fileInfo.name());
+        if (blobMetaData != null) {
+            return blobMetaData.length() == fileInfo.length();
+        } else if (blobs.containsKey(fileInfo.partName(0))) {
+            // multi part file sum up the size and check
+            int part = 0;
+            long totalSize = 0;
+            while (true) {
+                blobMetaData = blobs.get(fileInfo.partName(part++));
+                if (blobMetaData == null) {
+                    break;
+                }
+                totalSize += blobMetaData.length();
             }
+            return totalSize == fileInfo.length();
         }
+        // no file, not exact and not multipart
+        return false;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
index d216fe3234e83..0f059a89e6c8c 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressorFactory;
+import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.store.ByteArrayIndexInput;
@@ -42,7 +43,6 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.core.internal.io.Streams;
 import org.elasticsearch.gateway.CorruptStateException;
 import org.elasticsearch.snapshots.SnapshotInfo;
 
@@ -149,24 +149,21 @@ public String blobName(String name) {
      * @param blobName blob name
      */
     public T readBlob(BlobContainer blobContainer, String blobName) throws IOException {
-        try (InputStream inputStream = blobContainer.readBlob(blobName)) {
-            ByteArrayOutputStream out = new ByteArrayOutputStream();
-            Streams.copy(inputStream, out);
-            final byte[] bytes = out.toByteArray();
-            final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")";
-            try (ByteArrayIndexInput indexInput = new ByteArrayIndexInput(resourceDesc, bytes)) {
-                CodecUtil.checksumEntireFile(indexInput);
-                CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION);
-                long filePointer = indexInput.getFilePointer();
-                long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
-                try (XContentParser parser = XContentHelper.createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE,
-                        new BytesArray(bytes, (int) filePointer, (int) contentSize))) {
-                    return reader.apply(parser);
-                }
-            } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
-                // we trick this into a dedicated exception with the original stacktrace
-                throw new CorruptStateException(ex);
+        final BytesReference bytes = Streams.readFully(blobContainer.readBlob(blobName));
+        final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")";
+        try (ByteArrayIndexInput indexInput =
+                 new ByteArrayIndexInput(resourceDesc, BytesReference.toBytes(bytes))) {
+            CodecUtil.checksumEntireFile(indexInput);
+            CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION);
+            long filePointer = indexInput.getFilePointer();
+            long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
+            try (XContentParser parser = XContentHelper.createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE,
+                bytes.slice((int) filePointer, (int) contentSize))) {
+                return reader.apply(parser);
             }
+        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
+            // we trick this into a dedicated exception with the original stacktrace
+            throw new CorruptStateException(ex);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/package-info.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/package-info.java
index 9d6d72f0458c9..0a66df3cf8521 100644
--- a/server/src/main/java/org/elasticsearch/repositories/blobstore/package-info.java
+++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/package-info.java
@@ -20,7 +20,7 @@
 /**
  * <p>This package exposes the blobstore repository used by Elasticsearch Snapshots.</p>
  *
- * <h1>Preliminaries</h1>
+ * <h2>Preliminaries</h2>
  *
  * <p>The {@link org.elasticsearch.repositories.blobstore.BlobStoreRepository} forms the basis of implementations of
  * {@link org.elasticsearch.repositories.Repository} on top of a blob store. A blobstore can be used as the basis for an implementation
@@ -84,7 +84,7 @@
  * }
  * </pre>
  *
- * <h1>Getting the Repository's RepositoryData</h1>
+ * <h2>Getting the Repository's RepositoryData</h2>
  *
  * <p>Loading the {@link org.elasticsearch.repositories.RepositoryData} that holds the list of all snapshots as well as the mapping of
  * indices' names to their repository {@link org.elasticsearch.repositories.IndexId} is done by invoking
@@ -103,7 +103,7 @@
  * <li>First, find the most recent {@code RepositoryData} by getting a list of all index-N blobs through listing all blobs with prefix
  * "index-" under the repository root and then selecting the one with the highest value for N.</li>
  * <li>If this operation fails because the repository's {@code BlobContainer} does not support list operations (in the case of read-only
- * repositories), read the highest value of N from the the index.latest blob.</li>
+ * repositories), read the highest value of N from the index.latest blob.</li>
  * </ol>
  * </li>
  * <li>
@@ -114,11 +114,11 @@
  * </ol>
  * </li>
  * </ol>
- * <h1>Creating a Snapshot</h1>
+ * <h2>Creating a Snapshot</h2>
  *
  * <p>Creating a snapshot in the repository happens in the three steps described in detail below.</p>
  *
- * <h2>Initializing a Snapshot in the Repository</h2>
+ * <h3>Initializing a Snapshot in the Repository</h3>
  *
  * <p>Creating a snapshot in the repository starts with a call to {@link org.elasticsearch.repositories.Repository#initializeSnapshot} which
  * the blob store repository implements via the following actions:</p>
@@ -130,7 +130,7 @@
  * </ol>
  * TODO: This behavior is problematic, adjust these docs once https://github.com/elastic/elasticsearch/issues/41581 is fixed
  *
- * <h2>Writing Shard Data (Segments)</h2>
+ * <h3>Writing Shard Data (Segments)</h3>
  *
  * <p>Once all the metadata has been written by the snapshot initialization, the snapshot process moves on to writing the actual shard data
  * to the repository by invoking {@link org.elasticsearch.repositories.Repository#snapshotShard} on the data-nodes that hold the primaries
@@ -158,7 +158,7 @@
  * {@code BlobStoreIndexShardSnapshots} blob is written to the shard's path with name {@code index-${N+1}}.</li>
  * </ol>
  *
- * <h2>Finalizing the Snapshot</h2>
+ * <h3>Finalizing the Snapshot</h3>
  *
  * <p>After all primaries have finished writing the necessary segment files to the blob store in the previous step, the master node moves on
  * to finalizing the snapshot by invoking {@link org.elasticsearch.repositories.Repository#finalizeSnapshot}. This method executes the
@@ -173,7 +173,7 @@
  * <li>Write the updated {@code /index.latest} blob containing the new repository generation {@code N + 1}.</li>
  * </ol>
  *
- * <h1>Deleting a Snapshot</h1>
+ * <h2>Deleting a Snapshot</h2>
  *
  * <p>Deleting a snapshot is an operation that is exclusively executed on the master node that runs through the following sequence of
  * action when {@link org.elasticsearch.repositories.blobstore.BlobStoreRepository#deleteSnapshot} is invoked:</p>
diff --git a/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java b/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
index 8f495f2d4842a..f338e0ee4cb08 100644
--- a/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
+++ b/server/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
@@ -66,14 +66,12 @@ public class FsRepository extends BlobStoreRepository {
 
     private final ByteSizeValue chunkSize;
 
-    private final BlobPath basePath;
-
     /**
      * Constructs a shared file system repository.
      */
     public FsRepository(RepositoryMetaData metadata, Environment environment, NamedXContentRegistry namedXContentRegistry,
                         ThreadPool threadPool) {
-        super(metadata, environment.settings(), namedXContentRegistry, threadPool);
+        super(metadata, environment.settings(), namedXContentRegistry, threadPool, BlobPath.cleanPath());
         this.environment = environment;
         String location = REPOSITORIES_LOCATION_SETTING.get(metadata.settings());
         if (location.isEmpty()) {
@@ -101,7 +99,6 @@ public FsRepository(RepositoryMetaData metadata, Environment environment, NamedX
         } else {
             this.chunkSize = REPOSITORIES_CHUNK_SIZE_SETTING.get(environment.settings());
         }
-        this.basePath = BlobPath.cleanPath();
     }
 
     @Override
@@ -115,9 +112,4 @@ protected BlobStore createBlobStore() throws Exception {
     protected ByteSizeValue chunkSize() {
         return chunkSize;
     }
-
-    @Override
-    protected BlobPath basePath() {
-        return basePath;
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java
index a55005454f629..3f32d281918a3 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestController.java
@@ -53,13 +53,12 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.UnaryOperator;
 
+import static org.elasticsearch.rest.BytesRestResponse.TEXT_CONTENT_TYPE;
 import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
-import static org.elasticsearch.rest.RestStatus.METHOD_NOT_ALLOWED;
-import static org.elasticsearch.rest.RestStatus.FORBIDDEN;
 import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR;
+import static org.elasticsearch.rest.RestStatus.METHOD_NOT_ALLOWED;
 import static org.elasticsearch.rest.RestStatus.NOT_ACCEPTABLE;
 import static org.elasticsearch.rest.RestStatus.OK;
-import static org.elasticsearch.rest.BytesRestResponse.TEXT_CONTENT_TYPE;
 
 public class RestController implements HttpServerTransport.Dispatcher {
 
@@ -253,7 +252,7 @@ boolean dispatchRequest(final RestRequest request, final RestChannel channel, fi
                 // If an alternative handler for an explicit path is registered to a
                 // different HTTP method than the one supplied - return a 405 Method
                 // Not Allowed error.
-                handleUnsupportedHttpMethod(request, channel, validMethodSet);
+                handleUnsupportedHttpMethod(request, channel, validMethodSet, null);
                 requestHandled = true;
             } else if (validMethodSet.contains(request.method()) == false
                 && (request.method() == RestRequest.Method.OPTIONS)) {
@@ -330,16 +329,28 @@ void tryAllHandlers(final RestRequest request, final RestChannel channel, final
             return;
         }
 
-        // Loop through all possible handlers, attempting to dispatch the request
-        Iterator<MethodHandlers> allHandlers = getAllHandlers(request);
-        for (Iterator<MethodHandlers> it = allHandlers; it.hasNext(); ) {
-            final Optional<RestHandler> mHandler = Optional.ofNullable(it.next()).flatMap(mh -> mh.getHandler(request.method()));
-            requestHandled = dispatchRequest(request, channel, client, mHandler);
-            if (requestHandled) {
-                break;
+        try {
+            // Resolves the HTTP method and fails if the method is invalid
+            final RestRequest.Method requestMethod = request.method();
+
+            // Loop through all possible handlers, attempting to dispatch the request
+            Iterator<MethodHandlers> allHandlers = getAllHandlers(request);
+            for (Iterator<MethodHandlers> it = allHandlers; it.hasNext(); ) {
+                Optional<RestHandler> mHandler = Optional.empty();
+                if (requestMethod != null) {
+                    mHandler = Optional.ofNullable(it.next()).flatMap(mh -> mh.getHandler(requestMethod));
+                }
+                requestHandled = dispatchRequest(request, channel, client, mHandler);
+                if (requestHandled) {
+                    break;
+                }
             }
+        } catch (final IllegalArgumentException e) {
+            handleUnsupportedHttpMethod(request, channel, getValidHandlerMethodSet(request), e);
+            requestHandled = true;
         }
 
+
         // If request has not been handled, fallback to a bad request error.
         if (requestHandled == false) {
             handleBadRequest(request, channel);
@@ -365,11 +376,25 @@ Iterator<MethodHandlers> getAllHandlers(final RestRequest request) {
      * <a href="https://tools.ietf.org/html/rfc2616#section-10.4.6">HTTP/1.1 -
      * 10.4.6 - 405 Method Not Allowed</a>).
      */
-    private void handleUnsupportedHttpMethod(RestRequest request, RestChannel channel, Set<RestRequest.Method> validMethodSet) {
+    private void handleUnsupportedHttpMethod(final RestRequest request,
+                                             final RestChannel channel,
+                                             final Set<RestRequest.Method> validMethodSet,
+                                             @Nullable final IllegalArgumentException exception) {
         try {
-            BytesRestResponse bytesRestResponse = BytesRestResponse.createSimpleErrorResponse(channel, METHOD_NOT_ALLOWED,
-                "Incorrect HTTP method for uri [" + request.uri() + "] and method [" + request.method() + "], allowed: " + validMethodSet);
-            bytesRestResponse.addHeader("Allow", Strings.collectionToDelimitedString(validMethodSet, ","));
+            final StringBuilder msg = new StringBuilder();
+            if (exception != null) {
+                msg.append(exception.getMessage());
+            } else {
+                msg.append("Incorrect HTTP method for uri [").append(request.uri());
+                msg.append("] and method [").append(request.method()).append("]");
+            }
+            if (validMethodSet.isEmpty() == false) {
+                msg.append(", allowed: ").append(validMethodSet);
+            }
+            BytesRestResponse bytesRestResponse = BytesRestResponse.createSimpleErrorResponse(channel, METHOD_NOT_ALLOWED, msg.toString());
+            if (validMethodSet.isEmpty() == false) {
+                bytesRestResponse.addHeader("Allow", Strings.collectionToDelimitedString(validMethodSet, ","));
+            }
             channel.sendResponse(bytesRestResponse);
         } catch (final IOException e) {
             logger.warn("failed to send bad request response", e);
@@ -385,11 +410,12 @@ private void handleUnsupportedHttpMethod(RestRequest request, RestChannel channe
      * - Options</a>).
      */
     private void handleOptionsRequest(RestRequest request, RestChannel channel, Set<RestRequest.Method> validMethodSet) {
-        if (request.method() == RestRequest.Method.OPTIONS && validMethodSet.size() > 0) {
+        assert request.method() == RestRequest.Method.OPTIONS;
+        if (validMethodSet.isEmpty() == false) {
             BytesRestResponse bytesRestResponse = new BytesRestResponse(OK, TEXT_CONTENT_TYPE, BytesArray.EMPTY);
             bytesRestResponse.addHeader("Allow", Strings.collectionToDelimitedString(validMethodSet, ","));
             channel.sendResponse(bytesRestResponse);
-        } else if (request.method() == RestRequest.Method.OPTIONS && validMethodSet.size() == 0) {
+        } else {
             /*
              * When we have an OPTIONS HTTP request and no valid handlers,
              * simply send OK by default (with the Access Control Origin header
@@ -433,20 +459,25 @@ private String getPath(RestRequest request) {
         return request.rawPath();
     }
 
-    void handleFavicon(RestRequest request, RestChannel channel) {
-        if (request.method() == RestRequest.Method.GET) {
-            try {
-                try (InputStream stream = getClass().getResourceAsStream("/config/favicon.ico")) {
-                    ByteArrayOutputStream out = new ByteArrayOutputStream();
-                    Streams.copy(stream, out);
-                    BytesRestResponse restResponse = new BytesRestResponse(RestStatus.OK, "image/x-icon", out.toByteArray());
-                    channel.sendResponse(restResponse);
+    private void handleFavicon(final RestRequest request, final RestChannel channel) {
+        try {
+            if (request.method() != RestRequest.Method.GET) {
+                handleUnsupportedHttpMethod(request, channel, Set.of(RestRequest.Method.GET), null);
+            } else {
+                try {
+                    try (InputStream stream = getClass().getResourceAsStream("/config/favicon.ico")) {
+                        ByteArrayOutputStream out = new ByteArrayOutputStream();
+                        Streams.copy(stream, out);
+                        BytesRestResponse restResponse = new BytesRestResponse(RestStatus.OK, "image/x-icon", out.toByteArray());
+                        channel.sendResponse(restResponse);
+                    }
+                } catch (IOException e) {
+                    channel.sendResponse(
+                        new BytesRestResponse(INTERNAL_SERVER_ERROR, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
                 }
-            } catch (IOException e) {
-                channel.sendResponse(new BytesRestResponse(INTERNAL_SERVER_ERROR, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
             }
-        } else {
-            channel.sendResponse(new BytesRestResponse(FORBIDDEN, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+        } catch (final IllegalArgumentException e) {
+            handleUnsupportedHttpMethod(request, channel, Set.of(RestRequest.Method.GET), e);
         }
     }
 
@@ -512,5 +543,4 @@ private static CircuitBreaker inFlightRequestsBreaker(CircuitBreakerService circ
         // We always obtain a fresh breaker to reflect changes to the breaker configuration.
         return circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
     }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
index fe976ee4ddce6..4fd8515caba03 100644
--- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -153,6 +153,12 @@ public enum Method {
         GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH, TRACE, CONNECT
     }
 
+    /**
+     * Returns the HTTP method used in the REST request.
+     *
+     * @return the {@link Method} used in the REST request
+     * @throws IllegalArgumentException if the HTTP method is invalid
+     */
     public Method method() {
         return httpRequest.method();
     }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java
index f42180b5029b8..1346757520dfb 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java
@@ -49,10 +49,10 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        String repository = request.param("repository");
+        String[] repositories = request.paramAsStringArray("repository", Strings.EMPTY_ARRAY);
         String[] snapshots = request.paramAsStringArray("snapshot", Strings.EMPTY_ARRAY);
 
-        GetSnapshotsRequest getSnapshotsRequest = getSnapshotsRequest(repository).snapshots(snapshots);
+        GetSnapshotsRequest getSnapshotsRequest = getSnapshotsRequest(repositories).snapshots(snapshots);
         getSnapshotsRequest.ignoreUnavailable(request.paramAsBoolean("ignore_unavailable", getSnapshotsRequest.ignoreUnavailable()));
         getSnapshotsRequest.verbose(request.paramAsBoolean("verbose", getSnapshotsRequest.verbose()));
         getSnapshotsRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getSnapshotsRequest.masterNodeTimeout()));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java
index d9d6bbcfee98d..99c8598106843 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.rest.action.admin.indices;
 
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.settings.Settings;
@@ -29,8 +29,6 @@
 import org.elasticsearch.rest.action.RestToXContentListener;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
@@ -64,106 +62,10 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-
-        AnalyzeRequest analyzeRequest = new AnalyzeRequest(request.param("index"));
-
         try (XContentParser parser = request.contentOrSourceParamParser()) {
-            buildFromContent(parser, analyzeRequest);
-        } catch (IOException e) {
-            throw new IllegalArgumentException("Failed to parse request body", e);
+            AnalyzeAction.Request analyzeRequest = AnalyzeAction.Request.fromXContent(parser, request.param("index"));
+            return channel -> client.admin().indices().analyze(analyzeRequest, new RestToXContentListener<>(channel));
         }
-
-        return channel -> client.admin().indices().analyze(analyzeRequest, new RestToXContentListener<>(channel));
     }
 
-    static void buildFromContent(XContentParser parser, AnalyzeRequest analyzeRequest)
-            throws IOException {
-        if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
-            throw new IllegalArgumentException("Malformed content, must start with an object");
-        } else {
-            XContentParser.Token token;
-            String currentFieldName = null;
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    currentFieldName = parser.currentName();
-                } else if (Fields.TEXT.match(currentFieldName, parser.getDeprecationHandler()) &&
-                    token == XContentParser.Token.VALUE_STRING) {
-                    analyzeRequest.text(parser.text());
-                } else if (Fields.TEXT.match(currentFieldName, parser.getDeprecationHandler()) &&
-                    token == XContentParser.Token.START_ARRAY) {
-                    List<String> texts = new ArrayList<>();
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        if (token.isValue() == false) {
-                            throw new IllegalArgumentException(currentFieldName + " array element should only contain text");
-                        }
-                        texts.add(parser.text());
-                    }
-                    analyzeRequest.text(texts.toArray(new String[texts.size()]));
-                } else if (Fields.ANALYZER.match(currentFieldName, parser.getDeprecationHandler())
-                        && token == XContentParser.Token.VALUE_STRING) {
-                    analyzeRequest.analyzer(parser.text());
-                } else if (Fields.FIELD.match(currentFieldName, parser.getDeprecationHandler()) &&
-                    token == XContentParser.Token.VALUE_STRING) {
-                    analyzeRequest.field(parser.text());
-                } else if (Fields.TOKENIZER.match(currentFieldName, parser.getDeprecationHandler())) {
-                    if (token == XContentParser.Token.VALUE_STRING) {
-                        analyzeRequest.tokenizer(parser.text());
-                    } else if (token == XContentParser.Token.START_OBJECT) {
-                        analyzeRequest.tokenizer(parser.map());
-                    } else {
-                        throw new IllegalArgumentException(currentFieldName + " should be tokenizer's name or setting");
-                    }
-                } else if (Fields.TOKEN_FILTERS.match(currentFieldName, parser.getDeprecationHandler())
-                        && token == XContentParser.Token.START_ARRAY) {
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        if (token == XContentParser.Token.VALUE_STRING) {
-                            analyzeRequest.addTokenFilter(parser.text());
-                        } else if (token == XContentParser.Token.START_OBJECT) {
-                            analyzeRequest.addTokenFilter(parser.map());
-                        } else {
-                            throw new IllegalArgumentException(currentFieldName
-                                    + " array element should contain filter's name or setting");
-                        }
-                    }
-                } else if (Fields.CHAR_FILTERS.match(currentFieldName, parser.getDeprecationHandler())
-                        && token == XContentParser.Token.START_ARRAY) {
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        if (token == XContentParser.Token.VALUE_STRING) {
-                            analyzeRequest.addCharFilter(parser.text());
-                        } else if (token == XContentParser.Token.START_OBJECT) {
-                            analyzeRequest.addCharFilter(parser.map());
-                        } else {
-                            throw new IllegalArgumentException(currentFieldName
-                                    + " array element should contain char filter's name or setting");
-                        }
-                    }
-                } else if (Fields.EXPLAIN.match(currentFieldName, parser.getDeprecationHandler())) {
-                    if (parser.isBooleanValue()) {
-                        analyzeRequest.explain(parser.booleanValue());
-                    } else {
-                        throw new IllegalArgumentException(currentFieldName + " must be either 'true' or 'false'");
-                    }
-                } else if (Fields.ATTRIBUTES.match(currentFieldName, parser.getDeprecationHandler()) &&
-                    token == XContentParser.Token.START_ARRAY) {
-                    List<String> attributes = new ArrayList<>();
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        if (token.isValue() == false) {
-                            throw new IllegalArgumentException(currentFieldName + " array element should only contain attribute name");
-                        }
-                        attributes.add(parser.text());
-                    }
-                    analyzeRequest.attributes(attributes.toArray(new String[attributes.size()]));
-                } else if (Fields.NORMALIZER.match(currentFieldName, parser.getDeprecationHandler())) {
-                    if (token == XContentParser.Token.VALUE_STRING) {
-                        analyzeRequest.normalizer(parser.text());
-                    } else {
-                        throw new IllegalArgumentException(currentFieldName + " should be normalizer's name");
-                    }
-                } else {
-                    throw new IllegalArgumentException("Unknown parameter ["
-                            + currentFieldName + "] in request body or parameter is of the wrong type[" + token + "] ");
-                }
-            }
-        }
-    }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index 788de8e715480..8c7d34c1ca1ff 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -19,54 +19,58 @@
 
 package org.elasticsearch.rest.action.cat;
 
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.elasticsearch.action.support.GroupedActionListener;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Requests;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterIndexHealth;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
-import org.elasticsearch.index.Index;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.action.RestActionListener;
 import org.elasticsearch.rest.action.RestResponseListener;
 
 import java.time.Instant;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
+import static org.elasticsearch.action.support.master.MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestIndicesAction extends AbstractCatAction {
 
     private static final DateFormatter STRICT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_time");
-    private final IndexNameExpressionResolver indexNameExpressionResolver;
 
-    public RestIndicesAction(Settings settings, RestController controller, IndexNameExpressionResolver indexNameExpressionResolver) {
+    public RestIndicesAction(Settings settings, RestController controller) {
         super(settings);
-        this.indexNameExpressionResolver = indexNameExpressionResolver;
         controller.registerHandler(GET, "/_cat/indices", this);
         controller.registerHandler(GET, "/_cat/indices/{index}", this);
     }
@@ -85,50 +89,147 @@ protected void documentation(StringBuilder sb) {
     @Override
     public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
-        final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
-        clusterStateRequest.clear().indices(indices).metaData(true);
-        clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
-        clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout()));
-        final IndicesOptions strictExpandIndicesOptions = IndicesOptions.strictExpand();
-        clusterStateRequest.indicesOptions(strictExpandIndicesOptions);
-
-        return channel -> client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
+        final IndicesOptions indicesOptions = IndicesOptions.strictExpand();
+        final boolean local = request.paramAsBoolean("local", false);
+        final TimeValue masterNodeTimeout = request.paramAsTime("master_timeout", DEFAULT_MASTER_NODE_TIMEOUT);
+        final boolean includeUnloadedSegments = request.paramAsBoolean("include_unloaded_segments", false);
+
+        return channel -> {
+            final ActionListener<Table> listener = ActionListener.notifyOnce(new RestResponseListener<>(channel) {
+                @Override
+                public RestResponse buildResponse(final Table table) throws Exception {
+                    return RestTable.buildResponse(table, channel);
+                }
+            });
+
+            sendGetSettingsRequest(indices, indicesOptions, local, masterNodeTimeout, client, new ActionListener<>() {
+                @Override
+                public void onResponse(final GetSettingsResponse getSettingsResponse) {
+                    final GroupedActionListener<ActionResponse> groupedListener = createGroupedListener(request, 4, listener);
+                    groupedListener.onResponse(getSettingsResponse);
+
+                    // Indices that were successfully resolved during the get settings request might be deleted when the subsequent cluster
+                    // state, cluster health and indices stats requests execute. We have to distinguish two cases:
+                    // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we want the
+                    //    subsequent requests to fail.
+                    // 2) the deleted index was resolved as part of a wildcard or _all. In this case, we want the subsequent requests not to
+                    //    fail on the deleted index (as we want to ignore wildcards that cannot be resolved).
+                    // This behavior can be ensured by letting the cluster state, cluster health and indices stats requests re-resolve the
+                    // index names with the same indices options that we used for the initial cluster state request (strictExpand).
+                    sendIndicesStatsRequest(indices, indicesOptions, includeUnloadedSegments, client,
+                        ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure));
+                    sendClusterStateRequest(indices, indicesOptions, local, masterNodeTimeout, client,
+                        ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure));
+                    sendClusterHealthRequest(indices, indicesOptions, local, masterNodeTimeout, client,
+                        ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure));
+                }
+
+                @Override
+                public void onFailure(final Exception e) {
+                    listener.onFailure(e);
+                }
+            });
+        };
+    }
+
+    /**
+     * We're using the Get Settings API here to resolve the authorized indices for the user.
+     * This is because the Cluster State and Cluster Health APIs do not filter output based
+     * on index privileges, so they can't be used to determine which indices are authorized
+     * or not. On top of this, the Indices Stats API cannot be used either to resolve indices
+     * as it does not provide information for all existing indices (for example recovering
+     * indices or non replicated closed indices are not reported in indices stats response).
+     */
+    private void sendGetSettingsRequest(final String[] indices,
+                                        final IndicesOptions indicesOptions,
+                                        final boolean local,
+                                        final TimeValue masterNodeTimeout,
+                                        final NodeClient client,
+                                        final ActionListener<GetSettingsResponse> listener) {
+        final GetSettingsRequest request = new GetSettingsRequest();
+        request.indices(indices);
+        request.indicesOptions(indicesOptions);
+        request.local(local);
+        request.masterNodeTimeout(masterNodeTimeout);
+        request.names(IndexSettings.INDEX_SEARCH_THROTTLED.getKey());
+
+        client.admin().indices().getSettings(request, listener);
+    }
+
+    private void sendClusterStateRequest(final String[] indices,
+                                         final IndicesOptions indicesOptions,
+                                         final boolean local,
+                                         final TimeValue masterNodeTimeout,
+                                         final NodeClient client,
+                                         final ActionListener<ClusterStateResponse> listener) {
+
+        final ClusterStateRequest request = new ClusterStateRequest();
+        request.indices(indices);
+        request.indicesOptions(indicesOptions);
+        request.local(local);
+        request.masterNodeTimeout(masterNodeTimeout);
+
+        client.admin().cluster().state(request, listener);
+    }
+
+    private void sendClusterHealthRequest(final String[] indices,
+                                          final IndicesOptions indicesOptions,
+                                          final boolean local,
+                                          final TimeValue masterNodeTimeout,
+                                          final NodeClient client,
+                                          final ActionListener<ClusterHealthResponse> listener) {
+
+        final ClusterHealthRequest request = new ClusterHealthRequest();
+        request.indices(indices);
+        request.indicesOptions(indicesOptions);
+        request.local(local);
+        request.masterNodeTimeout(masterNodeTimeout);
+
+        client.admin().cluster().health(request, listener);
+    }
+
+    private void sendIndicesStatsRequest(final String[] indices,
+                                         final IndicesOptions indicesOptions,
+                                         final boolean includeUnloadedSegments,
+                                         final NodeClient client,
+                                         final ActionListener<IndicesStatsResponse> listener) {
+
+        final IndicesStatsRequest request = new IndicesStatsRequest();
+        request.indices(indices);
+        request.indicesOptions(indicesOptions);
+        request.all();
+        request.includeUnloadedSegments(includeUnloadedSegments);
+
+        client.admin().indices().stats(request, listener);
+    }
+
+    private GroupedActionListener<ActionResponse> createGroupedListener(final RestRequest request, final int size,
+                                                                        final ActionListener<Table> listener) {
+        return new GroupedActionListener<>(new ActionListener<>() {
             @Override
-            public void processResponse(final ClusterStateResponse clusterStateResponse) {
-                final ClusterState clusterState = clusterStateResponse.getState();
-                final IndexMetaData[] indicesMetaData = getOrderedIndexMetaData(indices, clusterState, strictExpandIndicesOptions);
-                // Indices that were successfully resolved during the cluster state request might be deleted when the subsequent cluster
-                // health and indices stats requests execute. We have to distinguish two cases:
-                // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we want the subsequent
-                //    requests to fail.
-                // 2) the deleted index was resolved as part of a wildcard or _all. In this case, we want the subsequent requests not to
-                //    fail on the deleted index (as we want to ignore wildcards that cannot be resolved).
-                // This behavior can be ensured by letting the cluster health and indices stats requests re-resolve the index names with the
-                // same indices options that we used for the initial cluster state request (strictExpand).
-                final ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest(indices);
-                clusterHealthRequest.indicesOptions(strictExpandIndicesOptions);
-                clusterHealthRequest.local(request.paramAsBoolean("local", clusterHealthRequest.local()));
-
-                client.admin().cluster().health(clusterHealthRequest, new RestActionListener<ClusterHealthResponse>(channel) {
-                    @Override
-                    public void processResponse(final ClusterHealthResponse clusterHealthResponse) {
-                        final IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
-                        indicesStatsRequest.indices(indices);
-                        indicesStatsRequest.indicesOptions(strictExpandIndicesOptions);
-                        indicesStatsRequest.all();
-                        indicesStatsRequest.includeUnloadedSegments(request.paramAsBoolean("include_unloaded_segments", false));
-
-                        client.admin().indices().stats(indicesStatsRequest, new RestResponseListener<IndicesStatsResponse>(channel) {
-                            @Override
-                            public RestResponse buildResponse(IndicesStatsResponse indicesStatsResponse) throws Exception {
-                                final Table tab = buildTable(request, indicesMetaData, clusterHealthResponse, indicesStatsResponse);
-                                return RestTable.buildResponse(tab, channel);
-                            }
-                        });
-                    }
-                });
+            public void onResponse(final Collection<ActionResponse> responses) {
+                GetSettingsResponse settingsResponse = extractResponse(responses, GetSettingsResponse.class);
+                Map<String, Settings> indicesSettings = StreamSupport.stream(settingsResponse.getIndexToSettings().spliterator(), false)
+                    .collect(Collectors.toMap(cursor -> cursor.key, cursor -> cursor.value));
+
+                ClusterStateResponse stateResponse = extractResponse(responses, ClusterStateResponse.class);
+                Map<String, IndexMetaData> indicesStates = StreamSupport.stream(stateResponse.getState().getMetaData().spliterator(), false)
+                    .collect(Collectors.toMap(indexMetaData -> indexMetaData.getIndex().getName(), Function.identity()));
+
+                ClusterHealthResponse healthResponse = extractResponse(responses, ClusterHealthResponse.class);
+                Map<String, ClusterIndexHealth> indicesHealths = healthResponse.getIndices();
+
+                IndicesStatsResponse statsResponse = extractResponse(responses, IndicesStatsResponse.class);
+                Map<String, IndexStats> indicesStats = statsResponse.getIndices();
+
+                listener.onResponse(buildTable(request, indicesSettings, indicesHealths, indicesStats, indicesStates));
             }
-        });
+
+            @Override
+            public void onFailure(final Exception e) {
+                listener.onFailure(e);
+            }
+        }, size);
     }
 
     private static final Set<String> RESPONSE_PARAMS;
@@ -393,18 +494,35 @@ protected Table getTableWithHeader(final RestRequest request) {
 
     // package private for testing
     Table buildTable(final RestRequest request,
-                     final IndexMetaData[] indicesMetaData,
-                     final ClusterHealthResponse clusterHealthResponse,
-                     final IndicesStatsResponse indicesStatsResponse) {
-        final String healthParam = request.param("health");
+                     final Map<String, Settings> indicesSettings,
+                     final Map<String, ClusterIndexHealth> indicesHealths,
+                     final Map<String, IndexStats> indicesStats,
+                     final Map<String, IndexMetaData> indicesMetaDatas) {
 
+        final String healthParam = request.param("health");
         final Table table = getTableWithHeader(request);
-        for (IndexMetaData indexMetaData : indicesMetaData) {
-            final String indexName = indexMetaData.getIndex().getName();
-            final ClusterIndexHealth indexHealth = clusterHealthResponse.getIndices().get(indexName);
-            final IndexStats indexStats = indicesStatsResponse.getIndices().get(indexName);
+
+        indicesSettings.forEach((indexName, settings) -> {
+            if (indicesMetaDatas.containsKey(indexName) == false) {
+                // the index exists in the Get Indices response but is not present in the cluster state:
+                // it is likely that the index was deleted in the meanwhile, so we ignore it.
+                return;
+            }
+
+            final IndexMetaData indexMetaData = indicesMetaDatas.get(indexName);
             final IndexMetaData.State indexState = indexMetaData.getState();
-            final boolean searchThrottled = IndexSettings.INDEX_SEARCH_THROTTLED.get(indexMetaData.getSettings());
+            final IndexStats indexStats = indicesStats.get(indexName);
+            final boolean searchThrottled = IndexSettings.INDEX_SEARCH_THROTTLED.get(settings);
+
+            final String health;
+            final ClusterIndexHealth indexHealth = indicesHealths.get(indexName);
+            if (indexHealth != null) {
+                health = indexHealth.getStatus().toString().toLowerCase(Locale.ROOT);
+            } else if (indexStats != null) {
+                health = "red*";
+            } else {
+                health = "";
+            }
 
             if (healthParam != null) {
                 final ClusterHealthStatus healthStatusFilter = ClusterHealthStatus.fromString(healthParam);
@@ -413,44 +531,26 @@ Table buildTable(final RestRequest request,
                     // index health is known but does not match the one requested
                     skip = indexHealth.getStatus() != healthStatusFilter;
                 } else {
-                    // index health is unknown, skip if we don't explicitly request RED health or if the index is closed but not replicated
-                    skip = ClusterHealthStatus.RED != healthStatusFilter || indexState == IndexMetaData.State.CLOSE;
+                    // index health is unknown, skip if we don't explicitly request RED health
+                    skip = ClusterHealthStatus.RED != healthStatusFilter;
                 }
                 if (skip) {
-                    continue;
+                    return;
                 }
             }
 
-            // the open index is present in the cluster state but is not returned in the indices stats API
-            if (indexStats == null && indexState != IndexMetaData.State.CLOSE) {
-                // the index stats API is called last, after cluster state and cluster health. If the index stats
-                // has not resolved the same open indices as the initial cluster state call, then the indices might
-                // have been removed in the meantime or, more likely, are unauthorized. This is because the cluster
-                // state exposes everything, even unauthorized indices, which are not exposed in APIs.
-                // We ignore such an index instead of displaying it with an empty stats.
-                continue;
-            }
-
             final CommonStats primaryStats;
             final CommonStats totalStats;
 
-            if (indexState == IndexMetaData.State.CLOSE) {
-                // empty stats for closed indices, but their names are displayed
+            if (indexStats == null || indexState == IndexMetaData.State.CLOSE) {
+                // TODO: expose docs stats for replicated closed indices
                 primaryStats = new CommonStats();
                 totalStats = new CommonStats();
             } else {
                 primaryStats = indexStats.getPrimaries();
                 totalStats = indexStats.getTotal();
             }
-
             table.startRow();
-
-            String health = null;
-            if (indexHealth != null) {
-                health = indexHealth.getStatus().toString().toLowerCase(Locale.ROOT);
-            } else if (indexStats != null) {
-                health = "red*";
-            }
             table.addCell(health);
             table.addCell(indexState.toString().toLowerCase(Locale.ROOT));
             table.addCell(indexName);
@@ -648,25 +748,13 @@ Table buildTable(final RestRequest request,
             table.addCell(searchThrottled);
 
             table.endRow();
-        }
+        });
 
         return table;
     }
 
-    // package private for testing
-    IndexMetaData[] getOrderedIndexMetaData(String[] indicesExpression, ClusterState clusterState, IndicesOptions indicesOptions) {
-        final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, indicesOptions, indicesExpression);
-        // concreteIndices should contain exactly the indices in state.metaData() that were selected by clusterStateRequest using the
-        // same indices option (IndicesOptions.strictExpand()). We select the indices again here so that they can be displayed in the
-        // resulting table in the requesting order.
-        assert concreteIndices.length == clusterState.metaData().getIndices().size();
-        final ImmutableOpenMap<String, IndexMetaData> indexMetaDataMap = clusterState.metaData().getIndices();
-        final IndexMetaData[] indicesMetaData = new IndexMetaData[concreteIndices.length];
-        // select the index metadata in the requested order, so that the response can display the indices in the resulting table
-        // in the requesting order.
-        for (int i = 0; i < concreteIndices.length; i++) {
-            indicesMetaData[i] = indexMetaDataMap.get(concreteIndices[i].getName());
-        }
-        return indicesMetaData;
+    @SuppressWarnings("unchecked")
+    private static <A extends ActionResponse> A extractResponse(final Collection<? extends ActionResponse> responses, Class<A> c) {
+        return (A) responses.stream().filter(c::isInstance).findFirst().get();
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
index 74a103dcb540c..f623408597382 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
@@ -324,7 +325,7 @@ Table buildTable(boolean fullId, RestRequest req, ClusterStateResponse state, No
             if (node.getRoles().isEmpty()) {
                 roles = "-";
             } else {
-                roles = node.getRoles().stream().map(DiscoveryNode.Role::getAbbreviation).collect(Collectors.joining());
+                roles = node.getRoles().stream().map(DiscoveryNodeRole::roleNameAbbreviation).sorted().collect(Collectors.joining());
             }
             table.addCell(roles);
             table.addCell(masterId == null ? "x" : masterId.equals(node.getId()) ? "*" : "-");
diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
index 22258ce2d8878..4b679cbf475eb 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
@@ -20,9 +20,11 @@
 package org.elasticsearch.rest.action.cat;
 
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
@@ -36,6 +38,8 @@
 
 import java.time.Instant;
 import java.time.ZoneOffset;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
@@ -58,7 +62,7 @@ public String getName() {
     @Override
     protected RestChannelConsumer doCatRequest(final RestRequest request, NodeClient client) {
         GetSnapshotsRequest getSnapshotsRequest = new GetSnapshotsRequest()
-                .repository(request.param("repository"))
+                .repositories(request.paramAsStringArray("repository", new String[]{"_all"}))
                 .snapshots(new String[]{GetSnapshotsRequest.ALL_SNAPSHOTS});
 
         getSnapshotsRequest.ignoreUnavailable(request.paramAsBoolean("ignore_unavailable", getSnapshotsRequest.ignoreUnavailable()));
@@ -66,7 +70,7 @@ protected RestChannelConsumer doCatRequest(final RestRequest request, NodeClient
         getSnapshotsRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getSnapshotsRequest.masterNodeTimeout()));
 
         return channel ->
-            client.admin().cluster().getSnapshots(getSnapshotsRequest, new RestResponseListener<GetSnapshotsResponse>(channel) {
+            client.admin().cluster().getSnapshots(getSnapshotsRequest, new RestResponseListener<>(channel) {
                 @Override
                 public RestResponse buildResponse(GetSnapshotsResponse getSnapshotsResponse) throws Exception {
                     return RestTable.buildResponse(buildTable(request, getSnapshotsResponse), channel);
@@ -84,6 +88,7 @@ protected Table getTableWithHeader(RestRequest request) {
         return new Table()
                 .startHeaders()
                 .addCell("id", "alias:id,snapshot;desc:unique snapshot")
+                .addCell("repository", "alias:re,repo;desc:repository name")
                 .addCell("status", "alias:s,status;text-align:right;desc:snapshot name")
                 .addCell("start_epoch", "alias:ste,startEpoch;desc:start time in seconds since 1970-01-01 00:00:00")
                 .addCell("start_time", "alias:sti,startTime;desc:start time in HH:MM:SS")
@@ -102,29 +107,50 @@ protected Table getTableWithHeader(RestRequest request) {
 
     private Table buildTable(RestRequest req, GetSnapshotsResponse getSnapshotsResponse) {
         Table table = getTableWithHeader(req);
-        for (SnapshotInfo snapshotStatus : getSnapshotsResponse.getSnapshots()) {
-            table.startRow();
-
-            table.addCell(snapshotStatus.snapshotId().getName());
-            table.addCell(snapshotStatus.state());
-            table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.startTime(), TimeUnit.MILLISECONDS));
-            table.addCell(FORMATTER.format(Instant.ofEpochMilli(snapshotStatus.startTime())));
-            table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.endTime(), TimeUnit.MILLISECONDS));
-            table.addCell(FORMATTER.format(Instant.ofEpochMilli(snapshotStatus.endTime())));
-            final long durationMillis;
-            if (snapshotStatus.state() == SnapshotState.IN_PROGRESS) {
-                durationMillis = System.currentTimeMillis() - snapshotStatus.startTime();
-            } else {
-                durationMillis = snapshotStatus.endTime() - snapshotStatus.startTime();
+
+        if (getSnapshotsResponse.isFailed()) {
+            ElasticsearchException causes = null;
+
+            for (ElasticsearchException e : getSnapshotsResponse.getFailedResponses().values()) {
+                if (causes == null) {
+                    causes = e;
+                } else {
+                    causes.addSuppressed(e);
+                }
+            }
+            throw new ElasticsearchException(
+                    "Repositories [" +
+                            Strings.collectionToCommaDelimitedString(getSnapshotsResponse.getFailedResponses().keySet()) +
+                    "] failed to retrieve snapshots", causes);
+        }
+
+        for (Map.Entry<String, List<SnapshotInfo>> response : getSnapshotsResponse.getSuccessfulResponses().entrySet()) {
+            String repository = response.getKey();
+            for (SnapshotInfo snapshotStatus : response.getValue()) {
+                table.startRow();
+
+                table.addCell(snapshotStatus.snapshotId().getName());
+                table.addCell(repository);
+                table.addCell(snapshotStatus.state());
+                table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.startTime(), TimeUnit.MILLISECONDS));
+                table.addCell(FORMATTER.format(Instant.ofEpochMilli(snapshotStatus.startTime())));
+                table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.endTime(), TimeUnit.MILLISECONDS));
+                table.addCell(FORMATTER.format(Instant.ofEpochMilli(snapshotStatus.endTime())));
+                final long durationMillis;
+                if (snapshotStatus.state() == SnapshotState.IN_PROGRESS) {
+                    durationMillis = System.currentTimeMillis() - snapshotStatus.startTime();
+                } else {
+                    durationMillis = snapshotStatus.endTime() - snapshotStatus.startTime();
+                }
+                table.addCell(TimeValue.timeValueMillis(durationMillis));
+                table.addCell(snapshotStatus.indices().size());
+                table.addCell(snapshotStatus.successfulShards());
+                table.addCell(snapshotStatus.failedShards());
+                table.addCell(snapshotStatus.totalShards());
+                table.addCell(snapshotStatus.reason());
+
+                table.endRow();
             }
-            table.addCell(TimeValue.timeValueMillis(durationMillis));
-            table.addCell(snapshotStatus.indices().size());
-            table.addCell(snapshotStatus.successfulShards());
-            table.addCell(snapshotStatus.failedShards());
-            table.addCell(snapshotStatus.totalShards());
-            table.addCell(snapshotStatus.reason());
-
-            table.endRow();
         }
 
         return table;
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java
index 075167e97ccb2..1896314aa4a06 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -56,9 +55,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         TermVectorsRequest template = new TermVectorsRequest()
             .index(request.param("index"));
 
-
-        template.type(MapperService.SINGLE_MAPPING_NAME);
-
         RestTermVectorsAction.readURIParameters(template, request);
         multiTermVectorsRequest.ids(Strings.commaDelimitedListToStringArray(request.param("ids")));
         request.withContentOrSourceParamParserOrNull(p -> multiTermVectorsRequest.add(template, p));
diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java
index 3e136b368280d..2a77a2b1afe19 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.VersionType;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -63,7 +62,7 @@ public String getName() {
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         TermVectorsRequest termVectorsRequest;
-        termVectorsRequest = new TermVectorsRequest(request.param("index"), MapperService.SINGLE_MAPPING_NAME, request.param("id"));
+        termVectorsRequest = new TermVectorsRequest(request.param("index"), request.param("id"));
 
         if (request.hasContentOrSourceParam()) {
             try (XContentParser parser = request.contentOrSourceParamParser()) {
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java
index ecdd34ca07c88..df744978ab198 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java
@@ -19,13 +19,11 @@
 
 package org.elasticsearch.rest.action.search;
 
-import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -46,10 +44,6 @@
 import static org.elasticsearch.search.internal.SearchContext.DEFAULT_TERMINATE_AFTER;
 
 public class RestCountAction extends BaseRestHandler {
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(
-        LogManager.getLogger(RestCountAction.class));
-    static final String TYPES_DEPRECATION_MESSAGE = "[types removal]" +
-        " Specifying types in count requests is deprecated.";
 
     public RestCountAction(Settings settings, RestController controller) {
         super(settings);
@@ -57,10 +51,6 @@ public RestCountAction(Settings settings, RestController controller) {
         controller.registerHandler(GET, "/_count", this);
         controller.registerHandler(POST, "/{index}/_count", this);
         controller.registerHandler(GET, "/{index}/_count", this);
-
-        // Deprecated typed endpoints.
-        controller.registerHandler(POST, "/{index}/{type}/_count", this);
-        controller.registerHandler(GET, "/{index}/{type}/_count", this);
     }
 
     @Override
@@ -90,11 +80,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             searchSourceBuilder.minScore(minScore);
         }
 
-        if (request.hasParam("type")) {
-            deprecationLogger.deprecatedAndMaybeLog("count_with_types", TYPES_DEPRECATION_MESSAGE);
-            countRequest.types(Strings.splitStringByCommaToArray(request.param("type")));
-        }
-
         countRequest.preference(request.param("preference"));
 
         final int terminateAfter = request.paramAsInt("terminate_after", DEFAULT_TERMINATE_AFTER);
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
index d9beba089857f..1aef4aa5254a4 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
@@ -134,7 +134,7 @@ public static void parseMultiLineRequest(RestRequest request, IndicesOptions ind
         final Tuple<XContentType, BytesReference> sourceTuple = request.contentOrSourceParam();
         final XContent xContent = sourceTuple.v1().xContent();
         final BytesReference data = sourceTuple.v2();
-        MultiSearchRequest.readMultiLineFormat(data, xContent, consumer, indices, indicesOptions, Strings.EMPTY_ARRAY, routing,
+        MultiSearchRequest.readMultiLineFormat(data, xContent, consumer, indices, indicesOptions, routing,
                 searchType, ccsMinimizeRoundtrips, request.getXContentRegistry(), allowExplicitIndex);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index f0eaa0d51dadf..c2c089ffe31ab 100644
--- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -43,7 +43,6 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ObjectMapper;
-import org.elasticsearch.index.mapper.TypeFieldMapper;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -76,7 +75,6 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -175,13 +173,13 @@ final class DefaultSearchContext extends SearchContext {
         this.indexShard = indexShard;
         this.indexService = indexService;
         this.clusterService = clusterService;
-        this.searcher = new ContextIndexSearcher(engineSearcher, indexService.cache().query(), indexShard.getQueryCachingPolicy());
+        this.searcher = new ContextIndexSearcher(engineSearcher.reader(), engineSearcher.searcher().getSimilarity(),
+            indexService.cache().query(), indexShard.getQueryCachingPolicy());
         this.relativeTimeSupplier = relativeTimeSupplier;
         this.timeout = timeout;
         this.minNodeVersion = minNodeVersion;
         queryShardContext = indexService.newQueryShardContext(request.shardId().id(), searcher.getIndexReader(), request::nowInMillis,
             shardTarget.getClusterAlias());
-        queryShardContext.setTypes(request.types());
         queryBoost = request.indexBoost();
     }
 
@@ -269,13 +267,8 @@ public void preProcess(boolean rewrite) {
     @Override
     public Query buildFilteredQuery(Query query) {
         List<Query> filters = new ArrayList<>();
-        Query typeFilter = createTypeFilter(queryShardContext.getTypes());
-        if (typeFilter != null) {
-            filters.add(typeFilter);
-        }
 
         if (mapperService().hasNested()
-                && typeFilter == null // when a _type filter is set, it will automatically exclude nested docs
                 && new NestedHelper(mapperService()).mightMatchNestedDocs(query)
                 && (aliasFilter == null || new NestedHelper(mapperService()).mightMatchNestedDocs(aliasFilter))) {
             filters.add(Queries.newNonNestedFilter());
@@ -301,17 +294,6 @@ && new NestedHelper(mapperService()).mightMatchNestedDocs(query)
         }
     }
 
-    private Query createTypeFilter(String[] types) {
-        if (types != null && types.length >= 1) {
-            MappedFieldType ft = mapperService().fullName(TypeFieldMapper.NAME);
-            if (ft != null) {
-                // ft might be null if no documents have been indexed yet
-                return ft.termsQuery(Arrays.asList(types), queryShardContext);
-            }
-        }
-        return null;
-    }
-
     @Override
     public long id() {
         return this.id;
diff --git a/server/src/main/java/org/elasticsearch/search/SearchHits.java b/server/src/main/java/org/elasticsearch/search/SearchHits.java
index e5021b47a3e36..7d884c028b548 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchHits.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchHits.java
@@ -47,7 +47,7 @@ public static SearchHits empty() {
     }
 
     public static SearchHits empty(boolean withTotalHits) {
-        // We shouldn't use static final instance, since that could directly be returned by native transport clients
+        // TODO: consider using static final instance
         return new SearchHits(EMPTY, withTotalHits ? new TotalHits(0, Relation.EQUAL_TO) : null, 0);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java
index 4c6ba07c631af..01a4aa66810c8 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -32,9 +32,7 @@
 import org.elasticsearch.common.xcontent.ParseFieldRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.MatchBoolPrefixQueryBuilder;
 import org.elasticsearch.index.query.BoostingQueryBuilder;
-import org.elasticsearch.index.query.CommonTermsQueryBuilder;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
 import org.elasticsearch.index.query.DisMaxQueryBuilder;
 import org.elasticsearch.index.query.DistanceFeatureQueryBuilder;
@@ -49,6 +47,7 @@
 import org.elasticsearch.index.query.IntervalQueryBuilder;
 import org.elasticsearch.index.query.IntervalsSourceProvider;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.MatchBoolPrefixQueryBuilder;
 import org.elasticsearch.index.query.MatchNoneQueryBuilder;
 import org.elasticsearch.index.query.MatchPhrasePrefixQueryBuilder;
 import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
@@ -154,9 +153,13 @@
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicParser;
 import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.LongRareTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.RareTermsAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.StringRareTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.UnmappedRareTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.UnmappedTerms;
 import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder;
@@ -280,7 +283,6 @@ public class SearchModule {
     public static final Setting<Integer> INDICES_MAX_CLAUSE_COUNT_SETTING = Setting.intSetting("indices.query.bool.max_clause_count",
             1024, 1, Integer.MAX_VALUE, Setting.Property.NodeScope);
 
-    private final boolean transportClient;
     private final Map<String, Highlighter> highlighters;
     private final ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry = new ParseFieldRegistry<>(
             "significance_heuristic");
@@ -296,14 +298,11 @@ public class SearchModule {
      *
      * NOTE: This constructor should not be called in production unless an accurate {@link Settings} object is provided.
      *       When constructed, a static flag is set in Lucene {@link BooleanQuery#setMaxClauseCount} according to the settings.
-     *
-     * @param settings Current settings
-     * @param transportClient Is this being constructed in the TransportClient or not
+     *  @param settings Current settings
      * @param plugins List of included {@link SearchPlugin} objects.
      */
-    public SearchModule(Settings settings, boolean transportClient, List<SearchPlugin> plugins) {
+    public SearchModule(Settings settings, List<SearchPlugin> plugins) {
         this.settings = settings;
-        this.transportClient = transportClient;
         registerSuggesters(plugins);
         highlighters = setupHighlighters(settings, plugins);
         registerScoreFunctions(plugins);
@@ -395,6 +394,11 @@ private void registerAggregations(List<SearchPlugin> plugins) {
                     .addResultReader(UnmappedTerms.NAME, UnmappedTerms::new)
                     .addResultReader(LongTerms.NAME, LongTerms::new)
                     .addResultReader(DoubleTerms.NAME, DoubleTerms::new));
+        registerAggregation(new AggregationSpec(RareTermsAggregationBuilder.NAME, RareTermsAggregationBuilder::new,
+                RareTermsAggregationBuilder::parse)
+                    .addResultReader(StringRareTerms.NAME, StringRareTerms::new)
+                    .addResultReader(UnmappedRareTerms.NAME, UnmappedRareTerms::new)
+                    .addResultReader(LongRareTerms.NAME, LongRareTerms::new));
         registerAggregation(new AggregationSpec(SignificantTermsAggregationBuilder.NAME, SignificantTermsAggregationBuilder::new,
                 SignificantTermsAggregationBuilder.getParser(significanceHeuristicParserRegistry))
                     .addResultReader(SignificantStringTerms.NAME, SignificantStringTerms::new)
@@ -438,12 +442,10 @@ private void registerAggregations(List<SearchPlugin> plugins) {
     }
 
     private void registerAggregation(AggregationSpec spec) {
-        if (false == transportClient) {
-            namedXContents.add(new NamedXContentRegistry.Entry(BaseAggregationBuilder.class, spec.getName(), (p, c) -> {
-                AggregatorFactories.AggParseContext context = (AggregatorFactories.AggParseContext) c;
-                return spec.getParser().parse(context.name, p);
-            }));
-        }
+        namedXContents.add(new NamedXContentRegistry.Entry(BaseAggregationBuilder.class, spec.getName(), (p, c) -> {
+            AggregatorFactories.AggParseContext context = (AggregatorFactories.AggParseContext) c;
+            return spec.getParser().parse(context.name, p);
+        }));
         namedWriteables.add(
                 new NamedWriteableRegistry.Entry(AggregationBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
         for (Map.Entry<String, Writeable.Reader<? extends InternalAggregation>> t : spec.getResultReaders().entrySet()) {
@@ -539,12 +541,10 @@ private void registerPipelineAggregations(List<SearchPlugin> plugins) {
     }
 
     private void registerPipelineAggregation(PipelineAggregationSpec spec) {
-        if (false == transportClient) {
-            namedXContents.add(new NamedXContentRegistry.Entry(BaseAggregationBuilder.class, spec.getName(), (p, c) -> {
-                AggregatorFactories.AggParseContext context = (AggregatorFactories.AggParseContext) c;
-                return spec.getParser().parse(context.name, p);
-            }));
-        }
+        namedXContents.add(new NamedXContentRegistry.Entry(BaseAggregationBuilder.class, spec.getName(), (p, c) -> {
+            AggregatorFactories.AggParseContext context = (AggregatorFactories.AggParseContext) c;
+            return spec.getParser().parse(context.name, p);
+        }));
         namedWriteables.add(
                 new NamedWriteableRegistry.Entry(PipelineAggregationBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
         namedWriteables.add(
@@ -567,9 +567,7 @@ private void registerRescorers(List<SearchPlugin> plugins) {
     }
 
     private void registerRescorer(RescorerSpec<?> spec) {
-        if (false == transportClient) {
-            namedXContents.add(new NamedXContentRegistry.Entry(RescorerBuilder.class, spec.getName(), (p, c) -> spec.getParser().apply(p)));
-        }
+        namedXContents.add(new NamedXContentRegistry.Entry(RescorerBuilder.class, spec.getName(), (p, c) -> spec.getParser().apply(p)));
         namedWriteables.add(new NamedWriteableRegistry.Entry(RescorerBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
     }
 
@@ -767,7 +765,6 @@ private void registerQueryParsers(List<SearchPlugin> plugins) {
         registerQuery(new QuerySpec<>(MoreLikeThisQueryBuilder.NAME, MoreLikeThisQueryBuilder::new,
                 MoreLikeThisQueryBuilder::fromXContent));
         registerQuery(new QuerySpec<>(WrapperQueryBuilder.NAME, WrapperQueryBuilder::new, WrapperQueryBuilder::fromXContent));
-        registerQuery(new QuerySpec<>(CommonTermsQueryBuilder.NAME, CommonTermsQueryBuilder::new, CommonTermsQueryBuilder::fromXContent));
         registerQuery(
                 new QuerySpec<>(SpanMultiTermQueryBuilder.NAME, SpanMultiTermQueryBuilder::new, SpanMultiTermQueryBuilder::fromXContent));
         registerQuery(new QuerySpec<>(FunctionScoreQueryBuilder.NAME, FunctionScoreQueryBuilder::new,
@@ -804,6 +801,10 @@ private void registerIntervalsSourceProviders() {
             IntervalsSourceProvider.Combine.NAME, IntervalsSourceProvider.Combine::new));
         namedWriteables.add(new NamedWriteableRegistry.Entry(IntervalsSourceProvider.class,
             IntervalsSourceProvider.Disjunction.NAME, IntervalsSourceProvider.Disjunction::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(IntervalsSourceProvider.class,
+            IntervalsSourceProvider.Prefix.NAME, IntervalsSourceProvider.Prefix::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(IntervalsSourceProvider.class,
+            IntervalsSourceProvider.Wildcard.NAME, IntervalsSourceProvider.Wildcard::new));
     }
 
     private void registerQuery(QuerySpec<?> spec) {
diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java
index daf8e1faf7bb8..19ae52cb51099 100644
--- a/server/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/server/src/main/java/org/elasticsearch/search/SearchService.java
@@ -548,19 +548,35 @@ final SearchContext createAndPutContext(ShardSearchRequest request) throws IOExc
         }
 
         SearchContext context = createContext(request);
+        onNewContext(context);
         boolean success = false;
         try {
             putContext(context);
-            if (request.scroll() != null) {
+            success = true;
+            return context;
+        } finally {
+            if (success == false) {
+                freeContext(context.id());
+            }
+        }
+    }
+
+    private void onNewContext(SearchContext context) {
+        boolean success = false;
+        try {
+            if (context.scrollContext() != null) {
                 openScrollContexts.incrementAndGet();
                 context.indexShard().getSearchOperationListener().onNewScrollContext(context);
             }
             context.indexShard().getSearchOperationListener().onNewContext(context);
             success = true;
-            return context;
         } finally {
-            if (!success) {
-                freeContext(context.id());
+            // currently, the concrete listener is CompositeListener, which swallows exceptions, but here we anyway try to do the
+            // right thing by closing and notifying onFreeXXX in case one of the listeners fails with an exception in the future.
+            if (success == false) {
+                try (context) {
+                    onFreeContext(context);
+                }
             }
         }
     }
@@ -648,18 +664,23 @@ private void freeAllContextForIndex(Index index) {
     public boolean freeContext(long id) {
         try (SearchContext context = removeContext(id)) {
             if (context != null) {
-                assert context.refCount() > 0 : " refCount must be > 0: " + context.refCount();
-                context.indexShard().getSearchOperationListener().onFreeContext(context);
-                if (context.scrollContext() != null) {
-                    openScrollContexts.decrementAndGet();
-                    context.indexShard().getSearchOperationListener().onFreeScrollContext(context);
-                }
+                onFreeContext(context);
                 return true;
             }
             return false;
         }
     }
 
+    private void onFreeContext(SearchContext context) {
+        assert context.refCount() > 0 : " refCount must be > 0: " + context.refCount();
+        assert activeContexts.containsKey(context.id()) == false;
+        context.indexShard().getSearchOperationListener().onFreeContext(context);
+        if (context.scrollContext() != null) {
+            openScrollContexts.decrementAndGet();
+            context.indexShard().getSearchOperationListener().onFreeScrollContext(context);
+        }
+    }
+
     public void freeAllScrollContexts() {
         for (SearchContext searchContext : activeContexts.values()) {
             if (searchContext.scrollContext() != null) {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java
index f07d9abfcb035..117f3a0772a71 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java
@@ -135,12 +135,12 @@ public final String getWriteableName() {
     }
 
     @Override
-    public final AggregatorFactory<?> build(SearchContext context, AggregatorFactory<?> parent) throws IOException {
-        AggregatorFactory<?> factory = doBuild(context, parent, factoriesBuilder);
+    public final AggregatorFactory build(SearchContext context, AggregatorFactory parent) throws IOException {
+        AggregatorFactory factory = doBuild(context, parent, factoriesBuilder);
         return factory;
     }
 
-    protected abstract AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent,
+    protected abstract AggregatorFactory doBuild(SearchContext context, AggregatorFactory parent,
             AggregatorFactories.Builder subfactoriesBuilder) throws IOException;
 
     @Override
@@ -164,28 +164,17 @@ public final XContentBuilder toXContent(XContentBuilder builder, Params params)
 
     @Override
     public int hashCode() {
-        return Objects.hash(factoriesBuilder, metaData, name, doHashCode());
+        return Objects.hash(factoriesBuilder, metaData, name);
     }
 
-    protected abstract int doHashCode();
-
     @Override
     public boolean equals(Object obj) {
-        if (obj == null)
-            return false;
-        if (getClass() != obj.getClass())
-            return false;
-        @SuppressWarnings("unchecked")
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
         AbstractAggregationBuilder<AB> other = (AbstractAggregationBuilder<AB>) obj;
-        if (!Objects.equals(name, other.name))
-            return false;
-        if (!Objects.equals(metaData, other.metaData))
-            return false;
-        if (!Objects.equals(factoriesBuilder, other.factoriesBuilder))
-            return false;
-        return doEquals(obj);
-    }
-
-    protected abstract boolean doEquals(Object obj);
 
+        return Objects.equals(name, other.name)
+            && Objects.equals(metaData, other.metaData)
+            && Objects.equals(factoriesBuilder, other.factoriesBuilder);
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
index 359c8dd571e25..6a7b955542480 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
@@ -63,7 +63,7 @@ public String getName() {
     }
 
     /** Internal: build an {@link AggregatorFactory} based on the configuration of this builder. */
-    protected abstract AggregatorFactory<?> build(SearchContext context, AggregatorFactory<?> parent) throws IOException;
+    protected abstract AggregatorFactory build(SearchContext context, AggregatorFactory parent) throws IOException;
 
     /** Associate metadata with this {@link AggregationBuilder}. */
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
index 7f296daa5d18c..5e818fe69c7ad 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
@@ -161,16 +161,16 @@ public AggParseContext(String name) {
         }
     }
 
-    public static final AggregatorFactories EMPTY = new AggregatorFactories(new AggregatorFactory<?>[0], new ArrayList<>());
+    public static final AggregatorFactories EMPTY = new AggregatorFactories(new AggregatorFactory[0], new ArrayList<>());
 
-    private AggregatorFactory<?>[] factories;
+    private AggregatorFactory[] factories;
     private List<PipelineAggregationBuilder> pipelineAggregatorFactories;
 
     public static Builder builder() {
         return new Builder();
     }
 
-    private AggregatorFactories(AggregatorFactory<?>[] factories, List<PipelineAggregationBuilder> pipelineAggregators) {
+    private AggregatorFactories(AggregatorFactory[] factories, List<PipelineAggregationBuilder> pipelineAggregators) {
         this.factories = factories;
         this.pipelineAggregatorFactories = pipelineAggregators;
     }
@@ -314,7 +314,7 @@ Builder skipResolveOrder() {
             return this;
         }
 
-        public AggregatorFactories build(SearchContext context, AggregatorFactory<?> parent) throws IOException {
+        public AggregatorFactories build(SearchContext context, AggregatorFactory parent) throws IOException {
             if (aggregationBuilders.isEmpty() && pipelineAggregatorBuilders.isEmpty()) {
                 return EMPTY;
             }
@@ -325,7 +325,7 @@ public AggregatorFactories build(SearchContext context, AggregatorFactory<?> par
                 orderedpipelineAggregators = resolvePipelineAggregatorOrder(this.pipelineAggregatorBuilders, this.aggregationBuilders,
                         parent);
             }
-            AggregatorFactory<?>[] aggFactories = new AggregatorFactory<?>[aggregationBuilders.size()];
+            AggregatorFactory[] aggFactories = new AggregatorFactory[aggregationBuilders.size()];
 
             int i = 0;
             for (AggregationBuilder agg : aggregationBuilders) {
@@ -337,7 +337,7 @@ public AggregatorFactories build(SearchContext context, AggregatorFactory<?> par
 
         private List<PipelineAggregationBuilder> resolvePipelineAggregatorOrder(
                 Collection<PipelineAggregationBuilder> pipelineAggregatorBuilders, Collection<AggregationBuilder> aggregationBuilders,
-                AggregatorFactory<?> parent) {
+                AggregatorFactory parent) {
             Map<String, PipelineAggregationBuilder> pipelineAggregatorBuildersMap = new HashMap<>();
             for (PipelineAggregationBuilder builder : pipelineAggregatorBuilders) {
                 pipelineAggregatorBuildersMap.put(builder.getName(), builder);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
index d6e7aca46a63b..196f4faea3e2d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
@@ -33,17 +33,17 @@
 import java.util.List;
 import java.util.Map;
 
-public abstract class AggregatorFactory<AF extends AggregatorFactory<AF>> {
+public abstract class AggregatorFactory {
 
     public static final class MultiBucketAggregatorWrapper extends Aggregator {
         private final BigArrays bigArrays;
         private final Aggregator parent;
-        private final AggregatorFactory<?> factory;
+        private final AggregatorFactory factory;
         private final Aggregator first;
         ObjectArray<Aggregator> aggregators;
         ObjectArray<LeafBucketCollector> collectors;
 
-        MultiBucketAggregatorWrapper(BigArrays bigArrays, SearchContext context, Aggregator parent, AggregatorFactory<?> factory,
+        MultiBucketAggregatorWrapper(BigArrays bigArrays, SearchContext context, Aggregator parent, AggregatorFactory factory,
                 Aggregator first) {
             this.bigArrays = bigArrays;
             this.parent = parent;
@@ -167,7 +167,7 @@ public void close() {
     }
 
     protected final String name;
-    protected final AggregatorFactory<?> parent;
+    protected final AggregatorFactory parent;
     protected final AggregatorFactories factories;
     protected final Map<String, Object> metaData;
     protected final SearchContext context;
@@ -180,7 +180,7 @@ public void close() {
      * @throws IOException
      *             if an error occurs creating the factory
      */
-    public AggregatorFactory(String name, SearchContext context, AggregatorFactory<?> parent,
+    public AggregatorFactory(String name, SearchContext context, AggregatorFactory parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         this.name = name;
         this.context = context;
@@ -217,7 +217,7 @@ public final Aggregator create(Aggregator parent, boolean collectsFromSingleBuck
         return createInternal(parent, collectsFromSingleBucket, this.factories.createPipelineAggregators(), this.metaData);
     }
 
-    public AggregatorFactory<?> getParent() {
+    public AggregatorFactory getParent() {
         return parent;
     }
 
@@ -226,7 +226,7 @@ public AggregatorFactory<?> getParent() {
      * {@link Aggregator}s that only know how to collect bucket {@code 0}, this
      * returns an aggregator that can collect any bucket.
      */
-    protected static Aggregator asMultiBucketAggregator(final AggregatorFactory<?> factory, final SearchContext context,
+    protected static Aggregator asMultiBucketAggregator(final AggregatorFactory factory, final SearchContext context,
             final Aggregator parent) throws IOException {
         final Aggregator first = factory.create(parent, true);
         final BigArrays bigArrays = context.bigArrays();
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
index b525fd32d918a..d664b10cfa16e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
@@ -218,40 +218,22 @@ public final XContentBuilder toXContent(XContentBuilder builder, Params params)
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, metaData, pipelineAggregators, doHashCode());
+        return Objects.hash(name, metaData, pipelineAggregators);
     }
 
-    /**
-     * Opportunity for subclasses to the {@link #hashCode()} for this
-     * class.
-     **/
-    protected abstract int doHashCode();
-
     @Override
     public boolean equals(Object obj) {
-        if (obj == null) {
-            return false;
-        }
-        if (obj.getClass() != getClass()) {
+        if (obj == null || getClass() != obj.getClass()) {
             return false;
         }
+        if (obj == this) { return true; }
+
         InternalAggregation other = (InternalAggregation) obj;
         return Objects.equals(name, other.name) &&
                 Objects.equals(pipelineAggregators, other.pipelineAggregators) &&
-                Objects.equals(metaData, other.metaData) &&
-                doEquals(obj);
+                Objects.equals(metaData, other.metaData);
     }
 
-    /**
-     * Opportunity for subclasses to add criteria to the {@link #equals(Object)}
-     * method for this class.
-     *
-     * This method can safely cast <code>obj</code> to the subclass since the
-     * {@link #equals(Object)} method checks that <code>obj</code> is the same
-     * class as <code>this</code>
-     */
-    protected abstract boolean doEquals(Object obj);
-
     @Override
     public String toString() {
         return Strings.toString(this);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
index 08938a5b9b9fe..85b9a7dcb0ca5 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
@@ -67,7 +67,7 @@ public final String[] getBucketsPaths() {
      * Internal: Validates the state of this factory (makes sure the factory is properly
      * configured)
      */
-    protected abstract void validate(AggregatorFactory<?> parent, Collection<AggregationBuilder> aggregationBuilders,
+    protected abstract void validate(AggregatorFactory parent, Collection<AggregationBuilder> aggregationBuilders,
             Collection<PipelineAggregationBuilder> pipelineAggregatorBuilders);
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
index 32695ac69a88e..5e5e60760d790 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
@@ -40,6 +40,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * A specialization of {@link DeferringBucketCollector} that collects all
@@ -48,28 +49,28 @@
  * this collector.
  */
 public class BestBucketsDeferringCollector extends DeferringBucketCollector {
-    private static class Entry {
+    static class Entry {
         final LeafReaderContext context;
         final PackedLongValues docDeltas;
         final PackedLongValues buckets;
 
         Entry(LeafReaderContext context, PackedLongValues docDeltas, PackedLongValues buckets) {
-            this.context = context;
-            this.docDeltas = docDeltas;
-            this.buckets = buckets;
+            this.context = Objects.requireNonNull(context);
+            this.docDeltas = Objects.requireNonNull(docDeltas);
+            this.buckets = Objects.requireNonNull(buckets);
         }
     }
 
-    final List<Entry> entries = new ArrayList<>();
-    BucketCollector collector;
-    final SearchContext searchContext;
-    final boolean isGlobal;
-    LeafReaderContext context;
-    PackedLongValues.Builder docDeltas;
-    PackedLongValues.Builder buckets;
-    long maxBucket = -1;
-    boolean finished = false;
-    LongHash selectedBuckets;
+    protected List<Entry> entries = new ArrayList<>();
+    protected BucketCollector collector;
+    protected final SearchContext searchContext;
+    protected final boolean isGlobal;
+    protected LeafReaderContext context;
+    protected PackedLongValues.Builder docDeltasBuilder;
+    protected PackedLongValues.Builder bucketsBuilder;
+    protected long maxBucket = -1;
+    protected boolean finished = false;
+    protected LongHash selectedBuckets;
 
     /**
      * Sole constructor.
@@ -97,28 +98,32 @@ public void setDeferredCollector(Iterable<BucketCollector> deferredCollectors) {
 
     private void finishLeaf() {
         if (context != null) {
-            entries.add(new Entry(context, docDeltas.build(), buckets.build()));
+            assert docDeltasBuilder != null && bucketsBuilder != null;
+            entries.add(new Entry(context, docDeltasBuilder.build(), bucketsBuilder.build()));
         }
-        context = null;
-        docDeltas = null;
-        buckets = null;
     }
 
     @Override
     public LeafBucketCollector getLeafCollector(LeafReaderContext ctx) throws IOException {
         finishLeaf();
 
-        context = ctx;
-        docDeltas = PackedLongValues.packedBuilder(PackedInts.DEFAULT);
-        buckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT);
+        context = null;
+        // allocates the builder lazily in case this segment doesn't contain any match
+        docDeltasBuilder = null;
+        bucketsBuilder = null;
 
         return new LeafBucketCollector() {
             int lastDoc = 0;
 
             @Override
             public void collect(int doc, long bucket) throws IOException {
-                docDeltas.add(doc - lastDoc);
-                buckets.add(bucket);
+                if (context == null) {
+                    context = ctx;
+                    docDeltasBuilder = PackedLongValues.packedBuilder(PackedInts.DEFAULT);
+                    bucketsBuilder = PackedLongValues.packedBuilder(PackedInts.DEFAULT);
+                }
+                docDeltasBuilder.add(doc - lastDoc);
+                bucketsBuilder.add(bucket);
                 lastDoc = doc;
                 maxBucket = Math.max(maxBucket, bucket);
             }
@@ -141,7 +146,7 @@ public void postCollection() throws IOException {
      */
     @Override
     public void prepareSelectedBuckets(long... selectedBuckets) throws IOException {
-        if (!finished) {
+        if (finished == false) {
             throw new IllegalStateException("Cannot replay yet, collection is not finished: postCollect() has not been called");
         }
         if (this.selectedBuckets != null) {
@@ -160,14 +165,16 @@ public void prepareSelectedBuckets(long... selectedBuckets) throws IOException {
             Query query = isGlobal ? new MatchAllDocsQuery() : searchContext.query();
             weight = searchContext.searcher().createWeight(searchContext.searcher().rewrite(query), ScoreMode.COMPLETE, 1f);
         }
+
         for (Entry entry : entries) {
+            assert entry.docDeltas.size() > 0 : "segment should have at least one document to replay, got 0";
             final LeafBucketCollector leafCollector = collector.getLeafCollector(entry.context);
-            DocIdSetIterator docIt = null;
-            if (needsScores && entry.docDeltas.size() > 0) {
+            DocIdSetIterator scoreIt = null;
+            if (needsScores) {
                 Scorer scorer = weight.scorer(entry.context);
                 // We don't need to check if the scorer is null
                 // since we are sure that there are documents to replay (entry.docDeltas it not empty).
-                docIt = scorer.iterator();
+                scoreIt = scorer.iterator();
                 leafCollector.setScorer(scorer);
             }
             final PackedLongValues.Iterator docDeltaIterator = entry.docDeltas.iterator();
@@ -179,17 +186,16 @@ public void prepareSelectedBuckets(long... selectedBuckets) throws IOException {
                 final long rebasedBucket = hash.find(bucket);
                 if (rebasedBucket != -1) {
                     if (needsScores) {
-                        if (docIt.docID() < doc) {
-                            docIt.advance(doc);
+                        if (scoreIt.docID() < doc) {
+                            scoreIt.advance(doc);
                         }
                         // aggregations should only be replayed on matching documents
-                        assert docIt.docID() == doc;
+                        assert scoreIt.docID() == doc;
                     }
                     leafCollector.collect(doc, rebasedBucket);
                 }
             }
         }
-
         collector.postCollection();
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
index 71dacc698bee6..a4ef4286447c1 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java
@@ -90,7 +90,9 @@ public final void mergeBuckets(long[] mergeMap, long newNumBuckets) {
             docCounts.fill(0, newNumBuckets, 0);
             for (int i = 0; i < oldDocCounts.size(); i++) {
                 int docCount = oldDocCounts.get(i);
-                if (docCount != 0) {
+
+                // Skip any in the map which have been "removed", signified with -1
+                if (docCount != 0 && mergeMap[i] != -1) {
                     docCounts.increment(mergeMap[i], docCount);
                 }
             }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
index 5626c8f21859d..cf5ddc54884d0 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
@@ -137,14 +137,18 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalSingleBucketAggregation other = (InternalSingleBucketAggregation) obj;
         return Objects.equals(docCount, other.docCount) &&
                 Objects.equals(aggregations, other.aggregations);
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(docCount, aggregations);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), docCount, aggregations);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/MergingBucketsDeferringCollector.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/MergingBucketsDeferringCollector.java
index 53049d0301c2d..bff5015846951 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/MergingBucketsDeferringCollector.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/MergingBucketsDeferringCollector.java
@@ -19,222 +19,106 @@
 
 package org.elasticsearch.search.aggregations.bucket;
 
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.packed.PackedInts;
 import org.apache.lucene.util.packed.PackedLongValues;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.LongHash;
-import org.elasticsearch.search.aggregations.Aggregator;
-import org.elasticsearch.search.aggregations.BucketCollector;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.LeafBucketCollector;
-import org.elasticsearch.search.aggregations.MultiBucketCollector;
 import org.elasticsearch.search.internal.SearchContext;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
 /**
- * A specialization of {@link DeferringBucketCollector} that collects all
+ * A specialization of {@link BestBucketsDeferringCollector} that collects all
  * matches and then is able to replay a given subset of buckets. Exposes
  * mergeBuckets, which can be invoked by the aggregator when increasing the
  * rounding interval.
  */
-public class MergingBucketsDeferringCollector extends DeferringBucketCollector {
-
-    List<Entry> entries = new ArrayList<>();
-    BucketCollector collector;
-    final SearchContext searchContext;
-    LeafReaderContext context;
-    PackedLongValues.Builder docDeltas;
-    PackedLongValues.Builder buckets;
-    long maxBucket = -1;
-    boolean finished = false;
-    LongHash selectedBuckets;
-
-    public MergingBucketsDeferringCollector(SearchContext context) {
-        this.searchContext = context;
-    }
-
-    @Override
-    public void setDeferredCollector(Iterable<BucketCollector> deferredCollectors) {
-        this.collector = MultiBucketCollector.wrap(deferredCollectors);
-    }
-
-    @Override
-    public ScoreMode scoreMode() {
-        if (collector == null) {
-            throw new IllegalStateException();
-        }
-        return collector.scoreMode();
-    }
-
-    @Override
-    public void preCollection() throws IOException {
-        collector.preCollection();
-    }
-
-    private void finishLeaf() {
-        if (context != null) {
-            entries.add(new Entry(context, docDeltas.build(), buckets.build()));
-        }
-        context = null;
-        docDeltas = null;
-        buckets = null;
-    }
-
-    @Override
-    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx) throws IOException {
-        finishLeaf();
-
-        context = ctx;
-        docDeltas = PackedLongValues.packedBuilder(PackedInts.DEFAULT);
-        buckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT);
-
-        return new LeafBucketCollector() {
-            int lastDoc = 0;
-
-            @Override
-            public void collect(int doc, long bucket) {
-                docDeltas.add(doc - lastDoc);
-                buckets.add(bucket);
-                lastDoc = doc;
-                maxBucket = Math.max(maxBucket, bucket);
-            }
-        };
+public class MergingBucketsDeferringCollector extends BestBucketsDeferringCollector {
+    public MergingBucketsDeferringCollector(SearchContext context, boolean isGlobal) {
+        super(context, isGlobal);
     }
 
+/**
+     * Merges/prunes the existing bucket ordinals and docDeltas according to the provided mergeMap.
+     *
+     * The mergeMap is an array where the index position represents the current bucket ordinal, and
+     * the value at that position represents the ordinal the bucket should be merged with.  If
+     * the value is set to -1 it is removed entirely.
+     *
+     * For example, if the mergeMap [1,1,3,-1,3] is provided:
+     *  - Buckets `0` and `1` will be merged to bucket ordinal `1`
+     *  - Bucket `2` and `4` will be merged to ordinal `3`
+     *  - Bucket `3` will be removed entirely
+     *
+     *  This process rebuilds the ordinals and docDeltas according to the mergeMap, so it should
+     *  not be called unless there are actually changes to be made, to avoid unnecessary work.
+     */
     public void mergeBuckets(long[] mergeMap) {
-
         List<Entry> newEntries = new ArrayList<>(entries.size());
         for (Entry sourceEntry : entries) {
             PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT);
+            PackedLongValues.Builder newDocDeltas = PackedLongValues.packedBuilder(PackedInts.DEFAULT);
+            PackedLongValues.Iterator docDeltasItr = sourceEntry.docDeltas.iterator();
+
+            long lastGoodDelta = 0;
             for (PackedLongValues.Iterator itr = sourceEntry.buckets.iterator(); itr.hasNext();) {
                 long bucket = itr.next();
-                newBuckets.add(mergeMap[Math.toIntExact(bucket)]);
+                assert docDeltasItr.hasNext();
+                long delta = docDeltasItr.next();
+
+                // Only merge in the ordinal if it hasn't been "removed", signified with -1
+                long ordinal = mergeMap[Math.toIntExact(bucket)];
+
+                if (ordinal != -1) {
+                    newBuckets.add(ordinal);
+                    newDocDeltas.add(delta + lastGoodDelta);
+                    lastGoodDelta = 0;
+                } else {
+                    // we are skipping this ordinal, which means we need to accumulate the
+                    // doc delta's since the last "good" delta
+                    lastGoodDelta += delta;
+                }
+            }
+            // Only create an entry if this segment has buckets after merging
+            if (newBuckets.size() > 0) {
+                assert newDocDeltas.size() > 0 : "docDeltas was empty but we had buckets";
+                newEntries.add(new Entry(sourceEntry.context, newDocDeltas.build(), newBuckets.build()));
             }
-            newEntries.add(new Entry(sourceEntry.context, sourceEntry.docDeltas, newBuckets.build()));
         }
         entries = newEntries;
 
         // if there are buckets that have been collected in the current segment
         // we need to update the bucket ordinals there too
-        if (buckets.size() > 0) {
-            PackedLongValues currentBuckets = buckets.build();
+        if (bucketsBuilder != null && bucketsBuilder.size() > 0) {
+            PackedLongValues currentBuckets = bucketsBuilder.build();
             PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT);
-            for (PackedLongValues.Iterator itr = currentBuckets.iterator(); itr.hasNext();) {
-                long bucket = itr.next();
-                newBuckets.add(mergeMap[Math.toIntExact(bucket)]);
-            }
-            buckets = newBuckets;
-        }
-    }
-
-    @Override
-    public void postCollection() {
-        finishLeaf();
-        finished = true;
-    }
-
-    /**
-     * Replay the wrapped collector, but only on a selection of buckets.
-     */
-    @Override
-    public void prepareSelectedBuckets(long... selectedBuckets) throws IOException {
-        if (finished == false) {
-            throw new IllegalStateException("Cannot replay yet, collection is not finished: postCollect() has not been called");
-        }
-        if (this.selectedBuckets != null) {
-            throw new IllegalStateException("Already been replayed");
-        }
+            PackedLongValues.Builder newDocDeltas = PackedLongValues.packedBuilder(PackedInts.DEFAULT);
 
-        final LongHash hash = new LongHash(selectedBuckets.length, BigArrays.NON_RECYCLING_INSTANCE);
-        for (long bucket : selectedBuckets) {
-            hash.add(bucket);
-        }
-        this.selectedBuckets = hash;
-
-        boolean needsScores = collector.scoreMode().needsScores();
-        Weight weight = null;
-        if (needsScores) {
-            weight = searchContext.searcher().createWeight(
-                    searchContext.searcher().rewrite(searchContext.query()),
-                    ScoreMode.COMPLETE, 1f);
-        }
-        for (Entry entry : entries) {
-            final LeafBucketCollector leafCollector = collector.getLeafCollector(entry.context);
-            DocIdSetIterator docIt = null;
-            if (needsScores && entry.docDeltas.size() > 0) {
-                Scorer scorer = weight.scorer(entry.context);
-                // We don't need to check if the scorer is null
-                // since we are sure that there are documents to replay
-                // (entry.docDeltas it not empty).
-                docIt = scorer.iterator();
-                leafCollector.setScorer(scorer);
-            }
-            final PackedLongValues.Iterator docDeltaIterator = entry.docDeltas.iterator();
-            final PackedLongValues.Iterator buckets = entry.buckets.iterator();
-            int doc = 0;
-            for (long i = 0, end = entry.docDeltas.size(); i < end; ++i) {
-                doc += docDeltaIterator.next();
-                final long bucket = buckets.next();
-                final long rebasedBucket = hash.find(bucket);
-                if (rebasedBucket != -1) {
-                    if (needsScores) {
-                        if (docIt.docID() < doc) {
-                            docIt.advance(doc);
-                        }
-                        // aggregations should only be replayed on matching
-                        // documents
-                        assert docIt.docID() == doc;
-                    }
-                    leafCollector.collect(doc, rebasedBucket);
-                }
-            }
-        }
+            // The current segment's deltas aren't built yet, so build to a temp object
+            PackedLongValues currentDeltas = docDeltasBuilder.build();
+            PackedLongValues.Iterator docDeltasItr = currentDeltas.iterator();
 
-        collector.postCollection();
-    }
-
-    /**
-     * Wrap the provided aggregator so that it behaves (almost) as if it had
-     * been collected directly.
-     */
-    @Override
-    public Aggregator wrap(final Aggregator in) {
-
-        return new WrappedAggregator(in) {
-
-            @Override
-            public InternalAggregation buildAggregation(long bucket) throws IOException {
-                if (selectedBuckets == null) {
-                    throw new IllegalStateException("Collection has not been replayed yet.");
-                }
-                final long rebasedBucket = selectedBuckets.find(bucket);
-                if (rebasedBucket == -1) {
-                    throw new IllegalStateException("Cannot build for a bucket which has not been collected [" + bucket + "]");
+            long lastGoodDelta = 0;
+            for (PackedLongValues.Iterator itr = currentBuckets.iterator(); itr.hasNext();) {
+                long bucket = itr.next();
+                assert docDeltasItr.hasNext();
+                long delta = docDeltasItr.next();
+                long ordinal = mergeMap[Math.toIntExact(bucket)];
+
+                // Only merge in the ordinal if it hasn't been "removed", signified with -1
+                if (ordinal != -1) {
+                    newBuckets.add(ordinal);
+                    newDocDeltas.add(delta + lastGoodDelta);
+                    lastGoodDelta = 0;
+                } else {
+                    // we are skipping this ordinal, which means we need to accumulate the
+                    // doc delta's since the last "good" delta.
+                    // The first is skipped because the original deltas are stored as offsets from first doc,
+                    // not offsets from 0
+                    lastGoodDelta += delta;
                 }
-                return in.buildAggregation(rebasedBucket);
             }
-
-        };
-    }
-
-    private static class Entry {
-        final LeafReaderContext context;
-        final PackedLongValues docDeltas;
-        final PackedLongValues buckets;
-
-        Entry(LeafReaderContext context, PackedLongValues docDeltas, PackedLongValues buckets) {
-            this.context = context;
-            this.docDeltas = docDeltas;
-            this.buckets = buckets;
+            docDeltasBuilder = newDocDeltas;
+            bucketsBuilder = newBuckets;
         }
     }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java
index fa33d64a1fb91..6b0530eb62e07 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java
@@ -196,7 +196,7 @@ public Map<String, QueryBuilder> filters() {
 
 
     @Override
-    protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+    protected AggregatorFactory doBuild(SearchContext context, AggregatorFactory parent, Builder subFactoriesBuilder)
             throws IOException {
         int maxFilters = context.indexShard().indexSettings().getMaxAdjacencyMatrixFilters();
         if (filters.size() > maxFilters){
@@ -230,12 +230,15 @@ protected XContentBuilder internalXContent(XContentBuilder builder, Params param
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(filters, separator);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), filters, separator);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         AdjacencyMatrixAggregationBuilder other = (AdjacencyMatrixAggregationBuilder) obj;
         return Objects.equals(filters, other.filters) && Objects.equals(separator, other.separator);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorFactory.java
index 7be588bcb5607..541a8e482c634 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorFactory.java
@@ -34,14 +34,14 @@
 import java.util.List;
 import java.util.Map;
 
-public class AdjacencyMatrixAggregatorFactory extends AggregatorFactory<AdjacencyMatrixAggregatorFactory> {
+public class AdjacencyMatrixAggregatorFactory extends AggregatorFactory {
 
     private final String[] keys;
     private final Weight[] weights;
     private final String separator;
 
     public AdjacencyMatrixAggregatorFactory(String name, List<KeyedFilter> filters, String separator, 
-            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactories, 
+            SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactories,
             Map<String, Object> metaData) throws IOException {
         super(name, context, parent, subFactories, metaData);
         IndexSearcher contextSearcher = context.searcher();
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java
index c862f458939ed..46eac6ce55dda 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java
@@ -239,12 +239,16 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(buckets);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), buckets);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalAdjacencyMatrix that = (InternalAdjacencyMatrix) obj;
         return Objects.equals(buckets, that.buckets);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java
index 809dec737ff2b..c91e40d4609f8 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java
@@ -162,7 +162,7 @@ public int size() {
      * this aggregator or the instance of the parent's factory that is incompatible with
      * the composite aggregation.
      */
-    private AggregatorFactory<?> checkParentIsNullOrNested(AggregatorFactory<?> factory) {
+    private AggregatorFactory checkParentIsNullOrNested(AggregatorFactory factory) {
         if (factory == null) {
             return null;
         } else if (factory instanceof NestedAggregatorFactory) {
@@ -195,9 +195,9 @@ private static void validateSources(List<CompositeValuesSourceBuilder<?>> source
     }
 
     @Override
-    protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent,
+    protected AggregatorFactory doBuild(SearchContext context, AggregatorFactory parent,
                                            AggregatorFactories.Builder subfactoriesBuilder) throws IOException {
-        AggregatorFactory<?> invalid = checkParentIsNullOrNested(parent);
+        AggregatorFactory invalid = checkParentIsNullOrNested(parent);
         if (invalid != null) {
             throw new IllegalArgumentException("[composite] aggregation cannot be used with a parent aggregation of" +
                 " type: [" + invalid.getClass().getSimpleName() + "]");
@@ -256,12 +256,15 @@ protected XContentBuilder internalXContent(XContentBuilder builder, Params param
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(sources, size, after);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), sources, size, after);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         CompositeAggregationBuilder other = (CompositeAggregationBuilder) obj;
         return size == other.size &&
             Objects.equals(sources, other.sources) &&
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationFactory.java
index 2b2fa4fb7e3eb..d79a00ce4c011 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationFactory.java
@@ -29,12 +29,12 @@
 import java.util.List;
 import java.util.Map;
 
-class CompositeAggregationFactory extends AggregatorFactory<CompositeAggregationFactory> {
+class CompositeAggregationFactory extends AggregatorFactory {
     private final int size;
     private final CompositeValuesSourceConfig[] sources;
     private final CompositeKey afterKey;
 
-    CompositeAggregationFactory(String name, SearchContext context, AggregatorFactory<?> parent,
+    CompositeAggregationFactory(String name, SearchContext context, AggregatorFactory parent,
                                 AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData,
                                 int size, CompositeValuesSourceConfig[] sources, CompositeKey afterKey) throws IOException {
         super(name, context, parent, subFactoriesBuilder, metaData);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java
index 5c028e8f124b9..dc7331aa58745 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java
@@ -117,12 +117,10 @@ public final XContentBuilder toXContent(XContentBuilder builder, Params params)
     }
 
     @Override
-    public final int hashCode() {
-        return Objects.hash(field, missingBucket, script, valueType, order, format, innerHashCode());
+    public int hashCode() {
+        return Objects.hash(field, missingBucket, script, valueType, order, format);
     }
 
-    protected abstract int innerHashCode();
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -135,12 +133,9 @@ public boolean equals(Object o) {
             Objects.equals(valueType, that.valueType()) &&
             Objects.equals(missingBucket, that.missingBucket()) &&
             Objects.equals(order, that.order()) &&
-            Objects.equals(format, that.format()) &&
-            innerEquals(that);
+            Objects.equals(format, that.format());
     }
 
-    protected abstract boolean innerEquals(AB builder);
-
     public String name() {
         return name;
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java
index bb7632278de91..71cc5fcd4b2b0 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/DateHistogramValuesSourceBuilder.java
@@ -97,12 +97,16 @@ protected void doXContentBody(XContentBuilder builder, Params params) throws IOE
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(dateHistogramInterval, timeZone);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), dateHistogramInterval, timeZone);
     }
 
     @Override
-    protected boolean innerEquals(DateHistogramValuesSourceBuilder other) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+        DateHistogramValuesSourceBuilder other = (DateHistogramValuesSourceBuilder) obj;
         return Objects.equals(dateHistogramInterval, other.dateHistogramInterval)
             && Objects.equals(timeZone, other.timeZone);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/HistogramValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/HistogramValuesSourceBuilder.java
index 75e2b8a3aaede..6ec1c76d459ab 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/HistogramValuesSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/HistogramValuesSourceBuilder.java
@@ -73,12 +73,16 @@ protected void doXContentBody(XContentBuilder builder, Params params) throws IOE
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(interval);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), interval);
     }
 
     @Override
-    protected boolean innerEquals(HistogramValuesSourceBuilder other) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+        HistogramValuesSourceBuilder other = (HistogramValuesSourceBuilder) obj;
         return Objects.equals(interval, other.interval);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java
index e6a7edd8c217d..ac51301766cb4 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java
@@ -182,7 +182,11 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalComposite that = (InternalComposite) obj;
         return Objects.equals(size, that.size) &&
             Objects.equals(buckets, that.buckets) &&
@@ -191,8 +195,8 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(size, buckets, afterKey, Arrays.hashCode(reverseMuls));
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), size, buckets, afterKey, Arrays.hashCode(reverseMuls));
     }
 
     private static class BucketIterator implements Comparable<BucketIterator> {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/TermsValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/TermsValuesSourceBuilder.java
index 3ef871f59b93c..061b8c5f5dec3 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/TermsValuesSourceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/TermsValuesSourceBuilder.java
@@ -64,16 +64,6 @@ protected void innerWriteTo(StreamOutput out) throws IOException {}
     @Override
     protected void doXContentBody(XContentBuilder builder, Params params) throws IOException {}
 
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(TermsValuesSourceBuilder builder) {
-        return true;
-    }
-
     @Override
     public String type() {
         return TYPE;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
index 71c1c8fa3deb2..e53e8ed3fc2c4 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
@@ -93,7 +93,7 @@ protected AggregationBuilder doRewrite(QueryRewriteContext queryShardContext) th
     }
 
     @Override
-    protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent,
+    protected AggregatorFactory doBuild(SearchContext context, AggregatorFactory parent,
             AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
         return new FilterAggregatorFactory(name, filter, context, parent, subFactoriesBuilder, metaData);
     }
@@ -112,12 +112,15 @@ public static FilterAggregationBuilder parse(String aggregationName, XContentPar
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(filter);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), filter);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         FilterAggregationBuilder other = (FilterAggregationBuilder) obj;
         return Objects.equals(filter, other.filter);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java
index 04dd8d3a53cea..c9773ef632616 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java
@@ -35,13 +35,13 @@
 import java.util.List;
 import java.util.Map;
 
-public class FilterAggregatorFactory extends AggregatorFactory<FilterAggregatorFactory> {
+public class FilterAggregatorFactory extends AggregatorFactory {
 
     private Weight weight;
     private Query filter;
 
     public FilterAggregatorFactory(String name, QueryBuilder filterBuilder, SearchContext context,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
+            AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, context, parent, subFactoriesBuilder, metaData);
         filter = filterBuilder.toQuery(context.getQueryShardContext());
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java
index 54dfc301b2dbc..add11f16483ec 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java
@@ -220,7 +220,7 @@ protected AggregationBuilder doRewrite(QueryRewriteContext queryShardContext) th
     }
 
     @Override
-    protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+    protected AggregatorFactory doBuild(SearchContext context, AggregatorFactory parent, Builder subFactoriesBuilder)
         throws IOException {
         return new FiltersAggregatorFactory(name, filters, keyed, otherBucket, otherBucketKey, context, parent,
             subFactoriesBuilder, metaData);
@@ -331,12 +331,15 @@ public static FiltersAggregationBuilder parse(String aggregationName, XContentPa
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(filters, keyed, otherBucket, otherBucketKey);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), filters, keyed, otherBucket, otherBucketKey);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         FiltersAggregationBuilder other = (FiltersAggregationBuilder) obj;
         return Objects.equals(filters, other.filters)
             && Objects.equals(keyed, other.keyed)
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorFactory.java
index cc299765621aa..4e9eec14c0089 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorFactory.java
@@ -35,7 +35,7 @@
 import java.util.List;
 import java.util.Map;
 
-public class FiltersAggregatorFactory extends AggregatorFactory<FiltersAggregatorFactory> {
+public class FiltersAggregatorFactory extends AggregatorFactory {
 
     private final String[] keys;
     private final Query[] filters;
@@ -45,7 +45,7 @@ public class FiltersAggregatorFactory extends AggregatorFactory<FiltersAggregato
     private final String otherBucketKey;
 
     public FiltersAggregatorFactory(String name, List<KeyedFilter> filters, boolean keyed, boolean otherBucket,
-            String otherBucketKey, SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactories,
+            String otherBucketKey, SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactories,
             Map<String, Object> metaData) throws IOException {
         super(name, context, parent, subFactories, metaData);
         this.keyed = keyed;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilters.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilters.java
index f6ebfa459c02d..d866754c5626e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilters.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilters.java
@@ -251,12 +251,16 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(buckets, keyed);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), buckets, keyed);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalFilters that = (InternalFilters) obj;
         return Objects.equals(buckets, that.buckets)
                 && Objects.equals(keyed, that.keyed);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
index 76fa6621d01cc..bae95c84c00ed 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
@@ -108,9 +108,9 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
     /**
      * Creates a new instance of the {@link ValuesSourceAggregatorFactory}-derived class specific to the geo aggregation.
      */
-    protected abstract ValuesSourceAggregatorFactory<ValuesSource.GeoPoint,?> createFactory(
+    protected abstract ValuesSourceAggregatorFactory<ValuesSource.GeoPoint> createFactory(
         String name, ValuesSourceConfig<ValuesSource.GeoPoint> config, int precision, int requiredSize, int shardSize,
-        SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder, Map<String, Object> metaData
+        SearchContext context, AggregatorFactory parent, Builder subFactoriesBuilder, Map<String, Object> metaData
     ) throws IOException;
 
     public int precision() {
@@ -144,8 +144,8 @@ public int shardSize() {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, ?> innerBuild(SearchContext context,
-            ValuesSourceConfig<ValuesSource.GeoPoint> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+    protected ValuesSourceAggregatorFactory<ValuesSource.GeoPoint> innerBuild(SearchContext context,
+            ValuesSourceConfig<ValuesSource.GeoPoint> config, AggregatorFactory parent, Builder subFactoriesBuilder)
                     throws IOException {
         int shardSize = this.shardSize;
 
@@ -180,22 +180,18 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         GeoGridAggregationBuilder other = (GeoGridAggregationBuilder) obj;
-        if (precision != other.precision) {
-            return false;
-        }
-        if (requiredSize != other.requiredSize) {
-            return false;
-        }
-        if (shardSize != other.shardSize) {
-            return false;
-        }
-        return true;
+        return precision == other.precision
+            && requiredSize == other.requiredSize
+            && shardSize == other.shardSize;
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(precision, requiredSize, shardSize);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), precision, requiredSize, shardSize);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java
index 4e560e681c796..cd7e8d41c6419 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java
@@ -59,9 +59,9 @@ public GeoGridAggregationBuilder precision(int precision) {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, ?> createFactory(
+    protected ValuesSourceAggregatorFactory<ValuesSource.GeoPoint> createFactory(
         String name, ValuesSourceConfig<ValuesSource.GeoPoint> config, int precision, int requiredSize, int shardSize,
-        SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+        SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder,
         Map<String, Object> metaData
     ) throws IOException {
         return new GeoHashGridAggregatorFactory(name, config, precision, requiredSize, shardSize, context, parent,
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java
index 54253b9dea25c..7f7127d13f041 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java
@@ -37,14 +37,14 @@
 import java.util.List;
 import java.util.Map;
 
-public class GeoHashGridAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, GeoHashGridAggregatorFactory> {
+public class GeoHashGridAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.GeoPoint> {
 
     private final int precision;
     private final int requiredSize;
     private final int shardSize;
 
     GeoHashGridAggregatorFactory(String name, ValuesSourceConfig<GeoPoint> config, int precision, int requiredSize,
-            int shardSize, SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+            int shardSize, SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.precision = precision;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java
index 33efeeb5d38b6..374b7ddf70735 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java
@@ -58,9 +58,9 @@ public GeoGridAggregationBuilder precision(int precision) {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, ?> createFactory(
+    protected ValuesSourceAggregatorFactory<ValuesSource.GeoPoint> createFactory(
         String name, ValuesSourceConfig<ValuesSource.GeoPoint> config, int precision, int requiredSize, int shardSize,
-        SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+        SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder,
         Map<String, Object> metaData
     ) throws IOException {
         return new GeoTileGridAggregatorFactory(name, config, precision, requiredSize, shardSize, context, parent,
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java
index 87077a89d6c23..34a1bdc35ca0c 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java
@@ -36,14 +36,14 @@
 import java.util.List;
 import java.util.Map;
 
-public class GeoTileGridAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, GeoTileGridAggregatorFactory> {
+public class GeoTileGridAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.GeoPoint> {
 
     private final int precision;
     private final int requiredSize;
     private final int shardSize;
 
     GeoTileGridAggregatorFactory(String name, ValuesSourceConfig<GeoPoint> config, int precision, int requiredSize,
-                                 int shardSize, SearchContext context, AggregatorFactory<?> parent,
+                                 int shardSize, SearchContext context, AggregatorFactory parent,
                                  AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData
     ) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoGrid.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoGrid.java
index 36908f51d70de..ae77e69b3e1ec 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoGrid.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoGrid.java
@@ -133,15 +133,19 @@ int getRequiredSize() {
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(requiredSize, buckets);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), requiredSize, buckets);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalGeoGrid other = (InternalGeoGrid) obj;
-        return Objects.equals(requiredSize, other.requiredSize) &&
-            Objects.equals(buckets, other.buckets);
+        return Objects.equals(requiredSize, other.requiredSize)
+            && Objects.equals(buckets, other.buckets);
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
index cae359d9ecb1a..a6e85a031573a 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
@@ -61,7 +61,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+    protected AggregatorFactory doBuild(SearchContext context, AggregatorFactory parent, Builder subFactoriesBuilder)
             throws IOException {
         return new GlobalAggregatorFactory(name, context, parent, subFactoriesBuilder, metaData);
     }
@@ -78,16 +78,6 @@ public static GlobalAggregationBuilder parse(String aggregationName, XContentPar
         return new GlobalAggregationBuilder(aggregationName);
     }
 
-    @Override
-    protected boolean doEquals(Object obj) {
-        return true;
-    }
-
-    @Override
-    protected int doHashCode() {
-        return 0;
-    }
-
     @Override
     public String getType() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregatorFactory.java
index b11ffde6d7bc4..ffe66894d103a 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregatorFactory.java
@@ -30,9 +30,9 @@
 import java.util.List;
 import java.util.Map;
 
-public class GlobalAggregatorFactory extends AggregatorFactory<GlobalAggregatorFactory> {
+public class GlobalAggregatorFactory extends AggregatorFactory {
 
-    public GlobalAggregatorFactory(String name, SearchContext context, AggregatorFactory<?> parent,
+    public GlobalAggregatorFactory(String name, SearchContext context, AggregatorFactory parent,
             AggregatorFactories.Builder subFactories, Map<String, Object> metaData) throws IOException {
         super(name, context, parent, subFactories, metaData);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
index 794ce066ed76e..1219fc24606fd 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Rounding;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -48,42 +49,65 @@
 import java.util.Map;
 import java.util.Objects;
 
+import static java.util.Map.entry;
+
 public class AutoDateHistogramAggregationBuilder
         extends ValuesSourceAggregationBuilder<ValuesSource.Numeric, AutoDateHistogramAggregationBuilder> {
 
     public static final String NAME = "auto_date_histogram";
 
     private static final ParseField NUM_BUCKETS_FIELD = new ParseField("buckets");
+    private static final ParseField MINIMUM_INTERVAL_FIELD = new ParseField("minimum_interval");
 
     private static final ObjectParser<AutoDateHistogramAggregationBuilder, Void> PARSER;
     static {
         PARSER = new ObjectParser<>(AutoDateHistogramAggregationBuilder.NAME);
         ValuesSourceParserHelper.declareNumericFields(PARSER, true, true, true);
-
         PARSER.declareInt(AutoDateHistogramAggregationBuilder::setNumBuckets, NUM_BUCKETS_FIELD);
+        PARSER.declareStringOrNull(AutoDateHistogramAggregationBuilder::setMinimumIntervalExpression, MINIMUM_INTERVAL_FIELD);
     }
 
+    public static final Map<Rounding.DateTimeUnit, String> ALLOWED_INTERVALS = Map.ofEntries(
+        entry(Rounding.DateTimeUnit.YEAR_OF_CENTURY, "year"),
+        entry(Rounding.DateTimeUnit.MONTH_OF_YEAR, "month"),
+        entry(Rounding.DateTimeUnit.DAY_OF_MONTH, "day"),
+        entry( Rounding.DateTimeUnit.HOUR_OF_DAY, "hour"),
+        entry(Rounding.DateTimeUnit.MINUTES_OF_HOUR, "minute"),
+        entry(Rounding.DateTimeUnit.SECOND_OF_MINUTE, "second")
+    );
+
     /**
      *
      * Build roundings, computed dynamically as roundings are time zone dependent.
      * The current implementation probably should not be invoked in a tight loop.
      * @return Array of RoundingInfo
      */
-    static RoundingInfo[] buildRoundings(ZoneId timeZone) {
+    static RoundingInfo[] buildRoundings(ZoneId timeZone, String minimumInterval) {
+
+        int indexToSliceFrom = 0;
+
         RoundingInfo[] roundings = new RoundingInfo[6];
-        roundings[0] = new RoundingInfo(createRounding(Rounding.DateTimeUnit.SECOND_OF_MINUTE, timeZone),
-            1000L, "s", 1, 5, 10, 30);
-        roundings[1] = new RoundingInfo(createRounding(Rounding.DateTimeUnit.MINUTES_OF_HOUR, timeZone),
+        roundings[0] = new RoundingInfo(Rounding.DateTimeUnit.SECOND_OF_MINUTE,
+            timeZone, 1000L, "s",1, 5, 10, 30);
+        roundings[1] = new RoundingInfo(Rounding.DateTimeUnit.MINUTES_OF_HOUR, timeZone,
             60 * 1000L, "m", 1, 5, 10, 30);
-        roundings[2] = new RoundingInfo(createRounding(Rounding.DateTimeUnit.HOUR_OF_DAY, timeZone),
-            60 * 60 * 1000L, "h",1, 3, 12);
-        roundings[3] = new RoundingInfo(createRounding(Rounding.DateTimeUnit.DAY_OF_MONTH, timeZone),
+        roundings[2] = new RoundingInfo(Rounding.DateTimeUnit.HOUR_OF_DAY, timeZone,
+            60 * 60 * 1000L, "h", 1, 3, 12);
+        roundings[3] = new RoundingInfo(Rounding.DateTimeUnit.DAY_OF_MONTH, timeZone,
             24 * 60 * 60 * 1000L, "d", 1, 7);
-        roundings[4] = new RoundingInfo(createRounding(Rounding.DateTimeUnit.MONTH_OF_YEAR, timeZone),
+        roundings[4] = new RoundingInfo(Rounding.DateTimeUnit.MONTH_OF_YEAR, timeZone,
             30 * 24 * 60 * 60 * 1000L, "M", 1, 3);
-        roundings[5] = new RoundingInfo(createRounding(Rounding.DateTimeUnit.YEAR_OF_CENTURY, timeZone),
+        roundings[5] = new RoundingInfo(Rounding.DateTimeUnit.YEAR_OF_CENTURY, timeZone,
             365 * 24 * 60 * 60 * 1000L, "y", 1, 5, 10, 20, 50, 100);
-        return roundings;
+
+        for (int i = 0; i < roundings.length; i++) {
+            RoundingInfo roundingInfo = roundings[i];
+            if (roundingInfo.getDateTimeUnit().equals(minimumInterval)) {
+                indexToSliceFrom = i;
+                break;
+            }
+        }
+        return Arrays.copyOfRange(roundings, indexToSliceFrom, roundings.length);
     }
 
     public static AutoDateHistogramAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
@@ -92,6 +116,22 @@ public static AutoDateHistogramAggregationBuilder parse(String aggregationName,
 
     private int numBuckets = 10;
 
+    private String minimumIntervalExpression;
+
+    public String getMinimumIntervalExpression() {
+        return minimumIntervalExpression;
+    }
+
+    public AutoDateHistogramAggregationBuilder setMinimumIntervalExpression(String minimumIntervalExpression) {
+        if (minimumIntervalExpression != null && !ALLOWED_INTERVALS.containsValue(minimumIntervalExpression)) {
+            throw new IllegalArgumentException(MINIMUM_INTERVAL_FIELD.getPreferredName() +
+                " must be one of [" + ALLOWED_INTERVALS.values().toString() + "]");
+        }
+        this.minimumIntervalExpression = minimumIntervalExpression;
+        return this;
+    }
+
+
     /** Create a new builder with the given name. */
     public AutoDateHistogramAggregationBuilder(String name) {
         super(name, ValuesSourceType.NUMERIC, ValueType.DATE);
@@ -101,12 +141,16 @@ public AutoDateHistogramAggregationBuilder(String name) {
     public AutoDateHistogramAggregationBuilder(StreamInput in) throws IOException {
         super(in, ValuesSourceType.NUMERIC, ValueType.DATE);
         numBuckets = in.readVInt();
+        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+            minimumIntervalExpression = in.readOptionalString();
+        }
     }
 
     protected AutoDateHistogramAggregationBuilder(AutoDateHistogramAggregationBuilder clone, Builder factoriesBuilder,
             Map<String, Object> metaData) {
         super(clone, factoriesBuilder, metaData);
         this.numBuckets = clone.numBuckets;
+        this.minimumIntervalExpression = clone.minimumIntervalExpression;
     }
 
     @Override
@@ -117,6 +161,9 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map<String, O
     @Override
     protected void innerWriteTo(StreamOutput out) throws IOException {
         out.writeVInt(numBuckets);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeOptionalString(minimumIntervalExpression);
+        }
     }
 
     @Override
@@ -137,9 +184,9 @@ public int getNumBuckets() {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        RoundingInfo[] roundings = buildRoundings(timeZone());
+    protected ValuesSourceAggregatorFactory<Numeric> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
+        RoundingInfo[] roundings = buildRoundings(timeZone(), getMinimumIntervalExpression());
         int maxRoundingInterval = Arrays.stream(roundings,0, roundings.length-1)
             .map(rounding -> rounding.innerIntervals)
             .flatMapToInt(Arrays::stream)
@@ -152,7 +199,9 @@ public int getNumBuckets() {
             throw new IllegalArgumentException(NUM_BUCKETS_FIELD.getPreferredName()+
                 " must be less than " + bucketCeiling);
         }
-        return new AutoDateHistogramAggregatorFactory(name, config, numBuckets, roundings, context, parent, subFactoriesBuilder, metaData);
+        return new AutoDateHistogramAggregatorFactory(name, config, numBuckets, roundings, context, parent,
+            subFactoriesBuilder,
+            metaData);
     }
 
     static Rounding createRounding(Rounding.DateTimeUnit interval, ZoneId timeZone) {
@@ -167,18 +216,22 @@ static Rounding createRounding(Rounding.DateTimeUnit interval, ZoneId timeZone)
     @Override
     protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         builder.field(NUM_BUCKETS_FIELD.getPreferredName(), numBuckets);
+        builder.field(MINIMUM_INTERVAL_FIELD.getPreferredName(), minimumIntervalExpression);
         return builder;
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(numBuckets);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), numBuckets, minimumIntervalExpression);
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         AutoDateHistogramAggregationBuilder other = (AutoDateHistogramAggregationBuilder) obj;
-        return Objects.equals(numBuckets, other.numBuckets);
+        return Objects.equals(numBuckets, other.numBuckets) && Objects.equals(minimumIntervalExpression, other.minimumIntervalExpression);
     }
 
     public static class RoundingInfo implements Writeable {
@@ -186,12 +239,22 @@ public static class RoundingInfo implements Writeable {
         final int[] innerIntervals;
         final long roughEstimateDurationMillis;
         final String unitAbbreviation;
-
-        public RoundingInfo(Rounding rounding, long roughEstimateDurationMillis, String unitAbbreviation, int... innerIntervals) {
-            this.rounding = rounding;
+        final String dateTimeUnit;
+
+        public RoundingInfo(Rounding.DateTimeUnit dateTimeUnit,
+                            ZoneId timeZone,
+                            long roughEstimateDurationMillis,
+                            String unitAbbreviation,
+                            int... innerIntervals) {
+            this.rounding = createRounding(dateTimeUnit, timeZone);
             this.roughEstimateDurationMillis = roughEstimateDurationMillis;
             this.unitAbbreviation = unitAbbreviation;
             this.innerIntervals = innerIntervals;
+            Objects.requireNonNull(dateTimeUnit, "dateTimeUnit cannot be null");
+            if (!ALLOWED_INTERVALS.containsKey(dateTimeUnit)) {
+                throw new IllegalArgumentException("dateTimeUnit must be one of " + ALLOWED_INTERVALS.keySet().toString());
+            }
+            this.dateTimeUnit = ALLOWED_INTERVALS.get(dateTimeUnit);
         }
 
         public RoundingInfo(StreamInput in) throws IOException {
@@ -199,6 +262,7 @@ public RoundingInfo(StreamInput in) throws IOException {
             roughEstimateDurationMillis = in.readVLong();
             innerIntervals = in.readIntArray();
             unitAbbreviation = in.readString();
+            dateTimeUnit = in.readString();
         }
 
         @Override
@@ -207,19 +271,22 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVLong(roughEstimateDurationMillis);
             out.writeIntArray(innerIntervals);
             out.writeString(unitAbbreviation);
+            out.writeString(dateTimeUnit);
         }
 
         public int getMaximumInnerInterval() {
             return innerIntervals[innerIntervals.length - 1];
         }
 
+        public String getDateTimeUnit() { return this.dateTimeUnit; }
+
         public long getRoughEstimateDurationMillis() {
             return roughEstimateDurationMillis;
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(rounding, Arrays.hashCode(innerIntervals));
+            return Objects.hash(rounding, Arrays.hashCode(innerIntervals), dateTimeUnit);
         }
 
         @Override
@@ -231,8 +298,10 @@ public boolean equals(Object obj) {
                 return false;
             }
             RoundingInfo other = (RoundingInfo) obj;
-            return Objects.equals(rounding, other.rounding) &&
-                    Objects.deepEquals(innerIntervals, other.innerIntervals);
+            return Objects.equals(rounding, other.rounding)
+                && Objects.deepEquals(innerIntervals, other.innerIntervals)
+                && Objects.equals(dateTimeUnit, other.dateTimeUnit)
+                ;
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
index 1b982ea9deca2..b10507cd2ce65 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
@@ -93,7 +93,7 @@ protected boolean shouldDefer(Aggregator aggregator) {
 
     @Override
     public DeferringBucketCollector getDeferringCollector() {
-        deferringCollector = new MergingBucketsDeferringCollector(context);
+        deferringCollector = new MergingBucketsDeferringCollector(context, descendsFromGlobalAggregator(parent()));
         return deferringCollector;
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java
index 051f2f9f6e7c7..c2ca3b15059df 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregatorFactory.java
@@ -35,14 +35,19 @@
 import java.util.Map;
 
 public final class AutoDateHistogramAggregatorFactory
-        extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, AutoDateHistogramAggregatorFactory> {
+        extends ValuesSourceAggregatorFactory<ValuesSource.Numeric> {
 
     private final int numBuckets;
     private RoundingInfo[] roundingInfos;
 
-    public AutoDateHistogramAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, int numBuckets, RoundingInfo[] roundingInfos,
-            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
-            Map<String, Object> metaData) throws IOException {
+    public AutoDateHistogramAggregatorFactory(String name,
+                                              ValuesSourceConfig<Numeric> config,
+                                              int numBuckets,
+                                              RoundingInfo[] roundingInfos,
+                                              SearchContext context,
+                                              AggregatorFactory parent,
+                                              AggregatorFactories.Builder subFactoriesBuilder,
+                                              Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.numBuckets = numBuckets;
         this.roundingInfos = roundingInfos;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
index 49b4275f4abc8..1fd2b3b5133a0 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
@@ -478,8 +478,8 @@ ZoneId rewriteTimeZone(QueryShardContext context) throws IOException {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+    protected ValuesSourceAggregatorFactory<Numeric> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         final ZoneId tz = timeZone();
         final Rounding rounding = dateHistogramInterval.createRounding(tz);
         final ZoneId rewrittenTimeZone = rewriteTimeZone(context.getQueryShardContext());
@@ -500,12 +500,15 @@ ZoneId rewriteTimeZone(QueryShardContext context) throws IOException {
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(order, keyed, minDocCount, dateHistogramInterval, minDocCount, extendedBounds);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), order, keyed, minDocCount, dateHistogramInterval, minDocCount, extendedBounds);
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         DateHistogramAggregationBuilder other = (DateHistogramAggregationBuilder) obj;
         return Objects.equals(order, other.order)
                 && Objects.equals(keyed, other.keyed)
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java
index 8c025eb34eeb3..419125b5f4e47 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java
@@ -36,7 +36,7 @@
 import java.util.Map;
 
 public final class DateHistogramAggregatorFactory
-        extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, DateHistogramAggregatorFactory> {
+        extends ValuesSourceAggregatorFactory<ValuesSource.Numeric> {
 
     private final long offset;
     private final BucketOrder order;
@@ -49,7 +49,7 @@ public final class DateHistogramAggregatorFactory
     public DateHistogramAggregatorFactory(String name, ValuesSourceConfig<Numeric> config,
             long offset, BucketOrder order, boolean keyed, long minDocCount,
             Rounding rounding, Rounding shardRounding, ExtendedBounds extendedBounds, SearchContext context,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+            AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.offset = offset;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
index fd38f6eb63a93..ecb65df433a69 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
@@ -295,26 +295,29 @@ public String getType() {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+    protected ValuesSourceAggregatorFactory<Numeric> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         return new HistogramAggregatorFactory(name, config, interval, offset, order, keyed, minDocCount, minBound, maxBound,
                 context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(order, keyed, minDocCount, interval, offset, minBound, maxBound);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), order, keyed, minDocCount, interval, offset, minBound, maxBound);
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         HistogramAggregationBuilder other = (HistogramAggregationBuilder) obj;
         return Objects.equals(order, other.order)
-                && Objects.equals(keyed, other.keyed)
-                && Objects.equals(minDocCount, other.minDocCount)
-                && Objects.equals(interval, other.interval)
-                && Objects.equals(offset, other.offset)
-                && Objects.equals(minBound, other.minBound)
-                && Objects.equals(maxBound, other.maxBound);
+            && Objects.equals(keyed, other.keyed)
+            && Objects.equals(minDocCount, other.minDocCount)
+            && Objects.equals(interval, other.interval)
+            && Objects.equals(offset, other.offset)
+            && Objects.equals(minBound, other.minBound)
+            && Objects.equals(maxBound, other.maxBound);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java
index 683ec1987c597..b6828c8e84d1b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java
@@ -34,7 +34,7 @@
 import java.util.List;
 import java.util.Map;
 
-public final class HistogramAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, HistogramAggregatorFactory> {
+public final class HistogramAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric> {
 
     private final double interval, offset;
     private final BucketOrder order;
@@ -44,7 +44,7 @@ public final class HistogramAggregatorFactory extends ValuesSourceAggregatorFact
 
     public HistogramAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double interval, double offset,
             BucketOrder order, boolean keyed, long minDocCount, double minBound, double maxBound,
-            SearchContext context, AggregatorFactory<?> parent,
+            SearchContext context, AggregatorFactory parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.interval = interval;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java
index dc0eefec45c82..f10318de02e43 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java
@@ -598,7 +598,11 @@ public Bucket createBucket(Number key, long docCount, InternalAggregations aggre
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalAutoDateHistogram that = (InternalAutoDateHistogram) obj;
         return Objects.equals(buckets, that.buckets)
                 && Objects.equals(format, that.format)
@@ -606,7 +610,7 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(buckets, format, bucketInfo);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), buckets, format, bucketInfo);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
index 162bfde5acf0f..8ac4ce5f27dc8 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
@@ -512,7 +512,11 @@ public Bucket createBucket(Number key, long docCount, InternalAggregations aggre
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalDateHistogram that = (InternalDateHistogram) obj;
         return Objects.equals(buckets, that.buckets)
                 && Objects.equals(order, that.order)
@@ -524,7 +528,7 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(buckets, order, format, keyed, minDocCount, offset, emptyBucketInfo);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), buckets, order, format, keyed, minDocCount, offset, emptyBucketInfo);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
index b324241e9bb6d..0bd18ab753e9f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
@@ -489,7 +489,11 @@ public Bucket createBucket(Number key, long docCount, InternalAggregations aggre
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalHistogram that = (InternalHistogram) obj;
         return Objects.equals(buckets, that.buckets)
                 && Objects.equals(emptyBucketInfo, that.emptyBucketInfo)
@@ -500,7 +504,7 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(buckets, emptyBucketInfo, format, keyed, minDocCount, order);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), buckets, emptyBucketInfo, format, keyed, minDocCount, order);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
index b72c004464427..de729b619dcd4 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
@@ -83,8 +83,8 @@ protected boolean serializeTargetValueType() {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(SearchContext context,
-            ValuesSourceConfig<ValuesSource> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+    protected ValuesSourceAggregatorFactory<ValuesSource> innerBuild(SearchContext context,
+            ValuesSourceConfig<ValuesSource> config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         return new MissingAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
@@ -92,17 +92,7 @@ protected boolean serializeTargetValueType() {
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         return builder;
     }
-
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(Object obj) {
-        return true;
-    }
-
+    
     @Override
     public String getType() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorFactory.java
index 5d05b79cef2d6..61705c773d1de 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorFactory.java
@@ -32,10 +32,10 @@
 import java.util.List;
 import java.util.Map;
 
-public class MissingAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource, MissingAggregatorFactory> {
+public class MissingAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource> {
 
     public MissingAggregatorFactory(String name, ValuesSourceConfig<ValuesSource> config, SearchContext context,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
+            AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
index 8429cf79ed903..e27b341003d97 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
@@ -87,7 +87,7 @@ public String path() {
     }
 
     @Override
-    protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+    protected AggregatorFactory doBuild(SearchContext context, AggregatorFactory parent, Builder subFactoriesBuilder)
             throws IOException {
         ObjectMapper childObjectMapper = context.getObjectMapper(path);
         if (childObjectMapper == null) {
@@ -143,14 +143,16 @@ public static NestedAggregationBuilder parse(String aggregationName, XContentPar
         return new NestedAggregationBuilder(aggregationName, path);
     }
 
-
     @Override
-    protected int doHashCode() {
-        return Objects.hash(path);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), path);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         NestedAggregationBuilder other = (NestedAggregationBuilder) obj;
         return Objects.equals(path, other.path);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java
index 6724ee7da30d6..769c5d024a94b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java
@@ -32,13 +32,13 @@
 import java.util.List;
 import java.util.Map;
 
-public class NestedAggregatorFactory extends AggregatorFactory<NestedAggregatorFactory> {
+public class NestedAggregatorFactory extends AggregatorFactory {
 
     private final ObjectMapper parentObjectMapper;
     private final ObjectMapper childObjectMapper;
 
     NestedAggregatorFactory(String name, ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper,
-            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactories,
+            SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactories,
                                    Map<String, Object> metaData) throws IOException {
         super(name, context, parent, subFactories, metaData);
         this.parentObjectMapper = parentObjectMapper;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
index 06c73ef797e28..62dfb4011769b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
@@ -92,7 +92,7 @@ public String path() {
     }
 
     @Override
-    protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+    protected AggregatorFactory doBuild(SearchContext context, AggregatorFactory parent, Builder subFactoriesBuilder)
             throws IOException {
         if (findNestedAggregatorFactory(parent) == null) {
             throw new SearchParseException(context,
@@ -120,7 +120,7 @@ protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<
         }
     }
 
-    private static NestedAggregatorFactory findNestedAggregatorFactory(AggregatorFactory<?> parent) {
+    private static NestedAggregatorFactory findNestedAggregatorFactory(AggregatorFactory parent) {
         if (parent == null) {
             return null;
         } else if (parent instanceof NestedAggregatorFactory) {
@@ -168,14 +168,16 @@ public static ReverseNestedAggregationBuilder parse(String aggregationName, XCon
         return factory;
     }
 
-
     @Override
-    protected int doHashCode() {
-        return Objects.hash(path);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), path);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         ReverseNestedAggregationBuilder other = (ReverseNestedAggregationBuilder) obj;
         return Objects.equals(path, other.path);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java
index 3ed6f21d99c8d..022a1cec77190 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java
@@ -32,13 +32,13 @@
 import java.util.List;
 import java.util.Map;
 
-public class ReverseNestedAggregatorFactory extends AggregatorFactory<ReverseNestedAggregatorFactory> {
+public class ReverseNestedAggregatorFactory extends AggregatorFactory {
 
     private final boolean unmapped;
     private final ObjectMapper parentObjectMapper;
 
     public ReverseNestedAggregatorFactory(String name, boolean unmapped, ObjectMapper parentObjectMapper,
-                                          SearchContext context, AggregatorFactory<?> parent,
+                                          SearchContext context, AggregatorFactory parent,
                                           AggregatorFactories.Builder subFactories,
                                           Map<String, Object> metaData) throws IOException {
         super(name, context, parent, subFactories, metaData);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java
index 156adcdc4f3ea..e1943e6af1170 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java
@@ -35,15 +35,14 @@
 import java.util.List;
 import java.util.Map;
 
-public class AbstractRangeAggregatorFactory<AF extends AbstractRangeAggregatorFactory<AF, R>, R extends Range>
-        extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, AF> {
+public class AbstractRangeAggregatorFactory<R extends Range> extends ValuesSourceAggregatorFactory<ValuesSource.Numeric> {
 
     private final InternalRange.Factory<?, ?> rangeFactory;
     private final R[] ranges;
     private final boolean keyed;
 
     public AbstractRangeAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, R[] ranges, boolean keyed,
-            InternalRange.Factory<?, ?> rangeFactory, SearchContext context, AggregatorFactory<?> parent,
+            InternalRange.Factory<?, ?> rangeFactory, SearchContext context, AggregatorFactory parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.ranges = ranges;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java
index e5b62d85ca17c..542929564e66f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java
@@ -140,12 +140,15 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(ranges, keyed);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), ranges, keyed);
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         AbstractRangeBuilder<AB, R> other = (AbstractRangeBuilder<AB, R>) obj;
         return Objects.equals(ranges, other.ranges)
                 && Objects.equals(keyed, other.keyed);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java
index a8d74a7d83615..41d30cc9bff81 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java
@@ -32,7 +32,7 @@
 import java.util.Map;
 
 public class BinaryRangeAggregatorFactory
-        extends ValuesSourceAggregatorFactory<ValuesSource.Bytes, BinaryRangeAggregatorFactory> {
+        extends ValuesSourceAggregatorFactory<ValuesSource.Bytes> {
 
     private final List<BinaryRangeAggregator.Range> ranges;
     private final boolean keyed;
@@ -41,7 +41,7 @@ public BinaryRangeAggregatorFactory(String name,
             ValuesSourceConfig<ValuesSource.Bytes> config,
             List<BinaryRangeAggregator.Range> ranges, boolean keyed,
             SearchContext context,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder,
+            AggregatorFactory parent, Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.ranges = ranges;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java
index 2b5e92ddcb3f9..f0896fb682bfb 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregationBuilder.java
@@ -289,7 +289,7 @@ public DateRangeAggregationBuilder addUnboundedFrom(ZonedDateTime from) {
 
     @Override
     protected DateRangeAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         // We need to call processRanges here so they are parsed and we know whether `now` has been used before we make
         // the decision of whether to cache the request
         RangeAggregator.Range[] ranges = processRanges(range -> {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorFactory.java
index 6dd51197ca823..361dcbb4c24d6 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/DateRangeAggregatorFactory.java
@@ -28,10 +28,10 @@
 import java.io.IOException;
 import java.util.Map;
 
-public class DateRangeAggregatorFactory extends AbstractRangeAggregatorFactory<DateRangeAggregatorFactory, RangeAggregator.Range> {
+public class DateRangeAggregatorFactory extends AbstractRangeAggregatorFactory<RangeAggregator.Range> {
 
     public DateRangeAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, RangeAggregator.Range[] ranges, boolean keyed,
-            InternalRange.Factory<?, ?> rangeFactory, SearchContext context, AggregatorFactory<?> parent,
+            InternalRange.Factory<?, ?> rangeFactory, SearchContext context, AggregatorFactory parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder, metaData);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java
index c612b303fb134..d9e29d0df469e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java
@@ -411,8 +411,8 @@ public boolean keyed() {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, ?> innerBuild(SearchContext context,
-            ValuesSourceConfig<ValuesSource.GeoPoint> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+    protected ValuesSourceAggregatorFactory<ValuesSource.GeoPoint> innerBuild(SearchContext context,
+            ValuesSourceConfig<ValuesSource.GeoPoint> config, AggregatorFactory parent, Builder subFactoriesBuilder)
                     throws IOException {
         Range[] ranges = this.ranges.toArray(new Range[this.range().size()]);
         if (ranges.length == 0) {
@@ -433,12 +433,15 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(origin, ranges, keyed, distanceType, unit);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), origin, ranges, keyed, distanceType, unit);
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         GeoDistanceAggregationBuilder other = (GeoDistanceAggregationBuilder) obj;
         return Objects.equals(origin, other.origin)
                 && Objects.equals(ranges, other.ranges)
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java
index 26847e1ba10c1..b99ae657aaee8 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java
@@ -43,7 +43,7 @@
 import java.util.Map;
 
 public class GeoDistanceRangeAggregatorFactory
-        extends ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, GeoDistanceRangeAggregatorFactory> {
+        extends ValuesSourceAggregatorFactory<ValuesSource.GeoPoint> {
 
     private final InternalRange.Factory<InternalGeoDistance.Bucket, InternalGeoDistance> rangeFactory = InternalGeoDistance.FACTORY;
     private final GeoPoint origin;
@@ -54,7 +54,7 @@ public class GeoDistanceRangeAggregatorFactory
 
     public GeoDistanceRangeAggregatorFactory(String name, ValuesSourceConfig<ValuesSource.GeoPoint> config, GeoPoint origin,
             Range[] ranges, DistanceUnit unit, GeoDistance distanceType, boolean keyed, SearchContext context,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
+            AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.origin = origin;
         this.ranges = ranges;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
index 35ee592d6c9ae..7051403b302a7 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
@@ -275,16 +275,20 @@ public XContentBuilder doXContentBody(XContentBuilder builder,
         return builder;
     }
 
+
     @Override
-    public boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalBinaryRange that = (InternalBinaryRange) obj;
         return Objects.equals(buckets, that.buckets)
             && Objects.equals(format, that.format)
             && Objects.equals(keyed, that.keyed);
     }
 
-    @Override
-    public int doHashCode() {
-        return Objects.hash(buckets, format, keyed);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), buckets, format, keyed);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
index 747cb22c87071..94b8d23617036 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
@@ -338,12 +338,16 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(ranges, format, keyed);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), ranges, format, keyed);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalRange<?,?> that = (InternalRange<?,?>) obj;
         return Objects.equals(ranges, that.ranges)
                 && Objects.equals(format, that.format)
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java
index 1a3eb4957e2c5..9ee76784d207e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregationBuilder.java
@@ -364,9 +364,9 @@ private static BytesRef toBytesRef(String ip) {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource.Bytes, ?> innerBuild(
+    protected ValuesSourceAggregatorFactory<ValuesSource.Bytes> innerBuild(
             SearchContext context, ValuesSourceConfig<ValuesSource.Bytes> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+            AggregatorFactory parent, Builder subFactoriesBuilder)
                     throws IOException {
         List<BinaryRangeAggregator.Range> ranges = new ArrayList<>();
         if(this.ranges.size() == 0){
@@ -387,14 +387,17 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(keyed, ranges);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), keyed, ranges);
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         IpRangeAggregationBuilder that = (IpRangeAggregationBuilder) obj;
         return keyed == that.keyed
-                && ranges.equals(that.ranges);
+            && ranges.equals(that.ranges);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
index 9a7edf7ef95dd..25c072a7a0656 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
@@ -147,7 +147,7 @@ public RangeAggregationBuilder addUnboundedFrom(double from) {
 
     @Override
     protected RangeAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         // We need to call processRanges here so they are parsed before we make the decision of whether to cache the request
         Range[] ranges = processRanges(range -> {
             DocValueFormat parser = config.format();
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorFactory.java
index d1dc3e71b5c11..66fde1554b922 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorFactory.java
@@ -30,10 +30,10 @@
 import java.io.IOException;
 import java.util.Map;
 
-public class RangeAggregatorFactory extends AbstractRangeAggregatorFactory<RangeAggregatorFactory, RangeAggregator.Range> {
+public class RangeAggregatorFactory extends AbstractRangeAggregatorFactory<RangeAggregator.Range> {
 
     public RangeAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, Range[] ranges, boolean keyed,
-            Factory<?, ?> rangeFactory, SearchContext context, AggregatorFactory<?> parent,
+            Factory<?, ?> rangeFactory, SearchContext context, AggregatorFactory parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder, metaData);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java
index d6ff4c3d18f5d..d4d7d4f8b01bc 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java
@@ -148,8 +148,8 @@ public String executionHint() {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(SearchContext context,
-            ValuesSourceConfig<ValuesSource> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+    protected ValuesSourceAggregatorFactory<ValuesSource> innerBuild(SearchContext context,
+            ValuesSourceConfig<ValuesSource> config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         return new DiversifiedAggregatorFactory(name, config, shardSize, maxDocsPerValue, executionHint, context, parent,
                 subFactoriesBuilder, metaData);
     }
@@ -165,16 +165,19 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(shardSize, maxDocsPerValue, executionHint);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), shardSize, maxDocsPerValue, executionHint);
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         DiversifiedAggregationBuilder other = (DiversifiedAggregationBuilder) obj;
         return Objects.equals(shardSize, other.shardSize)
-                && Objects.equals(maxDocsPerValue, other.maxDocsPerValue)
-                && Objects.equals(executionHint, other.executionHint);
+            && Objects.equals(maxDocsPerValue, other.maxDocsPerValue)
+            && Objects.equals(executionHint, other.executionHint);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java
index 2bd68296d7b3e..281e6b0e29d9f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java
@@ -37,14 +37,14 @@
 import java.util.List;
 import java.util.Map;
 
-public class DiversifiedAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource, DiversifiedAggregatorFactory> {
+public class DiversifiedAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource> {
 
     private final int shardSize;
     private final int maxDocsPerValue;
     private final String executionHint;
 
     DiversifiedAggregatorFactory(String name, ValuesSourceConfig<ValuesSource> config, int shardSize, int maxDocsPerValue,
-            String executionHint, SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+            String executionHint, SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.shardSize = shardSize;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
index 1aed0a0374aa9..afdb3e5e48cb6 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
@@ -84,7 +84,7 @@ public int shardSize() {
     }
 
     @Override
-    protected SamplerAggregatorFactory doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+    protected SamplerAggregatorFactory doBuild(SearchContext context, AggregatorFactory parent, Builder subFactoriesBuilder)
             throws IOException {
         return new SamplerAggregatorFactory(name, shardSize, context, parent, subFactoriesBuilder, metaData);
     }
@@ -126,12 +126,15 @@ public static SamplerAggregationBuilder parse(String aggregationName, XContentPa
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(shardSize);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), shardSize);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         SamplerAggregationBuilder other = (SamplerAggregationBuilder) obj;
         return Objects.equals(shardSize, other.shardSize);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorFactory.java
index c122cc95fa117..d78a2bc275a04 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorFactory.java
@@ -29,11 +29,11 @@
 import java.util.List;
 import java.util.Map;
 
-public class SamplerAggregatorFactory extends AggregatorFactory<SamplerAggregatorFactory> {
+public class SamplerAggregatorFactory extends AggregatorFactory {
 
     private final int shardSize;
 
-    SamplerAggregatorFactory(String name, int shardSize, SearchContext context, AggregatorFactory<?> parent,
+    SamplerAggregatorFactory(String name, int shardSize, SearchContext context, AggregatorFactory parent,
             AggregatorFactories.Builder subFactories, Map<String, Object> metaData) throws IOException {
         super(name, context, parent, subFactories, metaData);
         this.shardSize = shardSize;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalMappedSignificantTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalMappedSignificantTerms.java
index e1bd4defd3e08..849015163dc23 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalMappedSignificantTerms.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalMappedSignificantTerms.java
@@ -109,10 +109,13 @@ protected SignificanceHeuristic getSignificanceHeuristic() {
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalMappedSignificantTerms<?, ?> that = (InternalMappedSignificantTerms<?, ?>) obj;
-        return super.doEquals(obj)
-                && Objects.equals(format, that.format)
+        return Objects.equals(format, that.format)
                 && subsetSize == that.subsetSize
                 && supersetSize == that.supersetSize
                 && Objects.equals(significanceHeuristic, that.significanceHeuristic)
@@ -121,8 +124,8 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(super.doHashCode(), format, subsetSize, supersetSize, significanceHeuristic, buckets, bucketMap);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), format, subsetSize, supersetSize, significanceHeuristic, buckets, bucketMap);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java
index 42a3c6a849bc1..5d8bc893a2604 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java
@@ -272,12 +272,16 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
     protected abstract SignificanceHeuristic getSignificanceHeuristic();
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(minDocCount, requiredSize);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), minDocCount, requiredSize);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalSignificantTerms<?, ?> that = (InternalSignificantTerms<?, ?>) obj;
         return Objects.equals(minDocCount, that.minDocCount)
                 && Objects.equals(requiredSize, that.requiredSize);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java
index 551ecd6a9f23e..a2adbcffab2f8 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java
@@ -103,6 +103,10 @@ protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOExcept
 
         @Override
         public boolean equals(Object obj) {
+            if (this == obj) return true;
+            if (obj == null || getClass() != obj.getClass()) return false;
+            if (super.equals(obj) == false) return false;
+
             return super.equals(obj) && Objects.equals(termBytes, ((SignificantStringTerms.Bucket) obj).termBytes);
         }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java
index 02216c5a5cb35..75b32f8abe062 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java
@@ -283,8 +283,8 @@ public SignificanceHeuristic significanceHeuristic() {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+    protected ValuesSourceAggregatorFactory<ValuesSource> innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource> config,
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         SignificanceHeuristic executionHeuristic = this.significanceHeuristic.rewrite(context);
         return new SignificantTermsAggregatorFactory(name, config, includeExclude, executionHint, filterBuilder,
                 bucketCountThresholds, executionHeuristic, context, parent, subFactoriesBuilder, metaData);
@@ -307,12 +307,15 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(bucketCountThresholds, executionHint, filterBuilder, includeExclude, significanceHeuristic);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), bucketCountThresholds, executionHint, filterBuilder, includeExclude, significanceHeuristic);
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         SignificantTermsAggregationBuilder other = (SignificantTermsAggregationBuilder) obj;
         return Objects.equals(bucketCountThresholds, other.bucketCountThresholds)
                 && Objects.equals(executionHint, other.executionHint)
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
index 09fd5877344f6..dbe8873f88ad4 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
@@ -58,7 +58,7 @@
 import java.util.List;
 import java.util.Map;
 
-public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource, SignificantTermsAggregatorFactory>
+public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource>
         implements Releasable {
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(
             LogManager.getLogger(SignificantTermsAggregatorFactory.class));
@@ -82,7 +82,7 @@ public SignificantTermsAggregatorFactory(String name,
                                              TermsAggregator.BucketCountThresholds bucketCountThresholds,
                                              SignificanceHeuristic significanceHeuristic,
                                              SearchContext context,
-                                             AggregatorFactory<?> parent,
+                                             AggregatorFactory parent,
                                              AggregatorFactories.Builder subFactoriesBuilder,
                                              Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregationBuilder.java
index f0b85f979c233..7d53776e4fe7d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregationBuilder.java
@@ -340,7 +340,7 @@ protected void doWriteTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent,
+    protected AggregatorFactory doBuild(SearchContext context, AggregatorFactory parent,
             Builder subFactoriesBuilder) throws IOException {
         SignificanceHeuristic executionHeuristic = this.significanceHeuristic.rewrite(context);
 
@@ -378,13 +378,17 @@ protected XContentBuilder internalXContent(XContentBuilder builder, Params param
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(bucketCountThresholds, fieldName, filterDuplicateText, filterBuilder,
-                includeExclude, significanceHeuristic, Arrays.hashCode(sourceFieldNames));
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), bucketCountThresholds, fieldName,
+            filterDuplicateText, filterBuilder,
+            includeExclude, significanceHeuristic, Arrays.hashCode(sourceFieldNames));
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         SignificantTextAggregationBuilder other = (SignificantTextAggregationBuilder) obj;
         return Objects.equals(bucketCountThresholds, other.bucketCountThresholds)
                 && Objects.equals(fieldName, other.fieldName)
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java
index 92a136960395d..eb33c6dbc4493 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java
@@ -49,7 +49,7 @@
 import java.util.List;
 import java.util.Map;
 
-public class SignificantTextAggregatorFactory extends AggregatorFactory<SignificantTextAggregatorFactory>
+public class SignificantTextAggregatorFactory extends AggregatorFactory
         implements Releasable {
 
     private final IncludeExclude includeExclude;
@@ -67,7 +67,7 @@ public class SignificantTextAggregatorFactory extends AggregatorFactory<Signific
 
     public SignificantTextAggregatorFactory(String name, IncludeExclude includeExclude,
             QueryBuilder filterBuilder, TermsAggregator.BucketCountThresholds bucketCountThresholds,
-            SignificanceHeuristic significanceHeuristic, SearchContext context, AggregatorFactory<?> parent,
+            SignificanceHeuristic significanceHeuristic, SearchContext context, AggregatorFactory parent,
             AggregatorFactories.Builder subFactoriesBuilder, String fieldName, String [] sourceFieldNames,
             boolean filterDuplicateText, Map<String, Object> metaData) throws IOException {
         super(name, context, parent, subFactoriesBuilder, metaData);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractRareTermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractRareTermsAggregator.java
new file mode 100644
index 0000000000000..2bbe3c01988df
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractRareTermsAggregator.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.elasticsearch.common.util.SetBackedScalingCuckooFilter;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.BucketOrder;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.bucket.DeferableBucketAggregator;
+import org.elasticsearch.search.aggregations.bucket.DeferringBucketCollector;
+import org.elasticsearch.search.aggregations.bucket.MergingBucketsDeferringCollector;
+import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+public abstract class AbstractRareTermsAggregator<T extends ValuesSource,
+    U extends IncludeExclude.Filter, V> extends DeferableBucketAggregator {
+
+    static final BucketOrder ORDER = BucketOrder.compound(BucketOrder.count(true), BucketOrder.key(true)); // sort by count ascending
+
+    protected final long maxDocCount;
+    protected final double precision;
+    protected final DocValueFormat format;
+    protected final T valuesSource;
+    protected final U includeExclude;
+
+    MergingBucketsDeferringCollector deferringCollector;
+    LeafBucketCollector subCollectors;
+    final SetBackedScalingCuckooFilter filter;
+
+    AbstractRareTermsAggregator(String name, AggregatorFactories factories, SearchContext context,
+                                Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+                                Map<String, Object> metaData, long maxDocCount, double precision,
+                                DocValueFormat format, T valuesSource, U includeExclude) throws IOException {
+        super(name, factories, context, parent, pipelineAggregators, metaData);
+
+        // We seed the rng with the ShardID so results are deterministic and don't change randomly
+        this.filter = new SetBackedScalingCuckooFilter(10000, new Random(context.indexShard().shardId().hashCode()), precision);
+        this.filter.registerBreaker(this::addRequestCircuitBreakerBytes);
+
+        this.maxDocCount = maxDocCount;
+        this.precision = precision;
+        this.format = format;
+        this.valuesSource = valuesSource;
+        this.includeExclude = includeExclude;
+        String scoringAgg = subAggsNeedScore();
+        String nestedAgg = descendsFromNestedAggregator(parent);
+        if (scoringAgg != null && nestedAgg != null) {
+            /*
+             * Terms agg would force the collect mode to depth_first here, because
+             * we need to access the score of nested documents in a sub-aggregation
+             * and we are not able to generate this score while replaying deferred documents.
+             *
+             * But the RareTerms agg _must_ execute in breadth first since it relies on
+             * deferring execution, so we just have to throw up our hands and refuse
+             */
+            throw new IllegalStateException("RareTerms agg [" + name() + "] is the child of the nested agg [" + nestedAgg
+                + "], and also has a scoring child agg [" + scoringAgg + "].  This combination is not supported because " +
+                "it requires executing in [depth_first] mode, which the RareTerms agg cannot do.");
+        }
+    }
+
+    @Override
+    protected boolean shouldDefer(Aggregator aggregator) {
+        return true;
+    }
+
+    @Override
+    public DeferringBucketCollector getDeferringCollector() {
+        deferringCollector = new MergingBucketsDeferringCollector(context, descendsFromGlobalAggregator(parent()));
+        return deferringCollector;
+    }
+
+    private String subAggsNeedScore() {
+        for (Aggregator subAgg : subAggregators) {
+            if (subAgg.scoreMode().needsScores()) {
+                return subAgg.name();
+            }
+        }
+        return null;
+    }
+
+    private String descendsFromNestedAggregator(Aggregator parent) {
+        while (parent != null) {
+            if (parent.getClass() == NestedAggregator.class) {
+                return parent.name();
+            }
+            parent = parent.parent();
+        }
+        return null;
+    }
+
+    protected void doCollect(V val, int docId) throws IOException {
+        long bucketOrdinal = addValueToOrds(val);
+
+        if (bucketOrdinal < 0) { // already seen
+            bucketOrdinal = -1 - bucketOrdinal;
+            collectExistingBucket(subCollectors, docId, bucketOrdinal);
+        } else {
+            collectBucket(subCollectors, docId, bucketOrdinal);
+        }
+    }
+
+    /**
+     * Add's the value to the ordinal map.  Return the newly allocated id if it wasn't in the ordinal map yet,
+     * or <code>-1-id</code> if it was already present
+     */
+    abstract long addValueToOrds(V value);
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/IncludeExclude.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/IncludeExclude.java
index 8154108f9f0bc..30653f04a355a 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/IncludeExclude.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/IncludeExclude.java
@@ -137,10 +137,12 @@ public static IncludeExclude parseExclude(XContentParser parser) throws IOExcept
         }
     }
 
+    public abstract static class Filter {}
+
     // The includeValue and excludeValue ByteRefs which are the result of the parsing
     // process are converted into a LongFilter when used on numeric fields
     // in the index.
-    public abstract static class LongFilter {
+    public abstract static class LongFilter extends Filter {
         public abstract boolean accept(long value);
 
     }
@@ -183,7 +185,7 @@ private void addReject(long val) {
     }
 
     // Only used for the 'map' execution mode (ie. scripts)
-    public abstract static class StringFilter {
+    public abstract static class StringFilter extends Filter {
         public abstract boolean accept(BytesRef value);
     }
 
@@ -231,7 +233,7 @@ public boolean accept(BytesRef value) {
         }
     }
 
-    public abstract static class OrdinalsFilter {
+    public abstract static class OrdinalsFilter extends Filter {
         public abstract LongBitSet acceptedGlobalOrdinals(SortedSetDocValues globalOrdinals) throws IOException;
 
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedRareTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedRareTerms.java
new file mode 100644
index 0000000000000..d774d09fa1862
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedRareTerms.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.util.SetBackedScalingCuckooFilter;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.AggregationExecutionException;
+import org.elasticsearch.search.aggregations.BucketOrder;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+public abstract class InternalMappedRareTerms<A extends InternalRareTerms<A, B>, B extends InternalRareTerms.Bucket<B>>
+    extends InternalRareTerms<A, B> {
+
+    protected DocValueFormat format;
+    protected List<B> buckets;
+    protected Map<String, B> bucketMap;
+
+    final SetBackedScalingCuckooFilter filter;
+
+    protected final Logger logger = LogManager.getLogger(getClass());
+
+    InternalMappedRareTerms(String name, BucketOrder order, List<PipelineAggregator> pipelineAggregators,
+                            Map<String, Object> metaData, DocValueFormat format,
+                            List<B> buckets, long maxDocCount, SetBackedScalingCuckooFilter filter) {
+        super(name, order, maxDocCount, pipelineAggregators, metaData);
+        this.format = format;
+        this.buckets = buckets;
+        this.filter = filter;
+    }
+
+    public long getMaxDocCount() {
+        return maxDocCount;
+    }
+
+    SetBackedScalingCuckooFilter getFilter() {
+        return filter;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    InternalMappedRareTerms(StreamInput in, Bucket.Reader<B> bucketReader) throws IOException {
+        super(in);
+        format = in.readNamedWriteable(DocValueFormat.class);
+        buckets = in.readList(stream -> bucketReader.read(stream, format));
+        filter = new SetBackedScalingCuckooFilter(in, Randomness.get());
+    }
+
+    @Override
+    protected void writeTermTypeInfoTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(format);
+        out.writeList(buckets);
+        filter.writeTo(out);
+    }
+
+    @Override
+    public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+        Map<Object, List<B>> buckets = new HashMap<>();
+        InternalRareTerms<A, B> referenceTerms = null;
+        SetBackedScalingCuckooFilter filter = null;
+
+        for (InternalAggregation aggregation : aggregations) {
+            // Unmapped rare terms don't have a cuckoo filter so we'll skip all this work
+            // and save some type casting headaches later.
+            if (aggregation.isMapped() == false) {
+                continue;
+            }
+
+            @SuppressWarnings("unchecked")
+            InternalRareTerms<A, B> terms = (InternalRareTerms<A, B>) aggregation;
+            if (referenceTerms == null && aggregation.getClass().equals(UnmappedRareTerms.class) == false) {
+                referenceTerms = terms;
+            }
+            if (referenceTerms != null &&
+                referenceTerms.getClass().equals(terms.getClass()) == false &&
+                terms.getClass().equals(UnmappedRareTerms.class) == false) {
+                // control gets into this loop when the same field name against which the query is executed
+                // is of different types in different indices.
+                throw new AggregationExecutionException("Merging/Reducing the aggregations failed when computing the aggregation ["
+                    + referenceTerms.getName() + "] because the field you gave in the aggregation query existed as two different "
+                    + "types in two different indices");
+            }
+            for (B bucket : terms.getBuckets()) {
+                List<B> bucketList = buckets.computeIfAbsent(bucket.getKey(), k -> new ArrayList<>());
+                bucketList.add(bucket);
+            }
+
+            SetBackedScalingCuckooFilter otherFilter = ((InternalMappedRareTerms)aggregation).getFilter();
+            if (filter == null) {
+                filter = new SetBackedScalingCuckooFilter(otherFilter);
+            } else {
+                filter.merge(otherFilter);
+            }
+        }
+
+        final List<B> rare = new ArrayList<>();
+        for (List<B> sameTermBuckets : buckets.values()) {
+            final B b = sameTermBuckets.get(0).reduce(sameTermBuckets, reduceContext);
+            if ((b.getDocCount() <= maxDocCount && containsTerm(filter, b) == false)) {
+                rare.add(b);
+                reduceContext.consumeBucketsAndMaybeBreak(1);
+            } else if (b.getDocCount() > maxDocCount) {
+                // this term has gone over threshold while merging, so add it to the filter.
+                // Note this may happen during incremental reductions too
+                addToFilter(filter, b);
+            }
+        }
+        CollectionUtil.introSort(rare, order.comparator(null));
+        return createWithFilter(name, rare, filter);
+    }
+
+    public abstract boolean containsTerm(SetBackedScalingCuckooFilter filter, B bucket);
+
+    public abstract void addToFilter(SetBackedScalingCuckooFilter filter, B bucket);
+
+    @Override
+    public List<B> getBuckets() {
+        return buckets;
+    }
+
+    @Override
+    public B getBucketByKey(String term) {
+        if (bucketMap == null) {
+            bucketMap = buckets.stream().collect(Collectors.toMap(InternalRareTerms.Bucket::getKeyAsString, Function.identity()));
+        }
+        return bucketMap.get(term);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+        InternalMappedRareTerms<?,?> that = (InternalMappedRareTerms<?,?>) obj;
+        return Objects.equals(buckets, that.buckets)
+            && Objects.equals(format, that.format)
+            && Objects.equals(filter, that.filter);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), buckets, format, filter);
+    }
+
+    @Override
+    public final XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        return doXContentCommon(builder, params, buckets);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedTerms.java
index 547c9d0a80ec6..91c8c72cddb8b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedTerms.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedTerms.java
@@ -115,10 +115,13 @@ public B getBucketByKey(String term) {
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalMappedTerms<?,?> that = (InternalMappedTerms<?,?>) obj;
-        return super.doEquals(obj)
-                && Objects.equals(buckets, that.buckets)
+        return Objects.equals(buckets, that.buckets)
                 && Objects.equals(format, that.format)
                 && Objects.equals(otherDocCount, that.otherDocCount)
                 && Objects.equals(showTermDocCountError, that.showTermDocCountError)
@@ -127,8 +130,8 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(super.doHashCode(), buckets, format, otherDocCount, showTermDocCountError, shardSize);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), buckets, format, otherDocCount, showTermDocCountError, shardSize);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalRareTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalRareTerms.java
new file mode 100644
index 0000000000000..dd1a0c19200cf
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalRareTerms.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.util.SetBackedScalingCuckooFilter;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.BucketOrder;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.InternalOrder;
+import org.elasticsearch.search.aggregations.KeyComparable;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public abstract class InternalRareTerms<A extends InternalRareTerms<A, B>, B extends InternalRareTerms.Bucket<B>>
+    extends InternalMultiBucketAggregation<A, B> implements RareTerms {
+
+    public abstract static class Bucket<B extends Bucket<B>> extends InternalMultiBucketAggregation.InternalBucket
+        implements RareTerms.Bucket, KeyComparable<B> {
+        /**
+         * Reads a bucket. Should be a constructor reference.
+         */
+        @FunctionalInterface
+        public interface Reader<B extends Bucket<B>> {
+            B read(StreamInput in, DocValueFormat format) throws IOException;
+        }
+
+        long bucketOrd;
+
+        protected long docCount;
+        protected InternalAggregations aggregations;
+        protected final DocValueFormat format;
+
+        protected Bucket(long docCount, InternalAggregations aggregations, DocValueFormat formatter) {
+            this.format = formatter;
+            this.docCount = docCount;
+            this.aggregations = aggregations;
+        }
+
+        /**
+         * Read from a stream.
+         */
+        protected Bucket(StreamInput in, DocValueFormat formatter) throws IOException {
+            this.format = formatter;
+            docCount = in.readVLong();
+            aggregations = new InternalAggregations(in);
+        }
+
+        @Override
+        public final void writeTo(StreamOutput out) throws IOException {
+            out.writeVLong(getDocCount());
+            aggregations.writeTo(out);
+            writeTermTo(out);
+        }
+
+        protected abstract void writeTermTo(StreamOutput out) throws IOException;
+
+        @Override
+        public long getDocCount() {
+            return docCount;
+        }
+
+        @Override
+        public Aggregations getAggregations() {
+            return aggregations;
+        }
+
+        abstract B newBucket(long docCount, InternalAggregations aggs);
+
+        public B reduce(List<B> buckets, ReduceContext context) {
+            long docCount = 0;
+            List<InternalAggregations> aggregationsList = new ArrayList<>(buckets.size());
+            for (B bucket : buckets) {
+                docCount += bucket.docCount;
+                aggregationsList.add(bucket.aggregations);
+            }
+            InternalAggregations aggs = InternalAggregations.reduce(aggregationsList, context);
+            return newBucket(docCount, aggs);
+        }
+
+        @Override
+        public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            keyToXContent(builder);
+            builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount());
+            aggregations.toXContentInternal(builder, params);
+            builder.endObject();
+            return builder;
+        }
+
+        protected abstract XContentBuilder keyToXContent(XContentBuilder builder) throws IOException;
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null || getClass() != obj.getClass()) {
+                return false;
+            }
+            Bucket<?> that = (Bucket<?>) obj;
+            return Objects.equals(docCount, that.docCount)
+                && Objects.equals(aggregations, that.aggregations);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(getClass(), docCount, aggregations);
+        }
+    }
+
+    protected final BucketOrder order;
+    protected final long maxDocCount;
+
+    protected InternalRareTerms(String name, BucketOrder order, long maxDocCount,
+                            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        super(name, pipelineAggregators, metaData);
+        this.order = order;
+        this.maxDocCount = maxDocCount;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    protected InternalRareTerms(StreamInput in) throws IOException {
+        super(in);
+        order = InternalOrder.Streams.readOrder(in);
+        maxDocCount = in.readVLong();
+    }
+
+    @Override
+    protected final void doWriteTo(StreamOutput out) throws IOException {
+        order.writeTo(out);
+        out.writeVLong(maxDocCount);
+        writeTermTypeInfoTo(out);
+    }
+
+    protected abstract void writeTermTypeInfoTo(StreamOutput out) throws IOException;
+
+    @Override
+    public abstract List<B> getBuckets();
+
+    @Override
+    public abstract B getBucketByKey(String term);
+
+    @Override
+    public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+        throw new UnsupportedOperationException();
+    }
+
+    protected abstract A createWithFilter(String name, List<B> buckets, SetBackedScalingCuckooFilter filter);
+
+    /**
+     * Create an array to hold some buckets. Used in collecting the results.
+     */
+    protected abstract B[] createBucketsArray(int size);
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+        InternalRareTerms<?,?> that = (InternalRareTerms<?,?>) obj;
+        return Objects.equals(maxDocCount, that.maxDocCount)
+            && Objects.equals(order, that.order);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), maxDocCount, order);
+    }
+
+    protected static XContentBuilder doXContentCommon(XContentBuilder builder, Params params,
+                                                      List<? extends Bucket> buckets) throws IOException {
+        builder.startArray(CommonFields.BUCKETS.getPreferredName());
+        for (Bucket bucket : buckets) {
+            bucket.toXContent(builder, params);
+        }
+        builder.endArray();
+        return builder;
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
index f40ff84bf2130..fc607621ff7ed 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
@@ -326,7 +326,11 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
     protected abstract B[] createBucketsArray(int size);
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalTerms<?,?> that = (InternalTerms<?,?>) obj;
         return Objects.equals(minDocCount, that.minDocCount)
                 && Objects.equals(order, that.order)
@@ -334,8 +338,8 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(minDocCount, order, requiredSize);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), minDocCount, order, requiredSize);
     }
 
     protected static XContentBuilder doXContentCommon(XContentBuilder builder, Params params,
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongRareTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongRareTerms.java
new file mode 100644
index 0000000000000..29f84fb6030e1
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongRareTerms.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.util.SetBackedScalingCuckooFilter;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.BucketOrder;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Result of the RareTerms aggregation when the field is some kind of whole number like a integer, long, or a date.
+ */
+public class LongRareTerms extends InternalMappedRareTerms<LongRareTerms, LongRareTerms.Bucket> {
+    public static final String NAME = "lrareterms";
+
+    public static class Bucket extends InternalRareTerms.Bucket<Bucket> {
+        long term;
+
+        public Bucket(long term, long docCount, InternalAggregations aggregations, DocValueFormat format) {
+            super(docCount, aggregations, format);
+            this.term = term;
+        }
+
+        /**
+         * Read from a stream.
+         */
+        public Bucket(StreamInput in, DocValueFormat format) throws IOException {
+            super(in, format);
+            term = in.readLong();
+        }
+
+        @Override
+        protected void writeTermTo(StreamOutput out) throws IOException {
+            out.writeLong(term);
+        }
+
+        @Override
+        public String getKeyAsString() {
+            return format.format(term).toString();
+        }
+
+        @Override
+        public Object getKey() {
+            return term;
+        }
+
+        @Override
+        public Number getKeyAsNumber() {
+            return term;
+        }
+
+        @Override
+        public int compareKey(Bucket other) {
+            return Long.compare(term, other.term);
+        }
+
+        @Override
+        Bucket newBucket(long docCount, InternalAggregations aggs) {
+            return new Bucket(term, docCount, aggs, format);
+        }
+
+        @Override
+        protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+            builder.field(CommonFields.KEY.getPreferredName(), term);
+            if (format != DocValueFormat.RAW) {
+                builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), format.format(term).toString());
+            }
+            return builder;
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            return super.equals(obj) && Objects.equals(term, ((Bucket) obj).term);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(super.hashCode(), term);
+        }
+    }
+
+    LongRareTerms(String name, BucketOrder order, List<PipelineAggregator> pipelineAggregators,
+                  Map<String, Object> metaData, DocValueFormat format,
+                  List<LongRareTerms.Bucket> buckets, long maxDocCount, SetBackedScalingCuckooFilter filter) {
+        super(name, order, pipelineAggregators, metaData, format, buckets, maxDocCount, filter);
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public LongRareTerms(StreamInput in) throws IOException {
+        super(in, LongRareTerms.Bucket::new);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public LongRareTerms create(List<LongRareTerms.Bucket> buckets) {
+        return new LongRareTerms(name, order, pipelineAggregators(), metaData, format, buckets, maxDocCount, filter);
+    }
+
+    @Override
+    public LongRareTerms.Bucket createBucket(InternalAggregations aggregations, LongRareTerms.Bucket prototype) {
+        return new LongRareTerms.Bucket(prototype.term, prototype.getDocCount(), aggregations, prototype.format);
+    }
+
+    @Override
+    protected LongRareTerms createWithFilter(String name, List<LongRareTerms.Bucket> buckets, SetBackedScalingCuckooFilter filter) {
+        return new LongRareTerms(name, order, pipelineAggregators(), getMetaData(), format,
+            buckets, maxDocCount, filter);
+    }
+
+    @Override
+    protected LongRareTerms.Bucket[] createBucketsArray(int size) {
+        return new LongRareTerms.Bucket[size];
+    }
+
+    @Override
+    public boolean containsTerm(SetBackedScalingCuckooFilter filter, LongRareTerms.Bucket bucket) {
+        return filter.mightContain((long) bucket.getKey());
+    }
+
+    @Override
+    public void addToFilter(SetBackedScalingCuckooFilter filter, LongRareTerms.Bucket bucket) {
+        filter.add((long) bucket.getKey());
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongRareTermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongRareTermsAggregator.java
new file mode 100644
index 0000000000000..b1d294fefdcf6
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongRareTermsAggregator.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.util.LongHash;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyList;
+
+/**
+ * An aggregator that finds "rare" string values (e.g. terms agg that orders ascending)
+ */
+public class LongRareTermsAggregator extends AbstractRareTermsAggregator<ValuesSource.Numeric, IncludeExclude.LongFilter, Long> {
+
+    protected LongHash bucketOrds;
+
+    LongRareTermsAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, DocValueFormat format,
+                                   SearchContext aggregationContext, Aggregator parent, IncludeExclude.LongFilter longFilter,
+                                   int maxDocCount, double precision, List<PipelineAggregator> pipelineAggregators,
+                                   Map<String, Object> metaData) throws IOException {
+        super(name, factories, aggregationContext, parent, pipelineAggregators, metaData, maxDocCount, precision,
+            format, valuesSource, longFilter);
+        this.bucketOrds = new LongHash(1, aggregationContext.bigArrays());
+    }
+
+    protected SortedNumericDocValues getValues(ValuesSource.Numeric valuesSource, LeafReaderContext ctx) throws IOException {
+        return valuesSource.longValues(ctx);
+    }
+
+    @Override
+    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
+                                                final LeafBucketCollector sub) throws IOException {
+        final SortedNumericDocValues values = getValues(valuesSource, ctx);
+        if (subCollectors == null) {
+            subCollectors = sub;
+        }
+        return new LeafBucketCollectorBase(sub, values) {
+
+            @Override
+            public void collect(int docId, long owningBucketOrdinal) throws IOException {
+                if (values.advanceExact(docId)) {
+                    final int valuesCount = values.docValueCount();
+                    long previous = Long.MAX_VALUE;
+                    for (int i = 0; i < valuesCount; ++i) {
+                        final long val = values.nextValue();
+                        if (previous != val || i == 0) {
+                            if ((includeExclude == null) || (includeExclude.accept(val))) {
+                                doCollect(val, docId);
+                            }
+                            previous = val;
+                        }
+                    }
+                }
+            }
+        };
+    }
+
+    @Override
+    long addValueToOrds(Long value) {
+        return bucketOrds.add(value);
+    }
+
+    /**
+     * Merges the ordinals to a minimal set, populates the CuckooFilter and
+     * generates a final set of buckets.
+     *
+     * If a term is below the maxDocCount, it is turned into a Bucket.  Otherwise,
+     * the term is added to the filter, and pruned from the ordinal map.  If
+     * necessary the ordinal map is merged down to a minimal set to remove deletions
+     */
+    private List<LongRareTerms.Bucket> buildSketch() {
+        long deletionCount = 0;
+        LongHash newBucketOrds = new LongHash(1, context.bigArrays());
+        List<LongRareTerms.Bucket> buckets = new ArrayList<>();
+        try (LongHash oldBucketOrds = bucketOrds) {
+
+            long[] mergeMap = new long[(int) oldBucketOrds.size()];
+            for (int i = 0; i < oldBucketOrds.size(); i++) {
+                long oldKey = oldBucketOrds.get(i);
+                long newBucketOrd = -1;
+
+                long docCount = bucketDocCount(i);
+                // if the key is below threshold, reinsert into the new ords
+                if (docCount <= maxDocCount) {
+                    newBucketOrd = newBucketOrds.add(oldKey);
+                    LongRareTerms.Bucket bucket = new LongRareTerms.Bucket(oldKey, docCount, null, format);
+                    bucket.bucketOrd = newBucketOrd;
+                    buckets.add(bucket);
+
+                    consumeBucketsAndMaybeBreak(1);
+                } else {
+                    // Make a note when one of the ords has been deleted
+                    deletionCount += 1;
+                    filter.add(oldKey);
+                }
+                mergeMap[i] = newBucketOrd;
+            }
+
+            // Only merge/delete the ordinals if we have actually deleted one,
+            // to save on some redundant work
+            if (deletionCount > 0) {
+                mergeBuckets(mergeMap, newBucketOrds.size());
+                if (deferringCollector != null) {
+                    deferringCollector.mergeBuckets(mergeMap);
+                }
+            }
+        }
+        bucketOrds = newBucketOrds;
+        return buckets;
+    }
+
+    @Override
+    public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException {
+        assert owningBucketOrdinal == 0;
+        List<LongRareTerms.Bucket> buckets = buildSketch();
+        runDeferredCollections(buckets.stream().mapToLong(b -> b.bucketOrd).toArray());
+
+        // Finalize the buckets
+        for (LongRareTerms.Bucket bucket : buckets) {
+            bucket.aggregations = bucketAggregations(bucket.bucketOrd);
+        }
+
+        CollectionUtil.introSort(buckets, ORDER.comparator(this));
+        return new LongRareTerms(name, ORDER, pipelineAggregators(), metaData(), format, buckets, maxDocCount, filter);
+    }
+
+    @Override
+    public InternalAggregation buildEmptyAggregation() {
+        return new LongRareTerms(name, ORDER, pipelineAggregators(), metaData(), format, emptyList(), 0, filter);
+    }
+
+    @Override
+    public void doClose() {
+        Releasables.close(bucketOrds);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTerms.java
new file mode 100644
index 0000000000000..2248514783264
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTerms.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+
+import java.util.List;
+
+
+public interface RareTerms extends MultiBucketsAggregation {
+
+    /**
+     * A bucket that is associated with a single term
+     */
+    interface Bucket extends MultiBucketsAggregation.Bucket {
+
+        Number getKeyAsNumber();
+    }
+
+    /**
+     * Return the sorted list of the buckets in this terms aggregation.
+     */
+    @Override
+    List<? extends Bucket> getBuckets();
+
+    /**
+     * Get the bucket for the given term, or null if there is no such bucket.
+     */
+    Bucket getBucketByKey(String term);
+
+}
+
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java
new file mode 100644
index 0000000000000..285869dd2e0cf
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregationBuilder.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+public class RareTermsAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource, RareTermsAggregationBuilder> {
+    public static final String NAME = "rare_terms";
+
+    private static final ParseField MAX_DOC_COUNT_FIELD_NAME = new ParseField("max_doc_count");
+    private static final ParseField PRECISION = new ParseField("precision");
+
+    private static final int MAX_MAX_DOC_COUNT = 100;
+    private static final ObjectParser<RareTermsAggregationBuilder, Void> PARSER;
+    static {
+        PARSER = new ObjectParser<>(RareTermsAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareAnyFields(PARSER, true, true);
+        PARSER.declareLong(RareTermsAggregationBuilder::maxDocCount, MAX_DOC_COUNT_FIELD_NAME);
+
+        PARSER.declareField((b, v) -> b.includeExclude(IncludeExclude.merge(v, b.includeExclude())),
+            IncludeExclude::parseInclude, IncludeExclude.INCLUDE_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_OR_STRING);
+
+        PARSER.declareField((b, v) -> b.includeExclude(IncludeExclude.merge(b.includeExclude(), v)),
+            IncludeExclude::parseExclude, IncludeExclude.EXCLUDE_FIELD, ObjectParser.ValueType.STRING_ARRAY);
+
+        PARSER.declareDouble(RareTermsAggregationBuilder::setPrecision, PRECISION);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
+        return PARSER.parse(parser, new RareTermsAggregationBuilder(aggregationName, null), null);
+    }
+
+    private IncludeExclude includeExclude = null;
+    private int maxDocCount = 1;
+    private double precision = 0.001;
+
+    public RareTermsAggregationBuilder(String name, ValueType valueType) {
+        super(name, ValuesSourceType.ANY, valueType);
+    }
+
+    private RareTermsAggregationBuilder(RareTermsAggregationBuilder clone, Builder factoriesBuilder, Map<String, Object> metaData) {
+        super(clone, factoriesBuilder, metaData);
+        this.includeExclude = clone.includeExclude;
+    }
+
+    @Override
+    protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map<String, Object> metaData) {
+        return new RareTermsAggregationBuilder(this, factoriesBuilder, metaData);
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public RareTermsAggregationBuilder(StreamInput in) throws IOException {
+        super(in, ValuesSourceType.ANY);
+        includeExclude = in.readOptionalWriteable(IncludeExclude::new);
+        maxDocCount = in.readVInt();
+    }
+
+    @Override
+    protected boolean serializeTargetValueType() {
+        return true;
+    }
+
+    @Override
+    protected void innerWriteTo(StreamOutput out) throws IOException {
+        out.writeOptionalWriteable(includeExclude);
+        out.writeVInt(maxDocCount);
+    }
+
+    /**
+     * Set the maximum document count terms should have in order to appear in
+     * the response.
+     */
+    public RareTermsAggregationBuilder maxDocCount(long maxDocCount) {
+        if (maxDocCount <= 0) {
+            throw new IllegalArgumentException(
+                "[" + MAX_DOC_COUNT_FIELD_NAME.getPreferredName() + "] must be greater than 0. Found ["
+                    + maxDocCount + "] in [" + name + "]");
+        }
+        //TODO review: what size cap should we put on this?
+        if (maxDocCount > MAX_MAX_DOC_COUNT) {
+            throw new IllegalArgumentException("[" + MAX_DOC_COUNT_FIELD_NAME.getPreferredName() + "] must be smaller" +
+                "than " + MAX_MAX_DOC_COUNT + "in [" + name + "]");
+        }
+        this.maxDocCount = (int) maxDocCount;
+        return this;
+    }
+
+    /**
+     * Set terms to include and exclude from the aggregation results
+     */
+    public RareTermsAggregationBuilder includeExclude(IncludeExclude includeExclude) {
+        this.includeExclude = includeExclude;
+        return this;
+    }
+
+    /**
+     * Get terms to include and exclude from the aggregation results
+     */
+    public IncludeExclude includeExclude() {
+        return includeExclude;
+    }
+
+    /**
+     * Get the current false positive rate for individual cuckoo filters.
+     */
+    public double getPrecision() {
+        return precision;
+    }
+
+    /**
+     * Set's the false-positive rate for individual cuckoo filters.  Does not dictate the overall fpp rate
+     * since we use a "scaling" cuckoo filter which adds more filters as required, and the overall
+     * error rate grows differently than individual filters
+     *
+     * This value does, however, affect the overall space usage of the filter.  Coarser precisions provide
+     * more compact filters.  The default is 0.01
+     */
+    public void setPrecision(double precision) {
+        if (precision < 0.00001) {
+            throw new IllegalArgumentException("[precision] must be greater than 0.00001");
+        }
+        this.precision = precision;
+    }
+
+    @Override
+    protected ValuesSourceAggregatorFactory<ValuesSource> innerBuild(SearchContext context,
+                                                                        ValuesSourceConfig<ValuesSource> config,
+                                                                        AggregatorFactory parent,
+                                                                        Builder subFactoriesBuilder) throws IOException {
+        return new RareTermsAggregatorFactory(name, config, includeExclude,
+            context, parent, subFactoriesBuilder, metaData, maxDocCount, precision);
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        if (includeExclude != null) {
+            includeExclude.toXContent(builder, params);
+        }
+        builder.field(MAX_DOC_COUNT_FIELD_NAME.getPreferredName(), maxDocCount);
+        builder.field(PRECISION.getPreferredName(), precision);
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), includeExclude, maxDocCount, precision);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+        RareTermsAggregationBuilder other = (RareTermsAggregationBuilder) obj;
+        return Objects.equals(includeExclude, other.includeExclude)
+            && Objects.equals(maxDocCount, other.maxDocCount)
+            && Objects.equals(precision, other.precision);
+    }
+
+    @Override
+    public String getType() {
+        return NAME;
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorFactory.java
new file mode 100644
index 0000000000000..ab4c4bc00843a
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorFactory.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.AggregationExecutionException;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.NonCollectingAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class RareTermsAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource> {
+    private final IncludeExclude includeExclude;
+    private final int maxDocCount;
+    private final double precision;
+
+    RareTermsAggregatorFactory(String name, ValuesSourceConfig<ValuesSource> config,
+                                      IncludeExclude includeExclude,
+                                      SearchContext context,
+                                      AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder,
+                                      Map<String, Object> metaData, int maxDocCount, double precision) throws IOException {
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
+        this.includeExclude = includeExclude;
+        this.maxDocCount = maxDocCount;
+        this.precision = precision;
+    }
+
+    @Override
+    protected Aggregator createUnmapped(Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
+        throws IOException {
+        final InternalAggregation aggregation = new UnmappedRareTerms(name, pipelineAggregators, metaData);
+        return new NonCollectingAggregator(name, context, parent, factories, pipelineAggregators, metaData) {
+            @Override
+            public InternalAggregation buildEmptyAggregation() {
+                return aggregation;
+            }
+        };
+    }
+
+    @Override
+    protected Aggregator doCreateInternal(ValuesSource valuesSource, Aggregator parent, boolean collectsFromSingleBucket,
+                                          List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
+        if (collectsFromSingleBucket == false) {
+            return asMultiBucketAggregator(this, context, parent);
+        }
+        if (valuesSource instanceof ValuesSource.Bytes) {
+            ExecutionMode execution = ExecutionMode.MAP; //TODO global ords not implemented yet, only supports "map"
+
+            DocValueFormat format = config.format();
+            if ((includeExclude != null) && (includeExclude.isRegexBased()) && format != DocValueFormat.RAW) {
+                throw new AggregationExecutionException("Aggregation [" + name + "] cannot support " +
+                    "regular expression style include/exclude settings as they can only be applied to string fields. " +
+                    "Use an array of values for include/exclude clauses");
+            }
+
+            return execution.create(name, factories, valuesSource, format,
+                includeExclude, context, parent, pipelineAggregators, metaData, maxDocCount, precision);
+        }
+
+        if ((includeExclude != null) && (includeExclude.isRegexBased())) {
+            throw new AggregationExecutionException("Aggregation [" + name + "] cannot support regular expression style include/exclude "
+                + "settings as they can only be applied to string fields. Use an array of numeric values for include/exclude clauses " +
+                "used to filter numeric fields");
+        }
+
+        if (valuesSource instanceof ValuesSource.Numeric) {
+            IncludeExclude.LongFilter longFilter = null;
+            if (((ValuesSource.Numeric) valuesSource).isFloatingPoint()) {
+                throw new AggregationExecutionException("RareTerms aggregation does not support floating point fields.");
+            }
+            if (includeExclude != null) {
+                longFilter = includeExclude.convertToLongFilter(config.format());
+            }
+            return new LongRareTermsAggregator(name, factories, (ValuesSource.Numeric) valuesSource, config.format(),
+                context, parent, longFilter, maxDocCount, precision, pipelineAggregators, metaData);
+        }
+
+        throw new AggregationExecutionException("RareTerms aggregation cannot be applied to field [" + config.fieldContext().field()
+            + "]. It can only be applied to numeric or string fields.");
+    }
+
+    public enum ExecutionMode {
+
+        MAP(new ParseField("map")) {
+
+            @Override
+            Aggregator create(String name, AggregatorFactories factories, ValuesSource valuesSource,
+                              DocValueFormat format, IncludeExclude includeExclude,
+                              SearchContext context, Aggregator parent,
+                              List<PipelineAggregator> pipelineAggregators,
+                              Map<String, Object> metaData, long maxDocCount, double precision)
+                throws IOException {
+                final IncludeExclude.StringFilter filter = includeExclude == null ? null : includeExclude.convertToStringFilter(format);
+                return new StringRareTermsAggregator(name, factories, (ValuesSource.Bytes) valuesSource, format, filter,
+                    context, parent, pipelineAggregators, metaData, maxDocCount, precision);
+            }
+
+            @Override
+            boolean needsGlobalOrdinals() {
+                return false;
+            }
+
+        };
+
+        public static ExecutionMode fromString(String value, final DeprecationLogger deprecationLogger) {
+            switch (value) {
+                case "map":
+                    return MAP;
+                default:
+                    throw new IllegalArgumentException("Unknown `execution_hint`: [" + value + "], expected any of [map]");
+            }
+        }
+
+        private final ParseField parseField;
+
+        ExecutionMode(ParseField parseField) {
+            this.parseField = parseField;
+        }
+
+        abstract Aggregator create(String name, AggregatorFactories factories, ValuesSource valuesSource,
+                                   DocValueFormat format, IncludeExclude includeExclude,
+                                   SearchContext context, Aggregator parent,
+                                   List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData,
+                                   long maxDocCount, double precision)
+            throws IOException;
+
+        abstract boolean needsGlobalOrdinals();
+
+        @Override
+        public String toString() {
+            return parseField.getPreferredName();
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringRareTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringRareTerms.java
new file mode 100644
index 0000000000000..3c3e19664a631
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringRareTerms.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.util.SetBackedScalingCuckooFilter;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.BucketOrder;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class StringRareTerms extends InternalMappedRareTerms<StringRareTerms, StringRareTerms.Bucket> {
+    public static final String NAME = "srareterms";
+
+    public static class Bucket extends InternalRareTerms.Bucket<Bucket> {
+        BytesRef termBytes;
+
+        public Bucket(BytesRef term, long docCount, InternalAggregations aggregations, DocValueFormat format) {
+            super(docCount, aggregations, format);
+            this.termBytes = term;
+        }
+
+        /**
+         * Read from a stream.
+         */
+        public Bucket(StreamInput in, DocValueFormat format) throws IOException {
+            super(in, format);
+            termBytes = in.readBytesRef();
+        }
+
+        @Override
+        protected void writeTermTo(StreamOutput out) throws IOException {
+            out.writeBytesRef(termBytes);
+        }
+
+        @Override
+        public Object getKey() {
+            return getKeyAsString();
+        }
+
+        // this method is needed for scripted numeric aggs
+        @Override
+        public Number getKeyAsNumber() {
+            /*
+             * If the term is a long greater than 2^52 then parsing as a double would lose accuracy. Therefore, we first parse as a long and
+             * if this fails then we attempt to parse the term as a double.
+             */
+            try {
+                return Long.parseLong(termBytes.utf8ToString());
+            } catch (final NumberFormatException ignored) {
+                return Double.parseDouble(termBytes.utf8ToString());
+            }
+        }
+
+        @Override
+        public String getKeyAsString() {
+            return format.format(termBytes).toString();
+        }
+
+        @Override
+        public int compareKey(Bucket other) {
+            return termBytes.compareTo(other.termBytes);
+        }
+
+        @Override
+        Bucket newBucket(long docCount, InternalAggregations aggs) {
+            return new Bucket(termBytes, docCount, aggs, format);
+        }
+
+        @Override
+        protected final XContentBuilder keyToXContent(XContentBuilder builder) throws IOException {
+            return builder.field(CommonFields.KEY.getPreferredName(), getKeyAsString());
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            return super.equals(obj) && Objects.equals(termBytes, ((Bucket) obj).termBytes);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(super.hashCode(), termBytes);
+        }
+    }
+
+    StringRareTerms(String name, BucketOrder order, List<PipelineAggregator> pipelineAggregators,
+                           Map<String, Object> metaData, DocValueFormat format,
+                           List<StringRareTerms.Bucket> buckets, long maxDocCount, SetBackedScalingCuckooFilter filter) {
+        super(name, order, pipelineAggregators, metaData, format, buckets, maxDocCount, filter);
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public StringRareTerms(StreamInput in) throws IOException {
+        super(in, StringRareTerms.Bucket::new);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public StringRareTerms create(List<StringRareTerms.Bucket> buckets) {
+        return new StringRareTerms(name, order, pipelineAggregators(), metaData, format, buckets, maxDocCount, filter);
+    }
+
+    @Override
+    public StringRareTerms.Bucket createBucket(InternalAggregations aggregations, StringRareTerms.Bucket prototype) {
+        return new StringRareTerms.Bucket(prototype.termBytes, prototype.getDocCount(), aggregations, prototype.format);
+    }
+
+    @Override
+    protected StringRareTerms createWithFilter(String name, List<StringRareTerms.Bucket> buckets,
+                                               SetBackedScalingCuckooFilter filterFilter) {
+        return new StringRareTerms(name, order, pipelineAggregators(), metaData, format,
+            buckets, maxDocCount, filterFilter);
+    }
+
+    @Override
+    protected StringRareTerms.Bucket[] createBucketsArray(int size) {
+        return new StringRareTerms.Bucket[size];
+    }
+
+    @Override
+    public boolean containsTerm(SetBackedScalingCuckooFilter filter, StringRareTerms.Bucket bucket) {
+        return filter.mightContain(bucket.termBytes);
+    }
+
+    @Override
+    public void addToFilter(SetBackedScalingCuckooFilter filter, StringRareTerms.Bucket bucket) {
+        filter.add(bucket.termBytes);
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringRareTermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringRareTermsAggregator.java
new file mode 100644
index 0000000000000..0c200e96b242c
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringRareTermsAggregator.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.util.BytesRefHash;
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyList;
+
+/**
+ * An aggregator that finds "rare" string values (e.g. terms agg that orders ascending)
+ */
+public class StringRareTermsAggregator extends AbstractRareTermsAggregator<ValuesSource.Bytes, IncludeExclude.StringFilter, BytesRef> {
+    protected BytesRefHash bucketOrds;
+
+    StringRareTermsAggregator(String name, AggregatorFactories factories, ValuesSource.Bytes valuesSource,
+                                     DocValueFormat format,  IncludeExclude.StringFilter stringFilter,
+                                     SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+                                     Map<String, Object> metaData, long maxDocCount, double precision) throws IOException {
+        super(name, factories, context, parent, pipelineAggregators, metaData, maxDocCount, precision, format, valuesSource, stringFilter);
+        this.bucketOrds = new BytesRefHash(1, context.bigArrays());
+    }
+
+    @Override
+    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
+                                                final LeafBucketCollector sub) throws IOException {
+        final SortedBinaryDocValues values = valuesSource.bytesValues(ctx);
+        if (subCollectors == null) {
+            subCollectors = sub;
+        }
+        return new LeafBucketCollectorBase(sub, values) {
+            final BytesRefBuilder previous = new BytesRefBuilder();
+
+            @Override
+            public void collect(int docId, long bucket) throws IOException {
+                assert bucket == 0;
+                if (values.advanceExact(docId)) {
+                    final int valuesCount = values.docValueCount();
+                    previous.clear();
+
+                    // SortedBinaryDocValues don't guarantee uniqueness so we
+                    // need to take care of dups
+                    for (int i = 0; i < valuesCount; ++i) {
+                        final BytesRef bytes = values.nextValue();
+                        if (includeExclude != null && !includeExclude.accept(bytes)) {
+                            continue;
+                        }
+                        if (i > 0 && previous.get().equals(bytes)) {
+                            continue;
+                        }
+
+                        doCollect(bytes, docId);
+                        previous.copyBytes(bytes);
+                    }
+                }
+            }
+        };
+    }
+
+    @Override
+    long addValueToOrds(BytesRef value) {
+        return bucketOrds.add(value);
+    }
+
+    /**
+     * Merges the ordinals to a minimal set, populates the CuckooFilter and
+     * generates a final set of buckets.
+     *
+     * If a term is below the maxDocCount, it is turned into a Bucket.  Otherwise,
+     * the term is added to the filter, and pruned from the ordinal map.  If
+     * necessary the ordinal map is merged down to a minimal set to remove deletions
+     */
+    private List<StringRareTerms.Bucket> buildSketch() {
+        long deletionCount = 0;
+        BytesRefHash newBucketOrds = new BytesRefHash(1, context.bigArrays());
+        List<StringRareTerms.Bucket> buckets = new ArrayList<>();
+        try (BytesRefHash oldBucketOrds = bucketOrds) {
+
+            long[] mergeMap = new long[(int) oldBucketOrds.size()];
+            BytesRef scratch = new BytesRef();
+            for (int i = 0; i < oldBucketOrds.size(); i++) {
+                BytesRef oldKey = oldBucketOrds.get(i, scratch);
+                long newBucketOrd = -1;
+                long docCount = bucketDocCount(i);
+                // if the key is below threshold, reinsert into the new ords
+                if (docCount <= maxDocCount) {
+                    newBucketOrd = newBucketOrds.add(oldKey);
+                    StringRareTerms.Bucket bucket = new StringRareTerms.Bucket(BytesRef.deepCopyOf(oldKey), docCount, null, format);
+                    bucket.bucketOrd = newBucketOrd;
+                    buckets.add(bucket);
+
+                    consumeBucketsAndMaybeBreak(1);
+                } else {
+                    // Make a note when one of the ords has been deleted
+                    deletionCount += 1;
+                    filter.add(oldKey);
+                }
+                mergeMap[i] = newBucketOrd;
+            }
+
+            // Only merge/delete the ordinals if we have actually deleted one,
+            // to save on some redundant work
+            if (deletionCount > 0) {
+                mergeBuckets(mergeMap, newBucketOrds.size());
+                if (deferringCollector != null) {
+                    deferringCollector.mergeBuckets(mergeMap);
+                }
+            }
+        }
+        bucketOrds = newBucketOrds;
+        return buckets;
+    }
+
+    @Override
+    public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException {
+        assert owningBucketOrdinal == 0;
+
+        List<StringRareTerms.Bucket> buckets = buildSketch();
+        runDeferredCollections(buckets.stream().mapToLong(b -> b.bucketOrd).toArray());
+
+        // Finalize the buckets
+        for (StringRareTerms.Bucket bucket : buckets) {
+            bucket.aggregations = bucketAggregations(bucket.bucketOrd);
+        }
+
+        CollectionUtil.introSort(buckets, ORDER.comparator(this));
+        return new StringRareTerms(name, ORDER, pipelineAggregators(), metaData(), format, buckets, maxDocCount, filter);
+    }
+
+    @Override
+    public InternalAggregation buildEmptyAggregation() {
+        return new StringRareTerms(name, LongRareTermsAggregator.ORDER, pipelineAggregators(), metaData(), format, emptyList(), 0, filter);
+    }
+
+    @Override
+    public void doClose() {
+        Releasables.close(bucketOrds);
+    }
+}
+
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java
index 20162fd1bc78a..446aafa22d36b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java
@@ -114,8 +114,9 @@ public void collect(int doc, long bucket) throws IOException {
     public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException {
         assert owningBucketOrdinal == 0;
 
-        if (bucketCountThresholds.getMinDocCount() == 0 && (InternalOrder.isCountDesc(order) == false ||
-                bucketOrds.size() < bucketCountThresholds.getRequiredSize())) {
+        if (bucketCountThresholds.getMinDocCount() == 0
+            && (InternalOrder.isCountDesc(order) == false
+                    || bucketOrds.size() < bucketCountThresholds.getRequiredSize())) {
             // we need to fill-in the blanks
             for (LeafReaderContext ctx : context.searcher().getTopReaderContext().leaves()) {
                 final SortedBinaryDocValues values = valuesSource.bytesValues(ctx);
@@ -168,11 +169,10 @@ public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOE
         runDeferredCollections(survivingBucketOrds);
 
         // Now build the aggs
-        for (int i = 0; i < list.length; i++) {
-          final StringTerms.Bucket bucket = list[i];
-          bucket.termBytes = BytesRef.deepCopyOf(bucket.termBytes);
-          bucket.aggregations = bucketAggregations(bucket.bucketOrd);
-          bucket.docCountError = 0;
+        for (final StringTerms.Bucket bucket : list) {
+            bucket.termBytes = BytesRef.deepCopyOf(bucket.termBytes);
+            bucket.aggregations = bucketAggregations(bucket.bucketOrd);
+            bucket.docCountError = 0;
         }
 
         return new StringTerms(name, order, bucketCountThresholds.getRequiredSize(), bucketCountThresholds.getMinDocCount(),
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
index 5887f6b525891..a124feb115b19 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
@@ -333,8 +333,8 @@ public TermsAggregationBuilder showTermDocCountError(boolean showTermDocCountErr
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+    protected ValuesSourceAggregatorFactory<ValuesSource> innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource> config,
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         return new TermsAggregatorFactory(name, config, order, includeExclude, executionHint, collectMode,
                 bucketCountThresholds, showTermDocCountError, context, parent, subFactoriesBuilder, metaData);
     }
@@ -358,19 +358,23 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(bucketCountThresholds, collectMode, executionHint, includeExclude, order, showTermDocCountError);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), bucketCountThresholds, collectMode,
+            executionHint, includeExclude, order, showTermDocCountError);
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         TermsAggregationBuilder other = (TermsAggregationBuilder) obj;
         return Objects.equals(bucketCountThresholds, other.bucketCountThresholds)
-                && Objects.equals(collectMode, other.collectMode)
-                && Objects.equals(executionHint, other.executionHint)
-                && Objects.equals(includeExclude, other.includeExclude)
-                && Objects.equals(order, other.order)
-                && Objects.equals(showTermDocCountError, other.showTermDocCountError);
+            && Objects.equals(collectMode, other.collectMode)
+            && Objects.equals(executionHint, other.executionHint)
+            && Objects.equals(includeExclude, other.includeExclude)
+            && Objects.equals(order, other.order)
+            && Objects.equals(showTermDocCountError, other.showTermDocCountError);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
index 877a8e59bc2d3..96daeeb476a28 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
@@ -46,7 +46,7 @@
 import java.util.List;
 import java.util.Map;
 
-public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource, TermsAggregatorFactory> {
+public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource> {
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(TermsAggregatorFactory.class));
 
     static Boolean REMAP_GLOBAL_ORDS, COLLECT_SEGMENT_ORDS;
@@ -67,7 +67,7 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<Values
                                   TermsAggregator.BucketCountThresholds bucketCountThresholds,
                                   boolean showTermDocCountError,
                                   SearchContext context,
-                                  AggregatorFactory<?> parent,
+                                  AggregatorFactory parent,
                                   AggregatorFactories.Builder subFactoriesBuilder,
                                   Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
@@ -189,7 +189,7 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, Aggregator pare
         }
 
         throw new AggregationExecutionException("terms aggregation cannot be applied to field [" + config.fieldContext().field()
-                + "]. It can only be applied to numeric or string fields.");
+            + "]. It can only be applied to numeric or string fields.");
     }
 
     // return the SubAggCollectionMode that this aggregation should use based on the expected size
@@ -262,21 +262,26 @@ Aggregator create(String name,
                 assert maxOrd != -1;
                 final double ratio = maxOrd / ((double) context.searcher().getIndexReader().numDocs());
 
+                assert valuesSource instanceof ValuesSource.Bytes.WithOrdinals;
+                ValuesSource.Bytes.WithOrdinals ordinalsValuesSource = (ValuesSource.Bytes.WithOrdinals) valuesSource;
+
                 if (factories == AggregatorFactories.EMPTY &&
                         includeExclude == null &&
                         Aggregator.descendsFromBucketAggregator(parent) == false &&
+                        ordinalsValuesSource.supportsGlobalOrdinalsMapping() &&
                         // we use the static COLLECT_SEGMENT_ORDS to allow tests to force specific optimizations
                         (COLLECT_SEGMENT_ORDS!= null ? COLLECT_SEGMENT_ORDS.booleanValue() : ratio <= 0.5 && maxOrd <= 2048)) {
                     /**
                      * We can use the low cardinality execution mode iff this aggregator:
                      *  - has no sub-aggregator AND
                      *  - is not a child of a bucket aggregator AND
+                     *  - has a values source that can map from segment to global ordinals
                      *  - At least we reduce the number of global ordinals look-ups by half (ration <= 0.5) AND
                      *  - the maximum global ordinal is less than 2048 (LOW_CARDINALITY has additional memory usage,
                      *  which directly linked to maxOrd, so we need to limit).
                      */
                     return new GlobalOrdinalsStringTermsAggregator.LowCardinality(name, factories,
-                            (ValuesSource.Bytes.WithOrdinals) valuesSource, order, format, bucketCountThresholds, context, parent, false,
+                            ordinalsValuesSource, order, format, bucketCountThresholds, context, parent, false,
                             subAggCollectMode, showTermDocCountError, pipelineAggregators, metaData);
 
                 }
@@ -301,7 +306,7 @@ Aggregator create(String name,
                          remapGlobalOrds = false;
                     }
                 }
-                return new GlobalOrdinalsStringTermsAggregator(name, factories, (ValuesSource.Bytes.WithOrdinals) valuesSource, order,
+                return new GlobalOrdinalsStringTermsAggregator(name, factories, ordinalsValuesSource, order,
                         format, bucketCountThresholds, filter, context, parent, remapGlobalOrds, subAggCollectMode, showTermDocCountError,
                         pipelineAggregators, metaData);
             }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/UnmappedRareTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/UnmappedRareTerms.java
new file mode 100644
index 0000000000000..eff5441a1d7e7
--- /dev/null
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/UnmappedRareTerms.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.util.SetBackedScalingCuckooFilter;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyList;
+
+/**
+ * Result of the RareTerms aggregation when the field is unmapped.
+ */
+public class UnmappedRareTerms extends InternalRareTerms<UnmappedRareTerms, UnmappedRareTerms.Bucket> {
+    public static final String NAME = "umrareterms";
+
+    protected abstract static class Bucket extends InternalRareTerms.Bucket<Bucket> {
+        private Bucket(long docCount, InternalAggregations aggregations, DocValueFormat formatter) {
+            super(docCount, aggregations, formatter);
+        }
+    }
+
+    UnmappedRareTerms(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        super(name, LongRareTermsAggregator.ORDER, 0, pipelineAggregators, metaData);
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public UnmappedRareTerms(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    protected void writeTermTypeInfoTo(StreamOutput out) throws IOException {
+        // Nothing to write
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public String getType() {
+        return StringTerms.NAME;
+    }
+
+    @Override
+    public UnmappedRareTerms create(List<UnmappedRareTerms.Bucket> buckets) {
+        return new UnmappedRareTerms(name, pipelineAggregators(), metaData);
+    }
+
+    @Override
+    public UnmappedRareTerms.Bucket createBucket(InternalAggregations aggregations, UnmappedRareTerms.Bucket prototype) {
+        throw new UnsupportedOperationException("not supported for UnmappedRareTerms");
+    }
+
+    @Override
+    protected UnmappedRareTerms createWithFilter(String name, List<UnmappedRareTerms.Bucket> buckets, SetBackedScalingCuckooFilter filter) {
+        throw new UnsupportedOperationException("not supported for UnmappedRareTerms");
+    }
+
+    @Override
+    public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+        return new UnmappedRareTerms(name, pipelineAggregators(), metaData);
+    }
+
+    @Override
+    public boolean isMapped() {
+        return false;
+    }
+
+    @Override
+    public final XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        return doXContentCommon(builder, params, Collections.emptyList());
+    }
+
+    @Override
+    public List<UnmappedRareTerms.Bucket> getBuckets() {
+        return emptyList();
+    }
+
+    @Override
+    public UnmappedRareTerms.Bucket getBucketByKey(String term) {
+        return null;
+    }
+
+    @Override
+    protected UnmappedRareTerms.Bucket[] createBucketsArray(int size) {
+        return new UnmappedRareTerms.Bucket[size];
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java
index b3bbe6b911c48..d8e043ee9b5e9 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java
@@ -150,7 +150,11 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         AbstractInternalHDRPercentiles that = (AbstractInternalHDRPercentiles) obj;
         return keyed == that.keyed
                 && Arrays.equals(keys, that.keys)
@@ -158,10 +162,14 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    protected int doHashCode() {
+    public int hashCode() {
         // we cannot use state.hashCode at the moment because of:
         // https://github.com/HdrHistogram/HdrHistogram/issues/81
         // TODO: upgrade the HDRHistogram library
-        return Objects.hash(keyed, Arrays.hashCode(keys), state.getIntegerToDoubleValueConversionRatio(), state.getTotalCount());
+        return Objects.hash(super.hashCode(),
+            keyed,
+            Arrays.hashCode(keys),
+            state.getIntegerToDoubleValueConversionRatio(),
+            state.getTotalCount());
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalTDigestPercentiles.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalTDigestPercentiles.java
index cc63e5f7a4325..ca03e2aa2b1c9 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalTDigestPercentiles.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalTDigestPercentiles.java
@@ -133,7 +133,11 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         AbstractInternalTDigestPercentiles that = (AbstractInternalTDigestPercentiles) obj;
         return keyed == that.keyed
                 && Arrays.equals(keys, that.keys)
@@ -141,7 +145,7 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(keyed, Arrays.hashCode(keys), state);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), keyed, Arrays.hashCode(keys), state);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregationBuilder.java
index 1f57964f667fb..a6a5fb02f8363 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregationBuilder.java
@@ -79,7 +79,7 @@ protected void innerWriteTo(StreamOutput out) {
 
     @Override
     protected AvgAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         return new AvgAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
@@ -88,16 +88,6 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
         return builder;
     }
 
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(Object obj) {
-        return true;
-    }
-
     @Override
     public String getType() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java
index 817e40db26e18..2b1c53535c9ac 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorFactory.java
@@ -33,10 +33,10 @@
 import java.util.List;
 import java.util.Map;
 
-class AvgAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, AvgAggregatorFactory> {
+class AvgAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric> {
 
     AvgAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, SearchContext context,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
+            AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregationBuilder.java
index fa8a1118c88ba..8d927e2fa59eb 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregationBuilder.java
@@ -124,7 +124,7 @@ public Long precisionThreshold() {
 
     @Override
     protected CardinalityAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         return new CardinalityAggregatorFactory(name, config, precisionThreshold, context, parent, subFactoriesBuilder, metaData);
     }
 
@@ -137,12 +137,15 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(precisionThreshold);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), precisionThreshold);
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         CardinalityAggregationBuilder other = (CardinalityAggregationBuilder) obj;
         return Objects.equals(precisionThreshold, other.precisionThreshold);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java
index 413c896fbcb3f..8927697b90f41 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorFactory.java
@@ -32,12 +32,12 @@
 import java.util.List;
 import java.util.Map;
 
-class CardinalityAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource, CardinalityAggregatorFactory> {
+class CardinalityAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource> {
 
     private final Long precisionThreshold;
 
     CardinalityAggregatorFactory(String name, ValuesSourceConfig<ValuesSource> config, Long precisionThreshold,
-            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+            SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.precisionThreshold = precisionThreshold;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java
index 33caa5f840028..af9e8bbca8d56 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregationBuilder.java
@@ -99,7 +99,7 @@ public double sigma() {
 
     @Override
     protected ExtendedStatsAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         return new ExtendedStatsAggregatorFactory(name, config, sigma, context, parent, subFactoriesBuilder, metaData);
     }
 
@@ -110,12 +110,15 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(sigma);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), sigma);
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         ExtendedStatsAggregationBuilder other = (ExtendedStatsAggregationBuilder) obj;
         return Objects.equals(sigma, other.sigma);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorFactory.java
index 890f3199498b4..31e38a3bfd197 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorFactory.java
@@ -33,12 +33,12 @@
 import java.util.List;
 import java.util.Map;
 
-class ExtendedStatsAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, ExtendedStatsAggregatorFactory> {
+class ExtendedStatsAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric> {
 
     private final double sigma;
 
     ExtendedStatsAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double sigma,
-            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+            SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.sigma = sigma;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java
index 9955f62f80ab7..6f6101fc45ee4 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java
@@ -99,7 +99,7 @@ public boolean wrapLongitude() {
 
     @Override
     protected GeoBoundsAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource.GeoPoint> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         return new GeoBoundsAggregatorFactory(name, config, wrapLongitude, context, parent, subFactoriesBuilder, metaData);
     }
 
@@ -110,12 +110,15 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(wrapLongitude);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), wrapLongitude);
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         GeoBoundsAggregationBuilder other = (GeoBoundsAggregationBuilder) obj;
         return Objects.equals(wrapLongitude, other.wrapLongitude);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java
index e6080d16cbfd4..de8936079c236 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java
@@ -32,12 +32,12 @@
 import java.util.List;
 import java.util.Map;
 
-class GeoBoundsAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, GeoBoundsAggregatorFactory> {
+class GeoBoundsAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.GeoPoint> {
 
     private final boolean wrapLongitude;
 
     GeoBoundsAggregatorFactory(String name, ValuesSourceConfig<ValuesSource.GeoPoint> config, boolean wrapLongitude,
-            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+            SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.wrapLongitude = wrapLongitude;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java
index 088483656f8ff..98e8f2e9dbfbe 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java
@@ -79,7 +79,7 @@ protected void innerWriteTo(StreamOutput out) {
 
     @Override
     protected GeoCentroidAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource.GeoPoint> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         return new GeoCentroidAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
@@ -88,16 +88,6 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
         return builder;
     }
 
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(Object obj) {
-        return true;
-    }
-
     @Override
     public String getType() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorFactory.java
index 2bfb31c49930c..b12ce921b7d52 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorFactory.java
@@ -32,10 +32,10 @@
 import java.util.List;
 import java.util.Map;
 
-class GeoCentroidAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, GeoCentroidAggregatorFactory> {
+class GeoCentroidAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.GeoPoint> {
 
     GeoCentroidAggregatorFactory(String name, ValuesSourceConfig<ValuesSource.GeoPoint> config,
-            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+            SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorFactory.java
index 1bb96e17da7a5..cfea50bc31224 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksAggregatorFactory.java
@@ -34,14 +34,14 @@
 import java.util.Map;
 
 class HDRPercentileRanksAggregatorFactory
-        extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, HDRPercentileRanksAggregatorFactory> {
+        extends ValuesSourceAggregatorFactory<ValuesSource.Numeric> {
 
     private final double[] values;
     private final int numberOfSignificantValueDigits;
     private final boolean keyed;
 
     HDRPercentileRanksAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double[] values,
-            int numberOfSignificantValueDigits, boolean keyed, SearchContext context, AggregatorFactory<?> parent,
+            int numberOfSignificantValueDigits, boolean keyed, SearchContext context, AggregatorFactory parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.values = values;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesAggregatorFactory.java
index fe53f32889a7e..584ff30f53d9d 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesAggregatorFactory.java
@@ -33,14 +33,14 @@
 import java.util.List;
 import java.util.Map;
 
-class HDRPercentilesAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, HDRPercentilesAggregatorFactory> {
+class HDRPercentilesAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric> {
 
     private final double[] percents;
     private final int numberOfSignificantValueDigits;
     private final boolean keyed;
 
     HDRPercentilesAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double[] percents,
-            int numberOfSignificantValueDigits, boolean keyed, SearchContext context, AggregatorFactory<?> parent,
+            int numberOfSignificantValueDigits, boolean keyed, SearchContext context, AggregatorFactory parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.percents = percents;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalAvg.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalAvg.java
index 1b30afc087459..089407314de99 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalAvg.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalAvg.java
@@ -118,12 +118,15 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(sum, count, format.getWriteableName());
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), sum, count, format.getWriteableName());
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         InternalAvg other = (InternalAvg) obj;
         return Objects.equals(sum, other.sum) &&
                 Objects.equals(count, other.count) &&
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalCardinality.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalCardinality.java
index 144777379b5ad..bfe82c6bef659 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalCardinality.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalCardinality.java
@@ -30,6 +30,7 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 public final class InternalCardinality extends InternalNumericMetricsAggregation.SingleValue implements Cardinality {
     private final HyperLogLogPlusPlus counts;
@@ -117,12 +118,16 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected int doHashCode() {
-        return counts.hashCode(0);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), counts.hashCode(0));
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalCardinality other = (InternalCardinality) obj;
         return counts.equals(0, other.counts);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStats.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStats.java
index 26a244c8ddfb8..3fe2e75576aa4 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStats.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStats.java
@@ -215,15 +215,18 @@ protected XContentBuilder otherStatsToXContent(XContentBuilder builder, Params p
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(super.doHashCode(), sumOfSqrs, sigma);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), sumOfSqrs, sigma);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalExtendedStats other = (InternalExtendedStats) obj;
-        return super.doEquals(obj) &&
-            Double.compare(sumOfSqrs, other.sumOfSqrs) == 0 &&
+        return Double.compare(sumOfSqrs, other.sumOfSqrs) == 0 &&
             Double.compare(sigma, other.sigma) == 0;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoBounds.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoBounds.java
index 7f259baca693f..4d48e4ab8966b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoBounds.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoBounds.java
@@ -252,7 +252,11 @@ public GeoPoint bottomRight() {
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalGeoBounds other = (InternalGeoBounds) obj;
         return top == other.top &&
             bottom == other.bottom &&
@@ -264,7 +268,7 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(bottom, posLeft, posRight, negLeft, negRight, wrapLongitude);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), bottom, posLeft, posRight, negLeft, negRight, wrapLongitude);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java
index 63dd8b50f71ef..2172d15259b85 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java
@@ -180,15 +180,18 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    public boolean doEquals(Object o) {
-        InternalGeoCentroid that = (InternalGeoCentroid) o;
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+        InternalGeoCentroid that = (InternalGeoCentroid) obj;
         return count == that.count &&
                 Objects.equals(centroid, that.centroid);
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(centroid, count);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), centroid, count);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMax.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMax.java
index 300c82710f6d1..9a8458c85a690 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMax.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMax.java
@@ -90,12 +90,15 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(max);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), max);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         InternalMax other = (InternalMax) obj;
         return Objects.equals(max, other.max);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMedianAbsoluteDeviation.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMedianAbsoluteDeviation.java
index ace9edb13f515..871f387638dc3 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMedianAbsoluteDeviation.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMedianAbsoluteDeviation.java
@@ -106,12 +106,15 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(valuesSketch);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), valuesSketch);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         InternalMedianAbsoluteDeviation other = (InternalMedianAbsoluteDeviation) obj;
         return Objects.equals(valuesSketch, other.valuesSketch);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMin.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMin.java
index 60ed785edfe91..f68d5a46860bc 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMin.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMin.java
@@ -90,12 +90,15 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(min);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), min);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         InternalMin other = (InternalMin) obj;
         return Objects.equals(min, other.min);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java
index b3439671580e7..84f405c78cb11 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java
@@ -107,19 +107,16 @@ protected InternalNumericMetricsAggregation(StreamInput in) throws IOException {
 
     @Override
     public int hashCode() {
-        return Objects.hash(format, super.hashCode());
+        return Objects.hash(super.hashCode(), format);
     }
 
     @Override
     public boolean equals(Object obj) {
-        if (obj == null) {
-            return false;
-        }
-        if (obj.getClass() != getClass()) {
-            return false;
-        }
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalNumericMetricsAggregation other = (InternalNumericMetricsAggregation) obj;
-        return super.equals(obj) &&
-                Objects.equals(format, other.format);
+        return Objects.equals(format, other.format);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetric.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetric.java
index a0bb5e74fd5cb..ce051a1691b55 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetric.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalScriptedMetric.java
@@ -135,15 +135,19 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalScriptedMetric other = (InternalScriptedMetric) obj;
         return Objects.equals(reduceScript, other.reduceScript) &&
                 Objects.equals(aggregation, other.aggregation);
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(reduceScript, aggregation);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), reduceScript, aggregation);
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java
index a05d6db7024e6..3c0aa8c7313bc 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java
@@ -212,12 +212,16 @@ protected XContentBuilder otherStatsToXContent(XContentBuilder builder, Params p
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(count, min, max, sum);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), count, min, max, sum);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalStats other = (InternalStats) obj;
         return count == other.count &&
             Double.compare(min, other.min) == 0 &&
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalSum.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalSum.java
index c3bb7173b3f07..2e1c9635aa896 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalSum.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalSum.java
@@ -100,12 +100,16 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hashCode(sum);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), sum);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalSum that = (InternalSum) obj;
         return Objects.equals(sum, that.sum);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java
index 4a266ee0703bd..348e98302d2bc 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTopHits.java
@@ -180,7 +180,11 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
 
     // Equals and hashcode implemented for testing round trips
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalTopHits other = (InternalTopHits) obj;
         if (from != other.from) return false;
         if (size != other.size) return false;
@@ -207,9 +211,10 @@ protected boolean doEquals(Object obj) {
     }
 
     @Override
-    protected int doHashCode() {
-        int hashCode = from;
-        hashCode = 31 * hashCode + size;
+    public int hashCode() {
+        int hashCode = super.hashCode();
+        hashCode = 31 * hashCode + Integer.hashCode(from);
+        hashCode = 31 * hashCode + Integer.hashCode(size);
         hashCode = 31 * hashCode + Long.hashCode(topDocs.topDocs.totalHits.value);
         hashCode = 31 * hashCode + topDocs.topDocs.totalHits.relation.hashCode();
         for (int d = 0; d < topDocs.topDocs.scoreDocs.length; d++) {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalValueCount.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalValueCount.java
index 36f2749c791f4..32ee8bd36d120 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalValueCount.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalValueCount.java
@@ -90,12 +90,16 @@ public String toString() {
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(value);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), value);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalValueCount that = (InternalValueCount) obj;
         return Objects.equals(this.value, that.value);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalWeightedAvg.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalWeightedAvg.java
index e06ffbc7b4a5a..6165803299296 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalWeightedAvg.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalWeightedAvg.java
@@ -129,12 +129,15 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(sum, weight, format.getWriteableName());
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), sum, weight, format.getWriteableName());
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         InternalWeightedAvg other = (InternalWeightedAvg) obj;
         return Objects.equals(sum, other.sum) &&
                 Objects.equals(weight, other.weight) &&
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregationBuilder.java
index 0c3229f08fd51..b5d788cfc853f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregationBuilder.java
@@ -79,7 +79,7 @@ protected void innerWriteTo(StreamOutput out) {
 
     @Override
     protected MaxAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         return new MaxAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
@@ -88,16 +88,6 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
         return builder;
     }
 
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(Object obj) {
-        return true;
-    }
-
     @Override
     public String getType() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorFactory.java
index d64987d9cde4e..2ae762396818c 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorFactory.java
@@ -33,10 +33,10 @@
 import java.util.List;
 import java.util.Map;
 
-class MaxAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, MaxAggregatorFactory> {
+class MaxAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric> {
 
     MaxAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, SearchContext context,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
+            AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregationBuilder.java
index b5b119cc38f6a..6218fd3e52b0f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregationBuilder.java
@@ -107,9 +107,9 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource.Numeric, ?> innerBuild(SearchContext context,
+    protected ValuesSourceAggregatorFactory<ValuesSource.Numeric> innerBuild(SearchContext context,
                                                                                 ValuesSourceConfig<ValuesSource.Numeric> config,
-                                                                                AggregatorFactory<?> parent,
+                                                                                AggregatorFactory parent,
                                                                                 AggregatorFactories.Builder subFactoriesBuilder)
                                                                                 throws IOException {
 
@@ -123,12 +123,15 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(compression);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), compression);
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         MedianAbsoluteDeviationAggregationBuilder other = (MedianAbsoluteDeviationAggregationBuilder) obj;
         return Objects.equals(compression, other.compression);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorFactory.java
index 3e621e396de06..037e743ad32db 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregatorFactory.java
@@ -32,15 +32,14 @@
 import java.util.List;
 import java.util.Map;
 
-public class MedianAbsoluteDeviationAggregatorFactory extends
-        ValuesSourceAggregatorFactory<ValuesSource.Numeric, MedianAbsoluteDeviationAggregatorFactory> {
+public class MedianAbsoluteDeviationAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric> {
 
     private final double compression;
 
     MedianAbsoluteDeviationAggregatorFactory(String name,
                                                     ValuesSourceConfig<ValuesSource.Numeric> config,
                                                     SearchContext context,
-                                                    AggregatorFactory<?> parent,
+                                                    AggregatorFactory parent,
                                                     AggregatorFactories.Builder subFactoriesBuilder,
                                                     Map<String, Object> metaData,
                                                     double compression) throws IOException {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregationBuilder.java
index 2d23539189dc8..f195e10c65eb8 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregationBuilder.java
@@ -79,7 +79,7 @@ protected void innerWriteTo(StreamOutput out) {
 
     @Override
     protected MinAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         return new MinAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
@@ -88,16 +88,6 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
         return builder;
     }
 
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(Object obj) {
-        return true;
-    }
-
     @Override
     public String getType() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorFactory.java
index 240cf2ba715fa..23e55627babfd 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorFactory.java
@@ -33,10 +33,10 @@
 import java.util.List;
 import java.util.Map;
 
-class MinAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, MinAggregatorFactory> {
+class MinAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric> {
 
     MinAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, SearchContext context,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
+            AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregationBuilder.java
index 3bf70d20989d3..d4a600b11b5fc 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksAggregationBuilder.java
@@ -240,8 +240,8 @@ public PercentilesMethod method() {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+    protected ValuesSourceAggregatorFactory<Numeric> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         switch (method) {
         case TDIGEST:
             return new TDigestPercentileRanksAggregatorFactory(name, config, values, compression, keyed, context, parent,
@@ -269,37 +269,40 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         PercentileRanksAggregationBuilder other = (PercentileRanksAggregationBuilder) obj;
-        if (!Objects.equals(method, other.method)) {
+        if (Objects.equals(method, other.method) == false) {
             return false;
         }
         boolean equalSettings = false;
         switch (method) {
-        case HDR:
-            equalSettings = Objects.equals(numberOfSignificantValueDigits, other.numberOfSignificantValueDigits);
-            break;
-        case TDIGEST:
-            equalSettings = Objects.equals(compression, other.compression);
-            break;
-        default:
-            throw new IllegalStateException("Illegal method [" + method + "]");
+            case HDR:
+                equalSettings = Objects.equals(numberOfSignificantValueDigits, other.numberOfSignificantValueDigits);
+                break;
+            case TDIGEST:
+                equalSettings = Objects.equals(compression, other.compression);
+                break;
+            default:
+                throw new IllegalStateException("Illegal method [" + method + "]");
         }
         return equalSettings
-                && Objects.deepEquals(values, other.values)
-                && Objects.equals(keyed, other.keyed)
-                && Objects.equals(method, other.method);
+            && Objects.deepEquals(values, other.values)
+            && Objects.equals(keyed, other.keyed)
+            && Objects.equals(method, other.method);
     }
 
     @Override
-    protected int innerHashCode() {
+    public int hashCode() {
         switch (method) {
-        case HDR:
-            return Objects.hash(Arrays.hashCode(values), keyed, numberOfSignificantValueDigits, method);
-        case TDIGEST:
-            return Objects.hash(Arrays.hashCode(values), keyed, compression, method);
-        default:
-            throw new IllegalStateException("Illegal method [" + method + "]");
+            case HDR:
+                return Objects.hash(super.hashCode(), Arrays.hashCode(values), keyed, numberOfSignificantValueDigits, method);
+            case TDIGEST:
+                return Objects.hash(super.hashCode(), Arrays.hashCode(values), keyed, compression, method);
+            default:
+                throw new IllegalStateException("Illegal method [" + method + "]");
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregationBuilder.java
index 3a6f5f89622f0..302e05a600a67 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/PercentilesAggregationBuilder.java
@@ -263,8 +263,8 @@ public PercentilesMethod method() {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+    protected ValuesSourceAggregatorFactory<Numeric> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         switch (method) {
         case TDIGEST:
             return new TDigestPercentilesAggregatorFactory(name, config, percents, compression, keyed, context, parent,
@@ -292,35 +292,39 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
     }
 
     @Override
-    protected boolean innerEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         PercentilesAggregationBuilder other = (PercentilesAggregationBuilder) obj;
-        if (!Objects.equals(method, other.method)) {
+        if (Objects.equals(method, other.method) == false) {
             return false;
         }
         boolean equalSettings = false;
         switch (method) {
-        case HDR:
-            equalSettings = Objects.equals(numberOfSignificantValueDigits, other.numberOfSignificantValueDigits);
-            break;
-        case TDIGEST:
-            equalSettings = Objects.equals(compression, other.compression);
-            break;
-        default:
-            throw new IllegalStateException("Illegal method [" + method.toString() + "]");
+            case HDR:
+                equalSettings = Objects.equals(numberOfSignificantValueDigits, other.numberOfSignificantValueDigits);
+                break;
+            case TDIGEST:
+                equalSettings = Objects.equals(compression, other.compression);
+                break;
+            default:
+                throw new IllegalStateException("Illegal method [" + method.toString() + "]");
         }
         return equalSettings
-                && Objects.deepEquals(percents, other.percents)
-                && Objects.equals(keyed, other.keyed)
-                && Objects.equals(method, other.method);
+            && Objects.deepEquals(percents, other.percents)
+            && Objects.equals(keyed, other.keyed)
+            && Objects.equals(method, other.method);
     }
 
     @Override
-    protected int innerHashCode() {
+    public int hashCode() {
         switch (method) {
         case HDR:
-            return Objects.hash(Arrays.hashCode(percents), keyed, numberOfSignificantValueDigits, method);
+            return Objects.hash(super.hashCode(), Arrays.hashCode(percents), keyed, numberOfSignificantValueDigits, method);
         case TDIGEST:
-            return Objects.hash(Arrays.hashCode(percents), keyed, compression, method);
+            return Objects.hash(super.hashCode(), Arrays.hashCode(percents), keyed, compression, method);
         default:
             throw new IllegalStateException("Illegal method [" + method.toString() + "]");
         }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregationBuilder.java
index 25fcebc6aa52d..0944ade85f174 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregationBuilder.java
@@ -193,7 +193,7 @@ public Map<String, Object> params() {
     }
 
     @Override
-    protected ScriptedMetricAggregatorFactory doBuild(SearchContext context, AggregatorFactory<?> parent,
+    protected ScriptedMetricAggregatorFactory doBuild(SearchContext context, AggregatorFactory parent,
             Builder subfactoriesBuilder) throws IOException {
 
         if (combineScript == null) {
@@ -325,18 +325,21 @@ public String getType() {
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(initScript, mapScript, combineScript, reduceScript, params);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), initScript, mapScript, combineScript, reduceScript, params);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         ScriptedMetricAggregationBuilder other = (ScriptedMetricAggregationBuilder) obj;
         return Objects.equals(initScript, other.initScript)
-                && Objects.equals(mapScript, other.mapScript)
-                && Objects.equals(combineScript, other.combineScript)
-                && Objects.equals(reduceScript, other.reduceScript)
-                && Objects.equals(params, other.params);
+            && Objects.equals(mapScript, other.mapScript)
+            && Objects.equals(combineScript, other.combineScript)
+            && Objects.equals(reduceScript, other.reduceScript)
+            && Objects.equals(params, other.params);
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorFactory.java
index 01084ee0b7f8b..5c0ceeb1043d7 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorFactory.java
@@ -36,7 +36,7 @@
 import java.util.List;
 import java.util.Map;
 
-class ScriptedMetricAggregatorFactory extends AggregatorFactory<ScriptedMetricAggregatorFactory> {
+class ScriptedMetricAggregatorFactory extends AggregatorFactory {
 
     private final ScriptedMetricAggContexts.MapScript.Factory mapScript;
     private final Map<String, Object> mapScriptParams;
@@ -53,7 +53,7 @@ class ScriptedMetricAggregatorFactory extends AggregatorFactory<ScriptedMetricAg
                                     ScriptedMetricAggContexts.InitScript.Factory initScript, Map<String, Object> initScriptParams,
                                     ScriptedMetricAggContexts.CombineScript.Factory combineScript,
                                     Map<String, Object> combineScriptParams, Script reduceScript, Map<String, Object> aggParams,
-                                    SearchLookup lookup, SearchContext context, AggregatorFactory<?> parent,
+                                    SearchLookup lookup, SearchContext context, AggregatorFactory parent,
                                     AggregatorFactories.Builder subFactories, Map<String, Object> metaData) throws IOException {
         super(name, context, parent, subFactories, metaData);
         this.mapScript = mapScript;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregationBuilder.java
index d96bbba447580..34e5ab2864106 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregationBuilder.java
@@ -80,7 +80,7 @@ protected void innerWriteTo(StreamOutput out) {
 
     @Override
     protected StatsAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         return new StatsAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
@@ -89,16 +89,6 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
         return builder;
     }
 
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(Object obj) {
-        return true;
-    }
-
     @Override
     public String getType() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorFactory.java
index 82dce359037c2..bc0e8a44e77fb 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorFactory.java
@@ -33,10 +33,10 @@
 import java.util.List;
 import java.util.Map;
 
-class StatsAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, StatsAggregatorFactory> {
+class StatsAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric> {
 
     StatsAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, SearchContext context,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
+            AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregationBuilder.java
index 8035a3ad671f7..9f320b098108e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregationBuilder.java
@@ -79,7 +79,7 @@ protected void innerWriteTo(StreamOutput out) {
 
     @Override
     protected SumAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         return new SumAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
@@ -88,16 +88,6 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
         return builder;
     }
 
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(Object obj) {
-        return true;
-    }
-
     @Override
     public String getType() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorFactory.java
index d8fa88541cb9b..a59e6dbf1e7e6 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregatorFactory.java
@@ -33,10 +33,10 @@
 import java.util.List;
 import java.util.Map;
 
-class SumAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, SumAggregatorFactory> {
+class SumAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric> {
 
     SumAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, SearchContext context,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
+            AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorFactory.java
index 10913bf59d14f..e4b3a722c7cb7 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksAggregatorFactory.java
@@ -34,14 +34,14 @@
 import java.util.Map;
 
 class TDigestPercentileRanksAggregatorFactory
-        extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, TDigestPercentileRanksAggregatorFactory> {
+        extends ValuesSourceAggregatorFactory<ValuesSource.Numeric> {
 
     private final double[] percents;
     private final double compression;
     private final boolean keyed;
 
     TDigestPercentileRanksAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double[] percents,
-            double compression, boolean keyed, SearchContext context, AggregatorFactory<?> parent,
+            double compression, boolean keyed, SearchContext context, AggregatorFactory parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.percents = percents;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesAggregatorFactory.java
index 0c1396196fb62..eed8df5011660 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesAggregatorFactory.java
@@ -34,14 +34,14 @@
 import java.util.Map;
 
 class TDigestPercentilesAggregatorFactory
-        extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, TDigestPercentilesAggregatorFactory> {
+        extends ValuesSourceAggregatorFactory<ValuesSource.Numeric> {
 
     private final double[] percents;
     private final double compression;
     private final boolean keyed;
 
     TDigestPercentilesAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double[] percents,
-            double compression, boolean keyed, SearchContext context, AggregatorFactory<?> parent,
+            double compression, boolean keyed, SearchContext context, AggregatorFactory parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.percents = percents;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java
index 019fec82d0df4..87aa56fc15510 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java
@@ -572,7 +572,7 @@ public TopHitsAggregationBuilder subAggregations(Builder subFactories) {
     }
 
     @Override
-    protected TopHitsAggregatorFactory doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subfactoriesBuilder)
+    protected TopHitsAggregatorFactory doBuild(SearchContext context, AggregatorFactory parent, Builder subfactoriesBuilder)
             throws IOException {
         long innerResultWindow = from() + size();
         int maxInnerResultWindow = context.mapperService().getIndexSettings().getMaxInnerResultWindow();
@@ -770,26 +770,31 @@ public static TopHitsAggregationBuilder parse(String aggregationName, XContentPa
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(explain, fetchSourceContext, docValueFields, storedFieldsContext, from, highlightBuilder,
-            scriptFields, size, sorts, trackScores, version, seqNoAndPrimaryTerm);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), explain, fetchSourceContext, docValueFields,
+            storedFieldsContext, from, highlightBuilder,
+            scriptFields, size, sorts, trackScores, version,
+            seqNoAndPrimaryTerm);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         TopHitsAggregationBuilder other = (TopHitsAggregationBuilder) obj;
         return Objects.equals(explain, other.explain)
-                && Objects.equals(fetchSourceContext, other.fetchSourceContext)
-                && Objects.equals(docValueFields, other.docValueFields)
-                && Objects.equals(storedFieldsContext, other.storedFieldsContext)
-                && Objects.equals(from, other.from)
-                && Objects.equals(highlightBuilder, other.highlightBuilder)
-                && Objects.equals(scriptFields, other.scriptFields)
-                && Objects.equals(size, other.size)
-                && Objects.equals(sorts, other.sorts)
-                && Objects.equals(trackScores, other.trackScores)
-                && Objects.equals(version, other.version)
-                && Objects.equals(seqNoAndPrimaryTerm, other.seqNoAndPrimaryTerm);
+            && Objects.equals(fetchSourceContext, other.fetchSourceContext)
+            && Objects.equals(docValueFields, other.docValueFields)
+            && Objects.equals(storedFieldsContext, other.storedFieldsContext)
+            && Objects.equals(from, other.from)
+            && Objects.equals(highlightBuilder, other.highlightBuilder)
+            && Objects.equals(scriptFields, other.scriptFields)
+            && Objects.equals(size, other.size)
+            && Objects.equals(sorts, other.sorts)
+            && Objects.equals(trackScores, other.trackScores)
+            && Objects.equals(version, other.version)
+            && Objects.equals(seqNoAndPrimaryTerm, other.seqNoAndPrimaryTerm);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java
index 7edaccb66d4bd..72d8fa254c22f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java
@@ -38,7 +38,7 @@
 import java.util.Map;
 import java.util.Optional;
 
-class TopHitsAggregatorFactory extends AggregatorFactory<TopHitsAggregatorFactory> {
+class TopHitsAggregatorFactory extends AggregatorFactory {
 
     private final int from;
     private final int size;
@@ -56,7 +56,7 @@ class TopHitsAggregatorFactory extends AggregatorFactory<TopHitsAggregatorFactor
     TopHitsAggregatorFactory(String name, int from, int size, boolean explain, boolean version, boolean seqNoAndPrimaryTerm,
             boolean trackScores, Optional<SortAndFormats> sort, HighlightBuilder highlightBuilder, StoredFieldsContext storedFieldsContext,
             List<FieldAndFormat> docValueFields, List<ScriptFieldsContext.ScriptField> scriptFields, FetchSourceContext fetchSourceContext,
-            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactories, Map<String, Object> metaData)
+            SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactories, Map<String, Object> metaData)
             throws IOException {
         super(name, context, parent, subFactories, metaData);
         this.from = from;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregationBuilder.java
index 70243cb8bc47a..ccf8ef8ba3dca 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregationBuilder.java
@@ -84,7 +84,7 @@ protected boolean serializeTargetValueType() {
 
     @Override
     protected ValueCountAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource> config,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
+            AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
         return new ValueCountAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
@@ -93,16 +93,6 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
         return builder;
     }
 
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(Object obj) {
-        return true;
-    }
-
     @Override
     public String getType() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorFactory.java
index 26f1760940d20..b149dc9a7307c 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorFactory.java
@@ -32,10 +32,10 @@
 import java.util.List;
 import java.util.Map;
 
-class ValueCountAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource, ValueCountAggregatorFactory> {
+class ValueCountAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource> {
 
     ValueCountAggregatorFactory(String name, ValuesSourceConfig<ValuesSource> config, SearchContext context,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
+            AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java
index c3f67fb905254..4cc2ff97c4ce9 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregationBuilder.java
@@ -98,10 +98,10 @@ protected void innerWriteTo(StreamOutput out) {
     }
 
     @Override
-    protected MultiValuesSourceAggregatorFactory<Numeric, ?> innerBuild(SearchContext context,
+    protected MultiValuesSourceAggregatorFactory<Numeric> innerBuild(SearchContext context,
                                                                     Map<String, ValuesSourceConfig<Numeric>> configs,
                                                                     DocValueFormat format,
-                                                                    AggregatorFactory<?> parent,
+                                                                    AggregatorFactory parent,
                                                                     Builder subFactoriesBuilder) throws IOException {
         return new WeightedAvgAggregatorFactory(name, configs, format, context, parent, subFactoriesBuilder, metaData);
     }
@@ -111,16 +111,6 @@ public XContentBuilder doXContentBody(XContentBuilder builder, ToXContent.Params
         return builder;
     }
 
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(Object obj) {
-        return true;
-    }
-
     @Override
     public String getType() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorFactory.java
index afdb727c512b0..0e57e000cf116 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregatorFactory.java
@@ -34,10 +34,10 @@
 import java.util.List;
 import java.util.Map;
 
-class WeightedAvgAggregatorFactory extends MultiValuesSourceAggregatorFactory<Numeric, WeightedAvgAggregatorFactory> {
+class WeightedAvgAggregatorFactory extends MultiValuesSourceAggregatorFactory<Numeric> {
 
     WeightedAvgAggregatorFactory(String name,  Map<String, ValuesSourceConfig<Numeric>> configs,
-                                 DocValueFormat format, SearchContext context, AggregatorFactory<?> parent,
+                                 DocValueFormat format, SearchContext context, AggregatorFactory parent,
                                  AggregatorFactories.Builder subFactoriesBuilder,
                                  Map<String, Object> metaData) throws IOException {
         super(name, configs, format, context, parent, subFactoriesBuilder, metaData);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
index 8a278ead95a2b..fa2f8c8090bcd 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
@@ -84,7 +84,7 @@ public String type() {
      * configured)
      */
     @Override
-    public final void validate(AggregatorFactory<?> parent, Collection<AggregationBuilder> factories,
+    public final void validate(AggregatorFactory parent, Collection<AggregationBuilder> factories,
             Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
         doValidate(parent, factories, pipelineAggregatorFactories);
     }
@@ -102,14 +102,14 @@ public final PipelineAggregator create() {
         return aggregator;
     }
 
-    public void doValidate(AggregatorFactory<?> parent, Collection<AggregationBuilder> factories,
+    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> factories,
             Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
     }
     
     /**
      * Validates pipeline aggregations that need sequentially ordered data.
      */
-    public static void validateSequentiallyOrderedParentAggs(AggregatorFactory<?> parent, String type, String name) {
+    public static void validateSequentiallyOrderedParentAggs(AggregatorFactory parent, String type, String name) {
         if ((parent instanceof HistogramAggregatorFactory || parent instanceof DateHistogramAggregatorFactory
                 || parent instanceof AutoDateHistogramAggregatorFactory) == false) {
             throw new IllegalStateException(
@@ -171,32 +171,20 @@ protected boolean overrideBucketsPath() {
 
     @Override
     public int hashCode() {
-        return Objects.hash(Arrays.hashCode(bucketsPaths), metaData, name, type, doHashCode());
+        return Objects.hash(Arrays.hashCode(bucketsPaths), metaData, name, type);
     }
 
-    protected abstract int doHashCode();
-
     @Override
     public boolean equals(Object obj) {
-        if (obj == null)
-            return false;
-        if (getClass() != obj.getClass())
-            return false;
-        @SuppressWarnings("unchecked")
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
         AbstractPipelineAggregationBuilder<PAB> other = (AbstractPipelineAggregationBuilder<PAB>) obj;
-        if (!Objects.equals(name, other.name))
-            return false;
-        if (!Objects.equals(type, other.type))
-            return false;
-        if (!Objects.deepEquals(bucketsPaths, other.bucketsPaths))
-            return false;
-        if (!Objects.equals(metaData, other.metaData))
-            return false;
-        return doEquals(obj);
+        return Objects.equals(type, other.type)
+            && Objects.equals(name, other.name)
+            && Objects.equals(metaData, other.metaData)
+            && Objects.deepEquals(bucketsPaths, other.bucketsPaths);
     }
 
-    protected abstract boolean doEquals(Object obj);
-
     @Override
     public String getType() {
         return type;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketPipelineAggregationBuilder.java
index 1495c691c83d7..005c39416ca3b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketPipelineAggregationBuilder.java
@@ -63,16 +63,6 @@ protected AvgBucketPipelineAggregationBuilder buildFactory(String pipelineAggreg
         }
     };
 
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(BucketMetricsPipelineAggregationBuilder<AvgBucketPipelineAggregationBuilder> other) {
-        return true;
-    }
-
     @Override
     public String getWriteableName() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java
index eddc48c6fdcba..0c4aa6f1359bd 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java
@@ -107,7 +107,7 @@ public GapPolicy gapPolicy() {
     protected abstract PipelineAggregator createInternal(Map<String, Object> metaData);
 
     @Override
-    public void doValidate(AggregatorFactory<?> parent, Collection<AggregationBuilder> aggBuilders,
+    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggBuilders,
             Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
@@ -147,21 +147,19 @@ protected final XContentBuilder internalXContent(XContentBuilder builder, Params
     protected abstract XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException;
 
     @Override
-    protected final int doHashCode() {
-        return Objects.hash(format, gapPolicy, innerHashCode());
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), format, gapPolicy);
     }
 
-    protected abstract int innerHashCode();
-
     @Override
-    protected final boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         @SuppressWarnings("unchecked")
         BucketMetricsPipelineAggregationBuilder<AF> other = (BucketMetricsPipelineAggregationBuilder<AF>) obj;
         return Objects.equals(format, other.format)
-                && Objects.equals(gapPolicy, other.gapPolicy)
-                && innerEquals(other);
+            && Objects.equals(gapPolicy, other.gapPolicy);
     }
 
-    protected abstract boolean innerEquals(BucketMetricsPipelineAggregationBuilder<AF> other);
-
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java
index c8ea3553752f8..a0dabefed323f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java
@@ -240,15 +240,20 @@ protected boolean overrideBucketsPath() {
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(bucketsPathsMap, script, format, gapPolicy);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), bucketsPathsMap, script, format, gapPolicy);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         BucketScriptPipelineAggregationBuilder other = (BucketScriptPipelineAggregationBuilder) obj;
-        return Objects.equals(bucketsPathsMap, other.bucketsPathsMap) && Objects.equals(script, other.script)
-                && Objects.equals(format, other.format) && Objects.equals(gapPolicy, other.gapPolicy);
+        return Objects.equals(bucketsPathsMap, other.bucketsPathsMap)
+            && Objects.equals(script, other.script)
+            && Objects.equals(format, other.format)
+            && Objects.equals(gapPolicy, other.gapPolicy);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java
index a6627be1cfe6f..6bb83c60dea83 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java
@@ -199,15 +199,20 @@ protected boolean overrideBucketsPath() {
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(bucketsPathsMap, script, gapPolicy);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), bucketsPathsMap, script, gapPolicy);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         BucketSelectorPipelineAggregationBuilder other = (BucketSelectorPipelineAggregationBuilder) obj;
-        return Objects.equals(bucketsPathsMap, other.bucketsPathsMap) && Objects.equals(script, other.script)
-                && Objects.equals(gapPolicy, other.gapPolicy);
+        return Objects.equals(bucketsPathsMap, other.bucketsPathsMap)
+            && Objects.equals(script, other.script)
+            && Objects.equals(gapPolicy, other.gapPolicy);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java
index 4dcd42934fc96..1e80c6f78c3c5 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java
@@ -144,7 +144,7 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory<?> parent, Collection<AggregationBuilder> aggFactories,
+    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
                            Collection<PipelineAggregationBuilder> pipelineAggregatoractories) {
         if (sorts.isEmpty() && size == null && from == 0) {
             throw new IllegalStateException("[" + name + "] is configured to perform nothing. Please set either of "
@@ -174,12 +174,15 @@ protected boolean overrideBucketsPath() {
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(sorts, from, size, gapPolicy);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), sorts, from, size, gapPolicy);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         BucketSortPipelineAggregationBuilder other = (BucketSortPipelineAggregationBuilder) obj;
         return Objects.equals(sorts, other.sorts)
                 && Objects.equals(from, other.from)
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java
index a8b51d9c1a5fe..fc9b8caf72769 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java
@@ -93,7 +93,7 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory<?> parent, Collection<AggregationBuilder> aggFactories,
+    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
             Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(BUCKETS_PATH.getPreferredName()
@@ -162,12 +162,15 @@ public static CumulativeSumPipelineAggregationBuilder parse(String pipelineAggre
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(format);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), format);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         CumulativeSumPipelineAggregationBuilder other = (CumulativeSumPipelineAggregationBuilder) obj;
         return Objects.equals(format, other.format);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java
index 25f30bc8343cb..4aa4155ac063f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java
@@ -152,7 +152,7 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory<?> parent, Collection<AggregationBuilder> aggFactories,
+    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
             Collection<PipelineAggregationBuilder> pipelineAggregatoractories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
@@ -238,23 +238,19 @@ public static DerivativePipelineAggregationBuilder parse(String pipelineAggregat
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         DerivativePipelineAggregationBuilder other = (DerivativePipelineAggregationBuilder) obj;
-        if (!Objects.equals(format, other.format)) {
-            return false;
-        }
-        if (!Objects.equals(gapPolicy, other.gapPolicy)) {
-            return false;
-        }
-        if (!Objects.equals(units, other.units)) {
-            return false;
-        }
-        return true;
+        return Objects.equals(format, other.format) &&
+            gapPolicy == other.gapPolicy &&
+            Objects.equals(units, other.units);
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(format, gapPolicy, units);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), format, gapPolicy, units);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java
index 3d16cf91ee065..89816a1fb22e8 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java
@@ -80,7 +80,7 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory<?> parent, Collection<AggregationBuilder> aggBuilders,
+    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggBuilders,
             Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
         super.doValidate(parent, aggBuilders, pipelineAggregatorFactories);
 
@@ -97,12 +97,15 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
     }
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(sigma);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), sigma);
     }
 
     @Override
-    protected boolean innerEquals(BucketMetricsPipelineAggregationBuilder<ExtendedStatsBucketPipelineAggregationBuilder> obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         ExtendedStatsBucketPipelineAggregationBuilder other = (ExtendedStatsBucketPipelineAggregationBuilder) obj;
         return Objects.equals(sigma, other.sigma);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalBucketMetricValue.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalBucketMetricValue.java
index 8a3de634dbfd4..1acdb54080693 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalBucketMetricValue.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalBucketMetricValue.java
@@ -118,12 +118,15 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(value, Arrays.hashCode(keys));
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), value, Arrays.hashCode(keys));
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         InternalBucketMetricValue other = (InternalBucketMetricValue) obj;
         return Objects.equals(value, other.value)
                 && Arrays.equals(keys, other.keys);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalDerivative.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalDerivative.java
index b8d5245846aeb..6597689bb6d32 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalDerivative.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalDerivative.java
@@ -98,12 +98,15 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(normalizationFactor, value);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), normalizationFactor, value);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         InternalDerivative other = (InternalDerivative) obj;
         return Objects.equals(value, other.value)
                 && Objects.equals(normalizationFactor, other.normalizationFactor);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalPercentilesBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalPercentilesBucket.java
index d9a3908d22136..77493f66d643e 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalPercentilesBucket.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalPercentilesBucket.java
@@ -156,14 +156,18 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
         InternalPercentilesBucket that = (InternalPercentilesBucket) obj;
         return Arrays.equals(percents, that.percents) && Arrays.equals(percentiles, that.percentiles);
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(Arrays.hashCode(percents), Arrays.hashCode(percentiles));
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), Arrays.hashCode(percents), Arrays.hashCode(percentiles));
     }
 
     public static class Iter implements Iterator<Percentile> {
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValue.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValue.java
index 2aac262a0076b..4f7b51b6e3b38 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValue.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValue.java
@@ -91,12 +91,15 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(value);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), value);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         InternalSimpleValue other = (InternalSimpleValue) obj;
         return Objects.equals(value, other.value);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java
index b335c15865d70..b37c41d85febc 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java
@@ -63,16 +63,6 @@ protected MaxBucketPipelineAggregationBuilder buildFactory(String pipelineAggreg
         }
     };
 
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(BucketMetricsPipelineAggregationBuilder<MaxBucketPipelineAggregationBuilder> other) {
-        return true;
-    }
-
     @Override
     public String getWriteableName() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java
index 405285993c0ab..766a9c3849070 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java
@@ -63,16 +63,6 @@ protected MinBucketPipelineAggregationBuilder buildFactory(String pipelineAggreg
         }
     };
 
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(BucketMetricsPipelineAggregationBuilder<MinBucketPipelineAggregationBuilder> other) {
-        return true;
-    }
-
     @Override
     public String getWriteableName() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java
index 3f589d8a08949..44f26c3c32bf8 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java
@@ -169,7 +169,7 @@ public void setWindow(int window) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory<?> parent, Collection<AggregationBuilder> aggFactories,
+    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
                            Collection<PipelineAggregationBuilder> pipelineAggregatoractories) {
         if (window <= 0) {
             throw new IllegalArgumentException("[" + WINDOW.getPreferredName() + "] must be a positive, non-zero integer.");
@@ -224,12 +224,15 @@ protected boolean overrideBucketsPath() {
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(bucketsPathString, script, format, gapPolicy, window);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), bucketsPathString, script, format, gapPolicy, window);
     }
 
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         MovFnPipelineAggregationBuilder other = (MovFnPipelineAggregationBuilder) obj;
         return Objects.equals(bucketsPathString, other.bucketsPathString)
             && Objects.equals(script, other.script)
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java
index 18105c4518564..06a940af89c77 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java
@@ -109,7 +109,7 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
 
     @Override
-    public void doValidate(AggregatorFactory<?> parent, Collection<AggregationBuilder> aggFactories,
+    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
             Collection<PipelineAggregationBuilder> pipelineAggregatorFactories) {
         super.doValidate(parent, aggFactories, pipelineAggregatorFactories);
 
@@ -172,14 +172,18 @@ else if (KEYED_FIELD.match(field, parser.getDeprecationHandler()) && token == XC
     };
 
     @Override
-    protected int innerHashCode() {
-        return Objects.hash(Arrays.hashCode(percents), keyed);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), Arrays.hashCode(percents), keyed);
     }
 
     @Override
-    protected boolean innerEquals(BucketMetricsPipelineAggregationBuilder<PercentilesBucketPipelineAggregationBuilder> obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         PercentilesBucketPipelineAggregationBuilder other = (PercentilesBucketPipelineAggregationBuilder) obj;
-        return Objects.deepEquals(percents, other.percents) && Objects.equals(keyed, other.keyed);
+        return Objects.deepEquals(percents, other.percents)
+            && Objects.equals(keyed, other.keyed);
     }
 
     @Override
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java
index 019026740f813..3b02edf51579a 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java
@@ -140,7 +140,7 @@ protected PipelineAggregator createInternal(Map<String, Object> metaData) {
     }
     
     @Override
-    public void doValidate(AggregatorFactory<?> parent, Collection<AggregationBuilder> aggFactories,
+    public void doValidate(AggregatorFactory parent, Collection<AggregationBuilder> aggFactories,
             Collection<PipelineAggregationBuilder> pipelineAggregatoractories) {        
         validateSequentiallyOrderedParentAggs(parent, NAME, name);
     }
@@ -228,11 +228,15 @@ public static SerialDiffPipelineAggregationBuilder parse(String reducerName, XCo
     }
 
     @Override
-    protected int doHashCode() {
-        return Objects.hash(format, gapPolicy, lag);
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), format, gapPolicy, lag);
     }
+
     @Override
-    protected boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         SerialDiffPipelineAggregationBuilder other = (SerialDiffPipelineAggregationBuilder) obj;
         return Objects.equals(format, other.format)
                 && Objects.equals(gapPolicy, other.gapPolicy)
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SiblingPipelineAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SiblingPipelineAggregator.java
index c04bd9fa0dfa4..a131dde9c7673 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SiblingPipelineAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SiblingPipelineAggregator.java
@@ -36,7 +36,7 @@
 import java.util.stream.StreamSupport;
 
 public abstract class SiblingPipelineAggregator extends PipelineAggregator {
-    SiblingPipelineAggregator(String name, String[] bucketsPaths, Map<String, Object> metaData) {
+    protected SiblingPipelineAggregator(String name, String[] bucketsPaths, Map<String, Object> metaData) {
         super(name, bucketsPaths, metaData);
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java
index 904cc16c29076..4943349c5463f 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java
@@ -64,16 +64,6 @@ protected StatsBucketPipelineAggregationBuilder buildFactory(String pipelineAggr
         }
     };
 
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(BucketMetricsPipelineAggregationBuilder<StatsBucketPipelineAggregationBuilder> other) {
-        return true;
-    }
-
     @Override
     public String getWriteableName() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java
index eb075d4368981..eae58ee12f320 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java
@@ -63,16 +63,6 @@ protected SumBucketPipelineAggregationBuilder buildFactory(String pipelineAggreg
         }
     };
 
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(BucketMetricsPipelineAggregationBuilder<SumBucketPipelineAggregationBuilder> other) {
-        return true;
-    }
-
     @Override
     public String getWriteableName() {
         return NAME;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
index acc0d2ee20bcb..cd37d1a9969ed 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
@@ -164,7 +164,7 @@ public AB format(String format) {
     }
 
     @Override
-    protected final MultiValuesSourceAggregatorFactory<VS, ?> doBuild(SearchContext context, AggregatorFactory<?> parent,
+    protected final MultiValuesSourceAggregatorFactory<VS> doBuild(SearchContext context, AggregatorFactory parent,
             AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
         ValueType finalValueType = this.valueType != null ? this.valueType : targetValueType;
 
@@ -190,8 +190,8 @@ private static DocValueFormat resolveFormat(@Nullable String format, @Nullable V
         return valueFormat;
     }
 
-    protected abstract MultiValuesSourceAggregatorFactory<VS, ?> innerBuild(SearchContext context,
-        Map<String, ValuesSourceConfig<VS>> configs, DocValueFormat format, AggregatorFactory<?> parent,
+    protected abstract MultiValuesSourceAggregatorFactory<VS> innerBuild(SearchContext context,
+        Map<String, ValuesSourceConfig<VS>> configs, DocValueFormat format, AggregatorFactory parent,
         AggregatorFactories.Builder subFactoriesBuilder) throws IOException;
 
 
@@ -225,28 +225,20 @@ public final XContentBuilder internalXContent(XContentBuilder builder, Params pa
     protected abstract XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException;
 
     @Override
-    protected final int doHashCode() {
-        return Objects.hash(fields, format, targetValueType, valueType, innerHashCode());
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), fields, format, targetValueType, valueType);
     }
 
-    protected abstract int innerHashCode();
 
     @Override
-    protected final boolean doEquals(Object other) {
-        if (this == other) {
-            return true;
-        }
-
-        if (other == null || getClass() != other.getClass()) {
-            return false;
-        }
-
-        MultiValuesSourceAggregationBuilder that = (MultiValuesSourceAggregationBuilder) other;
-
-        return Objects.equals(this.fields, that.fields)
-            && Objects.equals(this.format, that.format)
-            && Objects.equals(this.valueType, that.valueType);
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
+
+        MultiValuesSourceAggregationBuilder other = (MultiValuesSourceAggregationBuilder) obj;
+        return Objects.equals(this.fields, other.fields)
+            && Objects.equals(this.format, other.format)
+            && Objects.equals(this.valueType, other.valueType);
     }
-
-    protected abstract boolean innerEquals(Object obj);
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
index 5de8fbd7561dc..9516c9db87537 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
@@ -30,15 +30,15 @@
 import java.util.List;
 import java.util.Map;
 
-public abstract class MultiValuesSourceAggregatorFactory<VS extends ValuesSource, AF extends MultiValuesSourceAggregatorFactory<VS, AF>>
-        extends AggregatorFactory<AF> {
+public abstract class MultiValuesSourceAggregatorFactory<VS extends ValuesSource>
+        extends AggregatorFactory {
 
     protected final Map<String, ValuesSourceConfig<VS>> configs;
     protected final DocValueFormat format;
 
     public MultiValuesSourceAggregatorFactory(String name, Map<String, ValuesSourceConfig<VS>> configs,
                                               DocValueFormat format, SearchContext context,
-                                              AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+                                              AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder,
                                               Map<String, Object> metaData) throws IOException {
         super(name, context, parent, subFactoriesBuilder, metaData);
         this.configs = configs;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
index b931cd81da954..7fd38288a821b 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
@@ -111,6 +111,15 @@ public abstract SortedSetDocValues ordinalsValues(LeafReaderContext context)
             public abstract SortedSetDocValues globalOrdinalsValues(LeafReaderContext context)
                     throws IOException;
 
+            /**
+             * Whether this values source is able to provide a mapping between global and segment ordinals,
+             * by returning the underlying {@link OrdinalMap}. If this method returns false, then calling
+             * {@link #globalOrdinalsMapping} will result in an {@link UnsupportedOperationException}.
+             */
+            public boolean supportsGlobalOrdinalsMapping() {
+                return true;
+            }
+
             /** Returns a mapping from segment ordinals to global ordinals. */
             public abstract LongUnaryOperator globalOrdinalsMapping(LeafReaderContext context)
                     throws IOException;
@@ -153,6 +162,11 @@ public SortedSetDocValues globalOrdinalsValues(LeafReaderContext context) {
                     return atomicFieldData.getOrdinalsValues();
                 }
 
+                @Override
+                public boolean supportsGlobalOrdinalsMapping() {
+                    return indexFieldData.supportsGlobalOrdinalsMapping();
+                }
+
                 @Override
                 public LongUnaryOperator globalOrdinalsMapping(LeafReaderContext context) throws IOException {
                     final IndexOrdinalsFieldData global = indexFieldData.loadGlobal((DirectoryReader)context.parent.reader());
@@ -452,6 +466,7 @@ public boolean advanceExact(int doc) throws IOException {
                 if (bytesValues.advanceExact(doc)) {
                     count = bytesValues.docValueCount();
                     grow();
+                    script.setDocument(doc);
                     for (int i = 0; i < count; ++i) {
                         final BytesRef value = bytesValues.nextValue();
                         script.setNextAggregationValue(value.utf8ToString());
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
index 57f2a214a41fb..11969b3f7dbeb 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
@@ -299,10 +299,10 @@ public ZoneId timeZone() {
     }
 
     @Override
-    protected final ValuesSourceAggregatorFactory<VS, ?> doBuild(SearchContext context, AggregatorFactory<?> parent,
+    protected final ValuesSourceAggregatorFactory<VS> doBuild(SearchContext context, AggregatorFactory parent,
             AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
         ValuesSourceConfig<VS> config = resolveConfig(context);
-        ValuesSourceAggregatorFactory<VS, ?> factory = innerBuild(context, config, parent, subFactoriesBuilder);
+        ValuesSourceAggregatorFactory<VS> factory = innerBuild(context, config, parent, subFactoriesBuilder);
         return factory;
     }
 
@@ -312,8 +312,8 @@ protected ValuesSourceConfig<VS> resolveConfig(SearchContext context) {
                 valueType, field, script, missing, timeZone, format);
     }
 
-    protected abstract ValuesSourceAggregatorFactory<VS, ?> innerBuild(SearchContext context, ValuesSourceConfig<VS> config,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException;
+    protected abstract ValuesSourceAggregatorFactory<VS> innerBuild(SearchContext context, ValuesSourceConfig<VS> config,
+            AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException;
 
     @Override
     public final XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
@@ -344,34 +344,24 @@ public final XContentBuilder internalXContent(XContentBuilder builder, Params pa
     protected abstract XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException;
 
     @Override
-    protected final int doHashCode() {
-        return Objects.hash(field, format, missing, script, targetValueType, timeZone, valueType, valuesSourceType,
-                innerHashCode());
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), field, format, missing, script,
+            targetValueType, timeZone, valueType, valuesSourceType);
     }
 
-    protected abstract int innerHashCode();
-
     @Override
-    protected final boolean doEquals(Object obj) {
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        if (super.equals(obj) == false) return false;
         ValuesSourceAggregationBuilder<?, ?> other = (ValuesSourceAggregationBuilder<?, ?>) obj;
-        if (!Objects.equals(field, other.field))
-            return false;
-        if (!Objects.equals(format, other.format))
-            return false;
-        if (!Objects.equals(missing, other.missing))
-            return false;
-        if (!Objects.equals(script, other.script))
-            return false;
-        if (!Objects.equals(targetValueType, other.targetValueType))
-            return false;
-        if (!Objects.equals(timeZone, other.timeZone))
-            return false;
-        if (!Objects.equals(valueType, other.valueType))
-            return false;
-        if (!Objects.equals(valuesSourceType, other.valuesSourceType))
-            return false;
-        return innerEquals(obj);
+        return Objects.equals(valuesSourceType, other.valuesSourceType)
+            && Objects.equals(field, other.field)
+            && Objects.equals(format, other.format)
+            && Objects.equals(missing, other.missing)
+            && Objects.equals(script, other.script)
+            && Objects.equals(targetValueType, other.targetValueType)
+            && Objects.equals(timeZone, other.timeZone)
+            && Objects.equals(valueType, other.valueType);
     }
-
-    protected abstract boolean innerEquals(Object obj);
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java
index 37260f9013314..17da14aabf8ec 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java
@@ -29,13 +29,12 @@
 import java.util.List;
 import java.util.Map;
 
-public abstract class ValuesSourceAggregatorFactory<VS extends ValuesSource, AF extends ValuesSourceAggregatorFactory<VS, AF>>
-        extends AggregatorFactory<AF> {
+public abstract class ValuesSourceAggregatorFactory<VS extends ValuesSource> extends AggregatorFactory {
 
     protected ValuesSourceConfig<VS> config;
 
     public ValuesSourceAggregatorFactory(String name, ValuesSourceConfig<VS> config, SearchContext context,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
+            AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, context, parent, subFactoriesBuilder, metaData);
         this.config = config;
     }
@@ -43,13 +42,28 @@ public ValuesSourceAggregatorFactory(String name, ValuesSourceConfig<VS> config,
     @Override
     public Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBucket,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
-        VS vs = config.toValuesSource(context.getQueryShardContext());
+        VS vs = config.toValuesSource(context.getQueryShardContext(), this::resolveMissingAny);
         if (vs == null) {
             return createUnmapped(parent, pipelineAggregators, metaData);
         }
         return doCreateInternal(vs, parent, collectsFromSingleBucket, pipelineAggregators, metaData);
     }
 
+    /**
+     * This method provides a hook for aggregations that need finer grained control over the ValuesSource selected when the user supplies a
+     * missing value and there is no mapped field to infer the type from.  This will only be called for aggregations that specify the
+     * ValuesSourceType.ANY in their constructors (On the builder class).  The user supplied object is passed as a parameter, so its type
+     * may be inspected as needed.
+     *
+     * Generally, only the type of the returned ValuesSource is used, so returning the EMPTY instance of the chosen type is recommended.
+     *
+     * @param missing The user supplied missing value
+     * @return A ValuesSource instance compatible with the supplied parameter
+     */
+    protected ValuesSource resolveMissingAny(Object missing) {
+        return ValuesSource.Bytes.WithOrdinals.EMPTY;
+    }
+
     protected abstract Aggregator createUnmapped(Aggregator parent,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException;
 
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
index c1c171e3e4fd5..919d1b752e22c 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
@@ -36,6 +36,7 @@
 
 import java.time.ZoneId;
 import java.time.ZoneOffset;
+import java.util.function.Function;
 
 /**
  * A configuration that tells aggregations how to retrieve data from the index
@@ -223,10 +224,15 @@ public DocValueFormat format() {
         return format;
     }
 
+    @Nullable
+    public VS toValuesSource(QueryShardContext context) {
+        return toValuesSource(context, value -> ValuesSource.Bytes.WithOrdinals.EMPTY);
+    }
+
     /** Get a value source given its configuration. A return value of null indicates that
      *  no value source could be built. */
     @Nullable
-    public VS toValuesSource(QueryShardContext context) {
+    public VS toValuesSource(QueryShardContext context, Function<Object, ValuesSource> resolveMissingAny) {
         if (!valid()) {
             throw new IllegalStateException(
                     "value source config is invalid; must have either a field context or a script or marked as unwrapped");
@@ -241,8 +247,10 @@ public VS toValuesSource(QueryShardContext context) {
                 vs = (VS) ValuesSource.Numeric.EMPTY;
             } else if (valueSourceType() == ValuesSourceType.GEOPOINT) {
                 vs = (VS) ValuesSource.GeoPoint.EMPTY;
-            } else if (valueSourceType() == ValuesSourceType.ANY || valueSourceType() == ValuesSourceType.BYTES) {
+            } else if (valueSourceType() == ValuesSourceType.BYTES) {
                 vs = (VS) ValuesSource.Bytes.WithOrdinals.EMPTY;
+            } else if (valueSourceType() == ValuesSourceType.ANY) {
+                vs = (VS) resolveMissingAny.apply(missing());
             } else {
                 throw new IllegalArgumentException("Can't deal with unmapped ValuesSource type " + valueSourceType());
             }
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
index c23be0f4cb994..196e74e0bbabd 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
@@ -31,7 +31,6 @@
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.BitSet;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.document.DocumentField;
@@ -383,32 +382,19 @@ private SearchHit.NestedIdentity getInternalNestedIdentity(SearchContext context
             BitSet parentBits = context.bitsetFilterCache().getBitSetProducer(parentFilter).getBitSet(subReaderContext);
 
             int offset = 0;
-            if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_6_5_0)) {
-                /**
-                 * Starts from the previous parent and finds the offset of the
-                 * <code>nestedSubDocID</code> within the nested children. Nested documents
-                 * are indexed in the same order than in the source array so the offset
-                 * of the nested child is the number of nested document with the same parent
-                 * that appear before him.
-                 */
-                int previousParent = parentBits.prevSetBit(currentParent);
-                for (int docId = childIter.advance(previousParent + 1); docId < nestedSubDocId && docId != DocIdSetIterator.NO_MORE_DOCS;
-                        docId = childIter.nextDoc()) {
-                    offset++;
-                }
-                currentParent = nestedSubDocId;
-            } else {
-                /**
-                 * Nested documents are in reverse order in this version so we start from the current nested document
-                 * and find the number of documents with the same parent that appear after it.
-                 */
-                int nextParent = parentBits.nextSetBit(currentParent);
-                for (int docId = childIter.advance(currentParent + 1); docId < nextParent && docId != DocIdSetIterator.NO_MORE_DOCS;
-                        docId = childIter.nextDoc()) {
-                    offset++;
-                }
-                currentParent = nextParent;
+            /**
+             * Starts from the previous parent and finds the offset of the
+             * <code>nestedSubDocID</code> within the nested children. Nested documents
+             * are indexed in the same order than in the source array so the offset
+             * of the nested child is the number of nested document with the same parent
+             * that appear before him.
+             */
+            int previousParent = parentBits.prevSetBit(currentParent);
+            for (int docId = childIter.advance(previousParent + 1); docId < nestedSubDocId && docId != DocIdSetIterator.NO_MORE_DOCS;
+                 docId = childIter.nextDoc()) {
+                offset++;
             }
+            currentParent = nestedSubDocId;
             current = nestedObjectMapper = nestedParentObjectMapper;
             int currentPrefix = current == null ? 0 : current.name().length() + 1;
             nestedIdentity = new SearchHit.NestedIdentity(originalName.substring(currentPrefix), offset, nestedIdentity);
diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java
index 583516c5cd4c2..d896d3a9d922d 100644
--- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java
+++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java
@@ -26,7 +26,7 @@
 import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo;
 import org.apache.lucene.search.vectorhighlight.FragmentsBuilder;
 import org.apache.lucene.util.CollectionUtil;
-import org.elasticsearch.index.analysis.CustomAnalyzer;
+import org.elasticsearch.index.analysis.AnalyzerComponentsProvider;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -81,9 +81,8 @@ private static boolean containsBrokenAnalysis(Analyzer analyzer) {
         if (analyzer instanceof NamedAnalyzer) {
             analyzer = ((NamedAnalyzer) analyzer).analyzer();
         }
-        if (analyzer instanceof CustomAnalyzer) {
-            final CustomAnalyzer a = (CustomAnalyzer) analyzer;
-            TokenFilterFactory[] tokenFilters = a.tokenFilters();
+        if (analyzer instanceof AnalyzerComponentsProvider) {
+            final TokenFilterFactory[] tokenFilters = ((AnalyzerComponentsProvider) analyzer).getComponents().getTokenFilters();
             for (TokenFilterFactory tokenFilterFactory : tokenFilters) {
                 if (tokenFilterFactory.breaksFastVectorHighlighter()) {
                     return true;
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
index 7c56796f3d24d..794c72ec14cfb 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
@@ -20,14 +20,19 @@
 package org.elasticsearch.search.internal;
 
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermStates;
 import org.apache.lucene.search.BulkScorer;
 import org.apache.lucene.search.CollectionStatistics;
+import org.apache.lucene.search.CollectionTerminatedException;
 import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.ConjunctionDISI;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryCache;
 import org.apache.lucene.search.QueryCachingPolicy;
@@ -35,9 +40,13 @@
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TermStatistics;
 import org.apache.lucene.search.Weight;
-import org.apache.lucene.search.XIndexSearcher;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.util.BitSet;
+import org.apache.lucene.util.BitSetIterator;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.CombinedBitSet;
+import org.apache.lucene.util.SparseFixedBitSet;
 import org.elasticsearch.common.lease.Releasable;
-import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.search.dfs.AggregatedDfs;
 import org.elasticsearch.search.profile.Timer;
 import org.elasticsearch.search.profile.query.ProfileWeight;
@@ -46,6 +55,7 @@
 import org.elasticsearch.search.profile.query.QueryTimingType;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
 
@@ -53,26 +63,19 @@
  * Context-aware extension of {@link IndexSearcher}.
  */
 public class ContextIndexSearcher extends IndexSearcher implements Releasable {
-
-    /** The wrapped {@link IndexSearcher}. The reason why we sometimes prefer delegating to this searcher instead of {@code super} is that
-     *  this instance may have more assertions, for example if it comes from MockInternalEngine which wraps the IndexSearcher into an
-     *  AssertingIndexSearcher. */
-    private final XIndexSearcher in;
+    /**
+     * The interval at which we check for search cancellation when we cannot use
+     * a {@link CancellableBulkScorer}. See {@link #intersectScorerAndBitSet}.
+     */
+    private static int CHECK_CANCELLED_SCORER_INTERVAL = 1 << 11;
 
     private AggregatedDfs aggregatedDfs;
-
-    private final Engine.Searcher engineSearcher;
-
-    // TODO revisit moving the profiler to inheritance or wrapping model in the future
     private QueryProfiler profiler;
-
     private Runnable checkCancelled;
 
-    public ContextIndexSearcher(Engine.Searcher searcher, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy) {
-        super(searcher.reader());
-        engineSearcher = searcher;
-        in = new XIndexSearcher(searcher.searcher());
-        setSimilarity(searcher.searcher().getSimilarity());
+    public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy) {
+        super(reader);
+        setSimilarity(similarity);
         setQueryCache(queryCache);
         setQueryCachingPolicy(queryCachingPolicy);
     }
@@ -104,7 +107,7 @@ public Query rewrite(Query original) throws IOException {
         }
 
         try {
-            return in.rewrite(original);
+            return super.rewrite(original);
         } finally {
             if (profiler != null) {
                 profiler.stopAndAddRewriteTime();
@@ -130,7 +133,6 @@ public Weight createWeight(Query query, ScoreMode scoreMode, float boost) throws
             }
             return new ProfileWeight(query, weight, profile);
         } else {
-            // needs to be 'super', not 'in' in order to use aggregated DFS
             return super.createWeight(query, scoreMode, boost);
         }
     }
@@ -152,13 +154,13 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
                 }
 
                 @Override
-                public Scorer scorer(LeafReaderContext context) throws IOException {
+                public boolean isCacheable(LeafReaderContext ctx) {
                     throw new UnsupportedOperationException();
                 }
 
                 @Override
-                public boolean isCacheable(LeafReaderContext ctx) {
-                    throw new UnsupportedOperationException();
+                public Scorer scorer(LeafReaderContext context) throws IOException {
+                    return weight.scorer(context);
                 }
 
                 @Override
@@ -174,16 +176,75 @@ public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
         } else {
             cancellableWeight = weight;
         }
-        in.search(leaves, cancellableWeight, collector);
+        searchInternal(leaves, cancellableWeight, collector);
     }
 
-    @Override
-    public Explanation explain(Query query, int doc) throws IOException {
-        if (aggregatedDfs != null) {
-            // dfs data is needed to explain the score
-            return super.explain(createWeight(rewrite(query), ScoreMode.COMPLETE, 1f), doc);
+    private void searchInternal(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
+        for (LeafReaderContext ctx : leaves) { // search each subreader
+            final LeafCollector leafCollector;
+            try {
+                leafCollector = collector.getLeafCollector(ctx);
+            } catch (CollectionTerminatedException e) {
+                // there is no doc of interest in this reader context
+                // continue with the following leaf
+                continue;
+            }
+            Bits liveDocs = ctx.reader().getLiveDocs();
+            BitSet liveDocsBitSet = getSparseBitSetOrNull(ctx.reader().getLiveDocs());
+            if (liveDocsBitSet == null) {
+                BulkScorer bulkScorer = weight.bulkScorer(ctx);
+                if (bulkScorer != null) {
+                    try {
+                        bulkScorer.score(leafCollector, liveDocs);
+                    } catch (CollectionTerminatedException e) {
+                        // collection was terminated prematurely
+                        // continue with the following leaf
+                    }
+                }
+            } else {
+                // if the role query result set is sparse then we should use the SparseFixedBitSet for advancing:
+                Scorer scorer = weight.scorer(ctx);
+                if (scorer != null) {
+                    try {
+                        intersectScorerAndBitSet(scorer, liveDocsBitSet, leafCollector,
+                            checkCancelled == null ? () -> {} : checkCancelled);
+                    } catch (CollectionTerminatedException e) {
+                        // collection was terminated prematurely
+                        // continue with the following leaf
+                    }
+                }
+            }
         }
-        return in.explain(query, doc);
+    }
+
+    private static BitSet getSparseBitSetOrNull(Bits liveDocs) {
+        if (liveDocs instanceof SparseFixedBitSet) {
+            return (BitSet) liveDocs;
+        } else if (liveDocs instanceof CombinedBitSet
+                        // if the underlying role bitset is sparse
+                        && ((CombinedBitSet) liveDocs).getFirst() instanceof SparseFixedBitSet) {
+            return (BitSet) liveDocs;
+        } else {
+            return null;
+        }
+
+    }
+
+    static void intersectScorerAndBitSet(Scorer scorer, BitSet acceptDocs,
+                                            LeafCollector collector, Runnable checkCancelled) throws IOException {
+        // ConjunctionDISI uses the DocIdSetIterator#cost() to order the iterators, so if roleBits has the lowest cardinality it should
+        // be used first:
+        DocIdSetIterator iterator = ConjunctionDISI.intersectIterators(Arrays.asList(new BitSetIterator(acceptDocs,
+            acceptDocs.approximateCardinality()), scorer.iterator()));
+        int seen = 0;
+        checkCancelled.run();
+        for (int docId = iterator.nextDoc(); docId < DocIdSetIterator.NO_MORE_DOCS; docId = iterator.nextDoc()) {
+            if (++seen % CHECK_CANCELLED_SCORER_INTERVAL == 0) {
+                checkCancelled.run();
+            }
+            collector.collect(docId);
+        }
+        checkCancelled.run();
     }
 
     @Override
@@ -215,10 +276,8 @@ public CollectionStatistics collectionStatistics(String field) throws IOExceptio
     }
 
     public DirectoryReader getDirectoryReader() {
-        return engineSearcher.getDirectoryReader();
-    }
-
-    public Engine.Searcher getEngineSearcher() {
-        return engineSearcher;
+        final IndexReader reader = getIndexReader();
+        assert reader instanceof DirectoryReader : "expected an instance of DirectoryReader, got " + reader.getClass();
+        return (DirectoryReader) reader;
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java
index 12eef5dcf29de..70c2aa6e5ac6d 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.internal;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.Nullable;
@@ -35,6 +36,7 @@
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 import java.io.IOException;
+import java.util.Arrays;
 
 /**
  * Shard level search request that gets created and consumed on the local node.
@@ -61,7 +63,6 @@ public class ShardSearchLocalRequest implements ShardSearchRequest {
     private final int numberOfShards;
     private final SearchType searchType;
     private final Scroll scroll;
-    private final String[] types;
     private final float indexBoost;
     private final Boolean requestCache;
     private final long nowInMillis;
@@ -74,7 +75,7 @@ public class ShardSearchLocalRequest implements ShardSearchRequest {
 
     public ShardSearchLocalRequest(SearchRequest searchRequest, ShardId shardId, int numberOfShards, AliasFilter aliasFilter,
                                    float indexBoost, long nowInMillis, @Nullable String clusterAlias, String[] indexRoutings) {
-        this(shardId, numberOfShards, searchRequest.searchType(), searchRequest.source(), searchRequest.types(),
+        this(shardId, numberOfShards, searchRequest.searchType(), searchRequest.source(),
             searchRequest.requestCache(), aliasFilter, indexBoost, searchRequest.allowPartialSearchResults(), indexRoutings,
             searchRequest.preference(), searchRequest.scroll(), nowInMillis, clusterAlias);
         // If allowPartialSearchResults is unset (ie null), the cluster-level default should have been substituted
@@ -82,11 +83,11 @@ public ShardSearchLocalRequest(SearchRequest searchRequest, ShardId shardId, int
         assert searchRequest.allowPartialSearchResults() != null;
     }
 
-    public ShardSearchLocalRequest(ShardId shardId, String[] types, long nowInMillis, AliasFilter aliasFilter) {
-        this(shardId, -1, null, null, types, null, aliasFilter, 1.0f, false, Strings.EMPTY_ARRAY, null, null, nowInMillis, null);
+    public ShardSearchLocalRequest(ShardId shardId, long nowInMillis, AliasFilter aliasFilter) {
+        this(shardId, -1, null, null, null, aliasFilter, 1.0f, false, Strings.EMPTY_ARRAY, null, null, nowInMillis, null);
     }
 
-    private ShardSearchLocalRequest(ShardId shardId, int numberOfShards, SearchType searchType, SearchSourceBuilder source, String[] types,
+    private ShardSearchLocalRequest(ShardId shardId, int numberOfShards, SearchType searchType, SearchSourceBuilder source,
                                     Boolean requestCache, AliasFilter aliasFilter, float indexBoost, boolean allowPartialSearchResults,
                                     String[] indexRoutings, String preference, Scroll scroll, long nowInMillis,
                                     @Nullable String clusterAlias) {
@@ -94,7 +95,6 @@ private ShardSearchLocalRequest(ShardId shardId, int numberOfShards, SearchType
         this.numberOfShards = numberOfShards;
         this.searchType = searchType;
         this.source = source;
-        this.types = types;
         this.requestCache = requestCache;
         this.aliasFilter = aliasFilter;
         this.indexBoost = indexBoost;
@@ -112,7 +112,14 @@ private ShardSearchLocalRequest(ShardId shardId, int numberOfShards, SearchType
         numberOfShards = in.readVInt();
         scroll = in.readOptionalWriteable(Scroll::new);
         source = in.readOptionalWriteable(SearchSourceBuilder::new);
-        types = in.readStringArray();
+        if (in.getVersion().before(Version.V_8_0_0)) {
+            // types no longer relevant so ignore
+            String[] types = in.readStringArray();
+            if (types.length > 0) {
+                throw new IllegalStateException(
+                        "types are no longer supported in search requests but found [" + Arrays.toString(types) + "]");
+            }
+        }
         aliasFilter = new AliasFilter(in);
         indexBoost = in.readFloat();
         nowInMillis = in.readVLong();
@@ -131,7 +138,10 @@ protected final void innerWriteTo(StreamOutput out, boolean asKey) throws IOExce
         }
         out.writeOptionalWriteable(scroll);
         out.writeOptionalWriteable(source);
-        out.writeStringArray(types);
+        if (out.getVersion().before(Version.V_8_0_0)) {
+            // types not supported so send an empty array to previous versions
+            out.writeStringArray(Strings.EMPTY_ARRAY);
+        }
         aliasFilter.writeTo(out);
         out.writeFloat(indexBoost);
         if (asKey == false) {
@@ -151,11 +161,6 @@ public ShardId shardId() {
         return shardId;
     }
 
-    @Override
-    public String[] types() {
-        return types;
-    }
-
     @Override
     public SearchSourceBuilder source() {
         return source;
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
index b88bda9009043..0e9d5de9788f3 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
@@ -49,8 +49,6 @@ public interface ShardSearchRequest {
 
     ShardId shardId();
 
-    String[] types();
-
     SearchSourceBuilder source();
 
     AliasFilter getAliasFilter();
diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java
index 9aae2df27779f..07557d9459973 100644
--- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java
+++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java
@@ -93,11 +93,6 @@ public ShardId shardId() {
         return shardSearchLocalRequest.shardId();
     }
 
-    @Override
-    public String[] types() {
-        return shardSearchLocalRequest.types();
-    }
-
     @Override
     public SearchSourceBuilder source() {
         return shardSearchLocalRequest.source();
diff --git a/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java
index 9cfe121fe7ebd..0022cdfdc9d99 100644
--- a/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java
+++ b/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.search.lookup;
 
 import org.apache.lucene.index.LeafReaderContext;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
@@ -31,13 +30,9 @@ public class DocLookup {
     private final MapperService mapperService;
     private final Function<MappedFieldType, IndexFieldData<?>> fieldDataLookup;
 
-    @Nullable
-    private final String[] types;
-
-    DocLookup(MapperService mapperService, Function<MappedFieldType, IndexFieldData<?>> fieldDataLookup, @Nullable String[] types) {
+    DocLookup(MapperService mapperService, Function<MappedFieldType, IndexFieldData<?>> fieldDataLookup) {
         this.mapperService = mapperService;
         this.fieldDataLookup = fieldDataLookup;
-        this.types = types;
     }
 
     public MapperService mapperService() {
@@ -49,10 +44,6 @@ public IndexFieldData<?> getForField(MappedFieldType fieldType) {
     }
 
     public LeafDocLookup getLeafDocLookup(LeafReaderContext context) {
-        return new LeafDocLookup(mapperService, fieldDataLookup, types, context);
-    }
-
-    public String[] getTypes() {
-        return types;
+        return new LeafDocLookup(mapperService, fieldDataLookup, context);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/lookup/FieldsLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/FieldsLookup.java
index feefb1fcb30e4..c089501e38596 100644
--- a/server/src/main/java/org/elasticsearch/search/lookup/FieldsLookup.java
+++ b/server/src/main/java/org/elasticsearch/search/lookup/FieldsLookup.java
@@ -19,22 +19,18 @@
 package org.elasticsearch.search.lookup;
 
 import org.apache.lucene.index.LeafReaderContext;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.mapper.MapperService;
 
 public class FieldsLookup {
 
     private final MapperService mapperService;
-    @Nullable
-    private final String[] types;
 
-    FieldsLookup(MapperService mapperService, @Nullable String[] types) {
+    FieldsLookup(MapperService mapperService) {
         this.mapperService = mapperService;
-        this.types = types;
     }
 
     public LeafFieldsLookup getLeafFieldsLookup(LeafReaderContext context) {
-        return new LeafFieldsLookup(mapperService, types, context.reader());
+        return new LeafFieldsLookup(mapperService, context.reader());
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java
index 04522834579e4..4fd6c78e47513 100644
--- a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java
+++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java
@@ -21,7 +21,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
@@ -31,7 +30,6 @@
 import java.io.IOException;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
@@ -51,18 +49,14 @@ public class LeafDocLookup implements Map<String, ScriptDocValues<?>> {
     private final MapperService mapperService;
     private final Function<MappedFieldType, IndexFieldData<?>> fieldDataLookup;
 
-    @Nullable
-    private final String[] types;
-
     private final LeafReaderContext reader;
 
     private int docId = -1;
 
-    LeafDocLookup(MapperService mapperService, Function<MappedFieldType, IndexFieldData<?>> fieldDataLookup, @Nullable String[] types,
+    LeafDocLookup(MapperService mapperService, Function<MappedFieldType, IndexFieldData<?>> fieldDataLookup,
                   LeafReaderContext reader) {
         this.mapperService = mapperService;
         this.fieldDataLookup = fieldDataLookup;
-        this.types = types;
         this.reader = reader;
     }
 
@@ -90,8 +84,7 @@ public ScriptDocValues<?> get(Object key) {
         if (scriptValues == null) {
             final MappedFieldType fieldType = mapperService.fullName(fieldName);
             if (fieldType == null) {
-                throw new IllegalArgumentException("No field found for [" + fieldName + "] in mapping with types " +
-                        Arrays.toString(types));
+                throw new IllegalArgumentException("No field found for [" + fieldName + "] in mapping");
             }
             // load fielddata on behalf of the script: otherwise it would need additional permissions
             // to deal with pagedbytes/ramusagestimator/etc
diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafFieldsLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafFieldsLookup.java
index d98a8585ecf6a..f614ce400ef36 100644
--- a/server/src/main/java/org/elasticsearch/search/lookup/LeafFieldsLookup.java
+++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafFieldsLookup.java
@@ -20,7 +20,6 @@
 
 import org.apache.lucene.index.LeafReader;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.fieldvisitor.SingleFieldsVisitor;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -29,7 +28,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
@@ -42,18 +40,14 @@ public class LeafFieldsLookup implements Map {
 
     private final MapperService mapperService;
 
-    @Nullable
-    private final String[] types;
-
     private final LeafReader reader;
 
     private int docId = -1;
 
     private final Map<String, FieldLookup> cachedFieldData = new HashMap<>();
 
-    LeafFieldsLookup(MapperService mapperService, @Nullable String[] types, LeafReader reader) {
+    LeafFieldsLookup(MapperService mapperService, LeafReader reader) {
         this.mapperService = mapperService;
-        this.types = types;
         this.reader = reader;
     }
 
@@ -136,7 +130,7 @@ private FieldLookup loadFieldData(String name) {
         if (data == null) {
             MappedFieldType fieldType = mapperService.fullName(name);
             if (fieldType == null) {
-                throw new IllegalArgumentException("No field found for [" + name + "] in mapping with types " + Arrays.toString(types));
+                throw new IllegalArgumentException("No field found for [" + name + "] in mapping");
             }
             data = new FieldLookup(fieldType);
             cachedFieldData.put(name, data);
diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java
index 8f4b5143dc6cd..04aef7d2e8f63 100644
--- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java
+++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.search.lookup;
 
 import org.apache.lucene.index.LeafReaderContext;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
@@ -35,11 +34,10 @@ public class SearchLookup {
 
     final FieldsLookup fieldsLookup;
 
-    public SearchLookup(MapperService mapperService, Function<MappedFieldType, IndexFieldData<?>> fieldDataLookup,
-                        @Nullable String[] types) {
-        docMap = new DocLookup(mapperService, fieldDataLookup, types);
+    public SearchLookup(MapperService mapperService, Function<MappedFieldType, IndexFieldData<?>> fieldDataLookup) {
+        docMap = new DocLookup(mapperService, fieldDataLookup);
         sourceLookup = new SourceLookup();
-        fieldsLookup = new FieldsLookup(mapperService, types);
+        fieldsLookup = new FieldsLookup(mapperService);
     }
 
     public LeafSearchLookup getLeafSearchLookup(LeafReaderContext context) {
diff --git a/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java b/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java
index 3c86b21a0873d..01924e938dcd0 100644
--- a/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java
@@ -217,19 +217,15 @@ public Query toFilter(ClusterService clusterService, ShardSearchRequest request,
 
         int shardId = request.shardId().id();
         int numShards = context.getIndexSettings().getNumberOfShards();
-        if (minNodeVersion.onOrAfter(Version.V_6_4_0) &&
-                (request.preference() != null || request.indexRoutings().length > 0)) {
+        if (request.preference() != null || request.indexRoutings().length > 0) {
             GroupShardsIterator<ShardIterator> group = buildShardIterator(clusterService, request);
             assert group.size() <= numShards : "index routing shards: " + group.size() +
                 " cannot be greater than total number of shards: " + numShards;
             if (group.size() < numShards) {
-                /**
+                /*
                  * The routing of this request targets a subset of the shards of this index so we need to we retrieve
                  * the original {@link GroupShardsIterator} and compute the request shard id and number of
                  * shards from it.
-                 * This behavior has been added in {@link Version#V_6_4_0} so if there is another node in the cluster
-                 * with an older version we use the original shard id and number of shards in order to ensure that all
-                 * slices use the same numbers.
                  */
                 numShards = group.size();
                 int ord = 0;
diff --git a/server/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java b/server/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java
index 52ac72d01b778..1ec3479424193 100644
--- a/server/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java
+++ b/server/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java
@@ -78,6 +78,9 @@ public boolean isCacheable(LeafReaderContext ctx) {
     private DocIdSet build(LeafReader reader) throws IOException {
         final DocIdSetBuilder builder = new DocIdSetBuilder(reader.maxDoc());
         final Terms terms = reader.terms(getField());
+        if (terms == null) {
+            return DocIdSet.EMPTY;
+        }
         final TermsEnum te = terms.iterator();
         PostingsEnum docsEnum = null;
         for (BytesRef term = te.next(); term != null; term = te.next()) {
diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
index 8abd4b9f40d5c..949a5a3ff441c 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
@@ -19,13 +19,11 @@
 
 package org.elasticsearch.search.sort;
 
-import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -36,6 +34,7 @@
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
 import org.elasticsearch.index.fielddata.plain.SortedNumericDVIndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -47,13 +46,13 @@
 import java.util.Locale;
 import java.util.Objects;
 
+import static org.elasticsearch.index.search.NestedHelper.parentObject;
 import static org.elasticsearch.search.sort.NestedSortBuilder.NESTED_FIELD;
 
 /**
  * A sort builder to sort based on a document field.
  */
 public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(FieldSortBuilder.class));
 
     public static final String NAME = "field_sort";
     public static final ParseField MISSING = new ParseField("missing");
@@ -80,10 +79,6 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
 
     private SortMode sortMode;
 
-    private QueryBuilder nestedFilter;
-
-    private String nestedPath;
-
     private NestedSortBuilder nestedSort;
 
     /** Copy constructor. */
@@ -95,8 +90,6 @@ public FieldSortBuilder(FieldSortBuilder template) {
         if (template.sortMode != null) {
             this.sortMode(template.sortMode());
         }
-        this.setNestedFilter(template.getNestedFilter());
-        this.setNestedPath(template.getNestedPath());
         if (template.getNestedSort() != null) {
             this.setNestedSort(template.getNestedSort());
         }
@@ -121,8 +114,12 @@ public FieldSortBuilder(String fieldName) {
      */
     public FieldSortBuilder(StreamInput in) throws IOException {
         fieldName = in.readString();
-        nestedFilter = in.readOptionalNamedWriteable(QueryBuilder.class);
-        nestedPath = in.readOptionalString();
+        if (in.getVersion().before(Version.V_8_0_0)) {
+            if (in.readOptionalNamedWriteable(QueryBuilder.class) != null || in.readOptionalString() != null) {
+                throw new IOException("the [sort] options [nested_path] and [nested_filter] are removed in 8.x, " +
+                    "please use [nested] instead");
+            }
+        }
         missing = in.readGenericValue();
         order = in.readOptionalWriteable(SortOrder::readFromStream);
         sortMode = in.readOptionalWriteable(SortMode::readFromStream);
@@ -136,8 +133,10 @@ public FieldSortBuilder(StreamInput in) throws IOException {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(fieldName);
-        out.writeOptionalNamedWriteable(nestedFilter);
-        out.writeOptionalString(nestedPath);
+        if (out.getVersion().before(Version.V_8_0_0)) {
+            out.writeOptionalNamedWriteable(null);
+            out.writeOptionalString(null);
+        }
         out.writeGenericValue(missing);
         out.writeOptionalWriteable(order);
         out.writeOptionalWriteable(sortMode);
@@ -210,58 +209,6 @@ public SortMode sortMode() {
         return this.sortMode;
     }
 
-    /**
-     * Sets the nested filter that the nested objects should match with in order
-     * to be taken into account for sorting.
-     *
-     * @deprecated set nested sort with {@link #setNestedSort(NestedSortBuilder)} and retrieve with {@link #getNestedSort()}
-     */
-    @Deprecated
-    public FieldSortBuilder setNestedFilter(QueryBuilder nestedFilter) {
-        if (this.nestedSort != null) {
-            throw new IllegalArgumentException("Setting both nested_path/nested_filter and nested not allowed");
-        }
-        this.nestedFilter = nestedFilter;
-        return this;
-    }
-
-    /**
-     * Returns the nested filter that the nested objects should match with in
-     * order to be taken into account for sorting.
-     *
-     * @deprecated set nested sort with {@link #setNestedSort(NestedSortBuilder)} and retrieve with {@link #getNestedSort()}
-     */
-    @Deprecated
-    public QueryBuilder getNestedFilter() {
-        return this.nestedFilter;
-    }
-
-    /**
-     * Sets the nested path if sorting occurs on a field that is inside a nested
-     * object. By default when sorting on a field inside a nested object, the
-     * nearest upper nested object is selected as nested path.
-     *
-     * @deprecated set nested sort with {@link #setNestedSort(NestedSortBuilder)} and retrieve with {@link #getNestedSort()}
-     */
-    @Deprecated
-    public FieldSortBuilder setNestedPath(String nestedPath) {
-        if (this.nestedSort != null) {
-            throw new IllegalArgumentException("Setting both nested_path/nested_filter and nested not allowed");
-        }
-        this.nestedPath = nestedPath;
-        return this;
-    }
-
-    /**
-     * Returns the nested path if sorting occurs in a field that is inside a
-     * nested object.
-     * @deprecated set nested sort with {@link #setNestedSort(NestedSortBuilder)} and retrieve with {@link #getNestedSort()}
-     */
-    @Deprecated
-    public String getNestedPath() {
-        return this.nestedPath;
-    }
-
     /**
      * Returns the {@link NestedSortBuilder}
      */
@@ -276,9 +223,6 @@ public NestedSortBuilder getNestedSort() {
      * order to be taken into account for sorting.
      */
     public FieldSortBuilder setNestedSort(final NestedSortBuilder nestedSort) {
-        if (this.nestedFilter != null || this.nestedPath != null) {
-            throw new IllegalArgumentException("Setting both nested_path/nested_filter and nested not allowed");
-        }
         this.nestedSort = nestedSort;
         return this;
     }
@@ -330,12 +274,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (sortMode != null) {
             builder.field(SORT_MODE.getPreferredName(), sortMode);
         }
-        if (nestedFilter != null) {
-            builder.field(NESTED_FILTER_FIELD.getPreferredName(), nestedFilter, params);
-        }
-        if (nestedPath != null) {
-            builder.field(NESTED_PATH_FIELD.getPreferredName(), nestedPath);
-        }
         if (nestedSort != null) {
             builder.field(NESTED_FIELD.getPreferredName(), nestedSort);
         }
@@ -367,69 +305,85 @@ private static NumericType resolveNumericType(String value) {
     @Override
     public SortFieldAndFormat build(QueryShardContext context) throws IOException {
         if (DOC_FIELD_NAME.equals(fieldName)) {
-            if (order == SortOrder.DESC) {
-                return SORT_DOC_REVERSE;
+            return order == SortOrder.DESC ? SORT_DOC_REVERSE : SORT_DOC;
+        }
+
+        boolean isUnmapped = false;
+        MappedFieldType fieldType = context.fieldMapper(fieldName);
+        if (fieldType == null) {
+            isUnmapped = true;
+            if (unmappedType != null) {
+                fieldType = context.getMapperService().unmappedFieldType(unmappedType);
             } else {
-                return SORT_DOC;
-            }
-        } else {
-            boolean isUnmapped = false;
-            MappedFieldType fieldType = context.fieldMapper(fieldName);
-            if (fieldType == null) {
-                isUnmapped = true;
-                if (unmappedType != null) {
-                    fieldType = context.getMapperService().unmappedFieldType(unmappedType);
-                } else {
-                    throw new QueryShardException(context, "No mapping found for [" + fieldName + "] in order to sort on");
-                }
+                throw new QueryShardException(context, "No mapping found for [" + fieldName + "] in order to sort on");
             }
+        }
 
-            MultiValueMode localSortMode = null;
-            if (sortMode != null) {
-                localSortMode = MultiValueMode.fromString(sortMode.toString());
-            }
+        MultiValueMode localSortMode = null;
+        if (sortMode != null) {
+            localSortMode = MultiValueMode.fromString(sortMode.toString());
+        }
 
-            boolean reverse = (order == SortOrder.DESC);
-            if (localSortMode == null) {
-                localSortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
-            }
+        boolean reverse = (order == SortOrder.DESC);
+        if (localSortMode == null) {
+            localSortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
+        }
 
-            Nested nested = null;
-            if (isUnmapped == false) {
-                if (nestedSort != null) {
-                    if (context.indexVersionCreated().before(Version.V_6_5_0) && nestedSort.getMaxChildren() != Integer.MAX_VALUE) {
-                        throw new QueryShardException(context,
-                            "max_children is only supported on v6.5.0 or higher");
-                    }
-                    if (nestedSort.getNestedSort() != null && nestedSort.getMaxChildren() != Integer.MAX_VALUE) {
-                        throw new QueryShardException(context,
-                            "max_children is only supported on last level of nested sort");
-                    }
-                    // new nested sorts takes priority
-                    nested = resolveNested(context, nestedSort);
-                } else {
-                    nested = resolveNested(context, nestedPath, nestedFilter);
+        Nested nested = null;
+        if (isUnmapped == false) {
+            if (nestedSort != null) {
+                if (nestedSort.getNestedSort() != null && nestedSort.getMaxChildren() != Integer.MAX_VALUE) {
+                    throw new QueryShardException(context,
+                        "max_children is only supported on last level of nested sort");
                 }
+                nested = resolveNested(context, nestedSort);
+            } else {
+                validateMissingNestedPath(context, fieldName);
             }
+        }
 
-            IndexFieldData<?> fieldData = context.getForField(fieldType);
-            if (fieldData instanceof IndexNumericFieldData == false
-                    && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) {
-                throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields");
+        IndexFieldData<?> fieldData = context.getForField(fieldType);
+        if (fieldData instanceof IndexNumericFieldData == false
+                && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) {
+            throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields");
+        }
+        final SortField field;
+        if (numericType != null) {
+            if (fieldData instanceof IndexNumericFieldData == false) {
+                throw new QueryShardException(context,
+                    "[numeric_type] option cannot be set on a non-numeric field, got " + fieldType.typeName());
             }
-            final SortField field;
-            if (numericType != null) {
-                if (fieldData instanceof IndexNumericFieldData == false) {
+            SortedNumericDVIndexFieldData numericFieldData = (SortedNumericDVIndexFieldData) fieldData;
+            NumericType resolvedType = resolveNumericType(numericType);
+            field = numericFieldData.sortField(resolvedType, missing, localSortMode, nested, reverse);
+        } else {
+            field = fieldData.sortField(missing, localSortMode, nested, reverse);
+        }
+        return new SortFieldAndFormat(field, fieldType.docValueFormat(null, null));
+    }
+
+    /**
+     * Throws an exception if the provided <code>field</code> requires a nested context.
+     */
+    static void validateMissingNestedPath(QueryShardContext context, String field) {
+        ObjectMapper contextMapper = context.nestedScope().getObjectMapper();
+        if (contextMapper != null && contextMapper.nested().isNested() == false) {
+            // already in nested context
+            return;
+        }
+        for (String parent = parentObject(field); parent != null; parent = parentObject(parent)) {
+            ObjectMapper parentMapper = context.getObjectMapper(parent);
+            if (parentMapper != null && parentMapper.nested().isNested()) {
+                if (contextMapper != null && contextMapper.fullPath().equals(parentMapper.fullPath())) {
+                    // we are in a nested context that matches the path of the provided field so the nested path
+                    // is not required
+                    return ;
+                }
+                if (parentMapper.nested().isIncludeInRoot() == false) {
                     throw new QueryShardException(context,
-                        "[numeric_type] option cannot be set on a non-numeric field, got " + fieldType.typeName());
+                        "it is mandatory to set the [nested] context on the nested sort field: [" + field + "].");
                 }
-                SortedNumericDVIndexFieldData numericFieldData = (SortedNumericDVIndexFieldData) fieldData;
-                NumericType resolvedType = resolveNumericType(numericType);
-                field = numericFieldData.sortField(resolvedType, missing, localSortMode, nested, reverse);
-            } else {
-                field = fieldData.sortField(missing, localSortMode, nested, reverse);
             }
-            return new SortFieldAndFormat(field, fieldType.docValueFormat(null, null));
         }
     }
 
@@ -444,8 +398,7 @@ public boolean equals(Object other) {
         }
 
         FieldSortBuilder builder = (FieldSortBuilder) other;
-        return (Objects.equals(this.fieldName, builder.fieldName) && Objects.equals(this.nestedFilter, builder.nestedFilter)
-                && Objects.equals(this.nestedPath, builder.nestedPath) && Objects.equals(this.missing, builder.missing)
+        return (Objects.equals(this.fieldName, builder.fieldName) && Objects.equals(this.missing, builder.missing)
                 && Objects.equals(this.order, builder.order) && Objects.equals(this.sortMode, builder.sortMode)
                 && Objects.equals(this.unmappedType, builder.unmappedType) && Objects.equals(this.nestedSort, builder.nestedSort))
                 && Objects.equals(this.numericType, builder.numericType);
@@ -453,7 +406,7 @@ public boolean equals(Object other) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(this.fieldName, this.nestedFilter, this.nestedPath, this.nestedSort, this.missing, this.order, this.sortMode,
+        return Objects.hash(this.fieldName, this.nestedSort, this.missing, this.order, this.sortMode,
             this.unmappedType, this.numericType);
     }
 
@@ -479,38 +432,22 @@ public static FieldSortBuilder fromXContent(XContentParser parser, String fieldN
 
     static {
         PARSER.declareField(FieldSortBuilder::missing, p -> p.objectText(),  MISSING, ValueType.VALUE);
-        PARSER.declareString((fieldSortBuilder, nestedPath) -> {
-            deprecationLogger.deprecated("[nested_path] has been deprecated in favor of the [nested] parameter");
-            fieldSortBuilder.setNestedPath(nestedPath);
-        }, NESTED_PATH_FIELD);
         PARSER.declareString(FieldSortBuilder::unmappedType , UNMAPPED_TYPE);
         PARSER.declareString((b, v) -> b.order(SortOrder.fromString(v)) , ORDER_FIELD);
         PARSER.declareString((b, v) -> b.sortMode(SortMode.fromString(v)), SORT_MODE);
-        PARSER.declareObject(FieldSortBuilder::setNestedFilter, (p, c) -> {
-            deprecationLogger.deprecated("[nested_filter] has been deprecated in favour for the [nested] parameter");
-            return SortBuilder.parseNestedFilter(p);
-        }, NESTED_FILTER_FIELD);
         PARSER.declareObject(FieldSortBuilder::setNestedSort, (p, c) -> NestedSortBuilder.fromXContent(p), NESTED_FIELD);
         PARSER.declareString((b, v) -> b.setNumericType(v), NUMERIC_TYPE);
     }
 
     @Override
     public FieldSortBuilder rewrite(QueryRewriteContext ctx) throws IOException {
-        if (nestedFilter == null && nestedSort == null) {
+        if (nestedSort == null) {
             return this;
         }
-        if (nestedFilter != null) {
-            QueryBuilder rewrite = nestedFilter.rewrite(ctx);
-            if (nestedFilter == rewrite) {
-                return this;
-            }
-            return new FieldSortBuilder(this).setNestedFilter(rewrite);
-        } else {
-            NestedSortBuilder rewrite = nestedSort.rewrite(ctx);
-            if (nestedSort == rewrite) {
-                return this;
-            }
-            return new FieldSortBuilder(this).setNestedSort(rewrite);
+        NestedSortBuilder rewrite = nestedSort.rewrite(ctx);
+        if (nestedSort == rewrite) {
+            return this;
         }
+        return new FieldSortBuilder(this).setNestedSort(rewrite);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
index 116a54a71103b..630b93b4f34b5 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.sort;
 
-import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.document.LatLonDocValuesField;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NumericDocValues;
@@ -36,7 +35,6 @@
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -65,14 +63,13 @@
 import java.util.Locale;
 import java.util.Objects;
 
-import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder;
+import static org.elasticsearch.search.sort.FieldSortBuilder.validateMissingNestedPath;
 import static org.elasticsearch.search.sort.NestedSortBuilder.NESTED_FIELD;
 
 /**
  * A geo distance based sorting on a geo point like field.
  */
 public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder> {
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(GeoDistanceSortBuilder.class));
 
     public static final String NAME = "_geo_distance";
     public static final String ALTERNATIVE_NAME = "_geoDistance";
@@ -91,8 +88,6 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
     private DistanceUnit unit = DistanceUnit.DEFAULT;
 
     private SortMode sortMode = null;
-    private QueryBuilder nestedFilter;
-    private String nestedPath;
 
     private NestedSortBuilder nestedSort;
 
@@ -151,8 +146,6 @@ public GeoDistanceSortBuilder(String fieldName, String ... geohashes) {
         this.unit = original.unit;
         this.order = original.order;
         this.sortMode = original.sortMode;
-        this.nestedFilter = original.nestedFilter;
-        this.nestedPath = original.nestedPath;
         this.validation = original.validation;
         this.nestedSort = original.nestedSort;
         this.ignoreUnmapped = original.ignoreUnmapped;
@@ -169,8 +162,13 @@ public GeoDistanceSortBuilder(StreamInput in) throws IOException {
         unit = DistanceUnit.readFromStream(in);
         order = SortOrder.readFromStream(in);
         sortMode = in.readOptionalWriteable(SortMode::readFromStream);
-        nestedFilter = in.readOptionalNamedWriteable(QueryBuilder.class);
-        nestedPath = in.readOptionalString();
+        if (in.getVersion().before(Version.V_8_0_0)) {
+            if (in.readOptionalNamedWriteable(QueryBuilder.class) != null || in.readOptionalString() != null) {
+                throw new IOException("the [sort] options [nested_path] and [nested_filter] are removed in 8.x, " +
+                    "please use [nested] instead");
+            }
+
+        }
         nestedSort = in.readOptionalWriteable(NestedSortBuilder::new);
         validation = GeoValidationMethod.readFromStream(in);
         ignoreUnmapped = in.readBoolean();
@@ -184,8 +182,10 @@ public void writeTo(StreamOutput out) throws IOException {
         unit.writeTo(out);
         order.writeTo(out);
         out.writeOptionalWriteable(sortMode);
-        out.writeOptionalNamedWriteable(nestedFilter);
-        out.writeOptionalString(nestedPath);
+        if (out.getVersion().before(Version.V_8_0_0)) {
+            out.writeOptionalNamedWriteable(null);
+            out.writeOptionalString(null);
+        }
         out.writeOptionalWriteable(nestedSort);
         validation.writeTo(out);
         out.writeBoolean(ignoreUnmapped);
@@ -289,59 +289,6 @@ public SortMode sortMode() {
         return this.sortMode;
     }
 
-    /**
-     * Sets the nested filter that the nested objects should match with in order to
-     * be taken into account for sorting.
-     *
-     * @deprecated set nested sort with {@link #setNestedSort(NestedSortBuilder)}
-     *             and retrieve with {@link #getNestedSort()}
-     **/
-   @Deprecated
-    public GeoDistanceSortBuilder setNestedFilter(QueryBuilder nestedFilter) {
-       if (this.nestedSort != null) {
-           throw new IllegalArgumentException("Setting both nested_path/nested_filter and nested not allowed");
-       }
-        this.nestedFilter = nestedFilter;
-        return this;
-    }
-
-    /**
-     * Returns the nested filter that the nested objects should match with in order to be taken into account
-     * for sorting.
-     * @deprecated set nested sort with {@link #setNestedSort(NestedSortBuilder)}
-     *             and retrieve with {@link #getNestedSort()}
-     **/
-    @Deprecated
-    public QueryBuilder getNestedFilter() {
-        return this.nestedFilter;
-    }
-
-    /**
-     * Sets the nested path if sorting occurs on a field that is inside a nested object. By default when sorting on a
-     * field inside a nested object, the nearest upper nested object is selected as nested path.
-     * @deprecated set nested sort with {@link #setNestedSort(NestedSortBuilder)}
-     *             and retrieve with {@link #getNestedSort()}
-     **/
-    @Deprecated
-    public GeoDistanceSortBuilder setNestedPath(String nestedPath) {
-        if (this.nestedSort != null) {
-            throw new IllegalArgumentException("Setting both nested_path/nested_filter and nested not allowed");
-        }
-        this.nestedPath = nestedPath;
-        return this;
-    }
-
-    /**
-     * Returns the nested path if sorting occurs on a field that is inside a nested object. By default when sorting on a
-     * field inside a nested object, the nearest upper nested object is selected as nested path.
-     * @deprecated set nested sort with {@link #setNestedSort(NestedSortBuilder)}
-     *             and retrieve with {@link #getNestedSort()}
-     **/
-    @Deprecated
-    public String getNestedPath() {
-        return this.nestedPath;
-    }
-
     /**
      * Returns the {@link NestedSortBuilder}
      */
@@ -356,9 +303,6 @@ public NestedSortBuilder getNestedSort() {
      * order to be taken into account for sorting.
      */
     public GeoDistanceSortBuilder setNestedSort(final NestedSortBuilder nestedSort) {
-        if (this.nestedFilter != null || this.nestedPath != null) {
-            throw new IllegalArgumentException("Setting both nested_path/nested_filter and nested not allowed");
-        }
         this.nestedSort = nestedSort;
         return this;
     }
@@ -394,12 +338,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(SORTMODE_FIELD.getPreferredName(), sortMode);
         }
 
-        if (nestedPath != null) {
-            builder.field(NESTED_PATH_FIELD.getPreferredName(), nestedPath);
-        }
-        if (nestedFilter != null) {
-            builder.field(NESTED_FILTER_FIELD.getPreferredName(), nestedFilter, params);
-        }
         if (nestedSort != null) {
             builder.field(NESTED_FIELD.getPreferredName(), nestedSort);
         }
@@ -433,8 +371,6 @@ public boolean equals(Object object) {
                 Objects.equals(unit, other.unit) &&
                 Objects.equals(sortMode, other.sortMode) &&
                 Objects.equals(order, other.order) &&
-                Objects.equals(nestedFilter, other.nestedFilter) &&
-                Objects.equals(nestedPath, other.nestedPath) &&
                 Objects.equals(validation, other.validation) &&
                 Objects.equals(nestedSort, other.nestedSort) &&
                 ignoreUnmapped == other.ignoreUnmapped;
@@ -443,8 +379,7 @@ public boolean equals(Object object) {
     @Override
     public int hashCode() {
         return Objects.hash(this.fieldName, this.points, this.geoDistance,
-                this.unit, this.sortMode, this.order, this.nestedFilter,
-                this.nestedPath, this.validation, this.nestedSort, this.ignoreUnmapped);
+                this.unit, this.sortMode, this.order, this.validation, this.nestedSort, this.ignoreUnmapped);
     }
 
     /**
@@ -464,8 +399,6 @@ public static GeoDistanceSortBuilder fromXContent(XContentParser parser, String
         GeoDistance geoDistance = GeoDistance.ARC;
         SortOrder order = SortOrder.ASC;
         SortMode sortMode = null;
-        QueryBuilder nestedFilter = null;
-        String nestedPath = null;
         NestedSortBuilder nestedSort = null;
         GeoValidationMethod validation = null;
         boolean ignoreUnmapped = false;
@@ -480,10 +413,7 @@ public static GeoDistanceSortBuilder fromXContent(XContentParser parser, String
 
                 fieldName = currentName;
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (NESTED_FILTER_FIELD.match(currentName, parser.getDeprecationHandler())) {
-                    deprecationLogger.deprecated("[nested_filter] has been deprecated in favour of the [nested] parameter");
-                    nestedFilter = parseInnerQueryBuilder(parser);
-                } else if (NESTED_FIELD.match(currentName, parser.getDeprecationHandler())) {
+                if (NESTED_FIELD.match(currentName, parser.getDeprecationHandler())) {
                     nestedSort = NestedSortBuilder.fromXContent(parser);
                 } else {
                     // the json in the format of -> field : { lat : 30, lon : 12 }
@@ -510,9 +440,6 @@ public static GeoDistanceSortBuilder fromXContent(XContentParser parser, String
                     validation = GeoValidationMethod.fromString(parser.text());
                 } else if (SORTMODE_FIELD.match(currentName, parser.getDeprecationHandler())) {
                     sortMode = SortMode.fromString(parser.text());
-                } else if (NESTED_PATH_FIELD.match(currentName, parser.getDeprecationHandler())) {
-                    deprecationLogger.deprecated("[nested_path] has been deprecated in favour of the [nested] parameter");
-                    nestedPath = parser.text();
                 } else if (IGNORE_UNMAPPED.match(currentName, parser.getDeprecationHandler())) {
                     ignoreUnmapped = parser.booleanValue();
                 } else if (token == Token.VALUE_STRING){
@@ -550,10 +477,6 @@ public static GeoDistanceSortBuilder fromXContent(XContentParser parser, String
         if (sortMode != null) {
             result.sortMode(sortMode);
         }
-        if (nestedFilter != null) {
-            result.setNestedFilter(nestedFilter);
-        }
-        result.setNestedPath(nestedPath);
         if (nestedSort != null) {
             result.setNestedSort(nestedSort);
         }
@@ -611,20 +534,15 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
         }
         final IndexGeoPointFieldData geoIndexFieldData = context.getForField(fieldType);
 
-        final Nested nested;
+        Nested nested = null;
         if (nestedSort != null) {
-            if (context.indexVersionCreated().before(Version.V_6_5_0) && nestedSort.getMaxChildren() != Integer.MAX_VALUE) {
-                throw new QueryShardException(context,
-                    "max_children is only supported on v6.5.0 or higher");
-            }
             if (nestedSort.getNestedSort() != null && nestedSort.getMaxChildren() != Integer.MAX_VALUE)  {
                 throw new QueryShardException(context,
                     "max_children is only supported on last level of nested sort");
             }
-            // new nested sorts takes priority
             nested = resolveNested(context, nestedSort);
         } else {
-            nested = resolveNested(context, nestedPath, nestedFilter);
+            validateMissingNestedPath(context, fieldName);
         }
 
         if (geoIndexFieldData.getClass() == LatLonPointDVIndexFieldData.class // only works with 5.x geo_point
@@ -703,21 +621,13 @@ static void parseGeoPoints(XContentParser parser, List<GeoPoint> geoPoints) thro
 
     @Override
     public GeoDistanceSortBuilder rewrite(QueryRewriteContext ctx) throws IOException {
-        if (nestedFilter == null && nestedSort == null) {
+        if (nestedSort == null) {
             return this;
         }
-        if (nestedFilter != null) {
-            QueryBuilder rewrite = nestedFilter.rewrite(ctx);
-            if (nestedFilter == rewrite) {
-                return this;
-            }
-            return new GeoDistanceSortBuilder(this).setNestedFilter(rewrite);
-        } else {
-            NestedSortBuilder rewrite = nestedSort.rewrite(ctx);
-            if (nestedSort == rewrite) {
-                return this;
-            }
-            return new GeoDistanceSortBuilder(this).setNestedSort(rewrite);
+        NestedSortBuilder rewrite = nestedSort.rewrite(ctx);
+        if (nestedSort == rewrite) {
+            return this;
         }
+        return new GeoDistanceSortBuilder(this).setNestedSort(rewrite);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
index d858f0f7c6ac3..17fed4d9ac19c 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.sort;
 
-import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Scorable;
@@ -31,7 +30,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -66,7 +64,6 @@
  * Script sort builder allows to sort based on a custom script expression.
  */
 public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(ScriptSortBuilder.class));
 
     public static final String NAME = "_script";
     public static final ParseField TYPE_FIELD = new ParseField("type");
@@ -79,10 +76,6 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
 
     private SortMode sortMode;
 
-    private QueryBuilder nestedFilter;
-
-    private String nestedPath;
-
     private NestedSortBuilder nestedSort;
 
     /**
@@ -106,8 +99,6 @@ public ScriptSortBuilder(Script script, ScriptSortType type) {
         this.type = original.type;
         this.order = original.order;
         this.sortMode = original.sortMode;
-        this.nestedFilter = original.nestedFilter;
-        this.nestedPath = original.nestedPath;
         this.nestedSort = original.nestedSort;
     }
 
@@ -119,8 +110,12 @@ public ScriptSortBuilder(StreamInput in) throws IOException {
         type = ScriptSortType.readFromStream(in);
         order = SortOrder.readFromStream(in);
         sortMode = in.readOptionalWriteable(SortMode::readFromStream);
-        nestedPath = in.readOptionalString();
-        nestedFilter = in.readOptionalNamedWriteable(QueryBuilder.class);
+        if (in.getVersion().before(Version.V_8_0_0)) {
+            if (in.readOptionalNamedWriteable(QueryBuilder.class) != null || in.readOptionalString() != null) {
+                throw new IOException("the [sort] options [nested_path] and [nested_filter] are removed in 8.x, " +
+                    "please use [nested] instead");
+            }
+        }
         nestedSort = in.readOptionalWriteable(NestedSortBuilder::new);
     }
 
@@ -130,8 +125,10 @@ public void writeTo(StreamOutput out) throws IOException {
         type.writeTo(out);
         order.writeTo(out);
         out.writeOptionalWriteable(sortMode);
-        out.writeOptionalString(nestedPath);
-        out.writeOptionalNamedWriteable(nestedFilter);
+        if (out.getVersion().before(Version.V_8_0_0)) {
+            out.writeOptionalString(null);
+            out.writeOptionalNamedWriteable(null);
+        }
         out.writeOptionalWriteable(nestedSort);
     }
 
@@ -170,56 +167,6 @@ public SortMode sortMode() {
         return this.sortMode;
     }
 
-    /**
-     * Sets the nested filter that the nested objects should match with in order to be taken into account
-     * for sorting.
-     *
-     * @deprecated set nested sort with {@link #setNestedSort(NestedSortBuilder)} and retrieve with {@link #getNestedSort()}
-     */
-    @Deprecated
-    public ScriptSortBuilder setNestedFilter(QueryBuilder nestedFilter) {
-        if (this.nestedSort != null) {
-            throw new IllegalArgumentException("Setting both nested_path/nested_filter and nested not allowed");
-        }
-        this.nestedFilter = nestedFilter;
-        return this;
-    }
-
-    /**
-     * Gets the nested filter.
-     *
-     * @deprecated set nested sort with {@link #setNestedSort(NestedSortBuilder)} and retrieve with {@link #getNestedSort()}
-     */
-    @Deprecated
-    public QueryBuilder getNestedFilter() {
-        return this.nestedFilter;
-    }
-
-    /**
-     * Sets the nested path if sorting occurs on a field that is inside a nested object. For sorting by script this
-     * needs to be specified.
-     *
-     * @deprecated set nested sort with {@link #setNestedSort(NestedSortBuilder)} and retrieve with {@link #getNestedSort()}
-     */
-    @Deprecated
-    public ScriptSortBuilder setNestedPath(String nestedPath) {
-        if (this.nestedSort != null) {
-            throw new IllegalArgumentException("Setting both nested_path/nested_filter and nested not allowed");
-        }
-        this.nestedPath = nestedPath;
-        return this;
-    }
-
-    /**
-     * Gets the nested path.
-     *
-     * @deprecated set nested sort with {@link #setNestedSort(NestedSortBuilder)} and retrieve with {@link #getNestedSort()}
-     */
-    @Deprecated
-    public String getNestedPath() {
-        return this.nestedPath;
-    }
-
     /**
      * Returns the {@link NestedSortBuilder}
      */
@@ -234,9 +181,6 @@ public NestedSortBuilder getNestedSort() {
      * order to be taken into account for sorting.
      */
     public ScriptSortBuilder setNestedSort(final NestedSortBuilder nestedSort) {
-        if (this.nestedFilter != null || this.nestedPath != null) {
-            throw new IllegalArgumentException("Setting both nested_path/nested_filter and nested not allowed");
-        }
         this.nestedSort = nestedSort;
         return this;
     }
@@ -251,12 +195,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params builderParams)
         if (sortMode != null) {
             builder.field(SORTMODE_FIELD.getPreferredName(), sortMode);
         }
-        if (nestedPath != null) {
-            builder.field(NESTED_PATH_FIELD.getPreferredName(), nestedPath);
-        }
-        if (nestedFilter != null) {
-            builder.field(NESTED_FILTER_FIELD.getPreferredName(), nestedFilter, builderParams);
-        }
         if (nestedSort != null) {
             builder.field(NESTED_FIELD.getPreferredName(), nestedSort);
         }
@@ -274,14 +212,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params builderParams)
         PARSER.declareField(constructorArg(), p -> ScriptSortType.fromString(p.text()), TYPE_FIELD, ValueType.STRING);
         PARSER.declareString((b, v) -> b.order(SortOrder.fromString(v)), ORDER_FIELD);
         PARSER.declareString((b, v) -> b.sortMode(SortMode.fromString(v)), SORTMODE_FIELD);
-        PARSER.declareString((fieldSortBuilder, nestedPath) -> {
-            deprecationLogger.deprecated("[nested_path] has been deprecated in favor of the [nested] parameter");
-            fieldSortBuilder.setNestedPath(nestedPath);
-        }, NESTED_PATH_FIELD);
-        PARSER.declareObject(ScriptSortBuilder::setNestedFilter, (p, c) -> {
-            deprecationLogger.deprecated("[nested_filter] has been deprecated in favour for the [nested] parameter");
-            return SortBuilder.parseNestedFilter(p);
-        }, NESTED_FILTER_FIELD);
         PARSER.declareObject(ScriptSortBuilder::setNestedSort, (p, c) -> NestedSortBuilder.fromXContent(p), NESTED_FIELD);
     }
 
@@ -310,20 +240,13 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
             valueMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
         }
 
-        final Nested nested;
+        Nested nested = null;
         if (nestedSort != null) {
-            if (context.indexVersionCreated().before(Version.V_6_5_0) && nestedSort.getMaxChildren() != Integer.MAX_VALUE) {
-                throw new QueryShardException(context,
-                    "max_children is only supported on v6.5.0 or higher");
-            }
             if (nestedSort.getNestedSort() != null && nestedSort.getMaxChildren() != Integer.MAX_VALUE)  {
                 throw new QueryShardException(context,
                     "max_children is only supported on last level of nested sort");
             }
-            // new nested sorts takes priority
             nested = resolveNested(context, nestedSort);
-        } else {
-            nested = resolveNested(context, nestedPath, nestedFilter);
         }
 
         final IndexFieldData.XFieldComparatorSource fieldComparatorSource;
@@ -404,14 +327,12 @@ public boolean equals(Object object) {
                 Objects.equals(type, other.type) &&
                 Objects.equals(order, other.order) &&
                 Objects.equals(sortMode, other.sortMode) &&
-                Objects.equals(nestedFilter, other.nestedFilter) &&
-                Objects.equals(nestedPath, other.nestedPath) &&
                 Objects.equals(nestedSort, other.nestedSort);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(script, type, order, sortMode, nestedFilter, nestedPath, nestedSort);
+        return Objects.hash(script, type, order, sortMode, nestedSort);
     }
 
     @Override
@@ -457,21 +378,13 @@ public String toString() {
 
     @Override
     public ScriptSortBuilder rewrite(QueryRewriteContext ctx) throws IOException {
-        if (nestedFilter == null && nestedSort == null) {
+        if (nestedSort == null) {
             return this;
         }
-        if (nestedFilter != null) {
-            QueryBuilder rewrite = nestedFilter.rewrite(ctx);
-            if (nestedFilter == rewrite) {
-                return this;
-            }
-            return new ScriptSortBuilder(this).setNestedFilter(rewrite);
-        } else {
-            NestedSortBuilder rewrite = nestedSort.rewrite(ctx);
-            if (nestedSort == rewrite) {
-                return this;
-            }
-            return new ScriptSortBuilder(this).setNestedSort(rewrite);
+        NestedSortBuilder rewrite = nestedSort.rewrite(ctx);
+        if (nestedSort == rewrite) {
+            return this;
         }
+        return new ScriptSortBuilder(this).setNestedSort(rewrite);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
index 4d793de18443d..881df666802d7 100644
--- a/server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
@@ -46,6 +46,7 @@
 import java.util.Optional;
 
 import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder;
+import static org.elasticsearch.search.sort.NestedSortBuilder.FILTER_FIELD;
 
 public abstract class SortBuilder<T extends SortBuilder<T>> implements NamedWriteable, ToXContentObject, Rewriteable<SortBuilder<?>> {
 
@@ -53,8 +54,6 @@ public abstract class SortBuilder<T extends SortBuilder<T>> implements NamedWrit
 
     // parse fields common to more than one SortBuilder
     public static final ParseField ORDER_FIELD = new ParseField("order");
-    public static final ParseField NESTED_FILTER_FIELD = new ParseField("nested_filter");
-    public static final ParseField NESTED_PATH_FIELD = new ParseField("nested_path");
 
     private static final Map<String, Parser<?>> PARSERS = Map.of(
             ScriptSortBuilder.NAME, ScriptSortBuilder::fromXContent,
@@ -171,12 +170,6 @@ public static Optional<SortAndFormats> buildSort(List<SortBuilder<?>> sortBuilde
         return Optional.empty();
     }
 
-    protected static Nested resolveNested(QueryShardContext context, String nestedPath, QueryBuilder nestedFilter) throws IOException {
-        NestedSortBuilder nestedSortBuilder = new NestedSortBuilder(nestedPath);
-        nestedSortBuilder.setFilter(nestedFilter);
-        return resolveNested(context, nestedSortBuilder);
-    }
-
     protected static Nested resolveNested(QueryShardContext context, NestedSortBuilder nestedSort) throws IOException {
         final Query childQuery = resolveNestedQuery(context, nestedSort, null);
         if (childQuery == null) {
@@ -189,7 +182,7 @@ protected static Nested resolveNested(QueryShardContext context, NestedSortBuild
         } else {
             parentQuery = objectMapper.nestedTypeFilter();
         }
-        return new Nested(context.bitsetFilter(parentQuery), childQuery, nestedSort);
+        return new Nested(context.bitsetFilter(parentQuery), childQuery, nestedSort, context::newCachedSearcher);
     }
 
     private static Query resolveNestedQuery(QueryShardContext context, NestedSortBuilder nestedSort, Query parentQuery) throws IOException {
@@ -207,7 +200,7 @@ private static Query resolveNestedQuery(QueryShardContext context, NestedSortBui
         if (nestedObjectMapper == null) {
             throw new QueryShardException(context, "[nested] failed to find nested object under path [" + nestedPath + "]");
         }
-        if (!nestedObjectMapper.nested().isNested()) {
+        if (nestedObjectMapper.nested().isNested() == false) {
             throw new QueryShardException(context, "[nested] nested object under path [" + nestedPath + "] is not of nested type");
         }
         ObjectMapper objectMapper = context.nestedScope().getObjectMapper();
@@ -256,7 +249,7 @@ protected static QueryBuilder parseNestedFilter(XContentParser parser) {
         try {
             return parseInnerQueryBuilder(parser);
         } catch (Exception e) {
-            throw new ParsingException(parser.getTokenLocation(), "Expected " + NESTED_FILTER_FIELD.getPreferredName() + " element.", e);
+            throw new ParsingException(parser.getTokenLocation(), "Expected " + FILTER_FIELD.getPreferredName() + " element.", e);
         }
     }
 
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/Suggester.java b/server/src/main/java/org/elasticsearch/search/suggest/Suggester.java
index 6c007bf4cb2b1..3146d7a0ffad9 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/Suggester.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/Suggester.java
@@ -29,12 +29,15 @@ public abstract class Suggester<T extends SuggestionSearchContext.SuggestionCont
     protected abstract Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
         innerExecute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException;
 
+    protected abstract Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
+        emptySuggestion(String name, T suggestion, CharsRefBuilder spare) throws IOException;
+
     public Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
                 execute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
-        // #3469 We want to ignore empty shards
 
+        // we only want to output an empty suggestion on empty shards
         if (searcher.getIndexReader().numDocs() == 0) {
-            return null;
+            return emptySuggestion(name, suggestion, spare);
         }
         return innerExecute(name, suggestion, searcher, spare);
     }
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java b/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
index 60ad277deb2f7..3763fd49629b9 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
@@ -49,12 +49,7 @@ protected Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Sugges
             final CompletionSuggestionContext suggestionContext, final IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
         if (suggestionContext.getFieldType() != null) {
             final CompletionFieldMapper.CompletionFieldType fieldType = suggestionContext.getFieldType();
-            CompletionSuggestion completionSuggestion =
-                new CompletionSuggestion(name, suggestionContext.getSize(), suggestionContext.isSkipDuplicates());
-            spare.copyUTF8Bytes(suggestionContext.getText());
-            CompletionSuggestion.Entry completionSuggestEntry = new CompletionSuggestion.Entry(
-                new Text(spare.toString()), 0, spare.length());
-            completionSuggestion.addTerm(completionSuggestEntry);
+            CompletionSuggestion completionSuggestion = emptySuggestion(name, suggestionContext, spare);
             int shardSize = suggestionContext.getShardSize() != null ? suggestionContext.getShardSize() : suggestionContext.getSize();
             TopSuggestGroupDocsCollector collector = new TopSuggestGroupDocsCollector(shardSize, suggestionContext.isSkipDuplicates());
             suggest(searcher, suggestionContext.toQuery(), collector);
@@ -71,7 +66,7 @@ protected Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Sugges
                 if (numResult++ < suggestionContext.getSize()) {
                     CompletionSuggestion.Entry.Option option = new CompletionSuggestion.Entry.Option(suggestDoc.doc,
                         new Text(suggestDoc.key.toString()), suggestDoc.score, contexts);
-                    completionSuggestEntry.addOption(option);
+                    completionSuggestion.getEntries().get(0).addOption(option);
                 } else {
                     break;
                 }
@@ -96,4 +91,14 @@ private static void suggest(IndexSearcher searcher, CompletionQuery query, TopSu
             }
         }
     }
+
+    @Override
+    protected CompletionSuggestion emptySuggestion(String name, CompletionSuggestionContext suggestion, CharsRefBuilder spare)
+            throws IOException {
+        CompletionSuggestion completionSuggestion = new CompletionSuggestion(name, suggestion.getSize(), suggestion.isSkipDuplicates());
+        spare.copyUTF8Bytes(suggestion.getText());
+        CompletionSuggestion.Entry completionSuggestEntry = new CompletionSuggestion.Entry(new Text(spare.toString()), 0, spare.length());
+        completionSuggestion.addTerm(completionSuggestEntry);
+        return completionSuggestion;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
index d80fd68dacbf8..10b9333ae78ee 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
@@ -156,4 +156,13 @@ private static PhraseSuggestion.Entry buildResultEntry(SuggestionContext suggest
         spare.copyUTF8Bytes(suggestion.getText());
         return new PhraseSuggestion.Entry(new Text(spare.toString()), 0, spare.length(), cutoffScore);
     }
+
+    @Override
+    protected Suggestion<? extends Entry<? extends Option>> emptySuggestion(String name, PhraseSuggestionContext suggestion,
+            CharsRefBuilder spare) throws IOException {
+        PhraseSuggestion phraseSuggestion = new PhraseSuggestion(name, suggestion.getSize());
+        spare.copyUTF8Bytes(suggestion.getText());
+        phraseSuggestion.addTerm(new PhraseSuggestion.Entry(new Text(spare.toString()), 0, spare.length()));
+        return phraseSuggestion;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
index 74b9437d67821..5b66badc733e0 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
@@ -31,7 +31,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.index.analysis.CustomAnalyzer;
+import org.elasticsearch.index.analysis.AnalyzerComponentsProvider;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
@@ -675,9 +675,8 @@ private static ShingleTokenFilterFactory.Factory getShingleFilterFactory(Analyze
         if (analyzer instanceof NamedAnalyzer) {
             analyzer = ((NamedAnalyzer)analyzer).analyzer();
         }
-        if (analyzer instanceof CustomAnalyzer) {
-            final CustomAnalyzer a = (CustomAnalyzer) analyzer;
-            final TokenFilterFactory[] tokenFilters = a.tokenFilters();
+        if (analyzer instanceof AnalyzerComponentsProvider) {
+            final TokenFilterFactory[] tokenFilters = ((AnalyzerComponentsProvider) analyzer).getComponents().getTokenFilters();
             for (TokenFilterFactory tokenFilterFactory : tokenFilters) {
                 if (tokenFilterFactory instanceof ShingleTokenFilterFactory) {
                     return ((ShingleTokenFilterFactory)tokenFilterFactory).getInnerFactory();
diff --git a/server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java b/server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
index d6edf352d1911..34ce79af87cb5 100644
--- a/server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
+++ b/server/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
@@ -94,4 +94,16 @@ private Token(Term term, int startOffset, int endOffset) {
         }
 
     }
+
+    @Override
+    protected TermSuggestion emptySuggestion(String name, TermSuggestionContext suggestion, CharsRefBuilder spare) throws IOException {
+        TermSuggestion termSuggestion = new TermSuggestion(name, suggestion.getSize(), suggestion.getDirectSpellCheckerSettings().sort());
+        List<Token> tokens = queryTerms(suggestion, spare);
+        for (Token token : tokens) {
+            Text key = new Text(new BytesArray(token.term.bytes()));
+            TermSuggestion.Entry resultEntry = new TermSuggestion.Entry(key, token.startOffset, token.endOffset - token.startOffset);
+            termSuggestion.addTerm(resultEntry);
+        }
+        return termSuggestion;
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
index 38aa945bcca47..3c7007bd27f17 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
@@ -42,6 +42,7 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 /**
@@ -51,6 +52,7 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
 
     public static final String CONTEXT_MODE_PARAM = "context_mode";
     public static final String CONTEXT_MODE_SNAPSHOT = "SNAPSHOT";
+    public static final Version METADATA_FIELD_INTRODUCED = Version.V_7_3_0;
     private static final DateFormatter DATE_TIME_FORMATTER = DateFormatter.forPattern("strictDateOptionalTime");
     private static final String SNAPSHOT = "snapshot";
     private static final String UUID = "uuid";
@@ -74,6 +76,7 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
     private static final String TOTAL_SHARDS = "total_shards";
     private static final String SUCCESSFUL_SHARDS = "successful_shards";
     private static final String INCLUDE_GLOBAL_STATE = "include_global_state";
+    private static final String USER_METADATA = "metadata";
 
     private static final Comparator<SnapshotInfo> COMPARATOR =
         Comparator.comparing(SnapshotInfo::startTime).thenComparing(SnapshotInfo::snapshotId);
@@ -88,6 +91,7 @@ public static final class SnapshotInfoBuilder {
         private long endTime = 0L;
         private ShardStatsBuilder shardStatsBuilder = null;
         private Boolean includeGlobalState = null;
+        private Map<String, Object> userMetadata = null;
         private int version = -1;
         private List<SnapshotShardFailure> shardFailures = null;
 
@@ -127,6 +131,10 @@ private void setIncludeGlobalState(Boolean includeGlobalState) {
             this.includeGlobalState = includeGlobalState;
         }
 
+        private void setUserMetadata(Map<String, Object> userMetadata) {
+            this.userMetadata = userMetadata;
+        }
+
         private void setVersion(int version) {
             this.version = version;
         }
@@ -153,7 +161,7 @@ public SnapshotInfo build() {
             }
 
             return new SnapshotInfo(snapshotId, indices, snapshotState, reason, version, startTime, endTime,
-                    totalShards, successfulShards, shardFailures, includeGlobalState);
+                    totalShards, successfulShards, shardFailures, includeGlobalState, userMetadata);
         }
     }
 
@@ -194,6 +202,7 @@ int getSuccessfulShards() {
         SNAPSHOT_INFO_PARSER.declareLong(SnapshotInfoBuilder::setEndTime, new ParseField(END_TIME_IN_MILLIS));
         SNAPSHOT_INFO_PARSER.declareObject(SnapshotInfoBuilder::setShardStatsBuilder, SHARD_STATS_PARSER, new ParseField(SHARDS));
         SNAPSHOT_INFO_PARSER.declareBoolean(SnapshotInfoBuilder::setIncludeGlobalState, new ParseField(INCLUDE_GLOBAL_STATE));
+        SNAPSHOT_INFO_PARSER.declareObject(SnapshotInfoBuilder::setUserMetadata, (p, c) -> p.map() , new ParseField(USER_METADATA));
         SNAPSHOT_INFO_PARSER.declareInt(SnapshotInfoBuilder::setVersion, new ParseField(VERSION_ID));
         SNAPSHOT_INFO_PARSER.declareObjectArray(SnapshotInfoBuilder::setShardFailures, SnapshotShardFailure.SNAPSHOT_SHARD_FAILURE_PARSER,
             new ParseField(FAILURES));
@@ -223,6 +232,9 @@ int getSuccessfulShards() {
     @Nullable
     private final Boolean includeGlobalState;
 
+    @Nullable
+    private final Map<String, Object> userMetadata;
+
     @Nullable
     private final Version version;
 
@@ -230,28 +242,30 @@ int getSuccessfulShards() {
 
     public SnapshotInfo(SnapshotId snapshotId, List<String> indices, SnapshotState state) {
         this(snapshotId, indices, state, null, null, 0L, 0L, 0, 0,
-            Collections.emptyList(), null);
+            Collections.emptyList(), null, null);
     }
 
     public SnapshotInfo(SnapshotId snapshotId, List<String> indices, SnapshotState state, Version version) {
         this(snapshotId, indices, state, null, version, 0L, 0L, 0, 0,
-            Collections.emptyList(), null);
+            Collections.emptyList(), null, null);
     }
 
-    public SnapshotInfo(SnapshotId snapshotId, List<String> indices, long startTime, Boolean includeGlobalState) {
+    public SnapshotInfo(SnapshotId snapshotId, List<String> indices, long startTime, Boolean includeGlobalState,
+                        Map<String, Object> userMetadata) {
         this(snapshotId, indices, SnapshotState.IN_PROGRESS, null, Version.CURRENT, startTime, 0L,
-            0, 0, Collections.emptyList(), includeGlobalState);
+            0, 0, Collections.emptyList(), includeGlobalState, userMetadata);
     }
 
     public SnapshotInfo(SnapshotId snapshotId, List<String> indices, long startTime, String reason, long endTime,
-                        int totalShards, List<SnapshotShardFailure> shardFailures, Boolean includeGlobalState) {
+                        int totalShards, List<SnapshotShardFailure> shardFailures, Boolean includeGlobalState,
+                        Map<String, Object> userMetadata) {
         this(snapshotId, indices, snapshotState(reason, shardFailures), reason, Version.CURRENT,
-             startTime, endTime, totalShards, totalShards - shardFailures.size(), shardFailures, includeGlobalState);
+             startTime, endTime, totalShards, totalShards - shardFailures.size(), shardFailures, includeGlobalState, userMetadata);
     }
 
     private SnapshotInfo(SnapshotId snapshotId, List<String> indices, SnapshotState state, String reason, Version version,
                          long startTime, long endTime, int totalShards, int successfulShards, List<SnapshotShardFailure> shardFailures,
-                         Boolean includeGlobalState) {
+                         Boolean includeGlobalState, Map<String, Object> userMetadata) {
         this.snapshotId = Objects.requireNonNull(snapshotId);
         this.indices = Collections.unmodifiableList(Objects.requireNonNull(indices));
         this.state = state;
@@ -263,6 +277,7 @@ private SnapshotInfo(SnapshotId snapshotId, List<String> indices, SnapshotState
         this.successfulShards = successfulShards;
         this.shardFailures = Objects.requireNonNull(shardFailures);
         this.includeGlobalState = includeGlobalState;
+        this.userMetadata = userMetadata;
     }
 
     /**
@@ -294,6 +309,11 @@ public SnapshotInfo(final StreamInput in) throws IOException {
         }
         version = in.readBoolean() ? Version.readVersion(in) : null;
         includeGlobalState = in.readOptionalBoolean();
+        if (in.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) {
+            userMetadata = in.readMap();
+        } else {
+            userMetadata = null;
+        }
     }
 
     /**
@@ -304,7 +324,7 @@ public static SnapshotInfo incompatible(SnapshotId snapshotId) {
         return new SnapshotInfo(snapshotId, Collections.emptyList(), SnapshotState.INCOMPATIBLE,
                                 "the snapshot is incompatible with the current version of Elasticsearch and its exact version is unknown",
                                 null, 0L, 0L, 0, 0,
-                                Collections.emptyList(), null);
+                                Collections.emptyList(), null, null);
     }
 
     /**
@@ -428,6 +448,15 @@ public Version version() {
         return version;
     }
 
+    /**
+     * Returns the custom metadata that was attached to this snapshot at creation time.
+     * @return custom metadata
+     */
+    @Nullable
+    public Map<String, Object> userMetadata() {
+        return userMetadata;
+    }
+
     /**
      * Compares two snapshots by their start time; if the start times are the same, then
      * compares the two snapshots by their snapshot ids.
@@ -492,6 +521,9 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
         if (includeGlobalState != null) {
             builder.field(INCLUDE_GLOBAL_STATE, includeGlobalState);
         }
+        if (userMetadata != null) {
+            builder.field(USER_METADATA, userMetadata);
+        }
         if (verbose || state != null) {
             builder.field(STATE, state);
         }
@@ -543,6 +575,7 @@ private XContentBuilder toXContentInternal(final XContentBuilder builder, final
         if (includeGlobalState != null) {
             builder.field(INCLUDE_GLOBAL_STATE, includeGlobalState);
         }
+        builder.field(USER_METADATA, userMetadata);
         builder.field(START_TIME, startTime);
         builder.field(END_TIME, endTime);
         builder.field(TOTAL_SHARDS, totalShards);
@@ -573,6 +606,7 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr
         int totalShards = 0;
         int successfulShards = 0;
         Boolean includeGlobalState = null;
+        Map<String, Object> userMetadata = null;
         List<SnapshotShardFailure> shardFailures = Collections.emptyList();
         if (parser.currentToken() == null) { // fresh parser? move to the first token
             parser.nextToken();
@@ -628,8 +662,12 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr
                                 parser.skipChildren();
                             }
                         } else if (token == XContentParser.Token.START_OBJECT) {
-                            // It was probably created by newer version - ignoring
-                            parser.skipChildren();
+                            if (USER_METADATA.equals(currentFieldName)) {
+                                userMetadata = parser.map();
+                            } else {
+                                // It was probably created by newer version - ignoring
+                                parser.skipChildren();
+                            }
                         }
                     }
                 }
@@ -651,7 +689,8 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr
                                 totalShards,
                                 successfulShards,
                                 shardFailures,
-                                includeGlobalState);
+                                includeGlobalState,
+                                userMetadata);
     }
 
     @Override
@@ -683,6 +722,9 @@ public void writeTo(final StreamOutput out) throws IOException {
             out.writeBoolean(false);
         }
         out.writeOptionalBoolean(includeGlobalState);
+        if (out.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) {
+            out.writeMap(userMetadata);
+        }
     }
 
     private static SnapshotState snapshotState(final String reason, final List<SnapshotShardFailure> shardFailures) {
@@ -712,13 +754,14 @@ public boolean equals(Object o) {
             Objects.equals(indices, that.indices) &&
             Objects.equals(includeGlobalState, that.includeGlobalState) &&
             Objects.equals(version, that.version) &&
-            Objects.equals(shardFailures, that.shardFailures);
+            Objects.equals(shardFailures, that.shardFailures) &&
+            Objects.equals(userMetadata, that.userMetadata);
     }
 
     @Override
     public int hashCode() {
 
         return Objects.hash(snapshotId, state, reason, indices, startTime, endTime,
-                totalShards, successfulShards, includeGlobalState, version, shardFailures);
+                totalShards, successfulShards, includeGlobalState, version, shardFailures, userMetadata);
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
index 65e1191211ec2..40ddc9a320056 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
@@ -66,6 +66,7 @@
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.Repository;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequestDeduplicator;
@@ -634,7 +635,7 @@ protected UpdateIndexShardSnapshotStatusResponse newResponse() {
         }
 
         @Override
-        protected void masterOperation(UpdateIndexShardSnapshotStatusRequest request, ClusterState state,
+        protected void masterOperation(Task task, UpdateIndexShardSnapshotStatusRequest request, ClusterState state,
                                        ActionListener<UpdateIndexShardSnapshotStatusResponse> listener) {
             innerUpdateSnapshotState(request, listener);
         }
diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index a6138b8f6052b..2783d635c90a4 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -287,7 +287,8 @@ public ClusterState execute(ClusterState currentState) {
                                                                 snapshotIndices,
                                                                 System.currentTimeMillis(),
                                                                 repositoryData.getGenId(),
-                                                                null);
+                                                                null,
+                                                                request.userMetadata());
                     initializingSnapshots.add(newSnapshot.snapshot());
                     snapshots = new SnapshotsInProgress(newSnapshot);
                 } else {
@@ -557,7 +558,8 @@ private void cleanupAfterError(Exception exception) {
                                                          0,
                                                          Collections.emptyList(),
                                                          snapshot.getRepositoryStateId(),
-                                                         snapshot.includeGlobalState());
+                                                         snapshot.includeGlobalState(),
+                                                         snapshot.userMetadata());
                 } catch (Exception inner) {
                     inner.addSuppressed(exception);
                     logger.warn(() -> new ParameterizedMessage("[{}] failed to close snapshot in repository",
@@ -572,7 +574,7 @@ private void cleanupAfterError(Exception exception) {
     private static SnapshotInfo inProgressSnapshot(SnapshotsInProgress.Entry entry) {
         return new SnapshotInfo(entry.snapshot().getSnapshotId(),
                                    entry.indices().stream().map(IndexId::getName).collect(Collectors.toList()),
-                                   entry.startTime(), entry.includeGlobalState());
+                                   entry.startTime(), entry.includeGlobalState(), entry.userMetadata());
     }
 
     /**
@@ -988,7 +990,8 @@ protected void doRun() {
                     entry.shards().size(),
                     unmodifiableList(shardFailures),
                     entry.getRepositoryStateId(),
-                    entry.includeGlobalState());
+                    entry.includeGlobalState(),
+                    entry.userMetadata());
                 removeSnapshotFromClusterState(snapshot, snapshotInfo, null);
                 logger.info("snapshot [{}] completed with state [{}]", snapshot, snapshotInfo.state());
             }
diff --git a/server/src/main/java/org/elasticsearch/snapshots/package-info.java b/server/src/main/java/org/elasticsearch/snapshots/package-info.java
index d5149c9bd3592..010e63eae6b4f 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/package-info.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/package-info.java
@@ -20,7 +20,7 @@
 /**
  * <p>This package exposes the Elasticsearch Snapshot functionality.</p>
  *
- * <h1>Preliminaries</h1>
+ * <h2>Preliminaries</h2>
  *
  * <p>There are two communication channels between all nodes and master in the snapshot functionality:</p>
  * <ul>
@@ -32,7 +32,7 @@
  * snapshot's entry in the cluster state accordingly.</li>
  * </ul>
  *
- * <h1>Snapshot Creation</h1>
+ * <h2>Snapshot Creation</h2>
  * <p>Snapshots are created by the following sequence of events:</p>
  * <ol>
  * <li>An invocation of {@link org.elasticsearch.snapshots.SnapshotsService#createSnapshot} enqueues a cluster state update to create
@@ -68,7 +68,7 @@
  * </li>
  * </ol>
  *
- * <h1>Deleting a Snapshot</h1>
+ * <h2>Deleting a Snapshot</h2>
  *
  * <p>Deleting a snapshot can take the form of either simply deleting it from the repository or (if it has not completed yet) aborting it
  * and subsequently deleting it from the repository.</p>
diff --git a/server/src/main/java/org/elasticsearch/threadpool/Scheduler.java b/server/src/main/java/org/elasticsearch/threadpool/Scheduler.java
index 588495dd27d32..04741cd87d04c 100644
--- a/server/src/main/java/org/elasticsearch/threadpool/Scheduler.java
+++ b/server/src/main/java/org/elasticsearch/threadpool/Scheduler.java
@@ -33,6 +33,7 @@
 import java.util.concurrent.Delayed;
 import java.util.concurrent.Future;
 import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.RunnableFuture;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.ThreadFactory;
@@ -276,7 +277,11 @@ protected void afterExecute(Runnable r, Throwable t) {
             if (t != null) return;
             // Scheduler only allows Runnable's so we expect no checked exceptions here. If anyone uses submit directly on `this`, we
             // accept the wrapped exception in the output.
-            ExceptionsHelper.reThrowIfNotNull(EsExecutors.rethrowErrors(r));
+            if (r instanceof RunnableFuture && ((RunnableFuture<?>) r).isDone()) {
+                // only check this if task is done, which it always is except for periodic tasks. Periodic tasks will hang on
+                // RunnableFuture.get()
+                ExceptionsHelper.reThrowIfNotNull(EsExecutors.rethrowErrors(r));
+            }
         }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
index 316fcd275a5a0..3126963f36930 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.ClusterNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.Tuple;
@@ -37,7 +38,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -136,7 +136,7 @@ static DiscoveryNode buildSeedNode(String clusterName, String address, boolean p
             TransportAddress transportAddress = new TransportAddress(TransportAddress.META_ADDRESS, 0);
             String hostName = address.substring(0, indexOfPortSeparator(address));
             return new DiscoveryNode("", clusterName + "#" + address, UUIDs.randomBase64UUID(), hostName, address,
-                    transportAddress, Collections.singletonMap("server_name", hostName), EnumSet.allOf(DiscoveryNode.Role.class),
+                    transportAddress, Collections.singletonMap("server_name", hostName), DiscoveryNodeRole.BUILT_IN_ROLES,
                     Version.CURRENT.minimumCompatibilityVersion());
         } else {
             TransportAddress transportAddress = new TransportAddress(RemoteClusterAware.parseSeedAddress(address));
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAwareClient.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAwareClient.java
index 2ca42ff85abdf..69b8a1f68a682 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAwareClient.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAwareClient.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.transport;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.ActionRequest;
@@ -44,7 +44,7 @@ final class RemoteClusterAwareClient extends AbstractClient {
 
     @Override
     protected <Request extends ActionRequest, Response extends ActionResponse>
-    void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+    void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
         remoteClusterService.ensureConnected(clusterAlias, ActionListener.wrap(res -> {
             Transport.Connection connection;
             if (request instanceof RemoteClusterAwareRequest) {
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
index 6b6ce18cfefbb..ca03e59a75858 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java
@@ -176,6 +176,7 @@ boolean isSkipUnavailable() {
     @Override
     public void onNodeDisconnected(DiscoveryNode node) {
         boolean remove = connectedNodes.remove(node);
+        logger.trace("node disconnected: {}, removed: {}", node, remove);
         if (remove && connectedNodes.size() < maxNumRemoteConnections) {
             // try to reconnect and fill up the slot of the disconnected node
             connectHandler.forceConnect();
diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
index 4f690d12acf1e..db4e0f021e05b 100644
--- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
+++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java
@@ -183,7 +183,8 @@ private synchronized void updateRemoteClusters(Map<String, Tuple<String, List<Tu
                     remote = new RemoteClusterConnection(settings, clusterAlias, seedList, transportService, numRemoteConnections,
                         getNodePredicate(settings), proxyAddress, connectionProfile);
                     remoteClusters.put(clusterAlias, remote);
-                } else if (connectionProfileChanged(remote.getConnectionManager().getConnectionProfile(), connectionProfile)) {
+                } else if (connectionProfileChanged(remote.getConnectionManager().getConnectionProfile(), connectionProfile)
+                        || seedsChanged(remote.getSeedNodes(), seedList)) {
                     // New ConnectionProfile. Must tear down existing connection
                     try {
                         IOUtils.close(remote);
@@ -421,6 +422,16 @@ private boolean connectionProfileChanged(ConnectionProfile oldProfile, Connectio
             || Objects.equals(oldProfile.getPingInterval(), newProfile.getPingInterval()) == false;
     }
 
+    private boolean seedsChanged(final List<Tuple<String, Supplier<DiscoveryNode>>> oldSeedNodes,
+                                 final List<Tuple<String, Supplier<DiscoveryNode>>> newSeedNodes) {
+        if (oldSeedNodes.size() != newSeedNodes.size()) {
+            return true;
+        }
+        Set<String> oldSeeds = oldSeedNodes.stream().map(Tuple::v1).collect(Collectors.toSet());
+        Set<String> newSeeds = newSeedNodes.stream().map(Tuple::v1).collect(Collectors.toSet());
+        return oldSeeds.equals(newSeeds) == false;
+    }
+
     /**
      * Collects all nodes of the given clusters and returns / passes a (clusterAlias, nodeId) to {@link DiscoveryNode}
      * function on success.
diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java
index eef9f4f42637c..ad9059dbc3757 100644
--- a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java
+++ b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java
@@ -280,7 +280,7 @@ private List<TcpChannel> initiateConnection(DiscoveryNode node, ConnectionProfil
         for (int i = 0; i < numConnections; ++i) {
             try {
                 TcpChannel channel = initiateChannel(node);
-                logger.trace(() -> new ParameterizedMessage("Tcp transport client channel opened: {}", channel));
+                logger.trace(() -> new ParameterizedMessage("Tcp transport channel opened: {}", channel));
                 channels.add(channel);
             } catch (ConnectTransportException e) {
                 CloseableChannel.closeChannels(channels, false);
@@ -933,34 +933,20 @@ public void onResponse(Void v) {
             if (countDown.countDown()) {
                 final TcpChannel handshakeChannel = channels.get(0);
                 try {
-                    executeHandshake(node, handshakeChannel, connectionProfile, new ActionListener<Version>() {
-                        @Override
-                        public void onResponse(Version version) {
-                            NodeChannels nodeChannels = new NodeChannels(node, channels, connectionProfile, version);
-                            long relativeMillisTime = threadPool.relativeTimeInMillis();
-                            nodeChannels.channels.forEach(ch -> {
-                                // Mark the channel init time
-                                ch.getChannelStats().markAccessed(relativeMillisTime);
-                                ch.addCloseListener(ActionListener.wrap(nodeChannels::close));
-                            });
-                            keepAlive.registerNodeConnection(nodeChannels.channels, connectionProfile);
-                            listener.onResponse(nodeChannels);
-                        }
-
-                        @Override
-                        public void onFailure(Exception e) {
-                            CloseableChannel.closeChannels(channels, false);
-
-                            if (e instanceof ConnectTransportException) {
-                                listener.onFailure(e);
-                            } else {
-                                listener.onFailure(new ConnectTransportException(node, "general node connection failure", e));
-                            }
-                        }
-                    });
+                    executeHandshake(node, handshakeChannel, connectionProfile, ActionListener.wrap(version -> {
+                        NodeChannels nodeChannels = new NodeChannels(node, channels, connectionProfile, version);
+                        long relativeMillisTime = threadPool.relativeTimeInMillis();
+                        nodeChannels.channels.forEach(ch -> {
+                            // Mark the channel init time
+                            ch.getChannelStats().markAccessed(relativeMillisTime);
+                            ch.addCloseListener(ActionListener.wrap(nodeChannels::close));
+                        });
+                        keepAlive.registerNodeConnection(nodeChannels.channels, connectionProfile);
+                        listener.onResponse(nodeChannels);
+                    }, e -> closeAndFail(e instanceof ConnectTransportException ?
+                        e : new ConnectTransportException(node, "general node connection failure", e))));
                 } catch (Exception ex) {
-                    CloseableChannel.closeChannels(channels, false);
-                    listener.onFailure(ex);
+                    closeAndFail(ex);
                 }
             }
         }
@@ -968,15 +954,23 @@ public void onFailure(Exception e) {
         @Override
         public void onFailure(Exception ex) {
             if (countDown.fastForward()) {
-                CloseableChannel.closeChannels(channels, false);
-                listener.onFailure(new ConnectTransportException(node, "connect_exception", ex));
+                closeAndFail(new ConnectTransportException(node, "connect_exception", ex));
             }
         }
 
         public void onTimeout() {
             if (countDown.fastForward()) {
+                closeAndFail(new ConnectTransportException(node, "connect_timeout[" + connectionProfile.getConnectTimeout() + "]"));
+            }
+        }
+
+        private void closeAndFail(Exception e) {
+            try {
                 CloseableChannel.closeChannels(channels, false);
-                listener.onFailure(new ConnectTransportException(node, "connect_timeout[" + connectionProfile.getConnectTimeout() + "]"));
+            } catch (Exception ex) {
+                e.addSuppressed(ex);
+            } finally {
+                listener.onFailure(e);
             }
         }
     }
diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java
index a89784945db93..dca7f52e60474 100644
--- a/server/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -24,8 +24,6 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.CheckedBiConsumer;
@@ -109,8 +107,6 @@ protected boolean removeEldestEntry(Map.Entry eldest) {
 
     private final RemoteClusterService remoteClusterService;
 
-    private final boolean validateConnections;
-
     /** if set will call requests sent to this id to shortcut and executed locally */
     volatile DiscoveryNode localNode = null;
     private final Transport.Connection localNodeConnection = new Transport.Connection() {
@@ -155,9 +151,6 @@ public TransportService(Settings settings, Transport transport, ThreadPool threa
     public TransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor transportInterceptor,
                             Function<BoundTransportAddress, DiscoveryNode> localNodeFactory, @Nullable ClusterSettings clusterSettings,
                             Set<String> taskHeaders, ConnectionManager connectionManager) {
-        // The only time we do not want to validate node connections is when this is a transport client using the simple node sampler
-        this.validateConnections = TransportClient.CLIENT_TYPE.equals(settings.get(Client.CLIENT_TYPE_SETTING_S.getKey())) == false ||
-            TransportClient.CLIENT_TRANSPORT_SNIFF.get(settings);
         this.transport = transport;
         this.threadPool = threadPool;
         this.localNodeFactory = localNodeFactory;
@@ -348,7 +341,7 @@ public CheckedBiConsumer<Transport.Connection, ConnectionProfile, IOException> c
         return (newConnection, actualProfile) -> {
             // We don't validate cluster names to allow for CCS connections.
             final DiscoveryNode remote = handshake(newConnection, actualProfile.getHandshakeTimeout().millis(), cn -> true).discoveryNode;
-            if (validateConnections && node.equals(remote) == false) {
+            if (node.equals(remote) == false) {
                 throw new ConnectTransportException(node, "handshake failed. unexpected remote node " + remote);
             }
         };
diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy
index 4df99ef6f8836..415828b8a533d 100644
--- a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy
+++ b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy
@@ -47,7 +47,7 @@ grant codeBase "${codebase.lucene-misc}" {
   permission java.nio.file.LinkPermission "hard";
 };
 
-grant codeBase "${codebase.plugin-classloader}" {
+grant codeBase "${codebase.elasticsearch-plugin-classloader}" {
   // needed to create the classloader which allows plugins to extend other plugins
   permission java.lang.RuntimePermission "createClassLoader";
 };
@@ -55,6 +55,8 @@ grant codeBase "${codebase.plugin-classloader}" {
 //// Everything else:
 
 grant {
+  // needed by vendored Guice
+  permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.vm.annotation";
 
   // checked by scripting engines, and before hacks and other issues in
   // third party code, to safeguard these against unprivileged code like scripts.
diff --git a/server/src/test/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java b/server/src/test/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java
index 4e4b04d1ff19c..3c24dc2d42b82 100644
--- a/server/src/test/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java
+++ b/server/src/test/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java
@@ -30,7 +30,6 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.CommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.IndexSearcher;
@@ -147,21 +146,6 @@ public void testMultiPhrasePrefixQuery() throws Exception {
             BreakIterator.getSentenceInstance(Locale.ROOT), 0, outputs);
     }
 
-    public void testCommonTermsQuery() throws Exception {
-        final String[] inputs = {
-            "The quick brown fox."
-        };
-        final String[] outputs = {
-            "The <b>quick</b> <b>brown</b> <b>fox</b>."
-        };
-        CommonTermsQuery query = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 128);
-        query.add(new Term("text", "quick"));
-        query.add(new Term("text", "brown"));
-        query.add(new Term("text", "fox"));
-        assertHighlightOneDoc("text", inputs, new StandardAnalyzer(), query, Locale.ROOT,
-            BreakIterator.getSentenceInstance(Locale.ROOT), 0, outputs);
-    }
-
     public void testSentenceBoundedBreakIterator() throws Exception {
         final String[] inputs = {
             "The quick brown fox in a long sentence with another quick brown fox. " +
diff --git a/server/src/test/java/org/apache/lucene/util/CombinedBitSetTests.java b/server/src/test/java/org/apache/lucene/util/CombinedBitSetTests.java
new file mode 100644
index 0000000000000..65165272f5469
--- /dev/null
+++ b/server/src/test/java/org/apache/lucene/util/CombinedBitSetTests.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.util;
+
+import org.apache.lucene.search.DocIdSetIterator;
+import org.elasticsearch.test.ESTestCase;
+
+public class CombinedBitSetTests extends ESTestCase {
+    public void testEmpty() {
+        for (float percent : new float[] {0f, 0.1f, 0.5f, 0.9f, 1f}) {
+            testCase(randomIntBetween(1, 10000), 0f, percent);
+            testCase(randomIntBetween(1, 10000), percent, 0f);
+        }
+    }
+
+    public void testSparse() {
+        for (float percent : new float[] {0f, 0.1f, 0.5f, 0.9f, 1f}) {
+            testCase(randomIntBetween(1, 10000), 0.1f, percent);
+            testCase(randomIntBetween(1, 10000), percent, 0.1f);
+        }
+    }
+
+    public void testDense() {
+        for (float percent : new float[] {0f, 0.1f, 0.5f, 0.9f, 1f}) {
+            testCase(randomIntBetween(1, 10000), 0.9f, percent);
+            testCase(randomIntBetween(1, 10000), percent, 0.9f);
+        }
+    }
+
+    public void testRandom() {
+        int iterations = atLeast(10);
+        for (int i = 0; i < iterations; i++) {
+            testCase(randomIntBetween(1, 10000), randomFloat(), randomFloat());
+        }
+    }
+
+    private void testCase(int numBits, float percent1, float percent2) {
+        BitSet first = randomSet(numBits, percent1);
+        BitSet second = randomSet(numBits, percent2);
+        CombinedBitSet actual = new CombinedBitSet(first, second);
+        FixedBitSet expected = new FixedBitSet(numBits);
+        or(expected, first);
+        and(expected, second);
+        assertEquals(expected.cardinality(), actual.cardinality());
+        assertEquals(expected, actual, numBits);
+        for (int i = 0; i < numBits; ++i) {
+            assertEquals(expected.nextSetBit(i), actual.nextSetBit(i));
+            assertEquals(Integer.toString(i), expected.prevSetBit(i), actual.prevSetBit(i));
+        }
+    }
+
+    private void or(BitSet set1, BitSet set2) {
+        int next = 0;
+        while (next < set2.length() && (next = set2.nextSetBit(next)) != DocIdSetIterator.NO_MORE_DOCS) {
+            set1.set(next);
+            next += 1;
+        }
+    }
+
+    private void and(BitSet set1, BitSet set2) {
+        int next = 0;
+        while (next < set1.length() && (next = set1.nextSetBit(next)) != DocIdSetIterator.NO_MORE_DOCS) {
+            if (set2.get(next) == false) {
+                set1.clear(next);
+            }
+            next += 1;
+        }
+    }
+
+    private void assertEquals(BitSet set1, BitSet set2, int maxDoc) {
+        for (int i = 0; i < maxDoc; ++i) {
+            assertEquals("Different at " + i, set1.get(i), set2.get(i));
+        }
+    }
+
+    private BitSet randomSet(int numBits, float percentSet) {
+        return randomSet(numBits, (int) (percentSet * numBits));
+    }
+
+    private BitSet randomSet(int numBits, int numBitsSet) {
+        assert numBitsSet <= numBits;
+        final BitSet set = randomBoolean() ? new SparseFixedBitSet(numBits) : new FixedBitSet(numBits);
+        for (int i = 0; i < numBitsSet; ++i) {
+            while (true) {
+                final int o = random().nextInt(numBits);
+                if (set.get(o) == false) {
+                    set.set(o);
+                    break;
+                }
+            }
+        }
+        return set;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/BuildTests.java b/server/src/test/java/org/elasticsearch/BuildTests.java
index 7a6f581bd7369..6e1c92a8a76bc 100644
--- a/server/src/test/java/org/elasticsearch/BuildTests.java
+++ b/server/src/test/java/org/elasticsearch/BuildTests.java
@@ -58,16 +58,6 @@ public void testIsProduction() {
         );
         assertTrue(build.getQualifiedVersion(), build.isProductionRelease());
 
-        assertFalse(new Build(
-            Build.CURRENT.flavor(), Build.CURRENT.type(), Build.CURRENT.shortHash(), Build.CURRENT.date(),
-            Build.CURRENT.isSnapshot(), "7.0.0-alpha1"
-        ).isProductionRelease());
-
-        assertFalse(new Build(
-            Build.CURRENT.flavor(), Build.CURRENT.type(), Build.CURRENT.shortHash(), Build.CURRENT.date(),
-            Build.CURRENT.isSnapshot(), "7.0.0-alpha1-SNAPSHOT"
-        ).isProductionRelease());
-
         assertFalse(new Build(
             Build.CURRENT.flavor(), Build.CURRENT.type(), Build.CURRENT.shortHash(), Build.CURRENT.date(),
             Build.CURRENT.isSnapshot(), "7.0.0-SNAPSHOT"
diff --git a/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java b/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java
index 2de2f259e6ff1..3b5d1ad43da97 100644
--- a/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/ExceptionsHelperTests.java
@@ -35,9 +35,9 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.RemoteClusterAware;
 
+import java.io.IOException;
 import java.util.Optional;
 
-import static org.elasticsearch.ExceptionsHelper.MAX_ITERATIONS;
 import static org.elasticsearch.ExceptionsHelper.maybeError;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
@@ -81,20 +81,14 @@ public void testMaybeError() {
         if (fatal) {
             assertError(cause, error);
         } else {
-            assertFalse(maybeError(cause, logger).isPresent());
+            assertFalse(maybeError(cause).isPresent());
         }
 
-        assertFalse(maybeError(new Exception(new DecoderException()), logger).isPresent());
-
-        Throwable chain = outOfMemoryError;
-        for (int i = 0; i < MAX_ITERATIONS; i++) {
-            chain = new Exception(chain);
-        }
-        assertFalse(maybeError(chain, logger).isPresent());
+        assertFalse(maybeError(new Exception(new DecoderException())).isPresent());
     }
 
     private void assertError(final Throwable cause, final Error error) {
-        final Optional<Error> maybeError = maybeError(cause, logger);
+        final Optional<Error> maybeError = maybeError(cause);
         assertTrue(maybeError.isPresent());
         assertThat(maybeError.get(), equalTo(error));
     }
@@ -211,4 +205,29 @@ public void testUnwrapCorruption() {
         withSuppressedException.addSuppressed(new RuntimeException());
         assertThat(ExceptionsHelper.unwrapCorruption(withSuppressedException), nullValue());
     }
+
+    public void testSuppressedCycle() {
+        RuntimeException e1 = new RuntimeException();
+        RuntimeException e2 = new RuntimeException();
+        e1.addSuppressed(e2);
+        e2.addSuppressed(e1);
+        ExceptionsHelper.unwrapCorruption(e1);
+
+        final CorruptIndexException corruptIndexException = new CorruptIndexException("corrupt", "resource");
+        RuntimeException e3 = new RuntimeException(corruptIndexException);
+        e3.addSuppressed(e1);
+        assertThat(ExceptionsHelper.unwrapCorruption(e3), equalTo(corruptIndexException));
+
+        RuntimeException e4 = new RuntimeException(e1);
+        e4.addSuppressed(corruptIndexException);
+        assertThat(ExceptionsHelper.unwrapCorruption(e4), equalTo(corruptIndexException));
+    }
+
+    public void testCauseCycle() {
+        RuntimeException e1 = new RuntimeException();
+        RuntimeException e2 = new RuntimeException(e1);
+        e1.initCause(e2);
+        ExceptionsHelper.unwrap(e1, IOException.class);
+        ExceptionsHelper.unwrapCorruption(e1);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/VersionTests.java b/server/src/test/java/org/elasticsearch/VersionTests.java
index e5149b9bce515..7705b435fb1b7 100644
--- a/server/src/test/java/org/elasticsearch/VersionTests.java
+++ b/server/src/test/java/org/elasticsearch/VersionTests.java
@@ -66,11 +66,6 @@ public void testVersionComparison() throws Exception {
         assertThat(V_7_2_0.onOrAfter(V_7_2_0), is(true));
         assertThat(V_8_0_0.onOrAfter(V_7_2_0), is(true));
 
-        assertTrue(Version.fromString("5.0.0-alpha2").onOrAfter(Version.fromString("5.0.0-alpha1")));
-        assertTrue(Version.fromString("5.0.0").onOrAfter(Version.fromString("5.0.0-beta2")));
-        assertTrue(Version.fromString("5.0.0-rc1").onOrAfter(Version.fromString("5.0.0-beta24")));
-        assertTrue(Version.fromString("5.0.0-alpha24").before(Version.fromString("5.0.0-beta0")));
-
         assertThat(V_7_2_0, is(lessThan(V_8_0_0)));
         assertThat(V_7_2_0.compareTo(V_7_2_0), is(0));
         assertThat(V_8_0_0, is(greaterThan(V_7_2_0)));
@@ -181,12 +176,12 @@ public void testMinCompatVersion() {
         assertThat(Version.fromString("5.3.0").minimumCompatibilityVersion(), equalTo(major5x));
 
         Version major56x = Version.fromString("5.6.0");
-        assertThat(Version.V_6_5_0.minimumCompatibilityVersion(), equalTo(major56x));
+        assertThat(Version.fromString("6.4.0").minimumCompatibilityVersion(), equalTo(major56x));
         assertThat(Version.fromString("6.3.1").minimumCompatibilityVersion(), equalTo(major56x));
 
         // from 7.0 on we are supporting the latest minor of the previous major... this might fail once we add a new version ie. 5.x is
         // released since we need to bump the supported minor in Version#minimumCompatibilityVersion()
-        Version lastVersion = Version.V_6_8_0; // TODO: remove this once min compat version is a constant instead of method
+        Version lastVersion = Version.fromString("6.8.0"); // TODO: remove this once min compat version is a constant instead of method
         assertEquals(lastVersion.major, Version.V_7_0_0.minimumCompatibilityVersion().major);
         assertEquals("did you miss to bump the minor in Version#minimumCompatibilityVersion()",
                 lastVersion.minor, Version.V_7_0_0.minimumCompatibilityVersion().minor);
@@ -194,41 +189,12 @@ public void testMinCompatVersion() {
     }
 
     public void testToString() {
-        // with 2.0.beta we lowercase
-        assertEquals("2.0.0-beta1", Version.fromString("2.0.0-beta1").toString());
-        assertEquals("5.0.0-alpha1", Version.fromId(5000001).toString());
+        assertEquals("5.0.0", Version.fromId(5000099).toString());
         assertEquals("2.3.0", Version.fromString("2.3.0").toString());
-        assertEquals("0.90.0.Beta1", Version.fromString("0.90.0.Beta1").toString());
-        assertEquals("1.0.0.Beta1", Version.fromString("1.0.0.Beta1").toString());
-        assertEquals("2.0.0-beta1", Version.fromString("2.0.0-beta1").toString());
-        assertEquals("5.0.0-beta1", Version.fromString("5.0.0-beta1").toString());
-        assertEquals("5.0.0-alpha1", Version.fromString("5.0.0-alpha1").toString());
-    }
-
-    public void testIsBeta() {
-        assertTrue(Version.fromString("2.0.0-beta1").isBeta());
-        assertTrue(Version.fromString("1.0.0.Beta1").isBeta());
-        assertTrue(Version.fromString("0.90.0.Beta1").isBeta());
-    }
-
-
-    public void testIsAlpha() {
-        assertTrue(new Version(5000001, org.apache.lucene.util.Version.LUCENE_7_0_0).isAlpha());
-        assertFalse(new Version(4000002, org.apache.lucene.util.Version.LUCENE_7_0_0).isAlpha());
-        assertTrue(new Version(4000002, org.apache.lucene.util.Version.LUCENE_7_0_0).isBeta());
-        assertTrue(Version.fromString("5.0.0-alpha14").isAlpha());
-        assertEquals(5000014, Version.fromString("5.0.0-alpha14").id);
-        assertTrue(Version.fromId(5000015).isAlpha());
-
-        for (int i = 0 ; i < 25; i++) {
-            assertEquals(Version.fromString("5.0.0-alpha" + i).id, Version.fromId(5000000 + i).id);
-            assertEquals("5.0.0-alpha" + i, Version.fromId(5000000 + i).toString());
-        }
-
-        for (int i = 0 ; i < 25; i++) {
-            assertEquals(Version.fromString("5.0.0-beta" + i).id, Version.fromId(5000000 + i + 25).id);
-            assertEquals("5.0.0-beta" + i, Version.fromId(5000000 + i + 25).toString());
-        }
+        assertEquals("0.90.0", Version.fromString("0.90.0").toString());
+        assertEquals("1.0.0", Version.fromString("1.0.0").toString());
+        assertEquals("2.0.0", Version.fromString("2.0.0").toString());
+        assertEquals("5.0.0", Version.fromString("5.0.0").toString());
     }
 
     public void testParseVersion() {
@@ -288,19 +254,8 @@ public void testAllVersionsMatchId() throws Exception {
                 }
                 assertEquals("Version id " + field.getName() + " does not point to " + constantName, v, Version.fromId(versionId));
                 assertEquals("Version " + constantName + " does not have correct id", versionId, v.id);
-                if (v.major >= 2) {
-                    String number = v.toString();
-                    if (v.isBeta()) {
-                        number = number.replace("-beta", "_beta");
-                    } else if (v.isRC()) {
-                        number = number.replace("-rc", "_rc");
-                    } else if (v.isAlpha()) {
-                        number = number.replace("-alpha", "_alpha");
-                    }
-                    assertEquals("V_" + number.replace('.', '_'), constantName);
-                } else {
-                    assertEquals("V_" + v.toString().replace('.', '_'), constantName);
-                }
+                String number = v.toString();
+                assertEquals("V_" + number.replace('.', '_'), constantName);
 
                 // only the latest version for a branch should be a snapshot (ie unreleased)
                 String branchName = "" + v.major + "." + v.minor;
@@ -328,8 +283,7 @@ public void testLuceneVersionIsSameOnMinorRelease() {
                     assertTrue("lucene versions must be "  + other + " >= " + version,
                         other.luceneVersion.onOrAfter(version.luceneVersion));
                 }
-                if (other.isAlpha() == false && version.isAlpha() == false
-                        && other.major == version.major && other.minor == version.minor) {
+                if (other.major == version.major && other.minor == version.minor) {
                     assertEquals(version + " vs. " + other, other.luceneVersion.major, version.luceneVersion.major);
                     assertEquals(version + " vs. " + other, other.luceneVersion.minor, version.luceneVersion.minor);
                     // should we also assert the lucene bugfix version?
@@ -345,10 +299,10 @@ public static void assertUnknownVersion(Version version) {
 
     public void testIsCompatible() {
         assertTrue(isCompatible(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()));
-        assertFalse(isCompatible(Version.V_6_6_0, Version.V_7_0_0));
-        assertTrue(isCompatible(Version.V_6_8_0, Version.V_7_0_0));
+        assertFalse(isCompatible(Version.V_7_0_0, Version.V_8_0_0));
+        assertTrue(isCompatible(Version.fromString("6.8.0"), Version.fromString("7.0.0")));
         assertFalse(isCompatible(Version.fromId(2000099), Version.V_7_0_0));
-        assertFalse(isCompatible(Version.fromId(2000099), Version.V_6_5_0));
+        assertFalse(isCompatible(Version.fromId(2000099), Version.fromString("6.5.0")));
 
         final Version currentMajorVersion = Version.fromId(Version.CURRENT.major * 1000000 + 99);
         final Version currentOrNextMajorVersion;
diff --git a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
index 1bde97d0b8ebd..979d0018d3c01 100644
--- a/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ActionModuleTests.java
@@ -88,7 +88,7 @@ protected FakeTransportAction(String actionName, ActionFilters actionFilters, Ta
             protected void doExecute(Task task, FakeRequest request, ActionListener<ActionResponse> listener) {
             }
         }
-        class FakeAction extends Action<ActionResponse> {
+        class FakeAction extends StreamableResponseActionType<ActionResponse> {
             protected FakeAction() {
                 super("fake");
             }
@@ -112,7 +112,7 @@ public ActionResponse newResponse() {
     public void testSetupRestHandlerContainsKnownBuiltin() {
         SettingsModule settings = new SettingsModule(Settings.EMPTY);
         UsageService usageService = new UsageService();
-        ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(),
+        ActionModule actionModule = new ActionModule(settings.getSettings(), new IndexNameExpressionResolver(),
                 settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), null, emptyList(), null,
                 null, usageService);
         actionModule.initRestHandlers(null);
@@ -135,7 +135,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
         ThreadPool threadPool = new TestThreadPool(getTestName());
         try {
             UsageService usageService = new UsageService();
-            ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(),
+            ActionModule actionModule = new ActionModule(settings.getSettings(), new IndexNameExpressionResolver(),
                     settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
                     singletonList(dupsMainAction), null, null, usageService);
             Exception e = expectThrows(IllegalArgumentException.class, () -> actionModule.initRestHandlers(null));
@@ -167,7 +167,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
         ThreadPool threadPool = new TestThreadPool(getTestName());
         try {
             UsageService usageService = new UsageService();
-            ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(),
+            ActionModule actionModule = new ActionModule(settings.getSettings(), new IndexNameExpressionResolver(),
                     settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
                     singletonList(registersFakeHandler), null, null, usageService);
             actionModule.initRestHandlers(null);
diff --git a/server/src/test/java/org/elasticsearch/action/ActionTests.java b/server/src/test/java/org/elasticsearch/action/ActionTests.java
index a7dca3f098d05..b9462ac68e749 100644
--- a/server/src/test/java/org/elasticsearch/action/ActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ActionTests.java
@@ -24,7 +24,7 @@
 public class ActionTests extends ESTestCase {
 
     public void testEquals() {
-        class FakeAction extends Action<ActionResponse> {
+        class FakeAction extends StreamableResponseActionType<ActionResponse> {
             protected FakeAction(String name) {
                 super(name);
             }
diff --git a/server/src/test/java/org/elasticsearch/action/IndicesRequestIT.java b/server/src/test/java/org/elasticsearch/action/IndicesRequestIT.java
index 2dbb52d547f97..24e2e68ed4fc6 100644
--- a/server/src/test/java/org/elasticsearch/action/IndicesRequestIT.java
+++ b/server/src/test/java/org/elasticsearch/action/IndicesRequestIT.java
@@ -21,7 +21,6 @@
 
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheAction;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
 import org.elasticsearch.action.admin.indices.close.CloseIndexAction;
@@ -207,7 +206,7 @@ public void testAnalyze() {
         String analyzeShardAction = AnalyzeAction.NAME + "[s]";
         interceptTransportActions(analyzeShardAction);
 
-        AnalyzeRequest analyzeRequest = new AnalyzeRequest(randomIndexOrAlias());
+        AnalyzeAction.Request analyzeRequest = new AnalyzeAction.Request(randomIndexOrAlias());
         analyzeRequest.text("text");
         internalCluster().coordOnlyNodeClient().admin().indices().analyze(analyzeRequest).actionGet();
 
@@ -341,7 +340,7 @@ public void testTermVector() {
         String termVectorShardAction = TermVectorsAction.NAME + "[s]";
         interceptTransportActions(termVectorShardAction);
 
-        TermVectorsRequest termVectorsRequest = new TermVectorsRequest(randomIndexOrAlias(), "type", "id");
+        TermVectorsRequest termVectorsRequest = new TermVectorsRequest(randomIndexOrAlias(), "id");
         internalCluster().coordOnlyNodeClient().termVectors(termVectorsRequest).actionGet();
 
         clearInterceptedActions();
@@ -357,7 +356,7 @@ public void testMultiTermVector() {
         int numDocs = iterations(1, 30);
         for (int i = 0; i < numDocs; i++) {
             String indexOrAlias = randomIndexOrAlias();
-            multiTermVectorsRequest.add(indexOrAlias, "type", Integer.toString(i));
+            multiTermVectorsRequest.add(indexOrAlias, Integer.toString(i));
             indices.add(indexOrAlias);
         }
         internalCluster().coordOnlyNodeClient().multiTermVectors(multiTermVectorsRequest).actionGet();
diff --git a/server/src/test/java/org/elasticsearch/action/ListenerActionIT.java b/server/src/test/java/org/elasticsearch/action/ListenerActionIT.java
index d0e8d997a1bb9..0dfc2fd58e86c 100644
--- a/server/src/test/java/org/elasticsearch/action/ListenerActionIT.java
+++ b/server/src/test/java/org/elasticsearch/action/ListenerActionIT.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Requests;
-import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.concurrent.CountDownLatch;
@@ -59,11 +58,6 @@ public void onFailure(Exception e) {
 
         latch.await();
 
-        boolean shouldBeThreaded = TransportClient.CLIENT_TYPE.equals(Client.CLIENT_TYPE_SETTING_S.get(client.settings()));
-        if (shouldBeThreaded) {
-            assertTrue(threadName.get().contains("listener"));
-        } else {
-            assertFalse(threadName.get().contains("listener"));
-        }
+        assertFalse(threadName.get().contains("listener"));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/RequestValidatorsTests.java b/server/src/test/java/org/elasticsearch/action/RequestValidatorsTests.java
new file mode 100644
index 0000000000000..f339a0b3e072e
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/RequestValidatorsTests.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action;
+
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.hamcrest.OptionalMatchers;
+import org.hamcrest.Matchers;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+public class RequestValidatorsTests extends ESTestCase {
+
+    private final RequestValidators.RequestValidator<PutMappingRequest> EMPTY = (request, state, indices) -> Optional.empty();
+    private final RequestValidators.RequestValidator<PutMappingRequest> FAIL =
+            (request, state, indices) -> Optional.of(new Exception("failure"));
+
+    public void testValidates() {
+        final int numberOfValidations = randomIntBetween(0, 8);
+        final List<RequestValidators.RequestValidator<PutMappingRequest>> validators = new ArrayList<>(numberOfValidations);
+        for (int i = 0; i < numberOfValidations; i++) {
+            validators.add(EMPTY);
+        }
+        final RequestValidators<PutMappingRequest> requestValidators = new RequestValidators<>(validators);
+        assertThat(requestValidators.validateRequest(null, null, null), OptionalMatchers.isEmpty());
+    }
+
+    public void testFailure() {
+        final RequestValidators<PutMappingRequest> validators = new RequestValidators<>(List.of(FAIL));
+        assertThat(validators.validateRequest(null, null, null), OptionalMatchers.isPresent());
+    }
+
+    public void testValidatesAfterFailure() {
+        final RequestValidators<PutMappingRequest> validators = new RequestValidators<>(List.of(FAIL, EMPTY));
+        assertThat(validators.validateRequest(null, null, null), OptionalMatchers.isPresent());
+    }
+
+    public void testMultipleFailures() {
+        final int numberOfFailures = randomIntBetween(2, 8);
+        final List<RequestValidators.RequestValidator<PutMappingRequest>> validators = new ArrayList<>(numberOfFailures);
+        for (int i = 0; i < numberOfFailures; i++) {
+            validators.add(FAIL);
+        }
+        final RequestValidators<PutMappingRequest> requestValidators = new RequestValidators<>(validators);
+        final Optional<Exception> e = requestValidators.validateRequest(null, null, null);
+        assertThat(e, OptionalMatchers.isPresent());
+        // noinspection OptionalGetWithoutIsPresent
+        assertThat(e.get().getSuppressed(), Matchers.arrayWithSize(numberOfFailures - 1));
+    }
+
+    public void testRandom() {
+        final int numberOfValidations = randomIntBetween(0, 8);
+        final int numberOfFailures = randomIntBetween(0, 8);
+        final List<RequestValidators.RequestValidator<PutMappingRequest>> validators =
+                new ArrayList<>(numberOfValidations + numberOfFailures);
+        for (int i = 0; i < numberOfValidations; i++) {
+            validators.add(EMPTY);
+        }
+        for (int i = 0; i < numberOfFailures; i++) {
+            validators.add(FAIL);
+        }
+        Randomness.shuffle(validators);
+        final RequestValidators<PutMappingRequest> requestValidators = new RequestValidators<>(validators);
+        final Optional<Exception> e = requestValidators.validateRequest(null, null, null);
+        if (numberOfFailures == 0) {
+            assertThat(e, OptionalMatchers.isEmpty());
+        } else {
+            assertThat(e, OptionalMatchers.isPresent());
+            // noinspection OptionalGetWithoutIsPresent
+            assertThat(e.get().getSuppressed(), Matchers.arrayWithSize(numberOfFailures - 1));
+        }
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/action/ShardValidateQueryRequestTests.java b/server/src/test/java/org/elasticsearch/action/ShardValidateQueryRequestTests.java
index c884d68a27606..5d7f903a4a3cd 100644
--- a/server/src/test/java/org/elasticsearch/action/ShardValidateQueryRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/ShardValidateQueryRequestTests.java
@@ -43,7 +43,7 @@ public class ShardValidateQueryRequestTests extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
         entries.addAll(indicesModule.getNamedWriteables());
         entries.addAll(searchModule.getNamedWriteables());
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java
index 52a5e7d3ce3dd..44880d5bdde59 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java
@@ -25,16 +25,16 @@
 import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfigExclusion;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNode.Role;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes.Builder;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Set;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
-import static java.util.Collections.singleton;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
@@ -55,14 +55,29 @@ public void testSerialization() throws IOException {
     }
 
     public void testResolve() {
-        final DiscoveryNode localNode
-            = new DiscoveryNode("local", "local", buildNewFakeTransportAddress(), emptyMap(), singleton(Role.MASTER), Version.CURRENT);
+        final DiscoveryNode localNode = new DiscoveryNode(
+                "local",
+                "local",
+                buildNewFakeTransportAddress(),
+                emptyMap(),
+                Set.of(DiscoveryNodeRole.MASTER_ROLE),
+                Version.CURRENT);
         final VotingConfigExclusion localNodeExclusion = new VotingConfigExclusion(localNode);
-        final DiscoveryNode otherNode1
-            = new DiscoveryNode("other1", "other1", buildNewFakeTransportAddress(), emptyMap(), singleton(Role.MASTER), Version.CURRENT);
+        final DiscoveryNode otherNode1 = new DiscoveryNode(
+                "other1",
+                "other1",
+                buildNewFakeTransportAddress(),
+                emptyMap(),
+                Set.of(DiscoveryNodeRole.MASTER_ROLE),
+                Version.CURRENT);
         final VotingConfigExclusion otherNode1Exclusion = new VotingConfigExclusion(otherNode1);
-        final DiscoveryNode otherNode2
-            = new DiscoveryNode("other2", "other2", buildNewFakeTransportAddress(), emptyMap(), singleton(Role.MASTER), Version.CURRENT);
+        final DiscoveryNode otherNode2 = new DiscoveryNode(
+                "other2",
+                "other2",
+                buildNewFakeTransportAddress(),
+                emptyMap(),
+                Set.of(DiscoveryNodeRole.MASTER_ROLE),
+                Version.CURRENT);
         final VotingConfigExclusion otherNode2Exclusion = new VotingConfigExclusion(otherNode2);
         final DiscoveryNode otherDataNode
             = new DiscoveryNode("data", "data", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
@@ -85,14 +100,29 @@ public void testResolve() {
     }
 
     public void testResolveAndCheckMaximum() {
-        final DiscoveryNode localNode
-            = new DiscoveryNode("local", "local", buildNewFakeTransportAddress(), emptyMap(), singleton(Role.MASTER), Version.CURRENT);
+        final DiscoveryNode localNode = new DiscoveryNode(
+                "local",
+                "local",
+                buildNewFakeTransportAddress(),
+                emptyMap(),
+                Set.of(DiscoveryNodeRole.MASTER_ROLE),
+                Version.CURRENT);
         final VotingConfigExclusion localNodeExclusion = new VotingConfigExclusion(localNode);
-        final DiscoveryNode otherNode1
-            = new DiscoveryNode("other1", "other1", buildNewFakeTransportAddress(), emptyMap(), singleton(Role.MASTER), Version.CURRENT);
+        final DiscoveryNode otherNode1 = new DiscoveryNode(
+                "other1",
+                "other1",
+                buildNewFakeTransportAddress(),
+                emptyMap(),
+                Set.of(DiscoveryNodeRole.MASTER_ROLE),
+                Version.CURRENT);
         final VotingConfigExclusion otherNode1Exclusion = new VotingConfigExclusion(otherNode1);
-        final DiscoveryNode otherNode2
-            = new DiscoveryNode("other2", "other2", buildNewFakeTransportAddress(), emptyMap(), singleton(Role.MASTER), Version.CURRENT);
+        final DiscoveryNode otherNode2 = new DiscoveryNode(
+                "other2",
+                "other2",
+                buildNewFakeTransportAddress(),
+                emptyMap(),
+                Set.of(DiscoveryNodeRole.MASTER_ROLE),
+                Version.CURRENT);
         final VotingConfigExclusion otherNode2Exclusion = new VotingConfigExclusion(otherNode2);
 
         final ClusterState.Builder builder = ClusterState.builder(new ClusterName("cluster")).nodes(new Builder()
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java
index 06d21fa4ba15b..d4dda501f5395 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java
@@ -28,12 +28,12 @@
 import org.elasticsearch.cluster.ClusterStateObserver.Listener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.coordination.CoordinationMetaData;
-import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
 import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfigExclusion;
+import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNode.Role;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes.Builder;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -60,7 +60,6 @@
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
-import static java.util.Collections.singleton;
 import static org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction.MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING;
 import static org.elasticsearch.cluster.ClusterState.builder;
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
@@ -96,7 +95,13 @@ public static void createThreadPoolAndClusterService() {
     }
 
     private static DiscoveryNode makeDiscoveryNode(String name) {
-        return new DiscoveryNode(name, name, buildNewFakeTransportAddress(), emptyMap(), singleton(Role.MASTER), Version.CURRENT);
+        return new DiscoveryNode(
+                name,
+                name,
+                buildNewFakeTransportAddress(),
+                emptyMap(),
+                Set.of(DiscoveryNodeRole.MASTER_ROLE),
+                Version.CURRENT);
     }
 
     @AfterClass
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
index c52fc58b90079..0a1e5c512df88 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
@@ -60,35 +60,30 @@ public class CancellableTasksTests extends TaskManagerTestCase {
 
     public static class CancellableNodeRequest extends BaseNodeRequest {
         protected String requestName;
-        protected String nodeId;
 
         public CancellableNodeRequest() {
             super();
         }
 
-        public CancellableNodeRequest(CancellableNodesRequest request, String nodeId) {
-            super(nodeId);
+        public CancellableNodeRequest(CancellableNodesRequest request) {
             requestName = request.requestName;
-            this.nodeId = nodeId;
         }
 
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             requestName = in.readString();
-            nodeId = in.readString();
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeString(requestName);
-            out.writeString(nodeId);
         }
 
         @Override
         public String getDescription() {
-            return "CancellableNodeRequest[" + requestName + ", " + nodeId + "]";
+            return "CancellableNodeRequest[" + requestName + "]";
         }
 
         @Override
@@ -161,19 +156,19 @@ class CancellableTestNodesAction extends AbstractTestNodesAction<CancellableNode
         }
 
         @Override
-        protected CancellableNodeRequest newNodeRequest(String nodeId, CancellableNodesRequest request) {
-            return new CancellableNodeRequest(request, nodeId);
+        protected CancellableNodeRequest newNodeRequest(CancellableNodesRequest request) {
+            return new CancellableNodeRequest(request);
         }
 
         @Override
         protected NodeResponse nodeOperation(CancellableNodeRequest request, Task task) {
             assert task instanceof CancellableTask;
-            debugDelay(request.nodeId, "op1");
+            debugDelay("op1");
             if (actionStartedLatch != null) {
                 actionStartedLatch.countDown();
             }
 
-            debugDelay(request.nodeId, "op2");
+            debugDelay("op2");
             if (shouldBlock) {
                 // Simulate a job that takes forever to finish
                 // Using periodic checks method to identify that the task was cancelled
@@ -189,15 +184,10 @@ protected NodeResponse nodeOperation(CancellableNodeRequest request, Task task)
                     Thread.currentThread().interrupt();
                 }
             }
-            debugDelay(request.nodeId, "op4");
+            debugDelay("op4");
 
             return new NodeResponse(clusterService.localNode());
         }
-
-        @Override
-        protected NodeResponse nodeOperation(CancellableNodeRequest request) {
-            throw new UnsupportedOperationException("the task parameter is required");
-        }
     }
 
     private Task startCancellableTestNodesAction(boolean waitForActionToStart, int blockedNodesCount, ActionListener<NodesResponse>
@@ -426,9 +416,9 @@ public void onFailure(Exception e) {
 
     }
 
-    private static void debugDelay(String nodeId, String name) {
+    private static void debugDelay(String name) {
         // Introduce an additional pseudo random repeatable race conditions
-        String delayName = RandomizedContext.current().getRunnerSeedAsString() + ":" + nodeId + ":" + name;
+        String delayName = RandomizedContext.current().getRunnerSeedAsString() + ":" + name;
         Random random = new Random(delayName.hashCode());
         if (RandomNumbers.randomIntBetween(random, 0, 10) < 1) {
             try {
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
index ecca51c7bbb83..bfaac83d51913 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
@@ -43,6 +43,7 @@
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.tasks.MockTaskManager;
@@ -156,7 +157,7 @@ protected NodeResponse newNodeResponse() {
         }
 
         @Override
-        protected abstract NodeResponse nodeOperation(NodeRequest request);
+        protected abstract NodeResponse nodeOperation(NodeRequest request, Task task);
 
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index fc758788e6197..40d5b028b4ba8 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -354,12 +354,12 @@ public void testSearchTaskDescriptions() {
         headers.put("Foo-Header", "bar");
         headers.put("Custom-Task-Header", "my_value");
         assertSearchResponse(
-            client().filterWithHeader(headers).prepareSearch("test").setTypes("doc").setQuery(QueryBuilders.matchAllQuery()).get());
+                client().filterWithHeader(headers).prepareSearch("test").setQuery(QueryBuilders.matchAllQuery()).get());
 
         // the search operation should produce one main task
         List<TaskInfo> mainTask = findEvents(SearchAction.NAME, Tuple::v1);
         assertEquals(1, mainTask.size());
-        assertThat(mainTask.get(0).getDescription(), startsWith("indices[test], types[doc], search_type["));
+        assertThat(mainTask.get(0).getDescription(), startsWith("indices[test], search_type["));
         assertThat(mainTask.get(0).getDescription(), containsString("\"query\":{\"match_all\""));
         assertTaskHeaders(mainTask.get(0));
 
@@ -747,13 +747,12 @@ public void testTaskStoringSuccesfulResult() throws Exception {
         assertNoFailures(client().admin().indices().prepareRefresh(TaskResultsService.TASK_INDEX).get());
 
         SearchResponse searchResponse = client().prepareSearch(TaskResultsService.TASK_INDEX)
-            .setTypes(TaskResultsService.TASK_TYPE)
             .setSource(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("task.action", taskInfo.getAction())))
             .get();
 
         assertEquals(1L, searchResponse.getHits().getTotalHits().value);
 
-        searchResponse = client().prepareSearch(TaskResultsService.TASK_INDEX).setTypes(TaskResultsService.TASK_TYPE)
+        searchResponse = client().prepareSearch(TaskResultsService.TASK_INDEX)
                 .setSource(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("task.node", taskInfo.getTaskId().getNodeId())))
                 .get();
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
index d39ab823dc490..d9c142498c917 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
@@ -18,13 +18,14 @@
  */
 package org.elasticsearch.action.admin.cluster.node.tasks;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
@@ -169,17 +170,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     public static class NodeRequest extends BaseNodeRequest {
         protected String requestName;
-        protected String nodeId;
         protected boolean shouldBlock;
 
         public NodeRequest() {
             super();
         }
 
-        public NodeRequest(NodesRequest request, String nodeId, boolean shouldBlock) {
-            super(nodeId);
+        public NodeRequest(NodesRequest request, boolean shouldBlock) {
             requestName = request.requestName;
-            this.nodeId = nodeId;
             this.shouldBlock = shouldBlock;
         }
 
@@ -187,7 +185,6 @@ public NodeRequest(NodesRequest request, String nodeId, boolean shouldBlock) {
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             requestName = in.readString();
-            nodeId = in.readString();
             shouldBlock = in.readBoolean();
         }
 
@@ -195,13 +192,12 @@ public void readFrom(StreamInput in) throws IOException {
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeString(requestName);
-            out.writeString(nodeId);
             out.writeBoolean(shouldBlock);
         }
 
         @Override
         public String getDescription() {
-            return "NodeRequest[" + requestName + ", " + nodeId + "]";
+            return "NodeRequest[" + requestName + "]";
         }
 
         @Override
@@ -301,8 +297,8 @@ protected NodesResponse newResponse(NodesRequest request, List<NodeResponse> res
         }
 
         @Override
-        protected NodeRequest newNodeRequest(String nodeId, NodesRequest request) {
-            return new NodeRequest(request, nodeId, request.getShouldBlock());
+        protected NodeRequest newNodeRequest(NodesRequest request) {
+            return new NodeRequest(request, request.getShouldBlock());
         }
 
         @Override
@@ -333,15 +329,9 @@ protected NodeResponse nodeOperation(NodeRequest request, Task task) {
             logger.info("Test task finished on the node {}", clusterService.localNode());
             return new NodeResponse(clusterService.localNode());
         }
-
-        @Override
-        protected NodeResponse nodeOperation(NodeRequest request) {
-            throw new UnsupportedOperationException("the task parameter is required");
-        }
-
     }
 
-    public static class TestTaskAction extends Action<NodesResponse> {
+    public static class TestTaskAction extends StreamableResponseActionType<NodesResponse> {
 
         public static final TestTaskAction INSTANCE = new TestTaskAction();
         public static final String NAME = "cluster:admin/tasks/test";
@@ -358,7 +348,7 @@ public NodesResponse newResponse() {
 
     public static class NodesRequestBuilder extends NodesOperationRequestBuilder<NodesRequest, NodesResponse, NodesRequestBuilder> {
 
-        protected NodesRequestBuilder(ElasticsearchClient client, Action<NodesResponse> action) {
+        protected NodesRequestBuilder(ElasticsearchClient client, ActionType<NodesResponse> action) {
             super(client, action, new NodesRequest("test"));
         }
 
@@ -466,7 +456,7 @@ protected void taskOperation(UnblockTestTasksRequest request, Task task, ActionL
 
     }
 
-    public static class UnblockTestTasksAction extends Action<UnblockTestTasksResponse> {
+    public static class UnblockTestTasksAction extends ActionType<UnblockTestTasksResponse> {
 
         public static final UnblockTestTasksAction INSTANCE = new UnblockTestTasksAction();
         public static final String NAME = "cluster:admin/tasks/testunblock";
@@ -475,11 +465,6 @@ private UnblockTestTasksAction() {
             super(NAME);
         }
 
-        @Override
-        public UnblockTestTasksResponse newResponse() {
-            throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-        }
-
         @Override
         public Writeable.Reader<UnblockTestTasksResponse> getResponseReader() {
             return UnblockTestTasksResponse::new;
@@ -489,7 +474,7 @@ public Writeable.Reader<UnblockTestTasksResponse> getResponseReader() {
     public static class UnblockTestTasksRequestBuilder extends ActionRequestBuilder<UnblockTestTasksRequest, UnblockTestTasksResponse> {
 
         protected UnblockTestTasksRequestBuilder(ElasticsearchClient client,
-                                                 Action<UnblockTestTasksResponse> action) {
+                                                 ActionType<UnblockTestTasksResponse> action) {
             super(client, action, new UnblockTestTasksRequest());
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
index b883d593352c2..6d8749ad4f4c8 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
@@ -84,8 +84,7 @@ public NodeRequest() {
             super();
         }
 
-        public NodeRequest(NodesRequest request, String nodeId) {
-            super(nodeId);
+        public NodeRequest(NodesRequest request) {
             requestName = request.requestName;
         }
 
@@ -157,8 +156,8 @@ abstract class TestNodesAction extends AbstractTestNodesAction<NodesRequest, Nod
         }
 
         @Override
-        protected NodeRequest newNodeRequest(String nodeId, NodesRequest request) {
-            return new NodeRequest(request, nodeId);
+        protected NodeRequest newNodeRequest(NodesRequest request) {
+            return new NodeRequest(request);
         }
 
         @Override
@@ -272,7 +271,7 @@ private Task startBlockingTestNodesAction(CountDownLatch checkLatch, NodesReques
             actions[i] = new TestNodesAction("internal:testAction", threadPool, testNodes[i].clusterService,
                     testNodes[i].transportService) {
                 @Override
-                protected NodeResponse nodeOperation(NodeRequest request) {
+                protected NodeResponse nodeOperation(NodeRequest request, Task task) {
                     logger.info("Action on node {}", node);
                     actionLatch.countDown();
                     try {
@@ -527,7 +526,7 @@ public void testFailedTasksCount() throws ExecutionException, InterruptedExcepti
             actions[i] = new TestNodesAction("internal:testAction", threadPool, testNodes[i].clusterService,
                     testNodes[i].transportService) {
                 @Override
-                protected NodeResponse nodeOperation(NodeRequest request) {
+                protected NodeResponse nodeOperation(NodeRequest request, Task task) {
                     logger.info("Action on node {}", node);
                     throw new RuntimeException("Test exception");
                 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java
index 485b2dc3fb4d1..da3157840e2e5 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsResponseTests.java
@@ -19,18 +19,19 @@
 
 package org.elasticsearch.action.admin.cluster.settings;
 
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.Settings.Builder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.test.AbstractSerializingTestCase;
 
 import java.util.List;
 import java.util.Set;
 import java.util.function.Predicate;
 
-public class ClusterUpdateSettingsResponseTests extends AbstractStreamableXContentTestCase<ClusterUpdateSettingsResponse> {
+public class ClusterUpdateSettingsResponseTests extends AbstractSerializingTestCase<ClusterUpdateSettingsResponse> {
 
     @Override
     protected ClusterUpdateSettingsResponse doParseInstance(XContentParser parser) {
@@ -93,8 +94,7 @@ protected ClusterUpdateSettingsResponse createTestInstance() {
     }
 
     @Override
-    protected ClusterUpdateSettingsResponse createBlankInstance() {
-        return new ClusterUpdateSettingsResponse();
+    protected Writeable.Reader<ClusterUpdateSettingsResponse> instanceReader() {
+        return ClusterUpdateSettingsResponse::new;
     }
-
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponseTests.java
index 534cea2ae4604..c83caae49a72f 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponseTests.java
@@ -73,7 +73,7 @@ public void testSerialization() throws Exception {
         ClusterSearchShardsResponse clusterSearchShardsResponse = new ClusterSearchShardsResponse(clusterSearchShardsGroups,
                 nodes.toArray(new DiscoveryNode[nodes.size()]), indicesAndFilters);
 
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
         entries.addAll(searchModule.getNamedWriteables());
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java
index 7dac3a38dddd7..9be5c23bb1bcd 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java
@@ -132,7 +132,8 @@ public void testDeleteSnapshotWithBlocks() {
 
     public void testRestoreSnapshotWithBlocks() {
         assertAcked(client().admin().indices().prepareDelete(INDEX_NAME, OTHER_INDEX_NAME));
-        assertFalse(client().admin().indices().prepareExists(INDEX_NAME, OTHER_INDEX_NAME).get().isExists());
+        assertFalse(indexExists(INDEX_NAME));
+        assertFalse(indexExists(OTHER_INDEX_NAME));
 
         logger.info("-->  restoring a snapshot is blocked when the cluster is read only");
         try {
@@ -148,8 +149,8 @@ public void testRestoreSnapshotWithBlocks() {
                 .setWaitForCompletion(true)
                 .execute().actionGet();
         assertThat(response.status(), equalTo(RestStatus.OK));
-        assertTrue(client().admin().indices().prepareExists(INDEX_NAME).get().isExists());
-        assertTrue(client().admin().indices().prepareExists(OTHER_INDEX_NAME).get().isExists());
+        assertTrue(indexExists(INDEX_NAME));
+        assertTrue(indexExists(OTHER_INDEX_NAME));
     }
 
     public void testGetSnapshotWithBlocks() {
@@ -157,8 +158,8 @@ public void testGetSnapshotWithBlocks() {
         try {
             setClusterReadOnly(true);
             GetSnapshotsResponse response = client().admin().cluster().prepareGetSnapshots(REPOSITORY_NAME).execute().actionGet();
-            assertThat(response.getSnapshots(), hasSize(1));
-            assertThat(response.getSnapshots().get(0).snapshotId().getName(), equalTo(SNAPSHOT_NAME));
+            assertThat(response.getSnapshots(REPOSITORY_NAME), hasSize(1));
+            assertThat(response.getSnapshots(REPOSITORY_NAME).get(0).snapshotId().getName(), equalTo(SNAPSHOT_NAME));
         } finally {
             setClusterReadOnly(false);
         }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java
index 0b598be6849cb..9f7bd5f6a0149 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java
@@ -19,10 +19,12 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.create;
 
+import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.IndicesOptions.Option;
 import org.elasticsearch.action.support.IndicesOptions.WildcardStates;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent.MapParams;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -41,6 +43,10 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.snapshots.SnapshotInfoTests.randomUserMetadata;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+
 public class CreateSnapshotRequestTests extends ESTestCase {
 
     // tests creating XContent and parsing with source(Map) equivalency
@@ -80,6 +86,10 @@ public void testToXContent() throws IOException {
             original.includeGlobalState(randomBoolean());
         }
 
+        if (randomBoolean()) {
+            original.userMetadata(randomUserMetadata());
+        }
+
         if (randomBoolean()) {
             Collection<WildcardStates> wildcardStates = randomSubsetOf(Arrays.asList(WildcardStates.values()));
             Collection<Option> options = randomSubsetOf(Arrays.asList(Option.ALLOW_NO_INDICES, Option.IGNORE_UNAVAILABLE));
@@ -108,4 +118,57 @@ public void testToXContent() throws IOException {
 
         assertEquals(original, processed);
     }
+
+    public void testSizeCheck() {
+        {
+            Map<String, Object> simple = new HashMap<>();
+            simple.put(randomAlphaOfLength(5), randomAlphaOfLength(25));
+            assertNull(createSnapshotRequestWithMetadata(simple).validate());
+        }
+
+        {
+            Map<String, Object> complex = new HashMap<>();
+            Map<String, Object> nested = new HashMap<>();
+            nested.put(randomAlphaOfLength(5), randomAlphaOfLength(5));
+            nested.put(randomAlphaOfLength(6), randomAlphaOfLength(5));
+            complex.put(randomAlphaOfLength(7), nested);
+            assertNull(createSnapshotRequestWithMetadata(complex).validate());
+        }
+
+        {
+            Map<String, Object> barelyFine = new HashMap<>();
+            barelyFine.put(randomAlphaOfLength(512), randomAlphaOfLength(505));
+            assertNull(createSnapshotRequestWithMetadata(barelyFine).validate());
+        }
+
+        {
+            Map<String, Object> barelyTooBig = new HashMap<>();
+            barelyTooBig.put(randomAlphaOfLength(512), randomAlphaOfLength(506));
+            ActionRequestValidationException validationException = createSnapshotRequestWithMetadata(barelyTooBig).validate();
+            assertNotNull(validationException);
+            assertThat(validationException.validationErrors(), hasSize(1));
+            assertThat(validationException.validationErrors().get(0), equalTo("metadata must be smaller than 1024 bytes, but was [1025]"));
+        }
+
+        {
+            Map<String, Object> tooBigOnlyIfNestedFieldsAreIncluded = new HashMap<>();
+            HashMap<Object, Object> nested = new HashMap<>();
+            nested.put(randomAlphaOfLength(500), randomAlphaOfLength(500));
+            tooBigOnlyIfNestedFieldsAreIncluded.put(randomAlphaOfLength(10), randomAlphaOfLength(10));
+            tooBigOnlyIfNestedFieldsAreIncluded.put(randomAlphaOfLength(11), nested);
+
+            ActionRequestValidationException validationException = createSnapshotRequestWithMetadata(tooBigOnlyIfNestedFieldsAreIncluded)
+                .validate();
+            assertNotNull(validationException);
+            assertThat(validationException.validationErrors(), hasSize(1));
+            assertThat(validationException.validationErrors().get(0), equalTo("metadata must be smaller than 1024 bytes, but was [1049]"));
+        }
+    }
+
+    private CreateSnapshotRequest createSnapshotRequestWithMetadata(Map<String, Object> metadata) {
+        return new CreateSnapshotRequest(randomAlphaOfLength(5), randomAlphaOfLength(5))
+            .indices(randomAlphaOfLength(5))
+            .settings(Settings.EMPTY)
+            .userMetadata(metadata);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java
index bbb11fc6feef0..5c4bc2ad2d7d3 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponseTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
+import org.elasticsearch.snapshots.SnapshotInfoTests;
 import org.elasticsearch.snapshots.SnapshotShardFailure;
 import org.elasticsearch.test.AbstractXContentTestCase;
 
@@ -30,6 +31,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.UUID;
+import java.util.function.Predicate;
 
 public class CreateSnapshotResponseTests extends AbstractXContentTestCase<CreateSnapshotResponse> {
 
@@ -64,6 +66,17 @@ protected CreateSnapshotResponse createTestInstance() {
         boolean globalState = randomBoolean();
 
         return new CreateSnapshotResponse(
-            new SnapshotInfo(snapshotId, indices, startTime, reason, endTime, totalShards, shardFailures, globalState));
+            new SnapshotInfo(snapshotId, indices, startTime, reason, endTime, totalShards, shardFailures,
+                globalState, SnapshotInfoTests.randomUserMetadata()));
+    }
+
+    @Override
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        // Don't inject random fields into the custom snapshot metadata, because the metadata map is equality-checked after doing a
+        // round-trip through xContent serialization/deserialization. Even though the rest of the object ignores unknown fields,
+        // `metadata` doesn't ignore unknown fields (it just includes them in the parsed object, because the keys are arbitrary), so any
+        // new fields added to the metadata before it gets deserialized that weren't in the serialized version will cause the equality
+        // check to fail.
+        return field -> field.startsWith("snapshot.metadata");
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java
index c5bd7d9f38ac1..dee5314b9672d 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java
@@ -19,44 +19,121 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.get;
 
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
+import org.elasticsearch.snapshots.SnapshotInfoTests;
 import org.elasticsearch.snapshots.SnapshotShardFailure;
-import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.regex.Pattern;
 
-public class GetSnapshotsResponseTests extends AbstractStreamableXContentTestCase<GetSnapshotsResponse> {
+import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
+import static org.hamcrest.CoreMatchers.containsString;
 
-    @Override
-    protected GetSnapshotsResponse doParseInstance(XContentParser parser) throws IOException {
+public class GetSnapshotsResponseTests extends ESTestCase {
+    // We can not subclass AbstractStreamableXContentTestCase because it
+    // can only be used for instances with equals and hashCode
+    // GetSnapshotResponse does not override equals and hashCode.
+    // It does not override equals and hashCode, because it
+    // contains ElasticsearchException, which does not override equals and hashCode.
+
+    private GetSnapshotsResponse doParseInstance(XContentParser parser) throws IOException {
         return GetSnapshotsResponse.fromXContent(parser);
     }
 
-    @Override
-    protected GetSnapshotsResponse createBlankInstance() {
-        return new GetSnapshotsResponse();
+    private GetSnapshotsResponse copyInstance(GetSnapshotsResponse instance, Version version) throws IOException {
+        return copyInstance(instance, new NamedWriteableRegistry(Collections.emptyList()), (out, value) -> value.writeTo(out),
+                in -> new GetSnapshotsResponse(in), version);
+
+    }
+
+    private void assertEqualInstances(GetSnapshotsResponse expectedInstance, GetSnapshotsResponse newInstance) {
+        assertEquals(expectedInstance.getSuccessfulResponses(), newInstance.getSuccessfulResponses());
+        assertEquals(expectedInstance.getFailedResponses().keySet(), newInstance.getFailedResponses().keySet());
+        for (Map.Entry<String, ElasticsearchException> expectedEntry : expectedInstance.getFailedResponses().entrySet()) {
+            ElasticsearchException expectedException = expectedEntry.getValue();
+            ElasticsearchException newException = newInstance.getFailedResponses().get(expectedEntry.getKey());
+            assertThat(newException.getMessage(), containsString(expectedException.getMessage()));
+        }
     }
 
-    @Override
-    protected GetSnapshotsResponse createTestInstance() {
+    private List<SnapshotInfo> createSnapshotInfos() {
         ArrayList<SnapshotInfo> snapshots = new ArrayList<>();
         for (int i = 0; i < randomIntBetween(5, 10); ++i) {
             SnapshotId snapshotId = new SnapshotId("snapshot " + i, UUIDs.base64UUID());
             String reason = randomBoolean() ? null : "reason";
             ShardId shardId = new ShardId("index", UUIDs.base64UUID(), 2);
             List<SnapshotShardFailure> shardFailures = Collections.singletonList(new SnapshotShardFailure("node-id", shardId, "reason"));
-            snapshots.add(new SnapshotInfo(snapshotId, Arrays.asList("indice1", "indice2"), System.currentTimeMillis(), reason,
-                System.currentTimeMillis(), randomIntBetween(2, 3), shardFailures, randomBoolean()));
+            snapshots.add(new SnapshotInfo(snapshotId, Arrays.asList("index1", "index2"), System.currentTimeMillis(), reason,
+                    System.currentTimeMillis(), randomIntBetween(2, 3), shardFailures, randomBoolean(),
+                    SnapshotInfoTests.randomUserMetadata()));
+
+        }
+        return snapshots;
+    }
+
+    private GetSnapshotsResponse createTestInstance() {
+        Set<String> repositories = new HashSet<>();
+        List<GetSnapshotsResponse.Response> responses = new ArrayList<>();
 
+        for (int i = 0; i < randomIntBetween(0, 5); i++) {
+            String repository = randomValueOtherThanMany(r -> repositories.contains(r), () -> randomAlphaOfLength(10));
+            repositories.add(repository);
+            responses.add(GetSnapshotsResponse.Response.snapshots(repository, createSnapshotInfos()));
         }
-        return new GetSnapshotsResponse(snapshots);
+
+        for (int i = 0; i < randomIntBetween(0, 5); i++) {
+            String repository = randomValueOtherThanMany(r -> repositories.contains(r), () -> randomAlphaOfLength(10));
+            repositories.add(repository);
+            responses.add(GetSnapshotsResponse.Response.error(repository, new ElasticsearchException(randomAlphaOfLength(10))));
+        }
+
+        return new GetSnapshotsResponse(responses);
+    }
+
+    public void testSerialization() throws IOException {
+        GetSnapshotsResponse testInstance = createTestInstance();
+        GetSnapshotsResponse deserializedInstance = copyInstance(testInstance, Version.CURRENT);
+        assertEqualInstances(testInstance, deserializedInstance);
     }
+
+    public void testFromXContent() throws IOException {
+        final Predicate<String> predicate = Pattern.compile("responses\\.\\d+\\.snapshots\\.\\d+\\.metadata.*").asMatchPredicate();
+        xContentTester(this::createParser, this::createTestInstance, ToXContent.EMPTY_PARAMS, this::doParseInstance)
+                .numberOfTestRuns(1)
+                .supportsUnknownFields(true)
+                .shuffleFieldsExceptions(Strings.EMPTY_ARRAY)
+                // Don't inject random fields into the custom snapshot metadata, because the metadata map is equality-checked after doing a
+                // round-trip through xContent serialization/deserialization. Even though the rest of the object ignores unknown fields,
+                // `metadata` doesn't ignore unknown fields (it just includes them in the parsed object, because the keys are arbitrary),
+                // so any new fields added to the metadata before it gets deserialized that weren't in the serialized version will
+                // cause the equality check to fail.
+
+                // The actual fields are nested in an array, so this regex matches fields with names of the form
+                // `responses.0.snapshots.3.metadata`
+                .randomFieldsExcludeFilter(predicate)
+                .assertEqualsConsumer(this::assertEqualInstances)
+                // We set it to false, because GetSnapshotsResponse contains
+                // ElasticsearchException, whose xContent creation/parsing are not stable.
+                .assertToXContentEquivalence(false)
+                .test();
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java
index f2214f4ba81fa..b219e208a6049 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java
@@ -67,10 +67,8 @@ public void testSerialization() throws Exception {
             assertThat(deserializedCSRequest.blocks(), equalTo(clusterStateRequest.blocks()));
             assertThat(deserializedCSRequest.indices(), equalTo(clusterStateRequest.indices()));
             assertOptionsMatch(deserializedCSRequest.indicesOptions(), clusterStateRequest.indicesOptions());
-            if (testVersion.onOrAfter(Version.V_6_6_0)) {
-                assertThat(deserializedCSRequest.waitForMetaDataVersion(), equalTo(clusterStateRequest.waitForMetaDataVersion()));
-                assertThat(deserializedCSRequest.waitForTimeout(), equalTo(clusterStateRequest.waitForTimeout()));
-            }
+            assertThat(deserializedCSRequest.waitForMetaDataVersion(), equalTo(clusterStateRequest.waitForMetaDataVersion()));
+            assertThat(deserializedCSRequest.waitForTimeout(), equalTo(clusterStateRequest.waitForTimeout()));
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
index 427aaa45cd678..ca3888aa7f807 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -63,9 +63,9 @@ public void testNodeCounts() {
         int total = 1;
         internalCluster().startNode();
         Map<String, Integer> expectedCounts = new HashMap<>();
-        expectedCounts.put(DiscoveryNode.Role.DATA.getRoleName(), 1);
-        expectedCounts.put(DiscoveryNode.Role.MASTER.getRoleName(), 1);
-        expectedCounts.put(DiscoveryNode.Role.INGEST.getRoleName(), 1);
+        expectedCounts.put(DiscoveryNodeRole.DATA_ROLE.roleName(), 1);
+        expectedCounts.put(DiscoveryNodeRole.MASTER_ROLE.roleName(), 1);
+        expectedCounts.put(DiscoveryNodeRole.INGEST_ROLE.roleName(), 1);
         expectedCounts.put(ClusterStatsNodes.Counts.COORDINATING_ONLY, 0);
         int numNodes = randomIntBetween(1, 5);
 
@@ -84,13 +84,13 @@ public void testNodeCounts() {
             waitForNodes(total);
 
             if (isDataNode) {
-                incrementCountForRole(DiscoveryNode.Role.DATA.getRoleName(), expectedCounts);
+                incrementCountForRole(DiscoveryNodeRole.DATA_ROLE.roleName(), expectedCounts);
             }
             if (isMasterNode) {
-                incrementCountForRole(DiscoveryNode.Role.MASTER.getRoleName(), expectedCounts);
+                incrementCountForRole(DiscoveryNodeRole.MASTER_ROLE.roleName(), expectedCounts);
             }
             if (isIngestNode) {
-                incrementCountForRole(DiscoveryNode.Role.INGEST.getRoleName(), expectedCounts);
+                incrementCountForRole(DiscoveryNodeRole.INGEST_ROLE.roleName(), expectedCounts);
             }
             if (!isDataNode && !isMasterNode && !isIngestNode) {
                 incrementCountForRole(ClusterStatsNodes.Counts.COORDINATING_ONLY, expectedCounts);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
index b0c2e34c30620..72830f79889c0 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
@@ -21,15 +21,17 @@
 import org.apache.lucene.analysis.MockTokenFilter;
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.util.automaton.Automata;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
 import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AbstractCharFilterFactory;
 import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
@@ -53,6 +55,8 @@
 
 import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 /**
  * Tests for {@link TransportAnalyzeAction}. See the rest tests in the {@code analysis-common} module for places where this code gets a ton
@@ -61,8 +65,8 @@
 public class TransportAnalyzeActionTests extends ESTestCase {
 
     private IndexAnalyzers indexAnalyzers;
+    private IndexSettings indexSettings;
     private AnalysisRegistry registry;
-    private Environment environment;
     private int maxTokenCount;
     private int idxMaxTokenCount;
 
@@ -77,30 +81,45 @@ public void setUp() throws Exception {
                 .put("index.analysis.analyzer.custom_analyzer.tokenizer", "standard")
                 .put("index.analysis.analyzer.custom_analyzer.filter", "mock")
                 .put("index.analysis.normalizer.my_normalizer.type", "custom")
+                .put("index.analysis.char_filter.my_append.type", "append")
+                .put("index.analysis.char_filter.my_append.suffix", "baz")
                 .put("index.analyze.max_token_count", 100)
                 .putList("index.analysis.normalizer.my_normalizer.filter", "lowercase").build();
-        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
-        environment = TestEnvironment.newEnvironment(settings);
+        this.indexSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
+        Environment environment = TestEnvironment.newEnvironment(settings);
         AnalysisPlugin plugin = new AnalysisPlugin() {
             class MockFactory extends AbstractTokenFilterFactory {
+
+                final CharacterRunAutomaton stopset;
+
                 MockFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
                     super(indexSettings, name, settings);
+                    if (settings.hasValue("stopword")) {
+                        this.stopset = new CharacterRunAutomaton(Automata.makeString(settings.get("stopword")));
+                    }
+                    else {
+                        this.stopset = MockTokenFilter.ENGLISH_STOPSET;
+                    }
                 }
 
                 @Override
                 public TokenStream create(TokenStream tokenStream) {
-                    return new MockTokenFilter(tokenStream, MockTokenFilter.ENGLISH_STOPSET);
+                    return new MockTokenFilter(tokenStream, this.stopset);
                 }
             }
 
             class AppendCharFilterFactory extends AbstractCharFilterFactory {
+
+                final String suffix;
+
                 AppendCharFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
                     super(indexSettings, name);
+                    this.suffix = settings.get("suffix", "bar");
                 }
 
                 @Override
                 public Reader create(Reader reader) {
-                    return new AppendCharFilter(reader, "bar");
+                    return new AppendCharFilter(reader, suffix);
                 }
             }
 
@@ -122,13 +141,20 @@ public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
 
             @Override
             public List<PreConfiguredCharFilter> getPreConfiguredCharFilters() {
-                return singletonList(PreConfiguredCharFilter.singleton("append_foo", false, reader -> new AppendCharFilter(reader, "foo")));
+                return singletonList(PreConfiguredCharFilter.singleton("append", false, reader -> new AppendCharFilter(reader, "foo")));
             }
         };
         registry = new AnalysisModule(environment, singletonList(plugin)).getAnalysisRegistry();
-        indexAnalyzers = registry.build(idxSettings);
+        indexAnalyzers = registry.build(this.indexSettings);
         maxTokenCount = IndexSettings.MAX_TOKEN_COUNT_SETTING.getDefault(settings);
-        idxMaxTokenCount = idxSettings.getMaxTokenCount();
+        idxMaxTokenCount = this.indexSettings.getMaxTokenCount();
+    }
+
+    private IndexService mockIndexService() {
+        IndexService is = mock(IndexService.class);
+        when(is.getIndexAnalyzers()).thenReturn(indexAnalyzers);
+        when(is.getIndexSettings()).thenReturn(indexSettings);
+        return is;
     }
 
     /**
@@ -136,62 +162,76 @@ public List<PreConfiguredCharFilter> getPreConfiguredCharFilters() {
      */
     public void testNoIndexAnalyzers() throws IOException {
         // Refer to an analyzer by its type so we get its default configuration
-        AnalyzeRequest request = new AnalyzeRequest();
+        AnalyzeAction.Request request = new AnalyzeAction.Request();
         request.text("the quick brown fox");
         request.analyzer("standard");
-        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, "text", null, null, registry, environment, maxTokenCount);
-        List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
+        AnalyzeAction.Response analyze
+            = TransportAnalyzeAction.analyze(request, registry, null, maxTokenCount);
+        List<AnalyzeAction.AnalyzeToken> tokens = analyze.getTokens();
         assertEquals(4, tokens.size());
 
-        // Refer to a token filter by its type so we get its default configuration
-        request = new AnalyzeRequest();
+        // We can refer to a pre-configured token filter by its name to get it
+        request = new AnalyzeAction.Request();
         request.text("the qu1ck brown fox");
         request.tokenizer("standard");
-        request.addTokenFilter("mock");
-        analyze = TransportAnalyzeAction.analyze(request, "text", null, randomBoolean() ? indexAnalyzers : null, registry, environment,
-            maxTokenCount);
+        request.addCharFilter("append");        // <-- no config, so use preconfigured filter
+        analyze
+            = TransportAnalyzeAction.analyze(request, registry, null, maxTokenCount);
+        tokens = analyze.getTokens();
+        assertEquals(4, tokens.size());
+        assertEquals("the", tokens.get(0).getTerm());
+        assertEquals("qu1ck", tokens.get(1).getTerm());
+        assertEquals("brown", tokens.get(2).getTerm());
+        assertEquals("foxfoo", tokens.get(3).getTerm());
+
+        // If the preconfigured filter doesn't exist, we use a global filter with no settings
+        request = new AnalyzeAction.Request();
+        request.text("the qu1ck brown fox");
+        request.tokenizer("standard");
+        request.addTokenFilter("mock");     // <-- not preconfigured, but a global one available
+        analyze
+            = TransportAnalyzeAction.analyze(request, registry, null, maxTokenCount);
         tokens = analyze.getTokens();
         assertEquals(3, tokens.size());
         assertEquals("qu1ck", tokens.get(0).getTerm());
         assertEquals("brown", tokens.get(1).getTerm());
         assertEquals("fox", tokens.get(2).getTerm());
 
-        // We can refer to a pre-configured token filter by its name to get it
-        request = new AnalyzeRequest();
+        // We can build a new char filter to get default values
+        request = new AnalyzeAction.Request();
         request.text("the qu1ck brown fox");
         request.tokenizer("standard");
-        request.addCharFilter("append_foo");
-        analyze = TransportAnalyzeAction.analyze(request, "text", null, randomBoolean() ? indexAnalyzers : null, registry, environment,
-            maxTokenCount);
+        request.addTokenFilter(Map.of("type", "mock", "stopword", "brown"));
+        request.addCharFilter(Map.of("type", "append"));    // <-- basic config, uses defaults
+        analyze
+            = TransportAnalyzeAction.analyze(request, registry, null, maxTokenCount);
         tokens = analyze.getTokens();
-        assertEquals(4, tokens.size());
+        assertEquals(3, tokens.size());
         assertEquals("the", tokens.get(0).getTerm());
         assertEquals("qu1ck", tokens.get(1).getTerm());
-        assertEquals("brown", tokens.get(2).getTerm());
-        assertEquals("foxfoo", tokens.get(3).getTerm());
+        assertEquals("foxbar", tokens.get(2).getTerm());
 
-        // We can refer to a token filter by its type to get its default configuration
-        request = new AnalyzeRequest();
+        // We can pass a new configuration
+        request = new AnalyzeAction.Request();
         request.text("the qu1ck brown fox");
         request.tokenizer("standard");
-        request.addCharFilter("append");
-        request.text("the qu1ck brown fox");
-        analyze = TransportAnalyzeAction.analyze(request, "text", null, randomBoolean() ? indexAnalyzers : null, registry, environment,
-            maxTokenCount);
+        request.addTokenFilter(Map.of("type", "mock", "stopword", "brown"));
+        request.addCharFilter(Map.of("type", "append", "suffix", "baz"));
+        analyze
+            = TransportAnalyzeAction.analyze(request, registry, null, maxTokenCount);
         tokens = analyze.getTokens();
-        assertEquals(4, tokens.size());
+        assertEquals(3, tokens.size());
         assertEquals("the", tokens.get(0).getTerm());
         assertEquals("qu1ck", tokens.get(1).getTerm());
-        assertEquals("brown", tokens.get(2).getTerm());
-        assertEquals("foxbar", tokens.get(3).getTerm());
+        assertEquals("foxbaz", tokens.get(2).getTerm());
     }
 
     public void testFillsAttributes() throws IOException {
-        AnalyzeRequest request = new AnalyzeRequest();
+        AnalyzeAction.Request request = new AnalyzeAction.Request();
         request.analyzer("standard");
         request.text("the 1 brown fox");
-        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, "text", null, null, registry, environment, maxTokenCount);
-        List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
+        AnalyzeAction.Response analyze = TransportAnalyzeAction.analyze(request, registry, null, maxTokenCount);
+        List<AnalyzeAction.AnalyzeToken> tokens = analyze.getTokens();
         assertEquals(4, tokens.size());
         assertEquals("the", tokens.get(0).getTerm());
         assertEquals(0, tokens.get(0).getStartOffset());
@@ -219,19 +259,19 @@ public void testFillsAttributes() throws IOException {
     }
 
     public void testWithIndexAnalyzers() throws IOException {
-        AnalyzeRequest request = new AnalyzeRequest();
+        AnalyzeAction.Request request = new AnalyzeAction.Request();
         request.text("the quick brown fox");
         request.analyzer("custom_analyzer");
-        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment,
-            maxTokenCount);
-        List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
+        AnalyzeAction.Response analyze
+            = TransportAnalyzeAction.analyze(request, registry, mockIndexService(), maxTokenCount);
+        List<AnalyzeAction.AnalyzeToken> tokens = analyze.getTokens();
         assertEquals(3, tokens.size());
         assertEquals("quick", tokens.get(0).getTerm());
         assertEquals("brown", tokens.get(1).getTerm());
         assertEquals("fox", tokens.get(2).getTerm());
 
         request.analyzer("standard");
-        analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment, maxTokenCount);
+        analyze = TransportAnalyzeAction.analyze(request, registry, mockIndexService(), maxTokenCount);
         tokens = analyze.getTokens();
         assertEquals(4, tokens.size());
         assertEquals("the", tokens.get(0).getTerm());
@@ -242,7 +282,7 @@ public void testWithIndexAnalyzers() throws IOException {
         // Switch the analyzer out for just a tokenizer
         request.analyzer(null);
         request.tokenizer("standard");
-        analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment, maxTokenCount);
+        analyze = TransportAnalyzeAction.analyze(request, registry, mockIndexService(), maxTokenCount);
         tokens = analyze.getTokens();
         assertEquals(4, tokens.size());
         assertEquals("the", tokens.get(0).getTerm());
@@ -252,32 +292,58 @@ public void testWithIndexAnalyzers() throws IOException {
 
         // Now try applying our token filter
         request.addTokenFilter("mock");
-        analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment, maxTokenCount);
+        analyze = TransportAnalyzeAction.analyze(request, registry, mockIndexService(), maxTokenCount);
         tokens = analyze.getTokens();
         assertEquals(3, tokens.size());
         assertEquals("quick", tokens.get(0).getTerm());
         assertEquals("brown", tokens.get(1).getTerm());
         assertEquals("fox", tokens.get(2).getTerm());
+
+        // Apply the char filter, checking that the correct configuration gets passed on
+        request.addCharFilter("my_append");
+        analyze = TransportAnalyzeAction.analyze(request, registry, mockIndexService(), maxTokenCount);
+        tokens = analyze.getTokens();
+        assertEquals(3, tokens.size());
+        assertEquals("quick", tokens.get(0).getTerm());
+        assertEquals("brown", tokens.get(1).getTerm());
+        assertEquals("foxbaz", tokens.get(2).getTerm());
+
+        // Apply a token filter with parameters
+        request.addTokenFilter(Map.of("type", "mock", "stopword", "brown"));
+        analyze = TransportAnalyzeAction.analyze(request, registry, mockIndexService(), maxTokenCount);
+        tokens = analyze.getTokens();
+        assertEquals(2, tokens.size());
+        assertEquals("quick", tokens.get(0).getTerm());
+        assertEquals("foxbaz", tokens.get(1).getTerm());
+
     }
 
-    public void testGetIndexAnalyserWithoutIndexAnalyzers() throws IOException {
+    public void testGetIndexAnalyserWithoutIndexAnalyzers() {
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
             () -> TransportAnalyzeAction.analyze(
-                new AnalyzeRequest()
+                new AnalyzeAction.Request()
                     .analyzer("custom_analyzer")
                     .text("the qu1ck brown fox-dog"),
-                "text", null, null, registry, environment, maxTokenCount));
+                registry, null, maxTokenCount));
         assertEquals(e.getMessage(), "failed to find global analyzer [custom_analyzer]");
     }
 
-    public void testUnknown() throws IOException {
+    public void testGetFieldAnalyzerWithoutIndexAnalyzers() {
+        AnalyzeAction.Request req = new AnalyzeAction.Request().field("field").text("text");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
+            TransportAnalyzeAction.analyze(req, registry, null, maxTokenCount);
+        });
+        assertEquals(e.getMessage(), "analysis based on a specific field requires an index");
+    }
+
+    public void testUnknown() {
         boolean notGlobal = randomBoolean();
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
             () -> TransportAnalyzeAction.analyze(
-                new AnalyzeRequest()
+                new AnalyzeAction.Request()
                     .analyzer("foobar")
                     .text("the qu1ck brown fox"),
-                "text", null, notGlobal ? indexAnalyzers : null, registry, environment, maxTokenCount));
+                registry, notGlobal ? mockIndexService() : null, maxTokenCount));
         if (notGlobal) {
             assertEquals(e.getMessage(), "failed to find analyzer [foobar]");
         } else {
@@ -286,10 +352,10 @@ public void testUnknown() throws IOException {
 
         e = expectThrows(IllegalArgumentException.class,
             () -> TransportAnalyzeAction.analyze(
-                new AnalyzeRequest()
+                new AnalyzeAction.Request()
                     .tokenizer("foobar")
                     .text("the qu1ck brown fox"),
-                "text", null, notGlobal ? indexAnalyzers : null, registry, environment, maxTokenCount));
+                registry, notGlobal ? mockIndexService() : null, maxTokenCount));
         if (notGlobal) {
             assertEquals(e.getMessage(), "failed to find tokenizer under [foobar]");
         } else {
@@ -298,61 +364,74 @@ public void testUnknown() throws IOException {
 
         e = expectThrows(IllegalArgumentException.class,
             () -> TransportAnalyzeAction.analyze(
-                new AnalyzeRequest()
+                new AnalyzeAction.Request()
                     .tokenizer("standard")
                     .addTokenFilter("foobar")
                     .text("the qu1ck brown fox"),
-                "text", null, notGlobal ? indexAnalyzers : null, registry, environment, maxTokenCount));
+                registry, notGlobal ? mockIndexService() : null, maxTokenCount));
         if (notGlobal) {
-            assertEquals(e.getMessage(), "failed to find token filter under [foobar]");
+            assertEquals(e.getMessage(), "failed to find filter under [foobar]");
         } else {
-            assertEquals(e.getMessage(), "failed to find global token filter under [foobar]");
+            assertEquals(e.getMessage(), "failed to find global filter under [foobar]");
         }
 
         e = expectThrows(IllegalArgumentException.class,
             () -> TransportAnalyzeAction.analyze(
-                new AnalyzeRequest()
+                new AnalyzeAction.Request()
                     .tokenizer("standard")
                     .addTokenFilter("lowercase")
                     .addCharFilter("foobar")
                     .text("the qu1ck brown fox"),
-                "text", null, notGlobal ? indexAnalyzers : null, registry, environment, maxTokenCount));
+                registry, notGlobal ? mockIndexService() : null, maxTokenCount));
         if (notGlobal) {
-            assertEquals(e.getMessage(), "failed to find char filter under [foobar]");
+            assertEquals(e.getMessage(), "failed to find char_filter under [foobar]");
         } else {
-            assertEquals(e.getMessage(), "failed to find global char filter under [foobar]");
+            assertEquals(e.getMessage(), "failed to find global char_filter under [foobar]");
         }
 
         e = expectThrows(IllegalArgumentException.class,
             () -> TransportAnalyzeAction.analyze(
-                new AnalyzeRequest()
+                new AnalyzeAction.Request()
                     .normalizer("foobar")
                     .text("the qu1ck brown fox"),
-                "text", null, indexAnalyzers, registry, environment, maxTokenCount));
+                registry, mockIndexService(), maxTokenCount));
         assertEquals(e.getMessage(), "failed to find normalizer under [foobar]");
     }
 
     public void testNonPreBuildTokenFilter() throws IOException {
-        AnalyzeRequest request = new AnalyzeRequest();
+        AnalyzeAction.Request request = new AnalyzeAction.Request();
         request.tokenizer("standard");
         request.addTokenFilter("stop"); // stop token filter is not prebuilt in AnalysisModule#setupPreConfiguredTokenFilters()
         request.text("the quick brown fox");
-        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment,
-            maxTokenCount);
-        List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
+        AnalyzeAction.Response analyze
+            = TransportAnalyzeAction.analyze(request, registry, mockIndexService(), maxTokenCount);
+        List<AnalyzeAction.AnalyzeToken> tokens = analyze.getTokens();
         assertEquals(3, tokens.size());
         assertEquals("quick", tokens.get(0).getTerm());
         assertEquals("brown", tokens.get(1).getTerm());
         assertEquals("fox", tokens.get(2).getTerm());
     }
 
+    public void testCustomCharFilterWithParameters() throws IOException {
+        AnalyzeAction.Request request = new AnalyzeAction.Request();
+        request.tokenizer("standard");
+        request.addCharFilter(Map.of("type", "append", "suffix", "foo"));
+        request.text("quick brown");
+        AnalyzeAction.Response analyze =
+            TransportAnalyzeAction.analyze(request, registry, mockIndexService(), maxTokenCount);
+        List<AnalyzeAction.AnalyzeToken> tokens = analyze.getTokens();
+        assertEquals(2, tokens.size());
+        assertEquals("quick", tokens.get(0).getTerm());
+        assertEquals("brownfoo", tokens.get(1).getTerm());
+    }
+
     public void testNormalizerWithIndex() throws IOException {
-        AnalyzeRequest request = new AnalyzeRequest("index");
+        AnalyzeAction.Request request = new AnalyzeAction.Request("index");
         request.normalizer("my_normalizer");
         request.text("ABc");
-        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, "text", null, indexAnalyzers, registry, environment,
-            maxTokenCount);
-        List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
+        AnalyzeAction.Response analyze
+            = TransportAnalyzeAction.analyze(request, registry, mockIndexService(), maxTokenCount);
+        List<AnalyzeAction.AnalyzeToken> tokens = analyze.getTokens();
 
         assertEquals(1, tokens.size());
         assertEquals("abc", tokens.get(0).getTerm());
@@ -362,7 +441,7 @@ public void testNormalizerWithIndex() throws IOException {
      * This test is equivalent of calling _analyze without a specific index.
      * The default value for the maximum token count is used.
      */
-    public void testExceedDefaultMaxTokenLimit() throws IOException{
+    public void testExceedDefaultMaxTokenLimit() {
         // create a string with No. words more than maxTokenCount
         StringBuilder sbText = new StringBuilder();
         for (int i = 0; i <= maxTokenCount; i++){
@@ -372,23 +451,21 @@ public void testExceedDefaultMaxTokenLimit() throws IOException{
         String text = sbText.toString();
 
         // request with explain=false to test simpleAnalyze path in TransportAnalyzeAction
-        AnalyzeRequest request = new AnalyzeRequest();
+        AnalyzeAction.Request request = new AnalyzeAction.Request();
         request.text(text);
         request.analyzer("standard");
         IllegalStateException e = expectThrows(IllegalStateException.class,
-            () -> TransportAnalyzeAction.analyze(
-                request, "text", null, null, registry, environment, maxTokenCount));
+            () -> TransportAnalyzeAction.analyze(request, registry, null, maxTokenCount));
         assertEquals(e.getMessage(), "The number of tokens produced by calling _analyze has exceeded the allowed maximum of ["
             + maxTokenCount + "]." + " This limit can be set by changing the [index.analyze.max_token_count] index level setting.");
 
         // request with explain=true to test detailAnalyze path in TransportAnalyzeAction
-        AnalyzeRequest request2 = new AnalyzeRequest();
+        AnalyzeAction.Request request2 = new AnalyzeAction.Request();
         request2.text(text);
         request2.analyzer("standard");
         request2.explain(true);
         IllegalStateException e2 = expectThrows(IllegalStateException.class,
-            () -> TransportAnalyzeAction.analyze(
-                request2, "text", null, null, registry, environment, maxTokenCount));
+            () -> TransportAnalyzeAction.analyze(request2, registry, null, maxTokenCount));
         assertEquals(e2.getMessage(), "The number of tokens produced by calling _analyze has exceeded the allowed maximum of ["
             + maxTokenCount + "]." + " This limit can be set by changing the [index.analyze.max_token_count] index level setting.");
     }
@@ -397,7 +474,7 @@ public void testExceedDefaultMaxTokenLimit() throws IOException{
      * This test is equivalent of calling _analyze against a specific index.
      * The index specific value for the maximum token count is used.
      */
-    public void testExceedSetMaxTokenLimit() throws IOException{
+    public void testExceedSetMaxTokenLimit() {
         // create a string with No. words more than idxMaxTokenCount
         StringBuilder sbText = new StringBuilder();
         for (int i = 0; i <= idxMaxTokenCount; i++){
@@ -406,12 +483,11 @@ public void testExceedSetMaxTokenLimit() throws IOException{
         }
         String text = sbText.toString();
 
-        AnalyzeRequest request = new AnalyzeRequest();
+        AnalyzeAction.Request request = new AnalyzeAction.Request();
         request.text(text);
         request.analyzer("standard");
         IllegalStateException e = expectThrows(IllegalStateException.class,
-            () -> TransportAnalyzeAction.analyze(
-                request, "text", null, indexAnalyzers, registry, environment, idxMaxTokenCount));
+            () -> TransportAnalyzeAction.analyze(request, registry, null, idxMaxTokenCount));
         assertEquals(e.getMessage(), "The number of tokens produced by calling _analyze has exceeded the allowed maximum of ["
             + idxMaxTokenCount + "]." + " This limit can be set by changing the [index.analyze.max_token_count] index level setting.");
     }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/ValidateIndicesAliasesRequestIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/ValidateIndicesAliasesRequestIT.java
new file mode 100644
index 0000000000000..3a2fd844fd406
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/ValidateIndicesAliasesRequestIT.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.alias;
+
+import org.elasticsearch.action.RequestValidators;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.function.Function;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.hasToString;
+
+public class ValidateIndicesAliasesRequestIT extends ESSingleNodeTestCase {
+
+    public static class IndicesAliasesPlugin extends Plugin implements ActionPlugin {
+
+        static final Setting<List<String>> ALLOWED_ORIGINS_SETTING = Setting.listSetting(
+                "index.aliases.allowed_origins",
+                Collections.emptyList(),
+                Function.identity(),
+                Setting.Property.IndexScope,
+                Setting.Property.Dynamic);
+
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Collections.singletonList(ALLOWED_ORIGINS_SETTING);
+        }
+
+        @Override
+        public Collection<RequestValidators.RequestValidator<IndicesAliasesRequest>> indicesAliasesRequestValidators() {
+            return Collections.singletonList((request, state, indices) -> {
+                for (final Index index : indices) {
+                    final List<String> allowedOrigins = ALLOWED_ORIGINS_SETTING.get(state.metaData().index(index).getSettings());
+                    if (allowedOrigins.contains(request.origin()) == false) {
+                        final String message = String.format(
+                                Locale.ROOT,
+                                "origin [%s] not allowed for index [%s]",
+                                request.origin(),
+                                index.getName());
+                        return Optional.of(new IllegalStateException(message));
+                    }
+                }
+                return Optional.empty();
+            });
+        }
+
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.singletonList(IndicesAliasesPlugin.class);
+    }
+
+    public void testAllowed() {
+        final Settings settings = Settings.builder()
+                .putList(IndicesAliasesPlugin.ALLOWED_ORIGINS_SETTING.getKey(), Collections.singletonList("allowed"))
+                .build();
+        createIndex("index", settings);
+        final IndicesAliasesRequest request = new IndicesAliasesRequest().origin("allowed");
+        request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("index").alias("alias"));
+        assertAcked(client().admin().indices().aliases(request).actionGet());
+        final GetAliasesResponse response = client().admin().indices().getAliases(new GetAliasesRequest("alias")).actionGet();
+        assertThat(response.getAliases().keys().size(), equalTo(1));
+        assertThat(response.getAliases().keys().iterator().next().value, equalTo("index"));
+        final List<AliasMetaData> aliasMetaData = response.getAliases().get("index");
+        assertThat(aliasMetaData, hasSize(1));
+        assertThat(aliasMetaData.get(0).alias(), equalTo("alias"));
+    }
+
+    public void testNotAllowed() {
+        final Settings settings = Settings.builder()
+                .putList(IndicesAliasesPlugin.ALLOWED_ORIGINS_SETTING.getKey(), Collections.singletonList("allowed"))
+                .build();
+        createIndex("index", settings);
+        final String origin = randomFrom("", "not-allowed");
+        final IndicesAliasesRequest request = new IndicesAliasesRequest().origin(origin);
+        request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("index").alias("alias"));
+        final Exception e = expectThrows(IllegalStateException.class, () -> client().admin().indices().aliases(request).actionGet());
+        assertThat(e, hasToString(containsString("origin [" + origin + "] not allowed for index [index]")));
+    }
+
+    public void testSomeAllowed() {
+        final Settings fooIndexSettings = Settings.builder()
+                .putList(IndicesAliasesPlugin.ALLOWED_ORIGINS_SETTING.getKey(), Collections.singletonList("foo_allowed"))
+                .build();
+        createIndex("foo", fooIndexSettings);
+        final Settings barIndexSettings = Settings.builder()
+                .putList(IndicesAliasesPlugin.ALLOWED_ORIGINS_SETTING.getKey(), Collections.singletonList("bar_allowed"))
+                .build();
+        createIndex("bar", barIndexSettings);
+        final String origin = randomFrom("foo_allowed", "bar_allowed");
+        final IndicesAliasesRequest request = new IndicesAliasesRequest().origin(origin);
+        request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("foo").alias("alias"));
+        request.addAliasAction(IndicesAliasesRequest.AliasActions.add().index("bar").alias("alias"));
+        final Exception e = expectThrows(IllegalStateException.class, () -> client().admin().indices().aliases(request).actionGet());
+        final String index = "foo_allowed".equals(origin) ? "bar" : "foo";
+        assertThat(e, hasToString(containsString("origin [" + origin + "] not allowed for index [" + index + "]")));
+        assertTrue(client().admin().indices().getAliases(new GetAliasesRequest("alias")).actionGet().getAliases().isEmpty());
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestTests.java
index d83b2fae0f917..6d6eef7a99882 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestTests.java
@@ -26,11 +26,12 @@
 
 import java.io.IOException;
 
+import static org.hamcrest.CoreMatchers.containsString;
 
 public class AnalyzeRequestTests extends ESTestCase {
 
-    public void testValidation() throws Exception {
-        AnalyzeRequest request = new AnalyzeRequest();
+    public void testValidation() {
+        AnalyzeAction.Request request = new AnalyzeAction.Request();
 
         ActionRequestValidationException e = request.validate();
         assertNotNull("text validation should fail", e);
@@ -60,16 +61,45 @@ public void testValidation() throws Exception {
         e = request.validate();
         assertTrue(e.getMessage().contains("tokenizer/analyze should be null if normalizer is specified"));
 
-        AnalyzeRequest requestAnalyzer = new AnalyzeRequest("index");
+        AnalyzeAction.Request requestAnalyzer = new AnalyzeAction.Request("index");
         requestAnalyzer.normalizer("some normalizer");
         requestAnalyzer.text("something");
         requestAnalyzer.analyzer("analyzer");
         e = requestAnalyzer.validate();
         assertTrue(e.getMessage().contains("tokenizer/analyze should be null if normalizer is specified"));
+
+        {
+            AnalyzeAction.Request analyzerPlusDefs = new AnalyzeAction.Request("index");
+            analyzerPlusDefs.text("text");
+            analyzerPlusDefs.analyzer("analyzer");
+            analyzerPlusDefs.addTokenFilter("tokenfilter");
+            e = analyzerPlusDefs.validate();
+            assertNotNull(e);
+            assertThat(e.getMessage(), containsString("cannot define extra components on a named analyzer"));
+        }
+
+        {
+            AnalyzeAction.Request analyzerPlusDefs = new AnalyzeAction.Request("index");
+            analyzerPlusDefs.text("text");
+            analyzerPlusDefs.normalizer("normalizer");
+            analyzerPlusDefs.addTokenFilter("tokenfilter");
+            e = analyzerPlusDefs.validate();
+            assertNotNull(e);
+            assertThat(e.getMessage(), containsString("cannot define extra components on a named normalizer"));
+        }
+        {
+            AnalyzeAction.Request analyzerPlusDefs = new AnalyzeAction.Request("index");
+            analyzerPlusDefs.text("text");
+            analyzerPlusDefs.field("field");
+            analyzerPlusDefs.addTokenFilter("tokenfilter");
+            e = analyzerPlusDefs.validate();
+            assertNotNull(e);
+            assertThat(e.getMessage(), containsString("cannot define extra components on a field-specific analyzer"));
+        }
     }
 
     public void testSerialization() throws IOException {
-        AnalyzeRequest request = new AnalyzeRequest("foo");
+        AnalyzeAction.Request request = new AnalyzeAction.Request("foo");
         request.text("a", "b");
         request.tokenizer("tokenizer");
         request.addTokenFilter("tokenfilter");
@@ -79,8 +109,7 @@ public void testSerialization() throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             request.writeTo(output);
             try (StreamInput in = output.bytes().streamInput()) {
-                AnalyzeRequest serialized = new AnalyzeRequest();
-                serialized.readFrom(in);
+                AnalyzeAction.Request serialized = new AnalyzeAction.Request(in);
                 assertArrayEquals(request.text(), serialized.text());
                 assertEquals(request.tokenizer().name, serialized.tokenizer().name);
                 assertEquals(request.tokenFilters().get(0).name, serialized.tokenFilters().get(0).name);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java
index a4cee7a4cde2a..95fc010f37f86 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java
@@ -20,124 +20,35 @@
 package org.elasticsearch.action.admin.indices.analyze;
 
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Predicate;
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class AnalyzeResponseTests extends AbstractSerializingTestCase<AnalyzeResponse> {
-
-    @Override
-    protected Predicate<String> getRandomFieldsExcludeFilter() {
-        return s -> s.contains("tokens.");
-    }
-
-    @Override
-    protected AnalyzeResponse doParseInstance(XContentParser parser) throws IOException {
-        return AnalyzeResponse.fromXContent(parser);
-    }
-
-    @Override
-    protected Writeable.Reader<AnalyzeResponse> instanceReader() {
-        return AnalyzeResponse::new;
-    }
-
-    @Override
-    protected AnalyzeResponse createTestInstance() {
-        int tokenCount = randomIntBetween(1, 30);
-        AnalyzeResponse.AnalyzeToken[] tokens = new AnalyzeResponse.AnalyzeToken[tokenCount];
-        for (int i = 0; i < tokenCount; i++) {
-            tokens[i] = randomToken();
-        }
-        if (randomBoolean()) {
-            DetailAnalyzeResponse.CharFilteredText[] charfilters = null;
-            DetailAnalyzeResponse.AnalyzeTokenList[] tokenfilters = null;
-            if (randomBoolean()) {
-                charfilters = new DetailAnalyzeResponse.CharFilteredText[]{
-                    new DetailAnalyzeResponse.CharFilteredText("my_charfilter", new String[]{"one two"})
-                };
-            }
-            if (randomBoolean()) {
-                tokenfilters = new DetailAnalyzeResponse.AnalyzeTokenList[]{
-                    new DetailAnalyzeResponse.AnalyzeTokenList("my_tokenfilter_1", tokens),
-                    new DetailAnalyzeResponse.AnalyzeTokenList("my_tokenfilter_2", tokens)
-                };
-            }
-            DetailAnalyzeResponse dar = new DetailAnalyzeResponse(
-                charfilters,
-                new DetailAnalyzeResponse.AnalyzeTokenList("my_tokenizer", tokens),
-                tokenfilters);
-            return new AnalyzeResponse(null, dar);
-        }
-        return new AnalyzeResponse(Arrays.asList(tokens), null);
-    }
-
-    private AnalyzeResponse.AnalyzeToken randomToken() {
-        String token = randomAlphaOfLengthBetween(1, 20);
-        int position = randomIntBetween(0, 1000);
-        int startOffset = randomIntBetween(0, 1000);
-        int endOffset = randomIntBetween(0, 1000);
-        int posLength = randomIntBetween(1, 5);
-        String type = randomAlphaOfLengthBetween(1, 20);
-        Map<String, Object> extras = new HashMap<>();
-        if (randomBoolean()) {
-            int entryCount = randomInt(6);
-            for (int i = 0; i < entryCount; i++) {
-                switch (randomInt(6)) {
-                    case 0:
-                    case 1:
-                    case 2:
-                    case 3:
-                        String key = randomAlphaOfLength(5);
-                        String value = randomAlphaOfLength(10);
-                        extras.put(key, value);
-                        break;
-                    case 4:
-                        String objkey = randomAlphaOfLength(5);
-                        Map<String, String> obj = new HashMap<>();
-                        obj.put(randomAlphaOfLength(5), randomAlphaOfLength(10));
-                        extras.put(objkey, obj);
-                        break;
-                    case 5:
-                        String listkey = randomAlphaOfLength(5);
-                        List<String> list = new ArrayList<>();
-                        list.add(randomAlphaOfLength(4));
-                        list.add(randomAlphaOfLength(6));
-                        extras.put(listkey, list);
-                        break;
-                }
-            }
-        }
-        return new AnalyzeResponse.AnalyzeToken(token, position, startOffset, endOffset, posLength, type, extras);
-    }
+public class AnalyzeResponseTests extends ESTestCase {
 
+    @SuppressWarnings("unchecked")
     public void testNullResponseToXContent() throws IOException {
-        DetailAnalyzeResponse.CharFilteredText[] charfilters = null;
+        AnalyzeAction.CharFilteredText[] charfilters = null;
 
         String name = "test_tokens_null";
-        AnalyzeResponse.AnalyzeToken[] tokens = null;
-        DetailAnalyzeResponse.AnalyzeTokenList tokenizer = null;
+        AnalyzeAction.AnalyzeToken[] tokens = null;
+        AnalyzeAction.AnalyzeTokenList tokenizer = null;
 
 
-        DetailAnalyzeResponse.AnalyzeTokenList tokenfiltersItem = new DetailAnalyzeResponse.AnalyzeTokenList(name, tokens);
-        DetailAnalyzeResponse.AnalyzeTokenList[] tokenfilters = {tokenfiltersItem};
+        AnalyzeAction.AnalyzeTokenList tokenfiltersItem = new AnalyzeAction.AnalyzeTokenList(name, tokens);
+        AnalyzeAction.AnalyzeTokenList[] tokenfilters = {tokenfiltersItem};
 
-        DetailAnalyzeResponse detail = new DetailAnalyzeResponse(charfilters, tokenizer, tokenfilters);
+        AnalyzeAction.DetailAnalyzeResponse detail = new AnalyzeAction.DetailAnalyzeResponse(charfilters, tokenizer, tokenfilters);
 
-        AnalyzeResponse response = new AnalyzeResponse(null, detail);
+        AnalyzeAction.Response response = new AnalyzeAction.Response(null, detail);
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
             response.toXContent(builder, ToXContent.EMPTY_PARAMS);
             Map<String, Object> converted = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2();
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheResponseTests.java
index 4b58d5fb70246..76e96b9b05574 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheResponseTests.java
@@ -20,8 +20,8 @@
 package org.elasticsearch.action.admin.indices.cache.clear;
 
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
-import org.elasticsearch.action.support.broadcast.AbstractBroadcastResponseTestCase;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractBroadcastResponseTestCase;
 
 import java.util.List;
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
index 985b4304a32f4..df940012bf24d 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java
@@ -28,8 +28,6 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 
-import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
-
 public class CloseIndexRequestTests extends ESTestCase {
 
     public void testSerialization() throws Exception {
@@ -54,7 +52,8 @@ public void testBwcSerialization() throws Exception {
         {
             final CloseIndexRequest request = randomRequest();
             try (BytesStreamOutput out = new BytesStreamOutput()) {
-                out.setVersion(randomVersionBetween(random(), Version.V_6_4_0, VersionUtils.getPreviousVersion(Version.V_7_2_0)));
+                out.setVersion(VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(),
+                    VersionUtils.getPreviousVersion(Version.V_7_2_0)));
                 request.writeTo(out);
 
                 try (StreamInput in = out.bytes().streamInput()) {
@@ -77,7 +76,8 @@ public void testBwcSerialization() throws Exception {
 
                 final CloseIndexRequest deserializedRequest = new CloseIndexRequest();
                 try (StreamInput in = out.bytes().streamInput()) {
-                    in.setVersion(randomVersionBetween(random(), Version.V_6_4_0, VersionUtils.getPreviousVersion(Version.V_7_2_0)));
+                    in.setVersion(VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(),
+                        VersionUtils.getPreviousVersion(Version.V_7_2_0)));
                     deserializedRequest.readFrom(in);
                 }
                 assertEquals(sample.getParentTask(), deserializedRequest.getParentTask());
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponseTests.java
index aa1c0b3bbb445..01fc1c0343e2c 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexResponseTests.java
@@ -22,17 +22,24 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.NoShardAvailableActionException;
+import org.elasticsearch.action.admin.indices.close.CloseIndexResponse.IndexResult;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.ActionNotFoundTransportException;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
 import static org.elasticsearch.test.VersionUtils.getPreviousVersion;
@@ -46,16 +53,47 @@
 
 public class CloseIndexResponseTests extends ESTestCase {
 
+    /**
+     * Test that random responses can be written to xcontent without errors.
+     * Also check some specific simple cases for output.
+     */
+    public void testToXContent() throws IOException {
+        CloseIndexResponse response = randomResponse();
+        XContentType xContentType = randomFrom(XContentType.values());
+        try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
+            response.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        }
+
+        Index index = new Index("test", "uuid");
+        IndexResult indexResult = new CloseIndexResponse.IndexResult(index);
+        CloseIndexResponse closeIndexResponse = new CloseIndexResponse(true, true,
+                Collections.singletonList(indexResult));
+        assertEquals("{\"acknowledged\":true,\"shards_acknowledged\":true,\"indices\":{\"test\":{\"closed\":true}}}",
+                Strings.toString(closeIndexResponse));
+
+        CloseIndexResponse.ShardResult[] shards = new CloseIndexResponse.ShardResult[1];
+        shards[0] = new CloseIndexResponse.ShardResult(0, new CloseIndexResponse.ShardResult.Failure[] {
+                new CloseIndexResponse.ShardResult.Failure("test", 0, new ActionNotFoundTransportException("test"), "nodeId") });
+        indexResult = new CloseIndexResponse.IndexResult(index, shards);
+        closeIndexResponse = new CloseIndexResponse(true, true,
+                Collections.singletonList(indexResult));
+        assertEquals("{\"acknowledged\":true,\"shards_acknowledged\":true,"
+                + "\"indices\":{\"test\":{\"closed\":false,\"failedShards\":{\"0\":{"
+                + "\"failures\":[{\"node\":\"nodeId\",\"shard\":0,\"index\":\"test\",\"status\":\"INTERNAL_SERVER_ERROR\","
+                + "\"reason\":{\"type\":\"action_not_found_transport_exception\","
+                + "\"reason\":\"No handler for action [test]\"}}]}}}}}",
+                Strings.toString(closeIndexResponse));
+    }
+
     public void testSerialization() throws Exception {
         final CloseIndexResponse response = randomResponse();
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             response.writeTo(out);
 
-            final CloseIndexResponse deserializedResponse = new CloseIndexResponse();
             try (StreamInput in = out.bytes().streamInput()) {
-                deserializedResponse.readFrom(in);
+                final CloseIndexResponse deserializedResponse = new CloseIndexResponse(in);
+                assertCloseIndexResponse(deserializedResponse, response);
             }
-            assertCloseIndexResponse(deserializedResponse, response);
         }
     }
 
@@ -66,12 +104,11 @@ public void testBwcSerialization() throws Exception {
                 out.setVersion(randomVersionBetween(random(), Version.V_7_0_0, getPreviousVersion(Version.V_7_2_0)));
                 response.writeTo(out);
 
-                final AcknowledgedResponse deserializedResponse = new AcknowledgedResponse();
                 try (StreamInput in = out.bytes().streamInput()) {
                     in.setVersion(out.getVersion());
-                    deserializedResponse.readFrom(in);
+                    final AcknowledgedResponse deserializedResponse = new AcknowledgedResponse(in);
+                    assertThat(deserializedResponse.isAcknowledged(), equalTo(response.isAcknowledged()));
                 }
-                assertThat(deserializedResponse.isAcknowledged(), equalTo(response.isAcknowledged()));
             }
         }
         {
@@ -79,12 +116,11 @@ public void testBwcSerialization() throws Exception {
             try (BytesStreamOutput out = new BytesStreamOutput()) {
                 response.writeTo(out);
 
-                final CloseIndexResponse deserializedResponse = new CloseIndexResponse();
                 try (StreamInput in = out.bytes().streamInput()) {
                     in.setVersion(randomVersionBetween(random(), Version.V_7_0_0, getPreviousVersion(Version.V_7_2_0)));
-                    deserializedResponse.readFrom(in);
+                    final CloseIndexResponse deserializedResponse = new CloseIndexResponse(in);
+                    assertThat(deserializedResponse.isAcknowledged(), equalTo(response.isAcknowledged()));
                 }
-                assertThat(deserializedResponse.isAcknowledged(), equalTo(response.isAcknowledged()));
             }
         }
         {
@@ -93,17 +129,16 @@ public void testBwcSerialization() throws Exception {
                 Version version = randomVersionBetween(random(), Version.V_7_2_0, Version.CURRENT);
                 out.setVersion(version);
                 response.writeTo(out);
-                final CloseIndexResponse deserializedResponse = new CloseIndexResponse();
                 try (StreamInput in = out.bytes().streamInput()) {
                     in.setVersion(version);
-                    deserializedResponse.readFrom(in);
-                }
-                assertThat(deserializedResponse.isAcknowledged(), equalTo(response.isAcknowledged()));
-                assertThat(deserializedResponse.isShardsAcknowledged(), equalTo(response.isShardsAcknowledged()));
-                if (version.onOrAfter(Version.V_7_3_0)) {
-                    assertThat(deserializedResponse.getIndices(), hasSize(response.getIndices().size()));
-                } else {
-                    assertThat(deserializedResponse.getIndices(), empty());
+                    final CloseIndexResponse deserializedResponse = new CloseIndexResponse(in);
+                    assertThat(deserializedResponse.isAcknowledged(), equalTo(response.isAcknowledged()));
+                    assertThat(deserializedResponse.isShardsAcknowledged(), equalTo(response.isShardsAcknowledged()));
+                    if (version.onOrAfter(Version.V_7_3_0)) {
+                        assertThat(deserializedResponse.getIndices(), hasSize(response.getIndices().size()));
+                    } else {
+                        assertThat(deserializedResponse.getIndices(), empty());
+                    }
                 }
             }
         }
@@ -131,7 +166,10 @@ private CloseIndexResponse randomResponse() {
                             acknowledged = false;
                             failures = new CloseIndexResponse.ShardResult.Failure[randomIntBetween(1, 3)];
                             for (int j = 0; j < failures.length; j++) {
-                                String nodeId = randomAlphaOfLength(5);
+                                String nodeId = null;
+                                if (frequently()) {
+                                    nodeId = randomAlphaOfLength(5);
+                                }
                                 failures[j] = new CloseIndexResponse.ShardResult.Failure(indexName, i, randomException(index, i), nodeId);
                             }
                         }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java
index d7974ed1c6365..6913b518d2464 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java
@@ -136,9 +136,13 @@ public static void afterClass() {
     }
 
     private void executeOnPrimaryOrReplica() throws Throwable {
+        executeOnPrimaryOrReplica(false);
+    }
+
+    private void executeOnPrimaryOrReplica(boolean phase1) throws Throwable {
         final TaskId taskId = new TaskId("_node_id", randomNonNegativeLong());
         final TransportVerifyShardBeforeCloseAction.ShardRequest request =
-            new TransportVerifyShardBeforeCloseAction.ShardRequest(indexShard.shardId(), clusterBlock, taskId);
+            new TransportVerifyShardBeforeCloseAction.ShardRequest(indexShard.shardId(), clusterBlock, phase1, taskId);
         final PlainActionFuture<Void> res = PlainActionFuture.newFuture();
         action.shardOperationOnPrimary(request, indexShard, ActionListener.wrap(
             r -> {
@@ -165,6 +169,11 @@ public void testShardIsFlushed() throws Throwable {
         assertThat(flushRequest.getValue().force(), is(true));
     }
 
+    public void testShardIsSynced() throws Throwable {
+        executeOnPrimaryOrReplica(true);
+        verify(indexShard, times(1)).sync();
+    }
+
     public void testOperationFailsWhenNotBlocked() {
         when(indexShard.getActiveOperationsCount()).thenReturn(randomIntBetween(0, 10));
 
@@ -227,7 +236,7 @@ public void testUnavailableShardsMarkedAsStale() throws Exception {
         final PlainActionFuture<PrimaryResult> listener = new PlainActionFuture<>();
         TaskId taskId = new TaskId(clusterService.localNode().getId(), 0L);
         TransportVerifyShardBeforeCloseAction.ShardRequest request =
-            new TransportVerifyShardBeforeCloseAction.ShardRequest(shardId, clusterBlock, taskId);
+            new TransportVerifyShardBeforeCloseAction.ShardRequest(shardId, clusterBlock, false, taskId);
         ReplicationOperation.Replicas<TransportVerifyShardBeforeCloseAction.ShardRequest> proxy = action.newReplicasProxy();
         ReplicationOperation<TransportVerifyShardBeforeCloseAction.ShardRequest,
             TransportVerifyShardBeforeCloseAction.ShardRequest, PrimaryResult> operation = new ReplicationOperation<>(
@@ -303,6 +312,11 @@ public long localCheckpoint() {
                         return 0;
                     }
 
+                    @Override
+                    public long computedGlobalCheckpoint() {
+                        return 0;
+                    }
+
                     @Override
                     public long globalCheckpoint() {
                         return 0;
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponseTests.java
index c0685d5d17d29..d93fd984079be 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponseTests.java
@@ -20,13 +20,14 @@
 package org.elasticsearch.action.admin.indices.create;
 
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.test.AbstractSerializingTestCase;
 
 import java.io.IOException;
 
-public class CreateIndexResponseTests extends AbstractStreamableXContentTestCase<CreateIndexResponse> {
+public class CreateIndexResponseTests extends AbstractSerializingTestCase<CreateIndexResponse> {
 
     @Override
     protected CreateIndexResponse createTestInstance() {
@@ -37,8 +38,8 @@ protected CreateIndexResponse createTestInstance() {
     }
 
     @Override
-    protected CreateIndexResponse createBlankInstance() {
-        return new CreateIndexResponse();
+    protected Writeable.Reader<CreateIndexResponse> instanceReader() {
+        return CreateIndexResponse::new;
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java
index b14bdd0ed9883..32cc6564e5ca0 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java
@@ -62,7 +62,6 @@
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.util.Arrays;
 import java.util.Map;
@@ -81,7 +80,6 @@ protected boolean forbidPrivateIndexSettings() {
         return false;
     }
 
-    @TestLogging("org.elasticsearch.index.store:DEBUG")
     public void testCreateShrinkIndexToN() {
 
         assumeFalse("https://github.com/elastic/elasticsearch/issues/34080", Constants.WINDOWS);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java
deleted file mode 100644
index 51d3ecc89afc7..0000000000000
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.admin.indices.exists;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.discovery.MasterNotDiscoveredException;
-import org.elasticsearch.gateway.GatewayService;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.InternalTestCluster;
-
-import java.io.IOException;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
-
-@ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0, autoMinMasterNodes = false)
-public class IndicesExistsIT extends ESIntegTestCase {
-
-    public void testIndexExistsWithBlocksInPlace() throws IOException {
-        internalCluster().setBootstrapMasterNodeIndex(0);
-        Settings settings = Settings.builder()
-            .put(GatewayService.RECOVER_AFTER_NODES_SETTING.getKey(), 99).build();
-        String node = internalCluster().startNode(settings);
-
-        assertThrows(client(node).admin().indices().prepareExists("test").setMasterNodeTimeout(TimeValue.timeValueSeconds(0)),
-            MasterNotDiscoveredException.class);
-
-        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node)); // shut down node so that test properly cleans up
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/flush/FlushResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/flush/FlushResponseTests.java
index 7b7cb8578e8b3..029426968f93c 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/flush/FlushResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/flush/FlushResponseTests.java
@@ -20,8 +20,8 @@
 package org.elasticsearch.action.admin.indices.flush;
 
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
-import org.elasticsearch.action.support.broadcast.AbstractBroadcastResponseTestCase;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractBroadcastResponseTestCase;
 
 import java.util.List;
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeResponseTests.java
index f5e86fdcdfe9b..1ce73edf4a914 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/forcemerge/ForceMergeResponseTests.java
@@ -20,8 +20,8 @@
 package org.elasticsearch.action.admin.indices.forcemerge;
 
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
-import org.elasticsearch.action.support.broadcast.AbstractBroadcastResponseTestCase;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractBroadcastResponseTestCase;
 
 import java.util.List;
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/ValidateMappingRequestPluginIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/ValidateMappingRequestPluginIT.java
index b25c9ecb5fc91..48398742e9dd6 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/ValidateMappingRequestPluginIT.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/ValidateMappingRequestPluginIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.indices.mapping.put;
 
+import org.elasticsearch.action.RequestValidators;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.plugins.ActionPlugin;
@@ -29,6 +30,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
@@ -38,14 +40,15 @@ public class ValidateMappingRequestPluginIT extends ESSingleNodeTestCase {
     static final Map<String, Collection<String>> allowedOrigins = ConcurrentCollections.newConcurrentMap();
     public static class TestPlugin extends Plugin implements ActionPlugin {
         @Override
-        public Collection<MappingRequestValidator> mappingRequestValidators() {
+        public Collection<RequestValidators.RequestValidator<PutMappingRequest>> mappingRequestValidators() {
             return Collections.singletonList((request, state, indices) -> {
                 for (Index index : indices) {
                     if (allowedOrigins.getOrDefault(index.getName(), Collections.emptySet()).contains(request.origin()) == false) {
-                        return new IllegalStateException("not allowed: index[" + index.getName() + "] origin[" + request.origin() + "]");
+                        return Optional.of(
+                                new IllegalStateException("not allowed: index[" + index.getName() + "] origin[" + request.origin() + "]"));
                     }
                 }
-                return null;
+                return Optional.empty();
             });
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponseTests.java
index 2ed86ca5d3be6..8e4326f169cb2 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/open/OpenIndexResponseTests.java
@@ -19,10 +19,11 @@
 
 package org.elasticsearch.action.admin.indices.open;
 
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.test.AbstractSerializingTestCase;
 
-public class OpenIndexResponseTests extends AbstractStreamableXContentTestCase<OpenIndexResponse> {
+public class OpenIndexResponseTests extends AbstractSerializingTestCase<OpenIndexResponse> {
 
     @Override
     protected OpenIndexResponse doParseInstance(XContentParser parser){
@@ -37,8 +38,8 @@ protected OpenIndexResponse createTestInstance() {
     }
 
     @Override
-    protected OpenIndexResponse createBlankInstance() {
-        return new OpenIndexResponse();
+    protected Writeable.Reader<OpenIndexResponse> instanceReader() {
+        return OpenIndexResponse::new;
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/refresh/RefreshResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/refresh/RefreshResponseTests.java
index b3d97f2edd448..dbe96aa387c99 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/refresh/RefreshResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/refresh/RefreshResponseTests.java
@@ -20,8 +20,8 @@
 package org.elasticsearch.action.admin.indices.refresh;
 
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
-import org.elasticsearch.action.support.broadcast.AbstractBroadcastResponseTestCase;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractBroadcastResponseTestCase;
 
 import java.util.List;
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java
index a38de844626dc..51bcbc6168f4e 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java
@@ -19,11 +19,11 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
-
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.test.AbstractSerializingTestCase;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -32,7 +32,7 @@
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 
-public class RolloverResponseTests extends AbstractStreamableXContentTestCase<RolloverResponse> {
+public class RolloverResponseTests extends AbstractSerializingTestCase<RolloverResponse> {
 
     @Override
     protected RolloverResponse createTestInstance() {
@@ -61,8 +61,8 @@ private static Map<String, Boolean> randomResults(boolean allowNoItems) {
     }
 
     @Override
-    protected RolloverResponse createBlankInstance() {
-        return new RolloverResponse();
+    protected Writeable.Reader<RolloverResponse> instanceReader() {
+        return RolloverResponse::new;
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
index 058dcc7243029..9dac4a38a36b6 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -21,18 +21,19 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
-import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsTests;
+import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.client.AdminClient;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.client.IndicesAdminClient;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.AliasAction;
@@ -43,6 +44,10 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexAliasesService;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
@@ -50,12 +55,30 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.index.cache.query.QueryCacheStats;
+import org.elasticsearch.index.cache.request.RequestCacheStats;
+import org.elasticsearch.index.engine.SegmentsStats;
+import org.elasticsearch.index.fielddata.FieldDataStats;
+import org.elasticsearch.index.flush.FlushStats;
+import org.elasticsearch.index.get.GetStats;
+import org.elasticsearch.index.merge.MergeStats;
+import org.elasticsearch.index.refresh.RefreshStats;
+import org.elasticsearch.index.search.stats.SearchStats;
 import org.elasticsearch.index.shard.DocsStats;
+import org.elasticsearch.index.shard.IndexingStats;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.shard.ShardPath;
+import org.elasticsearch.index.store.StoreStats;
+import org.elasticsearch.index.warmer.WarmerStats;
+import org.elasticsearch.search.suggest.completion.CompletionStats;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.mockito.ArgumentCaptor;
 
+import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -63,12 +86,14 @@
 import java.util.Map;
 import java.util.Set;
 
+import static java.util.Collections.emptyList;
 import static org.elasticsearch.action.admin.indices.rollover.TransportRolloverAction.evaluateConditions;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
@@ -165,6 +190,35 @@ public void testEvaluateWithoutDocStats() {
         }
     }
 
+    public void testEvaluateWithoutMetaData() {
+        MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L);
+        MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(2));
+        MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 100), ByteSizeUnit.MB));
+
+        long matchMaxDocs = randomIntBetween(100, 1000);
+        final Set<Condition<?>> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition);
+        Map<String, Boolean> results = evaluateConditions(conditions,
+            new DocsStats(matchMaxDocs, 0L, ByteSizeUnit.MB.toBytes(120)), null);
+        assertThat(results.size(), equalTo(3));
+        results.forEach((k, v) -> assertFalse(v));
+
+        final Settings settings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 1000))
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomInt(10))
+            .build();
+
+        final IndexMetaData metaData = IndexMetaData.builder(randomAlphaOfLength(10))
+            .creationDate(System.currentTimeMillis() - TimeValue.timeValueHours(randomIntBetween(5, 10)).getMillis())
+            .settings(settings)
+            .build();
+        IndicesStatsResponse indicesStats = randomIndicesStatsResponse(new IndexMetaData[]{metaData});
+        Map<String, Boolean> results2 = evaluateConditions(conditions, null, indicesStats);
+        assertThat(results2.size(), equalTo(3));
+        results2.forEach((k, v) -> assertFalse(v));
+    }
+
     public void testCreateUpdateAliasRequest() {
         String sourceAlias = randomAlphaOfLength(10);
         String sourceIndex = randomAlphaOfLength(10);
@@ -317,6 +371,9 @@ public void testRejectDuplicateAlias() {
     public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPrimariesFromWriteIndex() {
         final TransportService mockTransportService = mock(TransportService.class);
         final ClusterService mockClusterService = mock(ClusterService.class);
+        final DiscoveryNode mockNode = mock(DiscoveryNode.class);
+        when(mockNode.getId()).thenReturn("mocknode");
+        when(mockClusterService.localNode()).thenReturn(mockNode);
         final ThreadPool mockThreadPool = mock(ThreadPool.class);
         final MetaDataCreateIndexService mockCreateIndexService = mock(MetaDataCreateIndexService.class);
         final IndexNameExpressionResolver mockIndexNameExpressionResolver = mock(IndexNameExpressionResolver.class);
@@ -325,31 +382,22 @@ public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPr
         final MetaDataIndexAliasesService mdIndexAliasesService = mock(MetaDataIndexAliasesService.class);
 
         final Client mockClient = mock(Client.class);
-        final AdminClient mockAdminClient = mock(AdminClient.class);
-        final IndicesAdminClient mockIndicesAdminClient = mock(IndicesAdminClient.class);
-        when(mockClient.admin()).thenReturn(mockAdminClient);
-        when(mockAdminClient.indices()).thenReturn(mockIndicesAdminClient);
 
-        final IndicesStatsRequestBuilder mockIndicesStatsBuilder = mock(IndicesStatsRequestBuilder.class);
-        when(mockIndicesAdminClient.prepareStats(any())).thenReturn(mockIndicesStatsBuilder);
         final Map<String, IndexStats> indexStats = new HashMap<>();
         int total = randomIntBetween(500, 1000);
         indexStats.put("logs-index-000001", createIndexStats(200L, total));
         indexStats.put("logs-index-000002", createIndexStats(300L, total));
         final IndicesStatsResponse statsResponse = createAliasToMultipleIndicesStatsResponse(indexStats);
-        when(mockIndicesStatsBuilder.clear()).thenReturn(mockIndicesStatsBuilder);
-        when(mockIndicesStatsBuilder.setDocs(true)).thenReturn(mockIndicesStatsBuilder);
-
-        assert statsResponse.getPrimaries().getDocs().getCount() == 500L;
-        assert statsResponse.getTotal().getDocs().getCount() == (total + total);
-
         doAnswer(invocation -> {
             Object[] args = invocation.getArguments();
-            assert args.length == 1;
-            ActionListener<IndicesStatsResponse> listener = (ActionListener<IndicesStatsResponse>) args[0];
+            assert args.length == 3;
+            ActionListener<IndicesStatsResponse> listener = (ActionListener<IndicesStatsResponse>) args[2];
             listener.onResponse(statsResponse);
             return null;
-        }).when(mockIndicesStatsBuilder).execute(any(ActionListener.class));
+        }).when(mockClient).execute(eq(IndicesStatsAction.INSTANCE), any(ActionRequest.class), any(ActionListener.class));
+
+        assert statsResponse.getPrimaries().getDocs().getCount() == 500L;
+        assert statsResponse.getTotal().getDocs().getCount() == (total + total);
 
         final IndexMetaData.Builder indexMetaData = IndexMetaData.builder("logs-index-000001")
                 .putAlias(AliasMetaData.builder("logs-alias").writeIndex(false).build()).settings(settings(Version.CURRENT))
@@ -370,7 +418,7 @@ public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPr
         RolloverRequest rolloverRequest = new RolloverRequest("logs-alias", "logs-index-000003");
         rolloverRequest.addMaxIndexDocsCondition(500L);
         rolloverRequest.dryRun(true);
-        transportRolloverAction.masterOperation(rolloverRequest, stateBefore, future);
+        transportRolloverAction.masterOperation(mock(Task.class), rolloverRequest, stateBefore, future);
 
         RolloverResponse response = future.actionGet();
         assertThat(response.getOldIndex(), equalTo("logs-index-000002"));
@@ -386,7 +434,7 @@ public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPr
         rolloverRequest = new RolloverRequest("logs-alias", "logs-index-000003");
         rolloverRequest.addMaxIndexDocsCondition(300L);
         rolloverRequest.dryRun(true);
-        transportRolloverAction.masterOperation(rolloverRequest, stateBefore, future);
+        transportRolloverAction.masterOperation(mock(Task.class), rolloverRequest, stateBefore, future);
 
         response = future.actionGet();
         assertThat(response.getOldIndex(), equalTo("logs-index-000002"));
@@ -460,4 +508,42 @@ private static Condition<?> createTestCondition() {
         when(condition.evaluate(any())).thenReturn(new Condition.Result(condition, true));
         return condition;
     }
+
+    public static IndicesStatsResponse randomIndicesStatsResponse(final IndexMetaData[] indices) {
+        List<ShardStats> shardStats = new ArrayList<>();
+        for (final IndexMetaData index : indices) {
+            int numShards = randomIntBetween(1, 3);
+            int primaryIdx = randomIntBetween(-1, numShards - 1); // -1 means there is no primary shard.
+            for (int i = 0; i < numShards; i++) {
+                ShardId shardId = new ShardId(index.getIndex(), i);
+                boolean primary = (i == primaryIdx);
+                Path path = createTempDir().resolve("indices").resolve(index.getIndexUUID()).resolve(String.valueOf(i));
+                ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, primary,
+                    primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE,
+                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)
+                );
+                shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
+                shardRouting = shardRouting.moveToStarted();
+                CommonStats stats = new CommonStats();
+                stats.fieldData = new FieldDataStats();
+                stats.queryCache = new QueryCacheStats();
+                stats.docs = new DocsStats();
+                stats.store = new StoreStats();
+                stats.indexing = new IndexingStats();
+                stats.search = new SearchStats();
+                stats.segments = new SegmentsStats();
+                stats.merge = new MergeStats();
+                stats.refresh = new RefreshStats();
+                stats.completion = new CompletionStats();
+                stats.requestCache = new RequestCacheStats();
+                stats.get = new GetStats();
+                stats.flush = new FlushStats();
+                stats.warmer = new WarmerStats();
+                shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null));
+            }
+        }
+        return IndicesStatsTests.newIndicesStatsResponse(
+            shardStats.toArray(new ShardStats[shardStats.size()]), shardStats.size(), shardStats.size(), 0, emptyList()
+        );
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java
index 9ec3daba19bc4..d119dcb1135c4 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/settings/get/GetSettingsActionTests.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -63,9 +64,10 @@ GetSettingsActionTests.this.threadPool, settingsFilter, new ActionFilters(Collec
                 new Resolver(), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
         }
         @Override
-        protected void masterOperation(GetSettingsRequest request, ClusterState state, ActionListener<GetSettingsResponse> listener) {
+        protected void masterOperation(Task task, GetSettingsRequest request, ClusterState state,
+                                       ActionListener<GetSettingsResponse> listener) {
             ClusterState stateWithIndex = ClusterStateCreationUtils.state(indexName, 1, 1);
-            super.masterOperation(request, stateWithIndex, listener);
+            super.masterOperation(task, request, stateWithIndex, listener);
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
index 70d45d1db3ac6..80694a0a8b437 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
@@ -38,7 +38,6 @@
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.store.MockFSIndexStore;
 
 import java.util.Arrays;
@@ -59,8 +58,6 @@
 import static org.hamcrest.Matchers.nullValue;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
-@TestLogging("_root:DEBUG,org.elasticsearch.action.admin.indices.shards:TRACE,org.elasticsearch.cluster.service:TRACE," +
-    "org.elasticsearch.gateway.TransportNodesListGatewayStartedShards:TRACE")
 public class IndicesShardStoreRequestIT extends ESIntegTestCase {
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponseTests.java
index 3e522e451011b..b663e187c78b9 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponseTests.java
@@ -20,10 +20,11 @@
 package org.elasticsearch.action.admin.indices.shrink;
 
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.test.AbstractSerializingTestCase;
 
-public class ResizeResponseTests extends AbstractStreamableXContentTestCase<ResizeResponse> {
+public class ResizeResponseTests extends AbstractSerializingTestCase<ResizeResponse> {
 
     public void testToXContent() {
         ResizeResponse response = new ResizeResponse(true, false, "index_name");
@@ -45,8 +46,8 @@ protected ResizeResponse createTestInstance() {
     }
 
     @Override
-    protected ResizeResponse createBlankInstance() {
-        return new ResizeResponse();
+    protected Writeable.Reader<ResizeResponse> instanceReader() {
+        return ResizeResponse::new;
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java
index 580ea76429388..eaea5babe3b7b 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -219,7 +220,7 @@ private DiscoveryNode newNode(String nodeId) {
                 nodeId,
                 buildNewFakeTransportAddress(),
                 emptyMap(),
-                Set.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA),
+                Set.of(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.DATA_ROLE),
                 Version.CURRENT);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryResponseTests.java
index d72aae8fa2bd1..e5ba103e2d207 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryResponseTests.java
@@ -21,9 +21,9 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
-import org.elasticsearch.action.support.broadcast.AbstractBroadcastResponseTestCase;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractBroadcastResponseTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java
index 76a99994e04ee..f7869a4935a6f 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java
@@ -35,9 +35,7 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.transport.MockTransportClient;
 
 import java.util.Arrays;
 import java.util.HashSet;
@@ -158,48 +156,6 @@ public void testBulkProcessorConcurrentRequests() throws Exception {
         assertMultiGetResponse(multiGetRequestBuilder.get(), numDocs);
     }
 
-    //https://github.com/elastic/elasticsearch/issues/5038
-    public void testBulkProcessorConcurrentRequestsNoNodeAvailableException() throws Exception {
-        //we create a transport client with no nodes to make sure it throws NoNodeAvailableException
-        Settings settings = Settings.builder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                .build();
-        Client transportClient = new MockTransportClient(settings);
-
-        int bulkActions = randomIntBetween(10, 100);
-        int numDocs = randomIntBetween(bulkActions, bulkActions + 100);
-        int concurrentRequests = randomIntBetween(0, 10);
-
-        int expectedBulkActions = numDocs / bulkActions;
-
-        final CountDownLatch latch = new CountDownLatch(expectedBulkActions);
-        int totalExpectedBulkActions = numDocs % bulkActions == 0 ? expectedBulkActions : expectedBulkActions + 1;
-        final CountDownLatch closeLatch = new CountDownLatch(totalExpectedBulkActions);
-
-        BulkProcessorTestListener listener = new BulkProcessorTestListener(latch, closeLatch);
-
-        try (BulkProcessor processor = BulkProcessor.builder(transportClient, listener)
-                .setConcurrentRequests(concurrentRequests).setBulkActions(bulkActions)
-                //set interval and size to high values
-                .setFlushInterval(TimeValue.timeValueHours(24)).setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)).build()) {
-
-            indexDocs(transportClient, processor, numDocs);
-
-            latch.await();
-
-            assertThat(listener.beforeCounts.get(), equalTo(expectedBulkActions));
-            assertThat(listener.afterCounts.get(), equalTo(expectedBulkActions));
-            assertThat(listener.bulkFailures.size(), equalTo(expectedBulkActions));
-            assertThat(listener.bulkItems.size(), equalTo(0));
-        }
-
-        closeLatch.await();
-
-        assertThat(listener.bulkFailures.size(), equalTo(totalExpectedBulkActions));
-        assertThat(listener.bulkItems.size(), equalTo(0));
-        transportClient.close();
-    }
-
     public void testBulkProcessorWaitOnClose() throws Exception {
         BulkProcessorTestListener listener = new BulkProcessorTestListener();
 
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java
index e7285ff6f97ed..41e3e8d933fe2 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java
@@ -146,7 +146,6 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure)
 
         SearchResponse results = client()
                 .prepareSearch(INDEX_NAME)
-                .setTypes(TYPE_NAME)
                 .setQuery(QueryBuilders.matchAllQuery())
                 .setSize(0)
                 .get();
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkRejectionIT.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkRejectionIT.java
index 900f50a9be005..9104ae78810ae 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/BulkRejectionIT.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkRejectionIT.java
@@ -23,8 +23,13 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
 
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -41,6 +46,18 @@ protected Settings nodeSettings(int nodeOrdinal) {
             .build();
     }
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(InternalSettingsPlugin.class);
+    }
+
+    @Override
+    public Settings indexSettings() {
+        return Settings.builder().put(super.indexSettings())
+            // sync global checkpoint quickly so we can verify seq_no_stats aligned between all copies after tests.
+            .put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "1s").build();
+    }
+
     @Override
     protected int numberOfReplicas() {
         return 1;
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
index f213b523fbfaf..ee721377afe67 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java
@@ -110,7 +110,7 @@ private void indicesThatCannotBeCreatedTestCase(Set<String> expected,
         final ExecutorService direct = EsExecutors.newDirectExecutorService();
         when(threadPool.executor(anyString())).thenReturn(direct);
         TransportBulkAction action = new TransportBulkAction(threadPool, mock(TransportService.class), clusterService,
-                null, null, null, mock(ActionFilters.class), null, null) {
+                null, null, mock(ActionFilters.class), null, null) {
             @Override
             void executeBulk(Task task, BulkRequest bulkRequest, long startTimeNanos, ActionListener<BulkResponse> listener,
                     AtomicArray<BulkItemResponse> responses, Map<String, IndexNotFoundException> indicesThatCannotBeCreated) {
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
index b570ec8f781a6..95afbbf54f573 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
@@ -40,6 +40,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -132,7 +133,7 @@ class TestTransportBulkAction extends TransportBulkAction {
 
         TestTransportBulkAction() {
             super(threadPool, transportService, clusterService, ingestService,
-                null, null, new ActionFilters(Collections.emptySet()), null,
+                null, new ActionFilters(Collections.emptySet()), null,
                 new AutoCreateIndex(
                     SETTINGS, new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
                     new IndexNameExpressionResolver()
@@ -412,16 +413,29 @@ public void testUseDefaultPipelineWithAlias() throws Exception {
     }
 
     public void testUseDefaultPipelineWithBulkUpsert() throws Exception {
+        String indexRequestName = randomFrom(new String[]{null, WITH_DEFAULT_PIPELINE, WITH_DEFAULT_PIPELINE_ALIAS});
+        validatePipelineWithBulkUpsert(indexRequestName, WITH_DEFAULT_PIPELINE);
+    }
+
+    public void testUseDefaultPipelineWithBulkUpsertWithAlias() throws Exception {
+        String indexRequestName = randomFrom(new String[]{null, WITH_DEFAULT_PIPELINE, WITH_DEFAULT_PIPELINE_ALIAS});
+        validatePipelineWithBulkUpsert(indexRequestName, WITH_DEFAULT_PIPELINE_ALIAS);
+    }
+
+    private void validatePipelineWithBulkUpsert(@Nullable String indexRequestIndexName, String updateRequestIndexName) throws Exception {
         Exception exception = new Exception("fake exception");
         BulkRequest bulkRequest = new BulkRequest();
-        IndexRequest indexRequest1 = new IndexRequest(WITH_DEFAULT_PIPELINE, "type", "id1").source(Collections.emptyMap());
-        IndexRequest indexRequest2 = new IndexRequest(WITH_DEFAULT_PIPELINE, "type", "id2").source(Collections.emptyMap());
-        IndexRequest indexRequest3 = new IndexRequest(WITH_DEFAULT_PIPELINE, "type", "id3").source(Collections.emptyMap());
-        UpdateRequest upsertRequest = new UpdateRequest(WITH_DEFAULT_PIPELINE, "type", "id1").upsert(indexRequest1).script(mockScript("1"));
-        UpdateRequest docAsUpsertRequest = new UpdateRequest(WITH_DEFAULT_PIPELINE, "type", "id2").doc(indexRequest2).docAsUpsert(true);
+        IndexRequest indexRequest1 = new IndexRequest(indexRequestIndexName, "type", "id1").source(Collections.emptyMap());
+        IndexRequest indexRequest2 = new IndexRequest(indexRequestIndexName, "type", "id2").source(Collections.emptyMap());
+        IndexRequest indexRequest3 = new IndexRequest(indexRequestIndexName, "type", "id3").source(Collections.emptyMap());
+        UpdateRequest upsertRequest = new UpdateRequest(updateRequestIndexName, "type", "id1")
+            .upsert(indexRequest1).script(mockScript("1"));
+        UpdateRequest docAsUpsertRequest = new UpdateRequest(updateRequestIndexName, "type", "id2")
+            .doc(indexRequest2).docAsUpsert(true);
         // this test only covers the mechanics that scripted bulk upserts will execute a default pipeline. However, in practice scripted
         // bulk upserts with a default pipeline are a bit surprising since the script executes AFTER the pipeline.
-        UpdateRequest scriptedUpsert = new UpdateRequest(WITH_DEFAULT_PIPELINE, "type", "id2").upsert(indexRequest3).script(mockScript("1"))
+        UpdateRequest scriptedUpsert = new UpdateRequest(updateRequestIndexName, "type", "id2")
+            .upsert(indexRequest3).script(mockScript("1"))
             .scriptedUpsert(true);
         bulkRequest.add(upsertRequest).add(docAsUpsertRequest).add(scriptedUpsert);
 
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java
index 162ef56553df4..65e4b1ee195ec 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java
@@ -59,7 +59,7 @@ class TestTransportBulkAction extends TransportBulkAction {
         boolean indexCreated = false; // set when the "real" index is created
 
         TestTransportBulkAction() {
-            super(TransportBulkActionTests.this.threadPool, transportService, clusterService, null, null,
+            super(TransportBulkActionTests.this.threadPool, transportService, clusterService, null,
                     null, new ActionFilters(Collections.emptySet()), new Resolver(),
                     new AutoCreateIndex(Settings.EMPTY, clusterService.getClusterSettings(), new Resolver()));
         }
diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
index 89ccbefb00e59..d8fe82bf88e77 100644
--- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
+++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
@@ -21,7 +21,7 @@
 package org.elasticsearch.action.bulk;
 
 import org.apache.lucene.util.Constants;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -104,8 +104,8 @@ private TransportBulkAction createAction(boolean controlled, AtomicLong expected
         NodeClient client = new NodeClient(Settings.EMPTY, threadPool) {
             @Override
             public <Request extends ActionRequest, Response extends ActionResponse>
-            void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
-                listener.onResponse((Response)new CreateIndexResponse());
+            void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
+                listener.onResponse((Response)new CreateIndexResponse(false, false, null));
             }
         };
 
@@ -115,7 +115,6 @@ void doExecute(Action<Response> action, Request request, ActionListener<Response
                     threadPool,
                     transportService,
                     clusterService,
-                    null,
                     client,
                     actionFilters,
                     resolver,
@@ -139,7 +138,6 @@ void executeBulk(
                     threadPool,
                     transportService,
                     clusterService,
-                    null,
                     client,
                     actionFilters,
                     resolver,
@@ -218,7 +216,6 @@ static class TestTransportBulkAction extends TransportBulkAction {
                 ThreadPool threadPool,
                 TransportService transportService,
                 ClusterService clusterService,
-                TransportShardBulkAction shardBulkAction,
                 NodeClient client,
                 ActionFilters actionFilters,
                 IndexNameExpressionResolver indexNameExpressionResolver,
@@ -229,7 +226,6 @@ static class TestTransportBulkAction extends TransportBulkAction {
                     transportService,
                     clusterService,
                     null,
-                    shardBulkAction,
                     client,
                     actionFilters,
                     indexNameExpressionResolver,
diff --git a/server/src/test/java/org/elasticsearch/action/explain/ExplainRequestTests.java b/server/src/test/java/org/elasticsearch/action/explain/ExplainRequestTests.java
index 79c0765d7ac83..390d06a5058a1 100644
--- a/server/src/test/java/org/elasticsearch/action/explain/ExplainRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/explain/ExplainRequestTests.java
@@ -46,7 +46,7 @@ public class ExplainRequestTests extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
         entries.addAll(indicesModule.getNamedWriteables());
         entries.addAll(searchModule.getNamedWriteables());
@@ -64,8 +64,7 @@ public void testSerialize() throws IOException {
             request.routing("some_routing");
             request.writeTo(output);
             try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
-                ExplainRequest readRequest = new ExplainRequest();
-                readRequest.readFrom(in);
+                ExplainRequest readRequest = new ExplainRequest(in);
                 assertEquals(request.filteringAlias(), readRequest.filteringAlias());
                 assertArrayEquals(request.storedFields(), readRequest.storedFields());
                 assertEquals(request.preference(), readRequest.preference());
diff --git a/server/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java b/server/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java
index 782edb517cb13..4d43454e8dd25 100644
--- a/server/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java
@@ -71,9 +71,7 @@ public void testSerialization() throws IOException {
 
         StreamInput in = out.bytes().streamInput();
         in.setVersion(out.getVersion());
-        MultiGetShardRequest multiGetShardRequest2 = new MultiGetShardRequest();
-        multiGetShardRequest2.readFrom(in);
-
+        MultiGetShardRequest multiGetShardRequest2 = new MultiGetShardRequest(in);
         assertThat(multiGetShardRequest2.index(), equalTo(multiGetShardRequest.index()));
         assertThat(multiGetShardRequest2.preference(), equalTo(multiGetShardRequest.preference()));
         assertThat(multiGetShardRequest2.realtime(), equalTo(multiGetShardRequest.realtime()));
diff --git a/server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java b/server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java
index b028724a80edd..69a65a361f901 100644
--- a/server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/get/TransportMultiGetActionTests.java
@@ -170,7 +170,7 @@ public void testTransportMultiGetAction() {
         request.add(new MultiGetRequest.Item("index1", "_doc", "2"));
 
         final AtomicBoolean shardActionInvoked = new AtomicBoolean(false);
-        transportAction = new TransportMultiGetAction(transportService, clusterService, shardAction,
+        transportAction = new TransportMultiGetAction(transportService, clusterService, client,
             new ActionFilters(emptySet()), new Resolver()) {
             @Override
             protected void executeShardAction(final ActionListener<MultiGetResponse> listener,
@@ -195,7 +195,7 @@ public void testTransportMultiGetAction_withMissingRouting() {
         request.add(new MultiGetRequest.Item("index2", "_doc", "2"));
 
         final AtomicBoolean shardActionInvoked = new AtomicBoolean(false);
-        transportAction = new TransportMultiGetAction(transportService, clusterService, shardAction,
+        transportAction = new TransportMultiGetAction(transportService, clusterService, client,
             new ActionFilters(emptySet()), new Resolver()) {
             @Override
             protected void executeShardAction(final ActionListener<MultiGetResponse> listener,
diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
index d9de69a1c6c62..4701290fb65b8 100644
--- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java
@@ -40,7 +40,6 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
@@ -87,15 +86,14 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
                         assertThat(groupBuilder.must(), Matchers.contains(QueryBuilders.termQuery("foo", "bar")));
                     }
                     assertArrayEquals(mockSearchPhaseContext.getRequest().indices(), searchRequest.indices());
-                    assertArrayEquals(mockSearchPhaseContext.getRequest().types(), searchRequest.types());
 
 
                     List<MultiSearchResponse.Item> mSearchResponses = new ArrayList<>(numInnerHits);
                     for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) {
                         InternalSearchResponse internalSearchResponse = new InternalSearchResponse(collapsedHits.get(innerHitNum),
                             null, null, null, false, null, 1);
-                        SearchResponse response = mockSearchPhaseContext.buildSearchResponse(internalSearchResponse, null);
-                        mSearchResponses.add(new MultiSearchResponse.Item(response, null));
+                        mockSearchPhaseContext.sendSearchResponse(internalSearchResponse, null);
+                        mSearchResponses.add(new MultiSearchResponse.Item(mockSearchPhaseContext.searchResponse.get(), null));
                     }
 
                     listener.onResponse(
@@ -107,20 +105,19 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
                 Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(collapseValue))))},
                 new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F);
             InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1);
-            AtomicReference<SearchResponse> reference = new AtomicReference<>();
             ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, (r) ->
                 new SearchPhase("test") {
                     @Override
-                    public void run() throws IOException {
-                        reference.set(mockSearchPhaseContext.buildSearchResponse(r, null));
+                    public void run() {
+                        mockSearchPhaseContext.sendSearchResponse(r, null);
                     }
                 }
             );
 
             phase.run();
             mockSearchPhaseContext.assertNoFailure();
-            assertNotNull(reference.get());
-            SearchResponse theResponse = reference.get();
+            SearchResponse theResponse = mockSearchPhaseContext.searchResponse.get();
+            assertNotNull(theResponse);
             assertEquals(numInnerHits, theResponse.getHits().getHits()[0].getInnerHits().size());
 
             for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) {
@@ -148,11 +145,12 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
                 assertTrue(executedMultiSearch.compareAndSet(false, true));
                 InternalSearchResponse internalSearchResponse = new InternalSearchResponse(collapsedHits,
                     null, null, null, false, null, 1);
-                SearchResponse response = mockSearchPhaseContext.buildSearchResponse(internalSearchResponse, null);
+                SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, 1, 1, 0, 0,
+                    ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY);
                 listener.onResponse(new MultiSearchResponse(
                     new MultiSearchResponse.Item[]{
                             new MultiSearchResponse.Item(null, new RuntimeException("boom")),
-                            new MultiSearchResponse.Item(response, null)
+                            new MultiSearchResponse.Item(searchResponse, null)
                     }, randomIntBetween(1, 10000)));
             }
         };
@@ -163,12 +161,11 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
                 Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(collapseValue))))},
             new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F);
         InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1);
-        AtomicReference<SearchResponse> reference = new AtomicReference<>();
         ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, r ->
             new SearchPhase("test") {
                 @Override
-                public void run() throws IOException {
-                    reference.set(mockSearchPhaseContext.buildSearchResponse(r, null));
+                public void run() {
+                    mockSearchPhaseContext.sendSearchResponse(r, null);
                 }
             }
         );
@@ -176,7 +173,7 @@ public void run() throws IOException {
         assertThat(mockSearchPhaseContext.phaseFailure.get(), Matchers.instanceOf(RuntimeException.class));
         assertEquals("boom", mockSearchPhaseContext.phaseFailure.get().getMessage());
         assertNotNull(mockSearchPhaseContext.phaseFailure.get());
-        assertNull(reference.get());
+        assertNull(mockSearchPhaseContext.searchResponse.get());
         assertEquals(0, mockSearchPhaseContext.phasesExecuted.get());
     }
 
@@ -195,18 +192,17 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
                 Collections.singletonMap("someField", new DocumentField("someField", Collections.singletonList(null))))},
             new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F);
         InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1);
-        AtomicReference<SearchResponse> reference = new AtomicReference<>();
         ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, r ->
             new SearchPhase("test") {
                 @Override
-                public void run() throws IOException {
-                    reference.set(mockSearchPhaseContext.buildSearchResponse(r, null));
+                public void run() {
+                    mockSearchPhaseContext.sendSearchResponse(r, null);
                 }
             }
         );
         phase.run();
         mockSearchPhaseContext.assertNoFailure();
-        assertNotNull(reference.get());
+        assertNotNull(mockSearchPhaseContext.searchResponse.get());
         assertEquals(1, mockSearchPhaseContext.phasesExecuted.get());
     }
 
@@ -223,18 +219,17 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
 
         SearchHits hits = new SearchHits(new SearchHit[0], new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f);
         InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1);
-        AtomicReference<SearchResponse> reference = new AtomicReference<>();
         ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, r ->
             new SearchPhase("test") {
                 @Override
-                public void run() throws IOException {
-                    reference.set(mockSearchPhaseContext.buildSearchResponse(r, null));
+                public void run() {
+                    mockSearchPhaseContext.sendSearchResponse(r, null);
                 }
             }
         );
         phase.run();
         mockSearchPhaseContext.assertNoFailure();
-        assertNotNull(reference.get());
+        assertNotNull(mockSearchPhaseContext.searchResponse.get());
         assertEquals(1, mockSearchPhaseContext.phasesExecuted.get());
     }
 
@@ -269,18 +264,17 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL
 
         SearchHits hits = new SearchHits(new SearchHit[0], new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f);
         InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, null, null, null, false, null, 1);
-        AtomicReference<SearchResponse> reference = new AtomicReference<>();
         ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, internalSearchResponse, r ->
             new SearchPhase("test") {
                 @Override
-                public void run() throws IOException {
-                    reference.set(mockSearchPhaseContext.buildSearchResponse(r, null));
+                public void run() {
+                    mockSearchPhaseContext.sendSearchResponse(r, null);
                 }
             }
         );
         phase.run();
         mockSearchPhaseContext.assertNoFailure();
-        assertNotNull(reference.get());
+        assertNotNull(mockSearchPhaseContext.searchResponse.get());
         assertEquals(1, mockSearchPhaseContext.phasesExecuted.get());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
index 5614476c851b8..886e6dc1fe55e 100644
--- a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java
@@ -42,7 +42,6 @@
 import java.io.IOException;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
 
 public class FetchSearchPhaseTests extends ESTestCase {
 
@@ -52,7 +51,6 @@ public void testShortcutQueryAndFetchOptimization() throws IOException {
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1);
         InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 1);
-        AtomicReference<SearchResponse> responseRef = new AtomicReference<>();
         boolean hasHits = randomBoolean();
         final int numHits;
         if (hasHits) {
@@ -73,17 +71,18 @@ public void testShortcutQueryAndFetchOptimization() throws IOException {
         FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
             @Override
-            public void run() throws IOException {
-                responseRef.set(mockSearchPhaseContext.buildSearchResponse(searchResponse, null));
+            public void run() {
+                mockSearchPhaseContext.sendSearchResponse(searchResponse, null);
             }
         });
         assertEquals("fetch", phase.getName());
         phase.run();
         mockSearchPhaseContext.assertNoFailure();
-        assertNotNull(responseRef.get());
-        assertEquals(numHits, responseRef.get().getHits().getTotalHits().value);
+        SearchResponse searchResponse = mockSearchPhaseContext.searchResponse.get();
+        assertNotNull(searchResponse);
+        assertEquals(numHits, searchResponse.getHits().getTotalHits().value);
         if (numHits != 0) {
-            assertEquals(42, responseRef.get().getHits().getAt(0).docId());
+            assertEquals(42, searchResponse.getHits().getAt(0).docId());
         }
         assertTrue(mockSearchPhaseContext.releasedSearchContexts.isEmpty());
     }
@@ -94,7 +93,6 @@ public void testFetchTwoDocument() throws IOException {
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2);
-        AtomicReference<SearchResponse> responseRef = new AtomicReference<>();
         int resultSetSize = randomIntBetween(2, 10);
         QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0), 
             null, OriginalIndices.NONE));
@@ -111,7 +109,7 @@ public void testFetchTwoDocument() throws IOException {
         queryResult.setShardIndex(1);
         results.consumeResult(queryResult);
 
-        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
+        mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null) {
             @Override
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
@@ -127,23 +125,23 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                 listener.onResponse(fetchResult);
             }
         };
-        mockSearchPhaseContext.searchTransport = searchTransportService;
         FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
-                public void run() throws IOException {
-                    responseRef.set(mockSearchPhaseContext.buildSearchResponse(searchResponse, null));
+                public void run() {
+                    mockSearchPhaseContext.sendSearchResponse(searchResponse, null);
                 }
             });
         assertEquals("fetch", phase.getName());
         phase.run();
         mockSearchPhaseContext.assertNoFailure();
-        assertNotNull(responseRef.get());
-        assertEquals(2, responseRef.get().getHits().getTotalHits().value);
-        assertEquals(84, responseRef.get().getHits().getAt(0).docId());
-        assertEquals(42, responseRef.get().getHits().getAt(1).docId());
-        assertEquals(0, responseRef.get().getFailedShards());
-        assertEquals(2, responseRef.get().getSuccessfulShards());
+        SearchResponse searchResponse = mockSearchPhaseContext.searchResponse.get();
+        assertNotNull(searchResponse);
+        assertEquals(2, searchResponse.getHits().getTotalHits().value);
+        assertEquals(84, searchResponse.getHits().getAt(0).docId());
+        assertEquals(42, searchResponse.getHits().getAt(1).docId());
+        assertEquals(0, searchResponse.getFailedShards());
+        assertEquals(2, searchResponse.getSuccessfulShards());
         assertTrue(mockSearchPhaseContext.releasedSearchContexts.isEmpty());
     }
 
@@ -153,7 +151,6 @@ public void testFailFetchOneDoc() throws IOException {
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2);
-        AtomicReference<SearchResponse> responseRef = new AtomicReference<>();
         int resultSetSize = randomIntBetween(2, 10);
         QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0), 
             null, OriginalIndices.NONE));
@@ -170,7 +167,7 @@ public void testFailFetchOneDoc() throws IOException {
         queryResult.setShardIndex(1);
         results.consumeResult(queryResult);
 
-        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
+        mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null) {
             @Override
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
@@ -185,24 +182,24 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
 
             }
         };
-        mockSearchPhaseContext.searchTransport = searchTransportService;
         FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
-                public void run() throws IOException {
-                    responseRef.set(mockSearchPhaseContext.buildSearchResponse(searchResponse, null));
+                public void run() {
+                    mockSearchPhaseContext.sendSearchResponse(searchResponse, null);
                 }
             });
         assertEquals("fetch", phase.getName());
         phase.run();
         mockSearchPhaseContext.assertNoFailure();
-        assertNotNull(responseRef.get());
-        assertEquals(2, responseRef.get().getHits().getTotalHits().value);
-        assertEquals(84, responseRef.get().getHits().getAt(0).docId());
-        assertEquals(1, responseRef.get().getFailedShards());
-        assertEquals(1, responseRef.get().getSuccessfulShards());
-        assertEquals(1, responseRef.get().getShardFailures().length);
-        assertTrue(responseRef.get().getShardFailures()[0].getCause() instanceof MockDirectoryWrapper.FakeIOException);
+        SearchResponse searchResponse = mockSearchPhaseContext.searchResponse.get();
+        assertNotNull(searchResponse);
+        assertEquals(2, searchResponse.getHits().getTotalHits().value);
+        assertEquals(84, searchResponse.getHits().getAt(0).docId());
+        assertEquals(1, searchResponse.getFailedShards());
+        assertEquals(1, searchResponse.getSuccessfulShards());
+        assertEquals(1, searchResponse.getShardFailures().length);
+        assertTrue(searchResponse.getShardFailures()[0].getCause() instanceof MockDirectoryWrapper.FakeIOException);
         assertEquals(1, mockSearchPhaseContext.releasedSearchContexts.size());
         assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(123L));
     }
@@ -216,7 +213,6 @@ public void testFetchDocsConcurrently() throws IOException, InterruptedException
         MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(numHits);
         InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), numHits);
-        AtomicReference<SearchResponse> responseRef = new AtomicReference<>();
         for (int i = 0; i < numHits; i++) {
             QuerySearchResult queryResult = new QuerySearchResult(i, new SearchShardTarget("node1", new ShardId("test", "na", 0), 
                 null, OriginalIndices.NONE));
@@ -226,7 +222,7 @@ public void testFetchDocsConcurrently() throws IOException, InterruptedException
             queryResult.setShardIndex(i);
             results.consumeResult(queryResult);
         }
-        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
+        mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null) {
             @Override
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
@@ -238,13 +234,12 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                 }).start();
             }
         };
-        mockSearchPhaseContext.searchTransport = searchTransportService;
         CountDownLatch latch = new CountDownLatch(1);
         FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
-                public void run() throws IOException {
-                    responseRef.set(mockSearchPhaseContext.buildSearchResponse(searchResponse, null));
+                public void run() {
+                    mockSearchPhaseContext.sendSearchResponse(searchResponse, null);
                     latch.countDown();
                 }
             });
@@ -252,17 +247,18 @@ public void run() throws IOException {
         phase.run();
         latch.await();
         mockSearchPhaseContext.assertNoFailure();
-        assertNotNull(responseRef.get());
-        assertEquals(numHits, responseRef.get().getHits().getTotalHits().value);
-        assertEquals(Math.min(numHits, resultSetSize), responseRef.get().getHits().getHits().length);
-        SearchHit[] hits = responseRef.get().getHits().getHits();
+        SearchResponse searchResponse = mockSearchPhaseContext.searchResponse.get();
+        assertNotNull(searchResponse);
+        assertEquals(numHits, searchResponse.getHits().getTotalHits().value);
+        assertEquals(Math.min(numHits, resultSetSize), searchResponse.getHits().getHits().length);
+        SearchHit[] hits = searchResponse.getHits().getHits();
         for (int i = 0; i < hits.length; i++) {
             assertNotNull(hits[i]);
             assertEquals("index: " + i, numHits-i, hits[i].docId());
             assertEquals("index: " + i, numHits-1-i, (int)hits[i].getScore());
         }
-        assertEquals(0, responseRef.get().getFailedShards());
-        assertEquals(numHits, responseRef.get().getSuccessfulShards());
+        assertEquals(0, searchResponse.getFailedShards());
+        assertEquals(numHits, searchResponse.getSuccessfulShards());
         int sizeReleasedContexts = Math.max(0, numHits - resultSetSize); // all non fetched results will be freed
         assertEquals(mockSearchPhaseContext.releasedSearchContexts.toString(),
             sizeReleasedContexts, mockSearchPhaseContext.releasedSearchContexts.size());
@@ -274,7 +270,6 @@ public void testExceptionFailsPhase() throws IOException {
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2);
-        AtomicReference<SearchResponse> responseRef = new AtomicReference<>();
         int resultSetSize = randomIntBetween(2, 10);
         QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0), 
             null, OriginalIndices.NONE));
@@ -291,7 +286,7 @@ public void testExceptionFailsPhase() throws IOException {
         queryResult.setShardIndex(1);
         results.consumeResult(queryResult);
         AtomicInteger numFetches = new AtomicInteger(0);
-        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
+        mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null) {
             @Override
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
@@ -310,19 +305,18 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                 listener.onResponse(fetchResult);
             }
         };
-        mockSearchPhaseContext.searchTransport = searchTransportService;
         FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
-                public void run() throws IOException {
-                    responseRef.set(mockSearchPhaseContext.buildSearchResponse(searchResponse, null));
+                public void run() {
+                    mockSearchPhaseContext.sendSearchResponse(searchResponse, null);
                 }
             });
         assertEquals("fetch", phase.getName());
         phase.run();
         assertNotNull(mockSearchPhaseContext.phaseFailure.get());
         assertEquals(mockSearchPhaseContext.phaseFailure.get().getMessage(), "BOOM");
-        assertNull(responseRef.get());
+        assertNull(mockSearchPhaseContext.searchResponse.get());
         assertTrue(mockSearchPhaseContext.releasedSearchContexts.isEmpty());
     }
 
@@ -332,7 +326,6 @@ public void testCleanupIrrelevantContexts() throws IOException { // contexts tha
             (b) -> new InternalAggregation.ReduceContext(BigArrays.NON_RECYCLING_INSTANCE, null, b));
         InitialSearchPhase.ArraySearchPhaseResults<SearchPhaseResult> results =
             controller.newSearchPhaseResults(mockSearchPhaseContext.getRequest(), 2);
-        AtomicReference<SearchResponse> responseRef = new AtomicReference<>();
         int resultSetSize = 1;
         QuerySearchResult queryResult = new QuerySearchResult(123, new SearchShardTarget("node1", new ShardId("test", "na", 0), 
             null, OriginalIndices.NONE));
@@ -349,7 +342,7 @@ public void testCleanupIrrelevantContexts() throws IOException { // contexts tha
         queryResult.setShardIndex(1);
         results.consumeResult(queryResult);
 
-        SearchTransportService searchTransportService = new SearchTransportService(null, null) {
+        mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null) {
             @Override
             public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRequest request, SearchTask task,
                                          SearchActionListener<FetchSearchResult> listener) {
@@ -363,23 +356,23 @@ public void sendExecuteFetch(Transport.Connection connection, ShardFetchSearchRe
                 listener.onResponse(fetchResult);
             }
         };
-        mockSearchPhaseContext.searchTransport = searchTransportService;
         FetchSearchPhase phase = new FetchSearchPhase(results, controller, mockSearchPhaseContext,
             (searchResponse, scrollId) -> new SearchPhase("test") {
                 @Override
-                public void run() throws IOException {
-                    responseRef.set(mockSearchPhaseContext.buildSearchResponse(searchResponse, null));
+                public void run() {
+                    mockSearchPhaseContext.sendSearchResponse(searchResponse, null);
                 }
             });
         assertEquals("fetch", phase.getName());
         phase.run();
         mockSearchPhaseContext.assertNoFailure();
-        assertNotNull(responseRef.get());
-        assertEquals(2, responseRef.get().getHits().getTotalHits().value);
-        assertEquals(1, responseRef.get().getHits().getHits().length);
-        assertEquals(84, responseRef.get().getHits().getAt(0).docId());
-        assertEquals(0, responseRef.get().getFailedShards());
-        assertEquals(2, responseRef.get().getSuccessfulShards());
+        SearchResponse searchResponse = mockSearchPhaseContext.searchResponse.get();
+        assertNotNull(searchResponse);
+        assertEquals(2, searchResponse.getHits().getTotalHits().value);
+        assertEquals(1, searchResponse.getHits().getHits().length);
+        assertEquals(84, searchResponse.getHits().getAt(0).docId());
+        assertEquals(0, searchResponse.getFailedShards());
+        assertEquals(2, searchResponse.getSuccessfulShards());
         assertEquals(1, mockSearchPhaseContext.releasedSearchContexts.size());
         assertTrue(mockSearchPhaseContext.releasedSearchContexts.contains(123L));
     }
diff --git a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
index 2a155d2e3adbb..40c3ad0afc0d4 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java
@@ -49,6 +49,7 @@ public final class MockSearchPhaseContext implements SearchPhaseContext {
     Set<Long> releasedSearchContexts = new HashSet<>();
     SearchRequest searchRequest = new SearchRequest();
     AtomicInteger phasesExecuted = new AtomicInteger();
+    AtomicReference<SearchResponse> searchResponse = new AtomicReference<>();
 
     public MockSearchPhaseContext(int numShards) {
         this.numShards = numShards;
@@ -82,9 +83,9 @@ public SearchRequest getRequest() {
     }
 
     @Override
-    public SearchResponse buildSearchResponse(InternalSearchResponse internalSearchResponse, String scrollId) {
-        return new SearchResponse(internalSearchResponse, scrollId, numShards, numSuccess.get(), 0, 0,
-            failures.toArray(new ShardSearchFailure[failures.size()]), SearchResponse.Clusters.EMPTY);
+    public void sendSearchResponse(InternalSearchResponse internalSearchResponse, String scrollId) {
+        searchResponse.set(new SearchResponse(internalSearchResponse, scrollId, numShards, numSuccess.get(), 0, 0,
+            failures.toArray(ShardSearchFailure.EMPTY_ARRAY), SearchResponse.Clusters.EMPTY));
     }
 
     @Override
@@ -130,11 +131,6 @@ public void execute(Runnable command) {
         command.run();
     }
 
-    @Override
-    public void onResponse(SearchResponse response) {
-        Assert.fail("should not be called");
-    }
-
     @Override
     public void onFailure(Exception e) {
         Assert.fail("should not be called");
diff --git a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
index afe957e2bf3ee..35f60546bb023 100644
--- a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
@@ -61,14 +61,10 @@ public void testSimpleAdd() throws Exception {
                 equalTo("test"));
         assertThat(request.requests().get(0).indicesOptions(),
                 equalTo(IndicesOptions.fromOptions(true, true, true, true, SearchRequest.DEFAULT_INDICES_OPTIONS)));
-        assertThat(request.requests().get(0).types().length,
-                equalTo(0));
         assertThat(request.requests().get(1).indices()[0],
                 equalTo("test"));
         assertThat(request.requests().get(1).indicesOptions(),
                 equalTo(IndicesOptions.fromOptions(false, true, true, true, SearchRequest.DEFAULT_INDICES_OPTIONS)));
-        assertThat(request.requests().get(1).types()[0],
-                equalTo("type1"));
         assertThat(request.requests().get(2).indices()[0],
                 equalTo("test"));
         assertThat(request.requests().get(2).indicesOptions(),
@@ -83,12 +79,9 @@ public void testSimpleAdd() throws Exception {
                 equalTo(IndicesOptions.fromOptions(true, false, false, true, SearchRequest.DEFAULT_INDICES_OPTIONS)));
 
         assertThat(request.requests().get(5).indices(), is(Strings.EMPTY_ARRAY));
-        assertThat(request.requests().get(5).types().length, equalTo(0));
         assertThat(request.requests().get(6).indices(), is(Strings.EMPTY_ARRAY));
-        assertThat(request.requests().get(6).types().length, equalTo(0));
         assertThat(request.requests().get(6).searchType(), equalTo(SearchType.DFS_QUERY_THEN_FETCH));
         assertThat(request.requests().get(7).indices(), is(Strings.EMPTY_ARRAY));
-        assertThat(request.requests().get(7).types().length, equalTo(0));
     }
 
     public void testFailWithUnknownKey() {
@@ -111,7 +104,6 @@ public void testSimpleAddWithCarriageReturn() throws Exception {
         assertThat(request.requests().get(0).indices()[0], equalTo("test"));
         assertThat(request.requests().get(0).indicesOptions(),
             equalTo(IndicesOptions.fromOptions(true, true, true, true, SearchRequest.DEFAULT_INDICES_OPTIONS)));
-        assertThat(request.requests().get(0).types().length, equalTo(0));
     }
 
     public void testDefaultIndicesOptions() throws IOException {
@@ -126,23 +118,17 @@ public void testDefaultIndicesOptions() throws IOException {
         assertThat(request.requests().get(0).indices()[0], equalTo("test"));
         assertThat(request.requests().get(0).indicesOptions(),
             equalTo(IndicesOptions.fromOptions(true, true, true, true, SearchRequest.DEFAULT_INDICES_OPTIONS)));
-        assertThat(request.requests().get(0).types().length, equalTo(0));
     }
 
     public void testSimpleAdd2() throws Exception {
         MultiSearchRequest request = parseMultiSearchRequest("/org/elasticsearch/action/search/simple-msearch2.json");
         assertThat(request.requests().size(), equalTo(5));
         assertThat(request.requests().get(0).indices()[0], equalTo("test"));
-        assertThat(request.requests().get(0).types().length, equalTo(0));
         assertThat(request.requests().get(1).indices()[0], equalTo("test"));
-        assertThat(request.requests().get(1).types()[0], equalTo("type1"));
         assertThat(request.requests().get(2).indices(), is(Strings.EMPTY_ARRAY));
-        assertThat(request.requests().get(2).types().length, equalTo(0));
         assertThat(request.requests().get(3).indices(), is(Strings.EMPTY_ARRAY));
-        assertThat(request.requests().get(3).types().length, equalTo(0));
         assertThat(request.requests().get(3).searchType(), equalTo(SearchType.DFS_QUERY_THEN_FETCH));
         assertThat(request.requests().get(4).indices(), is(Strings.EMPTY_ARRAY));
-        assertThat(request.requests().get(4).types().length, equalTo(0));
     }
 
     public void testSimpleAdd3() throws Exception {
@@ -152,13 +138,9 @@ public void testSimpleAdd3() throws Exception {
         assertThat(request.requests().get(0).indices()[1], equalTo("test1"));
         assertThat(request.requests().get(1).indices()[0], equalTo("test2"));
         assertThat(request.requests().get(1).indices()[1], equalTo("test3"));
-        assertThat(request.requests().get(1).types()[0], equalTo("type1"));
         assertThat(request.requests().get(2).indices()[0], equalTo("test4"));
         assertThat(request.requests().get(2).indices()[1], equalTo("test1"));
-        assertThat(request.requests().get(2).types()[0], equalTo("type2"));
-        assertThat(request.requests().get(2).types()[1], equalTo("type1"));
         assertThat(request.requests().get(3).indices(), is(Strings.EMPTY_ARRAY));
-        assertThat(request.requests().get(3).types().length, equalTo(0));
         assertThat(request.requests().get(3).searchType(), equalTo(SearchType.DFS_QUERY_THEN_FETCH));
     }
 
@@ -171,13 +153,10 @@ public void testSimpleAdd4() throws Exception {
         assertThat(request.requests().get(0).preference(), nullValue());
         assertThat(request.requests().get(1).indices()[0], equalTo("test2"));
         assertThat(request.requests().get(1).indices()[1], equalTo("test3"));
-        assertThat(request.requests().get(1).types()[0], equalTo("type1"));
         assertThat(request.requests().get(1).requestCache(), nullValue());
         assertThat(request.requests().get(1).preference(), equalTo("_local"));
         assertThat(request.requests().get(2).indices()[0], equalTo("test4"));
         assertThat(request.requests().get(2).indices()[1], equalTo("test1"));
-        assertThat(request.requests().get(2).types()[0], equalTo("type2"));
-        assertThat(request.requests().get(2).types()[1], equalTo("type1"));
         assertThat(request.requests().get(2).routing(), equalTo("123"));
     }
 
@@ -272,7 +251,7 @@ public void testMultiLineSerialization() throws IOException {
                 parsedRequest.add(r);
             };
             MultiSearchRequest.readMultiLineFormat(new BytesArray(originalBytes), xContentType.xContent(),
-                    consumer, null, null, null, null, null, null, xContentRegistry(), true);
+                    consumer, null, null, null, null, null, xContentRegistry(), true);
             assertEquals(originalRequest, parsedRequest);
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
index 6a6b1c54db225..8cd74b80c32f9 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
@@ -56,6 +56,7 @@
 
 import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap;
 import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentSet;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 public class SearchAsyncActionTests extends ESTestCase {
 
@@ -371,6 +372,102 @@ protected void executeNext(Runnable runnable, Thread originalThread) {
         executor.shutdown();
     }
 
+    public void testAllowPartialResults() throws InterruptedException {
+        SearchRequest request = new SearchRequest();
+        request.allowPartialSearchResults(false);
+        int numConcurrent = randomIntBetween(1, 5);
+        request.setMaxConcurrentShardRequests(numConcurrent);
+        int numShards = randomIntBetween(5, 10);
+        AtomicBoolean searchPhaseDidRun = new AtomicBoolean(false);
+        ActionListener<SearchResponse> responseListener = ActionListener.wrap(response -> {},
+            (e) -> { throw new AssertionError("unexpected", e);} );
+        DiscoveryNode primaryNode = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
+        // for the sake of this test we place the replica on the same node. ie. this is not a mistake since we limit per node now
+        DiscoveryNode replicaNode = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
+
+        AtomicInteger contextIdGenerator = new AtomicInteger(0);
+        GroupShardsIterator<SearchShardIterator> shardsIter = getShardsIter("idx",
+            new OriginalIndices(new String[]{"idx"}, SearchRequest.DEFAULT_INDICES_OPTIONS),
+            numShards, true, primaryNode, replicaNode);
+        int numShardAttempts = 0;
+        for (SearchShardIterator it : shardsIter) {
+            numShardAttempts += it.remaining();
+        }
+        CountDownLatch latch = new CountDownLatch(numShardAttempts);
+
+        SearchTransportService transportService = new SearchTransportService(null, null);
+        Map<String, Transport.Connection> lookup = new HashMap<>();
+        Map<ShardId, Boolean> seenShard = new ConcurrentHashMap<>();
+        lookup.put(primaryNode.getId(), new MockConnection(primaryNode));
+        lookup.put(replicaNode.getId(), new MockConnection(replicaNode));
+        Map<String, AliasFilter> aliasFilters = Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY));
+        AtomicInteger numRequests = new AtomicInteger(0);
+        AtomicInteger numFailReplicas = new AtomicInteger(0);
+        AbstractSearchAsyncAction<TestSearchPhaseResult> asyncAction =
+            new AbstractSearchAsyncAction<>(
+                "test",
+                logger,
+                transportService,
+                (cluster, node) -> {
+                    assert cluster == null : "cluster was not null: " + cluster;
+                    return lookup.get(node); },
+                aliasFilters,
+                Collections.emptyMap(),
+                Collections.emptyMap(),
+                null,
+                request,
+                responseListener,
+                shardsIter,
+                new TransportSearchAction.SearchTimeProvider(0, 0, () -> 0),
+                0,
+                null,
+                new InitialSearchPhase.ArraySearchPhaseResults<>(shardsIter.size()),
+                request.getMaxConcurrentShardRequests(),
+                SearchResponse.Clusters.EMPTY) {
+
+                @Override
+                protected void executePhaseOnShard(SearchShardIterator shardIt, ShardRouting shard,
+                                                   SearchActionListener<TestSearchPhaseResult> listener) {
+                    seenShard.computeIfAbsent(shard.shardId(), (i) -> {
+                        numRequests.incrementAndGet(); // only count this once per shard copy
+                        return Boolean.TRUE;
+                    });
+                    new Thread(() -> {
+                        Transport.Connection connection = getConnection(null, shard.currentNodeId());
+                        TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(contextIdGenerator.incrementAndGet(),
+                            connection.getNode());
+                        if (shardIt.remaining() > 0) {
+                            numFailReplicas.incrementAndGet();
+                            listener.onFailure(new RuntimeException());
+                        } else {
+                            listener.onResponse(testSearchPhaseResult);
+                        }
+                    }).start();
+                }
+
+                @Override
+                protected SearchPhase getNextPhase(SearchPhaseResults<TestSearchPhaseResult> results, SearchPhaseContext context) {
+                    return new SearchPhase("test") {
+                        @Override
+                        public void run() {
+                            assertTrue(searchPhaseDidRun.compareAndSet(false, true));
+                        }
+                    };
+                }
+
+                @Override
+                protected void executeNext(Runnable runnable, Thread originalThread) {
+                    super.executeNext(runnable, originalThread);
+                    latch.countDown();
+                }
+            };
+        asyncAction.start();
+        latch.await();
+        assertTrue(searchPhaseDidRun.get());
+        assertEquals(numShards, numRequests.get());
+        assertThat(numFailReplicas.get(), greaterThanOrEqualTo(1));
+    }
+
     static GroupShardsIterator<SearchShardIterator> getShardsIter(String index, OriginalIndices originalIndices, int numShards,
                                                      boolean doReplicas, DiscoveryNode primaryNode, DiscoveryNode replicaNode) {
         ArrayList<SearchShardIterator> list = new ArrayList<>();
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestBuilderTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestBuilderTests.java
index 38f872dea2f4c..d9064185955e4 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestBuilderTests.java
@@ -19,57 +19,41 @@
 
 package org.elasticsearch.action.search;
 
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
+import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.transport.MockTransportClient;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.mockito.Mockito;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 
 public class SearchRequestBuilderTests extends ESTestCase {
-    private static Client client;
-
-    @BeforeClass
-    public static void initClient() {
-        //this client will not be hit by any request, but it needs to be a non null proper client
-        //that is why we create it but we don't add any transport address to it
-        Settings settings = Settings.builder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                .build();
-        client = new MockTransportClient(settings);
-    }
-
-    @AfterClass
-    public static void closeClient() {
-        client.close();
-        client = null;
+    
+    private SearchRequestBuilder makeBuilder() {
+        ElasticsearchClient client = Mockito.mock(ElasticsearchClient.class);
+        return new SearchRequestBuilder(client, SearchAction.INSTANCE);
     }
 
     public void testEmptySourceToString() {
-        SearchRequestBuilder searchRequestBuilder = client.prepareSearch();
+        SearchRequestBuilder searchRequestBuilder = makeBuilder();
         assertThat(searchRequestBuilder.toString(), equalTo(new SearchSourceBuilder().toString()));
     }
 
     public void testQueryBuilderQueryToString() {
-        SearchRequestBuilder searchRequestBuilder = client.prepareSearch();
+        SearchRequestBuilder searchRequestBuilder = makeBuilder();
         searchRequestBuilder.setQuery(QueryBuilders.matchAllQuery());
         assertThat(searchRequestBuilder.toString(), equalTo(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).toString()));
     }
 
     public void testSearchSourceBuilderToString() {
-        SearchRequestBuilder searchRequestBuilder = client.prepareSearch();
+        SearchRequestBuilder searchRequestBuilder = makeBuilder();
         searchRequestBuilder.setSource(new SearchSourceBuilder().query(QueryBuilders.termQuery("field", "value")));
         assertThat(searchRequestBuilder.toString(), equalTo(new SearchSourceBuilder()
             .query(QueryBuilders.termQuery("field", "value")).toString()));
     }
 
     public void testThatToStringDoesntWipeRequestSource() {
-        SearchRequestBuilder searchRequestBuilder = client.prepareSearch()
+        SearchRequestBuilder searchRequestBuilder = makeBuilder()
             .setSource(new SearchSourceBuilder().query(QueryBuilders.termQuery("field", "value")));
         String preToString = searchRequestBuilder.request().toString();
         assertThat(searchRequestBuilder.toString(), equalTo(new SearchSourceBuilder()
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
index 06231db26d67e..7ba20d24efd8f 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java
@@ -88,7 +88,6 @@ public void testIllegalArguments() {
         SearchRequest searchRequest = new SearchRequest();
         assertNotNull(searchRequest.indices());
         assertNotNull(searchRequest.indicesOptions());
-        assertNotNull(searchRequest.types());
         assertNotNull(searchRequest.searchType());
 
         NullPointerException e = expectThrows(NullPointerException.class, () -> searchRequest.indices((String[]) null));
@@ -99,11 +98,6 @@ public void testIllegalArguments() {
         e = expectThrows(NullPointerException.class, () -> searchRequest.indicesOptions(null));
         assertEquals("indicesOptions must not be null", e.getMessage());
 
-        e = expectThrows(NullPointerException.class, () -> searchRequest.types((String[]) null));
-        assertEquals("types must not be null", e.getMessage());
-        e = expectThrows(NullPointerException.class, () -> searchRequest.types((String) null));
-        assertEquals("type must not be null", e.getMessage());
-
         e = expectThrows(NullPointerException.class, () -> searchRequest.searchType((SearchType)null));
         assertEquals("searchType must not be null", e.getMessage());
 
@@ -187,7 +181,6 @@ private SearchRequest mutate(SearchRequest searchRequest) {
         mutators.add(() -> mutation.indices(ArrayUtils.concat(searchRequest.indices(), new String[] { randomAlphaOfLength(10) })));
         mutators.add(() -> mutation.indicesOptions(randomValueOtherThan(searchRequest.indicesOptions(),
                 () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()))));
-        mutators.add(() -> mutation.types(ArrayUtils.concat(searchRequest.types(), new String[] { randomAlphaOfLength(10) })));
         mutators.add(() -> mutation.preference(randomValueOtherThan(searchRequest.preference(), () -> randomAlphaOfLengthBetween(3, 10))));
         mutators.add(() -> mutation.routing(randomValueOtherThan(searchRequest.routing(), () -> randomAlphaOfLengthBetween(3, 10))));
         mutators.add(() -> mutation.requestCache((randomValueOtherThan(searchRequest.requestCache(), ESTestCase::randomBoolean))));
diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java
index ed5104cabe5d2..16e34ffec3fd6 100644
--- a/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java
@@ -69,7 +69,7 @@ public class SearchResponseTests extends ESTestCase {
     }
 
     private final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(
-            new SearchModule(Settings.EMPTY, false, emptyList()).getNamedWriteables());
+            new SearchModule(Settings.EMPTY, emptyList()).getNamedWriteables());
     private AggregationsTests aggregationsTests = new AggregationsTests();
 
     @Before
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
index 7ecc172924b1c..b405b0790debc 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Randomness;
@@ -154,12 +155,12 @@ public void testDefaultMaxConcurrentSearches() {
         DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
         for (int i = 0; i < numDataNodes; i++) {
             builder.add(new DiscoveryNode("_id" + i, buildNewFakeTransportAddress(), Collections.emptyMap(),
-                    Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT));
+                    Collections.singleton(DiscoveryNodeRole.DATA_ROLE), Version.CURRENT));
         }
         builder.add(new DiscoveryNode("master", buildNewFakeTransportAddress(), Collections.emptyMap(),
-                Collections.singleton(DiscoveryNode.Role.MASTER), Version.CURRENT));
+                Collections.singleton(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT));
         builder.add(new DiscoveryNode("ingest", buildNewFakeTransportAddress(), Collections.emptyMap(),
-                Collections.singleton(DiscoveryNode.Role.INGEST), Version.CURRENT));
+                Collections.singleton(DiscoveryNodeRole.INGEST_ROLE), Version.CURRENT));
 
         ClusterState state = ClusterState.builder(new ClusterName("_name")).nodes(builder).build();
         int result = TransportMultiSearchAction.defaultMaxConcurrentSearches(10, state);
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionSingleNodeTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionSingleNodeTests.java
index 10f252c30dc3b..fa6160839d2a9 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionSingleNodeTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionSingleNodeTests.java
@@ -19,11 +19,14 @@
 
 package org.elasticsearch.action.search;
 
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
@@ -174,4 +177,62 @@ public void testFinalReduce()  {
             assertEquals(2, longTerms.getBuckets().size());
         }
     }
+
+    public void testSplitIndices() {
+        {
+            CreateIndexResponse response = client().admin().indices().prepareCreate("write").get();
+            assertTrue(response.isAcknowledged());
+        }
+        {
+            CreateIndexResponse response = client().admin().indices().prepareCreate("readonly").get();
+            assertTrue(response.isAcknowledged());
+        }
+        {
+            SearchResponse response = client().prepareSearch("readonly").get();
+            assertEquals(1, response.getTotalShards());
+            assertEquals(1, response.getSuccessfulShards());
+            assertEquals(1, response.getNumReducePhases());
+        }
+        {
+            SearchResponse response = client().prepareSearch("write").get();
+            assertEquals(1, response.getTotalShards());
+            assertEquals(1, response.getSuccessfulShards());
+            assertEquals(1, response.getNumReducePhases());
+        }
+        {
+            SearchResponse response = client().prepareSearch("readonly", "write").get();
+            assertEquals(2, response.getTotalShards());
+            assertEquals(2, response.getSuccessfulShards());
+            assertEquals(1, response.getNumReducePhases());
+        }
+        {
+            Settings settings = Settings.builder().put("index.blocks.read_only", "true").build();
+            AcknowledgedResponse response = client().admin().indices().prepareUpdateSettings("readonly").setSettings(settings).get();
+            assertTrue(response.isAcknowledged());
+        }
+        try {
+            {
+                SearchResponse response = client().prepareSearch("readonly").get();
+                assertEquals(1, response.getTotalShards());
+                assertEquals(1, response.getSuccessfulShards());
+                assertEquals(1, response.getNumReducePhases());
+            }
+            {
+                SearchResponse response = client().prepareSearch("write").get();
+                assertEquals(1, response.getTotalShards());
+                assertEquals(1, response.getSuccessfulShards());
+                assertEquals(1, response.getNumReducePhases());
+            }
+            {
+                SearchResponse response = client().prepareSearch("readonly", "write").get();
+                assertEquals(2, response.getTotalShards());
+                assertEquals(2, response.getSuccessfulShards());
+                assertEquals(3, response.getNumReducePhases());
+            }
+        } finally {
+            Settings settings = Settings.builder().put("index.blocks.read_only", "false").build();
+            AcknowledgedResponse response = client().admin().indices().prepareUpdateSettings("readonly").setSettings(settings).get();
+            assertTrue(response.isAcknowledged());
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
index 369d71f05ffb8..60078486335c9 100644
--- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java
@@ -29,6 +29,10 @@
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.GroupShardsIteratorTests;
@@ -837,4 +841,75 @@ public void testShouldMinimizeRoundtrips() throws Exception {
             assertFalse(TransportSearchAction.shouldMinimizeRoundtrips(searchRequest));
         }
     }
+
+    public void testShouldSplitIndices() {
+        {
+            SearchRequest searchRequest = new SearchRequest();
+            assertTrue(TransportSearchAction.shouldSplitIndices(searchRequest));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest();
+            searchRequest.source(new SearchSourceBuilder());
+            assertTrue(TransportSearchAction.shouldSplitIndices(searchRequest));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest();
+            searchRequest.source(new SearchSourceBuilder().size(randomIntBetween(1, 100)));
+            assertTrue(TransportSearchAction.shouldSplitIndices(searchRequest));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest();
+            searchRequest.scroll("5s");
+            assertFalse(TransportSearchAction.shouldSplitIndices(searchRequest));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest();
+            searchRequest.source(new SearchSourceBuilder().size(0));
+            assertFalse(TransportSearchAction.shouldSplitIndices(searchRequest));
+        }
+        {
+            SearchRequest searchRequest = new SearchRequest();
+            searchRequest.searchType(SearchType.DFS_QUERY_THEN_FETCH);
+            assertFalse(TransportSearchAction.shouldSplitIndices(searchRequest));
+        }
+    }
+
+    public void testSplitIndices() {
+        int numIndices = randomIntBetween(1, 10);
+        Index[] indices = new Index[numIndices];
+        for (int i = 0; i < numIndices; i++) {
+            String indexName = randomAlphaOfLengthBetween(5, 10);
+            indices[i] = new Index(indexName, indexName + "-uuid");
+        }
+        {
+            ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).build();
+            List<String> writeIndices = new ArrayList<>();
+            List<String> readOnlyIndices = new ArrayList<>();
+            TransportSearchAction.splitIndices(indices, clusterState, writeIndices, readOnlyIndices);
+            assertEquals(0, readOnlyIndices.size());
+            assertEquals(numIndices, writeIndices.size());
+        }
+        {
+            List<String> expectedWrite = new ArrayList<>();
+            List<String> expectedReadOnly = new ArrayList<>();
+            ClusterBlocks.Builder blocksBuilder = ClusterBlocks.builder();
+            for (Index index : indices) {
+                if (randomBoolean()) {
+                    blocksBuilder.addIndexBlock(index.getName(), IndexMetaData.INDEX_WRITE_BLOCK);
+                    expectedReadOnly.add(index.getName());
+                } else if(randomBoolean() ){
+                    blocksBuilder.addIndexBlock(index.getName(), IndexMetaData.INDEX_READ_ONLY_BLOCK);
+                    expectedReadOnly.add(index.getName());
+                } else {
+                    expectedWrite.add(index.getName());
+                }
+            }
+            ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).blocks(blocksBuilder).build();
+            List<String> writeIndices = new ArrayList<>();
+            List<String> readOnlyIndices = new ArrayList<>();
+            TransportSearchAction.splitIndices(indices, clusterState, writeIndices, readOnlyIndices);
+            assertEquals(writeIndices, expectedWrite);
+            assertEquals(readOnlyIndices, expectedReadOnly);
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/support/ActiveShardsObserverIT.java b/server/src/test/java/org/elasticsearch/action/support/ActiveShardsObserverIT.java
index 2f0dd64b7ec02..b1269f7e916cc 100644
--- a/server/src/test/java/org/elasticsearch/action/support/ActiveShardsObserverIT.java
+++ b/server/src/test/java/org/elasticsearch/action/support/ActiveShardsObserverIT.java
@@ -118,7 +118,7 @@ public void testCreateIndexWaitsForAllActiveShards() throws Exception {
                        .get()
                        .isShardsAcknowledged());
         waitForIndexCreationToComplete(indexName);
-        if (client().admin().indices().prepareExists(indexName).get().isExists()) {
+        if (indexExists(indexName)) {
             client().admin().indices().prepareDelete(indexName).get();
         }
 
diff --git a/server/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java b/server/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java
index 1317183f286b3..45d8f4c8c0bf0 100644
--- a/server/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java
+++ b/server/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java
@@ -39,7 +39,7 @@
 
 import static org.hamcrest.Matchers.equalTo;
 
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false)
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class IndexingMasterFailoverIT extends ESIntegTestCase {
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponseTests.java b/server/src/test/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponseTests.java
new file mode 100644
index 0000000000000..ca998c268fb86
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/action/support/master/ShardsAcknowledgedResponseTests.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.support.master;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.Matchers.is;
+
+public class ShardsAcknowledgedResponseTests extends ESTestCase {
+
+    public void testSerialization() throws Exception {
+        ShardsAcknowledgedResponse testInstance = new TestImpl(true, true);
+
+        ShardsAcknowledgedResponse result =
+            copyWriteable(testInstance, new NamedWriteableRegistry(List.of()), in -> new TestImpl(in, true), Version.CURRENT);
+        assertThat(result.isAcknowledged(), is(true));
+        assertThat(result.isShardsAcknowledged(), is(true));
+
+        result = copyWriteable(testInstance, new NamedWriteableRegistry(List.of()), in -> new TestImpl(in, false), Version.CURRENT);
+        assertThat(result.isAcknowledged(), is(true));
+        assertThat(result.isShardsAcknowledged(), is(false));
+    }
+
+    private static class TestImpl extends ShardsAcknowledgedResponse {
+
+        private TestImpl(StreamInput in, boolean readShardsAcknowledged) throws IOException {
+            super(in, readShardsAcknowledged);
+        }
+
+        private TestImpl(boolean acknowledged, boolean shardsAcknowledged) {
+            super(acknowledged, shardsAcknowledged);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            writeShardsAcknowledged(out);
+        }
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
index 3967123ad5065..fc33ee7686715 100644
--- a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
@@ -34,14 +34,15 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
@@ -95,9 +96,9 @@ public void setUp() throws Exception {
         transportService.start();
         transportService.acceptIncomingRequests();
         localNode = new DiscoveryNode("local_node", buildNewFakeTransportAddress(), Collections.emptyMap(),
-                Collections.singleton(DiscoveryNode.Role.MASTER), Version.CURRENT);
+                Collections.singleton(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT);
         remoteNode = new DiscoveryNode("remote_node", buildNewFakeTransportAddress(), Collections.emptyMap(),
-                Collections.singleton(DiscoveryNode.Role.MASTER), Version.CURRENT);
+                Collections.singleton(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT);
         allNodes = new DiscoveryNode[]{localNode, remoteNode};
     }
 
@@ -184,7 +185,7 @@ protected Response newResponse() {
         }
 
         @Override
-        protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) throws Exception {
+        protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<Response> listener) throws Exception {
             listener.onResponse(new Response()); // default implementation, overridden in specific tests
         }
 
@@ -446,7 +447,8 @@ public void testMasterFailoverAfterStepDown() throws ExecutionException, Interru
 
         new Action( "internal:testAction", transportService, clusterService, threadPool) {
             @Override
-            protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) throws Exception {
+            protected void masterOperation(Task task, Request request, ClusterState state,
+                                           ActionListener<Response> listener) throws Exception {
                 // The other node has become master, simulate failures of this node while publishing cluster state through ZenDiscovery
                 setState(clusterService, ClusterStateCreationUtils.state(localNode, remoteNode, allNodes));
                 Exception failure = randomBoolean()
diff --git a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionUtils.java b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionUtils.java
index be4a7b29703e1..fa5b5fa33bff0 100644
--- a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionUtils.java
+++ b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionUtils.java
@@ -26,13 +26,15 @@
 public class TransportMasterNodeActionUtils {
 
     /**
-     * Allows to directly call {@link TransportMasterNodeAction#masterOperation(MasterNodeRequest, ClusterState, ActionListener)} which is
-     * a protected method.
+     * Allows to directly call
+     * {@link TransportMasterNodeAction#masterOperation(org.elasticsearch.tasks.Task, MasterNodeRequest, ClusterState, ActionListener)}
+     * which is a protected method.
      */
     public static <Request extends MasterNodeRequest<Request>, Response extends ActionResponse> void runMasterOperation(
         TransportMasterNodeAction<Request, Response> masterNodeAction, Request request, ClusterState clusterState,
         ActionListener<Response> actionListener) throws Exception {
         assert masterNodeAction.checkBlock(request, clusterState) == null;
-        masterNodeAction.masterOperation(request, clusterState, actionListener);
+        // TODO: pass through task here?
+        masterNodeAction.masterOperation(null, request, clusterState, actionListener);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
index 0a2b495e2b820..59ed774dbb06b 100644
--- a/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
@@ -27,10 +27,12 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -43,7 +45,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -188,7 +189,7 @@ public void setUp() throws Exception {
         List<DiscoveryNode> discoveryNodes = new ArrayList<>();
         for (int i = 0; i < numNodes; i++) {
             Map<String, String> attributes = new HashMap<>();
-            Set<DiscoveryNode.Role> roles = new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values())));
+            Set<DiscoveryNodeRole> roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES));
             if (frequently()) {
                 attributes.put("custom", randomBoolean() ? "match" : randomAlphaOfLengthBetween(3, 5));
             }
@@ -235,7 +236,7 @@ public DataNodesOnlyTransportNodesAction getDataNodesOnlyTransportNodesAction(Tr
         );
     }
 
-    private static DiscoveryNode newNode(int nodeId, Map<String, String> attributes, Set<DiscoveryNode.Role> roles) {
+    private static DiscoveryNode newNode(int nodeId, Map<String, String> attributes, Set<DiscoveryNodeRole> roles) {
         String node = "node_" + nodeId;
         return new DiscoveryNode(node, node, buildNewFakeTransportAddress(), attributes, roles, Version.CURRENT);
     }
@@ -257,7 +258,7 @@ protected TestNodesResponse newResponse(TestNodesRequest request,
         }
 
         @Override
-        protected TestNodeRequest newNodeRequest(String nodeId, TestNodesRequest request) {
+        protected TestNodeRequest newNodeRequest(TestNodesRequest request) {
             return new TestNodeRequest();
         }
 
@@ -267,7 +268,7 @@ protected TestNodeResponse newNodeResponse() {
         }
 
         @Override
-        protected TestNodeResponse nodeOperation(TestNodeRequest request) {
+        protected TestNodeResponse nodeOperation(TestNodeRequest request, Task task) {
             return new TestNodeResponse();
         }
 
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
index 6b628d88c59d4..a427cd7297bf3 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.AllocationId;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -41,7 +42,6 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -407,7 +407,7 @@ public static ClusterState state(DiscoveryNode localNode, DiscoveryNode masterNo
 
     private static DiscoveryNode newNode(int nodeId) {
         return new DiscoveryNode("node_" + nodeId, ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(),
-                new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
+                new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT);
     }
 
     private static String selectAndRemove(Set<String> strings) {
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
index c959e3ed45d1a..9f86d190a644a 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
@@ -137,6 +137,7 @@ public void testReplication() throws Exception {
 
         assertThat(primary.knownLocalCheckpoints.remove(primaryShard.allocationId().getId()), equalTo(primary.localCheckpoint));
         assertThat(primary.knownLocalCheckpoints, equalTo(replicasProxy.generatedLocalCheckpoints));
+        assertThat(primary.knownGlobalCheckpoints.remove(primaryShard.allocationId().getId()), equalTo(primary.globalCheckpoint));
         assertThat(primary.knownGlobalCheckpoints, equalTo(replicasProxy.generatedGlobalCheckpoints));
     }
 
@@ -533,6 +534,11 @@ public long globalCheckpoint() {
             return globalCheckpoint;
         }
 
+        @Override
+        public long computedGlobalCheckpoint() {
+            return globalCheckpoint;
+        }
+
         @Override
         public long maxSeqNoOfUpdatesOrDeletes() {
             return maxSeqNoOfUpdatesOrDeletes;
diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java
index 28373347b19ca..d5eb6a81bf24c 100644
--- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
@@ -64,7 +65,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
@@ -128,7 +128,7 @@ public void setUp() throws Exception {
         clusterService = createClusterService(threadPool);
 
         final ClusterState.Builder state = ClusterState.builder(clusterService.state());
-        Set<DiscoveryNode.Role> roles = new HashSet<>(Arrays.asList(DiscoveryNode.Role.values()));
+        Set<DiscoveryNodeRole> roles = new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES);
         DiscoveryNode node1 = new DiscoveryNode("_name1", "_node1", buildNewFakeTransportAddress(), emptyMap(), roles, Version.CURRENT);
         DiscoveryNode node2 = new DiscoveryNode("_name2", "_node2", buildNewFakeTransportAddress(), emptyMap(), roles, Version.CURRENT);
         state.nodes(DiscoveryNodes.builder()
diff --git a/server/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java b/server/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java
index 0393f5929ce38..f0e94cbd1694a 100644
--- a/server/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java
+++ b/server/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java
@@ -403,7 +403,7 @@ protected void validateResponse(TermVectorsResponse esResponse, Fields luceneFie
     }
 
     protected TermVectorsRequestBuilder getRequestForConfig(TestConfig config) {
-        return client().prepareTermVectors(randomBoolean() ? config.doc.index : config.doc.alias, config.doc.type, config.doc.id)
+        return client().prepareTermVectors(randomBoolean() ? config.doc.index : config.doc.alias, config.doc.id)
             .setPayloads(config.requestPayloads)
             .setOffsets(config.requestOffsets).setPositions(config.requestPositions).setFieldStatistics(true).setTermStatistics(true)
             .setSelectedFields(config.selectedFields).setRealtime(false);
diff --git a/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java b/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
index 442e27c0867b9..934f2563574d0 100644
--- a/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
+++ b/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
@@ -81,7 +81,7 @@ public void testNoSuchDoc() throws Exception {
         client().prepareIndex("test", "type1", "666").setSource("field", "foo bar").execute().actionGet();
         refresh();
         for (int i = 0; i < 20; i++) {
-            ActionFuture<TermVectorsResponse> termVector = client().termVectors(new TermVectorsRequest(indexOrAlias(), "type1", "" + i));
+            ActionFuture<TermVectorsResponse> termVector = client().termVectors(new TermVectorsRequest(indexOrAlias(), "" + i));
             TermVectorsResponse actionGet = termVector.actionGet();
             assertThat(actionGet, notNullValue());
             assertThat(actionGet.getIndex(), equalTo("test"));
@@ -105,7 +105,7 @@ public void testExistingFieldWithNoTermVectorsNoNPE() throws Exception {
         // when indexing a field that simply has a question mark, the term vectors will be null
         client().prepareIndex("test", "type1", "0").setSource("existingfield", "?").execute().actionGet();
         refresh();
-        ActionFuture<TermVectorsResponse> termVector = client().termVectors(new TermVectorsRequest(indexOrAlias(), "type1", "0")
+        ActionFuture<TermVectorsResponse> termVector = client().termVectors(new TermVectorsRequest(indexOrAlias(), "0")
                 .selectedFields(new String[]{"existingfield"}));
 
         // lets see if the null term vectors are caught...
@@ -130,7 +130,7 @@ public void testExistingFieldButNotInDocNPE() throws Exception {
         // when indexing a field that simply has a question mark, the term vectors will be null
         client().prepareIndex("test", "type1", "0").setSource("anotherexistingfield", 1).execute().actionGet();
         refresh();
-        ActionFuture<TermVectorsResponse> termVectors = client().termVectors(new TermVectorsRequest(indexOrAlias(), "type1", "0")
+        ActionFuture<TermVectorsResponse> termVectors = client().termVectors(new TermVectorsRequest(indexOrAlias(), "0")
                 .selectedFields(randomBoolean() ? new String[]{"existingfield"} : null)
                 .termStatistics(true)
                 .fieldStatistics(true));
@@ -166,7 +166,7 @@ public void testNotIndexedField() throws Exception {
         indexRandom(true, indexBuilders);
 
         for (int i = 0; i < 4; i++) {
-            TermVectorsResponse resp = client().prepareTermVectors(indexOrAlias(), "type1", String.valueOf(i))
+            TermVectorsResponse resp = client().prepareTermVectors(indexOrAlias(), String.valueOf(i))
                     .setSelectedFields("field" + i)
                     .get();
             assertThat(resp, notNullValue());
@@ -176,7 +176,7 @@ public void testNotIndexedField() throws Exception {
         }
 
         for (int i = 4; i < 6; i++) {
-            TermVectorsResponse resp = client().prepareTermVectors(indexOrAlias(), "type1", String.valueOf(i))
+            TermVectorsResponse resp = client().prepareTermVectors(indexOrAlias(), String.valueOf(i))
                     .setSelectedFields("field" + i).get();
             assertThat(resp.getIndex(), equalTo("test"));
             assertThat("field" + i + " :", resp.getFields().terms("field" + i), notNullValue());
@@ -208,7 +208,7 @@ public void testSimpleTermVectors() throws IOException {
             refresh();
         }
         for (int i = 0; i < 10; i++) {
-            TermVectorsRequestBuilder resp = client().prepareTermVectors(indexOrAlias(), "type1", Integer.toString(i)).setPayloads(true)
+            TermVectorsRequestBuilder resp = client().prepareTermVectors(indexOrAlias(), Integer.toString(i)).setPayloads(true)
                     .setOffsets(true).setPositions(true).setSelectedFields();
             TermVectorsResponse response = resp.execute().actionGet();
             assertThat(response.getIndex(), equalTo("test"));
@@ -289,7 +289,7 @@ public void testRandomSingleTermVectors() throws IOException {
         boolean isPositionsRequested = randomBoolean();
         String infoString = createInfoString(isPositionsRequested, isOffsetRequested, optionString);
         for (int i = 0; i < 10; i++) {
-            TermVectorsRequestBuilder resp = client().prepareTermVectors("test", "type1", Integer.toString(i))
+            TermVectorsRequestBuilder resp = client().prepareTermVectors("test", Integer.toString(i))
                     .setOffsets(isOffsetRequested).setPositions(isPositionsRequested).setSelectedFields();
             TermVectorsResponse response = resp.execute().actionGet();
             assertThat(infoString + "doc id: " + i + " doesn't exists but should", response.isExists(), equalTo(true));
@@ -416,7 +416,7 @@ public void testSimpleTermVectorsWithGenerate() throws IOException {
         }
 
         for (int i = 0; i < 10; i++) {
-            TermVectorsResponse response = client().prepareTermVectors("test", "type1", Integer.toString(i))
+            TermVectorsResponse response = client().prepareTermVectors("test", Integer.toString(i))
                     .setPayloads(true)
                     .setOffsets(true)
                     .setPositions(true)
@@ -506,7 +506,7 @@ public void testDuelWithAndWithoutTermVectors() throws IOException, ExecutionExc
         for (int id = 0; id < content.length; id++) {
             Fields[] fields = new Fields[2];
             for (int j = 0; j < indexNames.length; j++) {
-                TermVectorsResponse resp = client().prepareTermVectors(indexNames[j], "type1", String.valueOf(id))
+                TermVectorsResponse resp = client().prepareTermVectors(indexNames[j], String.valueOf(id))
                         .setOffsets(true)
                         .setPositions(true)
                         .setSelectedFields("field1")
@@ -580,7 +580,7 @@ public void testSimpleWildCards() throws IOException {
         client().prepareIndex("test", "type1", "0").setSource(source).get();
         refresh();
 
-        TermVectorsResponse response = client().prepareTermVectors(indexOrAlias(), "type1", "0").setSelectedFields("field*").get();
+        TermVectorsResponse response = client().prepareTermVectors(indexOrAlias(), "0").setSelectedFields("field*").get();
         assertThat("Doc doesn't exists but should", response.isExists(), equalTo(true));
         assertThat(response.getIndex(), equalTo("test"));
         assertThat("All term vectors should have been generated", response.getFields().size(), equalTo(numFields));
@@ -615,7 +615,7 @@ public void testArtificialVsExisting() throws ExecutionException, InterruptedExc
 
         for (int i = 0; i < content.length; i++) {
             // request tvs from existing document
-            TermVectorsResponse respExisting = client().prepareTermVectors("test", "type1", String.valueOf(i))
+            TermVectorsResponse respExisting = client().prepareTermVectors("test", String.valueOf(i))
                     .setOffsets(true)
                     .setPositions(true)
                     .setFieldStatistics(true)
@@ -626,7 +626,6 @@ public void testArtificialVsExisting() throws ExecutionException, InterruptedExc
             // request tvs from artificial document
             TermVectorsResponse respArtificial = client().prepareTermVectors()
                     .setIndex("test")
-                    .setType("type1")
                     .setRouting(String.valueOf(i)) // ensure we get the stats from the same shard as existing doc
                     .setDoc(jsonBuilder()
                             .startObject()
@@ -658,7 +657,6 @@ public void testArtificialNoDoc() throws IOException {
         String text = "the quick brown fox jumps over the lazy dog";
         TermVectorsResponse resp = client().prepareTermVectors()
                 .setIndex("test")
-                .setType("type1")
                 .setDoc(jsonBuilder()
                         .startObject()
                         .field("field1", text)
@@ -736,7 +734,7 @@ public void testPerFieldAnalyzer() throws IOException {
         }
 
         // selected fields not specified
-        TermVectorsResponse response = client().prepareTermVectors(indexOrAlias(), "type1", "0")
+        TermVectorsResponse response = client().prepareTermVectors(indexOrAlias(), "0")
                 .setPerFieldAnalyzer(perFieldAnalyzer)
                 .get();
 
@@ -744,7 +742,7 @@ public void testPerFieldAnalyzer() throws IOException {
         checkAnalyzedFields(response.getFields(), withTermVectors, perFieldAnalyzer);
 
         // selected fields specified including some not in the mapping
-        response = client().prepareTermVectors(indexOrAlias(), "type1", "0")
+        response = client().prepareTermVectors(indexOrAlias(), "0")
                 .setSelectedFields(selectedFields.toArray(Strings.EMPTY_ARRAY))
                 .setPerFieldAnalyzer(perFieldAnalyzer)
                 .get();
@@ -784,7 +782,7 @@ public void testTermVectorsWithVersion() {
                 .setSettings(Settings.builder().put("index.refresh_interval", -1)));
         ensureGreen();
 
-        TermVectorsResponse response = client().prepareTermVectors("test", "type1", "1").get();
+        TermVectorsResponse response = client().prepareTermVectors("test", "1").get();
         assertThat(response.isExists(), equalTo(false));
 
         logger.info("--> index doc 1");
@@ -793,12 +791,12 @@ public void testTermVectorsWithVersion() {
         // From translog:
 
         // version 0 means ignore version, which is the default
-        response = client().prepareTermVectors(indexOrAlias(), "type1", "1").setVersion(Versions.MATCH_ANY).get();
+        response = client().prepareTermVectors(indexOrAlias(), "1").setVersion(Versions.MATCH_ANY).get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getId(), equalTo("1"));
         assertThat(response.getVersion(), equalTo(1L));
 
-        response = client().prepareTermVectors(indexOrAlias(), "type1", "1").setVersion(1).get();
+        response = client().prepareTermVectors(indexOrAlias(), "1").setVersion(1).get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getId(), equalTo("1"));
         assertThat(response.getVersion(), equalTo(1L));
@@ -814,13 +812,13 @@ public void testTermVectorsWithVersion() {
         refresh();
 
         // version 0 means ignore version, which is the default
-        response = client().prepareTermVectors(indexOrAlias(), "type1", "1").setVersion(Versions.MATCH_ANY).setRealtime(false).get();
+        response = client().prepareTermVectors(indexOrAlias(), "1").setVersion(Versions.MATCH_ANY).setRealtime(false).get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getId(), equalTo("1"));
         assertThat(response.getIndex(), equalTo("test"));
         assertThat(response.getVersion(), equalTo(1L));
 
-        response = client().prepareTermVectors(indexOrAlias(), "type1", "1").setVersion(1).setRealtime(false).get();
+        response = client().prepareTermVectors(indexOrAlias(), "1").setVersion(1).setRealtime(false).get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getId(), equalTo("1"));
         assertThat(response.getIndex(), equalTo("test"));
@@ -839,7 +837,7 @@ public void testTermVectorsWithVersion() {
         // From translog:
 
         // version 0 means ignore version, which is the default
-        response = client().prepareTermVectors(indexOrAlias(), "type1", "1").setVersion(Versions.MATCH_ANY).get();
+        response = client().prepareTermVectors(indexOrAlias(), "1").setVersion(Versions.MATCH_ANY).get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getId(), equalTo("1"));
         assertThat(response.getIndex(), equalTo("test"));
@@ -852,7 +850,7 @@ public void testTermVectorsWithVersion() {
             //all good
         }
 
-        response = client().prepareTermVectors(indexOrAlias(), "type1", "1").setVersion(2).get();
+        response = client().prepareTermVectors(indexOrAlias(), "1").setVersion(2).get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getId(), equalTo("1"));
         assertThat(response.getIndex(), equalTo("test"));
@@ -862,7 +860,7 @@ public void testTermVectorsWithVersion() {
         refresh();
 
         // version 0 means ignore version, which is the default
-        response = client().prepareTermVectors(indexOrAlias(), "type1", "1").setVersion(Versions.MATCH_ANY).setRealtime(false).get();
+        response = client().prepareTermVectors(indexOrAlias(), "1").setVersion(Versions.MATCH_ANY).setRealtime(false).get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getId(), equalTo("1"));
         assertThat(response.getIndex(), equalTo("test"));
@@ -875,7 +873,7 @@ public void testTermVectorsWithVersion() {
             //all good
         }
 
-        response = client().prepareTermVectors(indexOrAlias(), "type1", "1").setVersion(2).setRealtime(false).get();
+        response = client().prepareTermVectors(indexOrAlias(), "1").setVersion(2).setRealtime(false).get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getId(), equalTo("1"));
         assertThat(response.getIndex(), equalTo("test"));
@@ -909,7 +907,7 @@ public void testFilterLength() throws ExecutionException, InterruptedException,
         TermVectorsResponse response;
         for (int i = 0; i < numTerms; i++) {
             filterSettings.minWordLength = numTerms - i;
-            response = client().prepareTermVectors("test", "type1", "1")
+            response = client().prepareTermVectors("test", "1")
                     .setSelectedFields("tags")
                     .setFieldStatistics(true)
                     .setTermStatistics(true)
@@ -948,7 +946,7 @@ public void testFilterTermFreq() throws ExecutionException, InterruptedException
         TermVectorsResponse response;
         for (int i = 0; i < numTerms; i++) {
             filterSettings.maxNumTerms = i + 1;
-            response = client().prepareTermVectors("test", "type1", "1")
+            response = client().prepareTermVectors("test", "1")
                     .setSelectedFields("tags")
                     .setFieldStatistics(true)
                     .setTermStatistics(true)
@@ -983,7 +981,7 @@ public void testFilterDocFreq() throws ExecutionException, InterruptedException,
         TermVectorsResponse response;
         for (int i = 0; i < numDocs; i++) {
             filterSettings.maxNumTerms = i + 1;
-            response = client().prepareTermVectors("test", "type1", (numDocs - 1) + "")
+            response = client().prepareTermVectors("test", (numDocs - 1) + "")
                     .setSelectedFields("tags")
                     .setFieldStatistics(true)
                     .setTermStatistics(true)
@@ -1016,7 +1014,6 @@ public void testArtificialDocWithPreference() throws ExecutionException, Interru
         for (Integer shardId : shardIds) {
             TermVectorsResponse tvResponse = client().prepareTermVectors()
                     .setIndex("test")
-                    .setType("type1")
                     .setPreference("_shards:" + shardId)
                     .setDoc(jsonBuilder().startObject().field("field1", "random permutation").endObject())
                     .setFieldStatistics(true)
@@ -1069,7 +1066,7 @@ public void testWithKeywordAndNormalizer() throws IOException, ExecutionExceptio
         for (int id = 0; id < content.length; id++) {
             Fields[] fields = new Fields[2];
             for (int j = 0; j < indexNames.length; j++) {
-                TermVectorsResponse resp = client().prepareTermVectors(indexNames[j], "type1", String.valueOf(id))
+                TermVectorsResponse resp = client().prepareTermVectors(indexNames[j], String.valueOf(id))
                     .setOffsets(true)
                     .setPositions(true)
                     .setSelectedFields("field1", "field2")
diff --git a/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java b/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java
index 6379867d51604..6fa6540785ce3 100644
--- a/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java
+++ b/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java
@@ -169,7 +169,7 @@ public void testRandomPayloadWithDelimitedPayloadTokenFilter() throws IOExceptio
         client().prepareIndex("test", "type1", Integer.toString(1))
                 .setSource(jsonBuilder().startObject().field("field", queryString).endObject()).execute().actionGet();
         client().admin().indices().prepareRefresh().get();
-        TermVectorsRequestBuilder resp = client().prepareTermVectors("test", "type1", Integer.toString(1))
+        TermVectorsRequestBuilder resp = client().prepareTermVectors("test", Integer.toString(1))
                 .setPayloads(true).setOffsets(true).setPositions(true).setSelectedFields();
         TermVectorsResponse response = resp.execute().actionGet();
         assertThat("doc id 1 doesn't exists but should", response.isExists(), equalTo(true));
diff --git a/server/src/test/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java b/server/src/test/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java
index 08751ffe058ec..964ac56692b9a 100644
--- a/server/src/test/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java
+++ b/server/src/test/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java
@@ -70,7 +70,7 @@ public void testDuelESLucene() throws Exception {
     }
 
     public void testMissingIndexThrowsMissingIndex() throws Exception {
-        TermVectorsRequestBuilder requestBuilder = client().prepareTermVectors("testX", "typeX", Integer.toString(1));
+        TermVectorsRequestBuilder requestBuilder = client().prepareTermVectors("testX", Integer.toString(1));
         MultiTermVectorsRequestBuilder mtvBuilder = client().prepareMultiTermVectors();
         mtvBuilder.add(requestBuilder.request());
         MultiTermVectorsResponse response = mtvBuilder.execute().actionGet();
@@ -84,7 +84,7 @@ public void testMultiTermVectorsWithVersion() throws Exception {
                 .setSettings(Settings.builder().put("index.refresh_interval", -1)));
         ensureGreen();
 
-        MultiTermVectorsResponse response = client().prepareMultiTermVectors().add(indexOrAlias(), "type1", "1").get();
+        MultiTermVectorsResponse response = client().prepareMultiTermVectors().add(indexOrAlias(), "1").get();
         assertThat(response.getResponses().length, equalTo(1));
         assertThat(response.getResponses()[0].getResponse().isExists(), equalTo(false));
 
@@ -94,9 +94,9 @@ public void testMultiTermVectorsWithVersion() throws Exception {
 
         // Version from translog
         response = client().prepareMultiTermVectors()
-                .add(new TermVectorsRequest(indexOrAlias(), "type1", "1").selectedFields("field").version(Versions.MATCH_ANY))
-                .add(new TermVectorsRequest(indexOrAlias(), "type1", "1").selectedFields("field").version(1))
-                .add(new TermVectorsRequest(indexOrAlias(), "type1", "1").selectedFields("field").version(2))
+                .add(new TermVectorsRequest(indexOrAlias(), "1").selectedFields("field").version(Versions.MATCH_ANY))
+                .add(new TermVectorsRequest(indexOrAlias(), "1").selectedFields("field").version(1))
+                .add(new TermVectorsRequest(indexOrAlias(), "1").selectedFields("field").version(2))
                 .get();
         assertThat(response.getResponses().length, equalTo(3));
         // [0] version doesn't matter, which is the default
@@ -118,10 +118,10 @@ public void testMultiTermVectorsWithVersion() throws Exception {
         //Version from Lucene index
         refresh();
         response = client().prepareMultiTermVectors()
-                .add(new TermVectorsRequest(indexOrAlias(), "type1", "1").selectedFields("field")
+                .add(new TermVectorsRequest(indexOrAlias(), "1").selectedFields("field")
                     .version(Versions.MATCH_ANY).realtime(false))
-                .add(new TermVectorsRequest(indexOrAlias(), "type1", "1").selectedFields("field").version(1).realtime(false))
-                .add(new TermVectorsRequest(indexOrAlias(), "type1", "1").selectedFields("field").version(2).realtime(false))
+                .add(new TermVectorsRequest(indexOrAlias(), "1").selectedFields("field").version(1).realtime(false))
+                .add(new TermVectorsRequest(indexOrAlias(), "1").selectedFields("field").version(2).realtime(false))
                 .get();
         assertThat(response.getResponses().length, equalTo(3));
         // [0] version doesn't matter, which is the default
@@ -145,9 +145,9 @@ public void testMultiTermVectorsWithVersion() throws Exception {
 
         // Version from translog
         response = client().prepareMultiTermVectors()
-                .add(new TermVectorsRequest(indexOrAlias(), "type1", "2").selectedFields("field").version(Versions.MATCH_ANY))
-                .add(new TermVectorsRequest(indexOrAlias(), "type1", "2").selectedFields("field").version(1))
-                .add(new TermVectorsRequest(indexOrAlias(), "type1", "2").selectedFields("field").version(2))
+                .add(new TermVectorsRequest(indexOrAlias(), "2").selectedFields("field").version(Versions.MATCH_ANY))
+                .add(new TermVectorsRequest(indexOrAlias(), "2").selectedFields("field").version(1))
+                .add(new TermVectorsRequest(indexOrAlias(), "2").selectedFields("field").version(2))
                 .get();
         assertThat(response.getResponses().length, equalTo(3));
         // [0] version doesn't matter, which is the default
@@ -171,9 +171,9 @@ public void testMultiTermVectorsWithVersion() throws Exception {
         //Version from Lucene index
         refresh();
         response = client().prepareMultiTermVectors()
-                .add(new TermVectorsRequest(indexOrAlias(), "type1", "2").selectedFields("field").version(Versions.MATCH_ANY))
-                .add(new TermVectorsRequest(indexOrAlias(), "type1", "2").selectedFields("field").version(1))
-                .add(new TermVectorsRequest(indexOrAlias(), "type1", "2").selectedFields("field").version(2))
+                .add(new TermVectorsRequest(indexOrAlias(), "2").selectedFields("field").version(Versions.MATCH_ANY))
+                .add(new TermVectorsRequest(indexOrAlias(), "2").selectedFields("field").version(1))
+                .add(new TermVectorsRequest(indexOrAlias(), "2").selectedFields("field").version(2))
                 .get();
         assertThat(response.getResponses().length, equalTo(3));
         // [0] version doesn't matter, which is the default
diff --git a/server/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java b/server/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java
index 8ab452950a87a..b14ccb82738ed 100644
--- a/server/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java
+++ b/server/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java
@@ -36,6 +36,7 @@
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.termvectors.TermVectorsRequest.Flag;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -48,7 +49,9 @@
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.mapper.TypeParsers;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.action.document.RestTermVectorsAction;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.StreamsUtils;
 import org.hamcrest.Matchers;
@@ -65,7 +68,7 @@
 
 public class TermVectorsUnitTests extends ESTestCase {
     public void testStreamResponse() throws Exception {
-        TermVectorsResponse outResponse = new TermVectorsResponse("a", "b", "c");
+        TermVectorsResponse outResponse = new TermVectorsResponse("a", "c");
         outResponse.setExists(true);
         writeStandardTermVector(outResponse);
 
@@ -82,7 +85,7 @@ public void testStreamResponse() throws Exception {
         // see if correct
         checkIfStandardTermVector(inResponse);
 
-        outResponse = new TermVectorsResponse("a", "b", "c");
+        outResponse = new TermVectorsResponse("a", "c");
         writeEmptyTermVector(outResponse);
         // write
         outBuffer = new ByteArrayOutputStream();
@@ -175,7 +178,7 @@ public void testRestRequestParsing() throws Exception {
         BytesReference inputBytes = new BytesArray(
                 " {\"fields\" : [\"a\",  \"b\",\"c\"], \"offsets\":false, \"positions\":false, \"payloads\":true}");
 
-        TermVectorsRequest tvr = new TermVectorsRequest(null, null, null);
+        TermVectorsRequest tvr = new TermVectorsRequest(null, null);
         XContentParser parser = createParser(JsonXContent.jsonXContent, inputBytes);
         TermVectorsRequest.parseRequest(tvr, parser);
 
@@ -196,7 +199,7 @@ public void testRestRequestParsing() throws Exception {
         RestTermVectorsAction.addFieldStringsFromParameter(tvr, additionalFields);
 
         inputBytes = new BytesArray(" {\"offsets\":false, \"positions\":false, \"payloads\":true}");
-        tvr = new TermVectorsRequest(null, null, null);
+        tvr = new TermVectorsRequest(null, null);
         parser = createParser(JsonXContent.jsonXContent, inputBytes);
         TermVectorsRequest.parseRequest(tvr, parser);
         additionalFields = "";
@@ -211,7 +214,7 @@ public void testRestRequestParsing() throws Exception {
     public void testRequestParsingThrowsException() throws Exception {
         BytesReference inputBytes = new BytesArray(
                 " {\"fields\" : \"a,  b,c   \", \"offsets\":false, \"positions\":false, \"payloads\":true, \"meaningless_term\":2}");
-        TermVectorsRequest tvr = new TermVectorsRequest(null, null, null);
+        TermVectorsRequest tvr = new TermVectorsRequest(null, null);
         boolean threwException = false;
         try {
             XContentParser parser = createParser(JsonXContent.jsonXContent, inputBytes);
@@ -225,7 +228,7 @@ public void testRequestParsingThrowsException() throws Exception {
 
     public void testStreamRequest() throws IOException {
         for (int i = 0; i < 10; i++) {
-            TermVectorsRequest request = new TermVectorsRequest("index", "type", "id");
+            TermVectorsRequest request = new TermVectorsRequest("index", "id");
             request.offsets(random().nextBoolean());
             request.fieldStatistics(random().nextBoolean());
             request.payloads(random().nextBoolean());
@@ -243,8 +246,53 @@ public void testStreamRequest() throws IOException {
             // read
             ByteArrayInputStream esInBuffer = new ByteArrayInputStream(outBuffer.toByteArray());
             InputStreamStreamInput esBuffer = new InputStreamStreamInput(esInBuffer);
-            TermVectorsRequest req2 = new TermVectorsRequest(null, null, null);
-            req2.readFrom(esBuffer);
+            TermVectorsRequest req2 = new TermVectorsRequest(esBuffer);
+
+            assertThat(request.offsets(), equalTo(req2.offsets()));
+            assertThat(request.fieldStatistics(), equalTo(req2.fieldStatistics()));
+            assertThat(request.payloads(), equalTo(req2.payloads()));
+            assertThat(request.positions(), equalTo(req2.positions()));
+            assertThat(request.termStatistics(), equalTo(req2.termStatistics()));
+            assertThat(request.preference(), equalTo(pref));
+            assertThat(request.routing(), equalTo(null));
+            assertEquals(new BytesArray("{}"), request.doc());
+            assertEquals(XContentType.JSON, request.xContentType());
+        }
+    }
+
+    public void testStreamRequestLegacyVersion() throws IOException {
+        for (int i = 0; i < 10; i++) {
+            TermVectorsRequest request = new TermVectorsRequest("index", "id");
+            request.offsets(random().nextBoolean());
+            request.fieldStatistics(random().nextBoolean());
+            request.payloads(random().nextBoolean());
+            request.positions(random().nextBoolean());
+            request.termStatistics(random().nextBoolean());
+            String pref = random().nextBoolean() ? "somePreference" : null;
+            request.preference(pref);
+            request.doc(new BytesArray("{}"), randomBoolean(), XContentType.JSON);
+
+            // write using older version which contains types
+            ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
+            OutputStreamStreamOutput out = new OutputStreamStreamOutput(outBuffer);
+            out.setVersion(Version.V_7_2_0);
+            request.writeTo(out);
+
+            // First check the type on the stream was written as "_doc" by manually parsing the stream until the type
+            ByteArrayInputStream esInBuffer = new ByteArrayInputStream(outBuffer.toByteArray());
+            InputStreamStreamInput esBuffer = new InputStreamStreamInput(esInBuffer);
+            TaskId.readFromStream(esBuffer);
+            if (esBuffer.readBoolean()) {
+                new ShardId(esBuffer);
+            }
+            esBuffer.readOptionalString();
+            assertThat(esBuffer.readString(), equalTo("_doc"));
+
+            // now read the stream as normal to check it is parsed correct if received from an older node
+            esInBuffer = new ByteArrayInputStream(outBuffer.toByteArray());
+            esBuffer = new InputStreamStreamInput(esInBuffer);
+            esBuffer.setVersion(Version.V_7_2_0);
+            TermVectorsRequest req2 = new TermVectorsRequest(esBuffer);
 
             assertThat(request.offsets(), equalTo(req2.offsets()));
             assertThat(request.fieldStatistics(), equalTo(req2.fieldStatistics()));
@@ -300,7 +348,6 @@ public void testMultiParser() throws Exception {
         request.add(new TermVectorsRequest(), data);
 
         checkParsedParameters(request);
-        assertWarnings(RestTermVectorsAction.TYPES_DEPRECATION_MESSAGE);
     }
 
     void checkParsedParameters(MultiTermVectorsRequest request) {
@@ -313,7 +360,6 @@ void checkParsedParameters(MultiTermVectorsRequest request) {
         fields.add("c");
         for (TermVectorsRequest singleRequest : request.requests) {
             assertThat(singleRequest.index(), equalTo("testidx"));
-            assertThat(singleRequest.type(), equalTo("test"));
             assertThat(singleRequest.payloads(), equalTo(false));
             assertThat(singleRequest.positions(), equalTo(false));
             assertThat(singleRequest.offsets(), equalTo(false));
@@ -332,14 +378,12 @@ public void testMultiParserFilter() throws Exception {
         request.add(new TermVectorsRequest(), data);
 
         checkParsedFilterParameters(request);
-        assertWarnings(RestTermVectorsAction.TYPES_DEPRECATION_MESSAGE);
     }
 
     void checkParsedFilterParameters(MultiTermVectorsRequest multiRequest) {
         Set<String> ids = new HashSet<>(Arrays.asList("1", "2"));
         for (TermVectorsRequest request : multiRequest.requests) {
             assertThat(request.index(), equalTo("testidx"));
-            assertThat(request.type(), equalTo("test"));
             assertTrue(ids.remove(request.id()));
             assertNotNull(request.filterSettings());
             assertThat(request.filterSettings().maxNumTerms, equalTo(20));
diff --git a/server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java b/server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java
index db50f75272820..6ea24123b4796 100644
--- a/server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java
+++ b/server/src/test/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsActionTests.java
@@ -168,11 +168,11 @@ public void testTransportMultiGetAction() {
         final Task task = createTask();
         final NodeClient client = new NodeClient(Settings.EMPTY, threadPool);
         final MultiTermVectorsRequestBuilder request = new MultiTermVectorsRequestBuilder(client, MultiTermVectorsAction.INSTANCE);
-        request.add(new TermVectorsRequest("index1", "_doc", "1"));
-        request.add(new TermVectorsRequest("index2", "_doc", "2"));
+        request.add(new TermVectorsRequest("index1", "1"));
+        request.add(new TermVectorsRequest("index2", "2"));
 
         final AtomicBoolean shardActionInvoked = new AtomicBoolean(false);
-        transportAction = new TransportMultiTermVectorsAction(transportService, clusterService, shardAction,
+        transportAction = new TransportMultiTermVectorsAction(transportService, clusterService, client,
             new ActionFilters(emptySet()), new Resolver()) {
             @Override
             protected void executeShardAction(final ActionListener<MultiTermVectorsResponse> listener,
@@ -193,11 +193,11 @@ public void testTransportMultiGetAction_withMissingRouting() {
         final Task task = createTask();
         final NodeClient client = new NodeClient(Settings.EMPTY, threadPool);
         final MultiTermVectorsRequestBuilder request = new MultiTermVectorsRequestBuilder(client, MultiTermVectorsAction.INSTANCE);
-        request.add(new TermVectorsRequest("index2", "_doc", "1").routing("1"));
-        request.add(new TermVectorsRequest("index2", "_doc", "2"));
+        request.add(new TermVectorsRequest("index2", "1").routing("1"));
+        request.add(new TermVectorsRequest("index2", "2"));
 
         final AtomicBoolean shardActionInvoked = new AtomicBoolean(false);
-        transportAction = new TransportMultiTermVectorsAction(transportService, clusterService, shardAction,
+        transportAction = new TransportMultiTermVectorsAction(transportService, clusterService, client,
             new ActionFilters(emptySet()), new Resolver()) {
             @Override
             protected void executeShardAction(final ActionListener<MultiTermVectorsResponse> listener,
diff --git a/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
index ce7ea7fbb6bd0..a10829fa47322 100644
--- a/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
+++ b/server/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
@@ -22,7 +22,6 @@
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
-import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistResponse;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.delete.DeleteResponse;
@@ -548,8 +547,9 @@ public void testDeleteAliases() throws Exception {
 
         assertAliasesVersionIncreases(indices, () -> admin().indices().prepareAliases().removeAlias(indices, aliases).get());
 
-        AliasesExistResponse response = admin().indices().prepareAliasesExist(aliases).get();
-        assertThat(response.exists(), equalTo(false));
+        for (String alias : aliases) {
+            assertTrue(admin().indices().prepareGetAliases(alias).get().getAliases().isEmpty());
+        }
 
         logger.info("--> creating index [foo_foo] and [bar_bar]");
         assertAcked(prepareCreate("foo_foo"));
@@ -564,9 +564,9 @@ public void testDeleteAliases() throws Exception {
                 "foo_foo",
                 () -> assertAcked(admin().indices().prepareAliases().addAliasAction(AliasActions.remove().index("foo*").alias("foo"))));
 
-        assertTrue(admin().indices().prepareAliasesExist("foo").get().exists());
-        assertFalse(admin().indices().prepareAliasesExist("foo").setIndices("foo_foo").get().exists());
-        assertTrue(admin().indices().prepareAliasesExist("foo").setIndices("bar_bar").get().exists());
+        assertFalse(admin().indices().prepareGetAliases("foo").get().getAliases().isEmpty());
+        assertTrue(admin().indices().prepareGetAliases("foo").setIndices("foo_foo").get().getAliases().isEmpty());
+        assertFalse(admin().indices().prepareGetAliases("foo").setIndices("bar_bar").get().getAliases().isEmpty());
         IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> admin().indices().prepareAliases()
                 .addAliasAction(AliasActions.remove().index("foo").alias("foo")).execute().actionGet());
         assertEquals("The provided expression [foo] matches an alias, specify the corresponding concrete indices instead.",
@@ -725,8 +725,7 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().get("foobar").get(0).getFilter(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getIndexRouting(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getSearchRouting(), nullValue());
-        AliasesExistResponse existsResponse = admin().indices().prepareAliasesExist("alias1").get();
-        assertThat(existsResponse.exists(), equalTo(true));
+        assertFalse(admin().indices().prepareGetAliases("alias1").get().getAliases().isEmpty());
 
         logger.info("--> getting all aliases that start with alias*");
         getResponse = admin().indices().prepareGetAliases("alias*").get();
@@ -743,8 +742,7 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().get("foobar").get(1).getFilter(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(1).getIndexRouting(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(1).getSearchRouting(), nullValue());
-        existsResponse = admin().indices().prepareAliasesExist("alias*").get();
-        assertThat(existsResponse.exists(), equalTo(true));
+        assertFalse(admin().indices().prepareGetAliases("alias*").get().getAliases().isEmpty());
 
 
         logger.info("--> creating aliases [bar, baz, foo]");
@@ -779,9 +777,10 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().get("bazbar").get(1).getFilter(), nullValue());
         assertThat(getResponse.getAliases().get("bazbar").get(1).getIndexRouting(), nullValue());
         assertThat(getResponse.getAliases().get("bazbar").get(1).getSearchRouting(), nullValue());
-        existsResponse = admin().indices().prepareAliasesExist("bar", "bac")
-                .addIndices("bazbar").get();
-        assertThat(existsResponse.exists(), equalTo(true));
+        assertFalse(admin().indices().prepareGetAliases("bar").get().getAliases().isEmpty());
+        assertFalse(admin().indices().prepareGetAliases("bac").get().getAliases().isEmpty());
+        assertFalse(admin().indices().prepareGetAliases("bar").addIndices("bazbar").get().getAliases().isEmpty());
+        assertFalse(admin().indices().prepareGetAliases("bac").addIndices("bazbar").get().getAliases().isEmpty());
 
         logger.info("--> getting *b* for index baz*");
         getResponse = admin().indices().prepareGetAliases("*b*").addIndices("baz*").get();
@@ -800,9 +799,7 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().get("bazbar").get(1).getFilter(), nullValue());
         assertThat(getResponse.getAliases().get("bazbar").get(1).getIndexRouting(), nullValue());
         assertThat(getResponse.getAliases().get("bazbar").get(1).getSearchRouting(), nullValue());
-        existsResponse = admin().indices().prepareAliasesExist("*b*")
-                .addIndices("baz*").get();
-        assertThat(existsResponse.exists(), equalTo(true));
+        assertFalse(admin().indices().prepareGetAliases("*b*").addIndices("baz*").get().getAliases().isEmpty());
 
         logger.info("--> getting *b* for index *bar");
         getResponse = admin().indices().prepareGetAliases("b*").addIndices("*bar").get();
@@ -826,9 +823,7 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().get("foobar").get(0).getFilter(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getIndexRouting(), equalTo("bla"));
         assertThat(getResponse.getAliases().get("foobar").get(0).getSearchRouting(), equalTo("bla"));
-        existsResponse = admin().indices().prepareAliasesExist("b*")
-                .addIndices("*bar").get();
-        assertThat(existsResponse.exists(), equalTo(true));
+        assertFalse(admin().indices().prepareGetAliases("b*").addIndices("*bar").get().getAliases().isEmpty());
 
         logger.info("--> getting f* for index *bar");
         getResponse = admin().indices().prepareGetAliases("f*").addIndices("*bar").get();
@@ -839,9 +834,7 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().get("foobar").get(0).getFilter(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getIndexRouting(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getSearchRouting(), nullValue());
-        existsResponse = admin().indices().prepareAliasesExist("f*")
-                .addIndices("*bar").get();
-        assertThat(existsResponse.exists(), equalTo(true));
+        assertFalse(admin().indices().prepareGetAliases("f*").addIndices("*bar").get().getAliases().isEmpty());
 
         // alias at work
         logger.info("--> getting f* for index *bac");
@@ -854,9 +847,7 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().get("foobar").get(0).getFilter(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getIndexRouting(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getSearchRouting(), nullValue());
-        existsResponse = admin().indices().prepareAliasesExist("foo")
-                .addIndices("*bac").get();
-        assertThat(existsResponse.exists(), equalTo(true));
+        assertFalse(admin().indices().prepareGetAliases("foo").addIndices("*bac").get().getAliases().isEmpty());
 
         logger.info("--> getting foo for index foobar");
         getResponse = admin().indices().prepareGetAliases("foo").addIndices("foobar").get();
@@ -867,9 +858,7 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().get("foobar").get(0).getFilter(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getIndexRouting(), nullValue());
         assertThat(getResponse.getAliases().get("foobar").get(0).getSearchRouting(), nullValue());
-        existsResponse = admin().indices().prepareAliasesExist("foo")
-                .addIndices("foobar").get();
-        assertThat(existsResponse.exists(), equalTo(true));
+        assertFalse(admin().indices().prepareGetAliases("foo").addIndices("foobar").get().getAliases().isEmpty());
 
         for (String aliasName : new String[]{null, "_all", "*"}) {
             logger.info("--> getting {} alias for index foobar", aliasName);
@@ -891,9 +880,7 @@ public void testIndicesGetAliases() throws Exception {
         assertThat(getResponse.getAliases().size(), equalTo(2));
         assertThat(getResponse.getAliases().get("foobar").size(), equalTo(4));
         assertThat(getResponse.getAliases().get("bazbar").size(), equalTo(2));
-        existsResponse = admin().indices().prepareAliasesExist("*")
-                .addIndices("*bac").get();
-        assertThat(existsResponse.exists(), equalTo(true));
+        assertFalse(admin().indices().prepareGetAliases("*").addIndices("*bac").get().getAliases().isEmpty());
 
         assertAcked(admin().indices().prepareAliases()
                 .removeAlias("foobar", "foo"));
@@ -902,8 +889,7 @@ public void testIndicesGetAliases() throws Exception {
         for (final ObjectObjectCursor<String, List<AliasMetaData>> entry : getResponse.getAliases()) {
             assertTrue(entry.value.isEmpty());
         }
-        existsResponse = admin().indices().prepareAliasesExist("foo").addIndices("foobar").get();
-        assertThat(existsResponse.exists(), equalTo(false));
+        assertTrue(admin().indices().prepareGetAliases("foo").addIndices("foobar").get().getAliases().isEmpty());
     }
 
     public void testGetAllAliasesWorks() {
@@ -1060,7 +1046,7 @@ public void testAliasesWithBlocks() {
                         () -> assertAcked(admin().indices().prepareAliases().addAlias("test", "alias1").addAlias("test", "alias2")));
                 assertAliasesVersionIncreases("test", () -> assertAcked(admin().indices().prepareAliases().removeAlias("test", "alias1")));
                 assertThat(admin().indices().prepareGetAliases("alias2").execute().actionGet().getAliases().get("test").size(), equalTo(1));
-                assertThat(admin().indices().prepareAliasesExist("alias2").get().exists(), equalTo(true));
+                assertFalse(admin().indices().prepareGetAliases("alias2").get().getAliases().isEmpty());
             } finally {
                 disableIndexBlock("test", block);
             }
@@ -1076,7 +1062,7 @@ public void testAliasesWithBlocks() {
                     "test",
                     () -> assertBlocked(admin().indices().prepareAliases().removeAlias("test", "alias2"), INDEX_READ_ONLY_BLOCK));
             assertThat(admin().indices().prepareGetAliases("alias2").execute().actionGet().getAliases().get("test").size(), equalTo(1));
-            assertThat(admin().indices().prepareAliasesExist("alias2").get().exists(), equalTo(true));
+            assertFalse(admin().indices().prepareGetAliases("alias2").get().getAliases().isEmpty());
 
         } finally {
             disableIndexBlock("test", SETTING_READ_ONLY);
@@ -1092,7 +1078,7 @@ public void testAliasesWithBlocks() {
                     "test",
                     () -> assertBlocked(admin().indices().prepareAliases().removeAlias("test", "alias2"), INDEX_METADATA_BLOCK));
             assertBlocked(admin().indices().prepareGetAliases("alias2"), INDEX_METADATA_BLOCK);
-            assertBlocked(admin().indices().prepareAliasesExist("alias2"), INDEX_METADATA_BLOCK);
+            assertBlocked(admin().indices().prepareGetAliases("alias2"), INDEX_METADATA_BLOCK);
 
         } finally {
             disableIndexBlock("test", SETTING_BLOCKS_METADATA);
@@ -1115,14 +1101,14 @@ public void testAliasActionRemoveIndex() throws InterruptedException, ExecutionE
                 iae.getMessage());
 
         assertAcked(client().admin().indices().prepareAliases().removeIndex("foo*"));
-        assertFalse(client().admin().indices().prepareExists("foo_foo").execute().actionGet().isExists());
-        assertTrue(admin().indices().prepareAliasesExist("foo").get().exists());
-        assertTrue(client().admin().indices().prepareExists("bar_bar").execute().actionGet().isExists());
-        assertTrue(admin().indices().prepareAliasesExist("foo").setIndices("bar_bar").get().exists());
+        assertFalse(indexExists("foo_foo"));
+        assertFalse(admin().indices().prepareGetAliases("foo").get().getAliases().isEmpty());
+        assertTrue(indexExists("bar_bar"));
+        assertFalse(admin().indices().prepareGetAliases("foo").setIndices("bar_bar").get().getAliases().isEmpty());
 
         assertAcked(client().admin().indices().prepareAliases().removeIndex("bar_bar"));
-        assertFalse(admin().indices().prepareAliasesExist("foo").get().exists());
-        assertFalse(client().admin().indices().prepareExists("bar_bar").execute().actionGet().isExists());
+        assertTrue(admin().indices().prepareGetAliases("foo").get().getAliases().isEmpty());
+        assertFalse(indexExists("bar_bar"));
     }
 
     public void testRemoveIndexAndReplaceWithAlias() throws InterruptedException, ExecutionException {
diff --git a/server/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java b/server/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java
index 9009b3592f895..e68480e58219d 100644
--- a/server/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java
+++ b/server/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.blocks;
 
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.index.IndexResponse;
@@ -41,42 +40,42 @@ public void testVerifyIndexAndClusterReadOnly() throws Exception {
         canCreateIndex("test1");
         canIndexDocument("test1");
         setIndexReadOnly("test1", "false");
-        canIndexExists("test1");
+        assertTrue(indexExists("test1"));
 
         // cluster.read_only = true: block write and metadata
         setClusterReadOnly(true);
         canNotCreateIndex("test2");
         // even if index has index.read_only = false
         canNotIndexDocument("test1");
-        canIndexExists("test1");
+        assertTrue(indexExists("test1"));
 
         // cluster.read_only = false: removes the block
         setClusterReadOnly(false);
         canCreateIndex("test2");
         canIndexDocument("test2");
         canIndexDocument("test1");
-        canIndexExists("test1");
+        assertTrue(indexExists("test1"));
 
 
         // newly created an index has no blocks
         canCreateIndex("ro");
         canIndexDocument("ro");
-        canIndexExists("ro");
+        assertTrue(indexExists("ro"));
 
         // adds index write and metadata block
         setIndexReadOnly( "ro", "true");
         canNotIndexDocument("ro");
-        canIndexExists("ro");
+        assertTrue(indexExists("ro"));
 
         // other indices not blocked
         canCreateIndex("rw");
         canIndexDocument("rw");
-        canIndexExists("rw");
+        assertTrue(indexExists("rw"));
 
         // blocks can be removed
         setIndexReadOnly("ro", "false");
         canIndexDocument("ro");
-        canIndexExists("ro");
+        assertTrue(indexExists("ro"));
     }
 
     public void testIndexReadWriteMetaDataBlocks() {
@@ -132,15 +131,6 @@ private void canNotIndexDocument(String index) {
         }
     }
 
-    private void canIndexExists(String index) {
-        try {
-            IndicesExistsResponse r = client().admin().indices().prepareExists(index).execute().actionGet();
-            assertThat(r, notNullValue());
-        } catch (ClusterBlockException e) {
-            fail();
-        }
-    }
-
     private void setIndexReadOnly(String index, Object value) {
         HashMap<String, Object> newSettings = new HashMap<>();
         newSettings.put(IndexMetaData.SETTING_READ_ONLY, value);
diff --git a/server/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java b/server/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java
index 53efeb393e4b4..720439768fabc 100644
--- a/server/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java
+++ b/server/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java
@@ -18,6 +18,12 @@
  */
 package org.elasticsearch.bwcompat;
 
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESIntegTestCase;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.DirectoryStream;
@@ -26,13 +32,6 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TestUtil;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.NodeEnvironment;
-import org.elasticsearch.test.ESIntegTestCase;
-
 import static org.hamcrest.Matchers.containsString;
 
 @LuceneTestCase.SuppressCodecs("*")
@@ -69,8 +68,8 @@ protected Settings prepareBackwardsDataDir(Path backwardsIndex) throws IOExcepti
             }
             throw new IllegalStateException(builder.toString());
         }
-        Path src = list[0].resolve(NodeEnvironment.NODES_FOLDER);
-        Path dest = dataDir.resolve(NodeEnvironment.NODES_FOLDER);
+        Path src = list[0].resolve("nodes");
+        Path dest = dataDir.resolve("nodes");
         assertTrue(Files.exists(src));
         Files.move(src, dest);
         assertFalse(Files.exists(src));
diff --git a/server/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java b/server/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java
index e7f32aa524972..ce65ed063bc08 100644
--- a/server/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java
+++ b/server/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.client;
 
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteAction;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotAction;
@@ -56,7 +56,7 @@ public abstract class AbstractClientHeadersTestCase extends ESTestCase {
             .put(ThreadContext.PREFIX + ".key2", "val 2")
             .build();
 
-    private static final Action<?>[] ACTIONS = new Action[] {
+    private static final ActionType<?>[] ACTIONS = new ActionType[] {
                 // client actions
                 GetAction.INSTANCE, SearchAction.INSTANCE, DeleteAction.INSTANCE, DeleteStoredScriptAction.INSTANCE,
                 IndexAction.INSTANCE,
@@ -92,7 +92,7 @@ public void tearDown() throws Exception {
         terminate(threadPool);
     }
 
-    protected abstract Client buildClient(Settings headersSettings, Action<?>[] testedActions);
+    protected abstract Client buildClient(Settings headersSettings, ActionType<?>[] testedActions);
 
 
     public void testActions() {
diff --git a/server/src/test/java/org/elasticsearch/client/OriginSettingClientTests.java b/server/src/test/java/org/elasticsearch/client/OriginSettingClientTests.java
index 8347cc1c1dd1b..f13b5e4c5c098 100644
--- a/server/src/test/java/org/elasticsearch/client/OriginSettingClientTests.java
+++ b/server/src/test/java/org/elasticsearch/client/OriginSettingClientTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.client;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -41,7 +41,7 @@ public void testSetsParentId() {
         NoOpClient mock = new NoOpClient(getTestName()) {
             @Override
             protected <Request extends ActionRequest, Response extends ActionResponse>
-                    void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+                    void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
                 assertEquals(origin, threadPool().getThreadContext().getTransient(ThreadContext.ACTION_ORIGIN_TRANSIENT_NAME));
                 super.doExecute(action, request, listener);
             }
diff --git a/server/src/test/java/org/elasticsearch/client/ParentTaskAssigningClientTests.java b/server/src/test/java/org/elasticsearch/client/ParentTaskAssigningClientTests.java
index eb1338ad78816..9f93bea56ca83 100644
--- a/server/src/test/java/org/elasticsearch/client/ParentTaskAssigningClientTests.java
+++ b/server/src/test/java/org/elasticsearch/client/ParentTaskAssigningClientTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.client;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -38,7 +38,7 @@ public void testSetsParentId() {
         NoOpClient mock = new NoOpClient(getTestName()) {
             @Override
             protected <Request extends ActionRequest, Response extends ActionResponse>
-            void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+            void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
                 assertEquals(parentTaskId[0], request.getParentTask());
                 super.doExecute(action, request, listener);
             }
diff --git a/server/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java b/server/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java
index 63c09f1e41dfd..31705bfa5f720 100644
--- a/server/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java
+++ b/server/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.client.node;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.support.ActionFilters;
@@ -39,7 +39,7 @@ public class NodeClientHeadersTests extends AbstractClientHeadersTestCase {
     private static final ActionFilters EMPTY_FILTERS = new ActionFilters(Collections.emptySet());
 
     @Override
-    protected Client buildClient(Settings headersSettings, Action[] testedActions) {
+    protected Client buildClient(Settings headersSettings, ActionType[] testedActions) {
         Settings settings = HEADER_SETTINGS;
         Actions actions = new Actions(settings, threadPool, testedActions);
         NodeClient client = new NodeClient(settings, threadPool);
@@ -47,10 +47,10 @@ protected Client buildClient(Settings headersSettings, Action[] testedActions) {
         return client;
     }
 
-    private static class Actions extends HashMap<Action, TransportAction> {
+    private static class Actions extends HashMap<ActionType, TransportAction> {
 
-        private Actions(Settings settings, ThreadPool threadPool, Action[] actions) {
-            for (Action action : actions) {
+        private Actions(Settings settings, ThreadPool threadPool, ActionType[] actions) {
+            for (ActionType action : actions) {
                 put(action, new InternalTransportAction(settings, action.name(), threadPool));
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
index f79fef74e917f..6de7da8c70991 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.common.UUIDs;
@@ -422,24 +423,24 @@ private static DiscoveryNodes createDiscoveryNodes(final int numNodes, final boo
         final int localNodeIndex = isLocalMaster ? 0 : randomIntBetween(1, numNodes - 1); // randomly assign the local node if not master
         for (int i = 0; i < numNodes; i++) {
             final String nodeId = NODE_ID_PREFIX + i;
-            Set<DiscoveryNode.Role> roles = new HashSet<>();
+            Set<DiscoveryNodeRole> roles = new HashSet<>();
             if (i == 0) {
                 // the master node
                 builder.masterNodeId(nodeId);
-                roles.add(DiscoveryNode.Role.MASTER);
+                roles.add(DiscoveryNodeRole.MASTER_ROLE);
             } else if (i == 1) {
                 // the alternate master node
-                roles.add(DiscoveryNode.Role.MASTER);
+                roles.add(DiscoveryNodeRole.MASTER_ROLE);
             } else if (i == 2) {
                 // we need at least one data node
-                roles.add(DiscoveryNode.Role.DATA);
+                roles.add(DiscoveryNodeRole.DATA_ROLE);
             } else {
                 // remaining nodes can be anything (except for master)
                 if (randomBoolean()) {
-                    roles.add(DiscoveryNode.Role.MASTER);
+                    roles.add(DiscoveryNodeRole.MASTER_ROLE);
                 }
                 if (randomBoolean()) {
-                    roles.add(DiscoveryNode.Role.DATA);
+                    roles.add(DiscoveryNodeRole.DATA_ROLE);
                 }
             }
             final DiscoveryNode node = newNode(nodeId, roles);
@@ -452,7 +453,7 @@ private static DiscoveryNodes createDiscoveryNodes(final int numNodes, final boo
     }
 
     // Create a new DiscoveryNode
-    private static DiscoveryNode newNode(final String nodeId, Set<DiscoveryNode.Role> roles) {
+    private static DiscoveryNode newNode(final String nodeId, Set<DiscoveryNodeRole> roles) {
         return new DiscoveryNode(nodeId, nodeId, nodeId, "host", "host_address", buildNewFakeTransportAddress(),
             Collections.emptyMap(), roles, Version.CURRENT);
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
index ad54d46585f7c..b1e452147c164 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
@@ -200,29 +198,4 @@ public void testAllocationDeciderOrder() {
             assertSame(decider.getClass(), expectedDeciders.get(idx++));
         }
     }
-
-    public void testPre63CustomsFiltering() {
-        final String whiteListedClusterCustom = randomFrom(ClusterModule.PRE_6_3_CLUSTER_CUSTOMS_WHITE_LIST);
-        final String whiteListedMetaDataCustom = randomFrom(ClusterModule.PRE_6_3_METADATA_CUSTOMS_WHITE_LIST);
-        final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
-            .putCustom(whiteListedClusterCustom, new RestoreInProgress.Builder().build())
-            .putCustom("other", new RestoreInProgress.Builder().build())
-            .metaData(MetaData.builder()
-                .putCustom(whiteListedMetaDataCustom, new RepositoriesMetaData(Collections.emptyList()))
-                .putCustom("other", new RepositoriesMetaData(Collections.emptyList()))
-                .build())
-            .build();
-
-        assertNotNull(clusterState.custom(whiteListedClusterCustom));
-        assertNotNull(clusterState.custom("other"));
-        assertNotNull(clusterState.metaData().custom(whiteListedMetaDataCustom));
-        assertNotNull(clusterState.metaData().custom("other"));
-
-        final ClusterState fixedClusterState = ClusterModule.filterCustomsForPre63Clients(clusterState);
-
-        assertNotNull(fixedClusterState.custom(whiteListedClusterCustom));
-        assertNull(fixedClusterState.custom("other"));
-        assertNotNull(fixedClusterState.metaData().custom(whiteListedMetaDataCustom));
-        assertNull(fixedClusterState.metaData().custom("other"));
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
index 313bf1d47c771..3ac34dc66fb7d 100644
--- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
@@ -32,11 +32,6 @@
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -47,15 +42,21 @@
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotInfoTests;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Collections;
@@ -719,7 +720,8 @@ public ClusterState.Custom randomCreate(String name) {
                                 Collections.emptyList(),
                                 Math.abs(randomLong()),
                                 (long) randomIntBetween(0, 1000),
-                                ImmutableOpenMap.of()));
+                                ImmutableOpenMap.of(),
+                                SnapshotInfoTests.randomUserMetadata()));
                     case 1:
                         return new RestoreInProgress.Builder().add(
                             new RestoreInProgress.Entry(
diff --git a/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
index fcccaf6c0f021..55eae6fc0e95f 100644
--- a/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
@@ -119,7 +119,7 @@ public void testFillShardLevelInfo() {
         ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
         ImmutableOpenMap.Builder<ShardRouting, String> routingToPath = ImmutableOpenMap.builder();
         ClusterState state = ClusterState.builder(new ClusterName("blarg")).version(0).build();
-        InternalClusterInfoService.buildShardLevelInfo(logger, stats, shardSizes, routingToPath, state);
+        InternalClusterInfoService.buildShardLevelInfo(logger, stats, shardSizes, routingToPath);
         assertEquals(2, shardSizes.size());
         assertTrue(shardSizes.containsKey(ClusterInfo.shardIdentifierFromRouting(test_0)));
         assertTrue(shardSizes.containsKey(ClusterInfo.shardIdentifierFromRouting(test_1)));
diff --git a/server/src/test/java/org/elasticsearch/cluster/FeatureAwareTests.java b/server/src/test/java/org/elasticsearch/cluster/FeatureAwareTests.java
index b25d8ced1806d..50bddc8677b62 100644
--- a/server/src/test/java/org/elasticsearch/cluster/FeatureAwareTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/FeatureAwareTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.cluster;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.ClusterState.FeatureAware;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -33,7 +32,6 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
-import java.util.HashSet;
 import java.util.Optional;
 
 import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
@@ -136,21 +134,13 @@ public void testFeature() {
         final Version afterVersion = randomVersionBetween(random(), version, Version.CURRENT);
         final Custom custom = new RequiredFeatureCustom(version);
         {
-            // the feature is present and the client is not a transport client
+            // the feature is present
             final BytesStreamOutput out = new BytesStreamOutput();
             out.setVersion(afterVersion);
             assertTrue(custom.getRequiredFeature().isPresent());
             out.setFeatures(Collections.singleton(custom.getRequiredFeature().get()));
             assertTrue(FeatureAware.shouldSerialize(out, custom));
         }
-        {
-            // the feature is present and the client is a transport client
-            final BytesStreamOutput out = new BytesStreamOutput();
-            out.setVersion(afterVersion);
-            assertTrue(custom.getRequiredFeature().isPresent());
-            out.setFeatures(new HashSet<>(Arrays.asList(custom.getRequiredFeature().get(), TransportClient.TRANSPORT_CLIENT_FEATURE)));
-            assertTrue(FeatureAware.shouldSerialize(out, custom));
-        }
     }
 
     public void testMissingFeature() {
@@ -158,18 +148,11 @@ public void testMissingFeature() {
         final Version afterVersion = randomVersionBetween(random(), version, Version.CURRENT);
         final Custom custom = new RequiredFeatureCustom(version);
         {
-            // the feature is missing but we should serialize it anyway because the client is not a transport client
+            // the feature is missing but we should serialize it anyway
             final BytesStreamOutput out = new BytesStreamOutput();
             out.setVersion(afterVersion);
             assertTrue(FeatureAware.shouldSerialize(out, custom));
         }
-        {
-            // the feature is missing and we should not serialize it because the client is a transport client
-            final BytesStreamOutput out = new BytesStreamOutput();
-            out.setVersion(afterVersion);
-            out.setFeatures(Collections.singleton(TransportClient.TRANSPORT_CLIENT_FEATURE));
-            assertFalse(FeatureAware.shouldSerialize(out, custom));
-        }
     }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
index cb1443bdf3765..a78bd4a67ece4 100644
--- a/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
@@ -41,7 +41,6 @@
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.disruption.NetworkDisruption;
 import org.elasticsearch.test.disruption.NetworkDisruption.TwoPartitions;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 
 import java.util.ArrayList;
@@ -62,8 +61,7 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false)
-@TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.cluster.coordination:TRACE")
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class MinimumMasterNodesIT extends ESIntegTestCase {
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java b/server/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java
index 78b1fef91f277..1555e9fd4cfd2 100644
--- a/server/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java
@@ -163,9 +163,7 @@ public void testNoMasterActions() throws Exception {
         bulkRequestBuilder.setTimeout(timeout);
         checkWriteAction(bulkRequestBuilder);
 
-        disruptionScheme.stopDisrupting();
-
-        client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForNodes("3").execute().actionGet();
+        internalCluster().clearDisruptionScheme(true);
     }
 
     void checkUpdateAction(boolean autoCreateIndex, TimeValue timeout, ActionRequestBuilder<?, ?> builder) {
@@ -192,7 +190,6 @@ void checkWriteAction(ActionRequestBuilder<?, ?> builder) {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/39688")
     public void testNoMasterActionsWriteMasterBlock() throws Exception {
         Settings settings = Settings.builder()
             .put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), false)
@@ -264,8 +261,6 @@ public void testNoMasterActionsWriteMasterBlock() throws Exception {
             assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
         }
 
-        disruptionScheme.stopDisrupting();
-
-        client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForNodes("3").get();
+        internalCluster().clearDisruptionScheme(true);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
index 193cde3180db8..151f4d9268523 100644
--- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.coordination.DeterministicTaskQueue;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.UUIDs;
@@ -51,7 +52,6 @@
 import org.junit.Before;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -80,7 +80,7 @@ public class NodeConnectionsServiceTests extends ESTestCase {
     private List<DiscoveryNode> generateNodes() {
         List<DiscoveryNode> nodes = new ArrayList<>();
         for (int i = randomIntBetween(20, 50); i > 0; i--) {
-            Set<DiscoveryNode.Role> roles = new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values())));
+            Set<DiscoveryNodeRole> roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES));
             nodes.add(new DiscoveryNode("node_" + i, "" + i, buildNewFakeTransportAddress(), Collections.emptyMap(),
                 roles, Version.CURRENT));
         }
@@ -255,7 +255,7 @@ public void testOnlyBlocksOnConnectionsToNewNodes() throws Exception {
 
             // once the connection is unblocked we successfully connect to it.
             nodeConnectionBlocks.clear();
-            connectionBarrier.await(0, TimeUnit.SECONDS);
+            connectionBarrier.await(10, TimeUnit.SECONDS);
             future3.actionGet();
             assertConnectedExactlyToNodes(nodes01);
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java b/server/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
index abfa0f692239c..27032b1a50c61 100644
--- a/server/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
@@ -51,7 +51,6 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.hamcrest.CollectionAssertions;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.junit.Before;
@@ -254,7 +253,6 @@ public void testLargeClusterStatePublishing() throws Exception {
         }
     }
 
-    @TestLogging("org.elasticsearch.action.admin.indices.close:DEBUG,org.elasticsearch.cluster.metadata:DEBUG")
     public void testIndicesOptions() throws Exception {
         ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().clear().setMetaData(true).setIndices("f*")
                 .get();
diff --git a/server/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java
index d5a986b8affff..82744aa86f93b 100644
--- a/server/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java
@@ -20,9 +20,12 @@
 package org.elasticsearch.cluster;
 
 import org.elasticsearch.action.UnavailableShardsException;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.Requests;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -35,13 +38,16 @@
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.hamcrest.Matchers.equalTo;
 
-@ClusterScope(scope= Scope.TEST, numDataNodes =0)
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
 public class SimpleDataNodesIT extends ESIntegTestCase {
-    public void testDataNodes() throws Exception {
+
+    private static final String SOURCE = "{\"type1\":{\"id\":\"1\",\"name\":\"test\"}}";
+
+    public void testIndexingBeforeAndAfterDataNodesStart() {
         internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).build());
         client().admin().indices().create(createIndexRequest("test").waitForActiveShards(ActiveShardCount.NONE)).actionGet();
         try {
-            client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test"), XContentType.JSON)
+            client().index(Requests.indexRequest("test").id("1").source(SOURCE, XContentType.JSON)
                 .timeout(timeValueSeconds(1))).actionGet();
             fail("no allocation should happen");
         } catch (UnavailableShardsException e) {
@@ -54,7 +60,7 @@ public void testDataNodes() throws Exception {
 
         // still no shard should be allocated
         try {
-            client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test"), XContentType.JSON)
+            client().index(Requests.indexRequest("test").id("1").source(SOURCE, XContentType.JSON)
                 .timeout(timeValueSeconds(1))).actionGet();
             fail("no allocation should happen");
         } catch (UnavailableShardsException e) {
@@ -66,13 +72,43 @@ public void testDataNodes() throws Exception {
         assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("3")
             .setLocal(true).execute().actionGet().isTimedOut(), equalTo(false));
 
-        IndexResponse indexResponse = client().index(Requests.indexRequest("test").type("type1").id("1")
-            .source(source("1", "test"), XContentType.JSON)).actionGet();
+        IndexResponse indexResponse = client().index(Requests.indexRequest("test").id("1")
+            .source(SOURCE, XContentType.JSON)).actionGet();
         assertThat(indexResponse.getId(), equalTo("1"));
-        assertThat(indexResponse.getType(), equalTo("type1"));
     }
 
-    private String source(String id, String nameValue) {
-        return "{ \"type1\" : { \"id\" : \"" + id + "\", \"name\" : \"" + nameValue + "\" } }";
+    public void testShardsAllocatedAfterDataNodesStart() {
+        internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).build());
+        client().admin().indices().create(createIndexRequest("test")
+            .settings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).waitForActiveShards(ActiveShardCount.NONE))
+            .actionGet();
+        final ClusterHealthResponse healthResponse1 = client().admin().cluster().prepareHealth()
+            .setWaitForEvents(Priority.LANGUID).execute().actionGet();
+        assertThat(healthResponse1.isTimedOut(), equalTo(false));
+        assertThat(healthResponse1.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); // TODO should be RED, see #41073
+        assertThat(healthResponse1.getActiveShards(), equalTo(0));
+
+        internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), true).build());
+
+        assertThat(client().admin().cluster().prepareHealth()
+            .setWaitForEvents(Priority.LANGUID).setWaitForNodes("2").setWaitForGreenStatus().execute().actionGet().isTimedOut(),
+            equalTo(false));
+    }
+
+    public void testAutoExpandReplicasAdjustedWhenDataNodeJoins() {
+        internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).build());
+        client().admin().indices().create(createIndexRequest("test")
+            .settings(Settings.builder().put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-all"))
+            .waitForActiveShards(ActiveShardCount.NONE))
+            .actionGet();
+        final ClusterHealthResponse healthResponse1 = client().admin().cluster().prepareHealth()
+            .setWaitForEvents(Priority.LANGUID).execute().actionGet();
+        assertThat(healthResponse1.isTimedOut(), equalTo(false));
+        assertThat(healthResponse1.getStatus(), equalTo(ClusterHealthStatus.YELLOW)); // TODO should be RED, see #41073
+        assertThat(healthResponse1.getActiveShards(), equalTo(0));
+
+        internalCluster().startNode();
+        internalCluster().startNode();
+        client().admin().cluster().prepareReroute().setRetryFailed(true).get();
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/SnapshotsInProgressTests.java b/server/src/test/java/org/elasticsearch/cluster/SnapshotsInProgressTests.java
index eac06c786a26b..087366d1188c8 100644
--- a/server/src/test/java/org/elasticsearch/cluster/SnapshotsInProgressTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/SnapshotsInProgressTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotInfoTests;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
@@ -65,7 +66,7 @@ public void testWaitingIndices() {
         // test no waiting shards in an index
         shards.put(new ShardId(idx3Name, idx3UUID, 0), new ShardSnapshotStatus(randomAlphaOfLength(2), randomNonWaitingState(), ""));
         Entry entry = new Entry(snapshot, randomBoolean(), randomBoolean(), State.INIT,
-                                indices, System.currentTimeMillis(), randomLong(), shards.build());
+                                indices, System.currentTimeMillis(), randomLong(), shards.build(), SnapshotInfoTests.randomUserMetadata());
 
         ImmutableOpenMap<String, List<ShardId>> waitingIndices = entry.waitingIndices();
         assertEquals(2, waitingIndices.get(idx1Name).size());
diff --git a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
index 38b9579eff046..cad4f51d7eac0 100644
--- a/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.InternalTestCluster;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.io.IOException;
 
@@ -39,8 +38,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 
-@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false)
-@TestLogging("_root:DEBUG,org.elasticsearch.action.admin.cluster.state:TRACE")
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class SpecificMasterNodesIT extends ESIntegTestCase {
 
     public void testSimpleOnlyMasterNodeElection() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
index 7eb995e04f3fa..3ccff88b9f987 100644
--- a/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
@@ -32,7 +32,7 @@
 import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
-import org.elasticsearch.cluster.routing.RoutingService;
+import org.elasticsearch.cluster.routing.RerouteService;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
@@ -91,8 +91,8 @@ public class ShardStateActionTests extends ESTestCase {
 
     private static class TestShardStateAction extends ShardStateAction {
         TestShardStateAction(ClusterService clusterService, TransportService transportService,
-                             AllocationService allocationService, RoutingService routingService) {
-            super(clusterService, transportService, allocationService, routingService, THREAD_POOL);
+                             AllocationService allocationService, RerouteService rerouteService) {
+            super(clusterService, transportService, allocationService, rerouteService, THREAD_POOL);
         }
 
         private Runnable onBeforeWaitForNewMasterAndRetry;
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterBootstrapServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterBootstrapServiceTests.java
index 4dd3e3e33c39f..6a7562ab8038a 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterBootstrapServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterBootstrapServiceTests.java
@@ -21,7 +21,7 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNode.Role;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.node.Node;
@@ -33,6 +33,7 @@
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
@@ -43,7 +44,6 @@
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
-import static java.util.Collections.singleton;
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.BOOTSTRAP_PLACEHOLDER_PREFIX;
 import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING;
@@ -88,8 +88,13 @@ protected void onSendRequest(long requestId, String action, TransportRequest req
     }
 
     private DiscoveryNode newDiscoveryNode(String nodeName) {
-        return new DiscoveryNode(nodeName, randomAlphaOfLength(10), buildNewFakeTransportAddress(), emptyMap(), singleton(Role.MASTER),
-            Version.CURRENT);
+        return new DiscoveryNode(
+                nodeName,
+                randomAlphaOfLength(10),
+                buildNewFakeTransportAddress(),
+                emptyMap(),
+                Set.of(DiscoveryNodeRole.MASTER_ROLE),
+                Version.CURRENT);
     }
 
     public void testBootstrapsAutomaticallyWithDefaultConfiguration() {
@@ -390,10 +395,22 @@ public void testCancelsBootstrapIfNodeMatchesMultipleRequirements() {
         clusterBootstrapService.onFoundPeersUpdated();
         deterministicTaskQueue.runAllTasks();
 
-        discoveredNodes.set(Stream.of(new DiscoveryNode(otherNode1.getName(), randomAlphaOfLength(10), buildNewFakeTransportAddress(),
-                emptyMap(), singleton(Role.MASTER), Version.CURRENT),
-            new DiscoveryNode("yet-another-node", randomAlphaOfLength(10), otherNode1.getAddress(), emptyMap(), singleton(Role.MASTER),
-                Version.CURRENT)).collect(Collectors.toList()));
+        discoveredNodes.set(Stream.of(
+                new DiscoveryNode(
+                        otherNode1.getName(),
+                        randomAlphaOfLength(10),
+                        buildNewFakeTransportAddress(),
+                        emptyMap(),
+                        Set.of(DiscoveryNodeRole.MASTER_ROLE),
+                        Version.CURRENT),
+                new DiscoveryNode(
+                        "yet-another-node",
+                        randomAlphaOfLength(10),
+                        otherNode1.getAddress(),
+                        emptyMap(),
+                        Set.of(DiscoveryNodeRole.MASTER_ROLE),
+                        Version.CURRENT))
+                .collect(Collectors.toList()));
 
         clusterBootstrapService.onFoundPeersUpdated();
         deterministicTaskQueue.runAllTasks();
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java
index 16740b0761ff5..850a1ec0b7abd 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -44,9 +45,12 @@
 import static org.elasticsearch.node.Node.NODE_NAME_SETTING;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.isOneOf;
+import static org.hamcrest.Matchers.oneOf;
 
 public class ClusterFormationFailureHelperTests extends ESTestCase {
+
+    private static final ElectionStrategy electionStrategy = ElectionStrategy.DEFAULT_INSTANCE;
+
     public void testScheduling() {
         final long expectedDelayMillis;
         final Settings.Builder settingsBuilder = Settings.builder();
@@ -72,7 +76,7 @@ public void testScheduling() {
         final ClusterFormationFailureHelper clusterFormationFailureHelper = new ClusterFormationFailureHelper(settingsBuilder.build(),
             () -> {
                 warningCount.incrementAndGet();
-                return new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), emptyList(), 0L);
+                return new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), emptyList(), 0L, electionStrategy);
             },
             deterministicTaskQueue.getThreadPool(), logLastFailedJoinAttemptWarningCount::incrementAndGet);
 
@@ -140,17 +144,20 @@ public void testDescriptionOnMasterIneligibleNodes() {
         final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
             .version(12L).nodes(DiscoveryNodes.builder().add(localNode).localNodeId(localNode.getId())).build();
 
-        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), emptyList(), 15L).getDescription(),
+        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), emptyList(), 15L, electionStrategy)
+                .getDescription(),
             is("master not discovered yet: have discovered []; discovery will continue using [] from hosts providers " +
                 "and [] from last-known cluster state; node term 15, last-accepted version 12 in term 0"));
 
         final TransportAddress otherAddress = buildNewFakeTransportAddress();
-        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, singletonList(otherAddress), emptyList(), 16L).getDescription(),
+        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, singletonList(otherAddress), emptyList(), 16L, electionStrategy)
+                .getDescription(),
             is("master not discovered yet: have discovered []; discovery will continue using [" + otherAddress +
                 "] from hosts providers and [] from last-known cluster state; node term 16, last-accepted version 12 in term 0"));
 
         final DiscoveryNode otherNode = new DiscoveryNode("other", buildNewFakeTransportAddress(), Version.CURRENT);
-        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), singletonList(otherNode), 17L).getDescription(),
+        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), singletonList(otherNode), 17L, electionStrategy)
+                .getDescription(),
             is("master not discovered yet: have discovered [" + otherNode + "]; discovery will continue using [] from hosts providers " +
                 "and [] from last-known cluster state; node term 17, last-accepted version 12 in term 0"));
     }
@@ -162,29 +169,31 @@ public void testDescriptionBeforeBootstrapping() {
             .metaData(MetaData.builder().coordinationMetaData(CoordinationMetaData.builder().term(4L).build()))
             .nodes(DiscoveryNodes.builder().add(localNode).localNodeId(localNode.getId())).build();
 
-        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), emptyList(), 1L).getDescription(),
-            is("master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and " +
+        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), emptyList(), 1L, electionStrategy).getDescription(),
+            is("master not discovered yet, this node has not previously joined a bootstrapped cluster, and " +
                 "[cluster.initial_master_nodes] is empty on this node: have discovered []; " +
                 "discovery will continue using [] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 1, last-accepted version 7 in term 4"));
 
         final TransportAddress otherAddress = buildNewFakeTransportAddress();
-        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, singletonList(otherAddress), emptyList(), 2L).getDescription(),
-            is("master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and " +
+        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, singletonList(otherAddress), emptyList(), 2L, electionStrategy)
+                .getDescription(),
+            is("master not discovered yet, this node has not previously joined a bootstrapped cluster, and " +
                 "[cluster.initial_master_nodes] is empty on this node: have discovered []; " +
                 "discovery will continue using [" + otherAddress + "] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 2, last-accepted version 7 in term 4"));
 
         final DiscoveryNode otherNode = new DiscoveryNode("other", buildNewFakeTransportAddress(), Version.CURRENT);
-        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), singletonList(otherNode), 3L).getDescription(),
-            is("master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and " +
+        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), singletonList(otherNode), 3L, electionStrategy)
+                .getDescription(),
+            is("master not discovered yet, this node has not previously joined a bootstrapped cluster, and " +
                 "[cluster.initial_master_nodes] is empty on this node: have discovered [" + otherNode + "]; " +
                 "discovery will continue using [] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 3, last-accepted version 7 in term 4"));
 
         assertThat(new ClusterFormationState(Settings.builder().putList(INITIAL_MASTER_NODES_SETTING.getKey(), "other").build(),
-                clusterState, emptyList(), emptyList(), 4L).getDescription(),
-            is("master not discovered yet, this node has not previously joined a bootstrapped (v7+) cluster, and " +
+                clusterState, emptyList(), emptyList(), 4L, electionStrategy).getDescription(),
+            is("master not discovered yet, this node has not previously joined a bootstrapped cluster, and " +
                 "this node must discover master-eligible nodes [other] to bootstrap a cluster: have discovered []; " +
                 "discovery will continue using [] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 4, last-accepted version 7 in term 4"));
@@ -213,28 +222,31 @@ public void testDescriptionAfterDetachCluster() {
         final ClusterState clusterState = state(localNode,
                 VotingConfiguration.MUST_JOIN_ELECTED_MASTER.getNodeIds().toArray(new String[0]));
 
-        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), emptyList(), 0L).getDescription(),
+        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), emptyList(), 0L, electionStrategy).getDescription(),
                 is("master not discovered yet and this node was detached from its previous cluster, " +
                         "have discovered []; " +
                         "discovery will continue using [] from hosts providers and [" + localNode +
                         "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
         final TransportAddress otherAddress = buildNewFakeTransportAddress();
-        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, singletonList(otherAddress), emptyList(), 0L).getDescription(),
+        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, singletonList(otherAddress), emptyList(), 0L, electionStrategy)
+                .getDescription(),
                 is("master not discovered yet and this node was detached from its previous cluster, " +
                         "have discovered []; " +
                         "discovery will continue using [" + otherAddress + "] from hosts providers and [" + localNode +
                         "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
         final DiscoveryNode otherNode = new DiscoveryNode("otherNode", buildNewFakeTransportAddress(), Version.CURRENT);
-        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), singletonList(otherNode), 0L).getDescription(),
+        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), singletonList(otherNode), 0L, electionStrategy)
+                .getDescription(),
                 is("master not discovered yet and this node was detached from its previous cluster, " +
                         "have discovered [" + otherNode + "]; " +
                         "discovery will continue using [] from hosts providers and [" + localNode +
                         "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
         final DiscoveryNode yetAnotherNode = new DiscoveryNode("yetAnotherNode", buildNewFakeTransportAddress(), Version.CURRENT);
-        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), singletonList(yetAnotherNode), 0L).getDescription(),
+        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), singletonList(yetAnotherNode), 0L, electionStrategy)
+                .getDescription(),
                 is("master not discovered yet and this node was detached from its previous cluster, " +
                         "have discovered [" + yetAnotherNode + "]; " +
                         "discovery will continue using [] from hosts providers and [" + localNode +
@@ -247,104 +259,109 @@ public void testDescriptionAfterBootstrapping() {
 
         final ClusterState clusterState = state(localNode, "otherNode");
 
-        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), emptyList(), 0L).getDescription(),
+        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), emptyList(), 0L, electionStrategy).getDescription(),
             is("master not discovered or elected yet, an election requires a node with id [otherNode], " +
                 "have discovered [] which is not a quorum; " +
                 "discovery will continue using [] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
         final TransportAddress otherAddress = buildNewFakeTransportAddress();
-        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, singletonList(otherAddress), emptyList(), 0L).getDescription(),
+        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, singletonList(otherAddress), emptyList(), 0L, electionStrategy)
+                .getDescription(),
             is("master not discovered or elected yet, an election requires a node with id [otherNode], " +
                 "have discovered [] which is not a quorum; " +
                 "discovery will continue using [" + otherAddress + "] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
         final DiscoveryNode otherNode = new DiscoveryNode("otherNode", buildNewFakeTransportAddress(), Version.CURRENT);
-        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), singletonList(otherNode), 0L).getDescription(),
+        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), singletonList(otherNode), 0L, electionStrategy)
+                .getDescription(),
             is("master not discovered or elected yet, an election requires a node with id [otherNode], " +
                 "have discovered [" + otherNode + "] which is a quorum; " +
                 "discovery will continue using [] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
         final DiscoveryNode yetAnotherNode = new DiscoveryNode("yetAnotherNode", buildNewFakeTransportAddress(), Version.CURRENT);
-        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), singletonList(yetAnotherNode), 0L).getDescription(),
+        assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), singletonList(yetAnotherNode), 0L, electionStrategy)
+                .getDescription(),
             is("master not discovered or elected yet, an election requires a node with id [otherNode], " +
                 "have discovered [" + yetAnotherNode + "] which is not a quorum; " +
                 "discovery will continue using [] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
-        assertThat(new ClusterFormationState(Settings.EMPTY, state(localNode, "n1", "n2"), emptyList(), emptyList(), 0L).getDescription(),
+        assertThat(new ClusterFormationState(Settings.EMPTY, state(localNode, "n1", "n2"), emptyList(), emptyList(), 0L, electionStrategy)
+                .getDescription(),
             is("master not discovered or elected yet, an election requires two nodes with ids [n1, n2], " +
                 "have discovered [] which is not a quorum; " +
                 "discovery will continue using [] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
-        assertThat(new ClusterFormationState(Settings.EMPTY, state(localNode, "n1", "n2", "n3"), emptyList(), emptyList(), 0L)
-                .getDescription(),
+        assertThat(new ClusterFormationState(Settings.EMPTY, state(localNode, "n1", "n2", "n3"), emptyList(), emptyList(), 0L,
+                electionStrategy).getDescription(),
             is("master not discovered or elected yet, an election requires at least 2 nodes with ids from [n1, n2, n3], " +
                 "have discovered [] which is not a quorum; " +
                 "discovery will continue using [] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
         assertThat(new ClusterFormationState(Settings.EMPTY, state(localNode, "n1", "n2", BOOTSTRAP_PLACEHOLDER_PREFIX + "n3"),
-                emptyList(), emptyList(), 0L).getDescription(),
+                emptyList(), emptyList(), 0L, electionStrategy).getDescription(),
             is("master not discovered or elected yet, an election requires 2 nodes with ids [n1, n2], " +
                 "have discovered [] which is not a quorum; " +
                 "discovery will continue using [] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
-        assertThat(new ClusterFormationState(Settings.EMPTY, state(localNode, "n1", "n2", "n3", "n4"), emptyList(), emptyList(), 0L)
-                .getDescription(),
+        assertThat(new ClusterFormationState(Settings.EMPTY, state(localNode, "n1", "n2", "n3", "n4"), emptyList(), emptyList(), 0L,
+                electionStrategy).getDescription(),
             is("master not discovered or elected yet, an election requires at least 3 nodes with ids from [n1, n2, n3, n4], " +
                 "have discovered [] which is not a quorum; " +
                 "discovery will continue using [] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
-        assertThat(new ClusterFormationState(Settings.EMPTY, state(localNode, "n1", "n2", "n3", "n4", "n5"), emptyList(), emptyList(), 0L)
-                .getDescription(),
+        assertThat(new ClusterFormationState(Settings.EMPTY, state(localNode, "n1", "n2", "n3", "n4", "n5"), emptyList(), emptyList(), 0L,
+                electionStrategy).getDescription(),
             is("master not discovered or elected yet, an election requires at least 3 nodes with ids from [n1, n2, n3, n4, n5], " +
                 "have discovered [] which is not a quorum; " +
                 "discovery will continue using [] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
         assertThat(new ClusterFormationState(Settings.EMPTY, state(localNode, "n1", "n2", "n3", "n4", BOOTSTRAP_PLACEHOLDER_PREFIX + "n5"),
-                emptyList(), emptyList(), 0L).getDescription(),
+                emptyList(), emptyList(), 0L, electionStrategy).getDescription(),
             is("master not discovered or elected yet, an election requires at least 3 nodes with ids from [n1, n2, n3, n4], " +
                 "have discovered [] which is not a quorum; " +
                 "discovery will continue using [] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
         assertThat(new ClusterFormationState(Settings.EMPTY, state(localNode, "n1", "n2", "n3",
-            BOOTSTRAP_PLACEHOLDER_PREFIX + "n4", BOOTSTRAP_PLACEHOLDER_PREFIX + "n5"), emptyList(), emptyList(), 0L).getDescription(),
+            BOOTSTRAP_PLACEHOLDER_PREFIX + "n4", BOOTSTRAP_PLACEHOLDER_PREFIX + "n5"), emptyList(), emptyList(), 0L, electionStrategy)
+                .getDescription(),
             is("master not discovered or elected yet, an election requires 3 nodes with ids [n1, n2, n3], " +
                 "have discovered [] which is not a quorum; " +
                 "discovery will continue using [] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
         assertThat(new ClusterFormationState(Settings.EMPTY, state(localNode, new String[]{"n1"}, new String[]{"n1"}), emptyList(),
-                emptyList(), 0L).getDescription(),
+                emptyList(), 0L, electionStrategy).getDescription(),
             is("master not discovered or elected yet, an election requires a node with id [n1], " +
                 "have discovered [] which is not a quorum; " +
                 "discovery will continue using [] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
         assertThat(new ClusterFormationState(Settings.EMPTY, state(localNode, new String[]{"n1"}, new String[]{"n2"}), emptyList(),
-                emptyList(), 0L).getDescription(),
+                emptyList(), 0L, electionStrategy).getDescription(),
             is("master not discovered or elected yet, an election requires a node with id [n1] and a node with id [n2], " +
                 "have discovered [] which is not a quorum; " +
                 "discovery will continue using [] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
         assertThat(new ClusterFormationState(Settings.EMPTY, state(localNode, new String[]{"n1"}, new String[]{"n2", "n3"}), emptyList(),
-                emptyList(), 0L).getDescription(),
+                emptyList(), 0L, electionStrategy).getDescription(),
             is("master not discovered or elected yet, an election requires a node with id [n1] and two nodes with ids [n2, n3], " +
                 "have discovered [] which is not a quorum; " +
                 "discovery will continue using [] from hosts providers and [" + localNode +
                 "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
 
         assertThat(new ClusterFormationState(Settings.EMPTY, state(localNode, new String[]{"n1"}, new String[]{"n2", "n3", "n4"}),
-                emptyList(), emptyList(), 0L).getDescription(),
+                emptyList(), emptyList(), 0L, electionStrategy).getDescription(),
             is("master not discovered or elected yet, an election requires a node with id [n1] and " +
                 "at least 2 nodes with ids from [n2, n3, n4], " +
                 "have discovered [] which is not a quorum; " +
@@ -353,8 +370,8 @@ public void testDescriptionAfterBootstrapping() {
 
         final DiscoveryNode otherMasterNode = new DiscoveryNode("other-master", buildNewFakeTransportAddress(), Version.CURRENT);
         final DiscoveryNode otherNonMasterNode = new DiscoveryNode("other-non-master", buildNewFakeTransportAddress(), emptyMap(),
-            new HashSet<>(randomSubsetOf(Arrays.stream(DiscoveryNode.Role.values())
-                .filter(r -> r != DiscoveryNode.Role.MASTER).collect(Collectors.toList()))),
+            new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES).stream()
+                .filter(r -> r != DiscoveryNodeRole.MASTER_ROLE).collect(Collectors.toList())),
             Version.CURRENT);
 
         String[] configNodeIds = new String[]{"n1", "n2"};
@@ -364,18 +381,19 @@ public void testDescriptionAfterBootstrapping() {
                 .lastAcceptedConfiguration(config(configNodeIds))
                 .lastCommittedConfiguration(config(configNodeIds)).build())).build();
 
-        assertThat(new ClusterFormationState(Settings.EMPTY, stateWithOtherNodes, emptyList(), emptyList(), 0L).getDescription(), isOneOf(
+        assertThat(
+            new ClusterFormationState(Settings.EMPTY, stateWithOtherNodes, emptyList(), emptyList(), 0L, electionStrategy).getDescription(),
 
             // nodes from last-known cluster state could be in either order
-
-            "master not discovered or elected yet, an election requires two nodes with ids [n1, n2], " +
-                "have discovered [] which is not a quorum; " +
-                "discovery will continue using [] from hosts providers and [" + localNode + ", " + otherMasterNode +
-                "] from last-known cluster state; node term 0, last-accepted version 0 in term 0",
-
-            "master not discovered or elected yet, an election requires two nodes with ids [n1, n2], " +
-                "have discovered [] which is not a quorum; " +
-                "discovery will continue using [] from hosts providers and [" + otherMasterNode + ", " + localNode +
-                "] from last-known cluster state; node term 0, last-accepted version 0 in term 0"));
+            is(oneOf(
+                "master not discovered or elected yet, an election requires two nodes with ids [n1, n2], " +
+                    "have discovered [] which is not a quorum; " +
+                    "discovery will continue using [] from hosts providers and [" + localNode + ", " + otherMasterNode +
+                    "] from last-known cluster state; node term 0, last-accepted version 0 in term 0",
+
+                "master not discovered or elected yet, an election requires two nodes with ids [n1, n2], " +
+                    "have discovered [] which is not a quorum; " +
+                    "discovery will continue using [] from hosts providers and [" + otherMasterNode + ", " + localNode +
+                    "] from last-known cluster state; node term 0, last-accepted version 0 in term 0")));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinationStateTests.java
index afe2e065a875e..99f04015867f5 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinationStateTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinationStateTests.java
@@ -23,33 +23,26 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
+import org.elasticsearch.cluster.coordination.CoordinationState.PersistedState;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNode.Role;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.coordination.CoordinationState.PersistedState;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.EqualsHashCodeTestUtils;
 import org.junit.Before;
 
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.EnumSet;
-import java.util.List;
 import java.util.Optional;
-import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
-import static java.util.stream.Collectors.toSet;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasSize;
 
 public class CoordinationStateTests extends ESTestCase {
 
@@ -89,7 +82,7 @@ public static DiscoveryNode createNode(String id) {
         return new DiscoveryNode("", id,
             UUIDs.randomBase64UUID(random()), // generated deterministically for repeatable tests
             address.address().getHostString(), address.getAddress(), address, Collections.emptyMap(),
-            EnumSet.allOf(Role.class), Version.CURRENT);
+            DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT);
     }
 
     public void testSetInitialState() {
@@ -241,6 +234,7 @@ public void testJoinWithLowerLastAcceptedTermWinsElection() {
         assertTrue(cs1.handleJoin(join));
         assertTrue(cs1.electionWon());
         assertTrue(cs1.containsJoinVoteFor(node1));
+        assertTrue(cs1.containsJoin(join));
         assertFalse(cs1.containsJoinVoteFor(node2));
         assertEquals(cs1.getLastPublishedVersion(), cs1.getLastAcceptedVersion());
         assertFalse(cs1.handleJoin(join));
@@ -323,7 +317,10 @@ public void testHandleClientValue() {
         Join v2 = cs2.handleStartJoin(startJoinRequest1);
         assertTrue(cs1.handleJoin(v1));
         assertTrue(cs1.electionWon());
+        assertTrue(cs1.containsJoin(v1));
+        assertFalse(cs1.containsJoin(v2));
         assertTrue(cs1.handleJoin(v2));
+        assertTrue(cs1.containsJoin(v2));
 
         VotingConfiguration newConfig = new VotingConfiguration(Collections.singleton(node2.getId()));
 
@@ -767,12 +764,14 @@ public void testVoteCollection() {
     }
 
     public void testSafety() {
-        new Cluster(randomIntBetween(1, 5)).runRandomly();
+        new CoordinationStateTestCluster(IntStream.range(0, randomIntBetween(1, 5))
+            .mapToObj(i -> new DiscoveryNode("node_" + i, buildNewFakeTransportAddress(), Version.CURRENT))
+            .collect(Collectors.toList()), ElectionStrategy.DEFAULT_INSTANCE)
+            .runRandomly();
     }
 
     public static CoordinationState createCoordinationState(PersistedState storage, DiscoveryNode localNode) {
-        final Settings initialSettings = Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), localNode.getId()).build();
-        return new CoordinationState(initialSettings, localNode, storage);
+        return new CoordinationState(localNode, storage, ElectionStrategy.DEFAULT_INSTANCE);
     }
 
     public static ClusterState clusterState(long term, long version, DiscoveryNode localNode, VotingConfiguration lastCommittedConfig,
@@ -811,181 +810,4 @@ public static ClusterState setValue(ClusterState clusterState, long value) {
     public static long value(ClusterState clusterState) {
         return clusterState.metaData().persistentSettings().getAsLong("value", 0L);
     }
-    
-    static class ClusterNode {
-
-        final DiscoveryNode localNode;
-        final PersistedState persistedState;
-        CoordinationState state;
-
-        ClusterNode(DiscoveryNode localNode) {
-            this.localNode = localNode;
-            persistedState = new InMemoryPersistedState(0L,
-                clusterState(0L, 0L, localNode, VotingConfiguration.EMPTY_CONFIG, VotingConfiguration.EMPTY_CONFIG, 0L));
-            state = new CoordinationState(Settings.EMPTY, localNode, persistedState);
-        }
-
-        void reboot() {
-            state = new CoordinationState(Settings.EMPTY, localNode, persistedState);
-        }
-
-        void setInitialState(VotingConfiguration initialConfig, long initialValue) {
-            final ClusterState.Builder builder = ClusterState.builder(state.getLastAcceptedState());
-            builder.metaData(MetaData.builder()
-                    .coordinationMetaData(CoordinationMetaData.builder()
-                        .lastAcceptedConfiguration(initialConfig)
-                        .lastCommittedConfiguration(initialConfig)
-                    .build()));
-            state.setInitialState(setValue(builder.build(), initialValue));
-        }
-    }
-
-    static class Cluster {
-
-        final List<Message> messages;
-        final List<ClusterNode> clusterNodes;
-        final VotingConfiguration initialConfiguration;
-        final long initialValue;
-
-        Cluster(int numNodes) {
-            messages = new ArrayList<>();
-
-            clusterNodes = IntStream.range(0, numNodes)
-                .mapToObj(i -> new DiscoveryNode("node_" + i, buildNewFakeTransportAddress(), Version.CURRENT))
-                .map(ClusterNode::new)
-                .collect(Collectors.toList());
-
-            initialConfiguration = randomVotingConfig();
-            initialValue = randomLong();
-        }
-
-        static class Message {
-            final DiscoveryNode sourceNode;
-            final DiscoveryNode targetNode;
-            final Object payload;
-
-            Message(DiscoveryNode sourceNode, DiscoveryNode targetNode, Object payload) {
-                this.sourceNode = sourceNode;
-                this.targetNode = targetNode;
-                this.payload = payload;
-            }
-        }
-
-        void reply(Message m, Object payload) {
-            messages.add(new Message(m.targetNode, m.sourceNode, payload));
-        }
-
-        void broadcast(DiscoveryNode sourceNode, Object payload) {
-            messages.addAll(clusterNodes.stream().map(cn -> new Message(sourceNode, cn.localNode, payload)).collect(Collectors.toList()));
-        }
-
-        Optional<ClusterNode> getNode(DiscoveryNode node) {
-            return clusterNodes.stream().filter(cn -> cn.localNode.equals(node)).findFirst();
-        }
-
-        VotingConfiguration randomVotingConfig() {
-            return new VotingConfiguration(
-                randomSubsetOf(randomIntBetween(1, clusterNodes.size()), clusterNodes).stream()
-                    .map(cn -> cn.localNode.getId()).collect(toSet()));
-        }
-
-        void applyMessage(Message message) {
-            final Optional<ClusterNode> maybeNode = getNode(message.targetNode);
-            if (maybeNode.isPresent() == false) {
-                throw new CoordinationStateRejectedException("node not available");
-            } else {
-                final Object payload = message.payload;
-                if (payload instanceof StartJoinRequest) {
-                    reply(message, maybeNode.get().state.handleStartJoin((StartJoinRequest) payload));
-                } else if (payload instanceof Join) {
-                    maybeNode.get().state.handleJoin((Join) payload);
-                } else if (payload instanceof PublishRequest) {
-                    reply(message, maybeNode.get().state.handlePublishRequest((PublishRequest) payload));
-                } else if (payload instanceof PublishResponse) {
-                    maybeNode.get().state.handlePublishResponse(message.sourceNode, (PublishResponse) payload)
-                        .ifPresent(ac -> broadcast(message.targetNode, ac));
-                } else if (payload instanceof ApplyCommitRequest) {
-                    maybeNode.get().state.handleCommit((ApplyCommitRequest) payload);
-                } else {
-                    throw new AssertionError("unknown message type");
-                }
-            }
-        }
-
-        void runRandomly() {
-            final int iterations = 10000;
-            final long maxTerm = 4;
-            long nextTerm = 1;
-            for (int i = 0; i < iterations; i++) {
-                try {
-                    if (rarely() && nextTerm < maxTerm) {
-                        final long term = rarely() ? randomLongBetween(0, maxTerm + 1) : nextTerm++;
-                        final StartJoinRequest startJoinRequest = new StartJoinRequest(randomFrom(clusterNodes).localNode, term);
-                        broadcast(startJoinRequest.getSourceNode(), startJoinRequest);
-                    } else if (rarely()) {
-                        randomFrom(clusterNodes).setInitialState(initialConfiguration, initialValue);
-                    } else if (rarely() && rarely()) {
-                        randomFrom(clusterNodes).reboot();
-                    } else if (rarely()) {
-                        final List<ClusterNode> masterNodes = clusterNodes.stream().filter(cn -> cn.state.electionWon())
-                            .collect(Collectors.toList());
-                        if (masterNodes.isEmpty() == false) {
-                            final ClusterNode clusterNode = randomFrom(masterNodes);
-                            final long term = rarely() ? randomLongBetween(0, maxTerm + 1) : clusterNode.state.getCurrentTerm();
-                            final long version = rarely() ? randomIntBetween(0, 5) : clusterNode.state.getLastPublishedVersion() + 1;
-                            final VotingConfiguration acceptedConfig = rarely() ? randomVotingConfig() :
-                                clusterNode.state.getLastAcceptedConfiguration();
-                            final PublishRequest publishRequest = clusterNode.state.handleClientValue(
-                                clusterState(term, version, clusterNode.localNode, clusterNode.state.getLastCommittedConfiguration(),
-                                    acceptedConfig, randomLong()));
-                            broadcast(clusterNode.localNode, publishRequest);
-                        }
-                    } else if (messages.isEmpty() == false) {
-                        applyMessage(randomFrom(messages));
-                    }
-
-                    // check node invariants after each iteration
-                    clusterNodes.forEach(cn -> cn.state.invariant());
-                } catch (CoordinationStateRejectedException e) {
-                    // ignore
-                }
-            }
-
-            // check system invariants. It's sufficient to do this at the end as these invariants are monotonic.
-            invariant();
-        }
-
-        void invariant() {
-            // one master per term
-            messages.stream().filter(m -> m.payload instanceof PublishRequest)
-                .collect(Collectors.groupingBy(m -> ((PublishRequest) m.payload).getAcceptedState().term()))
-                .forEach((term, publishMessages) -> {
-                    Set<DiscoveryNode> mastersForTerm = publishMessages.stream().collect(Collectors.groupingBy(m -> m.sourceNode)).keySet();
-                    assertThat("Multiple masters " + mastersForTerm + " for term " + term, mastersForTerm, hasSize(1));
-                });
-
-            // unique cluster state per (term, version) pair
-            messages.stream().filter(m -> m.payload instanceof PublishRequest)
-                .map(m -> ((PublishRequest) m.payload).getAcceptedState())
-                .collect(Collectors.groupingBy(ClusterState::term))
-                .forEach((term, clusterStates) -> {
-                    clusterStates.stream().collect(Collectors.groupingBy(ClusterState::version))
-                    .forEach((version, clusterStates1) -> {
-                        Set<String> clusterStateUUIDsForTermAndVersion = clusterStates1.stream().collect(Collectors.groupingBy(
-                            ClusterState::stateUUID
-                        )).keySet();
-                        assertThat("Multiple cluster states " + clusterStates1 + " for term " + term + " and version " + version,
-                            clusterStateUUIDsForTermAndVersion, hasSize(1));
-
-                        Set<Long> clusterStateValuesForTermAndVersion = clusterStates1.stream().collect(Collectors.groupingBy(
-                            CoordinationStateTests::value
-                        )).keySet();
-
-                        assertThat("Multiple cluster states " + clusterStates1 + " for term " + term + " and version " + version,
-                            clusterStateValuesForTermAndVersion, hasSize(1));
-                    });
-                });
-        }
-
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java
index 5daa863402b2a..d7832f04d9589 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java
@@ -18,105 +18,45 @@
  */
 package org.elasticsearch.cluster.coordination;
 
-import com.carrotsearch.randomizedtesting.RandomizedContext;
-import org.apache.logging.log4j.CloseableThreadContext;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.core.LogEvent;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
-import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateTaskListener;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
-import org.elasticsearch.cluster.ESAllocationTestCase;
-import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.block.ClusterBlock;
-import org.elasticsearch.cluster.coordination.ClusterStatePublisher.AckListener;
+import org.elasticsearch.cluster.coordination.AbstractCoordinatorTestCase.Cluster.ClusterNode;
 import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
-import org.elasticsearch.cluster.coordination.CoordinationState.PersistedState;
 import org.elasticsearch.cluster.coordination.Coordinator.Mode;
-import org.elasticsearch.cluster.coordination.CoordinatorTests.Cluster.ClusterNode;
-import org.elasticsearch.cluster.coordination.LinearizabilityChecker.History;
-import org.elasticsearch.cluster.coordination.LinearizabilityChecker.SequentialSpec;
-import org.elasticsearch.cluster.metadata.Manifest;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNode.Role;
-import org.elasticsearch.cluster.service.ClusterApplierService;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Randomness;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.Settings.Builder;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.discovery.DiscoveryModule;
-import org.elasticsearch.discovery.SeedHostsProvider.HostsResolver;
-import org.elasticsearch.env.NodeEnvironment;
-import org.elasticsearch.gateway.MetaStateService;
-import org.elasticsearch.gateway.MockGatewayMetaState;
-import org.elasticsearch.indices.cluster.FakeThreadPoolMasterService;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.test.MockLogAppender;
-import org.elasticsearch.test.disruption.DisruptableMockTransport;
-import org.elasticsearch.test.disruption.DisruptableMockTransport.ConnectionStatus;
-import org.elasticsearch.transport.TransportService;
-import org.hamcrest.Matcher;
-import org.hamcrest.core.IsCollectionContaining;
-import org.junit.After;
-import org.junit.Before;
 
 import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
-import java.util.concurrent.Callable;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.BiConsumer;
-import java.util.function.Consumer;
 import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
-import static java.util.Collections.emptyList;
-import static java.util.Collections.emptySet;
-import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.BOOTSTRAP_PLACEHOLDER_PREFIX;
-import static org.elasticsearch.cluster.coordination.CoordinationStateTests.clusterState;
+import static org.elasticsearch.cluster.coordination.AbstractCoordinatorTestCase.Cluster.DEFAULT_DELAY_VARIABILITY;
 import static org.elasticsearch.cluster.coordination.Coordinator.Mode.CANDIDATE;
-import static org.elasticsearch.cluster.coordination.Coordinator.Mode.FOLLOWER;
-import static org.elasticsearch.cluster.coordination.Coordinator.Mode.LEADER;
 import static org.elasticsearch.cluster.coordination.Coordinator.PUBLISH_TIMEOUT_SETTING;
-import static org.elasticsearch.cluster.coordination.CoordinatorTests.Cluster.DEFAULT_DELAY_VARIABILITY;
-import static org.elasticsearch.cluster.coordination.ElectionSchedulerFactory.ELECTION_BACK_OFF_TIME_SETTING;
-import static org.elasticsearch.cluster.coordination.ElectionSchedulerFactory.ELECTION_DURATION_SETTING;
 import static org.elasticsearch.cluster.coordination.ElectionSchedulerFactory.ELECTION_INITIAL_TIMEOUT_SETTING;
 import static org.elasticsearch.cluster.coordination.FollowersChecker.FOLLOWER_CHECK_INTERVAL_SETTING;
 import static org.elasticsearch.cluster.coordination.FollowersChecker.FOLLOWER_CHECK_RETRY_COUNT_SETTING;
@@ -125,69 +65,58 @@
 import static org.elasticsearch.cluster.coordination.LeaderChecker.LEADER_CHECK_RETRY_COUNT_SETTING;
 import static org.elasticsearch.cluster.coordination.LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING;
 import static org.elasticsearch.cluster.coordination.NoMasterBlockService.NO_MASTER_BLOCK_ALL;
-import static org.elasticsearch.cluster.coordination.NoMasterBlockService.NO_MASTER_BLOCK_ID;
 import static org.elasticsearch.cluster.coordination.NoMasterBlockService.NO_MASTER_BLOCK_SETTING;
 import static org.elasticsearch.cluster.coordination.NoMasterBlockService.NO_MASTER_BLOCK_WRITES;
 import static org.elasticsearch.cluster.coordination.Reconfigurator.CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION;
 import static org.elasticsearch.discovery.PeerFinder.DISCOVERY_FIND_PEERS_INTERVAL_SETTING;
-import static org.elasticsearch.node.Node.NODE_NAME_SETTING;
-import static org.elasticsearch.transport.TransportService.NOOP_TRANSPORT_INTERCEPTOR;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.nullValue;
-import static org.hamcrest.Matchers.sameInstance;
 import static org.hamcrest.Matchers.startsWith;
 
-public class CoordinatorTests extends ESTestCase {
+public class CoordinatorTests extends AbstractCoordinatorTestCase {
 
-    private final List<NodeEnvironment> nodeEnvironments = new ArrayList<>();
-
-    private final AtomicInteger nextNodeIndex = new AtomicInteger();
+    /**
+     * This test was added to verify that state recovery is properly reset on a node after it has become master and successfully
+     * recovered a state (see {@link GatewayService}). The situation which triggers this with a decent likelihood is as follows:
+     * 3 master-eligible nodes (leader, follower1, follower2), the followers are shut down (leader remains), when followers come back
+     * one of them becomes leader and publishes first state (with STATE_NOT_RECOVERED_BLOCK) to old leader, which accepts it.
+     * Old leader is initiating an election at the same time, and wins election. It becomes leader again, but as it previously
+     * successfully completed state recovery, is never reset to a state where state recovery can be retried.
+     */
+    public void testStateRecoveryResetAfterPreviousLeadership() {
+        final Cluster cluster = new Cluster(3);
+        cluster.runRandomly();
+        cluster.stabilise();
 
-    @Before
-    public void resetNodeIndexBeforeEachTest() {
-        nextNodeIndex.set(0);
-    }
+        final ClusterNode leader = cluster.getAnyLeader();
+        final ClusterNode follower1 = cluster.getAnyNodeExcept(leader);
+        final ClusterNode follower2 = cluster.getAnyNodeExcept(leader, follower1);
+
+        // restart follower1 and follower2
+        for (ClusterNode clusterNode : Arrays.asList(follower1, follower2)) {
+            clusterNode.close();
+            cluster.clusterNodes.forEach(
+                cn -> cluster.deterministicTaskQueue.scheduleNow(cn.onNode(
+                    new Runnable() {
+                        @Override
+                        public void run() {
+                            cn.transportService.disconnectFromNode(clusterNode.getLocalNode());
+                        }
 
-    @After
-    public void closeNodeEnvironmentsAfterEachTest() {
-        for (NodeEnvironment nodeEnvironment : nodeEnvironments) {
-            nodeEnvironment.close();
+                        @Override
+                        public String toString() {
+                            return "disconnect from " + clusterNode.getLocalNode() + " after shutdown";
+                        }
+                    })));
+            cluster.clusterNodes.replaceAll(cn -> cn == clusterNode ? cn.restartedNode() : cn);
         }
-        nodeEnvironments.clear();
-    }
-
-    @Before
-    public void resetPortCounterBeforeEachTest() {
-        resetPortCounter();
-    }
 
-    // check that runRandomly leads to reproducible results
-    public void testRepeatableTests() throws Exception {
-        final Callable<Long> test = () -> {
-            resetNodeIndexBeforeEachTest();
-            final Cluster cluster = new Cluster(randomIntBetween(1, 5));
-            cluster.runRandomly();
-            final long afterRunRandomly = value(cluster.getAnyNode().getLastAppliedClusterState());
-            cluster.stabilise();
-            final long afterStabilisation = value(cluster.getAnyNode().getLastAppliedClusterState());
-            return afterRunRandomly ^ afterStabilisation;
-        };
-        final long seed = randomLong();
-        logger.info("First run with seed [{}]", seed);
-        final long result1 = RandomizedContext.current().runWithPrivateRandomness(seed, test);
-        logger.info("Second run with seed [{}]", seed);
-        final long result2 = RandomizedContext.current().runWithPrivateRandomness(seed, test);
-        assertEquals(result1, result2);
+        cluster.stabilise();
     }
 
     public void testCanUpdateClusterStateAfterStabilisation() {
@@ -215,7 +144,7 @@ public void testDoesNotElectNonMasterNode() {
         cluster.stabilise();
 
         final ClusterNode leader = cluster.getAnyLeader();
-        assertTrue(leader.localNode.isMasterNode());
+        assertTrue(leader.getLocalNode().isMasterNode());
     }
 
     public void testNodesJoinAfterStableCluster() {
@@ -1215,1160 +1144,79 @@ public void testClusterRecoversAfterExceptionDuringSerialization() {
         }
     }
 
-    private static long defaultMillis(Setting<TimeValue> setting) {
-        return setting.get(Settings.EMPTY).millis() + Cluster.DEFAULT_DELAY_VARIABILITY;
-    }
-
-    private static int defaultInt(Setting<Integer> setting) {
-        return setting.get(Settings.EMPTY);
-    }
-
-    // Updating the cluster state involves up to 7 delays:
-    // 1. submit the task to the master service
-    // 2. send PublishRequest
-    // 3. receive PublishResponse
-    // 4. send ApplyCommitRequest
-    // 5. apply committed cluster state
-    // 6. receive ApplyCommitResponse
-    // 7. apply committed state on master (last one to apply cluster state)
-    private static final long DEFAULT_CLUSTER_STATE_UPDATE_DELAY = 7 * DEFAULT_DELAY_VARIABILITY;
-
-    private static final int ELECTION_RETRIES = 10;
-
-    // The time it takes to complete an election
-    private static final long DEFAULT_ELECTION_DELAY
-        // Pinging all peers twice should be enough to discover all nodes
-        = defaultMillis(DISCOVERY_FIND_PEERS_INTERVAL_SETTING) * 2
-        // Then wait for an election to be scheduled; we allow enough time for retries to allow for collisions
-        + defaultMillis(ELECTION_INITIAL_TIMEOUT_SETTING) * ELECTION_RETRIES
-        + defaultMillis(ELECTION_BACK_OFF_TIME_SETTING) * ELECTION_RETRIES * (ELECTION_RETRIES - 1) / 2
-        + defaultMillis(ELECTION_DURATION_SETTING) * ELECTION_RETRIES
-        // Allow two round-trip for pre-voting and voting
-        + 4 * DEFAULT_DELAY_VARIABILITY
-        // Then a commit of the new leader's first cluster state
-        + DEFAULT_CLUSTER_STATE_UPDATE_DELAY;
-
-    private static final long DEFAULT_STABILISATION_TIME =
-        // If leader just blackholed, need to wait for this to be detected
-        (defaultMillis(LEADER_CHECK_INTERVAL_SETTING) + defaultMillis(LEADER_CHECK_TIMEOUT_SETTING))
-            * defaultInt(LEADER_CHECK_RETRY_COUNT_SETTING)
-            // then wait for a follower to be promoted to leader
-            + DEFAULT_ELECTION_DELAY
-            // perhaps there is an election collision requiring another publication (which times out) and a term bump
-            + defaultMillis(PUBLISH_TIMEOUT_SETTING) + DEFAULT_ELECTION_DELAY
-            // then wait for the new leader to notice that the old leader is unresponsive
-            + (defaultMillis(FOLLOWER_CHECK_INTERVAL_SETTING) + defaultMillis(FOLLOWER_CHECK_TIMEOUT_SETTING))
-            * defaultInt(FOLLOWER_CHECK_RETRY_COUNT_SETTING)
-            // then wait for the new leader to commit a state without the old leader
-            + DEFAULT_CLUSTER_STATE_UPDATE_DELAY;
-
-    class Cluster {
-
-        static final long EXTREME_DELAY_VARIABILITY = 10000L;
-        static final long DEFAULT_DELAY_VARIABILITY = 100L;
-
-        final List<ClusterNode> clusterNodes;
-        final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(
-            // TODO does ThreadPool need a node name any more?
-            Settings.builder().put(NODE_NAME_SETTING.getKey(), "deterministic-task-queue").build(), random());
-        private boolean disruptStorage;
-
-        private final VotingConfiguration initialConfiguration;
-
-        private final Set<String> disconnectedNodes = new HashSet<>();
-        private final Set<String> blackholedNodes = new HashSet<>();
-        private final Set<Tuple<String,String>> blackholedConnections = new HashSet<>();
-        private final Map<Long, ClusterState> committedStatesByVersion = new HashMap<>();
-        private final LinearizabilityChecker linearizabilityChecker = new LinearizabilityChecker();
-        private final History history = new History();
-
-        private final Function<DiscoveryNode, MockPersistedState> defaultPersistedStateSupplier = MockPersistedState::new;
-
-        @Nullable // null means construct a list from all the current nodes
-        private List<TransportAddress> seedHostsList;
-
-        Cluster(int initialNodeCount) {
-            this(initialNodeCount, true, Settings.EMPTY);
-        }
-
-        Cluster(int initialNodeCount, boolean allNodesMasterEligible, Settings nodeSettings) {
-            deterministicTaskQueue.setExecutionDelayVariabilityMillis(DEFAULT_DELAY_VARIABILITY);
-
-            assertThat(initialNodeCount, greaterThan(0));
-
-            final Set<String> masterEligibleNodeIds = new HashSet<>(initialNodeCount);
-            clusterNodes = new ArrayList<>(initialNodeCount);
-            for (int i = 0; i < initialNodeCount; i++) {
-                final ClusterNode clusterNode = new ClusterNode(nextNodeIndex.getAndIncrement(),
-                    allNodesMasterEligible || i == 0 || randomBoolean(), nodeSettings);
-                clusterNodes.add(clusterNode);
-                if (clusterNode.getLocalNode().isMasterNode()) {
-                    masterEligibleNodeIds.add(clusterNode.getId());
-                }
-            }
-
-            initialConfiguration = new VotingConfiguration(new HashSet<>(
-                randomSubsetOf(randomIntBetween(1, masterEligibleNodeIds.size()), masterEligibleNodeIds)));
-
-            logger.info("--> creating cluster of {} nodes (master-eligible nodes: {}) with initial configuration {}",
-                initialNodeCount, masterEligibleNodeIds, initialConfiguration);
-        }
-
-        List<ClusterNode> addNodesAndStabilise(int newNodesCount) {
-            final List<ClusterNode> addedNodes = addNodes(newNodesCount);
-            stabilise(
-                // The first pinging discovers the master
-                defaultMillis(DISCOVERY_FIND_PEERS_INTERVAL_SETTING)
-                    // One message delay to send a join
-                    + DEFAULT_DELAY_VARIABILITY
-                    // Commit a new cluster state with the new node(s). Might be split into multiple commits, and each might need a
-                    // followup reconfiguration
-                    + newNodesCount * 2 * DEFAULT_CLUSTER_STATE_UPDATE_DELAY);
-            // TODO Investigate whether 4 publications is sufficient due to batching? A bound linear in the number of nodes isn't great.
-            return addedNodes;
-        }
-
-        List<ClusterNode> addNodes(int newNodesCount) {
-            logger.info("--> adding {} nodes", newNodesCount);
-
-            final List<ClusterNode> addedNodes = new ArrayList<>();
-            for (int i = 0; i < newNodesCount; i++) {
-                final ClusterNode clusterNode = new ClusterNode(nextNodeIndex.getAndIncrement(), true, Settings.EMPTY);
-                addedNodes.add(clusterNode);
-            }
-            clusterNodes.addAll(addedNodes);
-            return addedNodes;
-        }
-
-        int size() {
-            return clusterNodes.size();
-        }
-
-        void runRandomly() {
-            runRandomly(true);
-        }
-
-        void runRandomly(boolean allowReboots) {
-
-            // TODO supporting (preserving?) existing disruptions needs implementing if needed, for now we just forbid it
-            assertThat("may reconnect disconnected nodes, probably unexpected", disconnectedNodes, empty());
-            assertThat("may reconnect blackholed nodes, probably unexpected", blackholedNodes, empty());
-
-            final List<Runnable> cleanupActions = new ArrayList<>();
-            cleanupActions.add(disconnectedNodes::clear);
-            cleanupActions.add(blackholedNodes::clear);
-            cleanupActions.add(() -> disruptStorage = false);
-
-            final int randomSteps = scaledRandomIntBetween(10, 10000);
-            final int keyRange = randomSteps / 50; // for randomized writes and reads
-            logger.info("--> start of safety phase of at least [{}] steps", randomSteps);
-
-            deterministicTaskQueue.setExecutionDelayVariabilityMillis(EXTREME_DELAY_VARIABILITY);
-            disruptStorage = true;
-            int step = 0;
-            long finishTime = -1;
-
-            while (finishTime == -1 || deterministicTaskQueue.getCurrentTimeMillis() <= finishTime) {
-                step++;
-                final int thisStep = step; // for lambdas
-
-                if (randomSteps <= step && finishTime == -1) {
-                    finishTime = deterministicTaskQueue.getLatestDeferredExecutionTime();
-                    deterministicTaskQueue.setExecutionDelayVariabilityMillis(DEFAULT_DELAY_VARIABILITY);
-                    logger.debug("----> [runRandomly {}] reducing delay variability and running until [{}ms]", step, finishTime);
-                }
-
-                try {
-                    if (finishTime == -1 && randomBoolean() && randomBoolean() && randomBoolean()) {
-                        final ClusterNode clusterNode = getAnyNodePreferringLeaders();
-                        final int key = randomIntBetween(0, keyRange);
-                        final int newValue = randomInt();
-                        clusterNode.onNode(() -> {
-                            logger.debug("----> [runRandomly {}] proposing new value [{}] to [{}]",
-                                thisStep, newValue, clusterNode.getId());
-                            clusterNode.submitValue(key, newValue);
-                        }).run();
-                    } else if (finishTime == -1 && randomBoolean() && randomBoolean() && randomBoolean()) {
-                        final ClusterNode clusterNode = getAnyNodePreferringLeaders();
-                        final int key = randomIntBetween(0, keyRange);
-                        clusterNode.onNode(() -> {
-                            logger.debug("----> [runRandomly {}] reading value from [{}]",
-                                thisStep, clusterNode.getId());
-                            clusterNode.readValue(key);
-                        }).run();
-                    } else if (rarely()) {
-                        final ClusterNode clusterNode = getAnyNodePreferringLeaders();
-                        final boolean autoShrinkVotingConfiguration = randomBoolean();
-                        clusterNode.onNode(
-                            () -> {
-                                logger.debug("----> [runRandomly {}] setting auto-shrink configuration to {} on {}",
-                                    thisStep, autoShrinkVotingConfiguration, clusterNode.getId());
-                                clusterNode.submitSetAutoShrinkVotingConfiguration(autoShrinkVotingConfiguration);
-                            }).run();
-                    } else if (allowReboots && rarely()) {
-                        // reboot random node
-                        final ClusterNode clusterNode = getAnyNode();
-                        logger.debug("----> [runRandomly {}] rebooting [{}]", thisStep, clusterNode.getId());
-                        clusterNode.close();
-                        clusterNodes.forEach(
-                            cn -> deterministicTaskQueue.scheduleNow(cn.onNode(
-                                new Runnable() {
-                                    @Override
-                                    public void run() {
-                                        cn.transportService.disconnectFromNode(clusterNode.getLocalNode());
-                                    }
-
-                                    @Override
-                                    public String toString() {
-                                        return "disconnect from " + clusterNode.getLocalNode() + " after shutdown";
-                                    }
-                                })));
-                        clusterNodes.replaceAll(cn -> cn == clusterNode ? cn.restartedNode() : cn);
-                    } else if (rarely()) {
-                        final ClusterNode clusterNode = getAnyNode();
-                        clusterNode.onNode(() -> {
-                            logger.debug("----> [runRandomly {}] forcing {} to become candidate", thisStep, clusterNode.getId());
-                            synchronized (clusterNode.coordinator.mutex) {
-                                clusterNode.coordinator.becomeCandidate("runRandomly");
-                            }
-                        }).run();
-                    } else if (rarely()) {
-                        final ClusterNode clusterNode = getAnyNode();
-
-                        switch (randomInt(2)) {
-                            case 0:
-                                if (clusterNode.heal()) {
-                                    logger.debug("----> [runRandomly {}] healing {}", step, clusterNode.getId());
-                                }
-                                break;
-                            case 1:
-                                if (clusterNode.disconnect()) {
-                                    logger.debug("----> [runRandomly {}] disconnecting {}", step, clusterNode.getId());
-                                }
-                                break;
-                            case 2:
-                                if (clusterNode.blackhole()) {
-                                    logger.debug("----> [runRandomly {}] blackholing {}", step, clusterNode.getId());
-                                }
-                                break;
-                        }
-                    } else if (rarely()) {
-                        final ClusterNode clusterNode = getAnyNode();
-                        logger.debug("----> [runRandomly {}] applying initial configuration on {}", step, clusterNode.getId());
-                        clusterNode.applyInitialConfiguration();
-                    } else {
-                        if (deterministicTaskQueue.hasDeferredTasks() && randomBoolean()) {
-                            deterministicTaskQueue.advanceTime();
-                        } else if (deterministicTaskQueue.hasRunnableTasks()) {
-                            deterministicTaskQueue.runRandomTask();
-                        }
-                    }
-
-                    // TODO other random steps:
-                    // - reboot a node
-                    // - abdicate leadership
-
-                } catch (CoordinationStateRejectedException | UncheckedIOException ignored) {
-                    // This is ok: it just means a message couldn't currently be handled.
-                }
-
-                assertConsistentStates();
-            }
-
-            logger.debug("running {} cleanup actions", cleanupActions.size());
-            cleanupActions.forEach(Runnable::run);
-            logger.debug("finished running cleanup actions");
-        }
-
-        private void assertConsistentStates() {
-            for (final ClusterNode clusterNode : clusterNodes) {
-                clusterNode.coordinator.invariant();
-            }
-            updateCommittedStates();
-        }
-
-        private void updateCommittedStates() {
-            for (final ClusterNode clusterNode : clusterNodes) {
-                ClusterState applierState = clusterNode.coordinator.getApplierState();
-                ClusterState storedState = committedStatesByVersion.get(applierState.getVersion());
-                if (storedState == null) {
-                    committedStatesByVersion.put(applierState.getVersion(), applierState);
-                } else {
-                    assertEquals("expected " + applierState + " but got " + storedState,
-                        value(applierState), value(storedState));
-                }
-            }
-        }
-
-        void stabilise() {
-            stabilise(DEFAULT_STABILISATION_TIME);
-        }
-
-        void stabilise(long stabilisationDurationMillis) {
-            assertThat("stabilisation requires default delay variability (and proper cleanup of raised variability)",
-                deterministicTaskQueue.getExecutionDelayVariabilityMillis(), lessThanOrEqualTo(DEFAULT_DELAY_VARIABILITY));
-            assertFalse("stabilisation requires stable storage", disruptStorage);
-
-            bootstrapIfNecessary();
-
-            runFor(stabilisationDurationMillis, "stabilising");
-
-            final ClusterNode leader = getAnyLeader();
-            final long leaderTerm = leader.coordinator.getCurrentTerm();
-
-            final int pendingTaskCount = leader.masterService.getFakeMasterServicePendingTaskCount();
-            runFor((pendingTaskCount + 1) * DEFAULT_CLUSTER_STATE_UPDATE_DELAY, "draining task queue");
-
-            final Matcher<Long> isEqualToLeaderVersion = equalTo(leader.coordinator.getLastAcceptedState().getVersion());
-            final String leaderId = leader.getId();
-
-            assertTrue(leaderId + " has been bootstrapped", leader.coordinator.isInitialConfigurationSet());
-            assertTrue(leaderId + " exists in its last-applied state", leader.getLastAppliedClusterState().getNodes().nodeExists(leaderId));
-            assertThat(leaderId + " has applied its state ", leader.getLastAppliedClusterState().getVersion(), isEqualToLeaderVersion);
-
-            for (final ClusterNode clusterNode : clusterNodes) {
-                final String nodeId = clusterNode.getId();
-                assertFalse(nodeId + " should not have an active publication", clusterNode.coordinator.publicationInProgress());
-
-                if (clusterNode == leader) {
-                    assertThat(nodeId + " is still the leader", clusterNode.coordinator.getMode(), is(LEADER));
-                    assertThat(nodeId + " did not change term", clusterNode.coordinator.getCurrentTerm(), is(leaderTerm));
-                    continue;
-                }
-
-                if (isConnectedPair(leader, clusterNode)) {
-                    assertThat(nodeId + " is a follower of " + leaderId, clusterNode.coordinator.getMode(), is(FOLLOWER));
-                    assertThat(nodeId + " has the same term as " + leaderId, clusterNode.coordinator.getCurrentTerm(), is(leaderTerm));
-                    assertTrue(nodeId + " has voted for " + leaderId, leader.coordinator.hasJoinVoteFrom(clusterNode.getLocalNode()));
-                    assertThat(nodeId + " has the same accepted state as " + leaderId,
-                        clusterNode.coordinator.getLastAcceptedState().getVersion(), isEqualToLeaderVersion);
-                    if (clusterNode.getClusterStateApplyResponse() == ClusterStateApplyResponse.SUCCEED) {
-                        assertThat(nodeId + " has the same applied state as " + leaderId,
-                            clusterNode.getLastAppliedClusterState().getVersion(), isEqualToLeaderVersion);
-                        assertTrue(nodeId + " is in its own latest applied state",
-                            clusterNode.getLastAppliedClusterState().getNodes().nodeExists(nodeId));
-                    }
-                    assertTrue(nodeId + " is in the latest applied state on " + leaderId,
-                        leader.getLastAppliedClusterState().getNodes().nodeExists(nodeId));
-                    assertTrue(nodeId + " has been bootstrapped", clusterNode.coordinator.isInitialConfigurationSet());
-                    assertThat(nodeId + " has correct master", clusterNode.getLastAppliedClusterState().nodes().getMasterNode(),
-                        equalTo(leader.getLocalNode()));
-                    assertThat(nodeId + " has no NO_MASTER_BLOCK",
-                        clusterNode.getLastAppliedClusterState().blocks().hasGlobalBlockWithId(NO_MASTER_BLOCK_ID), equalTo(false));
-                } else {
-                    assertThat(nodeId + " is not following " + leaderId, clusterNode.coordinator.getMode(), is(CANDIDATE));
-                    assertThat(nodeId + " has no master", clusterNode.getLastAppliedClusterState().nodes().getMasterNode(), nullValue());
-                    assertThat(nodeId + " has NO_MASTER_BLOCK",
-                        clusterNode.getLastAppliedClusterState().blocks().hasGlobalBlockWithId(NO_MASTER_BLOCK_ID), equalTo(true));
-                    assertFalse(nodeId + " is not in the applied state on " + leaderId,
-                        leader.getLastAppliedClusterState().getNodes().nodeExists(nodeId));
-                }
-            }
-
-            final Set<String> connectedNodeIds
-                = clusterNodes.stream().filter(n -> isConnectedPair(leader, n)).map(ClusterNode::getId).collect(Collectors.toSet());
-
-            assertThat(leader.getLastAppliedClusterState().getNodes().getSize(), equalTo(connectedNodeIds.size()));
-
-            final ClusterState lastAcceptedState = leader.coordinator.getLastAcceptedState();
-            final VotingConfiguration lastCommittedConfiguration = lastAcceptedState.getLastCommittedConfiguration();
-            assertTrue(connectedNodeIds + " should be a quorum of " + lastCommittedConfiguration,
-                lastCommittedConfiguration.hasQuorum(connectedNodeIds));
-            assertThat("leader " + leader.getLocalNode() + " should be part of voting configuration " + lastCommittedConfiguration,
-                lastCommittedConfiguration.getNodeIds(), IsCollectionContaining.hasItem(leader.getLocalNode().getId()));
-
-            assertThat("no reconfiguration is in progress",
-                lastAcceptedState.getLastCommittedConfiguration(), equalTo(lastAcceptedState.getLastAcceptedConfiguration()));
-            assertThat("current configuration is already optimal",
-                leader.improveConfiguration(lastAcceptedState), sameInstance(lastAcceptedState));
-
-            logger.info("checking linearizability of history with size {}: {}", history.size(), history);
-            assertTrue("history not linearizable: " + history, linearizabilityChecker.isLinearizable(spec, history, i -> null));
-            logger.info("linearizability check completed");
-        }
-
-        void bootstrapIfNecessary() {
-            if (clusterNodes.stream().allMatch(ClusterNode::isNotUsefullyBootstrapped)) {
-                assertThat("setting initial configuration may fail with disconnected nodes", disconnectedNodes, empty());
-                assertThat("setting initial configuration may fail with blackholed nodes", blackholedNodes, empty());
-                runFor(defaultMillis(DISCOVERY_FIND_PEERS_INTERVAL_SETTING) * 2, "discovery prior to setting initial configuration");
-                final ClusterNode bootstrapNode = getAnyBootstrappableNode();
-                bootstrapNode.applyInitialConfiguration();
-            } else {
-                logger.info("setting initial configuration not required");
-            }
-        }
-
-        void runFor(long runDurationMillis, String description) {
-            final long endTime = deterministicTaskQueue.getCurrentTimeMillis() + runDurationMillis;
-            logger.info("--> runFor({}ms) running until [{}ms]: {}", runDurationMillis, endTime, description);
-
-            while (deterministicTaskQueue.getCurrentTimeMillis() < endTime) {
-
-                while (deterministicTaskQueue.hasRunnableTasks()) {
-                    try {
-                        deterministicTaskQueue.runRandomTask();
-                    } catch (CoordinationStateRejectedException e) {
-                        logger.debug("ignoring benign exception thrown when stabilising", e);
-                    }
-                    for (final ClusterNode clusterNode : clusterNodes) {
-                        clusterNode.coordinator.invariant();
-                    }
-                    updateCommittedStates();
-                }
-
-                if (deterministicTaskQueue.hasDeferredTasks() == false) {
-                    // A 1-node cluster has no need for fault detection etc so will eventually run out of things to do.
-                    assert clusterNodes.size() == 1 : clusterNodes.size();
-                    break;
-                }
-
-                deterministicTaskQueue.advanceTime();
-            }
-
-            logger.info("--> runFor({}ms) completed run until [{}ms]: {}", runDurationMillis, endTime, description);
-        }
-
-        private boolean isConnectedPair(ClusterNode n1, ClusterNode n2) {
-            return n1 == n2 ||
-                (getConnectionStatus(n1.getLocalNode(), n2.getLocalNode()) == ConnectionStatus.CONNECTED
-                    && getConnectionStatus(n2.getLocalNode(), n1.getLocalNode()) == ConnectionStatus.CONNECTED);
-        }
-
-        ClusterNode getAnyLeader() {
-            List<ClusterNode> allLeaders = clusterNodes.stream().filter(ClusterNode::isLeader).collect(Collectors.toList());
-            assertThat("leaders", allLeaders, not(empty()));
-            return randomFrom(allLeaders);
-        }
-
-        private final ConnectionStatus preferredUnknownNodeConnectionStatus =
-            randomFrom(ConnectionStatus.DISCONNECTED, ConnectionStatus.BLACK_HOLE);
-
-        private ConnectionStatus getConnectionStatus(DiscoveryNode sender, DiscoveryNode destination) {
-            ConnectionStatus connectionStatus;
-            if (blackholedNodes.contains(sender.getId()) || blackholedNodes.contains(destination.getId())) {
-                connectionStatus = ConnectionStatus.BLACK_HOLE;
-            } else if (disconnectedNodes.contains(sender.getId()) || disconnectedNodes.contains(destination.getId())) {
-                connectionStatus = ConnectionStatus.DISCONNECTED;
-            } else if (blackholedConnections.contains(Tuple.tuple(sender.getId(), destination.getId()))) {
-                connectionStatus = ConnectionStatus.BLACK_HOLE_REQUESTS_ONLY;
-            } else if (nodeExists(sender) && nodeExists(destination)) {
-                connectionStatus = ConnectionStatus.CONNECTED;
-            } else {
-                connectionStatus = usually() ? preferredUnknownNodeConnectionStatus :
-                    randomFrom(ConnectionStatus.DISCONNECTED, ConnectionStatus.BLACK_HOLE);
-            }
-            return connectionStatus;
-        }
-
-        boolean nodeExists(DiscoveryNode node) {
-            return clusterNodes.stream().anyMatch(cn -> cn.getLocalNode().equals(node));
-        }
-
-        ClusterNode getAnyBootstrappableNode() {
-            return randomFrom(clusterNodes.stream().filter(n -> n.getLocalNode().isMasterNode())
-                .filter(n -> initialConfiguration.getNodeIds().contains(n.getLocalNode().getId()))
-                .collect(Collectors.toList()));
-        }
-
-        ClusterNode getAnyNode() {
-            return getAnyNodeExcept();
-        }
-
-        ClusterNode getAnyNodeExcept(ClusterNode... clusterNodes) {
-            List<ClusterNode> filteredNodes = getAllNodesExcept(clusterNodes);
-            assert filteredNodes.isEmpty() == false;
-            return randomFrom(filteredNodes);
-        }
-
-        List<ClusterNode> getAllNodesExcept(ClusterNode... clusterNodes) {
-            Set<String> forbiddenIds = Arrays.stream(clusterNodes).map(ClusterNode::getId).collect(Collectors.toSet());
-            List<ClusterNode> acceptableNodes
-                = this.clusterNodes.stream().filter(n -> forbiddenIds.contains(n.getId()) == false).collect(Collectors.toList());
-            return acceptableNodes;
-        }
-
-        ClusterNode getAnyNodePreferringLeaders() {
-            for (int i = 0; i < 3; i++) {
-                ClusterNode clusterNode = getAnyNode();
-                if (clusterNode.coordinator.getMode() == LEADER) {
-                    return clusterNode;
-                }
-            }
-            return getAnyNode();
-        }
-
-        void setEmptySeedHostsList() {
-            seedHostsList = emptyList();
-        }
-
-        void blackholeConnectionsFrom(ClusterNode sender, ClusterNode destination) {
-            blackholedConnections.add(Tuple.tuple(sender.getId(), destination.getId()));
-        }
-
-        void clearBlackholedConnections() {
-            blackholedConnections.clear();
+    public void testLogsWarningPeriodicallyIfClusterNotFormed() {
+        final long warningDelayMillis;
+        final Settings settings;
+        if (randomBoolean()) {
+            settings = Settings.EMPTY;
+            warningDelayMillis = ClusterFormationFailureHelper.DISCOVERY_CLUSTER_FORMATION_WARNING_TIMEOUT_SETTING.get(settings).millis();
+        } else {
+            warningDelayMillis = randomLongBetween(1, 100000);
+            settings = Settings.builder()
+                .put(ClusterFormationFailureHelper.DISCOVERY_CLUSTER_FORMATION_WARNING_TIMEOUT_SETTING.getKey(), warningDelayMillis + "ms")
+                .build();
         }
+        logger.info("--> emitting warnings every [{}ms]", warningDelayMillis);
 
-        class MockPersistedState implements PersistedState {
-            private final PersistedState delegate;
-            private final NodeEnvironment nodeEnvironment;
-
-            MockPersistedState(DiscoveryNode localNode) {
-                try {
-                    if (rarely()) {
-                        nodeEnvironment = newNodeEnvironment();
-                        nodeEnvironments.add(nodeEnvironment);
-                        delegate = new MockGatewayMetaState(Settings.EMPTY, nodeEnvironment, xContentRegistry(), localNode)
-                                .getPersistedState(Settings.EMPTY, null);
-                    } else {
-                        nodeEnvironment = null;
-                        delegate = new InMemoryPersistedState(0L,
-                                clusterState(0L, 0L, localNode, VotingConfiguration.EMPTY_CONFIG, VotingConfiguration.EMPTY_CONFIG, 0L));
-                    }
-                } catch (IOException e) {
-                    throw new UncheckedIOException("Unable to create MockPersistedState", e);
-                }
-            }
-
-            MockPersistedState(DiscoveryNode newLocalNode, MockPersistedState oldState,
-                               Function<MetaData, MetaData> adaptGlobalMetaData, Function<Long, Long> adaptCurrentTerm) {
-                try {
-                    if (oldState.nodeEnvironment != null) {
-                        nodeEnvironment = oldState.nodeEnvironment;
-                        final MetaStateService metaStateService = new MetaStateService(nodeEnvironment, xContentRegistry());
-                        final MetaData updatedMetaData = adaptGlobalMetaData.apply(oldState.getLastAcceptedState().metaData());
-                        if (updatedMetaData != oldState.getLastAcceptedState().metaData()) {
-                            metaStateService.writeGlobalStateAndUpdateManifest("update global state", updatedMetaData);
-                        }
-                        final long updatedTerm = adaptCurrentTerm.apply(oldState.getCurrentTerm());
-                        if (updatedTerm != oldState.getCurrentTerm()) {
-                            final Manifest manifest = metaStateService.loadManifestOrEmpty();
-                            metaStateService.writeManifestAndCleanup("update term",
-                                new Manifest(updatedTerm, manifest.getClusterStateVersion(), manifest.getGlobalGeneration(),
-                                    manifest.getIndexGenerations()));
-                        }
-                        delegate = new MockGatewayMetaState(Settings.EMPTY, nodeEnvironment, xContentRegistry(), newLocalNode)
-                                .getPersistedState(Settings.EMPTY, null);
-                    } else {
-                        nodeEnvironment = null;
-                        BytesStreamOutput outStream = new BytesStreamOutput();
-                        outStream.setVersion(Version.CURRENT);
-                        final MetaData updatedMetaData = adaptGlobalMetaData.apply(oldState.getLastAcceptedState().metaData());
-                        final ClusterState clusterState;
-                        if (updatedMetaData != oldState.getLastAcceptedState().metaData()) {
-                            clusterState = ClusterState.builder(oldState.getLastAcceptedState()).metaData(updatedMetaData).build();
-                        } else {
-                            clusterState = oldState.getLastAcceptedState();
-                        }
-                        clusterState.writeTo(outStream);
-                        StreamInput inStream = new NamedWriteableAwareStreamInput(outStream.bytes().streamInput(),
-                                new NamedWriteableRegistry(ClusterModule.getNamedWriteables()));
-                        delegate = new InMemoryPersistedState(adaptCurrentTerm.apply(oldState.getCurrentTerm()),
-                                ClusterState.readFrom(inStream, newLocalNode)); // adapts it to new localNode instance
-                    }
-                } catch (IOException e) {
-                    throw new UncheckedIOException("Unable to create MockPersistedState", e);
-                }
-            }
-
-            private void possiblyFail(String description) {
-                if (disruptStorage && rarely()) {
-                    logger.trace("simulating IO exception [{}]", description);
-                    // In the real-life IOError might be thrown, for example if state fsync fails.
-                    // This will require node restart and we're not emulating it here.
-                    throw new UncheckedIOException(new IOException("simulated IO exception [" + description + ']'));
-                }
-            }
-
-            @Override
-            public long getCurrentTerm() {
-                return delegate.getCurrentTerm();
-            }
-
-            @Override
-            public ClusterState getLastAcceptedState() {
-                return delegate.getLastAcceptedState();
-            }
+        final Cluster cluster = new Cluster(3, true, settings);
+        cluster.runRandomly();
+        cluster.stabilise();
 
-            @Override
-            public void setCurrentTerm(long currentTerm) {
-                possiblyFail("before writing term of " + currentTerm);
-                delegate.setCurrentTerm(currentTerm);
-            }
+        logger.info("--> disconnecting all nodes");
 
-            @Override
-            public void setLastAcceptedState(ClusterState clusterState) {
-                possiblyFail("before writing last-accepted state of term=" + clusterState.term() + ", version=" + clusterState.version());
-                delegate.setLastAcceptedState(clusterState);
-            }
+        for (final ClusterNode clusterNode : cluster.clusterNodes) {
+            clusterNode.disconnect();
         }
 
-        class ClusterNode {
-            private final Logger logger = LogManager.getLogger(ClusterNode.class);
-
-            private final int nodeIndex;
-            private Coordinator coordinator;
-            private final DiscoveryNode localNode;
-            private final MockPersistedState persistedState;
-            private final Settings nodeSettings;
-            private AckedFakeThreadPoolMasterService masterService;
-            private DisruptableClusterApplierService clusterApplierService;
-            private ClusterService clusterService;
-            private TransportService transportService;
-            private DisruptableMockTransport mockTransport;
-            private List<BiConsumer<DiscoveryNode, ClusterState>> extraJoinValidators = new ArrayList<>();
-
-            ClusterNode(int nodeIndex, boolean masterEligible, Settings nodeSettings) {
-                this(nodeIndex, createDiscoveryNode(nodeIndex, masterEligible), defaultPersistedStateSupplier, nodeSettings);
-            }
-
-            ClusterNode(int nodeIndex, DiscoveryNode localNode, Function<DiscoveryNode, MockPersistedState> persistedStateSupplier,
-                        Settings nodeSettings) {
-                this.nodeIndex = nodeIndex;
-                this.localNode = localNode;
-                this.nodeSettings = nodeSettings;
-                persistedState = persistedStateSupplier.apply(localNode);
-                onNodeLog(localNode, this::setUp).run();
-            }
-
-            private void setUp() {
-                mockTransport = new DisruptableMockTransport(localNode, logger) {
-                    @Override
-                    protected void execute(Runnable runnable) {
-                        deterministicTaskQueue.scheduleNow(onNode(runnable));
-                    }
-
-                    @Override
-                    protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) {
-                        return Cluster.this.getConnectionStatus(getLocalNode(), destination);
-                    }
-
-                    @Override
-                    protected Optional<DisruptableMockTransport> getDisruptableMockTransport(TransportAddress address) {
-                        return clusterNodes.stream().map(cn -> cn.mockTransport)
-                            .filter(transport -> transport.getLocalNode().getAddress().equals(address)).findAny();
-                    }
-                };
-
-                final Settings settings = nodeSettings.hasValue(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey()) ?
-                    nodeSettings : Settings.builder().put(nodeSettings)
-                    .putList(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(),
-                        ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.get(Settings.EMPTY)).build(); // suppress auto-bootstrap
-                transportService = mockTransport.createTransportService(
-                    settings, deterministicTaskQueue.getThreadPool(this::onNode), NOOP_TRANSPORT_INTERCEPTOR,
-                    a -> localNode, null, emptySet());
-                masterService = new AckedFakeThreadPoolMasterService(localNode.getId(), "test",
-                    runnable -> deterministicTaskQueue.scheduleNow(onNode(runnable)));
-                final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-                clusterApplierService = new DisruptableClusterApplierService(localNode.getId(), settings, clusterSettings,
-                    deterministicTaskQueue, this::onNode);
-                clusterService = new ClusterService(settings, clusterSettings, masterService, clusterApplierService);
-                clusterService.setNodeConnectionsService(
-                    new NodeConnectionsService(clusterService.getSettings(), deterministicTaskQueue.getThreadPool(this::onNode),
-                        transportService));
-                final Collection<BiConsumer<DiscoveryNode, ClusterState>> onJoinValidators =
-                    Collections.singletonList((dn, cs) -> extraJoinValidators.forEach(validator -> validator.accept(dn, cs)));
-                coordinator = new Coordinator("test_node", settings, clusterSettings, transportService, writableRegistry(),
-                    ESAllocationTestCase.createAllocationService(Settings.EMPTY), masterService, this::getPersistedState,
-                    Cluster.this::provideSeedHosts, clusterApplierService, onJoinValidators, Randomness.get());
-                masterService.setClusterStatePublisher(coordinator);
-
-                logger.trace("starting up [{}]", localNode);
-                transportService.start();
-                transportService.acceptIncomingRequests();
-                coordinator.start();
-                clusterService.start();
-                coordinator.startInitialJoin();
-            }
-
-            void close() {
-                onNode(() -> {
-                    logger.trace("taking down [{}]", localNode);
-                    coordinator.stop();
-                    clusterService.stop();
-                    //transportService.stop(); // does blocking stuff :/
-                    clusterService.close();
-                    coordinator.close();
-                    //transportService.close(); // does blocking stuff :/
-                });
-            }
-
-            ClusterNode restartedNode() {
-                return restartedNode(Function.identity(), Function.identity(), nodeSettings);
-            }
+        cluster.runFor(defaultMillis(LEADER_CHECK_INTERVAL_SETTING) + defaultMillis(LEADER_CHECK_TIMEOUT_SETTING),
+            "waiting for leader failure");
 
-            ClusterNode restartedNode(Function<MetaData, MetaData> adaptGlobalMetaData, Function<Long, Long> adaptCurrentTerm,
-                                      Settings nodeSettings) {
-                final TransportAddress address = randomBoolean() ? buildNewFakeTransportAddress() : localNode.getAddress();
-                final DiscoveryNode newLocalNode = new DiscoveryNode(localNode.getName(), localNode.getId(),
-                    UUIDs.randomBase64UUID(random()), // generated deterministically for repeatable tests
-                    address.address().getHostString(), address.getAddress(), address, Collections.emptyMap(),
-                    localNode.isMasterNode() ? EnumSet.allOf(Role.class) : emptySet(), Version.CURRENT);
-                return new ClusterNode(nodeIndex, newLocalNode,
-                    node -> new MockPersistedState(newLocalNode, persistedState, adaptGlobalMetaData, adaptCurrentTerm), nodeSettings);
+        for (int i = scaledRandomIntBetween(1, 10); i >= 0; i--) {
+            final MockLogAppender mockLogAppender;
+            try {
+                mockLogAppender = new MockLogAppender();
+            } catch (IllegalAccessException e) {
+                throw new AssertionError(e);
             }
 
-            private PersistedState getPersistedState() {
-                return persistedState;
-            }
-
-            String getId() {
-                return localNode.getId();
-            }
+            try {
+                Loggers.addAppender(LogManager.getLogger(ClusterFormationFailureHelper.class), mockLogAppender);
+                mockLogAppender.start();
+                mockLogAppender.addExpectation(new MockLogAppender.LoggingExpectation() {
+                    final Set<DiscoveryNode> nodesLogged = new HashSet<>();
 
-            DiscoveryNode getLocalNode() {
-                return localNode;
-            }
-
-            boolean isLeader() {
-                return coordinator.getMode() == LEADER;
-            }
-
-            boolean isCandidate() {
-                return coordinator.getMode() == CANDIDATE;
-            }
-
-            ClusterState improveConfiguration(ClusterState currentState) {
-                synchronized (coordinator.mutex) {
-                    return coordinator.improveConfiguration(currentState);
-                }
-            }
-
-            void setClusterStateApplyResponse(ClusterStateApplyResponse clusterStateApplyResponse) {
-                clusterApplierService.clusterStateApplyResponse = clusterStateApplyResponse;
-            }
-
-            ClusterStateApplyResponse getClusterStateApplyResponse() {
-                return clusterApplierService.clusterStateApplyResponse;
-            }
-
-            Runnable onNode(Runnable runnable) {
-                final Runnable wrapped = onNodeLog(localNode, runnable);
-                return new Runnable() {
                     @Override
-                    public void run() {
-                        if (clusterNodes.contains(ClusterNode.this) == false) {
-                            logger.trace("ignoring runnable {} from node {} as node has been removed from cluster", runnable, localNode);
-                            return;
-                        }
-                        wrapped.run();
-                    }
-
-                    @Override
-                    public String toString() {
-                        return wrapped.toString();
-                    }
-                };
-            }
+                    public void match(LogEvent event) {
+                        final String message = event.getMessage().getFormattedMessage();
+                        assertThat(message,
+                            startsWith("master not discovered or elected yet, an election requires at least 2 nodes with ids from ["));
 
-            void submitSetAutoShrinkVotingConfiguration(final boolean autoShrinkVotingConfiguration) {
-                submitUpdateTask("set master nodes failure tolerance [" + autoShrinkVotingConfiguration + "]", cs ->
-                    ClusterState.builder(cs).metaData(
-                        MetaData.builder(cs.metaData())
-                            .persistentSettings(Settings.builder()
-                                .put(cs.metaData().persistentSettings())
-                                .put(CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION.getKey(), autoShrinkVotingConfiguration)
-                                .build())
-                            .build())
-                        .build(), (source, e) -> {});
-            }
+                        final List<ClusterNode> matchingNodes = cluster.clusterNodes.stream()
+                            .filter(n -> event.getContextData().<String>getValue(NODE_ID_LOG_CONTEXT_KEY)
+                                .equals(getNodeIdForLogContext(n.getLocalNode()))).collect(Collectors.toList());
+                        assertThat(matchingNodes, hasSize(1));
 
-            AckCollector submitValue(final long value) {
-                return submitValue(0, value);
-            }
+                        assertTrue(Regex.simpleMatch("*have discovered *" + matchingNodes.get(0).toString() + "*discovery will continue*",
+                            message));
 
-            AckCollector submitValue(final int key, final long value) {
-                final int eventId = history.invoke(new Tuple<>(key, value));
-                return submitUpdateTask("new value [" + value + "]", cs -> setValue(cs, key, value), new ClusterStateTaskListener() {
-                    @Override
-                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                        history.respond(eventId, value(oldState, key));
+                        nodesLogged.add(matchingNodes.get(0).getLocalNode());
                     }
 
                     @Override
-                    public void onNoLongerMaster(String source) {
-                        // in this case, we know for sure that event was not processed by the system and will not change history
-                        // remove event to help avoid bloated history and state space explosion in linearizability checker
-                        history.remove(eventId);
-                    }
-
-                    @Override
-                    public void onFailure(String source, Exception e) {
-                        // do not remove event from history, the write might still take place
-                        // instead, complete history when checking for linearizability
+                    public void assertMatched() {
+                        assertThat(nodesLogged + " vs " + cluster.clusterNodes, nodesLogged,
+                            equalTo(cluster.clusterNodes.stream().map(ClusterNode::getLocalNode).collect(Collectors.toSet())));
                     }
                 });
-            }
-
-            void readValue(int key) {
-                final int eventId = history.invoke(new Tuple<>(key, null));
-                submitUpdateTask("read value", cs -> ClusterState.builder(cs).build(), new ClusterStateTaskListener() {
-                    @Override
-                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                        history.respond(eventId, value(newState, key));
-                    }
-
-                    @Override
-                    public void onFailure(String source, Exception e) {
-                        // reads do not change state
-                        // remove event to help avoid bloated history and state space explosion in linearizability checker
-                        history.remove(eventId);
-                    }
-                });
-            }
-
-            AckCollector submitUpdateTask(String source, UnaryOperator<ClusterState> clusterStateUpdate,
-                                          ClusterStateTaskListener taskListener) {
-                final AckCollector ackCollector = new AckCollector();
-                onNode(() -> {
-                    logger.trace("[{}] submitUpdateTask: enqueueing [{}]", localNode.getId(), source);
-                    final long submittedTerm = coordinator.getCurrentTerm();
-                    masterService.submitStateUpdateTask(source,
-                        new ClusterStateUpdateTask() {
-                            @Override
-                            public ClusterState execute(ClusterState currentState) {
-                                assertThat(currentState.term(), greaterThanOrEqualTo(submittedTerm));
-                                masterService.nextAckCollector = ackCollector;
-                                return clusterStateUpdate.apply(currentState);
-                            }
-
-                            @Override
-                            public void onFailure(String source, Exception e) {
-                                logger.debug(() -> new ParameterizedMessage("failed to publish: [{}]", source), e);
-                                taskListener.onFailure(source, e);
-                            }
-
-                            @Override
-                            public void onNoLongerMaster(String source) {
-                                logger.trace("no longer master: [{}]", source);
-                                taskListener.onNoLongerMaster(source);
-                            }
-
-                            @Override
-                            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                                updateCommittedStates();
-                                ClusterState state = committedStatesByVersion.get(newState.version());
-                                assertNotNull("State not committed : " + newState.toString(), state);
-                                assertStateEquals(state, newState);
-                                logger.trace("successfully published: [{}]", newState);
-                                taskListener.clusterStateProcessed(source, oldState, newState);
-                            }
-                        });
-                }).run();
-                return ackCollector;
-            }
-
-            @Override
-            public String toString() {
-                return localNode.toString();
-            }
-
-            boolean heal() {
-                boolean unBlackholed = blackholedNodes.remove(localNode.getId());
-                boolean unDisconnected = disconnectedNodes.remove(localNode.getId());
-                assert unBlackholed == false || unDisconnected == false;
-                return unBlackholed || unDisconnected;
-            }
-
-            boolean disconnect() {
-                boolean unBlackholed = blackholedNodes.remove(localNode.getId());
-                boolean disconnected = disconnectedNodes.add(localNode.getId());
-                assert disconnected || unBlackholed == false;
-                return disconnected;
-            }
-
-            boolean blackhole() {
-                boolean unDisconnected = disconnectedNodes.remove(localNode.getId());
-                boolean blackholed = blackholedNodes.add(localNode.getId());
-                assert blackholed || unDisconnected == false;
-                return blackholed;
-            }
-
-            void onDisconnectEventFrom(ClusterNode clusterNode) {
-                transportService.disconnectFromNode(clusterNode.localNode);
-            }
-
-            ClusterState getLastAppliedClusterState() {
-                return clusterApplierService.state();
-            }
-
-            void applyInitialConfiguration() {
-                onNode(() -> {
-                    final Set<String> nodeIdsWithPlaceholders = new HashSet<>(initialConfiguration.getNodeIds());
-                    Stream.generate(() -> BOOTSTRAP_PLACEHOLDER_PREFIX + UUIDs.randomBase64UUID(random()))
-                        .limit((Math.max(initialConfiguration.getNodeIds().size(), 2) - 1) / 2)
-                        .forEach(nodeIdsWithPlaceholders::add);
-                    final Set<String> nodeIds = new HashSet<>(
-                        randomSubsetOf(initialConfiguration.getNodeIds().size(), nodeIdsWithPlaceholders));
-                    // initial configuration should not have a place holder for local node
-                    if (initialConfiguration.getNodeIds().contains(localNode.getId()) && nodeIds.contains(localNode.getId()) == false) {
-                        nodeIds.remove(nodeIds.iterator().next());
-                        nodeIds.add(localNode.getId());
-                    }
-                    final VotingConfiguration configurationWithPlaceholders = new VotingConfiguration(nodeIds);
-                    try {
-                        coordinator.setInitialConfiguration(configurationWithPlaceholders);
-                        logger.info("successfully set initial configuration to {}", configurationWithPlaceholders);
-                    } catch (CoordinationStateRejectedException e) {
-                        logger.info(new ParameterizedMessage("failed to set initial configuration to {}",
-                            configurationWithPlaceholders), e);
-                    }
-                }).run();
-            }
-
-            private boolean isNotUsefullyBootstrapped() {
-                return getLocalNode().isMasterNode() == false || coordinator.isInitialConfigurationSet() == false;
+                cluster.runFor(warningDelayMillis + DEFAULT_DELAY_VARIABILITY, "waiting for warning to be emitted");
+                mockLogAppender.assertAllExpectationsMatched();
+            } finally {
+                mockLogAppender.stop();
+                Loggers.removeAppender(LogManager.getLogger(ClusterFormationFailureHelper.class), mockLogAppender);
             }
         }
-
-        private List<TransportAddress> provideSeedHosts(HostsResolver ignored) {
-            return seedHostsList != null ? seedHostsList
-                : clusterNodes.stream().map(ClusterNode::getLocalNode).map(DiscoveryNode::getAddress).collect(Collectors.toList());
-        }
     }
 
-    public static Runnable onNodeLog(DiscoveryNode node, Runnable runnable) {
-        final String nodeId = "{" + node.getId() + "}{" + node.getEphemeralId() + "}";
-        return new Runnable() {
-            @Override
-            public void run() {
-                try (CloseableThreadContext.Instance ignored = CloseableThreadContext.put("nodeId", nodeId)) {
-                    runnable.run();
-                }
-            }
 
-            @Override
-            public String toString() {
-                return nodeId + ": " + runnable.toString();
-            }
-        };
-    }
-
-    static class AckCollector implements AckListener {
-
-        private final Set<DiscoveryNode> ackedNodes = new HashSet<>();
-        private final List<DiscoveryNode> successfulNodes = new ArrayList<>();
-        private final List<DiscoveryNode> unsuccessfulNodes = new ArrayList<>();
-
-        @Override
-        public void onCommit(TimeValue commitTime) {
-            // TODO we only currently care about per-node acks
-        }
-
-        @Override
-        public void onNodeAck(DiscoveryNode node, Exception e) {
-            assertTrue("duplicate ack from " + node, ackedNodes.add(node));
-            if (e == null) {
-                successfulNodes.add(node);
-            } else {
-                unsuccessfulNodes.add(node);
-            }
-        }
-
-        boolean hasAckedSuccessfully(ClusterNode clusterNode) {
-            return successfulNodes.contains(clusterNode.localNode);
-        }
-
-        boolean hasAckedUnsuccessfully(ClusterNode clusterNode) {
-            return unsuccessfulNodes.contains(clusterNode.localNode);
-        }
-
-        boolean hasAcked(ClusterNode clusterNode) {
-            return ackedNodes.contains(clusterNode.localNode);
-        }
-
-        int getSuccessfulAckIndex(ClusterNode clusterNode) {
-            assert successfulNodes.contains(clusterNode.localNode) : "get index of " + clusterNode;
-            return successfulNodes.indexOf(clusterNode.localNode);
-        }
-    }
-
-    static class AckedFakeThreadPoolMasterService extends FakeThreadPoolMasterService {
-
-        AckCollector nextAckCollector = new AckCollector();
-
-        AckedFakeThreadPoolMasterService(String nodeName, String serviceName, Consumer<Runnable> onTaskAvailableToRun) {
-            super(nodeName, serviceName, onTaskAvailableToRun);
-        }
-
-        @Override
-        protected AckListener wrapAckListener(AckListener ackListener) {
-            final AckCollector ackCollector = nextAckCollector;
-            nextAckCollector = new AckCollector();
-            return new AckListener() {
-                @Override
-                public void onCommit(TimeValue commitTime) {
-                    ackCollector.onCommit(commitTime);
-                    ackListener.onCommit(commitTime);
-                }
-
-                @Override
-                public void onNodeAck(DiscoveryNode node, Exception e) {
-                    ackCollector.onNodeAck(node, e);
-                    ackListener.onNodeAck(node, e);
-                }
-            };
-        }
-    }
-
-    static class DisruptableClusterApplierService extends ClusterApplierService {
-        private final String nodeName;
-        private final DeterministicTaskQueue deterministicTaskQueue;
-        ClusterStateApplyResponse clusterStateApplyResponse = ClusterStateApplyResponse.SUCCEED;
-
-        DisruptableClusterApplierService(String nodeName, Settings settings, ClusterSettings clusterSettings,
-                                         DeterministicTaskQueue deterministicTaskQueue, Function<Runnable, Runnable> runnableWrapper) {
-            super(nodeName, settings, clusterSettings, deterministicTaskQueue.getThreadPool(runnableWrapper));
-            this.nodeName = nodeName;
-            this.deterministicTaskQueue = deterministicTaskQueue;
-            addStateApplier(event -> {
-                switch (clusterStateApplyResponse) {
-                    case SUCCEED:
-                    case HANG:
-                        final ClusterState oldClusterState = event.previousState();
-                        final ClusterState newClusterState = event.state();
-                        assert oldClusterState.version() <= newClusterState.version() : "updating cluster state from version "
-                            + oldClusterState.version() + " to stale version " + newClusterState.version();
-                        break;
-                    case FAIL:
-                        throw new ElasticsearchException("simulated cluster state applier failure");
-                }
-            });
-        }
-
-        @Override
-        protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() {
-            return new MockSinglePrioritizingExecutor(nodeName, deterministicTaskQueue);
-        }
-
-        @Override
-        public void onNewClusterState(String source, Supplier<ClusterState> clusterStateSupplier, ClusterApplyListener listener) {
-            if (clusterStateApplyResponse == ClusterStateApplyResponse.HANG) {
-                if (randomBoolean()) {
-                    // apply cluster state, but don't notify listener
-                    super.onNewClusterState(source, clusterStateSupplier, (source1, e) -> {
-                        // ignore result
-                    });
-                }
-            } else {
-                super.onNewClusterState(source, clusterStateSupplier, listener);
-            }
-        }
-
-        @Override
-        protected void connectToNodesAndWait(ClusterState newClusterState) {
-            // don't do anything, and don't block
-        }
-    }
-
-    private static DiscoveryNode createDiscoveryNode(int nodeIndex, boolean masterEligible) {
-        final TransportAddress address = buildNewFakeTransportAddress();
-        return new DiscoveryNode("", "node" + nodeIndex,
-            UUIDs.randomBase64UUID(random()), // generated deterministically for repeatable tests
-            address.address().getHostString(), address.getAddress(), address, Collections.emptyMap(),
-            masterEligible ? EnumSet.allOf(Role.class) : emptySet(), Version.CURRENT);
-    }
-
-    /**
-     * How to behave with a new cluster state
-     */
-    enum ClusterStateApplyResponse {
-        /**
-         * Apply the state (default)
-         */
-        SUCCEED,
-
-        /**
-         * Reject the state with an exception.
-         */
-        FAIL,
-
-        /**
-         * Never respond either way.
-         */
-        HANG,
-    }
-
-    public ClusterState setValue(ClusterState clusterState, int key, long value) {
-        return ClusterState.builder(clusterState).metaData(
-            MetaData.builder(clusterState.metaData())
-                .persistentSettings(Settings.builder()
-                    .put(clusterState.metaData().persistentSettings())
-                    .put("value_" + key, value)
-                    .build())
-                .build())
-            .build();
-    }
-
-    public long value(ClusterState clusterState) {
-        return value(clusterState, 0);
-    }
-
-    public long value(ClusterState clusterState, int key) {
-        return clusterState.metaData().persistentSettings().getAsLong("value_" + key, 0L);
-    }
-
-    public void assertStateEquals(ClusterState clusterState1, ClusterState clusterState2) {
-        assertEquals(clusterState1.version(), clusterState2.version());
-        assertEquals(clusterState1.term(), clusterState2.term());
-        assertEquals(keySet(clusterState1), keySet(clusterState2));
-        for (int key : keySet(clusterState1)) {
-            assertEquals(value(clusterState1, key), value(clusterState2, key));
-        }
-    }
-
-    public Set<Integer> keySet(ClusterState clusterState) {
-        return clusterState.metaData().persistentSettings().keySet().stream()
-            .filter(s -> s.startsWith("value_")).map(s -> Integer.valueOf(s.substring("value_".length()))).collect(Collectors.toSet());
-    }
-
-    /**
-     * Simple register model. Writes are modeled by providing an integer input. Reads are modeled by providing null as input.
-     * Responses that time out are modeled by returning null. Successful writes return the previous value of the register.
-     */
-    private final SequentialSpec spec = new LinearizabilityChecker.KeyedSpec() {
-        @Override
-        public Object getKey(Object value) {
-            return ((Tuple) value).v1();
-        }
-
-        @Override
-        public Object getValue(Object value) {
-            return ((Tuple) value).v2();
-        }
-
-        @Override
-        public Object initialState() {
-            return 0L;
-        }
-
-        @Override
-        public Optional<Object> nextState(Object currentState, Object input, Object output) {
-            // null input is read, non-null is write
-            if (input == null) {
-                // history is completed with null, simulating timeout, which assumes that read went through
-                if (output == null || currentState.equals(output)) {
-                    return Optional.of(currentState);
-                }
-                return Optional.empty();
-            } else {
-                if (output == null || currentState.equals(output)) {
-                    // history is completed with null, simulating timeout, which assumes that write went through
-                    return Optional.of(input);
-                }
-                return Optional.empty();
-            }
-        }
-    };
-
-    public void testRegisterSpecConsistency() {
-        assertThat(spec.initialState(), equalTo(0L));
-        assertThat(spec.nextState(7, 42, 7), equalTo(Optional.of(42))); // successful write 42 returns previous value 7
-        assertThat(spec.nextState(7, 42, null), equalTo(Optional.of(42))); // write 42 times out
-        assertThat(spec.nextState(7, null, 7), equalTo(Optional.of(7))); // successful read
-        assertThat(spec.nextState(7, null, null), equalTo(Optional.of(7))); // read times out
-        assertThat(spec.nextState(7, null, 42), equalTo(Optional.empty()));
-    }
 
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/FollowersCheckerTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/FollowersCheckerTests.java
index a57810b88e500..dc763f403d257 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/FollowersCheckerTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/FollowersCheckerTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.cluster.coordination.Coordinator.Mode;
 import org.elasticsearch.cluster.coordination.FollowersChecker.FollowerCheckRequest;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
@@ -43,7 +44,6 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -577,13 +577,13 @@ private static List<DiscoveryNode> randomNodes(final int numNodes) {
                 attributes.put("custom", randomBoolean() ? "match" : randomAlphaOfLengthBetween(3, 5));
             }
             final DiscoveryNode node = newNode(i, attributes,
-                new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))));
+                new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)));
             nodesList.add(node);
         }
         return nodesList;
     }
 
-    private static DiscoveryNode newNode(int nodeId, Map<String, String> attributes, Set<DiscoveryNode.Role> roles) {
+    private static DiscoveryNode newNode(int nodeId, Map<String, String> attributes, Set<DiscoveryNodeRole> roles) {
         return new DiscoveryNode("name_" + nodeId, "node_" + nodeId, buildNewFakeTransportAddress(), attributes, roles,
             Version.CURRENT);
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java
index 08501c4f55326..70567da1aadfb 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinHelperTests.java
@@ -57,7 +57,7 @@ public void testJoinDeduplication() {
             x -> localNode, null, Collections.emptySet());
         JoinHelper joinHelper = new JoinHelper(Settings.EMPTY, null, null, transportService, () -> 0L, () -> null,
             (joinRequest, joinCallback) -> { throw new AssertionError(); }, startJoinRequest -> { throw new AssertionError(); },
-            Collections.emptyList());
+            Collections.emptyList(), (s, r) -> {});
         transportService.start();
 
         DiscoveryNode node1 = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT);
@@ -153,7 +153,7 @@ public void testJoinValidationRejectsMismatchedClusterUUID() {
             x -> localNode, null, Collections.emptySet());
         new JoinHelper(Settings.EMPTY, null, null, transportService, () -> 0L, () -> localClusterState,
             (joinRequest, joinCallback) -> { throw new AssertionError(); }, startJoinRequest -> { throw new AssertionError(); },
-            Collections.emptyList()); // registers request handler
+            Collections.emptyList(), (s, r) -> {}); // registers request handler
         transportService.start();
         transportService.acceptIncomingRequests();
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinTaskExecutorTests.java
index e20559ca00561..0225ffe8fe6cd 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinTaskExecutorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinTaskExecutorTests.java
@@ -28,10 +28,8 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 
-import static org.elasticsearch.test.VersionUtils.getPreviousVersion;
 import static org.elasticsearch.test.VersionUtils.maxCompatibleVersion;
 import static org.elasticsearch.test.VersionUtils.randomCompatibleVersion;
-import static org.elasticsearch.test.VersionUtils.randomVersion;
 import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
 
 public class JoinTaskExecutorTests extends ESTestCase {
@@ -56,8 +54,7 @@ public void testPreventJoinClusterWithUnsupportedIndices() {
         Settings.builder().build();
         MetaData.Builder metaBuilder = MetaData.builder();
         IndexMetaData indexMetaData = IndexMetaData.builder("test")
-            .settings(settings(VersionUtils.getPreviousVersion(Version.CURRENT
-                .minimumIndexCompatibilityVersion())))
+            .settings(settings(Version.fromString("6.8.0"))) // latest V6 released version
             .numberOfShards(1)
             .numberOfReplicas(1).build();
         metaBuilder.put(indexMetaData, false);
@@ -69,27 +66,25 @@ public void testPreventJoinClusterWithUnsupportedIndices() {
 
     public void testPreventJoinClusterWithUnsupportedNodeVersions() {
         DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
-        final Version version = randomVersion(random());
+        final Version version = randomCompatibleVersion(random(), Version.CURRENT);
         builder.add(new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), version));
         builder.add(new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), randomCompatibleVersion(random(), version)));
         DiscoveryNodes nodes = builder.build();
 
         final Version maxNodeVersion = nodes.getMaxNodeVersion();
         final Version minNodeVersion = nodes.getMinNodeVersion();
-        if (maxNodeVersion.onOrAfter(Version.V_7_0_0)) {
-            final Version tooLow = getPreviousVersion(maxNodeVersion.minimumCompatibilityVersion());
-            expectThrows(IllegalStateException.class, () -> {
-                if (randomBoolean()) {
-                    JoinTaskExecutor.ensureNodesCompatibility(tooLow, nodes);
-                } else {
-                    JoinTaskExecutor.ensureNodesCompatibility(tooLow, minNodeVersion, maxNodeVersion);
-                }
-            });
-        }
 
-        Version oldMajor = Version.V_6_4_0.minimumCompatibilityVersion();
-        expectThrows(IllegalStateException.class, () -> JoinTaskExecutor.ensureMajorVersionBarrier(oldMajor, minNodeVersion));
+        final Version tooLow = Version.fromId(maxNodeVersion.minimumCompatibilityVersion().id - 100);
+        expectThrows(IllegalStateException.class, () -> {
+            if (randomBoolean()) {
+                JoinTaskExecutor.ensureNodesCompatibility(tooLow, nodes);
+            } else {
+                JoinTaskExecutor.ensureNodesCompatibility(tooLow, minNodeVersion, maxNodeVersion);
+            }
+        });
 
+        Version oldMajor = minNodeVersion.minimumCompatibilityVersion();
+        expectThrows(IllegalStateException.class, () -> JoinTaskExecutor.ensureMajorVersionBarrier(oldMajor, minNodeVersion));
 
         final Version minGoodVersion = maxNodeVersion.major == minNodeVersion.major ?
             // we have to stick with the same major
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/LeaderCheckerTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/LeaderCheckerTests.java
index a806cb84a6818..ce25d24bce6ba 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/LeaderCheckerTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/LeaderCheckerTests.java
@@ -52,9 +52,12 @@
 import static org.elasticsearch.node.Node.NODE_NAME_SETTING;
 import static org.elasticsearch.transport.TransportService.HANDSHAKE_ACTION_NAME;
 import static org.elasticsearch.transport.TransportService.NOOP_TRANSPORT_INTERCEPTOR;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.matchesRegex;
 import static org.hamcrest.Matchers.nullValue;
 
 public class LeaderCheckerTests extends ESTestCase {
@@ -146,7 +149,10 @@ public String toString() {
         final AtomicBoolean leaderFailed = new AtomicBoolean();
 
         final LeaderChecker leaderChecker = new LeaderChecker(settings, transportService,
-            () -> assertTrue(leaderFailed.compareAndSet(false, true)));
+            e -> {
+                assertThat(e.getMessage(), matchesRegex("node \\[.*\\] failed \\[[1-9][0-9]*\\] consecutive checks"));
+                assertTrue(leaderFailed.compareAndSet(false, true));
+            });
 
         logger.info("--> creating first checker");
         leaderChecker.updateLeader(leader1);
@@ -247,7 +253,10 @@ public String toString() {
 
         final AtomicBoolean leaderFailed = new AtomicBoolean();
         final LeaderChecker leaderChecker = new LeaderChecker(settings, transportService,
-            () -> assertTrue(leaderFailed.compareAndSet(false, true)));
+            e -> {
+                assertThat(e.getMessage(), anyOf(endsWith("disconnected"), endsWith("disconnected during check")));
+                assertTrue(leaderFailed.compareAndSet(false, true));
+            });
 
         leaderChecker.updateLeader(leader);
         {
@@ -316,7 +325,7 @@ public void testLeaderBehaviour() {
         transportService.start();
         transportService.acceptIncomingRequests();
 
-        final LeaderChecker leaderChecker = new LeaderChecker(settings, transportService, () -> fail("shouldn't be checking anything"));
+        final LeaderChecker leaderChecker = new LeaderChecker(settings, transportService, e -> fail("shouldn't be checking anything"));
 
         final DiscoveryNodes discoveryNodes
             = DiscoveryNodes.builder().add(localNode).localNodeId(localNode.getId()).masterNodeId(localNode.getId()).build();
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
index 108b9e6dd7f1e..37145e991b6c1 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java
@@ -22,11 +22,12 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.MasterService;
 import org.elasticsearch.cluster.service.MasterServiceTests;
@@ -39,7 +40,6 @@
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -55,7 +55,6 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Random;
@@ -76,7 +75,6 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
-@TestLogging("org.elasticsearch.cluster.service:TRACE,org.elasticsearch.cluster.coordination:TRACE")
 public class NodeJoinTests extends ESTestCase {
 
     private static ThreadPool threadPool;
@@ -176,7 +174,7 @@ transportService, writableRegistry(),
             () -> new InMemoryPersistedState(term, initialState), r -> emptyList(),
             new NoOpClusterApplier(),
             Collections.emptyList(),
-            random);
+            random, (s, r) -> {}, ElectionStrategy.DEFAULT_INSTANCE);
         transportService.start();
         transportService.acceptIncomingRequests();
         transport = capturingTransport;
@@ -189,9 +187,11 @@ protected DiscoveryNode newNode(int i) {
     }
 
     protected DiscoveryNode newNode(int i, boolean master) {
-        Set<DiscoveryNode.Role> roles = new HashSet<>();
+        final Set<DiscoveryNodeRole> roles;
         if (master) {
-            roles.add(DiscoveryNode.Role.MASTER);
+            roles = Set.of(DiscoveryNodeRole.MASTER_ROLE);
+        } else {
+            roles = Set.of();
         }
         final String prefix = master ? "master_" : "data_";
         return new DiscoveryNode(prefix + i, i + "", buildNewFakeTransportAddress(), emptyMap(), roles, Version.CURRENT);
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/PreVoteCollectorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/PreVoteCollectorTests.java
index 78896752bae4d..4d6578e5c5d12 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/PreVoteCollectorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/PreVoteCollectorTests.java
@@ -114,7 +114,7 @@ public String toString() {
             assert electionOccurred == false;
             electionOccurred = true;
         }, l -> {
-        }); // TODO need tests that check that the max term seen is updated
+        }, ElectionStrategy.DEFAULT_INSTANCE); // TODO need tests that check that the max term seen is updated
         preVoteCollector.update(getLocalPreVoteResponse(), null);
     }
 
@@ -233,8 +233,8 @@ public void testPrevotingIndicatesElectionSuccess() {
         DiscoveryNode[] votingNodes = votingNodesSet.toArray(new DiscoveryNode[0]);
         startAndRunCollector(votingNodes);
 
-        final CoordinationState coordinationState = new CoordinationState(Settings.EMPTY, localNode,
-            new InMemoryPersistedState(currentTerm, makeClusterState(votingNodes)));
+        final CoordinationState coordinationState = new CoordinationState(localNode,
+            new InMemoryPersistedState(currentTerm, makeClusterState(votingNodes)), ElectionStrategy.DEFAULT_INSTANCE);
 
         final long newTerm = randomLongBetween(currentTerm + 1, Long.MAX_VALUE);
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTests.java
index d4cab5110eef2..5050b12531859 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTests.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.Tuple;
@@ -71,7 +72,8 @@ class MockNode {
             this.localNode = localNode;
             ClusterState initialState = CoordinationStateTests.clusterState(0L, 0L, localNode,
                 CoordinationMetaData.VotingConfiguration.EMPTY_CONFIG, CoordinationMetaData.VotingConfiguration.EMPTY_CONFIG, 0L);
-            coordinationState = new CoordinationState(settings, localNode, new InMemoryPersistedState(0L, initialState));
+            coordinationState = new CoordinationState(localNode, new InMemoryPersistedState(0L, initialState),
+                ElectionStrategy.DEFAULT_INSTANCE);
         }
 
         final DiscoveryNode localNode;
@@ -464,13 +466,13 @@ private static List<DiscoveryNode> randomNodes(final int numNodes) {
                 attributes.put("custom", randomBoolean() ? "match" : randomAlphaOfLengthBetween(3, 5));
             }
             final DiscoveryNode node = newNode(i, attributes,
-                new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))));
+                new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)));
             nodesList.add(node);
         }
         return nodesList;
     }
 
-    private static DiscoveryNode newNode(int nodeId, Map<String, String> attributes, Set<DiscoveryNode.Role> roles) {
+    private static DiscoveryNode newNode(int nodeId, Map<String, String> attributes, Set<DiscoveryNodeRole> roles) {
         return new DiscoveryNode("name_" + nodeId, "node_" + nodeId, buildNewFakeTransportAddress(), attributes, roles,
             Version.CURRENT);
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java
index 27036680880b2..9658900f4ebe9 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java
@@ -50,7 +50,6 @@
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.disruption.BlockClusterStateProcessing;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.transport.TransportSettings;
 
 import java.util.List;
@@ -68,7 +67,6 @@
 import static org.hamcrest.Matchers.instanceOf;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
-@TestLogging("_root:DEBUG")
 public class RareClusterStateIT extends ESIntegTestCase {
 
     @Override
@@ -141,6 +139,10 @@ public void onFailure(String source, Exception e) {
 
     private <Req extends ActionRequest, Res extends ActionResponse> ActionFuture<Res> executeAndCancelCommittedPublication(
             ActionRequestBuilder<Req, Res> req) throws Exception {
+        // Wait for no publication in progress to not accidentally cancel a publication different from the one triggered by the given
+        // request.
+        assertBusy(
+            () -> assertFalse(((Coordinator) internalCluster().getCurrentMasterNodeInstance(Discovery.class)).publicationInProgress()));
         ActionFuture<Res> future = req.execute();
         assertBusy(
             () -> assertTrue(((Coordinator)internalCluster().getCurrentMasterNodeInstance(Discovery.class)).cancelCommittedPublication()));
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ReconfiguratorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ReconfiguratorTests.java
index bbd9514222c77..a1d12d98398ca 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/ReconfiguratorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ReconfiguratorTests.java
@@ -53,7 +53,7 @@ public void testReconfigurationExamples() {
 
         check(nodes("a"), conf("a"), true, conf("a"));
         check(nodes("a", "b"), conf("a"), true, conf("a"));
-        check(nodes("a", "b"), conf("b"), true, conf("b"));
+        check(nodes("a", "b"), conf("b"), true, conf("a"));
         check(nodes("a", "b"), conf("a", "c"), true, conf("a"));
         check(nodes("a", "b"), conf("a", "b"), true, conf("a"));
         check(nodes("a", "b"), conf("a", "b", "e"), true, conf("a", "b", "e"));
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java
index 44f4d7bf4aa53..6f6dde9f4f3f7 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java
@@ -36,7 +36,6 @@
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -52,8 +51,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false)
-@TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.cluster.coordination:TRACE")
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class UnsafeBootstrapAndDetachCommandIT extends ESIntegTestCase {
 
     private MockTerminal executeCommand(ElasticsearchNodeCommand command, Environment environment, boolean abort)
@@ -404,7 +402,7 @@ public boolean clearData(String nodeName) {
 
         logger.info("--> verify that the dangling index exists and has green status");
         assertBusy(() -> {
-            assertThat(client().admin().indices().prepareExists("test").execute().actionGet().isExists(), equalTo(true));
+            assertThat(indexExists("test"), equalTo(true));
         });
         ensureGreen("test");
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java
index 8c6775cb6c91e..c3c3abf8866b7 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java
@@ -18,17 +18,38 @@
  */
 package org.elasticsearch.cluster.coordination;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction;
 import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Priority;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.transport.TransportService;
 
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.nullValue;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class VotingConfigurationIT extends ESIntegTestCase {
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singletonList(MockTransportService.TestPlugin.class);
+    }
+
     public void testAbdicateAfterVotingConfigExclusionAdded() throws ExecutionException, InterruptedException {
+        internalCluster().setBootstrapMasterNodeIndex(0);
         internalCluster().startNodes(2);
         final String originalMaster = internalCluster().getMasterName();
 
@@ -38,4 +59,56 @@ public void testAbdicateAfterVotingConfigExclusionAdded() throws ExecutionExcept
         client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).get();
         assertNotEquals(originalMaster, internalCluster().getMasterName());
     }
+
+    public void testElectsNodeNotInVotingConfiguration() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        final List<String> nodeNames = internalCluster().startNodes(4);
+
+        // a 4-node cluster settles on a 3-node configuration; we then prevent the nodes in the configuration from winning an election
+        // by failing at the pre-voting stage, so that the extra node must be elected instead when the master shuts down. This extra node
+        // should then add itself into the voting configuration.
+
+        assertFalse(internalCluster().client().admin().cluster().prepareHealth()
+            .setWaitForNodes("4").setWaitForEvents(Priority.LANGUID).get().isTimedOut());
+
+        String excludedNodeName = null;
+        final ClusterState clusterState
+            = internalCluster().client().admin().cluster().prepareState().clear().setNodes(true).setMetaData(true).get().getState();
+        final Set<String> votingConfiguration = clusterState.getLastCommittedConfiguration().getNodeIds();
+        assertThat(votingConfiguration, hasSize(3));
+        assertThat(clusterState.nodes().getSize(), equalTo(4));
+        assertThat(votingConfiguration, hasItem(clusterState.nodes().getMasterNodeId()));
+        for (DiscoveryNode discoveryNode : clusterState.nodes()) {
+            if (votingConfiguration.contains(discoveryNode.getId()) == false) {
+                assertThat(excludedNodeName, nullValue());
+                excludedNodeName = discoveryNode.getName();
+            }
+        }
+
+        for (final String sender : nodeNames) {
+            if (sender.equals(excludedNodeName)) {
+                continue;
+            }
+            final MockTransportService senderTransportService
+                = (MockTransportService) internalCluster().getInstance(TransportService.class, sender);
+            for (final String receiver : nodeNames) {
+                senderTransportService.addSendBehavior(internalCluster().getInstance(TransportService.class, receiver),
+                    (connection, requestId, action, request, options) -> {
+                        if (action.equals(PreVoteCollector.REQUEST_PRE_VOTE_ACTION_NAME)) {
+                            throw new ElasticsearchException("rejected");
+                        }
+                        connection.sendRequest(requestId, action, request, options);
+                    });
+            }
+        }
+
+        internalCluster().stopCurrentMasterNode();
+        assertFalse(internalCluster().client().admin().cluster().prepareHealth()
+            .setWaitForNodes("3").setWaitForEvents(Priority.LANGUID).get().isTimedOut());
+
+        final ClusterState newClusterState
+            = internalCluster().client().admin().cluster().prepareState().clear().setNodes(true).setMetaData(true).get().getState();
+        assertThat(newClusterState.nodes().getMasterNode().getName(), equalTo(excludedNodeName));
+        assertThat(newClusterState.getLastCommittedConfiguration().getNodeIds(), hasItem(newClusterState.nodes().getMasterNodeId()));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ZenDiscoveryIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ZenDiscoveryIT.java
index feffbfc792656..61bf1fdcc7896 100644
--- a/server/src/test/java/org/elasticsearch/cluster/coordination/ZenDiscoveryIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ZenDiscoveryIT.java
@@ -38,7 +38,6 @@
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.TestCustomMetaData;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.transport.RemoteTransportException;
 
 import java.util.EnumSet;
@@ -55,7 +54,6 @@
 import static org.hamcrest.Matchers.notNullValue;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
-@TestLogging("_root:DEBUG")
 public class ZenDiscoveryIT extends ESIntegTestCase {
 
     public void testNoShardRelocationsOccurWhenElectedMasterNodeFails() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java
index b96d5eacb15d1..f78104201c9fb 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java
@@ -25,11 +25,11 @@
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.indices.cluster.ClusterStateChanges;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -104,11 +104,9 @@ public void testInvalidValues() {
 
     private static final AtomicInteger nodeIdGenerator = new AtomicInteger();
 
-    protected DiscoveryNode createNode(DiscoveryNode.Role... mustHaveRoles) {
-        Set<DiscoveryNode.Role> roles = new HashSet<>(randomSubsetOf(Sets.newHashSet(DiscoveryNode.Role.values())));
-        for (DiscoveryNode.Role mustHaveRole : mustHaveRoles) {
-            roles.add(mustHaveRole);
-        }
+    protected DiscoveryNode createNode(DiscoveryNodeRole... mustHaveRoles) {
+        Set<DiscoveryNodeRole> roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES));
+        Collections.addAll(roles, mustHaveRoles);
         final String id = String.format(Locale.ROOT, "node_%03d", nodeIdGenerator.incrementAndGet());
         return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), roles,
             Version.CURRENT);
@@ -126,12 +124,12 @@ public void testAutoExpandWhenNodeLeavesAndPossiblyRejoins() throws InterruptedE
 
         try {
             List<DiscoveryNode> allNodes = new ArrayList<>();
-            DiscoveryNode localNode = createNode(DiscoveryNode.Role.MASTER); // local node is the master
+            DiscoveryNode localNode = createNode(DiscoveryNodeRole.MASTER_ROLE); // local node is the master
             allNodes.add(localNode);
             int numDataNodes = randomIntBetween(3, 5);
             List<DiscoveryNode> dataNodes = new ArrayList<>(numDataNodes);
             for (int i = 0; i < numDataNodes; i++) {
-                dataNodes.add(createNode(DiscoveryNode.Role.DATA));
+                dataNodes.add(createNode(DiscoveryNodeRole.DATA_ROLE));
             }
             allNodes.addAll(dataNodes);
             ClusterState state = ClusterStateCreationUtils.state(localNode, localNode, allNodes.toArray(new DiscoveryNode[0]));
@@ -179,7 +177,7 @@ public void testAutoExpandWhenNodeLeavesAndPossiblyRejoins() throws InterruptedE
                     .collect(Collectors.toList());
 
                 if (randomBoolean()) {
-                    nodesToAdd.add(createNode(DiscoveryNode.Role.DATA));
+                    nodesToAdd.add(createNode(DiscoveryNodeRole.DATA_ROLE));
                 }
 
                 state = cluster.joinNodesAndBecomeMaster(state, nodesToAdd);
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
index 3de716acfee44..3eef8ff686eef 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -397,7 +398,7 @@ private DiscoveryNode newNode(String nodeId) {
                 nodeId,
                 buildNewFakeTransportAddress(),
                 emptyMap(),
-                Set.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA), Version.CURRENT);
+                Set.of(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.DATA_ROLE), Version.CURRENT);
     }
 
     public void testValidateIndexName() throws Exception {
@@ -433,12 +434,6 @@ public void testCalculateNumRoutingShards() {
         assertEquals(2048, MetaDataCreateIndexService.calculateNumRoutingShards(1024, Version.CURRENT));
         assertEquals(4096, MetaDataCreateIndexService.calculateNumRoutingShards(2048, Version.CURRENT));
 
-        Version latestV6 = VersionUtils.getPreviousVersion(Version.V_7_0_0);
-        int numShards = randomIntBetween(1, 1000);
-        assertEquals(numShards, MetaDataCreateIndexService.calculateNumRoutingShards(numShards, latestV6));
-        assertEquals(numShards, MetaDataCreateIndexService.calculateNumRoutingShards(numShards,
-            VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), latestV6)));
-
         for (int i = 0; i < 1000; i++) {
             int randomNumShards = randomIntBetween(1, 10000);
             int numRoutingShards = MetaDataCreateIndexService.calculateNumRoutingShards(randomNumShards, Version.CURRENT);
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexServiceTests.java
index 5905d528ff43f..70d1ef34b092b 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexServiceTests.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInProgressException;
+import org.elasticsearch.snapshots.SnapshotInfoTests;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 
@@ -60,7 +61,8 @@ public void testDeleteSnapshotting() {
         Snapshot snapshot = new Snapshot("doesn't matter", new SnapshotId("snapshot name", "snapshot uuid"));
         SnapshotsInProgress snaps = new SnapshotsInProgress(new SnapshotsInProgress.Entry(snapshot, true, false,
                 SnapshotsInProgress.State.INIT, singletonList(new IndexId(index, "doesn't matter")),
-                System.currentTimeMillis(), (long) randomIntBetween(0, 1000), ImmutableOpenMap.of()));
+                System.currentTimeMillis(), (long) randomIntBetween(0, 1000), ImmutableOpenMap.of(),
+                SnapshotInfoTests.randomUserMetadata()));
         ClusterState state = ClusterState.builder(clusterState(index))
                 .putCustom(SnapshotsInProgress.TYPE, snaps)
                 .build();
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java
index b655a98379553..96ce35dda77df 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
@@ -48,6 +49,7 @@
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInProgressException;
+import org.elasticsearch.snapshots.SnapshotInfoTests;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
@@ -150,9 +152,9 @@ public void testCloseRoutingTableRemovesRoutingTable() {
         state = ClusterState.builder(state)
             .nodes(DiscoveryNodes.builder(state.nodes())
                 .add(new DiscoveryNode("old_node", buildNewFakeTransportAddress(), emptyMap(),
-                    new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.V_7_0_0))
+                    new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.V_7_0_0))
                 .add(new DiscoveryNode("new_node", buildNewFakeTransportAddress(), emptyMap(),
-                    new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.V_7_2_0)))
+                    new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.V_7_2_0)))
             .build();
 
         state = MetaDataIndexStateService.closeRoutingTable(state, blockedIndices, results).v1();
@@ -432,7 +434,8 @@ private static ClusterState addSnapshotIndex(final String index, final int numSh
         final Snapshot snapshot = new Snapshot(randomAlphaOfLength(10), new SnapshotId(randomAlphaOfLength(5), randomAlphaOfLength(5)));
         final SnapshotsInProgress.Entry entry =
             new SnapshotsInProgress.Entry(snapshot, randomBoolean(), false, SnapshotsInProgress.State.INIT,
-                Collections.singletonList(new IndexId(index, index)), randomNonNegativeLong(), randomLong(), shardsBuilder.build());
+                Collections.singletonList(new IndexId(index, index)), randomNonNegativeLong(), randomLong(), shardsBuilder.build(),
+                SnapshotInfoTests.randomUserMetadata());
         return ClusterState.builder(newState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(entry)).build();
     }
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java
index 4bff7f8dc6198..b6b812b5219ee 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -46,7 +47,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -247,8 +247,8 @@ public void testUpdateTemplates() {
         assertThat(service.upgradesInProgress.get(), equalTo(2));
     }
 
-    private static final Set<DiscoveryNode.Role> MASTER_DATA_ROLES =
-        Collections.unmodifiableSet(EnumSet.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA));
+    private static final Set<DiscoveryNodeRole> MASTER_DATA_ROLES =
+            Set.of(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.DATA_ROLE);
 
     @SuppressWarnings("unchecked")
     public void testClusterStateUpdate() throws InterruptedException {
diff --git a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleIT.java b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleIT.java
new file mode 100644
index 0000000000000..06c4b14cae417
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleIT.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.node;
+
+import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.hamcrest.Matcher;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.not;
+
+public class DiscoveryNodeRoleIT extends ESIntegTestCase {
+
+    public static class AdditionalRolePlugin extends Plugin {
+
+        public AdditionalRolePlugin() {
+
+        }
+
+        static final Setting<Boolean> NODE_ADDITIONAL_SETTING =
+                Setting.boolSetting("node.additional", true, Setting.Property.NodeScope);
+
+        static DiscoveryNodeRole ADDITIONAL_ROLE = new DiscoveryNodeRole("additional", "a") {
+
+            @Override
+            protected Setting<Boolean> roleSetting() {
+                return NODE_ADDITIONAL_SETTING;
+            }
+
+        };
+
+        @Override
+        public Set<DiscoveryNodeRole> getRoles() {
+            return Set.of(ADDITIONAL_ROLE);
+        }
+
+        @Override
+        public List<Setting<?>> getSettings() {
+            return List.of(NODE_ADDITIONAL_SETTING);
+        }
+
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(AdditionalRolePlugin.class);
+    }
+
+    public void testDefaultHasAdditionalRole() {
+        runTestNodeHasAdditionalRole(Settings.EMPTY);
+    }
+
+    public void testExplicitlyHasAdditionalRole() {
+        runTestNodeHasAdditionalRole(Settings.builder().put(AdditionalRolePlugin.NODE_ADDITIONAL_SETTING.getKey(), true).build());
+    }
+
+    public void testDoesNotHaveAdditionalRole() {
+        runTestNodeHasAdditionalRole(Settings.builder().put(AdditionalRolePlugin.NODE_ADDITIONAL_SETTING.getKey(), false).build());
+    }
+
+    private void runTestNodeHasAdditionalRole(final Settings settings) {
+        final String name = internalCluster().startNode(settings);
+        final NodesInfoResponse response = client().admin().cluster().prepareNodesInfo(name).get();
+        assertThat(response.getNodes(), hasSize(1));
+        final Matcher<Iterable<? super DiscoveryNodeRole>> matcher;
+        if (AdditionalRolePlugin.NODE_ADDITIONAL_SETTING.get(settings)) {
+            matcher = hasItem(AdditionalRolePlugin.ADDITIONAL_ROLE);
+        } else {
+            matcher = not(hasItem(AdditionalRolePlugin.ADDITIONAL_ROLE));
+        }
+        assertThat(response.getNodes().get(0).getNode().getRoles(), matcher);
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleTests.java b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleTests.java
new file mode 100644
index 0000000000000..7e03a32b7b163
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleTests.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.node;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+
+import java.util.Set;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasToString;
+
+public class DiscoveryNodeRoleTests extends ESTestCase {
+
+    public void testDiscoveryNodeSetPossibleRolesRejectsDuplicateRoleNames() {
+        final IllegalStateException e = expectThrows(
+                IllegalStateException.class,
+                () -> DiscoveryNode.setPossibleRoles(Set.of(
+                        new DiscoveryNodeRole("foo", "f") {
+
+                            @Override
+                            protected Setting<Boolean> roleSetting() {
+                                return null;
+                            }
+
+                        },
+                        new DiscoveryNodeRole("foo", "f") {
+
+                            @Override
+                            protected Setting<Boolean> roleSetting() {
+                                return null;
+                            }
+
+                        })));
+        assertThat(e, hasToString(containsString("Duplicate key")));
+    }
+
+    public void testDiscoveryNodeSetPossibleRolesRejectsDuplicateRoleNameAbbreviations() {
+        final IllegalStateException e = expectThrows(
+                IllegalStateException.class,
+                () -> DiscoveryNode.setPossibleRoles(Set.of(
+                        new DiscoveryNodeRole("foo_1", "f") {
+
+                            @Override
+                            protected Setting<Boolean> roleSetting() {
+                                return null;
+                            }
+
+                        },
+                        new DiscoveryNodeRole("foo_2", "f") {
+
+                            @Override
+                            protected Setting<Boolean> roleSetting() {
+                                return null;
+                            }
+
+                        })));
+        assertThat(e, hasToString(containsString("Duplicate key")));
+    }
+
+    public void testDiscoveryNodeRoleEqualsHashCode() {
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(new DiscoveryNodeRole.UnknownRole(randomAlphaOfLength(10), randomAlphaOfLength(1)),
+            r -> new DiscoveryNodeRole.UnknownRole(r.roleName(), r.roleNameAbbreviation()),
+            r -> {
+                if (randomBoolean()) {
+                    return new DiscoveryNodeRole.UnknownRole(randomAlphaOfLength(21 - r.roleName().length()), r.roleNameAbbreviation());
+                } else {
+                    return new DiscoveryNodeRole.UnknownRole(r.roleName(), randomAlphaOfLength(3 - r.roleNameAbbreviation().length()));
+                }
+            });
+
+    }
+
+    public void testUnknownRoleIsDistinctFromKnownRoles() {
+        for (DiscoveryNodeRole buildInRole : DiscoveryNodeRole.BUILT_IN_ROLES) {
+            final DiscoveryNodeRole.UnknownRole unknownDataRole
+                = new DiscoveryNodeRole.UnknownRole(buildInRole.roleName(), buildInRole.roleNameAbbreviation());
+            assertNotEquals(buildInRole, unknownDataRole);
+            assertNotEquals(buildInRole.toString(), unknownDataRole.toString());
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java
index 5c92029520e8d..2cdbd95fc6358 100644
--- a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java
@@ -21,6 +21,7 @@
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import org.elasticsearch.Version;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.test.ESTestCase;
 
@@ -240,8 +241,16 @@ private static List<DiscoveryNode> randomNodes(final int numNodes) {
             if (frequently()) {
                 attributes.put("custom", randomBoolean() ? "match" : randomAlphaOfLengthBetween(3, 5));
             }
-            final DiscoveryNode node = newNode(idGenerator.getAndIncrement(), attributes,
-                new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))));
+            final Set<DiscoveryNodeRole> roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES));
+            if (frequently()) {
+                roles.add(new DiscoveryNodeRole("custom_role", "cr") {
+                    @Override
+                    protected Setting<Boolean> roleSetting() {
+                        return null;
+                    }
+                });
+            }
+            final DiscoveryNode node = newNode(idGenerator.getAndIncrement(), attributes, roles);
             nodesList.add(node);
         }
         return nodesList;
@@ -259,7 +268,7 @@ private static DiscoveryNodes buildDiscoveryNodes() {
         return discoBuilder.build();
     }
 
-    private static DiscoveryNode newNode(int nodeId, Map<String, String> attributes, Set<DiscoveryNode.Role> roles) {
+    private static DiscoveryNode newNode(int nodeId, Map<String, String> attributes, Set<DiscoveryNodeRole> roles) {
         return new DiscoveryNode("name_" + nodeId, "node_" + nodeId, buildNewFakeTransportAddress(), attributes, roles,
             Version.CURRENT);
     }
@@ -275,21 +284,21 @@ Set<String> matchingNodeIds(DiscoveryNodes nodes) {
             Set<String> matchingNodeIds(DiscoveryNodes nodes) {
                 return Collections.singleton(nodes.getMasterNodeId());
             }
-        }, MASTER_ELIGIBLE(DiscoveryNode.Role.MASTER.getRoleName() + ":true") {
+        }, MASTER_ELIGIBLE(DiscoveryNodeRole.MASTER_ROLE.roleName() + ":true") {
             @Override
             Set<String> matchingNodeIds(DiscoveryNodes nodes) {
                 Set<String> ids = new HashSet<>();
                 nodes.getMasterNodes().keysIt().forEachRemaining(ids::add);
                 return ids;
             }
-        }, DATA(DiscoveryNode.Role.DATA.getRoleName() + ":true") {
+        }, DATA(DiscoveryNodeRole.DATA_ROLE.roleName() + ":true") {
             @Override
             Set<String> matchingNodeIds(DiscoveryNodes nodes) {
                 Set<String> ids = new HashSet<>();
                 nodes.getDataNodes().keysIt().forEachRemaining(ids::add);
                 return ids;
             }
-        }, INGEST(DiscoveryNode.Role.INGEST.getRoleName() + ":true") {
+        }, INGEST(DiscoveryNodeRole.INGEST_ROLE.roleName() + ":true") {
             @Override
             Set<String> matchingNodeIds(DiscoveryNodes nodes) {
                 Set<String> ids = new HashSet<>();
@@ -314,6 +323,17 @@ Set<String> matchingNodeIds(DiscoveryNodes nodes) {
                 });
                 return ids;
             }
+        }, CUSTOM_ROLE("custom_role:true") {
+            @Override
+            Set<String> matchingNodeIds(DiscoveryNodes nodes) {
+                Set<String> ids = new HashSet<>();
+                nodes.getNodes().valuesIt().forEachRemaining(node -> {
+                    if (node.getRoles().stream().anyMatch(role -> role.roleName().equals("custom_role"))) {
+                        ids.add(node.getId());
+                    }
+                });
+                return ids;
+            }
         };
 
         private final String selector;
@@ -328,13 +348,13 @@ Set<String> matchingNodeIds(DiscoveryNodes nodes) {
     public void testMaxMinNodeVersion() {
         DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
         discoBuilder.add(new DiscoveryNode("name_" + 1, "node_" + 1, buildNewFakeTransportAddress(), Collections.emptyMap(),
-            new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))),
+            new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)),
             Version.fromString("5.1.0")));
         discoBuilder.add(new DiscoveryNode("name_" + 2, "node_" + 2, buildNewFakeTransportAddress(), Collections.emptyMap(),
-            new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))),
+            new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)),
             Version.fromString("6.3.0")));
         discoBuilder.add(new DiscoveryNode("name_" + 3, "node_" + 3, buildNewFakeTransportAddress(), Collections.emptyMap(),
-            new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))),
+            new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)),
             Version.fromString("1.1.0")));
         discoBuilder.localNodeId("name_1");
         discoBuilder.masterNodeId("name_2");
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/BatchedRerouteServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/BatchedRerouteServiceTests.java
new file mode 100644
index 0000000000000..966ac1e60650d
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/BatchedRerouteServiceTests.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.routing;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.test.ClusterServiceUtils;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.hamcrest.Matchers.lessThan;
+
+public class BatchedRerouteServiceTests extends ESTestCase {
+
+    private ThreadPool threadPool;
+    private ClusterService clusterService;
+
+    @Before
+    public void beforeTest() {
+        threadPool = new TestThreadPool("test");
+        clusterService = ClusterServiceUtils.createClusterService(threadPool);
+    }
+
+    @After
+    public void afterTest() {
+        clusterService.stop();
+        threadPool.shutdown();
+    }
+
+    public void testReroutesWhenRequested() throws InterruptedException {
+        final AtomicLong rerouteCount = new AtomicLong();
+        final BatchedRerouteService batchedRerouteService = new BatchedRerouteService(clusterService, (s, r) -> {
+            rerouteCount.incrementAndGet();
+            return s;
+        });
+
+        long rerouteCountBeforeReroute = 0L;
+        final int iterations = between(1, 100);
+        final CountDownLatch countDownLatch = new CountDownLatch(iterations);
+        for (int i = 0; i < iterations; i++) {
+            rerouteCountBeforeReroute = Math.max(rerouteCountBeforeReroute, rerouteCount.get());
+            batchedRerouteService.reroute("iteration " + i, ActionListener.wrap(countDownLatch::countDown));
+        }
+        countDownLatch.await(10, TimeUnit.SECONDS);
+        assertThat(rerouteCountBeforeReroute, lessThan(rerouteCount.get()));
+    }
+
+    public void testBatchesReroutesTogether() throws BrokenBarrierException, InterruptedException {
+        final CyclicBarrier cyclicBarrier = new CyclicBarrier(2);
+        clusterService.submitStateUpdateTask("block master service", new ClusterStateUpdateTask() {
+            @Override
+            public ClusterState execute(ClusterState currentState) throws Exception {
+                cyclicBarrier.await(); // notify test that we are blocked
+                cyclicBarrier.await(); // wait to be unblocked by test
+                return currentState;
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                throw new AssertionError(source, e);
+            }
+        });
+
+        cyclicBarrier.await(); // wait for master thread to be blocked
+
+        final AtomicBoolean rerouteExecuted = new AtomicBoolean();
+        final BatchedRerouteService batchedRerouteService = new BatchedRerouteService(clusterService, (s, r) -> {
+            assertTrue(rerouteExecuted.compareAndSet(false, true)); // only called once
+            return s;
+        });
+
+        final int iterations = between(1, 100);
+        final CountDownLatch countDownLatch = new CountDownLatch(iterations);
+        for (int i = 0; i < iterations; i++) {
+            batchedRerouteService.reroute("iteration " + i, ActionListener.wrap(countDownLatch::countDown));
+        }
+
+        cyclicBarrier.await(); // allow master thread to continue;
+        countDownLatch.await(); // wait for reroute to complete
+        assertTrue(rerouteExecuted.get()); // see above for assertion that it's only called once
+    }
+
+    public void testNotifiesOnFailure() throws InterruptedException {
+
+        final BatchedRerouteService batchedRerouteService = new BatchedRerouteService(clusterService, (s, r) -> {
+            if (rarely()) {
+                throw new ElasticsearchException("simulated");
+            }
+            return randomBoolean() ? s : ClusterState.builder(s).build();
+        });
+
+        final int iterations = between(1, 100);
+        final CountDownLatch countDownLatch = new CountDownLatch(iterations);
+        for (int i = 0; i < iterations; i++) {
+            batchedRerouteService.reroute("iteration " + i, ActionListener.wrap(countDownLatch::countDown));
+            if (rarely()) {
+                clusterService.getMasterService().setClusterStatePublisher(
+                    randomBoolean()
+                        ? ClusterServiceUtils.createClusterStatePublisher(clusterService.getClusterApplierService())
+                        : (event, publishListener, ackListener)
+                        -> publishListener.onFailure(new FailedToCommitClusterStateException("simulated")));
+            }
+
+            if (rarely()) {
+                clusterService.getClusterApplierService().onNewClusterState("simulated", () -> {
+                    ClusterState state = clusterService.state();
+                    return ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes())
+                        .masterNodeId(randomBoolean() ? null : state.nodes().getLocalNodeId())).build();
+                }, (source, e) -> { });
+            }
+        }
+
+        assertTrue(countDownLatch.await(10, TimeUnit.SECONDS)); // i.e. it doesn't leak any listeners
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
index 6897380ee9f01..098803e3cfd53 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -223,7 +223,7 @@ public void testDelayedUnassignedScheduleRerouteAfterDelayedReroute() throws Exc
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData)
             .routingTable(RoutingTable.builder().addAsNew(metaData.index("short_delay")).addAsNew(metaData.index("long_delay")).build())
             .nodes(DiscoveryNodes.builder()
-                .add(newNode("node0", singleton(DiscoveryNode.Role.MASTER))).localNodeId("node0").masterNodeId("node0")
+                .add(newNode("node0", singleton(DiscoveryNodeRole.MASTER_ROLE))).localNodeId("node0").masterNodeId("node0")
                 .add(newNode("node1")).add(newNode("node2")).add(newNode("node3")).add(newNode("node4"))).build();
         // allocate shards
         clusterState = allocationService.reroute(clusterState, "reroute");
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java b/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java
index 0e6b24c45d169..5ffa7dbfe444c 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java
@@ -53,7 +53,6 @@
 import org.elasticsearch.test.disruption.NetworkDisruption;
 import org.elasticsearch.test.disruption.NetworkDisruption.NetworkDisconnect;
 import org.elasticsearch.test.disruption.NetworkDisruption.TwoPartitions;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 
 import java.util.ArrayList;
@@ -491,8 +490,6 @@ public void testForceAllocatePrimaryOnNoDecision() throws Exception {
     /**
      * This test asserts that replicas failed to execute resync operations will be failed but not marked as stale.
      */
-    @TestLogging("_root:DEBUG, org.elasticsearch.cluster.routing.allocation:TRACE, org.elasticsearch.cluster.action.shard:TRACE," +
-        "org.elasticsearch.indices.recovery:TRACE, org.elasticsearch.cluster.routing.allocation.allocator:TRACE")
     public void testPrimaryReplicaResyncFailed() throws Exception {
         String master = internalCluster().startMasterOnlyNode(Settings.EMPTY);
         final int numberOfReplicas = between(2, 3);
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
index c966e3cac27dc..f3dbdd0eb2552 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RoutingNode;
@@ -60,7 +61,6 @@
 import org.elasticsearch.index.shard.ShardNotFoundException;
 
 import java.util.Collections;
-import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -160,7 +160,7 @@ public void testAllocateCommand() {
                 .add(newNode("node1"))
                 .add(newNode("node2"))
                 .add(newNode("node3"))
-                .add(newNode("node4", singleton(DiscoveryNode.Role.MASTER)))
+                .add(newNode("node4", singleton(DiscoveryNodeRole.MASTER_ROLE)))
         ).build();
         clusterState = allocation.reroute(clusterState, "reroute");
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0));
@@ -620,7 +620,7 @@ public void testMoveShardToNonDataNode() {
         DiscoveryNode node1 = new DiscoveryNode("node1", "node1", "node1", "test1", "test1", buildNewFakeTransportAddress(), emptyMap(),
             MASTER_DATA_ROLES, Version.CURRENT);
         DiscoveryNode node2 = new DiscoveryNode("node2", "node2", "node2", "test2", "test2", buildNewFakeTransportAddress(), emptyMap(),
-            new HashSet<>(randomSubsetOf(EnumSet.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.INGEST))), Version.CURRENT);
+            new HashSet<>(randomSubsetOf(Set.of(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.INGEST_ROLE))), Version.CURRENT);
 
         clusterState = ClusterState.builder(clusterState).nodes(
             DiscoveryNodes.builder()
@@ -659,7 +659,7 @@ public void testMoveShardFromNonDataNode() {
         DiscoveryNode node1 = new DiscoveryNode("node1", "node1", "node1", "test1", "test1", buildNewFakeTransportAddress(), emptyMap(),
             MASTER_DATA_ROLES, Version.CURRENT);
         DiscoveryNode node2 = new DiscoveryNode("node2", "node2", "node2", "test2", "test2", buildNewFakeTransportAddress(), emptyMap(),
-            new HashSet<>(randomSubsetOf(EnumSet.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.INGEST))), Version.CURRENT);
+            new HashSet<>(randomSubsetOf(Set.of(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.INGEST_ROLE))), Version.CURRENT);
 
         clusterState = ClusterState.builder(clusterState).nodes(
             DiscoveryNodes.builder()
@@ -677,4 +677,76 @@ public void testMoveShardFromNonDataNode() {
         assertEquals("[move_allocation] can't move [test][0] from " + node2 + " to " + node1 +
             ": source [" + node2.getName() + "] is not a data node.", e.getMessage());
     }
+
+    public void testConflictingCommandsInSingleRequest() {
+        AllocationService allocation = createAllocationService(Settings.builder()
+            .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none")
+            .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")
+            .build());
+
+        final String index1 = "test1";
+        final String index2 = "test2";
+        final String index3 = "test3";
+        logger.info("--> building initial routing table");
+        MetaData metaData = MetaData.builder()
+            .put(IndexMetaData.builder(index1).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)
+                .putInSyncAllocationIds(0, Collections.singleton("randomAllocID"))
+                .putInSyncAllocationIds(1, Collections.singleton("randomAllocID2")))
+            .put(IndexMetaData.builder(index2).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)
+                .putInSyncAllocationIds(0, Collections.singleton("randomAllocID"))
+                .putInSyncAllocationIds(1, Collections.singleton("randomAllocID2")))
+            .put(IndexMetaData.builder(index3).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)
+                .putInSyncAllocationIds(0, Collections.singleton("randomAllocID"))
+                .putInSyncAllocationIds(1, Collections.singleton("randomAllocID2")))
+            .build();
+        RoutingTable routingTable = RoutingTable.builder()
+            .addAsRecovery(metaData.index(index1))
+            .addAsRecovery(metaData.index(index2))
+            .addAsRecovery(metaData.index(index3))
+            .build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .metaData(metaData).routingTable(routingTable).build();
+
+        final String node1 = "node1";
+        final String node2 = "node2";
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
+            .add(newNode(node1))
+            .add(newNode(node2))
+        ).build();
+        final ClusterState finalClusterState = allocation.reroute(clusterState, "reroute");
+
+        logger.info("--> allocating same index primary in multiple commands should fail");
+        assertThat(expectThrows(IllegalArgumentException.class, () -> {
+            allocation.reroute(finalClusterState,
+                new AllocationCommands(
+                    new AllocateStalePrimaryAllocationCommand(index1, 0, node1, true),
+                    new AllocateStalePrimaryAllocationCommand(index1, 0, node2, true)
+                ), false, false);
+        }).getMessage(), containsString("primary [" + index1 + "][0] is already assigned"));
+
+        assertThat(expectThrows(IllegalArgumentException.class, () -> {
+            allocation.reroute(finalClusterState,
+                new AllocationCommands(
+                    new AllocateEmptyPrimaryAllocationCommand(index2, 0, node1, true),
+                    new AllocateEmptyPrimaryAllocationCommand(index2, 0, node2, true)
+                ), false, false);
+        }).getMessage(), containsString("primary [" + index2 + "][0] is already assigned"));
+
+
+        clusterState = allocation.reroute(clusterState,
+            new AllocationCommands(new AllocateEmptyPrimaryAllocationCommand(index3, 0, node1, true)), false, false).getClusterState();
+        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+
+        final ClusterState updatedClusterState = clusterState;
+        assertThat(updatedClusterState.getRoutingNodes().node(node1).shardsWithState(STARTED).size(), equalTo(1));
+
+        logger.info("--> subsequent replica allocation fails as all configured replicas have been allocated");
+        assertThat(expectThrows(IllegalArgumentException.class, () -> {
+            allocation.reroute(updatedClusterState,
+                new AllocationCommands(
+                    new AllocateReplicaAllocationCommand(index3, 0, node2),
+                    new AllocateReplicaAllocationCommand(index3, 0, node2)
+                ), false, false);
+        }).getMessage(), containsString("all copies of [" + index3 + "][0] are already assigned. Use the move allocation command instead"));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
index 10fa01c2a2b12..ba18bc0f6d42d 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
@@ -64,28 +64,28 @@ public void testIndexBalance() {
         /* Tests balance over indices only */
         final float indexBalance = 1.0f;
         final float replicaBalance = 0.0f;
-        final float balanceTreshold = 1.0f;
+        final float balanceThreshold = 1.0f;
 
         Settings.Builder settings = Settings.builder();
         settings.put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(),
             ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS.toString());
         settings.put(BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING.getKey(), indexBalance);
         settings.put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), replicaBalance);
-        settings.put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), balanceTreshold);
+        settings.put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), balanceThreshold);
 
         AllocationService strategy = createAllocationService(settings.build(), new TestGatewayAllocator());
 
         ClusterState clusterState = initCluster(strategy);
         assertIndexBalance(clusterState.getRoutingTable(), clusterState.getRoutingNodes(), numberOfNodes, numberOfIndices,
-            numberOfReplicas, numberOfShards, balanceTreshold);
+            numberOfReplicas, numberOfShards, balanceThreshold);
 
         clusterState = addNode(clusterState, strategy);
         assertIndexBalance(clusterState.getRoutingTable(), clusterState.getRoutingNodes(), numberOfNodes + 1,
-            numberOfIndices, numberOfReplicas, numberOfShards, balanceTreshold);
+            numberOfIndices, numberOfReplicas, numberOfShards, balanceThreshold);
 
         clusterState = removeNodes(clusterState, strategy);
         assertIndexBalance(clusterState.getRoutingTable(), clusterState.getRoutingNodes(),
-            (numberOfNodes + 1) - (numberOfNodes + 1) / 2, numberOfIndices, numberOfReplicas, numberOfShards, balanceTreshold);
+            (numberOfNodes + 1) - (numberOfNodes + 1) / 2, numberOfIndices, numberOfReplicas, numberOfShards, balanceThreshold);
     }
 
     public void testReplicaBalance() {
@@ -201,7 +201,7 @@ private ClusterState removeNodes(ClusterState clusterState, AllocationService st
 
 
     private void assertReplicaBalance(RoutingNodes nodes, int numberOfNodes, int numberOfIndices, int numberOfReplicas,
-        int numberOfShards, float treshold) {
+        int numberOfShards, float threshold) {
         final int unassigned = nodes.unassigned().size();
 
         if (unassigned > 0) {
@@ -220,8 +220,8 @@ private void assertReplicaBalance(RoutingNodes nodes, int numberOfNodes, int num
 
         final int numShards = numberOfIndices * numberOfShards * (numberOfReplicas + 1) - unassigned;
         final float avgNumShards = (float) (numShards) / (float) (numberOfNodes);
-        final int minAvgNumberOfShards = Math.round(Math.round(Math.floor(avgNumShards - treshold)));
-        final int maxAvgNumberOfShards = Math.round(Math.round(Math.ceil(avgNumShards + treshold)));
+        final int minAvgNumberOfShards = Math.round(Math.round(Math.floor(avgNumShards - threshold)));
+        final int maxAvgNumberOfShards = Math.round(Math.round(Math.ceil(avgNumShards + threshold)));
 
         for (RoutingNode node : nodes) {
             assertThat(node.shardsWithState(STARTED).size(), Matchers.greaterThanOrEqualTo(minAvgNumberOfShards));
@@ -230,12 +230,12 @@ private void assertReplicaBalance(RoutingNodes nodes, int numberOfNodes, int num
     }
 
     private void assertIndexBalance(RoutingTable routingTable, RoutingNodes nodes, int numberOfNodes, int numberOfIndices,
-                                    int numberOfReplicas, int numberOfShards, float treshold) {
+                                    int numberOfReplicas, int numberOfShards, float threshold) {
 
         final int numShards = numberOfShards * (numberOfReplicas + 1);
         final float avgNumShards = (float) (numShards) / (float) (numberOfNodes);
-        final int minAvgNumberOfShards = Math.round(Math.round(Math.floor(avgNumShards - treshold)));
-        final int maxAvgNumberOfShards = Math.round(Math.round(Math.ceil(avgNumShards + treshold)));
+        final int minAvgNumberOfShards = Math.round(Math.round(Math.floor(avgNumShards - threshold)));
+        final int maxAvgNumberOfShards = Math.round(Math.round(Math.ceil(avgNumShards + threshold)));
 
         for (ObjectCursor<String> index : routingTable.indicesRouting().keys()) {
             for (RoutingNode node : nodes) {
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java
index 2e56ae6297b89..6f8f75d48fe37 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.cluster.routing.allocation;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
@@ -33,6 +32,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
@@ -40,7 +40,6 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.indices.cluster.ClusterStateChanges;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -210,11 +209,11 @@ private static Version getNodeVersion(ShardRouting shardRouting, ClusterState st
 
     public ClusterState randomInitialClusterState() {
         List<DiscoveryNode> allNodes = new ArrayList<>();
-        DiscoveryNode localNode = createNode(DiscoveryNode.Role.MASTER); // local node is the master
+        DiscoveryNode localNode = createNode(DiscoveryNodeRole.MASTER_ROLE); // local node is the master
         allNodes.add(localNode);
         // at least two nodes that have the data role so that we can allocate shards
-        allNodes.add(createNode(DiscoveryNode.Role.DATA));
-        allNodes.add(createNode(DiscoveryNode.Role.DATA));
+        allNodes.add(createNode(DiscoveryNodeRole.DATA_ROLE));
+        allNodes.add(createNode(DiscoveryNodeRole.DATA_ROLE));
         for (int i = 0; i < randomIntBetween(2, 5); i++) {
             allNodes.add(createNode());
         }
@@ -223,11 +222,9 @@ public ClusterState randomInitialClusterState() {
     }
 
 
-    protected DiscoveryNode createNode(DiscoveryNode.Role... mustHaveRoles) {
-        Set<DiscoveryNode.Role> roles = new HashSet<>(randomSubsetOf(Sets.newHashSet(DiscoveryNode.Role.values())));
-        for (DiscoveryNode.Role mustHaveRole : mustHaveRoles) {
-            roles.add(mustHaveRole);
-        }
+    protected DiscoveryNode createNode(DiscoveryNodeRole... mustHaveRoles) {
+        Set<DiscoveryNodeRole> roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES));
+        Collections.addAll(roles, mustHaveRoles);
         final String id = String.format(Locale.ROOT, "node_%03d", nodeIdGenerator.incrementAndGet());
         return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), roles,
             VersionUtils.randomIndexCompatibleVersion(random()));
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RetryFailedAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RetryFailedAllocationTests.java
new file mode 100644
index 0000000000000..1ca516da26286
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RetryFailedAllocationTests.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.allocation.command.AllocateReplicaAllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
+import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.sameInstance;
+
+public class RetryFailedAllocationTests extends ESAllocationTestCase {
+
+    private MockAllocationService strategy;
+    private ClusterState clusterState;
+    private final String INDEX_NAME = "index";
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        MetaData metaData = MetaData.builder().put(IndexMetaData.builder(INDEX_NAME)
+            .settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).build();
+        RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index(INDEX_NAME)).build();
+        clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable)
+            .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
+        strategy = createAllocationService(Settings.EMPTY);
+    }
+
+    private ShardRouting getPrimary() {
+        return clusterState.getRoutingTable().index(INDEX_NAME).shard(0).primaryShard();
+    }
+
+    private ShardRouting getReplica() {
+        return clusterState.getRoutingTable().index(INDEX_NAME).shard(0).replicaShards().get(0);
+    }
+
+    public void testRetryFailedResetForAllocationCommands() {
+        final int retries = MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY.get(Settings.EMPTY);
+        clusterState = strategy.reroute(clusterState, "initial allocation");
+        clusterState = strategy.applyStartedShards(clusterState, Collections.singletonList(getPrimary()));
+
+        // Exhaust all replica allocation attempts with shard failures
+        for (int i = 0; i < retries; i++) {
+            List<FailedShard> failedShards = Collections.singletonList(
+                new FailedShard(getReplica(), "failing-shard::attempt-" + i,
+                    new ElasticsearchException("simulated"), randomBoolean()));
+            clusterState = strategy.applyFailedShards(clusterState, failedShards);
+            clusterState = strategy.reroute(clusterState, "allocation retry attempt-" + i);
+        }
+        assertThat("replica should not be assigned", getReplica().state(), equalTo(ShardRoutingState.UNASSIGNED));
+        assertThat("reroute should be a no-op", strategy.reroute(clusterState, "test"), sameInstance(clusterState));
+
+        // Now allocate replica with retry_failed flag set
+        AllocationService.CommandsResult result = strategy.reroute(clusterState,
+            new AllocationCommands(new AllocateReplicaAllocationCommand(INDEX_NAME, 0,
+                getPrimary().currentNodeId().equals("node1") ? "node2" : "node1")),
+            false, true);
+        clusterState = result.getClusterState();
+
+        assertEquals(ShardRoutingState.INITIALIZING, getReplica().state());
+        clusterState = strategy.applyStartedShards(clusterState, Collections.singletonList(getReplica()));
+        assertEquals(ShardRoutingState.STARTED, getReplica().state());
+        assertFalse(clusterState.getRoutingNodes().hasUnassignedShards());
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
index 24838b22d47e2..cf8abf6ccd6e5 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
@@ -912,9 +913,9 @@ public void testForSingleDataNode() {
 
         logger.info("--> adding one master node, one data node");
         DiscoveryNode discoveryNode1 = new DiscoveryNode("", "node1", buildNewFakeTransportAddress(), emptyMap(),
-                singleton(DiscoveryNode.Role.MASTER), Version.CURRENT);
+                singleton(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT);
         DiscoveryNode discoveryNode2 = new DiscoveryNode("", "node2", buildNewFakeTransportAddress(), emptyMap(),
-                singleton(DiscoveryNode.Role.DATA), Version.CURRENT);
+                singleton(DiscoveryNodeRole.DATA_ROLE), Version.CURRENT);
 
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(discoveryNode1).add(discoveryNode2).build();
         ClusterState baseClusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
@@ -977,7 +978,7 @@ Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLU
         // Add another datanode, it should relocate.
         logger.info("--> adding node3");
         DiscoveryNode discoveryNode3 = new DiscoveryNode("", "node3", buildNewFakeTransportAddress(), emptyMap(),
-                singleton(DiscoveryNode.Role.DATA), Version.CURRENT);
+                singleton(DiscoveryNodeRole.DATA_ROLE), Version.CURRENT);
         ClusterState updateClusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
                 .add(discoveryNode3)).build();
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
index 77967788adffb..47c4cb60331c4 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RecoverySource.EmptyStoreRecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.LocalShardsRecoverySource;
@@ -47,7 +48,6 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 
@@ -73,9 +73,9 @@ public void testCanAllocateUsesMaxAvailableSpace() {
         ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true, EmptyStoreRecoverySource.INSTANCE,
             new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         DiscoveryNode node_0 = new DiscoveryNode("node_0", buildNewFakeTransportAddress(), Collections.emptyMap(),
-                new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
+                new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT);
         DiscoveryNode node_1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Collections.emptyMap(),
-                new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
+                new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT);
 
         RoutingTable routingTable = RoutingTable.builder()
                 .addAsNew(metaData.index("test"))
@@ -129,9 +129,9 @@ public void testCannotAllocateDueToLackOfDiskResources() {
         ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true, EmptyStoreRecoverySource.INSTANCE,
             new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         DiscoveryNode node_0 = new DiscoveryNode("node_0", buildNewFakeTransportAddress(), Collections.emptyMap(),
-            new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
+            new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT);
         DiscoveryNode node_1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Collections.emptyMap(),
-            new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
+            new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT);
 
         RoutingTable routingTable = RoutingTable.builder()
             .addAsNew(metaData.index("test"))
@@ -178,9 +178,9 @@ public void testCanRemainUsesLeastAvailableSpace() {
         ImmutableOpenMap.Builder<ShardRouting, String> shardRoutingMap = ImmutableOpenMap.builder();
 
         DiscoveryNode node_0 = new DiscoveryNode("node_0", buildNewFakeTransportAddress(), Collections.emptyMap(),
-                new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
+                new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT);
         DiscoveryNode node_1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Collections.emptyMap(),
-                new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
+                new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT);
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
index efbd1232013f9..3d9e77e48b2b9 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
@@ -102,10 +102,10 @@ public void testRerouteOccursOnDiskPassingHighWatermark() throws Exception {
         cis.setN2Usage(realNodeNames.get(1), new DiskUsage(nodes.get(1), "n2", "_na_", 100, 50));
         cis.setN3Usage(realNodeNames.get(2), new DiskUsage(nodes.get(2), "n3", "_na_", 100, 0)); // nothing free on node3
 
+        logger.info("--> waiting for shards to relocate off node [{}]", realNodeNames.get(2));
+
         assertBusy(() -> {
             final ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
-            logger.info("--> {}", clusterState.routingTable());
-
             final Map<String, Integer> nodesToShardCount = new HashMap<>();
             for (final RoutingNode node : clusterState.getRoutingNodes()) {
                 logger.info("--> node {} has {} shards",
@@ -122,6 +122,8 @@ public void testRerouteOccursOnDiskPassingHighWatermark() throws Exception {
         cis.setN2Usage(realNodeNames.get(1), new DiskUsage(nodes.get(1), "n2", "_na_", 100, 50));
         cis.setN3Usage(realNodeNames.get(2), new DiskUsage(nodes.get(2), "n3", "_na_", 100, 50)); // node3 has free space now
 
+        logger.info("--> waiting for shards to rebalance back onto node [{}]", realNodeNames.get(2));
+
         assertBusy(() -> {
             final Map<String, Integer> nodesToShardCount = new HashMap<>();
             final ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java
index 60e3e2438c19f..cfb0979a9ee40 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
@@ -171,7 +172,7 @@ private ClusterState createInitialClusterState() {
             .build();
 
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
-            .add(newNode("master", Collections.singleton(DiscoveryNode.Role.MASTER)))
+            .add(newNode("master", Collections.singleton(DiscoveryNodeRole.MASTER_ROLE)))
             .localNodeId("master")
             .masterNodeId("master")
             .build();
diff --git a/server/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java b/server/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
index 6b33b7eb3e2a8..0e732d5fb0502 100644
--- a/server/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
@@ -30,7 +30,6 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.util.Arrays;
 import java.util.HashSet;
@@ -330,7 +329,6 @@ public void onFailure(String source, Exception e) {
         assertThat(processedLatch.await(1, TimeUnit.SECONDS), equalTo(true));
     }
 
-    @TestLogging("_root:debug,org.elasticsearch.action.admin.cluster.tasks:trace")
     public void testPendingUpdateTask() throws Exception {
         String node_0 = internalCluster().startNode();
         internalCluster().startCoordinatingOnlyNode(Settings.EMPTY);
diff --git a/server/src/test/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java b/server/src/test/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java
index 434c371278e06..e79bf35dda335 100644
--- a/server/src/test/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java
@@ -230,7 +230,7 @@ public void testRestoreSnapshotOverLimit() {
             equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
 
         List<SnapshotInfo> snapshotInfos = client.admin().cluster().prepareGetSnapshots("test-repo")
-            .setSnapshots("test-snap").get().getSnapshots();
+            .setSnapshots("test-snap").get().getSnapshots("test-repo");
         assertThat(snapshotInfos.size(), equalTo(1));
         SnapshotInfo snapshotInfo = snapshotInfos.get(0);
         assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS));
diff --git a/server/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java b/server/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java
index 7f6c56855ec70..9548d14cca9a1 100644
--- a/server/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java
+++ b/server/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.common.geo.parsers.ShapeParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.geo.utils.GeographyValidator;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.hamcrest.ElasticsearchGeoAssertions;
 import org.locationtech.jts.geom.Geometry;
@@ -70,7 +71,7 @@ protected void assertGeometryEquals(Object expected, XContentBuilder geoJson, bo
     protected void assertGeometryEquals(org.elasticsearch.geo.geometry.Geometry expected, XContentBuilder geoJson) throws IOException {
         try (XContentParser parser = createParser(geoJson)) {
             parser.nextToken();
-            assertEquals(expected, GeoJson.fromXContent(parser, true, false, false));
+            assertEquals(expected, new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java
index 7442d3c8d803d..dfc01e4c64e16 100644
--- a/server/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java
+++ b/server/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.geo.geometry.Rectangle;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -67,22 +66,6 @@ public void testInvalidReadFrom() throws Exception {
         }
     }
 
-    public void testDistanceCheck() {
-        // Note, is within is an approximation, so, even though 0.52 is outside 50mi, we still get "true"
-        double radius = DistanceUnit.convert(50, DistanceUnit.MILES, DistanceUnit.METERS);
-        org.apache.lucene.geo.Rectangle r = org.apache.lucene.geo.Rectangle.fromPointDistance(0, 0, radius);
-        Rectangle box = new Rectangle(r.minLat, r.maxLat, r.minLon, r.maxLon);
-        assertThat(box.containsPoint(0.5, 0.5), equalTo(true));
-        assertThat(box.containsPoint(0.52, 0.52), equalTo(true));
-        assertThat(box.containsPoint(1, 1), equalTo(false));
-
-        radius = DistanceUnit.convert(200, DistanceUnit.MILES, DistanceUnit.METERS);
-        r = org.apache.lucene.geo.Rectangle.fromPointDistance(0, 179, radius);
-        box = new Rectangle(r.minLat, r.maxLat, r.minLon, r.maxLon);
-        assertThat(box.containsPoint(0, -179), equalTo(true));
-        assertThat(box.containsPoint(0, -178), equalTo(false));
-    }
-
     private static double arcDistance(GeoPoint p1, GeoPoint p2) {
         return GeoDistance.ARC.calculate(p1.lat(), p1.lon(), p2.lat(), p2.lon(), DistanceUnit.METERS);
     }
diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonParserTests.java
index 4e2c2e50e2cf1..4146adb2d299a 100644
--- a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonParserTests.java
+++ b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonParserTests.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.geo.geometry.Point;
 import org.elasticsearch.geo.geometry.Polygon;
 import org.elasticsearch.geo.geometry.Rectangle;
+import org.elasticsearch.geo.utils.GeographyValidator;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -72,7 +73,7 @@ public void testParseLineString() throws IOException {
         Line expected = new Line(new double[] {0.0, 1.0}, new double[] { 100.0, 101.0});
         try (XContentParser parser = createParser(lineGeoJson)) {
             parser.nextToken();
-            assertEquals(expected, GeoJson.fromXContent(parser, false, false, true));
+            assertEquals(expected, new GeoJson(false, false, new GeographyValidator(true)).fromXContent(parser));
         }
     }
 
@@ -124,7 +125,7 @@ public void testParseMultiDimensionShapes() throws IOException {
 
         try (XContentParser parser = createParser(pointGeoJson)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, false, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(false, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
 
@@ -140,7 +141,7 @@ public void testParseMultiDimensionShapes() throws IOException {
 
         try (XContentParser parser = createParser(lineGeoJson)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, false, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(false, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
     }
@@ -178,7 +179,7 @@ public void testParseEnvelope() throws IOException {
                 .endObject();
         try (XContentParser parser = createParser(multilinesGeoJson)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, false, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(false, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
 
@@ -189,7 +190,7 @@ public void testParseEnvelope() throws IOException {
                 .endObject();
         try (XContentParser parser = createParser(multilinesGeoJson)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, false, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(false, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
     }
@@ -239,7 +240,7 @@ public void testParse3DPolygon() throws IOException {
         ));
         try (XContentParser parser = createParser(polygonGeoJson)) {
             parser.nextToken();
-            assertEquals(expected, GeoJson.fromXContent(parser, true, false, true));
+            assertEquals(expected, new GeoJson(true, false, new GeographyValidator(true)).fromXContent(parser));
         }
     }
 
@@ -259,7 +260,7 @@ public void testInvalidDimensionalPolygon() throws IOException {
             .endObject();
         try (XContentParser parser = createParser(polygonGeoJson)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, true));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(true)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
     }
@@ -275,7 +276,7 @@ public void testParseInvalidPoint() throws IOException {
                 .endObject();
         try (XContentParser parser = createParser(invalidPoint1)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
 
@@ -288,7 +289,7 @@ public void testParseInvalidPoint() throws IOException {
                 .endObject();
         try (XContentParser parser = createParser(invalidPoint2)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
     }
@@ -302,7 +303,7 @@ public void testParseInvalidMultipoint() throws IOException {
                 .endObject();
         try (XContentParser parser = createParser(invalidMultipoint1)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
 
@@ -315,7 +316,7 @@ public void testParseInvalidMultipoint() throws IOException {
                 .endObject();
         try (XContentParser parser = createParser(invalidMultipoint2)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
 
@@ -329,7 +330,7 @@ public void testParseInvalidMultipoint() throws IOException {
                 .endObject();
         try (XContentParser parser = createParser(invalidMultipoint3)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
     }
@@ -370,7 +371,7 @@ public void testParseInvalidDimensionalMultiPolygon() throws IOException {
 
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, multiPolygonGeoJson)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
    }
@@ -391,7 +392,7 @@ public void testParseInvalidPolygon() throws IOException {
                 .endObject());
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, invalidPoly)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
 
@@ -406,7 +407,7 @@ public void testParseInvalidPolygon() throws IOException {
 
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, invalidPoly)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
 
@@ -421,7 +422,7 @@ public void testParseInvalidPolygon() throws IOException {
 
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, invalidPoly)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
 
@@ -436,7 +437,7 @@ public void testParseInvalidPolygon() throws IOException {
 
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, invalidPoly)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
 
@@ -449,7 +450,7 @@ public void testParseInvalidPolygon() throws IOException {
 
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, invalidPoly)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
 
@@ -460,7 +461,7 @@ public void testParseInvalidPolygon() throws IOException {
 
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, invalidPoly)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
 
@@ -473,7 +474,7 @@ public void testParseInvalidPolygon() throws IOException {
 
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, invalidPoly)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
     }
@@ -710,7 +711,7 @@ public void testParseInvalidShapes() throws IOException {
 
         try (XContentParser parser = createParser(tooLittlePointGeoJson)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
 
@@ -723,7 +724,7 @@ public void testParseInvalidShapes() throws IOException {
 
         try (XContentParser parser = createParser(emptyPointGeoJson)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
             assertNull(parser.nextToken());
         }
     }
@@ -749,7 +750,7 @@ public void testParseInvalidGeometryCollectionShapes() throws IOException {
             parser.nextToken(); // foo
             parser.nextToken(); // start object
             parser.nextToken(); // start object
-            expectThrows(XContentParseException.class, () -> GeoJson.fromXContent(parser, true, false, false));
+            expectThrows(XContentParseException.class, () -> new GeoJson(true, false, new GeographyValidator(false)).fromXContent(parser));
             assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken()); // end of the document
             assertNull(parser.nextToken()); // no more elements afterwards
         }
diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonSerializationTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonSerializationTests.java
index ab6e3242654f5..46766b4e11f4b 100644
--- a/server/src/test/java/org/elasticsearch/common/geo/GeoJsonSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/common/geo/GeoJsonSerializationTests.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.geo.geometry.Point;
 import org.elasticsearch.geo.geometry.Polygon;
 import org.elasticsearch.geo.geometry.Rectangle;
+import org.elasticsearch.geo.utils.GeographyValidator;
 import org.elasticsearch.test.AbstractXContentTestCase;
 import org.elasticsearch.test.ESTestCase;
 
@@ -49,6 +50,7 @@ public class GeoJsonSerializationTests extends ESTestCase {
     private static class GeometryWrapper implements ToXContentObject {
 
         private Geometry geometry;
+        private static GeoJson PARSER = new GeoJson(true, false, new GeographyValidator(true));
 
         GeometryWrapper(Geometry geometry) {
             this.geometry = geometry;
@@ -61,7 +63,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
         public static GeometryWrapper fromXContent(XContentParser parser) throws IOException {
             parser.nextToken();
-            return new GeometryWrapper(GeoJson.fromXContent(parser, true, false, true));
+            return new GeometryWrapper(PARSER.fromXContent(parser));
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryParserTests.java
index 24ba7780cefd6..e3db70fc24e2f 100644
--- a/server/src/test/java/org/elasticsearch/common/geo/GeometryParserTests.java
+++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryParserTests.java
@@ -20,6 +20,8 @@
 package org.elasticsearch.common.geo;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParseException;
@@ -44,7 +46,11 @@ public void testGeoJsonParsing() throws Exception {
 
         try (XContentParser parser = createParser(pointGeoJson)) {
             parser.nextToken();
-            assertEquals(new Point(0, 100), GeometryParser.parse(parser, true, randomBoolean(), randomBoolean()));
+            GeometryFormat format = new GeometryParser(true, randomBoolean(), randomBoolean()).geometryFormat(parser);
+            assertEquals(new Point(0, 100), format.fromXContent(parser));
+            XContentBuilder newGeoJson = XContentFactory.jsonBuilder();
+            format.toXContent(new Point(10, 100), newGeoJson, ToXContent.EMPTY_PARAMS);
+            assertEquals("{\"type\":\"Point\",\"coordinates\":[100.0,10.0]}", Strings.toString(newGeoJson));
         }
 
         XContentBuilder pointGeoJsonWithZ = XContentFactory.jsonBuilder()
@@ -55,13 +61,13 @@ public void testGeoJsonParsing() throws Exception {
 
         try (XContentParser parser = createParser(pointGeoJsonWithZ)) {
             parser.nextToken();
-            assertEquals(new Point(0, 100, 10.0), GeometryParser.parse(parser, true, randomBoolean(), true));
+            assertEquals(new Point(0, 100, 10.0), new GeometryParser(true, randomBoolean(), true).parse(parser));
         }
 
 
         try (XContentParser parser = createParser(pointGeoJsonWithZ)) {
             parser.nextToken();
-            expectThrows(XContentParseException.class, () -> GeometryParser.parse(parser, true, randomBoolean(), false));
+            expectThrows(IllegalArgumentException.class, () -> new GeometryParser(true, randomBoolean(), false).parse(parser));
         }
 
         XContentBuilder polygonGeoJson = XContentFactory.jsonBuilder()
@@ -77,17 +83,17 @@ public void testGeoJsonParsing() throws Exception {
                     .endArray()
                 .endObject();
 
-        Polygon p = new Polygon(new LinearRing(new double[] {1d, 1d, 0d, 0d, 1d}, new double[] {100d, 101d, 101d, 100d, 100d}));
+        Polygon p = new Polygon(new LinearRing(new double[]{1d, 1d, 0d, 0d, 1d}, new double[]{100d, 101d, 101d, 100d, 100d}));
         try (XContentParser parser = createParser(polygonGeoJson)) {
             parser.nextToken();
             // Coerce should automatically close the polygon
-            assertEquals(p, GeometryParser.parse(parser, true, true, randomBoolean()));
+            assertEquals(p, new GeometryParser(true, true, randomBoolean()).parse(parser));
         }
 
         try (XContentParser parser = createParser(polygonGeoJson)) {
             parser.nextToken();
             // No coerce - the polygon parsing should fail
-            expectThrows(XContentParseException.class, () -> GeometryParser.parse(parser, true, false, randomBoolean()));
+            expectThrows(XContentParseException.class, () -> new GeometryParser(true, false, randomBoolean()).parse(parser));
         }
     }
 
@@ -101,7 +107,12 @@ public void testWKTParsing() throws Exception {
             parser.nextToken(); // Start object
             parser.nextToken(); // Field Name
             parser.nextToken(); // Field Value
-            assertEquals(new Point(0, 100), GeometryParser.parse(parser, true, randomBoolean(), randomBoolean()));
+            GeometryFormat format = new GeometryParser(true, randomBoolean(), randomBoolean()).geometryFormat(parser);
+            assertEquals(new Point(0, 100), format.fromXContent(parser));
+            XContentBuilder newGeoJson = XContentFactory.jsonBuilder().startObject().field("val");
+            format.toXContent(new Point(10, 100), newGeoJson, ToXContent.EMPTY_PARAMS);
+            newGeoJson.endObject();
+            assertEquals("{\"val\":\"point (100.0 10.0)\"}", Strings.toString(newGeoJson));
         }
     }
 
@@ -115,7 +126,20 @@ public void testNullParsing() throws Exception {
             parser.nextToken(); // Start object
             parser.nextToken(); // Field Name
             parser.nextToken(); // Field Value
-            assertNull(GeometryParser.parse(parser, true, randomBoolean(), randomBoolean()));
+            GeometryFormat format = new GeometryParser(true, randomBoolean(), randomBoolean()).geometryFormat(parser);
+            assertNull(format.fromXContent(parser));
+
+            XContentBuilder newGeoJson = XContentFactory.jsonBuilder().startObject().field("val");
+            // if we serialize non-null value - it should be serialized as geojson
+            format.toXContent(new Point(10, 100), newGeoJson, ToXContent.EMPTY_PARAMS);
+            newGeoJson.endObject();
+            assertEquals("{\"val\":{\"type\":\"Point\",\"coordinates\":[100.0,10.0]}}", Strings.toString(newGeoJson));
+
+            newGeoJson = XContentFactory.jsonBuilder().startObject().field("val");
+            format.toXContent(null, newGeoJson, ToXContent.EMPTY_PARAMS);
+            newGeoJson.endObject();
+            assertEquals("{\"val\":null}", Strings.toString(newGeoJson));
+
         }
     }
 
@@ -130,7 +154,7 @@ public void testUnsupportedValueParsing() throws Exception {
             parser.nextToken(); // Field Name
             parser.nextToken(); // Field Value
             ElasticsearchParseException ex = expectThrows(ElasticsearchParseException.class,
-                () -> GeometryParser.parse(parser, true, randomBoolean(), randomBoolean()));
+                () -> new GeometryParser(true, randomBoolean(), randomBoolean()).parse(parser));
             assertEquals("shape must be an object consisting of type and coordinates", ex.getMessage());
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
index 837c0202faf92..ce0bce03b0335 100644
--- a/server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
+++ b/server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.ByteArrayInputStream;
@@ -49,7 +50,9 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasToString;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.iterableWithSize;
+import static org.hamcrest.Matchers.nullValue;
 
 public class StreamTests extends ESTestCase {
 
@@ -405,4 +408,30 @@ public void writeTo(StreamOutput out) throws IOException {
         }
     }
 
+    public void testSecureStringSerialization() throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            final SecureString secureString = new SecureString("super secret".toCharArray());
+            output.writeSecureString(secureString);
+
+            final BytesReference bytesReference = output.bytes();
+            final StreamInput input = bytesReference.streamInput();
+
+            assertThat(secureString, is(equalTo(input.readSecureString())));
+        }
+
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            final SecureString secureString = randomBoolean() ? null : new SecureString("super secret".toCharArray());
+            output.writeOptionalSecureString(secureString);
+
+            final BytesReference bytesReference = output.bytes();
+            final StreamInput input = bytesReference.streamInput();
+
+            if (secureString != null) {
+                assertThat(input.readOptionalSecureString(), is(equalTo(secureString)));
+            } else {
+                assertThat(input.readOptionalSecureString(), is(nullValue()));
+            }
+        }
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/common/joda/JodaDateMathParserTests.java b/server/src/test/java/org/elasticsearch/common/joda/JodaDateMathParserTests.java
index 4440c4e1b11a9..f6382b92343ec 100644
--- a/server/src/test/java/org/elasticsearch/common/joda/JodaDateMathParserTests.java
+++ b/server/src/test/java/org/elasticsearch/common/joda/JodaDateMathParserTests.java
@@ -151,8 +151,13 @@ public void testNow() {
 
         assertDateMathEquals("now", "2014-11-18T14:27:32", now, false, null);
         assertDateMathEquals("now+M", "2014-12-18T14:27:32", now, false, null);
+        assertDateMathEquals("now+M", "2014-12-18T14:27:32", now, true, null);
         assertDateMathEquals("now-2d", "2014-11-16T14:27:32", now, false, null);
+        assertDateMathEquals("now-2d", "2014-11-16T14:27:32", now, true, null);
         assertDateMathEquals("now/m", "2014-11-18T14:27", now, false, null);
+        assertDateMathEquals("now/m", "2014-11-18T14:27:59.999Z", now, true, null);
+        assertDateMathEquals("now/M", "2014-11-01T00:00:00", now, false, null);
+        assertDateMathEquals("now/M", "2014-11-30T23:59:59.999Z", now, true, null);
 
         // timezone does not affect now
         assertDateMathEquals("now/m", "2014-11-18T14:27", now, false, DateTimeZone.forID("+02:00"));
diff --git a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java
index 28b02de80f63c..1c55848cb04b2 100644
--- a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java
+++ b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndVersion;
 import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.VersionFieldMapper;
 import org.elasticsearch.test.ESTestCase;
 
@@ -52,6 +53,8 @@ public void testSimple() throws Exception {
         Document doc = new Document();
         doc.add(new Field(IdFieldMapper.NAME, "6", IdFieldMapper.Defaults.FIELD_TYPE));
         doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong()));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE)));
         writer.addDocument(doc);
         writer.addDocument(new Document());
         DirectoryReader reader = DirectoryReader.open(writer);
@@ -86,6 +89,8 @@ public void testTwoDocuments() throws Exception {
         Document doc = new Document();
         doc.add(new Field(IdFieldMapper.NAME, "6", IdFieldMapper.Defaults.FIELD_TYPE));
         doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong()));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE)));
         writer.addDocument(doc);
         writer.addDocument(doc);
         writer.addDocument(new Document());
diff --git a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java
index 94945dc92c952..e3e558395ca78 100644
--- a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java
+++ b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.VersionFieldMapper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
@@ -68,6 +69,8 @@ public void testVersions() throws Exception {
         Document doc = new Document();
         doc.add(new Field(IdFieldMapper.NAME, "1", IdFieldMapper.Defaults.FIELD_TYPE));
         doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 1));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong()));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE)));
         writer.updateDocument(new Term(IdFieldMapper.NAME, "1"), doc);
         directoryReader = reopen(directoryReader);
         assertThat(loadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(1L));
@@ -77,6 +80,8 @@ public void testVersions() throws Exception {
         Field version = new NumericDocValuesField(VersionFieldMapper.NAME, 2);
         doc.add(uid);
         doc.add(version);
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong()));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE)));
         writer.updateDocument(new Term(IdFieldMapper.NAME, "1"), doc);
         directoryReader = reopen(directoryReader);
         assertThat(loadDocIdAndVersion(directoryReader, new Term(IdFieldMapper.NAME, "1"), randomBoolean()).version, equalTo(2L));
@@ -86,6 +91,8 @@ public void testVersions() throws Exception {
         version.setLongValue(3);
         doc.add(uid);
         doc.add(version);
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong()));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE)));
         writer.updateDocument(new Term(IdFieldMapper.NAME, "1"), doc);
 
         directoryReader = reopen(directoryReader);
@@ -115,6 +122,8 @@ public void testNestedDocuments() throws IOException {
         doc.add(new Field(IdFieldMapper.NAME, "1", IdFieldMapper.Defaults.FIELD_TYPE));
         NumericDocValuesField version = new NumericDocValuesField(VersionFieldMapper.NAME, 5L);
         doc.add(version);
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong()));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE)));
         docs.add(doc);
 
         writer.updateDocuments(new Term(IdFieldMapper.NAME, "1"), docs);
@@ -145,6 +154,8 @@ public void testCache() throws Exception {
         Document doc = new Document();
         doc.add(new Field(IdFieldMapper.NAME, "6", IdFieldMapper.Defaults.FIELD_TYPE));
         doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong()));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE)));
         writer.addDocument(doc);
         DirectoryReader reader = DirectoryReader.open(writer);
         // should increase cache size by 1
@@ -170,6 +181,8 @@ public void testCacheFilterReader() throws Exception {
         Document doc = new Document();
         doc.add(new Field(IdFieldMapper.NAME, "6", IdFieldMapper.Defaults.FIELD_TYPE));
         doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, randomNonNegativeLong()));
+        doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, randomLongBetween(1, Long.MAX_VALUE)));
         writer.addDocument(doc);
         DirectoryReader reader = DirectoryReader.open(writer);
         assertEquals(87, loadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "6"), randomBoolean()).version);
diff --git a/server/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java b/server/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
index 32cc6785d57a2..cc5d024da9a76 100644
--- a/server/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
@@ -136,14 +136,8 @@ public Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadP
                 return Collections.singletonMap("custom", custom);
             }
         };
-        NetworkModule module = newNetworkModule(settings, false, plugin);
-        assertFalse(module.isTransportClient());
+        NetworkModule module = newNetworkModule(settings, plugin);
         assertSame(custom, module.getTransportSupplier());
-
-        // check it works with transport only as well
-        module = newNetworkModule(settings, true, plugin);
-        assertSame(custom, module.getTransportSupplier());
-        assertTrue(module.isTransportClient());
     }
 
     public void testRegisterHttpTransport() {
@@ -152,7 +146,7 @@ public void testRegisterHttpTransport() {
             .put(NetworkModule.TRANSPORT_TYPE_KEY, "local").build();
         Supplier<HttpServerTransport> custom = FakeHttpTransport::new;
 
-        NetworkModule module = newNetworkModule(settings, false, new NetworkPlugin() {
+        NetworkModule module = newNetworkModule(settings, new NetworkPlugin() {
             @Override
             public Map<String, Supplier<HttpServerTransport>> getHttpTransports(Settings settings, ThreadPool threadPool,
                                                                                 BigArrays bigArrays,
@@ -165,11 +159,9 @@ public Map<String, Supplier<HttpServerTransport>> getHttpTransports(Settings set
             }
         });
         assertSame(custom, module.getHttpServerTransportSupplier());
-        assertFalse(module.isTransportClient());
 
         settings = Settings.builder().put(NetworkModule.TRANSPORT_TYPE_KEY, "local").build();
-        NetworkModule newModule = newNetworkModule(settings, false);
-        assertFalse(newModule.isTransportClient());
+        NetworkModule newModule = newNetworkModule(settings);
         expectThrows(IllegalStateException.class, () -> newModule.getHttpServerTransportSupplier());
     }
 
@@ -182,7 +174,7 @@ public void testOverrideDefault() {
         Supplier<Transport> customTransport = () -> null;  // content doesn't matter we check reference equality
         Supplier<HttpServerTransport> custom = FakeHttpTransport::new;
         Supplier<HttpServerTransport> def = FakeHttpTransport::new;
-        NetworkModule module = newNetworkModule(settings, false, new NetworkPlugin() {
+        NetworkModule module = newNetworkModule(settings, new NetworkPlugin() {
             @Override
             public Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool,
                                                                   PageCacheRecycler pageCacheRecycler,
@@ -217,7 +209,7 @@ public void testDefaultKeys() {
         Supplier<HttpServerTransport> custom = FakeHttpTransport::new;
         Supplier<HttpServerTransport> def = FakeHttpTransport::new;
         Supplier<Transport> customTransport = () -> null;
-        NetworkModule module = newNetworkModule(settings, false, new NetworkPlugin() {
+        NetworkModule module = newNetworkModule(settings, new NetworkPlugin() {
             @Override
             public Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool,
                                                                   PageCacheRecycler pageCacheRecycler,
@@ -265,7 +257,7 @@ public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(
                 return actualHandler;
             }
         };
-        NetworkModule module = newNetworkModule(settings, false, new NetworkPlugin() {
+        NetworkModule module = newNetworkModule(settings, new NetworkPlugin() {
                 @Override
                 public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry,
                                                                            ThreadContext threadContext) {
@@ -285,7 +277,7 @@ public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistr
         assertSame(((NetworkModule.CompositeTransportInterceptor)transportInterceptor).transportInterceptors.get(0), interceptor);
 
         NullPointerException nullPointerException = expectThrows(NullPointerException.class, () -> {
-            newNetworkModule(settings, false, new NetworkPlugin() {
+            newNetworkModule(settings, new NetworkPlugin() {
                 @Override
                 public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry,
                                                                            ThreadContext threadContext) {
@@ -297,8 +289,8 @@ public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistr
         assertEquals("interceptor must not be null", nullPointerException.getMessage());
     }
 
-    private NetworkModule newNetworkModule(Settings settings, boolean transportClient, NetworkPlugin... plugins) {
-        return new NetworkModule(settings, transportClient, Arrays.asList(plugins), threadPool, null, null, null, null,
+    private NetworkModule newNetworkModule(Settings settings, NetworkPlugin... plugins) {
+        return new NetworkModule(settings, Arrays.asList(plugins), threadPool, null, null, null, null,
             xContentRegistry(), null, new NullDispatcher());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/network/NetworkUtilsTests.java b/server/src/test/java/org/elasticsearch/common/network/NetworkUtilsTests.java
index 85a7472416166..48fc767cf9980 100644
--- a/server/src/test/java/org/elasticsearch/common/network/NetworkUtilsTests.java
+++ b/server/src/test/java/org/elasticsearch/common/network/NetworkUtilsTests.java
@@ -20,12 +20,15 @@
 package org.elasticsearch.common.network;
 
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.hamcrest.OptionalMatchers;
 
 import java.net.InetAddress;
 import java.net.NetworkInterface;
-import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 /**
  * Tests for network utils. Please avoid using any methods that cause DNS lookups!
@@ -79,23 +82,14 @@ public void testFilter() throws Exception {
         assertArrayEquals(new InetAddress[] { InetAddress.getByName("::1") }, NetworkUtils.filterIPV6(addresses));
     }
 
-    /**
-     * Test that selecting by name is possible and properly matches the addresses on all interfaces and virtual
-     * interfaces.
-     *
-     * Note that to avoid that this test fails when interfaces are down or they do not have addresses assigned to them,
-     * they are ignored.
-     */
-    public void testAddressInterfaceLookup() throws Exception {
-        for (NetworkInterface netIf : NetworkUtils.getInterfaces()) {
-            if (!netIf.isUp() || Collections.list(netIf.getInetAddresses()).isEmpty()) {
-                continue;
-            }
-
-            String name = netIf.getName();
-            InetAddress[] expectedAddresses = Collections.list(netIf.getInetAddresses()).toArray(new InetAddress[0]);
-            InetAddress[] foundAddresses = NetworkUtils.getAddressesForInterface(name);
-            assertArrayEquals(expectedAddresses, foundAddresses);
+    // test that selecting by name is possible
+    public void testMaybeGetInterfaceByName() throws Exception {
+        final List<NetworkInterface> networkInterfaces = NetworkUtils.getInterfaces();
+        for (NetworkInterface netIf : networkInterfaces) {
+            final Optional<NetworkInterface> maybeNetworkInterface =
+                NetworkUtils.maybeGetInterfaceByName(networkInterfaces, netIf.getName());
+            assertThat(maybeNetworkInterface, OptionalMatchers.isPresent());
+            assertThat(maybeNetworkInterface.get().getName(), equalTo(netIf.getName()));
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/common/regex/RegexTests.java b/server/src/test/java/org/elasticsearch/common/regex/RegexTests.java
index 033b4f4ad696a..dd80db7bb11b4 100644
--- a/server/src/test/java/org/elasticsearch/common/regex/RegexTests.java
+++ b/server/src/test/java/org/elasticsearch/common/regex/RegexTests.java
@@ -63,4 +63,24 @@ public void testDoubleWildcardMatch() {
         assertTrue(Regex.simpleMatch("fff*******ddd", "fffabcddd"));
         assertFalse(Regex.simpleMatch("fff******ddd", "fffabcdd"));
     }
+
+    public void testSimpleMatch() {
+        for (int i = 0; i < 1000; i++) {
+            final String matchingString = randomAlphaOfLength(between(0, 50));
+
+            // construct a pattern that matches this string by repeatedly replacing random substrings with '*' characters
+            String pattern = matchingString;
+            for (int shrink = between(0, 5); shrink > 0; shrink--) {
+                final int shrinkStart = between(0, pattern.length());
+                final int shrinkEnd = between(shrinkStart, pattern.length());
+                pattern = pattern.substring(0, shrinkStart) + "*" + pattern.substring(shrinkEnd);
+            }
+            assertTrue("[" + pattern + "] should match [" + matchingString + "]", Regex.simpleMatch(pattern, matchingString));
+
+            // construct a pattern that does not match this string by inserting a non-matching character (a digit)
+            final int insertPos = between(0, pattern.length());
+            pattern = pattern.substring(0, insertPos) + between(0, 9) + pattern.substring(insertPos);
+            assertFalse("[" + pattern + "] should not match [" + matchingString + "]", Regex.simpleMatch(pattern, matchingString));
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/settings/ConsistentSettingsIT.java b/server/src/test/java/org/elasticsearch/common/settings/ConsistentSettingsIT.java
new file mode 100644
index 0000000000000..4ee0c6849c2bc
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/common/settings/ConsistentSettingsIT.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Setting.AffixSetting;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
+public class ConsistentSettingsIT extends ESIntegTestCase {
+
+    static final Setting<SecureString> DUMMY_STRING_CONSISTENT_SETTING = SecureSetting
+            .secureString("dummy.consistent.secure.string.setting", null, Setting.Property.Consistent);
+    static final AffixSetting<SecureString> DUMMY_AFFIX_STRING_CONSISTENT_SETTING = Setting.affixKeySetting(
+            "dummy.consistent.secure.string.affix.setting.", "suffix",
+            key -> SecureSetting.secureString(key, null, Setting.Property.Consistent));
+    private final AtomicReference<Function<Integer, Settings>> nodeSettingsOverride = new AtomicReference<>(null);
+
+    public void testAllConsistentOnAllNodesSuccess() throws Exception {
+        for (String nodeName : internalCluster().getNodeNames()) {
+            Environment environment = internalCluster().getInstance(Environment.class, nodeName);
+            ClusterService clusterService = internalCluster().getInstance(ClusterService.class, nodeName);
+            assertTrue("Empty settings list always consistent.",
+                    new ConsistentSettingsService(environment.settings(), clusterService, Collections.emptyList()).areAllConsistent());
+            assertTrue(
+                    "Simple consistent secure setting is consistent [" + clusterService.state().metaData().hashesOfConsistentSettings()
+                            + "].",
+                    new ConsistentSettingsService(environment.settings(), clusterService,
+                            Collections.singletonList(DUMMY_STRING_CONSISTENT_SETTING)).areAllConsistent());
+            assertTrue(
+                    "Affix consistent secure setting is consistent [" + clusterService.state().metaData().hashesOfConsistentSettings()
+                            + "].",
+                    new ConsistentSettingsService(environment.settings(), clusterService,
+                            Collections.singletonList(DUMMY_AFFIX_STRING_CONSISTENT_SETTING)).areAllConsistent());
+            assertTrue("All secure settings are consistent [" + clusterService.state().metaData().hashesOfConsistentSettings() + "].",
+                    new ConsistentSettingsService(environment.settings(), clusterService,
+                            List.of(DUMMY_STRING_CONSISTENT_SETTING, DUMMY_AFFIX_STRING_CONSISTENT_SETTING)).areAllConsistent());
+        }
+    }
+
+    public void testConsistencyFailures() throws Exception {
+        nodeSettingsOverride.set(nodeOrdinal -> {
+            Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal));
+            MockSecureSettings secureSettings = new MockSecureSettings();
+            if (randomBoolean()) {
+                // different value
+                secureSettings.setString("dummy.consistent.secure.string.setting", "DIFFERENT_VALUE");
+            } else {
+                // missing value
+                // secureSettings.setString("dummy.consistent.secure.string.setting", "string_value");
+            }
+            secureSettings.setString("dummy.consistent.secure.string.affix.setting." + "affix1" + ".suffix", "affix_value_1");
+            secureSettings.setString("dummy.consistent.secure.string.affix.setting." + "affix2" + ".suffix", "affix_value_2");
+            assert builder.getSecureSettings() == null : "Deal with the settings merge";
+            builder.setSecureSettings(secureSettings);
+            return builder.build();
+        });
+        String newNodeName = internalCluster().startNode();
+        Environment environment = internalCluster().getInstance(Environment.class, newNodeName);
+        ClusterService clusterService = internalCluster().getInstance(ClusterService.class, newNodeName);
+        assertTrue("Empty settings list always consistent.",
+                new ConsistentSettingsService(environment.settings(), clusterService, Collections.emptyList()).areAllConsistent());
+        assertFalse(
+                "Simple consistent secure setting is NOT consistent [" + clusterService.state().metaData().hashesOfConsistentSettings()
+                        + "].",
+                new ConsistentSettingsService(environment.settings(), clusterService,
+                        Collections.singletonList(DUMMY_STRING_CONSISTENT_SETTING)).areAllConsistent());
+        assertTrue(
+                "Affix consistent secure setting is consistent [" + clusterService.state().metaData().hashesOfConsistentSettings()
+                        + "].",
+                new ConsistentSettingsService(environment.settings(), clusterService,
+                        Collections.singletonList(DUMMY_AFFIX_STRING_CONSISTENT_SETTING)).areAllConsistent());
+        assertFalse("All secure settings are NOT consistent [" + clusterService.state().metaData().hashesOfConsistentSettings() + "].",
+                new ConsistentSettingsService(environment.settings(), clusterService,
+                        List.of(DUMMY_STRING_CONSISTENT_SETTING, DUMMY_AFFIX_STRING_CONSISTENT_SETTING)).areAllConsistent());
+        nodeSettingsOverride.set(nodeOrdinal -> {
+            Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal));
+            MockSecureSettings secureSettings = new MockSecureSettings();
+            secureSettings.setString("dummy.consistent.secure.string.setting", "string_value");
+            if (randomBoolean()) {
+                secureSettings.setString("dummy.consistent.secure.string.affix.setting." + "affix1" + ".suffix", "affix_value_1");
+                if (randomBoolean()) {
+                    secureSettings.setString("dummy.consistent.secure.string.affix.setting." + "affix2" + ".suffix", "DIFFERENT_VALUE");
+                } else {
+                    // missing value
+                    // "dummy.consistent.secure.string.affix.setting.affix2.suffix"
+                }
+            } else {
+                if (randomBoolean()) {
+                    secureSettings.setString("dummy.consistent.secure.string.affix.setting." + "affix1" + ".suffix", "DIFFERENT_VALUE_1");
+                    secureSettings.setString("dummy.consistent.secure.string.affix.setting." + "affix2" + ".suffix", "DIFFERENT_VALUE_2");
+                } else {
+                    // missing values
+                    // dummy.consistent.secure.string.affix.setting.affix1.suffix
+                    // dummy.consistent.secure.string.affix.setting.affix2.suffix
+                }
+            }
+            assert builder.getSecureSettings() == null : "Deal with the settings merge";
+            builder.setSecureSettings(secureSettings);
+            return builder.build();
+        });
+        newNodeName = internalCluster().startNode();
+        environment = internalCluster().getInstance(Environment.class, newNodeName);
+        clusterService = internalCluster().getInstance(ClusterService.class, newNodeName);
+        assertTrue("Empty settings list always consistent.",
+                new ConsistentSettingsService(environment.settings(), clusterService, Collections.emptyList()).areAllConsistent());
+        assertTrue(
+                "Simple consistent secure setting is consistent [" + clusterService.state().metaData().hashesOfConsistentSettings()
+                        + "].",
+                new ConsistentSettingsService(environment.settings(), clusterService,
+                        Collections.singletonList(DUMMY_STRING_CONSISTENT_SETTING)).areAllConsistent());
+        assertFalse(
+                "Affix consistent secure setting is NOT consistent [" + clusterService.state().metaData().hashesOfConsistentSettings()
+                        + "].",
+                new ConsistentSettingsService(environment.settings(), clusterService,
+                        Collections.singletonList(DUMMY_AFFIX_STRING_CONSISTENT_SETTING)).areAllConsistent());
+        assertFalse("All secure settings are NOT consistent [" + clusterService.state().metaData().hashesOfConsistentSettings() + "].",
+                new ConsistentSettingsService(environment.settings(), clusterService,
+                        List.of(DUMMY_STRING_CONSISTENT_SETTING, DUMMY_AFFIX_STRING_CONSISTENT_SETTING)).areAllConsistent());
+        nodeSettingsOverride.set(null);
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        Function<Integer, Settings> nodeSettingsOverrideFunction = nodeSettingsOverride.get();
+        if (nodeSettingsOverrideFunction != null) {
+            final Settings overrideSettings = nodeSettingsOverrideFunction.apply(nodeOrdinal);
+            if (overrideSettings != null) {
+                return overrideSettings;
+            }
+        }
+        Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal));
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("dummy.consistent.secure.string.setting", "string_value");
+        secureSettings.setString("dummy.consistent.secure.string.affix.setting." + "affix1" + ".suffix", "affix_value_1");
+        secureSettings.setString("dummy.consistent.secure.string.affix.setting." + "affix2" + ".suffix", "affix_value_2");
+        assert builder.getSecureSettings() == null : "Deal with the settings merge";
+        builder.setSecureSettings(secureSettings);
+        return builder.build();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        Collection<Class<? extends Plugin>> classes = new ArrayList<>(super.nodePlugins());
+        classes.add(DummyPlugin.class);
+        return classes;
+    }
+
+    public static final class DummyPlugin extends Plugin {
+
+        public DummyPlugin() {
+        }
+
+        @Override
+        public List<Setting<?>> getSettings() {
+            List<Setting<?>> settings = new ArrayList<>(super.getSettings());
+            settings.add(DUMMY_STRING_CONSISTENT_SETTING);
+            settings.add(DUMMY_AFFIX_STRING_CONSISTENT_SETTING);
+            return settings;
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/common/settings/ConsistentSettingsServiceTests.java b/server/src/test/java/org/elasticsearch/common/settings/ConsistentSettingsServiceTests.java
new file mode 100644
index 0000000000000..687b74e3397cb
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/common/settings/ConsistentSettingsServiceTests.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.mock.orig.Mockito;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+import org.mockito.stubbing.Answer;
+
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.mockito.Mockito.mock;
+import static org.hamcrest.Matchers.is;
+
+public class ConsistentSettingsServiceTests extends ESTestCase {
+
+    private AtomicReference<ClusterState> clusterState = new AtomicReference<>();
+    private ClusterService clusterService;
+
+    @Before
+    public void init() throws Exception {
+        clusterState.set(ClusterState.EMPTY_STATE);
+        clusterService = mock(ClusterService.class);
+        Mockito.doAnswer((Answer) invocation -> {
+            return clusterState.get();
+        }).when(clusterService).state();
+        Mockito.doAnswer((Answer) invocation -> {
+            final ClusterStateUpdateTask arg0 = (ClusterStateUpdateTask) invocation.getArguments()[1];
+            this.clusterState.set(arg0.execute(this.clusterState.get()));
+            return null;
+        }).when(clusterService).submitStateUpdateTask(Mockito.isA(String.class), Mockito.isA(ClusterStateUpdateTask.class));
+    }
+
+    public void testSingleStringSetting() throws Exception {
+        Setting<?> stringSetting = SecureSetting.secureString("test.simple.foo", null, Setting.Property.Consistent);
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString(stringSetting.getKey(), "somethingsecure");
+        secureSettings.setString("test.noise.setting", "noise");
+        Settings.Builder builder = Settings.builder();
+        builder.setSecureSettings(secureSettings);
+        Settings settings = builder.build();
+        // hashes not yet published
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).areAllConsistent(), is(false));
+        // publish
+        new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).newHashPublisher().onMaster();
+        ConsistentSettingsService consistentService = new ConsistentSettingsService(settings, clusterService, List.of(stringSetting));
+        assertThat(consistentService.areAllConsistent(), is(true));
+        // change value
+        secureSettings.setString(stringSetting.getKey(), "_TYPO_somethingsecure");
+        assertThat(consistentService.areAllConsistent(), is(false));
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).areAllConsistent(), is(false));
+        // publish change
+        new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).newHashPublisher().onMaster();
+        assertThat(consistentService.areAllConsistent(), is(true));
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).areAllConsistent(), is(true));
+    }
+
+    public void testSingleAffixSetting() throws Exception {
+        Setting.AffixSetting<?> affixStringSetting = Setting.affixKeySetting("test.affix.", "bar",
+                (key) -> SecureSetting.secureString(key, null, Setting.Property.Consistent));
+        // add two affix settings to the keystore
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("test.noise.setting", "noise");
+        secureSettings.setString("test.affix.first.bar", "first_secure");
+        secureSettings.setString("test.affix.second.bar", "second_secure");
+        Settings.Builder builder = Settings.builder();
+        builder.setSecureSettings(secureSettings);
+        Settings settings = builder.build();
+        // hashes not yet published
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(false));
+        // publish
+        new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).newHashPublisher().onMaster();
+        ConsistentSettingsService consistentService = new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting));
+        assertThat(consistentService.areAllConsistent(), is(true));
+        // change value
+        secureSettings.setString("test.affix.second.bar", "_TYPO_second_secure");
+        assertThat(consistentService.areAllConsistent(), is(false));
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(false));
+        // publish change
+        new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).newHashPublisher().onMaster();
+        assertThat(consistentService.areAllConsistent(), is(true));
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(true));
+        // add value
+        secureSettings.setString("test.affix.third.bar", "third_secure");
+        builder = Settings.builder();
+        builder.setSecureSettings(secureSettings);
+        settings = builder.build();
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(false));
+        // publish
+        new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).newHashPublisher().onMaster();
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(true));
+        // remove value
+        secureSettings = new MockSecureSettings();
+        secureSettings.setString("test.another.noise.setting", "noise");
+        // missing value test.affix.first.bar
+        secureSettings.setString("test.affix.second.bar", "second_secure");
+        secureSettings.setString("test.affix.third.bar", "third_secure");
+        builder = Settings.builder();
+        builder.setSecureSettings(secureSettings);
+        settings = builder.build();
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(false));
+    }
+
+    public void testStringAndAffixSettings() throws Exception {
+        Setting<?> stringSetting = SecureSetting.secureString("mock.simple.foo", null, Setting.Property.Consistent);
+        Setting.AffixSetting<?> affixStringSetting = Setting.affixKeySetting("mock.affix.", "bar",
+                (key) -> SecureSetting.secureString(key, null, Setting.Property.Consistent));
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString(randomAlphaOfLength(8).toLowerCase(Locale.ROOT), "noise");
+        secureSettings.setString(stringSetting.getKey(), "somethingsecure");
+        secureSettings.setString("mock.affix.foo.bar", "another_secure");
+        Settings.Builder builder = Settings.builder();
+        builder.setSecureSettings(secureSettings);
+        Settings settings = builder.build();
+        // hashes not yet published
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting, affixStringSetting)).areAllConsistent(),
+                is(false));
+        // publish only the simple string setting
+        new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).newHashPublisher().onMaster();
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).areAllConsistent(), is(true));
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(false));
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting, affixStringSetting)).areAllConsistent(),
+                is(false));
+        // publish only the affix string setting
+        new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).newHashPublisher().onMaster();
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).areAllConsistent(), is(false));
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(true));
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting, affixStringSetting)).areAllConsistent(),
+                is(false));
+        // publish both settings
+        new ConsistentSettingsService(settings, clusterService, List.of(stringSetting, affixStringSetting)).newHashPublisher().onMaster();
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting)).areAllConsistent(), is(true));
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(affixStringSetting)).areAllConsistent(), is(true));
+        assertThat(new ConsistentSettingsService(settings, clusterService, List.of(stringSetting, affixStringSetting)).areAllConsistent(),
+                is(true));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
index c6182eac8f680..c374984eb5d15 100644
--- a/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
@@ -21,11 +21,13 @@
 
 import org.elasticsearch.common.inject.ModuleTestCase;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.hamcrest.Matchers;
 
 import java.util.Arrays;
 
 import static java.util.Collections.emptySet;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
 
 public class SettingsModuleTests extends ModuleTestCase {
 
@@ -85,6 +87,40 @@ public void testRegisterSettings() {
         }
     }
 
+    public void testRegisterConsistentSettings() {
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("some.custom.secure.consistent.setting", "secure_value");
+        final Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+        final Setting<?> concreteConsistentSetting = SecureSetting.secureString("some.custom.secure.consistent.setting", null,
+                Setting.Property.Consistent);
+        SettingsModule module = new SettingsModule(settings, concreteConsistentSetting);
+        assertInstanceBinding(module, Settings.class, (s) -> s == settings);
+        assertThat(module.getConsistentSettings(), Matchers.containsInAnyOrder(concreteConsistentSetting));
+
+        final Setting<?> concreteUnsecureConsistentSetting = Setting.simpleString("some.custom.UNSECURE.consistent.setting",
+                Property.Consistent, Property.NodeScope);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> new SettingsModule(Settings.builder().build(), concreteUnsecureConsistentSetting));
+        assertThat(e.getMessage(), is("Invalid consistent secure setting [some.custom.UNSECURE.consistent.setting]"));
+
+        secureSettings = new MockSecureSettings();
+        secureSettings.setString("some.custom.secure.consistent.afix.wow.setting", "secure_value");
+        final Settings settings2 = Settings.builder().setSecureSettings(secureSettings).build();
+        final Setting<?> afixConcreteConsistentSetting = Setting.affixKeySetting(
+                "some.custom.secure.consistent.afix.", "setting",
+                key -> SecureSetting.secureString(key, null, Setting.Property.Consistent));
+        module = new SettingsModule(settings2,afixConcreteConsistentSetting);
+        assertInstanceBinding(module, Settings.class, (s) -> s == settings2);
+        assertThat(module.getConsistentSettings(), Matchers.containsInAnyOrder(afixConcreteConsistentSetting));
+
+        final Setting<?> concreteUnsecureConsistentAfixSetting = Setting.affixKeySetting(
+                "some.custom.secure.consistent.afix.", "setting",
+                key -> Setting.simpleString(key, Setting.Property.Consistent, Property.NodeScope));
+        e = expectThrows(IllegalArgumentException.class,
+                () -> new SettingsModule(Settings.builder().build(), concreteUnsecureConsistentAfixSetting));
+        assertThat(e.getMessage(), is("Invalid consistent secure setting [some.custom.secure.consistent.afix.*.setting]"));
+    }
+
     public void testLoggerSettings() {
         {
             Settings settings = Settings.builder().put("logger._root", "TRACE").put("logger.transport", "INFO").build();
diff --git a/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java b/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java
index 7fc7a83523c66..2fb524608968d 100644
--- a/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java
+++ b/server/src/test/java/org/elasticsearch/common/time/JavaDateMathParserTests.java
@@ -37,6 +37,19 @@ public class JavaDateMathParserTests extends ESTestCase {
     private final DateFormatter formatter = DateFormatter.forPattern("dateOptionalTime||epoch_millis");
     private final DateMathParser parser = formatter.toDateMathParser();
 
+    public void testOverridingLocaleOrZoneAndCompositeRoundUpParser() {
+        //the pattern has to be composite and the match should not be on the first one
+        DateFormatter formatter = DateFormatter.forPattern("date||epoch_millis").withLocale(randomLocale(random()));
+        DateMathParser parser = formatter.toDateMathParser();
+        long gotMillis = parser.parse("297276785531", () -> 0, true, (ZoneId) null).toEpochMilli();
+        assertDateEquals(gotMillis, "297276785531", "297276785531");
+
+        formatter = DateFormatter.forPattern("date||epoch_millis").withZone(randomZone());
+        parser = formatter.toDateMathParser();
+        gotMillis = parser.parse("297276785531", () -> 0, true, (ZoneId) null).toEpochMilli();
+        assertDateEquals(gotMillis, "297276785531", "297276785531");
+    }
+
     public void testBasicDates() {
         assertDateMathEquals("2014-05-30", "2014-05-30T00:00:00.000");
         assertDateMathEquals("2014-05-30T20", "2014-05-30T20:00:00.000");
@@ -126,8 +139,13 @@ public void testNow() {
 
         assertDateMathEquals("now", "2014-11-18T14:27:32", now, false, null);
         assertDateMathEquals("now+M", "2014-12-18T14:27:32", now, false, null);
+        assertDateMathEquals("now+M", "2014-12-18T14:27:32", now, true, null);
         assertDateMathEquals("now-2d", "2014-11-16T14:27:32", now, false, null);
+        assertDateMathEquals("now-2d", "2014-11-16T14:27:32", now, true, null);
         assertDateMathEquals("now/m", "2014-11-18T14:27", now, false, null);
+        assertDateMathEquals("now/m", "2014-11-18T14:27:59.999Z", now, true, null);
+        assertDateMathEquals("now/M", "2014-11-01T00:00:00", now, false, null);
+        assertDateMathEquals("now/M", "2014-11-30T23:59:59.999Z", now, true, null);
 
         // timezone does not affect now
         assertDateMathEquals("now/m", "2014-11-18T14:27", now, false, ZoneId.of("+02:00"));
diff --git a/server/src/test/java/org/elasticsearch/common/util/CuckooFilterTests.java b/server/src/test/java/org/elasticsearch/common/util/CuckooFilterTests.java
new file mode 100644
index 0000000000000..47e9081d815b6
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/common/util/CuckooFilterTests.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.util;
+
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.hash.MurmurHash3;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class CuckooFilterTests extends AbstractWireSerializingTestCase<CuckooFilter> {
+
+    @Override
+    protected CuckooFilter createTestInstance() {
+        CuckooFilter filter = new CuckooFilter(randomIntBetween(1, 100000),
+            ((float)randomIntBetween(1, 50)) / 100.0, Randomness.get());
+
+        int num = randomIntBetween(0, 10);
+        for (int i = 0; i < num; i++) {
+            filter.add(hash(randomLong()));
+        }
+
+        return filter;
+    }
+
+    @Override
+    protected Writeable.Reader<CuckooFilter> instanceReader() {
+        return in -> new CuckooFilter(in, Randomness.get());
+    }
+
+    @Override
+    protected CuckooFilter mutateInstance(CuckooFilter instance) {
+        CuckooFilter newInstance = new CuckooFilter(instance);
+        int num = randomIntBetween(1, 10);
+        for (int i = 0; i < num; i++) {
+            newInstance.add(hash(randomLong()));
+        }
+        return newInstance;
+    }
+
+    public void testExact() {
+        CuckooFilter filter = new CuckooFilter(10000, 0.03, Randomness.get());
+
+        for (int i = 0; i < 100; i++) {
+            filter.add(hash(i));
+        }
+
+        // Was sized sufficiently large that all of these values should be retained
+        for (int i = 0; i < 100; i++) {
+            assertThat(filter.mightContain(hash(i)), equalTo(true));
+        }
+    }
+
+    public void testSaturate() {
+        CuckooFilter filter = new CuckooFilter(10, 0.03, Randomness.get());
+        int counter = 0;
+        boolean saturated = false;
+        for (int i = 0; i < 100; i++) {
+            logger.info("Value: " + i);
+            if (filter.add(hash(i)) == false) {
+                saturated = true;
+            }
+            counter += 1;
+            if (saturated) {
+                break;
+            }
+        }
+        // Unclear when it will saturate exactly, but should be before 100 given the configuration
+        assertTrue(saturated);
+        logger.info("Saturated at: " + counter);
+
+        for (int i = 0; i < counter; i++) {
+            logger.info("Value: " + i);
+            assertThat(filter.mightContain(hash(i)), equalTo(true));
+        }
+    }
+
+    public void testHash() {
+        CuckooFilter.hashToIndex(-10, 32);
+    }
+
+    public void testBig() {
+        CuckooFilter filter = new CuckooFilter(1000000, 0.001, Randomness.get());
+
+        for (int i = 0; i < 10000; i++) {
+            filter.add(hash(i));
+        }
+
+        int correct = 0;
+        int incorrect = 0;
+        for (int i = 0; i < 10000; i++) {
+            if (filter.mightContain(hash(i))) {
+                correct += 1;
+            } else {
+                incorrect += 1;
+            }
+        }
+
+        assertThat(correct, equalTo(10000));
+        assertThat(incorrect, equalTo(0));
+
+        for (int i = 10000; i < 100000; i++) {
+            if (filter.mightContain(hash(i))) {
+                incorrect += 1;
+            } else {
+                correct += 1;
+            }
+        }
+
+        double fppRate = (double) incorrect / 100000;
+        assertThat(fppRate, lessThanOrEqualTo(0.001));
+    }
+
+    private long hash(long i) {
+        return MurmurHash3.murmur64(i);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/common/util/SetBackedScalingCuckooFilterTests.java b/server/src/test/java/org/elasticsearch/common/util/SetBackedScalingCuckooFilterTests.java
new file mode 100644
index 0000000000000..20ffaa00998a1
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/common/util/SetBackedScalingCuckooFilterTests.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.util;
+
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.hash.MurmurHash3;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class SetBackedScalingCuckooFilterTests extends AbstractWireSerializingTestCase<SetBackedScalingCuckooFilter> {
+
+    @Override
+    protected SetBackedScalingCuckooFilter createTestInstance() {
+        SetBackedScalingCuckooFilter bloom = new SetBackedScalingCuckooFilter(1000, Randomness.get(), 0.01);
+
+        int num = randomIntBetween(0, 10);
+        for (int i = 0; i < num; i++) {
+            bloom.add(randomLong());
+        }
+
+        return bloom;
+    }
+
+    @Override
+    protected Writeable.Reader<SetBackedScalingCuckooFilter> instanceReader() {
+        return in -> new SetBackedScalingCuckooFilter(in, Randomness.get());
+    }
+
+    @Override
+    protected SetBackedScalingCuckooFilter mutateInstance(SetBackedScalingCuckooFilter instance) throws IOException {
+        SetBackedScalingCuckooFilter newInstance = new SetBackedScalingCuckooFilter(instance);
+        int num = randomIntBetween(1, 10);
+        for (int i = 0; i < num; i++) {
+            newInstance.add(randomLong());
+        }
+        return newInstance;
+    }
+
+    public void testExact() {
+        int threshold = randomIntBetween(1000, 10000);
+        SetBackedScalingCuckooFilter filter = new SetBackedScalingCuckooFilter(threshold, Randomness.get(), 0.01);
+
+        int size = 0;
+        Set<Long> values = new HashSet<>();
+        Set<Long> hashed = new HashSet<>(values.size());
+        while (size < threshold - 100) {
+            long value = randomLong();
+            filter.add(value);
+            boolean newValue = values.add(value);
+            if (newValue) {
+                Long hash = MurmurHash3.murmur64(value);
+                hashed.add(hash);
+
+                size += 16;
+            }
+        }
+        assertThat(filter.hashes.size(), equalTo(hashed.size()));
+        assertThat(filter.hashes, equalTo(hashed));
+        assertNull(filter.filters);
+
+        for (Long value : values) {
+            assertThat(filter.mightContain(value), equalTo(true));
+        }
+    }
+
+    public void testConvert() {
+        int threshold = randomIntBetween(1000, 10000);
+        SetBackedScalingCuckooFilter filter = new SetBackedScalingCuckooFilter(threshold, Randomness.get(), 0.01);
+
+        int counter = 0;
+        Set<Long> values = new HashSet<>();
+        while (counter < threshold + 100) {
+            long value = randomLong();
+            filter.add(value);
+            boolean newValue = values.add(value);
+            if (newValue) {
+                counter += 1;
+            }
+        }
+        assertNull(filter.hashes);
+        assertThat(filter.filters.size(), greaterThan(0));
+
+        int incorrect = 0;
+        for (Long v : values) {
+            if (filter.mightContain(v) == false) {
+                incorrect += 1;
+            }
+        }
+        double fppRate = (double) incorrect / values.size();
+        assertThat(fppRate, lessThanOrEqualTo(0.001));
+    }
+
+    public void testConvertTwice() {
+        int threshold = randomIntBetween(1000, 10000);
+        SetBackedScalingCuckooFilter filter = new SetBackedScalingCuckooFilter(threshold, Randomness.get(), 0.01);
+
+        int counter = 0;
+        Set<Long> values = new HashSet<>();
+        while (counter < threshold + 100) {
+            long value = randomLong();
+            filter.add(value);
+            boolean newValue = values.add(value);
+            if (newValue) {
+                counter += 1;
+            }
+        }
+        assertNull(filter.hashes);
+        assertThat(filter.filters.size(), greaterThan(0));
+        IllegalStateException e = expectThrows(IllegalStateException.class, filter::convert);
+        assertThat(e.getMessage(), equalTo("Cannot convert SetBackedScalingCuckooFilter to approximate " +
+            "when it has already been converted."));
+    }
+
+    public void testMergeSmall() {
+        int threshold = 1000;
+
+        // Setup the first filter
+        SetBackedScalingCuckooFilter filter = new SetBackedScalingCuckooFilter(threshold, Randomness.get(), 0.01);
+
+        int counter = 0;
+        Set<Long> values = new HashSet<>();
+        while (counter < threshold + 1) {
+            long value = randomLong();
+            filter.add(value);
+            boolean newValue = values.add(value);
+            if (newValue) {
+                counter += 1;
+            }
+        }
+        assertNull(filter.hashes);
+        assertThat(filter.filters.size(), greaterThan(0));
+
+        int incorrect = 0;
+        for (Long v : values) {
+            if (filter.mightContain(v) == false) {
+                incorrect += 1;
+            }
+        }
+        double fppRate = (double) incorrect / values.size();
+        assertThat(fppRate, lessThanOrEqualTo(0.001));
+
+        // Setup the second filter
+        SetBackedScalingCuckooFilter filter2 = new SetBackedScalingCuckooFilter(threshold, Randomness.get(), 0.01);
+        counter = 0;
+        Set<Long> values2 = new HashSet<>();
+        while (counter < threshold + 1) {
+            long value = randomLong();
+            filter2.add(value);
+            boolean newValue = values2.add(value);
+            if (newValue) {
+                counter += 1;
+            }
+        }
+        assertNull(filter2.hashes);
+        assertThat(filter2.filters.size(), greaterThan(0));
+
+        incorrect = 0;
+        for (Long v : values2) {
+            if (filter2.mightContain(v) == false) {
+                incorrect += 1;
+            }
+        }
+        fppRate = (double) incorrect / values2.size();
+        assertThat(fppRate, lessThanOrEqualTo(0.001));
+
+        // now merge and verify the combined set
+        filter.merge(filter2);
+        incorrect = 0;
+        for (Long v : values) {
+            if (filter.mightContain(v) == false) {
+                incorrect += 1;
+            }
+        }
+        for (Long v : values2) {
+            if (filter.mightContain(v) == false) {
+                incorrect += 1;
+            }
+        }
+        fppRate = (double) incorrect / (values.size() + values2.size());
+        assertThat(fppRate, lessThanOrEqualTo(0.001));
+    }
+
+    public void testMergeIncompatible() {
+        SetBackedScalingCuckooFilter filter1 = new SetBackedScalingCuckooFilter(100, Randomness.get(), 0.01);
+        SetBackedScalingCuckooFilter filter2 = new SetBackedScalingCuckooFilter(1000, Randomness.get(), 0.01);
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> filter1.merge(filter2));
+        assertThat(e.getMessage(), equalTo("Cannot merge other CuckooFilter because thresholds do not match: [100] vs [1000]"));
+
+        SetBackedScalingCuckooFilter filter3 = new SetBackedScalingCuckooFilter(100, Randomness.get(), 0.001);
+        e = expectThrows(IllegalStateException.class, () -> filter1.merge(filter3));
+        assertThat(e.getMessage(), equalTo("Cannot merge other CuckooFilter because precisions do not match: [0.01] vs [0.001]"));
+    }
+
+    public void testBadParameters() {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> new SetBackedScalingCuckooFilter(-1, Randomness.get(), 0.11));
+        assertThat(e.getMessage(), equalTo("[threshold] must be a positive integer"));
+
+        e = expectThrows(IllegalArgumentException.class,
+            () -> new SetBackedScalingCuckooFilter(1000000, Randomness.get(), 0.11));
+        assertThat(e.getMessage(), equalTo("[threshold] must be smaller than [500000]"));
+
+        e = expectThrows(IllegalArgumentException.class,
+            () -> new SetBackedScalingCuckooFilter(100, Randomness.get(), -1.0));
+        assertThat(e.getMessage(), equalTo("[fpp] must be a positive double"));
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/AsyncIOProcessorTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/AsyncIOProcessorTests.java
index 72a1e21d78865..56aa797caa6ef 100644
--- a/server/src/test/java/org/elasticsearch/common/util/concurrent/AsyncIOProcessorTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/AsyncIOProcessorTests.java
@@ -19,22 +19,43 @@
 package org.elasticsearch.common.util.concurrent;
 
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.After;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 public class AsyncIOProcessorTests extends ESTestCase {
 
+    private ThreadContext threadContext;
+
+    @Before
+    public void setUpThreadContext() {
+        threadContext = new ThreadContext(Settings.EMPTY);
+    }
+
+    @After
+    public void tearDownThreadContext() {
+        threadContext.close();
+    }
+
     public void testPut() throws InterruptedException {
         boolean blockInternal = randomBoolean();
         AtomicInteger received = new AtomicInteger(0);
-        AsyncIOProcessor<Object> processor = new AsyncIOProcessor<Object>(logger, scaledRandomIntBetween(1, 2024)) {
+        AsyncIOProcessor<Object> processor = new AsyncIOProcessor<Object>(logger, scaledRandomIntBetween(1, 2024), threadContext) {
             @Override
             protected void write(List<Tuple<Object, Consumer<Exception>>> candidates) throws IOException {
                 if (blockInternal) {
@@ -83,7 +104,7 @@ public void testRandomFail() throws InterruptedException {
         AtomicInteger received = new AtomicInteger(0);
         AtomicInteger failed = new AtomicInteger(0);
         AtomicInteger actualFailed = new AtomicInteger(0);
-        AsyncIOProcessor<Object> processor = new AsyncIOProcessor<Object>(logger, scaledRandomIntBetween(1, 2024)) {
+        AsyncIOProcessor<Object> processor = new AsyncIOProcessor<Object>(logger, scaledRandomIntBetween(1, 2024), threadContext) {
             @Override
             protected void write(List<Tuple<Object, Consumer<Exception>>> candidates) throws IOException {
                 received.addAndGet(candidates.size());
@@ -137,7 +158,7 @@ public void testConsumerCanThrowExceptions() {
         AtomicInteger received = new AtomicInteger(0);
         AtomicInteger notified = new AtomicInteger(0);
 
-        AsyncIOProcessor<Object> processor = new AsyncIOProcessor<Object>(logger, scaledRandomIntBetween(1, 2024)) {
+        AsyncIOProcessor<Object> processor = new AsyncIOProcessor<Object>(logger, scaledRandomIntBetween(1, 2024), threadContext) {
             @Override
             protected void write(List<Tuple<Object, Consumer<Exception>>> candidates) throws IOException {
                 received.addAndGet(candidates.size());
@@ -156,7 +177,7 @@ protected void write(List<Tuple<Object, Consumer<Exception>>> candidates) throws
     }
 
     public void testNullArguments() {
-        AsyncIOProcessor<Object> processor = new AsyncIOProcessor<Object>(logger, scaledRandomIntBetween(1, 2024)) {
+        AsyncIOProcessor<Object> processor = new AsyncIOProcessor<Object>(logger, scaledRandomIntBetween(1, 2024), threadContext) {
             @Override
             protected void write(List<Tuple<Object, Consumer<Exception>>> candidates) throws IOException {
             }
@@ -165,4 +186,109 @@ protected void write(List<Tuple<Object, Consumer<Exception>>> candidates) throws
         expectThrows(NullPointerException.class, () -> processor.put(null, (e) -> {}));
         expectThrows(NullPointerException.class, () -> processor.put(new Object(), null));
     }
+
+    public void testPreserveThreadContext() throws InterruptedException {
+        final int threadCount = randomIntBetween(2, 10);
+        final String testHeader = "testheader";
+
+        AtomicInteger received = new AtomicInteger(0);
+        AtomicInteger notified = new AtomicInteger(0);
+
+        CountDownLatch writeDelay = new CountDownLatch(1);
+        AsyncIOProcessor<Object> processor = new AsyncIOProcessor<Object>(logger, scaledRandomIntBetween(threadCount - 1, 2024),
+            threadContext) {
+            @Override
+            protected void write(List<Tuple<Object, Consumer<Exception>>> candidates) throws IOException {
+                try {
+                    assertTrue(writeDelay.await(10, TimeUnit.SECONDS));
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+                received.addAndGet(candidates.size());
+            }
+        };
+
+        // first thread blocks, the rest should be non blocking.
+        CountDownLatch nonBlockingDone = new CountDownLatch(randomIntBetween(0, threadCount - 1));
+        List<Thread> threads = IntStream.range(0, threadCount).mapToObj(i -> new Thread(getTestName() + "_" + i) {
+            private final String response = randomAlphaOfLength(10);
+            {
+                setDaemon(true);
+            }
+
+            @Override
+            public void run() {
+                threadContext.addResponseHeader(testHeader, response);
+                processor.put(new Object(), (e) -> {
+                    assertEquals(Map.of(testHeader, List.of(response)), threadContext.getResponseHeaders());
+                    notified.incrementAndGet();
+                });
+                nonBlockingDone.countDown();
+            }
+        }).collect(Collectors.toList());
+        threads.forEach(Thread::start);
+        assertTrue(nonBlockingDone.await(10, TimeUnit.SECONDS));
+        writeDelay.countDown();
+        threads.forEach(t -> {
+            try {
+                t.join(20000);
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+        assertEquals(threadCount, notified.get());
+        assertEquals(threadCount, received.get());
+        threads.forEach(t -> assertFalse(t.isAlive()));
+    }
+
+    public void testSlowConsumer() {
+        AtomicInteger received = new AtomicInteger(0);
+        AtomicInteger notified = new AtomicInteger(0);
+
+        AsyncIOProcessor<Object> processor = new AsyncIOProcessor<Object>(logger, scaledRandomIntBetween(1, 2024), threadContext) {
+            @Override
+            protected void write(List<Tuple<Object, Consumer<Exception>>> candidates) throws IOException {
+                received.addAndGet(candidates.size());
+            }
+        };
+
+        int threadCount = randomIntBetween(2, 10);
+        CyclicBarrier barrier = new CyclicBarrier(threadCount);
+        Semaphore serializePutSemaphore = new Semaphore(1);
+        List<Thread> threads = IntStream.range(0, threadCount).mapToObj(i -> new Thread(getTestName() + "_" + i) {
+            {
+                setDaemon(true);
+            }
+
+            @Override
+            public void run() {
+                try {
+                    assertTrue(serializePutSemaphore.tryAcquire(10, TimeUnit.SECONDS));
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+                processor.put(new Object(), (e) -> {
+                    serializePutSemaphore.release();
+                    try {
+                        barrier.await(10, TimeUnit.SECONDS);
+                    } catch (InterruptedException | BrokenBarrierException | TimeoutException ex) {
+                        throw new RuntimeException(ex);
+                    }
+                    notified.incrementAndGet();
+                });
+            }
+        }).collect(Collectors.toList());
+        threads.forEach(Thread::start);
+        threads.forEach(t -> {
+            try {
+                t.join(20000);
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        });
+        assertEquals(threadCount, notified.get());
+        assertEquals(threadCount, received.get());
+        threads.forEach(t -> assertFalse(t.isAlive()));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/QueueResizingEsThreadPoolExecutorTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/QueueResizingEsThreadPoolExecutorTests.java
index 6b5f7d95700d1..7ffe6a4090a29 100644
--- a/server/src/test/java/org/elasticsearch/common/util/concurrent/QueueResizingEsThreadPoolExecutorTests.java
+++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/QueueResizingEsThreadPoolExecutorTests.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.common.util.concurrent;
 
-import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -154,7 +154,6 @@ public void testAutoQueueSizingWithMin() throws Exception {
         context.close();
     }
 
-    @TestLogging("org.elasticsearch.common.util.concurrent:DEBUG")
     public void testAutoQueueSizingWithMax() throws Exception {
         ThreadContext context = new ThreadContext(Settings.EMPTY);
         ResizableBlockingQueue<Runnable> queue =
diff --git a/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java
index 3215325e835bf..c973d89c4d729 100644
--- a/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java
+++ b/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java
@@ -84,7 +84,6 @@
 /**
  * Tests various cluster operations (e.g., indexing) during disruptions.
  */
-@TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE")
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class ClusterDisruptionIT extends AbstractDisruptionTestCase {
 
@@ -110,6 +109,7 @@ static ConflictMode randomMode() {
         "org.elasticsearch.discovery:TRACE,org.elasticsearch.action.support.replication:TRACE," +
         "org.elasticsearch.cluster.service:TRACE,org.elasticsearch.indices.recovery:TRACE," +
         "org.elasticsearch.indices.cluster:TRACE,org.elasticsearch.index.shard:TRACE")
+    // TestLogging for https://github.com/elastic/elasticsearch/issues/41068
     public void testAckedIndexing() throws Exception {
 
         final int seconds = !(TEST_NIGHTLY && rarely()) ? 1 : 5;
@@ -459,7 +459,7 @@ public void testIndicesDeleted() throws Exception {
         });
         internalCluster().restartNode(masterNode1, InternalTestCluster.EMPTY_CALLBACK);
         ensureYellow();
-        assertFalse(client().admin().indices().prepareExists(idxName).get().isExists());
+        assertFalse(indexExists(idxName));
     }
 
     public void testRestartNodeWhileIndexing() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java
index 29a75ea744cd8..87de5acba7d73 100644
--- a/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java
+++ b/server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.test.disruption.NetworkDisruption.NetworkDisconnect;
 import org.elasticsearch.test.disruption.ServiceDisruptionScheme;
 import org.elasticsearch.test.disruption.SlowClusterStateProcessing;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
@@ -47,7 +46,6 @@
 /**
  * Tests for discovery during disruptions.
  */
-@TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE")
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class DiscoveryDisruptionIT extends AbstractDisruptionTestCase {
 
diff --git a/server/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java b/server/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java
index 97e966533122d..4ba865a2495e2 100644
--- a/server/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.coordination.Coordinator;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RerouteService;
 import org.elasticsearch.cluster.service.ClusterApplier;
 import org.elasticsearch.cluster.service.MasterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -90,7 +91,8 @@ public void clearDummyServices() throws IOException {
 
     private DiscoveryModule newModule(Settings settings, List<DiscoveryPlugin> plugins) {
         return new DiscoveryModule(settings, threadPool, transportService, namedWriteableRegistry, null, masterService,
-            clusterApplier, clusterSettings, plugins, null, createTempDir().toAbsolutePath(), gatewayMetaState);
+            clusterApplier, clusterSettings, plugins, null, createTempDir().toAbsolutePath(), gatewayMetaState,
+            mock(RerouteService.class));
     }
 
     public void testDefaults() {
diff --git a/server/src/test/java/org/elasticsearch/discovery/DiskDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/DiskDisruptionIT.java
new file mode 100644
index 0000000000000..dac48d89e78b3
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/discovery/DiskDisruptionIT.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.discovery;
+
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import org.apache.lucene.mockfile.FilterFileSystemProvider;
+import org.elasticsearch.action.admin.indices.stats.ShardStats;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.io.PathUtilsForTesting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.seqno.SequenceNumbers;
+import org.elasticsearch.test.BackgroundIndexer;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalTestCluster;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.file.FileSystem;
+import java.nio.file.OpenOption;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileAttribute;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class DiskDisruptionIT extends AbstractDisruptionTestCase {
+
+    private static DisruptTranslogFileSystemProvider disruptTranslogFileSystemProvider;
+
+    @BeforeClass
+    public static void installDisruptTranslogFS() {
+        FileSystem current = PathUtils.getDefaultFileSystem();
+        disruptTranslogFileSystemProvider = new DisruptTranslogFileSystemProvider(current);
+        PathUtilsForTesting.installMock(disruptTranslogFileSystemProvider.getFileSystem(null));
+    }
+
+    @AfterClass
+    public static void removeDisruptTranslogFS() {
+        PathUtilsForTesting.teardown();
+    }
+
+    void injectTranslogFailures() {
+        disruptTranslogFileSystemProvider.injectFailures.set(true);
+    }
+
+    @After
+    void stopTranslogFailures() {
+        disruptTranslogFileSystemProvider.injectFailures.set(false);
+    }
+
+    static class DisruptTranslogFileSystemProvider extends FilterFileSystemProvider {
+
+        AtomicBoolean injectFailures = new AtomicBoolean();
+
+        DisruptTranslogFileSystemProvider(FileSystem inner) {
+            super("disrupttranslog://", inner);
+        }
+
+        @Override
+        public FileChannel newFileChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
+            if (injectFailures.get() && path.toString().endsWith(".ckp")) {
+                // prevents checkpoint file to be updated
+                throw new IOException("fake IOException");
+            }
+            return super.newFileChannel(path, options, attrs);
+        }
+
+    }
+
+    /**
+     * This test checks that all operations below the global checkpoint are properly persisted.
+     * It simulates a full power outage by preventing translog checkpoint files to be written and restart the cluster. This means that
+     * all un-fsynced data will be lost.
+     */
+    public void testGlobalCheckpointIsSafe() throws Exception {
+        startCluster(rarely() ? 5 : 3);
+
+        final int numberOfShards = 1 + randomInt(2);
+        assertAcked(prepareCreate("test")
+            .setSettings(Settings.builder()
+                .put(indexSettings())
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomInt(2))
+            ));
+        ensureGreen();
+
+        AtomicBoolean stopGlobalCheckpointFetcher = new AtomicBoolean();
+
+        Map<Integer, Long> shardToGcp = new ConcurrentHashMap<>();
+        for (int i = 0; i < numberOfShards; i++) {
+            shardToGcp.put(i, SequenceNumbers.NO_OPS_PERFORMED);
+        }
+        final Thread globalCheckpointSampler = new Thread(() -> {
+            while (stopGlobalCheckpointFetcher.get() == false) {
+                try {
+                    for (ShardStats shardStats : client().admin().indices().prepareStats("test").clear().get().getShards()) {
+                        final int shardId = shardStats.getShardRouting().id();
+                        final long globalCheckpoint = shardStats.getSeqNoStats().getGlobalCheckpoint();
+                        shardToGcp.compute(shardId, (i, v) -> Math.max(v, globalCheckpoint));
+                    }
+                } catch (Exception e) {
+                    // ignore
+                    logger.debug("failed to fetch shard stats", e);
+                }
+            }
+        });
+
+        globalCheckpointSampler.start();
+
+        try (BackgroundIndexer indexer = new BackgroundIndexer("test", "_doc", client(), -1, RandomizedTest.scaledRandomIntBetween(2, 5),
+            false, random())) {
+            indexer.setRequestTimeout(TimeValue.ZERO);
+            indexer.setIgnoreIndexingFailures(true);
+            indexer.setAssertNoFailuresOnStop(false);
+            indexer.start(-1);
+
+            waitForDocs(randomIntBetween(1, 100), indexer);
+
+            logger.info("injecting failures");
+            injectTranslogFailures();
+            logger.info("stopping indexing");
+        }
+
+        logger.info("full cluster restart");
+        internalCluster().fullRestart(new InternalTestCluster.RestartCallback() {
+
+            @Override
+            public void onAllNodesStopped() {
+                logger.info("stopping failures");
+                stopTranslogFailures();
+            }
+
+        });
+
+        stopGlobalCheckpointFetcher.set(true);
+
+        logger.info("waiting for global checkpoint sampler");
+        globalCheckpointSampler.join();
+
+        logger.info("waiting for green");
+        ensureGreen("test");
+
+        for (ShardStats shardStats : client().admin().indices().prepareStats("test").clear().get().getShards()) {
+            final int shardId = shardStats.getShardRouting().id();
+            final long maxSeqNo = shardStats.getSeqNoStats().getMaxSeqNo();
+            if (shardStats.getShardRouting().active()) {
+                assertThat(maxSeqNo, greaterThanOrEqualTo(shardToGcp.get(shardId)));
+            }
+        }
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java
index 37d8efd72c72f..a20ace529bc10 100644
--- a/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java
+++ b/server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java
@@ -36,7 +36,6 @@
 import org.elasticsearch.test.disruption.NetworkDisruption.TwoPartitions;
 import org.elasticsearch.test.disruption.ServiceDisruptionScheme;
 import org.elasticsearch.test.disruption.SingleNodeDisruption;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -50,7 +49,6 @@
 /**
  * Tests relating to the loss of the master.
  */
-@TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE")
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class MasterDisruptionIT extends AbstractDisruptionTestCase {
 
@@ -94,11 +92,6 @@ public void testMasterNodeGCs() throws Exception {
      * This test isolates the master from rest of the cluster, waits for a new master to be elected, restores the partition
      * and verifies that all node agree on the new cluster state
      */
-    @TestLogging(
-            "_root:DEBUG,"
-                    + "org.elasticsearch.cluster.service:TRACE,"
-                    + "org.elasticsearch.gateway:TRACE,"
-                    + "org.elasticsearch.indices.store:TRACE")
     public void testIsolateMasterAndVerifyClusterStateConsensus() throws Exception {
         final List<String> nodes = startCluster(3);
 
@@ -244,16 +237,6 @@ public void testVerifyApiBlocksDuringPartition() throws Exception {
 
     }
 
-    @TestLogging(
-        "_root:DEBUG,"
-            + "org.elasticsearch.action.bulk:TRACE,"
-            + "org.elasticsearch.action.get:TRACE,"
-            + "org.elasticsearch.cluster.service:TRACE,"
-            + "org.elasticsearch.discovery:TRACE,"
-            + "org.elasticsearch.indices.cluster:TRACE,"
-            + "org.elasticsearch.indices.recovery:TRACE,"
-            + "org.elasticsearch.index.seqno:TRACE,"
-            + "org.elasticsearch.index.shard:TRACE")
     public void testMappingTimeout() throws Exception {
         startCluster(3);
         createIndex("test", Settings.builder()
diff --git a/server/src/test/java/org/elasticsearch/discovery/SeedHostsResolverTests.java b/server/src/test/java/org/elasticsearch/discovery/SeedHostsResolverTests.java
index 3d303ef47c462..9ccb949b3da90 100644
--- a/server/src/test/java/org/elasticsearch/discovery/SeedHostsResolverTests.java
+++ b/server/src/test/java/org/elasticsearch/discovery/SeedHostsResolverTests.java
@@ -23,6 +23,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkAddress;
@@ -32,6 +33,7 @@
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
@@ -98,10 +100,14 @@ public void startResolver() {
     }
 
     private void recreateSeedHostsResolver(TransportService transportService) {
+        recreateSeedHostsResolver(transportService, Settings.EMPTY);
+    }
+
+    private void recreateSeedHostsResolver(TransportService transportService, Settings settings) {
         if (seedHostsResolver != null) {
             seedHostsResolver.stop();
         }
-        seedHostsResolver = new SeedHostsResolver("test_node", Settings.EMPTY, transportService, hostsResolver -> transportAddresses);
+        seedHostsResolver = new SeedHostsResolver("test_node", settings, transportService, hostsResolver -> transportAddresses);
         seedHostsResolver.start();
     }
 
@@ -321,6 +327,62 @@ public TransportAddress[] addressesFromString(String address) throws UnknownHost
         }
     }
 
+    public void testCancellationOnClose() throws InterruptedException {
+        final NetworkService networkService = new NetworkService(Collections.emptyList());
+        final CountDownLatch latch = new CountDownLatch(1);
+        final CountDownLatch conditionLatch = new CountDownLatch(1);
+        final Transport transport = new MockNioTransport(
+            Settings.EMPTY,
+            Version.CURRENT,
+            threadPool,
+            networkService,
+            PageCacheRecycler.NON_RECYCLING_INSTANCE,
+            new NamedWriteableRegistry(Collections.emptyList()),
+            new NoneCircuitBreakerService()) {
+
+            @Override
+            public BoundTransportAddress boundAddress() {
+                return new BoundTransportAddress(
+                    new TransportAddress[]{new TransportAddress(InetAddress.getLoopbackAddress(), 9500)},
+                    new TransportAddress(InetAddress.getLoopbackAddress(), 9500)
+                );
+            }
+
+            @Override
+            public TransportAddress[] addressesFromString(String address) throws UnknownHostException {
+                if ("hostname1".equals(address)) {
+                    return new TransportAddress[]{new TransportAddress(TransportAddress.META_ADDRESS, 9300)};
+                } else if ("hostname2".equals(address)) {
+                    try {
+                        conditionLatch.countDown();
+                        latch.await();
+                        throw new AssertionError("should never be called");
+                    } catch (InterruptedException e) {
+                        throw new RuntimeException(e);
+                    }
+                } else {
+                    throw new UnknownHostException(address);
+                }
+            }
+
+        };
+        closeables.push(transport);
+
+        final TransportService transportService =
+            new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null,
+                Collections.emptySet());
+        closeables.push(transportService);
+        recreateSeedHostsResolver(transportService,
+            Settings.builder().put(SeedHostsResolver.DISCOVERY_SEED_RESOLVER_TIMEOUT_SETTING.getKey(), "10m").build());
+
+        final PlainActionFuture<List<TransportAddress>> fut = new PlainActionFuture<>();
+        threadPool.generic().execute((() -> fut.onResponse(seedHostsResolver.resolveHosts(Arrays.asList("hostname1", "hostname2")))));
+
+        conditionLatch.await();
+        seedHostsResolver.stop();
+        assertThat(FutureUtils.get(fut, 10, TimeUnit.SECONDS), hasSize(0));
+    }
+
     public void testInvalidHosts() throws IllegalAccessException {
         final Transport transport = new MockNioTransport(
             Settings.EMPTY,
diff --git a/server/src/test/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java
index c8bb9e33c6eac..ceab9a4cb2d83 100644
--- a/server/src/test/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java
+++ b/server/src/test/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java
@@ -37,7 +37,6 @@
 import org.elasticsearch.snapshots.SnapshotState;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.disruption.NetworkDisruption;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 
 import java.util.Arrays;
@@ -54,7 +53,6 @@
 /**
  * Tests snapshot operations during disruptions.
  */
-@TestLogging("org.elasticsearch.snapshot:TRACE")
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class SnapshotDisruptionIT extends ESIntegTestCase {
 
@@ -179,7 +177,7 @@ private void assertAllSnapshotsCompleted() throws Exception {
     private void assertSnapshotExists(String repository, String snapshot) {
         GetSnapshotsResponse snapshotsStatusResponse = dataNodeClient().admin().cluster().prepareGetSnapshots(repository)
                 .setSnapshots(snapshot).get();
-        SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
+        SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots(repository).get(0);
         assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
         assertEquals(snapshotInfo.totalShards(), snapshotInfo.successfulShards());
         assertEquals(0, snapshotInfo.failedShards());
diff --git a/server/src/test/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java
index 1bb93123309eb..0e2f34b8e557e 100644
--- a/server/src/test/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java
+++ b/server/src/test/java/org/elasticsearch/discovery/StableMasterDisruptionIT.java
@@ -65,7 +65,6 @@
  * Tests relating to the loss of the master, but which work with the default fault detection settings which are rather lenient and will
  * not detect a master failure too quickly.
  */
-@TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE")
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class StableMasterDisruptionIT extends ESIntegTestCase {
 
@@ -174,6 +173,8 @@ private void testFollowerCheckerAfterMasterReelection(NetworkLinkDisruptionType
      * Tests that emulates a frozen elected master node that unfreezes and pushes its cluster state to other nodes that already are
      * following another elected master node. These nodes should reject this cluster state and prevent them from following the stale master.
      */
+    @TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE")
+    // TestLogging for https://github.com/elastic/elasticsearch/issues/43392
     public void testStaleMasterNotHijackingMajority() throws Exception {
         final List<String> nodes = internalCluster().startNodes(3, Settings.builder()
             .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "1s")
diff --git a/server/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java b/server/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java
index 1b90ea691c1cd..1d8abcc0e6f11 100644
--- a/server/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java
+++ b/server/src/test/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java
@@ -50,7 +50,7 @@
         numDataNodes = 1,
         numClientNodes = 0,
         supportsDedicatedMasters = false,
-        autoMinMasterNodes = false)
+        autoManageMasterNodes = false)
 public class SingleNodeDiscoveryIT extends ESIntegTestCase {
 
     @Override
@@ -189,7 +189,6 @@ public Path nodeConfigPath(int nodeOrdinal) {
     public void testStatePersistence() throws Exception {
         createIndex("test");
         internalCluster().fullRestart();
-        assertTrue(client().admin().indices().prepareExists("test").get().isExists());
+        assertTrue(indexExists("test"));
     }
-
 }
diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentIT.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentIT.java
index 74de578426f2c..8976815f092ef 100644
--- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentIT.java
+++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentIT.java
@@ -21,13 +21,25 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
 
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
 
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
@@ -35,7 +47,7 @@
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class NodeEnvironmentIT extends ESIntegTestCase {
-    public void testStartFailureOnDataForNonDataNode() throws Exception {
+    public void testStartFailureOnDataForNonDataNode() {
         final String indexName = "test-fail-on-data";
 
         logger.info("--> starting one node");
@@ -123,4 +135,107 @@ public void testFailsToStartIfUpgradedTooFar() {
         assertThat(illegalStateException.getMessage(),
             allOf(startsWith("cannot upgrade a node from version ["), endsWith("] directly to version [" + Version.CURRENT + "]")));
     }
+
+    public void testUpgradeDataFolder() throws IOException, InterruptedException {
+        String node = internalCluster().startNode();
+        prepareCreate("test").get();
+        indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("{}", XContentType.JSON));
+        String nodeId = client().admin().cluster().prepareState().get().getState().nodes().getMasterNodeId();
+
+        final Settings dataPathSettings = internalCluster().dataPathSettings(node);
+        internalCluster().stopRandomDataNode();
+
+        // simulate older data path layout by moving data under "nodes/0" folder
+        final List<Path> dataPaths = Environment.PATH_DATA_SETTING.get(dataPathSettings)
+            .stream().map(PathUtils::get).collect(Collectors.toList());
+        dataPaths.forEach(path -> {
+                final Path targetPath = path.resolve("nodes").resolve("0");
+                try {
+                    Files.createDirectories(targetPath);
+
+                    try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) {
+                        for (Path subPath : stream) {
+                            String fileName = subPath.getFileName().toString();
+                            Path targetSubPath = targetPath.resolve(fileName);
+                            if (fileName.equals("nodes") == false) {
+                                Files.move(subPath, targetSubPath, StandardCopyOption.ATOMIC_MOVE);
+                            }
+                        }
+                    }
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            });
+
+        dataPaths.forEach(path -> assertTrue(Files.exists(path.resolve("nodes"))));
+
+        // create extra file/folder, and check that upgrade fails
+        if (dataPaths.isEmpty() == false) {
+            final Path badFileInNodesDir = Files.createTempFile(randomFrom(dataPaths).resolve("nodes"), "bad", "file");
+            IllegalStateException ise = expectThrows(IllegalStateException.class, () -> internalCluster().startNode(dataPathSettings));
+            assertThat(ise.getMessage(), containsString("unexpected file/folder encountered during data folder upgrade"));
+            Files.delete(badFileInNodesDir);
+
+            final Path badFolderInNodesDir = Files.createDirectories(randomFrom(dataPaths).resolve("nodes").resolve("bad-folder"));
+            ise = expectThrows(IllegalStateException.class, () -> internalCluster().startNode(dataPathSettings));
+            assertThat(ise.getMessage(), containsString("unexpected file/folder encountered during data folder upgrade"));
+            Files.delete(badFolderInNodesDir);
+
+            final Path badFile = Files.createTempFile(randomFrom(dataPaths).resolve("nodes").resolve("0"), "bad", "file");
+            ise = expectThrows(IllegalStateException.class, () -> internalCluster().startNode(dataPathSettings));
+            assertThat(ise.getMessage(), containsString("unexpected file/folder encountered during data folder upgrade"));
+            Files.delete(badFile);
+
+            final Path badFolder = Files.createDirectories(randomFrom(dataPaths).resolve("nodes").resolve("0").resolve("bad-folder"));
+            ise = expectThrows(IllegalStateException.class, () -> internalCluster().startNode(dataPathSettings));
+            assertThat(ise.getMessage(), containsString("unexpected folder encountered during data folder upgrade"));
+            Files.delete(badFolder);
+
+            final Path conflictingFolder = randomFrom(dataPaths).resolve("indices");
+            if (Files.exists(conflictingFolder) == false) {
+                Files.createDirectories(conflictingFolder);
+                ise = expectThrows(IllegalStateException.class, () -> internalCluster().startNode(dataPathSettings));
+                assertThat(ise.getMessage(), containsString("target folder already exists during data folder upgrade"));
+                Files.delete(conflictingFolder);
+            }
+        }
+
+        // check that upgrade works
+        dataPaths.forEach(path -> assertTrue(Files.exists(path.resolve("nodes"))));
+        internalCluster().startNode(dataPathSettings);
+        dataPaths.forEach(path -> assertFalse(Files.exists(path.resolve("nodes"))));
+        assertEquals(nodeId, client().admin().cluster().prepareState().get().getState().nodes().getMasterNodeId());
+        assertTrue(indexExists("test"));
+        ensureYellow("test");
+        assertHitCount(client().prepareSearch().setQuery(matchAllQuery()).get(), 1L);
+    }
+
+    public void testFailsToStartOnDataPathsFromMultipleNodes() throws IOException {
+        final List<String> nodes = internalCluster().startNodes(2);
+        ensureStableCluster(2);
+
+        final List<String> node0DataPaths = Environment.PATH_DATA_SETTING.get(internalCluster().dataPathSettings(nodes.get(0)));
+        final List<String> node1DataPaths = Environment.PATH_DATA_SETTING.get(internalCluster().dataPathSettings(nodes.get(1)));
+
+        final List<String> allDataPaths = new ArrayList<>(node0DataPaths);
+        allDataPaths.addAll(node1DataPaths);
+
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(1)));
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(0)));
+
+        final IllegalStateException illegalStateException = expectThrows(IllegalStateException.class,
+            () -> internalCluster().startNode(Settings.builder().putList(Environment.PATH_DATA_SETTING.getKey(), allDataPaths)));
+
+        assertThat(illegalStateException.getMessage(), containsString("belong to multiple nodes with IDs"));
+
+        final List<String> node0DataPathsPlusOne = new ArrayList<>(node0DataPaths);
+        node0DataPathsPlusOne.add(createTempDir().toString());
+        internalCluster().startNode(Settings.builder().putList(Environment.PATH_DATA_SETTING.getKey(), node0DataPathsPlusOne));
+
+        final List<String> node1DataPathsPlusOne = new ArrayList<>(node1DataPaths);
+        node1DataPathsPlusOne.add(createTempDir().toString());
+        internalCluster().startNode(Settings.builder().putList(Environment.PATH_DATA_SETTING.getKey(), node1DataPathsPlusOne));
+
+        ensureStableCluster(2);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
index f21b55b9aee8f..5bb1152bcbe45 100644
--- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
+++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
@@ -373,10 +373,10 @@ public void testCustomDataPaths() throws Exception {
 
         assertThat("shard paths with a custom data_path should contain only regular paths",
                 env.availableShardPaths(sid),
-                equalTo(stringsToPaths(dataPaths, "nodes/0/indices/" + index.getUUID() + "/0")));
+                equalTo(stringsToPaths(dataPaths, "indices/" + index.getUUID() + "/0")));
 
         assertThat("index paths uses the regular template",
-                env.indexPaths(index), equalTo(stringsToPaths(dataPaths, "nodes/0/indices/" + index.getUUID())));
+                env.indexPaths(index), equalTo(stringsToPaths(dataPaths, "indices/" + index.getUUID())));
 
         IndexSettings s3 = new IndexSettings(s2.getIndexMetaData(), Settings.builder().build());
 
@@ -385,10 +385,10 @@ public void testCustomDataPaths() throws Exception {
 
         assertThat("shard paths with a custom data_path should contain only regular paths",
                 env.availableShardPaths(sid),
-                equalTo(stringsToPaths(dataPaths, "nodes/0/indices/" + index.getUUID() + "/0")));
+                equalTo(stringsToPaths(dataPaths, "indices/" + index.getUUID() + "/0")));
 
         assertThat("index paths uses the regular template",
-                env.indexPaths(index), equalTo(stringsToPaths(dataPaths, "nodes/0/indices/" + index.getUUID())));
+                env.indexPaths(index), equalTo(stringsToPaths(dataPaths, "indices/" + index.getUUID())));
 
         env.close();
     }
@@ -418,7 +418,7 @@ public void testExistingTempFiles() throws IOException {
         String[] paths = tmpPaths();
         // simulate some previous left over temp files
         for (String path : randomSubsetOf(randomIntBetween(1, paths.length), paths)) {
-            final Path nodePath = NodeEnvironment.resolveNodePath(PathUtils.get(path));
+            final Path nodePath = PathUtils.get(path);
             Files.createDirectories(nodePath);
             Files.createFile(nodePath.resolve(NodeEnvironment.TEMP_FILE_NAME));
             if (randomBoolean()) {
@@ -433,7 +433,7 @@ public void testExistingTempFiles() throws IOException {
 
         // check we clean up
         for (String path: paths) {
-            final Path nodePath = NodeEnvironment.resolveNodePath(PathUtils.get(path));
+            final Path nodePath = PathUtils.get(path);
             final Path tempFile = nodePath.resolve(NodeEnvironment.TEMP_FILE_NAME);
             assertFalse(tempFile + " should have been cleaned", Files.exists(tempFile));
             final Path srcTempFile = nodePath.resolve(NodeEnvironment.TEMP_FILE_NAME + ".src");
diff --git a/server/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java b/server/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
index 2a45947706298..2a38942eb5a22 100644
--- a/server/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
@@ -43,11 +44,11 @@
 
 public class AsyncShardFetchTests extends ESTestCase {
     private final DiscoveryNode node1 = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Collections.emptyMap(),
-            Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT);
+            Collections.singleton(DiscoveryNodeRole.DATA_ROLE), Version.CURRENT);
     private final Response response1 = new Response(node1);
     private final Throwable failure1 = new Throwable("simulated failure 1");
     private final DiscoveryNode node2 = new DiscoveryNode("node2", buildNewFakeTransportAddress(), Collections.emptyMap(),
-            Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT);
+            Collections.singleton(DiscoveryNodeRole.DATA_ROLE), Version.CURRENT);
     private final Response response2 = new Response(node2);
     private final Throwable failure2 = new Throwable("simulate failure 2");
 
diff --git a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java
index cf4067694c2d7..fd985132bebbd 100644
--- a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -279,7 +280,7 @@ public void testSetLocalNode() {
                 .metaData(metaData)
                 .build();
         final DiscoveryNode localNode = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Collections.emptyMap(),
-                Sets.newHashSet(DiscoveryNode.Role.MASTER), Version.CURRENT);
+                Sets.newHashSet(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT);
 
         final ClusterState updatedState = setLocalNode(initialState, localNode);
 
@@ -322,7 +323,7 @@ public void testHideStateIfNotRecovered() {
                 .blocks(ClusterBlocks.builder().addGlobalBlock(STATE_NOT_RECOVERED_BLOCK))
                 .build();
         final DiscoveryNode localNode = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Collections.emptyMap(),
-                Sets.newHashSet(DiscoveryNode.Role.MASTER), Version.CURRENT);
+                Sets.newHashSet(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT);
         final ClusterState updatedState = Function.<ClusterState>identity()
                 .andThen(state -> setLocalNode(state, localNode))
                 .andThen(ClusterStateUpdaters::recoverClusterBlocks)
diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
index 3402622c78ab5..bfc45b3118800 100644
--- a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
+++ b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -54,7 +55,11 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -70,6 +75,12 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
 
     private final Logger logger = LogManager.getLogger(GatewayIndexStateIT.class);
 
+    @Override
+    protected boolean addMockInternalEngine() {
+        // testRecoverBrokenIndexMetadata replies on the flushing on shutdown behavior which can be randomly disabled in MockInternalEngine.
+        return false;
+    }
+
     public void testMappingMetaDataParsed() throws Exception {
         logger.info("--> starting 1 nodes");
         internalCluster().startNode();
@@ -309,7 +320,7 @@ public Settings onNodeStopped(final String nodeName) throws Exception {
                 logger.info("--> delete index and verify it is deleted");
                 final Client client = client(otherNode);
                 client.admin().indices().prepareDelete(indexName).execute().actionGet();
-                assertFalse(client.admin().indices().prepareExists(indexName).execute().actionGet().isExists());
+                assertFalse(indexExists(indexName, client));
                 logger.info("--> index deleted");
                 return super.onNodeStopped(nodeName);
             }
@@ -323,7 +334,7 @@ public Settings onNodeStopped(final String nodeName) throws Exception {
         ensureGreen();
 
         logger.info("--> verify that the deleted index is removed from the cluster and not reimported as dangling by the restarted node");
-        assertFalse(client().admin().indices().prepareExists(indexName).execute().actionGet().isExists());
+        assertFalse(indexExists(indexName));
         assertBusy(() -> {
             final NodeEnvironment nodeEnv = internalCluster().getInstance(NodeEnvironment.class);
             try {
@@ -340,7 +351,6 @@ public Settings onNodeStopped(final String nodeName) throws Exception {
      * allocated in our metadata that we recover. In that case we now have the ability to check the index on local recovery from disk
      * if it is sane and if we can successfully create an IndexService. This also includes plugins etc.
      */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/40867")
     public void testRecoverBrokenIndexMetadata() throws Exception {
         logger.info("--> starting one node");
         internalCluster().startNode();
@@ -367,14 +377,7 @@ public void testRecoverBrokenIndexMetadata() throws Exception {
                 // this one is not validated ahead of time and breaks allocation
                 .put("index.analysis.filter.myCollator.type", "icu_collation")
         ).build();
-        internalCluster().fullRestart(new RestartCallback(){
-            @Override
-            public Settings onNodeStopped(String nodeName) throws Exception {
-                final MetaStateService metaStateService = internalCluster().getInstance(MetaStateService.class, nodeName);
-                metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMeta);
-                return super.onNodeStopped(nodeName);
-            }
-        });
+        writeBrokenMeta(metaStateService -> metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMeta));
 
         // check that the cluster does not keep reallocating shards
         assertBusy(() -> {
@@ -407,7 +410,6 @@ public Settings onNodeStopped(String nodeName) throws Exception {
      * if it is sane and if we can successfully create an IndexService.
      * This also includes plugins etc.
      */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/40867")
     public void testRecoverMissingAnalyzer() throws Exception {
         logger.info("--> starting one node");
         internalCluster().startNode();
@@ -442,14 +444,7 @@ public void testRecoverMissingAnalyzer() throws Exception {
         final IndexMetaData metaData = state.getMetaData().index("test");
         final IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(metaData.getSettings()
                 .filter((s) -> "index.analysis.analyzer.test.tokenizer".equals(s) == false)).build();
-        internalCluster().fullRestart(new RestartCallback(){
-            @Override
-            public Settings onNodeStopped(String nodeName) throws Exception {
-                final MetaStateService metaStateService = internalCluster().getInstance(MetaStateService.class, nodeName);
-                metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMeta);
-                return super.onNodeStopped(nodeName);
-            }
-        });
+        writeBrokenMeta(metaStateService -> metaStateService.writeIndexAndUpdateManifest("broken metadata", brokenMeta));
 
         // check that the cluster does not keep reallocating shards
         assertBusy(() -> {
@@ -494,14 +489,7 @@ public void testArchiveBrokenClusterSettings() throws Exception {
         final MetaData brokenMeta = MetaData.builder(metaData).persistentSettings(Settings.builder()
                 .put(metaData.persistentSettings()).put("this.is.unknown", true)
                 .put(MetaData.SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey(), "broken").build()).build();
-        internalCluster().fullRestart(new RestartCallback(){
-            @Override
-            public Settings onNodeStopped(String nodeName) throws Exception {
-                final MetaStateService metaStateService = internalCluster().getInstance(MetaStateService.class, nodeName);
-                metaStateService.writeGlobalStateAndUpdateManifest("broken metadata", brokenMeta);
-                return super.onNodeStopped(nodeName);
-            }
-        });
+        writeBrokenMeta(metaStateService -> metaStateService.writeGlobalStateAndUpdateManifest("broken metadata", brokenMeta));
 
         ensureYellow("test"); // wait for state recovery
         state = client().admin().cluster().prepareState().get().getState();
@@ -518,4 +506,17 @@ public Settings onNodeStopped(String nodeName) throws Exception {
             + MetaData.SETTING_CLUSTER_MAX_SHARDS_PER_NODE.getKey()));
         assertHitCount(client().prepareSearch().setQuery(matchAllQuery()).get(), 1L);
     }
+
+    private void writeBrokenMeta(CheckedConsumer<MetaStateService, IOException> writer) throws Exception {
+        Map<String, MetaStateService> metaStateServices = Stream.of(internalCluster().getNodeNames())
+            .collect(Collectors.toMap(Function.identity(), nodeName -> internalCluster().getInstance(MetaStateService.class, nodeName)));
+        internalCluster().fullRestart(new RestartCallback(){
+            @Override
+            public Settings onNodeStopped(String nodeName) throws Exception {
+                final MetaStateService metaStateService = metaStateServices.get(nodeName);
+                writer.accept(metaStateService);
+                return super.onNodeStopped(nodeName);
+            }
+        });
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java
index 8ccfa5e406ae2..13348cef75fd4 100644
--- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.cluster.metadata.Manifest;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
@@ -50,7 +51,7 @@ public class GatewayMetaStatePersistedStateTests extends ESTestCase {
     public void setUp() throws Exception {
         nodeEnvironment = newNodeEnvironment();
         localNode = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Collections.emptyMap(),
-                Sets.newHashSet(DiscoveryNode.Role.MASTER), Version.CURRENT);
+                Sets.newHashSet(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT);
         clusterName = new ClusterName(randomAlphaOfLength(10));
         settings = Settings.builder().put(ClusterName.CLUSTER_NAME_SETTING.getKey(), clusterName.value()).build();
         super.setUp();
diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
index 22259b919ec6f..2f9c290717a72 100644
--- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
+++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.cluster.metadata.Manifest;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
-import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
@@ -144,7 +144,7 @@ private ClusterState clusterStateWithJustOpenedIndex(IndexMetaData indexMetaData
     }
 
     private DiscoveryNodes.Builder generateDiscoveryNodes(boolean masterEligible) {
-        Set<DiscoveryNode.Role> dataOnlyRoles = Collections.singleton(DiscoveryNode.Role.DATA);
+        Set<DiscoveryNodeRole> dataOnlyRoles = Set.of(DiscoveryNodeRole.DATA_ROLE);
         return DiscoveryNodes.builder().add(newNode("node1", masterEligible ? MASTER_DATA_ROLES : dataOnlyRoles))
                 .add(newNode("master_node", MASTER_DATA_ROLES)).localNodeId("node1").masterNodeId(masterEligible ? "node1" : "master_node");
     }
diff --git a/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java b/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java
index 86976d553fa2a..84188f80aaed3 100644
--- a/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java
+++ b/server/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java
@@ -34,7 +34,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 
-@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false)
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class RecoverAfterNodesIT extends ESIntegTestCase {
     private static final TimeValue BLOCK_WAIT_TIMEOUT = TimeValue.timeValueSeconds(10);
 
diff --git a/server/src/test/java/org/elasticsearch/http/CorsHandlerTests.java b/server/src/test/java/org/elasticsearch/http/CorsHandlerTests.java
new file mode 100644
index 0000000000000..92ea69e0a3d17
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/http/CorsHandlerTests.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.http;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Set;
+import java.util.regex.PatternSyntaxException;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.common.Strings.collectionToDelimitedString;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_MAX_AGE;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class CorsHandlerTests extends ESTestCase {
+
+    public void testCorsConfigWithBadRegex() {
+        final Settings settings = Settings.builder()
+            .put(SETTING_CORS_ENABLED.getKey(), true)
+            .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "/[*/")
+            .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
+            .build();
+        SettingsException e = expectThrows(SettingsException.class, () -> CorsHandler.fromSettings(settings));
+        assertThat(e.getMessage(), containsString("Bad regex in [http.cors.allow-origin]: [/[*/]"));
+        assertThat(e.getCause(), instanceOf(PatternSyntaxException.class));
+    }
+
+    public void testCorsConfig() {
+        final Set<String> methods = new HashSet<>(Arrays.asList("get", "options", "post"));
+        final Set<String> headers = new HashSet<>(Arrays.asList("Content-Type", "Content-Length"));
+        final String prefix = randomBoolean() ? " " : ""; // sometimes have a leading whitespace between comma delimited elements
+        final Settings settings = Settings.builder()
+            .put(SETTING_CORS_ENABLED.getKey(), true)
+            .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "*")
+            .put(SETTING_CORS_ALLOW_METHODS.getKey(), collectionToDelimitedString(methods, ",", prefix, ""))
+            .put(SETTING_CORS_ALLOW_HEADERS.getKey(), collectionToDelimitedString(headers, ",", prefix, ""))
+            .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
+            .build();
+        final CorsHandler.Config corsConfig = CorsHandler.fromSettings(settings);
+        assertTrue(corsConfig.isAnyOriginSupported());
+        assertEquals(headers, corsConfig.allowedRequestHeaders());
+        assertEquals(methods.stream().map(s -> s.toUpperCase(Locale.ENGLISH)).collect(Collectors.toSet()),
+            corsConfig.allowedRequestMethods().stream().map(RestRequest.Method::name).collect(Collectors.toSet()));
+    }
+
+    public void testCorsConfigWithDefaults() {
+        final Set<String> methods = Strings.commaDelimitedListToSet(SETTING_CORS_ALLOW_METHODS.getDefault(Settings.EMPTY));
+        final Set<String> headers = Strings.commaDelimitedListToSet(SETTING_CORS_ALLOW_HEADERS.getDefault(Settings.EMPTY));
+        final long maxAge = SETTING_CORS_MAX_AGE.getDefault(Settings.EMPTY);
+        final Settings settings = Settings.builder().put(SETTING_CORS_ENABLED.getKey(), true).build();
+        final CorsHandler.Config corsConfig = CorsHandler.fromSettings(settings);
+        assertFalse(corsConfig.isAnyOriginSupported());
+        assertEquals(Collections.emptySet(), corsConfig.origins().get());
+        assertEquals(headers, corsConfig.allowedRequestHeaders());
+        assertEquals(methods, corsConfig.allowedRequestMethods().stream().map(RestRequest.Method::name).collect(Collectors.toSet()));
+        assertEquals(maxAge, corsConfig.maxAge());
+        assertFalse(corsConfig.isCredentialsAllowed());
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java b/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java
index 1a19ccf9b4e08..c58ec6a4becbb 100644
--- a/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java
+++ b/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java
@@ -22,10 +22,13 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.bytes.ReleasablePagedBytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.ByteArray;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -53,6 +56,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
@@ -303,6 +307,72 @@ public void testConnectionClose() throws Exception {
         }
     }
 
+    public void testUnsupportedHttpMethod() {
+        final boolean close = randomBoolean();
+        final HttpRequest.HttpVersion httpVersion = close ? HttpRequest.HttpVersion.HTTP_1_0 : HttpRequest.HttpVersion.HTTP_1_1;
+        final String httpConnectionHeaderValue = close ? DefaultRestChannel.CLOSE : DefaultRestChannel.KEEP_ALIVE;
+        final RestRequest request = RestRequest.request(xContentRegistry(), new TestRequest(httpVersion, null, "/") {
+            @Override
+            public RestRequest.Method method() {
+                throw new IllegalArgumentException("test");
+            }
+        }, httpChannel);
+        request.getHttpRequest().getHeaders().put(DefaultRestChannel.CONNECTION, Collections.singletonList(httpConnectionHeaderValue));
+
+        DefaultRestChannel channel = new DefaultRestChannel(httpChannel, request.getHttpRequest(), request, bigArrays,
+            HttpHandlingSettings.fromSettings(Settings.EMPTY), threadPool.getThreadContext());
+
+        // ESTestCase#after will invoke ensureAllArraysAreReleased which will fail if the response content was not released
+        final BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
+        final ByteArray byteArray = bigArrays.newByteArray(0, false);
+        final BytesReference content = new ReleasablePagedBytesReference(byteArray, 0 , byteArray);
+        channel.sendResponse(new TestRestResponse(RestStatus.METHOD_NOT_ALLOWED, content));
+
+        Class<ActionListener<Void>> listenerClass = (Class<ActionListener<Void>>) (Class) ActionListener.class;
+        ArgumentCaptor<ActionListener<Void>> listenerCaptor = ArgumentCaptor.forClass(listenerClass);
+        verify(httpChannel).sendResponse(any(), listenerCaptor.capture());
+        ActionListener<Void> listener = listenerCaptor.getValue();
+        if (randomBoolean()) {
+            listener.onResponse(null);
+        } else {
+            listener.onFailure(new ClosedChannelException());
+        }
+        if (close) {
+            verify(httpChannel, times(1)).close();
+        } else {
+            verify(httpChannel, times(0)).close();
+        }
+    }
+
+    public void testCloseOnException() {
+        final boolean close = randomBoolean();
+        final HttpRequest.HttpVersion httpVersion = close ? HttpRequest.HttpVersion.HTTP_1_0 : HttpRequest.HttpVersion.HTTP_1_1;
+        final String httpConnectionHeaderValue = close ? DefaultRestChannel.CLOSE : DefaultRestChannel.KEEP_ALIVE;
+        final RestRequest request = RestRequest.request(xContentRegistry(), new TestRequest(httpVersion, null, "/") {
+            @Override
+            public HttpResponse createResponse(RestStatus status, BytesReference content) {
+                throw new IllegalArgumentException("test");
+            }
+        }, httpChannel);
+        request.getHttpRequest().getHeaders().put(DefaultRestChannel.CONNECTION, Collections.singletonList(httpConnectionHeaderValue));
+
+        DefaultRestChannel channel = new DefaultRestChannel(httpChannel, request.getHttpRequest(), request, bigArrays,
+            HttpHandlingSettings.fromSettings(Settings.EMPTY), threadPool.getThreadContext());
+
+        // ESTestCase#after will invoke ensureAllArraysAreReleased which will fail if the response content was not released
+        final BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
+        final ByteArray byteArray = bigArrays.newByteArray(0, false);
+        final BytesReference content = new ReleasablePagedBytesReference(byteArray, 0 , byteArray);
+
+        expectThrows(IllegalArgumentException.class, () -> channel.sendResponse(new TestRestResponse(RestStatus.OK, content)));
+
+        if (close) {
+            verify(httpChannel, times(1)).close();
+        } else {
+            verify(httpChannel, times(0)).close();
+        }
+    }
+
     private TestResponse executeRequest(final Settings settings, final String host) {
         return executeRequest(settings, null, host);
     }
@@ -424,10 +494,16 @@ public boolean containsHeader(String name) {
 
     private static class TestRestResponse extends RestResponse {
 
+        private final RestStatus status;
         private final BytesReference content;
 
+        TestRestResponse(final RestStatus status, final BytesReference content) {
+            this.status = Objects.requireNonNull(status);
+            this.content = Objects.requireNonNull(content);
+        }
+
         TestRestResponse() {
-            content = new BytesArray("content".getBytes(StandardCharsets.UTF_8));
+            this(RestStatus.OK, new BytesArray("content".getBytes(StandardCharsets.UTF_8)));
         }
 
         public String contentType() {
@@ -439,7 +515,7 @@ public BytesReference content() {
         }
 
         public RestStatus status() {
-            return RestStatus.OK;
+            return status;
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index d0f811007a6fa..d052fa365beb2 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -23,7 +23,6 @@
 import org.apache.lucene.index.FieldInvertState;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.CollectionStatistics;
-import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.QueryCachingPolicy;
 import org.apache.lucene.search.TermStatistics;
 import org.apache.lucene.search.Weight;
@@ -34,6 +33,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -50,14 +50,12 @@
 import org.elasticsearch.index.cache.query.IndexQueryCache;
 import org.elasticsearch.index.cache.query.QueryCache;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.engine.EngineException;
 import org.elasticsearch.index.engine.InternalEngineFactory;
 import org.elasticsearch.index.engine.InternalEngineTests;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.shard.IndexEventListener;
-import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.IndexingOperationListener;
 import org.elasticsearch.index.shard.SearchOperationListener;
 import org.elasticsearch.index.shard.ShardId;
@@ -159,10 +157,10 @@ private IndexService newIndexService(IndexModule module) throws IOException {
     public void testWrapperIsBound() throws IOException {
         final MockEngineFactory engineFactory = new MockEngineFactory(AssertingDirectoryReader.class);
         IndexModule module = new IndexModule(indexSettings, emptyAnalysisRegistry, engineFactory, Collections.emptyMap());
-        module.setSearcherWrapper((s) -> new Wrapper());
+        module.setReaderWrapper(s -> new Wrapper());
 
         IndexService indexService = newIndexService(module);
-        assertTrue(indexService.getSearcherWrapper() instanceof Wrapper);
+        assertTrue(indexService.getReaderWrapper() instanceof Wrapper);
         assertSame(indexService.getEngineFactory(), module.getEngineFactory());
         indexService.close("simon says", false);
     }
@@ -321,7 +319,7 @@ public void testFrozen() {
         assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.addIndexEventListener(null)).getMessage());
         assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.addIndexOperationListener(null)).getMessage());
         assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.addSimilarity(null, null)).getMessage());
-        assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.setSearcherWrapper(null)).getMessage());
+        assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.setReaderWrapper(null)).getMessage());
         assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.forceQueryCacheProvider(null)).getMessage());
     }
 
@@ -455,15 +453,9 @@ public Directory newDirectory(IndexSettings indexSettings, ShardPath shardPath)
         }
     }
 
-    public static final class Wrapper extends IndexSearcherWrapper {
-
-        @Override
-        public DirectoryReader wrap(DirectoryReader reader) {
-            return null;
-        }
-
+    public static final class Wrapper implements CheckedFunction<DirectoryReader, DirectoryReader, IOException> {
         @Override
-        public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
+        public DirectoryReader apply(DirectoryReader reader) {
             return null;
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java
index 2d4030a51ce3d..af4d621448ad7 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java
@@ -42,12 +42,15 @@
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
+import static org.elasticsearch.index.shard.IndexShardTestCase.getEngine;
 import static org.elasticsearch.test.InternalSettingsPlugin.TRANSLOG_RETENTION_CHECK_INTERVAL_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.core.IsEqual.equalTo;
@@ -370,7 +373,7 @@ public void testAsyncTranslogTrimActuallyWorks() throws Exception {
             .build();
         IndexService indexService = createIndex("test", settings);
         ensureGreen("test");
-        assertTrue(indexService.getRefreshTask().mustReschedule());
+        assertTrue(indexService.getTrimTranslogTask().mustReschedule());
         client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get();
         client().admin().indices().prepareFlush("test").get();
         client().admin().indices().prepareUpdateSettings("test")
@@ -382,6 +385,48 @@ public void testAsyncTranslogTrimActuallyWorks() throws Exception {
         assertBusy(() -> assertThat(IndexShardTestCase.getTranslog(shard).totalOperations(), equalTo(0)));
     }
 
+    public void testAsyncTranslogTrimTaskOnClosedIndex() throws Exception {
+        final String indexName = "test";
+        IndexService indexService = createIndex(indexName, Settings.builder()
+            .put(TRANSLOG_RETENTION_CHECK_INTERVAL_SETTING.getKey(), "100ms")
+            .build());
+
+        Translog translog = IndexShardTestCase.getTranslog(indexService.getShard(0));
+        final Path translogPath = translog.getConfig().getTranslogPath();
+        final String translogUuid = translog.getTranslogUUID();
+
+        final int numDocs = scaledRandomIntBetween(10, 100);
+        for (int i = 0; i < numDocs; i++) {
+            client().prepareIndex().setIndex(indexName).setId(String.valueOf(i)).setSource("{\"foo\": \"bar\"}", XContentType.JSON).get();
+            if (randomBoolean()) {
+                client().admin().indices().prepareFlush(indexName).get();
+            }
+        }
+        assertThat(translog.totalOperations(), equalTo(numDocs));
+        assertThat(translog.stats().estimatedNumberOfOperations(), equalTo(numDocs));
+        assertAcked(client().admin().indices().prepareClose("test"));
+
+        indexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(indexService.index());
+        assertTrue(indexService.getTrimTranslogTask().mustReschedule());
+
+        final long lastCommitedTranslogGeneration;
+        try (Engine.IndexCommitRef indexCommitRef = getEngine(indexService.getShard(0)).acquireLastIndexCommit(false)) {
+            Map<String, String> lastCommittedUserData = indexCommitRef.getIndexCommit().getUserData();
+            lastCommitedTranslogGeneration = Long.parseLong(lastCommittedUserData.get(Translog.TRANSLOG_GENERATION_KEY));
+        }
+        assertBusy(() -> {
+            long minTranslogGen = Translog.readMinTranslogGeneration(translogPath, translogUuid);
+            assertThat(minTranslogGen, equalTo(lastCommitedTranslogGeneration));
+        });
+
+        assertAcked(client().admin().indices().prepareOpen("test"));
+
+        indexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(indexService.index());
+        translog = IndexShardTestCase.getTranslog(indexService.getShard(0));
+        assertThat(translog.totalOperations(), equalTo(0));
+        assertThat(translog.stats().estimatedNumberOfOperations(), equalTo(0));
+    }
+
     public void testIllegalFsyncInterval() {
         Settings settings = Settings.builder()
             .put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(), "0ms") // disable
diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
index b3e6557b187ae..31a92745a4245 100644
--- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
+++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
@@ -564,17 +564,8 @@ public void testUpdateSoftDeletesFails() {
 
     public void testSoftDeletesDefaultSetting() {
         // enabled by default on 7.0+ or later
-        {
-            Version createdVersion = VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.CURRENT);
-            Settings settings = Settings.builder().put(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey(), createdVersion).build();
-            assertTrue(IndexSettings.INDEX_SOFT_DELETES_SETTING.get(settings));
-        }
-        // disabled by default on the previous versions
-        {
-            Version prevVersion = VersionUtils.randomVersionBetween(
-                random(), Version.V_6_5_0, VersionUtils.getPreviousVersion(Version.V_7_0_0));
-            Settings settings = Settings.builder().put(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey(), prevVersion).build();
-            assertFalse(IndexSettings.INDEX_SOFT_DELETES_SETTING.get(settings));
-        }
+        Version createdVersion = VersionUtils.randomIndexCompatibleVersion(random());
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey(), createdVersion).build();
+        assertTrue(IndexSettings.INDEX_SOFT_DELETES_SETTING.get(settings));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
index c054b2008dba6..2f682f437a2cb 100644
--- a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
+++ b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
@@ -63,11 +63,6 @@ public ShardId shardId() {
                     return new ShardId(indexService.index(), 0);
                 }
 
-                @Override
-                public String[] types() {
-                    return new String[0];
-                }
-
                 @Override
                 public SearchSourceBuilder source() {
                     return searchSourceBuilder;
diff --git a/server/src/test/java/org/elasticsearch/index/SettingsListenerIT.java b/server/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
index 8b603420dfd05..21f15cc46907c 100644
--- a/server/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
+++ b/server/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
@@ -18,14 +18,22 @@
  */
 package org.elasticsearch.index;
 
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -60,8 +68,11 @@ public void onIndexModule(IndexModule module) {
         }
 
         @Override
-        public Collection<Module> createGuiceModules() {
-            return Collections.<Module>singletonList(new SettingsListenerModule(service));
+        public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
+                                                   ResourceWatcherService resourceWatcherService, ScriptService scriptService,
+                                                   NamedXContentRegistry xContentRegistry, Environment environment,
+                                                   NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) {
+            return Collections.singletonList(service);
         }
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/analysis/IndexAnalyzersTests.java b/server/src/test/java/org/elasticsearch/index/analysis/IndexAnalyzersTests.java
index f5808c4b0da5c..4f397f2039cae 100644
--- a/server/src/test/java/org/elasticsearch/index/analysis/IndexAnalyzersTests.java
+++ b/server/src/test/java/org/elasticsearch/index/analysis/IndexAnalyzersTests.java
@@ -19,15 +19,16 @@
 
 package org.elasticsearch.index.analysis;
 
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 public class IndexAnalyzersTests extends ESTestCase {
 
@@ -38,16 +39,14 @@ public void testAnalyzerMapChecks() {
         Map<String, NamedAnalyzer> analyzers = new HashMap<>();
         {
             NullPointerException ex = expectThrows(NullPointerException.class,
-                    () -> new IndexAnalyzers(IndexSettingsModule.newIndexSettings("index", Settings.EMPTY), analyzers,
-                            Collections.emptyMap(), Collections.emptyMap()));
+                    () -> new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap()));
             assertEquals("the default analyzer must be set", ex.getMessage());
         }
         {
             analyzers.put(AnalysisRegistry.DEFAULT_ANALYZER_NAME,
                     new NamedAnalyzer("otherName", AnalyzerScope.INDEX, new StandardAnalyzer()));
             IllegalStateException ex = expectThrows(IllegalStateException.class,
-                    () -> new IndexAnalyzers(IndexSettingsModule.newIndexSettings("index", Settings.EMPTY), analyzers,
-                            Collections.emptyMap(), Collections.emptyMap()));
+                    () -> new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap()));
             assertEquals("default analyzer must have the name [default] but was: [otherName]", ex.getMessage());
         }
     }
@@ -58,8 +57,7 @@ public void testAnalyzerDefaults() throws IOException {
         analyzers.put(AnalysisRegistry.DEFAULT_ANALYZER_NAME, analyzer);
 
         // if only "default" is set in the map, all getters should return the same analyzer
-        try (IndexAnalyzers indexAnalyzers = new IndexAnalyzers(IndexSettingsModule.newIndexSettings("index", Settings.EMPTY), analyzers,
-                Collections.emptyMap(), Collections.emptyMap())) {
+        try (IndexAnalyzers indexAnalyzers = new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap())) {
             assertSame(analyzer, indexAnalyzers.getDefaultIndexAnalyzer());
             assertSame(analyzer, indexAnalyzers.getDefaultSearchAnalyzer());
             assertSame(analyzer, indexAnalyzers.getDefaultSearchQuoteAnalyzer());
@@ -67,8 +65,7 @@ public void testAnalyzerDefaults() throws IOException {
 
         analyzers.put(AnalysisRegistry.DEFAULT_SEARCH_ANALYZER_NAME,
                 new NamedAnalyzer("my_search_analyzer", AnalyzerScope.INDEX, new StandardAnalyzer()));
-        try (IndexAnalyzers indexAnalyzers = new IndexAnalyzers(IndexSettingsModule.newIndexSettings("index", Settings.EMPTY), analyzers,
-                Collections.emptyMap(), Collections.emptyMap())) {
+        try (IndexAnalyzers indexAnalyzers = new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap())) {
             assertSame(analyzer, indexAnalyzers.getDefaultIndexAnalyzer());
             assertEquals("my_search_analyzer", indexAnalyzers.getDefaultSearchAnalyzer().name());
             assertEquals("my_search_analyzer", indexAnalyzers.getDefaultSearchQuoteAnalyzer().name());
@@ -76,12 +73,44 @@ public void testAnalyzerDefaults() throws IOException {
 
         analyzers.put(AnalysisRegistry.DEFAULT_SEARCH_QUOTED_ANALYZER_NAME,
                 new NamedAnalyzer("my_search_quote_analyzer", AnalyzerScope.INDEX, new StandardAnalyzer()));
-        try (IndexAnalyzers indexAnalyzers = new IndexAnalyzers(IndexSettingsModule.newIndexSettings("index", Settings.EMPTY), analyzers,
-                Collections.emptyMap(), Collections.emptyMap())) {
+        try (IndexAnalyzers indexAnalyzers = new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap())) {
             assertSame(analyzer, indexAnalyzers.getDefaultIndexAnalyzer());
             assertEquals("my_search_analyzer", indexAnalyzers.getDefaultSearchAnalyzer().name());
             assertEquals("my_search_quote_analyzer", indexAnalyzers.getDefaultSearchQuoteAnalyzer().name());
         }
     }
 
+    public void testClose() throws IOException {
+
+        AtomicInteger closes = new AtomicInteger(0);
+        NamedAnalyzer a = new NamedAnalyzer("default", AnalyzerScope.INDEX, new WhitespaceAnalyzer()){
+            @Override
+            public void close() {
+                super.close();
+                closes.incrementAndGet();
+            }
+        };
+
+        NamedAnalyzer n = new NamedAnalyzer("keyword_normalizer", AnalyzerScope.INDEX, new KeywordAnalyzer()){
+            @Override
+            public void close() {
+                super.close();
+                closes.incrementAndGet();
+            }
+        };
+
+        NamedAnalyzer w = new NamedAnalyzer("whitespace_normalizer", AnalyzerScope.INDEX, new WhitespaceAnalyzer()){
+            @Override
+            public void close() {
+                super.close();
+                closes.incrementAndGet();
+            }
+        };
+
+        IndexAnalyzers ia = new IndexAnalyzers(Map.of("default", a), Map.of("n", n), Map.of("w", w));
+        ia.close();
+        assertEquals(3, closes.get());
+
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzerTests.java b/server/src/test/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzerTests.java
new file mode 100644
index 0000000000000..e60df7e2ce1a4
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/analysis/ReloadableCustomAnalyzerTests.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.index.analysis.AnalyzerComponents.createComponents;
+
+public class ReloadableCustomAnalyzerTests extends ESTestCase {
+
+    private static TestAnalysis testAnalysis;
+    private static Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
+
+    private static TokenFilterFactory NO_OP_SEARCH_TIME_FILTER = new AbstractTokenFilterFactory(
+            IndexSettingsModule.newIndexSettings("index", settings), "my_filter", Settings.EMPTY) {
+        @Override
+        public AnalysisMode getAnalysisMode() {
+            return AnalysisMode.SEARCH_TIME;
+        }
+
+        @Override
+        public TokenStream create(TokenStream tokenStream) {
+            return tokenStream;
+        }
+    };
+
+    private static TokenFilterFactory LOWERCASE_SEARCH_TIME_FILTER = new AbstractTokenFilterFactory(
+            IndexSettingsModule.newIndexSettings("index", settings), "my_other_filter", Settings.EMPTY) {
+        @Override
+        public AnalysisMode getAnalysisMode() {
+            return AnalysisMode.SEARCH_TIME;
+        }
+
+        @Override
+        public TokenStream create(TokenStream tokenStream) {
+            return new LowerCaseFilter(tokenStream);
+        }
+    };
+
+    @BeforeClass
+    public static void setup() throws IOException {
+        testAnalysis = createTestAnalysis(new Index("test", "_na_"), settings);
+    }
+
+    /**
+     * test constructor and getters
+     */
+    public void testBasicCtor() {
+        int positionIncrementGap = randomInt();
+        int offsetGap = randomInt();
+
+        Settings analyzerSettings = Settings.builder()
+                .put("tokenizer", "standard")
+                .putList("filter", "my_filter")
+                .build();
+
+        AnalyzerComponents components = createComponents("my_analyzer", analyzerSettings, testAnalysis.tokenizer, testAnalysis.charFilter,
+                Collections.singletonMap("my_filter", NO_OP_SEARCH_TIME_FILTER));
+
+        try (ReloadableCustomAnalyzer analyzer = new ReloadableCustomAnalyzer(components, positionIncrementGap, offsetGap)) {
+            assertEquals(positionIncrementGap, analyzer.getPositionIncrementGap(randomAlphaOfLength(5)));
+            assertEquals(offsetGap >= 0 ? offsetGap : 1, analyzer.getOffsetGap(randomAlphaOfLength(5)));
+            assertEquals("standard", analyzer.getComponents().getTokenizerName());
+            assertEquals(0, analyzer.getComponents().getCharFilters().length);
+            assertSame(testAnalysis.tokenizer.get("standard"), analyzer.getComponents().getTokenizerFactory());
+            assertEquals(1, analyzer.getComponents().getTokenFilters().length);
+            assertSame(NO_OP_SEARCH_TIME_FILTER, analyzer.getComponents().getTokenFilters()[0]);
+        }
+
+        // check that when using regular non-search time filters only, we get an exception
+        final Settings indexAnalyzerSettings = Settings.builder()
+                .put("tokenizer", "standard")
+                .putList("filter", "lowercase")
+                .build();
+        AnalyzerComponents indexAnalyzerComponents = createComponents("my_analyzer", indexAnalyzerSettings, testAnalysis.tokenizer,
+                testAnalysis.charFilter, testAnalysis.tokenFilter);
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+                () -> new ReloadableCustomAnalyzer(indexAnalyzerComponents, positionIncrementGap, offsetGap));
+        assertEquals("ReloadableCustomAnalyzer must only be initialized with analysis components in AnalysisMode.SEARCH_TIME mode",
+                ex.getMessage());
+    }
+
+    /**
+     * start multiple threads that create token streams from this analyzer until reloaded tokenfilter takes effect
+     */
+    public void testReloading() throws IOException, InterruptedException {
+        Settings analyzerSettings = Settings.builder()
+                .put("tokenizer", "standard")
+                .putList("filter", "my_filter")
+                .build();
+
+        AnalyzerComponents components = createComponents("my_analyzer", analyzerSettings, testAnalysis.tokenizer, testAnalysis.charFilter,
+                Collections.singletonMap("my_filter", NO_OP_SEARCH_TIME_FILTER));
+        int numThreads = randomIntBetween(5, 10);
+
+        ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
+        CountDownLatch firstCheckpoint = new CountDownLatch(numThreads);
+        CountDownLatch secondCheckpoint = new CountDownLatch(numThreads);
+
+        try (ReloadableCustomAnalyzer analyzer = new ReloadableCustomAnalyzer(components, 0, 0)) {
+            executorService.submit(() -> {
+                while (secondCheckpoint.getCount() > 0) {
+                    try (TokenStream firstTokenStream = analyzer.tokenStream("myField", "TEXT")) {
+                        firstTokenStream.reset();
+                        CharTermAttribute term = firstTokenStream.addAttribute(CharTermAttribute.class);
+                        assertTrue(firstTokenStream.incrementToken());
+                        if (term.toString().equals("TEXT")) {
+                            firstCheckpoint.countDown();
+                        }
+                        if (term.toString().equals("text")) {
+                            secondCheckpoint.countDown();
+                        }
+                        assertFalse(firstTokenStream.incrementToken());
+                        firstTokenStream.end();
+                    } catch (Exception e) {
+                        throw ExceptionsHelper.convertToRuntime(e);
+                    }
+                }
+            });
+
+            // wait until all running threads have seen the unaltered upper case analysis at least once
+            assertTrue(firstCheckpoint.await(5, TimeUnit.SECONDS));
+
+            analyzer.reload("my_analyzer", analyzerSettings, testAnalysis.tokenizer, testAnalysis.charFilter,
+                    Collections.singletonMap("my_filter", LOWERCASE_SEARCH_TIME_FILTER));
+
+            // wait until all running threads have seen the new lower case analysis at least once
+            assertTrue(secondCheckpoint.await(5, TimeUnit.SECONDS));
+
+            executorService.shutdown();
+            executorService.awaitTermination(1, TimeUnit.SECONDS);
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java b/server/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java
index 617d23c44e1c6..110a27ff5510f 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java
@@ -30,7 +30,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -38,7 +37,6 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 
-import static java.util.Collections.singletonList;
 import static org.elasticsearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED;
 import static org.elasticsearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy;
 import static org.hamcrest.Matchers.equalTo;
@@ -180,41 +178,6 @@ public void testAcquireIndexCommit() throws Exception {
             Math.max(NO_OPS_PERFORMED, Math.min(getLocalCheckpoint(safeCommit) + 1, globalCheckpoint.get() + 1 - extraRetainedOps))));
     }
 
-    public void testLegacyIndex() throws Exception {
-        final AtomicLong globalCheckpoint = new AtomicLong();
-        final SoftDeletesPolicy softDeletesPolicy = new SoftDeletesPolicy(globalCheckpoint::get, -1, 0, () -> RetentionLeases.EMPTY);
-        final UUID translogUUID = UUID.randomUUID();
-
-        TranslogDeletionPolicy translogPolicy = createTranslogDeletionPolicy();
-        CombinedDeletionPolicy indexPolicy = new CombinedDeletionPolicy(logger, translogPolicy, softDeletesPolicy, globalCheckpoint::get);
-
-        long legacyTranslogGen = randomNonNegativeLong();
-        IndexCommit legacyCommit = mockLegacyIndexCommit(translogUUID, legacyTranslogGen);
-        assertThat(CombinedDeletionPolicy.findSafeCommitPoint(singletonList(legacyCommit), globalCheckpoint.get()),
-            equalTo(legacyCommit));
-
-        long safeTranslogGen = randomLongBetween(legacyTranslogGen, Long.MAX_VALUE);
-        long maxSeqNo = randomLongBetween(1, Long.MAX_VALUE);
-        final IndexCommit freshCommit = mockIndexCommit(randomLongBetween(-1, maxSeqNo), maxSeqNo, translogUUID, safeTranslogGen);
-
-        globalCheckpoint.set(randomLongBetween(0, maxSeqNo - 1));
-        indexPolicy.onCommit(Arrays.asList(legacyCommit, freshCommit));
-        verify(legacyCommit, times(1)).delete(); // Do not keep the legacy commit once we have a new commit.
-        verify(freshCommit, times(0)).delete();
-        assertThat(translogPolicy.getMinTranslogGenerationForRecovery(), equalTo(safeTranslogGen));
-        assertThat(translogPolicy.getTranslogGenerationOfLastCommit(), equalTo(safeTranslogGen));
-
-        // Make the fresh commit safe.
-        resetDeletion(legacyCommit);
-        globalCheckpoint.set(randomLongBetween(maxSeqNo, Long.MAX_VALUE));
-        indexPolicy.onCommit(Arrays.asList(legacyCommit, freshCommit));
-        verify(legacyCommit, times(2)).delete();
-        verify(freshCommit, times(0)).delete();
-        assertThat(translogPolicy.getMinTranslogGenerationForRecovery(), equalTo(safeTranslogGen));
-        assertThat(translogPolicy.getTranslogGenerationOfLastCommit(), equalTo(safeTranslogGen));
-        assertThat(softDeletesPolicy.getMinRetainedSeqNo(), equalTo(getLocalCheckpoint(freshCommit) + 1));
-    }
-
     public void testDeleteInvalidCommits() throws Exception {
         final AtomicLong globalCheckpoint = new AtomicLong(randomNonNegativeLong());
         final SoftDeletesPolicy softDeletesPolicy = new SoftDeletesPolicy(globalCheckpoint::get, -1, 0, () -> RetentionLeases.EMPTY);
@@ -317,15 +280,4 @@ void resetDeletion(IndexCommit commit) {
     private long getLocalCheckpoint(IndexCommit commit) throws IOException {
         return Long.parseLong(commit.getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
     }
-
-    IndexCommit mockLegacyIndexCommit(UUID translogUUID, long translogGen) throws IOException {
-        final Map<String, String> userData = new HashMap<>();
-        userData.put(Translog.TRANSLOG_UUID_KEY, translogUUID.toString());
-        userData.put(Translog.TRANSLOG_GENERATION_KEY, Long.toString(translogGen));
-        final IndexCommit commit = mock(IndexCommit.class);
-        when(commit.getUserData()).thenReturn(userData);
-        resetDeletion(commit);
-        return commit;
-    }
-
 }
diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java
index 1bf928fcb8060..a072ca880e01e 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
@@ -40,7 +39,6 @@
 @ClusterScope(supportsDedicatedMasters = false, numDataNodes = 1, scope = Scope.SUITE)
 public class InternalEngineMergeIT extends ESIntegTestCase {
 
-    @TestLogging("_root:DEBUG")
     public void testMergesHappening() throws InterruptedException, IOException, ExecutionException {
         final int numOfShards = randomIntBetween(1, 5);
         // some settings to keep num segments low
diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index b213da097ce5e..09872b418cc22 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -21,7 +21,6 @@
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
-
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -51,6 +50,8 @@
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.index.SoftDeletesRetentionMergePolicy;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.TieredMergePolicy;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
@@ -61,8 +62,11 @@
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TotalHitCountCollector;
 import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.BaseDirectoryWrapper;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
 import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -98,6 +102,7 @@
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.codec.CodecService;
@@ -115,15 +120,18 @@
 import org.elasticsearch.index.mapper.RootObjectMapper;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.mapper.VersionFieldMapper;
 import org.elasticsearch.index.seqno.LocalCheckpointTracker;
 import org.elasticsearch.index.seqno.ReplicationTracker;
 import org.elasticsearch.index.seqno.RetentionLease;
 import org.elasticsearch.index.seqno.RetentionLeases;
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.seqno.SequenceNumbers;
-import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.shard.ShardPath;
 import org.elasticsearch.index.shard.ShardUtils;
+import org.elasticsearch.index.store.FsDirectoryFactory;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.SnapshotMatchers;
 import org.elasticsearch.index.translog.Translog;
@@ -213,7 +221,7 @@ public void testVersionMapAfterAutoIDDocument() throws IOException {
         Engine.Index update = indexForDoc(doc);
         engine.index(update);
         assertTrue(engine.isSafeAccessRequired());
-        assertEquals(1, engine.getVersionMapSize());
+        assertThat(engine.getVersionMap().values(), hasSize(1));
         try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
             assertEquals(0, searcher.reader().numDocs());
         }
@@ -245,7 +253,7 @@ public void testVersionMapAfterAutoIDDocument() throws IOException {
             : appendOnlyReplica(doc, false, 1, generateNewSeqNo(engine));
         engine.index(operation);
         assertTrue("safe access should be required", engine.isSafeAccessRequired());
-        assertEquals(1, engine.getVersionMapSize()); // now we add this to the map
+        assertThat(engine.getVersionMap().values(), hasSize(1)); // now we add this to the map
         engine.refresh("test");
         if (randomBoolean()) { // randomly refresh here again
             engine.refresh("test");
@@ -263,7 +271,7 @@ public void testVersionMapAfterAutoIDDocument() throws IOException {
         }
     }
 
-    public void testSegments() throws Exception {
+    public void testSegmentsWithoutSoftDeletes() throws Exception {
         Settings settings = Settings.builder()
             .put(defaultSettings.getSettings())
             .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false).build();
@@ -591,6 +599,68 @@ public void testSegmentsStatsIncludingFileSizes() throws Exception {
         }
     }
 
+    public void testSegmentsWithSoftDeletes() throws Exception {
+        Settings.Builder settings = Settings.builder()
+            .put(defaultSettings.getSettings())
+            .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
+        final IndexMetaData indexMetaData = IndexMetaData.builder(defaultSettings.getIndexMetaData()).settings(settings).build();
+        final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetaData);
+        final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
+        try (Store store = createStore();
+             InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null,
+                 null, globalCheckpoint::get))) {
+            assertThat(engine.segments(false), empty());
+            int numDocsFirstSegment = randomIntBetween(5, 50);
+            Set<String> liveDocsFirstSegment = new HashSet<>();
+            for (int i = 0; i < numDocsFirstSegment; i++) {
+                String id = Integer.toString(i);
+                ParsedDocument doc = testParsedDocument(id, null, testDocument(), B_1, null);
+                engine.index(indexForDoc(doc));
+                liveDocsFirstSegment.add(id);
+            }
+            engine.refresh("test");
+            List<Segment> segments = engine.segments(randomBoolean());
+            assertThat(segments, hasSize(1));
+            assertThat(segments.get(0).getNumDocs(), equalTo(liveDocsFirstSegment.size()));
+            assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
+            assertFalse(segments.get(0).committed);
+            int deletes = 0;
+            int updates = 0;
+            int appends = 0;
+            int iterations = scaledRandomIntBetween(1, 50);
+            for (int i = 0; i < iterations && liveDocsFirstSegment.isEmpty() == false; i++) {
+                String idToUpdate = randomFrom(liveDocsFirstSegment);
+                liveDocsFirstSegment.remove(idToUpdate);
+                ParsedDocument doc = testParsedDocument(idToUpdate, null, testDocument(), B_1, null);
+                if (randomBoolean()) {
+                    engine.delete(new Engine.Delete(doc.type(), doc.id(), newUid(doc), primaryTerm.get()));
+                    deletes++;
+                } else {
+                    engine.index(indexForDoc(doc));
+                    updates++;
+                }
+                if (randomBoolean()) {
+                    engine.index(indexForDoc(testParsedDocument(UUIDs.randomBase64UUID(), null, testDocument(), B_1, null)));
+                    appends++;
+                }
+            }
+            boolean committed = randomBoolean();
+            if (committed) {
+                engine.flush();
+            }
+            engine.refresh("test");
+            segments = engine.segments(randomBoolean());
+            assertThat(segments, hasSize(2));
+            assertThat(segments.get(0).getNumDocs(), equalTo(liveDocsFirstSegment.size()));
+            assertThat(segments.get(0).getDeletedDocs(), equalTo(updates + deletes));
+            assertThat(segments.get(0).committed, equalTo(committed));
+
+            assertThat(segments.get(1).getNumDocs(), equalTo(updates + appends));
+            assertThat(segments.get(1).getDeletedDocs(), equalTo(deletes)); // delete tombstones
+            assertThat(segments.get(1).committed, equalTo(committed));
+        }
+    }
+
     public void testCommitStats() throws IOException {
         final AtomicLong maxSeqNo = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
         final AtomicLong localCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
@@ -606,7 +676,7 @@ public long getMaxSeqNo() {
                         }
 
                         @Override
-                        public long getCheckpoint() {
+                        public long getProcessedCheckpoint() {
                             return localCheckpoint.get();
                         }
                     }
@@ -652,36 +722,6 @@ public long getCheckpoint() {
         }
     }
 
-    public void testIndexSearcherWrapper() throws Exception {
-        final AtomicInteger counter = new AtomicInteger();
-        IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {
-
-            @Override
-            public DirectoryReader wrap(DirectoryReader reader) {
-                counter.incrementAndGet();
-                return reader;
-            }
-
-            @Override
-            public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
-                counter.incrementAndGet();
-                return searcher;
-            }
-        };
-        Store store = createStore();
-        Path translog = createTempDir("translog-test");
-        InternalEngine engine = createEngine(store, translog);
-        engine.close();
-
-        engine = new InternalEngine(engine.config());
-        assertTrue(engine.isRecovering());
-        engine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
-        Engine.Searcher searcher = wrapper.wrap(engine.acquireSearcher("test"));
-        assertThat(counter.get(), equalTo(2));
-        searcher.close();
-        IOUtils.close(store, engine);
-    }
-
     public void testFlushIsDisabledDuringTranslogRecovery() throws IOException {
         assertFalse(engine.isRecovering());
         ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), SOURCE, null);
@@ -820,18 +860,18 @@ public void testRecoveryFromTranslogUpToSeqNo() throws IOException {
                     }
                 }
                 maxSeqNo = engine.getLocalCheckpointTracker().getMaxSeqNo();
-                globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getLocalCheckpoint()));
+                globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getProcessedLocalCheckpoint()));
                 engine.syncTranslog();
             }
             try (InternalEngine engine = new InternalEngine(config)) {
                 engine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
-                assertThat(engine.getLocalCheckpoint(), equalTo(maxSeqNo));
+                assertThat(engine.getProcessedLocalCheckpoint(), equalTo(maxSeqNo));
                 assertThat(engine.getLocalCheckpointTracker().getMaxSeqNo(), equalTo(maxSeqNo));
             }
             try (InternalEngine engine = new InternalEngine(config)) {
                 long upToSeqNo = randomLongBetween(globalCheckpoint.get(), maxSeqNo);
                 engine.recoverFromTranslog(translogHandler, upToSeqNo);
-                assertThat(engine.getLocalCheckpoint(), equalTo(upToSeqNo));
+                assertThat(engine.getProcessedLocalCheckpoint(), equalTo(upToSeqNo));
                 assertThat(engine.getLocalCheckpointTracker().getMaxSeqNo(), equalTo(upToSeqNo));
             }
         }
@@ -1101,7 +1141,8 @@ public void testCommitAdvancesMinTranslogForRecovery() throws IOException {
         engine.index(indexForDoc(doc));
         boolean inSync = randomBoolean();
         if (inSync) {
-            globalCheckpoint.set(engine.getLocalCheckpoint());
+            engine.syncTranslog(); // to advance persisted local checkpoint
+            globalCheckpoint.set(engine.getPersistedLocalCheckpoint());
         }
 
         engine.flush();
@@ -1119,7 +1160,7 @@ public void testCommitAdvancesMinTranslogForRecovery() throws IOException {
         assertThat(engine.getTranslog().getDeletionPolicy().getMinTranslogGenerationForRecovery(), equalTo(inSync ? 4L : 1L));
         assertThat(engine.getTranslog().getDeletionPolicy().getTranslogGenerationOfLastCommit(), equalTo(4L));
 
-        globalCheckpoint.set(engine.getLocalCheckpoint());
+        globalCheckpoint.set(engine.getPersistedLocalCheckpoint());
         engine.flush(true, true);
         assertThat(engine.getTranslog().currentFileGeneration(), equalTo(5L));
         assertThat(engine.getTranslog().getDeletionPolicy().getMinTranslogGenerationForRecovery(), equalTo(5L));
@@ -1452,6 +1493,61 @@ public void testForceMergeWithoutSoftDeletes() throws IOException {
         }
     }
 
+    /*
+     * we are testing an edge case here where we have a fully deleted segment that is retained but has all it's IDs pruned away.
+     */
+    public void testLookupVersionWithPrunedAwayIds() throws IOException {
+        try (Directory dir = newDirectory()) {
+            IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Lucene.STANDARD_ANALYZER);
+            indexWriterConfig.setSoftDeletesField(Lucene.SOFT_DELETES_FIELD);
+            try (IndexWriter writer = new IndexWriter(dir,
+                indexWriterConfig.setMergePolicy(new SoftDeletesRetentionMergePolicy(Lucene.SOFT_DELETES_FIELD,
+                    MatchAllDocsQuery::new, new PrunePostingsMergePolicy(indexWriterConfig.getMergePolicy(), "_id"))))) {
+                org.apache.lucene.document.Document doc = new org.apache.lucene.document.Document();
+                doc.add(new Field(IdFieldMapper.NAME, "1", IdFieldMapper.Defaults.FIELD_TYPE));
+                doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, -1));
+                doc.add(new NumericDocValuesField(SeqNoFieldMapper.NAME, 1));
+                doc.add(new NumericDocValuesField(SeqNoFieldMapper.PRIMARY_TERM_NAME, 1));
+                writer.addDocument(doc);
+                writer.flush();
+                writer.softUpdateDocument(new Term(IdFieldMapper.NAME, "1"), doc, new NumericDocValuesField(Lucene.SOFT_DELETES_FIELD, 1));
+                writer.updateNumericDocValue(new Term(IdFieldMapper.NAME, "1"), Lucene.SOFT_DELETES_FIELD, 1);
+                writer.forceMerge(1);
+                try (DirectoryReader reader = DirectoryReader.open(writer)) {
+                    assertEquals(1, reader.leaves().size());
+                    assertNull(VersionsAndSeqNoResolver.loadDocIdAndVersion(reader, new Term(IdFieldMapper.NAME, "1"), false));
+                }
+            }
+        }
+    }
+
+    public void testUpdateWithFullyDeletedSegments() throws IOException {
+        Settings.Builder settings = Settings.builder()
+            .put(defaultSettings.getSettings())
+            .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
+            .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), Integer.MAX_VALUE);
+        final IndexMetaData indexMetaData = IndexMetaData.builder(defaultSettings.getIndexMetaData()).settings(settings).build();
+        final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetaData);
+        final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
+        final Set<String> liveDocs = new HashSet<>();
+        try (Store store = createStore();
+             InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), newMergePolicy(), null,
+                 null, globalCheckpoint::get))) {
+            int numDocs = scaledRandomIntBetween(10, 100);
+            for (int i = 0; i < numDocs; i++) {
+                ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), B_1, null);
+                engine.index(indexForDoc(doc));
+                liveDocs.add(doc.id());
+            }
+
+            for (int i = 0; i < numDocs; i++) {
+                ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), B_1, null);
+                engine.index(indexForDoc(doc));
+                liveDocs.add(doc.id());
+            }
+        }
+    }
+
     public void testForceMergeWithSoftDeletesRetention() throws Exception {
         final long retainedExtraOps = randomLongBetween(0, 10);
         Settings.Builder settings = Settings.builder()
@@ -1488,7 +1584,7 @@ public void testForceMergeWithSoftDeletesRetention() throws Exception {
             }
             engine.flush();
 
-            long localCheckpoint = engine.getLocalCheckpoint();
+            long localCheckpoint = engine.getProcessedLocalCheckpoint();
             globalCheckpoint.set(randomLongBetween(0, localCheckpoint));
             engine.syncTranslog();
             final long safeCommitCheckpoint;
@@ -1576,7 +1672,7 @@ public void testForceMergeWithSoftDeletesRetentionAndRecoverySource() throws Exc
                 }
             }
             engine.flush();
-            globalCheckpoint.set(randomLongBetween(0, engine.getLocalCheckpoint()));
+            globalCheckpoint.set(randomLongBetween(0, engine.getPersistedLocalCheckpoint()));
             engine.syncTranslog();
             final long minSeqNoToRetain;
             try (Engine.IndexCommitRef safeCommit = engine.acquireSafeIndexCommit()) {
@@ -1588,7 +1684,7 @@ public void testForceMergeWithSoftDeletesRetentionAndRecoverySource() throws Exc
             assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, mapperService);
             Map<Long, Translog.Operation> ops = readAllOperationsInLucene(engine, mapperService)
                 .stream().collect(Collectors.toMap(Translog.Operation::seqNo, Function.identity()));
-            for (long seqno = 0; seqno <= engine.getLocalCheckpoint(); seqno++) {
+            for (long seqno = 0; seqno <= engine.getPersistedLocalCheckpoint(); seqno++) {
                 String msg = "seq# [" + seqno + "], global checkpoint [" + globalCheckpoint + "], retained-ops [" + retainedExtraOps + "]";
                 if (seqno < minSeqNoToRetain) {
                     Translog.Operation op = ops.get(seqno);
@@ -1610,14 +1706,14 @@ public void testForceMergeWithSoftDeletesRetentionAndRecoverySource() throws Exc
             // If the global checkpoint equals to the local checkpoint, the next force-merge will be a noop
             // because all deleted documents are expunged in the previous force-merge already. We need to flush
             // a new segment to make merge happen so that we can verify that all _recovery_source are pruned.
-            if (globalCheckpoint.get() == engine.getLocalCheckpoint() && liveDocs.isEmpty() == false) {
+            if (globalCheckpoint.get() == engine.getPersistedLocalCheckpoint() && liveDocs.isEmpty() == false) {
                 String deleteId = randomFrom(liveDocs);
                 engine.delete(new Engine.Delete("test", deleteId, newUid(deleteId), primaryTerm.get()));
                 liveDocsWithSource.remove(deleteId);
                 liveDocs.remove(deleteId);
                 engine.flush();
             }
-            globalCheckpoint.set(engine.getLocalCheckpoint());
+            globalCheckpoint.set(engine.getPersistedLocalCheckpoint());
             engine.syncTranslog();
             engine.forceMerge(true, 1, false, false, false);
             assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, mapperService);
@@ -2323,12 +2419,14 @@ public void testSeqNoAndCheckpoints() throws IOException {
                     }
                 }
 
+                initialEngine.syncTranslog(); // to advance persisted local checkpoint
+
                 if (randomInt(10) < 3) {
                     // only update rarely as we do it every doc
                     replicaLocalCheckpoint = randomIntBetween(Math.toIntExact(replicaLocalCheckpoint), Math.toIntExact(primarySeqNo));
                 }
                 gcpTracker.updateLocalCheckpoint(primary.allocationId().getId(),
-                    initialEngine.getLocalCheckpoint());
+                    initialEngine.getPersistedLocalCheckpoint());
                 gcpTracker.updateLocalCheckpoint(replica.allocationId().getId(), replicaLocalCheckpoint);
 
                 if (rarely()) {
@@ -2342,7 +2440,7 @@ public void testSeqNoAndCheckpoints() throws IOException {
             globalCheckpoint = gcpTracker.getGlobalCheckpoint();
 
             assertEquals(primarySeqNo, initialEngine.getSeqNoStats(-1).getMaxSeqNo());
-            assertEquals(primarySeqNo, initialEngine.getLocalCheckpoint());
+            assertEquals(primarySeqNo, initialEngine.getPersistedLocalCheckpoint());
             assertThat(globalCheckpoint, equalTo(replicaLocalCheckpoint));
 
             assertThat(
@@ -2376,7 +2474,8 @@ public void testSeqNoAndCheckpoints() throws IOException {
                 // that the committed max seq no is equivalent to what the current primary seq no is, as all data
                 // we have assigned sequence numbers to should be in the commit
                 equalTo(primarySeqNo));
-            assertThat(recoveringEngine.getLocalCheckpoint(), equalTo(primarySeqNo));
+            assertThat(recoveringEngine.getProcessedLocalCheckpoint(), equalTo(primarySeqNo));
+            assertThat(recoveringEngine.getPersistedLocalCheckpoint(), equalTo(primarySeqNo));
             assertThat(recoveringEngine.getSeqNoStats(-1).getMaxSeqNo(), equalTo(primarySeqNo));
             assertThat(generateNewSeqNo(recoveringEngine), equalTo(primarySeqNo + 1));
         }
@@ -2693,7 +2792,9 @@ public void testCurrentTranslogIDisCommitted() throws IOException {
 
                 try (InternalEngine engine = createEngine(config)) {
                     engine.index(firstIndexRequest);
-                    globalCheckpoint.set(engine.getLocalCheckpoint());
+                    engine.syncTranslog(); // to advance persisted local checkpoint
+                    assertEquals(engine.getProcessedLocalCheckpoint(), engine.getPersistedLocalCheckpoint());
+                    globalCheckpoint.set(engine.getPersistedLocalCheckpoint());
                     expectThrows(IllegalStateException.class, () -> engine.recoverFromTranslog(translogHandler, Long.MAX_VALUE));
                     Map<String, String> userData = engine.getLastCommittedSegmentInfos().getUserData();
                     assertEquals("1", userData.get(Translog.TRANSLOG_GENERATION_KEY));
@@ -2855,7 +2956,9 @@ protected void commitIndexWriter(IndexWriter writer, Translog translog, String s
                 final ParsedDocument doc1 = testParsedDocument("1", null,
                     testDocumentWithTextField(), SOURCE, null);
                 engine.index(indexForDoc(doc1));
-                globalCheckpoint.set(engine.getLocalCheckpoint());
+                engine.syncTranslog(); // to advance local checkpoint
+                assertEquals(engine.getProcessedLocalCheckpoint(), engine.getPersistedLocalCheckpoint());
+                globalCheckpoint.set(engine.getPersistedLocalCheckpoint());
                 throwErrorOnCommit.set(true);
                 FlushFailedEngineException e = expectThrows(FlushFailedEngineException.class, engine::flush);
                 assertThat(e.getCause().getMessage(), equalTo("power's out"));
@@ -2915,7 +3018,7 @@ private Path[] filterExtraFSFiles(Path[] files) {
     }
 
     public void testTranslogReplay() throws IOException {
-        final LongSupplier inSyncGlobalCheckpointSupplier = () -> this.engine.getLocalCheckpoint();
+        final LongSupplier inSyncGlobalCheckpointSupplier = () -> this.engine.getProcessedLocalCheckpoint();
         final int numDocs = randomIntBetween(1, 10);
         for (int i = 0; i < numDocs; i++) {
             ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), new BytesArray("{}"), null);
@@ -3009,7 +3112,7 @@ public void testRecoverFromForeignTranslog() throws IOException {
         final String badUUID = Translog.createEmptyTranslog(badTranslogLog, SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get());
         Translog translog = new Translog(
             new TranslogConfig(shardId, badTranslogLog, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE),
-            badUUID, createTranslogDeletionPolicy(INDEX_SETTINGS), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get);
+            badUUID, createTranslogDeletionPolicy(INDEX_SETTINGS), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {});
         translog.add(new Translog.Index("test", "SomeBogusId", 0, primaryTerm.get(),
             "{}".getBytes(Charset.forName("UTF-8"))));
         assertEquals(generation.translogFileGeneration, translog.currentFileGeneration());
@@ -3742,7 +3845,7 @@ public void run() {
                     } catch (InterruptedException e) {
                         throw new AssertionError(e);
                     }
-                    assertEquals(0, engine.getVersionMapSize());
+                    assertThat(engine.getVersionMap().values(), empty());
                     int docOffset;
                     while ((docOffset = offset.incrementAndGet()) < docs.size()) {
                         try {
@@ -3934,20 +4037,23 @@ public void testLookupSeqNoByIdInLucene() throws Exception {
              InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), newMergePolicy(), null))) {
             CheckedRunnable<IOException> lookupAndCheck = () -> {
                 try (Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
-                    for (String id : latestOps.keySet()) {
-                        String msg = "latestOps=" + latestOps + " op=" + id;
-                        DocIdAndSeqNo docIdAndSeqNo = VersionsAndSeqNoResolver.loadDocIdAndSeqNo(searcher.reader(), newUid(id));
-                        assertThat(msg, docIdAndSeqNo.seqNo, equalTo(latestOps.get(id).seqNo()));
-                        assertThat(msg, docIdAndSeqNo.isLive,
-                            equalTo(latestOps.get(id).operationType() == Engine.Operation.TYPE.INDEX));
-                    }
-                    assertThat(VersionsAndSeqNoResolver.loadDocIdAndVersion(
-                        searcher.reader(), newUid("any-" + between(1, 10)), randomBoolean()), nullValue());
                     Map<String, Long> liveOps = latestOps.entrySet().stream()
                         .filter(e -> e.getValue().operationType() == Engine.Operation.TYPE.INDEX)
                         .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().seqNo()));
                     assertThat(getDocIds(engine, true).stream().collect(Collectors.toMap(e -> e.getId(), e -> e.getSeqNo())),
                         equalTo(liveOps));
+                    for (String id : latestOps.keySet()) {
+                        String msg = "latestOps=" + latestOps + " op=" + id;
+                        DocIdAndSeqNo docIdAndSeqNo = VersionsAndSeqNoResolver.loadDocIdAndSeqNo(searcher.reader(), newUid(id));
+                        if (liveOps.containsKey(id) == false) {
+                            assertNull(msg, docIdAndSeqNo);
+                        } else {
+                            assertNotNull(msg, docIdAndSeqNo);
+                            assertThat(msg, docIdAndSeqNo.seqNo, equalTo(latestOps.get(id).seqNo()));
+                        }
+                    }
+                    String notFoundId = randomValueOtherThanMany(liveOps::containsKey, () -> Long.toString(randomNonNegativeLong()));
+                    assertNull(VersionsAndSeqNoResolver.loadDocIdAndSeqNo(searcher.reader(), newUid(notFoundId)));
                 }
             };
             for (Engine.Operation op : operations) {
@@ -4048,9 +4154,10 @@ public void testSequenceNumberAdvancesToMaxSeqOnEngineOpenOnPrimary() throws Bro
                 }
             }
 
-            assertThat(initialEngine.getLocalCheckpoint(), equalTo(expectedLocalCheckpoint.get()));
+            assertThat(initialEngine.getProcessedLocalCheckpoint(), equalTo(expectedLocalCheckpoint.get()));
             assertThat(initialEngine.getSeqNoStats(-1).getMaxSeqNo(), equalTo((long) (docs - 1)));
             initialEngine.flush(true, true);
+            assertEquals(initialEngine.getProcessedLocalCheckpoint(), initialEngine.getPersistedLocalCheckpoint());
 
             latchReference.get().countDown();
             for (final Thread thread : threads) {
@@ -4059,10 +4166,11 @@ public void testSequenceNumberAdvancesToMaxSeqOnEngineOpenOnPrimary() throws Bro
         } finally {
             IOUtils.close(initialEngine);
         }
-        try (Engine recoveringEngine = new InternalEngine(initialEngine.config())) {
+        try (InternalEngine recoveringEngine = new InternalEngine(initialEngine.config())) {
             recoveringEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
             recoveringEngine.fillSeqNoGaps(2);
-            assertThat(recoveringEngine.getLocalCheckpoint(), greaterThanOrEqualTo((long) (docs - 1)));
+            assertEquals(recoveringEngine.getProcessedLocalCheckpoint(), recoveringEngine.getPersistedLocalCheckpoint());
+            assertThat(recoveringEngine.getProcessedLocalCheckpoint(), greaterThanOrEqualTo((long) (docs - 1)));
         }
     }
 
@@ -4143,7 +4251,7 @@ public void testOutOfOrderSequenceNumbersWithVersionConflict() throws IOExceptio
             expectedLocalCheckpoint = numberOfOperations - 1;
         }
 
-        assertThat(engine.getLocalCheckpoint(), equalTo(expectedLocalCheckpoint));
+        assertThat(engine.getProcessedLocalCheckpoint(), equalTo(expectedLocalCheckpoint));
         try (Engine.GetResult result = engine.get(new Engine.Get(true, false,
             "type", "2", uid), searcherFactory)) {
             assertThat(result.exists(), equalTo(exists));
@@ -4175,12 +4283,12 @@ protected long doGenerateSeqNoForOperation(Operation operation) {
             final int gapsFilled = noOpEngine.fillSeqNoGaps(primaryTerm.get());
             final String reason = "filling gaps";
             noOpEngine.noOp(new Engine.NoOp(maxSeqNo + 1, primaryTerm.get(), LOCAL_TRANSLOG_RECOVERY, System.nanoTime(), reason));
-            assertThat(noOpEngine.getLocalCheckpoint(), equalTo((long) (maxSeqNo + 1)));
+            assertThat(noOpEngine.getProcessedLocalCheckpoint(), equalTo((long) (maxSeqNo + 1)));
             assertThat(noOpEngine.getTranslog().stats().getUncommittedOperations(), equalTo(gapsFilled));
             noOpEngine.noOp(
                 new Engine.NoOp(maxSeqNo + 2, primaryTerm.get(),
                     randomFrom(PRIMARY, REPLICA, PEER_RECOVERY), System.nanoTime(), reason));
-            assertThat(noOpEngine.getLocalCheckpoint(), equalTo((long) (maxSeqNo + 2)));
+            assertThat(noOpEngine.getProcessedLocalCheckpoint(), equalTo((long) (maxSeqNo + 2)));
             assertThat(noOpEngine.getTranslog().stats().getUncommittedOperations(), equalTo(gapsFilled + 1));
             // skip to the op that we added to the translog
             Translog.Operation op;
@@ -4399,7 +4507,7 @@ public void testRestoreLocalHistoryFromTranslog() throws IOException {
                         engine.flush();
                     }
                 }
-                globalCheckpoint.set(randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, engine.getLocalCheckpoint()));
+                globalCheckpoint.set(randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, engine.getPersistedLocalCheckpoint()));
                 engine.syncTranslog();
                 prevSeqNoStats = engine.getSeqNoStats(globalCheckpoint.get());
                 prevDocs = getDocIds(engine, true);
@@ -4436,7 +4544,9 @@ public void testFillUpSequenceIdGapsOnRecovery() throws IOException {
                     replicaEngine.index(replicaIndexForDoc(doc, 1, indexResult.getSeqNo(), false));
                 }
             }
-            checkpointOnReplica = replicaEngine.getLocalCheckpoint();
+            engine.syncTranslog(); // to advance local checkpoint
+            replicaEngine.syncTranslog(); // to advance local checkpoint
+            checkpointOnReplica = replicaEngine.getProcessedLocalCheckpoint();
         } finally {
             IOUtils.close(replicaEngine);
         }
@@ -4444,17 +4554,17 @@ public void testFillUpSequenceIdGapsOnRecovery() throws IOException {
 
         boolean flushed = false;
         AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
-        Engine recoveringEngine = null;
+        InternalEngine recoveringEngine = null;
         try {
             assertEquals(docs - 1, engine.getSeqNoStats(-1).getMaxSeqNo());
-            assertEquals(docs - 1, engine.getLocalCheckpoint());
+            assertEquals(docs - 1, engine.getProcessedLocalCheckpoint());
             assertEquals(maxSeqIDOnReplica, replicaEngine.getSeqNoStats(-1).getMaxSeqNo());
-            assertEquals(checkpointOnReplica, replicaEngine.getLocalCheckpoint());
+            assertEquals(checkpointOnReplica, replicaEngine.getProcessedLocalCheckpoint());
             recoveringEngine = new InternalEngine(copy(replicaEngine.config(), globalCheckpoint::get));
             assertEquals(numDocsOnReplica, getTranslog(recoveringEngine).stats().getUncommittedOperations());
             recoveringEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
             assertEquals(maxSeqIDOnReplica, recoveringEngine.getSeqNoStats(-1).getMaxSeqNo());
-            assertEquals(checkpointOnReplica, recoveringEngine.getLocalCheckpoint());
+            assertEquals(checkpointOnReplica, recoveringEngine.getProcessedLocalCheckpoint());
             assertEquals((maxSeqIDOnReplica + 1) - numDocsOnReplica, recoveringEngine.fillSeqNoGaps(2));
 
             // now snapshot the tlog and ensure the primary term is updated
@@ -4470,7 +4580,7 @@ public void testFillUpSequenceIdGapsOnRecovery() throws IOException {
 
                 }
                 assertEquals(maxSeqIDOnReplica, recoveringEngine.getSeqNoStats(-1).getMaxSeqNo());
-                assertEquals(maxSeqIDOnReplica, recoveringEngine.getLocalCheckpoint());
+                assertEquals(maxSeqIDOnReplica, recoveringEngine.getProcessedLocalCheckpoint());
                 if ((flushed = randomBoolean())) {
                     globalCheckpoint.set(recoveringEngine.getSeqNoStats(-1).getMaxSeqNo());
                     getTranslog(recoveringEngine).sync();
@@ -4489,10 +4599,10 @@ public void testFillUpSequenceIdGapsOnRecovery() throws IOException {
             }
             recoveringEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
             assertEquals(maxSeqIDOnReplica, recoveringEngine.getSeqNoStats(-1).getMaxSeqNo());
-            assertEquals(maxSeqIDOnReplica, recoveringEngine.getLocalCheckpoint());
+            assertEquals(maxSeqIDOnReplica, recoveringEngine.getProcessedLocalCheckpoint());
             assertEquals(0, recoveringEngine.fillSeqNoGaps(3));
             assertEquals(maxSeqIDOnReplica, recoveringEngine.getSeqNoStats(-1).getMaxSeqNo());
-            assertEquals(maxSeqIDOnReplica, recoveringEngine.getLocalCheckpoint());
+            assertEquals(maxSeqIDOnReplica, recoveringEngine.getProcessedLocalCheckpoint());
         } finally {
             IOUtils.close(recoveringEngine);
         }
@@ -4676,7 +4786,7 @@ protected void commitIndexWriter(IndexWriter writer, Translog translog, String s
                     // Advance the global checkpoint during the flush to create a lag between a persisted global checkpoint in the translog
                     // (this value is visible to the deletion policy) and an in memory global checkpoint in the SequenceNumbersService.
                     if (rarely()) {
-                        globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), getLocalCheckpoint()));
+                        globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), getPersistedLocalCheckpoint()));
                     }
                     super.commitIndexWriter(writer, translog, syncId);
                 }
@@ -4688,7 +4798,7 @@ protected void commitIndexWriter(IndexWriter writer, Translog translog, String s
                 document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
                 engine.index(indexForDoc(testParsedDocument(Integer.toString(docId), null, document, B_1, null)));
                 if (frequently()) {
-                    globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getLocalCheckpoint()));
+                    globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
                     engine.syncTranslog();
                 }
                 if (frequently()) {
@@ -4828,11 +4938,11 @@ public void testCleanUpCommitsWhenGlobalCheckpointAdvanced() throws Exception {
             engine.flush(false, randomBoolean());
             List<IndexCommit> commits = DirectoryReader.listCommits(store.directory());
             // Global checkpoint advanced but not enough - all commits are kept.
-            globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getLocalCheckpoint() - 1));
+            globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint() - 1));
             engine.syncTranslog();
             assertThat(DirectoryReader.listCommits(store.directory()), equalTo(commits));
             // Global checkpoint advanced enough - only the last commit is kept.
-            globalCheckpoint.set(randomLongBetween(engine.getLocalCheckpoint(), Long.MAX_VALUE));
+            globalCheckpoint.set(randomLongBetween(engine.getPersistedLocalCheckpoint(), Long.MAX_VALUE));
             engine.syncTranslog();
             assertThat(DirectoryReader.listCommits(store.directory()), contains(commits.get(commits.size() - 1)));
             assertThat(engine.getTranslog().totalOperations(), equalTo(0));
@@ -4857,7 +4967,7 @@ public void testCleanupCommitsWhenReleaseSnapshot() throws Exception {
             for (int i = 0; i < numSnapshots; i++) {
                 snapshots.add(engine.acquireSafeIndexCommit()); // taking snapshots from the safe commit.
             }
-            globalCheckpoint.set(engine.getLocalCheckpoint());
+            globalCheckpoint.set(engine.getPersistedLocalCheckpoint());
             engine.syncTranslog();
             final List<IndexCommit> commits = DirectoryReader.listCommits(store.directory());
             for (int i = 0; i < numSnapshots - 1; i++) {
@@ -4938,7 +5048,7 @@ public void testStressShouldPeriodicallyFlush() throws Exception {
         engine.onSettingsChanged();
         final int numOps = scaledRandomIntBetween(100, 10_000);
         for (int i = 0; i < numOps; i++) {
-            final long localCheckPoint = engine.getLocalCheckpoint();
+            final long localCheckPoint = engine.getProcessedLocalCheckpoint();
             final long seqno = randomLongBetween(Math.max(0, localCheckPoint), localCheckPoint + 5);
             final ParsedDocument doc =
                 testParsedDocument(Long.toString(seqno), null, testDocumentWithTextField(), SOURCE, null);
@@ -5040,18 +5150,19 @@ public void testPruneOnlyDeletesAtMostLocalCheckpoint() throws Exception {
                     engine.delete(replicaDeleteForDoc(UUIDs.randomBase64UUID(), 1, seqno, threadPool.relativeTimeInMillis()));
                 }
             }
-            List<DeleteVersionValue> tombstones = new ArrayList<>(engine.getDeletedTombstones());
+
+            List<DeleteVersionValue> tombstones = new ArrayList<>(tombstonesInVersionMap(engine).values());
             engine.config().setEnableGcDeletes(true);
             // Prune tombstones whose seqno < gap_seqno and timestamp < clock-gcInterval.
             clock.set(randomLongBetween(gcInterval, deleteBatch + gcInterval));
             engine.refresh("test");
             tombstones.removeIf(v -> v.seqNo < gapSeqNo && v.time < clock.get() - gcInterval);
-            assertThat(engine.getDeletedTombstones(), containsInAnyOrder(tombstones.toArray()));
+            assertThat(tombstonesInVersionMap(engine).values(), containsInAnyOrder(tombstones.toArray()));
             // Prune tombstones whose seqno at most the local checkpoint (eg. seqno < gap_seqno).
             clock.set(randomLongBetween(deleteBatch + gcInterval * 4/3, 100)); // Need a margin for gcInterval/4.
             engine.refresh("test");
             tombstones.removeIf(v -> v.seqNo < gapSeqNo);
-            assertThat(engine.getDeletedTombstones(), containsInAnyOrder(tombstones.toArray()));
+            assertThat(tombstonesInVersionMap(engine).values(), containsInAnyOrder(tombstones.toArray()));
             // Fill the seqno gap - should prune all tombstones.
             clock.set(between(0, 100));
             if (randomBoolean()) {
@@ -5063,7 +5174,7 @@ public void testPruneOnlyDeletesAtMostLocalCheckpoint() throws Exception {
             }
             clock.set(randomLongBetween(100 + gcInterval * 4/3, Long.MAX_VALUE)); // Need a margin for gcInterval/4.
             engine.refresh("test");
-            assertThat(engine.getDeletedTombstones(), empty());
+            assertThat(tombstonesInVersionMap(engine).values(), empty());
         }
     }
 
@@ -5120,8 +5231,8 @@ public void testTrackMaxSeqNoOfNonAppendOnlyOperations() throws Exception {
             }
             appendOnlyIndexer.join(120_000);
             assertThat(engine.getMaxSeqNoOfNonAppendOnlyOperations(), equalTo(maxSeqNoOfNonAppendOnly));
-            globalCheckpoint.set(engine.getLocalCheckpoint());
             engine.syncTranslog();
+            globalCheckpoint.set(engine.getPersistedLocalCheckpoint());
             engine.flush();
         }
         try (InternalEngine engine = createEngine(store, translogPath, globalCheckpoint::get)) {
@@ -5305,7 +5416,10 @@ public void testKeepMinRetainedSeqNoByMergePolicy() throws IOException {
             }
             existingSeqNos.add(result.getSeqNo());
             if (randomBoolean()) {
-                globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getLocalCheckpointTracker().getCheckpoint()));
+                engine.syncTranslog(); // advance persisted local checkpoint
+                assertEquals(engine.getProcessedLocalCheckpoint(), engine.getPersistedLocalCheckpoint());
+                globalCheckpoint.set(
+                    randomLongBetween(globalCheckpoint.get(), engine.getLocalCheckpointTracker().getPersistedCheckpoint()));
             }
             if (randomBoolean()) {
                 retentionLeasesVersion.incrementAndGet();
@@ -5369,7 +5483,7 @@ public void testLastRefreshCheckpoint() throws Exception {
             latch.countDown();
             refreshThreads[i] = new Thread(() -> {
                 while (done.get() == false) {
-                    long checkPointBeforeRefresh = engine.getLocalCheckpoint();
+                    long checkPointBeforeRefresh = engine.getProcessedLocalCheckpoint();
                     engine.refresh("test", randomFrom(Engine.SearcherScope.values()), true);
                     assertThat(engine.lastRefreshedCheckpoint(), greaterThanOrEqualTo(checkPointBeforeRefresh));
                 }
@@ -5385,7 +5499,7 @@ public void testLastRefreshCheckpoint() throws Exception {
             thread.join();
         }
         engine.refresh("test");
-        assertThat(engine.lastRefreshedCheckpoint(), equalTo(engine.getLocalCheckpoint()));
+        assertThat(engine.lastRefreshedCheckpoint(), equalTo(engine.getProcessedLocalCheckpoint()));
     }
 
     public void testLuceneSnapshotRefreshesOnlyOnce() throws Exception {
@@ -5450,6 +5564,37 @@ public void testAcquireSearcherOnClosingEngine() throws Exception {
         expectThrows(AlreadyClosedException.class, () -> engine.acquireSearcher("test"));
     }
 
+    public void testNoOpOnClosingEngine() throws Exception {
+        engine.close();
+        Settings settings = Settings.builder()
+            .put(defaultSettings.getSettings())
+            .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).build();
+        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(
+            IndexMetaData.builder(defaultSettings.getIndexMetaData()).settings(settings).build());
+        assertTrue(indexSettings.isSoftDeleteEnabled());
+        try (Store store = createStore();
+             InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null))) {
+            engine.close();
+            expectThrows(AlreadyClosedException.class, () -> engine.noOp(
+                new Engine.NoOp(2, primaryTerm.get(), LOCAL_TRANSLOG_RECOVERY, System.nanoTime(), "reason")));
+        }
+    }
+
+    public void testSoftDeleteOnClosingEngine() throws Exception {
+        engine.close();
+        Settings settings = Settings.builder()
+            .put(defaultSettings.getSettings())
+            .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).build();
+        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(
+            IndexMetaData.builder(defaultSettings.getIndexMetaData()).settings(settings).build());
+        assertTrue(indexSettings.isSoftDeleteEnabled());
+        try (Store store = createStore();
+             InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null))) {
+            engine.close();
+            expectThrows(AlreadyClosedException.class, () -> engine.delete(replicaDeleteForDoc("test", 42, 7, System.nanoTime())));
+        }
+    }
+
     public void testTrackMaxSeqNoOfUpdatesOrDeletesOnPrimary() throws Exception {
         engine.close();
         Set<String> liveDocIds = new HashSet<>();
@@ -5477,9 +5622,10 @@ public void testTrackMaxSeqNoOfUpdatesOrDeletesOnPrimary() throws Exception {
         }
     }
 
-    public void testRebuildLocalCheckpointTracker() throws Exception {
+    public void testRebuildLocalCheckpointTrackerAndVersionMap() throws Exception {
         Settings.Builder settings = Settings.builder()
             .put(defaultSettings.getSettings())
+            .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), 10000)
             .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
         final IndexMetaData indexMetaData = IndexMetaData.builder(defaultSettings.getIndexMetaData()).settings(settings).build();
         final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetaData);
@@ -5496,33 +5642,55 @@ public void testRebuildLocalCheckpointTracker() throws Exception {
                 for (Engine.Operation op : operations) {
                     flushedOperations.add(op);
                     applyOperation(engine, op);
+                    if (randomBoolean()) {
+                        engine.syncTranslog();
+                        globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
+                    }
                     if (randomInt(100) < 10) {
                         engine.refresh("test");
                     }
                     if (randomInt(100) < 5) {
-                        engine.flush();
+                        engine.flush(true, true);
+                        flushedOperations.sort(Comparator.comparing(Engine.Operation::seqNo));
                         commits.add(new ArrayList<>(flushedOperations));
-                        globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getLocalCheckpoint()));
                     }
                 }
-                globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getLocalCheckpoint()));
-                engine.syncTranslog();
                 docs = getDocIds(engine, true);
             }
-            Set<Long> seqNosInSafeCommit = null;
+            List<Engine.Operation> operationsInSafeCommit = null;
             for (int i = commits.size() - 1; i >= 0; i--) {
                 if (commits.get(i).stream().allMatch(op -> op.seqNo() <= globalCheckpoint.get())) {
-                    seqNosInSafeCommit = commits.get(i).stream().map(Engine.Operation::seqNo).collect(Collectors.toSet());
+                    operationsInSafeCommit = commits.get(i);
                     break;
                 }
             }
-            assertThat(seqNosInSafeCommit, notNullValue());
+            assertThat(operationsInSafeCommit, notNullValue());
             try (InternalEngine engine = new InternalEngine(config)) { // do not recover from translog
+                final Map<BytesRef, Engine.Operation> deletesAfterCheckpoint = new HashMap<>();
+                for (Engine.Operation op : operationsInSafeCommit) {
+                    if (op instanceof Engine.NoOp == false && op.seqNo() > engine.getPersistedLocalCheckpoint()) {
+                        deletesAfterCheckpoint.put(new Term(IdFieldMapper.NAME, Uid.encodeId(op.id())).bytes(), op);
+                    }
+                }
+                deletesAfterCheckpoint.values().removeIf(o -> o instanceof Engine.Delete == false);
+                final Map<BytesRef, VersionValue> versionMap = engine.getVersionMap();
+                for (BytesRef uid : deletesAfterCheckpoint.keySet()) {
+                    final VersionValue versionValue = versionMap.get(uid);
+                    final Engine.Operation op = deletesAfterCheckpoint.get(uid);
+                    final String msg = versionValue + " vs " +
+                        "op[" + op.operationType() + "id=" + op.id() + " seqno=" + op.seqNo() + " term=" + op.primaryTerm() + "]";
+                    assertThat(versionValue, instanceOf(DeleteVersionValue.class));
+                    assertThat(msg, versionValue.seqNo, equalTo(op.seqNo()));
+                    assertThat(msg, versionValue.term, equalTo(op.primaryTerm()));
+                    assertThat(msg, versionValue.version, equalTo(op.version()));
+                }
+                assertThat(versionMap.keySet(), equalTo(deletesAfterCheckpoint.keySet()));
                 final LocalCheckpointTracker tracker = engine.getLocalCheckpointTracker();
+                final Set<Long> seqNosInSafeCommit = operationsInSafeCommit.stream().map(op -> op.seqNo()).collect(Collectors.toSet());
                 for (Engine.Operation op : operations) {
                     assertThat(
-                        "seq_no=" + op.seqNo() + " max_seq_no=" + tracker.getMaxSeqNo() + "checkpoint=" + tracker.getCheckpoint(),
-                        tracker.contains(op.seqNo()), equalTo(seqNosInSafeCommit.contains(op.seqNo())));
+                        "seq_no=" + op.seqNo() + " max_seq_no=" + tracker.getMaxSeqNo() + "checkpoint=" + tracker.getProcessedCheckpoint(),
+                        tracker.hasProcessed(op.seqNo()), equalTo(seqNosInSafeCommit.contains(op.seqNo())));
                 }
                 engine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
                 assertThat(getDocIds(engine, true), equalTo(docs));
@@ -5542,8 +5710,8 @@ public void testOpenSoftDeletesIndexWithSoftDeletesDisabled() throws Exception {
                 config(softDeletesEnabled, store, translogPath, newMergePolicy(), null, null, globalCheckpoint::get))) {
                 List<Engine.Operation> ops = generateHistoryOnReplica(between(1, 100), randomBoolean(), randomBoolean(), randomBoolean());
                 applyOperations(engine, ops);
-                globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getLocalCheckpoint()));
-                engine.syncTranslog();
+                engine.syncTranslog(); // to advance persisted checkpoint
+                globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
                 engine.flush();
                 docs = getDocIds(engine, true);
             }
@@ -5678,4 +5846,155 @@ public void testRefreshAndFailEngineConcurrently() throws Exception {
         }
         assertThat(engine.config().getCircuitBreakerService().getBreaker(CircuitBreaker.ACCOUNTING).getUsed(), equalTo(0L));
     }
+
+    public void testGetReaderAttributes() throws IOException {
+        try(BaseDirectoryWrapper dir = newFSDirectory(createTempDir())) {
+            Directory unwrap = FilterDirectory.unwrap(dir);
+            boolean isMMap = unwrap instanceof MMapDirectory;
+            Map<String, String> readerAttributes = InternalEngine.getReaderAttributes(dir);
+            assertEquals(2, readerAttributes.size());
+            assertEquals("ON_HEAP", readerAttributes.get("blocktree.terms.fst._id"));
+            if (isMMap) {
+                assertEquals("OFF_HEAP", readerAttributes.get("blocktree.terms.fst"));
+            } else {
+                assertEquals("ON_HEAP", readerAttributes.get("blocktree.terms.fst"));
+            }
+        }
+
+        try(MMapDirectory dir = new MMapDirectory(createTempDir())) {
+            Map<String, String> readerAttributes =
+                InternalEngine.getReaderAttributes(randomBoolean() ? dir :
+                    new MockDirectoryWrapper(random(), dir));
+            assertEquals(2, readerAttributes.size());
+            assertEquals("ON_HEAP", readerAttributes.get("blocktree.terms.fst._id"));
+            assertEquals("OFF_HEAP", readerAttributes.get("blocktree.terms.fst"));
+        }
+
+        Settings.Builder settingsBuilder = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
+        Settings settings = settingsBuilder.build();
+        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
+        FsDirectoryFactory service = new FsDirectoryFactory();
+        Path tempDir = createTempDir().resolve(indexSettings.getUUID()).resolve("0");
+        ShardPath path = new ShardPath(false, tempDir, tempDir, new ShardId(indexSettings.getIndex(), 0));
+        try (Directory directory = service.newDirectory(indexSettings, path)) {
+
+            Map<String, String> readerAttributes =
+                InternalEngine.getReaderAttributes(randomBoolean() ? directory :
+                    new MockDirectoryWrapper(random(), directory));
+            assertEquals(2, readerAttributes.size());
+
+            switch (IndexModule.defaultStoreType(true)) {
+                case HYBRIDFS:
+                case MMAPFS:
+                    assertEquals("ON_HEAP", readerAttributes.get("blocktree.terms.fst._id"));
+                    assertEquals("OFF_HEAP", readerAttributes.get("blocktree.terms.fst"));
+                    break;
+                case NIOFS:
+                case SIMPLEFS:
+                case FS:
+                    assertEquals("ON_HEAP", readerAttributes.get("blocktree.terms.fst._id"));
+                    assertEquals("ON_HEAP", readerAttributes.get("blocktree.terms.fst"));
+                    break;
+                default:
+                    fail("unknownw type");
+            }
+        }
+    }
+
+    public void testPruneAwayDeletedButRetainedIds() throws Exception {
+        IOUtils.close(engine, store);
+        Settings settings = Settings.builder()
+            .put(defaultSettings.getSettings())
+            .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true).build();
+        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(
+            IndexMetaData.builder(defaultSettings.getIndexMetaData()).settings(settings).build());
+        store = createStore(indexSettings, newDirectory());
+        LogDocMergePolicy policy = new LogDocMergePolicy();
+        policy.setMinMergeDocs(10000);
+        try (InternalEngine engine = createEngine(indexSettings, store, createTempDir(), policy)) {
+            int numDocs = between(1, 20);
+            for (int i = 0; i < numDocs; i++) {
+                index(engine, i);
+            }
+            engine.forceMerge(true);
+            engine.delete(new Engine.Delete("_doc", "0", newUid("0"), primaryTerm.get()));
+            engine.refresh("test");
+            // now we have 2 segments since we now added a tombstone plus the old segment with the delete
+            try (Searcher searcher = engine.acquireSearcher("test")) {
+                IndexReader reader = searcher.reader();
+                assertEquals(2, reader.leaves().size());
+                LeafReaderContext leafReaderContext = reader.leaves().get(0);
+                LeafReader leafReader = leafReaderContext.reader();
+                assertEquals("the delete and the tombstone", 1, leafReader.numDeletedDocs());
+                assertEquals(numDocs, leafReader.maxDoc());
+                Terms id = leafReader.terms("_id");
+                assertNotNull(id);
+                assertEquals("deleted IDs are NOT YET pruned away", reader.numDocs() + 1, id.size());
+                TermsEnum iterator = id.iterator();
+                assertTrue(iterator.seekExact(Uid.encodeId("0")));
+            }
+
+            // lets force merge the tombstone and the original segment and make sure the doc is still there but the ID term is gone
+            engine.forceMerge(true);
+            engine.refresh("test");
+            try (Searcher searcher = engine.acquireSearcher("test")) {
+                IndexReader reader = searcher.reader();
+                assertEquals(1, reader.leaves().size());
+                LeafReaderContext leafReaderContext = reader.leaves().get(0);
+                LeafReader leafReader = leafReaderContext.reader();
+                assertEquals("the delete and the tombstone", 2, leafReader.numDeletedDocs());
+                assertEquals(numDocs + 1, leafReader.maxDoc());
+                Terms id = leafReader.terms("_id");
+                if (numDocs == 1) {
+                    assertNull(id); // everything is pruned away
+                    assertEquals(0, leafReader.numDocs());
+                } else {
+                    assertNotNull(id);
+                    assertEquals("deleted IDs are pruned away", reader.numDocs(), id.size());
+                    TermsEnum iterator = id.iterator();
+                    assertFalse(iterator.seekExact(Uid.encodeId("0")));
+                }
+            }
+        }
+    }
+
+    public void testRecoverFromLocalTranslog() throws Exception {
+        final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
+        Path translogPath = createTempDir();
+        List<Engine.Operation> operations = generateHistoryOnReplica(between(1, 500), randomBoolean(), randomBoolean(), randomBoolean());
+        try (Store store = createStore()) {
+            EngineConfig config = config(defaultSettings, store, translogPath, newMergePolicy(), null, null, globalCheckpoint::get);
+            final List<DocIdSeqNoAndSource> docs;
+            try (InternalEngine engine = createEngine(config)) {
+                for (Engine.Operation op : operations) {
+                    applyOperation(engine, op);
+                    if (randomBoolean()) {
+                        engine.syncTranslog();
+                        globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
+                    }
+                    if (randomInt(100) < 10) {
+                        engine.refresh("test");
+                    }
+                    if (randomInt(100) < 5) {
+                        engine.flush();
+                    }
+                    if (randomInt(100) < 5) {
+                        engine.forceMerge(randomBoolean(), 1, false, false, false);
+                    }
+                }
+                docs = getDocIds(engine, true);
+            }
+            try (InternalEngine engine = new InternalEngine(config)) {
+                engine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
+                assertThat(getDocIds(engine, randomBoolean()), equalTo(docs));
+            }
+        }
+    }
+
+    private Map<BytesRef, DeleteVersionValue> tombstonesInVersionMap(InternalEngine engine) {
+        return engine.getVersionMap().entrySet().stream()
+            .filter(e -> e.getValue() instanceof DeleteVersionValue)
+            .collect(Collectors.toMap(e -> e.getKey(), e -> (DeleteVersionValue) e.getValue()));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/engine/LuceneChangesSnapshotTests.java b/server/src/test/java/org/elasticsearch/index/engine/LuceneChangesSnapshotTests.java
index d1840c4d97cff..f6327e8132cea 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/LuceneChangesSnapshotTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/LuceneChangesSnapshotTests.java
@@ -161,7 +161,7 @@ public void testSkipStaleOrNonRootOfNestedDocuments() throws Exception {
         int totalOps = 0;
         for (Engine.Operation op : operations) {
             // Engine skips deletes or indexes below the local checkpoint
-            if (engine.getLocalCheckpoint() < op.seqNo() || op instanceof Engine.NoOp) {
+            if (engine.getProcessedLocalCheckpoint() < op.seqNo() || op instanceof Engine.NoOp) {
                 seqNoToTerm.put(op.seqNo(), op.primaryTerm());
                 if (op instanceof Engine.Index) {
                     totalOps += ((Engine.Index) op).docs().size();
@@ -228,7 +228,7 @@ public void testUpdateAndReadChangesConcurrently() throws Exception {
         readyLatch.countDown();
         readyLatch.await();
         concurrentlyApplyOps(operations, engine);
-        assertThat(engine.getLocalCheckpointTracker().getCheckpoint(), equalTo(operations.size() - 1L));
+        assertThat(engine.getLocalCheckpointTracker().getProcessedCheckpoint(), equalTo(operations.size() - 1L));
         isDone.set(true);
         for (Follower follower : followers) {
             follower.join();
@@ -237,13 +237,13 @@ public void testUpdateAndReadChangesConcurrently() throws Exception {
     }
 
     class Follower extends Thread {
-        private final Engine leader;
+        private final InternalEngine leader;
         private final InternalEngine engine;
         private final TranslogHandler translogHandler;
         private final AtomicBoolean isDone;
         private final CountDownLatch readLatch;
 
-        Follower(Engine leader, AtomicBoolean isDone, CountDownLatch readLatch) throws IOException {
+        Follower(InternalEngine leader, AtomicBoolean isDone, CountDownLatch readLatch) throws IOException {
             this.leader = leader;
             this.isDone = isDone;
             this.readLatch = readLatch;
@@ -252,9 +252,9 @@ class Follower extends Thread {
             this.engine = createEngine(createStore(), createTempDir());
         }
 
-        void pullOperations(Engine follower) throws IOException {
-            long leaderCheckpoint = leader.getLocalCheckpoint();
-            long followerCheckpoint = follower.getLocalCheckpoint();
+        void pullOperations(InternalEngine follower) throws IOException {
+            long leaderCheckpoint = leader.getLocalCheckpointTracker().getProcessedCheckpoint();
+            long followerCheckpoint = follower.getLocalCheckpointTracker().getProcessedCheckpoint();
             if (followerCheckpoint < leaderCheckpoint) {
                 long fromSeqNo = followerCheckpoint + 1;
                 long batchSize = randomLongBetween(0, 100);
@@ -271,7 +271,8 @@ public void run() {
                 readLatch.countDown();
                 readLatch.await();
                 while (isDone.get() == false ||
-                    engine.getLocalCheckpointTracker().getCheckpoint() < leader.getLocalCheckpoint()) {
+                    engine.getLocalCheckpointTracker().getProcessedCheckpoint() <
+                        leader.getLocalCheckpointTracker().getProcessedCheckpoint()) {
                     pullOperations(engine);
                 }
                 assertConsistentHistoryBetweenTranslogAndLuceneIndex(engine, mapperService);
diff --git a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java
index f03500e6e1250..bd934f683fb94 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.store.Store;
+import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogDeletionPolicy;
 import org.elasticsearch.test.IndexSettingsModule;
 
@@ -42,6 +43,7 @@
 import java.io.UncheckedIOException;
 import java.nio.file.Path;
 import java.util.Collections;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -83,14 +85,14 @@ public void testNoopAfterRegularEngine() throws IOException {
             tracker.updateLocalCheckpoint(allocationId.getId(), i);
         }
 
-        flushAndTrimTranslog(engine);
+        engine.flush(true, true);
 
-        long localCheckpoint = engine.getLocalCheckpoint();
+        long localCheckpoint = engine.getPersistedLocalCheckpoint();
         long maxSeqNo = engine.getSeqNoStats(100L).getMaxSeqNo();
         engine.close();
 
         final NoOpEngine noOpEngine = new NoOpEngine(noOpConfig(INDEX_SETTINGS, store, primaryTranslogDir, tracker));
-        assertThat(noOpEngine.getLocalCheckpoint(), equalTo(localCheckpoint));
+        assertThat(noOpEngine.getPersistedLocalCheckpoint(), equalTo(localCheckpoint));
         assertThat(noOpEngine.getSeqNoStats(100L).getMaxSeqNo(), equalTo(maxSeqNo));
         try (Engine.IndexCommitRef ref = noOpEngine.acquireLastIndexCommit(false)) {
             try (IndexReader reader = DirectoryReader.open(ref.getIndexCommit())) {
@@ -114,7 +116,8 @@ public void testNoOpEngineStats() throws Exception {
                     if (rarely()) {
                         engine.flush();
                     }
-                    globalCheckpoint.set(engine.getLocalCheckpoint());
+                    engine.syncTranslog(); // advance persisted local checkpoint
+                    globalCheckpoint.set(engine.getPersistedLocalCheckpoint());
                 }
 
                 for (int i = 0; i < numDocs; i++) {
@@ -122,11 +125,12 @@ public void testNoOpEngineStats() throws Exception {
                         String delId = Integer.toString(i);
                         Engine.DeleteResult result = engine.delete(new Engine.Delete("test", delId, newUid(delId), primaryTerm.get()));
                         assertTrue(result.isFound());
-                        globalCheckpoint.set(engine.getLocalCheckpoint());
+                        engine.syncTranslog(); // advance persisted local checkpoint
+                        globalCheckpoint.set(engine.getPersistedLocalCheckpoint());
                         deletions += 1;
                     }
                 }
-                engine.getLocalCheckpointTracker().waitForOpsToComplete(numDocs + deletions - 1);
+                engine.getLocalCheckpointTracker().waitForProcessedOpsToComplete(numDocs + deletions - 1);
                 flushAndTrimTranslog(engine);
             }
 
@@ -144,7 +148,7 @@ public void testNoOpEngineStats() throws Exception {
                 assertEquals(expectedDocStats.getTotalSizeInBytes(), noOpEngine.docStats().getTotalSizeInBytes());
                 assertEquals(expectedDocStats.getAverageSizeInBytes(), noOpEngine.docStats().getAverageSizeInBytes());
                 assertEquals(expectedSegmentStats.getCount(), noOpEngine.segmentsStats(includeFileSize, true).getCount());
-                assertEquals(expectedSegmentStats.getMemoryInBytes(), noOpEngine.segmentsStats(includeFileSize, true).getMemoryInBytes());
+                // don't compare memory in bytes since we load the index with term-dict off-heap
                 assertEquals(expectedSegmentStats.getFileSizes().size(),
                     noOpEngine.segmentsStats(includeFileSize, true).getFileSizes().size());
 
@@ -157,6 +161,52 @@ public void testNoOpEngineStats() throws Exception {
         }
     }
 
+    public void testTrimUnreferencedTranslogFiles() throws Exception {
+        final ReplicationTracker tracker = (ReplicationTracker) engine.config().getGlobalCheckpointSupplier();
+        ShardRouting routing = TestShardRouting.newShardRouting("test", shardId.id(), "node",
+            null, true, ShardRoutingState.STARTED, allocationId);
+        IndexShardRoutingTable table = new IndexShardRoutingTable.Builder(shardId).addShard(routing).build();
+        tracker.updateFromMaster(1L, Collections.singleton(allocationId.getId()), table);
+        tracker.activatePrimaryMode(SequenceNumbers.NO_OPS_PERFORMED);
+
+        final int numDocs = scaledRandomIntBetween(10, 3000);
+        for (int i = 0; i < numDocs; i++) {
+            engine.index(indexForDoc(createParsedDoc(Integer.toString(i), null)));
+            if (rarely()) {
+                engine.flush();
+            }
+            tracker.updateLocalCheckpoint(allocationId.getId(), i);
+        }
+        engine.flush(true, true);
+
+        final String translogUuid = engine.getTranslog().getTranslogUUID();
+        final long minFileGeneration = engine.getTranslog().getMinFileGeneration();
+        final long currentFileGeneration = engine.getTranslog().currentFileGeneration();
+        engine.close();
+
+        final NoOpEngine noOpEngine = new NoOpEngine(noOpConfig(INDEX_SETTINGS, store, primaryTranslogDir, tracker));
+        final Path translogPath = noOpEngine.config().getTranslogConfig().getTranslogPath();
+
+        final long lastCommitedTranslogGeneration;
+        try (Engine.IndexCommitRef indexCommitRef = noOpEngine.acquireLastIndexCommit(false)) {
+            Map<String, String> lastCommittedUserData = indexCommitRef.getIndexCommit().getUserData();
+            lastCommitedTranslogGeneration = Long.parseLong(lastCommittedUserData.get(Translog.TRANSLOG_GENERATION_KEY));
+            assertThat(lastCommitedTranslogGeneration, equalTo(currentFileGeneration));
+        }
+
+        assertThat(Translog.readMinTranslogGeneration(translogPath, translogUuid), equalTo(minFileGeneration));
+        assertThat(noOpEngine.getTranslogStats().estimatedNumberOfOperations(), equalTo(numDocs));
+        assertThat(noOpEngine.getTranslogStats().getUncommittedOperations(), equalTo(0));
+
+        noOpEngine.trimUnreferencedTranslogFiles();
+
+        assertThat(Translog.readMinTranslogGeneration(translogPath, translogUuid), equalTo(lastCommitedTranslogGeneration));
+        assertThat(noOpEngine.getTranslogStats().estimatedNumberOfOperations(), equalTo(0));
+        assertThat(noOpEngine.getTranslogStats().getUncommittedOperations(), equalTo(0));
+
+        noOpEngine.close();
+    }
+
     private void flushAndTrimTranslog(final InternalEngine engine) {
         engine.flush(true, true);
         final TranslogDeletionPolicy deletionPolicy = engine.getTranslog().getDeletionPolicy();
diff --git a/server/src/test/java/org/elasticsearch/index/engine/PrunePostingsMergePolicyTests.java b/server/src/test/java/org/elasticsearch/index/engine/PrunePostingsMergePolicyTests.java
new file mode 100644
index 0000000000000..9fb9445181327
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/index/engine/PrunePostingsMergePolicyTests.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.engine;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SoftDeletesRetentionMergePolicy;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class PrunePostingsMergePolicyTests extends ESTestCase {
+
+    public void testPrune() throws IOException {
+        try (Directory dir = newDirectory()) {
+            IndexWriterConfig iwc = newIndexWriterConfig();
+            iwc.setSoftDeletesField("_soft_deletes");
+            MergePolicy mp = new SoftDeletesRetentionMergePolicy("_soft_deletes", MatchAllDocsQuery::new,
+                new PrunePostingsMergePolicy(newLogMergePolicy(), "id"));
+            iwc.setMergePolicy(mp);
+            boolean sorted = randomBoolean();
+            if (sorted) {
+                iwc.setIndexSort(new Sort(new SortField("sort", SortField.Type.INT)));
+            }
+            int numUniqueDocs = randomIntBetween(1, 100);
+            int numDocs = randomIntBetween(numUniqueDocs, numUniqueDocs * 5);
+
+            try (IndexWriter writer = new IndexWriter(dir, iwc)) {
+                for (int i = 0; i < numDocs ; i++) {
+                    if (rarely()) {
+                        writer.flush();
+                    }
+                    if (rarely()) {
+                        writer.forceMerge(1, false);
+                    }
+                    int id = i % numUniqueDocs;
+                    Document doc = new Document();
+                    doc.add(new StringField("id", "" + id, Field.Store.NO));
+                    doc.add(newTextField("text", "the quick brown fox", Field.Store.YES));
+                    doc.add(new NumericDocValuesField("sort", i));
+                    writer.softUpdateDocument(new Term("id", "" + id), doc, new NumericDocValuesField("_soft_deletes", 1));
+                    if (i == 0) {
+                        // make sure we have at least 2 segments to ensure we do an actual merge to kick out all postings for
+                        // soft deletes
+                        writer.flush();
+                    }
+                }
+
+                writer.forceMerge(1);
+                try (DirectoryReader reader = DirectoryReader.open(writer)) {
+                    LeafReader leafReader = reader.leaves().get(0).reader();
+                    assertEquals(numDocs, leafReader.maxDoc());
+                    Terms id = leafReader.terms("id");
+                    TermsEnum iterator = id.iterator();
+                    for (int i = 0; i < numUniqueDocs; i++) {
+                        assertTrue(iterator.seekExact(new BytesRef("" + i)));
+                        assertEquals(1, iterator.docFreq());
+                    }
+                    iterator = leafReader.terms("text").iterator();
+                    assertTrue(iterator.seekExact(new BytesRef("quick")));
+                    assertEquals(leafReader.maxDoc(), iterator.docFreq());
+                    int numValues = 0;
+                    NumericDocValues sort = leafReader.getNumericDocValues("sort");
+                    while (sort.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+                        if (sorted) {
+                            assertEquals(sort.docID(), sort.longValue());
+                        } else {
+                            assertTrue(sort.longValue() >= 0);
+                            assertTrue(sort.longValue() < numDocs);
+                        }
+                        numValues++;
+                    }
+                    assertEquals(numValues, numDocs);
+                }
+                {
+                    // prune away a single ID
+                    Document doc = new Document();
+                    doc.add(new StringField("id", "test", Field.Store.NO));
+                    writer.deleteDocuments(new Term("id", "test"));
+                    writer.flush();
+                    writer.forceMerge(1);
+                    writer.updateNumericDocValue(new Term("id", "test"), "_soft_deletes", 1);// delete it
+                    writer.flush();
+                    writer.forceMerge(1);
+
+                    try (DirectoryReader reader = DirectoryReader.open(writer)) {
+                        LeafReader leafReader = reader.leaves().get(0).reader();
+                        assertEquals(numDocs, leafReader.maxDoc());
+                        Terms id = leafReader.terms("id");
+                        TermsEnum iterator = id.iterator();
+                        assertEquals(numUniqueDocs, id.size());
+                        for (int i = 0; i < numUniqueDocs; i++) {
+                            assertTrue(iterator.seekExact(new BytesRef("" + i)));
+                            assertEquals(1, iterator.docFreq());
+                        }
+                        assertFalse(iterator.seekExact(new BytesRef("test")));
+                        iterator = leafReader.terms("text").iterator();
+                        assertTrue(iterator.seekExact(new BytesRef("quick")));
+                        assertEquals(leafReader.maxDoc(), iterator.docFreq());
+                    }
+                }
+
+                { // drop all ids
+                    // first add a doc such that we can force merge
+                    Document doc = new Document();
+                    doc.add(new StringField("id", "" + 0, Field.Store.NO));
+                    doc.add(newTextField("text", "the quick brown fox", Field.Store.YES));
+                    doc.add(new NumericDocValuesField("sort", 0));
+                    writer.softUpdateDocument(new Term("id", "" + 0), doc, new NumericDocValuesField("_soft_deletes", 1));
+                    for (int i = 0; i < numUniqueDocs; i++) {
+                        writer.updateNumericDocValue(new Term("id", "" + i), "_soft_deletes", 1);
+                    }
+                    writer.flush();
+                    writer.forceMerge(1);
+
+
+                    try (DirectoryReader reader = DirectoryReader.open(writer)) {
+                        LeafReader leafReader = reader.leaves().get(0).reader();
+                        assertEquals(numDocs+1, leafReader.maxDoc());
+                        assertEquals(0, leafReader.numDocs());
+                        assertNull(leafReader.terms("id"));
+                        TermsEnum iterator = leafReader.terms("text").iterator();
+                        assertTrue(iterator.seekExact(new BytesRef("quick")));
+                        assertEquals(leafReader.maxDoc(), iterator.docFreq());
+                    }
+                }
+            }
+
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java
index e0ad514e6dbb9..0a94ece4f8f7e 100644
--- a/server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java
+++ b/server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.store.Store;
+import org.elasticsearch.index.translog.TranslogStats;
 
 import java.io.IOException;
 import java.util.List;
@@ -37,6 +38,7 @@
 
 import static org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader.getElasticsearchDirectoryReader;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
 
 public class ReadOnlyEngineTests extends EngineTestCase {
@@ -62,15 +64,16 @@ public void testReadOnlyEngine() throws Exception {
                     if (rarely()) {
                         engine.flush();
                     }
-                    globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getLocalCheckpoint()));
+                    globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
                 }
                 engine.syncTranslog();
+                globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
                 engine.flush();
                 readOnlyEngine = new ReadOnlyEngine(engine.engineConfig, engine.getSeqNoStats(globalCheckpoint.get()),
                     engine.getTranslogStats(), false, Function.identity());
                 lastSeqNoStats = engine.getSeqNoStats(globalCheckpoint.get());
                 lastDocIds = getDocIds(engine, true);
-                assertThat(readOnlyEngine.getLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
+                assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
                 assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo()));
                 assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds));
                 for (int i = 0; i < numDocs; i++) {
@@ -94,7 +97,7 @@ public void testReadOnlyEngine() throws Exception {
 
                 IOUtils.close(external, internal);
                 // the locked down engine should still point to the previous commit
-                assertThat(readOnlyEngine.getLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
+                assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
                 assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo()));
                 assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds));
                 try (Engine.GetResult getResult = readOnlyEngine.get(get, readOnlyEngine::acquireSearcher)) {
@@ -105,7 +108,7 @@ public void testReadOnlyEngine() throws Exception {
             try (InternalEngine recoveringEngine = new InternalEngine(config)) {
                 recoveringEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
                 // the locked down engine should still point to the previous commit
-                assertThat(readOnlyEngine.getLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
+                assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
                 assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo()));
                 assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds));
             }
@@ -129,9 +132,10 @@ public void testFlushes() throws IOException {
                     if (rarely()) {
                         engine.flush();
                     }
-                    globalCheckpoint.set(engine.getLocalCheckpoint());
+                    engine.syncTranslog(); // advance persisted local checkpoint
+                    globalCheckpoint.set(engine.getPersistedLocalCheckpoint());
                 }
-                globalCheckpoint.set(engine.getLocalCheckpoint());
+                globalCheckpoint.set(engine.getPersistedLocalCheckpoint());
                 engine.syncTranslog();
                 engine.flushAndClose();
                 readOnlyEngine = new ReadOnlyEngine(engine.engineConfig, null , null, true, Function.identity());
@@ -155,10 +159,10 @@ public void testEnsureMaxSeqNoIsEqualToGlobalCheckpoint() throws IOException {
                     ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), new BytesArray("{}"), null);
                     engine.index(new Engine.Index(newUid(doc), doc, i, primaryTerm.get(), 1, null, Engine.Operation.Origin.REPLICA,
                         System.nanoTime(), -1, false, SequenceNumbers.UNASSIGNED_SEQ_NO, 0));
-                    maxSeqNo = engine.getLocalCheckpoint();
+                    maxSeqNo = engine.getProcessedLocalCheckpoint();
                 }
-                globalCheckpoint.set(engine.getLocalCheckpoint() - 1);
                 engine.syncTranslog();
+                globalCheckpoint.set(engine.getPersistedLocalCheckpoint() - 1);
                 engine.flushAndClose();
 
                 IllegalStateException exception = expectThrows(IllegalStateException.class,
@@ -181,7 +185,7 @@ public void testReadOnly() throws IOException {
         try (Store store = createStore()) {
             EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
             store.createEmpty(Version.CURRENT.luceneVersion);
-            try (ReadOnlyEngine readOnlyEngine = new ReadOnlyEngine(config, null , null, true, Function.identity())) {
+            try (ReadOnlyEngine readOnlyEngine = new ReadOnlyEngine(config, null , new TranslogStats(), true, Function.identity())) {
                 Class<? extends Throwable> expectedException = LuceneTestCase.TEST_ASSERTS_ENABLED ? AssertionError.class :
                     UnsupportedOperationException.class;
                 expectThrows(expectedException, () -> readOnlyEngine.index(null));
@@ -202,7 +206,7 @@ public void testVerifyShardBeforeIndexClosingIsNoOp() throws IOException {
         try (Store store = createStore()) {
             EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
             store.createEmpty(Version.CURRENT.luceneVersion);
-            try (ReadOnlyEngine readOnlyEngine = new ReadOnlyEngine(config, null , null, true, Function.identity())) {
+            try (ReadOnlyEngine readOnlyEngine = new ReadOnlyEngine(config, null , new TranslogStats(), true, Function.identity())) {
                 globalCheckpoint.set(randomNonNegativeLong());
                 try {
                     readOnlyEngine.verifyEngineBeforeIndexClosing();
@@ -240,4 +244,46 @@ public void testRecoverFromTranslogAppliesNoOperations() throws IOException {
             }
         }
     }
+
+    public void testTranslogStats() throws IOException {
+        IOUtils.close(engine, store);
+        try (Store store = createStore()) {
+            final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
+            EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
+
+            final int numDocs = frequently() ? scaledRandomIntBetween(10, 200) : 0;
+            int uncommittedDocs = 0;
+
+            try (InternalEngine engine = createEngine(config)) {
+                for (int i = 0; i < numDocs; i++) {
+                    ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), new BytesArray("{}"), null);
+                    engine.index(new Engine.Index(newUid(doc), doc, i, primaryTerm.get(), 1, null, Engine.Operation.Origin.REPLICA,
+                        System.nanoTime(), -1, false, SequenceNumbers.UNASSIGNED_SEQ_NO, 0));
+                    if (rarely()) {
+                        engine.flush();
+                        uncommittedDocs = 0;
+                    } else {
+                        uncommittedDocs += 1;
+                    }
+                    globalCheckpoint.set(i);
+                }
+
+                assertThat(engine.getTranslogStats().estimatedNumberOfOperations(), equalTo(numDocs));
+                assertThat(engine.getTranslogStats().getUncommittedOperations(), equalTo(uncommittedDocs));
+                assertThat(engine.getTranslogStats().getTranslogSizeInBytes(), greaterThan(0L));
+                assertThat(engine.getTranslogStats().getUncommittedSizeInBytes(), greaterThan(0L));
+                assertThat(engine.getTranslogStats().getEarliestLastModifiedAge(), greaterThan(0L));
+
+                engine.flush(true, true);
+            }
+
+            try (ReadOnlyEngine readOnlyEngine = new ReadOnlyEngine(config, null, null, true, Function.identity())) {
+                assertThat(readOnlyEngine.getTranslogStats().estimatedNumberOfOperations(), equalTo(numDocs));
+                assertThat(readOnlyEngine.getTranslogStats().getUncommittedOperations(), equalTo(0));
+                assertThat(readOnlyEngine.getTranslogStats().getTranslogSizeInBytes(), greaterThan(0L));
+                assertThat(readOnlyEngine.getTranslogStats().getUncommittedSizeInBytes(), greaterThan(0L));
+                assertThat(readOnlyEngine.getTranslogStats().getEarliestLastModifiedAge(), greaterThan(0L));
+            }
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
index 6d9ec95af16cd..cb3af392d0bdc 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
@@ -163,7 +163,7 @@ public void tearDown() throws Exception {
 
     protected Nested createNested(IndexSearcher searcher, Query parentFilter, Query childFilter) throws IOException {
         BitsetFilterCache s = indexService.cache().bitsetFilterCache();
-        return new Nested(s.getBitSetProducer(parentFilter), childFilter, null);
+        return new Nested(s.getBitSetProducer(parentFilter), childFilter, null, IndexSearcher::new);
     }
 
     public void testEmpty() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java
index 21d84203f6df3..1539a800f2ae3 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java
@@ -465,7 +465,7 @@ public void testGlobalOrdinals() throws Exception {
         assertThat(topLevelReader.leaves().size(), equalTo(3));
 
         // First segment
-        assertThat(globalOrdinals, instanceOf(GlobalOrdinalsIndexFieldData.class));
+        assertThat(globalOrdinals, instanceOf(GlobalOrdinalsIndexFieldData.Consumer.class));
         LeafReaderContext leaf = topLevelReader.leaves().get(0);
         AtomicOrdinalsFieldData afd = globalOrdinals.load(leaf);
         SortedSetDocValues values = afd.getOrdinalsValues();
@@ -590,7 +590,7 @@ public void testGlobalOrdinalsGetRemovedOnceIndexReaderCloses() throws Exception
         IndexOrdinalsFieldData ifd = getForField("string", "value", hasDocValues());
         IndexOrdinalsFieldData globalOrdinals = ifd.loadGlobal(topLevelReader);
         assertNotNull(globalOrdinals.getOrdinalMap());
-        assertThat(ifd.loadGlobal(topLevelReader), sameInstance(globalOrdinals));
+        assertThat(ifd.loadGlobal(topLevelReader).getOrdinalMap(), sameInstance(globalOrdinals.getOrdinalMap()));
         // 3 b/c 1 segment level caches and 1 top level cache
         // in case of doc values, we don't cache atomic FD, so only the top-level cache is there
         assertThat(indicesFieldDataCache.getCache().weight(), equalTo(hasDocValues() ? 1L : 4L));
@@ -602,7 +602,8 @@ public void testGlobalOrdinalsGetRemovedOnceIndexReaderCloses() throws Exception
                 break;
             }
         }
-        assertThat(cachedInstance, sameInstance(globalOrdinals));
+        assertNotSame(cachedInstance, globalOrdinals);
+        assertThat(cachedInstance.getOrdinalMap(), sameInstance(globalOrdinals.getOrdinalMap()));
         topLevelReader.close();
         // Now only 3 segment level entries, only the toplevel reader has been closed, but the segment readers are still used by IW
         assertThat(indicesFieldDataCache.getCache().weight(), equalTo(hasDocValues() ? 0L : 3L));
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
index 7b17b727be2b1..bfa7cf9a9e1f1 100644
--- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
@@ -39,9 +39,9 @@
 import org.elasticsearch.index.mapper.ContentPath;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper.BuilderContext;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
-import org.elasticsearch.index.mapper.Mapper.BuilderContext;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
index 6822298c2228e..e69cf09de09a7 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
@@ -135,16 +135,48 @@ public void testParsesBooleansStrict() throws IOException {
                 .endObject()
             .endObject());
         DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping));
+        // omit "false"/"true" here as they should still be parsed correctly
+        String randomValue = randomFrom("off", "no", "0", "on", "yes", "1");
         BytesReference source = BytesReference.bytes(XContentFactory.jsonBuilder()
                 .startObject()
-                    // omit "false"/"true" here as they should still be parsed correctly
-                    .field("field", randomFrom("off", "no", "0", "on", "yes", "1"))
+                    .field("field", randomValue)
                 .endObject());
         MapperParsingException ex = expectThrows(MapperParsingException.class,
                 () -> defaultMapper.parse(new SourceToParse("test", "type", "1", source, XContentType.JSON)));
-        assertEquals("failed to parse field [field] of type [boolean] in document with id '1'", ex.getMessage());
+        assertEquals("failed to parse field [field] of type [boolean] in document with id '1'. " +
+            "Preview of field's value: '" + randomValue + "'", ex.getMessage());
     }
 
+
+    public void testParsesBooleansNestedStrict() throws IOException {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", "boolean")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject());
+        DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping));
+        // omit "false"/"true" here as they should still be parsed correctly
+        String randomValue = "no";
+        BytesReference source = BytesReference.bytes(XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("field")
+                    .field("inner_field", randomValue)
+                .endObject()
+            .endObject());
+        MapperParsingException ex = expectThrows(MapperParsingException.class,
+                () -> defaultMapper.parse(new SourceToParse("test", "type", "1", source, XContentType.JSON)));
+        assertEquals("failed to parse field [field] of type [boolean] in document with id '1'. " +
+            "Preview of field's value: '{inner_field=" + randomValue + "}'", ex.getMessage());
+    }
+
+
+
+
     public void testMultiFields() throws IOException {
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties")
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
index fab34efbb00e2..fa3c9c3e4bd78 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
@@ -178,7 +178,7 @@ public void testTermQuery() {
         QueryShardContext context = new QueryShardContext(0,
                 new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(),
                         indexSettings),
-                null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null);
+                null, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null);
         MappedFieldType ft = createDefaultFieldType();
         ft.setName("field");
         String date = "2015-10-12T14:10:55";
@@ -200,7 +200,7 @@ public void testRangeQuery() throws IOException {
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build();
         QueryShardContext context = new QueryShardContext(0,
                 new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings),
-                null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null);
+                null, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null);
         MappedFieldType ft = createDefaultFieldType();
         ft.setName("field");
         String date1 = "2015-10-12T14:10:55";
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index f7f3432aa6ad2..9373a5ceeae20 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Strings;
@@ -37,6 +39,8 @@
 import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -706,6 +710,62 @@ public void testMappedNullValue() throws Exception {
         assertEquals(0, doc.rootDoc().getFields("foo").length);
     }
 
+    public void testDynamicBigInteger() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+            .startObject("type")
+                .startArray("dynamic_templates").startObject()
+                    .startObject("big-integer-to-keyword")
+                        .field("match", "big-*")
+                        .field("match_mapping_type", "long")
+                        .startObject("mapping").field("type", "keyword").endObject()
+                    .endObject()
+                .endObject().endArray()
+            .endObject()
+        .endObject());
+
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+        BigInteger value = BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE);
+        BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+            .field("big-integer", value)
+        .endObject());
+        ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", bytes, XContentType.JSON));
+
+        IndexableField[] fields = doc.rootDoc().getFields("big-integer");
+        assertEquals(2, fields.length);
+        assertTrue(fields[0].fieldType() instanceof KeywordFieldMapper.KeywordFieldType);
+        assertEquals(new BytesRef(value.toString()), fields[0].binaryValue());
+    }
+
+    public void testDynamicBigDecimal() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+            .startObject("type")
+                .startArray("dynamic_templates").startObject()
+                    .startObject("big-decimal-to-scaled-float")
+                        .field("match", "big-*")
+                        .field("match_mapping_type", "double")
+                        .startObject("mapping")
+                            .field("type", "keyword")
+                        .endObject()
+                    .endObject()
+                .endObject().endArray()
+            .endObject()
+        .endObject());
+
+        BigDecimal value = BigDecimal.valueOf(Double.MAX_VALUE).add(BigDecimal.valueOf(10.1));
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+        BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+            .field("big-decimal", value)
+        .endObject());
+        ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", bytes, XContentType.JSON));
+
+        IndexableField[] fields = doc.rootDoc().getFields("big-decimal");
+        assertEquals(2, fields.length);
+        assertTrue(fields[0].fieldType() instanceof KeywordFieldMapper.KeywordFieldType);
+        assertEquals(new BytesRef(value.toString()), fields[0].binaryValue());
+    }
+
     public void testDynamicDottedFieldNameLongArray() throws Exception {
         DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java
index e5d3040f7a3bc..5515603db3476 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Strings;
@@ -132,12 +131,6 @@ public void testExternalValuesWithMultifield() throws Exception {
                         .startObject("field")
                             .field("type", "text")
                             .field("store", true)
-                            .startObject("fields")
-                                .startObject("raw")
-                                    .field("type", "keyword")
-                                    .field("store", true)
-                                .endObject()
-                            .endObject()
                         .endObject()
                     .endObject()
                 .endObject()
@@ -164,87 +157,5 @@ public void testExternalValuesWithMultifield() throws Exception {
         IndexableField field = doc.rootDoc().getField("field.field");
         assertThat(field, notNullValue());
         assertThat(field.stringValue(), is("foo"));
-
-        IndexableField raw = doc.rootDoc().getField("field.field.raw");
-
-        assertThat(raw, notNullValue());
-        assertThat(raw.binaryValue(), is(new BytesRef("foo")));
-
-        assertWarnings("At least one multi-field, [field], was " +
-            "encountered that itself contains a multi-field. Defining multi-fields within a multi-field is deprecated and will " +
-            "no longer be supported in 8.0. To resolve the issue, all instances of [fields] that occur within a [fields] block " +
-            "should be removed from the mappings, either by flattening the chained [fields] blocks into a single level, or " +
-            "switching to [copy_to] if appropriate.");
-    }
-
-    public void testExternalValuesWithMultifieldTwoLevels() throws Exception {
-        IndexService indexService = createIndex("test");
-        Map<String, Mapper.TypeParser> mapperParsers = new HashMap<>();
-        mapperParsers.put(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo"));
-        mapperParsers.put(ExternalMapperPlugin.EXTERNAL_BIS, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "bar"));
-        mapperParsers.put(TextFieldMapper.CONTENT_TYPE, new TextFieldMapper.TypeParser());
-        MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER);
-
-        Supplier<QueryShardContext> queryShardContext = () -> {
-            return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null);
-        };
-        DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(),
-                indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext);
-
-        DocumentMapper documentMapper = parser.parse("type", new CompressedXContent(
-                Strings
-                .toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
-                .startObject("field")
-                    .field("type", ExternalMapperPlugin.EXTERNAL)
-                    .startObject("fields")
-                        .startObject("field")
-                            .field("type", "text")
-                            .startObject("fields")
-                                .startObject("generated")
-                                    .field("type", ExternalMapperPlugin.EXTERNAL_BIS)
-                                .endObject()
-                                .startObject("raw")
-                                    .field("type", "text")
-                                .endObject()
-                            .endObject()
-                        .endObject()
-                        .startObject("raw")
-                            .field("type", "text")
-                        .endObject()
-                    .endObject()
-                .endObject()
-                .endObject().endObject().endObject())));
-
-        ParsedDocument doc = documentMapper.parse(new SourceToParse("test", "type", "1", BytesReference
-                .bytes(XContentFactory.jsonBuilder()
-                        .startObject()
-                        .field("field", "1234")
-                        .endObject()),
-                XContentType.JSON));
-
-        assertThat(doc.rootDoc().getField("field.bool"), notNullValue());
-        assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T"));
-
-        assertThat(doc.rootDoc().getField("field.point"), notNullValue());
-
-        assertThat(doc.rootDoc().getField("field.shape"), notNullValue());
-
-        assertThat(doc.rootDoc().getField("field.field"), notNullValue());
-        assertThat(doc.rootDoc().getField("field.field").stringValue(), is("foo"));
-
-        assertThat(doc.rootDoc().getField("field.field.generated.generated"), notNullValue());
-        assertThat(doc.rootDoc().getField("field.field.generated.generated").stringValue(), is("bar"));
-
-        assertThat(doc.rootDoc().getField("field.field.raw"), notNullValue());
-        assertThat(doc.rootDoc().getField("field.field.raw").stringValue(), is("foo"));
-
-        assertThat(doc.rootDoc().getField("field.raw"), notNullValue());
-        assertThat(doc.rootDoc().getField("field.raw").stringValue(), is("foo"));
-
-        assertWarnings("At least one multi-field, [field], was " +
-            "encountered that itself contains a multi-field. Defining multi-fields within a multi-field is deprecated and will " +
-            "no longer be supported in 8.0. To resolve the issue, all instances of [fields] that occur within a [fields] block " +
-            "should be removed from the mappings, either by flattening the chained [fields] blocks into a single level, or " +
-            "switching to [copy_to] if appropriate.");
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java
index 6d47e4a784e06..7e7764d9514fe 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java
@@ -139,14 +139,8 @@ public void testExternalValuesWithMultifield() throws Exception {
                     .field("type", ExternalMapperPlugin.EXTERNAL_UPPER)
                     .startObject("fields")
                         .startObject("g")
-                            .field("type", "text")
+                            .field("type", "keyword")
                             .field("store", true)
-                            .startObject("fields")
-                                .startObject("raw")
-                                    .field("type", "keyword")
-                                    .field("store", true)
-                                .endObject()
-                            .endObject()
                         .endObject()
                     .endObject()
                 .endObject()
@@ -156,7 +150,7 @@ public void testExternalValuesWithMultifield() throws Exception {
         refresh();
 
         SearchResponse response = client().prepareSearch("test-idx")
-                .setQuery(QueryBuilders.termQuery("f.g.raw", "FOO BAR"))
+                .setQuery(QueryBuilders.termQuery("f.g", "FOO BAR"))
                 .execute().actionGet();
 
         assertThat(response.getHits().getTotalHits().value, equalTo((long) 1));
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java
index 4ae03138764d7..d01c8efe6c7c6 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java
@@ -63,10 +63,10 @@ public void testTermQuery() {
         MapperService mapperService = mock(MapperService.class);
         when(mapperService.fullName("_field_names")).thenReturn(fieldNamesFieldType);
         when(mapperService.fullName("field_name")).thenReturn(fieldType);
-        when(mapperService.simpleMatchToFullName("field_name")).thenReturn(Collections.singletonList("field_name"));
+        when(mapperService.simpleMatchToFullName("field_name")).thenReturn(Collections.singleton("field_name"));
 
         QueryShardContext queryShardContext = new QueryShardContext(0,
-                indexSettings, null, null, mapperService, null, null, null, null, null, null, () -> 0L, null);
+                indexSettings, null, null, null, mapperService, null, null, null, null, null, null, () -> 0L, null);
         fieldNamesFieldType.setEnabled(true);
         Query termQuery = fieldNamesFieldType.termQuery("field_name", queryShardContext);
         assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.CONTENT_TYPE, "field_name")), termQuery);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java
index f53610d23aaad..28d712e9854ab 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldTypeTests.java
@@ -28,9 +28,6 @@
 import org.elasticsearch.index.query.QueryShardContext;
 import org.mockito.Mockito;
 
-import java.util.Collection;
-import java.util.Collections;
-
 public class IdFieldTypeTests extends FieldTypeTestCase {
     @Override
     protected MappedFieldType createDefaultFieldType() {
@@ -58,8 +55,6 @@ public void testTermsQuery() throws Exception {
         Mockito.when(context.indexVersionCreated()).thenReturn(indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null));
 
         MapperService mapperService = Mockito.mock(MapperService.class);
-        Collection<String> types = Collections.emptySet();
-        Mockito.when(context.queryTypes()).thenReturn(types);
         Mockito.when(context.getMapperService()).thenReturn(mapperService);
 
         MappedFieldType ft = IdFieldMapper.defaultFieldType(mockSettings);
@@ -67,8 +62,6 @@ public void testTermsQuery() throws Exception {
         Query query = ft.termQuery("id", context);
         assertEquals(new TermInSetQuery("_id", Uid.encodeId("id")), query);
 
-        types = Collections.singleton("type");
-        Mockito.when(context.queryTypes()).thenReturn(types);
         query = ft.termQuery("id", context);
         assertEquals(new TermInSetQuery("_id", Uid.encodeId("id")), query);
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
index 1bdf40bcc6708..a076a60231f36 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
@@ -386,6 +386,85 @@ public void testNormalizer() throws IOException {
         assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
     }
 
+    public void testParsesKeywordNestedEmptyObjectStrict() throws IOException {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", "keyword")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject());
+        DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference source = BytesReference.bytes(XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("field")
+                .endObject()
+            .endObject());
+        MapperParsingException ex = expectThrows(MapperParsingException.class,
+                () -> defaultMapper.parse(new SourceToParse("test", "type", "1", source, XContentType.JSON)));
+        assertEquals("failed to parse field [field] of type [keyword] in document with id '1'. " +
+            "Preview of field's value: '{}'", ex.getMessage());
+    }
+
+    public void testParsesKeywordNestedListStrict() throws IOException {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", "keyword")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject());
+        DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference source = BytesReference.bytes(XContentFactory.jsonBuilder()
+            .startObject()
+                .startArray("field")
+                    .startObject()
+                        .startArray("array_name")
+                            .value("inner_field_first")
+                            .value("inner_field_second")
+                        .endArray()
+                    .endObject()
+                .endArray()
+            .endObject());
+        MapperParsingException ex = expectThrows(MapperParsingException.class,
+                () -> defaultMapper.parse(new SourceToParse("test", "type", "1", source, XContentType.JSON)));
+        assertEquals("failed to parse field [field] of type [keyword] in document with id '1'. " +
+            "Preview of field's value: '{array_name=[inner_field_first, inner_field_second]}'", ex.getMessage());
+    }
+
+    public void testParsesKeywordNullStrict() throws IOException {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", "keyword")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject());
+        DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference source = BytesReference.bytes(XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("field")
+                    .nullField("field_name")
+                .endObject()
+            .endObject());
+        MapperParsingException ex = expectThrows(MapperParsingException.class,
+                () -> defaultMapper.parse(new SourceToParse("test", "type", "1", source, XContentType.JSON)));
+        assertEquals("failed to parse field [field] of type [keyword] in document with id '1'. " +
+            "Preview of field's value: '{field_name=null}'", ex.getMessage());
+    }
+
     public void testUpdateNormalizer() throws IOException {
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("field")
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
index d8c120e492d31..6bdfc167dec8b 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
@@ -19,7 +19,9 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.analysis.TokenStream;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -27,11 +29,21 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AnalysisMode;
+import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.analysis.ReloadableCustomAnalyzer;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType;
 import org.elasticsearch.indices.InvalidTypeNameException;
+import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
+import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -39,6 +51,8 @@
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ExecutionException;
 
 import static org.hamcrest.CoreMatchers.containsString;
@@ -49,7 +63,7 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return Collections.singleton(InternalSettingsPlugin.class);
+        return List.of(InternalSettingsPlugin.class, ReloadableFilterPlugin.class);
     }
 
     public void testTypeNameStartsWithIllegalDot() {
@@ -434,4 +448,97 @@ public void testMappingRecoverySkipFieldNameLengthLimit() throws Throwable {
         assertEquals(testString, documentMapper.mappers().getMapper(testString).simpleName());
     }
 
+    public void testReloadSearchAnalyzers() throws IOException {
+        Settings settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+                .put("index.analysis.analyzer.reloadableAnalyzer.type", "custom")
+                .put("index.analysis.analyzer.reloadableAnalyzer.tokenizer", "standard")
+                .putList("index.analysis.analyzer.reloadableAnalyzer.filter", "myReloadableFilter").build();
+
+        MapperService mapperService = createIndex("test_index", settings).mapperService();
+        CompressedXContent mapping = new CompressedXContent(BytesReference.bytes(
+                XContentFactory.jsonBuilder().startObject().startObject("_doc")
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", "text")
+                            .field("analyzer", "simple")
+                            .field("search_analyzer", "reloadableAnalyzer")
+                            .field("search_quote_analyzer", "stop")
+                        .endObject()
+                        .startObject("otherField")
+                            .field("type", "text")
+                            .field("analyzer", "standard")
+                            .field("search_analyzer", "simple")
+                            .field("search_quote_analyzer", "reloadableAnalyzer")
+                        .endObject()
+                    .endObject()
+                .endObject().endObject()));
+
+        mapperService.merge("_doc", mapping, MergeReason.MAPPING_UPDATE);
+        IndexAnalyzers current = mapperService.getIndexAnalyzers();
+
+        ReloadableCustomAnalyzer originalReloadableAnalyzer = (ReloadableCustomAnalyzer) current.get("reloadableAnalyzer").analyzer();
+        TokenFilterFactory[] originalTokenFilters = originalReloadableAnalyzer.getComponents().getTokenFilters();
+        assertEquals(1, originalTokenFilters.length);
+        assertEquals("myReloadableFilter", originalTokenFilters[0].name());
+
+        // now reload, this should change the tokenfilterFactory inside the analyzer
+        mapperService.reloadSearchAnalyzers(getInstanceFromNode(AnalysisRegistry.class));
+        IndexAnalyzers updatedAnalyzers = mapperService.getIndexAnalyzers();
+        assertSame(current, updatedAnalyzers);
+        assertSame(current.getDefaultIndexAnalyzer(), updatedAnalyzers.getDefaultIndexAnalyzer());
+        assertSame(current.getDefaultSearchAnalyzer(), updatedAnalyzers.getDefaultSearchAnalyzer());
+        assertSame(current.getDefaultSearchQuoteAnalyzer(), updatedAnalyzers.getDefaultSearchQuoteAnalyzer());
+
+        assertFalse(assertSameContainedFilters(originalTokenFilters, current.get("reloadableAnalyzer")));
+        assertFalse(assertSameContainedFilters(originalTokenFilters, mapperService.fullName("field").searchAnalyzer()));
+        assertFalse(assertSameContainedFilters(originalTokenFilters, mapperService.fullName("otherField").searchQuoteAnalyzer()));
+    }
+
+    private boolean assertSameContainedFilters(TokenFilterFactory[] originalTokenFilter, NamedAnalyzer updatedAnalyzer) {
+        ReloadableCustomAnalyzer updatedReloadableAnalyzer = (ReloadableCustomAnalyzer) updatedAnalyzer.analyzer();
+        TokenFilterFactory[] newTokenFilters = updatedReloadableAnalyzer.getComponents().getTokenFilters();
+        assertEquals(originalTokenFilter.length, newTokenFilters.length);
+        int i = 0;
+        for (TokenFilterFactory tf : newTokenFilters ) {
+            assertEquals(originalTokenFilter[i].name(), tf.name());
+            if (originalTokenFilter[i] != tf) {
+                return false;
+            }
+            i++;
+        }
+        return true;
+    }
+
+    public static final class ReloadableFilterPlugin extends Plugin implements AnalysisPlugin {
+
+        @Override
+        public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
+            return Collections.singletonMap("myReloadableFilter", new AnalysisProvider<TokenFilterFactory>() {
+
+                @Override
+                public TokenFilterFactory get(IndexSettings indexSettings, Environment environment, String name, Settings settings)
+                        throws IOException {
+                    return new TokenFilterFactory() {
+
+                        @Override
+                        public String name() {
+                            return "myReloadableFilter";
+                        }
+
+                        @Override
+                        public TokenStream create(TokenStream tokenStream) {
+                            return tokenStream;
+                        }
+
+                        @Override
+                        public AnalysisMode getAnalysisMode() {
+                            return AnalysisMode.SEARCH_TIME;
+                        }
+                    };
+                }
+            });
+        }
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
index edca517830833..5a2fe0233ef05 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.index.IndexableField;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -32,6 +33,7 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.VersionUtils;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -695,4 +697,44 @@ protected boolean forbidPrivateIndexSettings() {
          */
         return false;
     }
+
+    public void testReorderParent() throws IOException {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
+            .startObject("nested1").field("type", "nested").endObject()
+            .endObject().endObject().endObject());
+
+        DocumentMapper docMapper = createIndex("test",
+            Settings.builder().put(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey(),
+                VersionUtils.randomIndexCompatibleVersion(random())).build())
+            .mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+
+        assertThat(docMapper.hasNestedObjects(), equalTo(true));
+        ObjectMapper nested1Mapper = docMapper.objectMappers().get("nested1");
+        assertThat(nested1Mapper.nested().isNested(), equalTo(true));
+
+        ParsedDocument doc = docMapper.parse(new SourceToParse("test", "type", "1",
+            BytesReference.bytes(XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field", "value")
+                .startArray("nested1")
+                .startObject()
+                .field("field1", "1")
+                .field("field2", "2")
+                .endObject()
+                .startObject()
+                .field("field1", "3")
+                .field("field2", "4")
+                .endObject()
+                .endArray()
+                .endObject()),
+            XContentType.JSON));
+
+        assertThat(doc.docs().size(), equalTo(3));
+        assertThat(doc.docs().get(0).get(TypeFieldMapper.NAME), equalTo(nested1Mapper.nestedTypePathAsString()));
+        assertThat(doc.docs().get(0).get("nested1.field1"), equalTo("1"));
+        assertThat(doc.docs().get(0).get("nested1.field2"), equalTo("2"));
+        assertThat(doc.docs().get(1).get("nested1.field1"), equalTo("3"));
+        assertThat(doc.docs().get(1).get("nested1.field2"), equalTo("4"));
+        assertThat(doc.docs().get(2).get("field"), equalTo("value"));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
index a26999fa3a6f5..7783c90de82a4 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
@@ -228,7 +228,7 @@ private QueryShardContext createContext() {
         Settings indexSettings = Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings);
-        return new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(),
+        return new QueryShardContext(0, idxSettings, null, null, null, null, null, null, xContentRegistry(),
             writableRegistry(), null, null, () -> nowInMillis, null);
     }
     
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
index 449d17a5b9bda..e7bec7ff95523 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
@@ -22,7 +22,6 @@
 import org.apache.lucene.analysis.MockSynonymAnalyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
@@ -41,7 +40,6 @@
 import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -636,7 +634,8 @@ public void testIndexPrefixIndexTypes() throws IOException {
             DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
 
             FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix");
-            FieldType ft = prefix.fieldType;
+            MappedFieldType ft = prefix.fieldType;
+            assertEquals(ft.name(), "field._index_prefix");
             assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, ft.indexOptions());
         }
 
@@ -652,7 +651,8 @@ public void testIndexPrefixIndexTypes() throws IOException {
             DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
 
             FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix");
-            FieldType ft = prefix.fieldType;
+            MappedFieldType ft = prefix.fieldType;
+            assertEquals(ft.name(), "field._index_prefix");
             assertEquals(IndexOptions.DOCS, ft.indexOptions());
             assertFalse(ft.storeTermVectors());
         }
@@ -669,12 +669,9 @@ public void testIndexPrefixIndexTypes() throws IOException {
             DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
 
             FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix");
-            FieldType ft = prefix.fieldType;
-            if (indexService.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_4_0)) {
-                assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, ft.indexOptions());
-            } else {
-                assertEquals(IndexOptions.DOCS, ft.indexOptions());
-            }
+            MappedFieldType ft = prefix.fieldType;
+            assertEquals(ft.name(), "field._index_prefix");
+            assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, ft.indexOptions());
             assertFalse(ft.storeTermVectors());
         }
 
@@ -690,12 +687,9 @@ public void testIndexPrefixIndexTypes() throws IOException {
             DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
 
             FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix");
-            FieldType ft = prefix.fieldType;
-            if (indexService.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_4_0)) {
-                assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, ft.indexOptions());
-            } else {
-                assertEquals(IndexOptions.DOCS, ft.indexOptions());
-            }
+            MappedFieldType ft = prefix.fieldType;
+            assertEquals(ft.name(), "field._index_prefix");
+            assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, ft.indexOptions());
             assertTrue(ft.storeTermVectorOffsets());
         }
 
@@ -711,16 +705,67 @@ public void testIndexPrefixIndexTypes() throws IOException {
             DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
 
             FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix");
-            FieldType ft = prefix.fieldType;
-            if (indexService.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_4_0)) {
-                assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, ft.indexOptions());
-            } else {
-                assertEquals(IndexOptions.DOCS, ft.indexOptions());
-            }
+            MappedFieldType ft = prefix.fieldType;
+            assertEquals(ft.name(), "field._index_prefix");
+            assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, ft.indexOptions());
             assertFalse(ft.storeTermVectorOffsets());
         }
     }
 
+    public void testNestedIndexPrefixes() throws IOException {
+        {
+            String mapping = Strings.toString(XContentFactory.jsonBuilder()
+                    .startObject()
+                        .startObject("properties")
+                            .startObject("object")
+                                .field("type", "object")
+                                .startObject("properties")
+                                    .startObject("field")
+                                        .field("type", "text")
+                                        .startObject("index_prefixes").endObject()
+                                    .endObject()
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject());
+
+            indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE);
+            MappedFieldType textField = indexService.mapperService().fullName("object.field");
+            assertNotNull(textField);
+            assertThat(textField, instanceOf(TextFieldType.class));
+            MappedFieldType prefix = ((TextFieldType) textField).getPrefixFieldType();
+            assertEquals(prefix.name(), "object.field._index_prefix");
+            assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, prefix.indexOptions());
+            assertFalse(prefix.storeTermVectorOffsets());
+        }
+
+        {
+            String mapping = Strings.toString(XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("properties")
+                        .startObject("body")
+                            .field("type", "text")
+                            .startObject("fields")
+                                .startObject("with_prefix")
+                                    .field("type", "text")
+                                    .startObject("index_prefixes").endObject()
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endObject());
+
+            indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE);
+            MappedFieldType textField = indexService.mapperService().fullName("body.with_prefix");
+            assertNotNull(textField);
+            assertThat(textField, instanceOf(TextFieldType.class));
+            MappedFieldType prefix = ((TextFieldType) textField).getPrefixFieldType();
+            assertEquals(prefix.name(), "body.with_prefix._index_prefix");
+            assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, prefix.indexOptions());
+            assertFalse(prefix.storeTermVectorOffsets());
+        }
+    }
+
     public void testFastPhraseMapping() throws IOException {
 
         QueryShardContext queryShardContext = indexService.newQueryShardContext(
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java
index 70f469b96370c..6f1a1119796f9 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -48,6 +49,7 @@
 import static org.elasticsearch.index.analysis.AnalysisRegistry.DEFAULT_ANALYZER_NAME;
 import static org.elasticsearch.index.analysis.AnalysisRegistry.DEFAULT_SEARCH_ANALYZER_NAME;
 import static org.elasticsearch.index.analysis.AnalysisRegistry.DEFAULT_SEARCH_QUOTED_ANALYZER_NAME;
+import static org.hamcrest.core.IsEqual.equalTo;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -69,7 +71,7 @@ public void testParseTextFieldCheckAnalyzerAnalysisMode() {
         analyzers.put("my_analyzer",
                 new NamedAnalyzer("my_named_analyzer", AnalyzerScope.INDEX, createAnalyzerWithMode("my_analyzer", AnalysisMode.ALL)));
 
-        IndexAnalyzers indexAnalyzers = new IndexAnalyzers(indexSettings, analyzers, Collections.emptyMap(), Collections.emptyMap());
+        IndexAnalyzers indexAnalyzers = new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap());
         when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers);
         TypeParsers.parseTextField(builder, "name", new HashMap<>(fieldNode), parserContext);
 
@@ -78,7 +80,7 @@ public void testParseTextFieldCheckAnalyzerAnalysisMode() {
         analyzers = defaultAnalyzers();
         analyzers.put("my_analyzer", new NamedAnalyzer("my_named_analyzer", AnalyzerScope.INDEX,
                 createAnalyzerWithMode("my_analyzer", mode)));
-        indexAnalyzers = new IndexAnalyzers(indexSettings, analyzers, Collections.emptyMap(), Collections.emptyMap());
+        indexAnalyzers = new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap());
         when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers);
         MapperException ex = expectThrows(MapperException.class,
                 () -> TypeParsers.parseTextField(builder, "name", new HashMap<>(fieldNode), parserContext));
@@ -112,7 +114,7 @@ public void testParseTextFieldCheckSearchAnalyzerAnalysisMode() {
                     new NamedAnalyzer("my_named_analyzer", AnalyzerScope.INDEX, createAnalyzerWithMode("my_analyzer", mode)));
             analyzers.put("standard", new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer()));
 
-            IndexAnalyzers indexAnalyzers = new IndexAnalyzers(indexSettings, analyzers, Collections.emptyMap(), Collections.emptyMap());
+            IndexAnalyzers indexAnalyzers = new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap());
             when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers);
             TypeParsers.parseTextField(builder, "name", new HashMap<>(fieldNode), parserContext);
 
@@ -122,7 +124,7 @@ public void testParseTextFieldCheckSearchAnalyzerAnalysisMode() {
             analyzers.put("my_analyzer",
                     new NamedAnalyzer("my_named_analyzer", AnalyzerScope.INDEX, createAnalyzerWithMode("my_analyzer", mode)));
             analyzers.put("standard", new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer()));
-            indexAnalyzers = new IndexAnalyzers(indexSettings, analyzers, Collections.emptyMap(), Collections.emptyMap());
+            indexAnalyzers = new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap());
             when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers);
             MapperException ex = expectThrows(MapperException.class,
                     () -> TypeParsers.parseTextField(builder, "name", new HashMap<>(fieldNode), parserContext));
@@ -142,7 +144,7 @@ public void testParseTextFieldCheckAnalyzerWithSearchAnalyzerAnalysisMode() {
         Map<String, NamedAnalyzer> analyzers = defaultAnalyzers();
         analyzers.put("my_analyzer",
                 new NamedAnalyzer("my_named_analyzer", AnalyzerScope.INDEX, createAnalyzerWithMode("my_analyzer", mode)));
-        IndexAnalyzers indexAnalyzers = new IndexAnalyzers(indexSettings, analyzers, Collections.emptyMap(), Collections.emptyMap());
+        IndexAnalyzers indexAnalyzers = new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap());
         when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers);
         MapperException ex = expectThrows(MapperException.class,
                 () -> TypeParsers.parseTextField(builder, "name", new HashMap<>(fieldNode), parserContext));
@@ -157,7 +159,7 @@ public void testParseTextFieldCheckAnalyzerWithSearchAnalyzerAnalysisMode() {
                 new NamedAnalyzer("my_named_analyzer", AnalyzerScope.INDEX, createAnalyzerWithMode("my_analyzer", mode)));
         analyzers.put("standard", new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer()));
 
-        indexAnalyzers = new IndexAnalyzers(indexSettings, analyzers, Collections.emptyMap(), Collections.emptyMap());
+        indexAnalyzers = new IndexAnalyzers(analyzers, Collections.emptyMap(), Collections.emptyMap());
         when(parserContext.getIndexAnalyzers()).thenReturn(indexAnalyzers);
         TypeParsers.parseTextField(builder, "name", new HashMap<>(fieldNode), parserContext);
     }
@@ -179,19 +181,35 @@ public void testMultiFieldWithinMultiField() throws IOException {
             .endObject()
         .endObject();
 
+        Mapper.TypeParser typeParser = new KeywordFieldMapper.TypeParser();
+
+        // For indices created prior to 8.0, we should only emit a warning and not fail parsing.
         Map<String, Object> fieldNode = XContentHelper.convertToMap(
             BytesReference.bytes(mapping), true, mapping.contentType()).v2();
 
-        Mapper.TypeParser typeParser = new KeywordFieldMapper.TypeParser();
-        Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext("type",
-            null, null, type -> typeParser, Version.CURRENT, null);
-
-        TypeParsers.parseField(builder, "some-field", fieldNode, parserContext);
-        assertWarnings("At least one multi-field, [sub-field], was " +
-            "encountered that itself contains a multi-field. Defining multi-fields within a multi-field is deprecated and will " +
-            "no longer be supported in 8.0. To resolve the issue, all instances of [fields] that occur within a [fields] block " +
-            "should be removed from the mappings, either by flattening the chained [fields] blocks into a single level, or " +
-            "switching to [copy_to] if appropriate.");
+        Version olderVersion = VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0);
+        Mapper.TypeParser.ParserContext olderContext = new Mapper.TypeParser.ParserContext("type",
+            null, null, type -> typeParser, olderVersion, null);
+
+        TypeParsers.parseField(builder, "some-field", fieldNode, olderContext);
+        assertWarnings("At least one multi-field, [sub-field], " +
+            "was encountered that itself contains a multi-field. Defining multi-fields within a multi-field is deprecated " +
+            "and is not supported for indices created in 8.0 and later. To migrate the mappings, all instances of [fields] " +
+            "that occur within a [fields] block should be removed from the mappings, either by flattening the chained " +
+            "[fields] blocks into a single level, or switching to [copy_to] if appropriate.");
+
+        // For indices created in 8.0 or later, we should throw an error.
+        Map<String, Object> fieldNodeCopy = XContentHelper.convertToMap(
+            BytesReference.bytes(mapping), true, mapping.contentType()).v2();
+
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_8_0_0, Version.CURRENT);
+        Mapper.TypeParser.ParserContext context = new Mapper.TypeParser.ParserContext("type",
+            null, null, type -> typeParser, version, null);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> TypeParsers.parseField(builder, "some-field", fieldNodeCopy, context));
+        assertThat(e.getMessage(), equalTo("Encountered a multi-field [sub-field] which itself contains a " +
+            "multi-field. Defining chained multi-fields is not supported."));
     }
 
     private Analyzer createAnalyzerWithMode(String name, AnalysisMode mode) {
diff --git a/server/src/test/java/org/elasticsearch/index/query/AbstractQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/AbstractQueryBuilderTests.java
index 68f964e2c8b82..96b9336679f40 100644
--- a/server/src/test/java/org/elasticsearch/index/query/AbstractQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/AbstractQueryBuilderTests.java
@@ -40,7 +40,7 @@ public class AbstractQueryBuilderTests extends ESTestCase {
 
     @BeforeClass
     public static void init() {
-        xContentRegistry = new NamedXContentRegistry(new SearchModule(Settings.EMPTY, false, emptyList()).getNamedXContents());
+        xContentRegistry = new NamedXContentRegistry(new SearchModule(Settings.EMPTY, emptyList()).getNamedXContents());
     }
 
     @AfterClass
diff --git a/server/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java
deleted file mode 100644
index 5e443ec41ede9..0000000000000
--- a/server/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.query;
-
-import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.ExtendedCommonTermsQuery;
-import org.apache.lucene.search.Query;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.test.AbstractQueryTestCase;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery;
-import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.nullValue;
-
-public class CommonTermsQueryBuilderTests extends AbstractQueryTestCase<CommonTermsQueryBuilder> {
-
-    @Override
-    protected CommonTermsQueryBuilder doCreateTestQueryBuilder() {
-        int numberOfTerms = randomIntBetween(0, 10);
-        StringBuilder text = new StringBuilder("");
-        for (int i = 0; i < numberOfTerms; i++) {
-            text.append(randomAlphaOfLengthBetween(1, 10)).append(" ");
-        }
-
-        String fieldName = randomFrom(STRING_FIELD_NAME,
-            STRING_ALIAS_FIELD_NAME,
-            randomAlphaOfLengthBetween(1, 10));
-        CommonTermsQueryBuilder query = new CommonTermsQueryBuilder(fieldName, text.toString());
-
-        if (randomBoolean()) {
-            query.cutoffFrequency(randomIntBetween(1, 10));
-        }
-
-        if (randomBoolean()) {
-            query.lowFreqOperator(randomFrom(Operator.values()));
-        }
-
-        // number of low frequency terms that must match
-        if (randomBoolean()) {
-            query.lowFreqMinimumShouldMatch("" + randomIntBetween(1, 5));
-        }
-
-        if (randomBoolean()) {
-            query.highFreqOperator(randomFrom(Operator.values()));
-        }
-
-        // number of high frequency terms that must match
-        if (randomBoolean()) {
-            query.highFreqMinimumShouldMatch("" + randomIntBetween(1, 5));
-        }
-
-        if (randomBoolean()) {
-            query.analyzer(randomAnalyzer());
-        }
-
-        return query;
-    }
-
-    @Override
-    protected Map<String, CommonTermsQueryBuilder> getAlternateVersions() {
-        Map<String, CommonTermsQueryBuilder> alternateVersions = new HashMap<>();
-        CommonTermsQueryBuilder commonTermsQuery = new CommonTermsQueryBuilder(randomAlphaOfLengthBetween(1, 10),
-                randomAlphaOfLengthBetween(1, 10));
-        String contentString = "{\n" +
-                "    \"common\" : {\n" +
-                "        \"" + commonTermsQuery.fieldName() + "\" : \"" + commonTermsQuery.value() + "\"\n" +
-                "    }\n" +
-                "}";
-        alternateVersions.put(contentString, commonTermsQuery);
-        return alternateVersions;
-    }
-
-    @Override
-    protected void doAssertLuceneQuery(CommonTermsQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
-        assertThat(query, instanceOf(ExtendedCommonTermsQuery.class));
-        ExtendedCommonTermsQuery extendedCommonTermsQuery = (ExtendedCommonTermsQuery) query;
-
-        List<Term> terms = extendedCommonTermsQuery.getTerms();
-        if (!terms.isEmpty()) {
-            String expectedFieldName = expectedFieldName(queryBuilder.fieldName());
-            String actualFieldName = terms.iterator().next().field();
-            assertThat(actualFieldName, equalTo(expectedFieldName));
-        }
-
-        assertThat(extendedCommonTermsQuery.getHighFreqMinimumNumberShouldMatchSpec(), equalTo(queryBuilder.highFreqMinimumShouldMatch()));
-        assertThat(extendedCommonTermsQuery.getLowFreqMinimumNumberShouldMatchSpec(), equalTo(queryBuilder.lowFreqMinimumShouldMatch()));
-    }
-
-    public void testIllegalArguments() {
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new CommonTermsQueryBuilder(null, "text"));
-        assertEquals("field name is null or empty", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> new CommonTermsQueryBuilder("", "text"));
-        assertEquals("field name is null or empty", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> new CommonTermsQueryBuilder("fieldName", null));
-        assertEquals("text cannot be null", e.getMessage());
-    }
-
-    public void testFromJson() throws IOException {
-        String query =
-                "{\n" +
-                "  \"common\" : {\n" +
-                "    \"body\" : {\n" +
-                "      \"query\" : \"nelly the elephant not as a cartoon\",\n" +
-                "      \"high_freq_operator\" : \"AND\",\n" +
-                "      \"low_freq_operator\" : \"OR\",\n" +
-                "      \"cutoff_frequency\" : 0.001,\n" +
-                "      \"minimum_should_match\" : {\n" +
-                "        \"low_freq\" : \"2\",\n" +
-                "        \"high_freq\" : \"3\"\n" +
-                "      },\n" +
-                "      \"boost\" : 42.0\n" +
-                "    }\n" +
-                "  }\n" +
-                "}";
-
-        CommonTermsQueryBuilder queryBuilder = (CommonTermsQueryBuilder) parseQuery(query);
-        checkGeneratedJson(query, queryBuilder);
-
-        assertEquals(query, 42, queryBuilder.boost, 0.00001);
-        assertEquals(query, 0.001, queryBuilder.cutoffFrequency(), 0.0001);
-        assertEquals(query, Operator.OR, queryBuilder.lowFreqOperator());
-        assertEquals(query, Operator.AND, queryBuilder.highFreqOperator());
-        assertEquals(query, "nelly the elephant not as a cartoon", queryBuilder.value());
-    }
-
-    public void testCommonTermsQuery1() throws IOException {
-        String query = copyToStringFromClasspath("/org/elasticsearch/index/query/commonTerms-query1.json");
-        Query parsedQuery = parseQuery(query).toQuery(createShardContext());
-        assertThat(parsedQuery, instanceOf(ExtendedCommonTermsQuery.class));
-        ExtendedCommonTermsQuery ectQuery = (ExtendedCommonTermsQuery) parsedQuery;
-        assertThat(ectQuery.getHighFreqMinimumNumberShouldMatchSpec(), nullValue());
-        assertThat(ectQuery.getLowFreqMinimumNumberShouldMatchSpec(), equalTo("2"));
-    }
-
-    public void testCommonTermsQuery2() throws IOException {
-        String query = copyToStringFromClasspath("/org/elasticsearch/index/query/commonTerms-query2.json");
-        Query parsedQuery = parseQuery(query).toQuery(createShardContext());
-        assertThat(parsedQuery, instanceOf(ExtendedCommonTermsQuery.class));
-        ExtendedCommonTermsQuery ectQuery = (ExtendedCommonTermsQuery) parsedQuery;
-        assertThat(ectQuery.getHighFreqMinimumNumberShouldMatchSpec(), equalTo("50%"));
-        assertThat(ectQuery.getLowFreqMinimumNumberShouldMatchSpec(), equalTo("5<20%"));
-    }
-
-    public void testCommonTermsQuery3() throws IOException {
-        String query = copyToStringFromClasspath("/org/elasticsearch/index/query/commonTerms-query3.json");
-        Query parsedQuery = parseQuery(query).toQuery(createShardContext());
-        assertThat(parsedQuery, instanceOf(ExtendedCommonTermsQuery.class));
-        ExtendedCommonTermsQuery ectQuery = (ExtendedCommonTermsQuery) parsedQuery;
-        assertThat(ectQuery.getHighFreqMinimumNumberShouldMatchSpec(), nullValue());
-        assertThat(ectQuery.getLowFreqMinimumNumberShouldMatchSpec(), equalTo("2"));
-    }
-
-    // see #11730
-    public void testCommonTermsQuery4() throws IOException {
-        Query parsedQuery = parseQuery(commonTermsQuery("field", "text")).toQuery(createShardContext());
-        assertThat(parsedQuery, instanceOf(ExtendedCommonTermsQuery.class));
-    }
-
-    public void testParseFailsWithMultipleFields() throws IOException {
-        String json = "{\n" +
-                "  \"common\" : {\n" +
-                "    \"message1\" : {\n" +
-                "      \"query\" : \"nelly the elephant not as a cartoon\"\n" +
-                "    },\n" +
-                "    \"message2\" : {\n" +
-                "      \"query\" : \"nelly the elephant not as a cartoon\"\n" +
-                "    }\n" +
-                "  }\n" +
-                "}";
-
-        ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(json));
-        assertEquals("[common] query doesn't support multiple fields, found [message1] and [message2]", e.getMessage());
-
-        String shortJson = "{\n" +
-                "  \"common\" : {\n" +
-                "    \"message1\" : \"nelly the elephant not as a cartoon\",\n" +
-                "    \"message2\" : \"nelly the elephant not as a cartoon\"\n" +
-                "  }\n" +
-                "}";
-        e = expectThrows(ParsingException.class, () -> parseQuery(shortJson));
-        assertEquals("[common] query doesn't support multiple fields, found [message1] and [message2]", e.getMessage());
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/index/query/CommonTermsQueryParserTests.java b/server/src/test/java/org/elasticsearch/index/query/CommonTermsQueryParserTests.java
deleted file mode 100644
index f4e737ea4b024..0000000000000
--- a/server/src/test/java/org/elasticsearch/index/query/CommonTermsQueryParserTests.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.query;
-
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.test.ESSingleNodeTestCase;
-
-import java.io.IOException;
-
-public class CommonTermsQueryParserTests extends ESSingleNodeTestCase {
-    public void testWhenParsedQueryIsNullNoNullPointerExceptionIsThrown() throws IOException {
-        final String index = "test-index";
-        final String type = "test-type";
-        client()
-                .admin()
-                .indices()
-                .prepareCreate(index)
-                .addMapping(type, "name", "type=text,analyzer=stop")
-                .execute()
-                .actionGet();
-        ensureGreen();
-
-        CommonTermsQueryBuilder commonTermsQueryBuilder =
-                new CommonTermsQueryBuilder("name", "the").queryName("query-name");
-
-        // the named query parses to null; we are testing this does not cause a NullPointerException
-        SearchResponse response =
-                client().prepareSearch(index).setTypes(type).setQuery(commonTermsQueryBuilder).execute().actionGet();
-
-        assertNotNull(response);
-        assertEquals(response.getHits().getHits().length, 0);
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/index/query/FullTextQueryTestCase.java b/server/src/test/java/org/elasticsearch/index/query/FullTextQueryTestCase.java
deleted file mode 100644
index 7f9d85e3e26b7..0000000000000
--- a/server/src/test/java/org/elasticsearch/index/query/FullTextQueryTestCase.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.query;
-
-import org.elasticsearch.index.mapper.DateFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.test.AbstractQueryTestCase;
-
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Set;
-
-public abstract class FullTextQueryTestCase<QB extends AbstractQueryBuilder<QB>> extends AbstractQueryTestCase<QB> {
-    protected abstract boolean isCacheable(QB queryBuilder);
-
-    /**
-     * Full text queries that start with "now" are not cacheable if they
-     * target a {@link DateFieldMapper.DateFieldType} field.
-     */
-    protected final boolean isCacheable(Collection<String> fields, String value) {
-        if (value.length() < 3
-                || value.substring(0, 3).equalsIgnoreCase("now") == false) {
-            return true;
-        }
-        Set<String> dateFields = new HashSet<>();
-        getMapping().forEach(ft -> {
-            if (ft instanceof DateFieldMapper.DateFieldType) {
-                dateFields.add(ft.name());
-            }
-        });
-        for (MappedFieldType ft : getMapping()) {
-            if (ft instanceof DateFieldMapper.DateFieldType) {
-                dateFields.add(ft.name());
-            }
-        }
-        if (fields.isEmpty()) {
-            // special case: all fields are requested
-            return dateFields.isEmpty();
-        }
-        return fields.stream()
-            .anyMatch(dateFields::contains) == false;
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
index 62cc7a43cd2c2..a5311ed157c62 100644
--- a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
@@ -72,7 +72,7 @@ protected String fieldName() {
     @Override
     protected Settings createTestIndexSettings() {
         // force the new shape impl
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_6_0, Version.CURRENT);
+        Version version = VersionUtils.randomIndexCompatibleVersion(random());
         return Settings.builder()
                 .put(super.createTestIndexSettings())
                 .put(IndexMetaData.SETTING_VERSION_CREATED, version)
diff --git a/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java
index 2aed8202dd698..1083248b850d1 100644
--- a/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java
@@ -23,15 +23,12 @@
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermInSetQuery;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
-import java.util.Arrays;
 
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.Matchers.contains;
@@ -41,44 +38,19 @@ public class IdsQueryBuilderTests extends AbstractQueryTestCase<IdsQueryBuilder>
 
     @Override
     protected IdsQueryBuilder doCreateTestQueryBuilder() {
-        final String type;
-        if (randomBoolean()) {
-            if (frequently()) {
-                type = "_doc";
-            } else {
-                type = randomAlphaOfLengthBetween(1, 10);
-            }
-        } else if (randomBoolean()) {
-            type = MetaData.ALL;
-        } else {
-            type = null;
-        }
         int numberOfIds = randomIntBetween(0, 10);
         String[] ids = new String[numberOfIds];
         for (int i = 0; i < numberOfIds; i++) {
             ids[i] = randomAlphaOfLengthBetween(1, 10);
         }
-        IdsQueryBuilder query;
-        if (type != null && randomBoolean()) {
-            query = new IdsQueryBuilder().types(type);
-            query.addIds(ids);
-        } else {
-            query = new IdsQueryBuilder();
-            query.addIds(ids);
-        }
+        IdsQueryBuilder query = new IdsQueryBuilder();
+        query.addIds(ids);
         return query;
     }
 
     @Override
     protected void doAssertLuceneQuery(IdsQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
-        boolean allTypes = queryBuilder.types().length == 0 ||
-                queryBuilder.types().length == 1 && "_all".equals(queryBuilder.types()[0]);
-        if (queryBuilder.ids().size() == 0
-                // no types
-                || context.getQueryShardContext().fieldMapper(IdFieldMapper.NAME) == null
-                // there are types, but disjoint from the query
-                || (allTypes == false &&
-                    Arrays.asList(queryBuilder.types()).indexOf(context.mapperService().documentMapper().type()) == -1)) {
+        if (queryBuilder.ids().size() == 0) {
             assertThat(query, instanceOf(MatchNoDocsQuery.class));
         } else {
             assertThat(query, instanceOf(TermInSetQuery.class));
@@ -86,11 +58,8 @@ protected void doAssertLuceneQuery(IdsQueryBuilder queryBuilder, Query query, Se
     }
 
     public void testIllegalArguments() {
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new IdsQueryBuilder().types((String[]) null));
-        assertEquals("[ids] types cannot be null", e.getMessage());
-
         IdsQueryBuilder idsQueryBuilder = new IdsQueryBuilder();
-        e = expectThrows(IllegalArgumentException.class, () -> idsQueryBuilder.addIds((String[]) null));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> idsQueryBuilder.addIds((String[]) null));
         assertEquals("[ids] ids cannot be null", e.getMessage());
     }
 
@@ -105,7 +74,6 @@ public void testFromJson() throws IOException {
         String json =
                 "{\n" +
                 "  \"ids\" : {\n" +
-                "    \"type\" : [ \"my_type\" ],\n" +
                 "    \"values\" : [ \"1\", \"100\", \"4\" ],\n" +
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
@@ -113,56 +81,23 @@ public void testFromJson() throws IOException {
         IdsQueryBuilder parsed = (IdsQueryBuilder) parseQuery(json);
         checkGeneratedJson(json, parsed);
         assertThat(parsed.ids(), contains("1","100","4"));
-        assertEquals(json, "my_type", parsed.types()[0]);
 
         // check that type that is not an array and also ids that are numbers are parsed
         json =
                 "{\n" +
                 "  \"ids\" : {\n" +
-                "    \"type\" : \"my_type\",\n" +
                 "    \"values\" : [ 1, 100, 4 ],\n" +
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
         parsed = (IdsQueryBuilder) parseQuery(json);
         assertThat(parsed.ids(), contains("1","100","4"));
-        assertEquals(json, "my_type", parsed.types()[0]);
-
-        // check with empty type array
-        json =
-                "{\n" +
-                "  \"ids\" : {\n" +
-                "    \"type\" : [ ],\n" +
-                "    \"values\" : [ \"1\", \"100\", \"4\" ],\n" +
-                "    \"boost\" : 1.0\n" +
-                "  }\n" +
-                "}";
-        parsed = (IdsQueryBuilder) parseQuery(json);
-        assertThat(parsed.ids(), contains("1","100","4"));
-        assertEquals(json, 0, parsed.types().length);
-
-        // check without type
-        json =
-                "{\n" +
-                "  \"ids\" : {\n" +
-                "    \"values\" : [ \"1\", \"100\", \"4\" ],\n" +
-                "    \"boost\" : 1.0\n" +
-                "  }\n" +
-                "}";
-        parsed = (IdsQueryBuilder) parseQuery(json);
-        assertThat(parsed.ids(), contains("1","100","4"));
-        assertEquals(json, 0, parsed.types().length);
     }
 
     @Override
     protected QueryBuilder parseQuery(XContentParser parser) throws IOException {
         QueryBuilder query = super.parseQuery(parser);
         assertThat(query, instanceOf(IdsQueryBuilder.class));
-
-        IdsQueryBuilder idsQuery = (IdsQueryBuilder) query;
-        if (idsQuery.types().length > 0) {
-            assertWarnings(IdsQueryBuilder.TYPES_DEPRECATION_MESSAGE);
-        }
-        return query;
+        return (IdsQueryBuilder) query;
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
index db32c251fd3f4..8c6f3a75b7ae2 100644
--- a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
@@ -67,7 +67,7 @@ public class InnerHitBuilderTests extends ESTestCase {
 
     @BeforeClass
     public static void init() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
         xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
diff --git a/server/src/test/java/org/elasticsearch/index/query/IntervalBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/IntervalBuilderTests.java
index 15ec8af0af2c5..69464edb51332 100644
--- a/server/src/test/java/org/elasticsearch/index/query/IntervalBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/IntervalBuilderTests.java
@@ -110,6 +110,12 @@ public void testPhraseWithStopword() throws IOException {
 
     }
 
+    public void testEmptyTokenStream() throws IOException {
+        CannedTokenStream ts = new CannedTokenStream();
+        IntervalsSource source = BUILDER.analyzeText(new CachingTokenFilter(ts), 0, true);
+        assertSame(IntervalBuilder.NO_INTERVALS, source);
+    }
+
     public void testSimpleSynonyms() throws IOException {
 
         CannedTokenStream ts = new CannedTokenStream(
diff --git a/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java
index 03e64eb7ac415..c480b52c6dc00 100644
--- a/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/IntervalQueryBuilderTests.java
@@ -59,6 +59,7 @@ protected IntervalQueryBuilder doCreateTestQueryBuilder() {
 
     private static final String MASKED_FIELD = "masked_field";
     private static final String NO_POSITIONS_FIELD = "no_positions_field";
+    private static final String PREFIXED_FIELD = "prefixed_field";
 
     @Override
     protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
@@ -70,6 +71,10 @@ protected void initializeAdditionalMappings(MapperService mapperService) throws
             .field("type", "text")
             .field("index_options", "freqs")
             .endObject()
+            .startObject(PREFIXED_FIELD)
+            .field("type", "text")
+            .startObject("index_prefixes").endObject()
+            .endObject()
             .endObject().endObject().endObject();
 
         mapperService.merge("_doc",
@@ -364,7 +369,7 @@ public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryTyp
 
         QueryShardContext baseContext = createShardContext();
         QueryShardContext context = new QueryShardContext(baseContext.getShardId(), baseContext.getIndexSettings(),
-            null, null, baseContext.getMapperService(), null,
+            null, null, null, baseContext.getMapperService(), null,
             scriptService,
             null, null, null, null, null, null);
 
@@ -385,4 +390,107 @@ public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryTyp
 
     }
 
+    public void testPrefixes() throws IOException {
+
+        String json = "{ \"intervals\" : { \"" + STRING_FIELD_NAME + "\": { " +
+            "\"prefix\" : { \"prefix\" : \"term\" } } } }";
+        IntervalQueryBuilder builder = (IntervalQueryBuilder) parseQuery(json);
+        Query expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.prefix("term"));
+        assertEquals(expected, builder.toQuery(createShardContext()));
+
+        String no_positions_json = "{ \"intervals\" : { \"" + NO_POSITIONS_FIELD + "\": { " +
+            "\"prefix\" : { \"prefix\" : \"term\" } } } }";
+        expectThrows(IllegalArgumentException.class, () -> {
+            IntervalQueryBuilder builder1 = (IntervalQueryBuilder) parseQuery(no_positions_json);
+            builder1.toQuery(createShardContext());
+            });
+
+        String no_positions_fixed_field_json = "{ \"intervals\" : { \"" + STRING_FIELD_NAME + "\": { " +
+            "\"prefix\" : { \"prefix\" : \"term\", \"use_field\" : \"" + NO_POSITIONS_FIELD + "\" } } } }";
+        expectThrows(IllegalArgumentException.class, () -> {
+            IntervalQueryBuilder builder1 = (IntervalQueryBuilder) parseQuery(no_positions_fixed_field_json);
+            builder1.toQuery(createShardContext());
+        });
+
+        String prefix_json = "{ \"intervals\" : { \"" + PREFIXED_FIELD + "\": { " +
+            "\"prefix\" : { \"prefix\" : \"term\" } } } }";
+        builder = (IntervalQueryBuilder) parseQuery(prefix_json);
+        expected = new IntervalQuery(PREFIXED_FIELD, Intervals.fixField(PREFIXED_FIELD + "._index_prefix", Intervals.term("term")));
+        assertEquals(expected, builder.toQuery(createShardContext()));
+
+        String short_prefix_json = "{ \"intervals\" : { \"" + PREFIXED_FIELD + "\": { " +
+            "\"prefix\" : { \"prefix\" : \"t\" } } } }";
+        builder = (IntervalQueryBuilder) parseQuery(short_prefix_json);
+        expected = new IntervalQuery(PREFIXED_FIELD, Intervals.or(
+            Intervals.fixField(PREFIXED_FIELD + "._index_prefix", Intervals.wildcard("t?")),
+            Intervals.term("t")));
+        assertEquals(expected, builder.toQuery(createShardContext()));
+
+        String fix_field_prefix_json =  "{ \"intervals\" : { \"" + STRING_FIELD_NAME + "\": { " +
+            "\"prefix\" : { \"prefix\" : \"term\", \"use_field\" : \"" + PREFIXED_FIELD + "\" } } } }";
+        builder = (IntervalQueryBuilder) parseQuery(fix_field_prefix_json);
+        // This looks weird, but it's fine, because the innermost fixField wins
+        expected = new IntervalQuery(STRING_FIELD_NAME,
+            Intervals.fixField(PREFIXED_FIELD, Intervals.fixField(PREFIXED_FIELD + "._index_prefix", Intervals.term("term"))));
+        assertEquals(expected, builder.toQuery(createShardContext()));
+
+        String keyword_json = "{ \"intervals\" : { \"" + PREFIXED_FIELD + "\": { " +
+            "\"prefix\" : { \"prefix\" : \"Term\", \"analyzer\" : \"keyword\" } } } }";
+        builder = (IntervalQueryBuilder) parseQuery(keyword_json);
+        expected = new IntervalQuery(PREFIXED_FIELD, Intervals.fixField(PREFIXED_FIELD + "._index_prefix", Intervals.term("Term")));
+        assertEquals(expected, builder.toQuery(createShardContext()));
+
+        String keyword_fix_field_json = "{ \"intervals\" : { \"" + STRING_FIELD_NAME + "\": { " +
+            "\"prefix\" : { \"prefix\" : \"Term\", \"analyzer\" : \"keyword\", \"use_field\" : \"" + PREFIXED_FIELD + "\" } } } }";
+        builder = (IntervalQueryBuilder) parseQuery(keyword_fix_field_json);
+        expected = new IntervalQuery(STRING_FIELD_NAME,
+            Intervals.fixField(PREFIXED_FIELD, Intervals.fixField(PREFIXED_FIELD + "._index_prefix", Intervals.term("Term"))));
+        assertEquals(expected, builder.toQuery(createShardContext()));
+    }
+
+    public void testWildcard() throws IOException {
+
+        String json = "{ \"intervals\" : { \"" + STRING_FIELD_NAME + "\": { " +
+            "\"wildcard\" : { \"pattern\" : \"Te?m\" } } } }";
+
+        IntervalQueryBuilder builder = (IntervalQueryBuilder) parseQuery(json);
+        Query expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.wildcard("te?m"));
+        assertEquals(expected, builder.toQuery(createShardContext()));
+
+        String no_positions_json = "{ \"intervals\" : { \"" + NO_POSITIONS_FIELD + "\": { " +
+            "\"wildcard\" : { \"pattern\" : \"term\" } } } }";
+        expectThrows(IllegalArgumentException.class, () -> {
+            IntervalQueryBuilder builder1 = (IntervalQueryBuilder) parseQuery(no_positions_json);
+            builder1.toQuery(createShardContext());
+        });
+
+        String keyword_json = "{ \"intervals\" : { \"" + STRING_FIELD_NAME + "\": { " +
+            "\"wildcard\" : { \"pattern\" : \"Te?m\", \"analyzer\" : \"keyword\" } } } }";
+
+        builder = (IntervalQueryBuilder) parseQuery(keyword_json);
+        expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.wildcard("Te?m"));
+        assertEquals(expected, builder.toQuery(createShardContext()));
+
+        String fixed_field_json = "{ \"intervals\" : { \"" + STRING_FIELD_NAME + "\": { " +
+            "\"wildcard\" : { \"pattern\" : \"Te?m\", \"use_field\" : \"masked_field\" } } } }";
+
+        builder = (IntervalQueryBuilder) parseQuery(fixed_field_json);
+        expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.fixField(MASKED_FIELD, Intervals.wildcard("te?m")));
+        assertEquals(expected, builder.toQuery(createShardContext()));
+
+        String fixed_field_json_no_positions = "{ \"intervals\" : { \"" + STRING_FIELD_NAME + "\": { " +
+            "\"wildcard\" : { \"pattern\" : \"Te?m\", \"use_field\" : \"" + NO_POSITIONS_FIELD + "\" } } } }";
+        expectThrows(IllegalArgumentException.class, () -> {
+            IntervalQueryBuilder builder1 = (IntervalQueryBuilder) parseQuery(fixed_field_json_no_positions);
+            builder1.toQuery(createShardContext());
+        });
+
+        String fixed_field_analyzer_json = "{ \"intervals\" : { \"" + STRING_FIELD_NAME + "\": { " +
+            "\"wildcard\" : { \"pattern\" : \"Te?m\", \"use_field\" : \"masked_field\", \"analyzer\" : \"keyword\" } } } }";
+
+        builder = (IntervalQueryBuilder) parseQuery(fixed_field_analyzer_json);
+        expected = new IntervalQuery(STRING_FIELD_NAME, Intervals.fixField(MASKED_FIELD, Intervals.wildcard("Te?m")));
+        assertEquals(expected, builder.toQuery(createShardContext()));
+    }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
index a7aad3dbc3e42..0b870127b9794 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
@@ -23,7 +23,6 @@
 import org.apache.lucene.analysis.CannedBinaryTokenStream;
 import org.apache.lucene.analysis.MockSynonymAnalyzer;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.ExtendedCommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.FuzzyQuery;
@@ -46,6 +45,7 @@
 import org.elasticsearch.index.search.MatchQuery.Type;
 import org.elasticsearch.index.search.MatchQuery.ZeroTermsQuery;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matcher;
 
 import java.io.IOException;
@@ -55,19 +55,13 @@
 import java.util.Locale;
 import java.util.Map;
 
-import static java.util.Collections.singletonList;
 import static org.hamcrest.CoreMatchers.either;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class MatchQueryBuilderTests extends FullTextQueryTestCase<MatchQueryBuilder> {
-    @Override
-    protected boolean isCacheable(MatchQueryBuilder queryBuilder) {
-        return queryBuilder.fuzziness() != null
-                ||  isCacheable(singletonList(queryBuilder.fieldName()), queryBuilder.value().toString());
-    }
+public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuilder> {
 
     @Override
     protected MatchQueryBuilder doCreateTestQueryBuilder() {
@@ -124,10 +118,6 @@ protected MatchQueryBuilder doCreateTestQueryBuilder() {
             matchQuery.zeroTermsQuery(randomFrom(ZeroTermsQuery.ALL, ZeroTermsQuery.NONE));
         }
 
-        if (randomBoolean()) {
-            matchQuery.cutoffFrequency((float) 10 / randomIntBetween(1, 100));
-        }
-
         if (randomBoolean()) {
             matchQuery.autoGenerateSynonymsPhraseQuery(randomBoolean());
         }
@@ -160,7 +150,8 @@ protected void doAssertLuceneQuery(MatchQueryBuilder queryBuilder, Query query,
         MappedFieldType fieldType = context.fieldMapper(queryBuilder.fieldName());
         if (query instanceof TermQuery && fieldType != null) {
             String queryValue = queryBuilder.value().toString();
-            if (queryBuilder.analyzer() == null || queryBuilder.analyzer().equals("simple")) {
+            if (isTextField(queryBuilder.fieldName())
+                  && (queryBuilder.analyzer() == null || queryBuilder.analyzer().equals("simple"))) {
                 queryValue = queryValue.toLowerCase(Locale.ROOT);
             }
             Query expectedTermQuery = fieldType.termQuery(queryValue, context);
@@ -185,18 +176,6 @@ protected void doAssertLuceneQuery(MatchQueryBuilder queryBuilder, Query query,
             }
         }
 
-        if (query instanceof ExtendedCommonTermsQuery) {
-            assertTrue(queryBuilder.cutoffFrequency() != null);
-            ExtendedCommonTermsQuery ectq = (ExtendedCommonTermsQuery) query;
-            List<Term> terms = ectq.getTerms();
-            if (!terms.isEmpty()) {
-                Term term = terms.iterator().next();
-                String expectedFieldName = expectedFieldName(queryBuilder.fieldName());
-                assertThat(term.field(), equalTo(expectedFieldName));
-            }
-            assertEquals(queryBuilder.cutoffFrequency(), ectq.getMaxTermFrequency(), Float.MIN_VALUE);
-        }
-
         if (query instanceof FuzzyQuery) {
             assertTrue(queryBuilder.fuzziness() != null);
             FuzzyQuery fuzzyQuery = (FuzzyQuery) query;
@@ -478,7 +457,7 @@ public void testMaxBooleanClause() {
         query.setAnalyzer(new MockGraphAnalyzer(createGiantGraphMultiTerms()));
         expectThrows(BooleanQuery.TooManyClauses.class, () -> query.parse(Type.PHRASE, STRING_FIELD_NAME, ""));
     }
-    
+
     private static class MockGraphAnalyzer extends Analyzer {
 
         CannedBinaryTokenStream tokenStream;
@@ -530,4 +509,22 @@ private static CannedBinaryTokenStream.BinaryToken[] createGiantGraphMultiTerms(
         }
         return tokens.toArray(new CannedBinaryTokenStream.BinaryToken[0]);
     }
+
+    /**
+     * "now" on date fields should make the query non-cachable.
+     */
+    public void testCachingStrategiesWithNow() throws IOException {
+        // if we hit a date field with "now", this should diable cachability
+        MatchQueryBuilder queryBuilder = new MatchQueryBuilder(DATE_FIELD_NAME, "now");
+        QueryShardContext context = createShardContext();
+        assert context.isCacheable();
+        /*
+         * We use a private rewrite context here since we want the most realistic way of asserting that we are cacheable or not. We do it
+         * this way in SearchService where we first rewrite the query with a private context, then reset the context and then build the
+         * actual lucene query
+         */
+        QueryBuilder rewritten = rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(rewritten.toQuery(context));
+        assertFalse("query should not be cacheable: " + queryBuilder.toString(), context.isCacheable());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
index 56e7a32347265..98c2a162faf06 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
@@ -96,18 +96,7 @@ private static String[] randomStringFields() {
     private Item generateRandomItem() {
         String index = randomBoolean() ? getIndex().getName() : null;
         // indexed item or artificial document
-        Item item;
-
-        if (randomBoolean()) {
-            item = randomBoolean()
-                ? new Item(index, randomAlphaOfLength(10))
-                : new Item(index, randomArtificialDoc());
-        } else {
-            String type = "doc";
-            item = randomBoolean()
-                ? new Item(index, type, randomAlphaOfLength(10))
-                : new Item(index, type, randomArtificialDoc());
-        }
+        Item item = randomBoolean() ? new Item(index, randomAlphaOfLength(10)) : new Item(index, randomArtificialDoc());
 
         // if no field is specified MLT uses all mapped fields for this item
         if (randomBoolean()) {
@@ -235,7 +224,7 @@ protected MultiTermVectorsResponse executeMultiTermVectors(MultiTermVectorsReque
             MultiTermVectorsItemResponse[] responses = new MultiTermVectorsItemResponse[mtvRequest.size()];
             int i = 0;
             for (TermVectorsRequest request : mtvRequest) {
-                TermVectorsResponse response = new TermVectorsResponse(request.index(), request.type(), request.id());
+                TermVectorsResponse response = new TermVectorsResponse(request.index(), request.id());
                 response.setExists(true);
                 Fields generatedFields;
                 if (request.doc() != null) {
@@ -374,9 +363,34 @@ public void testItemFromXContent() throws IOException {
         assertEquals(expectedItem, newItem);
     }
 
+    /**
+     * Check that this query is generally not cacheable, except when we fetch 0 items
+     */
     @Override
-    protected boolean isCacheable(MoreLikeThisQueryBuilder queryBuilder) {
-        return queryBuilder.likeItems().length == 0; // items are always fetched
+    public void testCacheability() throws IOException {
+        MoreLikeThisQueryBuilder queryBuilder = createTestQueryBuilder();
+        boolean isCacheable = queryBuilder.likeItems().length == 0; // items are always fetched
+        QueryShardContext context = createShardContext();
+        QueryBuilder rewriteQuery = rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(rewriteQuery.toQuery(context));
+        assertEquals("query should " + (isCacheable ? "" : "not") + " be cacheable: " + queryBuilder.toString(), isCacheable,
+                context.isCacheable());
+
+        // specifically trigger case where query is cacheable
+        queryBuilder = new MoreLikeThisQueryBuilder(randomStringFields(), new String[] {"some text"}, null);
+        context = createShardContext();
+        rewriteQuery(queryBuilder, new QueryShardContext(context));
+        rewriteQuery = rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(rewriteQuery.toQuery(context));
+        assertTrue("query should be cacheable: " + queryBuilder.toString(), context.isCacheable());
+
+        // specifically trigger case where query is not cacheable
+        queryBuilder = new MoreLikeThisQueryBuilder(randomStringFields(), null, new Item[] { new Item("foo", "1") });
+        context = createShardContext();
+        rewriteQuery(queryBuilder, new QueryShardContext(context));
+        rewriteQuery = rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(rewriteQuery.toQuery(context));
+        assertFalse("query should be cacheable: " + queryBuilder.toString(), context.isCacheable());
     }
 
     public void testFromJson() throws IOException {
@@ -386,11 +400,9 @@ public void testFromJson() throws IOException {
                 "    \"fields\" : [ \"title\", \"description\" ],\n" +
                 "    \"like\" : [ \"and potentially some more text here as well\", {\n" +
                 "      \"_index\" : \"imdb\",\n" +
-                "      \"_type\" : \"movies\",\n" +
                 "      \"_id\" : \"1\"\n" +
                 "    }, {\n" +
                 "      \"_index\" : \"imdb\",\n" +
-                "      \"_type\" : \"movies\",\n" +
                 "      \"_id\" : \"2\"\n" +
                 "    } ],\n" +
                 "    \"max_query_terms\" : 12,\n" +
@@ -418,11 +430,6 @@ public void testFromJson() throws IOException {
     protected QueryBuilder parseQuery(XContentParser parser) throws IOException {
         QueryBuilder query = super.parseQuery(parser);
         assertThat(query, instanceOf(MoreLikeThisQueryBuilder.class));
-
-        MoreLikeThisQueryBuilder mltQuery = (MoreLikeThisQueryBuilder) query;
-        if (mltQuery.isTypeless() == false) {
-            assertWarnings(MoreLikeThisQueryBuilder.TYPES_DEPRECATION_MESSAGE);
-        }
         return query;
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
index 6590a5609353a..3bbcca56beca7 100644
--- a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.index.Term;
-import org.apache.lucene.queries.ExtendedCommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
@@ -42,6 +41,7 @@
 import org.elasticsearch.index.query.MultiMatchQueryBuilder.Type;
 import org.elasticsearch.index.search.MatchQuery;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -59,16 +59,10 @@
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.collection.IsCollectionWithSize.hasSize;
 
-public class MultiMatchQueryBuilderTests extends FullTextQueryTestCase<MultiMatchQueryBuilder> {
+public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatchQueryBuilder> {
     private static final String MISSING_WILDCARD_FIELD_NAME = "missing_*";
     private static final String MISSING_FIELD_NAME = "missing";
 
-    @Override
-    protected boolean isCacheable(MultiMatchQueryBuilder queryBuilder) {
-        return queryBuilder.fuzziness() != null
-                || isCacheable(queryBuilder.fields().keySet(), queryBuilder.value().toString());
-    }
-
     @Override
     protected MultiMatchQueryBuilder doCreateTestQueryBuilder() {
         String fieldName = randomFrom(STRING_FIELD_NAME, INT_FIELD_NAME, DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME,
@@ -134,9 +128,6 @@ protected MultiMatchQueryBuilder doCreateTestQueryBuilder() {
         if (randomBoolean()) {
             query.tieBreaker(randomFloat());
         }
-        if (randomBoolean() && query.type() != Type.BOOL_PREFIX) {
-            query.cutoffFrequency((float) 10 / randomIntBetween(1, 100));
-        }
         if (randomBoolean()) {
             query.zeroTermsQuery(randomFrom(MatchQuery.ZeroTermsQuery.NONE, MatchQuery.ZeroTermsQuery.ALL));
         }
@@ -174,7 +165,6 @@ protected void doAssertLuceneQuery(MultiMatchQueryBuilder queryBuilder, Query qu
             instanceOf(FuzzyQuery.class),
             instanceOf(MultiPhrasePrefixQuery.class),
             instanceOf(MatchAllDocsQuery.class),
-            instanceOf(ExtendedCommonTermsQuery.class),
             instanceOf(MatchNoDocsQuery.class),
             instanceOf(PhraseQuery.class),
             instanceOf(PointRangeQuery.class),
@@ -244,10 +234,9 @@ public void testToQueryFieldsWildcard() throws Exception {
         assertThat(query, instanceOf(DisjunctionMaxQuery.class));
         DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query;
         assertThat(dQuery.getTieBreakerMultiplier(), equalTo(1.0f));
-        assertThat(dQuery.getDisjuncts().size(), equalTo(3));
+        assertThat(dQuery.getDisjuncts().size(), equalTo(2));
         assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 0).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test")));
         assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 1).getTerm(), equalTo(new Term(STRING_FIELD_NAME_2, "test")));
-        assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 2).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "test")));
     }
 
     public void testToQueryFieldMissing() throws Exception {
@@ -269,23 +258,6 @@ public void testToQueryBooleanPrefixSingleField() throws IOException {
     }
 
     public void testToQueryBooleanPrefixMultipleFields() throws IOException {
-        {
-            final MultiMatchQueryBuilder builder = new MultiMatchQueryBuilder("foo bar", STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME);
-            builder.type(Type.BOOL_PREFIX);
-            final Query query = builder.toQuery(createShardContext());
-            assertThat(query, instanceOf(DisjunctionMaxQuery.class));
-            final DisjunctionMaxQuery disMaxQuery = (DisjunctionMaxQuery) query;
-            assertThat(disMaxQuery.getDisjuncts(), hasSize(2));
-            for (Query disjunctQuery : disMaxQuery.getDisjuncts()) {
-                assertThat(disjunctQuery, instanceOf(BooleanQuery.class));
-                final BooleanQuery booleanQuery = (BooleanQuery) disjunctQuery;
-                assertThat(booleanQuery.clauses(), hasSize(2));
-                assertThat(assertBooleanSubQuery(booleanQuery, TermQuery.class, 0).getTerm(), equalTo(new Term(STRING_FIELD_NAME, "foo")));
-                assertThat(assertBooleanSubQuery(booleanQuery, PrefixQuery.class, 1).getPrefix(),
-                    equalTo(new Term(STRING_FIELD_NAME, "bar")));
-            }
-        }
-
         {
             // STRING_FIELD_NAME_2 is a keyword field
             final MultiMatchQueryBuilder queryBuilder = new MultiMatchQueryBuilder("foo bar", STRING_FIELD_NAME, STRING_FIELD_NAME_2);
@@ -454,9 +426,9 @@ public void testDefaultField() throws Exception {
             query = qb.toQuery(context);
             expected = new DisjunctionMaxQuery(
                 Arrays.asList(
+                    new MatchNoDocsQuery("failed [mapped_int] query, caused by number_format_exception:[For input string: \"hello\"]"),
                     new TermQuery(new Term(STRING_FIELD_NAME, "hello")),
-                    new BoostQuery(new TermQuery(new Term(STRING_FIELD_NAME_2, "hello")), 5.0f),
-                    new MatchNoDocsQuery("failed [mapped_int] query, caused by number_format_exception:[For input string: \"hello\"]")
+                    new BoostQuery(new TermQuery(new Term(STRING_FIELD_NAME_2, "hello")), 5.0f)
                 ), 0.0f
             );
             assertEquals(expected, query);
@@ -574,8 +546,26 @@ private void assertQueryWithAllFieldsWildcard(Query query) {
                 noMatchNoDocsQueries++;
             }
         }
-        assertEquals(11, noMatchNoDocsQueries);
+        assertEquals(9, noMatchNoDocsQueries);
         assertThat(disjunctionMaxQuery.getDisjuncts(), hasItems(new TermQuery(new Term(STRING_FIELD_NAME, "hello")),
             new TermQuery(new Term(STRING_FIELD_NAME_2, "hello"))));
     }
+
+    /**
+     * "now" on date fields should make the query non-cachable.
+     */
+    public void testCachingStrategiesWithNow() throws IOException {
+        // if we hit a date field with "now", this should diable cachability
+        MultiMatchQueryBuilder queryBuilder = new MultiMatchQueryBuilder("now", DATE_FIELD_NAME);
+        QueryShardContext context = createShardContext();
+        assert context.isCacheable();
+        /*
+         * We use a private rewrite context here since we want the most realistic way of asserting that we are cacheable or not. We do it
+         * this way in SearchService where we first rewrite the query with a private context, then reset the context and then build the
+         * actual lucene query
+         */
+        QueryBuilder rewritten = rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(rewritten.toQuery(context));
+        assertFalse("query should not be cacheable: " + queryBuilder.toString(), context.isCacheable());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
index b27b9e26ab7dc..6305f6aff92b5 100644
--- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
@@ -128,7 +128,7 @@ public static QueryShardContext createQueryShardContext(String indexUuid, String
         final long nowInMillis = randomNonNegativeLong();
 
         return new QueryShardContext(
-            0, indexSettings, null, (mappedFieldType, idxName) ->
+            0, indexSettings, null, null, (mappedFieldType, idxName) ->
             mappedFieldType.fielddataBuilder(idxName).build(indexSettings, mappedFieldType, null, null, null)
             , mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), null, null,
             () -> nowInMillis, clusterAlias);
diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
index ee4e0f9540451..3e9da2f2e5099 100644
--- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
@@ -61,6 +61,7 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.search.QueryStringQueryParser;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.CoreMatchers;
 import org.hamcrest.Matchers;
 
@@ -71,6 +72,7 @@
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -83,12 +85,7 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.instanceOf;
 
-public class QueryStringQueryBuilderTests extends FullTextQueryTestCase<QueryStringQueryBuilder> {
-    @Override
-    protected boolean isCacheable(QueryStringQueryBuilder queryBuilder) {
-        return queryBuilder.fuzziness() != null
-                || isCacheable(queryBuilder.fields().keySet(), queryBuilder.queryString());
-    }
+public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStringQueryBuilder> {
 
     @Override
     protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
@@ -108,8 +105,11 @@ protected QueryStringQueryBuilder doCreateTestQueryBuilder() {
         int numTerms = randomIntBetween(0, 5);
         String query = "";
         for (int i = 0; i < numTerms; i++) {
-            //min length 4 makes sure that the text is not an operator (AND/OR) so toQuery won't break
-            query += (randomBoolean() ? STRING_FIELD_NAME + ":" : "") + randomAlphaOfLengthBetween(4, 10) + " ";
+            // min length 4 makes sure that the text is not an operator (AND/OR) so toQuery won't break
+            // also avoid "now" since we might hit dqte fields later and this complicates caching checks
+            String term = randomValueOtherThanMany(s -> s.toLowerCase(Locale.ROOT).contains("now"),
+                    () -> randomAlphaOfLengthBetween(4, 10));
+            query += (randomBoolean() ? STRING_FIELD_NAME + ":" : "") + term + " ";
         }
         QueryStringQueryBuilder queryStringQueryBuilder = new QueryStringQueryBuilder(query);
         if (randomBoolean()) {
@@ -518,13 +518,11 @@ public void testToQueryFieldsWildcard() throws Exception {
         Query query = queryStringQuery("test").field("mapped_str*").toQuery(createShardContext());
         assertThat(query, instanceOf(DisjunctionMaxQuery.class));
         DisjunctionMaxQuery dQuery = (DisjunctionMaxQuery) query;
-        assertThat(dQuery.getDisjuncts().size(), equalTo(3));
+        assertThat(dQuery.getDisjuncts().size(), equalTo(2));
         assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 0).getTerm(),
             equalTo(new Term(STRING_FIELD_NAME, "test")));
         assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 1).getTerm(),
             equalTo(new Term(STRING_FIELD_NAME_2, "test")));
-        assertThat(assertDisjunctionSubQuery(query, TermQuery.class, 2).getTerm(),
-            equalTo(new Term(STRING_FIELD_NAME, "test")));
     }
 
     public void testToQueryDisMaxQuery() throws Exception {
@@ -767,50 +765,40 @@ public void testToQueryRegExpQueryMaxDeterminizedStatesParsing() throws Exceptio
         assertThat(e.getMessage(), containsString("would result in more than 10 states"));
     }
 
-    /**
-     * Validates that {@code max_determinized_states} can be parsed and lowers the allowed number of determinized states.
-     */
-    public void testEnabledPositionIncrements() throws Exception {
-
-        XContentBuilder builder = JsonXContent.contentBuilder();
-        builder.startObject(); {
-            builder.startObject("query_string"); {
-                builder.field("query", "text");
-                builder.field("default_field", STRING_FIELD_NAME);
-                builder.field("enable_position_increments", false);
-            }
-            builder.endObject();
-        }
-        builder.endObject();
-
-        QueryStringQueryBuilder queryBuilder = (QueryStringQueryBuilder) parseInnerQueryBuilder(createParser(builder));
-        assertFalse(queryBuilder.enablePositionIncrements());
-    }
-
     public void testToQueryFuzzyQueryAutoFuziness() throws Exception {
-        int length = randomIntBetween(1, 10);
-        StringBuilder queryString = new StringBuilder();
-        for (int i = 0; i < length; i++) {
-            queryString.append("a");
-        }
-        queryString.append("~");
-
-        int expectedEdits;
-        if (length <= 2) {
-            expectedEdits = 0;
-        } else if (3 <= length && length <= 5) {
-            expectedEdits = 1;
-        } else {
-            expectedEdits = 2;
+        for (int i = 0; i < 3; i++) {
+            final int len;
+            final int expectedEdits;
+            switch (i) {
+                case 0:
+                    len = randomIntBetween(1, 2);
+                    expectedEdits = 0;
+                    break;
+
+                case 1:
+                    len = randomIntBetween(3, 5);
+                    expectedEdits = 1;
+                    break;
+
+                default:
+                    len = randomIntBetween(6, 20);
+                    expectedEdits = 2;
+                    break;
+            }
+            char[] bytes = new char[len];
+            Arrays.fill(bytes, 'a');
+            String queryString = new String(bytes);
+            for (int j = 0; j < 2; j++) {
+                Query query = queryStringQuery(queryString + (j == 0 ? "~" : "~auto"))
+                    .defaultField(STRING_FIELD_NAME)
+                    .fuzziness(Fuzziness.AUTO)
+                    .toQuery(createShardContext());
+                assertThat(query, instanceOf(FuzzyQuery.class));
+                FuzzyQuery fuzzyQuery = (FuzzyQuery) query;
+                assertEquals(expectedEdits, fuzzyQuery.getMaxEdits());
+            }
         }
-
-        Query query = queryStringQuery(queryString.toString()).defaultField(STRING_FIELD_NAME).fuzziness(Fuzziness.AUTO)
-            .toQuery(createShardContext());
-        assertThat(query, instanceOf(FuzzyQuery.class));
-        FuzzyQuery fuzzyQuery = (FuzzyQuery) query;
-        assertEquals(expectedEdits, fuzzyQuery.getMaxEdits());
     }
-
     public void testFuzzyNumeric() throws Exception {
         QueryStringQueryBuilder query = queryStringQuery("12~1.0").defaultField(INT_FIELD_NAME);
         QueryShardContext context = createShardContext();
@@ -1430,6 +1418,19 @@ public void testWithStopWords() throws Exception {
         assertEquals(expected, query);
     }
 
+    public void testEnablePositionIncrement() throws Exception {
+        Query query = new QueryStringQueryBuilder("\"quick the fox\"")
+            .field(STRING_FIELD_NAME)
+            .analyzer("stop")
+            .enablePositionIncrements(false)
+            .toQuery(createShardContext());
+        PhraseQuery expected = new PhraseQuery.Builder()
+            .add(new Term(STRING_FIELD_NAME, "quick"))
+            .add(new Term(STRING_FIELD_NAME, "fox"))
+            .build();
+        assertEquals(expected, query);
+    }
+
     public void testWithPrefixStopWords() throws Exception {
         Query query = new QueryStringQueryBuilder("the* quick fox")
             .field(STRING_FIELD_NAME)
@@ -1538,6 +1539,19 @@ public void testNegativeFieldBoost() {
         assertThat(exc.getMessage(), CoreMatchers.containsString("negative [boost]"));
     }
 
+    public void testMergeBoosts() throws IOException {
+        Query query = new QueryStringQueryBuilder("first")
+            .type(MultiMatchQueryBuilder.Type.MOST_FIELDS)
+            .field(STRING_FIELD_NAME, 0.3f)
+            .field(STRING_FIELD_NAME.substring(0, STRING_FIELD_NAME.length()-2) + "*", 0.5f)
+            .toQuery(createShardContext());
+        List<Query> terms = new ArrayList<>();
+        terms.add(new BoostQuery(new TermQuery(new Term(STRING_FIELD_NAME, "first")), 0.075f));
+        terms.add(new BoostQuery(new TermQuery(new Term(STRING_FIELD_NAME_2, "first")), 0.5f));
+        Query expected = new DisjunctionMaxQuery(terms, 1.0f);
+        assertEquals(expected, query);
+    }
+
     private static IndexMetaData newIndexMeta(String name, Settings oldIndexSettings, Settings indexSettings) {
         Settings build = Settings.builder().put(oldIndexSettings)
             .put(indexSettings)
@@ -1554,8 +1568,43 @@ private void assertQueryWithAllFieldsWildcard(Query query) {
                 noMatchNoDocsQueries++;
             }
         }
-        assertEquals(11, noMatchNoDocsQueries);
+        assertEquals(9, noMatchNoDocsQueries);
         assertThat(disjunctionMaxQuery.getDisjuncts(), hasItems(new TermQuery(new Term(STRING_FIELD_NAME, "hello")),
             new TermQuery(new Term(STRING_FIELD_NAME_2, "hello"))));
     }
+
+    /**
+     * Query terms that contain "now" can trigger a query to not be cacheable.
+     * This test checks the search context cacheable flag is updated accordingly.
+     */
+    public void testCachingStrategiesWithNow() throws IOException {
+        // if we hit all fields, this should contain a date field and should diable cachability
+        String query = "now " + randomAlphaOfLengthBetween(4, 10);
+        QueryStringQueryBuilder queryStringQueryBuilder = new QueryStringQueryBuilder(query);
+        assertQueryCachability(queryStringQueryBuilder, false);
+
+        // if we hit a date field with "now", this should diable cachability
+        queryStringQueryBuilder = new QueryStringQueryBuilder("now");
+        queryStringQueryBuilder.field(DATE_FIELD_NAME);
+        assertQueryCachability(queryStringQueryBuilder, false);
+
+        // everything else is fine on all fields
+        query = randomFrom("NoW", "nOw", "NOW") + " " + randomAlphaOfLengthBetween(4, 10);
+        queryStringQueryBuilder = new QueryStringQueryBuilder(query);
+        assertQueryCachability(queryStringQueryBuilder, true);
+    }
+
+    private void assertQueryCachability(QueryStringQueryBuilder qb, boolean cachingExpected) throws IOException {
+        QueryShardContext context = createShardContext();
+        assert context.isCacheable();
+        /*
+         * We use a private rewrite context here since we want the most realistic way of asserting that we are cacheable or not. We do it
+         * this way in SearchService where we first rewrite the query with a private context, then reset the context and then build the
+         * actual lucene query
+         */
+        QueryBuilder rewritten = rewriteQuery(qb, new QueryShardContext(context));
+        assertNotNull(rewritten.toQuery(context));
+        assertEquals("query should " + (cachingExpected ? "" : "not") + " be cacheable: " + qb.toString(), cachingExpected,
+                context.isCacheable());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java b/server/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
index 04d2d2c347bbf..1609b6a511d22 100644
--- a/server/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
+++ b/server/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
@@ -21,7 +21,6 @@
 
 import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
-import org.elasticsearch.common.Strings;
 
 import java.util.Random;
 
@@ -50,7 +49,7 @@ public static QueryBuilder createQuery(Random r) {
             case 2:
                 // We make sure this query has no types to avoid deprecation warnings in the
                 // tests that use this method.
-                return new IdsQueryBuilderTests().createTestQueryBuilder().types(Strings.EMPTY_ARRAY);
+            return new IdsQueryBuilderTests().createTestQueryBuilder();
             case 3:
                 return createMultiTermQuery(r);
             default:
diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
index d270a8c7113b5..8c129d689a86e 100644
--- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
@@ -601,4 +601,25 @@ public void testTypeField() throws IOException {
         builder.doToQuery(createShardContext());
         assertWarnings(QueryShardContext.TYPES_DEPRECATION_MESSAGE);
     }
+
+    /**
+     * Range queries should generally be cacheable, at least the ones we create randomly.
+     * This test makes sure we also test the non-cacheable cases regularly.
+     */
+    @Override
+    public void testCacheability() throws IOException {
+        RangeQueryBuilder queryBuilder = createTestQueryBuilder();
+        QueryShardContext context = createShardContext();
+        QueryBuilder rewriteQuery = rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(rewriteQuery.toQuery(context));
+        assertTrue("query should be cacheable: " + queryBuilder.toString(), context.isCacheable());
+
+        // queries on date fields using "now" should not be cached
+        queryBuilder = new RangeQueryBuilder(randomFrom(DATE_FIELD_NAME, DATE_RANGE_FIELD_NAME, DATE_ALIAS_FIELD_NAME));
+        queryBuilder.to("now");
+        context = createShardContext();
+        rewriteQuery = rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(rewriteQuery.toQuery(context));
+        assertFalse("query should not be cacheable: " + queryBuilder.toString(), context.isCacheable());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java
index 573f60f34f7c0..95a2640266808 100644
--- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java
@@ -37,8 +37,8 @@ public class RangeQueryRewriteTests extends ESSingleNodeTestCase {
     public void testRewriteMissingField() throws Exception {
         IndexService indexService = createIndex("test");
         IndexReader reader = new MultiReader();
-        QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, indexService.mapperService(),
-                null, null, xContentRegistry(), writableRegistry(), null, reader, null, null);
+        QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, null,
+                indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), null, reader, null, null);
         RangeQueryBuilder range = new RangeQueryBuilder("foo");
         assertEquals(Relation.DISJOINT, range.getRelation(context));
     }
@@ -54,8 +54,8 @@ public void testRewriteMissingReader() throws Exception {
             .endObject().endObject());
         indexService.mapperService().merge("type",
                 new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE);
-        QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, indexService.mapperService(),
-                null, null, xContentRegistry(), writableRegistry(), null, null, null, null);
+        QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, null,
+                indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), null, null, null, null);
         RangeQueryBuilder range = new RangeQueryBuilder("foo");
         // can't make assumptions on a missing reader, so it must return INTERSECT
         assertEquals(Relation.INTERSECTS, range.getRelation(context));
@@ -73,8 +73,8 @@ public void testRewriteEmptyReader() throws Exception {
         indexService.mapperService().merge("type",
                 new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE);
         IndexReader reader = new MultiReader();
-        QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, indexService.mapperService(),
-                null, null, xContentRegistry(), writableRegistry(), null, reader, null, null);
+        QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, null,
+                indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), null, reader, null, null);
         RangeQueryBuilder range = new RangeQueryBuilder("foo");
         // no values -> DISJOINT
         assertEquals(Relation.DISJOINT, range.getRelation(context));
diff --git a/server/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java
index b0bbca3266bab..b3cac936d3ae2 100644
--- a/server/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java
@@ -116,8 +116,15 @@ protected Set<String> getObjectsHoldingArbitraryContent() {
         return Collections.singleton(Script.PARAMS_PARSE_FIELD.getPreferredName());
     }
 
+    /**
+     * Check that this query is generally not cacheable
+     */
     @Override
-    protected boolean isCacheable(ScriptQueryBuilder queryBuilder) {
-        return false;
+    public void testCacheability() throws IOException {
+        ScriptQueryBuilder queryBuilder = createTestQueryBuilder();
+        QueryShardContext context = createShardContext();
+        QueryBuilder rewriteQuery = rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(rewriteQuery.toQuery(context));
+        assertFalse("query should not be cacheable: " + queryBuilder.toString(), context.isCacheable());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/ScriptScoreQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/ScriptScoreQueryBuilderTests.java
index ad9af8c49c391..ad420e8bbdc33 100644
--- a/server/src/test/java/org/elasticsearch/index/query/ScriptScoreQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/ScriptScoreQueryBuilderTests.java
@@ -88,8 +88,15 @@ public void testIllegalArguments() {
         );
     }
 
+    /**
+     * Check that this query is generally not cacheable
+     */
     @Override
-    protected boolean isCacheable(ScriptScoreQueryBuilder queryBuilder) {
-        return false;
+    public void testCacheability() throws IOException {
+        ScriptScoreQueryBuilder queryBuilder = createTestQueryBuilder();
+        QueryShardContext context = createShardContext();
+        QueryBuilder rewriteQuery = rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(rewriteQuery.toQuery(context));
+        assertFalse("query should not be cacheable: " + queryBuilder.toString(), context.isCacheable());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
index ab479d89fe9d6..fbd707b4c2ae1 100644
--- a/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
@@ -43,6 +43,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.search.SimpleQueryStringQueryParser;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -64,15 +65,13 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class SimpleQueryStringBuilderTests extends FullTextQueryTestCase<SimpleQueryStringBuilder> {
-    @Override
-    protected boolean isCacheable(SimpleQueryStringBuilder queryBuilder) {
-        return isCacheable(queryBuilder.fields().keySet(), queryBuilder.value());
-    }
+public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQueryStringBuilder> {
 
     @Override
     protected SimpleQueryStringBuilder doCreateTestQueryBuilder() {
-        String queryText = randomAlphaOfLengthBetween(1, 10);
+        // we avoid strings with "now" since those can have different caching policies that are checked elsewhere
+        String queryText = randomValueOtherThanMany(s -> s.toLowerCase(Locale.ROOT).contains("now"),
+                () -> randomAlphaOfLengthBetween(1, 10));
         SimpleQueryStringBuilder result = new SimpleQueryStringBuilder(queryText);
         if (randomBoolean()) {
             result.analyzeWildcard(randomBoolean());
@@ -103,7 +102,7 @@ protected SimpleQueryStringBuilder doCreateTestQueryBuilder() {
         int fieldCount = randomIntBetween(0, 2);
         Map<String, Float> fields = new HashMap<>();
         for (int i = 0; i < fieldCount; i++) {
-            if (randomBoolean()) {
+            if (i == 0) {
                 String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME);
                 fields.put(fieldName, AbstractQueryBuilder.DEFAULT_BOOST);
             } else {
@@ -782,8 +781,43 @@ private void assertQueryWithAllFieldsWildcard(Query query) {
                 noMatchNoDocsQueries++;
             }
         }
-        assertEquals(11, noMatchNoDocsQueries);
+        assertEquals(9, noMatchNoDocsQueries);
         assertThat(disjunctionMaxQuery.getDisjuncts(), hasItems(new TermQuery(new Term(STRING_FIELD_NAME, "hello")),
             new TermQuery(new Term(STRING_FIELD_NAME_2, "hello"))));
     }
+
+    /**
+     * Query terms that contain "now" can trigger a query to not be cacheable.
+     * This test checks the search context cacheable flag is updated accordingly.
+     */
+    public void testCachingStrategiesWithNow() throws IOException {
+        // if we hit all fields, this should contain a date field and should diable cachability
+        String query = "now " + randomAlphaOfLengthBetween(4, 10);
+        SimpleQueryStringBuilder queryBuilder = new SimpleQueryStringBuilder(query);
+        assertQueryCachability(queryBuilder, false);
+
+        // if we hit a date field with "now", this should diable cachability
+        queryBuilder = new SimpleQueryStringBuilder("now");
+        queryBuilder.field(DATE_FIELD_NAME);
+        assertQueryCachability(queryBuilder, false);
+
+        // everything else is fine on all fields
+        query = randomFrom("NoW", "nOw", "NOW") + " " + randomAlphaOfLengthBetween(4, 10);
+        queryBuilder = new SimpleQueryStringBuilder(query);
+        assertQueryCachability(queryBuilder, true);
+    }
+
+    private void assertQueryCachability(SimpleQueryStringBuilder qb, boolean cachingExpected) throws IOException {
+        QueryShardContext context = createShardContext();
+        assert context.isCacheable();
+        /*
+         * We use a private rewrite context here since we want the most realistic way of asserting that we are cacheable or not. We do it
+         * this way in SearchService where we first rewrite the query with a private context, then reset the context and then build the
+         * actual lucene query
+         */
+        QueryBuilder rewritten = rewriteQuery(qb, new QueryShardContext(context));
+        assertNotNull(rewritten.toQuery(context));
+        assertEquals("query should " + (cachingExpected ? "" : "not") + " be cacheable: " + qb.toString(), cachingExpected,
+                context.isCacheable());
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
index 4c59e25804a55..163c730294867 100644
--- a/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
@@ -38,7 +38,6 @@
 import org.apache.lucene.search.spans.SpanQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.store.Directory;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -193,22 +192,13 @@ public void testToQueryInnerTermQuery() throws IOException {
         final QueryShardContext context = createShardContext();
         {
             Query query = new SpanMultiTermQueryBuilder(new PrefixQueryBuilder(fieldName, "foo")).toQuery(context);
-            if (context.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_4_0)) {
-                assertThat(query, instanceOf(FieldMaskingSpanQuery.class));
-                FieldMaskingSpanQuery fieldQuery = (FieldMaskingSpanQuery) query;
-                assertThat(fieldQuery.getMaskedQuery(), instanceOf(SpanTermQuery.class));
-                assertThat(fieldQuery.getField(), equalTo("prefix_field"));
-                SpanTermQuery termQuery = (SpanTermQuery) fieldQuery.getMaskedQuery();
-                assertThat(termQuery.getTerm().field(), equalTo("prefix_field._index_prefix"));
-                assertThat(termQuery.getTerm().text(), equalTo("foo"));
-            } else {
-                assertThat(query, instanceOf(SpanMultiTermQueryWrapper.class));
-                SpanMultiTermQueryWrapper wrapper = (SpanMultiTermQueryWrapper) query;
-                assertThat(wrapper.getWrappedQuery(), instanceOf(PrefixQuery.class));
-                PrefixQuery prefixQuery = (PrefixQuery) wrapper.getWrappedQuery();
-                assertThat(prefixQuery.getField(), equalTo("prefix_field"));
-                assertThat(prefixQuery.getPrefix().text(), equalTo("foo"));
-            }
+            assertThat(query, instanceOf(FieldMaskingSpanQuery.class));
+            FieldMaskingSpanQuery fieldQuery = (FieldMaskingSpanQuery) query;
+            assertThat(fieldQuery.getMaskedQuery(), instanceOf(SpanTermQuery.class));
+            assertThat(fieldQuery.getField(), equalTo("prefix_field"));
+            SpanTermQuery termQuery = (SpanTermQuery) fieldQuery.getMaskedQuery();
+            assertThat(termQuery.getTerm().field(), equalTo("prefix_field._index_prefix"));
+            assertThat(termQuery.getTerm().text(), equalTo("foo"));
         }
 
         {
diff --git a/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
index 40e32b91d7e55..f782c45c03127 100644
--- a/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
@@ -281,13 +281,6 @@ public void testGeo() throws Exception {
                 e.getMessage());
     }
 
-    @Override
-    protected boolean isCacheable(TermsQueryBuilder queryBuilder) {
-        // even though we use a terms lookup here we do this during rewrite and that means we are cacheable on toQuery
-        // that's why we return true here all the time
-        return super.isCacheable(queryBuilder);
-    }
-
     public void testSerializationFailsUnlessFetched() throws IOException {
         QueryBuilder builder = new TermsQueryBuilder(STRING_FIELD_NAME, randomTermsLookup());
         QueryBuilder termsQueryBuilder = Rewriteable.rewrite(builder, createShardContext());
diff --git a/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java
index f68769bb89cb5..84e90f8901674 100644
--- a/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java
@@ -110,10 +110,42 @@ protected void doAssertLuceneQuery(TermsSetQueryBuilder queryBuilder, Query quer
         }
     }
 
+    /**
+     * Check that this query is generally not cacheable and explicitly testing the two conditions when it is not as well
+     */
     @Override
-    protected boolean isCacheable(TermsSetQueryBuilder queryBuilder) {
-        return queryBuilder.getMinimumShouldMatchField() != null ||
+    public void testCacheability() throws IOException {
+        TermsSetQueryBuilder queryBuilder = createTestQueryBuilder();
+        boolean isCacheable = queryBuilder.getMinimumShouldMatchField() != null ||
                 (queryBuilder.getMinimumShouldMatchScript() != null && queryBuilder.getValues().isEmpty());
+        QueryShardContext context = createShardContext();
+        rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(queryBuilder.doToQuery(context));
+        assertEquals("query should " + (isCacheable ? "" : "not") + " be cacheable: " + queryBuilder.toString(), isCacheable,
+                context.isCacheable());
+
+        // specifically trigger the two cases where query is cacheable
+        queryBuilder = new TermsSetQueryBuilder(STRING_FIELD_NAME, Collections.singletonList("foo"));
+        queryBuilder.setMinimumShouldMatchField("m_s_m");
+        context = createShardContext();
+        rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(queryBuilder.doToQuery(context));
+        assertTrue("query should be cacheable: " + queryBuilder.toString(), context.isCacheable());
+
+        queryBuilder = new TermsSetQueryBuilder(STRING_FIELD_NAME, Collections.emptyList());
+        queryBuilder.setMinimumShouldMatchScript(new Script(ScriptType.INLINE, MockScriptEngine.NAME, "_script", emptyMap()));
+        context = createShardContext();
+        rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(queryBuilder.doToQuery(context));
+        assertTrue("query should be cacheable: " + queryBuilder.toString(), context.isCacheable());
+
+        // also test one case where query is not cacheable
+        queryBuilder = new TermsSetQueryBuilder(STRING_FIELD_NAME, Collections.singletonList("foo"));
+        queryBuilder.setMinimumShouldMatchScript(new Script(ScriptType.INLINE, MockScriptEngine.NAME, "_script", emptyMap()));
+        context = createShardContext();
+        rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(queryBuilder.doToQuery(context));
+        assertFalse("query should be cacheable: " + queryBuilder.toString(), context.isCacheable());
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/index/query/TypeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TypeQueryBuilderTests.java
index 67916e52789c5..be2592f01fcf7 100644
--- a/server/src/test/java/org/elasticsearch/index/query/TypeQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/TypeQueryBuilderTests.java
@@ -83,4 +83,10 @@ public void testMustRewrite() throws IOException {
         super.testMustRewrite();
         assertWarnings(TypeQueryBuilder.TYPES_DEPRECATION_MESSAGE);
     }
+
+    @Override
+    public void testCacheability() throws IOException {
+        super.testCacheability();
+        assertWarnings(TypeQueryBuilder.TYPES_DEPRECATION_MESSAGE);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
index 8f177cac863b3..c21c9a56a4c3c 100644
--- a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.query.functionscore;
 
 import com.fasterxml.jackson.core.JsonParseException;
+
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
@@ -40,6 +41,7 @@
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.RandomQueryBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.query.WrapperQueryBuilder;
@@ -676,7 +678,7 @@ public void testRewriteWithFunction() throws IOException {
      */
     public void testSingleScriptFunction() throws IOException {
         QueryBuilder queryBuilder = RandomQueryBuilder.createQuery(random());
-        ScoreFunctionBuilder functionBuilder = new ScriptScoreFunctionBuilder(
+        ScoreFunctionBuilder<ScriptScoreFunctionBuilder> functionBuilder = new ScriptScoreFunctionBuilder(
             new Script(ScriptType.INLINE, MockScriptEngine.NAME, "1", Collections.emptyMap()));
 
         FunctionScoreQueryBuilder builder = functionScoreQuery(queryBuilder, functionBuilder);
@@ -796,8 +798,38 @@ public List<ScoreFunctionSpec<?>> getScoreFunctions() {
         }
     }
 
+    /**
+     * Check that this query is generally cacheable except for builders using {@link ScriptScoreFunctionBuilder} or
+     * {@link RandomScoreFunctionBuilder} without a seed
+     */
     @Override
-    protected boolean isCacheable(FunctionScoreQueryBuilder queryBuilder) {
+    public void testCacheability() throws IOException {
+        FunctionScoreQueryBuilder queryBuilder = createTestQueryBuilder();
+        boolean isCacheable = isCacheable(queryBuilder);
+        QueryShardContext context = createShardContext();
+        QueryBuilder rewriteQuery = rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(rewriteQuery.toQuery(context));
+        assertEquals("query should " + (isCacheable ? "" : "not") + " be cacheable: " + queryBuilder.toString(), isCacheable,
+                context.isCacheable());
+
+        // check the two non-cacheable cases explicitly
+        ScoreFunctionBuilder<?> scriptScoreFunction = new ScriptScoreFunctionBuilder(
+                new Script(ScriptType.INLINE, MockScriptEngine.NAME, "1", Collections.emptyMap()));
+        RandomScoreFunctionBuilder randomScoreFunctionBuilder = new RandomScoreFunctionBuilderWithFixedSeed();
+
+        for (ScoreFunctionBuilder<?> scoreFunction : List.of(scriptScoreFunction, randomScoreFunctionBuilder)) {
+            FilterFunctionBuilder[] functions = new FilterFunctionBuilder[] {
+                    new FilterFunctionBuilder(RandomQueryBuilder.createQuery(random()), scoreFunction) };
+            queryBuilder = new FunctionScoreQueryBuilder(functions);
+
+            context = createShardContext();
+            rewriteQuery = rewriteQuery(queryBuilder, new QueryShardContext(context));
+            assertNotNull(rewriteQuery.toQuery(context));
+            assertFalse("query should not be cacheable: " + queryBuilder.toString(), context.isCacheable());
+        }
+    }
+
+    private boolean isCacheable(FunctionScoreQueryBuilder queryBuilder) {
         FilterFunctionBuilder[] filterFunctionBuilders = queryBuilder.filterFunctionBuilders();
         for (FilterFunctionBuilder builder : filterFunctionBuilders) {
             if (builder.getScoreFunction() instanceof ScriptScoreFunctionBuilder) {
diff --git a/server/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java b/server/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java
index 143e2416b8880..dc7423c68596c 100644
--- a/server/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java
+++ b/server/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java
@@ -21,7 +21,9 @@
 
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.test.AbstractXContentTestCase;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
@@ -29,9 +31,12 @@
 /**
  * Shared superclass for testing reindex and friends. In particular it makes sure to test the slice features.
  */
-public abstract class AbstractBulkByScrollRequestTestCase<R extends AbstractBulkByScrollRequest<R>> extends ESTestCase {
+public abstract class AbstractBulkByScrollRequestTestCase<R extends AbstractBulkByScrollRequest<R> & ToXContent>
+    extends AbstractXContentTestCase<R> {
+
     public void testForSlice() {
         R original = newRequest();
+        extraRandomizationForSlice(original);
         original.setAbortOnVersionConflict(randomBoolean());
         original.setRefresh(randomBoolean());
         original.setTimeout(parseTimeValue(randomPositiveTimeValue(), "timeout"));
@@ -42,7 +47,7 @@ public void testForSlice() {
         original.setRequestsPerSecond(
                 randomBoolean() ? Float.POSITIVE_INFINITY : randomValueOtherThanMany(r -> r < 0, ESTestCase::randomFloat));
         if (randomBoolean()) {
-            original.setSize(between(0, Integer.MAX_VALUE));
+            original.setMaxDocs(between(0, Integer.MAX_VALUE));
         }
 
         // it's not important how many slices there are, we just need a number for forSlice
@@ -64,8 +69,10 @@ public void testForSlice() {
         assertEquals("slice requests always have a single worker", 1, forSliced.getSlices());
         assertEquals("requests_per_second is split between all workers", original.getRequestsPerSecond() / actualSlices,
                 forSliced.getRequestsPerSecond(), Float.MIN_NORMAL);
-        assertEquals("size is split evenly between all workers", original.getSize() == AbstractBulkByScrollRequest.SIZE_ALL_MATCHES
-                ? AbstractBulkByScrollRequest.SIZE_ALL_MATCHES : original.getSize() / actualSlices, forSliced.getSize());
+        assertEquals("max_docs is split evenly between all workers",
+            original.getMaxDocs() == AbstractBulkByScrollRequest.MAX_DOCS_ALL_MATCHES
+                ? AbstractBulkByScrollRequest.MAX_DOCS_ALL_MATCHES : original.getMaxDocs() / actualSlices,
+            forSliced.getMaxDocs());
         assertEquals(slicingTask, forSliced.getParentTask());
 
         extraForSliceAssertions(original, forSliced);
diff --git a/server/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryRequestTests.java b/server/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryRequestTests.java
index 76c6dc03b5a49..44cd8d71b2dd5 100644
--- a/server/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryRequestTests.java
@@ -22,8 +22,11 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilders;
 
+import java.io.IOException;
+
 import static org.apache.lucene.util.TestUtil.randomSimpleString;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
@@ -79,30 +82,6 @@ protected void extraForSliceAssertions(DeleteByQueryRequest original, DeleteByQu
         // No extra assertions needed
     }
 
-    public void testTypesGetter() {
-        int numTypes = between(1, 50);
-        String[] types = new String[numTypes];
-        for (int i = 0; i < numTypes; i++) {
-            types[i] = randomSimpleString(random(), 1, 30);
-        }
-        SearchRequest searchRequest = new SearchRequest();
-        searchRequest.types(types);
-        DeleteByQueryRequest request = new DeleteByQueryRequest(searchRequest);
-        assertArrayEquals(request.types(), types);
-    }
-
-    public void testTypesSetter() {
-        int numTypes = between(1, 50);
-        String[] types = new String[numTypes];
-        for (int i = 0; i < numTypes; i++) {
-            types[i] = randomSimpleString(random(), 1, 30);
-        }
-        SearchRequest searchRequest = new SearchRequest();
-        DeleteByQueryRequest request = new DeleteByQueryRequest(searchRequest);
-        request.types(types);
-        assertArrayEquals(request.types(), types);
-    }
-
     public void testValidateGivenNoQuery() {
         SearchRequest searchRequest = new SearchRequest();
         DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(searchRequest);
@@ -124,4 +103,28 @@ public void testValidateGivenValid() {
 
         assertThat(e, is(nullValue()));
     }
+
+    // TODO: Implement standard to/from x-content parsing tests
+
+    @Override
+    protected DeleteByQueryRequest createTestInstance() {
+        return newRequest();
+    }
+
+    @Override
+    protected DeleteByQueryRequest doParseInstance(XContentParser parser) throws IOException {
+        XContentParser.Token token;
+        while ((token = parser.nextToken()) != null) {
+        }
+        return newRequest();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected void assertEqualInstances(DeleteByQueryRequest expectedInstance, DeleteByQueryRequest newInstance) {
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java b/server/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java
index 1c3d539263e7d..23a9880046001 100644
--- a/server/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java
@@ -21,10 +21,26 @@
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.slice.SliceBuilder;
 
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
+import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 
 /**
@@ -32,12 +48,101 @@
  */
 public class ReindexRequestTests extends AbstractBulkByScrollRequestTestCase<ReindexRequest> {
 
+    private final BytesReference matchAll = new BytesArray("{ \"foo\" : \"bar\" }");
+
+    @Override
+    protected NamedWriteableRegistry writableRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
+        return new NamedWriteableRegistry(searchModule.getNamedWriteables());
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
+        return new NamedXContentRegistry(searchModule.getNamedXContents());
+    }
+
+    @Override
+    protected boolean enableWarningsCheck() {
+        // There sometimes will be a warning about specifying types in reindex requests being deprecated.
+        return false;
+    }
+
+    @Override
+    protected ReindexRequest createTestInstance() {
+        ReindexRequest reindexRequest = new ReindexRequest();
+        reindexRequest.setSourceIndices("source");
+        reindexRequest.setDestIndex("destination");
+
+        if (randomBoolean()) {
+            try (XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint()) {
+                BytesReference query = BytesReference.bytes(matchAllQuery().toXContent(builder, ToXContent.EMPTY_PARAMS));
+                reindexRequest.setRemoteInfo(new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, Integer.MAX_VALUE),
+                    null, query, "user", "pass", emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT));
+            } catch (IOException e) {
+                throw new AssertionError(e);
+            }
+        }
+
+        if (randomBoolean()) {
+            reindexRequest.setSourceBatchSize(randomInt(100));
+        }
+        if (randomBoolean()) {
+            reindexRequest.setDestDocType("type");
+        }
+        if (randomBoolean()) {
+            reindexRequest.setDestOpType("create");
+        }
+        if (randomBoolean()) {
+            reindexRequest.setDestPipeline("my_pipeline");
+        }
+        if (randomBoolean()) {
+            reindexRequest.setDestRouting("=cat");
+        }
+        if (randomBoolean()) {
+            reindexRequest.setMaxDocs(randomIntBetween(100, 1000));
+        }
+        if (randomBoolean()) {
+            reindexRequest.setAbortOnVersionConflict(false);
+        }
+
+        if (reindexRequest.getRemoteInfo() == null && randomBoolean()) {
+            reindexRequest.setSourceQuery(new TermQueryBuilder("foo", "fooval"));
+        }
+
+        return reindexRequest;
+    }
+
+    @Override
+    protected ReindexRequest doParseInstance(XContentParser parser) throws IOException {
+        return ReindexRequest.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected void assertEqualInstances(ReindexRequest expectedInstance, ReindexRequest newInstance) {
+        assertNotSame(newInstance, expectedInstance);
+        assertArrayEquals(expectedInstance.getSearchRequest().indices(), newInstance.getSearchRequest().indices());
+        assertEquals(expectedInstance.getSearchRequest(), newInstance.getSearchRequest());
+        assertEquals(expectedInstance.getMaxDocs(), newInstance.getMaxDocs());
+        assertEquals(expectedInstance.getSlices(), newInstance.getSlices());
+        assertEquals(expectedInstance.isAbortOnVersionConflict(), newInstance.isAbortOnVersionConflict());
+        assertEquals(expectedInstance.getRemoteInfo(), newInstance.getRemoteInfo());
+        assertEquals(expectedInstance.getDestination().getPipeline(), newInstance.getDestination().getPipeline());
+        assertEquals(expectedInstance.getDestination().routing(), newInstance.getDestination().routing());
+        assertEquals(expectedInstance.getDestination().opType(), newInstance.getDestination().opType());
+        assertEquals(expectedInstance.getDestination().type(), newInstance.getDestination().type());
+    }
+
     public void testReindexFromRemoteDoesNotSupportSearchQuery() {
         ReindexRequest reindex = newRequest();
         reindex.setRemoteInfo(
                 new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, Integer.MAX_VALUE), null,
-                    new BytesArray("real_query"), null, null, emptyMap(),
-                    RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT));
+                    matchAll, null, null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT));
         reindex.getSearchRequest().source().query(matchAllQuery()); // Unsupported place to put query
         ActionRequestValidationException e = reindex.validate();
         assertEquals("Validation Failed: 1: reindex from remote sources should use RemoteInfo's query instead of source's query;",
@@ -48,8 +153,7 @@ public void testReindexFromRemoteDoesNotSupportSlices() {
         ReindexRequest reindex = newRequest();
         reindex.setRemoteInfo(
                 new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, Integer.MAX_VALUE), null,
-                    new BytesArray("real_query"), null, null, emptyMap(),
-                    RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT));
+                    matchAll, null, null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT));
         reindex.setSlices(between(2, Integer.MAX_VALUE));
         ActionRequestValidationException e = reindex.validate();
         assertEquals(
@@ -71,10 +175,9 @@ protected void extraRandomizationForSlice(ReindexRequest original) {
             original.setScript(mockScript(randomAlphaOfLength(5)));
         }
         if (randomBoolean()) {
-            original.setRemoteInfo(new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, 10000),
-                    null, new BytesArray(randomAlphaOfLength(5)), null, null, emptyMap(),
-                    parseTimeValue(randomPositiveTimeValue(), "socket_timeout"),
-                    parseTimeValue(randomPositiveTimeValue(), "connect_timeout")));
+            original.setRemoteInfo(new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, 10000), null, matchAll, null,
+                null, emptyMap(), parseTimeValue(randomPositiveTimeValue(), "socket_timeout"),
+                parseTimeValue(randomPositiveTimeValue(), "connect_timeout")));
         }
     }
 
@@ -92,4 +195,133 @@ protected ReindexRequest newRequest() {
         reindex.setDestIndex("dest");
         return reindex;
     }
+
+    public void testBuildRemoteInfoNoRemote() throws IOException {
+        assertNull(ReindexRequest.buildRemoteInfo(new HashMap<>()));
+    }
+
+    public void testBuildRemoteInfoFullyLoaded() throws IOException {
+        Map<String, String> headers = new HashMap<>();
+        headers.put("first", "a");
+        headers.put("second", "b");
+        headers.put("third", "");
+
+        Map<String, Object> remote = new HashMap<>();
+        remote.put("host", "https://example.com:9200");
+        remote.put("username", "testuser");
+        remote.put("password", "testpass");
+        remote.put("headers", headers);
+        remote.put("socket_timeout", "90s");
+        remote.put("connect_timeout", "10s");
+
+        Map<String, Object> query = new HashMap<>();
+        query.put("a", "b");
+
+        Map<String, Object> source = new HashMap<>();
+        source.put("remote", remote);
+        source.put("query", query);
+
+        RemoteInfo remoteInfo = ReindexRequest.buildRemoteInfo(source);
+        assertEquals("https", remoteInfo.getScheme());
+        assertEquals("example.com", remoteInfo.getHost());
+        assertEquals(9200, remoteInfo.getPort());
+        assertEquals("{\n  \"a\" : \"b\"\n}", remoteInfo.getQuery().utf8ToString());
+        assertEquals("testuser", remoteInfo.getUsername());
+        assertEquals("testpass", remoteInfo.getPassword());
+        assertEquals(headers, remoteInfo.getHeaders());
+        assertEquals(timeValueSeconds(90), remoteInfo.getSocketTimeout());
+        assertEquals(timeValueSeconds(10), remoteInfo.getConnectTimeout());
+    }
+
+    public void testBuildRemoteInfoWithoutAllParts() throws IOException {
+        expectThrows(IllegalArgumentException.class, () -> buildRemoteInfoHostTestCase("example.com"));
+        expectThrows(IllegalArgumentException.class, () -> buildRemoteInfoHostTestCase(":9200"));
+        expectThrows(IllegalArgumentException.class, () -> buildRemoteInfoHostTestCase("http://:9200"));
+        expectThrows(IllegalArgumentException.class, () -> buildRemoteInfoHostTestCase("example.com:9200"));
+        expectThrows(IllegalArgumentException.class, () -> buildRemoteInfoHostTestCase("http://example.com"));
+    }
+
+    public void testBuildRemoteInfoWithAllHostParts() throws IOException {
+        RemoteInfo info = buildRemoteInfoHostTestCase("http://example.com:9200");
+        assertEquals("http", info.getScheme());
+        assertEquals("example.com", info.getHost());
+        assertEquals(9200, info.getPort());
+        assertNull(info.getPathPrefix());
+        assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout()); // Didn't set the timeout so we should get the default
+        assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout()); // Didn't set the timeout so we should get the default
+
+        info = buildRemoteInfoHostTestCase("https://other.example.com:9201");
+        assertEquals("https", info.getScheme());
+        assertEquals("other.example.com", info.getHost());
+        assertEquals(9201, info.getPort());
+        assertNull(info.getPathPrefix());
+        assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout());
+        assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout());
+
+        info = buildRemoteInfoHostTestCase("https://[::1]:9201");
+        assertEquals("https", info.getScheme());
+        assertEquals("[::1]", info.getHost());
+        assertEquals(9201, info.getPort());
+        assertNull(info.getPathPrefix());
+        assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout());
+        assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout());
+
+        info = buildRemoteInfoHostTestCase("https://other.example.com:9201/");
+        assertEquals("https", info.getScheme());
+        assertEquals("other.example.com", info.getHost());
+        assertEquals(9201, info.getPort());
+        assertEquals("/", info.getPathPrefix());
+        assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout());
+        assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout());
+
+        info = buildRemoteInfoHostTestCase("https://other.example.com:9201/proxy-path/");
+        assertEquals("https", info.getScheme());
+        assertEquals("other.example.com", info.getHost());
+        assertEquals(9201, info.getPort());
+        assertEquals("/proxy-path/", info.getPathPrefix());
+        assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout());
+        assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout());
+
+        final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> buildRemoteInfoHostTestCase("https"));
+        assertEquals("[host] must be of the form [scheme]://[host]:[port](/[pathPrefix])? but was [https]",
+            exception.getMessage());
+    }
+
+    public void testReindexFromRemoteRequestParsing() throws IOException {
+        BytesReference request;
+        try (XContentBuilder b = JsonXContent.contentBuilder()) {
+            b.startObject(); {
+                b.startObject("source"); {
+                    b.startObject("remote"); {
+                        b.field("host", "http://localhost:9200");
+                    }
+                    b.endObject();
+                    b.field("index", "source");
+                }
+                b.endObject();
+                b.startObject("dest"); {
+                    b.field("index", "dest");
+                }
+                b.endObject();
+            }
+            b.endObject();
+            request = BytesReference.bytes(b);
+        }
+        try (XContentParser p = createParser(JsonXContent.jsonXContent, request)) {
+            ReindexRequest r = ReindexRequest.fromXContent(p);
+            assertEquals("localhost", r.getRemoteInfo().getHost());
+            assertArrayEquals(new String[] {"source"}, r.getSearchRequest().indices());
+        }
+    }
+
+    private RemoteInfo buildRemoteInfoHostTestCase(String hostInRest) throws IOException {
+        Map<String, Object> remote = new HashMap<>();
+        remote.put("host", hostInRest);
+
+        Map<String, Object> source = new HashMap<>();
+        source.put("remote", remote);
+
+        return ReindexRequest.buildRemoteInfo(source);
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java b/server/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java
index 207f0f12ff23d..bc4ccd83023fe 100644
--- a/server/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java
@@ -20,6 +20,9 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.apache.lucene.util.TestUtil.randomSimpleString;
 
@@ -77,4 +80,28 @@ protected void extraForSliceAssertions(UpdateByQueryRequest original, UpdateByQu
         assertEquals(original.getScript(), forSliced.getScript());
         assertEquals(original.getPipeline(), forSliced.getPipeline());
     }
+
+    // TODO: Implement standard to/from x-content parsing tests
+
+    @Override
+    protected UpdateByQueryRequest createTestInstance() {
+        return newRequest();
+    }
+
+    @Override
+    protected UpdateByQueryRequest doParseInstance(XContentParser parser) throws IOException {
+        XContentParser.Token token;
+        while ((token = parser.nextToken()) != null) {
+        }
+        return newRequest();
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected void assertEqualInstances(UpdateByQueryRequest expectedInstance, UpdateByQueryRequest newInstance) {
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
index e25557eaabcf6..97e5210c9d09e 100644
--- a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
+++ b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
@@ -122,14 +122,15 @@ public void run() {
                 (indexShard, node) -> new RecoveryTarget(indexShard, node, recoveryListener) {
                     @Override
                     public void cleanFiles(int totalTranslogOps, long globalCheckpoint,
-                                           Store.MetadataSnapshot sourceMetaData) throws IOException {
-                        super.cleanFiles(totalTranslogOps, globalCheckpoint, sourceMetaData);
-                        latch.countDown();
-                        try {
-                            latch.await();
-                        } catch (InterruptedException e) {
-                            throw new AssertionError(e);
-                        }
+                                           Store.MetadataSnapshot sourceMetaData, ActionListener<Void> listener) {
+                        super.cleanFiles(totalTranslogOps, globalCheckpoint, sourceMetaData, ActionListener.runAfter(listener, () -> {
+                            latch.countDown();
+                            try {
+                                latch.await();
+                            } catch (InterruptedException e) {
+                                throw new AssertionError(e);
+                            }
+                        }));
                     }
                 });
             future.get();
diff --git a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
index d499cf6e83f90..c60f32132c646 100644
--- a/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
+++ b/server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.bulk.BulkShardRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
@@ -58,7 +59,6 @@
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.indices.recovery.RecoveryTarget;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -227,7 +227,6 @@ public void testRecoveryToReplicaThatReceivedExtraDocument() throws Exception {
         }
     }
 
-    @TestLogging("org.elasticsearch.index.shard:TRACE,org.elasticsearch.indices.recovery:TRACE")
     public void testRecoveryAfterPrimaryPromotion() throws Exception {
         try (ReplicationGroup shards = createGroup(2)) {
             shards.startAll();
@@ -364,7 +363,6 @@ public void testReplicaRollbackStaleDocumentsInPeerRecovery() throws Exception {
         }
     }
 
-    @TestLogging("org.elasticsearch.index.shard:TRACE,org.elasticsearch.action.resync:TRACE")
     public void testResyncAfterPrimaryPromotion() throws Exception {
         // TODO: check translog trimming functionality once rollback is implemented in Lucene (ES trimming is done)
         Map<String, String> mappings =
@@ -521,16 +519,6 @@ public void finalizeRecovery(long globalCheckpoint, ActionListener<Void> listene
         }
     }
 
-    @TestLogging(
-            "_root:DEBUG,"
-                    + "org.elasticsearch.action.bulk:TRACE,"
-                    + "org.elasticsearch.action.get:TRACE,"
-                    + "org.elasticsearch.cluster.service:TRACE,"
-                    + "org.elasticsearch.discovery:TRACE,"
-                    + "org.elasticsearch.indices.cluster:TRACE,"
-                    + "org.elasticsearch.indices.recovery:TRACE,"
-                    + "org.elasticsearch.index.seqno:TRACE,"
-                    + "org.elasticsearch.index.shard:TRACE")
     public void testCheckpointsAndMarkingInSync() throws Exception {
         final IndexMetaData metaData = buildIndexMetaData(0);
         final BlockingEngineFactory replicaEngineFactory = new BlockingEngineFactory();
@@ -608,10 +596,10 @@ public void indexTranslogOperations(
                 final long expectedDocs = docs + 2L;
                 assertThat(shards.getPrimary().getLocalCheckpoint(), equalTo(expectedDocs - 1));
                 // recovery has not completed, therefore the global checkpoint can have advanced on the primary
-                assertThat(shards.getPrimary().getGlobalCheckpoint(), equalTo(expectedDocs - 1));
+                assertThat(shards.getPrimary().getLastKnownGlobalCheckpoint(), equalTo(expectedDocs - 1));
                 // the pending document is not done, the checkpoints can not have advanced on the replica
                 assertThat(replica.getLocalCheckpoint(), lessThan(expectedDocs - 1));
-                assertThat(replica.getGlobalCheckpoint(), lessThan(expectedDocs - 1));
+                assertThat(replica.getLastKnownGlobalCheckpoint(), lessThan(expectedDocs - 1));
             }
 
             // wait for recovery to enter the translog phase
@@ -624,9 +612,9 @@ public void indexTranslogOperations(
                 final long expectedDocs = docs + 3L;
                 assertThat(shards.getPrimary().getLocalCheckpoint(), equalTo(expectedDocs - 1));
                 // recovery is now in the process of being completed, therefore the global checkpoint can not have advanced on the primary
-                assertThat(shards.getPrimary().getGlobalCheckpoint(), equalTo(expectedDocs - 2));
+                assertThat(shards.getPrimary().getLastKnownGlobalCheckpoint(), equalTo(expectedDocs - 2));
                 assertThat(replica.getLocalCheckpoint(), lessThan(expectedDocs - 2));
-                assertThat(replica.getGlobalCheckpoint(), lessThan(expectedDocs - 2));
+                assertThat(replica.getLastKnownGlobalCheckpoint(), lessThan(expectedDocs - 2));
             }
 
             replicaEngineFactory.releaseLatchedIndexers();
@@ -636,10 +624,10 @@ public void indexTranslogOperations(
                 final long expectedDocs = docs + 3L;
                 assertBusy(() -> {
                     assertThat(shards.getPrimary().getLocalCheckpoint(), equalTo(expectedDocs - 1));
-                    assertThat(shards.getPrimary().getGlobalCheckpoint(), equalTo(expectedDocs - 1));
+                    assertThat(shards.getPrimary().getLastKnownGlobalCheckpoint(), equalTo(expectedDocs - 1));
                     assertThat(replica.getLocalCheckpoint(), equalTo(expectedDocs - 1));
                     // the global checkpoint advances can only advance here if a background global checkpoint sync fires
-                    assertThat(replica.getGlobalCheckpoint(), anyOf(equalTo(expectedDocs - 1), equalTo(expectedDocs - 2)));
+                    assertThat(replica.getLastKnownGlobalCheckpoint(), anyOf(equalTo(expectedDocs - 1), equalTo(expectedDocs - 2)));
                 });
             }
         }
@@ -725,6 +713,9 @@ public void testAddNewReplicas() throws Exception {
                             if (randomInt(100) < 10) {
                                 shards.getPrimary().flush(new FlushRequest());
                             }
+                            if (randomInt(100) < 5) {
+                                shards.getPrimary().forceMerge(new ForceMergeRequest().flush(randomBoolean()).maxNumSegments(1));
+                            }
                         } catch (Exception ex) {
                             throw new AssertionError(ex);
                         }
@@ -771,7 +762,7 @@ public void testRollbackOnPromotion() throws Exception {
             }
             shards.refresh("test");
             List<DocIdSeqNoAndSource> docsBelowGlobalCheckpoint = EngineTestCase.getDocIds(getEngine(newPrimary), randomBoolean())
-                .stream().filter(doc -> doc.getSeqNo() <= newPrimary.getGlobalCheckpoint()).collect(Collectors.toList());
+                .stream().filter(doc -> doc.getSeqNo() <= newPrimary.getLastKnownGlobalCheckpoint()).collect(Collectors.toList());
             CountDownLatch latch = new CountDownLatch(1);
             final AtomicBoolean done = new AtomicBoolean();
             Thread thread = new Thread(() -> {
@@ -857,9 +848,10 @@ public void indexTranslogOperations(
         }
 
         @Override
-        public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetaData) throws IOException {
+        public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetaData,
+                               ActionListener<Void> listener) {
             blockIfNeeded(RecoveryState.Stage.INDEX);
-            super.cleanFiles(totalTranslogOps, globalCheckpoint, sourceMetaData);
+            super.cleanFiles(totalTranslogOps, globalCheckpoint, sourceMetaData, listener);
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/index/search/MatchPhraseQueryIT.java b/server/src/test/java/org/elasticsearch/index/search/MatchPhraseQueryIT.java
index ebd0bf0460aeb..5bbc4cb389f29 100644
--- a/server/src/test/java/org/elasticsearch/index/search/MatchPhraseQueryIT.java
+++ b/server/src/test/java/org/elasticsearch/index/search/MatchPhraseQueryIT.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.search.MatchQuery.ZeroTermsQuery;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.junit.Before;
@@ -33,6 +32,7 @@
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 
+import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 
@@ -55,7 +55,7 @@ public void testZeroTermsQuery() throws ExecutionException, InterruptedException
         List<IndexRequestBuilder> indexRequests = getIndexRequests();
         indexRandom(true, false, indexRequests);
 
-        MatchPhraseQueryBuilder baseQuery = QueryBuilders.matchPhraseQuery("name", "the who")
+        MatchPhraseQueryBuilder baseQuery = matchPhraseQuery("name", "the who")
             .analyzer("standard_stopwords");
 
         MatchPhraseQueryBuilder matchNoneQuery = baseQuery.zeroTermsQuery(ZeroTermsQuery.NONE);
@@ -67,7 +67,6 @@ public void testZeroTermsQuery() throws ExecutionException, InterruptedException
         assertHitCount(matchAllResponse, 2L);
     }
 
-
     private List<IndexRequestBuilder> getIndexRequests() {
         List<IndexRequestBuilder> requests = new ArrayList<>();
         requests.add(client().prepareIndex(INDEX, "band").setSource("name", "the beatles"));
diff --git a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java
index 58baadd83573d..d42806aeeabd3 100644
--- a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java
@@ -111,8 +111,8 @@ public void testCrossFieldMultiMatchQuery() throws IOException {
                 .toQuery(queryShardContext);
             try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) {
                 Query rewrittenQuery = searcher.searcher().rewrite(parsedQuery);
-                Query tq1 = new BoostQuery(new TermQuery(new Term("name.first", "banon")), 2);
-                Query tq2 = new BoostQuery(new TermQuery(new Term("name.last", "banon")), 3);
+                Query tq1 = new BoostQuery(new TermQuery(new Term("name.last", "banon")), 3);
+                Query tq2 = new BoostQuery(new TermQuery(new Term("name.first", "banon")), 2);
                 Query expected = new DisjunctionMaxQuery(Arrays.asList(tq2, tq1), tieBreaker);
                 assertEquals(expected, rewrittenQuery);
             }
@@ -129,7 +129,7 @@ public void testBlendTerms() {
         Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f);
         Query actual = MultiMatchQuery.blendTerm(
                 indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null),
-                new BytesRef("baz"), null, 1f, false, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)));
+                new BytesRef("baz"), 1f, false, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)));
         assertEquals(expected, actual);
     }
 
@@ -145,7 +145,7 @@ public void testBlendTermsWithFieldBoosts() {
         Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f);
         Query actual = MultiMatchQuery.blendTerm(
                 indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null),
-                new BytesRef("baz"), null, 1f, false, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)));
+                new BytesRef("baz"), 1f, false, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)));
         assertEquals(expected, actual);
     }
 
@@ -167,7 +167,7 @@ public Query termQuery(Object value, QueryShardContext context) {
         ), 1f);
         Query actual = MultiMatchQuery.blendTerm(
                 indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null),
-                new BytesRef("baz"), null, 1f, true, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)));
+                new BytesRef("baz"), 1f, true, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)));
         assertEquals(expected, actual);
     }
 
@@ -181,7 +181,7 @@ public Query termQuery(Object value, QueryShardContext context) {
         ft.setName("bar");
         expectThrows(IllegalArgumentException.class, () -> MultiMatchQuery.blendTerm(
             indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null),
-            new BytesRef("baz"), null, 1f, false, Arrays.asList(new FieldAndBoost(ft, 1))));
+            new BytesRef("baz"), 1f, false, Arrays.asList(new FieldAndBoost(ft, 1))));
     }
 
     public void testBlendNoTermQuery() {
@@ -205,7 +205,7 @@ public Query termQuery(Object value, QueryShardContext context) {
             ), 1.0f);
         Query actual = MultiMatchQuery.blendTerm(
                 indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null),
-                new BytesRef("baz"), null, 1f, false, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)));
+                new BytesRef("baz"), 1f, false, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)));
         assertEquals(expected, actual);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java
index dd44a386329f9..2d4445c523709 100644
--- a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java
+++ b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java
@@ -792,6 +792,8 @@ public void testMultiLevelNestedSorting() throws IOException {
             assertThat(searcher.doc(topFields.scoreDocs[0].doc).get("_id"), equalTo("4"));
             assertThat(((FieldDoc) topFields.scoreDocs[0]).fields[0], equalTo(87L));
         }
+
+        searcher.getIndexReader().close();
     }
 
     private static TopFieldDocs search(QueryBuilder queryBuilder, FieldSortBuilder sortBuilder, QueryShardContext queryShardContext,
diff --git a/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java b/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
index cec3c05b28438..79b9b231b48d4 100644
--- a/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
@@ -100,7 +100,7 @@ public void testTranslogSyncAfterGlobalCheckpointSync() throws Exception {
             lastSyncedGlobalCheckpoint = globalCheckpoint;
         }
 
-        when(indexShard.getGlobalCheckpoint()).thenReturn(globalCheckpoint);
+        when(indexShard.getLastKnownGlobalCheckpoint()).thenReturn(globalCheckpoint);
         when(indexShard.getLastSyncedGlobalCheckpoint()).thenReturn(lastSyncedGlobalCheckpoint);
 
         final GlobalCheckpointSyncAction action = new GlobalCheckpointSyncAction(
diff --git a/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java b/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java
index 4f4f39c614687..65dce061a3daf 100644
--- a/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java
+++ b/server/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncIT.java
@@ -30,6 +30,8 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -42,6 +44,7 @@
 import java.util.Optional;
 import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -58,9 +61,37 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
                 .collect(Collectors.toList());
     }
 
+    public void testGlobalCheckpointSyncWithAsyncDurability() throws Exception {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        prepareCreate(
+            "test",
+            Settings.builder()
+                .put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "1s")
+                .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC)
+                .put(IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.getKey(), "1s")
+                .put("index.number_of_replicas", 1))
+            .get();
+
+        for (int j = 0; j < 10; j++) {
+            final String id = Integer.toString(j);
+            client().prepareIndex("test", "test", id).setSource("{\"foo\": " + id + "}", XContentType.JSON).get();
+        }
+
+        assertBusy(() -> {
+            SeqNoStats seqNoStats = client().admin().indices().prepareStats("test").get().getIndex("test").getShards()[0].getSeqNoStats();
+            assertThat(seqNoStats.getGlobalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
+        });
+    }
+
     public void testPostOperationGlobalCheckpointSync() throws Exception {
-        // set the sync interval high so it does not execute during this test
-        runGlobalCheckpointSyncTest(TimeValue.timeValueHours(24), client -> {}, client -> {});
+        // set the sync interval high so it does not execute during this test. This only allows the global checkpoint to catch up
+        // on a post-operation background sync if translog durability is set to sync. Async durability relies on a scheduled global
+        // checkpoint sync to allow the information about persisted local checkpoints to be transferred to the primary.
+        runGlobalCheckpointSyncTest(TimeValue.timeValueHours(24),
+            client ->
+                client.admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder()
+                    .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.REQUEST)).get(),
+            client -> {});
     }
 
     /*
@@ -188,7 +219,7 @@ private void runGlobalCheckpointSyncTest(
                     assertThat(seqNoStats.getGlobalCheckpoint(), equalTo(primarySeqNoStats.getGlobalCheckpoint()));
                 }
             }
-        });
+        }, 30, TimeUnit.SECONDS);
 
         for (final Thread thread : threads) {
             thread.join();
diff --git a/server/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointTrackerTests.java b/server/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointTrackerTests.java
index 44b3794ea6d42..a11e29097cc48 100644
--- a/server/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointTrackerTests.java
+++ b/server/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointTrackerTests.java
@@ -55,43 +55,79 @@ public void setUp() throws Exception {
         tracker = createEmptyTracker();
     }
 
-    public void testSimplePrimary() {
+    public void testSimplePrimaryProcessed() {
         long seqNo1, seqNo2;
-        assertThat(tracker.getCheckpoint(), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
+        assertThat(tracker.getProcessedCheckpoint(), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
         seqNo1 = tracker.generateSeqNo();
         assertThat(seqNo1, equalTo(0L));
-        tracker.markSeqNoAsCompleted(seqNo1);
-        assertThat(tracker.getCheckpoint(), equalTo(0L));
-        assertThat(tracker.contains(0L), equalTo(true));
-        assertThat(tracker.contains(atLeast(1)), equalTo(false));
+        tracker.markSeqNoAsProcessed(seqNo1);
+        assertThat(tracker.getProcessedCheckpoint(), equalTo(0L));
+        assertThat(tracker.hasProcessed(0L), equalTo(true));
+        assertThat(tracker.hasProcessed(atLeast(1)), equalTo(false));
         seqNo1 = tracker.generateSeqNo();
         seqNo2 = tracker.generateSeqNo();
         assertThat(seqNo1, equalTo(1L));
         assertThat(seqNo2, equalTo(2L));
-        tracker.markSeqNoAsCompleted(seqNo2);
-        assertThat(tracker.getCheckpoint(), equalTo(0L));
-        assertThat(tracker.contains(seqNo1), equalTo(false));
-        assertThat(tracker.contains(seqNo2), equalTo(true));
-        tracker.markSeqNoAsCompleted(seqNo1);
-        assertThat(tracker.getCheckpoint(), equalTo(2L));
-        assertThat(tracker.contains(between(0, 2)), equalTo(true));
-        assertThat(tracker.contains(atLeast(3)), equalTo(false));
+        tracker.markSeqNoAsProcessed(seqNo2);
+        assertThat(tracker.getProcessedCheckpoint(), equalTo(0L));
+        assertThat(tracker.hasProcessed(seqNo1), equalTo(false));
+        assertThat(tracker.hasProcessed(seqNo2), equalTo(true));
+        tracker.markSeqNoAsProcessed(seqNo1);
+        assertThat(tracker.getProcessedCheckpoint(), equalTo(2L));
+        assertThat(tracker.hasProcessed(between(0, 2)), equalTo(true));
+        assertThat(tracker.hasProcessed(atLeast(3)), equalTo(false));
+        assertThat(tracker.getPersistedCheckpoint(), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
+        assertThat(tracker.getMaxSeqNo(), equalTo(2L));
     }
 
-    public void testSimpleReplica() {
-        assertThat(tracker.getCheckpoint(), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
-        assertThat(tracker.contains(randomNonNegativeLong()), equalTo(false));
-        tracker.markSeqNoAsCompleted(0L);
-        assertThat(tracker.getCheckpoint(), equalTo(0L));
-        assertThat(tracker.contains(0), equalTo(true));
-        tracker.markSeqNoAsCompleted(2L);
-        assertThat(tracker.getCheckpoint(), equalTo(0L));
-        assertThat(tracker.contains(1L), equalTo(false));
-        assertThat(tracker.contains(2L), equalTo(true));
-        tracker.markSeqNoAsCompleted(1L);
-        assertThat(tracker.getCheckpoint(), equalTo(2L));
-        assertThat(tracker.contains(between(0, 2)), equalTo(true));
-        assertThat(tracker.contains(atLeast(3)), equalTo(false));
+    public void testSimplePrimaryPersisted() {
+        long seqNo1, seqNo2;
+        assertThat(tracker.getPersistedCheckpoint(), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
+        seqNo1 = tracker.generateSeqNo();
+        assertThat(seqNo1, equalTo(0L));
+        tracker.markSeqNoAsPersisted(seqNo1);
+        assertThat(tracker.getPersistedCheckpoint(), equalTo(0L));
+        seqNo1 = tracker.generateSeqNo();
+        seqNo2 = tracker.generateSeqNo();
+        assertThat(seqNo1, equalTo(1L));
+        assertThat(seqNo2, equalTo(2L));
+        tracker.markSeqNoAsPersisted(seqNo2);
+        assertThat(tracker.getPersistedCheckpoint(), equalTo(0L));
+        tracker.markSeqNoAsPersisted(seqNo1);
+        assertThat(tracker.getPersistedCheckpoint(), equalTo(2L));
+        assertThat(tracker.getProcessedCheckpoint(), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
+        assertThat(tracker.getMaxSeqNo(), equalTo(2L));
+    }
+
+    public void testSimpleReplicaProcessed() {
+        assertThat(tracker.getProcessedCheckpoint(), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
+        assertThat(tracker.hasProcessed(randomNonNegativeLong()), equalTo(false));
+        tracker.markSeqNoAsProcessed(0L);
+        assertThat(tracker.getProcessedCheckpoint(), equalTo(0L));
+        assertThat(tracker.hasProcessed(0), equalTo(true));
+        tracker.markSeqNoAsProcessed(2L);
+        assertThat(tracker.getProcessedCheckpoint(), equalTo(0L));
+        assertThat(tracker.hasProcessed(1L), equalTo(false));
+        assertThat(tracker.hasProcessed(2L), equalTo(true));
+        tracker.markSeqNoAsProcessed(1L);
+        assertThat(tracker.getProcessedCheckpoint(), equalTo(2L));
+        assertThat(tracker.hasProcessed(between(0, 2)), equalTo(true));
+        assertThat(tracker.hasProcessed(atLeast(3)), equalTo(false));
+        assertThat(tracker.getPersistedCheckpoint(), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
+        assertThat(tracker.getMaxSeqNo(), equalTo(2L));
+    }
+
+    public void testSimpleReplicaPersisted() {
+        assertThat(tracker.getPersistedCheckpoint(), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
+        assertThat(tracker.hasProcessed(randomNonNegativeLong()), equalTo(false));
+        tracker.markSeqNoAsPersisted(0L);
+        assertThat(tracker.getPersistedCheckpoint(), equalTo(0L));
+        tracker.markSeqNoAsPersisted(2L);
+        assertThat(tracker.getPersistedCheckpoint(), equalTo(0L));
+        tracker.markSeqNoAsPersisted(1L);
+        assertThat(tracker.getPersistedCheckpoint(), equalTo(2L));
+        assertThat(tracker.getProcessedCheckpoint(), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
+        assertThat(tracker.getMaxSeqNo(), equalTo(2L));
     }
 
     public void testLazyInitialization() {
@@ -100,10 +136,10 @@ public void testLazyInitialization() {
          * sequence numbers this could lead to excessive memory usage resulting in out of memory errors.
          */
         long seqNo = randomNonNegativeLong();
-        tracker.markSeqNoAsCompleted(seqNo);
+        tracker.markSeqNoAsProcessed(seqNo);
         assertThat(tracker.processedSeqNo.size(), equalTo(1));
-        assertThat(tracker.contains(seqNo), equalTo(true));
-        assertThat(tracker.contains(randomValueOtherThan(seqNo, ESTestCase::randomNonNegativeLong)), equalTo(false));
+        assertThat(tracker.hasProcessed(seqNo), equalTo(true));
+        assertThat(tracker.hasProcessed(randomValueOtherThan(seqNo, ESTestCase::randomNonNegativeLong)), equalTo(false));
         assertThat(tracker.processedSeqNo.size(), equalTo(1));
     }
 
@@ -117,16 +153,16 @@ public void testSimpleOverFlow() {
         }
         Collections.shuffle(seqNoList, random());
         for (Long seqNo : seqNoList) {
-            tracker.markSeqNoAsCompleted(seqNo);
+            tracker.markSeqNoAsProcessed(seqNo);
         }
-        assertThat(tracker.checkpoint, equalTo(maxOps - 1L));
+        assertThat(tracker.processedCheckpoint.get(), equalTo(maxOps - 1L));
         assertThat(tracker.processedSeqNo.size(), equalTo(aligned ? 0 : 1));
         if (aligned == false) {
-            assertThat(tracker.processedSeqNo.keys().iterator().next().value, equalTo(tracker.checkpoint / BIT_SET_SIZE));
+            assertThat(tracker.processedSeqNo.keys().iterator().next().value, equalTo(tracker.processedCheckpoint.get() / BIT_SET_SIZE));
         }
-        assertThat(tracker.contains(randomFrom(seqNoList)), equalTo(true));
+        assertThat(tracker.hasProcessed(randomFrom(seqNoList)), equalTo(true));
         final long notCompletedSeqNo = randomValueOtherThanMany(seqNoList::contains, ESTestCase::randomNonNegativeLong);
-        assertThat(tracker.contains(notCompletedSeqNo), equalTo(false));
+        assertThat(tracker.hasProcessed(notCompletedSeqNo), equalTo(false));
     }
 
     public void testConcurrentPrimary() throws InterruptedException {
@@ -151,7 +187,7 @@ protected void doRun() throws Exception {
                         long seqNo = tracker.generateSeqNo();
                         logger.info("[t{}] started   [{}]", threadId, seqNo);
                         if (seqNo != unFinishedSeq) {
-                            tracker.markSeqNoAsCompleted(seqNo);
+                            tracker.markSeqNoAsProcessed(seqNo);
                             logger.info("[t{}] completed [{}]", threadId, seqNo);
                         }
                     }
@@ -163,12 +199,12 @@ protected void doRun() throws Exception {
             thread.join();
         }
         assertThat(tracker.getMaxSeqNo(), equalTo(maxOps - 1L));
-        assertThat(tracker.getCheckpoint(), equalTo(unFinishedSeq - 1L));
-        tracker.markSeqNoAsCompleted(unFinishedSeq);
-        assertThat(tracker.getCheckpoint(), equalTo(maxOps - 1L));
+        assertThat(tracker.getProcessedCheckpoint(), equalTo(unFinishedSeq - 1L));
+        tracker.markSeqNoAsProcessed(unFinishedSeq);
+        assertThat(tracker.getProcessedCheckpoint(), equalTo(maxOps - 1L));
         assertThat(tracker.processedSeqNo.size(), isOneOf(0, 1));
         if (tracker.processedSeqNo.size() == 1) {
-            assertThat(tracker.processedSeqNo.keys().iterator().next().value, equalTo(tracker.checkpoint / BIT_SET_SIZE));
+            assertThat(tracker.processedSeqNo.keys().iterator().next().value, equalTo(tracker.processedCheckpoint.get() / BIT_SET_SIZE));
         }
     }
 
@@ -202,7 +238,7 @@ protected void doRun() throws Exception {
                     Integer[] ops = seqNoPerThread[threadId];
                     for (int seqNo : ops) {
                         if (seqNo != unFinishedSeq) {
-                            tracker.markSeqNoAsCompleted(seqNo);
+                            tracker.markSeqNoAsProcessed(seqNo);
                             logger.info("[t{}] completed [{}]", threadId, seqNo);
                         }
                     }
@@ -214,15 +250,15 @@ protected void doRun() throws Exception {
             thread.join();
         }
         assertThat(tracker.getMaxSeqNo(), equalTo(maxOps - 1L));
-        assertThat(tracker.getCheckpoint(), equalTo(unFinishedSeq - 1L));
-        assertThat(tracker.contains(unFinishedSeq), equalTo(false));
-        tracker.markSeqNoAsCompleted(unFinishedSeq);
-        assertThat(tracker.getCheckpoint(), equalTo(maxOps - 1L));
-        assertThat(tracker.contains(unFinishedSeq), equalTo(true));
-        assertThat(tracker.contains(randomLongBetween(maxOps, Long.MAX_VALUE)), equalTo(false));
+        assertThat(tracker.getProcessedCheckpoint(), equalTo(unFinishedSeq - 1L));
+        assertThat(tracker.hasProcessed(unFinishedSeq), equalTo(false));
+        tracker.markSeqNoAsProcessed(unFinishedSeq);
+        assertThat(tracker.getProcessedCheckpoint(), equalTo(maxOps - 1L));
+        assertThat(tracker.hasProcessed(unFinishedSeq), equalTo(true));
+        assertThat(tracker.hasProcessed(randomLongBetween(maxOps, Long.MAX_VALUE)), equalTo(false));
         assertThat(tracker.processedSeqNo.size(), isOneOf(0, 1));
         if (tracker.processedSeqNo.size() == 1) {
-            assertThat(tracker.processedSeqNo.keys().iterator().next().value, equalTo(tracker.checkpoint / BIT_SET_SIZE));
+            assertThat(tracker.processedSeqNo.keys().iterator().next().value, equalTo(tracker.processedCheckpoint.get() / BIT_SET_SIZE));
         }
     }
 
@@ -234,7 +270,7 @@ public void testWaitForOpsToComplete() throws BrokenBarrierException, Interrupte
             try {
                 // sychronize starting with the test thread
                 barrier.await();
-                tracker.waitForOpsToComplete(seqNo);
+                tracker.waitForProcessedOpsToComplete(seqNo);
                 complete.set(true);
                 // synchronize with the test thread checking if we are no longer waiting
                 barrier.await();
@@ -251,11 +287,11 @@ public void testWaitForOpsToComplete() throws BrokenBarrierException, Interrupte
         final List<Integer> elements = IntStream.rangeClosed(0, seqNo).boxed().collect(Collectors.toList());
         Randomness.shuffle(elements);
         for (int i = 0; i < elements.size() - 1; i++) {
-            tracker.markSeqNoAsCompleted(elements.get(i));
+            tracker.markSeqNoAsProcessed(elements.get(i));
             assertFalse(complete.get());
         }
 
-        tracker.markSeqNoAsCompleted(elements.get(elements.size() - 1));
+        tracker.markSeqNoAsProcessed(elements.get(elements.size() - 1));
         // synchronize with the waiting thread to mark that it is complete
         barrier.await();
         assertTrue(complete.get());
@@ -268,17 +304,17 @@ public void testContains() {
         final long localCheckpoint = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, maxSeqNo);
         final LocalCheckpointTracker tracker = new LocalCheckpointTracker(maxSeqNo, localCheckpoint);
         if (localCheckpoint >= 0) {
-            assertThat(tracker.contains(randomLongBetween(0, localCheckpoint)), equalTo(true));
+            assertThat(tracker.hasProcessed(randomLongBetween(0, localCheckpoint)), equalTo(true));
         }
-        assertThat(tracker.contains(randomLongBetween(localCheckpoint + 1, Long.MAX_VALUE)), equalTo(false));
+        assertThat(tracker.hasProcessed(randomLongBetween(localCheckpoint + 1, Long.MAX_VALUE)), equalTo(false));
         final int numOps = between(1, 100);
         final List<Long> seqNos = new ArrayList<>();
         for (int i = 0; i < numOps; i++) {
             long seqNo = randomLongBetween(0, 1000);
             seqNos.add(seqNo);
-            tracker.markSeqNoAsCompleted(seqNo);
+            tracker.markSeqNoAsProcessed(seqNo);
         }
         final long seqNo = randomNonNegativeLong();
-        assertThat(tracker.contains(seqNo), equalTo(seqNo <= localCheckpoint || seqNos.contains(seqNo)));
+        assertThat(tracker.hasProcessed(seqNo), equalTo(seqNo <= localCheckpoint || seqNos.contains(seqNo)));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java
index 05ca0a5ea3006..70b728a4a2022 100644
--- a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java
+++ b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java
@@ -716,7 +716,7 @@ public void testPrimaryContextHandoff() throws IOException {
                 newPrimary.shardAllocationId,
                 Math.max(SequenceNumbers.NO_OPS_PERFORMED, oldPrimary.getGlobalCheckpoint() + randomInt(5)));
         oldPrimary.updateGlobalCheckpointForShard(newPrimary.shardAllocationId, oldPrimary.getGlobalCheckpoint());
-        ReplicationTracker.PrimaryContext primaryContext = oldPrimary.startRelocationHandoff();
+        ReplicationTracker.PrimaryContext primaryContext = oldPrimary.startRelocationHandoff(newPrimary.shardAllocationId);
 
         if (randomBoolean()) {
             // cluster state update after primary context handoff
@@ -742,7 +742,7 @@ public void testPrimaryContextHandoff() throws IOException {
             }
 
             // do another handoff
-            primaryContext = oldPrimary.startRelocationHandoff();
+            primaryContext = oldPrimary.startRelocationHandoff(newPrimary.shardAllocationId);
         }
 
         // send primary context through the wire
diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java
index f1c05c565f8ef..2be10a80fe793 100644
--- a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java
@@ -17,14 +17,10 @@
 
 package org.elasticsearch.index.seqno;
 
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.ActionTestUtils;
-import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -35,29 +31,21 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.IndexShardClosedException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportService;
-import org.mockito.ArgumentCaptor;
 
 import java.util.Collections;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import static org.elasticsearch.mock.orig.Mockito.verifyNoMoreInteractions;
 import static org.elasticsearch.mock.orig.Mockito.when;
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
-import static org.hamcrest.Matchers.arrayContaining;
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.sameInstance;
-import static org.mockito.Matchers.same;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 
@@ -173,73 +161,6 @@ public void testRetentionLeaseBackgroundSyncActionOnReplica() throws WriteStateE
         assertTrue(success.get());
     }
 
-    public void testRetentionLeaseSyncExecution() {
-        final IndicesService indicesService = mock(IndicesService.class);
-
-        final Index index = new Index("index", "uuid");
-        final IndexService indexService = mock(IndexService.class);
-        when(indicesService.indexServiceSafe(index)).thenReturn(indexService);
-
-        final int id = randomIntBetween(0, 4);
-        final IndexShard indexShard = mock(IndexShard.class);
-        when(indexService.getShard(id)).thenReturn(indexShard);
-
-        final ShardId shardId = new ShardId(index, id);
-        when(indexShard.shardId()).thenReturn(shardId);
-
-        final Logger retentionLeaseSyncActionLogger = mock(Logger.class);
-
-        final RetentionLeases retentionLeases = mock(RetentionLeases.class);
-        final AtomicBoolean invoked = new AtomicBoolean();
-        final RetentionLeaseBackgroundSyncAction action = new RetentionLeaseBackgroundSyncAction(
-                Settings.EMPTY,
-                transportService,
-                clusterService,
-                indicesService,
-                threadPool,
-                shardStateAction,
-                new ActionFilters(Collections.emptySet()),
-                new IndexNameExpressionResolver()) {
-
-            @Override
-            protected void doExecute(Task task, Request request, ActionListener<ReplicationResponse> listener) {
-                assertTrue(threadPool.getThreadContext().isSystemContext());
-                assertThat(request.shardId(), sameInstance(indexShard.shardId()));
-                assertThat(request.getRetentionLeases(), sameInstance(retentionLeases));
-                if (randomBoolean()) {
-                    listener.onResponse(new ReplicationResponse());
-                } else {
-                    final Exception e = randomFrom(
-                            new AlreadyClosedException("closed"),
-                            new IndexShardClosedException(indexShard.shardId()),
-                            new TransportException(randomFrom(
-                                    "failed",
-                                    "TransportService is closed stopped can't send request",
-                                    "transport stopped, action: indices:admin/seq_no/retention_lease_background_sync[p]")),
-                            new RuntimeException("failed"));
-                    listener.onFailure(e);
-                    if (e.getMessage().equals("failed")) {
-                        final ArgumentCaptor<ParameterizedMessage> captor = ArgumentCaptor.forClass(ParameterizedMessage.class);
-                        verify(retentionLeaseSyncActionLogger).warn(captor.capture(), same(e));
-                        final ParameterizedMessage message = captor.getValue();
-                        assertThat(message.getFormat(), equalTo("{} retention lease background sync failed"));
-                        assertThat(message.getParameters(), arrayContaining(indexShard.shardId()));
-                    }
-                    verifyNoMoreInteractions(retentionLeaseSyncActionLogger);
-                }
-                invoked.set(true);
-            }
-
-            @Override
-            protected Logger getLogger() {
-                return retentionLeaseSyncActionLogger;
-            }
-        };
-
-        action.backgroundSync(indexShard.shardId(), retentionLeases);
-        assertTrue(invoked.get());
-    }
-
     public void testBlocks() {
         final IndicesService indicesService = mock(IndicesService.class);
 
diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionTests.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionTests.java
index 73387d69b1e4c..14dbb1cc5eed8 100644
--- a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionTests.java
+++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionTests.java
@@ -17,9 +17,6 @@
 
 package org.elasticsearch.index.seqno;
 
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.ActionTestUtils;
@@ -33,27 +30,20 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.IndexShardClosedException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.mockito.ArgumentCaptor;
 
 import java.util.Collections;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import static org.elasticsearch.mock.orig.Mockito.verifyNoMoreInteractions;
 import static org.elasticsearch.mock.orig.Mockito.when;
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
-import static org.hamcrest.Matchers.arrayContaining;
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.sameInstance;
-import static org.mockito.Matchers.same;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 
@@ -166,70 +156,6 @@ public void testRetentionLeaseSyncActionOnReplica() throws WriteStateException {
         assertTrue(success.get());
     }
 
-    public void testRetentionLeaseSyncExecution() {
-        final IndicesService indicesService = mock(IndicesService.class);
-
-        final Index index = new Index("index", "uuid");
-        final IndexService indexService = mock(IndexService.class);
-        when(indicesService.indexServiceSafe(index)).thenReturn(indexService);
-
-        final int id = randomIntBetween(0, 4);
-        final IndexShard indexShard = mock(IndexShard.class);
-        when(indexService.getShard(id)).thenReturn(indexShard);
-
-        final ShardId shardId = new ShardId(index, id);
-        when(indexShard.shardId()).thenReturn(shardId);
-
-        final Logger retentionLeaseSyncActionLogger = mock(Logger.class);
-
-        final RetentionLeases retentionLeases = mock(RetentionLeases.class);
-        final AtomicBoolean invoked = new AtomicBoolean();
-        final RetentionLeaseSyncAction action = new RetentionLeaseSyncAction(
-                Settings.EMPTY,
-                transportService,
-                clusterService,
-                indicesService,
-                threadPool,
-                shardStateAction,
-                new ActionFilters(Collections.emptySet()),
-                new IndexNameExpressionResolver()) {
-
-            @Override
-            protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
-                assertTrue(threadPool.getThreadContext().isSystemContext());
-                assertThat(request.shardId(), sameInstance(indexShard.shardId()));
-                assertThat(request.getRetentionLeases(), sameInstance(retentionLeases));
-                if (randomBoolean()) {
-                    listener.onResponse(new Response());
-                } else {
-                    final Exception e = randomFrom(
-                            new AlreadyClosedException("closed"),
-                            new IndexShardClosedException(indexShard.shardId()),
-                            new RuntimeException("failed"));
-                    listener.onFailure(e);
-                    if (e instanceof AlreadyClosedException == false && e instanceof IndexShardClosedException == false) {
-                        final ArgumentCaptor<ParameterizedMessage> captor = ArgumentCaptor.forClass(ParameterizedMessage.class);
-                        verify(retentionLeaseSyncActionLogger).warn(captor.capture(), same(e));
-                        final ParameterizedMessage message = captor.getValue();
-                        assertThat(message.getFormat(), equalTo("{} retention lease sync failed"));
-                        assertThat(message.getParameters(), arrayContaining(indexShard.shardId()));
-                    }
-                    verifyNoMoreInteractions(retentionLeaseSyncActionLogger);
-                }
-                invoked.set(true);
-            }
-
-            @Override
-            protected Logger getLogger() {
-                return retentionLeaseSyncActionLogger;
-            }
-        };
-
-        // execution happens on the test thread, so no need to register an actual listener to callback
-        action.sync(indexShard.shardId(), retentionLeases, ActionListener.wrap(() -> {}));
-        assertTrue(invoked.get());
-    }
-
     public void testBlocks() {
         final IndicesService indicesService = mock(IndicesService.class);
 
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexReaderWrapperTests.java
similarity index 65%
rename from server/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java
rename to server/src/test/java/org/elasticsearch/index/shard/IndexReaderWrapperTests.java
index 7a422e82c2202..6abcacc1581c7 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexReaderWrapperTests.java
@@ -28,32 +28,24 @@
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.TotalHitCountCollector;
-import org.apache.lucene.search.Weight;
 import org.apache.lucene.store.Directory;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.engine.EngineException;
-import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import static org.hamcrest.Matchers.equalTo;
-
-public class IndexSearcherWrapperTests extends ESTestCase {
+public class IndexReaderWrapperTests extends ESTestCase {
 
     public void testReaderCloseListenerIsCalled() throws IOException {
         Directory dir = newDirectory();
@@ -67,23 +59,13 @@ public void testReaderCloseListenerIsCalled() throws IOException {
         IndexSearcher searcher = new IndexSearcher(open);
         assertEquals(1, searcher.search(new TermQuery(new Term("field", "doc")), 1).totalHits.value);
         final AtomicInteger closeCalls = new AtomicInteger(0);
-        IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {
-            @Override
-            public DirectoryReader wrap(DirectoryReader reader) throws IOException {
-                return new FieldMaskingReader("field", reader, closeCalls);
-            }
-
-            @Override
-            public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
-                return searcher;
-            }
-
-        };
+        CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper =
+            reader -> new FieldMaskingReader("field", reader, closeCalls);
         final int sourceRefCount = open.getRefCount();
         final AtomicInteger count = new AtomicInteger();
         final AtomicInteger outerCount = new AtomicInteger();
         final AtomicBoolean closeCalled = new AtomicBoolean(false);
-        final Engine.Searcher wrap =  wrapper.wrap(new Engine.Searcher("foo", searcher, () -> closeCalled.set(true)));
+        final Engine.Searcher wrap =  IndexShard.wrapSearcher(new Engine.Searcher("foo", searcher, () -> closeCalled.set(true)), wrapper);
         assertEquals(1, wrap.reader().getRefCount());
         ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), key -> {
             if (key == open.getReaderCacheHelper().getKey()) {
@@ -118,20 +100,11 @@ public void testIsCacheable() throws IOException {
         assertEquals(1, searcher.search(new TermQuery(new Term("field", "doc")), 1).totalHits.value);
         searcher.setSimilarity(iwc.getSimilarity());
         final AtomicInteger closeCalls = new AtomicInteger(0);
-        IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {
-            @Override
-            public DirectoryReader wrap(DirectoryReader reader) throws IOException {
-                return new FieldMaskingReader("field", reader, closeCalls);
-            }
-
-            @Override
-            public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
-                return searcher;
-            }
-        };
+        CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper =
+            reader -> new FieldMaskingReader("field", reader, closeCalls);
         final ConcurrentHashMap<Object, TopDocs> cache = new ConcurrentHashMap<>();
         AtomicBoolean closeCalled = new AtomicBoolean(false);
-        try (Engine.Searcher wrap = wrapper.wrap(new Engine.Searcher("foo", searcher, () -> closeCalled.set(true)))) {
+        try (Engine.Searcher wrap = IndexShard.wrapSearcher(new Engine.Searcher("foo", searcher, () -> closeCalled.set(true)), wrapper)) {
             ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), key -> {
                 cache.remove(key);
             });
@@ -159,58 +132,14 @@ public void testNoWrap() throws IOException {
         IndexSearcher searcher = new IndexSearcher(open);
         assertEquals(1, searcher.search(new TermQuery(new Term("field", "doc")), 1).totalHits.value);
         searcher.setSimilarity(iwc.getSimilarity());
-        IndexSearcherWrapper wrapper = new IndexSearcherWrapper();
+        CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper = directoryReader -> directoryReader;
         try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher, open::close)) {
-            final Engine.Searcher wrap = wrapper.wrap(engineSearcher);
+            final Engine.Searcher wrap = IndexShard.wrapSearcher(engineSearcher, wrapper);
             assertSame(wrap, engineSearcher);
         }
         IOUtils.close(writer, dir);
     }
 
-    public void testWrapVisibility() throws IOException {
-        Directory dir = newDirectory();
-        IndexWriterConfig iwc = newIndexWriterConfig();
-        IndexWriter writer = new IndexWriter(dir, iwc);
-        Document doc = new Document();
-        doc.add(new StringField("id", "1", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
-        doc.add(new TextField("field", "doc", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
-        writer.addDocument(doc);
-        DirectoryReader open = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1));
-        IndexSearcher searcher = new IndexSearcher(open);
-        assertEquals(1, searcher.search(new TermQuery(new Term("field", "doc")), 1).totalHits.value);
-        IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {
-            @Override
-            public DirectoryReader wrap(DirectoryReader reader) throws IOException {
-                return reader;
-            }
-
-            @Override
-            public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
-                return new IndexSearcher(searcher.getIndexReader()) {
-                    @Override
-                    protected void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
-                        throw new IllegalStateException("boum");
-                    }
-                };
-            }
-
-        };
-        final AtomicBoolean closeCalled = new AtomicBoolean(false);
-        final Engine.Searcher wrap =  wrapper.wrap(new Engine.Searcher("foo", searcher, () -> closeCalled.set(true)));
-        assertEquals(1, wrap.reader().getRefCount());
-        ContextIndexSearcher contextSearcher = new ContextIndexSearcher(wrap, wrap.searcher().getQueryCache(),
-            wrap.searcher().getQueryCachingPolicy());
-        IllegalStateException exc = expectThrows(IllegalStateException.class,
-            () -> contextSearcher.search(new TermQuery(new Term("field", "doc")), new TotalHitCountCollector()));
-        assertThat(exc.getMessage(), equalTo("boum"));
-        wrap.close();
-        assertFalse("wrapped reader is closed", wrap.reader().tryIncRef());
-        assertTrue(closeCalled.get());
-
-        IOUtils.close(open, writer, dir);
-        assertEquals(0, open.getRefCount());
-    }
-
     private static class FieldMaskingReader extends FilterDirectoryReader {
         private final String field;
         private final AtomicInteger closeCalls;
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java
index d0bc06ab1cc7f..c55a1dd61be1c 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.index.shard;
 
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
@@ -40,6 +41,7 @@
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
@@ -81,7 +83,6 @@
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPoolStats;
 import org.junit.Assert;
@@ -424,7 +425,6 @@ public void testMaybeRollTranslogGeneration() throws Exception {
         }
     }
 
-    @TestLogging("org.elasticsearch.index.shard:TRACE,org.elasticsearch.index.engine:TRACE")
     public void testStressMaybeFlushOrRollTranslogGeneration() throws Exception {
         createIndex("test");
         ensureGreen();
@@ -530,7 +530,7 @@ public void testShardHasMemoryBufferOnTranslogRecover() throws Throwable {
         client().prepareIndex("test", "test", "1").setSource("{\"foo\" : \"bar\"}", XContentType.JSON)
             .setRefreshPolicy(IMMEDIATE).get();
 
-        IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {};
+        CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper = directoryReader -> directoryReader;
         shard.close("simon says", false);
         AtomicReference<IndexShard> shardRef = new AtomicReference<>();
         List<Exception> failures = new ArrayList<>();
@@ -648,10 +648,10 @@ public static final IndexShard recoverShard(IndexShard newShard) throws IOExcept
     }
 
     public static final IndexShard newIndexShard(
-            final IndexService indexService,
-            final IndexShard shard,IndexSearcherWrapper wrapper,
-            final CircuitBreakerService cbs,
-            final IndexingOperationListener... listeners) throws IOException {
+        final IndexService indexService,
+        final IndexShard shard, CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper,
+        final CircuitBreakerService cbs,
+        final IndexingOperationListener... listeners) throws IOException {
         ShardRouting initializingShardRouting = getInitializingShardRouting(shard.routingEntry());
         return new IndexShard(
                 initializingShardRouting,
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 5187ef37fcdf8..264a8840d5ff2 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -23,7 +23,6 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.AlreadyClosedException;
@@ -54,6 +53,7 @@
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.breaker.CircuitBreaker;
@@ -82,7 +82,6 @@
 import org.elasticsearch.index.engine.DocIdSeqNoAndSource;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.Engine.DeleteResult;
-import org.elasticsearch.index.engine.EngineException;
 import org.elasticsearch.index.engine.EngineTestCase;
 import org.elasticsearch.index.engine.InternalEngine;
 import org.elasticsearch.index.engine.InternalEngineFactory;
@@ -419,7 +418,7 @@ public void testPrimaryPromotionDelaysOperations() throws IOException, BrokenBar
                 }
                 indexShard.acquireReplicaOperationPermit(
                         indexShard.getPendingPrimaryTerm(),
-                        indexShard.getGlobalCheckpoint(),
+                        indexShard.getLastKnownGlobalCheckpoint(),
                         indexShard.getMaxSeqNoOfUpdatesOrDeletes(),
                         new ActionListener<Releasable>() {
                             @Override
@@ -716,7 +715,7 @@ public void onFailure(final Exception e) {
 
         if (Assertions.ENABLED && indexShard.routingEntry().isRelocationTarget() == false) {
             assertThat(expectThrows(AssertionError.class, () -> indexShard.acquireReplicaOperationPermit(pendingPrimaryTerm,
-                indexShard.getGlobalCheckpoint(), indexShard.getMaxSeqNoOfUpdatesOrDeletes(), new ActionListener<Releasable>() {
+                indexShard.getLastKnownGlobalCheckpoint(), indexShard.getMaxSeqNoOfUpdatesOrDeletes(), new ActionListener<Releasable>() {
                     @Override
                     public void onResponse(Releasable releasable) {
                         fail();
@@ -842,7 +841,7 @@ private Releasable acquirePrimaryOperationPermitBlockingly(IndexShard indexShard
     private Releasable acquireReplicaOperationPermitBlockingly(IndexShard indexShard, long opPrimaryTerm)
         throws ExecutionException, InterruptedException {
         PlainActionFuture<Releasable> fut = new PlainActionFuture<>();
-        indexShard.acquireReplicaOperationPermit(opPrimaryTerm, indexShard.getGlobalCheckpoint(),
+        indexShard.acquireReplicaOperationPermit(opPrimaryTerm, indexShard.getLastKnownGlobalCheckpoint(),
             randomNonNegativeLong(), fut, ThreadPool.Names.WRITE, "");
         return fut.get();
     }
@@ -876,7 +875,7 @@ public void testOperationPermitOnReplicaShards() throws Exception {
                 routing = newShardRouting(routing.shardId(), routing.currentNodeId(), "otherNode",
                     true, ShardRoutingState.RELOCATING, AllocationId.newRelocation(routing.allocationId()));
                 IndexShardTestCase.updateRoutingEntry(indexShard, routing);
-                indexShard.relocated(primaryContext -> {});
+                indexShard.relocated(routing.getTargetRelocatingShard().allocationId().getId(), primaryContext -> {});
                 engineClosed = false;
                 break;
             }
@@ -921,18 +920,19 @@ public void testOperationPermitOnReplicaShards() throws Exception {
             final long newPrimaryTerm = primaryTerm + 1 + randomInt(20);
             if (engineClosed == false) {
                 assertThat(indexShard.getLocalCheckpoint(), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
-                assertThat(indexShard.getGlobalCheckpoint(), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
+                assertThat(indexShard.getLastKnownGlobalCheckpoint(), equalTo(SequenceNumbers.NO_OPS_PERFORMED));
             }
             final long newGlobalCheckPoint;
             if (engineClosed || randomBoolean()) {
                 newGlobalCheckPoint = SequenceNumbers.NO_OPS_PERFORMED;
             } else {
-                long localCheckPoint = indexShard.getGlobalCheckpoint() + randomInt(100);
+                long localCheckPoint = indexShard.getLastKnownGlobalCheckpoint() + randomInt(100);
                 // advance local checkpoint
                 for (int i = 0; i <= localCheckPoint; i++) {
                     indexShard.markSeqNoAsNoop(i, "dummy doc");
                 }
-                newGlobalCheckPoint = randomIntBetween((int) indexShard.getGlobalCheckpoint(), (int) localCheckPoint);
+                indexShard.sync(); // advance local checkpoint
+                newGlobalCheckPoint = randomIntBetween((int) indexShard.getLastKnownGlobalCheckpoint(), (int) localCheckPoint);
             }
             final long expectedLocalCheckpoint;
             if (newGlobalCheckPoint == UNASSIGNED_SEQ_NO) {
@@ -953,7 +953,7 @@ public void onResponse(Releasable releasable) {
                         assertThat(indexShard.getPendingPrimaryTerm(), equalTo(newPrimaryTerm));
                         assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(newPrimaryTerm));
                         assertThat(indexShard.getLocalCheckpoint(), equalTo(expectedLocalCheckpoint));
-                        assertThat(indexShard.getGlobalCheckpoint(), equalTo(newGlobalCheckPoint));
+                        assertThat(indexShard.getLastKnownGlobalCheckpoint(), equalTo(newGlobalCheckPoint));
                         onResponse.set(true);
                         releasable.close();
                         finish();
@@ -1022,7 +1022,7 @@ private void finish() {
                         // and one after replaying translog (upto the global checkpoint); otherwise we roll translog once.
                         either(equalTo(translogGen + 1)).or(equalTo(translogGen + 2)));
                     assertThat(indexShard.getLocalCheckpoint(), equalTo(expectedLocalCheckpoint));
-                    assertThat(indexShard.getGlobalCheckpoint(), equalTo(newGlobalCheckPoint));
+                    assertThat(indexShard.getLastKnownGlobalCheckpoint(), equalTo(newGlobalCheckPoint));
                 }
             }
             thread.join();
@@ -1050,7 +1050,7 @@ public void onFailure(Exception e) {
             };
 
             final long oldPrimaryTerm = indexShard.getPendingPrimaryTerm() - 1;
-            randomReplicaOperationPermitAcquisition(indexShard, oldPrimaryTerm, indexShard.getGlobalCheckpoint(),
+            randomReplicaOperationPermitAcquisition(indexShard, oldPrimaryTerm, indexShard.getLastKnownGlobalCheckpoint(),
                 randomNonNegativeLong(), onLockAcquired, "");
             latch.await();
             assertFalse(onResponse.get());
@@ -1071,7 +1071,7 @@ public void testAcquireReplicaPermitAdvanceMaxSeqNoOfUpdates() throws Exception
 
         long newMaxSeqNoOfUpdates = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE);
         PlainActionFuture<Releasable> fut = new PlainActionFuture<>();
-        randomReplicaOperationPermitAcquisition(replica, replica.getOperationPrimaryTerm(), replica.getGlobalCheckpoint(),
+        randomReplicaOperationPermitAcquisition(replica, replica.getOperationPrimaryTerm(), replica.getLastKnownGlobalCheckpoint(),
             newMaxSeqNoOfUpdates, fut, "");
         try (Releasable ignored = fut.actionGet()) {
             assertThat(replica.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(Math.max(currentMaxSeqNoOfUpdates, newMaxSeqNoOfUpdates)));
@@ -1115,19 +1115,22 @@ public void testGlobalCheckpointSync() throws IOException {
         final String replicaAllocationId = replicaShard.routingEntry().allocationId().getId();
         primaryShard.updateLocalCheckpointForShard(replicaAllocationId, replicaLocalCheckpoint);
 
-        // initialize the local knowledge on the primary of the global checkpoint on the replica shard
-        final int replicaGlobalCheckpoint =
-                randomIntBetween(Math.toIntExact(SequenceNumbers.NO_OPS_PERFORMED), Math.toIntExact(primaryShard.getGlobalCheckpoint()));
+        // initialize the local knowledge on the primary of the persisted global checkpoint on the replica shard
+        final int replicaGlobalCheckpoint = randomIntBetween(Math.toIntExact(SequenceNumbers.NO_OPS_PERFORMED),
+            Math.toIntExact(primaryShard.getLastKnownGlobalCheckpoint()));
         primaryShard.updateGlobalCheckpointForShard(replicaAllocationId, replicaGlobalCheckpoint);
 
+        // initialize the local knowledge on the primary of the persisted global checkpoint on the primary
+        primaryShard.updateGlobalCheckpointForShard(shardRouting.allocationId().getId(), primaryShard.getLastKnownGlobalCheckpoint());
+
         // simulate a background maybe sync; it should only run if the knowledge on the replica of the global checkpoint lags the primary
         primaryShard.maybeSyncGlobalCheckpoint("test");
         assertThat(
                 synced.get(),
-                equalTo(maxSeqNo == primaryShard.getGlobalCheckpoint() && (replicaGlobalCheckpoint < checkpoint)));
+                equalTo(maxSeqNo == primaryShard.getLastKnownGlobalCheckpoint() && (replicaGlobalCheckpoint < checkpoint)));
 
         // simulate that the background sync advanced the global checkpoint on the replica
-        primaryShard.updateGlobalCheckpointForShard(replicaAllocationId, primaryShard.getGlobalCheckpoint());
+        primaryShard.updateGlobalCheckpointForShard(replicaAllocationId, primaryShard.getLastKnownGlobalCheckpoint());
 
         // reset our boolean so that we can assert after another simulated maybe sync
         synced.set(false);
@@ -1288,7 +1291,7 @@ public void testConcurrentTermIncreaseOnReplicaShard() throws BrokenBarrierExcep
             }
             indexShard.acquireReplicaOperationPermit(
                     primaryTerm + increment,
-                    indexShard.getGlobalCheckpoint(),
+                    indexShard.getLastKnownGlobalCheckpoint(),
                     randomNonNegativeLong(),
                     new ActionListener<Releasable>() {
                         @Override
@@ -1735,12 +1738,13 @@ public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) {
 
     public void testLockingBeforeAndAfterRelocated() throws Exception {
         final IndexShard shard = newStartedShard(true);
-        IndexShardTestCase.updateRoutingEntry(shard, ShardRoutingHelper.relocate(shard.routingEntry(), "other_node"));
+        final ShardRouting routing = ShardRoutingHelper.relocate(shard.routingEntry(), "other_node");
+        IndexShardTestCase.updateRoutingEntry(shard, routing);
         CountDownLatch latch = new CountDownLatch(1);
         Thread recoveryThread = new Thread(() -> {
             latch.countDown();
             try {
-                shard.relocated(primaryContext -> {});
+                shard.relocated(routing.getTargetRelocatingShard().allocationId().getId(), primaryContext -> {});
             } catch (InterruptedException e) {
                 throw new RuntimeException(e);
             }
@@ -1765,13 +1769,14 @@ public void testLockingBeforeAndAfterRelocated() throws Exception {
 
     public void testDelayedOperationsBeforeAndAfterRelocated() throws Exception {
         final IndexShard shard = newStartedShard(true);
-        IndexShardTestCase.updateRoutingEntry(shard, ShardRoutingHelper.relocate(shard.routingEntry(), "other_node"));
+        final ShardRouting routing = ShardRoutingHelper.relocate(shard.routingEntry(), "other_node");
+        IndexShardTestCase.updateRoutingEntry(shard, routing);
         final CountDownLatch startRecovery = new CountDownLatch(1);
         final CountDownLatch relocationStarted = new CountDownLatch(1);
         Thread recoveryThread = new Thread(() -> {
             try {
                 startRecovery.await();
-                shard.relocated(primaryContext -> relocationStarted.countDown());
+                shard.relocated(routing.getTargetRelocatingShard().allocationId().getId(), primaryContext -> relocationStarted.countDown());
             } catch (InterruptedException e) {
                 throw new RuntimeException(e);
             }
@@ -1836,7 +1841,8 @@ public void onFailure(Exception e) {
     public void testStressRelocated() throws Exception {
         final IndexShard shard = newStartedShard(true);
         assertFalse(shard.isRelocatedPrimary());
-        IndexShardTestCase.updateRoutingEntry(shard, ShardRoutingHelper.relocate(shard.routingEntry(), "other_node"));
+        final ShardRouting routing = ShardRoutingHelper.relocate(shard.routingEntry(), "other_node");
+        IndexShardTestCase.updateRoutingEntry(shard, routing);
         final int numThreads = randomIntBetween(2, 4);
         Thread[] indexThreads = new Thread[numThreads];
         CountDownLatch allPrimaryOperationLocksAcquired = new CountDownLatch(numThreads);
@@ -1858,7 +1864,7 @@ public void run() {
         AtomicBoolean relocated = new AtomicBoolean();
         final Thread recoveryThread = new Thread(() -> {
             try {
-                shard.relocated(primaryContext -> {});
+                shard.relocated(routing.getTargetRelocatingShard().allocationId().getId(), primaryContext -> {});
             } catch (InterruptedException e) {
                 throw new RuntimeException(e);
             }
@@ -1891,8 +1897,9 @@ public void run() {
     public void testRelocatedShardCanNotBeRevived() throws IOException, InterruptedException {
         final IndexShard shard = newStartedShard(true);
         final ShardRouting originalRouting = shard.routingEntry();
-        IndexShardTestCase.updateRoutingEntry(shard, ShardRoutingHelper.relocate(originalRouting, "other_node"));
-        shard.relocated(primaryContext -> {});
+        final ShardRouting routing = ShardRoutingHelper.relocate(originalRouting, "other_node");
+        IndexShardTestCase.updateRoutingEntry(shard, routing);
+        shard.relocated(routing.getTargetRelocatingShard().allocationId().getId(), primaryContext -> {});
         expectThrows(IllegalIndexShardStateException.class, () -> IndexShardTestCase.updateRoutingEntry(shard, originalRouting));
         closeShards(shard);
     }
@@ -1900,16 +1907,19 @@ public void testRelocatedShardCanNotBeRevived() throws IOException, InterruptedE
     public void testShardCanNotBeMarkedAsRelocatedIfRelocationCancelled() throws IOException {
         final IndexShard shard = newStartedShard(true);
         final ShardRouting originalRouting = shard.routingEntry();
-        IndexShardTestCase.updateRoutingEntry(shard, ShardRoutingHelper.relocate(originalRouting, "other_node"));
+        final ShardRouting relocationRouting = ShardRoutingHelper.relocate(originalRouting, "other_node");
+        IndexShardTestCase.updateRoutingEntry(shard, relocationRouting);
         IndexShardTestCase.updateRoutingEntry(shard, originalRouting);
-        expectThrows(IllegalIndexShardStateException.class, () ->  shard.relocated(primaryContext -> {}));
+        expectThrows(IllegalIndexShardStateException.class,
+            () ->  shard.relocated(relocationRouting.getTargetRelocatingShard().allocationId().getId(), primaryContext -> {}));
         closeShards(shard);
     }
 
     public void testRelocatedShardCanNotBeRevivedConcurrently() throws IOException, InterruptedException, BrokenBarrierException {
         final IndexShard shard = newStartedShard(true);
         final ShardRouting originalRouting = shard.routingEntry();
-        IndexShardTestCase.updateRoutingEntry(shard, ShardRoutingHelper.relocate(originalRouting, "other_node"));
+        final ShardRouting relocationRouting = ShardRoutingHelper.relocate(originalRouting, "other_node");
+        IndexShardTestCase.updateRoutingEntry(shard, relocationRouting);
         CyclicBarrier cyclicBarrier = new CyclicBarrier(3);
         AtomicReference<Exception> relocationException = new AtomicReference<>();
         Thread relocationThread = new Thread(new AbstractRunnable() {
@@ -1921,7 +1931,7 @@ public void onFailure(Exception e) {
             @Override
             protected void doRun() throws Exception {
                 cyclicBarrier.await();
-                shard.relocated(primaryContext -> {});
+                shard.relocated(relocationRouting.getTargetRelocatingShard().allocationId().getId(), primaryContext -> {});
             }
         });
         relocationThread.start();
@@ -1949,7 +1959,8 @@ protected void doRun() throws Exception {
             assertThat(relocationException.get(), nullValue());
         } else {
             logger.debug("shard relocation was cancelled");
-            assertThat(relocationException.get(), instanceOf(IllegalIndexShardStateException.class));
+            assertThat(relocationException.get(),
+                either(instanceOf(IllegalIndexShardStateException.class)).or(instanceOf(IllegalStateException.class)));
             assertThat("current routing:" + shard.routingEntry(), shard.routingEntry().relocating(), equalTo(false));
             assertThat(cancellingException.get(), nullValue());
 
@@ -1957,6 +1968,25 @@ protected void doRun() throws Exception {
         closeShards(shard);
     }
 
+    public void testRelocateMissingTarget() throws Exception {
+        final IndexShard shard = newStartedShard(true);
+        final ShardRouting original = shard.routingEntry();
+        final ShardRouting toNode1 = ShardRoutingHelper.relocate(original, "node_1");
+        IndexShardTestCase.updateRoutingEntry(shard, toNode1);
+        IndexShardTestCase.updateRoutingEntry(shard, original);
+        final ShardRouting toNode2 = ShardRoutingHelper.relocate(original, "node_2");
+        IndexShardTestCase.updateRoutingEntry(shard, toNode2);
+        final AtomicBoolean relocated = new AtomicBoolean();
+        final IllegalStateException error = expectThrows(IllegalStateException.class,
+            () -> shard.relocated(toNode1.getTargetRelocatingShard().allocationId().getId(), ctx -> relocated.set(true)));
+        assertThat(error.getMessage(), equalTo("relocation target [" + toNode1.getTargetRelocatingShard().allocationId().getId()
+            + "] is no longer part of the replication group"));
+        assertFalse(relocated.get());
+        shard.relocated(toNode2.getTargetRelocatingShard().allocationId().getId(), ctx -> relocated.set(true));
+        assertTrue(relocated.get());
+        closeShards(shard);
+    }
+
     public void testRecoverFromStoreWithOutOfOrderDelete() throws IOException {
         /*
          * The flow of this test:
@@ -1983,6 +2013,7 @@ public void testRecoverFromStoreWithOutOfOrderDelete() throws IOException {
             new SourceToParse(shard.shardId().getIndexName(), "_doc", "id-2", new BytesArray("{}"), XContentType.JSON));
         shard.applyIndexOperationOnReplica(5, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
             new SourceToParse(shard.shardId().getIndexName(), "_doc", "id-5", new BytesArray("{}"), XContentType.JSON));
+        shard.sync(); // advance local checkpoint
 
         final int translogOps;
         if (randomBoolean()) {
@@ -2261,7 +2292,7 @@ public void testRecoveryFailsAfterMovingToRelocatedState() throws InterruptedExc
         assertThat(shard.state(), equalTo(IndexShardState.STARTED));
         ShardRouting inRecoveryRouting = ShardRoutingHelper.relocate(origRouting, "some_node");
         IndexShardTestCase.updateRoutingEntry(shard, inRecoveryRouting);
-        shard.relocated(primaryContext -> {});
+        shard.relocated(inRecoveryRouting.getTargetRelocatingShard().allocationId().getId(), primaryContext -> {});
         assertTrue(shard.isRelocatedPrimary());
         try {
             IndexShardTestCase.updateRoutingEntry(shard, origRouting);
@@ -2329,7 +2360,7 @@ public void restoreShard(Store store, SnapshotId snapshotId,
         closeShards(target);
     }
 
-    public void testSearcherWrapperIsUsed() throws IOException {
+    public void testReaderWrapperIsUsed() throws IOException {
         IndexShard shard = newStartedShard(true);
         indexDoc(shard, "_doc", "0", "{\"foo\" : \"bar\"}");
         indexDoc(shard, "_doc", "1", "{\"foobar\" : \"bar\"}");
@@ -2347,17 +2378,7 @@ public void testSearcherWrapperIsUsed() throws IOException {
             search = searcher.searcher().search(new TermQuery(new Term("foobar", "bar")), 10);
             assertEquals(search.totalHits.value, 1);
         }
-        IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {
-            @Override
-            public DirectoryReader wrap(DirectoryReader reader) throws IOException {
-                return new FieldMaskingReader("foo", reader);
-            }
-
-            @Override
-            public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
-                return searcher;
-            }
-        };
+        CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper = reader -> new FieldMaskingReader("foo", reader);
         closeShards(shard);
         IndexShard newShard = newShard(
                 ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.ExistingStoreRecoverySource.INSTANCE),
@@ -2389,18 +2410,8 @@ public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
         closeShards(newShard);
     }
 
-    public void testSearcherWrapperWorksWithGlobalOrdinals() throws IOException {
-        IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {
-            @Override
-            public DirectoryReader wrap(DirectoryReader reader) throws IOException {
-                return new FieldMaskingReader("foo", reader);
-            }
-
-            @Override
-            public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
-                return searcher;
-            }
-        };
+    public void testReaderWrapperWorksWithGlobalOrdinals() throws IOException {
+        CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper = reader -> new FieldMaskingReader("foo", reader);
 
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
@@ -2501,16 +2512,8 @@ public void testSearchIsReleaseIfWrapperFails() throws IOException {
         IndexShard shard = newStartedShard(true);
         indexDoc(shard, "_doc", "0", "{\"foo\" : \"bar\"}");
         shard.refresh("test");
-        IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {
-            @Override
-            public DirectoryReader wrap(DirectoryReader reader) throws IOException {
-                throw new RuntimeException("boom");
-            }
-
-            @Override
-            public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
-                return searcher;
-            }
+        CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper = reader -> {
+            throw new RuntimeException("boom");
         };
 
         closeShards(shard);
@@ -2914,6 +2917,8 @@ public void testDocStats() throws Exception {
             // Need to update and sync the global checkpoint as the soft-deletes retention MergePolicy depends on it.
             if (indexShard.indexSettings.isSoftDeleteEnabled()) {
                 if (indexShard.routingEntry().primary()) {
+                    indexShard.updateLocalCheckpointForShard(indexShard.routingEntry().allocationId().getId(),
+                        indexShard.getLocalCheckpoint());
                     indexShard.updateGlobalCheckpointForShard(indexShard.routingEntry().allocationId().getId(),
                         indexShard.getLocalCheckpoint());
                 } else {
@@ -3299,6 +3304,7 @@ private Result indexOnReplicaWithGaps(
                 indexShard.flush(new FlushRequest());
             }
         }
+        indexShard.sync(); // advance local checkpoint
         assert localCheckpoint == indexShard.getLocalCheckpoint();
         assert !gap || (localCheckpoint != max);
         return new Result(localCheckpoint, max);
@@ -3746,7 +3752,7 @@ public void testResetEngine() throws Exception {
         IndexShard shard = newStartedShard(false);
         indexOnReplicaWithGaps(shard, between(0, 1000), Math.toIntExact(shard.getLocalCheckpoint()));
         long maxSeqNoBeforeRollback = shard.seqNoStats().getMaxSeqNo();
-        final long globalCheckpoint = randomLongBetween(shard.getGlobalCheckpoint(), shard.getLocalCheckpoint());
+        final long globalCheckpoint = randomLongBetween(shard.getLastKnownGlobalCheckpoint(), shard.getLocalCheckpoint());
         shard.updateGlobalCheckpointOnReplica(globalCheckpoint, "test");
         Set<String> docBelowGlobalCheckpoint = getShardDocUIDs(shard).stream()
             .filter(id -> Long.parseLong(id) <= globalCheckpoint).collect(Collectors.toSet());
@@ -3830,7 +3836,7 @@ public InternalEngine recoverFromTranslog(TranslogRecoveryRunner translogRecover
         closeShardThread.start();
 
         final CountDownLatch engineResetLatch = new CountDownLatch(1);
-        shard.acquireAllReplicaOperationsPermits(shard.getOperationPrimaryTerm(), shard.getGlobalCheckpoint(), 0L,
+        shard.acquireAllReplicaOperationsPermits(shard.getOperationPrimaryTerm(), shard.getLastKnownGlobalCheckpoint(), 0L,
             ActionListener.wrap(r -> {
                 try (r) {
                     shard.resetEngineToGlobalCheckpoint();
@@ -3870,7 +3876,7 @@ public InternalEngine recoverFromTranslog(TranslogRecoveryRunner translogRecover
         });
 
         indexOnReplicaWithGaps(shard, between(0, 1000), Math.toIntExact(shard.getLocalCheckpoint()));
-        final long globalCheckpoint = randomLongBetween(shard.getGlobalCheckpoint(), shard.getLocalCheckpoint());
+        final long globalCheckpoint = randomLongBetween(shard.getLastKnownGlobalCheckpoint(), shard.getLocalCheckpoint());
         shard.updateGlobalCheckpointOnReplica(globalCheckpoint, "test");
 
         Thread snapshotThread = new Thread(() -> {
@@ -3893,7 +3899,7 @@ public InternalEngine recoverFromTranslog(TranslogRecoveryRunner translogRecover
         snapshotThread.start();
 
         final CountDownLatch engineResetLatch = new CountDownLatch(1);
-        shard.acquireAllReplicaOperationsPermits(shard.getOperationPrimaryTerm(), shard.getGlobalCheckpoint(), 0L,
+        shard.acquireAllReplicaOperationsPermits(shard.getOperationPrimaryTerm(), shard.getLastKnownGlobalCheckpoint(), 0L,
             ActionListener.wrap(r -> {
                 try (r) {
                     shard.resetEngineToGlobalCheckpoint();
@@ -3917,7 +3923,7 @@ public void testConcurrentAcquireAllReplicaOperationsPermitsWithPrimaryTermUpdat
 
         for (int i = 0; i < nbTermUpdates; i++) {
             long opPrimaryTerm = replica.getOperationPrimaryTerm() + 1;
-            final long globalCheckpoint = replica.getGlobalCheckpoint();
+            final long globalCheckpoint = replica.getLastKnownGlobalCheckpoint();
             final long maxSeqNoOfUpdatesOrDeletes = replica.getMaxSeqNoOfUpdatesOrDeletes();
 
             final int operations = scaledRandomIntBetween(5, 32);
@@ -4070,7 +4076,7 @@ public void testDoNotTrimCommitsWhenOpenReadOnlyEngine() throws Exception {
         ShardRouting readonlyShardRouting = newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), true,
             ShardRoutingState.INITIALIZING, RecoverySource.ExistingStoreRecoverySource.INSTANCE);
         final IndexShard readonlyShard = reinitShard(shard, readonlyShardRouting,
-            engineConfig -> new ReadOnlyEngine(engineConfig, null, null, false, Function.identity()) {
+            engineConfig -> new ReadOnlyEngine(engineConfig, null, null, true, Function.identity()) {
                 @Override
                 protected void ensureMaxSeqNoEqualsToGlobalCheckpoint(SeqNoStats seqNoStats) {
                     // just like a following shard, we need to skip this check for now.
diff --git a/server/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java
index 4e6e3036f4c40..73ae826d7211f 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java
@@ -90,7 +90,7 @@ static class MockUsableSpaceFileSystemProvider extends FilterFileSystemProvider
 
         @Override
         public FileStore getFileStore(Path path) throws IOException {
-            if (path.toString().contains(aPathPart)) {
+            if (path.toString().contains(aPathPart) || (path.toString() + path.getFileSystem().getSeparator()).contains(aPathPart)) {
                 return aFileStore;
             } else {
                 return bFileStore;
diff --git a/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java b/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java
index e0825445bb8c2..481aaa233caed 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/PrimaryReplicaSyncerTests.java
@@ -101,7 +101,7 @@ public void testSyncerSendsOffCorrectDocuments() throws Exception {
         shard.updateShardState(shard.routingEntry(), shard.getPendingPrimaryTerm(), null, 1000L, Collections.singleton(allocationId),
             new IndexShardRoutingTable.Builder(shard.shardId()).addShard(shard.routingEntry()).build());
         shard.updateLocalCheckpointForShard(allocationId, globalCheckPoint);
-        assertEquals(globalCheckPoint, shard.getGlobalCheckpoint());
+        assertEquals(globalCheckPoint, shard.getLastKnownGlobalCheckpoint());
 
         logger.info("Total ops: {}, global checkpoint: {}", numDocs, globalCheckPoint);
 
@@ -197,7 +197,7 @@ public void onResponse(PrimaryReplicaSyncer.ResyncTask result) {
 
     public void testDoNotSendOperationsWithoutSequenceNumber() throws Exception {
         IndexShard shard = spy(newStartedShard(true));
-        when(shard.getGlobalCheckpoint()).thenReturn(SequenceNumbers.UNASSIGNED_SEQ_NO);
+        when(shard.getLastKnownGlobalCheckpoint()).thenReturn(SequenceNumbers.UNASSIGNED_SEQ_NO);
         int numOps = between(0, 20);
         List<Translog.Operation> operations = new ArrayList<>();
         for (int i = 0; i < numOps; i++) {
diff --git a/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java b/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java
index c7b1846356363..4e4b1756b2cbd 100644
--- a/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java
+++ b/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java
@@ -52,6 +52,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -67,9 +68,7 @@ public class RemoveCorruptedShardDataCommandTests extends IndexShardTestCase {
 
     private ShardId shardId;
     private ShardRouting routing;
-    private Path dataDir;
     private Environment environment;
-    private Settings settings;
     private ShardPath shardPath;
     private IndexMetaData indexMetaData;
     private IndexShard indexShard;
@@ -86,7 +85,7 @@ public void setup() throws IOException {
         routing = TestShardRouting.newShardRouting(shardId, nodeId, true, ShardRoutingState.INITIALIZING,
             RecoverySource.EmptyStoreRecoverySource.INSTANCE);
 
-        dataDir = createTempDir();
+        final Path dataDir = createTempDir();
 
         environment =
             TestEnvironment.newEnvironment(Settings.builder()
@@ -94,16 +93,15 @@ public void setup() throws IOException {
                 .putList(Environment.PATH_DATA_SETTING.getKey(), dataDir.toAbsolutePath().toString()).build());
 
         // create same directory structure as prod does
-        final Path path = NodeEnvironment.resolveNodePath(dataDir);
-        Files.createDirectories(path);
-        settings = Settings.builder()
+        Files.createDirectories(dataDir);
+        final Settings settings = Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
             .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
             .build();
 
-        final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(path);
+        final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(dataDir);
         shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId);
         final IndexMetaData.Builder metaData = IndexMetaData.builder(routing.getIndexName())
             .settings(settings)
@@ -217,7 +215,7 @@ public void testCorruptedTranslog() throws Exception {
         // close shard
         closeShards(indexShard);
 
-        TestTranslog.corruptRandomTranslogFile(logger, random(), Arrays.asList(translogPath));
+        TestTranslog.corruptRandomTranslogFile(logger, random(), Collections.singletonList(translogPath));
 
         // test corrupted shard
         final IndexShard corruptedShard = reopenIndexShard(true);
@@ -283,7 +281,7 @@ public void testCorruptedBothIndexAndTranslog() throws Exception {
             expectThrows(IndexShardRecoveryException.class, () -> newStartedShard(p -> corruptedShard, true));
             closeShards(corruptedShard);
         }
-        TestTranslog.corruptRandomTranslogFile(logger, random(), Arrays.asList(translogPath));
+        TestTranslog.corruptRandomTranslogFile(logger, random(), Collections.singletonList(translogPath));
 
         final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
         final MockTerminal t = new MockTerminal();
diff --git a/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
index cd0c90f50779c..3ca29b6b375b0 100644
--- a/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
+++ b/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
@@ -160,7 +160,9 @@ public void testCorruptFileAndRecover() throws ExecutionException, InterruptedEx
         }
         indexRandom(true, builders);
         ensureGreen();
-        assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).execute().actionGet());
+        // double flush to create safe commit in case of async durability
+        assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).get());
+        assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).get());
         // we have to flush at least once here since we don't corrupt the translog
         SearchResponse countResponse = client().prepareSearch().setSize(0).get();
         assertHitCount(countResponse, numDocs);
@@ -264,7 +266,9 @@ public void testCorruptPrimaryNoReplica() throws ExecutionException, Interrupted
         }
         indexRandom(true, builders);
         ensureGreen();
-        assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).execute().actionGet());
+        // double flush to create safe commit in case of async durability
+        assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).get());
+        assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).get());
         // we have to flush at least once here since we don't corrupt the translog
         SearchResponse countResponse = client().prepareSearch().setSize(0).get();
         assertHitCount(countResponse, numDocs);
diff --git a/server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java b/server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java
index 0f24f8f3a5a4f..dfba867bf3c1d 100644
--- a/server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java
+++ b/server/src/test/java/org/elasticsearch/index/store/FsDirectoryFactoryTests.java
@@ -18,8 +18,9 @@
  */
 package org.elasticsearch.index.store;
 
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FileSwitchDirectory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.store.NIOFSDirectory;
 import org.apache.lucene.store.NoLockFactory;
@@ -36,6 +37,7 @@
 import org.elasticsearch.index.shard.ShardPath;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
+import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -49,34 +51,66 @@ public void testPreload() throws IOException {
         doTestPreload();
         doTestPreload("nvd", "dvd", "tim");
         doTestPreload("*");
+        Settings build = Settings.builder()
+            .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.HYBRIDFS.name().toLowerCase(Locale.ROOT))
+            .putList(IndexModule.INDEX_STORE_PRE_LOAD_SETTING.getKey(), "dvd", "bar")
+            .build();
+        try (Directory directory = newDirectory(build)) {
+            assertTrue(FsDirectoryFactory.isHybridFs(directory));
+            FsDirectoryFactory.HybridDirectory hybridDirectory = (FsDirectoryFactory.HybridDirectory) directory;
+            assertTrue(hybridDirectory.useDelegate("foo.dvd"));
+            assertTrue(hybridDirectory.useDelegate("foo.nvd"));
+            assertTrue(hybridDirectory.useDelegate("foo.tim"));
+            assertTrue(hybridDirectory.useDelegate("foo.tip"));
+            assertTrue(hybridDirectory.useDelegate("foo.cfs"));
+            assertFalse(hybridDirectory.useDelegate("foo.bar"));
+            MMapDirectory delegate = hybridDirectory.getDelegate();
+            assertThat(delegate, Matchers.instanceOf(FsDirectoryFactory.PreLoadMMapDirectory.class));
+            FsDirectoryFactory.PreLoadMMapDirectory preLoadMMapDirectory = (FsDirectoryFactory.PreLoadMMapDirectory) delegate;
+            assertTrue(preLoadMMapDirectory.useDelegate("foo.dvd"));
+            assertTrue(preLoadMMapDirectory.useDelegate("foo.bar"));
+        }
+    }
+
+    private Directory newDirectory(Settings settings) throws IOException {
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("foo", settings);
+        Path tempDir = createTempDir().resolve(idxSettings.getUUID()).resolve("0");
+        Files.createDirectories(tempDir);
+        ShardPath path = new ShardPath(false, tempDir, tempDir, new ShardId(idxSettings.getIndex(), 0));
+        return new FsDirectoryFactory().newDirectory(idxSettings, path);
     }
 
     private void doTestPreload(String...preload) throws IOException {
         Settings build = Settings.builder()
-                .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "mmapfs")
-                .putList(IndexModule.INDEX_STORE_PRE_LOAD_SETTING.getKey(), preload)
-                .build();
-        IndexSettings settings = IndexSettingsModule.newIndexSettings("foo", build);
-        Path tempDir = createTempDir().resolve(settings.getUUID()).resolve("0");
-        Files.createDirectories(tempDir);
-        ShardPath path = new ShardPath(false, tempDir, tempDir, new ShardId(settings.getIndex(), 0));
-        FsDirectoryFactory fsDirectoryFactory = new FsDirectoryFactory();
-        Directory directory = fsDirectoryFactory.newDirectory(settings, path);
-        assertFalse(directory instanceof SleepingLockWrapper);
-        if (preload.length == 0) {
-            assertTrue(directory.toString(), directory instanceof MMapDirectory);
-            assertFalse(((MMapDirectory) directory).getPreload());
-        } else if (Arrays.asList(preload).contains("*")) {
-            assertTrue(directory.toString(), directory instanceof MMapDirectory);
-            assertTrue(((MMapDirectory) directory).getPreload());
-        } else {
-            assertTrue(directory.toString(), directory instanceof FileSwitchDirectory);
-            FileSwitchDirectory fsd = (FileSwitchDirectory) directory;
-            assertTrue(fsd.getPrimaryDir() instanceof MMapDirectory);
-            assertTrue(((MMapDirectory) fsd.getPrimaryDir()).getPreload());
-            assertTrue(fsd.getSecondaryDir() instanceof MMapDirectory);
-            assertFalse(((MMapDirectory) fsd.getSecondaryDir()).getPreload());
+            .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "mmapfs")
+            .putList(IndexModule.INDEX_STORE_PRE_LOAD_SETTING.getKey(), preload)
+            .build();
+        Directory directory = newDirectory(build);
+        try (Directory dir = directory){
+            assertSame(dir, directory); // prevent warnings
+            assertFalse(directory instanceof SleepingLockWrapper);
+            if (preload.length == 0) {
+                assertTrue(directory.toString(), directory instanceof MMapDirectory);
+                assertFalse(((MMapDirectory) directory).getPreload());
+            } else if (Arrays.asList(preload).contains("*")) {
+                assertTrue(directory.toString(), directory instanceof MMapDirectory);
+                assertTrue(((MMapDirectory) directory).getPreload());
+            } else {
+                assertTrue(directory.toString(), directory instanceof FsDirectoryFactory.PreLoadMMapDirectory);
+                FsDirectoryFactory.PreLoadMMapDirectory preLoadMMapDirectory = (FsDirectoryFactory.PreLoadMMapDirectory) directory;
+                for (String ext : preload) {
+                    assertTrue("ext: " + ext, preLoadMMapDirectory.useDelegate("foo." + ext));
+                    assertTrue("ext: " + ext, preLoadMMapDirectory.getDelegate().getPreload());
+                }
+                assertFalse(preLoadMMapDirectory.useDelegate("XXX"));
+                assertFalse(preLoadMMapDirectory.getPreload());
+                preLoadMMapDirectory.close();
+                expectThrows(AlreadyClosedException.class, () -> preLoadMMapDirectory.getDelegate().openInput("foo.bar",
+                    IOContext.DEFAULT));
+            }
         }
+        expectThrows(AlreadyClosedException.class, () -> directory.openInput(randomBoolean() && preload.length != 0 ?
+            "foo." + preload[0] : "foo.bar", IOContext.DEFAULT));
     }
 
     public void testStoreDirectory() throws IOException {
@@ -102,7 +136,7 @@ private void doTestStoreDirectory(Path tempDir, String typeSettingValue, IndexMo
         try (Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE, indexSettings)) {
             switch (type) {
                 case HYBRIDFS:
-                    assertHybridDirectory(directory);
+                    assertTrue(FsDirectoryFactory.isHybridFs(directory));
                     break;
                 case NIOFS:
                     assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory);
@@ -115,7 +149,7 @@ private void doTestStoreDirectory(Path tempDir, String typeSettingValue, IndexMo
                     break;
                 case FS:
                     if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
-                        assertHybridDirectory(directory);
+                        assertTrue(FsDirectoryFactory.isHybridFs(directory));
                     } else if (Constants.WINDOWS) {
                         assertTrue(directory.toString(), directory instanceof SimpleFSDirectory);
                     } else {
@@ -127,10 +161,4 @@ private void doTestStoreDirectory(Path tempDir, String typeSettingValue, IndexMo
             }
         }
     }
-
-    private void assertHybridDirectory(Directory directory) {
-        assertTrue(directory.toString(), directory instanceof FsDirectoryFactory.HybridDirectory);
-        Directory randomAccessDirectory = ((FsDirectoryFactory.HybridDirectory) directory).getRandomAccessDirectory();
-        assertTrue("randomAccessDirectory:  " +  randomAccessDirectory.toString(), randomAccessDirectory instanceof MMapDirectory);
-    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/store/StoreTests.java b/server/src/test/java/org/elasticsearch/index/store/StoreTests.java
index 9e8fae209dd81..dc2557a1f6e13 100644
--- a/server/src/test/java/org/elasticsearch/index/store/StoreTests.java
+++ b/server/src/test/java/org/elasticsearch/index/store/StoreTests.java
@@ -40,14 +40,12 @@
 import org.apache.lucene.index.SnapshotDeletionPolicy;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.store.BaseDirectoryWrapper;
-import org.apache.lucene.store.ByteBufferIndexInput;
 import org.apache.lucene.store.ChecksumIndexInput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FilterDirectory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.store.NIOFSDirectory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.BytesRef;
@@ -1081,48 +1079,6 @@ public void testHistoryUUIDCanBeForced() throws IOException {
         }
     }
 
-    public void testDeoptimizeMMap() throws IOException {
-        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("index",
-            Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, org.elasticsearch.Version.CURRENT)
-                .put(Store.FORCE_RAM_TERM_DICT.getKey(), true).build());
-        final ShardId shardId = new ShardId("index", "_na_", 1);
-        String file = "test." + (randomBoolean() ? "tip" : "cfs");
-        try (Store store = new Store(shardId, indexSettings, new MMapDirectory(createTempDir()), new DummyShardLock(shardId))) {
-            try (IndexOutput output = store.directory().createOutput(file, IOContext.DEFAULT)) {
-                output.writeInt(0);
-            }
-            try (IndexOutput output = store.directory().createOutput("someOtherFile.txt", IOContext.DEFAULT)) {
-                output.writeInt(0);
-            }
-            try (IndexInput input = store.directory().openInput(file, IOContext.DEFAULT)) {
-                assertFalse(input instanceof ByteBufferIndexInput);
-                assertFalse(input.clone() instanceof ByteBufferIndexInput);
-                assertFalse(input.slice("foo", 1, 1) instanceof ByteBufferIndexInput);
-            }
-
-            try (IndexInput input = store.directory().openInput("someOtherFile.txt", IOContext.DEFAULT)) {
-                assertTrue(input instanceof ByteBufferIndexInput);
-                assertTrue(input.clone() instanceof ByteBufferIndexInput);
-                assertTrue(input.slice("foo", 1, 1) instanceof ByteBufferIndexInput);
-            }
-        }
-
-        indexSettings = IndexSettingsModule.newIndexSettings("index",
-            Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, org.elasticsearch.Version.CURRENT)
-                .put(Store.FORCE_RAM_TERM_DICT.getKey(), false).build());
-
-        try (Store store = new Store(shardId, indexSettings, new MMapDirectory(createTempDir()), new DummyShardLock(shardId))) {
-            try (IndexOutput output = store.directory().createOutput(file, IOContext.DEFAULT)) {
-                output.writeInt(0);
-            }
-            try (IndexInput input = store.directory().openInput(file, IOContext.DEFAULT)) {
-                assertTrue(input instanceof ByteBufferIndexInput);
-                assertTrue(input.clone() instanceof ByteBufferIndexInput);
-                assertTrue(input.slice("foo", 1, 1) instanceof ByteBufferIndexInput);
-            }
-        }
-    }
-
     public void testGetPendingFiles() throws IOException {
         final ShardId shardId = new ShardId("index", "_na_", 1);
         final String testfile = "testfile";
diff --git a/server/src/test/java/org/elasticsearch/index/termvectors/TermVectorsServiceTests.java b/server/src/test/java/org/elasticsearch/index/termvectors/TermVectorsServiceTests.java
index 034e7daaf7f99..6d192f584f0de 100644
--- a/server/src/test/java/org/elasticsearch/index/termvectors/TermVectorsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/index/termvectors/TermVectorsServiceTests.java
@@ -69,7 +69,7 @@ public void testTook() throws Exception {
 
         List<Long> longs = Stream.of(abs(randomLong()), abs(randomLong())).sorted().collect(toList());
 
-        TermVectorsRequest request = new TermVectorsRequest("test", "type1", "0");
+        TermVectorsRequest request = new TermVectorsRequest("test", "0");
         TermVectorsResponse response = TermVectorsService.getTermVectors(shard, request, longs.iterator()::next);
 
         assertThat(response, notNullValue());
@@ -103,22 +103,22 @@ public void testDocFreqs() throws IOException {
         }
         bulk.get();
 
-        TermVectorsRequest request = new TermVectorsRequest("test", "_doc", "0").termStatistics(true);
+        TermVectorsRequest request = new TermVectorsRequest("test", "0").termStatistics(true);
 
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService test = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
         assertThat(shard, notNullValue());
-        TermVectorsResponse response = TermVectorsService.getTermVectors(shard, request);        
-        assertEquals(1, response.getFields().size());            
+        TermVectorsResponse response = TermVectorsService.getTermVectors(shard, request);
+        assertEquals(1, response.getFields().size());
 
         Terms terms = response.getFields().terms("text");
         TermsEnum iterator = terms.iterator();
         while (iterator.next() != null) {
             assertEquals(max, iterator.docFreq());
         }
-    }  
-    
+    }
+
     public void testWithIndexedPhrases() throws IOException {
         XContentBuilder mapping = jsonBuilder()
             .startObject()
@@ -126,7 +126,7 @@ public void testWithIndexedPhrases() throws IOException {
                     .startObject("properties")
                         .startObject("text")
                             .field("type", "text")
-                            .field("index_phrases", true)                            
+                .field("index_phrases", true)
                             .field("term_vector", "with_positions_offsets_payloads")
                         .endObject()
                     .endObject()
@@ -146,13 +146,13 @@ public void testWithIndexedPhrases() throws IOException {
         }
         bulk.get();
 
-        TermVectorsRequest request = new TermVectorsRequest("test", "_doc", "0").termStatistics(true);
+        TermVectorsRequest request = new TermVectorsRequest("test", "0").termStatistics(true);
 
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService test = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
         assertThat(shard, notNullValue());
-        TermVectorsResponse response = TermVectorsService.getTermVectors(shard, request);        
+        TermVectorsResponse response = TermVectorsService.getTermVectors(shard, request);
         assertEquals(2, response.getFields().size());
 
         Terms terms = response.getFields().terms("text");
@@ -160,11 +160,11 @@ public void testWithIndexedPhrases() throws IOException {
         while (iterator.next() != null) {
             assertEquals(max, iterator.docFreq());
         }
-                
+
         Terms phrases = response.getFields().terms("text._index_phrase");
         TermsEnum phraseIterator = phrases.iterator();
         while (phraseIterator.next() != null) {
             assertEquals(max, phraseIterator.docFreq());
         }
-    }  
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/translog/TestTranslog.java b/server/src/test/java/org/elasticsearch/index/translog/TestTranslog.java
index a3ebfff478e98..af1693a45bf9b 100644
--- a/server/src/test/java/org/elasticsearch/index/translog/TestTranslog.java
+++ b/server/src/test/java/org/elasticsearch/index/translog/TestTranslog.java
@@ -47,6 +47,8 @@
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.core.Is.is;
 import static org.hamcrest.core.IsNot.not;
 
@@ -54,7 +56,7 @@
  * Helpers for testing translog.
  */
 public class TestTranslog {
-    static final Pattern TRANSLOG_FILE_PATTERN = Pattern.compile("translog-(\\d+)\\.tlog");
+    private static final Pattern TRANSLOG_FILE_PATTERN = Pattern.compile("translog-(\\d+)\\.tlog");
 
     public static void corruptRandomTranslogFile(Logger logger, Random random, Collection<Path> translogDirs) throws IOException {
         for (Path translogDir : translogDirs) {
@@ -65,12 +67,11 @@ public static void corruptRandomTranslogFile(Logger logger, Random random, Colle
 
     /**
      * Corrupts random translog file (translog-N.tlog) from the given translog directory.
-     *
-     * @return a translog file which has been corrupted.
      */
-    public static Path corruptRandomTranslogFile(Logger logger, Random random, Path translogDir, long minGeneration) throws IOException {
+    public static void corruptRandomTranslogFile(Logger logger, Random random, Path translogDir, long minGeneration)
+            throws IOException {
         Set<Path> candidates = new TreeSet<>(); // TreeSet makes sure iteration order is deterministic
-        logger.info("--> Translog dir [{}], minUsedTranslogGen [{}]", translogDir, minGeneration);
+        logger.info("--> corruptRandomTranslogFile: translogDir [{}], minUsedTranslogGen [{}]", translogDir, minGeneration);
         try (DirectoryStream<Path> stream = Files.newDirectoryStream(translogDir)) {
             for (Path item : stream) {
                 if (Files.isRegularFile(item)) {
@@ -81,41 +82,51 @@ public static Path corruptRandomTranslogFile(Logger logger, Random random, Path
                 }
             }
         }
-        assertThat(candidates, is(not(empty())));
+        assertThat("no translog files found in " + translogDir, candidates, is(not(empty())));
 
         Path corruptedFile = RandomPicks.randomFrom(random, candidates);
         corruptFile(logger, random, corruptedFile);
-        return corruptedFile;
     }
 
-
-     static void corruptFile(Logger logger, Random random, Path fileToCorrupt) throws IOException {
-        try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) {
-            // read
-            raf.position(RandomNumbers.randomLongBetween(random, 0, raf.size() - 1));
-            long filePointer = raf.position();
-            ByteBuffer bb = ByteBuffer.wrap(new byte[1]);
-            raf.read(bb);
-            bb.flip();
-
-            // corrupt
-            byte oldValue = bb.get(0);
-            byte newValue = (byte) (oldValue + 1);
-            bb.put(0, newValue);
-
-            // rewrite
-            raf.position(filePointer);
-            raf.write(bb);
-            logger.info("--> corrupting file {} --  flipping at position {} from {} to {} file: {}",
-                fileToCorrupt, filePointer, Integer.toHexString(oldValue),
-                Integer.toHexString(newValue), fileToCorrupt);
+    static void corruptFile(Logger logger, Random random, Path fileToCorrupt) throws IOException {
+        final long fileSize = Files.size(fileToCorrupt);
+        assertThat("cannot corrupt empty file " + fileToCorrupt, fileSize, greaterThan(0L));
+
+        try (FileChannel fileChannel = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) {
+            final long corruptPosition = RandomNumbers.randomLongBetween(random, 0, fileSize - 1);
+
+            if (random.nextBoolean()) {
+                // read
+                fileChannel.position(corruptPosition);
+                assertThat(fileChannel.position(), equalTo(corruptPosition));
+                ByteBuffer bb = ByteBuffer.wrap(new byte[1]);
+                fileChannel.read(bb);
+                bb.flip();
+
+                // corrupt
+                byte oldValue = bb.get(0);
+                byte newValue;
+                do {
+                    newValue = (byte) random.nextInt(0x100);
+                } while (newValue == oldValue);
+                bb.put(0, newValue);
+
+                // rewrite
+                fileChannel.position(corruptPosition);
+                fileChannel.write(bb);
+                logger.info("--> corrupting file {} at position {} turning 0x{} into 0x{}", fileToCorrupt, corruptPosition,
+                    Integer.toHexString(oldValue & 0xff), Integer.toHexString(newValue & 0xff));
+            } else {
+                logger.info("--> truncating file {} from length {} to length {}", fileToCorrupt, fileSize, corruptPosition);
+                fileChannel.truncate(corruptPosition);
+            }
         }
     }
 
     /**
      * Lists all existing commits in a given index path, then read the minimum translog generation that will be used in recoverFromTranslog.
      */
-    public static long minTranslogGenUsedInRecovery(Path translogPath) throws IOException {
+    private static long minTranslogGenUsedInRecovery(Path translogPath) throws IOException {
         try (NIOFSDirectory directory = new NIOFSDirectory(translogPath.getParent().resolve("index"))) {
             List<IndexCommit> commits = DirectoryReader.listCommits(directory);
             final String translogUUID = commits.get(commits.size() - 1).getUserData().get(Translog.TRANSLOG_UUID_KEY);
diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java
index c8d4dbd43df2f..da339ff5c8ec0 100644
--- a/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java
+++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogDeletionPolicyTests.java
@@ -171,7 +171,7 @@ private Tuple<List<TranslogReader>, TranslogWriter> createReadersAndWriter(final
             }
             writer = TranslogWriter.create(new ShardId("index", "uuid", 0), translogUUID, gen,
                 tempDir.resolve(Translog.getFilename(gen)), FileChannel::open, TranslogConfig.DEFAULT_BUFFER_SIZE, 1L, 1L, () -> 1L,
-                () -> 1L, randomNonNegativeLong(), new TragicExceptionHolder());
+                () -> 1L, randomNonNegativeLong(), new TragicExceptionHolder(), seqNo -> {});
             writer = Mockito.spy(writer);
             Mockito.doReturn(now - (numberOfReaders - gen + 1) * 1000).when(writer).getLastModifiedTime();
 
diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java
index 7d06e25519ac9..90960693f8af7 100644
--- a/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogHeaderTests.java
@@ -19,24 +19,18 @@
 
 package org.elasticsearch.index.translog;
 
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.store.OutputStreamDataOutput;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
 import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.test.ESTestCase;
 
-import java.io.IOException;
-import java.nio.channels.Channels;
 import java.nio.channels.FileChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.lessThan;
 
 public class TranslogHeaderTests extends ESTestCase {
 
@@ -62,50 +56,26 @@ public void testCurrentHeaderVersion() throws Exception {
         });
         assertThat(mismatchUUID.getMessage(), containsString("this translog file belongs to a different translog"));
         int corruptions = between(1, 10);
-        for (int i = 0; i < corruptions; i++) {
+        for (int i = 0; i < corruptions && Files.size(translogFile) > 0; i++) {
             TestTranslog.corruptFile(logger, random(), translogFile);
         }
         expectThrows(TranslogCorruptedException.class, () -> {
             try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.READ)) {
                 TranslogHeader.read(outHeader.getTranslogUUID(), translogFile, channel);
+            } catch (IllegalStateException e) {
+                // corruption corrupted the version byte making this look like a v2, v1 or v0 translog
+                assertThat("version " + TranslogHeader.VERSION_CHECKPOINTS + "-or-earlier translog",
+                    e.getMessage(), anyOf(containsString("pre-2.0 translog found"), containsString("pre-1.4 translog found"),
+                        containsString("pre-6.3 translog found")));
+                throw new TranslogCorruptedException(translogFile.toString(), "adjusted translog version", e);
             }
         });
     }
 
-    public void testHeaderWithoutPrimaryTerm() throws Exception {
-        final String translogUUID = UUIDs.randomBase64UUID();
-        final long generation = randomNonNegativeLong();
-        final Path translogFile = createTempDir().resolve(Translog.getFilename(generation));
-        try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
-            writeHeaderWithoutTerm(channel, translogUUID);
-            assertThat((int)channel.position(), lessThan(TranslogHeader.headerSizeInBytes(translogUUID)));
-        }
-        try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.READ)) {
-            final TranslogHeader inHeader = TranslogHeader.read(translogUUID, translogFile, channel);
-            assertThat(inHeader.getTranslogUUID(), equalTo(translogUUID));
-            assertThat(inHeader.getPrimaryTerm(), equalTo(SequenceNumbers.UNASSIGNED_PRIMARY_TERM));
-            assertThat(inHeader.sizeInBytes(), equalTo((int)channel.position()));
-        }
-        expectThrows(TranslogCorruptedException.class, () -> {
-            try (FileChannel channel = FileChannel.open(translogFile, StandardOpenOption.READ)) {
-                TranslogHeader.read(UUIDs.randomBase64UUID(), translogFile, channel);
-            }
-        });
-    }
-
-    static void writeHeaderWithoutTerm(FileChannel channel, String translogUUID) throws IOException {
-        final OutputStreamStreamOutput out = new OutputStreamStreamOutput(Channels.newOutputStream(channel));
-        CodecUtil.writeHeader(new OutputStreamDataOutput(out), TranslogHeader.TRANSLOG_CODEC, TranslogHeader.VERSION_CHECKPOINTS);
-        final BytesRef uuid = new BytesRef(translogUUID);
-        out.writeInt(uuid.length);
-        out.writeBytes(uuid.bytes, uuid.offset, uuid.length);
-        channel.force(true);
-        assertThat(channel.position(), equalTo(43L));
-    }
-
-    public void testLegacyTranslogVersions() throws Exception {
+    public void testLegacyTranslogVersions() {
         checkFailsToOpen("/org/elasticsearch/index/translog/translog-v0.binary", IllegalStateException.class, "pre-1.4 translog");
         checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1.binary", IllegalStateException.class, "pre-2.0 translog");
+        checkFailsToOpen("/org/elasticsearch/index/translog/translog-v2.binary", IllegalStateException.class, "pre-6.3 translog");
         checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1-truncated.binary", IllegalStateException.class, "pre-2.0 translog");
         checkFailsToOpen("/org/elasticsearch/index/translog/translog-v1-corrupted-magic.binary",
             TranslogCorruptedException.class, "translog looks like version 1 or later, but has corrupted header");
diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index ceca8a811a67f..c99fee9dcb8a7 100644
--- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -113,6 +113,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.LongConsumer;
 import java.util.function.LongSupplier;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
@@ -128,6 +129,7 @@
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasToString;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.isIn;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -146,6 +148,7 @@ public class TranslogTests extends ESTestCase {
     protected Path translogDir;
     // A default primary term is used by translog instances created in this test.
     private final AtomicLong primaryTerm = new AtomicLong();
+    private final AtomicReference<LongConsumer> persistedSeqNoConsumer = new AtomicReference<>();
 
     @Override
     protected void afterIfSuccessful() throws Exception {
@@ -164,16 +167,25 @@ protected void afterIfSuccessful() throws Exception {
 
     }
 
+    private LongConsumer getPersistedSeqNoConsumer() {
+        return seqNo -> {
+            final LongConsumer consumer = persistedSeqNoConsumer.get();
+            if (consumer != null) {
+                consumer.accept(seqNo);
+            }
+        };
+    }
+
     protected Translog createTranslog(TranslogConfig config) throws IOException {
         String translogUUID =
             Translog.createEmptyTranslog(config.getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get());
         return new Translog(config, translogUUID, createTranslogDeletionPolicy(config.getIndexSettings()),
-            () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get);
+            () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, getPersistedSeqNoConsumer());
     }
 
     protected Translog openTranslog(TranslogConfig config, String translogUUID) throws IOException {
         return new Translog(config, translogUUID, createTranslogDeletionPolicy(config.getIndexSettings()),
-            () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get);
+            () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, getPersistedSeqNoConsumer());
     }
 
 
@@ -225,7 +237,8 @@ private Translog create(Path path) throws IOException {
         final TranslogConfig translogConfig = getTranslogConfig(path);
         final TranslogDeletionPolicy deletionPolicy = createTranslogDeletionPolicy(translogConfig.getIndexSettings());
         final String translogUUID = Translog.createEmptyTranslog(path, SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get());
-        return new Translog(translogConfig, translogUUID, deletionPolicy, () -> globalCheckpoint.get(), primaryTerm::get);
+        return new Translog(translogConfig, translogUUID, deletionPolicy, () -> globalCheckpoint.get(), primaryTerm::get,
+            getPersistedSeqNoConsumer());
     }
 
     private TranslogConfig getTranslogConfig(final Path path) {
@@ -882,7 +895,8 @@ public void testTruncatedTranslogs() throws Exception {
             for (int i = 0; i < locations.size(); i++) {
                 try {
                     assertNotNull(snap.next());
-                } catch (EOFException e) {
+                } catch (TranslogCorruptedException e) {
+                    assertThat(e.getCause(), instanceOf(EOFException.class));
                     truncations.incrementAndGet();
                 }
             }
@@ -980,7 +994,7 @@ public void doRun() throws BrokenBarrierException, InterruptedException, IOExcep
                                 throw new AssertionError("unsupported operation type [" + type + "]");
                         }
                         Translog.Location location = translog.add(op);
-                        tracker.markSeqNoAsCompleted(id);
+                        tracker.markSeqNoAsProcessed(id);
                         Translog.Location existing = writtenOps.put(op, location);
                         if (existing != null) {
                             fail("duplicate op [" + op + "], old entry at " + location);
@@ -992,7 +1006,7 @@ public void doRun() throws BrokenBarrierException, InterruptedException, IOExcep
                             synchronized (flushMutex) {
                                 // we need not do this concurrently as we need to make sure that the generation
                                 // we're committing - is still present when we're committing
-                                long localCheckpoint = tracker.getCheckpoint();
+                                long localCheckpoint = tracker.getProcessedCheckpoint();
                                 translog.rollGeneration();
                                 // expose the new checkpoint (simulating a commit), before we trim the translog
                                 lastCommittedLocalCheckpoint.set(localCheckpoint);
@@ -1277,6 +1291,8 @@ public void testBasicCheckpoint() throws IOException {
 
     public void testTranslogWriter() throws IOException {
         final TranslogWriter writer = translog.createWriter(translog.currentFileGeneration() + 1);
+        final Set<Long> persistedSeqNos = new HashSet<>();
+        persistedSeqNoConsumer.set(persistedSeqNos::add);
         final int numOps = randomIntBetween(8, 128);
         byte[] bytes = new byte[4];
         ByteArrayDataOutput out = new ByteArrayDataOutput(bytes);
@@ -1295,7 +1311,10 @@ public void testTranslogWriter() throws IOException {
             }
             writer.add(new BytesArray(bytes), seqNo);
         }
+        assertThat(persistedSeqNos, empty());
         writer.sync();
+        persistedSeqNos.remove(SequenceNumbers.UNASSIGNED_SEQ_NO);
+        assertEquals(seenSeqNos, persistedSeqNos);
 
         final BaseTranslogReader reader = randomBoolean() ? writer :
             translog.openReader(writer.path(), Checkpoint.read(translog.location().resolve(Translog.CHECKPOINT_FILE_NAME)));
@@ -1399,7 +1418,7 @@ public void testBasicRecovery() throws IOException {
             }
         } else {
             translog = new Translog(config, translogGeneration.translogUUID, translog.getDeletionPolicy(),
-                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get);
+                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {});
             assertEquals("lastCommitted must be 1 less than current",
                 translogGeneration.translogFileGeneration + 1, translog.currentFileGeneration());
             assertFalse(translog.syncNeeded());
@@ -1441,7 +1460,7 @@ public void testRecoveryUncommitted() throws IOException {
         final String translogUUID = translog.getTranslogUUID();
         final TranslogDeletionPolicy deletionPolicy = translog.getDeletionPolicy();
         try (Translog translog = new Translog(config, translogUUID, deletionPolicy,
-                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) {
+                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {})) {
             assertNotNull(translogGeneration);
             assertEquals("lastCommitted must be 2 less than current - we never finished the commit",
                 translogGeneration.translogFileGeneration + 2, translog.currentFileGeneration());
@@ -1457,7 +1476,7 @@ public void testRecoveryUncommitted() throws IOException {
         }
         if (randomBoolean()) { // recover twice
             try (Translog translog = new Translog(config, translogUUID, deletionPolicy,
-                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) {
+                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {})) {
                 assertNotNull(translogGeneration);
                 assertEquals("lastCommitted must be 3 less than current - we never finished the commit and run recovery twice",
                     translogGeneration.translogFileGeneration + 3, translog.currentFileGeneration());
@@ -1506,7 +1525,7 @@ public void testRecoveryUncommittedFileExists() throws IOException {
         final String translogUUID = translog.getTranslogUUID();
         final TranslogDeletionPolicy deletionPolicy = translog.getDeletionPolicy();
         try (Translog translog = new Translog(config, translogUUID, deletionPolicy,
-                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) {
+                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {})) {
             assertNotNull(translogGeneration);
             assertEquals("lastCommitted must be 2 less than current - we never finished the commit",
                 translogGeneration.translogFileGeneration + 2, translog.currentFileGeneration());
@@ -1523,7 +1542,7 @@ public void testRecoveryUncommittedFileExists() throws IOException {
 
         if (randomBoolean()) { // recover twice
             try (Translog translog = new Translog(config, translogUUID, deletionPolicy,
-                    () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) {
+                    () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {})) {
                 assertNotNull(translogGeneration);
                 assertEquals("lastCommitted must be 3 less than current - we never finished the commit and run recovery twice",
                     translogGeneration.translogFileGeneration + 3, translog.currentFileGeneration());
@@ -1571,7 +1590,7 @@ public void testRecoveryUncommittedCorruptedCheckpoint() throws IOException {
         final String translogUUID = translog.getTranslogUUID();
         final TranslogDeletionPolicy deletionPolicy = translog.getDeletionPolicy();
         try (Translog ignored = new Translog(config, translogUUID, deletionPolicy,
-                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) {
+                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {})) {
             fail("corrupted");
         } catch (IllegalStateException ex) {
             assertEquals("Checkpoint file translog-3.ckp already exists but has corrupted content expected: Checkpoint{offset=3025, " +
@@ -1582,7 +1601,7 @@ public void testRecoveryUncommittedCorruptedCheckpoint() throws IOException {
         Checkpoint.write(FileChannel::open, config.getTranslogPath().resolve(Translog.getCommitCheckpointFileName(read.generation)),
             read, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
         try (Translog translog = new Translog(config, translogUUID, deletionPolicy,
-                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) {
+                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {})) {
             assertNotNull(translogGeneration);
             assertEquals("lastCommitted must be 2 less than current - we never finished the commit",
                 translogGeneration.translogFileGeneration + 2, translog.currentFileGeneration());
@@ -1851,12 +1870,14 @@ public void testOpenForeignTranslog() throws IOException {
         final String foreignTranslog = randomRealisticUnicodeOfCodepointLengthBetween(1,
             translogGeneration.translogUUID.length());
         try {
-            new Translog(config, foreignTranslog, createTranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get);
+            new Translog(config, foreignTranslog, createTranslogDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get,
+                seqNo -> {});
             fail("translog doesn't belong to this UUID");
         } catch (TranslogCorruptedException ex) {
 
         }
-        this.translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get);
+        this.translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get,
+            seqNo -> {});
         try (Translog.Snapshot snapshot = this.translog.newSnapshotFromGen(translogGeneration, Long.MAX_VALUE)) {
             for (int i = firstUncommitted; i < translogOperations; i++) {
                 Translog.Operation next = snapshot.next();
@@ -2050,7 +2071,7 @@ public void testFailFlush() throws IOException {
         final String translogUUID = translog.getTranslogUUID();
         final TranslogDeletionPolicy deletionPolicy = translog.getDeletionPolicy();
         try (Translog tlog = new Translog(config, translogUUID, deletionPolicy,
-                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) {
+                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {})) {
             assertEquals("lastCommitted must be 1 less than current",
                 translogGeneration.translogFileGeneration + 1, tlog.currentFileGeneration());
             assertFalse(tlog.syncNeeded());
@@ -2189,7 +2210,7 @@ protected void afterAdd() throws IOException {
             writtenOperations.removeIf(next -> checkpoint.offset < (next.location.translogLocation + next.location.size));
             try (Translog tlog =
                      new Translog(config, translogUUID, createTranslogDeletionPolicy(),
-                         () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get);
+                         () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {});
                  Translog.Snapshot snapshot = tlog.newSnapshot()) {
                 if (writtenOperations.size() != snapshot.totalOperations()) {
                     for (int i = 0; i < threadCount; i++) {
@@ -2239,7 +2260,7 @@ public void testRecoveryFromAFutureGenerationCleansUp() throws IOException {
         deletionPolicy.setTranslogGenerationOfLastCommit(randomLongBetween(comittedGeneration, Long.MAX_VALUE));
         deletionPolicy.setMinTranslogGenerationForRecovery(comittedGeneration);
         translog = new Translog(config, translog.getTranslogUUID(), deletionPolicy,
-            () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get);
+            () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {});
         assertThat(translog.getMinFileGeneration(), equalTo(1L));
         // no trimming done yet, just recovered
         for (long gen = 1; gen < translog.currentFileGeneration(); gen++) {
@@ -2298,7 +2319,7 @@ public void testRecoveryFromFailureOnTrimming() throws IOException {
         deletionPolicy.setTranslogGenerationOfLastCommit(randomLongBetween(comittedGeneration, Long.MAX_VALUE));
         deletionPolicy.setMinTranslogGenerationForRecovery(comittedGeneration);
         try (Translog translog = new Translog(config, translogUUID, deletionPolicy,
-            () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) {
+            () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {})) {
             // we don't know when things broke exactly
             assertThat(translog.getMinFileGeneration(), greaterThanOrEqualTo(1L));
             assertThat(translog.getMinFileGeneration(), lessThanOrEqualTo(comittedGeneration));
@@ -2380,7 +2401,8 @@ private Translog getFailableTranslog(final FailSwitch fail, final TranslogConfig
             translogUUID = Translog.createEmptyTranslog(
                 config.getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId, channelFactory, primaryTerm.get());
         }
-        return new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get) {
+        return new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get,
+            seqNo -> {}) {
             @Override
             ChannelFactory getChannelFactory() {
                 return channelFactory;
@@ -2494,9 +2516,10 @@ public void testFailWhileCreateWriteWithRecoveredTLogs() throws IOException {
         translog.close();
         try {
             new Translog(config, translog.getTranslogUUID(), createTranslogDeletionPolicy(),
-                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get) {
+                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}) {
                 @Override
-                protected TranslogWriter createWriter(long fileGeneration, long initialMinTranslogGen, long initialGlobalCheckpoint)
+                protected TranslogWriter createWriter(long fileGeneration, long initialMinTranslogGen, long initialGlobalCheckpoint,
+                                                      LongConsumer persistedSequenceNumberConsumer)
                     throws IOException {
                     throw new MockDirectoryWrapper.FakeIOException();
                 }
@@ -2557,7 +2580,7 @@ public void testRecoverWithUnbackedNextGenInIllegalState() throws IOException {
         Files.createFile(config.getTranslogPath().resolve("translog-" + (read.generation + 1) + ".tlog"));
 
         TranslogException ex = expectThrows(TranslogException.class, () -> new Translog(config, translog.getTranslogUUID(),
-            translog.getDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get));
+            translog.getDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}));
         assertEquals(ex.getMessage(), "failed to create new translog file");
         assertEquals(ex.getCause().getClass(), FileAlreadyExistsException.class);
     }
@@ -2577,7 +2600,7 @@ public void testRecoverWithUnbackedNextGenAndFutureFile() throws IOException {
         // we add N+1 and N+2 to ensure we only delete the N+1 file and never jump ahead and wipe without the right condition
         Files.createFile(config.getTranslogPath().resolve("translog-" + (read.generation + 2) + ".tlog"));
         try (Translog tlog = new Translog(config, translogUUID, deletionPolicy,
-                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) {
+                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {})) {
             assertFalse(tlog.syncNeeded());
             try (Translog.Snapshot snapshot = tlog.newSnapshot()) {
                 for (int i = 0; i < 1; i++) {
@@ -2591,7 +2614,8 @@ public void testRecoverWithUnbackedNextGenAndFutureFile() throws IOException {
         }
 
         TranslogException ex = expectThrows(TranslogException.class,
-                () -> new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get));
+                () -> new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get,
+                    seqNo -> {}));
         assertEquals(ex.getMessage(), "failed to create new translog file");
         assertEquals(ex.getCause().getClass(), FileAlreadyExistsException.class);
     }
@@ -2704,7 +2728,7 @@ public void testWithRandomException() throws IOException {
                     SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get());
             }
             try (Translog translog = new Translog(config, generationUUID, deletionPolicy,
-                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get);
+                () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {});
                  Translog.Snapshot snapshot = translog.newSnapshotFromGen(
                      new Translog.TranslogGeneration(generationUUID, minGenForRecovery), Long.MAX_VALUE)) {
                 assertEquals(syncedDocs.size(), snapshot.totalOperations());
@@ -2771,14 +2795,16 @@ public void testPendingDelete() throws IOException {
         final String translogUUID = translog.getTranslogUUID();
         final TranslogDeletionPolicy deletionPolicy = createTranslogDeletionPolicy(config.getIndexSettings());
         translog.close();
-        translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get);
+        translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get,
+            seqNo -> {});
         translog.add(new Translog.Index("test", "2", 1, primaryTerm.get(), new byte[]{2}));
         translog.rollGeneration();
         Closeable lock = translog.acquireRetentionLock();
         translog.add(new Translog.Index("test", "3", 2, primaryTerm.get(), new byte[]{3}));
         translog.close();
         IOUtils.close(lock);
-        translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get);
+        translog = new Translog(config, translogUUID, deletionPolicy, () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get,
+            seqNo -> {});
     }
 
     public static Translog.Location randomTranslogLocation() {
@@ -3099,7 +3125,7 @@ public void testTranslogCloseInvariant() throws IOException {
         class MisbehavingTranslog extends Translog {
             MisbehavingTranslog(TranslogConfig config, String translogUUID, TranslogDeletionPolicy deletionPolicy,
                                     LongSupplier globalCheckpointSupplier, LongSupplier primaryTermSupplier) throws IOException {
-                super(config, translogUUID, deletionPolicy, globalCheckpointSupplier, primaryTermSupplier);
+                super(config, translogUUID, deletionPolicy, globalCheckpointSupplier, primaryTermSupplier, seqNo -> {});
             }
 
             void callCloseDirectly() throws IOException {
@@ -3221,7 +3247,7 @@ public void copy(Path source, Path target, CopyOption... options) throws IOExcep
             assertFalse(brokenTranslog.isOpen());
 
             try (Translog recoveredTranslog = new Translog(getTranslogConfig(path), brokenTranslog.getTranslogUUID(),
-                brokenTranslog.getDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) {
+                brokenTranslog.getDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {})) {
                 recoveredTranslog.rollGeneration();
                 assertFilePresences(recoveredTranslog);
             }
diff --git a/server/src/test/java/org/elasticsearch/indexing/IndexActionIT.java b/server/src/test/java/org/elasticsearch/indexing/IndexActionIT.java
index 36488addb3737..3194f81303767 100644
--- a/server/src/test/java/org/elasticsearch/indexing/IndexActionIT.java
+++ b/server/src/test/java/org/elasticsearch/indexing/IndexActionIT.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -50,7 +49,6 @@ public class IndexActionIT extends ESIntegTestCase {
      * while the index is being created.
      */
 
-    @TestLogging("_root:DEBUG,org.elasticsearch.index.shard.IndexShard:TRACE,org.elasticsearch.action.search:TRACE")
     public void testAutoGenerateIdNoDuplicates() throws Exception {
         int numberOfIterations = scaledRandomIntBetween(10, 50);
         for (int i = 0; i < numberOfIterations; i++) {
@@ -84,7 +82,7 @@ public void testAutoGenerateIdNoDuplicates() throws Exception {
                 }
                 try {
                     logger.debug("running search with a specific type");
-                    SearchResponse response = client().prepareSearch("test").setTypes("type").get();
+                    SearchResponse response = client().prepareSearch("test").get();
                     if (response.getHits().getTotalHits().value != numOfDocs) {
                         final String message =
                                 "Count is " + response.getHits().getTotalHits().value + " but " + numOfDocs + " was expected. " +
diff --git a/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
index 6b4ddcf1ae64b..a48196e0ff790 100644
--- a/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
@@ -18,17 +18,18 @@
  */
 package org.elasticsearch.indices;
 
+import org.apache.lucene.index.DirectoryReader;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardIT;
 import org.elasticsearch.index.shard.IndexShardTestCase;
@@ -426,7 +427,7 @@ public void testTranslogRecoveryWorksWithIMC() throws IOException {
             client().prepareIndex("test", "test", Integer.toString(i)).setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get();
         }
 
-        IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {};
+        CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper = directoryReader -> directoryReader;
         shard.close("simon says", false);
         AtomicReference<IndexShard> shardRef = new AtomicReference<>();
         Settings settings = Settings.builder().put("indices.memory.index_buffer_size", "50kb").build();
diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/server/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
index e6450a496617a..e922fcca9c5a7 100644
--- a/server/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
@@ -21,10 +21,8 @@
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequestBuilder;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequestBuilder;
-import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistRequestBuilder;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequestBuilder;
-import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequestBuilder;
 import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequestBuilder;
@@ -85,8 +83,6 @@ public void testSpecifiedIndexUnavailableMultipleIndices() throws Exception {
         verify(forceMerge("test1", "test2"), true);
         verify(refreshBuilder("test1", "test2"), true);
         verify(validateQuery("test1", "test2"), true);
-        verify(aliasExists("test1", "test2"), true);
-        verify(typesExists("test1", "test2"), true);
         verify(getAliases("test1", "test2"), true);
         verify(getFieldMapping("test1", "test2"), true);
         verify(getMapping("test1", "test2"), true);
@@ -102,8 +98,6 @@ public void testSpecifiedIndexUnavailableMultipleIndices() throws Exception {
         verify(forceMerge("test1", "test2").setIndicesOptions(options), true);
         verify(refreshBuilder("test1", "test2").setIndicesOptions(options), true);
         verify(validateQuery("test1", "test2").setIndicesOptions(options), true);
-        verify(aliasExists("test1", "test2").setIndicesOptions(options), true);
-        verify(typesExists("test1", "test2").setIndicesOptions(options), true);
         verify(getAliases("test1", "test2").setIndicesOptions(options), true);
         verify(getFieldMapping("test1", "test2").setIndicesOptions(options), true);
         verify(getMapping("test1", "test2").setIndicesOptions(options), true);
@@ -119,8 +113,6 @@ public void testSpecifiedIndexUnavailableMultipleIndices() throws Exception {
         verify(forceMerge("test1", "test2").setIndicesOptions(options), false);
         verify(refreshBuilder("test1", "test2").setIndicesOptions(options), false);
         verify(validateQuery("test1", "test2").setIndicesOptions(options), false);
-        verify(aliasExists("test1", "test2").setIndicesOptions(options), false);
-        verify(typesExists("test1", "test2").setIndicesOptions(options), false);
         verify(getAliases("test1", "test2").setIndicesOptions(options), false);
         verify(getFieldMapping("test1", "test2").setIndicesOptions(options), false);
         verify(getMapping("test1", "test2").setIndicesOptions(options), false);
@@ -137,8 +129,6 @@ public void testSpecifiedIndexUnavailableMultipleIndices() throws Exception {
         verify(forceMerge("test1", "test2").setIndicesOptions(options), false);
         verify(refreshBuilder("test1", "test2").setIndicesOptions(options), false);
         verify(validateQuery("test1", "test2").setIndicesOptions(options), false);
-        verify(aliasExists("test1", "test2").setIndicesOptions(options), false);
-        verify(typesExists("test1", "test2").setIndicesOptions(options), false);
         verify(getAliases("test1", "test2").setIndicesOptions(options), false);
         verify(getFieldMapping("test1", "test2").setIndicesOptions(options), false);
         verify(getMapping("test1", "test2").setIndicesOptions(options), false);
@@ -164,8 +154,6 @@ public void testSpecifiedIndexUnavailableSingleIndexThatIsClosed() throws Except
         verify(forceMerge("test1").setIndicesOptions(options), true);
         verify(refreshBuilder("test1").setIndicesOptions(options), true);
         verify(validateQuery("test1").setIndicesOptions(options), true);
-        verify(aliasExists("test1").setIndicesOptions(options), true);
-        verify(typesExists("test1").setIndicesOptions(options), true);
         verify(getAliases("test1").setIndicesOptions(options), true);
         verify(getFieldMapping("test1").setIndicesOptions(options), true);
         verify(getMapping("test1").setIndicesOptions(options), true);
@@ -182,8 +170,6 @@ public void testSpecifiedIndexUnavailableSingleIndexThatIsClosed() throws Except
         verify(forceMerge("test1").setIndicesOptions(options), false);
         verify(refreshBuilder("test1").setIndicesOptions(options), false);
         verify(validateQuery("test1").setIndicesOptions(options), false);
-        verify(aliasExists("test1").setIndicesOptions(options), false);
-        verify(typesExists("test1").setIndicesOptions(options), false);
         verify(getAliases("test1").setIndicesOptions(options), false);
         verify(getFieldMapping("test1").setIndicesOptions(options), false);
         verify(getMapping("test1").setIndicesOptions(options), false);
@@ -202,8 +188,6 @@ public void testSpecifiedIndexUnavailableSingleIndexThatIsClosed() throws Except
         verify(forceMerge("test1").setIndicesOptions(options), false);
         verify(refreshBuilder("test1").setIndicesOptions(options), false);
         verify(validateQuery("test1").setIndicesOptions(options), false);
-        verify(aliasExists("test1").setIndicesOptions(options), false);
-        verify(typesExists("test1").setIndicesOptions(options), false);
         verify(getAliases("test1").setIndicesOptions(options), false);
         verify(getFieldMapping("test1").setIndicesOptions(options), false);
         verify(getMapping("test1").setIndicesOptions(options), false);
@@ -221,8 +205,6 @@ public void testSpecifiedIndexUnavailableSingleIndex() throws Exception {
         verify(forceMerge("test1").setIndicesOptions(options), true);
         verify(refreshBuilder("test1").setIndicesOptions(options), true);
         verify(validateQuery("test1").setIndicesOptions(options), true);
-        verify(aliasExists("test1").setIndicesOptions(options), true);
-        verify(typesExists("test1").setIndicesOptions(options), true);
         verify(getAliases("test1").setIndicesOptions(options), true);
         verify(getFieldMapping("test1").setIndicesOptions(options), true);
         verify(getMapping("test1").setIndicesOptions(options), true);
@@ -239,8 +221,6 @@ public void testSpecifiedIndexUnavailableSingleIndex() throws Exception {
         verify(forceMerge("test1").setIndicesOptions(options), false);
         verify(refreshBuilder("test1").setIndicesOptions(options), false);
         verify(validateQuery("test1").setIndicesOptions(options), false);
-        verify(aliasExists("test1").setIndicesOptions(options), false);
-        verify(typesExists("test1").setIndicesOptions(options), false);
         verify(getAliases("test1").setIndicesOptions(options), false);
         verify(getFieldMapping("test1").setIndicesOptions(options), false);
         verify(getMapping("test1").setIndicesOptions(options), false);
@@ -258,8 +238,6 @@ public void testSpecifiedIndexUnavailableSingleIndex() throws Exception {
         verify(forceMerge("test1").setIndicesOptions(options), false);
         verify(refreshBuilder("test1").setIndicesOptions(options), false);
         verify(validateQuery("test1").setIndicesOptions(options), false);
-        verify(aliasExists("test1").setIndicesOptions(options), false);
-        verify(typesExists("test1").setIndicesOptions(options), false);
         verify(getAliases("test1").setIndicesOptions(options), false);
         verify(getFieldMapping("test1").setIndicesOptions(options), false);
         verify(getMapping("test1").setIndicesOptions(options), false);
@@ -308,8 +286,6 @@ public void testWildcardBehaviour() throws Exception {
         verify(forceMerge(indices), false);
         verify(refreshBuilder(indices), false);
         verify(validateQuery(indices), true);
-        verify(aliasExists(indices), false);
-        verify(typesExists(indices), false);
         verify(getAliases(indices), false);
         verify(getFieldMapping(indices), false);
         verify(getMapping(indices), false);
@@ -326,8 +302,6 @@ public void testWildcardBehaviour() throws Exception {
         verify(forceMerge(indices).setIndicesOptions(options), false);
         verify(refreshBuilder(indices).setIndicesOptions(options), false);
         verify(validateQuery(indices).setIndicesOptions(options), false);
-        verify(aliasExists(indices).setIndicesOptions(options), false);
-        verify(typesExists(indices).setIndicesOptions(options), false);
         verify(getAliases(indices).setIndicesOptions(options), false);
         verify(getFieldMapping(indices).setIndicesOptions(options), false);
         verify(getMapping(indices).setIndicesOptions(options), false);
@@ -347,8 +321,6 @@ public void testWildcardBehaviour() throws Exception {
         verify(forceMerge(indices), false);
         verify(refreshBuilder(indices), false);
         verify(validateQuery(indices), false);
-        verify(aliasExists(indices), false);
-        verify(typesExists(indices), false);
         verify(getAliases(indices), false);
         verify(getFieldMapping(indices), false);
         verify(getMapping(indices), false);
@@ -365,8 +337,6 @@ public void testWildcardBehaviour() throws Exception {
         verify(forceMerge(indices), false);
         verify(refreshBuilder(indices), false);
         verify(validateQuery(indices), true);
-        verify(aliasExists(indices), false);
-        verify(typesExists(indices), false);
         verify(getAliases(indices), false);
         verify(getFieldMapping(indices), false);
         verify(getMapping(indices), false);
@@ -383,8 +353,6 @@ public void testWildcardBehaviour() throws Exception {
         verify(forceMerge(indices).setIndicesOptions(options), false);
         verify(refreshBuilder(indices).setIndicesOptions(options), false);
         verify(validateQuery(indices).setIndicesOptions(options), false);
-        verify(aliasExists(indices).setIndicesOptions(options), false);
-        verify(typesExists(indices).setIndicesOptions(options), false);
         verify(getAliases(indices).setIndicesOptions(options), false);
         verify(getFieldMapping(indices).setIndicesOptions(options), false);
         verify(getMapping(indices).setIndicesOptions(options), false);
@@ -477,7 +445,6 @@ public void testCloseApiSpecifiedIndices() throws Exception {
         verify(search("t*"), false);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/39578")
     public void testOpenCloseApiWildcards() throws Exception {
         createIndex("foo", "foobar", "bar", "barbaz");
         ensureGreen();
@@ -514,9 +481,9 @@ public void testDeleteIndex() throws Exception {
         createIndex("foobar");
 
         verify(client().admin().indices().prepareDelete("foo"), true);
-        assertThat(client().admin().indices().prepareExists("foobar").get().isExists(), equalTo(true));
+        assertThat(indexExists("foobar"), equalTo(true));
         verify(client().admin().indices().prepareDelete("foobar"), false);
-        assertThat(client().admin().indices().prepareExists("foobar").get().isExists(), equalTo(false));
+        assertThat(indexExists("foobar"), equalTo(false));
     }
 
     public void testDeleteIndexWildcard() throws Exception {
@@ -525,24 +492,23 @@ public void testDeleteIndexWildcard() throws Exception {
         createIndex("foo", "foobar", "bar", "barbaz");
 
         verify(client().admin().indices().prepareDelete("foo*"), false);
-        assertThat(client().admin().indices().prepareExists("foo").get().isExists(), equalTo(false));
-        assertThat(client().admin().indices().prepareExists("foobar").get().isExists(), equalTo(false));
-        assertThat(client().admin().indices().prepareExists("bar").get().isExists(), equalTo(true));
-        assertThat(client().admin().indices().prepareExists("barbaz").get().isExists(), equalTo(true));
+        assertThat(indexExists("foobar"), equalTo(false));
+        assertThat(indexExists("bar"), equalTo(true));
+        assertThat(indexExists("barbaz"), equalTo(true));
 
         verify(client().admin().indices().prepareDelete("foo*"), false);
 
         verify(client().admin().indices().prepareDelete("_all"), false);
-        assertThat(client().admin().indices().prepareExists("foo").get().isExists(), equalTo(false));
-        assertThat(client().admin().indices().prepareExists("foobar").get().isExists(), equalTo(false));
-        assertThat(client().admin().indices().prepareExists("bar").get().isExists(), equalTo(false));
-        assertThat(client().admin().indices().prepareExists("barbaz").get().isExists(), equalTo(false));
+        assertThat(indexExists("foo"), equalTo(false));
+        assertThat(indexExists("foobar"), equalTo(false));
+        assertThat(indexExists("bar"), equalTo(false));
+        assertThat(indexExists("barbaz"), equalTo(false));
     }
 
     public void testPutAlias() throws Exception {
         createIndex("foobar");
         verify(client().admin().indices().prepareAliases().addAlias("foobar", "foobar_alias"), false);
-        assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("foobar").get().exists(), equalTo(true));
+        assertFalse(client().admin().indices().prepareGetAliases("foobar_alias").setIndices("foobar").get().getAliases().isEmpty());
 
     }
 
@@ -550,16 +516,16 @@ public void testPutAliasWildcard() throws Exception {
         createIndex("foo", "foobar", "bar", "barbaz");
 
         verify(client().admin().indices().prepareAliases().addAlias("foo*", "foobar_alias"), false);
-        assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("foo").get().exists(), equalTo(true));
-        assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("foobar").get().exists(), equalTo(true));
-        assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("bar").get().exists(), equalTo(false));
-        assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("barbaz").get().exists(), equalTo(false));
+        assertFalse(client().admin().indices().prepareGetAliases("foobar_alias").setIndices("foo").get().getAliases().isEmpty());
+        assertFalse(client().admin().indices().prepareGetAliases("foobar_alias").setIndices("foobar").get().getAliases().isEmpty());
+        assertTrue(client().admin().indices().prepareGetAliases("foobar_alias").setIndices("bar").get().getAliases().isEmpty());
+        assertTrue(client().admin().indices().prepareGetAliases("foobar_alias").setIndices("barbaz").get().getAliases().isEmpty());
 
         verify(client().admin().indices().prepareAliases().addAlias("*", "foobar_alias"), false);
-        assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("foo").get().exists(), equalTo(true));
-        assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("foobar").get().exists(), equalTo(true));
-        assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("bar").get().exists(), equalTo(true));
-        assertThat(client().admin().indices().prepareAliasesExist("foobar_alias").setIndices("barbaz").get().exists(), equalTo(true));
+        assertFalse(client().admin().indices().prepareGetAliases("foobar_alias").setIndices("foo").get().getAliases().isEmpty());
+        assertFalse(client().admin().indices().prepareGetAliases("foobar_alias").setIndices("foobar").get().getAliases().isEmpty());
+        assertFalse(client().admin().indices().prepareGetAliases("foobar_alias").setIndices("bar").get().getAliases().isEmpty());
+        assertFalse(client().admin().indices().prepareGetAliases("foobar_alias").setIndices("barbaz").get().getAliases().isEmpty());
 
     }
 
@@ -683,14 +649,6 @@ private static ValidateQueryRequestBuilder validateQuery(String... indices) {
         return client().admin().indices().prepareValidateQuery(indices);
     }
 
-    private static AliasesExistRequestBuilder aliasExists(String... indices) {
-        return client().admin().indices().prepareAliasesExist("dummy").addIndices(indices);
-    }
-
-    private static TypesExistsRequestBuilder typesExists(String... indices) {
-        return client().admin().indices().prepareTypesExists(indices).setTypes("dummy");
-    }
-
     private static GetAliasesRequestBuilder getAliases(String... indices) {
         return client().admin().indices().prepareGetAliases("dummy").addIndices(indices);
     }
@@ -748,5 +706,4 @@ private static void verify(ActionRequestBuilder requestBuilder, boolean fail, lo
             }
         }
     }
-
 }
diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
index 7995c5f5052b7..68653223cf554 100644
--- a/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java
@@ -34,7 +34,6 @@
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.time.ZoneId;
 import java.time.ZoneOffset;
@@ -51,7 +50,6 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
-@TestLogging(value = "org.elasticsearch.indices.IndicesRequestCache:TRACE,org.elasticsearch.index.engine.Engine:DEBUG")
 public class IndicesRequestCacheIT extends ESIntegTestCase {
 
     // One of the primary purposes of the query cache is to cache aggs results
diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
index 4a77160ce36d0..a8d3b09c9e0c5 100644
--- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
@@ -330,8 +330,10 @@ public void testPendingTasks() throws Exception {
                 fail(e.getMessage());
             }
         });
-        assertThat(indicesService.hasUncompletedPendingDeletes(), equalTo(hasBogus)); // "bogus" index has not been removed
-        assertFalse(shardPath.exists());
+        assertBusy(() -> {
+            assertThat(indicesService.hasUncompletedPendingDeletes(), equalTo(hasBogus)); // "bogus" index has not been removed
+            assertFalse(shardPath.exists());
+        });
     }
 
     public void testVerifyIfIndexContentDeleted() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java b/server/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java
index 4511c59c6b3f0..e5d0a5643f0e4 100644
--- a/server/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java
@@ -19,8 +19,8 @@
 package org.elasticsearch.indices.analyze;
 
 import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -53,9 +53,9 @@ public void testSimpleAnalyzerTests() throws Exception {
         ensureGreen();
 
         for (int i = 0; i < 10; i++) {
-            AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "this is a test").get();
+            AnalyzeAction.Response analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "this is a test").get();
             assertThat(analyzeResponse.getTokens().size(), equalTo(4));
-            AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0);
+            AnalyzeAction.AnalyzeToken token = analyzeResponse.getTokens().get(0);
             assertThat(token.getTerm(), equalTo("this"));
             assertThat(token.getStartOffset(), equalTo(0));
             assertThat(token.getEndOffset(), equalTo(4));
@@ -94,7 +94,7 @@ public void testAnalyzeNumericField() throws IOException {
     }
 
     public void testAnalyzeWithNoIndex() throws Exception {
-        AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setAnalyzer("simple").get();
+        AnalyzeAction.Response analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setAnalyzer("simple").get();
         assertThat(analyzeResponse.getTokens().size(), equalTo(4));
 
         analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setTokenizer("keyword").addTokenFilter("lowercase")
@@ -105,7 +105,7 @@ public void testAnalyzeWithNoIndex() throws Exception {
         analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setTokenizer("standard").addTokenFilter("lowercase")
             .get();
         assertThat(analyzeResponse.getTokens().size(), equalTo(4));
-        AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0);
+        AnalyzeAction.AnalyzeToken token = analyzeResponse.getTokens().get(0);
         assertThat(token.getTerm(), equalTo("this"));
         token = analyzeResponse.getTokens().get(1);
         assertThat(token.getTerm(), equalTo("is"));
@@ -134,9 +134,9 @@ public void testAnalyzerWithFieldOrTypeTests() throws Exception {
             final AnalyzeRequestBuilder requestBuilder = client().admin().indices().prepareAnalyze("THIS IS A TEST");
             requestBuilder.setIndex(indexOrAlias());
             requestBuilder.setField("document.simple");
-            AnalyzeResponse analyzeResponse = requestBuilder.get();
+            AnalyzeAction.Response analyzeResponse = requestBuilder.get();
             assertThat(analyzeResponse.getTokens().size(), equalTo(4));
-            AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(3);
+            AnalyzeAction.AnalyzeToken token = analyzeResponse.getTokens().get(3);
             assertThat(token.getTerm(), equalTo("test"));
             assertThat(token.getStartOffset(), equalTo(10));
             assertThat(token.getEndOffset(), equalTo(14));
@@ -146,7 +146,7 @@ public void testAnalyzerWithFieldOrTypeTests() throws Exception {
 
     // issue #5974
     public void testThatStandardAndDefaultAnalyzersAreSame() throws Exception {
-        AnalyzeResponse response = client().admin().indices().prepareAnalyze("this is a test").setAnalyzer("standard").get();
+        AnalyzeAction.Response response = client().admin().indices().prepareAnalyze("this is a test").setAnalyzer("standard").get();
         assertTokens(response, "this", "is", "a", "test");
 
         response = client().admin().indices().prepareAnalyze("this is a test").setAnalyzer("default").get();
@@ -156,7 +156,7 @@ public void testThatStandardAndDefaultAnalyzersAreSame() throws Exception {
         assertTokens(response, "this", "is", "a", "test");
     }
 
-    private void assertTokens(AnalyzeResponse response, String ... tokens) {
+    private void assertTokens(AnalyzeAction.Response response, String ... tokens) {
         assertThat(response.getTokens(), hasSize(tokens.length));
         for (int i = 0; i < tokens.length; i++) {
             assertThat(response.getTokens().get(i).getTerm(), is(tokens[i]));
@@ -180,9 +180,9 @@ public void testAnalyzerWithMultiValues() throws Exception {
         requestBuilder.setText(texts);
         requestBuilder.setIndex(indexOrAlias());
         requestBuilder.setField("simple");
-        AnalyzeResponse analyzeResponse = requestBuilder.get();
+        AnalyzeAction.Response analyzeResponse = requestBuilder.get();
         assertThat(analyzeResponse.getTokens().size(), equalTo(7));
-        AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(3);
+        AnalyzeAction.AnalyzeToken token = analyzeResponse.getTokens().get(3);
         assertThat(token.getTerm(), equalTo("test"));
         assertThat(token.getPosition(), equalTo(3));
         assertThat(token.getStartOffset(), equalTo(10));
@@ -199,7 +199,7 @@ public void testAnalyzerWithMultiValues() throws Exception {
 
     public void testDetailAnalyzeWithNoIndex() throws Exception {
         //analyzer only
-        AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST")
+        AnalyzeAction.Response analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST")
             .setExplain(true).setAnalyzer("simple").get();
 
         assertThat(analyzeResponse.detail().tokenizer(), IsNull.nullValue());
@@ -211,7 +211,7 @@ public void testDetailAnalyzeWithNoIndex() throws Exception {
 
     public void testDetailAnalyzeCustomAnalyzerWithNoIndex() throws Exception {
         //analyzer only
-        AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST")
+        AnalyzeAction.Response analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST")
             .setExplain(true).setAnalyzer("simple").get();
 
         assertThat(analyzeResponse.detail().tokenizer(), IsNull.nullValue());
@@ -257,12 +257,12 @@ public void testDetailAnalyzeWithMultiValues() throws Exception {
             .setType("document").setSource("simple", "type=text,analyzer=simple,position_increment_gap=100").get();
 
         String[] texts = new String[]{"THIS IS A TEST", "THE SECOND TEXT"};
-        AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze().setIndex(indexOrAlias()).setText(texts)
+        AnalyzeAction.Response analyzeResponse = client().admin().indices().prepareAnalyze().setIndex(indexOrAlias()).setText(texts)
             .setExplain(true).setField("simple").setText(texts).execute().get();
 
         assertThat(analyzeResponse.detail().analyzer().getName(), equalTo("simple"));
         assertThat(analyzeResponse.detail().analyzer().getTokens().length, equalTo(7));
-        AnalyzeResponse.AnalyzeToken token = analyzeResponse.detail().analyzer().getTokens()[3];
+        AnalyzeAction.AnalyzeToken token = analyzeResponse.detail().analyzer().getTokens()[3];
 
         assertThat(token.getTerm(), equalTo("test"));
         assertThat(token.getPosition(), equalTo(3));
@@ -292,7 +292,7 @@ public void testCustomTokenFilterInRequest() throws Exception {
         Map<String, Object> stopFilterSettings = new HashMap<>();
         stopFilterSettings.put("type", "stop");
         stopFilterSettings.put("stopwords", new String[]{"foo", "buzz"});
-        AnalyzeResponse analyzeResponse = client().admin().indices()
+        AnalyzeAction.Response analyzeResponse = client().admin().indices()
             .prepareAnalyze()
             .setText("Foo buzz test")
             .setTokenizer("standard")
@@ -345,7 +345,7 @@ public void testCustomTokenFilterInRequest() throws Exception {
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getPositionLength(), equalTo(1));
 
         // tokenfilter({"type": "stop", "stopwords": ["foo", "buzz"]})
-        assertThat(analyzeResponse.detail().tokenfilters()[1].getName(), equalTo("_anonymous_tokenfilter"));
+        assertThat(analyzeResponse.detail().tokenfilters()[1].getName(), equalTo("__anonymous__stop"));
         assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens().length, equalTo(1));
 
         assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getTerm(), equalTo("test"));
@@ -359,9 +359,9 @@ public void testAnalyzeKeywordField() throws IOException {
         assertAcked(prepareCreate("test").addAlias(new Alias("alias")).addMapping("test", "keyword", "type=keyword"));
         ensureGreen("test");
 
-        AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "ABC").setField("keyword").get();
+        AnalyzeAction.Response analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "ABC").setField("keyword").get();
         assertThat(analyzeResponse.getTokens().size(), equalTo(1));
-        AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0);
+        AnalyzeAction.AnalyzeToken token = analyzeResponse.getTokens().get(0);
         assertThat(token.getTerm(), equalTo("ABC"));
         assertThat(token.getStartOffset(), equalTo(0));
         assertThat(token.getEndOffset(), equalTo(3));
@@ -377,9 +377,9 @@ public void testAnalyzeNormalizedKeywordField() throws IOException {
             .addMapping("test", "keyword", "type=keyword,normalizer=my_normalizer"));
         ensureGreen("test");
 
-        AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "ABC").setField("keyword").get();
+        AnalyzeAction.Response analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "ABC").setField("keyword").get();
         assertThat(analyzeResponse.getTokens().size(), equalTo(1));
-        AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0);
+        AnalyzeAction.AnalyzeToken token = analyzeResponse.getTokens().get(0);
         assertThat(token.getTerm(), equalTo("abc"));
         assertThat(token.getStartOffset(), equalTo(0));
         assertThat(token.getEndOffset(), equalTo(3));
diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
index 433662f95d4e0..80aaae2049fd5 100644
--- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
+++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
@@ -213,7 +213,7 @@ allocationService, new AliasValidator(), environment,
             transportService, clusterService, threadPool, createIndexService, actionFilters, indexNameExpressionResolver);
 
         nodeRemovalExecutor = new NodeRemovalClusterStateTaskExecutor(allocationService, logger);
-        joinTaskExecutor = new JoinTaskExecutor(allocationService, logger);
+        joinTaskExecutor = new JoinTaskExecutor(allocationService, logger, (s, r) -> {});
     }
 
     public ClusterState createIndex(ClusterState state, CreateIndexRequest request) {
diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/FakeThreadPoolMasterServiceTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/FakeThreadPoolMasterServiceTests.java
index 978c5b739e0f2..64d10831c8e9e 100644
--- a/server/src/test/java/org/elasticsearch/indices/cluster/FakeThreadPoolMasterServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/cluster/FakeThreadPoolMasterServiceTests.java
@@ -26,10 +26,10 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -44,7 +44,7 @@ public void testFakeMasterService() {
         List<Runnable> runnableTasks = new ArrayList<>();
         AtomicReference<ClusterState> lastClusterStateRef = new AtomicReference<>();
         DiscoveryNode discoveryNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(),
-            new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
+            new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT);
         lastClusterStateRef.set(ClusterStateCreationUtils.state(discoveryNode, discoveryNode));
         long firstClusterStateVersion = lastClusterStateRef.get().version();
         AtomicReference<ActionListener<Void>> publishingCallback = new AtomicReference<>();
diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
index 9fd7f24db024b..6c28c7bc2dc2c 100644
--- a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
@@ -21,6 +21,7 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
@@ -29,6 +30,8 @@
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
@@ -38,6 +41,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -48,7 +52,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.seqno.RetentionLeaseSyncer;
+import org.elasticsearch.index.seqno.RetentionLeases;
 import org.elasticsearch.index.shard.PrimaryReplicaSyncer;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
@@ -257,11 +261,11 @@ public void testInitializingPrimaryRemovesInitializingReplicaWithSameAID() {
     public ClusterState randomInitialClusterState(Map<DiscoveryNode, IndicesClusterStateService> clusterStateServiceMap,
                                                   Supplier<MockIndicesService> indicesServiceSupplier) {
         List<DiscoveryNode> allNodes = new ArrayList<>();
-        DiscoveryNode localNode = createNode(DiscoveryNode.Role.MASTER); // local node is the master
+        DiscoveryNode localNode = createNode(DiscoveryNodeRole.MASTER_ROLE); // local node is the master
         allNodes.add(localNode);
         // at least two nodes that have the data role so that we can allocate shards
-        allNodes.add(createNode(DiscoveryNode.Role.DATA));
-        allNodes.add(createNode(DiscoveryNode.Role.DATA));
+        allNodes.add(createNode(DiscoveryNodeRole.DATA_ROLE));
+        allNodes.add(createNode(DiscoveryNodeRole.DATA_ROLE));
         for (int i = 0; i < randomIntBetween(2, 5); i++) {
             allNodes.add(createNode());
         }
@@ -436,11 +440,9 @@ public ClusterState randomlyUpdateClusterState(ClusterState state,
 
     private static final AtomicInteger nodeIdGenerator = new AtomicInteger();
 
-    protected DiscoveryNode createNode(DiscoveryNode.Role... mustHaveRoles) {
-        Set<DiscoveryNode.Role> roles = new HashSet<>(randomSubsetOf(Sets.newHashSet(DiscoveryNode.Role.values())));
-        for (DiscoveryNode.Role mustHaveRole : mustHaveRoles) {
-            roles.add(mustHaveRole);
-        }
+    protected DiscoveryNode createNode(DiscoveryNodeRole... mustHaveRoles) {
+        Set<DiscoveryNodeRole> roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES));
+        Collections.addAll(roles, mustHaveRoles);
         final String id = String.format(Locale.ROOT, "node_%03d", nodeIdGenerator.incrementAndGet());
         return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), roles,
             Version.CURRENT);
@@ -467,6 +469,7 @@ private IndicesClusterStateService createIndicesClusterStateService(DiscoveryNod
             transportService, null, clusterService);
         final ShardStateAction shardStateAction = mock(ShardStateAction.class);
         final PrimaryReplicaSyncer primaryReplicaSyncer = mock(PrimaryReplicaSyncer.class);
+        final NodeClient client = mock(NodeClient.class);
         return new IndicesClusterStateService(
                 settings,
                 indicesService,
@@ -481,8 +484,14 @@ private IndicesClusterStateService createIndicesClusterStateService(DiscoveryNod
                 null,
                 null,
                 primaryReplicaSyncer,
-                s -> {},
-                RetentionLeaseSyncer.EMPTY);
+                client) {
+            @Override
+            public void sync(ShardId shardId, RetentionLeases retentionLeases, ActionListener<ReplicationResponse> listener) {}
+            @Override
+            public void backgroundSync(ShardId shardId, RetentionLeases retentionLeases) {}
+            @Override
+            protected void updateGlobalCheckpointForShard(final ShardId shardId) {}
+        };
     }
 
     private class RecordingIndicesService extends MockIndicesService {
diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceTests.java
new file mode 100644
index 0000000000000..9eb93586392d6
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceTests.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.indices.cluster;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction;
+import org.elasticsearch.index.seqno.RetentionLeaseSyncAction;
+import org.elasticsearch.index.seqno.RetentionLeases;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.IndexShardClosedException;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskManager;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportService;
+import org.junit.Before;
+import org.mockito.ArgumentCaptor;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.elasticsearch.mock.orig.Mockito.verifyNoMoreInteractions;
+import static org.elasticsearch.mock.orig.Mockito.when;
+import static org.hamcrest.Matchers.arrayContaining;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Matchers.same;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+public class IndicesClusterStateServiceTests extends ESTestCase {
+
+    ThreadPool threadPool;
+
+    @Before
+    public void createServices() {
+        threadPool = mock(ThreadPool.class);
+        when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
+    }
+
+    public void testRetentionLeaseBackgroundSyncExecution() {
+        final IndicesService indicesService = mock(IndicesService.class);
+
+        final Index index = new Index("index", "uuid");
+        final IndexService indexService = mock(IndexService.class);
+        when(indicesService.indexServiceSafe(index)).thenReturn(indexService);
+
+        final int id = randomIntBetween(0, 4);
+        final IndexShard indexShard = mock(IndexShard.class);
+        when(indexService.getShard(id)).thenReturn(indexShard);
+
+        final ShardId shardId = new ShardId(index, id);
+        when(indexShard.shardId()).thenReturn(shardId);
+
+        final Logger mockLogger = mock(Logger.class);
+        final TransportService transportService = mock(TransportService.class);
+        when(transportService.getTaskManager()).thenReturn(mock(TaskManager.class));
+
+        final RetentionLeases retentionLeases = mock(RetentionLeases.class);
+        final AtomicBoolean invoked = new AtomicBoolean();
+        final RetentionLeaseBackgroundSyncAction action = new RetentionLeaseBackgroundSyncAction(
+            Settings.EMPTY,
+            transportService,
+            null,
+            indicesService,
+            threadPool,
+            null,
+            new ActionFilters(Collections.emptySet()),
+            new IndexNameExpressionResolver()) {
+
+            @Override
+            protected void doExecute(Task task, RetentionLeaseBackgroundSyncAction.Request request,
+                                     ActionListener<ReplicationResponse> listener) {
+                assertTrue(threadPool.getThreadContext().isSystemContext());
+                assertThat(request.shardId(), sameInstance(indexShard.shardId()));
+                assertThat(request.getRetentionLeases(), sameInstance(retentionLeases));
+                if (randomBoolean()) {
+                    listener.onResponse(new ReplicationResponse());
+                } else {
+                    final Exception e = randomFrom(
+                        new AlreadyClosedException("closed"),
+                        new IndexShardClosedException(indexShard.shardId()),
+                        new TransportException(randomFrom(
+                            "failed",
+                            "TransportService is closed stopped can't send request",
+                            "transport stopped, action: indices:admin/seq_no/retention_lease_background_sync[p]")),
+                        new RuntimeException("failed"));
+                    listener.onFailure(e);
+                    if (e.getMessage().equals("failed")) {
+                        final ArgumentCaptor<ParameterizedMessage> captor = ArgumentCaptor.forClass(ParameterizedMessage.class);
+                        verify(mockLogger).warn(captor.capture(), same(e));
+                        final ParameterizedMessage message = captor.getValue();
+                        assertThat(message.getFormat(), equalTo("{} retention lease background sync failed"));
+                        assertThat(message.getParameters(), arrayContaining(indexShard.shardId()));
+                    }
+                    verifyNoMoreInteractions(mockLogger);
+                }
+                invoked.set(true);
+            }
+        };
+        NodeClient client = new NodeClient(Settings.EMPTY, null);
+        Map<ActionType, TransportAction> actions = Collections.singletonMap(RetentionLeaseBackgroundSyncAction.TYPE, action);
+        client.initialize(actions, null, null);
+        IndicesClusterStateService service = new IndicesClusterStateService(Settings.EMPTY, null, null, threadPool, null, null, null,
+            null, null, null, null, null, null, client) {
+            @Override
+            protected Logger getLogger() {
+                return mockLogger;
+            }
+        };
+
+        service.backgroundSync(indexShard.shardId(), retentionLeases);
+        assertTrue(invoked.get());
+    }
+
+
+    public void testRetentionLeaseSyncExecution() {
+        final IndicesService indicesService = mock(IndicesService.class);
+
+        final Index index = new Index("index", "uuid");
+        final IndexService indexService = mock(IndexService.class);
+        when(indicesService.indexServiceSafe(index)).thenReturn(indexService);
+        final int id = randomIntBetween(0, 4);
+        final IndexShard indexShard = mock(IndexShard.class);
+        when(indexService.getShard(id)).thenReturn(indexShard);
+
+        final ShardId shardId = new ShardId(index, id);
+        when(indexShard.shardId()).thenReturn(shardId);
+
+        final Logger mockLogger = mock(Logger.class);
+        final TransportService transportService = mock(TransportService.class);
+        when(transportService.getTaskManager()).thenReturn(mock(TaskManager.class));
+
+        final RetentionLeases retentionLeases = mock(RetentionLeases.class);
+        final AtomicBoolean invoked = new AtomicBoolean();
+        final RetentionLeaseSyncAction action = new RetentionLeaseSyncAction(
+            Settings.EMPTY,
+            transportService,
+            null,
+            indicesService,
+            threadPool,
+            null,
+            new ActionFilters(Collections.emptySet()),
+            new IndexNameExpressionResolver()) {
+
+            @Override
+            protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
+                assertTrue(threadPool.getThreadContext().isSystemContext());
+                assertThat(request.shardId(), sameInstance(indexShard.shardId()));
+                assertThat(request.getRetentionLeases(), sameInstance(retentionLeases));
+                if (randomBoolean()) {
+                    listener.onResponse(new Response());
+                } else {
+                    final Exception e = randomFrom(
+                        new AlreadyClosedException("closed"),
+                        new IndexShardClosedException(indexShard.shardId()),
+                        new RuntimeException("failed"));
+                    listener.onFailure(e);
+                    if (e instanceof AlreadyClosedException == false && e instanceof IndexShardClosedException == false) {
+                        final ArgumentCaptor<ParameterizedMessage> captor = ArgumentCaptor.forClass(ParameterizedMessage.class);
+                        verify(mockLogger).warn(captor.capture(), same(e));
+                        final ParameterizedMessage message = captor.getValue();
+                        assertThat(message.getFormat(), equalTo("{} retention lease sync failed"));
+                        assertThat(message.getParameters(), arrayContaining(indexShard.shardId()));
+                    }
+                    verifyNoMoreInteractions(mockLogger);
+                }
+                invoked.set(true);
+            }
+        };
+        NodeClient client = new NodeClient(Settings.EMPTY, null);
+        Map<ActionType, TransportAction> actions = Collections.singletonMap(RetentionLeaseSyncAction.TYPE, action);
+        client.initialize(actions, null, null);
+        IndicesClusterStateService service = new IndicesClusterStateService(Settings.EMPTY, null, null, threadPool, null, null, null,
+            null, null, null, null, null, null, client) {
+            @Override
+            protected Logger getLogger() {
+                return mockLogger;
+            }
+        };
+
+        // execution happens on the test thread, so no need to register an actual listener to callback
+        service.sync(indexShard.shardId(), retentionLeases, ActionListener.wrap(() -> {}));
+        assertTrue(invoked.get());
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/indices/exists/indices/IndicesExistsIT.java b/server/src/test/java/org/elasticsearch/indices/exists/indices/IndicesExistsIT.java
deleted file mode 100644
index dbfa5fb2d077a..0000000000000
--- a/server/src/test/java/org/elasticsearch/indices/exists/indices/IndicesExistsIT.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.indices.exists.indices;
-
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder;
-import org.elasticsearch.cluster.block.ClusterBlockException;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import java.util.Arrays;
-
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.hamcrest.Matchers.equalTo;
-
-public class IndicesExistsIT extends ESIntegTestCase {
-    // Indices exists never throws IndexMissingException, the indices options control its behaviour (return true or false)
-    public void testIndicesExists() throws Exception {
-        assertFalse(client().admin().indices().prepareExists("foo").get().isExists());
-        assertFalse(client().admin().indices().prepareExists("foo*").get().isExists());
-        assertFalse(client().admin().indices().prepareExists("_all").get().isExists());
-
-        createIndex("foo", "foobar", "bar", "barbaz");
-
-        IndicesExistsRequestBuilder indicesExistsRequestBuilder = client().admin().indices().prepareExists("foo*")
-                .setExpandWildcardsOpen(false);
-        IndicesExistsRequest request = indicesExistsRequestBuilder.request();
-        //check that ignore unavailable and allow no indices are set to false. That is their only valid value as it can't be overridden
-        assertFalse(request.indicesOptions().ignoreUnavailable());
-        assertFalse(request.indicesOptions().allowNoIndices());
-        assertThat(indicesExistsRequestBuilder.get().isExists(), equalTo(false));
-
-        assertAcked(client().admin().indices().prepareClose("foobar").get());
-
-        assertThat(client().admin().indices().prepareExists("foo*").get().isExists(), equalTo(true));
-        assertThat(client().admin().indices().prepareExists("foo*").setExpandWildcardsOpen(false)
-                .setExpandWildcardsClosed(false).get().isExists(), equalTo(false));
-        assertThat(client().admin().indices().prepareExists("foobar").get().isExists(), equalTo(true));
-        assertThat(client().admin().indices().prepareExists("foob*").setExpandWildcardsClosed(false).get().isExists(), equalTo(false));
-        assertThat(client().admin().indices().prepareExists("bar*").get().isExists(), equalTo(true));
-        assertThat(client().admin().indices().prepareExists("bar").get().isExists(), equalTo(true));
-        assertThat(client().admin().indices().prepareExists("_all").get().isExists(), equalTo(true));
-    }
-
-    public void testIndicesExistsWithBlocks() {
-        createIndex("ro");
-
-        // Request is not blocked
-        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY,
-            SETTING_READ_ONLY_ALLOW_DELETE)) {
-            try {
-                enableIndexBlock("ro", blockSetting);
-                assertThat(client().admin().indices().prepareExists("ro").execute().actionGet().isExists(), equalTo(true));
-            } finally {
-                disableIndexBlock("ro", blockSetting);
-            }
-        }
-
-        // Request is blocked
-        try {
-            enableIndexBlock("ro", IndexMetaData.SETTING_BLOCKS_METADATA);
-            assertThat(client().admin().indices().prepareExists("ro").execute().actionGet().isExists(), equalTo(true));
-            fail("Exists should fail when " + IndexMetaData.SETTING_BLOCKS_METADATA + " is true");
-        } catch (ClusterBlockException e) {
-            // Ok, a ClusterBlockException is expected
-        } finally {
-            disableIndexBlock("ro", IndexMetaData.SETTING_BLOCKS_METADATA);
-        }
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/indices/exists/types/TypesExistsIT.java b/server/src/test/java/org/elasticsearch/indices/exists/types/TypesExistsIT.java
deleted file mode 100644
index 294df4960328e..0000000000000
--- a/server/src/test/java/org/elasticsearch/indices/exists/types/TypesExistsIT.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.indices.exists.types;
-
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
-import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.InternalSettingsPlugin;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WRITE;
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
-import static org.hamcrest.Matchers.equalTo;
-
-public class TypesExistsIT extends ESIntegTestCase {
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(InternalSettingsPlugin.class);
-    }
-
-    public void testSimple() throws Exception {
-        Client client = client();
-        CreateIndexResponse response1 = client.admin().indices().prepareCreate("test1")
-                .addMapping("type1", jsonBuilder().startObject().startObject("type1").endObject().endObject())
-                .execute().actionGet();
-        CreateIndexResponse response2 = client.admin().indices().prepareCreate("test2")
-                .addMapping("type2", jsonBuilder().startObject().startObject("type2").endObject().endObject())
-                .execute().actionGet();
-        client.admin().indices().prepareAliases().addAlias("test1", "alias1").execute().actionGet();
-        assertAcked(response1);
-        assertAcked(response2);
-
-        TypesExistsResponse response = client.admin().indices().prepareTypesExists("test1").setTypes("type1").execute().actionGet();
-        assertThat(response.isExists(), equalTo(true));
-        response = client.admin().indices().prepareTypesExists("test1").setTypes("type2").execute().actionGet();
-        assertThat(response.isExists(), equalTo(false));
-        try {
-            client.admin().indices().prepareTypesExists("notExist").setTypes("type1").execute().actionGet();
-            fail("Exception should have been thrown");
-        } catch (IndexNotFoundException e) {}
-        try {
-            client.admin().indices().prepareTypesExists("notExist").setTypes("type0").execute().actionGet();
-            fail("Exception should have been thrown");
-        } catch (IndexNotFoundException e) {}
-        response = client.admin().indices().prepareTypesExists("alias1").setTypes("type1").execute().actionGet();
-        assertThat(response.isExists(), equalTo(true));
-        response = client.admin().indices().prepareTypesExists("*").setTypes("type1").execute().actionGet();
-        assertThat(response.isExists(), equalTo(false));
-        response = client.admin().indices().prepareTypesExists("test1", "test2").setTypes("type1").execute().actionGet();
-        assertThat(response.isExists(), equalTo(false));
-        response = client.admin().indices().prepareTypesExists("test1", "test2").setTypes("type2").execute().actionGet();
-        assertThat(response.isExists(), equalTo(false));
-    }
-
-    public void testTypesExistsWithBlocks() throws IOException {
-        assertAcked(prepareCreate("ro").addMapping("type1", jsonBuilder().startObject().startObject("type1").endObject().endObject()));
-        ensureGreen("ro");
-
-        // Request is not blocked
-        for (String block : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY)) {
-            try {
-                enableIndexBlock("ro", block);
-                assertThat(client().admin().indices().prepareTypesExists("ro").setTypes("type1").execute().actionGet().isExists(),
-                    equalTo(true));
-            } finally {
-                disableIndexBlock("ro", block);
-            }
-        }
-
-        // Request is blocked
-        try {
-            enableIndexBlock("ro", IndexMetaData.SETTING_BLOCKS_METADATA);
-            assertBlocked(client().admin().indices().prepareTypesExists("ro").setTypes("type1"));
-        } finally {
-            disableIndexBlock("ro", IndexMetaData.SETTING_BLOCKS_METADATA);
-        }
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
index 0b0f71e69928d..3710988772af3 100644
--- a/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
@@ -18,16 +18,6 @@
  */
 package org.elasticsearch.indices.flush;
 
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-
 import org.apache.lucene.index.Term;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
@@ -49,6 +39,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.engine.InternalEngine;
 import org.elasticsearch.index.engine.InternalEngineTests;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.Uid;
@@ -59,6 +50,16 @@
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.ESIntegTestCase;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.emptyArray;
@@ -274,9 +275,10 @@ public void testUnallocatedShardsDoesNotHang() throws InterruptedException {
     private void indexDoc(Engine engine, String id) throws IOException {
         final ParsedDocument doc = InternalEngineTests.createParsedDoc(id, null);
         final Engine.IndexResult indexResult = engine.index(new Engine.Index(new Term("_id", Uid.encodeId(doc.id())), doc,
-            engine.getLocalCheckpoint() + 1, 1L, 1L, null, Engine.Operation.Origin.REPLICA, randomLong(), -1L, false,
-            SequenceNumbers.UNASSIGNED_SEQ_NO, 0));
+            ((InternalEngine) engine).getProcessedLocalCheckpoint() + 1, 1L, 1L, null, Engine.Operation.Origin.REPLICA, System.nanoTime(),
+            -1L, false, SequenceNumbers.UNASSIGNED_SEQ_NO, 0));
         assertThat(indexResult.getFailure(), nullValue());
+        engine.syncTranslog();
     }
 
     public void testSyncedFlushSkipOutOfSyncReplicas() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java b/server/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
index f379b7ee5229a..b86b622705e1e 100644
--- a/server/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
@@ -51,10 +51,8 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Random;
-import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
@@ -70,10 +68,8 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     @Override
-    protected Collection<Class<? extends Plugin>> getMockPlugins() {
-        Set<Class<? extends Plugin>> mocks = new HashSet<>(super.getMockPlugins());
-        mocks.remove(MockEngineFactoryPlugin.class);
-        return mocks;
+    protected boolean addMockInternalEngine() {
+        return false;
     }
 
     public void testBreakerWithRandomExceptions() throws IOException, InterruptedException, ExecutionException {
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java
index 0c056da34a56d..b020c3d6b88dc 100644
--- a/server/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java
@@ -31,21 +31,17 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.hamcrest.Matchers.equalTo;
 
-@TestLogging("_root:DEBUG")
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
 public class IndexPrimaryRelocationIT extends ESIntegTestCase {
 
     private static final int RELOCATION_COUNT = 15;
 
-    @TestLogging("_root:DEBUG,org.elasticsearch.action.bulk:TRACE,org.elasticsearch.index.shard:TRACE," +
-        "org.elasticsearch.cluster.service:TRACE")
     public void testPrimaryRelocationWhileIndexing() throws Exception {
         internalCluster().ensureAtLeastNumDataNodes(randomIntBetween(2, 3));
         client().admin().indices().prepareCreate("test")
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
index 4710c59647c25..18aef3720c31a 100644
--- a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
@@ -521,7 +521,7 @@ public void testSnapshotRecovery() throws Exception {
             equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
 
         assertThat(client().admin().cluster().prepareGetSnapshots(REPO_NAME).setSnapshots(SNAP_NAME).get()
-                .getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
+                .getSnapshots(REPO_NAME).get(0).state(), equalTo(SnapshotState.SUCCESS));
 
         client().admin().indices().prepareClose(INDEX_NAME).execute().actionGet();
 
@@ -832,7 +832,6 @@ public void sendRequest(Transport.Connection connection, long requestId, String
         }
     }
 
-    @TestLogging("org.elasticsearch.indices.recovery:TRACE")
     public void testHistoryRetention() throws Exception {
         internalCluster().startNodes(3);
 
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java
index bb4c25e6186de..1154ce99078b0 100644
--- a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java
@@ -28,6 +28,7 @@
 import org.apache.lucene.store.IndexInput;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.UUIDs;
@@ -144,11 +145,13 @@ public void testWriteFileChunksConcurrently() throws Exception {
         final DiscoveryNode rNode = getFakeDiscoNode(targetShard.routingEntry().currentNodeId());
         targetShard.markAsRecovering("test-peer-recovery", new RecoveryState(targetShard.routingEntry(), rNode, pNode));
         final RecoveryTarget recoveryTarget = new RecoveryTarget(targetShard, null, null);
+        final PlainActionFuture<Void> receiveFileInfoFuture = new PlainActionFuture<>();
         recoveryTarget.receiveFileInfo(
             mdFiles.stream().map(StoreFileMetaData::name).collect(Collectors.toList()),
             mdFiles.stream().map(StoreFileMetaData::length).collect(Collectors.toList()),
-            Collections.emptyList(), Collections.emptyList(), 0
+            Collections.emptyList(), Collections.emptyList(), 0, receiveFileInfoFuture
         );
+        receiveFileInfoFuture.actionGet();
         List<RecoveryFileChunkRequest> requests = new ArrayList<>();
         for (StoreFileMetaData md : mdFiles) {
             try (IndexInput in = sourceShard.store().directory().openInput(md.name(), IOContext.READONCE)) {
@@ -189,7 +192,10 @@ public void testWriteFileChunksConcurrently() throws Exception {
         for (Thread sender : senders) {
             sender.join();
         }
-        recoveryTarget.cleanFiles(0, Long.parseLong(sourceSnapshot.getCommitUserData().get(SequenceNumbers.MAX_SEQ_NO)), sourceSnapshot);
+        PlainActionFuture<Void> cleanFilesFuture = new PlainActionFuture<>();
+        recoveryTarget.cleanFiles(0, Long.parseLong(sourceSnapshot.getCommitUserData().get(SequenceNumbers.MAX_SEQ_NO)),
+            sourceSnapshot, cleanFilesFuture);
+        cleanFilesFuture.actionGet();
         recoveryTarget.decRef();
         Store.MetadataSnapshot targetSnapshot = targetShard.snapshotStoreMetadata();
         Store.RecoveryDiff diff = sourceSnapshot.recoveryDiff(targetSnapshot);
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
index b00e89575ccd5..215bf475a0c9b 100644
--- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
@@ -98,7 +98,6 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.IntSupplier;
-import java.util.function.Supplier;
 import java.util.zip.CRC32;
 
 import static java.util.Collections.emptyMap;
@@ -478,9 +477,9 @@ public void testThrowExceptionOnPrimaryRelocatedBeforePhase1Started() throws IOE
                 between(1, 8)) {
 
             @Override
-            public SendFileResult phase1(final IndexCommit snapshot, final long globalCheckpoint, final Supplier<Integer> translogOps) {
+            void phase1(IndexCommit snapshot, long globalCheckpoint, IntSupplier translogOps, ActionListener<SendFileResult> listener) {
                 phase1Called.set(true);
-                return super.phase1(snapshot, globalCheckpoint, translogOps);
+                super.phase1(snapshot, globalCheckpoint, translogOps, listener);
             }
 
             @Override
@@ -754,11 +753,13 @@ public void indexTranslogOperations(
 
         @Override
         public void receiveFileInfo(List<String> phase1FileNames, List<Long> phase1FileSizes, List<String> phase1ExistingFileNames,
-                                    List<Long> phase1ExistingFileSizes, int totalTranslogOps) {
+                                    List<Long> phase1ExistingFileSizes, int totalTranslogOps, ActionListener<Void> listener) {
+
         }
 
         @Override
-        public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetaData) {
+        public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetaData,
+                               ActionListener<Void> listener) {
         }
 
         @Override
diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
index 1dc2ba058b75e..28e84c1210a29 100644
--- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
+++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java
@@ -47,7 +47,6 @@
 import org.elasticsearch.index.translog.SnapshotMatchers;
 import org.elasticsearch.index.translog.Translog;
 
-import java.io.IOException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -143,6 +142,7 @@ public void testRecoveryWithOutOfOrderDeleteWithTranslog() throws Exception {
             // index #2
             orgReplica.applyIndexOperationOnReplica(2, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
                 new SourceToParse(indexName, "type", "id-2", new BytesArray("{}"), XContentType.JSON));
+            orgReplica.sync(); // advance local checkpoint
             orgReplica.updateGlobalCheckpointOnReplica(3L, "test");
             // index #5 -> force NoOp #4.
             orgReplica.applyIndexOperationOnReplica(5, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
@@ -207,6 +207,7 @@ public void testRecoveryWithOutOfOrderDeleteWithSoftDeletes() throws Exception {
             // index #2
             orgReplica.applyIndexOperationOnReplica(2, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
                 new SourceToParse(indexName, "type", "id-2", new BytesArray("{}"), XContentType.JSON));
+            orgReplica.sync(); // advance local checkpoint
             orgReplica.updateGlobalCheckpointOnReplica(3L, "test");
             // index #5 -> force NoOp #4.
             orgReplica.applyIndexOperationOnReplica(5, 1, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false,
@@ -330,12 +331,13 @@ public void testPeerRecoverySendSafeCommitInFileBased() throws Exception {
             @Override
             public void prepareForTranslogOperations(boolean fileBasedRecovery, int totalTranslogOps, ActionListener<Void> listener) {
                 super.prepareForTranslogOperations(fileBasedRecovery, totalTranslogOps, listener);
-                assertThat(replicaShard.getGlobalCheckpoint(), equalTo(primaryShard.getGlobalCheckpoint()));
+                assertThat(replicaShard.getLastKnownGlobalCheckpoint(), equalTo(primaryShard.getLastKnownGlobalCheckpoint()));
             }
             @Override
-            public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetaData) throws IOException {
-                assertThat(globalCheckpoint, equalTo(primaryShard.getGlobalCheckpoint()));
-                super.cleanFiles(totalTranslogOps, globalCheckpoint, sourceMetaData);
+            public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetaData,
+                                   ActionListener<Void> listener) {
+                assertThat(globalCheckpoint, equalTo(primaryShard.getLastKnownGlobalCheckpoint()));
+                super.cleanFiles(totalTranslogOps, globalCheckpoint, sourceMetaData, listener);
             }
         }, true, true);
         List<IndexCommit> commits = DirectoryReader.listCommits(replicaShard.store().directory());
diff --git a/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java b/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java
index 54750933ecd90..24cd6a88335de 100644
--- a/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java
+++ b/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java
@@ -19,11 +19,11 @@
 
 package org.elasticsearch.indices.settings;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
@@ -41,6 +41,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -62,7 +63,7 @@ public List<Setting<?>> getSettings() {
         return Arrays.asList(INDEX_INTERNAL_SETTING, INDEX_PRIVATE_SETTING);
     }
 
-    public static class UpdateInternalOrPrivateAction extends Action<UpdateInternalOrPrivateAction.Response> {
+    public static class UpdateInternalOrPrivateAction extends StreamableResponseActionType<UpdateInternalOrPrivateAction.Response> {
 
         public static final UpdateInternalOrPrivateAction INSTANCE = new UpdateInternalOrPrivateAction();
         private static final String NAME = "indices:admin/settings/update-internal-or-private-index";
@@ -153,9 +154,9 @@ protected UpdateInternalOrPrivateAction.Response newResponse() {
 
         @Override
         protected void masterOperation(
-                final UpdateInternalOrPrivateAction.Request request,
-                final ClusterState state,
-                final ActionListener<UpdateInternalOrPrivateAction.Response> listener) throws Exception {
+            Task task, final UpdateInternalOrPrivateAction.Request request,
+            final ClusterState state,
+            final ActionListener<UpdateInternalOrPrivateAction.Response> listener) throws Exception {
             clusterService.submitStateUpdateTask("update-index-internal-or-private", new ClusterStateUpdateTask() {
                 @Override
                 public ClusterState execute(final ClusterState currentState) throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/indices/state/CloseIndexIT.java b/server/src/test/java/org/elasticsearch/indices/state/CloseIndexIT.java
index b39a008de5f4f..2701bfc104c71 100644
--- a/server/src/test/java/org/elasticsearch/indices/state/CloseIndexIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/state/CloseIndexIT.java
@@ -74,10 +74,9 @@ public class CloseIndexIT extends ESIntegTestCase {
 
     @Override
     public Settings indexSettings() {
-        Settings.builder().put(super.indexSettings())
+        return Settings.builder().put(super.indexSettings())
             .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(),
-                new ByteSizeValue(randomIntBetween(1, 4096), ByteSizeUnit.KB));
-        return super.indexSettings();
+                new ByteSizeValue(randomIntBetween(1, 4096), ByteSizeUnit.KB)).build();
     }
 
     public void testCloseMissingIndex() {
diff --git a/server/src/test/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java b/server/src/test/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java
index 71db668c29f26..2125184baef63 100644
--- a/server/src/test/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/state/CloseWhileRelocatingShardsIT.java
@@ -40,7 +40,6 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.BackgroundIndexer;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.test.transport.StubbableTransport;
 import org.elasticsearch.transport.TransportService;
@@ -87,8 +86,6 @@ protected int maximumNumberOfShards() {
         return 3;
     }
 
-    @TestLogging("_root_:DEBUG,org.elasticsearch.indices.recovery:TRACE,org.elasticsearch.action.admin.indices.close:TRACE,"
-        + "org.elasticsearch.cluster.metadata.MetaDataIndexStateService:DEBUG")
     public void testCloseWhileRelocatingShards() throws Exception {
         final String[] indices = new String[randomIntBetween(3, 5)];
         final Map<String, Long> docsPerIndex = new HashMap<>();
diff --git a/server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java b/server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java
index e9e9108f5e8f1..cd9cb8c441375 100644
--- a/server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java
@@ -22,7 +22,9 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -34,6 +36,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.io.IOException;
@@ -54,6 +57,7 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
 
 public class OpenCloseIndexIT extends ESIntegTestCase {
     public void testSimpleCloseOpen() {
@@ -287,7 +291,7 @@ public void testOpenCloseWithDocs() throws IOException, ExecutionException, Inte
         // check the index still contains the records that we indexed
         client().admin().indices().prepareOpen("test").execute().get();
         ensureGreen();
-        SearchResponse searchResponse = client().prepareSearch().setTypes("type").setQuery(QueryBuilders.matchQuery("test", "init")).get();
+        SearchResponse searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchQuery("test", "init")).get();
         assertNoFailures(searchResponse);
         assertHitCount(searchResponse, docs);
     }
@@ -346,4 +350,38 @@ public void testOpenCloseIndexWithBlocks() {
             }
         }
     }
+
+    public void testTranslogStats()  {
+        final String indexName = "test";
+        createIndex(indexName, Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .build());
+
+        final int nbDocs = randomIntBetween(0, 50);
+        int uncommittedOps = 0;
+        for (long i = 0; i < nbDocs; i++) {
+            final IndexResponse indexResponse = client().prepareIndex(indexName, "_doc", Long.toString(i)).setSource("field", i).get();
+            assertThat(indexResponse.status(), is(RestStatus.CREATED));
+
+            if (rarely()) {
+                client().admin().indices().prepareFlush(indexName).get();
+                uncommittedOps = 0;
+            } else {
+                uncommittedOps += 1;
+            }
+        }
+
+        IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName).clear().setTranslog(true).get();
+        assertThat(stats.getIndex(indexName), notNullValue());
+        assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(nbDocs));
+        assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(uncommittedOps));
+
+        assertAcked(client().admin().indices().prepareClose("test"));
+
+        IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
+        stats = client().admin().indices().prepareStats(indexName).setIndicesOptions(indicesOptions).clear().setTranslog(true).get();
+        assertThat(stats.getIndex(indexName), notNullValue());
+        assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(nbDocs));
+        assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(0));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/indices/state/ReopenWhileClosingIT.java b/server/src/test/java/org/elasticsearch/indices/state/ReopenWhileClosingIT.java
index 0f57f518e70cd..8cf3b76184ae4 100644
--- a/server/src/test/java/org/elasticsearch/indices/state/ReopenWhileClosingIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/state/ReopenWhileClosingIT.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.Glob;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.RunOnce;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -50,7 +51,7 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, minNumDataNodes = 2)
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
 public class ReopenWhileClosingIT extends ESIntegTestCase {
 
     @Override
@@ -64,8 +65,9 @@ protected int minimumNumberOfShards() {
     }
 
     public void testReopenDuringClose() throws Exception {
+        List<String> dataOnlyNodes = internalCluster().startDataOnlyNodes(randomIntBetween(2, 3));
         final String indexName = "test";
-        createIndexWithDocs(indexName);
+        createIndexWithDocs(indexName, dataOnlyNodes);
 
         ensureYellowAndNoInitializingShards(indexName);
 
@@ -84,12 +86,12 @@ public void testReopenDuringClose() throws Exception {
         assertIndexIsOpened(indexName);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/39757")
     public void testReopenDuringCloseOnMultipleIndices() throws Exception {
+        List<String> dataOnlyNodes = internalCluster().startDataOnlyNodes(randomIntBetween(2, 3));
         final List<String> indices = new ArrayList<>();
         for (int i = 0; i < randomIntBetween(2, 10); i++) {
             indices.add("index-" + i);
-            createIndexWithDocs(indices.get(i));
+            createIndexWithDocs(indices.get(i), dataOnlyNodes);
         }
 
         ensureYellowAndNoInitializingShards(indices.toArray(Strings.EMPTY_ARRAY));
@@ -117,8 +119,9 @@ public void testReopenDuringCloseOnMultipleIndices() throws Exception {
         });
     }
 
-    private void createIndexWithDocs(final String indexName) {
-        createIndex(indexName);
+    private void createIndexWithDocs(final String indexName, final Collection<String> dataOnlyNodes) {
+        createIndex(indexName,
+            Settings.builder().put(indexSettings()).put("index.routing.allocation.include._name", String.join(",", dataOnlyNodes)).build());
         final int nbDocs =  scaledRandomIntBetween(1, 100);
         for (int i = 0; i < nbDocs; i++) {
             index(indexName, "_doc", String.valueOf(i), "num", i);
diff --git a/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
index 59e7c21a3e6e8..95c9b1adf6ada 100644
--- a/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
+++ b/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
@@ -1204,7 +1204,7 @@ private void persistGlobalCheckpoint(String index) throws Exception {
             for (IndexService indexService : indexServices) {
                 for (IndexShard indexShard : indexService) {
                     indexShard.sync();
-                    assertThat(indexShard.getLastSyncedGlobalCheckpoint(), equalTo(indexShard.getGlobalCheckpoint()));
+                    assertThat(indexShard.getLastSyncedGlobalCheckpoint(), equalTo(indexShard.getLastKnownGlobalCheckpoint()));
                 }
             }
         }
diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
index e5aea1f5d5ce1..43e2a8a584979 100644
--- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
@@ -153,10 +153,72 @@ public void testUpdatePipelines() {
             .build();
         ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
         assertThat(ingestService.pipelines().size(), is(1));
-        assertThat(ingestService.pipelines().get("_id").getId(), equalTo("_id"));
-        assertThat(ingestService.pipelines().get("_id").getDescription(), nullValue());
-        assertThat(ingestService.pipelines().get("_id").getProcessors().size(), equalTo(1));
-        assertThat(ingestService.pipelines().get("_id").getProcessors().get(0).getType(), equalTo("set"));
+        assertThat(ingestService.pipelines().get("_id").pipeline.getId(), equalTo("_id"));
+        assertThat(ingestService.pipelines().get("_id").pipeline.getDescription(), nullValue());
+        assertThat(ingestService.pipelines().get("_id").pipeline.getProcessors().size(), equalTo(1));
+        assertThat(ingestService.pipelines().get("_id").pipeline.getProcessors().get(0).getType(), equalTo("set"));
+    }
+
+    public void testInnerUpdatePipelines() {
+        IngestService ingestService = createWithProcessors();
+        assertThat(ingestService.pipelines().size(), is(0));
+
+        PipelineConfiguration pipeline1 = new PipelineConfiguration("_id1", new BytesArray("{\"processors\": []}"), XContentType.JSON);
+        IngestMetadata ingestMetadata = new IngestMetadata(Map.of("_id1", pipeline1));
+
+        ingestService.innerUpdatePipelines(ingestMetadata);
+        assertThat(ingestService.pipelines().size(), is(1));
+        assertThat(ingestService.pipelines().get("_id1").pipeline.getId(), equalTo("_id1"));
+        assertThat(ingestService.pipelines().get("_id1").pipeline.getProcessors().size(), equalTo(0));
+
+        PipelineConfiguration pipeline2 = new PipelineConfiguration("_id2", new BytesArray("{\"processors\": []}"), XContentType.JSON);
+        ingestMetadata = new IngestMetadata(Map.of("_id1", pipeline1, "_id2", pipeline2));
+
+        ingestService.innerUpdatePipelines(ingestMetadata);
+        assertThat(ingestService.pipelines().size(), is(2));
+        assertThat(ingestService.pipelines().get("_id1").pipeline.getId(), equalTo("_id1"));
+        assertThat(ingestService.pipelines().get("_id1").pipeline.getProcessors().size(), equalTo(0));
+        assertThat(ingestService.pipelines().get("_id2").pipeline.getId(), equalTo("_id2"));
+        assertThat(ingestService.pipelines().get("_id2").pipeline.getProcessors().size(), equalTo(0));
+
+        PipelineConfiguration pipeline3 = new PipelineConfiguration("_id3", new BytesArray("{\"processors\": []}"), XContentType.JSON);
+        ingestMetadata = new IngestMetadata(Map.of("_id1", pipeline1, "_id2", pipeline2, "_id3", pipeline3));
+
+        ingestService.innerUpdatePipelines(ingestMetadata);
+        assertThat(ingestService.pipelines().size(), is(3));
+        assertThat(ingestService.pipelines().get("_id1").pipeline.getId(), equalTo("_id1"));
+        assertThat(ingestService.pipelines().get("_id1").pipeline.getProcessors().size(), equalTo(0));
+        assertThat(ingestService.pipelines().get("_id2").pipeline.getId(), equalTo("_id2"));
+        assertThat(ingestService.pipelines().get("_id2").pipeline.getProcessors().size(), equalTo(0));
+        assertThat(ingestService.pipelines().get("_id3").pipeline.getId(), equalTo("_id3"));
+        assertThat(ingestService.pipelines().get("_id3").pipeline.getProcessors().size(), equalTo(0));
+
+        ingestMetadata = new IngestMetadata(Map.of("_id1", pipeline1, "_id3", pipeline3));
+
+        ingestService.innerUpdatePipelines(ingestMetadata);
+        assertThat(ingestService.pipelines().size(), is(2));
+        assertThat(ingestService.pipelines().get("_id1").pipeline.getId(), equalTo("_id1"));
+        assertThat(ingestService.pipelines().get("_id1").pipeline.getProcessors().size(), equalTo(0));
+        assertThat(ingestService.pipelines().get("_id3").pipeline.getId(), equalTo("_id3"));
+        assertThat(ingestService.pipelines().get("_id3").pipeline.getProcessors().size(), equalTo(0));
+
+        pipeline3 = new PipelineConfiguration(
+            "_id3",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON
+        );
+        ingestMetadata = new IngestMetadata(Map.of("_id1", pipeline1, "_id3", pipeline3));
+
+        ingestService.innerUpdatePipelines(ingestMetadata);
+        assertThat(ingestService.pipelines().size(), is(2));
+        assertThat(ingestService.pipelines().get("_id1").pipeline.getId(), equalTo("_id1"));
+        assertThat(ingestService.pipelines().get("_id1").pipeline.getProcessors().size(), equalTo(0));
+        assertThat(ingestService.pipelines().get("_id3").pipeline.getId(), equalTo("_id3"));
+        assertThat(ingestService.pipelines().get("_id3").pipeline.getProcessors().size(), equalTo(1));
+        assertThat(ingestService.pipelines().get("_id3").pipeline.getProcessors().get(0).getType(), equalTo("set"));
+
+        // Perform an update with no changes:
+        Map<String, IngestService.PipelineHolder> pipelines = ingestService.pipelines();
+        ingestService.innerUpdatePipelines(ingestMetadata);
+        assertThat(ingestService.pipelines(), sameInstance(pipelines));
     }
 
     public void testDelete() {
diff --git a/server/src/test/java/org/elasticsearch/node/NodeTests.java b/server/src/test/java/org/elasticsearch/node/NodeTests.java
index 6f0419421b868..777f4eb13eeea 100644
--- a/server/src/test/java/org/elasticsearch/node/NodeTests.java
+++ b/server/src/test/java/org/elasticsearch/node/NodeTests.java
@@ -153,35 +153,87 @@ public void testCloseOnOutstandingTask() throws Exception {
         node.start();
         ThreadPool threadpool = node.injector().getInstance(ThreadPool.class);
         AtomicBoolean shouldRun = new AtomicBoolean(true);
+        final CountDownLatch threadRunning = new CountDownLatch(1);
         threadpool.executor(ThreadPool.Names.SEARCH).execute(() -> {
+            threadRunning.countDown();
             while (shouldRun.get());
         });
+        threadRunning.await();
         node.close();
         shouldRun.set(false);
         assertTrue(node.awaitClose(1, TimeUnit.DAYS));
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/42577")
+    public void testCloseRaceWithTaskExecution() throws Exception {
+        Node node = new MockNode(baseSettings().build(), basePlugins());
+        node.start();
+        ThreadPool threadpool = node.injector().getInstance(ThreadPool.class);
+        AtomicBoolean shouldRun = new AtomicBoolean(true);
+        final CountDownLatch running = new CountDownLatch(3);
+        Thread submitThread = new Thread(() -> {
+            running.countDown();
+            try {
+                running.await();
+            } catch (InterruptedException e) {
+                throw new AssertionError("interrupted while waiting", e);
+            }
+            threadpool.executor(ThreadPool.Names.SEARCH).execute(() -> {
+                while (shouldRun.get());
+            });
+        });
+        Thread closeThread = new Thread(() -> {
+            running.countDown();
+            try {
+                running.await();
+            } catch (InterruptedException e) {
+                throw new AssertionError("interrupted while waiting", e);
+            }
+            try {
+                node.close();
+            } catch (IOException e) {
+                throw new AssertionError("node close failed", e);
+            }
+        });
+        submitThread.start();
+        closeThread.start();
+        running.countDown();
+        running.await();
+
+        submitThread.join();
+        closeThread.join();
+
+        shouldRun.set(false);
+        assertTrue(node.awaitClose(1, TimeUnit.DAYS));
+    }
+
     public void testAwaitCloseTimeoutsOnNonInterruptibleTask() throws Exception {
         Node node = new MockNode(baseSettings().build(), basePlugins());
         node.start();
         ThreadPool threadpool = node.injector().getInstance(ThreadPool.class);
         AtomicBoolean shouldRun = new AtomicBoolean(true);
+        final CountDownLatch threadRunning = new CountDownLatch(1);
         threadpool.executor(ThreadPool.Names.SEARCH).execute(() -> {
+            threadRunning.countDown();
             while (shouldRun.get());
         });
+        threadRunning.await();
         node.close();
         assertFalse(node.awaitClose(0, TimeUnit.MILLISECONDS));
         shouldRun.set(false);
+        assertTrue(node.awaitClose(1, TimeUnit.DAYS));
     }
 
     public void testCloseOnInterruptibleTask() throws Exception {
         Node node = new MockNode(baseSettings().build(), basePlugins());
         node.start();
         ThreadPool threadpool = node.injector().getInstance(ThreadPool.class);
-        CountDownLatch latch = new CountDownLatch(1);
+        final CountDownLatch threadRunning = new CountDownLatch(1);
+        final CountDownLatch latch = new CountDownLatch(1);
         final CountDownLatch finishLatch = new CountDownLatch(1);
         final AtomicBoolean interrupted = new AtomicBoolean(false);
         threadpool.executor(ThreadPool.Names.SEARCH).execute(() -> {
+            threadRunning.countDown();
             try {
                 latch.await();
             } catch (InterruptedException e) {
@@ -191,6 +243,7 @@ public void testCloseOnInterruptibleTask() throws Exception {
                 finishLatch.countDown();
             }
         });
+        threadRunning.await();
         node.close();
         // close should not interrput ongoing tasks
         assertFalse(interrupted.get());
diff --git a/server/src/test/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIT.java b/server/src/test/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIT.java
index fea2e4699d5e5..83a65444a78c8 100644
--- a/server/src/test/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIT.java
+++ b/server/src/test/java/org/elasticsearch/operateAllIndices/DestructiveOperationsIT.java
@@ -69,7 +69,7 @@ public void testDeleteIndexDefaultBehaviour() throws Exception {
             assertAcked(client().admin().indices().prepareDelete("*").get());
         }
 
-        assertThat(client().admin().indices().prepareExists("_all").get().isExists(), equalTo(false));
+        assertThat(indexExists("_all"), equalTo(false));
     }
 
     public void testCloseIndexIsRejected() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java
index f29953633a1bb..1aa3e607519f1 100644
--- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -793,7 +794,7 @@ private DiscoveryNode newNode(String nodeId) {
                 nodeId,
                 buildNewFakeTransportAddress(),
                 emptyMap(),
-                Set.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA),
+                Set.of(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.DATA_ROLE),
                 Version.CURRENT);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java
index 9118fad66b0dd..ce1f6a245f6a2 100644
--- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java
+++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java
@@ -20,7 +20,6 @@
 
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.Version;
-import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diff;
@@ -58,14 +57,12 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.Set;
 
 import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_GATEWAY;
 import static org.elasticsearch.cluster.metadata.MetaData.CONTEXT_MODE_SNAPSHOT;
 import static org.elasticsearch.persistent.PersistentTasksExecutor.NO_NODE_FOUND;
-import static org.elasticsearch.test.VersionUtils.allReleasedVersions;
 import static org.elasticsearch.test.VersionUtils.compatibleFutureVersion;
 import static org.elasticsearch.test.VersionUtils.getFirstVersion;
 import static org.elasticsearch.test.VersionUtils.getPreviousVersion;
@@ -259,7 +256,7 @@ public void testBuilder() {
     public void testMinVersionSerialization() throws IOException {
         PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder();
 
-        Version minVersion = allReleasedVersions().stream().filter(Version::isRelease).findFirst().orElseThrow(NoSuchElementException::new);
+        Version minVersion = getFirstVersion();
         final Version streamVersion = randomVersionBetween(random(), minVersion, getPreviousVersion(Version.CURRENT));
         tasks.addTask("test_compatible_version", TestPersistentTasksExecutor.NAME,
             new TestParams(null, randomVersionBetween(random(), minVersion, streamVersion),
@@ -270,14 +267,10 @@ public void testMinVersionSerialization() throws IOException {
                 randomBoolean() ? Optional.empty() : Optional.of("test")),
             randomAssignment());
         final BytesStreamOutput out = new BytesStreamOutput();
+
         out.setVersion(streamVersion);
         Set<String> features = new HashSet<>();
-        final boolean transportClient = randomBoolean();
-        if (transportClient) {
-            features.add(TransportClient.TRANSPORT_CLIENT_FEATURE);
-        }
-        // if a transport client, then it must have the feature otherwise we add the feature randomly
-        if (transportClient || randomBoolean()) {
+        if (randomBoolean()) {
             features.add("test");
         }
         out.setFeatures(features);
@@ -291,30 +284,6 @@ public void testMinVersionSerialization() throws IOException {
         assertThat(read.taskMap().keySet(), equalTo(Collections.singleton("test_compatible_version")));
     }
 
-    public void testFeatureSerialization() throws IOException {
-        PersistentTasksCustomMetaData.Builder tasks = PersistentTasksCustomMetaData.builder();
-
-        Version minVersion = getFirstVersion();
-        tasks.addTask("test_compatible", TestPersistentTasksExecutor.NAME,
-            new TestParams(null, randomVersionBetween(random(), minVersion, Version.CURRENT),
-                randomBoolean() ? Optional.empty() : Optional.of("existing")),
-            randomAssignment());
-        tasks.addTask("test_incompatible", TestPersistentTasksExecutor.NAME,
-            new TestParams(null, randomVersionBetween(random(), minVersion, Version.CURRENT), Optional.of("non_existing")),
-            randomAssignment());
-        final BytesStreamOutput out = new BytesStreamOutput();
-        out.setVersion(Version.CURRENT);
-        Set<String> features = new HashSet<>();
-        features.add("existing");
-        features.add(TransportClient.TRANSPORT_CLIENT_FEATURE);
-        out.setFeatures(features);
-        tasks.build().writeTo(out);
-
-        PersistentTasksCustomMetaData read = new PersistentTasksCustomMetaData(
-            new NamedWriteableAwareStreamInput(out.bytes().streamInput(), getNamedWriteableRegistry()));
-        assertThat(read.taskMap().keySet(), equalTo(Collections.singleton("test_compatible")));
-    }
-
     public void testDisassociateDeadNodes_givenNoPersistentTasks() {
         ClusterState originalState = ClusterState.builder(new ClusterName("persistent-tasks-tests")).build();
         ClusterState returnedState = PersistentTasksCustomMetaData.disassociateDeadNodes(originalState);
diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java
index 63ff7f9f97463..d7ee34a860bd7 100644
--- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java
+++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksExecutorFullRestartIT.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.persistent.TestPersistentTasksPlugin.TestPersistentTasksExecutor;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -48,7 +47,6 @@ protected boolean ignoreExternalCluster() {
         return true;
     }
 
-    @TestLogging("org.elasticsearch.persistent:TRACE,org.elasticsearch.cluster.service:DEBUG")
     public void testFullClusterRestart() throws Exception {
         PersistentTasksService service = internalCluster().getInstance(PersistentTasksService.class);
         int numberOfTasks = randomIntBetween(1, 10);
diff --git a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java
index 0b5c432d5978c..4cea01e65181a 100644
--- a/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java
+++ b/server/src/test/java/org/elasticsearch/persistent/TestPersistentTasksPlugin.java
@@ -22,7 +22,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -395,7 +395,7 @@ protected AllocatedPersistentTask createTask(long id, String type, String action
         }
     }
 
-    public static class TestTaskAction extends Action<TestTasksResponse> {
+    public static class TestTaskAction extends ActionType<TestTasksResponse> {
 
         public static final TestTaskAction INSTANCE = new TestTaskAction();
         public static final String NAME = "cluster:admin/persistent/task_test";
@@ -404,11 +404,6 @@ private TestTaskAction() {
             super(NAME);
         }
 
-        @Override
-        public TestTasksResponse newResponse() {
-            throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-        }
-
         @Override
         public Writeable.Reader<TestTasksResponse> getResponseReader() {
             return TestTasksResponse::new;
diff --git a/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java
index 562bb3b4d9ecb..deafe203ac8dc 100644
--- a/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.plugins;
 
-import org.apache.log4j.Level;
+import org.apache.logging.log4j.Level;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.Version;
@@ -171,7 +171,15 @@ public void testDesktopServicesStoreFiles() throws IOException {
             if (Constants.WINDOWS) {
                 assertThat(e.getCause(), instanceOf(NoSuchFileException.class));
             } else {
-                assertThat(e.getCause(), hasToString(containsString("Not a directory")));
+                // force a "Not a directory" exception to be thrown so that we can extract the locale-dependent message
+                final String expected;
+                try (InputStream ignored = Files.newInputStream(desktopServicesStore.resolve("not-a-directory"))) {
+                    throw new AssertionError();
+                } catch (final FileSystemException inner) {
+                    // locale-dependent translation of "Not a directory"
+                    expected = inner.getReason();
+                }
+                assertThat(e.getCause(), hasToString(containsString(expected)));
             }
         }
     }
diff --git a/server/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java b/server/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
index 7002a77b6ba10..9e5a170b6b334 100644
--- a/server/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
+++ b/server/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
@@ -42,7 +42,6 @@
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.BackgroundIndexer;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -61,8 +60,6 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout;
 
-@TestLogging("_root:DEBUG,org.elasticsearch.index.shard:TRACE,org.elasticsearch.cluster.service:TRACE," +
-        "org.elasticsearch.index.seqno:TRACE,org.elasticsearch.indices.recovery:TRACE")
 public class RecoveryWhileUnderLoadIT extends ESIntegTestCase {
     private final Logger logger = LogManager.getLogger(RecoveryWhileUnderLoadIT.class);
 
diff --git a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java
index c2d35279bdff4..369daef08d1c3 100644
--- a/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java
+++ b/server/src/test/java/org/elasticsearch/recovery/RelocationIT.java
@@ -60,7 +60,6 @@
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.MockIndexEventListener;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.test.transport.StubbableTransport;
 import org.elasticsearch.transport.Transport;
@@ -94,7 +93,6 @@
 import static org.hamcrest.Matchers.startsWith;
 
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0)
-@TestLogging("_root:DEBUG,org.elasticsearch.indices.recovery:TRACE,org.elasticsearch.index.shard.service:TRACE")
 public class RelocationIT extends ESIntegTestCase {
     private final TimeValue ACCEPTABLE_RELOCATION_TIME = new TimeValue(5, TimeUnit.MINUTES);
 
@@ -162,7 +160,6 @@ public void testSimpleRelocationNoIndexing() {
         assertThat(client().prepareSearch("test").setSize(0).execute().actionGet().getHits().getTotalHits().value, equalTo(20L));
     }
 
-    @TestLogging("org.elasticsearch.action.bulk:TRACE,org.elasticsearch.action.search:TRACE")
     public void testRelocationWhileIndexingRandom() throws Exception {
         int numberOfRelocations = scaledRandomIntBetween(1, rarely() ? 10 : 4);
         int numberOfReplicas = randomBoolean() ? 0 : 1;
@@ -266,7 +263,6 @@ public void testRelocationWhileIndexingRandom() throws Exception {
         }
     }
 
-    @TestLogging("org.elasticsearch.action.bulk:TRACE,org.elasticsearch.action.search:TRACE")
     public void testRelocationWhileRefreshing() throws Exception {
         int numberOfRelocations = scaledRandomIntBetween(1, rarely() ? 10 : 4);
         int numberOfReplicas = randomBoolean() ? 0 : 1;
@@ -450,11 +446,6 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
         }
     }
 
-    @TestLogging(
-            "org.elasticsearch.action.bulk:TRACE,"
-                    + "org.elasticsearch.action.search:TRACE,"
-                    + "org.elasticsearch.cluster.service:TRACE,"
-                    + "org.elasticsearch.index.seqno:TRACE")
     public void testIndexAndRelocateConcurrently() throws ExecutionException, InterruptedException {
         int halfNodes = randomIntBetween(1, 3);
         Settings[] nodeSettings = Stream.concat(
diff --git a/server/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java b/server/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java
index 973c687ebe84c..3000d7262db77 100644
--- a/server/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java
+++ b/server/src/test/java/org/elasticsearch/recovery/TruncatedRecoveryIT.java
@@ -108,6 +108,7 @@ public void testCancelRecoveryAndResume() throws Exception {
         ensureGreen();
         // ensure we have flushed segments and make them a big one via optimize
         client().admin().indices().prepareFlush().setForce(true).get();
+        client().admin().indices().prepareFlush().setForce(true).get(); // double flush to create safe commit in case of async durability
         client().admin().indices().prepareForceMerge().setMaxNumSegments(1).setFlush(true).get();
 
         final CountDownLatch latch = new CountDownLatch(1);
@@ -119,7 +120,7 @@ public void testCancelRecoveryAndResume() throws Exception {
                 (connection, requestId, action, request, options) -> {
                     if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
                         RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
-                        logger.debug("file chunk [{}] lastChunk: {}", req, req.lastChunk());
+                        logger.info("file chunk [{}] lastChunk: {}", req, req.lastChunk());
                         if ((req.name().endsWith("cfs") || req.name().endsWith("fdt")) && req.lastChunk() && truncate.get()) {
                             latch.countDown();
                             throw new RuntimeException("Caused some truncated files for fun and profit");
diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java
index ae703795ec622..d79763a9f6eab 100644
--- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java
@@ -49,6 +49,7 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 import static org.mockito.Mockito.mock;
 
@@ -160,7 +161,7 @@ public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, Met
         @Override
         public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<IndexId> indices, long startTime, String failure,
                                              int totalShards, List<SnapshotShardFailure> shardFailures, long repositoryStateId,
-                                             boolean includeGlobalState) {
+                                             boolean includeGlobalState, Map<String, Object> userMetadata) {
             return null;
         }
 
diff --git a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java
index ec8a444d84fae..1dc7a6263d37b 100644
--- a/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java
+++ b/server/src/test/java/org/elasticsearch/repositories/fs/FsRepositoryTests.java
@@ -148,7 +148,7 @@ public void testSnapshotAndRestore() throws IOException, InterruptedException {
                 secondState.getIndex().fileDetails().stream().filter(f -> f.reused() == false).collect(Collectors.toList());
             Collections.sort(recoveredFiles, Comparator.comparing(RecoveryState.File::name));
             assertTrue(recoveredFiles.get(0).name(), recoveredFiles.get(0).name().endsWith(".liv"));
-            assertTrue(recoveredFiles.get(1).name(), recoveredFiles.get(1).name().endsWith("segments_2"));
+            assertTrue(recoveredFiles.get(1).name(), recoveredFiles.get(1).name().endsWith("segments_" + incIndexCommit.getGeneration()));
         } finally {
             terminate(threadPool);
         }
diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index c04dc7be026b9..0f4944cb0c6da 100644
--- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -35,6 +35,8 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
 import org.elasticsearch.http.HttpInfo;
+import org.elasticsearch.http.HttpRequest;
+import org.elasticsearch.http.HttpResponse;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.HttpStats;
 import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
@@ -58,6 +60,8 @@
 import java.util.function.UnaryOperator;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.doCallRealMethod;
@@ -470,6 +474,89 @@ public void testDispatchBadRequestUnknownCause() {
         assertThat(channel.getRestResponse().content().utf8ToString(), containsString("unknown cause"));
     }
 
+    public void testFavicon() {
+        final FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withMethod(RestRequest.Method.GET)
+            .withPath("/favicon.ico")
+            .build();
+        final AssertingChannel channel = new AssertingChannel(fakeRestRequest, false, RestStatus.OK);
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+        assertThat(channel.getRestResponse().contentType(), containsString("image/x-icon"));
+    }
+
+    public void testFaviconWithWrongHttpMethod() {
+        final FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withMethod(randomValueOtherThan(RestRequest.Method.GET, () -> randomFrom(RestRequest.Method.values())))
+            .withPath("/favicon.ico")
+            .build();
+        final AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.METHOD_NOT_ALLOWED);
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+        assertThat(channel.getRestResponse().getHeaders().containsKey("Allow"), equalTo(true));
+        assertThat(channel.getRestResponse().getHeaders().get("Allow"), hasItem(equalTo(RestRequest.Method.GET.toString())));
+    }
+
+    public void testDispatchUnsupportedHttpMethod() {
+        final boolean hasContent = randomBoolean();
+        final RestRequest request = RestRequest.request(xContentRegistry(), new HttpRequest() {
+            @Override
+            public RestRequest.Method method() {
+                throw new IllegalArgumentException("test");
+            }
+
+            @Override
+            public String uri() {
+                return "/";
+            }
+
+            @Override
+            public BytesReference content() {
+                if (hasContent) {
+                    return new BytesArray("test");
+                }
+                return BytesArray.EMPTY;
+            }
+
+            @Override
+            public Map<String, List<String>> getHeaders() {
+                Map<String, List<String>> headers = new HashMap<>();
+                if (hasContent) {
+                    headers.put("Content-Type", Collections.singletonList("text/plain"));
+                }
+                return headers;
+            }
+
+            @Override
+            public List<String> strictCookies() {
+                return null;
+            }
+
+            @Override
+            public HttpVersion protocolVersion() {
+                return randomFrom(HttpVersion.values());
+            }
+
+            @Override
+            public HttpRequest removeHeader(String header) {
+                return this;
+            }
+
+            @Override
+            public HttpResponse createResponse(RestStatus status, BytesReference content) {
+                return null;
+            }
+        }, null);
+
+        final AssertingChannel channel = new AssertingChannel(request, true, RestStatus.METHOD_NOT_ALLOWED);
+        assertFalse(channel.getSendResponseCalled());
+        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.getSendResponseCalled());
+        assertThat(channel.getRestResponse().getHeaders().containsKey("Allow"), equalTo(true));
+        assertThat(channel.getRestResponse().getHeaders().get("Allow"), hasItem(equalTo(RestRequest.Method.GET.toString())));
+    }
+
+
     private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements
         HttpServerTransport {
 
diff --git a/server/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java b/server/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java
index d44b2b70a05ef..497cde82fb87a 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/RestActionsTests.java
@@ -54,7 +54,7 @@ public class RestActionsTests extends ESTestCase {
 
     @BeforeClass
     public static void init() {
-        xContentRegistry = new NamedXContentRegistry(new SearchModule(Settings.EMPTY, false, emptyList()).getNamedXContents());
+        xContentRegistry = new NamedXContentRegistry(new SearchModule(Settings.EMPTY, emptyList()).getNamedXContents());
     }
 
     @AfterClass
diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java
index 406e9b1d36c07..70e8903b0c3f8 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java
@@ -18,20 +18,23 @@
  */
 package org.elasticsearch.rest.action.admin.indices;
 
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.analysis.NameOrDefinition;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
 
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.startsWith;
 import static org.mockito.Mockito.mock;
 
 public class RestAnalyzeActionTests extends ESTestCase {
@@ -44,15 +47,13 @@ public void testParseXContentForAnalyzeRequest() throws Exception {
                 .array("filter", "lowercase")
             .endObject())) {
 
-            AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test");
-
-            RestAnalyzeAction.buildFromContent(content, analyzeRequest);
+            AnalyzeAction.Request analyzeRequest = AnalyzeAction.Request.fromXContent(content, "for test");
 
             assertThat(analyzeRequest.text().length, equalTo(1));
             assertThat(analyzeRequest.text(), equalTo(new String[]{"THIS IS A TEST"}));
             assertThat(analyzeRequest.tokenizer().name, equalTo("keyword"));
             assertThat(analyzeRequest.tokenFilters().size(), equalTo(1));
-            for (AnalyzeRequest.NameOrDefinition filter : analyzeRequest.tokenFilters()) {
+            for (NameOrDefinition filter : analyzeRequest.tokenFilters()) {
                 assertThat(filter.name, equalTo("lowercase"));
             }
         }
@@ -79,9 +80,7 @@ public void testParseXContentForAnalyzeRequestWithCustomFilters() throws Excepti
                 .field("normalizer", "normalizer")
             .endObject())) {
 
-            AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test");
-
-            RestAnalyzeAction.buildFromContent(content, analyzeRequest);
+            AnalyzeAction.Request analyzeRequest = AnalyzeAction.Request.fromXContent(content, "for test");
 
             assertThat(analyzeRequest.text().length, equalTo(1));
             assertThat(analyzeRequest.text(), equalTo(new String[]{"THIS IS A TEST"}));
@@ -95,48 +94,45 @@ public void testParseXContentForAnalyzeRequestWithCustomFilters() throws Excepti
         }
     }
 
-    public void testParseXContentForAnalyzeRequestWithInvalidJsonThrowsException() throws Exception {
+    public void testParseXContentForAnalyzeRequestWithInvalidJsonThrowsException() {
         RestAnalyzeAction action = new RestAnalyzeAction(Settings.EMPTY, mock(RestController.class));
         RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
             .withContent(new BytesArray("{invalid_json}"), XContentType.JSON).build();
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> action.handleRequest(request, null, null));
-        assertThat(e.getMessage(), equalTo("Failed to parse request body"));
+        IOException e = expectThrows(IOException.class, () -> action.handleRequest(request, null, null));
+        assertThat(e.getMessage(), containsString("expecting double-quote"));
     }
 
     public void testParseXContentForAnalyzeRequestWithUnknownParamThrowsException() throws Exception {
-        AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test");
         try (XContentParser invalidContent = createParser(XContentFactory.jsonBuilder()
             .startObject()
                 .field("text", "THIS IS A TEST")
                 .field("unknown", "keyword")
             .endObject())) {
             IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> RestAnalyzeAction.buildFromContent(invalidContent, analyzeRequest));
-            assertThat(e.getMessage(), startsWith("Unknown parameter [unknown]"));
+                () -> AnalyzeAction.Request.fromXContent(invalidContent, "for test"));
+            assertThat(e.getMessage(), containsString("unknown field [unknown]"));
         }
     }
 
     public void testParseXContentForAnalyzeRequestWithInvalidStringExplainParamThrowsException() throws Exception {
-        AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test");
         try (XContentParser invalidExplain = createParser(XContentFactory.jsonBuilder()
             .startObject()
                 .field("explain", "fals")
             .endObject())) {
             IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> RestAnalyzeAction.buildFromContent(invalidExplain, analyzeRequest));
-            assertThat(e.getMessage(), startsWith("explain must be either 'true' or 'false'"));
+                () -> AnalyzeAction.Request.fromXContent(invalidExplain, "for test"));
+            assertThat(e.getMessage(), containsString("failed to parse field [explain]"));
         }
     }
 
     public void testParseXContentForAnalyzeRequestWithInvalidNormalizerThrowsException() throws Exception {
-        AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test");
         try (XContentParser invalidExplain = createParser(XContentFactory.jsonBuilder()
             .startObject()
             .field("normalizer", true)
             .endObject())) {
             IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> RestAnalyzeAction.buildFromContent(invalidExplain, analyzeRequest));
-            assertThat(e.getMessage(), startsWith("normalizer should be normalizer's name"));
+                () -> AnalyzeAction.Request.fromXContent(invalidExplain, "for test"));
+            assertThat(e.getMessage(), containsString("normalizer doesn't support values of type: VALUE_BOOLEAN"));
         }
     }
 
@@ -147,9 +143,9 @@ public void testDeprecatedParamIn2xException() throws Exception {
             .field("tokenizer", "keyword")
             .array("filters", "lowercase")
             .endObject())) {
-            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser,
-                new AnalyzeRequest("for test")));
-            assertThat(e.getMessage(), startsWith("Unknown parameter [filters]"));
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> AnalyzeAction.Request.fromXContent(parser,"for test"));
+            assertThat(e.getMessage(), containsString("unknown field [filters]"));
         }
 
         try (XContentParser parser = createParser(XContentFactory.jsonBuilder()
@@ -158,9 +154,9 @@ public void testDeprecatedParamIn2xException() throws Exception {
             .field("tokenizer", "keyword")
             .array("token_filters", "lowercase")
             .endObject())) {
-            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser,
-                new AnalyzeRequest("for test")));
-            assertThat(e.getMessage(), startsWith("Unknown parameter [token_filters]"));
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> AnalyzeAction.Request.fromXContent(parser, "for test"));
+            assertThat(e.getMessage(), containsString("unknown field [token_filters]"));
         }
 
         try (XContentParser parser = createParser(XContentFactory.jsonBuilder()
@@ -169,9 +165,9 @@ public void testDeprecatedParamIn2xException() throws Exception {
             .field("tokenizer", "keyword")
             .array("char_filters", "lowercase")
             .endObject())) {
-            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser,
-                new AnalyzeRequest("for test")));
-            assertThat(e.getMessage(), startsWith("Unknown parameter [char_filters]"));
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> AnalyzeAction.Request.fromXContent(parser, "for test"));
+            assertThat(e.getMessage(), containsString("unknown field [char_filters]"));
         }
 
         try (XContentParser parser = createParser(XContentFactory.jsonBuilder()
@@ -180,9 +176,9 @@ public void testDeprecatedParamIn2xException() throws Exception {
             .field("tokenizer", "keyword")
             .array("token_filter", "lowercase")
             .endObject())) {
-            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser,
-                new AnalyzeRequest("for test")));
-            assertThat(e.getMessage(), startsWith("Unknown parameter [token_filter]"));
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> AnalyzeAction.Request.fromXContent(parser, "for test"));
+            assertThat(e.getMessage(), containsString("unknown field [token_filter]"));
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java
index 0d2e4723a7352..0e5dada0f7c64 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryActionTests.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.rest.action.admin.indices;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction;
@@ -79,7 +79,7 @@ protected void doExecute(Task task, ActionRequest request, ActionListener listen
             }
         };
 
-        final Map<Action, TransportAction> actions = new HashMap<>();
+        final Map<ActionType, TransportAction> actions = new HashMap<>();
         actions.put(ValidateQueryAction.INSTANCE, transportAction);
 
         client.initialize(actions, () -> "local", null);
diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
index ae387806796fc..37dc84c126e2e 100644
--- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
+++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
@@ -20,182 +20,154 @@
 package org.elasticsearch.rest.action.cat;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
-import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
-import org.elasticsearch.action.admin.indices.stats.IndicesStatsTests;
-import org.elasticsearch.action.admin.indices.stats.ShardStats;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.health.ClusterIndexHealth;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.routing.RecoverySource;
-import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
-import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.cache.query.QueryCacheStats;
-import org.elasticsearch.index.cache.request.RequestCacheStats;
-import org.elasticsearch.index.engine.SegmentsStats;
-import org.elasticsearch.index.fielddata.FieldDataStats;
-import org.elasticsearch.index.flush.FlushStats;
-import org.elasticsearch.index.get.GetStats;
-import org.elasticsearch.index.merge.MergeStats;
-import org.elasticsearch.index.refresh.RefreshStats;
-import org.elasticsearch.index.search.stats.SearchStats;
-import org.elasticsearch.index.shard.DocsStats;
-import org.elasticsearch.index.shard.IndexingStats;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.shard.ShardPath;
-import org.elasticsearch.index.store.StoreStats;
-import org.elasticsearch.index.warmer.WarmerStats;
 import org.elasticsearch.rest.RestController;
-import org.elasticsearch.search.suggest.completion.CompletionStats;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
 import org.elasticsearch.usage.UsageService;
 
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.IntStream;
 
-import static java.util.Collections.emptyList;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
-/**
- * Tests for {@link RestIndicesAction}
- */
 public class RestIndicesActionTests extends ESTestCase {
 
-    private IndexMetaData[] buildRandomIndicesMetaData(int numIndices) {
-        // build a (semi-)random table
-        final IndexMetaData[] indicesMetaData = new IndexMetaData[numIndices];
+    public void testBuildTable() {
+        final int numIndices = randomIntBetween(3, 20);
+        final Map<String, Settings> indicesSettings = new LinkedHashMap<>();
+        final Map<String, IndexMetaData> indicesMetaDatas = new LinkedHashMap<>();
+        final Map<String, ClusterIndexHealth> indicesHealths = new LinkedHashMap<>();
+        final Map<String, IndexStats> indicesStats = new LinkedHashMap<>();
+
         for (int i = 0; i < numIndices; i++) {
-            indicesMetaData[i] = IndexMetaData.builder(randomAlphaOfLength(5) + i)
-                                    .settings(Settings.builder()
-                                        .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                                        .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()))
-                                    .creationDate(System.currentTimeMillis())
-                                    .numberOfShards(1)
-                                    .numberOfReplicas(1)
-                                    .state(IndexMetaData.State.OPEN)
-                                    .build();
-        }
-        return indicesMetaData;
-    }
+            String indexName = "index-" + i;
 
-    private ClusterState buildClusterState(IndexMetaData[] indicesMetaData) {
-        final MetaData.Builder metaDataBuilder = MetaData.builder();
-        for (IndexMetaData indexMetaData : indicesMetaData) {
-            metaDataBuilder.put(indexMetaData, false);
-        }
-        final MetaData metaData = metaDataBuilder.build();
-        final ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
-                                              .metaData(metaData)
-                                              .build();
-        return clusterState;
-    }
+            Settings indexSettings = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+                .put(IndexSettings.INDEX_SEARCH_THROTTLED.getKey(), randomBoolean())
+                .build();
+            indicesSettings.put(indexName, indexSettings);
 
-    private ClusterHealthResponse buildClusterHealthResponse(ClusterState clusterState, IndexMetaData[] indicesMetaData) {
-        final String[] indicesStr = new String[indicesMetaData.length];
-        for (int i = 0; i < indicesMetaData.length; i++) {
-            indicesStr[i] = indicesMetaData[i].getIndex().getName();
-        }
-        final ClusterHealthResponse clusterHealthResponse = new ClusterHealthResponse(
-            clusterState.getClusterName().value(), indicesStr, clusterState, 0, 0, 0, TimeValue.timeValueMillis(1000L)
-        );
-        return clusterHealthResponse;
-    }
+            IndexMetaData.State indexState = randomBoolean() ? IndexMetaData.State.OPEN : IndexMetaData.State.CLOSE;
+            if (frequently()) {
+                ClusterHealthStatus healthStatus = randomFrom(ClusterHealthStatus.values());
+                int numberOfShards = randomIntBetween(1, 3);
+                int numberOfReplicas = healthStatus == ClusterHealthStatus.YELLOW ? 1 : randomInt(1);
+                IndexMetaData indexMetaData = IndexMetaData.builder(indexName)
+                    .settings(indexSettings)
+                    .creationDate(System.currentTimeMillis())
+                    .numberOfShards(numberOfShards)
+                    .numberOfReplicas(numberOfReplicas)
+                    .state(indexState)
+                    .build();
+                indicesMetaDatas.put(indexName, indexMetaData);
 
-    public void testBuildTable() {
-        final Settings settings = Settings.EMPTY;
-        UsageService usageService = new UsageService();
-        final RestController restController = new RestController(Collections.emptySet(), null, null, null, usageService);
-        final RestIndicesAction action = new RestIndicesAction(settings, restController, new IndexNameExpressionResolver());
+                if (frequently()) {
+                    Index index = indexMetaData.getIndex();
+                    IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index);
+                    switch (randomFrom(ClusterHealthStatus.values())) {
+                        case GREEN:
+                            IntStream.range(0, numberOfShards)
+                                .mapToObj(n -> new ShardId(index, n))
+                                .map(shardId -> TestShardRouting.newShardRouting(shardId, "nodeA", true, ShardRoutingState.STARTED))
+                                .forEach(indexRoutingTable::addShard);
+                            if (numberOfReplicas > 0) {
+                                IntStream.range(0, numberOfShards)
+                                    .mapToObj(n -> new ShardId(index, n))
+                                    .map(shardId -> TestShardRouting.newShardRouting(shardId, "nodeB", false, ShardRoutingState.STARTED))
+                                    .forEach(indexRoutingTable::addShard);
+                            }
+                            break;
+                        case YELLOW:
+                            IntStream.range(0, numberOfShards)
+                                .mapToObj(n -> new ShardId(index, n))
+                                .map(shardId -> TestShardRouting.newShardRouting(shardId, "nodeA", true, ShardRoutingState.STARTED))
+                                .forEach(indexRoutingTable::addShard);
+                            if (numberOfReplicas > 0) {
+                                IntStream.range(0, numberOfShards)
+                                    .mapToObj(n -> new ShardId(index, n))
+                                    .map(shardId -> TestShardRouting.newShardRouting(shardId, null, false, ShardRoutingState.UNASSIGNED))
+                                    .forEach(indexRoutingTable::addShard);
+                            }
+                            break;
+                        case RED:
+                            break;
+                    }
+                    indicesHealths.put(indexName, new ClusterIndexHealth(indexMetaData, indexRoutingTable.build()));
 
-        final IndexMetaData[] generatedIndicesMetaData = buildRandomIndicesMetaData(randomIntBetween(1, 5));
-        final ClusterState clusterState = buildClusterState(generatedIndicesMetaData);
-        final ClusterHealthResponse clusterHealthResponse = buildClusterHealthResponse(clusterState, generatedIndicesMetaData);
+                    if (frequently()) {
+                        IndexStats indexStats = mock(IndexStats.class);
+                        when(indexStats.getPrimaries()).thenReturn(new CommonStats());
+                        when(indexStats.getTotal()).thenReturn(new CommonStats());
+                        indicesStats.put(indexName, indexStats);
+                    }
+                }
+            }
+        }
 
-        final IndexMetaData[] sortedIndicesMetaData = action.getOrderedIndexMetaData(new String[0], clusterState,
-            IndicesOptions.strictExpand());
-        final IndexMetaData[] smallerSortedIndicesMetaData = removeRandomElement(sortedIndicesMetaData);
-        final Table table = action.buildTable(new FakeRestRequest(), sortedIndicesMetaData, clusterHealthResponse,
-                randomIndicesStatsResponse(smallerSortedIndicesMetaData));
+        final RestController restController = new RestController(Collections.emptySet(), null, null, null, new UsageService());
+        final RestIndicesAction action = new RestIndicesAction(Settings.EMPTY, restController);
+        final Table table = action.buildTable(new FakeRestRequest(), indicesSettings, indicesHealths, indicesStats, indicesMetaDatas);
 
         // now, verify the table is correct
-        int count = 0;
         List<Table.Cell> headers = table.getHeaders();
-        assertThat(headers.get(count++).value, equalTo("health"));
-        assertThat(headers.get(count++).value, equalTo("status"));
-        assertThat(headers.get(count++).value, equalTo("index"));
-        assertThat(headers.get(count++).value, equalTo("uuid"));
-
-        List<List<Table.Cell>> rows = table.getRows();
-        assertThat(rows.size(), equalTo(smallerSortedIndicesMetaData.length));
-        // TODO: more to verify (e.g. randomize cluster health, num primaries, num replicas, etc)
-        for (int i = 0; i < rows.size(); i++) {
-            count = 0;
-            final List<Table.Cell> row = rows.get(i);
-            assertThat(row.get(count++).value, equalTo("red*")); // all are red because cluster state doesn't have routing entries
-            assertThat(row.get(count++).value, equalTo("open")); // all are OPEN for now
-            assertThat(row.get(count++).value, equalTo(smallerSortedIndicesMetaData[i].getIndex().getName()));
-            assertThat(row.get(count++).value, equalTo(smallerSortedIndicesMetaData[i].getIndexUUID()));
-        }
-    }
+        assertThat(headers.get(0).value, equalTo("health"));
+        assertThat(headers.get(1).value, equalTo("status"));
+        assertThat(headers.get(2).value, equalTo("index"));
+        assertThat(headers.get(3).value, equalTo("uuid"));
+        assertThat(headers.get(4).value, equalTo("pri"));
+        assertThat(headers.get(5).value, equalTo("rep"));
+
+        final List<List<Table.Cell>> rows = table.getRows();
+        assertThat(rows.size(), equalTo(indicesMetaDatas.size()));
+
+        for (final List<Table.Cell> row : rows) {
+            final String indexName = (String) row.get(2).value;
 
-    private IndicesStatsResponse randomIndicesStatsResponse(final IndexMetaData[] indices) {
-        List<ShardStats> shardStats = new ArrayList<>();
-        for (final IndexMetaData index : indices) {
-            int numShards = randomIntBetween(1, 3);
-            int primaryIdx = randomIntBetween(-1, numShards - 1); // -1 means there is no primary shard.
-            for (int i = 0; i < numShards; i++) {
-                ShardId shardId = new ShardId(index.getIndex(), i);
-                boolean primary = (i == primaryIdx);
-                Path path = createTempDir().resolve("indices").resolve(index.getIndexUUID()).resolve(String.valueOf(i));
-                ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, primary,
-                    primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : PeerRecoverySource.INSTANCE,
-                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)
-                    );
-                shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
-                shardRouting = shardRouting.moveToStarted();
-                CommonStats stats = new CommonStats();
-                stats.fieldData = new FieldDataStats();
-                stats.queryCache = new QueryCacheStats();
-                stats.docs = new DocsStats();
-                stats.store = new StoreStats();
-                stats.indexing = new IndexingStats();
-                stats.search = new SearchStats();
-                stats.segments = new SegmentsStats();
-                stats.merge = new MergeStats();
-                stats.refresh = new RefreshStats();
-                stats.completion = new CompletionStats();
-                stats.requestCache = new RequestCacheStats();
-                stats.get = new GetStats();
-                stats.flush = new FlushStats();
-                stats.warmer = new WarmerStats();
-                shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null));
+            ClusterIndexHealth indexHealth = indicesHealths.get(indexName);
+            IndexStats indexStats = indicesStats.get(indexName);
+            IndexMetaData indexMetaData = indicesMetaDatas.get(indexName);
+
+            if (indexHealth != null) {
+                assertThat(row.get(0).value, equalTo(indexHealth.getStatus().toString().toLowerCase(Locale.ROOT)));
+            } else if (indexStats != null) {
+                assertThat(row.get(0).value, equalTo("red*"));
+            } else {
+                assertThat(row.get(0).value, equalTo(""));
             }
-        }
-        return IndicesStatsTests.newIndicesStatsResponse(
-            shardStats.toArray(new ShardStats[shardStats.size()]), shardStats.size(), shardStats.size(), 0, emptyList()
-        );
-    }
 
-    private IndexMetaData[] removeRandomElement(IndexMetaData[] array) {
-        assert array != null;
-        assert array.length > 0;
-        final List<IndexMetaData> collectionLessAnItem = new ArrayList<>();
-        collectionLessAnItem.addAll(Arrays.asList(array));
-        final int toRemoveIndex = randomIntBetween(0, array.length - 1);
-        collectionLessAnItem.remove(toRemoveIndex);
-        return collectionLessAnItem.toArray(new IndexMetaData[0]);
+            assertThat(row.get(1).value, equalTo(indexMetaData.getState().toString().toLowerCase(Locale.ROOT)));
+            assertThat(row.get(2).value, equalTo(indexName));
+            assertThat(row.get(3).value, equalTo(indexMetaData.getIndexUUID()));
+            if (indexHealth != null) {
+                assertThat(row.get(4).value, equalTo(indexMetaData.getNumberOfShards()));
+                assertThat(row.get(5).value, equalTo(indexMetaData.getNumberOfReplicas()));
+            } else {
+                assertThat(row.get(4).value, nullValue());
+                assertThat(row.get(5).value, nullValue());
+            }
+        }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/rest/action/search/RestCountActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/search/RestCountActionTests.java
deleted file mode 100644
index c6af3d12e2913..0000000000000
--- a/server/src/test/java/org/elasticsearch/rest/action/search/RestCountActionTests.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.rest.action.search;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestRequest.Method;
-import org.elasticsearch.test.rest.RestActionTestCase;
-import org.elasticsearch.test.rest.FakeRestRequest;
-import org.junit.Before;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class RestCountActionTests extends RestActionTestCase {
-
-    @Before
-    public void setUpAction() {
-        new RestCountAction(Settings.EMPTY, controller());
-    }
-
-    public void testTypeInPath() {
-        RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
-            .withMethod(Method.POST)
-            .withPath("/some_index/some_type/_count")
-            .build();
-
-        dispatchRequest(request);
-        assertWarnings(RestCountAction.TYPES_DEPRECATION_MESSAGE);
-    }
-
-    public void testTypeParameter() {
-        Map<String, String> params = new HashMap<>();
-        params.put("type", "some_type");
-
-        RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
-            .withMethod(Method.GET)
-            .withPath("/some_index/_count")
-            .withParams(params)
-            .build();
-
-        dispatchRequest(request);
-        assertWarnings(RestCountAction.TYPES_DEPRECATION_MESSAGE);
-    }
-}
diff --git a/server/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java b/server/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java
index b383a80296247..38ed0b819d4ab 100644
--- a/server/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java
+++ b/server/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java
@@ -584,17 +584,17 @@ public void testRequiredRoutingMappingVariousAPIs() throws Exception {
         }
 
         logger.info("--> verifying term vector with id [1], with routing [0], should succeed");
-        TermVectorsResponse termVectorsResponse = client().prepareTermVectors(indexOrAlias(), "type1", "1")
+        TermVectorsResponse termVectorsResponse = client().prepareTermVectors(indexOrAlias(), "1")
                                                           .setRouting(routingValue)
                                                           .get();
         assertThat(termVectorsResponse.isExists(), equalTo(true));
         assertThat(termVectorsResponse.getId(), equalTo("1"));
 
         try {
-            client().prepareTermVectors(indexOrAlias(), "type1", "1").get();
+            client().prepareTermVectors(indexOrAlias(), "1").get();
             fail();
         } catch (RoutingMissingException e) {
-            assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]"));
+            assertThat(e.getMessage(), equalTo("routing is required for [test]/[_doc]/[1]"));
         }
 
         UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting(routingValue)
@@ -632,9 +632,9 @@ public void testRequiredRoutingMappingVariousAPIs() throws Exception {
         assertThat(multiGetResponse.getResponses()[1].getFailure().getMessage(), equalTo("routing is required for [test]/[type1]/[2]"));
 
         MultiTermVectorsResponse multiTermVectorsResponse = client().prepareMultiTermVectors()
-                                                                    .add(new TermVectorsRequest(indexOrAlias(), "type1", "1")
+                .add(new TermVectorsRequest(indexOrAlias(), "1")
                                                                         .routing(routingValue))
-                                                                    .add(new TermVectorsRequest(indexOrAlias(), "type1", "2")
+                .add(new TermVectorsRequest(indexOrAlias(), "2")
                                                                         .routing(routingValue))
                                                                     .get();
         assertThat(multiTermVectorsResponse.getResponses().length, equalTo(2));
@@ -648,21 +648,20 @@ public void testRequiredRoutingMappingVariousAPIs() throws Exception {
         assertThat(multiTermVectorsResponse.getResponses()[1].getResponse().isExists(), equalTo(true));
 
         multiTermVectorsResponse = client().prepareMultiTermVectors()
-                                           .add(new TermVectorsRequest(indexOrAlias(), "type1", "1"))
-                                           .add(new TermVectorsRequest(indexOrAlias(), "type1", "2")).get();
+                .add(new TermVectorsRequest(indexOrAlias(), "1")).add(new TermVectorsRequest(indexOrAlias(), "2")).get();
         assertThat(multiTermVectorsResponse.getResponses().length, equalTo(2));
         assertThat(multiTermVectorsResponse.getResponses()[0].getId(), equalTo("1"));
         assertThat(multiTermVectorsResponse.getResponses()[0].isFailed(), equalTo(true));
         assertThat(multiTermVectorsResponse.getResponses()[0].getFailure()
                                                              .getCause()
-                                                             .getMessage(), equalTo("routing is required for [test]/[type1]/[1]"));
+                                                             .getMessage(), equalTo("routing is required for [test]/[_doc]/[1]"));
         assertThat(multiTermVectorsResponse.getResponses()[0].getResponse(), nullValue());
         assertThat(multiTermVectorsResponse.getResponses()[1].getId(), equalTo("2"));
         assertThat(multiTermVectorsResponse.getResponses()[1].isFailed(), equalTo(true));
         assertThat(multiTermVectorsResponse.getResponses()[1].getResponse(), nullValue());
         assertThat(multiTermVectorsResponse.getResponses()[1].getFailure()
                                                              .getCause()
-                                                             .getMessage(), equalTo("routing is required for [test]/[type1]/[2]"));
+                                                             .getMessage(), equalTo("routing is required for [test]/[_doc]/[2]"));
     }
 
     private static String indexOrAlias() {
diff --git a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java
index 2144ce057a53a..a2400bf3d3884 100644
--- a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java
@@ -61,7 +61,7 @@ public void setUp() throws Exception {
         super.setUp();
         IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
         searchExtPlugin = new TestSearchExtPlugin();
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.singletonList(searchExtPlugin));
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.singletonList(searchExtPlugin));
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
         entries.addAll(indicesModule.getNamedWriteables());
         entries.addAll(searchModule.getNamedWriteables());
diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
index 189929171a5d1..eda60f5cb2ba8 100644
--- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
@@ -72,7 +72,6 @@ public void testPreProcess() throws Exception {
         when(shardSearchRequest.searchType()).thenReturn(SearchType.DEFAULT);
         ShardId shardId = new ShardId("index", UUID.randomUUID().toString(), 1);
         when(shardSearchRequest.shardId()).thenReturn(shardId);
-        when(shardSearchRequest.types()).thenReturn(new String[]{});
 
         IndexShard indexShard = mock(IndexShard.class);
         QueryCachingPolicy queryCachingPolicy = mock(QueryCachingPolicy.class);
diff --git a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java
index b7755dd321416..2d616dacaad3c 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java
@@ -64,9 +64,12 @@
 import org.elasticsearch.search.rescore.RescoreContext;
 import org.elasticsearch.search.rescore.RescorerBuilder;
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
+import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry;
+import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
 import org.elasticsearch.search.suggest.Suggester;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
+import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.search.suggest.term.TermSuggestion;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 import org.elasticsearch.test.ESTestCase;
@@ -163,6 +166,7 @@ public List<PipelineAggregationSpec> getPipelineAggregations() {
         expectThrows(IllegalArgumentException.class, registryForPlugin(registersDupePipelineAggregation));
 
         SearchPlugin registersDupeRescorer = new SearchPlugin() {
+            @Override
             public List<RescorerSpec<?>> getRescorers() {
                 return singletonList(
                         new RescorerSpec<>(QueryRescorerBuilder.NAME, QueryRescorerBuilder::new, QueryRescorerBuilder::fromXContent));
@@ -172,11 +176,11 @@ public List<RescorerSpec<?>> getRescorers() {
     }
 
     private ThrowingRunnable registryForPlugin(SearchPlugin plugin) {
-        return () -> new NamedXContentRegistry(new SearchModule(Settings.EMPTY, false, singletonList(plugin)).getNamedXContents());
+        return () -> new NamedXContentRegistry(new SearchModule(Settings.EMPTY, singletonList(plugin)).getNamedXContents());
     }
 
     public void testRegisterSuggester() {
-        SearchModule module = new SearchModule(Settings.EMPTY, false, singletonList(new SearchPlugin() {
+        SearchModule module = new SearchModule(Settings.EMPTY, singletonList(new SearchPlugin() {
             @Override
             public List<SuggesterSpec<?>> getSuggesters() {
                 return singletonList(
@@ -222,7 +226,7 @@ public List<SuggesterSpec<?>> getSuggesters() {
 
     public void testRegisterHighlighter() {
         CustomHighlighter customHighlighter = new CustomHighlighter();
-        SearchModule module = new SearchModule(Settings.EMPTY, false, singletonList(new SearchPlugin() {
+        SearchModule module = new SearchModule(Settings.EMPTY, singletonList(new SearchPlugin() {
             @Override
             public Map<String, Highlighter> getHighlighters() {
                 return singletonMap("custom", customHighlighter);
@@ -236,14 +240,15 @@ public Map<String, Highlighter> getHighlighters() {
         assertSame(highlighters.get("custom"), customHighlighter);
     }
 
-    public void testRegisteredQueries() throws IOException {
+    public void testRegisteredQueries() {
         List<String> allSupportedQueries = new ArrayList<>();
         Collections.addAll(allSupportedQueries, NON_DEPRECATED_QUERIES);
         Collections.addAll(allSupportedQueries, DEPRECATED_QUERIES);
-        SearchModule module = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule module = new SearchModule(Settings.EMPTY, emptyList());
 
         Set<String> registeredNonDeprecated = module.getNamedXContents().stream()
                 .filter(e -> e.categoryClass.equals(QueryBuilder.class))
+                .filter(e -> e.name.getDeprecatedNames().length == 0)
                 .map(e -> e.name.getPreferredName())
                 .collect(toSet());
         Set<String> registeredAll = module.getNamedXContents().stream()
@@ -256,7 +261,7 @@ public void testRegisteredQueries() throws IOException {
     }
 
     public void testRegisterAggregation() {
-        SearchModule module = new SearchModule(Settings.EMPTY, false, singletonList(new SearchPlugin() {
+        SearchModule module = new SearchModule(Settings.EMPTY, singletonList(new SearchPlugin() {
             @Override
             public List<AggregationSpec> getAggregations() {
                 return singletonList(new AggregationSpec("test", TestAggregationBuilder::new, TestAggregationBuilder::fromXContent));
@@ -272,7 +277,7 @@ public List<AggregationSpec> getAggregations() {
     }
 
     public void testRegisterPipelineAggregation() {
-        SearchModule module = new SearchModule(Settings.EMPTY, false, singletonList(new SearchPlugin() {
+        SearchModule module = new SearchModule(Settings.EMPTY, singletonList(new SearchPlugin() {
             @Override
             public List<PipelineAggregationSpec> getPipelineAggregations() {
                 return singletonList(new PipelineAggregationSpec("test",
@@ -289,7 +294,7 @@ public List<PipelineAggregationSpec> getPipelineAggregations() {
     }
 
     public void testRegisterRescorer() {
-        SearchModule module = new SearchModule(Settings.EMPTY, false, singletonList(new SearchPlugin() {
+        SearchModule module = new SearchModule(Settings.EMPTY, singletonList(new SearchPlugin() {
             @Override
             public List<RescorerSpec<?>> getRescorers() {
                 return singletonList(new RescorerSpec<>("test", TestRescorerBuilder::new, TestRescorerBuilder::fromXContent));
@@ -306,7 +311,6 @@ public List<RescorerSpec<?>> getRescorers() {
     private static final String[] NON_DEPRECATED_QUERIES = new String[] {
             "bool",
             "boosting",
-            "common",
             "constant_score",
             "dis_max",
             "exists",
@@ -386,8 +390,8 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
         }
 
         @Override
-        protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(SearchContext context,
-                ValuesSourceConfig<ValuesSource> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+        protected ValuesSourceAggregatorFactory<ValuesSource> innerBuild(SearchContext context,
+                ValuesSourceConfig<ValuesSource> config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
             return null;
         }
 
@@ -396,16 +400,6 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
             return null;
         }
 
-        @Override
-        protected int innerHashCode() {
-            return 0;
-        }
-
-        @Override
-        protected boolean innerEquals(Object obj) {
-            return false;
-        }
-
         private static TestAggregationBuilder fromXContent(String name, XContentParser p) {
             return null;
         }
@@ -441,16 +435,6 @@ protected XContentBuilder internalXContent(XContentBuilder builder, Params param
             return null;
         }
 
-        @Override
-        protected int doHashCode() {
-            return 0;
-        }
-
-        @Override
-        protected boolean doEquals(Object obj) {
-            return false;
-        }
-
         private static TestPipelineAggregationBuilder fromXContent(String name, XContentParser p) {
             return null;
         }
@@ -515,6 +499,7 @@ public RescoreContext innerBuildContext(int windowSize, QueryShardContext contex
     }
 
     private static class TestSuggester extends Suggester<SuggestionSearchContext.SuggestionContext> {
+
         @Override
         protected Suggestion<? extends Suggestion.Entry<? extends Suggestion.Entry.Option>> innerExecute(
                 String name,
@@ -523,6 +508,12 @@ protected Suggestion<? extends Suggestion.Entry<? extends Suggestion.Entry.Optio
                 CharsRefBuilder spare) throws IOException {
             return null;
         }
+
+        @Override
+        protected Suggestion<? extends Entry<? extends Option>> emptySuggestion(String name, SuggestionContext suggestion,
+                CharsRefBuilder spare) throws IOException {
+            return null;
+        }
     }
 
     private static class TestSuggestionBuilder extends SuggestionBuilder<TestSuggestionBuilder> {
diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 90957c2779e8f..f8ef11abe9bf4 100644
--- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.search;
 
 import com.carrotsearch.hppc.IntArrayList;
-
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.ElasticsearchException;
@@ -51,6 +50,7 @@
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.search.stats.SearchStats;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.SearchOperationListener;
 import org.elasticsearch.index.shard.ShardId;
@@ -226,6 +226,7 @@ public void testSearchWhileIndexDeleted() throws InterruptedException {
         AtomicBoolean running = new AtomicBoolean(true);
         CountDownLatch startGun = new CountDownLatch(1);
         Semaphore semaphore = new Semaphore(Integer.MAX_VALUE);
+
         final Thread thread = new Thread() {
             @Override
             public void run() {
@@ -261,12 +262,16 @@ public void onFailure(Exception e) {
         try {
             final int rounds = scaledRandomIntBetween(100, 10000);
             SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true);
+            SearchRequest scrollSearchRequest = new SearchRequest().allowPartialSearchResults(true)
+                .scroll(new Scroll(TimeValue.timeValueMinutes(1)));
             for (int i = 0; i < rounds; i++) {
                 try {
                     try {
                         PlainActionFuture<SearchPhaseResult> result = new PlainActionFuture<>();
+                        final boolean useScroll = randomBoolean();
                         service.executeQueryPhase(
-                            new ShardSearchLocalRequest(searchRequest, indexShard.shardId(), 1,
+                            new ShardSearchLocalRequest(useScroll ? scrollSearchRequest : searchRequest,
+                                indexShard.shardId(), 1,
                                 new AliasFilter(null, Strings.EMPTY_ARRAY), 1.0f, -1, null, null),
                             new SearchTask(123L, "", "", "", null, Collections.emptyMap()), result);
                         SearchPhaseResult searchPhaseResult = result.get();
@@ -276,6 +281,9 @@ public void onFailure(Exception e) {
                         PlainActionFuture<FetchSearchResult> listener = new PlainActionFuture<>();
                         service.executeFetchPhase(req, new SearchTask(123L, "", "", "", null, Collections.emptyMap()), listener);
                         listener.get();
+                        if (useScroll) {
+                            service.freeContext(searchPhaseResult.getRequestId());
+                        }
                     } catch (ExecutionException ex) {
                         assertThat(ex.getCause(), instanceOf(RuntimeException.class));
                         throw ((RuntimeException)ex.getCause());
@@ -293,6 +301,13 @@ public void onFailure(Exception e) {
             thread.join();
             semaphore.acquire(Integer.MAX_VALUE);
         }
+
+        assertEquals(0, service.getActiveContexts());
+
+        SearchStats.Stats totalStats = indexShard.searchStats().getTotal();
+        assertEquals(0, totalStats.getQueryCurrent());
+        assertEquals(0, totalStats.getScrollCurrent());
+        assertEquals(0, totalStats.getFetchCurrent());
     }
 
     public void testTimeout() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesBuilderTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesBuilderTests.java
index 4cbc78acad1c2..9e3eb62c5e06d 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesBuilderTests.java
@@ -49,7 +49,7 @@ public void setUp() throws Exception {
         super.setUp();
 
         // register aggregations as NamedWriteable
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
         namedXContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
index 1fd8580e29027..e5b45d094e620 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
@@ -67,7 +67,7 @@ public void setUp() throws Exception {
             String type = randomAlphaOfLengthBetween(1, 10);
             currentTypes[i] = type;
         }
-        xContentRegistry = new NamedXContentRegistry(new SearchModule(settings, false, emptyList()).getNamedXContents());
+        xContentRegistry = new NamedXContentRegistry(new SearchModule(settings, emptyList()).getNamedXContents());
     }
 
     public void testGetAggregatorFactories_returnsUnmodifiableList() {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index ea25e86b3a284..39c5ae9c01580 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -77,7 +77,7 @@ public void setUp() throws Exception {
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
             .build();
         IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
-        SearchModule searchModule = new SearchModule(settings, false, emptyList());
+        SearchModule searchModule = new SearchModule(settings, emptyList());
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
         entries.addAll(indicesModule.getNamedWriteables());
         entries.addAll(searchModule.getNamedWriteables());
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationsTests.java
index e209a65cf3629..cf6a9f6d838ee 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationsTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationsTests.java
@@ -45,7 +45,7 @@
 public class InternalAggregationsTests extends ESTestCase {
 
     private final NamedWriteableRegistry registry = new NamedWriteableRegistry(
-        new SearchModule(Settings.EMPTY, false, Collections.emptyList()).getNamedWriteables());
+        new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedWriteables());
 
     public void testReduceEmptyAggs() {
         List<InternalAggregations> aggs = Collections.emptyList();
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/AutoDateHistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/AutoDateHistogramTests.java
index a54f30ffac0d1..eb16fc62fd491 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/AutoDateHistogramTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/AutoDateHistogramTests.java
@@ -29,6 +29,7 @@ protected AutoDateHistogramAggregationBuilder createTestAggregatorBuilder() {
         AutoDateHistogramAggregationBuilder builder = new AutoDateHistogramAggregationBuilder(randomAlphaOfLengthBetween(1, 10));
         builder.field(INT_FIELD_NAME);
         builder.setNumBuckets(randomIntBetween(1, 100000));
+        //TODO[PCS]: add builder pattern here
         if (randomBoolean()) {
             builder.format("###.##");
         }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java
index 8cbdcc41a9840..7272c55194d79 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java
@@ -83,7 +83,7 @@ public void setupSuiteScopeCluster() throws Exception {
     }
 
     public void testSingleValueField() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(terms("terms")
                         .field(SINGLE_VALUED_FIELD_NAME)
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
@@ -117,7 +117,7 @@ public void testSingleValueField() throws Exception {
     }
 
     public void testMultiValueField() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(terms("terms")
                         .field(MULTI_VALUED_FIELD_NAME)
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
@@ -151,7 +151,7 @@ public void testMultiValueField() throws Exception {
     }
 
     public void testUnmapped() throws Exception {
-        SearchResponse response = client().prepareSearch("idx_unmapped").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx_unmapped")
                 .addAggregation(terms("terms")
                         .field(SINGLE_VALUED_FIELD_NAME)
                         .size(between(1, 5))
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
index ad2939347edb1..fcf8d9a763b49 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
@@ -1549,7 +1549,6 @@ private void assertMultiSortResponse(int[] expectedDays, BucketOrder... order) {
         ZonedDateTime[] expectedKeys = Arrays.stream(expectedDays).mapToObj(d -> date(1, d)).toArray(ZonedDateTime[]::new);
         SearchResponse response = client()
             .prepareSearch("sort_idx")
-            .setTypes("type")
             .addAggregation(
                 dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY).order(BucketOrder.compound(order))
                     .subAggregation(avg("avg_l").field("l")).subAggregation(sum("sum_d").field("d"))).get();
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java
index 4ae523a168219..c65a1d327a93e 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.sampler.Sampler;
 import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregator;
@@ -30,7 +31,6 @@
 import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.Max;
-import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Collection;
@@ -104,7 +104,7 @@ public void testIssue10719() throws Exception {
         // Tests that we can refer to nested elements under a sample in a path
         // statement
         boolean asc = randomBoolean();
-        SearchResponse response = client().prepareSearch("test").setTypes("book").setSearchType(SearchType.QUERY_THEN_FETCH)
+        SearchResponse response = client().prepareSearch("test").setSearchType(SearchType.QUERY_THEN_FETCH)
                 .addAggregation(terms("genres")
                         .field("genre")
                         .order(BucketOrder.aggregation("sample>max_price.value", asc))
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java
index 23842a3f9df55..ad55c2c6866e4 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java
@@ -35,9 +35,9 @@
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
 import org.elasticsearch.search.aggregations.metrics.Avg;
+import org.elasticsearch.search.aggregations.metrics.ExtendedStats;
 import org.elasticsearch.search.aggregations.metrics.Max;
 import org.elasticsearch.search.aggregations.metrics.Stats;
-import org.elasticsearch.search.aggregations.metrics.ExtendedStats;
 import org.elasticsearch.search.aggregations.metrics.Sum;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -885,7 +885,6 @@ public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAsCompound
     private void assertMultiSortResponse(double[] expectedKeys, BucketOrder... order) {
         SearchResponse response = client()
                 .prepareSearch("sort_idx")
-                .setTypes("multi_sort_type")
                 .addAggregation(
                         terms("terms").field(SINGLE_VALUED_FIELD_NAME).collectMode(randomFrom(SubAggCollectionMode.values()))
                                 .order(BucketOrder.compound(order)).subAggregation(avg("avg_l").field("l"))
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java
index bee32d571b69f..94a89067285ba 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.search.aggregations.bucket;
 
 import com.carrotsearch.hppc.LongHashSet;
+
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
@@ -30,6 +31,7 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
+import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -38,7 +40,6 @@
 import org.elasticsearch.search.aggregations.metrics.Max;
 import org.elasticsearch.search.aggregations.metrics.Stats;
 import org.elasticsearch.search.aggregations.metrics.Sum;
-import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.hamcrest.Matchers;
 
@@ -1178,7 +1179,6 @@ public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAsCompound
     private void assertMultiSortResponse(long[] expectedKeys, BucketOrder... order) {
         SearchResponse response = client()
             .prepareSearch("sort_idx")
-            .setTypes("type")
             .addAggregation(
                 histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(1).order(BucketOrder.compound(order))
                     .subAggregation(avg("avg_l").field("l")).subAggregation(sum("sum_d").field("d"))).get();
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java
index 6b704a6711ad9..bacb67605bc53 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java
@@ -35,9 +35,9 @@
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
 import org.elasticsearch.search.aggregations.metrics.Avg;
+import org.elasticsearch.search.aggregations.metrics.ExtendedStats;
 import org.elasticsearch.search.aggregations.metrics.Max;
 import org.elasticsearch.search.aggregations.metrics.Stats;
-import org.elasticsearch.search.aggregations.metrics.ExtendedStats;
 import org.elasticsearch.search.aggregations.metrics.Sum;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -858,7 +858,7 @@ public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAsCompound
     }
 
     private void assertMultiSortResponse(long[] expectedKeys, BucketOrder... order) {
-        SearchResponse response = client().prepareSearch("sort_idx").setTypes("multi_sort_type")
+        SearchResponse response = client().prepareSearch("sort_idx")
                 .addAggregation(terms("terms")
                         .field(SINGLE_VALUED_FIELD_NAME)
                         .collectMode(randomFrom(SubAggCollectionMode.values()))
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java
index b09277aca6c6d..aa762c3a94392 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/MinDocCountIT.java
@@ -22,6 +22,7 @@
 import com.carrotsearch.hppc.LongHashSet;
 import com.carrotsearch.hppc.LongSet;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
@@ -308,7 +309,7 @@ private void testMinDocCountOnTerms(String field, Script script, BucketOrder ord
 
     private void testMinDocCountOnTerms(String field, Script script, BucketOrder order, String include, boolean retry) throws Exception {
         // all terms
-        final SearchResponse allTermsResponse = client().prepareSearch("idx").setTypes("type")
+        final SearchResponse allTermsResponse = client().prepareSearch("idx")
                 .setSize(0)
                 .setQuery(QUERY)
                 .addAggregation(script.apply(terms("terms"), field)
@@ -325,7 +326,7 @@ private void testMinDocCountOnTerms(String field, Script script, BucketOrder ord
 
         for (long minDocCount = 0; minDocCount < 20; ++minDocCount) {
             final int size = randomIntBetween(1, cardinality + 2);
-            final SearchRequest request = client().prepareSearch("idx").setTypes("type")
+            final SearchRequest request = client().prepareSearch("idx")
                     .setSize(0)
                     .setQuery(QUERY)
                     .addAggregation(script.apply(terms("terms"), field)
@@ -376,7 +377,7 @@ public void testDateHistogramKeyDesc() throws Exception {
 
     private void testMinDocCountOnHistogram(BucketOrder order) throws Exception {
         final int interval = randomIntBetween(1, 3);
-        final SearchResponse allResponse = client().prepareSearch("idx").setTypes("type")
+        final SearchResponse allResponse = client().prepareSearch("idx")
                 .setSize(0)
                 .setQuery(QUERY)
                 .addAggregation(histogram("histo").field("d").interval(interval).order(order).minDocCount(0))
@@ -385,7 +386,7 @@ private void testMinDocCountOnHistogram(BucketOrder order) throws Exception {
         final Histogram allHisto = allResponse.getAggregations().get("histo");
 
         for (long minDocCount = 0; minDocCount < 50; ++minDocCount) {
-            final SearchResponse response = client().prepareSearch("idx").setTypes("type")
+            final SearchResponse response = client().prepareSearch("idx")
                     .setSize(0)
                     .setQuery(QUERY)
                     .addAggregation(histogram("histo").field("d").interval(interval).order(order).minDocCount(minDocCount))
@@ -395,7 +396,7 @@ private void testMinDocCountOnHistogram(BucketOrder order) throws Exception {
     }
 
     private void testMinDocCountOnDateHistogram(BucketOrder order) throws Exception {
-        final SearchResponse allResponse = client().prepareSearch("idx").setTypes("type")
+        final SearchResponse allResponse = client().prepareSearch("idx")
                 .setSize(0)
                 .setQuery(QUERY)
                 .addAggregation(
@@ -409,7 +410,7 @@ private void testMinDocCountOnDateHistogram(BucketOrder order) throws Exception
         final Histogram allHisto = allResponse.getAggregations().get("histo");
 
         for (long minDocCount = 0; minDocCount < 50; ++minDocCount) {
-            final SearchResponse response = client().prepareSearch("idx").setTypes("type")
+            final SearchResponse response = client().prepareSearch("idx")
                     .setSize(0)
                     .setQuery(QUERY)
                     .addAggregation(
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
index 14fa6a9f565ef..fc76cf9f1d39c 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
@@ -414,7 +414,7 @@ public void testParentFilterResolvedCorrectly() throws Exception {
                         "{\"tid\" : 22, \"name\": \"DataChannels\"}], \"identifier\": \"29101\"}]}", XContentType.JSON));
         indexRandom(true, indexRequests);
 
-        SearchResponse response = client().prepareSearch("idx2").setTypes("provider")
+        SearchResponse response = client().prepareSearch("idx2")
                 .addAggregation(
                         terms("startDate").field("dates.month.start").subAggregation(
                                 terms("endDate").field("dates.month.end").subAggregation(
@@ -499,7 +499,7 @@ public void testNestedSameDocIdProcessedMultipleTime() throws Exception {
                 .endObject()).get();
         refresh();
 
-        SearchResponse response = client().prepareSearch("idx4").setTypes("product")
+        SearchResponse response = client().prepareSearch("idx4")
                 .addAggregation(terms("category").field("categories").subAggregation(
                         nested("property", "property").subAggregation(
                                 terms("property_id").field("property.id")
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RareTermsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RareTermsTests.java
new file mode 100644
index 0000000000000..31382f5df8e5c
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/RareTermsTests.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.automaton.RegExp;
+import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
+import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
+import org.elasticsearch.search.aggregations.bucket.terms.RareTermsAggregationBuilder;
+
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+public class RareTermsTests extends BaseAggregationTestCase<RareTermsAggregationBuilder> {
+
+    @Override
+    protected RareTermsAggregationBuilder createTestAggregatorBuilder() {
+        String name = randomAlphaOfLengthBetween(3, 20);
+        RareTermsAggregationBuilder factory = new RareTermsAggregationBuilder(name, null);
+        String field = randomAlphaOfLengthBetween(3, 20);
+        randomFieldOrScript(factory, field);
+        if (randomBoolean()) {
+            factory.missing("MISSING");
+        }
+        if (randomBoolean()) {
+            factory.format("###.##");
+        }
+        if (randomBoolean()) {
+            IncludeExclude incExc = null;
+            switch (randomInt(6)) {
+                case 0:
+                    incExc = new IncludeExclude(new RegExp("foobar"), null);
+                    break;
+                case 1:
+                    incExc = new IncludeExclude(null, new RegExp("foobaz"));
+                    break;
+                case 2:
+                    incExc = new IncludeExclude(new RegExp("foobar"), new RegExp("foobaz"));
+                    break;
+                case 3:
+                    SortedSet<BytesRef> includeValues = new TreeSet<>();
+                    int numIncs = randomIntBetween(1, 20);
+                    for (int i = 0; i < numIncs; i++) {
+                        includeValues.add(new BytesRef(randomAlphaOfLengthBetween(1, 30)));
+                    }
+                    SortedSet<BytesRef> excludeValues = null;
+                    incExc = new IncludeExclude(includeValues, excludeValues);
+                    break;
+                case 4:
+                    SortedSet<BytesRef> includeValues2 = null;
+                    SortedSet<BytesRef> excludeValues2 = new TreeSet<>();
+                    int numExcs2 = randomIntBetween(1, 20);
+                    for (int i = 0; i < numExcs2; i++) {
+                        excludeValues2.add(new BytesRef(randomAlphaOfLengthBetween(1, 30)));
+                    }
+                    incExc = new IncludeExclude(includeValues2, excludeValues2);
+                    break;
+                case 5:
+                    SortedSet<BytesRef> includeValues3 = new TreeSet<>();
+                    int numIncs3 = randomIntBetween(1, 20);
+                    for (int i = 0; i < numIncs3; i++) {
+                        includeValues3.add(new BytesRef(randomAlphaOfLengthBetween(1, 30)));
+                    }
+                    SortedSet<BytesRef> excludeValues3 = new TreeSet<>();
+                    int numExcs3 = randomIntBetween(1, 20);
+                    for (int i = 0; i < numExcs3; i++) {
+                        excludeValues3.add(new BytesRef(randomAlphaOfLengthBetween(1, 30)));
+                    }
+                    incExc = new IncludeExclude(includeValues3, excludeValues3);
+                    break;
+                case 6:
+                    final int numPartitions = randomIntBetween(1, 100);
+                    final int partition = randomIntBetween(0, numPartitions - 1);
+                    incExc = new IncludeExclude(partition, numPartitions);
+                    break;
+                default:
+                    fail();
+            }
+            factory.includeExclude(incExc);
+        }
+        return factory;
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java
index 42192bbebf209..852f929ffe53a 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java
@@ -23,13 +23,13 @@
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.bucket.sampler.Sampler;
-import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregator;
 import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregator;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
 import org.elasticsearch.search.aggregations.metrics.Max;
-import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.List;
@@ -102,7 +102,7 @@ public void testIssue10719() throws Exception {
         // Tests that we can refer to nested elements under a sample in a path
         // statement
         boolean asc = randomBoolean();
-        SearchResponse response = client().prepareSearch("test").setTypes("book").setSearchType(SearchType.QUERY_THEN_FETCH)
+        SearchResponse response = client().prepareSearch("test").setSearchType(SearchType.QUERY_THEN_FETCH)
                 .addAggregation(terms("genres")
                         .field("genre")
                         .order(BucketOrder.aggregation("sample>max_price.value", asc))
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java
index 2953324c58cfa..0b16714788eef 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java
@@ -20,8 +20,8 @@
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
-import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.BucketOrder;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 
 import java.util.HashMap;
 import java.util.List;
@@ -37,7 +37,7 @@ public void testNoShardSizeString() throws Exception {
 
         indexData();
 
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
                         .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.count(false)))
@@ -60,7 +60,7 @@ public void testShardSizeEqualsSizeString() throws Exception {
 
         indexData();
 
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3).shardSize(3)
                         .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.count(false)))
@@ -84,7 +84,7 @@ public void testWithShardSizeString() throws Exception {
 
         indexData();
 
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
                         .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(BucketOrder.count(false)))
@@ -108,7 +108,7 @@ public void testWithShardSizeStringSingleShard() throws Exception {
 
         indexData();
 
-        SearchResponse response = client().prepareSearch("idx").setTypes("type").setRouting(routing1)
+        SearchResponse response = client().prepareSearch("idx").setRouting(routing1)
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
                         .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(BucketOrder.count(false)))
@@ -131,7 +131,7 @@ public void testNoShardSizeTermOrderString() throws Exception {
 
         indexData();
 
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
                         .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.key(true)))
@@ -154,7 +154,7 @@ public void testNoShardSizeLong() throws Exception {
 
         indexData();
 
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
                         .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.count(false)))
@@ -177,7 +177,7 @@ public void testShardSizeEqualsSizeLong() throws Exception {
 
         indexData();
 
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3).shardSize(3)
                         .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.count(false)))
@@ -200,7 +200,7 @@ public void testWithShardSizeLong() throws Exception {
 
         indexData();
 
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
                         .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(BucketOrder.count(false)))
@@ -224,7 +224,7 @@ public void testWithShardSizeLongSingleShard() throws Exception {
 
         indexData();
 
-        SearchResponse response = client().prepareSearch("idx").setTypes("type").setRouting(routing1)
+        SearchResponse response = client().prepareSearch("idx").setRouting(routing1)
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
                         .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(BucketOrder.count(false)))
@@ -247,7 +247,7 @@ public void testNoShardSizeTermOrderLong() throws Exception {
 
         indexData();
 
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
                         .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.key(true)))
@@ -270,7 +270,7 @@ public void testNoShardSizeDouble() throws Exception {
 
         indexData();
 
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
                         .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.count(false)))
@@ -293,7 +293,7 @@ public void testShardSizeEqualsSizeDouble() throws Exception {
 
         indexData();
 
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3).shardSize(3)
                         .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.count(false)))
@@ -316,7 +316,7 @@ public void testWithShardSizeDouble() throws Exception {
 
         indexData();
 
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
                         .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(BucketOrder.count(false)))
@@ -339,7 +339,7 @@ public void testWithShardSizeDoubleSingleShard() throws Exception {
 
         indexData();
 
-        SearchResponse response = client().prepareSearch("idx").setTypes("type").setRouting(routing1)
+        SearchResponse response = client().prepareSearch("idx").setRouting(routing1)
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
                         .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(BucketOrder.count(false)))
@@ -362,7 +362,7 @@ public void testNoShardSizeTermOrderDouble() throws Exception {
 
         indexData();
 
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
                         .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.key(true)))
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTestCase.java
index 4e4bfce5ecbb2..6112163bb1faa 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTestCase.java
@@ -90,11 +90,11 @@ protected void indexData() throws Exception {
 
         indexRandom(true, docs);
 
-        SearchResponse resp = client().prepareSearch("idx").setTypes("type").setRouting(routing1).setQuery(matchAllQuery()).get();
+        SearchResponse resp = client().prepareSearch("idx").setRouting(routing1).setQuery(matchAllQuery()).get();
         assertSearchResponse(resp);
         long totalOnOne = resp.getHits().getTotalHits().value;
         assertThat(totalOnOne, is(15L));
-        resp = client().prepareSearch("idx").setTypes("type").setRouting(routing2).setQuery(matchAllQuery()).get();
+        resp = client().prepareSearch("idx").setRouting(routing2).setQuery(matchAllQuery()).get();
         assertSearchResponse(resp);
         long totalOnTwo = resp.getHits().getTotalHits().value;
         assertThat(totalOnTwo, is(12L));
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
index 81ea71621ae3e..dd7ecc8c1f1ed 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
@@ -106,7 +106,7 @@ public void testPlugin() throws Exception {
         if ("text".equals(type) && randomBoolean()) {
             // Use significant_text on text fields but occasionally run with alternative of
             // significant_terms on legacy fieldData=true too.
-            request = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE)
+            request = client().prepareSearch(INDEX_NAME)
                     .addAggregation(
                             terms("class")
                             .field(CLASS_FIELD)
@@ -117,7 +117,7 @@ public void testPlugin() throws Exception {
                     );
         }else
         {
-            request = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE)
+            request = client().prepareSearch(INDEX_NAME)
                     .addAggregation(
                             terms("class")
                             .field(CLASS_FIELD)
@@ -271,11 +271,11 @@ public void testXContentResponse() throws Exception {
         if ("text".equals(type) && randomBoolean() ) {
             // Use significant_text on text fields but occasionally run with alternative of
             // significant_terms on legacy fieldData=true too.
-            request = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE)
+            request = client().prepareSearch(INDEX_NAME)
                     .addAggregation(terms("class").field(CLASS_FIELD)
                             .subAggregation(significantText("sig_terms", TEXT_FIELD)));
         } else {
-            request = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE)
+            request = client().prepareSearch(INDEX_NAME)
                     .addAggregation(terms("class").field(CLASS_FIELD)
                             .subAggregation(significantTerms("sig_terms").field(TEXT_FIELD)));
         }
@@ -370,7 +370,7 @@ public void testDeletesIssue7951() throws Exception {
 
         SearchRequestBuilder request;
         if (randomBoolean() ) {
-            request = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE)
+            request = client().prepareSearch(INDEX_NAME)
                 .addAggregation(
                         terms("class")
                         .field(CLASS_FIELD)
@@ -380,7 +380,7 @@ public void testDeletesIssue7951() throws Exception {
                                         .minDocCount(1)));
         }else
         {
-            request = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE)
+            request = client().prepareSearch(INDEX_NAME)
                     .addAggregation(
                             terms("class")
                             .field(CLASS_FIELD)
@@ -413,7 +413,7 @@ public void testBackgroundVsSeparateSet(SignificanceHeuristic significanceHeuris
         final boolean useSigText = randomBoolean() && type.equals("text");
         SearchRequestBuilder request1;
         if (useSigText) {
-            request1 = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE)
+            request1 = client().prepareSearch(INDEX_NAME)
                     .addAggregation(terms("class")
                             .field(CLASS_FIELD)
                             .subAggregation(
@@ -423,7 +423,7 @@ public void testBackgroundVsSeparateSet(SignificanceHeuristic significanceHeuris
                                                     significanceHeuristicExpectingSuperset)));
         }else
         {
-            request1 = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE)
+            request1 = client().prepareSearch(INDEX_NAME)
                     .addAggregation(terms("class")
                             .field(CLASS_FIELD)
                             .subAggregation(
@@ -439,7 +439,7 @@ public void testBackgroundVsSeparateSet(SignificanceHeuristic significanceHeuris
 
         SearchRequestBuilder request2;
         if (useSigText) {
-            request2 = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE)
+            request2 = client().prepareSearch(INDEX_NAME)
                     .addAggregation(filter("0", QueryBuilders.termQuery(CLASS_FIELD, "0"))
                             .subAggregation(significantText("sig_terms", TEXT_FIELD)
                                     .minDocCount(1)
@@ -452,7 +452,7 @@ public void testBackgroundVsSeparateSet(SignificanceHeuristic significanceHeuris
                                     .significanceHeuristic(significanceHeuristicExpectingSeparateSets)));
         }else
         {
-            request2 = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE)
+            request2 = client().prepareSearch(INDEX_NAME)
                     .addAggregation(filter("0", QueryBuilders.termQuery(CLASS_FIELD, "0"))
                             .subAggregation(significantTerms("sig_terms")
                                     .field(TEXT_FIELD)
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java
index 04fe0d5751887..f3e6eb544ced8 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java
@@ -25,10 +25,10 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
+import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.ExecutionMode;
-import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.ArrayList;
@@ -263,7 +263,7 @@ private void assertUnboundedDocCountError(int size, SearchResponse accurateRespo
     public void testStringValueField() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
@@ -274,7 +274,7 @@ public void testStringValueField() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
@@ -292,7 +292,7 @@ public void testStringValueField() throws Exception {
     public void testStringValueFieldSingleShard() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
@@ -303,7 +303,7 @@ public void testStringValueFieldSingleShard() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
@@ -322,7 +322,7 @@ public void testStringValueFieldWithRouting() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
 
-        SearchResponse testResponse = client().prepareSearch("idx_with_routing").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_with_routing")
                 .setRouting(String.valueOf(between(1, numRoutingValues)))
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
@@ -341,7 +341,7 @@ public void testStringValueFieldWithRouting() throws Exception {
     public void testStringValueFieldDocCountAsc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
@@ -353,7 +353,7 @@ public void testStringValueFieldDocCountAsc() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
@@ -372,7 +372,7 @@ public void testStringValueFieldDocCountAsc() throws Exception {
     public void testStringValueFieldTermSortAsc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
@@ -384,7 +384,7 @@ public void testStringValueFieldTermSortAsc() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
@@ -403,7 +403,7 @@ public void testStringValueFieldTermSortAsc() throws Exception {
     public void testStringValueFieldTermSortDesc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
@@ -415,7 +415,7 @@ public void testStringValueFieldTermSortDesc() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
@@ -434,7 +434,7 @@ public void testStringValueFieldTermSortDesc() throws Exception {
     public void testStringValueFieldSubAggAsc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
@@ -447,7 +447,7 @@ public void testStringValueFieldSubAggAsc() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
@@ -467,7 +467,7 @@ public void testStringValueFieldSubAggAsc() throws Exception {
     public void testStringValueFieldSubAggDesc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
@@ -480,7 +480,7 @@ public void testStringValueFieldSubAggDesc() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
@@ -500,7 +500,7 @@ public void testStringValueFieldSubAggDesc() throws Exception {
     public void testLongValueField() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
@@ -511,7 +511,7 @@ public void testLongValueField() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
@@ -529,7 +529,7 @@ public void testLongValueField() throws Exception {
     public void testLongValueFieldSingleShard() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
@@ -540,7 +540,7 @@ public void testLongValueFieldSingleShard() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
@@ -559,7 +559,7 @@ public void testLongValueFieldWithRouting() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
 
-        SearchResponse testResponse = client().prepareSearch("idx_with_routing").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_with_routing")
                 .setRouting(String.valueOf(between(1, numRoutingValues)))
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
@@ -578,7 +578,7 @@ public void testLongValueFieldWithRouting() throws Exception {
     public void testLongValueFieldDocCountAsc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
@@ -590,7 +590,7 @@ public void testLongValueFieldDocCountAsc() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
@@ -609,7 +609,7 @@ public void testLongValueFieldDocCountAsc() throws Exception {
     public void testLongValueFieldTermSortAsc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
@@ -621,7 +621,7 @@ public void testLongValueFieldTermSortAsc() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
@@ -640,7 +640,7 @@ public void testLongValueFieldTermSortAsc() throws Exception {
     public void testLongValueFieldTermSortDesc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
@@ -652,7 +652,7 @@ public void testLongValueFieldTermSortDesc() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
@@ -671,7 +671,7 @@ public void testLongValueFieldTermSortDesc() throws Exception {
     public void testLongValueFieldSubAggAsc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
@@ -684,7 +684,7 @@ public void testLongValueFieldSubAggAsc() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
@@ -704,7 +704,7 @@ public void testLongValueFieldSubAggAsc() throws Exception {
     public void testLongValueFieldSubAggDesc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
@@ -717,7 +717,7 @@ public void testLongValueFieldSubAggDesc() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
@@ -737,7 +737,7 @@ public void testLongValueFieldSubAggDesc() throws Exception {
     public void testDoubleValueField() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
@@ -748,7 +748,7 @@ public void testDoubleValueField() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
@@ -766,7 +766,7 @@ public void testDoubleValueField() throws Exception {
     public void testDoubleValueFieldSingleShard() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
@@ -777,7 +777,7 @@ public void testDoubleValueFieldSingleShard() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
@@ -796,7 +796,7 @@ public void testDoubleValueFieldWithRouting() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
 
-        SearchResponse testResponse = client().prepareSearch("idx_with_routing").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_with_routing")
                 .setRouting(String.valueOf(between(1, numRoutingValues)))
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
@@ -815,7 +815,7 @@ public void testDoubleValueFieldWithRouting() throws Exception {
     public void testDoubleValueFieldDocCountAsc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
@@ -827,7 +827,7 @@ public void testDoubleValueFieldDocCountAsc() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
@@ -846,7 +846,7 @@ public void testDoubleValueFieldDocCountAsc() throws Exception {
     public void testDoubleValueFieldTermSortAsc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
@@ -858,7 +858,7 @@ public void testDoubleValueFieldTermSortAsc() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
@@ -877,7 +877,7 @@ public void testDoubleValueFieldTermSortAsc() throws Exception {
     public void testDoubleValueFieldTermSortDesc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
@@ -889,7 +889,7 @@ public void testDoubleValueFieldTermSortDesc() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
@@ -908,7 +908,7 @@ public void testDoubleValueFieldTermSortDesc() throws Exception {
     public void testDoubleValueFieldSubAggAsc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
@@ -921,7 +921,7 @@ public void testDoubleValueFieldSubAggAsc() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
@@ -941,7 +941,7 @@ public void testDoubleValueFieldSubAggAsc() throws Exception {
     public void testDoubleValueFieldSubAggDesc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
@@ -954,7 +954,7 @@ public void testDoubleValueFieldSubAggDesc() throws Exception {
 
         assertSearchResponse(accurateResponse);
 
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
@@ -977,7 +977,7 @@ public void testDoubleValueFieldSubAggDesc() throws Exception {
      * 3 one-shard indices.
      */
     public void testFixedDocs() throws Exception {
-        SearchResponse response = client().prepareSearch("idx_fixed_docs_0", "idx_fixed_docs_1", "idx_fixed_docs_2").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx_fixed_docs_0", "idx_fixed_docs_1", "idx_fixed_docs_2")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilderTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilderTests.java
index 643344bb3bb03..e5bed886bac1e 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilderTests.java
@@ -76,7 +76,7 @@ public void testFilterSizeLimitation() throws Exception {
         Map<String, QueryBuilder> emptyFilters = Collections.emptyMap();
 
         AdjacencyMatrixAggregationBuilder aggregationBuilder = new AdjacencyMatrixAggregationBuilder("dummy", emptyFilters);
-        AggregatorFactory<?> factory = aggregationBuilder.doBuild(context, null, new AggregatorFactories.Builder());
+        AggregatorFactory factory = aggregationBuilder.doBuild(context, null, new AggregatorFactories.Builder());
         assertThat(factory instanceof AdjacencyMatrixAggregatorFactory, is(true));
         assertThat(factory.name(), equalTo("dummy"));
     }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilderTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilderTests.java
new file mode 100644
index 0000000000000..095a335a7dc97
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilderTests.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.histogram;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class AutoDateHistogramAggregationBuilderTests extends ESTestCase {
+
+    public void testInvalidInterval() {
+        AutoDateHistogramAggregationBuilder builder = new AutoDateHistogramAggregationBuilder("name");
+        IllegalArgumentException wrongIntervalName = expectThrows(IllegalArgumentException.class,
+            () -> builder.setMinimumIntervalExpression("foobar"));
+        assertTrue(wrongIntervalName.getMessage().startsWith("minimum_interval must be one of"));
+    }
+
+    public void testBuildRoundingsWithNullParameter() {
+        int expectedLength = AutoDateHistogramAggregationBuilder.ALLOWED_INTERVALS.size();
+        AutoDateHistogramAggregationBuilder.RoundingInfo[] roundings = AutoDateHistogramAggregationBuilder.buildRoundings(null, null);
+        assertThat(roundings.length, equalTo(expectedLength));
+    }
+
+    public void testBuildRoundingsWithMinIntervalOfAYear() {
+        int[] expectedYearIntervals = { 1, 5, 10, 20, 50, 100 };
+        AutoDateHistogramAggregationBuilder.RoundingInfo[] roundings = AutoDateHistogramAggregationBuilder.buildRoundings(null, "year");
+        assertThat(roundings.length, equalTo(1));
+        AutoDateHistogramAggregationBuilder.RoundingInfo year = roundings[0];
+        assertEquals(year.unitAbbreviation, "y");
+        assertEquals(year.dateTimeUnit, "year");
+        assertEquals(year.roughEstimateDurationMillis, 31536000000L);
+        assertArrayEquals(year.innerIntervals, expectedYearIntervals);
+    }
+
+    public void testRoundingsMatchAllowedIntervals() {
+        AutoDateHistogramAggregationBuilder.RoundingInfo[] roundings = AutoDateHistogramAggregationBuilder.buildRoundings(
+            null, "second");
+        Set<String> actualDateTimeUnits = Arrays.stream(roundings)
+            .map(AutoDateHistogramAggregationBuilder.RoundingInfo::getDateTimeUnit)
+            .collect(Collectors.toSet());
+        Set<String> expectedDateTimeUnits = new HashSet<>(AutoDateHistogramAggregationBuilder.ALLOWED_INTERVALS.values());
+        assertEquals(actualDateTimeUnits, expectedDateTimeUnits);
+    }
+
+}
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java
index 4eded8c8155d1..d875224fb0614 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java
@@ -100,7 +100,7 @@ public void testParseAndValidate() {
         SearchContext context = mock(SearchContext.class);
         QueryShardContext qsc = new QueryShardContext(0,
                 new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings), null, null, null, null,
-                null, xContentRegistry(), writableRegistry(), null, null, () -> now, null);
+                null, null, xContentRegistry(), writableRegistry(), null, null, () -> now, null);
         when(context.getQueryShardContext()).thenReturn(qsc);
         DateFormatter formatter = DateFormatter.forPattern("dateOptionalTime");
         DocValueFormat format = new DocValueFormat.DateTime(formatter, ZoneOffset.UTC, DateFieldMapper.Resolution.MILLISECONDS);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorTests.java
index bdb99a4971ac1..624870f6e47a6 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorTests.java
@@ -21,17 +21,22 @@
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
 
+import static org.hamcrest.Matchers.containsString;
+
 public class HistogramAggregatorTests extends AggregatorTestCase {
 
     public void testLongs() throws Exception {
@@ -188,6 +193,83 @@ public void testMissing() throws Exception {
         }
     }
 
+    public void testMissingUnmappedField() throws Exception {
+        try (Directory dir = newDirectory();
+             RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
+            for (int i = 0; i < 7; i ++) {
+                Document doc = new Document();
+                w.addDocument(doc);
+            }
+
+            HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg")
+                .field("field")
+                .interval(5)
+                .missing(2d);
+            MappedFieldType type = null;
+            try (IndexReader reader = w.getReader()) {
+                IndexSearcher searcher = new IndexSearcher(reader);
+                InternalHistogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, type);
+
+                assertEquals(1, histogram.getBuckets().size());
+
+                assertEquals(0d, histogram.getBuckets().get(0).getKey());
+                assertEquals(7, histogram.getBuckets().get(0).getDocCount());
+
+                assertTrue(AggregationInspectionHelper.hasValue(histogram));
+            }
+        }
+    }
+
+    public void testMissingUnmappedFieldBadType() throws Exception {
+        try (Directory dir = newDirectory();
+             RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
+            for (int i = 0; i < 7; i ++) {
+                w.addDocument(new Document());
+            }
+
+            String missingValue = "🍌🍌🍌";
+            HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg")
+                .field("field")
+                .interval(5)
+                .missing(missingValue);
+            MappedFieldType type = null;
+            try (IndexReader reader = w.getReader()) {
+                IndexSearcher searcher = new IndexSearcher(reader);
+                Throwable t = expectThrows(IllegalArgumentException.class, () -> {
+                    search(searcher, new MatchAllDocsQuery(), aggBuilder, type);
+                });
+                // This throws a number format exception (which is a subclass of IllegalArgumentException) and might be ok?
+                assertThat(t.getMessage(), containsString(missingValue));
+            }
+        }
+    }
+
+    public void testIncorrectFieldType() throws Exception {
+        try (Directory dir = newDirectory();
+             RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
+            for (String value : new String[] {"foo", "bar", "baz", "quux"}) {
+                Document doc = new Document();
+                doc.add(new SortedSetDocValuesField("field", new BytesRef(value)));
+                w.addDocument(doc);
+            }
+
+            HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg")
+                .field("field")
+                .interval(5);
+            MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
+            fieldType.setName("field");
+            fieldType.setHasDocValues(true);
+            try (IndexReader reader = w.getReader()) {
+                IndexSearcher searcher = new IndexSearcher(reader);
+
+                expectThrows(IllegalArgumentException.class, () -> {
+                    search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType);
+                });
+            }
+        }
+
+    }
+
     public void testOffset() throws Exception {
         try (Directory dir = newDirectory();
                 RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java
index d99dc1a9490be..bc4c84282b262 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java
@@ -44,7 +44,6 @@
 import static org.elasticsearch.common.unit.TimeValue.timeValueHours;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
-import static org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder.createRounding;
 import static org.hamcrest.Matchers.equalTo;
 
 public class InternalAutoDateHistogramTests extends InternalMultiBucketAggregationTestCase<InternalAutoDateHistogram> {
@@ -64,7 +63,7 @@ protected InternalAutoDateHistogram createTestInstance(String name,
                                                        Map<String, Object> metaData,
                                                        InternalAggregations aggregations) {
 
-        roundingInfos = AutoDateHistogramAggregationBuilder.buildRoundings(null);
+        roundingInfos = AutoDateHistogramAggregationBuilder.buildRoundings(null, null);
         int nbBuckets = randomNumberOfBuckets();
         int targetBuckets = randomIntBetween(1, nbBuckets * 2 + 1);
         List<InternalAutoDateHistogram.Bucket> buckets = new ArrayList<>(nbBuckets);
@@ -93,12 +92,12 @@ public void testGetAppropriateRoundingUsesCorrectIntervals() {
         // Since we pass 0 as the starting index to getAppropriateRounding, we'll also use
         // an innerInterval that is quite large, such that targetBuckets * roundings[i].getMaximumInnerInterval()
         // will be larger than the estimate.
-        roundings[0] = new RoundingInfo(createRounding(Rounding.DateTimeUnit.SECOND_OF_MINUTE, timeZone),
-            1000L, "s",1000);
-        roundings[1] = new RoundingInfo(createRounding(Rounding.DateTimeUnit.MINUTES_OF_HOUR, timeZone),
-            60 * 1000L, "m",1, 5, 10, 30);
-        roundings[2] = new RoundingInfo(createRounding(Rounding.DateTimeUnit.HOUR_OF_DAY, timeZone),
-            60 * 60 * 1000L, "h",1, 3, 12);
+        roundings[0] = new RoundingInfo(Rounding.DateTimeUnit.SECOND_OF_MINUTE, timeZone,
+            1000L, "s", 1000);
+        roundings[1] = new RoundingInfo(Rounding.DateTimeUnit.MINUTES_OF_HOUR, timeZone,
+            60 * 1000L, "m", 1, 5, 10, 30);
+        roundings[2] = new RoundingInfo(Rounding.DateTimeUnit.HOUR_OF_DAY, timeZone,
+            60 * 60 * 1000L, "h", 1, 3, 12);
 
         OffsetDateTime timestamp = Instant.parse("2018-01-01T00:00:01.000Z").atOffset(ZoneOffset.UTC);
         // We want to pass a roundingIdx of zero, because in order to reproduce this bug, we need the function
@@ -109,6 +108,7 @@ public void testGetAppropriateRoundingUsesCorrectIntervals() {
         assertThat(result, equalTo(2));
     }
 
+
     @Override
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/39497")
     // TODO: When resolving the above AwaitsFix, just delete this override. Method is only overriden to apply the annotation.
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java
index 5d8c86a371582..f759d30cbbde5 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/IpRangeAggregatorTests.java
@@ -152,4 +152,45 @@ public void testRanges() throws Exception {
             }
         }
     }
+
+    public void testMissingUnmapped() throws Exception {
+        try (Directory dir = newDirectory();
+             RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
+            for (int i = 0; i < 7; i++) {
+                Document doc = new Document();
+                w.addDocument(doc);
+            }
+
+            IpRangeAggregationBuilder builder = new IpRangeAggregationBuilder("test_agg")
+                .field("field")
+                .addRange(new IpRangeAggregationBuilder.Range("foo", "192.168.100.0", "192.168.100.255"))
+                .missing("192.168.100.42"); // Apparently we expect a string here
+            try (IndexReader reader = w.getReader()) {
+                IndexSearcher searcher = new IndexSearcher(reader);
+                InternalBinaryRange range = search(searcher, new MatchAllDocsQuery(), builder, (MappedFieldType) null);
+                assertEquals(1, range.getBuckets().size());
+            }
+        }
+    }
+
+    public void testMissingUnmappedBadType() throws Exception {
+        try (Directory dir = newDirectory();
+             RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
+            for (int i = 0; i < 7; i++) {
+                Document doc = new Document();
+                w.addDocument(doc);
+            }
+
+            IpRangeAggregationBuilder builder = new IpRangeAggregationBuilder("test_agg")
+                .field("field")
+                .addRange(new IpRangeAggregationBuilder.Range("foo", "192.168.100.0", "192.168.100.255"))
+                .missing(1234);
+            try (IndexReader reader = w.getReader()) {
+                IndexSearcher searcher = new IndexSearcher(reader);
+                expectThrows(IllegalArgumentException.class, () -> {
+                    search(searcher, new MatchAllDocsQuery(), builder, (MappedFieldType) null);
+                });
+            }
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
index 63415d981e61d..49de4eb821115 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
@@ -107,7 +107,7 @@ public void testStreamResponse() throws Exception {
         // read
         ByteArrayInputStream inBuffer = new ByteArrayInputStream(outBuffer.toByteArray());
         StreamInput in = new InputStreamStreamInput(inBuffer);
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); // populates the registry through side effects
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); // populates the registry through side effects
         NamedWriteableRegistry registry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
         in = new NamedWriteableAwareStreamInput(in, registry);
         in.setVersion(version);
@@ -218,7 +218,7 @@ SignificantLongTerms.Bucket createBucket(long subsetDF, long subsetSize, long su
     // 1. The output of the builders can actually be parsed
     // 2. The parser does not swallow parameters after a significance heuristic was defined
     public void testBuilderAndParser() throws Exception {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         ParseFieldRegistry<SignificanceHeuristicParser> heuristicParserMapper = searchModule.getSignificanceHeuristicParserRegistry();
 
         // test jlh with string
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java
new file mode 100644
index 0000000000000..a0d48b7ab778f
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java
@@ -0,0 +1,600 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TotalHits;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.mapper.SeqNoFieldMapper;
+import org.elasticsearch.index.mapper.TypeFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.global.InternalGlobal;
+import org.elasticsearch.search.aggregations.bucket.nested.InternalNested;
+import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.InternalTopHits;
+import org.elasticsearch.search.aggregations.metrics.Max;
+import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder;
+import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.ScoreSortBuilder;
+import org.junit.Assert;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Consumer;
+
+import static org.elasticsearch.index.mapper.SeqNoFieldMapper.PRIMARY_TERM_NAME;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+public class RareTermsAggregatorTests extends AggregatorTestCase {
+
+    private static final String LONG_FIELD = "numeric";
+    private static final String KEYWORD_FIELD = "keyword";
+
+    private static final List<Long> dataset;
+    static {
+        List<Long> d = new ArrayList<>(45);
+        for (int i = 0; i < 10; i++) {
+            for (int j = 0; j < i; j++) {
+                d.add((long) i);
+            }
+        }
+        dataset  = d;
+    }
+
+    public void testMatchNoDocs() throws IOException {
+        testBothCases(new MatchNoDocsQuery(), dataset,
+            aggregation -> aggregation.field(KEYWORD_FIELD).maxDocCount(1),
+            agg -> assertEquals(0, agg.getBuckets().size()), ValueType.STRING
+        );
+        testBothCases(new MatchNoDocsQuery(), dataset,
+            aggregation -> aggregation.field(LONG_FIELD).maxDocCount(1),
+            agg -> assertEquals(0, agg.getBuckets().size()), ValueType.NUMERIC
+        );
+    }
+
+    public void testMatchAllDocs() throws IOException {
+        Query query = new MatchAllDocsQuery();
+
+        testBothCases(query, dataset,
+            aggregation -> aggregation.field(LONG_FIELD).maxDocCount(1),
+            agg -> {
+                assertEquals(1, agg.getBuckets().size());
+                LongRareTerms.Bucket bucket = (LongRareTerms.Bucket) agg.getBuckets().get(0);
+                assertThat(bucket.getKey(), equalTo(1L));
+                assertThat(bucket.getDocCount(), equalTo(1L));
+            }, ValueType.NUMERIC
+        );
+        testBothCases(query, dataset,
+            aggregation -> aggregation.field(KEYWORD_FIELD).maxDocCount(1),
+            agg -> {
+                assertEquals(1, agg.getBuckets().size());
+                StringRareTerms.Bucket bucket = (StringRareTerms.Bucket) agg.getBuckets().get(0);
+                assertThat(bucket.getKeyAsString(), equalTo("1"));
+                assertThat(bucket.getDocCount(), equalTo(1L));
+            }, ValueType.STRING
+        );
+    }
+
+    public void testManyDocsOneRare() throws IOException {
+        Query query = new MatchAllDocsQuery();
+
+        List<Long> d = new ArrayList<>(500);
+        for (int i = 1; i < 500; i++) {
+            d.add((long) i);
+            d.add((long) i);
+        }
+
+        // The one rare term
+        d.add(0L);
+
+        testSearchAndReduceCase(query, d,
+            aggregation -> aggregation.field(LONG_FIELD).maxDocCount(1),
+            agg -> {
+                assertEquals(1, agg.getBuckets().size());
+                LongRareTerms.Bucket bucket = (LongRareTerms.Bucket) agg.getBuckets().get(0);
+                assertThat(bucket.getKey(), equalTo(0L));
+                assertThat(bucket.getDocCount(), equalTo(1L));
+            }, ValueType.NUMERIC
+        );
+        testSearchAndReduceCase(query, d,
+            aggregation -> aggregation.field(KEYWORD_FIELD).maxDocCount(1),
+            agg -> {
+                assertEquals(1, agg.getBuckets().size());
+                StringRareTerms.Bucket bucket = (StringRareTerms.Bucket) agg.getBuckets().get(0);
+                assertThat(bucket.getKeyAsString(), equalTo("0"));
+                assertThat(bucket.getDocCount(), equalTo(1L));
+            }, ValueType.STRING
+        );
+    }
+
+    public void testIncludeExclude() throws IOException {
+        Query query = new MatchAllDocsQuery();
+
+        testBothCases(query, dataset,
+            aggregation -> aggregation.field(LONG_FIELD)
+                .maxDocCount(2) // bump to 2 since we're only including "2"
+                .includeExclude(new IncludeExclude(new long[]{2}, new long[]{})),
+            agg -> {
+                assertEquals(1, agg.getBuckets().size());
+                LongRareTerms.Bucket bucket = (LongRareTerms.Bucket) agg.getBuckets().get(0);
+                assertThat(bucket.getKey(), equalTo(2L));
+                assertThat(bucket.getDocCount(), equalTo(2L));
+            }, ValueType.NUMERIC
+        );
+        testBothCases(query, dataset,
+            aggregation -> aggregation.field(KEYWORD_FIELD)
+                .maxDocCount(2) // bump to 2 since we're only including "2"
+                .includeExclude(new IncludeExclude(new String[]{"2"}, new String[]{})),
+            agg -> {
+                assertEquals(1, agg.getBuckets().size());
+                StringRareTerms.Bucket bucket = (StringRareTerms.Bucket) agg.getBuckets().get(0);
+                assertThat(bucket.getKeyAsString(), equalTo("2"));
+                assertThat(bucket.getDocCount(), equalTo(2L));
+            }, ValueType.STRING
+        );
+    }
+
+    public void testEmbeddedMaxAgg() throws IOException {
+        Query query = new MatchAllDocsQuery();
+
+        testBothCases(query, dataset, aggregation -> {
+                MaxAggregationBuilder max = new MaxAggregationBuilder("the_max").field(LONG_FIELD);
+                aggregation.field(LONG_FIELD).maxDocCount(1).subAggregation(max);
+            },
+            agg -> {
+                assertEquals(1, agg.getBuckets().size());
+                LongRareTerms.Bucket bucket = (LongRareTerms.Bucket) agg.getBuckets().get(0);
+                assertThat(bucket.getKey(), equalTo(1L));
+                assertThat(bucket.getDocCount(), equalTo(1L));
+
+                Aggregations children = bucket.getAggregations();
+                assertThat(children.asList().size(), equalTo(1));
+                assertThat(children.asList().get(0).getName(), equalTo("the_max"));
+                assertThat(((Max)(children.asList().get(0))).getValue(), equalTo(1.0));
+            }, ValueType.NUMERIC
+        );
+        testBothCases(query, dataset, aggregation -> {
+                MaxAggregationBuilder max = new MaxAggregationBuilder("the_max").field(LONG_FIELD);
+                aggregation.field(KEYWORD_FIELD).maxDocCount(1).subAggregation(max);
+            },
+            agg -> {
+                assertEquals(1, agg.getBuckets().size());
+                StringRareTerms.Bucket bucket = (StringRareTerms.Bucket) agg.getBuckets().get(0);
+                assertThat(bucket.getKey(), equalTo("1"));
+                assertThat(bucket.getDocCount(), equalTo(1L));
+
+                Aggregations children = bucket.getAggregations();
+                assertThat(children.asList().size(), equalTo(1));
+                assertThat(children.asList().get(0).getName(), equalTo("the_max"));
+                assertThat(((Max)(children.asList().get(0))).getValue(), equalTo(1.0));
+            }, ValueType.STRING
+        );
+    }
+
+    public void testEmpty() throws IOException {
+        Query query = new MatchAllDocsQuery();
+
+        testSearchCase(query, Collections.emptyList(),
+            aggregation -> aggregation.field(LONG_FIELD).maxDocCount(1),
+            agg -> assertEquals(0, agg.getBuckets().size()), ValueType.NUMERIC
+        );
+        testSearchCase(query, Collections.emptyList(),
+            aggregation -> aggregation.field(KEYWORD_FIELD).maxDocCount(1),
+            agg -> assertEquals(0, agg.getBuckets().size()), ValueType.STRING
+        );
+
+        // Note: the search and reduce test will generate no segments (due to no docs)
+        // and so will return a null agg because the aggs aren't run/reduced
+        testSearchAndReduceCase(query, Collections.emptyList(),
+            aggregation -> aggregation.field(LONG_FIELD).maxDocCount(1),
+            Assert::assertNull, ValueType.NUMERIC
+        );
+        testSearchAndReduceCase(query, Collections.emptyList(),
+            aggregation -> aggregation.field(KEYWORD_FIELD).maxDocCount(1),
+            Assert::assertNull, ValueType.STRING
+        );
+    }
+
+    public void testUnmapped() throws Exception {
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                Document document = new Document();
+                document.add(new SortedDocValuesField("string", new BytesRef("a")));
+                document.add(new NumericDocValuesField("long", 0L));
+                indexWriter.addDocument(document);
+                MappedFieldType fieldType1 = new KeywordFieldMapper.KeywordFieldType();
+                fieldType1.setName("another_string");
+                fieldType1.setHasDocValues(true);
+
+                MappedFieldType fieldType2 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+                fieldType2.setName("another_long");
+                fieldType2.setHasDocValues(true);
+
+
+                try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
+                    IndexSearcher indexSearcher = newIndexSearcher(indexReader);
+                    ValueType[] valueTypes = new ValueType[]{ValueType.STRING, ValueType.LONG};
+                    String[] fieldNames = new String[]{"string", "long"};
+                    for (int i = 0; i < fieldNames.length; i++) {
+                        RareTermsAggregationBuilder aggregationBuilder = new RareTermsAggregationBuilder("_name", valueTypes[i])
+                            .field(fieldNames[i]);
+                        Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType1, fieldType2);
+                        aggregator.preCollection();
+                        indexSearcher.search(new MatchAllDocsQuery(), aggregator);
+                        aggregator.postCollection();
+                        RareTerms result = (RareTerms) aggregator.buildAggregation(0L);
+                        assertEquals("_name", result.getName());
+                        assertEquals(0, result.getBuckets().size());
+                    }
+                }
+            }
+        }
+    }
+
+    public void testNestedTerms() throws IOException {
+        Query query = new MatchAllDocsQuery();
+
+        testBothCases(query, dataset, aggregation -> {
+                TermsAggregationBuilder terms = new TermsAggregationBuilder("the_terms", ValueType.STRING).field(KEYWORD_FIELD);
+                aggregation.field(LONG_FIELD).maxDocCount(1).subAggregation(terms);
+            },
+            agg -> {
+                assertEquals(1, agg.getBuckets().size());
+                LongRareTerms.Bucket bucket = (LongRareTerms.Bucket) agg.getBuckets().get(0);
+                assertThat(bucket.getKey(), equalTo(1L));
+                assertThat(bucket.getDocCount(), equalTo(1L));
+
+                Aggregations children = bucket.getAggregations();
+                assertThat(children.asList().size(), equalTo(1));
+                assertThat(children.asList().get(0).getName(), equalTo("the_terms"));
+                assertThat(((Terms)(children.asList().get(0))).getBuckets().size(), equalTo(1));
+                assertThat(((Terms)(children.asList().get(0))).getBuckets().get(0).getKeyAsString(), equalTo("1"));
+            }, ValueType.NUMERIC
+        );
+
+        testBothCases(query, dataset, aggregation -> {
+                TermsAggregationBuilder terms = new TermsAggregationBuilder("the_terms", ValueType.STRING).field(KEYWORD_FIELD);
+                aggregation.field(KEYWORD_FIELD).maxDocCount(1).subAggregation(terms);
+            },
+            agg -> {
+                assertEquals(1, agg.getBuckets().size());
+                StringRareTerms.Bucket bucket = (StringRareTerms.Bucket) agg.getBuckets().get(0);
+                assertThat(bucket.getKey(), equalTo("1"));
+                assertThat(bucket.getDocCount(), equalTo(1L));
+
+                Aggregations children = bucket.getAggregations();
+                assertThat(children.asList().size(), equalTo(1));
+                assertThat(children.asList().get(0).getName(), equalTo("the_terms"));
+                assertThat(((Terms)(children.asList().get(0))).getBuckets().size(), equalTo(1));
+                assertThat(((Terms)(children.asList().get(0))).getBuckets().get(0).getKeyAsString(), equalTo("1"));
+            }, ValueType.STRING
+        );
+    }
+
+    public void testGlobalAggregationWithScore() throws IOException {
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                Document document = new Document();
+                document.add(new SortedDocValuesField("keyword", new BytesRef("a")));
+                indexWriter.addDocument(document);
+                document = new Document();
+                document.add(new SortedDocValuesField("keyword", new BytesRef("c")));
+                indexWriter.addDocument(document);
+                document = new Document();
+                document.add(new SortedDocValuesField("keyword", new BytesRef("e")));
+                indexWriter.addDocument(document);
+                try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
+                    IndexSearcher indexSearcher = newIndexSearcher(indexReader);
+                    Aggregator.SubAggCollectionMode collectionMode = randomFrom(Aggregator.SubAggCollectionMode.values());
+                    GlobalAggregationBuilder globalBuilder = new GlobalAggregationBuilder("global")
+                        .subAggregation(
+                            new RareTermsAggregationBuilder("terms", ValueType.STRING)
+                                .field("keyword")
+                                .subAggregation(
+                                    new RareTermsAggregationBuilder("sub_terms", ValueType.STRING)
+                                        .field("keyword")
+                                        .subAggregation(
+                                            new TopHitsAggregationBuilder("top_hits")
+                                                .storedField("_none_")
+                                        )
+                                )
+                        );
+
+                    MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
+                    fieldType.setName("keyword");
+                    fieldType.setHasDocValues(true);
+
+                    InternalGlobal result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), globalBuilder, fieldType);
+                    InternalMultiBucketAggregation<?, ?> terms = result.getAggregations().get("terms");
+                    assertThat(terms.getBuckets().size(), equalTo(3));
+                    for (MultiBucketsAggregation.Bucket bucket : terms.getBuckets()) {
+                        InternalMultiBucketAggregation<?, ?> subTerms = bucket.getAggregations().get("sub_terms");
+                        assertThat(subTerms.getBuckets().size(), equalTo(1));
+                        MultiBucketsAggregation.Bucket subBucket  = subTerms.getBuckets().get(0);
+                        InternalTopHits topHits = subBucket.getAggregations().get("top_hits");
+                        assertThat(topHits.getHits().getHits().length, equalTo(1));
+                        for (SearchHit hit : topHits.getHits()) {
+                            assertThat(hit.getScore(), greaterThan(0f));
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    public void testWithNestedAggregations() throws IOException {
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                for (int i = 0; i < 10; i++) {
+                    int[] nestedValues = new int[i];
+                    for (int j = 0; j < i; j++) {
+                        nestedValues[j] = j;
+                    }
+                    indexWriter.addDocuments(generateDocsWithNested(Integer.toString(i), i, nestedValues));
+                }
+                indexWriter.commit();
+
+                NestedAggregationBuilder nested = new NestedAggregationBuilder("nested", "nested_object")
+                    .subAggregation(new RareTermsAggregationBuilder("terms", ValueType.LONG)
+                        .field("nested_value")
+                        .maxDocCount(1)
+                    );
+                MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+                fieldType.setHasDocValues(true);
+                fieldType.setName("nested_value");
+                try (IndexReader indexReader = wrap(DirectoryReader.open(directory))) {
+                    InternalNested result = searchAndReduce(newIndexSearcher(indexReader),
+                        // match root document only
+                        new DocValuesFieldExistsQuery(PRIMARY_TERM_NAME), nested, fieldType);
+                    InternalMultiBucketAggregation<?, ?> terms = result.getAggregations().get("terms");
+                    assertThat(terms.getBuckets().size(), equalTo(1));
+                    assertThat(terms.getBuckets().get(0).getKeyAsString(), equalTo("8"));
+                }
+
+            }
+        }
+    }
+
+    public void testWithNestedScoringAggregations() throws IOException {
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                for (int i = 0; i < 10; i++) {
+                    int[] nestedValues = new int[i];
+                    for (int j = 0; j < i; j++) {
+                        nestedValues[j] = j;
+                    }
+                    indexWriter.addDocuments(generateDocsWithNested(Integer.toString(i), i, nestedValues));
+                }
+                indexWriter.commit();
+                for (boolean withScore : new boolean[]{true, false}) {
+                    NestedAggregationBuilder nested = new NestedAggregationBuilder("nested", "nested_object")
+                        .subAggregation(new RareTermsAggregationBuilder("terms", ValueType.LONG)
+                            .field("nested_value")
+                            .maxDocCount(2)
+                            .subAggregation(
+                                new TopHitsAggregationBuilder("top_hits")
+                                    .sort(withScore ? new ScoreSortBuilder() : new FieldSortBuilder("_doc"))
+                                    .storedField("_none_")
+                            )
+                        );
+                    MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+                    fieldType.setHasDocValues(true);
+                    fieldType.setName("nested_value");
+                    try (IndexReader indexReader = wrap(DirectoryReader.open(directory))) {
+
+                        if (withScore) {
+
+                            IllegalStateException e = expectThrows(IllegalStateException.class,
+                                () -> searchAndReduce(newIndexSearcher(indexReader),
+                                // match root document only
+                                new DocValuesFieldExistsQuery(PRIMARY_TERM_NAME), nested, fieldType));
+                            assertThat(e.getMessage(), equalTo("RareTerms agg [terms] is the child of the nested agg [nested], " +
+                                "and also has a scoring child agg [top_hits].  This combination is not supported because it requires " +
+                                "executing in [depth_first] mode, which the RareTerms agg cannot do."));
+                        } else {
+                            InternalNested result = searchAndReduce(newIndexSearcher(indexReader),
+                                // match root document only
+                                new DocValuesFieldExistsQuery(PRIMARY_TERM_NAME), nested, fieldType);
+                            InternalMultiBucketAggregation<?, ?> terms = result.getAggregations().get("terms");
+                            assertThat(terms.getBuckets().size(), equalTo(2));
+                            long counter = 1;
+                            for (MultiBucketsAggregation.Bucket bucket : terms.getBuckets()) {
+                                InternalTopHits topHits = bucket.getAggregations().get("top_hits");
+                                TotalHits hits = topHits.getHits().getTotalHits();
+                                assertNotNull(hits);
+                                assertThat(hits.value, equalTo(counter));
+                                assertThat(topHits.getHits().getMaxScore(), equalTo(Float.NaN));
+                                counter += 1;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    private final SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID();
+    private List<Document> generateDocsWithNested(String id, int value, int[] nestedValues) {
+        List<Document> documents = new ArrayList<>();
+
+        for (int nestedValue : nestedValues) {
+            Document document = new Document();
+            document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.NESTED_FIELD_TYPE));
+            document.add(new Field(TypeFieldMapper.NAME, "__nested_object", TypeFieldMapper.Defaults.FIELD_TYPE));
+            document.add(new SortedNumericDocValuesField("nested_value", nestedValue));
+            documents.add(document);
+        }
+
+        Document document = new Document();
+        document.add(new Field(IdFieldMapper.NAME, Uid.encodeId(id), IdFieldMapper.Defaults.FIELD_TYPE));
+        document.add(new Field(TypeFieldMapper.NAME, "docs", TypeFieldMapper.Defaults.FIELD_TYPE));
+        document.add(new SortedNumericDocValuesField("value", value));
+        document.add(sequenceIDFields.primaryTerm);
+        documents.add(document);
+
+        return documents;
+    }
+
+
+    private InternalAggregation buildInternalAggregation(RareTermsAggregationBuilder builder, MappedFieldType fieldType,
+                                                         IndexSearcher searcher) throws IOException {
+        AbstractRareTermsAggregator aggregator = createAggregator(builder, searcher, fieldType);
+        aggregator.preCollection();
+        searcher.search(new MatchAllDocsQuery(), aggregator);
+        aggregator.postCollection();
+        return aggregator.buildAggregation(0L);
+    }
+
+    private void testSearchCase(Query query, List<Long> dataset,
+                                Consumer<RareTermsAggregationBuilder> configure,
+                                Consumer<InternalMappedRareTerms> verify, ValueType valueType) throws IOException {
+        executeTestCase(false, query, dataset, configure, verify, valueType);
+    }
+
+    private void testSearchAndReduceCase(Query query, List<Long> dataset,
+                                         Consumer<RareTermsAggregationBuilder> configure,
+                                         Consumer<InternalMappedRareTerms> verify, ValueType valueType) throws IOException {
+        executeTestCase(true, query, dataset, configure, verify, valueType);
+    }
+
+    private void testBothCases(Query query, List<Long> dataset,
+                               Consumer<RareTermsAggregationBuilder> configure,
+                               Consumer<InternalMappedRareTerms> verify, ValueType valueType) throws IOException {
+        testSearchCase(query, dataset, configure, verify, valueType);
+        testSearchAndReduceCase(query, dataset, configure, verify, valueType);
+    }
+
+    @Override
+    protected IndexSettings createIndexSettings() {
+        Settings nodeSettings = Settings.builder()
+            .put("search.max_buckets", 100000).build();
+        return new IndexSettings(
+            IndexMetaData.builder("_index").settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+                .numberOfShards(1)
+                .numberOfReplicas(0)
+                .creationDate(System.currentTimeMillis())
+                .build(),
+            nodeSettings
+        );
+    }
+
+    private void executeTestCase(boolean reduced, Query query, List<Long> dataset,
+                                 Consumer<RareTermsAggregationBuilder> configure,
+                                 Consumer<InternalMappedRareTerms> verify, ValueType valueType) throws IOException {
+
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                Document document = new Document();
+                for (Long value : dataset) {
+                    if (frequently()) {
+                        indexWriter.commit();
+                    }
+
+                    document.add(new SortedNumericDocValuesField(LONG_FIELD, value));
+                    document.add(new LongPoint(LONG_FIELD, value));
+                    document.add(new SortedSetDocValuesField(KEYWORD_FIELD, new BytesRef(Long.toString(value))));
+                    indexWriter.addDocument(document);
+                    document.clear();
+                }
+            }
+
+            try (IndexReader indexReader = DirectoryReader.open(directory)) {
+                IndexSearcher indexSearcher = newIndexSearcher(indexReader);
+
+                RareTermsAggregationBuilder aggregationBuilder = new RareTermsAggregationBuilder("_name", valueType);
+                if (configure != null) {
+                    configure.accept(aggregationBuilder);
+                }
+
+                MappedFieldType keywordFieldType = new KeywordFieldMapper.KeywordFieldType();
+                keywordFieldType.setName(KEYWORD_FIELD);
+                keywordFieldType.setHasDocValues(true);
+
+                MappedFieldType longFieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+                longFieldType.setName(LONG_FIELD);
+                longFieldType.setHasDocValues(true);
+
+                InternalMappedRareTerms rareTerms;
+                if (reduced) {
+                    rareTerms = searchAndReduce(indexSearcher, query, aggregationBuilder, keywordFieldType, longFieldType);
+                } else {
+                    rareTerms = search(indexSearcher, query, aggregationBuilder, keywordFieldType, longFieldType);
+                }
+                verify.accept(rareTerms);
+            }
+        }
+    }
+
+    @Override
+    public void doAssertReducedMultiBucketConsumer(Aggregation agg, MultiBucketConsumerService.MultiBucketConsumer bucketConsumer) {
+        /*
+         * No-op.
+         *
+         * This is used in the aggregator tests to check that after a reduction, we have the correct number of buckets.
+         * This can be done during incremental reduces, and the final reduce.  Unfortunately, the number of buckets
+         * can _decrease_ during runtime as values are reduced together (e.g. 1 count on each shard, but when
+         * reduced it becomes 2 and is greater than the threshold).
+         *
+         * Because the incremental reduction test picks random subsets to reduce together, it's impossible
+         * to predict how the buckets will end up, and so this assertion will fail.
+         *
+         * If we want to put this assertion back in, we'll need this test to override the incremental reduce
+         * portion so that we can deterministically know which shards are being reduced together and which
+         * buckets we should have left after each reduction.
+         */
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java
index 11eed6f90e739..65952e7048ab0 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsIT.java
@@ -42,7 +42,6 @@
 import org.elasticsearch.search.aggregations.metrics.Stats;
 import org.elasticsearch.search.aggregations.metrics.Sum;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.hamcrest.Matchers;
 import org.junit.After;
 import org.junit.Before;
 
@@ -71,6 +70,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.startsWith;
 import static org.hamcrest.core.IsNull.notNullValue;
 
 @ESIntegTestCase.SuiteScopeTestCase
@@ -138,7 +138,8 @@ public void setupSuiteScopeCluster() throws Exception {
                             .startArray(MULTI_VALUED_FIELD_NAME)
                                 .value("val" + i)
                                 .value("val" + (i + 1))
-                            .endArray().endObject()));
+                            .endArray()
+                        .endObject()));
         }
 
         getMultiSortDocs(builders);
@@ -262,7 +263,7 @@ public void testMultiValueFieldWithPartitionedFiltering() throws Exception {
 
     private void runTestFieldWithPartitionedFiltering(String field) throws Exception {
         // Find total number of unique terms
-        SearchResponse allResponse = client().prepareSearch("idx").setTypes("type")
+        SearchResponse allResponse = client().prepareSearch("idx")
                 .addAggregation(terms("terms").field(field).size(10000).collectMode(randomFrom(SubAggCollectionMode.values())))
                 .get();
         assertSearchResponse(allResponse);
@@ -275,7 +276,7 @@ private void runTestFieldWithPartitionedFiltering(String field) throws Exception
         final int numPartitions = randomIntBetween(2, 4);
         Set<String> foundTerms = new HashSet<>();
         for (int partition = 0; partition < numPartitions; partition++) {
-            SearchResponse response = client().prepareSearch("idx").setTypes("type").addAggregation(terms("terms").field(field)
+            SearchResponse response = client().prepareSearch("idx").addAggregation(terms("terms").field(field)
                     .includeExclude(new IncludeExclude(partition, numPartitions)).collectMode(randomFrom(SubAggCollectionMode.values())))
                     .get();
             assertSearchResponse(response);
@@ -292,7 +293,7 @@ private void runTestFieldWithPartitionedFiltering(String field) throws Exception
     public void testSingleValuedFieldWithValueScript() throws Exception {
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("terms")
                                 .executionHint(randomExecutionHint())
@@ -319,7 +320,7 @@ public void testSingleValuedFieldWithValueScript() throws Exception {
     public void testMultiValuedFieldWithValueScriptNotUnique() throws Exception {
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("terms")
                                 .executionHint(randomExecutionHint())
@@ -345,7 +346,7 @@ public void testMultiValuedFieldWithValueScriptNotUnique() throws Exception {
     public void testMultiValuedScript() throws Exception {
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("terms")
                             .executionHint(randomExecutionHint())
@@ -376,7 +377,7 @@ public void testMultiValuedScript() throws Exception {
     public void testMultiValuedFieldWithValueScript() throws Exception {
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("terms")
                                 .executionHint(randomExecutionHint())
@@ -422,7 +423,7 @@ public void testScriptSingleValue() throws Exception {
 
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("terms")
                                 .collectMode(randomFrom(SubAggCollectionMode.values()))
@@ -451,7 +452,7 @@ public void testScriptSingleValueExplicitSingleValue() throws Exception {
 
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("terms")
                                 .collectMode(randomFrom(SubAggCollectionMode.values()))
@@ -477,7 +478,7 @@ public void testScriptSingleValueExplicitSingleValue() throws Exception {
     public void testScriptMultiValued() throws Exception {
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("terms")
                             .collectMode(randomFrom(SubAggCollectionMode.values()))
@@ -508,7 +509,7 @@ public void testScriptMultiValued() throws Exception {
     public void testPartiallyUnmapped() throws Exception {
         SearchResponse response = client()
                 .prepareSearch("idx", "idx_unmapped")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
                                 .collectMode(randomFrom(SubAggCollectionMode.values()))).get();
@@ -532,7 +533,7 @@ public void testStringTermsNestedIntoPerBucketAggregator() throws Exception {
         // no execution hint so that the logic that decides whether or not to use ordinals is executed
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         filter("filter", termQuery(MULTI_VALUED_FIELD_NAME, "val3")).subAggregation(
                                 terms("terms").field(MULTI_VALUED_FIELD_NAME).collectMode(randomFrom(SubAggCollectionMode.values()))))
@@ -560,7 +561,7 @@ public void testSingleValuedFieldOrderedByIllegalAgg() throws Exception {
         try {
             client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
                                 .collectMode(randomFrom(SubAggCollectionMode.values()))
@@ -574,7 +575,7 @@ public void testSingleValuedFieldOrderedByIllegalAgg() throws Exception {
                 ElasticsearchException rootCause = rootCauses[0];
                 if (rootCause instanceof AggregationExecutionException) {
                     AggregationExecutionException aggException = (AggregationExecutionException) rootCause;
-                    assertThat(aggException.getMessage(), Matchers.startsWith("Invalid aggregation order path"));
+                    assertThat(aggException.getMessage(), startsWith("Invalid aggregation order path"));
                 } else {
                     throw e;
                 }
@@ -588,7 +589,7 @@ public void testSingleValuedFieldOrderedBySingleBucketSubAggregationAsc() throws
         boolean asc = randomBoolean();
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("tags").executionHint(randomExecutionHint()).field("tag")
                                 .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.aggregation("filter", asc))
@@ -624,7 +625,7 @@ public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevels(
         boolean asc = randomBoolean();
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("tags")
                                 .executionHint(randomExecutionHint())
@@ -687,7 +688,7 @@ public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevelsS
         boolean asc = randomBoolean();
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("tags")
                                 .executionHint(randomExecutionHint())
@@ -750,7 +751,7 @@ public void testSingleValuedFieldOrderedBySubAggregationAscMultiHierarchyLevelsS
         boolean asc = randomBoolean();
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("tags")
                                 .executionHint(randomExecutionHint())
@@ -807,7 +808,7 @@ public void testSingleValuedFieldOrderedByMissingSubAggregation() throws Excepti
         for (String index : Arrays.asList("idx", "idx_unmapped")) {
             try {
                 client().prepareSearch(index)
-                        .setTypes("type")
+
                         .addAggregation(
                                 terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
                                         .collectMode(randomFrom(SubAggCollectionMode.values()))
@@ -825,7 +826,7 @@ public void testSingleValuedFieldOrderedByNonMetricsOrMultiBucketSubAggregation(
         for (String index : Arrays.asList("idx", "idx_unmapped")) {
             try {
                 client().prepareSearch(index)
-                        .setTypes("type")
+
                         .addAggregation(
                                 terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
                                         .collectMode(randomFrom(SubAggCollectionMode.values()))
@@ -847,7 +848,7 @@ public void testSingleValuedFieldOrderedByMultiValuedSubAggregationWithUnknownMe
             try {
                 SearchResponse response = client()
                         .prepareSearch(index)
-                        .setTypes("type")
+
                         .addAggregation(
                                 terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
                                         .collectMode(randomFrom(SubAggCollectionMode.values()))
@@ -866,7 +867,7 @@ public void testSingleValuedFieldOrderedByMultiValuedSubAggregationWithoutMetric
         for (String index : Arrays.asList("idx", "idx_unmapped")) {
             try {
                 client().prepareSearch(index)
-                        .setTypes("type")
+
                         .addAggregation(
                                 terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
                                         .collectMode(randomFrom(SubAggCollectionMode.values()))
@@ -886,7 +887,7 @@ public void testSingleValuedFieldOrderedByMultiValueSubAggregationAsc() throws E
         boolean asc = true;
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
                                 .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.aggregation("stats.avg", asc))
@@ -916,7 +917,7 @@ public void testSingleValuedFieldOrderedByMultiValueSubAggregationDesc() throws
         boolean asc = false;
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
                                 .collectMode(randomFrom(SubAggCollectionMode.values())).order(BucketOrder.aggregation("stats.avg", asc))
@@ -947,7 +948,7 @@ public void testSingleValuedFieldOrderedByMultiValueExtendedStatsAsc() throws Ex
         boolean asc = true;
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
                                 .collectMode(randomFrom(SubAggCollectionMode.values()))
@@ -979,7 +980,7 @@ public void testSingleValuedFieldOrderedByStatsAggAscWithTermsSubAgg() throws Ex
         boolean asc = true;
         SearchResponse response = client()
                 .prepareSearch("idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
                                 .collectMode(randomFrom(SubAggCollectionMode.values()))
@@ -1089,7 +1090,7 @@ private void assertMultiSortResponse(String[] expectedKeys, BucketOrder... order
     public void testIndexMetaField() throws Exception {
         SearchResponse response = client()
                 .prepareSearch("idx", "empty_bucket_idx")
-                .setTypes("type")
+
                 .addAggregation(
                         terms("terms").collectMode(randomFrom(SubAggCollectionMode.values())).executionHint(randomExecutionHint())
                                 .field(IndexFieldMapper.NAME)).get();
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
index 8acf78d301af5..18e633617e35e 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
@@ -21,6 +21,7 @@
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.document.LatLonDocValuesField;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
@@ -37,10 +38,12 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
+import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.util.MockPageCacheRecycler;
+import org.elasticsearch.index.mapper.GeoPointFieldMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.IpFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
@@ -77,6 +80,7 @@
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
+import org.elasticsearch.test.geo.RandomGeoGenerator;
 
 import java.io.IOException;
 import java.net.InetAddress;
@@ -825,42 +829,119 @@ public void testEmpty() throws Exception {
     public void testUnmapped() throws Exception {
         try (Directory directory = newDirectory()) {
             try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
+                    IndexSearcher indexSearcher = newIndexSearcher(indexReader);
+                    ValueType[] valueTypes = new ValueType[]{ValueType.STRING, ValueType.LONG, ValueType.DOUBLE};
+                    String[] fieldNames = new String[]{"string", "long", "double"};
+                    for (int i = 0; i < fieldNames.length; i++) {
+                        TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", valueTypes[i])
+                            .field(fieldNames[i]);
+                        Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, (MappedFieldType) null);
+                        aggregator.preCollection();
+                        indexSearcher.search(new MatchAllDocsQuery(), aggregator);
+                        aggregator.postCollection();
+                        Terms result = (Terms) aggregator.buildAggregation(0L);
+                        assertEquals("_name", result.getName());
+                        assertEquals(0, result.getBuckets().size());
+                        assertFalse(AggregationInspectionHelper.hasValue((InternalTerms)result));
+                    }
+                }
+            }
+        }
+    }
+
+    public void testUnmappedWithMissing() throws Exception {
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+
                 Document document = new Document();
-                document.add(new SortedDocValuesField("string", new BytesRef("a")));
-                document.add(new NumericDocValuesField("long", 0L));
-                document.add(new NumericDocValuesField("double", Double.doubleToRawLongBits(0L)));
+                document.add(new NumericDocValuesField("unrelated_value", 100));
                 indexWriter.addDocument(document);
-                MappedFieldType fieldType1 = new KeywordFieldMapper.KeywordFieldType();
-                fieldType1.setName("another_string");
-                fieldType1.setHasDocValues(true);
-
-                MappedFieldType fieldType2 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
-                fieldType2.setName("another_long");
-                fieldType2.setHasDocValues(true);
 
-                MappedFieldType fieldType3 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE);
-                fieldType3.setName("another_double");
-                fieldType3.setHasDocValues(true);
                 try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
+
+                    MappedFieldType fieldType1 = new KeywordFieldMapper.KeywordFieldType();
+                    fieldType1.setName("unrelated_value");
+                    fieldType1.setHasDocValues(true);
+
                     IndexSearcher indexSearcher = newIndexSearcher(indexReader);
                     ValueType[] valueTypes = new ValueType[]{ValueType.STRING, ValueType.LONG, ValueType.DOUBLE};
                     String[] fieldNames = new String[]{"string", "long", "double"};
+                    Object[] missingValues = new Object[]{"abc", 19L, 19.2};
+
+
                     for (int i = 0; i < fieldNames.length; i++) {
                         TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", valueTypes[i])
-                            .field(fieldNames[i]);
-                        Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType1, fieldType2, fieldType3);
+                            .field(fieldNames[i]).missing(missingValues[i]);
+                        Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType1);
                         aggregator.preCollection();
                         indexSearcher.search(new MatchAllDocsQuery(), aggregator);
                         aggregator.postCollection();
                         Terms result = (Terms) aggregator.buildAggregation(0L);
                         assertEquals("_name", result.getName());
-                        assertEquals(0, result.getBuckets().size());
+                        assertEquals(1, result.getBuckets().size());
+                        assertEquals(missingValues[i], result.getBuckets().get(0).getKey());
+                        assertEquals(1, result.getBuckets().get(0).getDocCount());
                     }
                 }
             }
         }
     }
 
+    public void testGeoPointField() throws Exception {
+        try (Directory directory = newDirectory()) {
+            GeoPoint point = RandomGeoGenerator.randomPoint(random());
+            final String field = "field";
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                Document document = new Document();
+                document.add(new LatLonDocValuesField(field, point.getLat(), point.getLon()));
+                indexWriter.addDocument(document);
+                try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
+                    MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType();
+                    fieldType.setHasDocValues(true);
+                    fieldType.setName("field");
+
+                    IndexSearcher indexSearcher = newIndexSearcher(indexReader);
+                    TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", null) .field(field);
+                    // Note - other places we throw IllegalArgumentException
+                    expectThrows(AggregationExecutionException.class, () -> {
+                        createAggregator(aggregationBuilder, indexSearcher, fieldType);
+                    });
+                }
+            }
+        }
+    }
+
+    public void testIpField() throws Exception {
+        try (Directory directory = newDirectory()) {
+            final String field = "field";
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                Document document = new Document();
+                document.add(new SortedSetDocValuesField("field",
+                    new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.100.42")))));
+                indexWriter.addDocument(document);
+                try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
+                    MappedFieldType fieldType = new IpFieldMapper.IpFieldType();
+                    fieldType.setHasDocValues(true);
+                    fieldType.setName("field");
+
+                    IndexSearcher indexSearcher = newIndexSearcher(indexReader);
+                    TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", null) .field(field);
+                    // Note - other places we throw IllegalArgumentException
+                    Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
+                    aggregator.preCollection();
+                    indexSearcher.search(new MatchAllDocsQuery(), aggregator);
+                    aggregator.postCollection();
+                    Terms result = (Terms) aggregator.buildAggregation(0L);
+                    assertEquals("_name", result.getName());
+                    assertEquals(1, result.getBuckets().size());
+                    assertEquals("192.168.100.42", result.getBuckets().get(0).getKey());
+                    assertEquals(1, result.getBuckets().get(0).getDocCount());
+                }
+            }
+        }
+    }
+
     public void testNestedTermsAgg() throws Exception {
         try (Directory directory = newDirectory()) {
             try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorTests.java
index 4f5a8bb1ea484..6cc2aa1a4593e 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityAggregatorTests.java
@@ -31,6 +31,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
@@ -107,8 +108,61 @@ public void testQueryFiltersAll() throws IOException {
         });
     }
 
+    public void testUnmappedMissingString() throws IOException {
+        CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name", null)
+            .field("number").missing("🍌🍌🍌");
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 8)));
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 9)));
+        }, card -> {
+            assertEquals(1, card.getValue(), 0);
+            assertTrue(AggregationInspectionHelper.hasValue(card));
+        }, null);
+    }
+
+    public void testUnmappedMissingNumber() throws IOException {
+        CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name", null)
+            .field("number").missing(1234);
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 8)));
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 9)));
+        }, card -> {
+            assertEquals(1, card.getValue(), 0);
+            assertTrue(AggregationInspectionHelper.hasValue(card));
+        }, null);
+    }
+
+    public void testUnmappedMissingGeoPoint() throws IOException {
+        CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder("name", null)
+            .field("number").missing(new GeoPoint(42.39561, -71.13051));
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 8)));
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 9)));
+        }, card -> {
+            assertEquals(1, card.getValue(), 0);
+            assertTrue(AggregationInspectionHelper.hasValue(card));
+        }, null);
+    }
+
     private void testCase(Query query, CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
-            Consumer<InternalCardinality> verify) throws IOException {
+                          Consumer<InternalCardinality> verify) throws IOException {
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(
+            NumberFieldMapper.NumberType.LONG);
+        fieldType.setName("number");
+        final CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder(
+            "_name", ValueType.NUMERIC).field("number");
+        testCase(aggregationBuilder, query, buildIndex, verify, fieldType);
+    }
+
+    private void testCase(CardinalityAggregationBuilder aggregationBuilder, Query query,
+                          CheckedConsumer<RandomIndexWriter, IOException> buildIndex, Consumer<InternalCardinality> verify,
+                          MappedFieldType fieldType) throws IOException {
         Directory directory = newDirectory();
         RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
         buildIndex.accept(indexWriter);
@@ -117,11 +171,6 @@ private void testCase(Query query, CheckedConsumer<RandomIndexWriter, IOExceptio
         IndexReader indexReader = DirectoryReader.open(directory);
         IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
 
-        CardinalityAggregationBuilder aggregationBuilder = new CardinalityAggregationBuilder(
-                "_name", ValueType.NUMERIC).field("number");
-        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(
-                NumberFieldMapper.NumberType.LONG);
-        fieldType.setName("number");
         CardinalityAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher,
             fieldType);
         aggregator.preCollection();
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java
index 759adddd9e890..174b40a154490 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java
@@ -172,7 +172,7 @@ private static String multiNumericField(boolean hash) {
     }
 
     public void testUnmapped() throws Exception {
-        SearchResponse response = client().prepareSearch("idx_unmapped").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx_unmapped")
                 .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_value"))
                 .get();
 
@@ -185,7 +185,7 @@ public void testUnmapped() throws Exception {
     }
 
     public void testPartiallyUnmapped() throws Exception {
-        SearchResponse response = client().prepareSearch("idx", "idx_unmapped").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx", "idx_unmapped")
                 .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_value"))
                 .get();
 
@@ -198,7 +198,7 @@ public void testPartiallyUnmapped() throws Exception {
     }
 
     public void testSingleValuedString() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_value"))
                 .get();
 
@@ -211,7 +211,7 @@ public void testSingleValuedString() throws Exception {
     }
 
     public void testSingleValuedNumeric() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField()))
                 .get();
 
@@ -250,7 +250,7 @@ public void testSingleValuedNumericGetProperty() throws Exception {
     }
 
     public void testSingleValuedNumericHashed() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField()))
                 .get();
 
@@ -263,7 +263,7 @@ public void testSingleValuedNumericHashed() throws Exception {
     }
 
     public void testMultiValuedString() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_values"))
                 .get();
 
@@ -276,7 +276,7 @@ public void testMultiValuedString() throws Exception {
     }
 
     public void testMultiValuedNumeric() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(multiNumericField(false)))
                 .get();
 
@@ -289,7 +289,7 @@ public void testMultiValuedNumeric() throws Exception {
     }
 
     public void testMultiValuedNumericHashed() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(multiNumericField(true)))
                 .get();
 
@@ -302,7 +302,7 @@ public void testMultiValuedNumericHashed() throws Exception {
     }
 
     public void testSingleValuedStringScript() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(
                         cardinality("cardinality")
                                 .precisionThreshold(precisionThreshold)
@@ -318,7 +318,7 @@ public void testSingleValuedStringScript() throws Exception {
     }
 
     public void testMultiValuedStringScript() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(
                         cardinality("cardinality")
                                 .precisionThreshold(precisionThreshold)
@@ -335,7 +335,7 @@ public void testMultiValuedStringScript() throws Exception {
 
     public void testSingleValuedNumericScript() throws Exception {
         Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc[' + singleNumericField() + '].value", emptyMap());
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).script(script))
                 .get();
 
@@ -350,7 +350,7 @@ public void testSingleValuedNumericScript() throws Exception {
     public void testMultiValuedNumericScript() throws Exception {
         Script script =
             new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc[' + multiNumericField(false) + ']", Collections.emptyMap());
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).script(script))
                 .get();
 
@@ -363,7 +363,7 @@ public void testMultiValuedNumericScript() throws Exception {
     }
 
     public void testSingleValuedStringValueScript() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(
                         cardinality("cardinality")
                                 .precisionThreshold(precisionThreshold)
@@ -380,7 +380,7 @@ public void testSingleValuedStringValueScript() throws Exception {
     }
 
     public void testMultiValuedStringValueScript() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(
                         cardinality("cardinality")
                                 .precisionThreshold(precisionThreshold)
@@ -397,7 +397,7 @@ public void testMultiValuedStringValueScript() throws Exception {
     }
 
     public void testSingleValuedNumericValueScript() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(
                         cardinality("cardinality")
                                 .precisionThreshold(precisionThreshold)
@@ -414,7 +414,7 @@ public void testSingleValuedNumericValueScript() throws Exception {
     }
 
     public void testMultiValuedNumericValueScript() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(
                         cardinality("cardinality")
                                 .precisionThreshold(precisionThreshold)
@@ -431,7 +431,7 @@ public void testMultiValuedNumericValueScript() throws Exception {
     }
 
     public void testAsSubAgg() throws Exception {
-        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+        SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(terms("terms").field("str_value")
                         .collectMode(randomFrom(SubAggCollectionMode.values()))
                         .subAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_values")))
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java
index d962178661272..66d2b9c6d5feb 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java
@@ -46,8 +46,15 @@
 import org.apache.lucene.util.FutureArrays;
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptEngine;
+import org.elasticsearch.script.ScriptModule;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
 
@@ -57,6 +64,7 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
@@ -65,6 +73,19 @@
 import static org.hamcrest.Matchers.equalTo;
 
 public class MaxAggregatorTests extends AggregatorTestCase {
+    private final String SCRIPT_NAME = "script_name";
+    private final long SCRIPT_VALUE = 19L;
+
+    @Override
+    protected ScriptService getMockScriptService() {
+        MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME,
+            Collections.singletonMap(SCRIPT_NAME, script -> SCRIPT_VALUE), // return 19 from script
+            Collections.emptyMap());
+        Map<String, ScriptEngine> engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine);
+
+        return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
+    }
+
     public void testNoDocs() throws IOException {
         testCase(new MatchAllDocsQuery(), iw -> {
             // Intentionally not writing any docs
@@ -124,9 +145,58 @@ public void testQueryFiltersAll() throws IOException {
         });
     }
 
+    public void testUnmappedField() throws IOException {
+        MaxAggregationBuilder aggregationBuilder = new MaxAggregationBuilder("_name").field("number");
+        testCase(aggregationBuilder, new DocValuesFieldExistsQuery("number"), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("number", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("number", 1)));
+        }, max -> {
+            assertEquals(max.getValue(), Double.NEGATIVE_INFINITY, 0);
+            assertFalse(AggregationInspectionHelper.hasValue(max));
+        }, null);
+    }
+
+    public void testUnmappedWithMissingField() throws IOException {
+        MaxAggregationBuilder aggregationBuilder = new MaxAggregationBuilder("_name").field("number").missing(19L);
+
+        testCase(aggregationBuilder, new DocValuesFieldExistsQuery("number"), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("number", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("number", 1)));
+        }, max -> {
+            assertEquals(max.getValue(), 19.0, 0);
+            assertTrue(AggregationInspectionHelper.hasValue(max));
+        }, null);
+    }
+
+    public void testScript() throws IOException {
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
+        fieldType.setName("number");
+
+        MaxAggregationBuilder aggregationBuilder = new MaxAggregationBuilder("_name")
+            .field("number")
+            .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, SCRIPT_NAME, Collections.emptyMap()));
+
+        testCase(aggregationBuilder, new DocValuesFieldExistsQuery("number"), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("number", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("number", 1)));
+        }, max -> {
+            assertEquals(max.getValue(), SCRIPT_VALUE, 0); // Note this is the script value (19L), not the doc values above
+            assertTrue(AggregationInspectionHelper.hasValue(max));
+        }, fieldType);
+    }
+
     private void testCase(Query query,
-                            CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
-                            Consumer<InternalMax> verify) throws IOException {
+                          CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
+                          Consumer<InternalMax> verify) throws IOException {
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
+        fieldType.setName("number");
+        MaxAggregationBuilder aggregationBuilder = new MaxAggregationBuilder("_name").field("number");
+        testCase(aggregationBuilder, query, buildIndex, verify, fieldType);
+    }
+
+    private void testCase(MaxAggregationBuilder aggregationBuilder, Query query,
+                          CheckedConsumer<RandomIndexWriter, IOException> buildIndex,
+                          Consumer<InternalMax> verify, MappedFieldType fieldType) throws IOException {
         Directory directory = newDirectory();
         RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
         buildIndex.accept(indexWriter);
@@ -135,10 +205,6 @@ private void testCase(Query query,
         IndexReader indexReader = DirectoryReader.open(directory);
         IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
 
-        MaxAggregationBuilder aggregationBuilder = new MaxAggregationBuilder("_name").field("number");
-        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
-        fieldType.setName("number");
-
         MaxAggregator aggregator = createAggregator(query, aggregationBuilder, indexSearcher, createIndexSettings(), fieldType);
         aggregator.preCollection();
         indexSearcher.search(query, aggregator);
@@ -254,4 +320,5 @@ public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue
         });
         assertTrue(seen[0]);
     }
+
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java
index 5f74937f6610b..54b1a33db0671 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java
@@ -26,8 +26,10 @@
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptEngine;
@@ -403,11 +405,12 @@ public void testSelfReferencingAggStateAfterCombine() throws IOException {
      * is final and cannot be mocked
      */
     @Override
-    protected QueryShardContext queryShardContextMock(MapperService mapperService) {
+    protected QueryShardContext queryShardContextMock(MapperService mapperService, IndexSettings indexSettings,
+                                                      CircuitBreakerService circuitBreakerService) {
         MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, SCRIPTS, Collections.emptyMap());
         Map<String, ScriptEngine> engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine);
         ScriptService scriptService =  new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
-        return new QueryShardContext(0, mapperService.getIndexSettings(), null, null, mapperService, null, scriptService,
+        return new QueryShardContext(0, indexSettings, null, null, null, mapperService, null, scriptService,
                 xContentRegistry(), writableRegistry(), null, null, System::currentTimeMillis, null);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java
index ea14d9ec671b2..fd3b930e407d5 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java
@@ -33,6 +33,7 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.index.mapper.BooleanFieldMapper;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.GeoPointFieldMapper;
@@ -119,11 +120,66 @@ public void testQueryFiltersAll() throws IOException {
         });
     }
 
+    public void testUnmappedMissingString() throws IOException {
+        ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name", null)
+            .field("number").missing("🍌🍌🍌");
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 8)));
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 9)));
+        }, card -> {
+            assertEquals(3, card.getValue(), 0);
+            assertTrue(AggregationInspectionHelper.hasValue(card));
+        }, null);
+    }
+
+    public void testUnmappedMissingNumber() throws IOException {
+        ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name", null)
+            .field("number").missing(1234);
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 8)));
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 9)));
+        }, card -> {
+            assertEquals(3, card.getValue(), 0);
+            assertTrue(AggregationInspectionHelper.hasValue(card));
+        }, null);
+    }
+
+    public void testUnmappedMissingGeoPoint() throws IOException {
+        ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("name", null)
+            .field("number").missing(new GeoPoint(42.39561, -71.13051));
+
+        testCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 8)));
+            iw.addDocument(singleton(new NumericDocValuesField("unrelatedField", 9)));
+        }, card -> {
+            assertEquals(3, card.getValue(), 0);
+            assertTrue(AggregationInspectionHelper.hasValue(card));
+        }, null);
+    }
+
     private void testCase(Query query,
                           ValueType valueType,
                           CheckedConsumer<RandomIndexWriter, IOException> indexer,
                           Consumer<InternalValueCount> verify) throws IOException {
+        MappedFieldType fieldType = createMappedFieldType(valueType);
+        fieldType.setName(FIELD_NAME);
+        fieldType.setHasDocValues(true);
+
+        ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("_name", valueType);
+        aggregationBuilder.field(FIELD_NAME);
 
+        testCase(aggregationBuilder, query, indexer, verify, fieldType);
+
+    }
+
+    private void testCase(ValueCountAggregationBuilder aggregationBuilder, Query query,
+                          CheckedConsumer<RandomIndexWriter, IOException> indexer,
+                          Consumer<InternalValueCount> verify, MappedFieldType fieldType) throws IOException {
         try (Directory directory = newDirectory()) {
             try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
                 indexer.accept(indexWriter);
@@ -132,13 +188,6 @@ private void testCase(Query query,
             try (IndexReader indexReader = DirectoryReader.open(directory)) {
                 IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
 
-                MappedFieldType fieldType = createMappedFieldType(valueType);
-                fieldType.setName(FIELD_NAME);
-                fieldType.setHasDocValues(true);
-
-                ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("_name", valueType);
-                aggregationBuilder.field(FIELD_NAME);
-
                 ValueCountAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType);
                 aggregator.preCollection();
                 indexSearcher.search(query, aggregator);
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregationBuilderTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregationBuilderTests.java
index 0b7c5cd0f86e0..5bc4eb7be9e37 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregationBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/weighted_avg/WeightedAvgAggregationBuilderTests.java
@@ -45,7 +45,7 @@ public void setupName() {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AbstractBucketMetricsTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AbstractBucketMetricsTestCase.java
index 8fdc23a025b95..8f268cecaedd2 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AbstractBucketMetricsTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/AbstractBucketMetricsTestCase.java
@@ -21,7 +21,6 @@
 
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
-import org.elasticsearch.search.aggregations.pipeline.BucketMetricsPipelineAggregationBuilder;
 
 public abstract class AbstractBucketMetricsTestCase<PAF extends BucketMetricsPipelineAggregationBuilder<PAF>>
         extends BasePipelineAggregationTestCase<PAF> {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java
index fd8c0d58caf9c..01ffdee417318 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java
@@ -172,7 +172,8 @@ static AggregatorFactory getRandomSequentiallyOrderedParentAgg() throws IOExcept
             case 2:
             default:
                 AutoDateHistogramAggregationBuilder.RoundingInfo[] roundings = new AutoDateHistogramAggregationBuilder.RoundingInfo[1];
-                factory = new AutoDateHistogramAggregatorFactory("name", mock(ValuesSourceConfig.class), 1, roundings,
+                factory = new AutoDateHistogramAggregatorFactory("name", mock(ValuesSourceConfig.class),
+                    1, roundings,
                     mock(SearchContext.class), null, new AggregatorFactories.Builder(), Collections.emptyMap());
         }
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffIT.java
index 9d5c790628fc6..cf771a9727003 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffIT.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffIT.java
@@ -228,7 +228,7 @@ private void setupExpected(MetricTarget target) {
 
     public void testBasicDiff() {
         SearchResponse response = client()
-                .prepareSearch("idx").setTypes("type")
+                .prepareSearch("idx")
                 .addAggregation(
                         histogram("histo").field(INTERVAL_FIELD).interval(interval)
                                 .extendedBounds(0L, (long) (interval * (numBuckets - 1)))
@@ -275,7 +275,7 @@ public void testBasicDiff() {
     public void testInvalidLagSize() {
         try {
             client()
-                .prepareSearch("idx").setTypes("type")
+                    .prepareSearch("idx")
                 .addAggregation(
                         histogram("histo").field(INTERVAL_FIELD).interval(interval)
                                 .extendedBounds(0L, (long) (interval * (numBuckets - 1)))
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java
index 2ac97a4408dcb..8d15ae2c01673 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java
@@ -20,11 +20,10 @@
 package org.elasticsearch.search.aggregations.support;
 
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
-import java.util.Collections;
+
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Scorable;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.script.AggregationScript;
 import org.elasticsearch.search.aggregations.support.values.ScriptBytesValues;
 import org.elasticsearch.search.aggregations.support.values.ScriptDoubleValues;
@@ -35,6 +34,7 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -47,7 +47,7 @@ private static class FakeAggregationScript extends AggregationScript {
         int index;
 
         FakeAggregationScript(Object[][] values) {
-            super(Collections.emptyMap(), new SearchLookup(null, null, Strings.EMPTY_ARRAY) {
+            super(Collections.emptyMap(), new SearchLookup(null, null) {
 
                 @Override
                 public LeafSearchLookup getLeafSearchLookup(LeafReaderContext context) {
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceTypeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceTypeTests.java
new file mode 100644
index 0000000000000..d2f73aab3aaa3
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceTypeTests.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.support;
+
+import org.elasticsearch.common.io.stream.AbstractWriteableEnumTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ValuesSourceTypeTests extends AbstractWriteableEnumTestCase {
+
+    public ValuesSourceTypeTests() {
+        super(ValuesSourceType::fromStream);
+    }
+
+    @Override
+    public void testValidOrdinals() {
+        assertThat(ValuesSourceType.ANY.ordinal(), equalTo(0));
+        assertThat(ValuesSourceType.NUMERIC.ordinal(), equalTo(1));
+        assertThat(ValuesSourceType.BYTES.ordinal(), equalTo(2));
+        assertThat(ValuesSourceType.GEOPOINT.ordinal(), equalTo(3));
+    }
+
+    @Override
+    public void testFromString() {
+        assertThat(ValuesSourceType.fromString("any"), equalTo(ValuesSourceType.ANY));
+        assertThat(ValuesSourceType.fromString("numeric"), equalTo(ValuesSourceType.NUMERIC));
+        assertThat(ValuesSourceType.fromString("bytes"), equalTo(ValuesSourceType.BYTES));
+        assertThat(ValuesSourceType.fromString("geopoint"), equalTo(ValuesSourceType.GEOPOINT));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ValuesSourceType.fromString("does_not_exist"));
+        assertThat(e.getMessage(),
+            equalTo("No enum constant org.elasticsearch.search.aggregations.support.ValuesSourceType.DOES_NOT_EXIST"));
+        expectThrows(NullPointerException.class, () -> ValuesSourceType.fromString(null));
+    }
+
+    @Override
+    public void testReadFrom() throws IOException {
+        assertReadFromStream(0, ValuesSourceType.ANY);
+        assertReadFromStream(1, ValuesSourceType.NUMERIC);
+        assertReadFromStream(2, ValuesSourceType.BYTES);
+        assertReadFromStream(3, ValuesSourceType.GEOPOINT);
+    }
+
+    @Override
+    public void testWriteTo() throws IOException {
+        assertWriteToStream(ValuesSourceType.ANY, 0);
+        assertWriteToStream(ValuesSourceType.NUMERIC, 1);
+        assertWriteToStream(ValuesSourceType.BYTES, 2);
+        assertWriteToStream(ValuesSourceType.GEOPOINT, 3);
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java b/server/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java
index b304bf2d79779..494a8139217c7 100644
--- a/server/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java
+++ b/server/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java
@@ -25,7 +25,6 @@
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -34,7 +33,6 @@
  * This test basically verifies that search with a single shard active (cause we indexed to it) and other
  * shards possibly not active at all (cause they haven't allocated) will still work.
  */
-@TestLogging("_root:DEBUG")
 public class SearchWhileCreatingIndexIT extends ESIntegTestCase {
     public void testIndexCausesIndexCreation() throws Exception {
         searchWhileCreatingIndex(false, 1); // 1 replica in our default...
diff --git a/server/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java b/server/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java
index 891e64f52372e..3d196d7a0d98e 100644
--- a/server/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java
@@ -49,10 +49,8 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Random;
-import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -65,10 +63,8 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     @Override
-    protected Collection<Class<? extends Plugin>> getMockPlugins() {
-        Set<Class<? extends Plugin>> mocks = new HashSet<>(super.getMockPlugins());
-        mocks.remove(MockEngineFactoryPlugin.class);
-        return mocks;
+    protected boolean addMockInternalEngine() {
+        return false;
     }
 
     public void testRandomExceptions() throws IOException, InterruptedException, ExecutionException {
diff --git a/server/src/test/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java b/server/src/test/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java
index 0a4d3201f5cc3..9aa1626b8646c 100644
--- a/server/src/test/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/basic/SearchWithRandomIOExceptionsIT.java
@@ -39,6 +39,7 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.store.MockFSDirectoryFactory;
 import org.elasticsearch.test.store.MockFSIndexStore;
+
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
@@ -169,7 +170,7 @@ public void testRandomDirectoryIOExceptions() throws IOException, InterruptedExc
                 int docToQuery = between(0, numDocs - 1);
                 int expectedResults = added[docToQuery] ? 1 : 0;
                 logger.info("Searching for [test:{}]", English.intToEnglish(docToQuery));
-                SearchResponse searchResponse = client().prepareSearch().setTypes("type")
+                SearchResponse searchResponse = client().prepareSearch()
                     .setQuery(QueryBuilders.matchQuery("test", English.intToEnglish(docToQuery)))
                     .setSize(expectedResults).get();
                 logger.info("Successful shards: [{}]  numShards: [{}]", searchResponse.getSuccessfulShards(), numShards.numPrimaries);
@@ -177,7 +178,7 @@ public void testRandomDirectoryIOExceptions() throws IOException, InterruptedExc
                     assertResultsAndLogOnFailure(expectedResults, searchResponse);
                 }
                 // check match all
-                searchResponse = client().prepareSearch().setTypes("type").setQuery(QueryBuilders.matchAllQuery())
+                searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchAllQuery())
                     .setSize(numCreated + numInitialDocs).addSort("_uid", SortOrder.ASC).get();
                 logger.info("Match all Successful shards: [{}]  numShards: [{}]", searchResponse.getSuccessfulShards(),
                         numShards.numPrimaries);
@@ -202,7 +203,7 @@ public void testRandomDirectoryIOExceptions() throws IOException, InterruptedExc
                 .put(MockFSDirectoryFactory.RANDOM_IO_EXCEPTION_RATE_ON_OPEN_SETTING.getKey(), 0));
             client().admin().indices().prepareOpen("test").execute().get();
             ensureGreen();
-            SearchResponse searchResponse = client().prepareSearch().setTypes("type")
+            SearchResponse searchResponse = client().prepareSearch()
                     .setQuery(QueryBuilders.matchQuery("test", "init")).get();
             assertNoFailures(searchResponse);
             assertHitCount(searchResponse, numInitialDocs);
diff --git a/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java b/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java
index 566df804f2bc9..5142f0c9b73f5 100644
--- a/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java
@@ -58,7 +58,7 @@ public class CollapseBuilderTests extends AbstractSerializingTestCase<CollapseBu
 
     @BeforeClass
     public static void init() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
         xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
diff --git a/server/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/server/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
index f2239d80d7f62..da16eeb039c8a 100644
--- a/server/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
+++ b/server/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
@@ -129,7 +129,7 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
                 hitContext.hit().getFields().put(NAME, hitField);
             }
             TermVectorsRequest termVectorsRequest = new TermVectorsRequest(context.indexShard().shardId().getIndex().getName(),
-                    hitContext.hit().getType(), hitContext.hit().getId());
+                    hitContext.hit().getId());
             TermVectorsResponse termVector = TermVectorsService.getTermVectors(context.indexShard(), termVectorsRequest);
             try {
                 Map<String, Integer> tv = new HashMap<>();
diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhaseTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhaseTests.java
index 7790e8d6576ca..4b62e77393865 100644
--- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhaseTests.java
@@ -25,8 +25,8 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESTestCase;
@@ -173,7 +173,7 @@ public FetchSourceContext fetchSourceContext() {
 
         @Override
         public SearchLookup lookup() {
-            SearchLookup lookup = new SearchLookup(this.mapperService(), this::getForField, null);
+            SearchLookup lookup = new SearchLookup(this.mapperService(), this::getForField);
             lookup.source().setSource(source);
             return lookup;
         }
diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/CustomHighlighterSearchIT.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/CustomHighlighterSearchIT.java
index b7eac260cd852..8faff9e1050ec 100644
--- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/CustomHighlighterSearchIT.java
+++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/CustomHighlighterSearchIT.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -57,7 +56,7 @@ protected void setup() throws Exception{
     }
 
     public void testThatCustomHighlightersAreSupported() throws IOException {
-        SearchResponse searchResponse = client().prepareSearch("test").setTypes("test")
+        SearchResponse searchResponse = client().prepareSearch("test")
                 .setQuery(QueryBuilders.matchAllQuery())
                 .highlighter(new HighlightBuilder().field("name").highlighterType("test-custom"))
                 .get();
@@ -71,7 +70,7 @@ public void testThatCustomHighlighterCanBeConfiguredPerField() throws Exception
         options.put("myFieldOption", "someValue");
         highlightConfig.options(options);
 
-        SearchResponse searchResponse = client().prepareSearch("test").setTypes("test")
+        SearchResponse searchResponse = client().prepareSearch("test")
                 .setQuery(QueryBuilders.matchAllQuery())
                 .highlighter(new HighlightBuilder().field(highlightConfig))
                 .get();
@@ -84,7 +83,7 @@ public void testThatCustomHighlighterCanBeConfiguredGlobally() throws Exception
         Map<String, Object> options = new HashMap<>();
         options.put("myGlobalOption", "someValue");
 
-        SearchResponse searchResponse = client().prepareSearch("test").setTypes("test").setQuery(QueryBuilders.matchAllQuery())
+        SearchResponse searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery())
                 .highlighter(new HighlightBuilder().field("name").highlighterType("test-custom").options(options))
                 .get();
 
@@ -93,7 +92,7 @@ public void testThatCustomHighlighterCanBeConfiguredGlobally() throws Exception
     }
 
     public void testThatCustomHighlighterReceivesFieldsInOrder() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("test").setTypes("test")
+        SearchResponse searchResponse = client().prepareSearch("test")
                 .setQuery(QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()).should(QueryBuilders
                         .termQuery("name", "arbitrary")))
                 .highlighter(
diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
index 46bca911e9c94..9b26e13d0df46 100644
--- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
@@ -84,7 +84,7 @@ public class HighlightBuilderTests extends ESTestCase {
      */
     @BeforeClass
     public static void init() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
         xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
@@ -277,8 +277,8 @@ public void testBuildSearchContextHighlight() throws IOException {
         Index index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_");
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings);
         // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter
-        QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(),
-            namedWriteableRegistry, null, null, System::currentTimeMillis, null) {
+        QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, null, null, null,
+            xContentRegistry(), namedWriteableRegistry, null, null, System::currentTimeMillis, null) {
             @Override
             public MappedFieldType fieldMapper(String name) {
                 TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name);
diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
index 3c21085fc905d..855eb7286010c 100644
--- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
+++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.search.fetch.subphase.highlight;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.MockTokenizer;
@@ -81,7 +80,6 @@
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.boostingQuery;
-import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery;
 import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
 import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
 import static org.elasticsearch.index.query.QueryBuilders.fuzzyQuery;
@@ -1440,41 +1438,6 @@ public void testBoostingQueryTermVector() throws IOException {
         assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick <x>brown</x> fox jumps over the lazy dog"));
     }
 
-    public void testCommonTermsQuery() {
-        createIndex("test");
-        ensureGreen();
-
-        client().prepareIndex("test", "type1")
-                .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog")
-                .get();
-        refresh();
-
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource()
-            .query(commonTermsQuery("field2", "quick brown").cutoffFrequency(100))
-            .highlighter(highlight().field("field2").order("score").preTags("<x>").postTags("</x>"));
-
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog"));
-    }
-
-    public void testCommonTermsTermVector() throws IOException {
-        assertAcked(prepareCreate("test").addMapping("type1", type1TermVectorMapping()));
-        ensureGreen();
-
-        client().prepareIndex("test", "type1").setSource(
-                "field1", "this is a test",
-                "field2", "The quick brown fox jumps over the lazy dog").get();
-        refresh();
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource().query(commonTermsQuery("field2", "quick brown").cutoffFrequency(100))
-                .highlighter(highlight().field("field2").order("score").preTags("<x>").postTags("</x>"));
-
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
-
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog"));
-    }
-
     public void testPlainHighlightDifferentFragmenter() throws Exception {
         assertAcked(prepareCreate("test")
                 .addMapping("type1", "tags", "type=text"));
@@ -2295,24 +2258,6 @@ public void testPostingsHighlighterBoostingQuery() throws IOException {
             equalTo("The quick <x>brown</x> fox jumps over the lazy dog! Second sentence."));
     }
 
-    public void testPostingsHighlighterCommonTermsQuery() throws IOException {
-        assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping()));
-        ensureGreen();
-
-        client().prepareIndex("test", "type1")
-            .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.").get();
-        refresh();
-
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource().query(commonTermsQuery("field2", "quick brown").cutoffFrequency(100))
-            .highlighter(highlight().field("field2").preTags("<x>").postTags("</x>"));
-        SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
-        assertHitCount(searchResponse, 1L);
-
-        assertHighlight(searchResponse, 0, "field2", 0, 1,
-            equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog! Second sentence."));
-    }
-
     private static XContentBuilder type1PostingsffsetsMapping() throws IOException {
         return XContentFactory.jsonBuilder().startObject().startObject("type1")
                 .startObject("properties")
@@ -2536,7 +2481,7 @@ public void testDoesNotHighlightTypeName() throws Exception {
         indexRandom(true, client().prepareIndex("test", "typename").setSource("foo", "test typename"));
 
         for (String highlighter : ALL_TYPES) {
-            SearchResponse response = client().prepareSearch("test").setTypes("typename").setQuery(matchQuery("foo", "test"))
+            SearchResponse response = client().prepareSearch("test").setQuery(matchQuery("foo", "test"))
                     .highlighter(new HighlightBuilder().field("foo").highlighterType(highlighter).requireFieldMatch(false)).get();
             assertHighlight(response, 0, "foo", 0, 1, equalTo("<em>test</em> typename"));
         }
@@ -2555,7 +2500,7 @@ public void testDoesNotHighlightAliasFilters() throws Exception {
         indexRandom(true, client().prepareIndex("test", "typename").setSource("foo", "test japanese"));
 
         for (String highlighter : ALL_TYPES) {
-            SearchResponse response = client().prepareSearch("filtered_alias").setTypes("typename").setQuery(matchQuery("foo", "test"))
+            SearchResponse response = client().prepareSearch("filtered_alias").setQuery(matchQuery("foo", "test"))
                     .highlighter(new HighlightBuilder().field("foo").highlighterType(highlighter).requireFieldMatch(false)).get();
             assertHighlight(response, 0, "foo", 0, 1, equalTo("<em>test</em> japanese"));
         }
diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
index 356fafbbb4de2..b401c28f5b622 100644
--- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
+++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
@@ -654,7 +654,6 @@ public void testSearchFieldsMetaData() throws Exception {
                 .get();
 
         SearchResponse searchResponse = client().prepareSearch("my-index")
-                .setTypes("my-type1")
                 .addStoredField("field1").addStoredField("_routing")
                 .get();
 
@@ -670,7 +669,7 @@ public void testSearchFieldsNonLeafField() throws Exception {
                 .setRefreshPolicy(IMMEDIATE)
                 .get();
 
-        assertFailures(client().prepareSearch("my-index").setTypes("my-type1").addStoredField("field1"),
+        assertFailures(client().prepareSearch("my-index").addStoredField("field1"),
                 RestStatus.BAD_REQUEST,
                 containsString("field [field1] isn't a leaf field"));
     }
@@ -748,7 +747,8 @@ public void testSingleValueFieldDatatField() throws ExecutionException, Interrup
                 .addMapping("type", "test_field", "type=keyword").get());
         indexRandom(true, client().prepareIndex("test", "type", "1").setSource("test_field", "foobar"));
         refresh();
-        SearchResponse searchResponse = client().prepareSearch("test").setTypes("type").setSource(
+        SearchResponse searchResponse = client().prepareSearch("test")
+                .setSource(
                 new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).docValueField("test_field")).get();
         assertHitCount(searchResponse, 1);
         Map<String, DocumentField> fields = searchResponse.getHits().getHits()[0].getFields();
diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java b/server/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java
index a8ef8c10116c4..f6803488d9a59 100644
--- a/server/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java
+++ b/server/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java
@@ -185,7 +185,6 @@ public void testGeoDistanceAggregation() throws IOException {
         String name = "TestPosition";
 
         search.setQuery(QueryBuilders.matchAllQuery())
-            .setTypes("type1")
             .addAggregation(AggregationBuilders.geoDistance(name, new GeoPoint(tgt_lat, tgt_lon))
             .field("location")
             .unit(DistanceUnit.MILES)
diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
index ef6bea10d749d..f9f2d8525fb4c 100644
--- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
@@ -123,7 +123,7 @@ public void testIndexPointsFilterRectangle() throws Exception {
 
         EnvelopeBuilder shape = new EnvelopeBuilder(new Coordinate(-45, 45), new Coordinate(45, -45));
 
-        SearchResponse searchResponse = client().prepareSearch("test").setTypes("type1")
+        SearchResponse searchResponse = client().prepareSearch("test")
                 .setQuery(geoIntersectionQuery("location", shape))
                 .get();
 
@@ -132,7 +132,7 @@ public void testIndexPointsFilterRectangle() throws Exception {
         assertThat(searchResponse.getHits().getHits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
 
-        searchResponse = client().prepareSearch("test").setTypes("type1")
+        searchResponse = client().prepareSearch("test")
                 .setQuery(geoShapeQuery("location", shape))
                 .get();
 
@@ -168,7 +168,7 @@ public void testEdgeCases() throws Exception {
 
         // This search would fail if both geoshape indexing and geoshape filtering
         // used the bottom-level optimization in SpatialPrefixTree#recursiveGetNodes.
-        SearchResponse searchResponse = client().prepareSearch("test").setTypes("type1")
+        SearchResponse searchResponse = client().prepareSearch("test")
                 .setQuery(geoIntersectionQuery("location", query))
                 .get();
 
@@ -626,7 +626,7 @@ public void testPointsOnly() throws Exception {
         }
 
         // test that point was inserted
-        SearchResponse response = client().prepareSearch("geo_points_only").setTypes("type1")
+        SearchResponse response = client().prepareSearch("geo_points_only")
                 .setQuery(geoIntersectionQuery("location", shape))
                 .get();
 
@@ -660,7 +660,7 @@ public void testPointsOnlyExplicit() throws Exception {
             .setRefreshPolicy(IMMEDIATE).get();
 
         // test that point was inserted
-        SearchResponse response = client().prepareSearch("geo_points_only").setTypes("type1")
+        SearchResponse response = client().prepareSearch("geo_points_only")
             .setQuery(matchAllQuery())
             .get();
 
diff --git a/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
new file mode 100644
index 0000000000000..ed2f972562c9c
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/search/internal/ContextIndexSearcherTests.java
@@ -0,0 +1,425 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.internal;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FilterDirectoryReader;
+import org.apache.lucene.index.FilterLeafReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.BulkScorer;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.BitSet;
+import org.apache.lucene.util.BitSetIterator;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.CombinedBitSet;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.SparseFixedBitSet;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.Set;
+
+import static org.elasticsearch.search.internal.ContextIndexSearcher.intersectScorerAndBitSet;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class ContextIndexSearcherTests extends ESTestCase {
+    public void testIntersectScorerAndRoleBits() throws Exception {
+        final Directory directory = newDirectory();
+        IndexWriter iw = new IndexWriter(
+            directory,
+            new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(NoMergePolicy.INSTANCE)
+        );
+
+        Document document = new Document();
+        document.add(new StringField("field1", "value1", Field.Store.NO));
+        document.add(new StringField("field2", "value1", Field.Store.NO));
+        iw.addDocument(document);
+
+        document = new Document();
+        document.add(new StringField("field1", "value2", Field.Store.NO));
+        document.add(new StringField("field2", "value1", Field.Store.NO));
+        iw.addDocument(document);
+
+        document = new Document();
+        document.add(new StringField("field1", "value3", Field.Store.NO));
+        document.add(new StringField("field2", "value1", Field.Store.NO));
+        iw.addDocument(document);
+
+        document = new Document();
+        document.add(new StringField("field1", "value4", Field.Store.NO));
+        document.add(new StringField("field2", "value1", Field.Store.NO));
+        iw.addDocument(document);
+
+        iw.commit();
+        iw.deleteDocuments(new Term("field1", "value3"));
+        iw.close();
+        DirectoryReader directoryReader = DirectoryReader.open(directory);
+        IndexSearcher searcher = new IndexSearcher(directoryReader);
+        Weight weight = searcher.createWeight(new TermQuery(new Term("field2", "value1")),
+            org.apache.lucene.search.ScoreMode.COMPLETE_NO_SCORES, 1f);
+
+        LeafReaderContext leaf = directoryReader.leaves().get(0);
+
+        CombinedBitSet bitSet = new CombinedBitSet(query(leaf, "field1", "value1"), leaf.reader().getLiveDocs());
+        LeafCollector leafCollector = new LeafBucketCollector() {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                assertThat(doc, equalTo(0));
+            }
+        };
+        intersectScorerAndBitSet(weight.scorer(leaf), bitSet, leafCollector, () -> {});
+
+        bitSet = new CombinedBitSet(query(leaf, "field1", "value2"), leaf.reader().getLiveDocs());
+        leafCollector = new LeafBucketCollector() {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                assertThat(doc, equalTo(1));
+            }
+        };
+        intersectScorerAndBitSet(weight.scorer(leaf), bitSet, leafCollector,  () -> {});
+
+
+        bitSet = new CombinedBitSet(query(leaf, "field1", "value3"), leaf.reader().getLiveDocs());
+        leafCollector = new LeafBucketCollector() {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                fail("docId [" + doc + "] should have been deleted");
+            }
+        };
+        intersectScorerAndBitSet(weight.scorer(leaf), bitSet, leafCollector,  () -> {});
+
+        bitSet = new CombinedBitSet(query(leaf, "field1", "value4"), leaf.reader().getLiveDocs());
+        leafCollector = new LeafBucketCollector() {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                assertThat(doc, equalTo(3));
+            }
+        };
+        intersectScorerAndBitSet(weight.scorer(leaf), bitSet, leafCollector,  () -> {});
+
+        directoryReader.close();
+        directory.close();
+    }
+
+    public void testContextIndexSearcherSparseNoDeletions() throws IOException {
+        doTestContextIndexSearcher(true, false);
+    }
+
+    public void testContextIndexSearcherDenseNoDeletions() throws IOException {
+        doTestContextIndexSearcher(false, false);
+    }
+
+    public void testContextIndexSearcherSparseWithDeletions() throws IOException {
+        doTestContextIndexSearcher(true, true);
+    }
+
+    public void testContextIndexSearcherDenseWithDeletions() throws IOException {
+        doTestContextIndexSearcher(false, true);
+    }
+
+    public void doTestContextIndexSearcher(boolean sparse, boolean deletions) throws IOException {
+        Directory dir = newDirectory();
+        IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(null));
+        Document doc = new Document();
+        StringField allowedField = new StringField("allowed", "yes", Field.Store.NO);
+        doc.add(allowedField);
+        StringField fooField = new StringField("foo", "bar", Field.Store.NO);
+        doc.add(fooField);
+        StringField deleteField = new StringField("delete", "no", Field.Store.NO);
+        doc.add(deleteField);
+        w.addDocument(doc);
+        if (deletions) {
+            // add a document that matches foo:bar but will be deleted
+            deleteField.setStringValue("yes");
+            w.addDocument(doc);
+            deleteField.setStringValue("no");
+        }
+        allowedField.setStringValue("no");
+        w.addDocument(doc);
+        if (sparse) {
+            for (int i = 0; i < 1000; ++i) {
+                w.addDocument(doc);
+            }
+            w.forceMerge(1);
+        }
+        w.deleteDocuments(new Term("delete", "yes"));
+
+        IndexSettings settings = IndexSettingsModule.newIndexSettings("_index", Settings.EMPTY);
+        BitsetFilterCache.Listener listener = new BitsetFilterCache.Listener() {
+            @Override
+            public void onCache(ShardId shardId, Accountable accountable) {
+
+            }
+
+            @Override
+            public void onRemoval(ShardId shardId, Accountable accountable) {
+
+            }
+        };
+        DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(w),
+            new ShardId(settings.getIndex(), 0));
+        BitsetFilterCache cache = new BitsetFilterCache(settings, listener);
+        Query roleQuery = new TermQuery(new Term("allowed", "yes"));
+        BitSet bitSet = cache.getBitSetProducer(roleQuery).getBitSet(reader.leaves().get(0));
+        if (sparse) {
+            assertThat(bitSet, instanceOf(SparseFixedBitSet.class));
+        } else {
+            assertThat(bitSet, instanceOf(FixedBitSet.class));
+        }
+
+        DocumentSubsetDirectoryReader filteredReader = new DocumentSubsetDirectoryReader(reader, cache, roleQuery);
+
+        ContextIndexSearcher searcher = new ContextIndexSearcher(filteredReader, IndexSearcher.getDefaultSimilarity(),
+            IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
+        searcher.setCheckCancelled(() -> {});
+
+        // Searching a non-existing term will trigger a null scorer
+        assertEquals(0, searcher.count(new TermQuery(new Term("non_existing_field", "non_existing_value"))));
+
+        assertEquals(1, searcher.count(new TermQuery(new Term("foo", "bar"))));
+
+        // make sure scorers are created only once, see #1725
+        assertEquals(1, searcher.count(new CreateScorerOnceQuery(new MatchAllDocsQuery())));
+        IOUtils.close(reader, w, dir);
+    }
+
+    private SparseFixedBitSet query(LeafReaderContext leaf, String field, String value) throws IOException {
+        SparseFixedBitSet sparseFixedBitSet = new SparseFixedBitSet(leaf.reader().maxDoc());
+        TermsEnum tenum = leaf.reader().terms(field).iterator();
+        while (tenum.next().utf8ToString().equals(value) == false) {
+        }
+        PostingsEnum penum = tenum.postings(null);
+        sparseFixedBitSet.or(penum);
+        return sparseFixedBitSet;
+    }
+
+    private static class DocumentSubsetDirectoryReader extends FilterDirectoryReader {
+        private final BitsetFilterCache bitsetFilterCache;
+        private final Query roleQuery;
+
+        DocumentSubsetDirectoryReader(DirectoryReader in, BitsetFilterCache bitsetFilterCache, Query roleQuery) throws IOException {
+            super(in, new SubReaderWrapper() {
+                @Override
+                public LeafReader wrap(LeafReader reader) {
+                    try {
+                        return new DocumentSubsetReader(reader, bitsetFilterCache, roleQuery);
+                    } catch (Exception e) {
+                        throw ExceptionsHelper.convertToElastic(e);
+                    }
+                }
+            });
+            this.bitsetFilterCache = bitsetFilterCache;
+            this.roleQuery = roleQuery;
+        }
+
+        @Override
+        protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
+            return new DocumentSubsetDirectoryReader(in, bitsetFilterCache, roleQuery);
+        }
+
+
+        @Override
+        public CacheHelper getReaderCacheHelper() {
+            return in.getReaderCacheHelper();
+        }
+    }
+
+    private static class DocumentSubsetReader extends FilterLeafReader {
+        private final BitSet roleQueryBits;
+        private final int numDocs;
+
+        /**
+         * <p>Construct a FilterLeafReader based on the specified base reader.
+         * <p>Note that base reader is closed if this FilterLeafReader is closed.</p>
+         *
+         * @param in specified base reader.
+         */
+        DocumentSubsetReader(LeafReader in, BitsetFilterCache bitsetFilterCache, Query roleQuery) throws IOException {
+            super(in);
+            this.roleQueryBits = bitsetFilterCache.getBitSetProducer(roleQuery).getBitSet(in.getContext());
+            this.numDocs = computeNumDocs(in, roleQueryBits);
+        }
+
+        @Override
+        public CacheHelper getCoreCacheHelper() {
+            return in.getCoreCacheHelper();
+        }
+
+        @Override
+        public CacheHelper getReaderCacheHelper() {
+            // Not delegated since we change the live docs
+            return null;
+        }
+
+        @Override
+        public int numDocs() {
+            return numDocs;
+        }
+
+        @Override
+        public Bits getLiveDocs() {
+            final Bits actualLiveDocs = in.getLiveDocs();
+            if (roleQueryBits == null) {
+                return new Bits.MatchNoBits(in.maxDoc());
+            } else if (actualLiveDocs == null) {
+                return roleQueryBits;
+            } else {
+                // apply deletes when needed:
+                return new CombinedBitSet(roleQueryBits, actualLiveDocs);
+            }
+        }
+
+        private static int computeNumDocs(LeafReader reader, BitSet roleQueryBits) {
+            final Bits liveDocs = reader.getLiveDocs();
+            if (roleQueryBits == null) {
+                return 0;
+            } else if (liveDocs == null) {
+                // slow
+                return roleQueryBits.cardinality();
+            } else {
+                // very slow, but necessary in order to be correct
+                int numDocs = 0;
+                DocIdSetIterator it = new BitSetIterator(roleQueryBits, 0L); // we don't use the cost
+                try {
+                    for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
+                        if (liveDocs.get(doc)) {
+                            numDocs++;
+                        }
+                    }
+                    return numDocs;
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            }
+        }
+    }
+
+    private static class CreateScorerOnceWeight extends Weight {
+
+        private final Weight weight;
+        private final Set<Object> seenLeaves = Collections.newSetFromMap(new IdentityHashMap<>());
+
+        CreateScorerOnceWeight(Weight weight) {
+            super(weight.getQuery());
+            this.weight = weight;
+        }
+
+        @Override
+        public void extractTerms(Set<Term> terms) {
+            weight.extractTerms(terms);
+        }
+
+        @Override
+        public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+            return weight.explain(context, doc);
+        }
+
+        @Override
+        public Scorer scorer(LeafReaderContext context) throws IOException {
+            assertTrue(seenLeaves.add(context.reader().getCoreCacheHelper().getKey()));
+            return weight.scorer(context);
+        }
+
+        @Override
+        public BulkScorer bulkScorer(LeafReaderContext context)
+            throws IOException {
+            assertTrue(seenLeaves.add(context.reader().getCoreCacheHelper().getKey()));
+            return weight.bulkScorer(context);
+        }
+
+        @Override
+        public boolean isCacheable(LeafReaderContext ctx) {
+            return true;
+        }
+    }
+
+    private static class CreateScorerOnceQuery extends Query {
+
+        private final Query query;
+
+        CreateScorerOnceQuery(Query query) {
+            this.query = query;
+        }
+
+        @Override
+        public String toString(String field) {
+            return query.toString(field);
+        }
+
+        @Override
+        public Query rewrite(IndexReader reader) throws IOException {
+            Query queryRewritten = query.rewrite(reader);
+            if (query != queryRewritten) {
+                return new CreateScorerOnceQuery(queryRewritten);
+            }
+            return super.rewrite(reader);
+        }
+
+        @Override
+        public Weight createWeight(IndexSearcher searcher, org.apache.lucene.search.ScoreMode scoreMode, float boost) throws IOException {
+            return new CreateScorerOnceWeight(query.createWeight(searcher, scoreMode, boost));
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            return sameClassAs(obj) && query.equals(((CreateScorerOnceQuery) obj).query);
+        }
+
+        @Override
+        public int hashCode() {
+            return 31 * classHash() + query.hashCode();
+        }
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java b/server/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java
index fa08a12485776..f4211ed32efe0 100644
--- a/server/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java
+++ b/server/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java
@@ -61,7 +61,6 @@ public void testSerialization() throws Exception {
         assertEquals(deserializedRequest.scroll(), shardSearchTransportRequest.scroll());
         assertEquals(deserializedRequest.getAliasFilter(), shardSearchTransportRequest.getAliasFilter());
         assertArrayEquals(deserializedRequest.indices(), shardSearchTransportRequest.indices());
-        assertArrayEquals(deserializedRequest.types(), shardSearchTransportRequest.types());
         assertEquals(deserializedRequest.indicesOptions(), shardSearchTransportRequest.indicesOptions());
         assertEquals(deserializedRequest.nowInMillis(), shardSearchTransportRequest.nowInMillis());
         assertEquals(deserializedRequest.source(), shardSearchTransportRequest.source());
diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java
index fca61bf2564b9..07ae5b40dce9c 100644
--- a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java
+++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
-import static org.elasticsearch.search.lookup.LeafDocLookup.TYPES_DEPRECATION_MESSAGE;
 import static org.mockito.AdditionalAnswers.returnsFirstArg;
 import static org.mockito.Matchers.anyObject;
 import static org.mockito.Mockito.doReturn;
@@ -46,22 +45,14 @@ public void setUp() throws Exception {
         when(fieldType.valueForDisplay(anyObject())).then(returnsFirstArg());
 
         MapperService mapperService = mock(MapperService.class);
-        when(mapperService.fullName("_type")).thenReturn(fieldType);
         when(mapperService.fullName("field")).thenReturn(fieldType);
         when(mapperService.fullName("alias")).thenReturn(fieldType);
 
         docValues = mock(ScriptDocValues.class);
-
-        AtomicFieldData atomicFieldData = mock(AtomicFieldData.class);
-        doReturn(docValues).when(atomicFieldData).getScriptValues();
-
-        IndexFieldData<?> fieldData = mock(IndexFieldData.class);
-        when(fieldData.getFieldName()).thenReturn("field");
-        doReturn(atomicFieldData).when(fieldData).load(anyObject());
+        IndexFieldData<?> fieldData = createFieldData(docValues);
 
         docLookup = new LeafDocLookup(mapperService,
             ignored -> fieldData,
-            new String[] { "type" },
             null);
     }
 
@@ -70,14 +61,19 @@ public void testBasicLookup() {
         assertEquals(docValues, fetchedDocValues);
     }
 
-    public void testLookupWithFieldAlias() {
+    public void testFieldAliases() {
         ScriptDocValues<?> fetchedDocValues = docLookup.get("alias");
         assertEquals(docValues, fetchedDocValues);
     }
 
-    public void testTypesDeprecation() {
-        ScriptDocValues<?> fetchedDocValues = docLookup.get("_type");
-        assertEquals(docValues, fetchedDocValues);
-        assertWarnings(TYPES_DEPRECATION_MESSAGE);
+    private IndexFieldData<?> createFieldData(ScriptDocValues scriptDocValues) {
+        AtomicFieldData atomicFieldData = mock(AtomicFieldData.class);
+        doReturn(scriptDocValues).when(atomicFieldData).getScriptValues();
+
+        IndexFieldData<?> fieldData = mock(IndexFieldData.class);
+        when(fieldData.getFieldName()).thenReturn("field");
+        doReturn(atomicFieldData).when(fieldData).load(anyObject());
+
+        return fieldData;
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafFieldsLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafFieldsLookupTests.java
index 72bd6d1fe2c87..6683adb9185f2 100644
--- a/server/src/test/java/org/elasticsearch/search/lookup/LeafFieldsLookupTests.java
+++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafFieldsLookupTests.java
@@ -66,9 +66,7 @@ public void setUp() throws Exception {
             return null;
         }).when(leafReader).document(anyInt(), any(StoredFieldVisitor.class));
 
-        fieldsLookup = new LeafFieldsLookup(mapperService,
-            new String[] { "type" },
-            leafReader);
+        fieldsLookup = new LeafFieldsLookup(mapperService, leafReader);
     }
 
     public void testBasicLookup() {
diff --git a/server/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java b/server/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
index 4492353f6f15b..3270dcc6212fc 100644
--- a/server/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
+++ b/server/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
@@ -112,7 +112,7 @@ public void testSimpleMoreLikeThisWithTypes() throws Exception {
 
         logger.info("Running moreLikeThis");
         SearchResponse response = client().prepareSearch().setQuery(
-            new MoreLikeThisQueryBuilder(null, new Item[] {new Item("test", "type1", "1")}).minTermFreq(1).minDocFreq(1)).get();
+            new MoreLikeThisQueryBuilder(null, new Item[] {new Item("test", "1")}).minTermFreq(1).minDocFreq(1)).get();
         assertHitCount(response, 1L);
     }
 
@@ -243,7 +243,7 @@ public void testMoreLikeThisWithAliasesInLikeDocuments() throws Exception {
         refresh(indexName);
 
         SearchResponse response = client().prepareSearch().setQuery(
-                new MoreLikeThisQueryBuilder(null, new Item[] {new Item(aliasName, typeName, "1")}).minTermFreq(1).minDocFreq(1)).get();
+                new MoreLikeThisQueryBuilder(null, new Item[] {new Item(aliasName, "1")}).minTermFreq(1).minDocFreq(1)).get();
         assertHitCount(response, 2L);
         assertThat(response.getHits().getAt(0).getId(), equalTo("3"));
     }
@@ -261,11 +261,11 @@ public void testMoreLikeThisIssue2197() throws Exception {
         assertThat(ensureGreen(), equalTo(ClusterHealthStatus.GREEN));
 
         SearchResponse response = client().prepareSearch().setQuery(
-                new MoreLikeThisQueryBuilder(null, new Item[] {new Item("foo", "bar", "1")})).get();
+                new MoreLikeThisQueryBuilder(null, new Item[] {new Item("foo", "1")})).get();
         assertNoFailures(response);
         assertThat(response, notNullValue());
         response = client().prepareSearch().setQuery(
-                new MoreLikeThisQueryBuilder(null, new Item[] {new Item("foo", "bar", "1")})).get();
+                new MoreLikeThisQueryBuilder(null, new Item[] {new Item("foo", "1")})).get();
         assertNoFailures(response);
         assertThat(response, notNullValue());
     }
@@ -286,7 +286,7 @@ public void testMoreLikeWithCustomRouting() throws Exception {
         client().admin().indices().prepareRefresh("foo").get();
 
         SearchResponse response = client().prepareSearch().setQuery(
-                new MoreLikeThisQueryBuilder(null, new Item[] {new Item("foo", "bar", "1").routing("2")})).get();
+                new MoreLikeThisQueryBuilder(null, new Item[] {new Item("foo", "1").routing("2")})).get();
         assertNoFailures(response);
         assertThat(response, notNullValue());
     }
@@ -308,7 +308,7 @@ public void testMoreLikeThisIssueRoutingNotSerialized() throws Exception {
                 .get();
         client().admin().indices().prepareRefresh("foo").get();
         SearchResponse response = client().prepareSearch().setQuery(
-                new MoreLikeThisQueryBuilder(null, new Item[] {new Item("foo", "bar", "1").routing("4000")})).get();
+                new MoreLikeThisQueryBuilder(null, new Item[] {new Item("foo", "1").routing("4000")})).get();
         assertNoFailures(response);
         assertThat(response, notNullValue());
     }
@@ -410,7 +410,7 @@ public void testMoreLikeThisWithFieldAlias() throws Exception {
         index("test", "_doc", "2", "text", "lucene release");
         refresh();
 
-        Item item = new Item("test", "_doc", "1");
+        Item item = new Item("test", "1");
         QueryBuilder query = QueryBuilders.moreLikeThisQuery(new String[] {"alias"}, null, new Item[] {item})
             .minTermFreq(1)
             .minDocFreq(1);
@@ -446,7 +446,7 @@ public void testSimpleMoreLikeInclude() throws Exception {
         assertOrderedSearchHits(response, "1", "2");
 
         response = client().prepareSearch().setQuery(
-                new MoreLikeThisQueryBuilder(null, new Item[] {new Item("test", "type1", "2")}).minTermFreq(1).minDocFreq(1).include(true)
+                new MoreLikeThisQueryBuilder(null, new Item[] {new Item("test", "2")}).minTermFreq(1).minDocFreq(1).include(true)
                 .minimumShouldMatch("0%")).get();
         assertOrderedSearchHits(response, "2", "1");
 
@@ -475,10 +475,10 @@ public void testSimpleMoreLikeThisIds() throws Exception {
         indexRandom(true, builders);
 
         logger.info("Running MoreLikeThis");
-        Item[] items = new Item[] { new Item(null, null, "1")};
+        Item[] items = new Item[] { new Item(null, "1")};
         MoreLikeThisQueryBuilder queryBuilder = QueryBuilders.moreLikeThisQuery(new String[] {"text"}, null, items).include(true)
                 .minTermFreq(1).minDocFreq(1);
-        SearchResponse mltResponse = client().prepareSearch().setTypes("type1").setQuery(queryBuilder).get();
+        SearchResponse mltResponse = client().prepareSearch().setQuery(queryBuilder).get();
         assertHitCount(mltResponse, 3L);
     }
 
@@ -504,10 +504,10 @@ public void testMoreLikeThisMultiValueFields() throws Exception {
         for (int i = 0; i < maxIters; i++) {
             int max_query_terms = randomIntBetween(1, values.length);
             logger.info("Running More Like This with max_query_terms = {}", max_query_terms);
-            MoreLikeThisQueryBuilder mltQuery = moreLikeThisQuery(new String[] {"text"}, null, new Item[] {new Item(null, null, "0")})
+            MoreLikeThisQueryBuilder mltQuery = moreLikeThisQuery(new String[] {"text"}, null, new Item[] {new Item(null, "0")})
                     .minTermFreq(1).minDocFreq(1)
                     .maxQueryTerms(max_query_terms).minimumShouldMatch("0%");
-            SearchResponse response = client().prepareSearch("test").setTypes("type1")
+            SearchResponse response = client().prepareSearch("test")
                     .setQuery(mltQuery).get();
             assertSearchResponse(response);
             assertHitCount(response, max_query_terms);
@@ -540,7 +540,7 @@ public void testMinimumShouldMatch() throws ExecutionException, InterruptedExcep
                     .minDocFreq(1)
                     .minimumShouldMatch(minimumShouldMatch);
             logger.info("Testing with minimum_should_match = {}", minimumShouldMatch);
-            SearchResponse response = client().prepareSearch("test").setTypes("type1")
+            SearchResponse response = client().prepareSearch("test")
                     .setQuery(mltQuery).get();
             assertSearchResponse(response);
             if (minimumShouldMatch.equals("0%")) {
@@ -567,12 +567,12 @@ public void testMoreLikeThisArtificialDocs() throws Exception {
 
         logger.info("Checking the document matches ...");
         // routing to ensure we hit the shard with the doc
-        MoreLikeThisQueryBuilder mltQuery = moreLikeThisQuery(new Item[] {new Item("test", "type1", doc).routing("0")})
+        MoreLikeThisQueryBuilder mltQuery = moreLikeThisQuery(new Item[] {new Item("test", doc).routing("0")})
                 .minTermFreq(0)
                 .minDocFreq(0)
                 .maxQueryTerms(100)
                 .minimumShouldMatch("100%"); // strict all terms must match!
-        SearchResponse response = client().prepareSearch("test").setTypes("type1")
+        SearchResponse response = client().prepareSearch("test")
                 .setQuery(mltQuery).get();
         assertSearchResponse(response);
         assertHitCount(response, 1);
@@ -597,22 +597,22 @@ public void testMoreLikeThisMalformedArtificialDocs() throws Exception {
                 .field("text", "Hello World!")
                 .field("date", "this is not a date!")
                 .endObject();
-        MoreLikeThisQueryBuilder mltQuery = moreLikeThisQuery(new Item[] {new Item("test", "type1", malformedFieldDoc)})
+        MoreLikeThisQueryBuilder mltQuery = moreLikeThisQuery(new Item[] {new Item("test", malformedFieldDoc)})
                 .minTermFreq(0)
                 .minDocFreq(0)
                 .minimumShouldMatch("0%");
-        SearchResponse response = client().prepareSearch("test").setTypes("type1")
+        SearchResponse response = client().prepareSearch("test")
                 .setQuery(mltQuery).get();
         assertSearchResponse(response);
         assertHitCount(response, 0);
 
         logger.info("Checking with an empty document ...");
         XContentBuilder emptyDoc = jsonBuilder().startObject().endObject();
-        mltQuery = moreLikeThisQuery(null, new Item[] {new Item("test", "type1", emptyDoc)})
+        mltQuery = moreLikeThisQuery(null, new Item[] {new Item("test", emptyDoc)})
                 .minTermFreq(0)
                 .minDocFreq(0)
                 .minimumShouldMatch("0%");
-        response = client().prepareSearch("test").setTypes("type1")
+        response = client().prepareSearch("test")
                 .setQuery(mltQuery).get();
         assertSearchResponse(response);
         assertHitCount(response, 0);
@@ -623,11 +623,11 @@ public void testMoreLikeThisMalformedArtificialDocs() throws Exception {
                 .field("text", "Hello World!")
                 .field("date", "1000-01-01") // should be properly parsed but ignored ...
                 .endObject();
-        mltQuery = moreLikeThisQuery(null, new Item[] {new Item("test", "type1", normalDoc)})
+        mltQuery = moreLikeThisQuery(null, new Item[] {new Item("test", normalDoc)})
                 .minTermFreq(0)
                 .minDocFreq(0)
                 .minimumShouldMatch("100%");  // strict all terms must match but date is ignored
-        response = client().prepareSearch("test").setTypes("type1")
+        response = client().prepareSearch("test")
                 .setQuery(mltQuery).get();
         assertSearchResponse(response);
         assertHitCount(response, 1);
@@ -653,12 +653,12 @@ public void testMoreLikeThisUnlike() throws ExecutionException, InterruptedExcep
         indexRandom(true, builders);
 
         logger.info("First check the document matches all indexed docs.");
-        MoreLikeThisQueryBuilder mltQuery = moreLikeThisQuery(new Item[] {new Item("test", "type1", doc)})
+        MoreLikeThisQueryBuilder mltQuery = moreLikeThisQuery(new Item[] {new Item("test", doc)})
                 .minTermFreq(0)
                 .minDocFreq(0)
                 .maxQueryTerms(100)
                 .minimumShouldMatch("0%");
-        SearchResponse response = client().prepareSearch("test").setTypes("type1")
+        SearchResponse response = client().prepareSearch("test")
                 .setQuery(mltQuery).get();
         assertSearchResponse(response);
         assertHitCount(response, numFields);
@@ -666,8 +666,8 @@ public void testMoreLikeThisUnlike() throws ExecutionException, InterruptedExcep
         logger.info("Now check like this doc, but ignore one doc in the index, then two and so on...");
         List<Item> docs = new ArrayList<>(numFields);
         for (int i = 0; i < numFields; i++) {
-            docs.add(new Item("test", "type1", i+""));
-            mltQuery = moreLikeThisQuery(null, new Item[] {new Item("test", "type1", doc)})
+            docs.add(new Item("test", i+""));
+            mltQuery = moreLikeThisQuery(null, new Item[] {new Item("test", doc)})
                     .unlike(docs.toArray(new Item[docs.size()]))
                     .minTermFreq(0)
                     .minDocFreq(0)
@@ -675,7 +675,7 @@ public void testMoreLikeThisUnlike() throws ExecutionException, InterruptedExcep
                     .include(true)
                     .minimumShouldMatch("0%");
 
-            response = client().prepareSearch("test").setTypes("type1").setQuery(mltQuery).get();
+            response = client().prepareSearch("test").setQuery(mltQuery).get();
             assertSearchResponse(response);
             assertHitCount(response, numFields - (i + 1));
         }
@@ -702,7 +702,7 @@ public void testSelectFields() throws IOException, ExecutionException, Interrupt
                 .minDocFreq(0)
                 .include(true)
                 .minimumShouldMatch("1%");
-        SearchResponse response = client().prepareSearch("test").setTypes("type1")
+        SearchResponse response = client().prepareSearch("test")
                 .setQuery(mltQuery).get();
         assertSearchResponse(response);
         assertHitCount(response, 2);
@@ -712,7 +712,7 @@ public void testSelectFields() throws IOException, ExecutionException, Interrupt
                 .minDocFreq(0)
                 .include(true)
                 .minimumShouldMatch("1%");
-        response = client().prepareSearch("test").setTypes("type1")
+        response = client().prepareSearch("test")
                 .setQuery(mltQuery).get();
         assertSearchResponse(response);
         assertHitCount(response, 1);
@@ -724,7 +724,7 @@ public void testWithRouting() throws IOException {
         client().prepareIndex("index", "type", "3").setRouting("4").setSource("text", "this is yet another document").get();
         refresh("index");
 
-        Item item = new Item("index", "type", "2").routing("1");
+        Item item = new Item("index", "2").routing("1");
         MoreLikeThisQueryBuilder moreLikeThisQueryBuilder = new MoreLikeThisQueryBuilder(new String[]{"text"}, null, new Item[]{item});
         moreLikeThisQueryBuilder.minTermFreq(1);
         moreLikeThisQueryBuilder.minDocFreq(1);
@@ -761,13 +761,13 @@ public void testWithMissingRouting() throws IOException {
             SearchPhaseExecutionException exception = expectThrows(SearchPhaseExecutionException.class, () ->
             client().prepareSearch().setQuery(new MoreLikeThisQueryBuilder(null, new Item[]{
                 new Item("test", "1").routing("1"),
-                new Item("test", "type1", "2"),
-                new Item("test", "type1", "3")
+                new Item("test", "2"),
+                new Item("test", "3")
             }).minTermFreq(1).minDocFreq(1)).get());
 
             Throwable cause = exception.getCause();
             assertThat(cause, instanceOf(RoutingMissingException.class));
-            assertThat(cause.getMessage(), equalTo("routing is required for [test]/[type1]/[2]"));
+            assertThat(cause.getMessage(), equalTo("routing is required for [test]/[_doc]/[2]"));
         }
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java b/server/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
index 5feb341fd6943..d80fa2c1e4cca 100644
--- a/server/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
+++ b/server/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
@@ -383,9 +383,10 @@ public void testSimpleNestedSorting() throws Exception {
         refresh();
 
         SearchResponse searchResponse = client().prepareSearch("test")
-                .setTypes("type1")
+
                 .setQuery(QueryBuilders.matchAllQuery())
-                .addSort(SortBuilders.fieldSort("nested1.field1").order(SortOrder.ASC).setNestedPath("nested1"))
+                .addSort(SortBuilders.fieldSort("nested1.field1").order(SortOrder.ASC)
+                    .setNestedSort(new NestedSortBuilder("nested1")))
                 .get();
 
         assertHitCount(searchResponse, 3);
@@ -397,9 +398,10 @@ public void testSimpleNestedSorting() throws Exception {
         assertThat(searchResponse.getHits().getHits()[2].getSortValues()[0].toString(), equalTo("4"));
 
         searchResponse = client().prepareSearch("test")
-                .setTypes("type1")
+
                 .setQuery(QueryBuilders.matchAllQuery())
-                .addSort(SortBuilders.fieldSort("nested1.field1").order(SortOrder.DESC).setNestedPath("nested1"))
+                .addSort(SortBuilders.fieldSort("nested1.field1").order(SortOrder.DESC)
+                    .setNestedSort(new NestedSortBuilder("nested1")))
                 .get();
 
         assertHitCount(searchResponse, 3);
@@ -474,10 +476,12 @@ public void testSimpleNestedSortingWithNestedFilterMissing() throws Exception {
                 .endObject()).get();
         refresh();
 
-        SearchRequestBuilder searchRequestBuilder = client().prepareSearch("test").setTypes("type1")
+        SearchRequestBuilder searchRequestBuilder = client().prepareSearch("test")
                 .setQuery(QueryBuilders.matchAllQuery())
-                .addSort(SortBuilders.fieldSort("nested1.field1").setNestedPath("nested1")
-                        .setNestedFilter(termQuery("nested1.field2", true)).missing(10).order(SortOrder.ASC));
+                .addSort(SortBuilders.fieldSort("nested1.field1")
+                    .setNestedSort(new NestedSortBuilder("nested1")
+                        .setFilter(termQuery("nested1.field2", true)))
+                    .missing(10).order(SortOrder.ASC));
 
         if (randomBoolean()) {
             searchRequestBuilder.setScroll("10m");
@@ -493,9 +497,11 @@ public void testSimpleNestedSortingWithNestedFilterMissing() throws Exception {
         assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("3"));
         assertThat(searchResponse.getHits().getHits()[2].getSortValues()[0].toString(), equalTo("10"));
 
-        searchRequestBuilder = client().prepareSearch("test").setTypes("type1").setQuery(QueryBuilders.matchAllQuery())
-                .addSort(SortBuilders.fieldSort("nested1.field1").setNestedPath("nested1")
-                        .setNestedFilter(termQuery("nested1.field2", true)).missing(10).order(SortOrder.DESC));
+        searchRequestBuilder = client().prepareSearch("test").setQuery(QueryBuilders.matchAllQuery())
+                .addSort(SortBuilders.fieldSort("nested1.field1")
+                    .setNestedSort(new NestedSortBuilder("nested1")
+                        .setFilter(termQuery("nested1.field2", true)))
+                    .missing(10).order(SortOrder.DESC));
 
         if (randomBoolean()) {
             searchRequestBuilder.setScroll("10m");
@@ -953,7 +959,7 @@ public void testSortNestedWithNestedFilter() throws Exception {
                 .setQuery(matchAllQuery())
                 .addSort(
                         SortBuilders.fieldSort("parent.child.child_values")
-                                .setNestedPath("parent.child")
+                                .setNestedSort(new NestedSortBuilder("parent.child"))
                                 .order(SortOrder.ASC)
                 )
                 .get();
@@ -967,12 +973,13 @@ public void testSortNestedWithNestedFilter() throws Exception {
         assertThat(searchResponse.getHits().getHits()[2].getSortValues()[0].toString(), equalTo("-1"));
 
         // With nested filter
+        NestedSortBuilder nestedSort = new NestedSortBuilder("parent.child");
+        nestedSort.setFilter(QueryBuilders.termQuery("parent.child.filter", true));
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
                 .addSort(
                         SortBuilders.fieldSort("parent.child.child_values")
-                                .setNestedPath("parent.child")
-                                .setNestedFilter(QueryBuilders.termQuery("parent.child.filter", true))
+                                .setNestedSort(nestedSort)
                                 .order(SortOrder.ASC)
                 )
                 .get();
@@ -990,8 +997,7 @@ public void testSortNestedWithNestedFilter() throws Exception {
                 .setQuery(matchAllQuery())
                 .addSort(
                         SortBuilders.fieldSort("parent.child.child_values")
-                                .setNestedPath("parent.child")
-                                .setNestedFilter(QueryBuilders.termQuery("parent.child.filter", true))
+                                .setNestedSort(nestedSort)
                                 .order(SortOrder.ASC)
                 )
                 .get();
@@ -1005,12 +1011,12 @@ public void testSortNestedWithNestedFilter() throws Exception {
         assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("3"));
         assertThat(searchResponse.getHits().getHits()[2].getSortValues()[0].toString(), equalTo("3"));
 
+        nestedSort.setFilter(QueryBuilders.termQuery("parent.filter", false));
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
                 .addSort(
                         SortBuilders.fieldSort("parent.parent_values")
-                                .setNestedPath("parent.child")
-                                .setNestedFilter(QueryBuilders.termQuery("parent.filter", false))
+                                .setNestedSort(nestedSort)
                                 .order(SortOrder.ASC)
                 )
                 .get();
@@ -1051,8 +1057,8 @@ public void testSortNestedWithNestedFilter() throws Exception {
                 .setQuery(matchAllQuery())
                 .addSort(
                         SortBuilders.fieldSort("parent.child.child_obj.value")
-                                .setNestedPath("parent.child")
-                                .setNestedFilter(QueryBuilders.termQuery("parent.child.filter", true))
+                                .setNestedSort(new NestedSortBuilder("parent.child")
+                                    .setFilter(QueryBuilders.termQuery("parent.child.filter", true)))
                                 .order(SortOrder.ASC)
                 )
                 .get();
@@ -1071,7 +1077,7 @@ public void testSortNestedWithNestedFilter() throws Exception {
                 .setQuery(matchAllQuery())
                 .addSort(
                         SortBuilders.fieldSort("parent.child.child_values")
-                                .setNestedPath("parent.child")
+                                .setNestedSort(new NestedSortBuilder("parent.child"))
                                 .sortMode(SortMode.SUM)
                                 .order(SortOrder.ASC)
                 )
@@ -1091,7 +1097,7 @@ public void testSortNestedWithNestedFilter() throws Exception {
                 .setQuery(matchAllQuery())
                 .addSort(
                         SortBuilders.fieldSort("parent.child.child_values")
-                                .setNestedPath("parent.child")
+                                .setNestedSort(new NestedSortBuilder("parent.child"))
                                 .sortMode(SortMode.SUM)
                                 .order(SortOrder.DESC)
                 )
@@ -1111,8 +1117,9 @@ public void testSortNestedWithNestedFilter() throws Exception {
                 .setQuery(matchAllQuery())
                 .addSort(
                         SortBuilders.fieldSort("parent.child.child_values")
-                                .setNestedPath("parent.child")
-                                .setNestedFilter(QueryBuilders.termQuery("parent.child.filter", true))
+                                .setNestedSort(new NestedSortBuilder("parent.child")
+                                    .setFilter(QueryBuilders.termQuery("parent.child.filter", true))
+                                )
                                 .sortMode(SortMode.SUM)
                                 .order(SortOrder.ASC)
                 )
@@ -1132,7 +1139,7 @@ public void testSortNestedWithNestedFilter() throws Exception {
                 .setQuery(matchAllQuery())
                 .addSort(
                         SortBuilders.fieldSort("parent.child.child_values")
-                                .setNestedPath("parent.child")
+                                .setNestedSort(new NestedSortBuilder("parent.child"))
                                 .sortMode(SortMode.AVG)
                                 .order(SortOrder.ASC)
                 )
@@ -1151,7 +1158,7 @@ public void testSortNestedWithNestedFilter() throws Exception {
                 .setQuery(matchAllQuery())
                 .addSort(
                         SortBuilders.fieldSort("parent.child.child_values")
-                                .setNestedPath("parent.child")
+                                .setNestedSort(new NestedSortBuilder("parent.child"))
                                 .sortMode(SortMode.AVG)
                                 .order(SortOrder.DESC)
                 )
@@ -1171,8 +1178,9 @@ public void testSortNestedWithNestedFilter() throws Exception {
                 .setQuery(matchAllQuery())
                 .addSort(
                         SortBuilders.fieldSort("parent.child.child_values")
-                                .setNestedPath("parent.child")
-                                .setNestedFilter(QueryBuilders.termQuery("parent.child.filter", true))
+                                .setNestedSort(new NestedSortBuilder("parent.child")
+                                    .setFilter(QueryBuilders.termQuery("parent.child.filter", true))
+                                )
                                 .sortMode(SortMode.AVG)
                                 .order(SortOrder.ASC)
                 )
@@ -1307,13 +1315,15 @@ public void testNestedSortingWithNestedFilterAsFilter() throws Exception {
 
         SearchResponse searchResponse = client().prepareSearch("test")
                 .addSort(SortBuilders.fieldSort("users.first")
-                        .setNestedPath("users")
+                        .setNestedSort(new NestedSortBuilder("users"))
                         .order(SortOrder.ASC))
                 .addSort(SortBuilders.fieldSort("users.first")
                         .order(SortOrder.ASC)
-                        .setNestedPath("users")
-                        .setNestedFilter(nestedQuery("users.workstations", termQuery("users.workstations.stationid", "s5"),
-                                ScoreMode.Avg)))
+                        .setNestedSort(new NestedSortBuilder("users")
+                            .setFilter(nestedQuery("users.workstations", termQuery("users.workstations.stationid", "s5"),
+                                    ScoreMode.Avg))
+                        )
+                )
                 .get();
         assertNoFailures(searchResponse);
         assertHitCount(searchResponse, 2);
diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
index 040e16b6e957f..6b851729332e0 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
@@ -571,7 +571,6 @@ public void testPhrase() throws Exception {
         SearchResponse resp = client().prepareSearch()
                 .setQuery(q)
                 .setIndices("test")
-                .setTypes("type1")
                 .setProfile(true)
                 .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .get();
diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
index 275349e15cdf1..b29d3ba3b7dd4 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
@@ -44,7 +44,6 @@
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.apache.lucene.util.TestUtil;
-import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.search.profile.ProfileResult;
 import org.elasticsearch.test.ESTestCase;
@@ -81,8 +80,8 @@ public static void setup() throws IOException {
         }
         reader = w.getReader();
         w.close();
-        Engine.Searcher engineSearcher = new Engine.Searcher("test", new IndexSearcher(reader), null);
-        searcher = new ContextIndexSearcher(engineSearcher, IndexSearcher.getDefaultQueryCache(), MAYBE_CACHE_POLICY);
+        searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
+            IndexSearcher.getDefaultQueryCache(), MAYBE_CACHE_POLICY);
     }
 
     @AfterClass
@@ -159,10 +158,10 @@ public void testUseIndexStats() throws IOException {
 
     public void testApproximations() throws IOException {
         QueryProfiler profiler = new QueryProfiler();
-        Engine.Searcher engineSearcher = new Engine.Searcher("test", new IndexSearcher(reader), reader::close);
         // disable query caching since we want to test approximations, which won't
         // be exposed on a cached entry
-        ContextIndexSearcher searcher = new ContextIndexSearcher(engineSearcher, null, MAYBE_CACHE_POLICY);
+        ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
+            null, MAYBE_CACHE_POLICY);
         searcher.setProfiler(profiler);
         Query query = new RandomApproximationQuery(new TermQuery(new Term("foo", "bar")), random());
         searcher.count(query);
diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java b/server/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java
index 00b859394c65f..ea9ef964153b7 100644
--- a/server/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java
+++ b/server/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java
@@ -22,11 +22,9 @@
 import org.apache.lucene.util.English;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.CommonTermsQueryBuilder;
 import org.elasticsearch.index.query.DisMaxQueryBuilder;
 import org.elasticsearch.index.query.FuzzyQueryBuilder;
 import org.elasticsearch.index.query.IdsQueryBuilder;
-import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.RangeQueryBuilder;
@@ -72,7 +70,7 @@ public static QueryBuilder randomQueryBuilder(List<String> stringFields, List<St
     }
 
     private static QueryBuilder randomTerminalQuery(List<String> stringFields, List<String> numericFields, int numDocs) {
-        switch (randomIntBetween(0,6)) {
+        switch (randomIntBetween(0,5)) {
             case 0:
                 return randomTermQuery(stringFields, numDocs);
             case 1:
@@ -82,10 +80,8 @@ private static QueryBuilder randomTerminalQuery(List<String> stringFields, List<
             case 3:
                 return QueryBuilders.matchAllQuery();
             case 4:
-                return randomCommonTermsQuery(stringFields, numDocs);
-            case 5:
                 return randomFuzzyQuery(stringFields);
-            case 6:
+            case 5:
                 return randomIDsQuery();
             default:
                 return randomTermQuery(stringFields, numDocs);
@@ -169,31 +165,6 @@ private static QueryBuilder randomConstantScoreQuery(List<String> stringFields,
         return QueryBuilders.constantScoreQuery(randomQueryBuilder(stringFields, numericFields, numDocs, depth - 1));
     }
 
-    private static QueryBuilder randomCommonTermsQuery(List<String> fields, int numDocs) {
-        int numTerms = randomInt(numDocs);
-
-        QueryBuilder q = QueryBuilders.commonTermsQuery(randomField(fields), randomQueryString(numTerms));
-        if (randomBoolean()) {
-            ((CommonTermsQueryBuilder)q).boost(randomFloat());
-        }
-
-        if (randomBoolean()) {
-            ((CommonTermsQueryBuilder)q).cutoffFrequency(randomFloat());
-        }
-
-        if (randomBoolean()) {
-            ((CommonTermsQueryBuilder)q).highFreqMinimumShouldMatch(Integer.toString(randomInt(numTerms)))
-                    .highFreqOperator(randomBoolean() ? Operator.AND : Operator.OR);
-        }
-
-        if (randomBoolean()) {
-            ((CommonTermsQueryBuilder)q).lowFreqMinimumShouldMatch(Integer.toString(randomInt(numTerms)))
-                    .lowFreqOperator(randomBoolean() ? Operator.AND : Operator.OR);
-        }
-
-        return q;
-    }
-
     private static QueryBuilder randomFuzzyQuery(List<String> fields) {
 
         QueryBuilder q = QueryBuilders.fuzzyQuery(randomField(fields), randomQueryString(1));
diff --git a/server/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java b/server/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
index 5fec898155487..415fa40ea9db8 100644
--- a/server/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
+++ b/server/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.search.query;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
@@ -72,8 +71,6 @@
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.lessThan;
 
 public class MultiMatchQueryIT extends ESIntegTestCase {
 
@@ -303,66 +300,6 @@ public void testSingleField() throws NoSuchFieldException, IllegalAccessExceptio
 
     }
 
-    public void testCutoffFreq() throws ExecutionException, InterruptedException {
-        final long numDocs = client().prepareSearch("test").setSize(0)
-                .setQuery(matchAllQuery()).get().getHits().getTotalHits().value;
-        MatchQuery.Type type = MatchQuery.Type.BOOLEAN;
-        Float cutoffFrequency = randomBoolean() ? Math.min(1, numDocs * 1.f / between(10, 20)) : 1.f / between(10, 20);
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .setQuery(randomizeType(multiMatchQuery("marvel hero captain america", "full_name", "first_name", "last_name", "category")
-                        .operator(Operator.OR).cutoffFrequency(cutoffFrequency))).get();
-        Set<String> topNIds = Sets.newHashSet("theone", "theother");
-        for (int i = 0; i < searchResponse.getHits().getHits().length; i++) {
-            topNIds.remove(searchResponse.getHits().getAt(i).getId());
-            // very likely that we hit a random doc that has the same score so orders are random since
-            // the doc id is the tie-breaker
-        }
-        assertThat(topNIds, empty());
-        assertThat(searchResponse.getHits().getHits()[0].getScore(),
-                greaterThanOrEqualTo(searchResponse.getHits().getHits()[1].getScore()));
-
-        cutoffFrequency = randomBoolean() ? Math.min(1, numDocs * 1.f / between(10, 20)) : 1.f / between(10, 20);
-        searchResponse = client().prepareSearch("test")
-                .setQuery(randomizeType(multiMatchQuery("marvel hero captain america", "full_name", "first_name", "last_name", "category")
-                        .operator(Operator.OR).cutoffFrequency(cutoffFrequency).type(type))).get();
-        assertFirstHit(searchResponse, anyOf(hasId("theone"), hasId("theother")));
-        assertThat(searchResponse.getHits().getHits()[0].getScore(), greaterThan(searchResponse.getHits().getHits()[1].getScore()));
-        long size = searchResponse.getHits().getTotalHits().value;
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(randomizeType(multiMatchQuery("marvel hero captain america", "full_name", "first_name", "last_name", "category")
-                        .operator(Operator.OR).type(type))).get();
-        assertFirstHit(searchResponse, anyOf(hasId("theone"), hasId("theother")));
-        assertThat("common terms expected to be a way smaller result set", size, lessThan(searchResponse.getHits().getTotalHits().value));
-
-        cutoffFrequency = randomBoolean() ? Math.min(1, numDocs * 1.f / between(10, 20)) : 1.f / between(10, 20);
-        searchResponse = client().prepareSearch("test")
-                .setQuery(randomizeType(multiMatchQuery("marvel hero", "full_name", "first_name", "last_name", "category")
-                        .operator(Operator.OR).cutoffFrequency(cutoffFrequency).type(type))).get();
-        assertFirstHit(searchResponse, hasId("theother"));
-
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(randomizeType(multiMatchQuery("captain america", "full_name", "first_name", "last_name", "category")
-                        .operator(Operator.AND).cutoffFrequency(cutoffFrequency).type(type))).get();
-        assertHitCount(searchResponse, 1L);
-        assertFirstHit(searchResponse, hasId("theone"));
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(randomizeType(multiMatchQuery("captain america", "full_name", "first_name", "last_name", "category")
-                        .operator(Operator.AND).cutoffFrequency(cutoffFrequency).type(type))).get();
-        assertHitCount(searchResponse, 1L);
-        assertFirstHit(searchResponse, hasId("theone"));
-
-        searchResponse = client().prepareSearch("test")
-                .setQuery(randomizeType(multiMatchQuery("marvel hero", "first_name", "last_name", "category")
-                        .operator(Operator.AND).cutoffFrequency(cutoffFrequency)
-                        .analyzer("category")
-                        .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS))).get();
-        assertHitCount(searchResponse, 1L);
-        assertFirstHit(searchResponse, hasId("theother"));
-    }
-
     public void testEquivalence() {
 
         final int numDocs = (int) client().prepareSearch("test").setSize(0)
@@ -559,21 +496,11 @@ public void testCrossFieldMode() throws ExecutionException, InterruptedException
                         .analyzer("category"))).get();
         assertFirstHit(searchResponse, hasId("theone"));
 
-        searchResponse = client().prepareSearch("test")
-                .setQuery(randomizeType(multiMatchQuery("captain america marvel hero", "first_name", "last_name", "category")
-                        .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)
-                        .cutoffFrequency(0.1f)
-                        .analyzer("category")
-                        .operator(Operator.OR))).get();
-        assertFirstHit(searchResponse, anyOf(hasId("theother"), hasId("theone")));
-        long numResults = searchResponse.getHits().getTotalHits().value;
-
         searchResponse = client().prepareSearch("test")
                 .setQuery(randomizeType(multiMatchQuery("captain america marvel hero", "first_name", "last_name", "category")
                         .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)
                         .analyzer("category")
                         .operator(Operator.OR))).get();
-        assertThat(numResults, lessThan(searchResponse.getHits().getTotalHits().value));
         assertFirstHit(searchResponse, hasId("theone"));
 
 
diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
index 582e1caa7ce87..192e54506495f 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java
@@ -681,6 +681,39 @@ public void testMaxScoreQueryVisitor() {
         }
     }
 
+    public void testMinScore() throws Exception {
+        Directory dir = newDirectory();
+        IndexWriterConfig iwc = newIndexWriterConfig();
+        RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+        for (int i = 0; i < 10; i++) {
+            Document doc = new Document();
+            doc.add(new StringField("foo", "bar", Store.NO));
+            doc.add(new StringField("filter", "f1", Store.NO));
+            w.addDocument(doc);
+        }
+        w.close();
+
+        IndexReader reader = DirectoryReader.open(dir);
+        IndexSearcher contextSearcher = new IndexSearcher(reader);
+        TestSearchContext context = new TestSearchContext(null, indexShard);
+        context.parsedQuery(new ParsedQuery(
+            new BooleanQuery.Builder()
+                .add(new TermQuery(new Term("foo", "bar")), Occur.MUST)
+                .add(new TermQuery(new Term("filter", "f1")), Occur.SHOULD)
+                .build()
+        ));
+        context.minimumScore(0.01f);
+        context.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap()));
+        context.setSize(1);
+        context.trackTotalHitsUpTo(5);
+
+        QueryPhase.execute(context, contextSearcher, checkCancelled -> {});
+        assertEquals(10, context.queryResult().topDocs().topDocs.totalHits.value);
+
+        reader.close();
+        dir.close();
+    }
+
     private static IndexSearcher getAssertingEarlyTerminationSearcher(IndexReader reader, int size) {
         return new IndexSearcher(reader) {
             @Override
diff --git a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
index efbe858e1899a..ef49d8e436e3e 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QuerySearchResultTests.java
@@ -48,7 +48,7 @@ public class QuerySearchResultTests extends ESTestCase {
     private final NamedWriteableRegistry namedWriteableRegistry;
 
     public QuerySearchResultTests() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         this.namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
     }
 
diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryStringIT.java b/server/src/test/java/org/elasticsearch/search/query/QueryStringIT.java
index 8d5933e6cfc75..0498cc63fb9f9 100644
--- a/server/src/test/java/org/elasticsearch/search/query/QueryStringIT.java
+++ b/server/src/test/java/org/elasticsearch/search/query/QueryStringIT.java
@@ -359,6 +359,7 @@ public void testFieldAliasOnDisallowedFieldType() throws Exception {
         assertHits(response.getHits(), "1");
     }
 
+
     private void assertHits(SearchHits hits, String... ids) {
         assertThat(hits.getTotalHits().value, equalTo((long) ids.length));
         Set<String> hitIds = new HashSet<>();
diff --git a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index 7e233b863076a..afba40e2cb752 100644
--- a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -68,7 +68,6 @@
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery;
 import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
 import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
 import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery;
@@ -101,7 +100,6 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSecondHit;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThirdHit;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasScore;
 import static org.hamcrest.Matchers.closeTo;
@@ -270,97 +268,6 @@ public void testAllDocsQueryString() throws InterruptedException, ExecutionExcep
         }
     }
 
-    public void testCommonTermsQuery() throws Exception {
-
-        client().admin().indices().prepareCreate("test")
-                .addMapping("type1", "field1", "type=text,analyzer=whitespace")
-                .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1)).get();
-        indexRandom(true, client().prepareIndex("test", "type1", "3").setSource("field1", "quick lazy huge brown pidgin", "field2",
-                "the quick lazy huge brown fox jumps over the tree"),
-                client().prepareIndex("test", "type1", "1").setSource("field1", "the quick brown fox"),
-                client().prepareIndex("test", "type1", "2").setSource("field1", "the quick lazy huge brown fox jumps over the tree") );
-
-
-        SearchResponse searchResponse = client().prepareSearch().setQuery(commonTermsQuery("field1", "the quick brown").cutoffFrequency(3)
-                .lowFreqOperator(Operator.OR)).get();
-        assertHitCount(searchResponse, 3L);
-        assertFirstHit(searchResponse, hasId("1"));
-        assertSecondHit(searchResponse, hasId("2"));
-        assertThirdHit(searchResponse, hasId("3"));
-
-        searchResponse = client().prepareSearch().setQuery(commonTermsQuery("field1", "the quick brown").cutoffFrequency(3)
-                .lowFreqOperator(Operator.AND)).get();
-        assertThat(searchResponse.getHits().getTotalHits().value, equalTo(2L));
-        assertFirstHit(searchResponse, hasId("1"));
-        assertSecondHit(searchResponse, hasId("2"));
-
-        // Default
-        searchResponse = client().prepareSearch().setQuery(commonTermsQuery("field1", "the quick brown").cutoffFrequency(3)).get();
-        assertHitCount(searchResponse, 3L);
-        assertFirstHit(searchResponse, hasId("1"));
-        assertSecondHit(searchResponse, hasId("2"));
-        assertThirdHit(searchResponse, hasId("3"));
-
-
-        searchResponse = client().prepareSearch().setQuery(commonTermsQuery("field1", "the huge fox").lowFreqMinimumShouldMatch("2")).get();
-        assertHitCount(searchResponse, 1L);
-        assertFirstHit(searchResponse, hasId("2"));
-
-        searchResponse = client().prepareSearch().setQuery(commonTermsQuery("field1", "the lazy fox brown").cutoffFrequency(1)
-                .highFreqMinimumShouldMatch("3")).get();
-        assertHitCount(searchResponse, 2L);
-        assertFirstHit(searchResponse, hasId("2"));
-        assertSecondHit(searchResponse, hasId("1"));
-
-        searchResponse = client().prepareSearch().setQuery(commonTermsQuery("field1", "the lazy fox brown").cutoffFrequency(1)
-                .highFreqMinimumShouldMatch("4")).get();
-        assertHitCount(searchResponse, 1L);
-        assertFirstHit(searchResponse, hasId("2"));
-
-        // Default
-        searchResponse = client().prepareSearch().setQuery(commonTermsQuery("field1", "the lazy fox brown").cutoffFrequency(1)).get();
-        assertHitCount(searchResponse, 1L);
-        assertFirstHit(searchResponse, hasId("2"));
-
-        searchResponse = client().prepareSearch().setQuery(commonTermsQuery("field1", "the quick brown").cutoffFrequency(3)
-                .analyzer("stop")).get();
-        assertHitCount(searchResponse, 3L);
-        // stop drops "the" since its a stopword
-        assertFirstHit(searchResponse, hasId("1"));
-        assertSecondHit(searchResponse, hasId("3"));
-        assertThirdHit(searchResponse, hasId("2"));
-
-        // try the same with match query
-        searchResponse = client().prepareSearch().setQuery(matchQuery("field1", "the quick brown").cutoffFrequency(3)
-                .operator(Operator.AND)).get();
-        assertHitCount(searchResponse, 2L);
-        assertFirstHit(searchResponse, hasId("1"));
-        assertSecondHit(searchResponse, hasId("2"));
-
-        searchResponse = client().prepareSearch().setQuery(matchQuery("field1", "the quick brown").cutoffFrequency(3)
-                .operator(Operator.OR)).get();
-        assertHitCount(searchResponse, 3L);
-        assertFirstHit(searchResponse, hasId("1"));
-        assertSecondHit(searchResponse, hasId("2"));
-        assertThirdHit(searchResponse, hasId("3"));
-
-        searchResponse = client().prepareSearch().setQuery(matchQuery("field1", "the quick brown").cutoffFrequency(3)
-                .operator(Operator.AND).analyzer("stop")).get();
-        assertHitCount(searchResponse, 3L);
-        // stop drops "the" since its a stopword
-        assertFirstHit(searchResponse, hasId("1"));
-        assertSecondHit(searchResponse, hasId("3"));
-        assertThirdHit(searchResponse, hasId("2"));
-
-        // try the same with multi match query
-        searchResponse = client().prepareSearch().setQuery(multiMatchQuery("the quick brown", "field1", "field2").cutoffFrequency(3)
-                .operator(Operator.AND)).get();
-        assertHitCount(searchResponse, 3L);
-        assertFirstHit(searchResponse, hasId("3"));
-        assertSecondHit(searchResponse, hasId("1"));
-        assertThirdHit(searchResponse, hasId("2"));
-    }
-
     public void testQueryStringAnalyzedWildcard() throws Exception {
         createIndex("test");
 
@@ -479,17 +386,6 @@ public void testDateRangeInQueryStringWithTimeZone_10477() {
         assertHitCount(searchResponse, 0L);
     }
 
-    public void testTypeFilter() throws Exception {
-        assertAcked(prepareCreate("test"));
-        indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("field1", "value1"),
-                client().prepareIndex("test", "type1", "2").setSource("field1", "value1"));
-
-        assertHitCount(client().prepareSearch().setTypes("type1").setQuery(matchAllQuery()).get(), 2L);
-        assertHitCount(client().prepareSearch().setTypes("type2").setQuery(matchAllQuery()).get(), 0L);
-
-        assertHitCount(client().prepareSearch().setTypes("type1", "type2").setQuery(matchAllQuery()).get(), 2L);
-    }
-
     public void testIdsQueryTestsIdIndexed() throws Exception {
         assertAcked(client().admin().indices().prepareCreate("test"));
 
@@ -501,16 +397,6 @@ public void testIdsQueryTestsIdIndexed() throws Exception {
         assertHitCount(searchResponse, 2L);
         assertSearchHits(searchResponse, "1", "3");
 
-        // no type
-        searchResponse = client().prepareSearch().setQuery(constantScoreQuery(idsQuery().addIds("1", "3"))).get();
-        assertHitCount(searchResponse, 2L);
-        assertSearchHits(searchResponse, "1", "3");
-
-        searchResponse = client().prepareSearch().setQuery(idsQuery().addIds("1", "3")).get();
-        assertHitCount(searchResponse, 2L);
-        assertSearchHits(searchResponse, "1", "3");
-
-        // no type
         searchResponse = client().prepareSearch().setQuery(idsQuery().addIds("1", "3")).get();
         assertHitCount(searchResponse, 2L);
         assertSearchHits(searchResponse, "1", "3");
@@ -519,7 +405,7 @@ public void testIdsQueryTestsIdIndexed() throws Exception {
         assertHitCount(searchResponse, 0L);
 
         // repeat..., with terms
-        searchResponse = client().prepareSearch().setTypes("type1").setQuery(constantScoreQuery(termsQuery("_id", "1", "3"))).get();
+        searchResponse = client().prepareSearch().setQuery(constantScoreQuery(termsQuery("_id", "1", "3"))).get();
         assertHitCount(searchResponse, 2L);
         assertSearchHits(searchResponse, "1", "3");
     }
diff --git a/server/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/server/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
index 7f8ab4aa51567..e7675a9cbb146 100644
--- a/server/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
+++ b/server/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
@@ -233,7 +233,7 @@ public void testNestedFieldSimpleQueryString() throws IOException {
         assertHitCount(searchResponse, 1L);
         assertSearchHits(searchResponse, "1");
 
-        searchResponse = client().prepareSearch().setTypes("type1").setQuery(
+        searchResponse = client().prepareSearch().setQuery(
                 simpleQueryStringQuery("foo bar baz").field("body")).get();
         assertHitCount(searchResponse, 1L);
         assertSearchHits(searchResponse, "1");
@@ -243,7 +243,7 @@ public void testNestedFieldSimpleQueryString() throws IOException {
         assertHitCount(searchResponse, 1L);
         assertSearchHits(searchResponse, "1");
 
-        searchResponse = client().prepareSearch().setTypes("type1").setQuery(
+        searchResponse = client().prepareSearch().setQuery(
                 simpleQueryStringQuery("foo bar baz").field("body.sub")).get();
         assertHitCount(searchResponse, 1L);
         assertSearchHits(searchResponse, "1");
@@ -642,7 +642,6 @@ public void testFieldAliasWithWildcardField() throws Exception {
         assertHits(response.getHits(), "2", "3");
     }
 
-
     public void testFieldAliasOnDisallowedFieldType() throws Exception {
         String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json");
         assertAcked(prepareCreate("test").setSource(indexBody, XContentType.JSON));
diff --git a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java
index 0f647353e95af..673a3fce95032 100644
--- a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java
@@ -69,7 +69,7 @@ public class QueryRescorerBuilderTests extends ESTestCase {
      */
     @BeforeClass
     public static void init() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
         xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
@@ -141,8 +141,8 @@ public void testBuildRescoreSearchContext() throws ElasticsearchParseException,
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings);
         // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer
-        QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(),
-            namedWriteableRegistry, null, null, () -> nowInMillis, null) {
+        QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, null, null, null,
+            xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null) {
             @Override
             public MappedFieldType fieldMapper(String name) {
                 TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name);
@@ -184,8 +184,8 @@ public void testRewritingKeepsSettings() throws IOException {
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings);
         // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer
-        QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(),
-            namedWriteableRegistry, null, null, () -> nowInMillis, null) {
+        QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, null, null, null,
+            xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null) {
             @Override
             public MappedFieldType fieldMapper(String name) {
                 TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name);
diff --git a/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java b/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
index 04349e846c78f..fb42d244f78ff 100644
--- a/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
+++ b/server/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.sort.NestedSortBuilder;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
@@ -163,9 +164,13 @@ private TestContext create(SearchType... searchTypes) throws Exception {
             }
         } else {
             if (randomBoolean()) {
-                sort = SortBuilders.fieldSort("nested.field3").missing(1);
+                sort = SortBuilders.fieldSort("nested.field3")
+                    .setNestedSort(new NestedSortBuilder("nested"))
+                    .missing(1);
             } else {
-                sort = SortBuilders.fieldSort("nested.field4").missing("1");
+                sort = SortBuilders.fieldSort("nested.field4")
+                    .setNestedSort(new NestedSortBuilder("nested"))
+                    .missing("1");
             }
         }
         sort.order(randomBoolean() ? SortOrder.ASC : SortOrder.DESC);
diff --git a/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
index 7ac48e03be688..64b621e0974a4 100644
--- a/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
+++ b/server/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
@@ -496,7 +496,7 @@ public void testStringSortMissingAscTerminates() throws Exception {
         refresh();
 
         SearchResponse response = client().prepareSearch("test")
-                .setTypes("test")
+
                 .addSort(new FieldSortBuilder("no_field").order(SortOrder.ASC).missing("_last"))
                 .setScroll("1m")
                 .get();
@@ -509,7 +509,7 @@ public void testStringSortMissingAscTerminates() throws Exception {
         assertNoSearchHits(response);
 
         response = client().prepareSearch("test")
-                .setTypes("test")
+
                 .addSort(new FieldSortBuilder("no_field").order(SortOrder.ASC).missing("_first"))
                 .setScroll("1m")
                 .get();
diff --git a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
index 6128f8d39fcf6..8802b35a694d6 100644
--- a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
@@ -112,11 +112,6 @@ public ShardId shardId() {
             return new ShardId(new Index(indices[0], indices[0]), shardId);
         }
 
-        @Override
-        public String[] types() {
-            return new String[0];
-        }
-
         @Override
         public SearchSourceBuilder source() {
             return null;
diff --git a/server/src/test/java/org/elasticsearch/search/slice/TermsSliceQueryTests.java b/server/src/test/java/org/elasticsearch/search/slice/TermsSliceQueryTests.java
index 881dc6f9587af..9093e3e87a689 100644
--- a/server/src/test/java/org/elasticsearch/search/slice/TermsSliceQueryTests.java
+++ b/server/src/test/java/org/elasticsearch/search/slice/TermsSliceQueryTests.java
@@ -46,7 +46,6 @@
 import static org.hamcrest.Matchers.equalTo;
 
 public class TermsSliceQueryTests extends ESTestCase {
-
     public void testBasics() {
         TermsSliceQuery query1 =
             new TermsSliceQuery("field1", 1, 10);
@@ -62,6 +61,24 @@ public void testBasics() {
         QueryUtils.checkUnequal(query1, query4);
     }
 
+    public void testEmpty() throws Exception {
+        final Directory dir = newDirectory();
+        final RandomIndexWriter w = new RandomIndexWriter(random(), dir, new KeywordAnalyzer());
+        for (int i = 0; i < 10; ++i) {
+            Document doc = new Document();
+            doc.add(new StringField("field", Integer.toString(i), Field.Store.YES));
+            w.addDocument(doc);
+        }
+        final IndexReader reader = w.getReader();
+        final IndexSearcher searcher = newSearcher(reader);
+        TermsSliceQuery query =
+            new TermsSliceQuery("unknown", 1, 1);
+        assertThat(searcher.count(query), equalTo(0));
+        w.close();
+        reader.close();
+        dir.close();
+    }
+
     public void testSearch() throws Exception {
         final int numDocs = randomIntBetween(100, 200);
         final Directory dir = newDirectory();
diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
index e301e8c11c3ad..7f18fbb19075c 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.sort;
 
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -87,7 +88,7 @@ public static void init() {
         ScriptEngine engine = new MockScriptEngine(MockScriptEngine.NAME, scripts, Collections.emptyMap());
         scriptService = new ScriptService(baseSettings, Collections.singletonMap(engine.getType(), engine), ScriptModule.CORE_CONTEXTS);
 
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
         xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
@@ -190,8 +191,8 @@ protected QueryShardContext createMockShardContext() {
             IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName);
             return builder.build(idxSettings, fieldType, new IndexFieldDataCache.None(), null, null);
         };
-        return new QueryShardContext(0, idxSettings, bitsetFilterCache, indexFieldDataLookup, null, null, scriptService,
-                xContentRegistry(), namedWriteableRegistry, null, null, () -> randomNonNegativeLong(), null) {
+        return new QueryShardContext(0, idxSettings, bitsetFilterCache, IndexSearcher::new, indexFieldDataLookup, null, null,
+                scriptService, xContentRegistry(), namedWriteableRegistry, null, null, () -> randomNonNegativeLong(), null) {
 
             @Override
             public MappedFieldType fieldMapper(String name) {
diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
index ee9c8f8ed1105..1623bc2bb378d 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
@@ -45,7 +45,6 @@
 import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
@@ -90,17 +89,7 @@ public FieldSortBuilder randomFieldSortBuilder() {
             builder.sortMode(randomFrom(SortMode.values()));
         }
         if (randomBoolean()) {
-            if (randomBoolean()) {
-                builder.setNestedSort(createRandomNestedSort(3));
-            } else {
-                // the following are alternative ways to setNestedSort for nested sorting
-                if (randomBoolean()) {
-                    builder.setNestedFilter(randomNestedFilter());
-                }
-                if (randomBoolean()) {
-                    builder.setNestedPath(randomAlphaOfLengthBetween(1, 10));
-                }
-            }
+            builder.setNestedSort(createRandomNestedSort(3));
         }
         if (randomBoolean()) {
             builder.setNumericType(randomFrom(random(), "long", "double", "date", "date_nanos"));
@@ -114,16 +103,8 @@ protected FieldSortBuilder mutate(FieldSortBuilder original) throws IOException
         int parameter = randomIntBetween(0, 5);
         switch (parameter) {
         case 0:
-            if (original.getNestedPath() == null && original.getNestedFilter() == null) {
-                mutated.setNestedSort(
-                        randomValueOtherThan(original.getNestedSort(), () -> NestedSortBuilderTests.createRandomNestedSort(3)));
-            } else {
-                if (randomBoolean()) {
-                    mutated.setNestedPath(randomValueOtherThan(original.getNestedPath(), () -> randomAlphaOfLengthBetween(1, 10)));
-                } else {
-                    mutated.setNestedFilter(randomValueOtherThan(original.getNestedFilter(), () -> randomNestedFilter()));
-                }
-            }
+            mutated.setNestedSort(
+                randomValueOtherThan(original.getNestedSort(), () -> NestedSortBuilderTests.createRandomNestedSort(3)));
             break;
         case 1:
             mutated.sortMode(randomValueOtherThan(original.sortMode(), () -> randomFrom(SortMode.values())));
@@ -285,7 +266,8 @@ public void testBuildNested() throws IOException {
         assertNotNull(nested);
         assertEquals(new TermQuery(new Term(MAPPED_STRING_FIELDNAME, "value")), nested.getInnerQuery());
 
-        sortBuilder = new FieldSortBuilder("fieldName").setNestedPath("path");
+        NestedSortBuilder nestedSort = new NestedSortBuilder("path");
+        sortBuilder = new FieldSortBuilder("fieldName").setNestedSort(nestedSort);
         sortField = sortBuilder.build(shardContextMock).field;
         assertThat(sortField.getComparatorSource(), instanceOf(XFieldComparatorSource.class));
         comparatorSource = (XFieldComparatorSource) sortField.getComparatorSource();
@@ -293,19 +275,14 @@ public void testBuildNested() throws IOException {
         assertNotNull(nested);
         assertEquals(new TermQuery(new Term(TypeFieldMapper.NAME, "__path")), nested.getInnerQuery());
 
-        sortBuilder = new FieldSortBuilder("fieldName").setNestedPath("path")
-                .setNestedFilter(QueryBuilders.termQuery(MAPPED_STRING_FIELDNAME, "value"));
+        nestedSort.setFilter(QueryBuilders.termQuery(MAPPED_STRING_FIELDNAME, "value"));
+        sortBuilder = new FieldSortBuilder("fieldName").setNestedSort(nestedSort);
         sortField = sortBuilder.build(shardContextMock).field;
         assertThat(sortField.getComparatorSource(), instanceOf(XFieldComparatorSource.class));
         comparatorSource = (XFieldComparatorSource) sortField.getComparatorSource();
         nested = comparatorSource.nested();
         assertNotNull(nested);
         assertEquals(new TermQuery(new Term(MAPPED_STRING_FIELDNAME, "value")), nested.getInnerQuery());
-
-        // if nested path is missing, we omit any filter and return a SortedNumericSortField
-        sortBuilder = new FieldSortBuilder("fieldName").setNestedFilter(QueryBuilders.termQuery(MAPPED_STRING_FIELDNAME, "value"));
-        sortField = sortBuilder.build(shardContextMock).field;
-        assertThat(sortField, instanceOf(SortedNumericSortField.class));
     }
 
     public void testUnknownOptionFails() throws IOException {
@@ -364,22 +341,6 @@ public void testModeNonNumericField() throws IOException {
         assertEquals(expectedError, e.getMessage());
     }
 
-    /**
-     * Test we can either set nested sort via path/filter or via nested sort builder, not both
-     */
-    public void testNestedSortBothThrows() throws IOException {
-        FieldSortBuilder sortBuilder = new FieldSortBuilder(MAPPED_STRING_FIELDNAME);
-        IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
-                () -> sortBuilder.setNestedPath("nestedPath").setNestedSort(new NestedSortBuilder("otherPath")));
-        assertEquals("Setting both nested_path/nested_filter and nested not allowed", iae.getMessage());
-        iae = expectThrows(IllegalArgumentException.class,
-                () -> sortBuilder.setNestedSort(new NestedSortBuilder("otherPath")).setNestedPath("nestedPath"));
-        assertEquals("Setting both nested_path/nested_filter and nested not allowed", iae.getMessage());
-        iae = expectThrows(IllegalArgumentException.class,
-                () -> sortBuilder.setNestedSort(new NestedSortBuilder("otherPath")).setNestedFilter(QueryBuilders.matchAllQuery()));
-        assertEquals("Setting both nested_path/nested_filter and nested not allowed", iae.getMessage());
-    }
-
     /**
      * Test the nested Filter gets rewritten
      */
@@ -391,10 +352,12 @@ public QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOEx
                 return new MatchNoneQueryBuilder();
             }
         };
-        sortBuilder.setNestedPath("path").setNestedFilter(rangeQuery);
+        NestedSortBuilder nestedSort = new NestedSortBuilder("path");
+        nestedSort.setFilter(rangeQuery);
+        sortBuilder.setNestedSort(nestedSort);
         FieldSortBuilder rewritten = sortBuilder
                 .rewrite(createMockShardContext());
-        assertNotSame(rangeQuery, rewritten.getNestedFilter());
+        assertNotSame(rangeQuery, rewritten.getNestedSort().getFilter());
     }
 
     /**
@@ -414,20 +377,6 @@ public QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOEx
         assertNotSame(rangeQuery, rewritten.getNestedSort().getFilter());
     }
 
-    @Override
-    protected void assertWarnings(FieldSortBuilder testItem) {
-        List<String> expectedWarnings = new ArrayList<>();
-        if (testItem.getNestedFilter() != null) {
-            expectedWarnings.add("[nested_filter] has been deprecated in favour for the [nested] parameter");
-        }
-        if (testItem.getNestedPath() != null) {
-            expectedWarnings.add("[nested_path] has been deprecated in favor of the [nested] parameter");
-        }
-        if (expectedWarnings.isEmpty() == false) {
-            assertWarnings(expectedWarnings.toArray(new String[expectedWarnings.size()]));
-        }
-    }
-
     @Override
     protected FieldSortBuilder fromXContent(XContentParser parser, String fieldName) throws IOException {
         return FieldSortBuilder.fromXContent(parser, fieldName);
diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
index d3f21867ab1d1..400dc30afa11f 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
@@ -1443,9 +1443,11 @@ public void testNestedSort() throws IOException, InterruptedException, Execution
         refresh();
 
         // We sort on nested field
+
         SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(SortBuilders.fieldSort("nested.foo").setNestedPath("nested").order(SortOrder.DESC))
+                .addSort(SortBuilders.fieldSort("nested.foo")
+                    .setNestedSort(new NestedSortBuilder("nested")).order(SortOrder.DESC))
                 .get();
         assertNoFailures(searchResponse);
         SearchHit[] hits = searchResponse.getHits().getHits();
@@ -1474,7 +1476,8 @@ public void testNestedSort() throws IOException, InterruptedException, Execution
         // We sort on nested sub field
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
-                .addSort(SortBuilders.fieldSort("nested.foo.sub").setNestedPath("nested").order(SortOrder.DESC))
+                .addSort(SortBuilders.fieldSort("nested.foo.sub")
+                    .setNestedSort(new NestedSortBuilder("nested")).order(SortOrder.DESC))
                 .get();
         assertNoFailures(searchResponse);
         hits = searchResponse.getHits().getHits();
@@ -1483,6 +1486,14 @@ public void testNestedSort() throws IOException, InterruptedException, Execution
         assertThat(hits[1].getSortValues().length, is(1));
         assertThat(hits[0].getSortValues()[0], is("cba bca"));
         assertThat(hits[1].getSortValues()[0], is("bar bar"));
+
+        // missing nested path
+        SearchPhaseExecutionException exc = expectThrows(SearchPhaseExecutionException.class,
+            () -> client().prepareSearch()
+                    .setQuery(matchAllQuery()).addSort(SortBuilders.fieldSort("nested.foo"))
+                    .get()
+        );
+        assertThat(exc.toString(), containsString("it is mandatory to set the [nested] context"));
     }
 
     public void testSortDuelBetweenSingleShardAndMultiShardIndex() throws Exception {
diff --git a/server/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java b/server/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java
index 87d84484aca0a..f3899785c499c 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java
@@ -272,7 +272,8 @@ public void testDistanceSortingNestedFields() throws Exception {
 
         // Order: Asc
         SearchResponse searchResponse = client().prepareSearch("companies").setQuery(matchAllQuery()).addSort(SortBuilders
-                .geoDistanceSort("branches.location", 40.7143528, -74.0059731).order(SortOrder.ASC).setNestedPath("branches"))
+                .geoDistanceSort("branches.location", 40.7143528, -74.0059731).order(SortOrder.ASC)
+                .setNestedSort(new NestedSortBuilder("branches")))
                 .get();
 
         assertHitCount(searchResponse, 4);
@@ -285,7 +286,8 @@ public void testDistanceSortingNestedFields() throws Exception {
         // Order: Asc, Mode: max
         searchResponse = client()
                 .prepareSearch("companies").setQuery(matchAllQuery()).addSort(SortBuilders.geoDistanceSort("branches.location",
-                        40.7143528, -74.0059731).order(SortOrder.ASC).sortMode(SortMode.MAX).setNestedPath("branches"))
+                        40.7143528, -74.0059731).order(SortOrder.ASC).sortMode(SortMode.MAX)
+                .setNestedSort(new NestedSortBuilder("branches")))
                 .get();
 
         assertHitCount(searchResponse, 4);
@@ -297,7 +299,8 @@ public void testDistanceSortingNestedFields() throws Exception {
 
         // Order: Desc
         searchResponse = client().prepareSearch("companies").setQuery(matchAllQuery()).addSort(SortBuilders
-                .geoDistanceSort("branches.location", 40.7143528, -74.0059731).order(SortOrder.DESC).setNestedPath("branches"))
+                .geoDistanceSort("branches.location", 40.7143528, -74.0059731).order(SortOrder.DESC)
+                .setNestedSort(new NestedSortBuilder("branches")))
                 .get();
 
         assertHitCount(searchResponse, 4);
@@ -310,7 +313,8 @@ public void testDistanceSortingNestedFields() throws Exception {
         // Order: Desc, Mode: min
         searchResponse = client()
                 .prepareSearch("companies").setQuery(matchAllQuery()).addSort(SortBuilders.geoDistanceSort("branches.location",
-                        40.7143528, -74.0059731).order(SortOrder.DESC).sortMode(SortMode.MIN).setNestedPath("branches"))
+                        40.7143528, -74.0059731).order(SortOrder.DESC).sortMode(SortMode.MIN)
+                .setNestedSort(new NestedSortBuilder("branches")))
                 .get();
 
         assertHitCount(searchResponse, 4);
@@ -322,7 +326,8 @@ public void testDistanceSortingNestedFields() throws Exception {
 
         searchResponse = client()
                 .prepareSearch("companies").setQuery(matchAllQuery()).addSort(SortBuilders.geoDistanceSort("branches.location",
-                        40.7143528, -74.0059731).sortMode(SortMode.AVG).order(SortOrder.ASC).setNestedPath("branches"))
+                        40.7143528, -74.0059731).sortMode(SortMode.AVG).order(SortOrder.ASC)
+                .setNestedSort(new NestedSortBuilder("branches")))
                 .get();
 
         assertHitCount(searchResponse, 4);
@@ -334,7 +339,8 @@ public void testDistanceSortingNestedFields() throws Exception {
 
         searchResponse = client().prepareSearch("companies")
                 .setQuery(matchAllQuery()).addSort(SortBuilders.geoDistanceSort("branches.location", 40.7143528, -74.0059731)
-                        .setNestedPath("branches").sortMode(SortMode.AVG).order(SortOrder.DESC).setNestedPath("branches"))
+                .setNestedSort(new NestedSortBuilder("branches"))
+                .sortMode(SortMode.AVG).order(SortOrder.DESC))
                 .get();
 
         assertHitCount(searchResponse, 4);
@@ -346,8 +352,10 @@ public void testDistanceSortingNestedFields() throws Exception {
 
         searchResponse = client().prepareSearch("companies").setQuery(matchAllQuery())
                 .addSort(SortBuilders.geoDistanceSort("branches.location", 40.7143528, -74.0059731)
-                        .setNestedFilter(termQuery("branches.name", "brooklyn"))
-                        .sortMode(SortMode.AVG).order(SortOrder.ASC).setNestedPath("branches"))
+                        .setNestedSort(new NestedSortBuilder("branches")
+                            .setFilter(termQuery("branches.name", "brooklyn"))
+                        )
+                        .sortMode(SortMode.AVG).order(SortOrder.ASC))
                 .get();
         assertHitCount(searchResponse, 4);
         assertFirstHit(searchResponse, hasId("4"));
@@ -360,7 +368,7 @@ public void testDistanceSortingNestedFields() throws Exception {
         try {
                 client().prepareSearch("companies").setQuery(matchAllQuery())
                         .addSort(SortBuilders.geoDistanceSort("branches.location", 40.7143528, -74.0059731).sortMode(SortMode.SUM)
-                                .setNestedPath("branches"));
+                                .setNestedSort(new NestedSortBuilder("branches")));
                 fail("Sum should not be allowed as sort mode");
         } catch (IllegalArgumentException e) {
             //expected
diff --git a/server/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
index ebe1118cc6f69..73be5837faa90 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
@@ -50,9 +50,7 @@
 import org.elasticsearch.test.geo.RandomGeoGenerator;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.List;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.instanceOf;
@@ -106,20 +104,10 @@ public static GeoDistanceSortBuilder randomGeoDistanceSortBuilder() {
             result.validation(randomValueOtherThan(result.validation(), () -> randomFrom(GeoValidationMethod.values())));
         }
         if (randomBoolean()) {
-            if (randomBoolean()) {
-                // don't fully randomize here, GeoDistanceSort is picky about the filters that are allowed
-                NestedSortBuilder nestedSort = new NestedSortBuilder(randomAlphaOfLengthBetween(3, 10));
-                nestedSort.setFilter(new MatchAllQueryBuilder());
-                result.setNestedSort(nestedSort);
-            } else {
-                // the following are alternative ways to setNestedSort for nested sorting
-                if (randomBoolean()) {
-                    result.setNestedFilter(new MatchAllQueryBuilder());
-                }
-                if (randomBoolean()) {
-                    result.setNestedPath(randomAlphaOfLengthBetween(1, 10));
-                }
-            }
+            // don't fully randomize here, GeoDistanceSort is picky about the filters that are allowed
+            NestedSortBuilder nestedSort = new NestedSortBuilder(randomAlphaOfLengthBetween(3, 10));
+            nestedSort.setFilter(new MatchAllQueryBuilder());
+            result.setNestedSort(nestedSort);
         }
         if (randomBoolean()) {
             result.ignoreUnmapped(result.ignoreUnmapped() == false);
@@ -183,16 +171,8 @@ protected GeoDistanceSortBuilder mutate(GeoDistanceSortBuilder original) throws
                     () -> randomFrom(SortMode.values())));
             break;
         case 6:
-            if (original.getNestedPath() == null && original.getNestedFilter() == null) {
-                result.setNestedSort(
-                        randomValueOtherThan(original.getNestedSort(), () -> NestedSortBuilderTests.createRandomNestedSort(3)));
-            } else {
-                if (randomBoolean()) {
-                    result.setNestedPath(randomValueOtherThan(original.getNestedPath(), () -> randomAlphaOfLengthBetween(1, 10)));
-                } else {
-                    result.setNestedFilter(randomValueOtherThan(original.getNestedFilter(), () -> randomNestedFilter()));
-                }
-            }
+            result.setNestedSort(
+                randomValueOtherThan(original.getNestedSort(), () -> NestedSortBuilderTests.createRandomNestedSort(3)));
             break;
         case 7:
             result.validation(randomValueOtherThan(result.validation(), () -> randomFrom(GeoValidationMethod.values())));
@@ -251,7 +231,6 @@ public void testGeoDistanceSortCanBeParsedFromGeoHash() throws IOException {
                 "    \"nested\" : {\n" +
                 "      \"filter\" : {\n" +
                 "        \"ids\" : {\n" +
-                "          \"type\" : [ ],\n" +
                 "          \"values\" : [ ],\n" +
                 "          \"boost\" : 5.711116\n" +
                 "        }\n" +
@@ -388,21 +367,6 @@ private GeoDistanceSortBuilder parse(XContentBuilder sortBuilder) throws Excepti
         }
     }
 
-    @Override
-    protected void assertWarnings(GeoDistanceSortBuilder testItem) {
-        List<String> expectedWarnings = new ArrayList<>();
-        if (testItem.getNestedFilter() != null) {
-            expectedWarnings.add("[nested_filter] has been deprecated in favour of the [nested] parameter");
-        }
-        if (testItem.getNestedPath() != null) {
-            expectedWarnings.add("[nested_path] has been deprecated in favour of the [nested] parameter");
-        }
-        if (expectedWarnings.isEmpty() == false) {
-            assertWarnings(expectedWarnings.toArray(new String[expectedWarnings.size()]));
-        }
-    }
-
-
     @Override
     protected GeoDistanceSortBuilder fromXContent(XContentParser parser, String fieldName) throws IOException {
         return GeoDistanceSortBuilder.fromXContent(parser, fieldName);
@@ -434,7 +398,7 @@ public void testCommonCaseIsOptimized() throws IOException {
         assertEquals(SortField.class, sort.field.getClass()); // descending means the max value should be considered rather than min
 
         builder = new GeoDistanceSortBuilder("random_field_name", new GeoPoint(3.5, 2.1));
-        builder.setNestedPath("some_nested_path");
+        builder.setNestedSort(new NestedSortBuilder("some_nested_path"));
         sort = builder.build(context);
         assertEquals(SortField.class, sort.field.getClass()); // can't use LatLon optimized sorting with nested fields
 
@@ -524,7 +488,8 @@ public void testBuildNested() throws IOException {
         assertNotNull(nested);
         assertEquals(new MatchAllDocsQuery(), nested.getInnerQuery());
 
-        sortBuilder = new GeoDistanceSortBuilder("fieldName", 1.0, 1.0).setNestedPath("path");
+        sortBuilder = new GeoDistanceSortBuilder("fieldName", 1.0, 1.0)
+            .setNestedSort(new NestedSortBuilder("path"));
         sortField = sortBuilder.build(shardContextMock).field;
         assertThat(sortField.getComparatorSource(), instanceOf(XFieldComparatorSource.class));
         comparatorSource = (XFieldComparatorSource) sortField.getComparatorSource();
@@ -532,20 +497,16 @@ public void testBuildNested() throws IOException {
         assertNotNull(nested);
         assertEquals(new TermQuery(new Term(TypeFieldMapper.NAME, "__path")), nested.getInnerQuery());
 
-        sortBuilder = new GeoDistanceSortBuilder("fieldName", 1.0, 1.0).setNestedPath("path")
-                .setNestedFilter(QueryBuilders.matchAllQuery());
+        sortBuilder = new GeoDistanceSortBuilder("fieldName", 1.0, 1.0)
+            .setNestedSort(new NestedSortBuilder("path")
+                .setFilter(QueryBuilders.matchAllQuery())
+            );
         sortField = sortBuilder.build(shardContextMock).field;
         assertThat(sortField.getComparatorSource(), instanceOf(XFieldComparatorSource.class));
         comparatorSource = (XFieldComparatorSource) sortField.getComparatorSource();
         nested = comparatorSource.nested();
         assertNotNull(nested);
         assertEquals(new MatchAllDocsQuery(), nested.getInnerQuery());
-
-        // if nested path is missing, we omit any filter and return a regular SortField
-        // (LatLonSortField)
-        sortBuilder = new GeoDistanceSortBuilder("fieldName", 1.0, 1.0).setNestedFilter(QueryBuilders.termQuery("fieldName", "value"));
-        sortField = sortBuilder.build(shardContextMock).field;
-        assertThat(sortField, instanceOf(SortField.class));
     }
 
     /**
@@ -580,22 +541,6 @@ public void testBuildInvalidPoints() throws IOException {
         }
     }
 
-    /**
-     * Test we can either set nested sort via path/filter or via nested sort builder, not both
-     */
-    public void testNestedSortBothThrows() throws IOException {
-        GeoDistanceSortBuilder sortBuilder = new GeoDistanceSortBuilder("fieldName", 0.0, 0.0);
-        IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
-                () -> sortBuilder.setNestedPath("nestedPath").setNestedSort(new NestedSortBuilder("otherPath")));
-        assertEquals("Setting both nested_path/nested_filter and nested not allowed", iae.getMessage());
-        iae = expectThrows(IllegalArgumentException.class,
-                () -> sortBuilder.setNestedSort(new NestedSortBuilder("otherPath")).setNestedPath("nestedPath"));
-        assertEquals("Setting both nested_path/nested_filter and nested not allowed", iae.getMessage());
-        iae = expectThrows(IllegalArgumentException.class,
-                () -> sortBuilder.setNestedSort(new NestedSortBuilder("otherPath")).setNestedFilter(QueryBuilders.matchAllQuery()));
-        assertEquals("Setting both nested_path/nested_filter and nested not allowed", iae.getMessage());
-    }
-
     /**
      * Test the nested Filter gets rewritten
      */
@@ -607,10 +552,12 @@ public QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOEx
                 return new MatchNoneQueryBuilder();
             }
         };
-        sortBuilder.setNestedPath("path").setNestedFilter(rangeQuery);
-        GeoDistanceSortBuilder rewritten = (GeoDistanceSortBuilder) sortBuilder
-                .rewrite(createMockShardContext());
-        assertNotSame(rangeQuery, rewritten.getNestedFilter());
+        sortBuilder.setNestedSort(
+            new NestedSortBuilder("path")
+                .setFilter(rangeQuery)
+        );
+        GeoDistanceSortBuilder rewritten = sortBuilder.rewrite(createMockShardContext());
+        assertNotSame(rangeQuery, rewritten.getNestedSort().getFilter());
     }
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/search/sort/NestedSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/NestedSortBuilderTests.java
index e644a070faecd..608c4f627871d 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/NestedSortBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/NestedSortBuilderTests.java
@@ -51,7 +51,7 @@ public class NestedSortBuilderTests extends ESTestCase {
 
     @BeforeClass
     public static void init() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
         xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
diff --git a/server/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
index 3017a2e0c067d..1d3b71bbf40db 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
@@ -324,7 +324,8 @@ public void testBuildNested() throws IOException {
         assertNotNull(nested);
         assertEquals(new MatchAllDocsQuery(), nested.getInnerQuery());
 
-        sortBuilder = new ScriptSortBuilder(mockScript(MOCK_SCRIPT_NAME), ScriptSortType.NUMBER).setNestedPath("path");
+        sortBuilder = new ScriptSortBuilder(mockScript(MOCK_SCRIPT_NAME), ScriptSortType.NUMBER)
+            .setNestedSort(new NestedSortBuilder("path"));
         sortField = sortBuilder.build(shardContextMock).field;
         assertThat(sortField.getComparatorSource(), instanceOf(XFieldComparatorSource.class));
         comparatorSource = (XFieldComparatorSource) sortField.getComparatorSource();
@@ -332,40 +333,17 @@ public void testBuildNested() throws IOException {
         assertNotNull(nested);
         assertEquals(new TermQuery(new Term(TypeFieldMapper.NAME, "__path")), nested.getInnerQuery());
 
-        sortBuilder = new ScriptSortBuilder(mockScript(MOCK_SCRIPT_NAME), ScriptSortType.NUMBER).setNestedPath("path")
-                .setNestedFilter(QueryBuilders.matchAllQuery());
+        sortBuilder = new ScriptSortBuilder(mockScript(MOCK_SCRIPT_NAME), ScriptSortType.NUMBER)
+            .setNestedSort(new NestedSortBuilder("path")
+                .setFilter(QueryBuilders.matchAllQuery()));
         sortField = sortBuilder.build(shardContextMock).field;
         assertThat(sortField.getComparatorSource(), instanceOf(XFieldComparatorSource.class));
         comparatorSource = (XFieldComparatorSource) sortField.getComparatorSource();
         nested = comparatorSource.nested();
         assertNotNull(nested);
         assertEquals(new MatchAllDocsQuery(), nested.getInnerQuery());
-
-        // if nested path is missing, we omit nested element in the comparator
-        sortBuilder = new ScriptSortBuilder(mockScript(MOCK_SCRIPT_NAME), ScriptSortType.NUMBER)
-                .setNestedFilter(QueryBuilders.matchAllQuery());
-        sortField = sortBuilder.build(shardContextMock).field;
-        assertThat(sortField.getComparatorSource(), instanceOf(XFieldComparatorSource.class));
-        comparatorSource = (XFieldComparatorSource) sortField.getComparatorSource();
-        assertNull(comparatorSource.nested());
     }
 
-    /**
-     * Test we can either set nested sort via path/filter or via nested sort builder, not both
-     */
-    public void testNestedSortBothThrows() throws IOException {
-        ScriptSortBuilder sortBuilder = new ScriptSortBuilder(mockScript(MOCK_SCRIPT_NAME), ScriptSortType.NUMBER);
-        IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
-                () -> sortBuilder.setNestedPath("nestedPath").setNestedSort(new NestedSortBuilder("otherPath")));
-        assertEquals("Setting both nested_path/nested_filter and nested not allowed", iae.getMessage());
-        iae = expectThrows(IllegalArgumentException.class,
-                () -> sortBuilder.setNestedSort(new NestedSortBuilder("otherPath")).setNestedPath("nestedPath"));
-        assertEquals("Setting both nested_path/nested_filter and nested not allowed", iae.getMessage());
-        iae = expectThrows(IllegalArgumentException.class,
-                () -> sortBuilder.setNestedSort(new NestedSortBuilder("otherPath")).setNestedFilter(QueryBuilders.matchAllQuery()));
-        assertEquals("Setting both nested_path/nested_filter and nested not allowed", iae.getMessage());
-     }
-
     /**
      * Test the nested Filter gets rewritten
      */
@@ -377,10 +355,10 @@ public QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOEx
                 return new MatchNoneQueryBuilder();
             }
         };
-        sortBuilder.setNestedPath("path").setNestedFilter(rangeQuery);
+        sortBuilder.setNestedSort(new NestedSortBuilder("path").setFilter(rangeQuery));
         ScriptSortBuilder rewritten = sortBuilder
                 .rewrite(createMockShardContext());
-        assertNotSame(rangeQuery, rewritten.getNestedFilter());
+        assertNotSame(rangeQuery, rewritten.getNestedSort().getFilter());
     }
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
index 5f5ea5e869450..7b1e56d5c69f2 100644
--- a/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
@@ -49,7 +49,7 @@ public class SortBuilderTests extends ESTestCase {
 
     @BeforeClass
     public static void init() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
@@ -143,16 +143,6 @@ public void testRandomSortBuilders() throws IOException {
                 xContentBuilder.field("sort");
             }
             for (SortBuilder<?> builder : testBuilders) {
-                if (builder instanceof GeoDistanceSortBuilder) {
-                    GeoDistanceSortBuilder gdsb = (GeoDistanceSortBuilder) builder;
-                    if (gdsb.getNestedFilter() != null) {
-                        expectedWarningHeaders.add("[nested_filter] has been deprecated in favour of the [nested] parameter");
-                    }
-                    if (gdsb.getNestedPath() != null) {
-                        expectedWarningHeaders.add("[nested_path] has been deprecated in favour of the [nested] parameter");
-                    }
-                }
-
                 if (builder instanceof ScoreSortBuilder || builder instanceof FieldSortBuilder) {
                     switch (randomIntBetween(0, 2)) {
                     case 0:
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
index 9d1f01fe3289b..171d3fa2bc1c1 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
@@ -69,7 +69,7 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
      */
     @BeforeClass
     public static void init() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
         xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
@@ -178,8 +178,8 @@ public void testBuild() throws IOException {
                     invocation -> new NamedAnalyzer((String) invocation.getArguments()[0], AnalyzerScope.INDEX, new SimpleAnalyzer()));
             when(scriptService.compile(any(Script.class), any())).then(invocation -> new TestTemplateService.MockTemplateScript.Factory(
                     ((Script) invocation.getArguments()[0]).getIdOrCode()));
-            QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, mapperService, null, scriptService,
-                    xContentRegistry(), namedWriteableRegistry, null, null, System::currentTimeMillis, null);
+            QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, mapperService, null,
+                    scriptService, xContentRegistry(), namedWriteableRegistry, null, null, System::currentTimeMillis, null);
 
             SuggestionContext suggestionContext = suggestionBuilder.build(mockShardContext);
             assertEquals(toBytesRef(suggestionBuilder.text()), suggestionContext.getText());
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java b/server/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
index c59b342b7c0d8..d4596f0885f5f 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.search.suggest;
 
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
 import org.apache.lucene.analysis.TokenStreamToAutomaton;
 import org.apache.lucene.search.suggest.document.ContextSuggestField;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
@@ -357,6 +358,26 @@ public void testSuggestDocumentSourceFiltering() throws Exception {
         }
     }
 
+    /**
+     * Suggestions run on an empty index should return a suggest element as part of the response. See #42473 for details.
+     */
+    public void testSuggestEmptyIndex() throws IOException, InterruptedException {
+        final CompletionMappingBuilder mapping = new CompletionMappingBuilder();
+        createIndexAndMapping(mapping);
+
+        CompletionSuggestionBuilder prefix = SuggestBuilders.completionSuggestion(FIELD).prefix("v");
+        SearchResponse searchResponse = client().prepareSearch(INDEX).suggest(new SuggestBuilder().addSuggestion("foo", prefix))
+                .setFetchSource("a", "b").get();
+        Suggest suggest = searchResponse.getSuggest();
+        assertNotNull(suggest);
+        CompletionSuggestion completionSuggestion = suggest.getSuggestion("foo");
+        CompletionSuggestion.Entry options = completionSuggestion.getEntries().get(0);
+        assertEquals("v", options.getText().string());
+        assertEquals(1, options.getLength());
+        assertEquals(0, options.getOffset());
+        assertEquals(0, options.options.size());
+    }
+
     public void testThatWeightsAreWorking() throws Exception {
         createIndexAndMapping(completionMappingBuilder);
 
@@ -813,7 +834,7 @@ public void testThatSortingOnCompletionFieldReturnsUsefulException() throws Exce
         refresh();
 
         SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class,
-            () -> client().prepareSearch(INDEX).setTypes(TYPE).addSort(new FieldSortBuilder(FIELD)).get());
+                () -> client().prepareSearch(INDEX).addSort(new FieldSortBuilder(FIELD)).get());
         assertThat(e.status().getStatus(), is(400));
         assertThat(e.toString(), containsString("Fielddata is not supported on field [" + FIELD + "] of type [completion]"));
     }
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java b/server/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java
index 2b99c62185b7c..378844afac559 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java
@@ -52,7 +52,7 @@ public class SuggestBuilderTests extends ESTestCase {
      */
     @BeforeClass
     public static void init() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
         xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java b/server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java
index 1813f17141b68..22c19765fed32 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java
@@ -345,6 +345,34 @@ public void testEmpty() throws Exception {
         assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
     }
 
+    public void testEmptyIndex() throws Exception {
+        assertAcked(prepareCreate("test").addMapping("type1", "text", "type=text"));
+        ensureGreen();
+
+        // use SuggestMode.ALWAYS, otherwise the results can vary between requests.
+        TermSuggestionBuilder termSuggest = termSuggestion("text")
+                .suggestMode(SuggestMode.ALWAYS)
+                .text("abcd");
+        Suggest suggest = searchSuggest("test", termSuggest);
+        assertSuggestionSize(suggest, 0, 0, "test");
+        assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
+
+        suggest = searchSuggest("test", termSuggest);
+        assertSuggestionSize(suggest, 0, 0, "test");
+        assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
+
+        index("test", "type1", "1", "text", "bar");
+        refresh();
+
+        suggest = searchSuggest("test", termSuggest);
+        assertSuggestionSize(suggest, 0, 0, "test");
+        assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
+
+        suggest = searchSuggest("test", termSuggest);
+        assertSuggestionSize(suggest, 0, 0, "test");
+        assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
+    }
+
     public void testWithMultipleCommands() throws Exception {
         assertAcked(prepareCreate("test").addMapping("typ1", "field1", "type=text", "field2", "type=text"));
         ensureGreen();
@@ -755,13 +783,43 @@ public void testEmptyShards() throws IOException, InterruptedException {
                 .put("index.analysis.filter.shingler.output_unigrams", true)).addMapping("type1", mappingBuilder));
         ensureGreen();
 
+        // test phrase suggestion on completely empty index
+        SearchResponse searchResponse = client().prepareSearch()
+                .setSize(0)
+                .suggest(
+                        new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean",
+                                phraseSuggestion("name").maxErrors(5.0f)))
+                .get();
+
+        assertNoFailures(searchResponse);
+        Suggest suggest = searchResponse.getSuggest();
+        assertSuggestionSize(suggest, 0, 0, "did_you_mean");
+        assertThat(suggest.getSuggestion("did_you_mean").getEntries().get(0).getText().string(), equalTo("tetsting sugestion"));
+
+
         index("test", "type1", "11", "foo", "bar");
         index("test", "type1", "12", "foo", "bar");
-        index("test", "type1", "1", "name", "Just testing the suggestions api");
         index("test", "type1", "2", "name", "An other title about equal length");
         refresh();
 
-        SearchResponse searchResponse = client().prepareSearch()
+        // test phrase suggestion but nothing matches
+        searchResponse = client().prepareSearch()
+                .setSize(0)
+                .suggest(
+                        new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean",
+                                phraseSuggestion("name").maxErrors(5.0f)))
+                .get();
+
+        assertNoFailures(searchResponse);
+        suggest = searchResponse.getSuggest();
+        assertSuggestionSize(suggest, 0, 0, "did_you_mean");
+        assertThat(suggest.getSuggestion("did_you_mean").getEntries().get(0).getText().string(), equalTo("tetsting sugestion"));
+
+        // finally indexing a document that will produce some meaningful suggestion
+        index("test", "type1", "1", "name", "Just testing the suggestions api");
+        refresh();
+
+        searchResponse = client().prepareSearch()
                 .setSize(0)
                 .suggest(
                         new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean",
@@ -769,7 +827,9 @@ public void testEmptyShards() throws IOException, InterruptedException {
                 .get();
 
         assertNoFailures(searchResponse);
-        assertSuggestion(searchResponse.getSuggest(), 0, 0, "did_you_mean", "testing suggestions");
+        suggest = searchResponse.getSuggest();
+        assertSuggestionSize(suggest, 0, 3, "did_you_mean");
+        assertSuggestion(suggest, 0, 0, "did_you_mean", "testing suggestions");
     }
 
     /**
diff --git a/server/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java b/server/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java
index 903d06a372e72..5c247630d6b7f 100644
--- a/server/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java
+++ b/server/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java
@@ -232,7 +232,7 @@ public void testSerialization() throws IOException {
         final Suggest bwcSuggest;
 
         NamedWriteableRegistry registry = new NamedWriteableRegistry
-            (new SearchModule(Settings.EMPTY, false, emptyList()).getNamedWriteables());
+            (new SearchModule(Settings.EMPTY, emptyList()).getNamedWriteables());
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.setVersion(bwcVersion);
diff --git a/server/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/server/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
index 9fe7356877c8e..9f680f11f2117 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
@@ -20,11 +20,13 @@
 
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.cluster.SnapshotsInProgress;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil;
 import org.elasticsearch.snapshots.mockstore.MockRepository;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.junit.After;
@@ -65,6 +67,32 @@ public void assertConsistentHistoryInLuceneIndex() throws Exception {
         internalCluster().assertConsistentHistoryBetweenTranslogAndLuceneIndex();
     }
 
+    private String skipRepoConsistencyCheckReason;
+
+    @After
+    public void assertRepoConsistency() {
+        if (skipRepoConsistencyCheckReason == null) {
+            client().admin().cluster().prepareGetRepositories().get().repositories()
+                .stream()
+                .map(RepositoryMetaData::name)
+                .forEach(name -> {
+                    final List<SnapshotInfo> snapshots = client().admin().cluster().prepareGetSnapshots(name).get().getSnapshots(name);
+                    // Delete one random snapshot to trigger repository cleanup.
+                    if (snapshots.isEmpty() == false) {
+                        client().admin().cluster().prepareDeleteSnapshot(name, randomFrom(snapshots).snapshotId().getName()).get();
+                    }
+                    BlobStoreTestUtil.assertRepoConsistency(internalCluster(), name);
+                });
+        } else {
+            logger.info("--> skipped repo consistency checks because [{}]", skipRepoConsistencyCheckReason);
+        }
+    }
+
+    protected void disableRepoConsistencyCheck(String reason) {
+        assertNotNull(reason);
+        skipRepoConsistencyCheckReason = reason;
+    }
+
     public static long getFailureCount(String repository) {
         long failureCount = 0;
         for (RepositoriesService repositoriesService :
@@ -120,7 +148,7 @@ public SnapshotInfo waitForCompletion(String repository, String snapshotName, Ti
         long start = System.currentTimeMillis();
         while (System.currentTimeMillis() - start < timeout.millis()) {
             List<SnapshotInfo> snapshotInfos = client().admin().cluster().prepareGetSnapshots(repository).setSnapshots(snapshotName)
-                .get().getSnapshots();
+                .get().getSnapshots(repository);
             assertThat(snapshotInfos.size(), equalTo(1));
             if (snapshotInfos.get(0).state().completed()) {
                 // Make sure that snapshot clean up operations are finished
diff --git a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
index 4a28b1eeea440..dba92dc09c9da 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
@@ -219,8 +219,7 @@ public void testExceptionWhenRestoringPersistentSettings() {
         assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0));
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0));
         assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute().actionGet()
-                .getSnapshots().get(0).state(),
-            equalTo(SnapshotState.SUCCESS));
+                .getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS));
 
         logger.info("--> change the test persistent setting and break it");
         setSettingValue.accept("new value 2");
@@ -272,7 +271,7 @@ public void testRestoreCustomMetadata() throws Exception {
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
             equalTo(createSnapshotResponse.getSnapshotInfo().successfulShards()));
         assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute().actionGet()
-                .getSnapshots().get(0).state(),
+                .getSnapshots("test-repo").get(0).state(),
             equalTo(SnapshotState.SUCCESS));
 
         logger.info("--> change custom persistent metadata");
@@ -420,7 +419,6 @@ public void testSnapshotDuringNodeShutdown() throws Exception {
         logger.info("--> done");
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/39852")
     public void testSnapshotWithStuckNode() throws Exception {
         logger.info("--> start 2 nodes");
         ArrayList<String> nodes = new ArrayList<>();
@@ -484,11 +482,21 @@ public void testSnapshotWithStuckNode() throws Exception {
         }
 
         logger.info("--> making sure that snapshot no longer exists");
-        assertThrows(client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute(),
-            SnapshotMissingException.class);
+        expectThrows(SnapshotMissingException.class,
+            () -> client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap")
+                .execute().actionGet().getSnapshots("test-repo"));
+
+        // TODO: Replace this by repository cleanup endpoint call once that's available
+        logger.info("--> Go through a loop of creating and deleting a snapshot to trigger repository cleanup");
+        client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-tmp")
+            .setWaitForCompletion(true)
+            .setIndices("test-idx")
+            .get();
+        client().admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-tmp").get();
+
         // Subtract four files that will remain in the repository:
-        //   (1) index-1
-        //   (2) index-0 (because we keep the previous version) and
+        //   (1) index-(N+1)
+        //   (2) index-N (because we keep the previous version) and
         //   (3) index-latest
         //   (4) incompatible-snapshots
         assertFileCount(repo, 4);
@@ -541,7 +549,7 @@ public void testRestoreIndexWithMissingShards() throws Exception {
         assertAcked(prepareCreate("test-idx-none", 1, Settings.builder().put("number_of_shards", 6)
                 .put("index.routing.allocation.include.tag", "nowhere")
                 .put("number_of_replicas", 0)).setWaitForActiveShards(ActiveShardCount.NONE).get());
-        assertTrue(client().admin().indices().prepareExists("test-idx-none").get().isExists());
+        assertTrue(indexExists("test-idx-none"));
 
         logger.info("--> creating repository");
         AcknowledgedResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo")
@@ -584,8 +592,8 @@ public void testRestoreIndexWithMissingShards() throws Exception {
             assertBusy(() -> {
                 GetSnapshotsResponse response = client().admin().cluster().prepareGetSnapshots("test-repo")
                     .setSnapshots("test-snap-2").get();
-                assertThat(response.getSnapshots().size(), equalTo(1));
-                SnapshotInfo snapshotInfo = response.getSnapshots().get(0);
+                assertThat(response.getSnapshots("test-repo").size(), equalTo(1));
+                SnapshotInfo snapshotInfo = response.getSnapshots("test-repo").get(0);
                 assertTrue(snapshotInfo.state().completed());
                 assertEquals(SnapshotState.PARTIAL, snapshotInfo.state());
             }, 1, TimeUnit.MINUTES);
@@ -600,7 +608,7 @@ public void testRestoreIndexWithMissingShards() throws Exception {
             assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), lessThan(16));
             assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(10));
             assertThat(client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-2").execute().actionGet()
-                    .getSnapshots().get(0).state(),
+                    .getSnapshots("test-repo").get(0).state(),
                 equalTo(SnapshotState.PARTIAL));
         }
 
@@ -723,6 +731,7 @@ public boolean clearData(String nodeName) {
     }
 
     public void testRegistrationFailure() {
+        disableRepoConsistencyCheck("This test does not create any data in the repository");
         logger.info("--> start first node");
         internalCluster().startNode();
         logger.info("--> start second node");
@@ -742,6 +751,7 @@ public void testRegistrationFailure() {
     }
 
     public void testThatSensitiveRepositorySettingsAreNotExposed() throws Exception {
+        disableRepoConsistencyCheck("This test does not create any data in the repository");
         Settings nodeSettings = Settings.EMPTY;
         logger.info("--> start two nodes");
         internalCluster().startNodes(2, nodeSettings);
@@ -841,7 +851,7 @@ public void testMasterShutdownDuringSnapshot() throws Exception {
         assertBusy(() -> {
             GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster().prepareGetSnapshots("test-repo")
                 .setSnapshots("test-snap").get();
-            SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
+            SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots("test-repo").get(0);
             assertTrue(snapshotInfo.state().completed());
         }, 1, TimeUnit.MINUTES);
 
@@ -849,7 +859,7 @@ public void testMasterShutdownDuringSnapshot() throws Exception {
 
         GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster().prepareGetSnapshots("test-repo")
             .setSnapshots("test-snap").get();
-        SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
+        SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots("test-repo").get(0);
         assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
         assertEquals(snapshotInfo.totalShards(), snapshotInfo.successfulShards());
         assertEquals(0, snapshotInfo.failedShards());
@@ -901,7 +911,7 @@ public void testMasterAndDataShutdownDuringSnapshot() throws Exception {
         assertBusy(() -> {
             GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster().prepareGetSnapshots("test-repo")
                 .setSnapshots("test-snap").get();
-            SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
+            SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots("test-repo").get(0);
             assertTrue(snapshotInfo.state().completed());
         }, 1, TimeUnit.MINUTES);
 
@@ -909,7 +919,7 @@ public void testMasterAndDataShutdownDuringSnapshot() throws Exception {
 
         GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster().prepareGetSnapshots("test-repo")
             .setSnapshots("test-snap").get();
-        SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
+        SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots("test-repo").get(0);
         assertEquals(SnapshotState.PARTIAL, snapshotInfo.state());
         assertNotEquals(snapshotInfo.totalShards(), snapshotInfo.successfulShards());
         assertThat(snapshotInfo.failedShards(), greaterThan(0));
@@ -967,8 +977,8 @@ public void testMasterShutdownDuringFailedSnapshot() throws Exception {
         assertBusy(() -> {
             GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster()
                 .prepareGetSnapshots("test-repo").setSnapshots("test-snap").setIgnoreUnavailable(true).get();
-            assertEquals(1, snapshotsStatusResponse.getSnapshots().size());
-            SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
+            assertEquals(1, snapshotsStatusResponse.getSnapshots("test-repo").size());
+            SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots("test-repo").get(0);
             assertTrue(snapshotInfo.state().completed());
             ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
             SnapshotsInProgress snapshotsInProgress = clusterState.custom(SnapshotsInProgress.TYPE);
@@ -978,7 +988,7 @@ public void testMasterShutdownDuringFailedSnapshot() throws Exception {
         logger.info("-->  verify that snapshot failed");
         GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster()
             .prepareGetSnapshots("test-repo").setSnapshots("test-snap").get();
-        SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
+        SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots("test-repo").get(0);
         assertEquals(SnapshotState.FAILED, snapshotInfo.state());
     }
 
@@ -1216,8 +1226,8 @@ public void testDataNodeRestartWithBusyMasterDuringSnapshot() throws Exception {
         assertBusy(() -> {
             GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster()
                 .prepareGetSnapshots("test-repo").setSnapshots("test-snap").setIgnoreUnavailable(true).get();
-            assertEquals(1, snapshotsStatusResponse.getSnapshots().size());
-            SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
+            assertEquals(1, snapshotsStatusResponse.getSnapshots("test-repo").size());
+            SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots("test-repo").get(0);
             assertTrue(snapshotInfo.state().toString(), snapshotInfo.state().completed());
         }, 60L, TimeUnit.SECONDS);
     }
diff --git a/server/src/test/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java
index aeea321b8536b..99cf9d14069b0 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java
@@ -89,7 +89,7 @@ public void testWhenMetadataAreLoaded() throws Exception {
         // Getting a snapshot does not load any metadata
         GetSnapshotsResponse getSnapshotsResponse =
             client().admin().cluster().prepareGetSnapshots("repository").addSnapshots("snap").setVerbose(randomBoolean()).get();
-        assertThat(getSnapshotsResponse.getSnapshots(), hasSize(1));
+        assertThat(getSnapshotsResponse.getSnapshots("repository"), hasSize(1));
         assertGlobalMetadataLoads("snap", 0);
         assertIndexMetadataLoads("snap", "docs", 0);
         assertIndexMetadataLoads("snap", "others", 0);
@@ -141,8 +141,8 @@ public void testWhenMetadataAreLoaded() throws Exception {
         // Deleting a snapshot does not load the global metadata state but loads each index metadata
         assertAcked(client().admin().cluster().prepareDeleteSnapshot("repository", "snap").get());
         assertGlobalMetadataLoads("snap", 1);
-        assertIndexMetadataLoads("snap", "docs", 5);
-        assertIndexMetadataLoads("snap", "others", 4);
+        assertIndexMetadataLoads("snap", "docs", 4);
+        assertIndexMetadataLoads("snap", "others", 3);
     }
 
     private void assertGlobalMetadataLoads(final String snapshot, final int times) {
diff --git a/server/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java
index 885baa883ed63..0dd8d6af585d8 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java
@@ -104,7 +104,7 @@ public void testConcurrentSnapshotDeletionsNotAllowed() throws Exception {
 
         logger.info("--> delete second snapshot, which should now work");
         client().admin().cluster().prepareDeleteSnapshot(repo, snapshot1).get();
-        assertTrue(client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots().isEmpty());
+        assertTrue(client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots(repo).isEmpty());
     }
 
     public void testSnapshottingWithInProgressDeletionNotAllowed() throws Exception {
@@ -150,7 +150,7 @@ public void testSnapshottingWithInProgressDeletionNotAllowed() throws Exception
 
         logger.info("--> creating second snapshot, which should now work");
         client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setWaitForCompletion(true).get();
-        assertEquals(1, client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots().size());
+        assertEquals(1, client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots(repo).size());
     }
 
     public void testRestoreWithInProgressDeletionsNotAllowed() throws Exception {
@@ -204,6 +204,6 @@ public void testRestoreWithInProgressDeletionsNotAllowed() throws Exception {
 
         logger.info("--> restoring snapshot, which should now work");
         client().admin().cluster().prepareRestoreSnapshot(repo, snapshot1).setWaitForCompletion(true).get();
-        assertEquals(1, client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots().size());
+        assertEquals(1, client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots(repo).size());
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java b/server/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
index 63e2c31ea3e96..455a3f18e5472 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
@@ -184,6 +184,8 @@ public void testRepositoryAckTimeout() throws Exception {
     }
 
     public void testRepositoryVerification() throws Exception {
+        disableRepoConsistencyCheck("This test does not create any data in the repository.");
+
         Client client = client();
 
         Settings settings = Settings.builder()
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index 0aa9fe1a9e2a6..4d475fc7b0b1b 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -95,7 +95,6 @@
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.StoredScriptsIT;
 import org.elasticsearch.snapshots.mockstore.MockRepository;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
@@ -130,8 +129,6 @@
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.index.shard.IndexShardTests.getEngineFromShard;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAliasesExist;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAliasesMissing;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertIndexTemplateExists;
@@ -240,7 +237,7 @@ public void testBasicWorkFlow() throws Exception {
             equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
 
         List<SnapshotInfo> snapshotInfos = client.admin().cluster().prepareGetSnapshots("test-repo")
-            .setSnapshots(randomFrom("test-snap", "_all", "*", "*-snap", "test*")).get().getSnapshots();
+                .setSnapshots(randomFrom("test-snap", "_all", "*", "*-snap", "test*")).get().getSnapshots("test-repo");
         assertThat(snapshotInfos.size(), equalTo(1));
         SnapshotInfo snapshotInfo = snapshotInfos.get(0);
         assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS));
@@ -363,7 +360,7 @@ public void testSingleGetAfterRestore() throws Exception {
         assertThat(client.prepareGet(restoredIndexName, typeName, docId).get().isExists(), equalTo(true));
     }
 
-    public void testFreshIndexUUID() {
+    public void testFreshIndexUUID() throws InterruptedException {
         Client client = client();
 
         logger.info("-->  creating repository");
@@ -479,8 +476,8 @@ public void testEmptySnapshot() throws Exception {
         assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0));
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0));
 
-        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(),
-            equalTo(SnapshotState.SUCCESS));
+        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap")
+                .get().getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS));
     }
 
     public void testRestoreAliases() throws Exception {
@@ -501,7 +498,8 @@ public void testRestoreAliases() throws Exception {
                 .addAlias("test-idx-3", "alias-123")
                 .addAlias("test-idx-1", "alias-1")
                 .get());
-        assertAliasesExist(client.admin().indices().prepareAliasesExist("alias-123").get());
+
+        assertFalse(client.admin().indices().prepareGetAliases("alias-123").get().getAliases().isEmpty());
 
         logger.info("--> snapshot");
         assertThat(client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
@@ -510,7 +508,8 @@ public void testRestoreAliases() throws Exception {
 
         logger.info("-->  delete all indices");
         cluster().wipeIndices("test-idx-1", "test-idx-2", "test-idx-3");
-        assertAliasesMissing(client.admin().indices().prepareAliasesExist("alias-123", "alias-1").get());
+        assertTrue(client.admin().indices().prepareGetAliases("alias-123").get().getAliases().isEmpty());
+        assertTrue(client.admin().indices().prepareGetAliases("alias-1").get().getAliases().isEmpty());
 
         logger.info("--> restore snapshot with aliases");
         RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap")
@@ -520,7 +519,8 @@ public void testRestoreAliases() throws Exception {
             equalTo(restoreSnapshotResponse.getRestoreInfo().totalShards())));
 
         logger.info("--> check that aliases are restored");
-        assertAliasesExist(client.admin().indices().prepareAliasesExist("alias-123", "alias-1").get());
+        assertFalse(client.admin().indices().prepareGetAliases("alias-123").get().getAliases().isEmpty());
+        assertFalse(client.admin().indices().prepareGetAliases("alias-1").get().getAliases().isEmpty());
 
         logger.info("-->  update aliases");
         assertAcked(client.admin().indices().prepareAliases().removeAlias("test-idx-3", "alias-123"));
@@ -529,7 +529,8 @@ public void testRestoreAliases() throws Exception {
         logger.info("-->  delete and close indices");
         cluster().wipeIndices("test-idx-1", "test-idx-2");
         assertAcked(client.admin().indices().prepareClose("test-idx-3"));
-        assertAliasesMissing(client.admin().indices().prepareAliasesExist("alias-123", "alias-1").get());
+        assertTrue(client.admin().indices().prepareGetAliases("alias-123").get().getAliases().isEmpty());
+        assertTrue(client.admin().indices().prepareGetAliases("alias-1").get().getAliases().isEmpty());
 
         logger.info("--> restore snapshot without aliases");
         restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true)
@@ -539,9 +540,9 @@ public void testRestoreAliases() throws Exception {
             equalTo(restoreSnapshotResponse.getRestoreInfo().totalShards())));
 
         logger.info("--> check that aliases are not restored and existing aliases still exist");
-        assertAliasesMissing(client.admin().indices().prepareAliasesExist("alias-123", "alias-1").get());
-        assertAliasesExist(client.admin().indices().prepareAliasesExist("alias-3").get());
-
+        assertTrue(client.admin().indices().prepareGetAliases("alias-123").get().getAliases().isEmpty());
+        assertTrue(client.admin().indices().prepareGetAliases("alias-1").get().getAliases().isEmpty());
+        assertFalse(client.admin().indices().prepareGetAliases("alias-3").get().getAliases().isEmpty());
     }
 
     public void testRestoreTemplates() throws Exception {
@@ -577,8 +578,8 @@ public void testRestoreTemplates() throws Exception {
             .setIndices().setWaitForCompletion(true).get();
         assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0));
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0));
-        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(),
-            equalTo(SnapshotState.SUCCESS));
+        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().
+                getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS));
 
         logger.info("-->  delete test template");
         assertThat(client.admin().indices().prepareDeleteTemplate("test-template").get().isAcknowledged(), equalTo(true));
@@ -594,7 +595,6 @@ public void testRestoreTemplates() throws Exception {
         logger.info("--> check that template is restored");
         getIndexTemplatesResponse = client().admin().indices().prepareGetTemplates().get();
         assertIndexTemplateExists(getIndexTemplatesResponse, "test-template");
-
     }
 
     public void testIncludeGlobalState() throws Exception {
@@ -661,7 +661,7 @@ public void testIncludeGlobalState() throws Exception {
         assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0));
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0));
         assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-no-global-state")
-            .get().getSnapshots().get(0).state(),
+                        .get().getSnapshots("test-repo").get(0).state(),
             equalTo(SnapshotState.SUCCESS));
         SnapshotsStatusResponse snapshotsStatusResponse = client.admin().cluster().prepareSnapshotStatus("test-repo")
             .addSnapshots("test-snap-no-global-state").get();
@@ -675,7 +675,7 @@ public void testIncludeGlobalState() throws Exception {
         assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0));
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0));
         assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-with-global-state")
-            .get().getSnapshots().get(0).state(),
+                        .get().getSnapshots("test-repo").get(0).state(),
             equalTo(SnapshotState.SUCCESS));
         snapshotsStatusResponse = client.admin().cluster().prepareSnapshotStatus("test-repo")
             .addSnapshots("test-snap-with-global-state").get();
@@ -750,7 +750,7 @@ public void testIncludeGlobalState() throws Exception {
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
             equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
         assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-no-global-state-with-index")
-                .get().getSnapshots().get(0).state(),
+                        .get().getSnapshots("test-repo").get(0).state(),
             equalTo(SnapshotState.SUCCESS));
 
         logger.info("-->  delete global state and index ");
@@ -781,10 +781,10 @@ public void testIncludeGlobalState() throws Exception {
         assertFalse(client().admin().cluster().prepareGetPipeline("barbaz").get().isFound());
         assertNull(client().admin().cluster().prepareGetStoredScript("foobar").get().getSource());
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L));
-
     }
 
-    public void testSnapshotFileFailureDuringSnapshot() {
+    public void testSnapshotFileFailureDuringSnapshot() throws InterruptedException {
+        disableRepoConsistencyCheck("This test uses a purposely broken repository so it would fail consistency checks");
         Client client = client();
 
         logger.info("-->  creating repository");
@@ -823,8 +823,8 @@ public void testSnapshotFileFailureDuringSnapshot() {
                 }
                 GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("test-repo")
                     .addSnapshots("test-snap").get();
-                assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1));
-                SnapshotInfo snapshotInfo = getSnapshotsResponse.getSnapshots().get(0);
+                assertThat(getSnapshotsResponse.getSnapshots("test-repo").size(), equalTo(1));
+                SnapshotInfo snapshotInfo = getSnapshotsResponse.getSnapshots("test-repo").get(0);
                 if (snapshotInfo.state() == SnapshotState.SUCCESS) {
                     assertThat(snapshotInfo.shardFailures().size(), greaterThan(0));
                     assertThat(snapshotInfo.totalShards(), greaterThan(snapshotInfo.successfulShards()));
@@ -843,6 +843,8 @@ public void testSnapshotFileFailureDuringSnapshot() {
     }
 
     public void testDataFileFailureDuringSnapshot() throws Exception {
+        disableRepoConsistencyCheck("This test intentionally leaves a broken repository");
+
         Client client = client();
         logger.info("-->  creating repository");
         assertAcked(client.admin().cluster().preparePutRepository("test-repo")
@@ -879,8 +881,8 @@ public void testDataFileFailureDuringSnapshot() throws Exception {
             }
             GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("test-repo")
                 .addSnapshots("test-snap").get();
-            assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1));
-            SnapshotInfo snapshotInfo = getSnapshotsResponse.getSnapshots().get(0);
+            assertThat(getSnapshotsResponse.getSnapshots("test-repo").size(), equalTo(1));
+            SnapshotInfo snapshotInfo = getSnapshotsResponse.getSnapshots("test-repo").get(0);
             assertThat(snapshotInfo.state(), equalTo(SnapshotState.PARTIAL));
             assertThat(snapshotInfo.shardFailures().size(), greaterThan(0));
             assertThat(snapshotInfo.totalShards(), greaterThan(snapshotInfo.successfulShards()));
@@ -911,6 +913,8 @@ public void testDataFileFailureDuringSnapshot() throws Exception {
     }
 
     public void testDataFileFailureDuringRestore() throws Exception {
+        disableRepoConsistencyCheck("This test intentionally leaves a broken repository");
+
         Path repositoryLocation = randomRepoPath();
         Client client = client();
         logger.info("-->  creating repository");
@@ -974,6 +978,8 @@ public void testDataFileFailureDuringRestore() throws Exception {
     }
 
     public void testDataFileCorruptionDuringRestore() throws Exception {
+        disableRepoConsistencyCheck("This test intentionally leaves a broken repository");
+
         Path repositoryLocation = randomRepoPath();
         Client client = client();
         logger.info("-->  creating repository");
@@ -1238,7 +1244,6 @@ public void testDeletionOfFailingToRecoverIndexShouldStopRestore() throws Except
         assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
         SearchResponse countResponse = client.prepareSearch("test-idx").setSize(0).get();
         assertThat(countResponse.getHits().getTotalHits().value, equalTo(100L));
-
     }
 
     public void testUnallocatedShards() throws Exception {
@@ -1323,6 +1328,99 @@ public void testDeleteSnapshot() throws Exception {
         assertThat(numberOfFiles(repo), equalTo(numberOfFiles[0] + 2));
     }
 
+    public void testGetSnapshotsNoRepos() {
+        ensureGreen();
+        GetSnapshotsResponse getSnapshotsResponse = client().admin().cluster()
+                .prepareGetSnapshots(new String[]{"_all"})
+                .setSnapshots(randomFrom("_all", "*"))
+                .get();
+
+        assertTrue(getSnapshotsResponse.getRepositories().isEmpty());
+        assertTrue(getSnapshotsResponse.getFailedResponses().isEmpty());
+        assertTrue(getSnapshotsResponse.getSuccessfulResponses().isEmpty());
+    }
+
+    public void testGetSnapshotsMultipleRepos() {
+        final Client client = client();
+
+        List<String> snapshotList = new ArrayList<>();
+        List<String> repoList = new ArrayList<>();
+        Map<String, List<String>> repo2SnapshotNames = new HashMap<>();
+
+        logger.info("--> create an index and index some documents");
+        final String indexName = "test-idx";
+        assertAcked(prepareCreate(indexName));
+        ensureGreen();
+        for (int i = 0; i < 10; i++) {
+            index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+
+        for (int repoIndex = 0; repoIndex < randomIntBetween(2, 5); repoIndex++) {
+            final String repoName = "repo" + repoIndex;
+            repoList.add(repoName);
+            final Path repoPath = randomRepoPath();
+            logger.info("--> create repository with name " + repoName);
+            assertAcked(client.admin().cluster().preparePutRepository(repoName)
+                    .setType("fs").setSettings(Settings.builder().put("location", repoPath).build()));
+            List<String> snapshotNames = new ArrayList<>();
+            repo2SnapshotNames.put(repoName, snapshotNames);
+
+            for (int snapshotIndex = 0; snapshotIndex < randomIntBetween(2, 5); snapshotIndex++) {
+                final String snapshotName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+                snapshotList.add(snapshotName);
+                logger.info("--> create snapshot with index {} and name {} in repository {}", snapshotIndex, snapshotName, repoName);
+                CreateSnapshotResponse createSnapshotResponse = client.admin()
+                        .cluster()
+                        .prepareCreateSnapshot(repoName, snapshotName)
+                        .setWaitForCompletion(true)
+                        .setIndices(indexName)
+                        .get();
+                assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
+                snapshotNames.add(snapshotName);
+            }
+        }
+
+        logger.info("--> get and verify snapshots");
+        GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster()
+                .prepareGetSnapshots(randomFrom(new String[]{"_all"}, new String[]{"repo*"}, repoList.toArray(new String[0])))
+                .setSnapshots(randomFrom("_all", "*"))
+                .get();
+
+        for (Map.Entry<String, List<String>> repo2Names : repo2SnapshotNames.entrySet()) {
+            String repo = repo2Names.getKey();
+            List<String> snapshotNames = repo2Names.getValue();
+            List<SnapshotInfo> snapshots = getSnapshotsResponse.getSnapshots(repo);
+            assertEquals(snapshotNames, snapshots.stream().map(s -> s.snapshotId().getName()).collect(Collectors.toList()));
+        }
+
+        logger.info("--> specify all snapshot names with ignoreUnavailable=false");
+        GetSnapshotsResponse getSnapshotsResponse2 = client.admin().cluster()
+                .prepareGetSnapshots(randomFrom("_all", "repo*"))
+                .setIgnoreUnavailable(false)
+                .setSnapshots(snapshotList.toArray(new String[0]))
+                .get();
+
+        for (String repo : repoList) {
+            expectThrows(SnapshotMissingException.class, () -> getSnapshotsResponse2.getSnapshots(repo));
+        }
+
+
+        logger.info("--> specify all snapshot names with ignoreUnavailable=true");
+        GetSnapshotsResponse getSnapshotsResponse3 = client.admin().cluster()
+                .prepareGetSnapshots(randomFrom("_all", "repo*"))
+                .setIgnoreUnavailable(true)
+                .setSnapshots(snapshotList.toArray(new String[0]))
+                .get();
+
+        for (Map.Entry<String, List<String>> repo2Names : repo2SnapshotNames.entrySet()) {
+            String repo = repo2Names.getKey();
+            List<String> snapshotNames = repo2Names.getValue();
+            List<SnapshotInfo> snapshots = getSnapshotsResponse3.getSnapshots(repo);
+            assertEquals(snapshotNames, snapshots.stream().map(s -> s.snapshotId().getName()).collect(Collectors.toList()));
+        }
+    }
+
     public void testDeleteSnapshotWithMissingIndexAndShardMetadata() throws Exception {
         Client client = client();
 
@@ -1367,8 +1465,9 @@ public void testDeleteSnapshotWithMissingIndexAndShardMetadata() throws Exceptio
         client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-1").get();
 
         logger.info("--> make sure snapshot doesn't exist");
-        assertThrows(client.admin().cluster().prepareGetSnapshots("test-repo")
-                                             .addSnapshots("test-snap-1"), SnapshotMissingException.class);
+
+        expectThrows(SnapshotMissingException.class, () -> client.admin().cluster().prepareGetSnapshots("test-repo")
+                .addSnapshots("test-snap-1").get().getSnapshots("test-repo"));
 
         for (String index : indices) {
             assertTrue(Files.notExists(indicesPath.resolve(indexIds.get(index).getId())));
@@ -1407,8 +1506,8 @@ public void testDeleteSnapshotWithMissingMetadata() throws Exception {
         client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-1").get();
 
         logger.info("--> make sure snapshot doesn't exist");
-        assertThrows(client.admin().cluster().prepareGetSnapshots("test-repo")
-                                             .addSnapshots("test-snap-1"), SnapshotMissingException.class);
+        expectThrows(SnapshotMissingException.class, () -> client.admin().cluster().prepareGetSnapshots("test-repo")
+                .addSnapshots("test-snap-1").get().getSnapshots("test-repo"));
     }
 
     public void testDeleteSnapshotWithCorruptedSnapshotFile() throws Exception {
@@ -1444,7 +1543,9 @@ public void testDeleteSnapshotWithCorruptedSnapshotFile() throws Exception {
         client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-1").get();
 
         logger.info("--> make sure snapshot doesn't exist");
-        assertThrows(client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap-1"), SnapshotMissingException.class);
+        expectThrows(SnapshotMissingException.class,
+                () -> client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap-1").get().
+                        getSnapshots("test-repo"));
 
         logger.info("--> make sure that we can create the snapshot again");
         createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1")
@@ -1490,7 +1591,7 @@ public void testDeleteSnapshotWithCorruptedGlobalState() throws Exception {
             }
         }
 
-        List<SnapshotInfo> snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo").get().getSnapshots();
+        List<SnapshotInfo> snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo").get().getSnapshots("test-repo");
         assertThat(snapshotInfos.size(), equalTo(1));
         assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS));
         assertThat(snapshotInfos.get(0).snapshotId().getName(), equalTo("test-snap"));
@@ -1501,8 +1602,8 @@ public void testDeleteSnapshotWithCorruptedGlobalState() throws Exception {
         assertThat(snapshotStatusResponse.getSnapshots().get(0).getSnapshot().getSnapshotId().getName(), equalTo("test-snap"));
 
         assertAcked(client().admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get());
-        assertThrows(client().admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap"),
-            SnapshotMissingException.class);
+        expectThrows(SnapshotMissingException.class, () -> client().admin().cluster()
+                .prepareGetSnapshots("test-repo").addSnapshots("test-snap").get().getSnapshots("test-repo"));
         assertThrows(client().admin().cluster().prepareSnapshotStatus("test-repo").addSnapshots("test-snap"),
             SnapshotMissingException.class);
 
@@ -1703,8 +1804,6 @@ public void testRenameOnRestore() throws Exception {
                 .setIndices("test-idx-1").setRenamePattern("test-idx").setRenameReplacement("alias")
                 .setWaitForCompletion(true).setIncludeAliases(false).execute().actionGet();
         assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
-
-
     }
 
     public void testMoveShardWhileSnapshotting() throws Exception {
@@ -1749,7 +1848,7 @@ public void testMoveShardWhileSnapshotting() throws Exception {
         logger.info("--> done");
 
         List<SnapshotInfo> snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo")
-            .setSnapshots("test-snap").get().getSnapshots();
+                .setSnapshots("test-snap").get().getSnapshots("test-repo");
 
         assertThat(snapshotInfos.size(), equalTo(1));
         assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS));
@@ -1771,6 +1870,7 @@ public void testMoveShardWhileSnapshotting() throws Exception {
     }
 
     public void testDeleteRepositoryWhileSnapshotting() throws Exception {
+        disableRepoConsistencyCheck("This test uses a purposely broken repository so it would fail consistency checks");
         Client client = client();
         Path repositoryLocation = randomRepoPath();
         logger.info("-->  creating repository");
@@ -1834,7 +1934,7 @@ public void testDeleteRepositoryWhileSnapshotting() throws Exception {
         logger.info("--> done");
 
         List<SnapshotInfo> snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo")
-            .setSnapshots("test-snap").get().getSnapshots();
+                .setSnapshots("test-snap").get().getSnapshots("test-repo");
 
         assertThat(snapshotInfos.size(), equalTo(1));
         assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS));
@@ -1855,8 +1955,6 @@ public void testDeleteRepositoryWhileSnapshotting() throws Exception {
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L));
     }
 
-    @TestLogging("_root:DEBUG")  // this fails every now and then: https://github.com/elastic/elasticsearch/issues/18121 but without
-    // more logs we cannot find out why
     public void testReadonlyRepository() throws Exception {
         Client client = client();
         logger.info("-->  creating repository");
@@ -1883,8 +1981,8 @@ public void testReadonlyRepository() throws Exception {
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
             equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
 
-        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(),
-            equalTo(SnapshotState.SUCCESS));
+        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get()
+                .getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS));
 
         logger.info("--> delete index");
         cluster().wipeIndices("test-idx");
@@ -1905,8 +2003,8 @@ public void testReadonlyRepository() throws Exception {
 
         logger.info("--> list available shapshots");
         GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("readonly-repo").get();
-        assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
-        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1));
+        assertThat(getSnapshotsResponse.getSnapshots("readonly-repo"), notNullValue());
+        assertThat(getSnapshotsResponse.getSnapshots("readonly-repo").size(), equalTo(1));
 
         logger.info("--> try deleting snapshot");
         assertThrows(client.admin().cluster().prepareDeleteSnapshot("readonly-repo", "test-snap"), RepositoryException.class,
@@ -2047,8 +2145,8 @@ public void testSnapshotStatus() throws Exception {
         logger.info("--> checking that _current returns the currently running snapshot");
         GetSnapshotsResponse getResponse = client.admin().cluster().prepareGetSnapshots("test-repo")
             .setCurrentSnapshot().execute().actionGet();
-        assertThat(getResponse.getSnapshots().size(), equalTo(1));
-        SnapshotInfo snapshotInfo = getResponse.getSnapshots().get(0);
+        assertThat(getResponse.getSnapshots("test-repo").size(), equalTo(1));
+        SnapshotInfo snapshotInfo = getResponse.getSnapshots("test-repo").get(0);
         assertThat(snapshotInfo.state(), equalTo(SnapshotState.IN_PROGRESS));
 
         logger.info("--> unblocking blocked node");
@@ -2078,7 +2176,7 @@ public void testSnapshotStatus() throws Exception {
 
         logger.info("--> checking that _current no longer returns the snapshot");
         assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("_current")
-                .execute().actionGet().getSnapshots().isEmpty(),
+                        .execute().actionGet().getSnapshots("test-repo").isEmpty(),
             equalTo(true));
 
         // test that getting an unavailable snapshot status throws an exception if ignoreUnavailable is false on the request
@@ -2168,8 +2266,8 @@ public void testSnapshotMoreThanOnce() throws ExecutionException, InterruptedExc
         assertThat(createSnapshotResponseFirst.getSnapshotInfo().successfulShards(), greaterThan(0));
         assertThat(createSnapshotResponseFirst.getSnapshotInfo().successfulShards(),
             equalTo(createSnapshotResponseFirst.getSnapshotInfo().totalShards()));
-        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test").get().getSnapshots().get(0).state(),
-            equalTo(SnapshotState.SUCCESS));
+        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test").get()
+                .getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS));
         {
             SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo")
                 .setSnapshots("test").get().getSnapshots().get(0);
@@ -2184,8 +2282,8 @@ public void testSnapshotMoreThanOnce() throws ExecutionException, InterruptedExc
         assertThat(createSnapshotResponseSecond.getSnapshotInfo().successfulShards(), greaterThan(0));
         assertThat(createSnapshotResponseSecond.getSnapshotInfo().successfulShards(),
             equalTo(createSnapshotResponseSecond.getSnapshotInfo().totalShards()));
-        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-1").get().getSnapshots().get(0).state(),
-            equalTo(SnapshotState.SUCCESS));
+        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-1").get()
+                .getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS));
         {
             SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo")
                 .setSnapshots("test-1").get().getSnapshots().get(0);
@@ -2201,8 +2299,8 @@ public void testSnapshotMoreThanOnce() throws ExecutionException, InterruptedExc
         assertThat(createSnapshotResponseThird.getSnapshotInfo().successfulShards(), greaterThan(0));
         assertThat(createSnapshotResponseThird.getSnapshotInfo().successfulShards(),
             equalTo(createSnapshotResponseThird.getSnapshotInfo().totalShards()));
-        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-2").get().getSnapshots().get(0).state(),
-            equalTo(SnapshotState.SUCCESS));
+        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-2").get()
+                .getSnapshots("test-repo").get(0).state(), equalTo(SnapshotState.SUCCESS));
         {
             SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo")
                 .setSnapshots("test-2").get().getSnapshots().get(0);
@@ -2331,7 +2429,6 @@ public void testChangeSettingsOnRestore() throws Exception {
 
         assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "Foo")).get(), numdocs);
         assertHitCount(client.prepareSearch("test-idx").setSize(0).setQuery(matchQuery("field1", "bar")).get(), numdocs);
-
     }
 
     public void testRecreateBlocksOnRestore() throws Exception {
@@ -2425,6 +2522,8 @@ public void testRecreateBlocksOnRestore() throws Exception {
     }
 
     public void testCloseOrDeleteIndexDuringSnapshot() throws Exception {
+        disableRepoConsistencyCheck("This test intentionally leaves a broken repository");
+
         Client client = client();
 
         boolean allowPartial = randomBoolean();
@@ -2714,7 +2813,8 @@ public ClusterState execute(ClusterState currentState) {
                                 Collections.singletonList(indexId),
                                 System.currentTimeMillis(),
                                 repositoryData.getGenId(),
-                                shards.build()))))
+                                shards.build(),
+                                SnapshotInfoTests.randomUserMetadata()))))
                         .build();
             }
 
@@ -2736,8 +2836,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, final Cl
     }
 
     private boolean waitForIndex(final String index, TimeValue timeout) throws InterruptedException {
-        return awaitBusy(() -> client().admin().indices().prepareExists(index).execute().actionGet().isExists(),
-            timeout.millis(), TimeUnit.MILLISECONDS);
+        return awaitBusy(() -> indexExists(index), timeout.millis(), TimeUnit.MILLISECONDS);
     }
 
     private boolean waitForRelocationsToStart(final String index, TimeValue timeout) throws InterruptedException {
@@ -2746,6 +2845,8 @@ private boolean waitForRelocationsToStart(final String index, TimeValue timeout)
     }
 
     public void testSnapshotName() throws Exception {
+        disableRepoConsistencyCheck("This test does not create any data in the repository");
+
         final Client client = client();
 
         logger.info("-->  creating repository");
@@ -2758,7 +2859,8 @@ public void testSnapshotName() throws Exception {
         expectThrows(InvalidSnapshotNameException.class,
                      () -> client.admin().cluster().prepareCreateSnapshot("test-repo", "_foo").get());
         expectThrows(SnapshotMissingException.class,
-                     () -> client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("_foo").get());
+                () -> client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("_foo")
+                        .get().getSnapshots("test-repo"));
         expectThrows(SnapshotMissingException.class,
                      () -> client.admin().cluster().prepareDeleteSnapshot("test-repo", "_foo").get());
         expectThrows(SnapshotMissingException.class,
@@ -2766,6 +2868,8 @@ public void testSnapshotName() throws Exception {
     }
 
     public void testListCorruptedSnapshot() throws Exception {
+        disableRepoConsistencyCheck("This test intentionally leaves a broken repository");
+
         Client client = client();
         Path repo = randomRepoPath();
         logger.info("-->  creating repository at {}", repo.toAbsolutePath());
@@ -2803,14 +2907,14 @@ public void testListCorruptedSnapshot() throws Exception {
         logger.info("--> get snapshots request should return both snapshots");
         List<SnapshotInfo> snapshotInfos = client.admin().cluster()
                 .prepareGetSnapshots("test-repo")
-                .setIgnoreUnavailable(true).get().getSnapshots();
+                .setIgnoreUnavailable(true).get().getSnapshots("test-repo");
 
         assertThat(snapshotInfos.size(), equalTo(1));
         assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS));
         assertThat(snapshotInfos.get(0).snapshotId().getName(), equalTo("test-snap-1"));
 
         final SnapshotException ex = expectThrows(SnapshotException.class, () ->
-            client.admin().cluster().prepareGetSnapshots("test-repo").setIgnoreUnavailable(false).get());
+                client.admin().cluster().prepareGetSnapshots("test-repo").setIgnoreUnavailable(false).get().getSnapshots("test-repo"));
         assertThat(ex.getRepositoryName(), equalTo("test-repo"));
         assertThat(ex.getSnapshotName(), equalTo("test-snap-2"));
     }
@@ -2845,7 +2949,7 @@ public void testRestoreSnapshotWithCorruptedGlobalState() throws Exception {
             outChan.truncate(randomInt(10));
         }
 
-        List<SnapshotInfo> snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo").get().getSnapshots();
+        List<SnapshotInfo> snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo").get().getSnapshots("test-repo");
         assertThat(snapshotInfos.size(), equalTo(1));
         assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS));
         assertThat(snapshotInfos.get(0).snapshotId().getName(), equalTo("test-snap"));
@@ -2935,7 +3039,7 @@ public void testRestoreSnapshotWithCorruptedIndexMetadata() throws Exception {
             outChan.truncate(randomInt(10));
         }
 
-        List<SnapshotInfo> snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo").get().getSnapshots();
+        List<SnapshotInfo> snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo").get().getSnapshots("test-repo");
         assertThat(snapshotInfos.size(), equalTo(1));
         assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS));
         assertThat(snapshotInfos.get(0).snapshotId().getName(), equalTo("test-snap"));
@@ -3014,7 +3118,7 @@ public void testSnapshotWithCorruptedShardIndexFile() throws Exception {
         }
 
         logger.info("-->  verifying snapshot state for [{}]", snapshot1);
-        List<SnapshotInfo> snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo").get().getSnapshots();
+        List<SnapshotInfo> snapshotInfos = client().admin().cluster().prepareGetSnapshots("test-repo").get().getSnapshots("test-repo");
         assertThat(snapshotInfos.size(), equalTo(1));
         assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS));
         assertThat(snapshotInfos.get(0).snapshotId().getName(), equalTo(snapshot1));
@@ -3146,7 +3250,8 @@ public void testGetSnapshotsRequest() throws Exception {
                                                                  .cluster()
                                                                  .prepareGetSnapshots(repositoryName)
                                                                  .addSnapshots("non-existent-snapshot")
-                                                                 .get());
+                .get()
+                .getSnapshots(repositoryName));
         // with ignore unavailable set to true, should not throw an exception
         GetSnapshotsResponse getSnapshotsResponse = client.admin()
                                                           .cluster()
@@ -3154,7 +3259,7 @@ public void testGetSnapshotsRequest() throws Exception {
                                                           .setIgnoreUnavailable(true)
                                                           .addSnapshots("non-existent-snapshot")
                                                           .get();
-        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0));
+        assertThat(getSnapshotsResponse.getSnapshots(repositoryName).size(), equalTo(0));
 
         logger.info("--> creating an index and indexing documents");
         // Create index on 2 nodes and make sure each node has a primary by setting no replicas
@@ -3179,8 +3284,8 @@ public void testGetSnapshotsRequest() throws Exception {
                                    .prepareGetSnapshots("test-repo")
                                    .setSnapshots(randomFrom("_all", "_current", "snap-on-*", "*-on-empty-repo", "snap-on-empty-repo"))
                                    .get();
-        assertEquals(1, getSnapshotsResponse.getSnapshots().size());
-        assertEquals("snap-on-empty-repo", getSnapshotsResponse.getSnapshots().get(0).snapshotId().getName());
+        assertEquals(1, getSnapshotsResponse.getSnapshots("test-repo").size());
+        assertEquals("snap-on-empty-repo", getSnapshotsResponse.getSnapshots("test-repo").get(0).snapshotId().getName());
         unblockNode(repositoryName, initialBlockedNode); // unblock node
         responseListener.actionGet(TimeValue.timeValueMillis(10000L)); // timeout after 10 seconds
         client.admin().cluster().prepareDeleteSnapshot(repositoryName, "snap-on-empty-repo").get();
@@ -3234,8 +3339,8 @@ public void testGetSnapshotsRequest() throws Exception {
                                              .get();
         List<String> sortedNames = Arrays.asList(snapshotNames);
         Collections.sort(sortedNames);
-        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(numSnapshots));
-        assertThat(getSnapshotsResponse.getSnapshots().stream()
+        assertThat(getSnapshotsResponse.getSnapshots(repositoryName).size(), equalTo(numSnapshots));
+        assertThat(getSnapshotsResponse.getSnapshots(repositoryName).stream()
                                                       .map(s -> s.snapshotId().getName())
                                                       .sorted()
                                                       .collect(Collectors.toList()), equalTo(sortedNames));
@@ -3246,8 +3351,8 @@ public void testGetSnapshotsRequest() throws Exception {
                                    .get();
         sortedNames = Arrays.asList(snapshotNames);
         Collections.sort(sortedNames);
-        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(numSnapshots));
-        assertThat(getSnapshotsResponse.getSnapshots().stream()
+        assertThat(getSnapshotsResponse.getSnapshots(repositoryName).size(), equalTo(numSnapshots));
+        assertThat(getSnapshotsResponse.getSnapshots(repositoryName).stream()
                                                       .map(s -> s.snapshotId().getName())
                                                       .sorted()
                                                       .collect(Collectors.toList()), equalTo(sortedNames));
@@ -3262,8 +3367,8 @@ public void testGetSnapshotsRequest() throws Exception {
                                              .addSnapshots(snapshotNames)
                                              .addSnapshots(firstRegex, secondRegex)
                                              .get();
-        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(numSnapshots));
-        assertThat(getSnapshotsResponse.getSnapshots().stream()
+        assertThat(getSnapshotsResponse.getSnapshots(repositoryName).size(), equalTo(numSnapshots));
+        assertThat(getSnapshotsResponse.getSnapshots(repositoryName).stream()
                                                       .map(s -> s.snapshotId().getName())
                                                       .sorted()
                                                       .collect(Collectors.toList()), equalTo(sortedNames));
@@ -3335,6 +3440,9 @@ public void testSnapshotCanceledOnRemovedShard() throws Exception {
     }
 
     public void testSnapshotSucceedsAfterSnapshotFailure() throws Exception {
+        // TODO: Fix repo cleanup logic to handle these leaked snap-file and only exclude test-repo (the mock repo) here.
+        disableRepoConsistencyCheck(
+            "This test uses a purposely broken repository implementation that results in leaking snap-{uuid}.dat files");
         logger.info("--> creating repository");
         final Path repoPath = randomRepoPath();
         final Client client = client();
@@ -3389,7 +3497,7 @@ public void testSnapshotSucceedsAfterSnapshotFailure() throws Exception {
         assertEquals(0, createSnapshotResponse.getSnapshotInfo().failedShards());
         GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("test-repo-2")
                                                         .addSnapshots("test-snap-2").get();
-        assertEquals(SnapshotState.SUCCESS, getSnapshotsResponse.getSnapshots().get(0).state());
+        assertEquals(SnapshotState.SUCCESS, getSnapshotsResponse.getSnapshots("test-repo-2").get(0).state());
     }
 
     public void testSnapshotStatusOnFailedIndex() throws Exception {
@@ -3518,8 +3626,8 @@ public void testGetSnapshotsFromIndexBlobOnly() throws Exception {
             .setSnapshots("_all")
             .setVerbose(false)
             .get();
-        assertEquals(indicesPerSnapshot.size(), response.getSnapshots().size());
-        verifySnapshotInfo(response, indicesPerSnapshot);
+        assertEquals(indicesPerSnapshot.size(), response.getSnapshots("test-repo").size());
+        verifySnapshotInfo("test-repo", response, indicesPerSnapshot);
 
         logger.info("--> verify wildcard returns snapshot info");
         response = client().admin().cluster()
@@ -3527,8 +3635,8 @@ public void testGetSnapshotsFromIndexBlobOnly() throws Exception {
             .setSnapshots("test-snap-*")
             .setVerbose(false)
             .get();
-        assertEquals(indicesPerSnapshot.size(), response.getSnapshots().size());
-        verifySnapshotInfo(response, indicesPerSnapshot);
+        assertEquals(indicesPerSnapshot.size(), response.getSnapshots("test-repo").size());
+        verifySnapshotInfo("test-repo", response, indicesPerSnapshot);
 
         logger.info("--> verify individual requests return snapshot info");
         for (int i = 0; i < numSnapshots; i++) {
@@ -3537,8 +3645,8 @@ public void testGetSnapshotsFromIndexBlobOnly() throws Exception {
                 .setSnapshots("test-snap-" + i)
                 .setVerbose(false)
                 .get();
-            assertEquals(1, response.getSnapshots().size());
-            verifySnapshotInfo(response, indicesPerSnapshot);
+            assertEquals(1, response.getSnapshots("test-repo").size());
+            verifySnapshotInfo("test-repo", response, indicesPerSnapshot);
         }
     }
 
@@ -3605,6 +3713,7 @@ public void testSnapshottingWithMissingSequenceNumbers() {
         for (int i = 10; i < 15; i++) {
             index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i);
         }
+        client().admin().indices().prepareFlush(indexName).setForce(true).setWaitIfOngoing(true).get();
 
         stats = client().admin().indices().prepareStats(indexName).clear().get();
         shardStats = stats.getShards()[0];
@@ -3735,7 +3844,6 @@ public void testParallelRestoreOperationsFromSingleSnapshot() throws Exception {
         assertThat(client.prepareGet(restoredIndexName2, typeName, sameSourceIndex ? docId : docId2).get().isExists(), equalTo(true));
     }
 
-    @TestLogging("org.elasticsearch.snapshots:TRACE")
     public void testAbortedSnapshotDuringInitDoesNotStart() throws Exception {
         final Client client = client();
 
@@ -3793,7 +3901,8 @@ public void testAbortedSnapshotDuringInitDoesNotStart() throws Exception {
 
             assertAcked(delete.get());
             expectThrows(SnapshotMissingException.class, () ->
-                client.admin().cluster().prepareGetSnapshots("repository").setSnapshots("snap").get());
+                    client.admin().cluster().prepareGetSnapshots("repository").setSnapshots("snap").get()
+                            .getSnapshots("repository"));
 
             assertFalse("Expecting snapshot state to be updated", states.isEmpty());
             assertFalse("Expecting snapshot to be aborted and not started at all", states.contains(State.STARTED));
@@ -3860,8 +3969,9 @@ private RepositoryData getRepositoryData(Repository repository) throws Interrupt
         return repositoryData.get();
     }
 
-    private void verifySnapshotInfo(final GetSnapshotsResponse response, final Map<String, List<String>> indicesPerSnapshot) {
-        for (SnapshotInfo snapshotInfo : response.getSnapshots()) {
+    private void verifySnapshotInfo(final String repo, final GetSnapshotsResponse response,
+                                    final Map<String, List<String>> indicesPerSnapshot) {
+        for (SnapshotInfo snapshotInfo : response.getSnapshots("test-repo")) {
             final List<String> expected = snapshotInfo.indices();
             assertEquals(expected, indicesPerSnapshot.get(snapshotInfo.snapshotId().getName()));
             assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java
new file mode 100644
index 0000000000000..723ce4ae751a7
--- /dev/null
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotInfoTests.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class SnapshotInfoTests extends AbstractWireSerializingTestCase<SnapshotInfo> {
+
+    @Override
+    protected SnapshotInfo createTestInstance() {
+        SnapshotId snapshotId = new SnapshotId(randomAlphaOfLength(5), randomAlphaOfLength(5));
+        List<String> indices = Arrays.asList(randomArray(1, 10, String[]::new, () -> randomAlphaOfLengthBetween(2, 20)));
+
+        String reason = randomBoolean() ? null : randomAlphaOfLengthBetween(5, 15);
+
+        long startTime = randomNonNegativeLong();
+        long endTime = randomNonNegativeLong();
+
+        int totalShards = randomIntBetween(0, 100);
+        int failedShards = randomIntBetween(0, totalShards);
+
+        List<SnapshotShardFailure> shardFailures = Arrays.asList(randomArray(failedShards, failedShards,
+            SnapshotShardFailure[]::new, () -> {
+                String indexName = randomAlphaOfLengthBetween(3, 50);
+                int id = randomInt();
+                ShardId shardId = ShardId.fromString("[" + indexName + "][" + id + "]");
+
+                return new SnapshotShardFailure(randomAlphaOfLengthBetween(5, 10), shardId, randomAlphaOfLengthBetween(5, 10));
+            }));
+
+        Boolean includeGlobalState = randomBoolean() ? null : randomBoolean();
+
+        Map<String, Object> userMetadata = randomUserMetadata();
+
+        return new SnapshotInfo(snapshotId, indices, startTime, reason, endTime, totalShards, shardFailures,
+            includeGlobalState, userMetadata);
+    }
+
+    @Override
+    protected Writeable.Reader<SnapshotInfo> instanceReader() {
+        return SnapshotInfo::new;
+    }
+
+    @Override
+    protected SnapshotInfo mutateInstance(SnapshotInfo instance) {
+        switch (randomIntBetween(0, 7)) {
+            case 0:
+                SnapshotId snapshotId = new SnapshotId(
+                    randomValueOtherThan(instance.snapshotId().getName(), () -> randomAlphaOfLength(5)),
+                    randomValueOtherThan(instance.snapshotId().getUUID(), () -> randomAlphaOfLength(5)));
+                return new SnapshotInfo(snapshotId, instance.indices(), instance.startTime(), instance.reason(),
+                    instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(),
+                    instance.userMetadata());
+            case 1:
+                int indicesSize = randomValueOtherThan(instance.indices().size(), () -> randomIntBetween(1, 10));
+                List<String> indices = Arrays.asList(randomArray(indicesSize, indicesSize, String[]::new,
+                    () -> randomAlphaOfLengthBetween(2, 20)));
+                return new SnapshotInfo(instance.snapshotId(), indices, instance.startTime(), instance.reason(),
+                    instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(),
+                    instance.userMetadata());
+            case 2:
+                return new SnapshotInfo(instance.snapshotId(), instance.indices(),
+                    randomValueOtherThan(instance.startTime(), ESTestCase::randomNonNegativeLong), instance.reason(),
+                    instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(),
+                    instance.userMetadata());
+            case 3:
+                return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.startTime(),
+                    randomValueOtherThan(instance.reason(), () -> randomAlphaOfLengthBetween(5, 15)), instance.endTime(),
+                    instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(), instance.userMetadata());
+            case 4:
+                return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.startTime(), instance.reason(),
+                    randomValueOtherThan(instance.endTime(), ESTestCase::randomNonNegativeLong), instance.totalShards(),
+                    instance.shardFailures(), instance.includeGlobalState(), instance.userMetadata());
+            case 5:
+                int totalShards = randomValueOtherThan(instance.totalShards(), () -> randomIntBetween(0, 100));
+                int failedShards = randomIntBetween(0, totalShards);
+
+                List<SnapshotShardFailure> shardFailures = Arrays.asList(randomArray(failedShards, failedShards,
+                    SnapshotShardFailure[]::new, () -> {
+                        String indexName = randomAlphaOfLengthBetween(3, 50);
+                        int id = randomInt();
+                        ShardId shardId = ShardId.fromString("[" + indexName + "][" + id + "]");
+
+                        return new SnapshotShardFailure(randomAlphaOfLengthBetween(5, 10), shardId, randomAlphaOfLengthBetween(5, 10));
+                    }));
+                return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.startTime(), instance.reason(),
+                    instance.endTime(), totalShards, shardFailures, instance.includeGlobalState(), instance.userMetadata());
+            case 6:
+                return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.startTime(), instance.reason(),
+                    instance.endTime(), instance.totalShards(), instance.shardFailures(),
+                    Boolean.FALSE.equals(instance.includeGlobalState()), instance.userMetadata());
+            case 7:
+                return new SnapshotInfo(instance.snapshotId(), instance.indices(), instance.startTime(), instance.reason(),
+                    instance.endTime(), instance.totalShards(), instance.shardFailures(), instance.includeGlobalState(),
+                    randomValueOtherThan(instance.userMetadata(), SnapshotInfoTests::randomUserMetadata));
+            default:
+                throw new IllegalArgumentException("invalid randomization case");
+        }
+    }
+
+    public static Map<String, Object> randomUserMetadata() {
+        if (randomBoolean()) {
+            return null;
+        }
+
+        Map<String, Object> metadata = new HashMap<>();
+        long fields = randomLongBetween(0, 25);
+        for (int i = 0; i < fields; i++) {
+            if (randomBoolean()) {
+                metadata.put(randomValueOtherThanMany(metadata::containsKey, () -> randomAlphaOfLengthBetween(2,10)),
+                    randomAlphaOfLengthBetween(5, 15));
+            } else {
+                Map<String, Object> nested = new HashMap<>();
+                long nestedFields = randomLongBetween(0, 25);
+                for (int j = 0; j < nestedFields; j++) {
+                    nested.put(randomValueOtherThanMany(nested::containsKey, () -> randomAlphaOfLengthBetween(2,10)),
+                        randomAlphaOfLengthBetween(5, 15));
+                }
+                metadata.put(randomValueOtherThanMany(metadata::containsKey, () -> randomAlphaOfLengthBetween(2,10)), nested);
+            }
+        }
+        return metadata;
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
index 285234999564e..4c02b205e76a3 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java
@@ -22,8 +22,9 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.RequestValidators;
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryAction;
 import org.elasticsearch.action.admin.cluster.repositories.put.TransportPutRepositoryAction;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteAction;
@@ -86,6 +87,7 @@
 import org.elasticsearch.cluster.coordination.Coordinator;
 import org.elasticsearch.cluster.coordination.CoordinatorTests;
 import org.elasticsearch.cluster.coordination.DeterministicTaskQueue;
+import org.elasticsearch.cluster.coordination.ElectionStrategy;
 import org.elasticsearch.cluster.coordination.InMemoryPersistedState;
 import org.elasticsearch.cluster.coordination.MockSinglePrioritizingExecutor;
 import org.elasticsearch.cluster.metadata.AliasValidator;
@@ -96,8 +98,9 @@
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.cluster.metadata.MetaDataMappingService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.RoutingService;
+import org.elasticsearch.cluster.routing.BatchedRerouteService;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
@@ -105,8 +108,6 @@
 import org.elasticsearch.cluster.service.ClusterApplierService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterService;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -117,11 +118,7 @@
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
-import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.TestEnvironment;
@@ -146,10 +143,10 @@
 import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.node.ResponseCollectorService;
 import org.elasticsearch.plugins.PluginsService;
-import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.Repository;
-import org.elasticsearch.repositories.RepositoryData;
+import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
+import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchService;
@@ -167,16 +164,8 @@
 import org.junit.After;
 import org.junit.Before;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.file.DirectoryNotEmptyException;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -222,9 +211,11 @@ public void createServices() {
     }
 
     @After
-    public void verifyReposThenStopServices() throws IOException {
+    public void verifyReposThenStopServices() {
         try {
-            assertNoStaleRepositoryData();
+            BlobStoreTestUtil.assertConsistency(
+                (BlobStoreRepository) testClusterNodes.randomMasterNodeSafe().repositoriesService.repository("repo"),
+                Runnable::run);
         } finally {
             testClusterNodes.nodes.values().forEach(TestClusterNode::stop);
         }
@@ -522,109 +513,6 @@ public void run() {
         assertThat(snapshotIds, either(hasSize(1)).or(hasSize(0)));
     }
 
-    /**
-     * Assert that there are no unreferenced indices or unreferenced root-level metadata blobs in any repository.
-     * TODO: Expand the logic here to also check for unreferenced segment blobs and shard level metadata
-     */
-    private void assertNoStaleRepositoryData() throws IOException {
-        final Path repoPath = tempDir.resolve("repo").toAbsolutePath();
-        final List<Path> repos;
-        try (Stream<Path> reposDir = repoFilesByPrefix(repoPath)) {
-            repos = reposDir.filter(s -> s.getFileName().toString().startsWith("extra") == false).collect(Collectors.toList());
-        }
-        for (Path repoRoot : repos) {
-            cleanupEmptyTrees(repoRoot);
-            final Path latestIndexGenBlob = repoRoot.resolve("index.latest");
-            assertTrue("Could not find index.latest blob for repo at [" + repoRoot + ']', Files.exists(latestIndexGenBlob));
-            final long latestGen = ByteBuffer.wrap(Files.readAllBytes(latestIndexGenBlob)).getLong(0);
-            assertIndexGenerations(repoRoot, latestGen);
-            final RepositoryData repositoryData;
-            try (XContentParser parser =
-                     XContentHelper.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE,
-                         new BytesArray(Files.readAllBytes(repoRoot.resolve("index-" + latestGen))), XContentType.JSON)) {
-                repositoryData = RepositoryData.snapshotsFromXContent(parser, latestGen);
-            }
-            assertIndexUUIDs(repoRoot, repositoryData);
-            assertSnapshotUUIDs(repoRoot, repositoryData);
-        }
-    }
-
-    // Lucene's mock file system randomly generates empty `extra0` files that break the deletion of blob-store directories.
-    // We clean those up here before checking a blob-store for stale files.
-    private void cleanupEmptyTrees(Path repoPath) {
-        try {
-            Files.walkFileTree(repoPath, new SimpleFileVisitor<>() {
-
-                @Override
-                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                    if (file.getFileName().toString().startsWith("extra")) {
-                        Files.delete(file);
-                    }
-                    return FileVisitResult.CONTINUE;
-                }
-
-                @Override
-                public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
-                    try {
-                        Files.delete(dir);
-                    } catch (DirectoryNotEmptyException e) {
-                        // We're only interested in deleting empty trees here, just ignore directories with content
-                    }
-                    return FileVisitResult.CONTINUE;
-                }
-            });
-        } catch (IOException e) {
-            throw new AssertionError(e);
-        }
-    }
-
-    private static void assertIndexGenerations(Path repoRoot, long latestGen) throws IOException {
-        try (Stream<Path> repoRootBlobs = repoFilesByPrefix(repoRoot)) {
-            final long[] indexGenerations = repoRootBlobs.filter(p -> p.getFileName().toString().startsWith("index-"))
-                .map(p -> p.getFileName().toString().replace("index-", ""))
-                .mapToLong(Long::parseLong).sorted().toArray();
-            assertEquals(latestGen, indexGenerations[indexGenerations.length - 1]);
-            assertTrue(indexGenerations.length <= 2);
-        }
-    }
-
-    private static void assertIndexUUIDs(Path repoRoot, RepositoryData repositoryData) throws IOException {
-        final List<String> expectedIndexUUIDs =
-            repositoryData.getIndices().values().stream().map(IndexId::getId).collect(Collectors.toList());
-        try (Stream<Path> indexRoots = repoFilesByPrefix(repoRoot.resolve("indices"))) {
-            final List<String> foundIndexUUIDs = indexRoots.filter(s -> s.getFileName().toString().startsWith("extra") == false)
-                .map(p -> p.getFileName().toString()).collect(Collectors.toList());
-            assertThat(foundIndexUUIDs, containsInAnyOrder(expectedIndexUUIDs.toArray(Strings.EMPTY_ARRAY)));
-        }
-    }
-
-    private static void assertSnapshotUUIDs(Path repoRoot, RepositoryData repositoryData) throws IOException {
-        final List<String> expectedSnapshotUUIDs =
-            repositoryData.getSnapshotIds().stream().map(SnapshotId::getUUID).collect(Collectors.toList());
-        for (String prefix : new String[]{"snap-", "meta-"}) {
-            try (Stream<Path> repoRootBlobs = repoFilesByPrefix(repoRoot)) {
-                final Collection<String> foundSnapshotUUIDs = repoRootBlobs.filter(p -> p.getFileName().toString().startsWith(prefix))
-                    .map(p -> p.getFileName().toString().replace(prefix, "").replace(".dat", ""))
-                    .collect(Collectors.toSet());
-                assertThat(foundSnapshotUUIDs, containsInAnyOrder(expectedSnapshotUUIDs.toArray(Strings.EMPTY_ARRAY)));
-            }
-        }
-    }
-
-    /**
-     * List contents of a blob path and return an empty stream if the path doesn't exist.
-     * @param prefix Path to find children for
-     * @return stream of child paths
-     * @throws IOException on failure
-     */
-    private static Stream<Path> repoFilesByPrefix(Path prefix) throws IOException {
-        try {
-            return Files.list(prefix);
-        } catch (FileNotFoundException | NoSuchFileException e) {
-            return Stream.empty();
-        }
-    }
-
     private void clearDisruptionsAndAwaitSync() {
         testClusterNodes.clearNetworkDisruptions();
         runUntil(() -> {
@@ -803,14 +691,14 @@ public TestClusterNode nodeById(final String nodeId) {
         }
 
         private TestClusterNode newMasterNode(String nodeName) throws IOException {
-            return newNode(nodeName, DiscoveryNode.Role.MASTER);
+            return newNode(nodeName, DiscoveryNodeRole.MASTER_ROLE);
         }
 
         private TestClusterNode newDataNode(String nodeName) throws IOException {
-            return newNode(nodeName, DiscoveryNode.Role.DATA);
+            return newNode(nodeName, DiscoveryNodeRole.DATA_ROLE);
         }
 
-        private TestClusterNode newNode(String nodeName, DiscoveryNode.Role role) throws IOException {
+        private TestClusterNode newNode(String nodeName, DiscoveryNodeRole role) throws IOException {
             return new TestClusterNode(
                 new DiscoveryNode(nodeName, randomAlphaOfLength(10), buildNewFakeTransportAddress(), emptyMap(),
                     Collections.singleton(role), Version.CURRENT), this::getDisruption);
@@ -860,13 +748,14 @@ public void disconnectNode(TestClusterNode node) {
         }
 
         public void clearNetworkDisruptions() {
-            disruptedLinks.disconnected.forEach(nodeName -> {
+            final Set<String> disconnectedNodes = new HashSet<>(disruptedLinks.disconnected);
+            disruptedLinks.clear();
+            disconnectedNodes.forEach(nodeName -> {
                 if (testClusterNodes.nodes.containsKey(nodeName)) {
                     final DiscoveryNode node = testClusterNodes.nodes.get(nodeName).node;
                     testClusterNodes.nodes.values().forEach(n -> n.transportService.getConnectionManager().openConnection(node, null));
                 }
             });
-            disruptedLinks.clear();
         }
 
         private NetworkDisruption.DisruptedLinks getDisruption() {
@@ -1061,9 +950,19 @@ protected void assertSnapshotOrGenericThread() {
                 transportService, indicesService, actionFilters, indexNameExpressionResolver);
             final ShardStateAction shardStateAction = new ShardStateAction(
                 clusterService, transportService, allocationService,
-                new RoutingService(clusterService, allocationService),
+                new BatchedRerouteService(clusterService, allocationService::reroute),
                 threadPool
             );
+            Map<ActionType, TransportAction> actions = new HashMap<>();
+            actions.put(GlobalCheckpointSyncAction.TYPE,
+                new GlobalCheckpointSyncAction(settings, transportService, clusterService, indicesService,
+                    threadPool, shardStateAction, actionFilters, indexNameExpressionResolver));
+            actions.put(RetentionLeaseBackgroundSyncAction.TYPE,
+                new RetentionLeaseBackgroundSyncAction(settings, transportService, clusterService, indicesService, threadPool,
+                    shardStateAction, actionFilters, indexNameExpressionResolver));
+            actions.put(RetentionLeaseSyncAction.TYPE,
+                new RetentionLeaseSyncAction(settings, transportService, clusterService, indicesService, threadPool,
+                    shardStateAction, actionFilters, indexNameExpressionResolver));
             final MetaDataMappingService metaDataMappingService = new MetaDataMappingService(clusterService, indicesService);
             indicesClusterStateService = new IndicesClusterStateService(
                 settings,
@@ -1089,34 +988,7 @@ protected void assertSnapshotOrGenericThread() {
                         shardStateAction,
                         actionFilters,
                         indexNameExpressionResolver)),
-                new GlobalCheckpointSyncAction(
-                    settings,
-                    transportService,
-                    clusterService,
-                    indicesService,
-                    threadPool,
-                    shardStateAction,
-                    actionFilters,
-                    indexNameExpressionResolver),
-                new RetentionLeaseSyncAction(
-                    settings,
-                    transportService,
-                    clusterService,
-                    indicesService,
-                    threadPool,
-                    shardStateAction,
-                    actionFilters,
-                    indexNameExpressionResolver),
-                    new RetentionLeaseBackgroundSyncAction(
-                            settings,
-                            transportService,
-                            clusterService,
-                            indicesService,
-                            threadPool,
-                            shardStateAction,
-                            actionFilters,
-                            indexNameExpressionResolver));
-            Map<Action, TransportAction> actions = new HashMap<>();
+                client);
             final MetaDataCreateIndexService metaDataCreateIndexService = new MetaDataCreateIndexService(settings, clusterService,
                 indicesService,
                 allocationService, new AliasValidator(), environment, indexScopedSettings,
@@ -1129,18 +1001,19 @@ allocationService, new AliasValidator(), environment, indexScopedSettings,
                 ));
             final MappingUpdatedAction mappingUpdatedAction = new MappingUpdatedAction(settings, clusterSettings);
             mappingUpdatedAction.setClient(client);
-            final TransportShardBulkAction transportShardBulkAction = new TransportShardBulkAction(settings, transportService,
-                clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedAction, new UpdateHelper(scriptService),
-                actionFilters, indexNameExpressionResolver);
             actions.put(BulkAction.INSTANCE,
                 new TransportBulkAction(threadPool, transportService, clusterService,
                     new IngestService(
                         clusterService, threadPool, environment, scriptService,
                         new AnalysisModule(environment, Collections.emptyList()).getAnalysisRegistry(),
                         Collections.emptyList()),
-                    transportShardBulkAction, client, actionFilters, indexNameExpressionResolver,
+                    client, actionFilters, indexNameExpressionResolver,
                     new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver)
                 ));
+            final TransportShardBulkAction transportShardBulkAction = new TransportShardBulkAction(settings, transportService,
+                clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedAction, new UpdateHelper(scriptService),
+                actionFilters, indexNameExpressionResolver);
+            actions.put(TransportShardBulkAction.TYPE, transportShardBulkAction);
             final RestoreService restoreService = new RestoreService(
                 clusterService, repositoriesService, allocationService,
                 metaDataCreateIndexService,
@@ -1154,7 +1027,7 @@ clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedActi
             );
             actions.put(PutMappingAction.INSTANCE,
                 new TransportPutMappingAction(transportService, clusterService, threadPool, metaDataMappingService,
-                    actionFilters, indexNameExpressionResolver, new TransportPutMappingAction.RequestValidators(Collections.emptyList())));
+                    actionFilters, indexNameExpressionResolver, new RequestValidators<>(Collections.emptyList())));
             final ResponseCollectorService responseCollectorService = new ResponseCollectorService(clusterService);
             final SearchTransportService searchTransportService = new SearchTransportService(transportService,
                 SearchExecutionStatsCollector.makeWrapper(responseCollectorService));
@@ -1243,7 +1116,8 @@ public void start(ClusterState initialState) {
                 allocationService, masterService, () -> persistedState,
                 hostsResolver -> testClusterNodes.nodes.values().stream().filter(n -> n.node.isMasterNode())
                     .map(n -> n.node.getAddress()).collect(Collectors.toList()),
-                clusterService.getClusterApplierService(), Collections.emptyList(), random());
+                clusterService.getClusterApplierService(), Collections.emptyList(), random(),
+                new BatchedRerouteService(clusterService, allocationService::reroute), ElectionStrategy.DEFAULT_INSTANCE);
             masterService.setClusterStatePublisher(coordinator);
             coordinator.start();
             masterService.start();
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java
index 20dd3693f78cd..0b18ce15abd26 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java
@@ -77,7 +77,7 @@ public void testRetryPostingSnapshotStatusMessages() throws Exception {
         waitForBlock(blockedNode, "test-repo", TimeValue.timeValueSeconds(60));
 
         final SnapshotId snapshotId = client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap")
-            .get().getSnapshots().get(0).snapshotId();
+            .get().getSnapshots("test-repo").get(0).snapshotId();
 
         logger.info("--> start disrupting cluster");
         final NetworkDisruption networkDisruption = new NetworkDisruption(new NetworkDisruption.TwoPartitions(masterNode, dataNode),
@@ -106,7 +106,7 @@ public void testRetryPostingSnapshotStatusMessages() throws Exception {
             GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster()
                 .prepareGetSnapshots("test-repo")
                 .setSnapshots("test-snap").get();
-            SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
+            SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots("test-repo").get(0);
             logger.info("Snapshot status [{}], successfulShards [{}]", snapshotInfo.state(), snapshotInfo.successfulShards());
             assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS));
             assertThat(snapshotInfo.successfulShards(), equalTo(shards));
diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java
index 6c8ddfb56c1cf..46404af9f4037 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java
@@ -71,7 +71,8 @@ private Entry randomSnapshot() {
                 shardState.failed() ? randomAlphaOfLength(10) : null));
         }
         ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards = builder.build();
-        return new Entry(snapshot, includeGlobalState, partial, state, indices, startTime, repositoryStateId, shards);
+        return new Entry(snapshot, includeGlobalState, partial, state, indices, startTime, repositoryStateId, shards,
+            SnapshotInfoTests.randomUserMetadata());
     }
 
     @Override
diff --git a/server/src/test/java/org/elasticsearch/snapshots/mockstore/BlobContainerWrapper.java b/server/src/test/java/org/elasticsearch/snapshots/mockstore/BlobContainerWrapper.java
index 5666869a1aa0b..df1d003aa55a5 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/mockstore/BlobContainerWrapper.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/mockstore/BlobContainerWrapper.java
@@ -64,6 +64,11 @@ public void deleteBlob(String blobName) throws IOException {
         delegate.deleteBlob(blobName);
     }
 
+    @Override
+    public void delete() throws IOException {
+        delegate.delete();
+    }
+
     @Override
     public void deleteBlobIgnoringIfNotExists(final String blobName) throws IOException {
         delegate.deleteBlobIgnoringIfNotExists(blobName);
@@ -74,6 +79,11 @@ public Map<String, BlobMetaData> listBlobs() throws IOException {
         return delegate.listBlobs();
     }
 
+    @Override
+    public Map<String, BlobContainer> children() throws IOException {
+        return delegate.children();
+    }
+
     @Override
     public Map<String, BlobMetaData> listBlobsByPrefix(String blobNamePrefix) throws IOException {
         return delegate.listBlobsByPrefix(blobNamePrefix);
diff --git a/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
index 9ce111e1d3011..21ade636f53df 100644
--- a/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
+++ b/server/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
@@ -52,12 +52,14 @@
 import java.security.NoSuchAlgorithmException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
 
 public class MockRepository extends FsRepository {
     private static final Logger logger = LogManager.getLogger(MockRepository.class);
@@ -332,12 +334,32 @@ public void deleteBlobIgnoringIfNotExists(String blobName) throws IOException {
                 super.deleteBlobIgnoringIfNotExists(blobName);
             }
 
+            @Override
+            public void delete() throws IOException {
+                for (BlobContainer child : children().values()) {
+                    child.delete();
+                }
+                for (String blob : listBlobs().values().stream().map(BlobMetaData::name).collect(Collectors.toList())) {
+                    deleteBlobIgnoringIfNotExists(blob);
+                }
+                blobStore().blobContainer(path().parent()).deleteBlob(path().toArray()[path().toArray().length - 1]);
+            }
+
             @Override
             public Map<String, BlobMetaData> listBlobs() throws IOException {
                 maybeIOExceptionOrBlock("");
                 return super.listBlobs();
             }
 
+            @Override
+            public Map<String, BlobContainer> children() throws IOException {
+                final Map<String, BlobContainer> res = new HashMap<>();
+                for (Map.Entry<String, BlobContainer> entry : super.children().entrySet()) {
+                    res.put(entry.getKey(), new MockBlobContainer(entry.getValue()));
+                }
+                return res;
+            }
+
             @Override
             public Map<String, BlobMetaData> listBlobsByPrefix(String blobNamePrefix) throws IOException {
                 maybeIOExceptionOrBlock(blobNamePrefix);
diff --git a/server/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java b/server/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java
index 07c2dac0e7cf5..f3e4e00e81976 100644
--- a/server/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java
+++ b/server/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java
@@ -58,7 +58,7 @@ public static void aggregateAndCheckFromSeveralShards(ESIntegTestCase testCase)
     }
 
     private static void checkSignificantTermsAggregationCorrect(ESIntegTestCase testCase) {
-        SearchResponse response = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE).addAggregation(
+        SearchResponse response = client().prepareSearch(INDEX_NAME).addAggregation(
                 terms("class").field(CLASS_FIELD).subAggregation(significantTerms("sig_terms").field(TEXT_FIELD)))
                 .execute().actionGet();
         assertSearchResponse(response);
diff --git a/server/src/test/java/org/elasticsearch/threadpool/SchedulerTests.java b/server/src/test/java/org/elasticsearch/threadpool/SchedulerTests.java
index 186f9e86b1e76..2f0251433091d 100644
--- a/server/src/test/java/org/elasticsearch/threadpool/SchedulerTests.java
+++ b/server/src/test/java/org/elasticsearch/threadpool/SchedulerTests.java
@@ -153,4 +153,16 @@ public void testScheduledOnScheduler() throws InterruptedException {
             Scheduler.terminate(executor, 10, TimeUnit.SECONDS);
         }
     }
+
+    public void testScheduleAtFixedRate() throws InterruptedException {
+        ScheduledThreadPoolExecutor executor = Scheduler.initScheduler(Settings.EMPTY);
+        try {
+            CountDownLatch missingExecutions = new CountDownLatch(randomIntBetween(1, 10));
+            executor.scheduleAtFixedRate(missingExecutions::countDown,
+                randomIntBetween(1, 10), randomIntBetween(1, 10), TimeUnit.MILLISECONDS);
+            assertTrue(missingExecutions.await(30, TimeUnit.SECONDS));
+        } finally {
+            Scheduler.terminate(executor, 10, TimeUnit.SECONDS);
+        }
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
index 5f6d31dce3e97..e93755873c917 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java
@@ -58,7 +58,6 @@
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.test.transport.StubbableConnectionManager;
 import org.elasticsearch.test.transport.StubbableTransport;
@@ -678,7 +677,6 @@ public void run() {
         }
     }
 
-    @TestLogging("_root:DEBUG, org.elasticsearch.transport:TRACE")
     public void testCloseWhileConcurrentlyConnecting() throws IOException, InterruptedException, BrokenBarrierException {
         List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
         try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
index 1105fe137e322..86c5e569f1eb5 100644
--- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.AbstractScopedSettings;
@@ -33,6 +34,7 @@
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -40,9 +42,9 @@
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -545,8 +547,8 @@ private ActionListener<Void> connectionListener(final CountDownLatch latch) {
         return ActionListener.wrap(x -> latch.countDown(), x -> fail());
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/41067")
-    public void testCollectNodes() throws InterruptedException, IOException {
+    @TestLogging("org.elasticsearch.transport:TRACE") // added for https://github.com/elastic/elasticsearch/issues/41067
+    public void testCollectNodes() throws Exception {
         final Settings settings = Settings.EMPTY;
         final List<DiscoveryNode> knownNodes_c1 = new CopyOnWriteArrayList<>();
         final List<DiscoveryNode> knownNodes_c2 = new CopyOnWriteArrayList<>();
@@ -665,6 +667,7 @@ public void onFailure(Exception e) {
                             new ActionListener<BiFunction<String, String, DiscoveryNode>>() {
                                 @Override
                                 public void onResponse(BiFunction<String, String, DiscoveryNode> stringStringDiscoveryNodeBiFunction) {
+                                    logger.warn("unexpected call", new Exception("just for the stack trace"));
                                     try {
                                         fail("should not be called");
                                     } finally {
@@ -739,49 +742,49 @@ public void testGetNodePredicateNodeRoles() {
         Predicate<DiscoveryNode> nodePredicate = RemoteClusterService.getNodePredicate(Settings.EMPTY);
         {
             DiscoveryNode all = new DiscoveryNode("id", address, Collections.emptyMap(),
-                    new HashSet<>(EnumSet.allOf(DiscoveryNode.Role.class)), Version.CURRENT);
+                    DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT);
             assertTrue(nodePredicate.test(all));
         }
         {
             DiscoveryNode dataMaster = new DiscoveryNode("id", address, Collections.emptyMap(),
-                    new HashSet<>(EnumSet.of(DiscoveryNode.Role.DATA, DiscoveryNode.Role.MASTER)), Version.CURRENT);
+                    Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT);
             assertTrue(nodePredicate.test(dataMaster));
         }
         {
             DiscoveryNode dedicatedMaster = new DiscoveryNode("id", address, Collections.emptyMap(),
-                    new HashSet<>(EnumSet.of(DiscoveryNode.Role.MASTER)), Version.CURRENT);
+                    Set.of(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT);
             assertFalse(nodePredicate.test(dedicatedMaster));
         }
         {
             DiscoveryNode dedicatedIngest = new DiscoveryNode("id", address, Collections.emptyMap(),
-                    new HashSet<>(EnumSet.of(DiscoveryNode.Role.INGEST)), Version.CURRENT);
+                    Set.of(DiscoveryNodeRole.INGEST_ROLE), Version.CURRENT);
             assertTrue(nodePredicate.test(dedicatedIngest));
         }
         {
             DiscoveryNode masterIngest = new DiscoveryNode("id", address, Collections.emptyMap(),
-                    new HashSet<>(EnumSet.of(DiscoveryNode.Role.INGEST, DiscoveryNode.Role.MASTER)), Version.CURRENT);
+                    Set.of(DiscoveryNodeRole.INGEST_ROLE, DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT);
             assertTrue(nodePredicate.test(masterIngest));
         }
         {
             DiscoveryNode dedicatedData = new DiscoveryNode("id", address, Collections.emptyMap(),
-                    new HashSet<>(EnumSet.of(DiscoveryNode.Role.DATA)), Version.CURRENT);
+                    Set.of(DiscoveryNodeRole.DATA_ROLE), Version.CURRENT);
             assertTrue(nodePredicate.test(dedicatedData));
         }
         {
             DiscoveryNode ingestData = new DiscoveryNode("id", address, Collections.emptyMap(),
-                    new HashSet<>(EnumSet.of(DiscoveryNode.Role.DATA, DiscoveryNode.Role.INGEST)), Version.CURRENT);
+                    Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.INGEST_ROLE), Version.CURRENT);
             assertTrue(nodePredicate.test(ingestData));
         }
         {
             DiscoveryNode coordOnly = new DiscoveryNode("id", address, Collections.emptyMap(),
-                    new HashSet<>(EnumSet.noneOf(DiscoveryNode.Role.class)), Version.CURRENT);
+                    Set.of(), Version.CURRENT);
             assertTrue(nodePredicate.test(coordOnly));
         }
     }
 
     public void testGetNodePredicateNodeVersion() {
         TransportAddress address = new TransportAddress(TransportAddress.META_ADDRESS, 0);
-        Set<DiscoveryNode.Role> roles = new HashSet<>(EnumSet.allOf(DiscoveryNode.Role.class));
+        Set<DiscoveryNodeRole> roles = DiscoveryNodeRole.BUILT_IN_ROLES;
         Predicate<DiscoveryNode> nodePredicate = RemoteClusterService.getNodePredicate(Settings.EMPTY);
         Version version = VersionUtils.randomVersion(random());
         DiscoveryNode node = new DiscoveryNode("id", address, Collections.emptyMap(), roles, version);
@@ -790,7 +793,7 @@ public void testGetNodePredicateNodeVersion() {
 
     public void testGetNodePredicateNodeAttrs() {
         TransportAddress address = new TransportAddress(TransportAddress.META_ADDRESS, 0);
-        Set<DiscoveryNode.Role> roles = new HashSet<>(EnumSet.allOf(DiscoveryNode.Role.class));
+        Set<DiscoveryNodeRole> roles = DiscoveryNodeRole.BUILT_IN_ROLES;
         Settings settings = Settings.builder().put("cluster.remote.node.attr", "gateway").build();
         Predicate<DiscoveryNode> nodePredicate = RemoteClusterService.getNodePredicate(settings);
         {
@@ -816,8 +819,8 @@ public void testGetNodePredicatesCombination() {
         TransportAddress address = new TransportAddress(TransportAddress.META_ADDRESS, 0);
         Settings settings = Settings.builder().put("cluster.remote.node.attr", "gateway").build();
         Predicate<DiscoveryNode> nodePredicate = RemoteClusterService.getNodePredicate(settings);
-        Set<DiscoveryNode.Role> allRoles = new HashSet<>(EnumSet.allOf(DiscoveryNode.Role.class));
-        Set<DiscoveryNode.Role> dedicatedMasterRoles = new HashSet<>(EnumSet.of(DiscoveryNode.Role.MASTER));
+        Set<DiscoveryNodeRole> allRoles = DiscoveryNodeRole.BUILT_IN_ROLES;
+        Set<DiscoveryNodeRole> dedicatedMasterRoles = Set.of(DiscoveryNodeRole.MASTER_ROLE);
         {
             DiscoveryNode node = new DiscoveryNode("id", address, Collections.singletonMap("gateway", "true"),
                     dedicatedMasterRoles, Version.CURRENT);
@@ -840,6 +843,81 @@ public void testGetNodePredicatesCombination() {
         }
     }
 
+    public void testReconnectWhenSeedsNodesAreUpdated() throws Exception {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService cluster_node_0 = startTransport("cluster_node_0", knownNodes, Version.CURRENT);
+             MockTransportService cluster_node_1 = startTransport("cluster_node_1", knownNodes, Version.CURRENT)) {
+
+            final DiscoveryNode node0 = cluster_node_0.getLocalDiscoNode();
+            final DiscoveryNode node1 = cluster_node_1.getLocalDiscoNode();
+            knownNodes.add(node0);
+            knownNodes.add(node1);
+            Collections.shuffle(knownNodes, random());
+
+            try (MockTransportService transportService =
+                     MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                transportService.start();
+                transportService.acceptIncomingRequests();
+
+                final Settings.Builder builder = Settings.builder();
+                builder.putList("cluster.remote.cluster_test.seeds", Collections.singletonList(node0.getAddress().toString()));
+                try (RemoteClusterService service = new RemoteClusterService(builder.build(), transportService)) {
+                    assertFalse(service.isCrossClusterSearchEnabled());
+                    service.initializeRemoteClusters();
+                    assertTrue(service.isCrossClusterSearchEnabled());
+
+                    final RemoteClusterConnection firstRemoteClusterConnection = service.getRemoteClusterConnection("cluster_test");
+                    assertTrue(firstRemoteClusterConnection.isNodeConnected(node0));
+                    assertTrue(firstRemoteClusterConnection.isNodeConnected(node1));
+                    assertEquals(2, firstRemoteClusterConnection.getNumNodesConnected());
+                    assertFalse(firstRemoteClusterConnection.isClosed());
+
+                    final CountDownLatch firstLatch = new CountDownLatch(1);
+                    service.updateRemoteCluster(
+                        "cluster_test",
+                        Collections.singletonList(node0.getAddress().toString()), null,
+                        genericProfile("cluster_test"), connectionListener(firstLatch));
+                    firstLatch.await();
+
+                    assertTrue(service.isCrossClusterSearchEnabled());
+                    assertTrue(firstRemoteClusterConnection.isNodeConnected(node0));
+                    assertTrue(firstRemoteClusterConnection.isNodeConnected(node1));
+                    assertEquals(2, firstRemoteClusterConnection.getNumNodesConnected());
+                    assertFalse(firstRemoteClusterConnection.isClosed());
+                    assertSame(firstRemoteClusterConnection, service.getRemoteClusterConnection("cluster_test"));
+
+                    final List<String> newSeeds = new ArrayList<>();
+                    newSeeds.add(node1.getAddress().toString());
+                    if (randomBoolean()) {
+                        newSeeds.add(node0.getAddress().toString());
+                        Collections.shuffle(newSeeds, random());
+                    }
+
+                    final CountDownLatch secondLatch = new CountDownLatch(1);
+                    service.updateRemoteCluster(
+                        "cluster_test",
+                        newSeeds, null,
+                        genericProfile("cluster_test"), connectionListener(secondLatch));
+                    secondLatch.await();
+
+                    assertTrue(service.isCrossClusterSearchEnabled());
+                    assertBusy(() -> {
+                        assertFalse(firstRemoteClusterConnection.isNodeConnected(node0));
+                        assertFalse(firstRemoteClusterConnection.isNodeConnected(node1));
+                        assertEquals(0, firstRemoteClusterConnection.getNumNodesConnected());
+                        assertTrue(firstRemoteClusterConnection.isClosed());
+                    });
+
+                    final RemoteClusterConnection secondRemoteClusterConnection = service.getRemoteClusterConnection("cluster_test");
+                    assertTrue(secondRemoteClusterConnection.isNodeConnected(node0));
+                    assertTrue(secondRemoteClusterConnection.isNodeConnected(node1));
+                    assertEquals(2, secondRemoteClusterConnection.getNumNodesConnected());
+                    assertFalse(secondRemoteClusterConnection.isClosed());
+                }
+            }
+        }
+    }
+
     public void testRemoteClusterWithProxy() throws Exception {
         List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
         try (MockTransportService cluster_1_node0 = startTransport("cluster_1_node0", knownNodes, Version.CURRENT);
diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java
index fa6b293adf7a7..65292918752b8 100644
--- a/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java
+++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java
@@ -20,8 +20,6 @@
 package org.elasticsearch.transport;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
@@ -177,43 +175,6 @@ public void testNodeConnectWithDifferentNodeId() {
         assertFalse(handleA.transportService.nodeConnected(discoveryNode));
     }
 
-    public void testNodeConnectWithDifferentNodeIdSucceedsIfThisIsTransportClientOfSimpleNodeSampler() {
-        Settings.Builder settings = Settings.builder().put("cluster.name", "test");
-        Settings transportClientSettings = settings.put(Client.CLIENT_TYPE_SETTING_S.getKey(), TransportClient.CLIENT_TYPE).build();
-        NetworkHandle handleA = startServices("TS_A", transportClientSettings, Version.CURRENT);
-        NetworkHandle handleB = startServices("TS_B", settings.build(), Version.CURRENT);
-        DiscoveryNode discoveryNode = new DiscoveryNode(
-            randomAlphaOfLength(10),
-            handleB.discoveryNode.getAddress(),
-            emptyMap(),
-            emptySet(),
-            handleB.discoveryNode.getVersion());
-
-        handleA.transportService.connectToNode(discoveryNode, TestProfiles.LIGHT_PROFILE);
-        assertTrue(handleA.transportService.nodeConnected(discoveryNode));
-    }
-
-    public void testNodeConnectWithDifferentNodeIdFailsWhenSnifferTransportClient() {
-        Settings.Builder settings = Settings.builder().put("cluster.name", "test");
-        Settings transportClientSettings = settings.put(Client.CLIENT_TYPE_SETTING_S.getKey(), TransportClient.CLIENT_TYPE)
-            .put(TransportClient.CLIENT_TRANSPORT_SNIFF.getKey(), true)
-            .build();
-        NetworkHandle handleA = startServices("TS_A", transportClientSettings, Version.CURRENT);
-        NetworkHandle handleB = startServices("TS_B", settings.build(), Version.CURRENT);
-        DiscoveryNode discoveryNode = new DiscoveryNode(
-            randomAlphaOfLength(10),
-            handleB.discoveryNode.getAddress(),
-            emptyMap(),
-            emptySet(),
-            handleB.discoveryNode.getVersion());
-        ConnectTransportException ex = expectThrows(ConnectTransportException.class, () -> {
-            handleA.transportService.connectToNode(discoveryNode, TestProfiles.LIGHT_PROFILE);
-        });
-        assertThat(ex.getMessage(), containsString("unexpected remote node"));
-        assertFalse(handleA.transportService.nodeConnected(discoveryNode));
-    }
-
-
     private static class NetworkHandle {
         private TransportService transportService;
         private DiscoveryNode discoveryNode;
diff --git a/server/src/test/java/org/elasticsearch/update/UpdateIT.java b/server/src/test/java/org/elasticsearch/update/UpdateIT.java
index 7652c503450ae..1a0df1c4a1014 100644
--- a/server/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/server/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.get.GetResponse;
@@ -700,7 +701,18 @@ public void run(){
                                         .setRetryOnConflict(retryOnConflict)
                                         .setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
                                         .request();
-                                client().update(ur, new UpdateListener(j));
+                                if (randomBoolean()) {
+                                    client().update(ur, new UpdateListener(j));
+                                } else {
+                                    client().prepareBulk().add(ur).execute(ActionListener.map(new UpdateListener(j), br -> {
+                                        final BulkItemResponse ir = br.getItems()[0];
+                                        if (ir.isFailed()) {
+                                            throw ir.getFailure().getCause();
+                                        } else {
+                                            return ir.getResponse();
+                                        }
+                                    }));
+                                }
                             } catch (NoNodeAvailableException nne) {
                                 updateRequestsOutstanding.release();
                                 synchronized (failedMap) {
diff --git a/server/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java b/server/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java
index 5f730ad138f96..c170ca76c49da 100644
--- a/server/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java
+++ b/server/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java
@@ -189,7 +189,7 @@ public void testExplainFilteredAlias() {
         assertThat(validateQueryResponse.getQueryExplanation().get(0).getExplanation(), containsString("field:value1"));
     }
 
-    public void testExplainWithRewriteValidateQuery() throws Exception {
+    public void testExplainWithRewriteValidateQuery() {
         client().admin().indices().prepareCreate("test")
                 .addMapping("type1", "field", "type=text,analyzer=whitespace")
                 .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1)).get();
@@ -205,18 +205,6 @@ public void testExplainWithRewriteValidateQuery() throws Exception {
         assertExplanation(QueryBuilders.matchPhrasePrefixQuery("field", "ju"),
                 containsString("field:jumps"), true);
 
-        // common terms queries
-        assertExplanation(QueryBuilders.commonTermsQuery("field", "huge brown pidgin").cutoffFrequency(1),
-                containsString("+field:pidgin field:huge field:brown"), true);
-        assertExplanation(QueryBuilders.commonTermsQuery("field", "the brown").analyzer("stop"),
-                containsString("field:brown"), true);
-
-        // match queries with cutoff frequency
-        assertExplanation(QueryBuilders.matchQuery("field", "huge brown pidgin").cutoffFrequency(1),
-                containsString("+field:pidgin field:huge field:brown"), true);
-        assertExplanation(QueryBuilders.matchQuery("field", "the brown").analyzer("stop"),
-                containsString("field:brown"), true);
-
         // fuzzy queries
         assertExplanation(QueryBuilders.fuzzyQuery("field", "the").fuzziness(Fuzziness.fromEdits(2)),
                 containsString("field:the (field:tree)^0.3333333"), true);
@@ -224,7 +212,7 @@ public void testExplainWithRewriteValidateQuery() throws Exception {
                 containsString("(field:jumps)^0.75"), true);
 
         // more like this queries
-        Item[] items = new Item[] { new Item(null, null, "1")};
+        Item[] items = new Item[] { new Item(null, "1") };
         assertExplanation(QueryBuilders.moreLikeThisQuery(new String[] { "field" }, null, items)
                         .include(true).minTermFreq(1).minDocFreq(1).maxQueryTerms(2),
                 containsString("field:huge field:pidgin"), true);
@@ -233,7 +221,7 @@ public void testExplainWithRewriteValidateQuery() throws Exception {
                 containsString("field:huge field:pidgin"), true);
     }
 
-    public void testExplainWithRewriteValidateQueryAllShards() throws Exception {
+    public void testExplainWithRewriteValidateQueryAllShards() {
         client().admin().indices().prepareCreate("test")
             .addMapping("type1", "field", "type=text,analyzer=whitespace")
             .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 2).put("index.number_of_routing_shards", 2)).get();
@@ -262,7 +250,7 @@ public void testExplainWithRewriteValidateQueryAllShards() throws Exception {
             ), true, true);
     }
 
-    public void testIrrelevantPropertiesBeforeQuery() throws IOException {
+    public void testIrrelevantPropertiesBeforeQuery() {
         createIndex("test");
         ensureGreen();
         refresh();
@@ -271,7 +259,7 @@ public void testIrrelevantPropertiesBeforeQuery() throws IOException {
             new BytesArray("{\"foo\": \"bar\", \"query\": {\"term\" : { \"user\" : \"kimchy\" }}}"))).get().isValid(), equalTo(false));
     }
 
-    public void testIrrelevantPropertiesAfterQuery() throws IOException {
+    public void testIrrelevantPropertiesAfterQuery() {
         createIndex("test");
         ensureGreen();
         refresh();
@@ -311,7 +299,7 @@ private static void assertExplanations(QueryBuilder queryBuilder,
         }
     }
 
-    public void testExplainTermsQueryWithLookup() throws Exception {
+    public void testExplainTermsQueryWithLookup() {
         client().admin().indices().prepareCreate("twitter")
             .addMapping("_doc", "user", "type=integer", "followers", "type=integer")
             .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 2).put("index.number_of_routing_shards", 2)).get();
diff --git a/server/src/test/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java b/server/src/test/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java
index a6b3865de247b..62048119b6974 100644
--- a/server/src/test/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java
+++ b/server/src/test/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java
@@ -38,7 +38,6 @@
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.disruption.ServiceDisruptionScheme;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -119,10 +118,6 @@
  * </ul>
  */
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, minNumDataNodes = 4, maxNumDataNodes = 6)
-@TestLogging("_root:DEBUG,org.elasticsearch.action.bulk:TRACE,org.elasticsearch.action.get:TRACE," +
-    "org.elasticsearch.discovery:TRACE,org.elasticsearch.action.support.replication:TRACE," +
-    "org.elasticsearch.cluster.service:TRACE,org.elasticsearch.indices.recovery:TRACE," +
-    "org.elasticsearch.indices.cluster:TRACE,org.elasticsearch.index.shard:TRACE")
 public class ConcurrentSeqNoVersioningIT extends AbstractDisruptionTestCase {
 
     private static final Pattern EXTRACT_VERSION = Pattern.compile("current document has seqNo \\[(\\d+)\\] and primary term \\[(\\d+)\\]");
@@ -136,6 +131,7 @@ public void testSeqNoCASLinearizability() {
 
         assertAcked(prepareCreate("test")
             .setSettings(Settings.builder()
+                .put(indexSettings())
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1 + randomInt(2))
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomInt(3))
             ));
@@ -445,13 +441,17 @@ public boolean isLinearizable() {
             logger.info("--> Linearizability checking history of size: {} for key: {} and initialVersion: {}: {}", history.size(),
                 id, initialVersion, history);
             LinearizabilityChecker.SequentialSpec spec = new CASSequentialSpec(initialVersion);
-            boolean linearizable = new LinearizabilityChecker().isLinearizable(spec, history, missingResponseGenerator());
-            // implicitly test that we can serialize all histories.
-            String serializedHistory = base64Serialize(history);
-            if (linearizable == false) {
-                // we dump base64 encoded data, since the nature of this test is that it does not reproduce even with same seed.
-                logger.error("Linearizability check failed. Spec: {}, initial version: {}, serialized history: {}", spec, initialVersion,
-                    serializedHistory);
+            boolean linearizable = false;
+            try {
+                linearizable = new LinearizabilityChecker().isLinearizable(spec, history, missingResponseGenerator());
+            } finally {
+                // implicitly test that we can serialize all histories.
+                String serializedHistory = base64Serialize(history);
+                if (linearizable == false) {
+                    // we dump base64 encoded data, since the nature of this test is that it does not reproduce even with same seed.
+                    logger.error("Linearizability check failed. Spec: {}, initial version: {}, serialized history: {}",
+                        spec, initialVersion, serializedHistory);
+                }
             }
             return linearizable;
         }
diff --git a/server/src/test/resources/org/elasticsearch/action/search/simple-msearch1.json b/server/src/test/resources/org/elasticsearch/action/search/simple-msearch1.json
index eefec530e1f71..6db9e4b996045 100644
--- a/server/src/test/resources/org/elasticsearch/action/search/simple-msearch1.json
+++ b/server/src/test/resources/org/elasticsearch/action/search/simple-msearch1.json
@@ -1,6 +1,6 @@
 {"index":"test", "ignore_unavailable" : true, "expand_wildcards" : "open,closed"}}
 {"query" : {"match_all" :{}}}
-{"index" : "test", "type" : "type1", "expand_wildcards" : ["open", "closed"]}
+{"index" : "test", "expand_wildcards" : ["open", "closed"]}
 {"query" : {"match_all" :{}}}
 {"index":"test", "ignore_unavailable" : false, "expand_wildcards" : ["open"]}}
 {"query" : {"match_all" :{}}}
diff --git a/server/src/test/resources/org/elasticsearch/action/search/simple-msearch2.json b/server/src/test/resources/org/elasticsearch/action/search/simple-msearch2.json
index 79330d80f7267..ef82fee039638 100644
--- a/server/src/test/resources/org/elasticsearch/action/search/simple-msearch2.json
+++ b/server/src/test/resources/org/elasticsearch/action/search/simple-msearch2.json
@@ -1,6 +1,6 @@
 {"index":"test"}
 {"query" : {"match_all" : {}}}
-{"index" : "test", "type" : "type1"}
+{"index" : "test"}
 {"query" : {"match_all" : {}}}
 {}
 {"query" : {"match_all" : {}}}
diff --git a/server/src/test/resources/org/elasticsearch/action/search/simple-msearch3.json b/server/src/test/resources/org/elasticsearch/action/search/simple-msearch3.json
index a6b52fd3bf93e..f7ff9a2b3f991 100644
--- a/server/src/test/resources/org/elasticsearch/action/search/simple-msearch3.json
+++ b/server/src/test/resources/org/elasticsearch/action/search/simple-msearch3.json
@@ -1,8 +1,8 @@
 {"index":["test0", "test1"]}
 {"query" : {"match_all" : {}}}
-{"index" : "test2,test3", "type" : "type1"}
+{"index" : "test2,test3"}
 {"query" : {"match_all" : {}}}
-{"index" : ["test4", "test1"], "type" :  [ "type2", "type1" ]}
+{"index" : ["test4", "test1"]}
 {"query" : {"match_all" : {}}}
 {"search_type" : "dfs_query_then_fetch"}
 {"query" : {"match_all" : {}}}
diff --git a/server/src/test/resources/org/elasticsearch/action/search/simple-msearch4.json b/server/src/test/resources/org/elasticsearch/action/search/simple-msearch4.json
index 844d8bea1f8ee..4dd2cfde569dd 100644
--- a/server/src/test/resources/org/elasticsearch/action/search/simple-msearch4.json
+++ b/server/src/test/resources/org/elasticsearch/action/search/simple-msearch4.json
@@ -1,6 +1,6 @@
 {"index":["test0", "test1"], "request_cache": true}
 {"query" : {"match_all" : {}}}
-{"index" : "test2,test3", "type" : "type1", "preference": "_local"}
+{"index" : "test2,test3", "preference": "_local"}
 {"query" : {"match_all" : {}}}
-{"index" : ["test4", "test1"], "type" :  [ "type2", "type1" ], "routing": "123"}
+{"index" : ["test4", "test1"], "routing": "123"}
 {"query" : {"match_all" : {}}}
diff --git a/server/src/test/resources/org/elasticsearch/action/termvectors/multiRequest1.json b/server/src/test/resources/org/elasticsearch/action/termvectors/multiRequest1.json
index fcb5e3a927ad9..aecbcdc9d69bb 100644
--- a/server/src/test/resources/org/elasticsearch/action/termvectors/multiRequest1.json
+++ b/server/src/test/resources/org/elasticsearch/action/termvectors/multiRequest1.json
@@ -7,7 +7,6 @@
         "offsets":false,
         "positions":false,
         "fields":["a","b","c"],
-        "_index": "testidx",
-        "_type":"test"
+        "_index": "testidx"
     }
 }
\ No newline at end of file
diff --git a/server/src/test/resources/org/elasticsearch/action/termvectors/multiRequest2.json b/server/src/test/resources/org/elasticsearch/action/termvectors/multiRequest2.json
index a0709effe709b..793bdda2a33b6 100644
--- a/server/src/test/resources/org/elasticsearch/action/termvectors/multiRequest2.json
+++ b/server/src/test/resources/org/elasticsearch/action/termvectors/multiRequest2.json
@@ -8,8 +8,7 @@
          "offsets": false,
          "positions": false,
          "fields":["a","b","c"],
-         "_index": "testidx",
-         "_type": "test"
+         "_index": "testidx"
       },
       {
          "_id": "2",
@@ -19,8 +18,7 @@
          "offsets": false,
          "positions": false,
          "fields":["a","b","c"],
-         "_index": "testidx",
-         "_type": "test"
+         "_index": "testidx"
       }
    ]
 }
\ No newline at end of file
diff --git a/server/src/test/resources/org/elasticsearch/action/termvectors/multiRequest3.json b/server/src/test/resources/org/elasticsearch/action/termvectors/multiRequest3.json
index 457f43cdc9aca..2c405919111aa 100644
--- a/server/src/test/resources/org/elasticsearch/action/termvectors/multiRequest3.json
+++ b/server/src/test/resources/org/elasticsearch/action/termvectors/multiRequest3.json
@@ -2,7 +2,6 @@
    "ids": ["1","2"],
    "parameters": {
       "_index": "testidx",
-      "_type": "test",
       "filter": {
          "max_num_terms": 20,
          "min_term_freq": 1,
diff --git a/server/src/test/resources/org/elasticsearch/index/query/commonTerms-query1.json b/server/src/test/resources/org/elasticsearch/index/query/commonTerms-query1.json
deleted file mode 100644
index b2728dac09df4..0000000000000
--- a/server/src/test/resources/org/elasticsearch/index/query/commonTerms-query1.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
-    "common" : {
-        "dogs" : {
-            "query" : "buck mia tom",
-            "cutoff_frequency" : 1,
-            "minimum_should_match" : {
-                "low_freq" : 2
-            }
-        }
-    }
-}
diff --git a/server/src/test/resources/org/elasticsearch/index/query/commonTerms-query2.json b/server/src/test/resources/org/elasticsearch/index/query/commonTerms-query2.json
deleted file mode 100644
index aeb281bb7592a..0000000000000
--- a/server/src/test/resources/org/elasticsearch/index/query/commonTerms-query2.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
-    "common" : {
-        "dogs" : {
-            "query" : "buck mia tom",
-            "minimum_should_match" : {
-                "high_freq" : "50%",
-                "low_freq" : "5<20%"
-            }
-        }
-    }
-}
diff --git a/server/src/test/resources/org/elasticsearch/index/query/commonTerms-query3.json b/server/src/test/resources/org/elasticsearch/index/query/commonTerms-query3.json
deleted file mode 100644
index f276209ffc7ed..0000000000000
--- a/server/src/test/resources/org/elasticsearch/index/query/commonTerms-query3.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
-    "common" : {
-        "dogs" : {
-            "query" : "buck mia tom",
-            "cutoff_frequency" : 1,
-            "minimum_should_match" : 2
-        }
-    }
-}
diff --git a/server/src/test/resources/org/elasticsearch/index/translog/translog-v2.binary b/server/src/test/resources/org/elasticsearch/index/translog/translog-v2.binary
new file mode 100644
index 0000000000000000000000000000000000000000..a61a02adee385e39fdc7954cd4cc4c557c8fa19f
GIT binary patch
literal 43
wcmcD&o+HjtQk0lioRgo<z`(!+#A05qkyXb29{C~8{{DFd<yqyPrBT`C01|->Bme*a

literal 0
HcmV?d00001

diff --git a/settings.gradle b/settings.gradle
index e0650f618ddf4..5408f23976f30 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -120,43 +120,47 @@ include projects.toArray(new String[0])
 
 project(':build-tools').projectDir = new File(rootProject.projectDir, 'buildSrc')
 
+project(":libs").children.each { libsProject ->
+    libsProject.name = "elasticsearch-${libsProject.name}"
+}
+
 if (isEclipse) {
   project(":server").projectDir = new File(rootProject.projectDir, 'server/src/main')
   project(":server").buildFileName = 'eclipse-build.gradle'
   project(":server-tests").projectDir = new File(rootProject.projectDir, 'server/src/test')
   project(":server-tests").buildFileName = 'eclipse-build.gradle'
-  project(":libs:core").projectDir = new File(rootProject.projectDir, 'libs/core/src/main')
-  project(":libs:core").buildFileName = 'eclipse-build.gradle'
-  project(":libs:core-tests").projectDir = new File(rootProject.projectDir, 'libs/core/src/test')
-  project(":libs:core-tests").buildFileName = 'eclipse-build.gradle'
-  project(":libs:dissect").projectDir = new File(rootProject.projectDir, 'libs/dissect/src/main')
-  project(":libs:dissect").buildFileName = 'eclipse-build.gradle'
-  project(":libs:dissect-tests").projectDir = new File(rootProject.projectDir, 'libs/dissect/src/test')
-  project(":libs:dissect-tests").buildFileName = 'eclipse-build.gradle'
-  project(":libs:nio").projectDir = new File(rootProject.projectDir, 'libs/nio/src/main')
-  project(":libs:nio").buildFileName = 'eclipse-build.gradle'
-  project(":libs:nio-tests").projectDir = new File(rootProject.projectDir, 'libs/nio/src/test')
-  project(":libs:nio-tests").buildFileName = 'eclipse-build.gradle'
-  project(":libs:x-content").projectDir = new File(rootProject.projectDir, 'libs/x-content/src/main')
-  project(":libs:x-content").buildFileName = 'eclipse-build.gradle'
-  project(":libs:x-content-tests").projectDir = new File(rootProject.projectDir, 'libs/x-content/src/test')
-  project(":libs:x-content-tests").buildFileName = 'eclipse-build.gradle'
-  project(":libs:secure-sm").projectDir = new File(rootProject.projectDir, 'libs/secure-sm/src/main')
-  project(":libs:secure-sm").buildFileName = 'eclipse-build.gradle'
-  project(":libs:secure-sm-tests").projectDir = new File(rootProject.projectDir, 'libs/secure-sm/src/test')
-  project(":libs:secure-sm-tests").buildFileName = 'eclipse-build.gradle'
-  project(":libs:grok").projectDir = new File(rootProject.projectDir, 'libs/grok/src/main')
-  project(":libs:grok").buildFileName = 'eclipse-build.gradle'
-  project(":libs:grok-tests").projectDir = new File(rootProject.projectDir, 'libs/grok/src/test')
-  project(":libs:grok-tests").buildFileName = 'eclipse-build.gradle'
-  project(":libs:geo").projectDir = new File(rootProject.projectDir, 'libs/geo/src/main')
-  project(":libs:geo").buildFileName = 'eclipse-build.gradle'
-  project(":libs:geo-tests").projectDir = new File(rootProject.projectDir, 'libs/geo/src/test')
-  project(":libs:geo-tests").buildFileName = 'eclipse-build.gradle'
-  project(":libs:ssl-config").projectDir = new File(rootProject.projectDir, 'libs/ssl-config/src/main')
-  project(":libs:ssl-config").buildFileName = 'eclipse-build.gradle'
-  project(":libs:ssl-config-tests").projectDir = new File(rootProject.projectDir, 'libs/ssl-config/src/test')
-  project(":libs:ssl-config-tests").buildFileName = 'eclipse-build.gradle'
+  project(":libs:elasticsearch-core").projectDir = new File(rootProject.projectDir, 'libs/core/src/main')
+  project(":libs:elasticsearch-core").buildFileName = 'eclipse-build.gradle'
+  project(":libs:elasticsearch-core-tests").projectDir = new File(rootProject.projectDir, 'libs/core/src/test')
+  project(":libs:elasticsearch-core-tests").buildFileName = 'eclipse-build.gradle'
+  project(":libs:elasticsearch-dissect").projectDir = new File(rootProject.projectDir, 'libs/dissect/src/main')
+  project(":libs:elasticsearch-dissect").buildFileName = 'eclipse-build.gradle'
+  project(":libs:elasticsearch-dissect-tests").projectDir = new File(rootProject.projectDir, 'libs/dissect/src/test')
+  project(":libs:elasticsearch-dissect-tests").buildFileName = 'eclipse-build.gradle'
+  project(":libs:elasticsearch-nio").projectDir = new File(rootProject.projectDir, 'libs/nio/src/main')
+  project(":libs:elasticsearch-nio").buildFileName = 'eclipse-build.gradle'
+  project(":libs:elasticsearch-nio-tests").projectDir = new File(rootProject.projectDir, 'libs/nio/src/test')
+  project(":libs:elasticsearch-nio-tests").buildFileName = 'eclipse-build.gradle'
+  project(":libs:elasticsearch-x-content").projectDir = new File(rootProject.projectDir, 'libs/x-content/src/main')
+  project(":libs:elasticsearch-x-content").buildFileName = 'eclipse-build.gradle'
+  project(":libs:elasticsearch-x-content-tests").projectDir = new File(rootProject.projectDir, 'libs/x-content/src/test')
+  project(":libs:elasticsearch-x-content-tests").buildFileName = 'eclipse-build.gradle'
+  project(":libs:elasticsearch-secure-sm").projectDir = new File(rootProject.projectDir, 'libs/secure-sm/src/main')
+  project(":libs:elasticsearch-secure-sm").buildFileName = 'eclipse-build.gradle'
+  project(":libs:elasticsearch-secure-sm-tests").projectDir = new File(rootProject.projectDir, 'libs/secure-sm/src/test')
+  project(":libs:elasticsearch-secure-sm-tests").buildFileName = 'eclipse-build.gradle'
+  project(":libs:elasticsearch-grok").projectDir = new File(rootProject.projectDir, 'libs/grok/src/main')
+  project(":libs:elasticsearch-grok").buildFileName = 'eclipse-build.gradle'
+  project(":libs:elasticsearch-grok-tests").projectDir = new File(rootProject.projectDir, 'libs/grok/src/test')
+  project(":libs:elasticsearch-grok-tests").buildFileName = 'eclipse-build.gradle'
+  project(":libs:elasticsearch-geo").projectDir = new File(rootProject.projectDir, 'libs/geo/src/main')
+  project(":libs:elasticsearch-geo").buildFileName = 'eclipse-build.gradle'
+  project(":libs:elasticsearch-geo-tests").projectDir = new File(rootProject.projectDir, 'libs/geo/src/test')
+  project(":libs:elasticsearch-geo-tests").buildFileName = 'eclipse-build.gradle'
+  project(":libs:elasticsearch-ssl-config").projectDir = new File(rootProject.projectDir, 'libs/ssl-config/src/main')
+  project(":libs:elasticsearch-ssl-config").buildFileName = 'eclipse-build.gradle'
+  project(":libs:elasticsearch-ssl-config-tests").projectDir = new File(rootProject.projectDir, 'libs/ssl-config/src/test')
+  project(":libs:elasticsearch-ssl-config-tests").buildFileName = 'eclipse-build.gradle'
   project(":client:rest-high-level").projectDir = new File(rootProject.projectDir, 'client/rest-high-level/src/main')
   project(":client:rest-high-level").buildFileName = 'eclipse-build.gradle'
   project(":client:rest-high-level-tests").projectDir = new File(rootProject.projectDir, 'client/rest-high-level/src/test')
@@ -175,7 +179,3 @@ if (extraProjects.exists()) {
     addSubProjects('', extraProjectDir)
   }
 }
-
-project(":libs:cli").name = 'elasticsearch-cli'
-project(":libs:geo").name = 'elasticsearch-geo'
-project(":libs:ssl-config").name = 'elasticsearch-ssl-config'
diff --git a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java b/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java
index 01315cdab01ca..b060d78b92b69 100644
--- a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java
+++ b/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java
@@ -19,18 +19,6 @@
 
 package hdfs;
 
-import java.io.File;
-import java.lang.management.ManagementFactory;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -45,6 +33,18 @@
 import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 
+import java.io.File;
+import java.lang.management.ManagementFactory;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
 /**
  * MiniHDFS test fixture. There is a CLI tool, but here we can
  * easily properly setup logging, avoid parsing JSON, etc.
@@ -174,4 +174,5 @@ public static void main(String[] args) throws Exception {
         Files.write(tmp, portFileContent.getBytes(StandardCharsets.UTF_8));
         Files.move(tmp, baseDir.resolve(PORT_FILE_NAME), StandardCopyOption.ATOMIC_MOVE);
     }
+
 }
diff --git a/test/fixtures/krb5kdc-fixture/build.gradle b/test/fixtures/krb5kdc-fixture/build.gradle
index 9787c3527af71..8f145ba909223 100644
--- a/test/fixtures/krb5kdc-fixture/build.gradle
+++ b/test/fixtures/krb5kdc-fixture/build.gradle
@@ -47,9 +47,6 @@ postProcessFixture {
     }
 }
 
-// https://github.com/elastic/elasticsearch/issues/40624
-buildFixture.enabled = false
-
 project.ext.krb5Conf = { service -> file("$buildDir/shared/${service}/krb5.conf") }
 project.ext.krb5Keytabs = { service, fileName -> file("$buildDir/shared/${service}/keytabs/${fileName}") }
 
diff --git a/test/fixtures/krb5kdc-fixture/docker-compose.yml b/test/fixtures/krb5kdc-fixture/docker-compose.yml
index 4d018dd6c3e08..8e7108d58e5c0 100644
--- a/test/fixtures/krb5kdc-fixture/docker-compose.yml
+++ b/test/fixtures/krb5kdc-fixture/docker-compose.yml
@@ -8,6 +8,8 @@ services:
     command: "bash /fixture/src/main/resources/provision/peppa.sh"
     volumes:
       - ./build/shared/peppa:/fixture/build
+      # containers have bad entropy so mount /dev/urandom. Less secure but this is a test fixture.
+      - /dev/urandom:/dev/random
     ports:
       - "4444"
       - "88/udp"
@@ -19,6 +21,8 @@ services:
     command: "bash /fixture/src/main/resources/provision/hdfs.sh"
     volumes:
       - ./build/shared/hdfs:/fixture/build
+      # containers have bad entropy so mount /dev/urandom. Less secure but this is a test fixture.
+      - /dev/urandom:/dev/random
     ports:
       - "4444"
       - "88/udp"
diff --git a/test/framework/build.gradle b/test/framework/build.gradle
index 686756c6e53ba..9cabdb82bf36a 100644
--- a/test/framework/build.gradle
+++ b/test/framework/build.gradle
@@ -18,11 +18,11 @@
  */
 
 dependencies {
-  compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
-  compile "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}"
-  compile "org.elasticsearch:elasticsearch-nio:${version}"
-  compile "org.elasticsearch:elasticsearch:${version}"
-  compile "org.elasticsearch:elasticsearch-cli:${version}"
+  compile project(":client:rest")
+  compile project(":client:sniffer")
+  compile project(':libs:elasticsearch-nio')
+  compile project(":server")
+  compile project(":libs:elasticsearch-cli")
   compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   compile "junit:junit:${versions.junit}"
   compile "org.hamcrest:hamcrest:${versions.hamcrest}"
@@ -52,7 +52,11 @@ thirdPartyAudit.ignoreMissingClasses (
   'javax.servlet.ServletContextListener',
   'org.apache.avalon.framework.logger.Logger',
   'org.apache.log.Hierarchy',
-  'org.apache.log.Logger'
+  'org.apache.log.Logger',
+  'org.apache.log4j.Category',
+  'org.apache.log4j.Level',
+  'org.apache.log4j.Logger',
+  'org.apache.log4j.Priority'
 )
 
 test {
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
index b3681247b4e9c..9be598bb384bd 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -43,7 +44,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -94,8 +94,8 @@ public static AllocationDeciders randomAllocationDeciders(Settings settings, Clu
         return new AllocationDeciders(deciders);
     }
 
-    protected static Set<DiscoveryNode.Role> MASTER_DATA_ROLES =
-            Collections.unmodifiableSet(EnumSet.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA));
+    protected static Set<DiscoveryNodeRole> MASTER_DATA_ROLES =
+            Collections.unmodifiableSet(Set.of(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.DATA_ROLE));
 
     protected static DiscoveryNode newNode(String nodeId) {
         return newNode(nodeId, Version.CURRENT);
@@ -109,7 +109,7 @@ protected static DiscoveryNode newNode(String nodeId, Map<String, String> attrib
         return new DiscoveryNode(nodeId, buildNewFakeTransportAddress(), attributes, MASTER_DATA_ROLES, Version.CURRENT);
     }
 
-    protected static DiscoveryNode newNode(String nodeId, Set<DiscoveryNode.Role> roles) {
+    protected static DiscoveryNode newNode(String nodeId, Set<DiscoveryNodeRole> roles) {
         return new DiscoveryNode(nodeId, buildNewFakeTransportAddress(), emptyMap(), roles, Version.CURRENT);
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java
new file mode 100644
index 0000000000000..0c27f84d7f1e4
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java
@@ -0,0 +1,1362 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import com.carrotsearch.randomizedtesting.RandomizedContext;
+import org.apache.logging.log4j.CloseableThreadContext;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateTaskListener;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.NodeConnectionsService;
+import org.elasticsearch.cluster.coordination.AbstractCoordinatorTestCase.Cluster.ClusterNode;
+import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
+import org.elasticsearch.cluster.coordination.LinearizabilityChecker.History;
+import org.elasticsearch.cluster.coordination.LinearizabilityChecker.SequentialSpec;
+import org.elasticsearch.cluster.metadata.Manifest;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.service.ClusterApplierService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
+import org.elasticsearch.discovery.DiscoveryModule;
+import org.elasticsearch.discovery.SeedHostsProvider;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.gateway.ClusterStateUpdaters;
+import org.elasticsearch.gateway.GatewayService;
+import org.elasticsearch.gateway.MetaStateService;
+import org.elasticsearch.gateway.MockGatewayMetaState;
+import org.elasticsearch.indices.cluster.FakeThreadPoolMasterService;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.disruption.DisruptableMockTransport;
+import org.elasticsearch.test.disruption.DisruptableMockTransport.ConnectionStatus;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportInterceptor;
+import org.elasticsearch.transport.TransportService;
+import org.hamcrest.Matcher;
+import org.hamcrest.core.IsCollectionContaining;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.function.UnaryOperator;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptySet;
+import static org.elasticsearch.cluster.coordination.AbstractCoordinatorTestCase.Cluster.DEFAULT_DELAY_VARIABILITY;
+import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.BOOTSTRAP_PLACEHOLDER_PREFIX;
+import static org.elasticsearch.cluster.coordination.CoordinationStateTestCluster.clusterState;
+import static org.elasticsearch.cluster.coordination.Coordinator.Mode.CANDIDATE;
+import static org.elasticsearch.cluster.coordination.Coordinator.Mode.FOLLOWER;
+import static org.elasticsearch.cluster.coordination.Coordinator.Mode.LEADER;
+import static org.elasticsearch.cluster.coordination.Coordinator.PUBLISH_TIMEOUT_SETTING;
+import static org.elasticsearch.cluster.coordination.ElectionSchedulerFactory.ELECTION_BACK_OFF_TIME_SETTING;
+import static org.elasticsearch.cluster.coordination.ElectionSchedulerFactory.ELECTION_DURATION_SETTING;
+import static org.elasticsearch.cluster.coordination.ElectionSchedulerFactory.ELECTION_INITIAL_TIMEOUT_SETTING;
+import static org.elasticsearch.cluster.coordination.FollowersChecker.FOLLOWER_CHECK_INTERVAL_SETTING;
+import static org.elasticsearch.cluster.coordination.FollowersChecker.FOLLOWER_CHECK_RETRY_COUNT_SETTING;
+import static org.elasticsearch.cluster.coordination.FollowersChecker.FOLLOWER_CHECK_TIMEOUT_SETTING;
+import static org.elasticsearch.cluster.coordination.LeaderChecker.LEADER_CHECK_INTERVAL_SETTING;
+import static org.elasticsearch.cluster.coordination.LeaderChecker.LEADER_CHECK_RETRY_COUNT_SETTING;
+import static org.elasticsearch.cluster.coordination.LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING;
+import static org.elasticsearch.cluster.coordination.NoMasterBlockService.NO_MASTER_BLOCK_ID;
+import static org.elasticsearch.cluster.coordination.Reconfigurator.CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION;
+import static org.elasticsearch.discovery.PeerFinder.DISCOVERY_FIND_PEERS_INTERVAL_SETTING;
+import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
+import static org.elasticsearch.node.Node.NODE_NAME_SETTING;
+import static org.elasticsearch.transport.TransportService.NOOP_TRANSPORT_INTERCEPTOR;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
+
+public class AbstractCoordinatorTestCase extends ESTestCase {
+
+    protected final List<NodeEnvironment> nodeEnvironments = new ArrayList<>();
+
+    protected final AtomicInteger nextNodeIndex = new AtomicInteger();
+
+    @Before
+    public void resetNodeIndexBeforeEachTest() {
+        nextNodeIndex.set(0);
+    }
+
+    @After
+    public void closeNodeEnvironmentsAfterEachTest() {
+        for (NodeEnvironment nodeEnvironment : nodeEnvironments) {
+            nodeEnvironment.close();
+        }
+        nodeEnvironments.clear();
+    }
+
+    @Before
+    public void resetPortCounterBeforeEachTest() {
+        resetPortCounter();
+    }
+
+    // check that runRandomly leads to reproducible results
+    public void testRepeatableTests() throws Exception {
+        final Callable<Long> test = () -> {
+            resetNodeIndexBeforeEachTest();
+            final Cluster cluster = new Cluster(randomIntBetween(1, 5));
+            cluster.runRandomly();
+            final long afterRunRandomly = value(cluster.getAnyNode().getLastAppliedClusterState());
+            cluster.stabilise();
+            final long afterStabilisation = value(cluster.getAnyNode().getLastAppliedClusterState());
+            return afterRunRandomly ^ afterStabilisation;
+        };
+        final long seed = randomLong();
+        logger.info("First run with seed [{}]", seed);
+        final long result1 = RandomizedContext.current().runWithPrivateRandomness(seed, test);
+        logger.info("Second run with seed [{}]", seed);
+        final long result2 = RandomizedContext.current().runWithPrivateRandomness(seed, test);
+        assertEquals(result1, result2);
+    }
+
+    protected static long defaultMillis(Setting<TimeValue> setting) {
+        return setting.get(Settings.EMPTY).millis() + DEFAULT_DELAY_VARIABILITY;
+    }
+
+    protected static int defaultInt(Setting<Integer> setting) {
+        return setting.get(Settings.EMPTY);
+    }
+
+    // Updating the cluster state involves up to 7 delays:
+    // 1. submit the task to the master service
+    // 2. send PublishRequest
+    // 3. receive PublishResponse
+    // 4. send ApplyCommitRequest
+    // 5. apply committed cluster state
+    // 6. receive ApplyCommitResponse
+    // 7. apply committed state on master (last one to apply cluster state)
+    public static final long DEFAULT_CLUSTER_STATE_UPDATE_DELAY = 7 * DEFAULT_DELAY_VARIABILITY;
+
+    private static final int ELECTION_RETRIES = 10;
+
+    // The time it takes to complete an election
+    public static final long DEFAULT_ELECTION_DELAY
+        // Pinging all peers twice should be enough to discover all nodes
+        = defaultMillis(DISCOVERY_FIND_PEERS_INTERVAL_SETTING) * 2
+        // Then wait for an election to be scheduled; we allow enough time for retries to allow for collisions
+        + defaultMillis(ELECTION_INITIAL_TIMEOUT_SETTING) * ELECTION_RETRIES
+        + defaultMillis(ELECTION_BACK_OFF_TIME_SETTING) * ELECTION_RETRIES * (ELECTION_RETRIES - 1) / 2
+        + defaultMillis(ELECTION_DURATION_SETTING) * ELECTION_RETRIES
+        // Allow two round-trip for pre-voting and voting
+        + 4 * DEFAULT_DELAY_VARIABILITY
+        // Then a commit of the new leader's first cluster state
+        + DEFAULT_CLUSTER_STATE_UPDATE_DELAY;
+
+    public static final long DEFAULT_STABILISATION_TIME =
+        // If leader just blackholed, need to wait for this to be detected
+        (defaultMillis(LEADER_CHECK_INTERVAL_SETTING) + defaultMillis(LEADER_CHECK_TIMEOUT_SETTING))
+            * defaultInt(LEADER_CHECK_RETRY_COUNT_SETTING)
+            // then wait for a follower to be promoted to leader
+            + DEFAULT_ELECTION_DELAY
+            // perhaps there is an election collision requiring another publication (which times out) and a term bump
+            + defaultMillis(PUBLISH_TIMEOUT_SETTING) + DEFAULT_ELECTION_DELAY
+            // then wait for the new leader to notice that the old leader is unresponsive
+            + (defaultMillis(FOLLOWER_CHECK_INTERVAL_SETTING) + defaultMillis(FOLLOWER_CHECK_TIMEOUT_SETTING))
+            * defaultInt(FOLLOWER_CHECK_RETRY_COUNT_SETTING)
+            // then wait for the new leader to commit a state without the old leader
+            + DEFAULT_CLUSTER_STATE_UPDATE_DELAY;
+
+    class Cluster {
+
+        static final long EXTREME_DELAY_VARIABILITY = 10000L;
+        static final long DEFAULT_DELAY_VARIABILITY = 100L;
+
+        final List<ClusterNode> clusterNodes;
+        final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(
+            // TODO does ThreadPool need a node name any more?
+            Settings.builder().put(NODE_NAME_SETTING.getKey(), "deterministic-task-queue").build(), random());
+        private boolean disruptStorage;
+
+        final VotingConfiguration initialConfiguration;
+
+        private final Set<String> disconnectedNodes = new HashSet<>();
+        private final Set<String> blackholedNodes = new HashSet<>();
+        private final Set<Tuple<String,String>> blackholedConnections = new HashSet<>();
+        private final Map<Long, ClusterState> committedStatesByVersion = new HashMap<>();
+        private final LinearizabilityChecker linearizabilityChecker = new LinearizabilityChecker();
+        private final History history = new History();
+
+        private final Function<DiscoveryNode, MockPersistedState> defaultPersistedStateSupplier = MockPersistedState::new;
+
+        @Nullable // null means construct a list from all the current nodes
+        private List<TransportAddress> seedHostsList;
+
+        Cluster(int initialNodeCount) {
+            this(initialNodeCount, true, Settings.EMPTY);
+        }
+
+        Cluster(int initialNodeCount, boolean allNodesMasterEligible, Settings nodeSettings) {
+            deterministicTaskQueue.setExecutionDelayVariabilityMillis(DEFAULT_DELAY_VARIABILITY);
+
+            assertThat(initialNodeCount, greaterThan(0));
+
+            final Set<String> masterEligibleNodeIds = new HashSet<>(initialNodeCount);
+            clusterNodes = new ArrayList<>(initialNodeCount);
+            for (int i = 0; i < initialNodeCount; i++) {
+                final ClusterNode clusterNode = new ClusterNode(nextNodeIndex.getAndIncrement(),
+                    allNodesMasterEligible || i == 0 || randomBoolean(), nodeSettings);
+                clusterNodes.add(clusterNode);
+                if (clusterNode.getLocalNode().isMasterNode()) {
+                    masterEligibleNodeIds.add(clusterNode.getId());
+                }
+            }
+
+            initialConfiguration = new VotingConfiguration(new HashSet<>(
+                randomSubsetOf(randomIntBetween(1, masterEligibleNodeIds.size()), masterEligibleNodeIds)));
+
+            logger.info("--> creating cluster of {} nodes (master-eligible nodes: {}) with initial configuration {}",
+                initialNodeCount, masterEligibleNodeIds, initialConfiguration);
+        }
+
+        List<ClusterNode> addNodesAndStabilise(int newNodesCount) {
+            final List<ClusterNode> addedNodes = addNodes(newNodesCount);
+            stabilise(
+                // The first pinging discovers the master
+                defaultMillis(DISCOVERY_FIND_PEERS_INTERVAL_SETTING)
+                    // One message delay to send a join
+                    + DEFAULT_DELAY_VARIABILITY
+                    // Commit a new cluster state with the new node(s). Might be split into multiple commits, and each might need a
+                    // followup reconfiguration
+                    + newNodesCount * 2 * DEFAULT_CLUSTER_STATE_UPDATE_DELAY);
+            // TODO Investigate whether 4 publications is sufficient due to batching? A bound linear in the number of nodes isn't great.
+            return addedNodes;
+        }
+
+        List<ClusterNode> addNodes(int newNodesCount) {
+            logger.info("--> adding {} nodes", newNodesCount);
+
+            final List<ClusterNode> addedNodes = new ArrayList<>();
+            for (int i = 0; i < newNodesCount; i++) {
+                final ClusterNode clusterNode = new ClusterNode(nextNodeIndex.getAndIncrement(), true, Settings.EMPTY);
+                addedNodes.add(clusterNode);
+            }
+            clusterNodes.addAll(addedNodes);
+            return addedNodes;
+        }
+
+        int size() {
+            return clusterNodes.size();
+        }
+
+        void runRandomly() {
+            runRandomly(true);
+        }
+
+        void runRandomly(boolean allowReboots) {
+
+            // TODO supporting (preserving?) existing disruptions needs implementing if needed, for now we just forbid it
+            assertThat("may reconnect disconnected nodes, probably unexpected", disconnectedNodes, empty());
+            assertThat("may reconnect blackholed nodes, probably unexpected", blackholedNodes, empty());
+
+            final List<Runnable> cleanupActions = new ArrayList<>();
+            cleanupActions.add(disconnectedNodes::clear);
+            cleanupActions.add(blackholedNodes::clear);
+            cleanupActions.add(() -> disruptStorage = false);
+
+            final int randomSteps = scaledRandomIntBetween(10, 10000);
+            final int keyRange = randomSteps / 50; // for randomized writes and reads
+            logger.info("--> start of safety phase of at least [{}] steps", randomSteps);
+
+            deterministicTaskQueue.setExecutionDelayVariabilityMillis(EXTREME_DELAY_VARIABILITY);
+            disruptStorage = true;
+            int step = 0;
+            long finishTime = -1;
+
+            while (finishTime == -1 || deterministicTaskQueue.getCurrentTimeMillis() <= finishTime) {
+                step++;
+                final int thisStep = step; // for lambdas
+
+                if (randomSteps <= step && finishTime == -1) {
+                    finishTime = deterministicTaskQueue.getLatestDeferredExecutionTime();
+                    deterministicTaskQueue.setExecutionDelayVariabilityMillis(DEFAULT_DELAY_VARIABILITY);
+                    logger.debug("----> [runRandomly {}] reducing delay variability and running until [{}ms]", step, finishTime);
+                }
+
+                try {
+                    if (finishTime == -1 && randomBoolean() && randomBoolean() && randomBoolean()) {
+                        final ClusterNode clusterNode = getAnyNodePreferringLeaders();
+                        final int key = randomIntBetween(0, keyRange);
+                        final int newValue = randomInt();
+                        clusterNode.onNode(() -> {
+                            logger.debug("----> [runRandomly {}] proposing new value [{}] to [{}]",
+                                thisStep, newValue, clusterNode.getId());
+                            clusterNode.submitValue(key, newValue);
+                        }).run();
+                    } else if (finishTime == -1 && randomBoolean() && randomBoolean() && randomBoolean()) {
+                        final ClusterNode clusterNode = getAnyNodePreferringLeaders();
+                        final int key = randomIntBetween(0, keyRange);
+                        clusterNode.onNode(() -> {
+                            logger.debug("----> [runRandomly {}] reading value from [{}]",
+                                thisStep, clusterNode.getId());
+                            clusterNode.readValue(key);
+                        }).run();
+                    } else if (rarely()) {
+                        final ClusterNode clusterNode = getAnyNodePreferringLeaders();
+                        final boolean autoShrinkVotingConfiguration = randomBoolean();
+                        clusterNode.onNode(
+                            () -> {
+                                logger.debug("----> [runRandomly {}] setting auto-shrink configuration to {} on {}",
+                                    thisStep, autoShrinkVotingConfiguration, clusterNode.getId());
+                                clusterNode.submitSetAutoShrinkVotingConfiguration(autoShrinkVotingConfiguration);
+                            }).run();
+                    } else if (allowReboots && rarely()) {
+                        // reboot random node
+                        final ClusterNode clusterNode = getAnyNode();
+                        logger.debug("----> [runRandomly {}] rebooting [{}]", thisStep, clusterNode.getId());
+                        clusterNode.close();
+                        clusterNodes.forEach(
+                            cn -> deterministicTaskQueue.scheduleNow(cn.onNode(
+                                new Runnable() {
+                                    @Override
+                                    public void run() {
+                                        cn.transportService.disconnectFromNode(clusterNode.getLocalNode());
+                                    }
+
+                                    @Override
+                                    public String toString() {
+                                        return "disconnect from " + clusterNode.getLocalNode() + " after shutdown";
+                                    }
+                                })));
+                        clusterNodes.replaceAll(cn -> cn == clusterNode ? cn.restartedNode() : cn);
+                    } else if (rarely()) {
+                        final ClusterNode clusterNode = getAnyNode();
+                        clusterNode.onNode(() -> {
+                            logger.debug("----> [runRandomly {}] forcing {} to become candidate", thisStep, clusterNode.getId());
+                            synchronized (clusterNode.coordinator.mutex) {
+                                clusterNode.coordinator.becomeCandidate("runRandomly");
+                            }
+                        }).run();
+                    } else if (rarely()) {
+                        final ClusterNode clusterNode = getAnyNode();
+
+                        switch (randomInt(2)) {
+                            case 0:
+                                if (clusterNode.heal()) {
+                                    logger.debug("----> [runRandomly {}] healing {}", step, clusterNode.getId());
+                                }
+                                break;
+                            case 1:
+                                if (clusterNode.disconnect()) {
+                                    logger.debug("----> [runRandomly {}] disconnecting {}", step, clusterNode.getId());
+                                }
+                                break;
+                            case 2:
+                                if (clusterNode.blackhole()) {
+                                    logger.debug("----> [runRandomly {}] blackholing {}", step, clusterNode.getId());
+                                }
+                                break;
+                        }
+                    } else if (rarely()) {
+                        final ClusterNode clusterNode = getAnyNode();
+                        logger.debug("----> [runRandomly {}] applying initial configuration on {}", step, clusterNode.getId());
+                        clusterNode.applyInitialConfiguration();
+                    } else {
+                        if (deterministicTaskQueue.hasDeferredTasks() && randomBoolean()) {
+                            deterministicTaskQueue.advanceTime();
+                        } else if (deterministicTaskQueue.hasRunnableTasks()) {
+                            deterministicTaskQueue.runRandomTask();
+                        }
+                    }
+
+                    // TODO other random steps:
+                    // - reboot a node
+                    // - abdicate leadership
+
+                } catch (CoordinationStateRejectedException | UncheckedIOException ignored) {
+                    // This is ok: it just means a message couldn't currently be handled.
+                }
+
+                assertConsistentStates();
+            }
+
+            logger.debug("running {} cleanup actions", cleanupActions.size());
+            cleanupActions.forEach(Runnable::run);
+            logger.debug("finished running cleanup actions");
+        }
+
+        private void assertConsistentStates() {
+            for (final ClusterNode clusterNode : clusterNodes) {
+                clusterNode.coordinator.invariant();
+            }
+            updateCommittedStates();
+        }
+
+        private void updateCommittedStates() {
+            for (final ClusterNode clusterNode : clusterNodes) {
+                ClusterState applierState = clusterNode.coordinator.getApplierState();
+                ClusterState storedState = committedStatesByVersion.get(applierState.getVersion());
+                if (storedState == null) {
+                    committedStatesByVersion.put(applierState.getVersion(), applierState);
+                } else {
+                    assertEquals("expected " + applierState + " but got " + storedState,
+                        value(applierState), value(storedState));
+                }
+            }
+        }
+
+        void stabilise() {
+            stabilise(DEFAULT_STABILISATION_TIME);
+        }
+
+        void stabilise(long stabilisationDurationMillis) {
+            assertThat("stabilisation requires default delay variability (and proper cleanup of raised variability)",
+                deterministicTaskQueue.getExecutionDelayVariabilityMillis(), lessThanOrEqualTo(DEFAULT_DELAY_VARIABILITY));
+            assertFalse("stabilisation requires stable storage", disruptStorage);
+
+            bootstrapIfNecessary();
+
+            runFor(stabilisationDurationMillis, "stabilising");
+
+            final ClusterNode leader = getAnyLeader();
+            final long leaderTerm = leader.coordinator.getCurrentTerm();
+
+            final int pendingTaskCount = leader.masterService.getFakeMasterServicePendingTaskCount();
+            runFor((pendingTaskCount + 1) * DEFAULT_CLUSTER_STATE_UPDATE_DELAY, "draining task queue");
+
+            final Matcher<Long> isEqualToLeaderVersion = equalTo(leader.coordinator.getLastAcceptedState().getVersion());
+            final String leaderId = leader.getId();
+
+            assertTrue(leaderId + " has been bootstrapped", leader.coordinator.isInitialConfigurationSet());
+            assertTrue(leaderId + " exists in its last-applied state", leader.getLastAppliedClusterState().getNodes().nodeExists(leaderId));
+            assertThat(leaderId + " has no NO_MASTER_BLOCK",
+                leader.getLastAppliedClusterState().blocks().hasGlobalBlockWithId(NO_MASTER_BLOCK_ID), equalTo(false));
+            assertThat(leaderId + " has no STATE_NOT_RECOVERED_BLOCK",
+                leader.getLastAppliedClusterState().blocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK), equalTo(false));
+            assertThat(leaderId + " has applied its state ", leader.getLastAppliedClusterState().getVersion(), isEqualToLeaderVersion);
+
+            for (final ClusterNode clusterNode : clusterNodes) {
+                final String nodeId = clusterNode.getId();
+                assertFalse(nodeId + " should not have an active publication", clusterNode.coordinator.publicationInProgress());
+
+                if (clusterNode == leader) {
+                    assertThat(nodeId + " is still the leader", clusterNode.coordinator.getMode(), is(LEADER));
+                    assertThat(nodeId + " did not change term", clusterNode.coordinator.getCurrentTerm(), is(leaderTerm));
+                    continue;
+                }
+
+                if (isConnectedPair(leader, clusterNode)) {
+                    assertThat(nodeId + " is a follower of " + leaderId, clusterNode.coordinator.getMode(), is(FOLLOWER));
+                    assertThat(nodeId + " has the same term as " + leaderId, clusterNode.coordinator.getCurrentTerm(), is(leaderTerm));
+                    assertTrue(nodeId + " has voted for " + leaderId, leader.coordinator.hasJoinVoteFrom(clusterNode.getLocalNode()));
+                    assertThat(nodeId + " has the same accepted state as " + leaderId,
+                        clusterNode.coordinator.getLastAcceptedState().getVersion(), isEqualToLeaderVersion);
+                    if (clusterNode.getClusterStateApplyResponse() == ClusterStateApplyResponse.SUCCEED) {
+                        assertThat(nodeId + " has the same applied state as " + leaderId,
+                            clusterNode.getLastAppliedClusterState().getVersion(), isEqualToLeaderVersion);
+                        assertTrue(nodeId + " is in its own latest applied state",
+                            clusterNode.getLastAppliedClusterState().getNodes().nodeExists(nodeId));
+                    }
+                    assertTrue(nodeId + " is in the latest applied state on " + leaderId,
+                        leader.getLastAppliedClusterState().getNodes().nodeExists(nodeId));
+                    assertTrue(nodeId + " has been bootstrapped", clusterNode.coordinator.isInitialConfigurationSet());
+                    assertThat(nodeId + " has correct master", clusterNode.getLastAppliedClusterState().nodes().getMasterNode(),
+                        equalTo(leader.getLocalNode()));
+                    assertThat(nodeId + " has no NO_MASTER_BLOCK",
+                        clusterNode.getLastAppliedClusterState().blocks().hasGlobalBlockWithId(NO_MASTER_BLOCK_ID), equalTo(false));
+                    assertThat(nodeId + " has no STATE_NOT_RECOVERED_BLOCK",
+                        clusterNode.getLastAppliedClusterState().blocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK), equalTo(false));
+                } else {
+                    assertThat(nodeId + " is not following " + leaderId, clusterNode.coordinator.getMode(), is(CANDIDATE));
+                    assertThat(nodeId + " has no master", clusterNode.getLastAppliedClusterState().nodes().getMasterNode(), nullValue());
+                    assertThat(nodeId + " has NO_MASTER_BLOCK",
+                        clusterNode.getLastAppliedClusterState().blocks().hasGlobalBlockWithId(NO_MASTER_BLOCK_ID), equalTo(true));
+                    assertFalse(nodeId + " is not in the applied state on " + leaderId,
+                        leader.getLastAppliedClusterState().getNodes().nodeExists(nodeId));
+                }
+            }
+
+            final Set<String> connectedNodeIds
+                = clusterNodes.stream().filter(n -> isConnectedPair(leader, n)).map(ClusterNode::getId).collect(Collectors.toSet());
+
+            assertThat(leader.getLastAppliedClusterState().getNodes().getSize(), equalTo(connectedNodeIds.size()));
+
+            final ClusterState lastAcceptedState = leader.coordinator.getLastAcceptedState();
+            final VotingConfiguration lastCommittedConfiguration = lastAcceptedState.getLastCommittedConfiguration();
+            assertTrue(connectedNodeIds + " should be a quorum of " + lastCommittedConfiguration,
+                lastCommittedConfiguration.hasQuorum(connectedNodeIds));
+            assertThat("leader " + leader.getLocalNode() + " should be part of voting configuration " + lastCommittedConfiguration,
+                lastCommittedConfiguration.getNodeIds(), IsCollectionContaining.hasItem(leader.getLocalNode().getId()));
+
+            assertThat("no reconfiguration is in progress",
+                lastAcceptedState.getLastCommittedConfiguration(), equalTo(lastAcceptedState.getLastAcceptedConfiguration()));
+            assertThat("current configuration is already optimal",
+                leader.improveConfiguration(lastAcceptedState), sameInstance(lastAcceptedState));
+
+            logger.info("checking linearizability of history with size {}: {}", history.size(), history);
+            assertTrue("history not linearizable: " + history, linearizabilityChecker.isLinearizable(spec, history, i -> null));
+            logger.info("linearizability check completed");
+        }
+
+        void bootstrapIfNecessary() {
+            if (clusterNodes.stream().allMatch(ClusterNode::isNotUsefullyBootstrapped)) {
+                assertThat("setting initial configuration may fail with disconnected nodes", disconnectedNodes, empty());
+                assertThat("setting initial configuration may fail with blackholed nodes", blackholedNodes, empty());
+                runFor(defaultMillis(DISCOVERY_FIND_PEERS_INTERVAL_SETTING) * 2, "discovery prior to setting initial configuration");
+                final ClusterNode bootstrapNode = getAnyBootstrappableNode();
+                bootstrapNode.applyInitialConfiguration();
+            } else {
+                logger.info("setting initial configuration not required");
+            }
+        }
+
+        void runFor(long runDurationMillis, String description) {
+            final long endTime = deterministicTaskQueue.getCurrentTimeMillis() + runDurationMillis;
+            logger.info("--> runFor({}ms) running until [{}ms]: {}", runDurationMillis, endTime, description);
+
+            while (deterministicTaskQueue.getCurrentTimeMillis() < endTime) {
+
+                while (deterministicTaskQueue.hasRunnableTasks()) {
+                    try {
+                        deterministicTaskQueue.runRandomTask();
+                    } catch (CoordinationStateRejectedException e) {
+                        logger.debug("ignoring benign exception thrown when stabilising", e);
+                    }
+                    for (final ClusterNode clusterNode : clusterNodes) {
+                        clusterNode.coordinator.invariant();
+                    }
+                    updateCommittedStates();
+                }
+
+                if (deterministicTaskQueue.hasDeferredTasks() == false) {
+                    // A 1-node cluster has no need for fault detection etc so will eventually run out of things to do.
+                    assert clusterNodes.size() == 1 : clusterNodes.size();
+                    break;
+                }
+
+                deterministicTaskQueue.advanceTime();
+            }
+
+            logger.info("--> runFor({}ms) completed run until [{}ms]: {}", runDurationMillis, endTime, description);
+        }
+
+        private boolean isConnectedPair(ClusterNode n1, ClusterNode n2) {
+            return n1 == n2 ||
+                (getConnectionStatus(n1.getLocalNode(), n2.getLocalNode()) == ConnectionStatus.CONNECTED
+                    && getConnectionStatus(n2.getLocalNode(), n1.getLocalNode()) == ConnectionStatus.CONNECTED);
+        }
+
+        ClusterNode getAnyLeader() {
+            List<ClusterNode> allLeaders = clusterNodes.stream().filter(ClusterNode::isLeader).collect(Collectors.toList());
+            assertThat("leaders", allLeaders, not(empty()));
+            return randomFrom(allLeaders);
+        }
+
+        private final ConnectionStatus preferredUnknownNodeConnectionStatus =
+            randomFrom(ConnectionStatus.DISCONNECTED, ConnectionStatus.BLACK_HOLE);
+
+        private ConnectionStatus getConnectionStatus(DiscoveryNode sender, DiscoveryNode destination) {
+            ConnectionStatus connectionStatus;
+            if (blackholedNodes.contains(sender.getId()) || blackholedNodes.contains(destination.getId())) {
+                connectionStatus = ConnectionStatus.BLACK_HOLE;
+            } else if (disconnectedNodes.contains(sender.getId()) || disconnectedNodes.contains(destination.getId())) {
+                connectionStatus = ConnectionStatus.DISCONNECTED;
+            } else if (blackholedConnections.contains(Tuple.tuple(sender.getId(), destination.getId()))) {
+                connectionStatus = ConnectionStatus.BLACK_HOLE_REQUESTS_ONLY;
+            } else if (nodeExists(sender) && nodeExists(destination)) {
+                connectionStatus = ConnectionStatus.CONNECTED;
+            } else {
+                connectionStatus = usually() ? preferredUnknownNodeConnectionStatus :
+                    randomFrom(ConnectionStatus.DISCONNECTED, ConnectionStatus.BLACK_HOLE);
+            }
+            return connectionStatus;
+        }
+
+        boolean nodeExists(DiscoveryNode node) {
+            return clusterNodes.stream().anyMatch(cn -> cn.getLocalNode().equals(node));
+        }
+
+        ClusterNode getAnyBootstrappableNode() {
+            return randomFrom(clusterNodes.stream().filter(n -> n.getLocalNode().isMasterNode())
+                .filter(n -> initialConfiguration.getNodeIds().contains(n.getLocalNode().getId()))
+                .collect(Collectors.toList()));
+        }
+
+        ClusterNode getAnyNode() {
+            return getAnyNodeExcept();
+        }
+
+        ClusterNode getAnyNodeExcept(ClusterNode... clusterNodes) {
+            List<ClusterNode> filteredNodes = getAllNodesExcept(clusterNodes);
+            assert filteredNodes.isEmpty() == false;
+            return randomFrom(filteredNodes);
+        }
+
+        List<ClusterNode> getAllNodesExcept(ClusterNode... clusterNodes) {
+            Set<String> forbiddenIds = Arrays.stream(clusterNodes).map(ClusterNode::getId).collect(Collectors.toSet());
+            List<ClusterNode> acceptableNodes
+                = this.clusterNodes.stream().filter(n -> forbiddenIds.contains(n.getId()) == false).collect(Collectors.toList());
+            return acceptableNodes;
+        }
+
+        ClusterNode getAnyNodePreferringLeaders() {
+            for (int i = 0; i < 3; i++) {
+                ClusterNode clusterNode = getAnyNode();
+                if (clusterNode.coordinator.getMode() == LEADER) {
+                    return clusterNode;
+                }
+            }
+            return getAnyNode();
+        }
+
+        void setEmptySeedHostsList() {
+            seedHostsList = emptyList();
+        }
+
+        void blackholeConnectionsFrom(ClusterNode sender, ClusterNode destination) {
+            blackholedConnections.add(Tuple.tuple(sender.getId(), destination.getId()));
+        }
+
+        void clearBlackholedConnections() {
+            blackholedConnections.clear();
+        }
+
+        class MockPersistedState implements CoordinationState.PersistedState {
+            private final CoordinationState.PersistedState delegate;
+            private final NodeEnvironment nodeEnvironment;
+
+            MockPersistedState(DiscoveryNode localNode) {
+                try {
+                    if (rarely()) {
+                        nodeEnvironment = newNodeEnvironment();
+                        nodeEnvironments.add(nodeEnvironment);
+                        delegate = new MockGatewayMetaState(Settings.EMPTY, nodeEnvironment, xContentRegistry(), localNode)
+                            .getPersistedState(Settings.EMPTY, null);
+                    } else {
+                        nodeEnvironment = null;
+                        delegate = new InMemoryPersistedState(0L,
+                            ClusterStateUpdaters.addStateNotRecoveredBlock(
+                                clusterState(0L, 0L, localNode, VotingConfiguration.EMPTY_CONFIG,
+                                    VotingConfiguration.EMPTY_CONFIG, 0L)));
+                    }
+                } catch (IOException e) {
+                    throw new UncheckedIOException("Unable to create MockPersistedState", e);
+                }
+            }
+
+            MockPersistedState(DiscoveryNode newLocalNode, MockPersistedState oldState,
+                               Function<MetaData, MetaData> adaptGlobalMetaData, Function<Long, Long> adaptCurrentTerm) {
+                try {
+                    if (oldState.nodeEnvironment != null) {
+                        nodeEnvironment = oldState.nodeEnvironment;
+                        final MetaStateService metaStateService = new MetaStateService(nodeEnvironment, xContentRegistry());
+                        final MetaData updatedMetaData = adaptGlobalMetaData.apply(oldState.getLastAcceptedState().metaData());
+                        if (updatedMetaData != oldState.getLastAcceptedState().metaData()) {
+                            metaStateService.writeGlobalStateAndUpdateManifest("update global state", updatedMetaData);
+                        }
+                        final long updatedTerm = adaptCurrentTerm.apply(oldState.getCurrentTerm());
+                        if (updatedTerm != oldState.getCurrentTerm()) {
+                            final Manifest manifest = metaStateService.loadManifestOrEmpty();
+                            metaStateService.writeManifestAndCleanup("update term",
+                                new Manifest(updatedTerm, manifest.getClusterStateVersion(), manifest.getGlobalGeneration(),
+                                    manifest.getIndexGenerations()));
+                        }
+                        delegate = new MockGatewayMetaState(Settings.EMPTY, nodeEnvironment, xContentRegistry(), newLocalNode)
+                            .getPersistedState(Settings.EMPTY, null);
+                    } else {
+                        nodeEnvironment = null;
+                        BytesStreamOutput outStream = new BytesStreamOutput();
+                        outStream.setVersion(Version.CURRENT);
+                        final MetaData updatedMetaData = adaptGlobalMetaData.apply(oldState.getLastAcceptedState().metaData());
+                        final ClusterState clusterState;
+                        if (updatedMetaData != oldState.getLastAcceptedState().metaData()) {
+                            clusterState = ClusterState.builder(oldState.getLastAcceptedState()).metaData(updatedMetaData).build();
+                        } else {
+                            clusterState = oldState.getLastAcceptedState();
+                        }
+                        clusterState.writeTo(outStream);
+                        StreamInput inStream = new NamedWriteableAwareStreamInput(outStream.bytes().streamInput(),
+                            new NamedWriteableRegistry(ClusterModule.getNamedWriteables()));
+                        // adapt cluster state to new localNode instance and add blocks
+                        delegate = new InMemoryPersistedState(adaptCurrentTerm.apply(oldState.getCurrentTerm()),
+                            ClusterStateUpdaters.addStateNotRecoveredBlock(ClusterState.readFrom(inStream, newLocalNode)));
+                    }
+                } catch (IOException e) {
+                    throw new UncheckedIOException("Unable to create MockPersistedState", e);
+                }
+            }
+
+            private void possiblyFail(String description) {
+                if (disruptStorage && rarely()) {
+                    logger.trace("simulating IO exception [{}]", description);
+                    // In the real-life IOError might be thrown, for example if state fsync fails.
+                    // This will require node restart and we're not emulating it here.
+                    throw new UncheckedIOException(new IOException("simulated IO exception [" + description + ']'));
+                }
+            }
+
+            @Override
+            public long getCurrentTerm() {
+                return delegate.getCurrentTerm();
+            }
+
+            @Override
+            public ClusterState getLastAcceptedState() {
+                return delegate.getLastAcceptedState();
+            }
+
+            @Override
+            public void setCurrentTerm(long currentTerm) {
+                possiblyFail("before writing term of " + currentTerm);
+                delegate.setCurrentTerm(currentTerm);
+            }
+
+            @Override
+            public void setLastAcceptedState(ClusterState clusterState) {
+                possiblyFail("before writing last-accepted state of term=" + clusterState.term() + ", version=" + clusterState.version());
+                delegate.setLastAcceptedState(clusterState);
+            }
+        }
+
+        class ClusterNode {
+            private final Logger logger = LogManager.getLogger(ClusterNode.class);
+
+            private final int nodeIndex;
+            Coordinator coordinator;
+            private final DiscoveryNode localNode;
+            final MockPersistedState persistedState;
+            final Settings nodeSettings;
+            private AckedFakeThreadPoolMasterService masterService;
+            private DisruptableClusterApplierService clusterApplierService;
+            private ClusterService clusterService;
+            TransportService transportService;
+            private DisruptableMockTransport mockTransport;
+            List<BiConsumer<DiscoveryNode, ClusterState>> extraJoinValidators = new ArrayList<>();
+
+            ClusterNode(int nodeIndex, boolean masterEligible, Settings nodeSettings) {
+                this(nodeIndex, createDiscoveryNode(nodeIndex, masterEligible), defaultPersistedStateSupplier, nodeSettings);
+            }
+
+            ClusterNode(int nodeIndex, DiscoveryNode localNode, Function<DiscoveryNode, MockPersistedState> persistedStateSupplier,
+                        Settings nodeSettings) {
+                this.nodeIndex = nodeIndex;
+                this.localNode = localNode;
+                this.nodeSettings = nodeSettings;
+                persistedState = persistedStateSupplier.apply(localNode);
+                onNodeLog(localNode, this::setUp).run();
+            }
+
+            private void setUp() {
+                mockTransport = new DisruptableMockTransport(localNode, logger) {
+                    @Override
+                    protected void execute(Runnable runnable) {
+                        deterministicTaskQueue.scheduleNow(onNode(runnable));
+                    }
+
+                    @Override
+                    protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) {
+                        return Cluster.this.getConnectionStatus(getLocalNode(), destination);
+                    }
+
+                    @Override
+                    protected Optional<DisruptableMockTransport> getDisruptableMockTransport(TransportAddress address) {
+                        return clusterNodes.stream().map(cn -> cn.mockTransport)
+                            .filter(transport -> transport.getLocalNode().getAddress().equals(address)).findAny();
+                    }
+                };
+
+                final Settings settings = nodeSettings.hasValue(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey()) ?
+                    nodeSettings : Settings.builder().put(nodeSettings)
+                    .putList(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(),
+                        ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.get(Settings.EMPTY)).build(); // suppress auto-bootstrap
+                transportService = mockTransport.createTransportService(
+                    settings, deterministicTaskQueue.getThreadPool(this::onNode),
+                    getTransportInterceptor(localNode, deterministicTaskQueue.getThreadPool(this::onNode)),
+                    a -> localNode, null, emptySet());
+                masterService = new AckedFakeThreadPoolMasterService(localNode.getId(), "test",
+                    runnable -> deterministicTaskQueue.scheduleNow(onNode(runnable)));
+                final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+                clusterApplierService = new DisruptableClusterApplierService(localNode.getId(), settings, clusterSettings,
+                    deterministicTaskQueue, this::onNode);
+                clusterService = new ClusterService(settings, clusterSettings, masterService, clusterApplierService);
+                clusterService.setNodeConnectionsService(
+                    new NodeConnectionsService(clusterService.getSettings(), deterministicTaskQueue.getThreadPool(this::onNode),
+                        transportService));
+                final Collection<BiConsumer<DiscoveryNode, ClusterState>> onJoinValidators =
+                    Collections.singletonList((dn, cs) -> extraJoinValidators.forEach(validator -> validator.accept(dn, cs)));
+                final AllocationService allocationService = ESAllocationTestCase.createAllocationService(Settings.EMPTY);
+                coordinator = new Coordinator("test_node", settings, clusterSettings, transportService, writableRegistry(),
+                    allocationService, masterService, this::getPersistedState,
+                    Cluster.this::provideSeedHosts, clusterApplierService, onJoinValidators, Randomness.get(), (s, r) -> {},
+                    getElectionStrategy());
+                masterService.setClusterStatePublisher(coordinator);
+                final GatewayService gatewayService = new GatewayService(settings, allocationService, clusterService,
+                    deterministicTaskQueue.getThreadPool(this::onNode), null, coordinator);
+
+                logger.trace("starting up [{}]", localNode);
+                transportService.start();
+                transportService.acceptIncomingRequests();
+                coordinator.start();
+                gatewayService.start();
+                clusterService.start();
+                coordinator.startInitialJoin();
+            }
+
+            void close() {
+                onNode(() -> {
+                    logger.trace("taking down [{}]", localNode);
+                    coordinator.stop();
+                    clusterService.stop();
+                    //transportService.stop(); // does blocking stuff :/
+                    clusterService.close();
+                    coordinator.close();
+                    //transportService.close(); // does blocking stuff :/
+                });
+            }
+
+            ClusterNode restartedNode() {
+                return restartedNode(Function.identity(), Function.identity(), nodeSettings);
+            }
+
+            ClusterNode restartedNode(Function<MetaData, MetaData> adaptGlobalMetaData, Function<Long, Long> adaptCurrentTerm,
+                                      Settings nodeSettings) {
+                final TransportAddress address = randomBoolean() ? buildNewFakeTransportAddress() : localNode.getAddress();
+                final DiscoveryNode newLocalNode = new DiscoveryNode(localNode.getName(), localNode.getId(),
+                    UUIDs.randomBase64UUID(random()), // generated deterministically for repeatable tests
+                    address.address().getHostString(), address.getAddress(), address, Collections.emptyMap(),
+                    localNode.isMasterNode() ? DiscoveryNodeRole.BUILT_IN_ROLES : emptySet(), Version.CURRENT);
+                return new ClusterNode(nodeIndex, newLocalNode,
+                    node -> new MockPersistedState(newLocalNode, persistedState, adaptGlobalMetaData, adaptCurrentTerm), nodeSettings);
+            }
+
+            private CoordinationState.PersistedState getPersistedState() {
+                return persistedState;
+            }
+
+            String getId() {
+                return localNode.getId();
+            }
+
+            DiscoveryNode getLocalNode() {
+                return localNode;
+            }
+
+            boolean isLeader() {
+                return coordinator.getMode() == LEADER;
+            }
+
+            boolean isCandidate() {
+                return coordinator.getMode() == CANDIDATE;
+            }
+
+            ClusterState improveConfiguration(ClusterState currentState) {
+                synchronized (coordinator.mutex) {
+                    return coordinator.improveConfiguration(currentState);
+                }
+            }
+
+            void setClusterStateApplyResponse(ClusterStateApplyResponse clusterStateApplyResponse) {
+                clusterApplierService.clusterStateApplyResponse = clusterStateApplyResponse;
+            }
+
+            ClusterStateApplyResponse getClusterStateApplyResponse() {
+                return clusterApplierService.clusterStateApplyResponse;
+            }
+
+            Runnable onNode(Runnable runnable) {
+                final Runnable wrapped = onNodeLog(localNode, runnable);
+                return new Runnable() {
+                    @Override
+                    public void run() {
+                        if (clusterNodes.contains(ClusterNode.this) == false) {
+                            logger.trace("ignoring runnable {} from node {} as node has been removed from cluster", runnable, localNode);
+                            return;
+                        }
+                        wrapped.run();
+                    }
+
+                    @Override
+                    public String toString() {
+                        return wrapped.toString();
+                    }
+                };
+            }
+
+            void submitSetAutoShrinkVotingConfiguration(final boolean autoShrinkVotingConfiguration) {
+                submitUpdateTask("set master nodes failure tolerance [" + autoShrinkVotingConfiguration + "]", cs ->
+                    ClusterState.builder(cs).metaData(
+                        MetaData.builder(cs.metaData())
+                            .persistentSettings(Settings.builder()
+                                .put(cs.metaData().persistentSettings())
+                                .put(CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION.getKey(), autoShrinkVotingConfiguration)
+                                .build())
+                            .build())
+                        .build(), (source, e) -> {});
+            }
+
+            AckCollector submitValue(final long value) {
+                return submitValue(0, value);
+            }
+
+            AckCollector submitValue(final int key, final long value) {
+                final int eventId = history.invoke(new Tuple<>(key, value));
+                return submitUpdateTask("new value [" + value + "]", cs -> setValue(cs, key, value), new ClusterStateTaskListener() {
+                    @Override
+                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                        history.respond(eventId, value(oldState, key));
+                    }
+
+                    @Override
+                    public void onNoLongerMaster(String source) {
+                        // in this case, we know for sure that event was not processed by the system and will not change history
+                        // remove event to help avoid bloated history and state space explosion in linearizability checker
+                        history.remove(eventId);
+                    }
+
+                    @Override
+                    public void onFailure(String source, Exception e) {
+                        // do not remove event from history, the write might still take place
+                        // instead, complete history when checking for linearizability
+                    }
+                });
+            }
+
+            void readValue(int key) {
+                final int eventId = history.invoke(new Tuple<>(key, null));
+                submitUpdateTask("read value", cs -> ClusterState.builder(cs).build(), new ClusterStateTaskListener() {
+                    @Override
+                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                        history.respond(eventId, value(newState, key));
+                    }
+
+                    @Override
+                    public void onFailure(String source, Exception e) {
+                        // reads do not change state
+                        // remove event to help avoid bloated history and state space explosion in linearizability checker
+                        history.remove(eventId);
+                    }
+                });
+            }
+
+            AckCollector submitUpdateTask(String source, UnaryOperator<ClusterState> clusterStateUpdate,
+                                          ClusterStateTaskListener taskListener) {
+                final AckCollector ackCollector = new AckCollector();
+                onNode(() -> {
+                    logger.trace("[{}] submitUpdateTask: enqueueing [{}]", localNode.getId(), source);
+                    final long submittedTerm = coordinator.getCurrentTerm();
+                    masterService.submitStateUpdateTask(source,
+                        new ClusterStateUpdateTask() {
+                            @Override
+                            public ClusterState execute(ClusterState currentState) {
+                                assertThat(currentState.term(), greaterThanOrEqualTo(submittedTerm));
+                                masterService.nextAckCollector = ackCollector;
+                                return clusterStateUpdate.apply(currentState);
+                            }
+
+                            @Override
+                            public void onFailure(String source, Exception e) {
+                                logger.debug(() -> new ParameterizedMessage("failed to publish: [{}]", source), e);
+                                taskListener.onFailure(source, e);
+                            }
+
+                            @Override
+                            public void onNoLongerMaster(String source) {
+                                logger.trace("no longer master: [{}]", source);
+                                taskListener.onNoLongerMaster(source);
+                            }
+
+                            @Override
+                            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                                updateCommittedStates();
+                                ClusterState state = committedStatesByVersion.get(newState.version());
+                                assertNotNull("State not committed : " + newState.toString(), state);
+                                assertStateEquals(state, newState);
+                                logger.trace("successfully published: [{}]", newState);
+                                taskListener.clusterStateProcessed(source, oldState, newState);
+                            }
+                        });
+                }).run();
+                return ackCollector;
+            }
+
+            @Override
+            public String toString() {
+                return localNode.toString();
+            }
+
+            boolean heal() {
+                boolean unBlackholed = blackholedNodes.remove(localNode.getId());
+                boolean unDisconnected = disconnectedNodes.remove(localNode.getId());
+                assert unBlackholed == false || unDisconnected == false;
+                return unBlackholed || unDisconnected;
+            }
+
+            boolean disconnect() {
+                boolean unBlackholed = blackholedNodes.remove(localNode.getId());
+                boolean disconnected = disconnectedNodes.add(localNode.getId());
+                assert disconnected || unBlackholed == false;
+                return disconnected;
+            }
+
+            boolean blackhole() {
+                boolean unDisconnected = disconnectedNodes.remove(localNode.getId());
+                boolean blackholed = blackholedNodes.add(localNode.getId());
+                assert blackholed || unDisconnected == false;
+                return blackholed;
+            }
+
+            void onDisconnectEventFrom(ClusterNode clusterNode) {
+                transportService.disconnectFromNode(clusterNode.localNode);
+            }
+
+            ClusterState getLastAppliedClusterState() {
+                return clusterApplierService.state();
+            }
+
+            void applyInitialConfiguration() {
+                onNode(() -> {
+                    final Set<String> nodeIdsWithPlaceholders = new HashSet<>(initialConfiguration.getNodeIds());
+                    Stream.generate(() -> BOOTSTRAP_PLACEHOLDER_PREFIX + UUIDs.randomBase64UUID(random()))
+                        .limit((Math.max(initialConfiguration.getNodeIds().size(), 2) - 1) / 2)
+                        .forEach(nodeIdsWithPlaceholders::add);
+                    final Set<String> nodeIds = new HashSet<>(
+                        randomSubsetOf(initialConfiguration.getNodeIds().size(), nodeIdsWithPlaceholders));
+                    // initial configuration should not have a place holder for local node
+                    if (initialConfiguration.getNodeIds().contains(localNode.getId()) && nodeIds.contains(localNode.getId()) == false) {
+                        nodeIds.remove(nodeIds.iterator().next());
+                        nodeIds.add(localNode.getId());
+                    }
+                    final VotingConfiguration configurationWithPlaceholders = new VotingConfiguration(nodeIds);
+                    try {
+                        coordinator.setInitialConfiguration(configurationWithPlaceholders);
+                        logger.info("successfully set initial configuration to {}", configurationWithPlaceholders);
+                    } catch (CoordinationStateRejectedException e) {
+                        logger.info(new ParameterizedMessage("failed to set initial configuration to {}",
+                            configurationWithPlaceholders), e);
+                    }
+                }).run();
+            }
+
+            private boolean isNotUsefullyBootstrapped() {
+                return getLocalNode().isMasterNode() == false || coordinator.isInitialConfigurationSet() == false;
+            }
+        }
+
+        private List<TransportAddress> provideSeedHosts(SeedHostsProvider.HostsResolver ignored) {
+            return seedHostsList != null ? seedHostsList
+                : clusterNodes.stream().map(ClusterNode::getLocalNode).map(DiscoveryNode::getAddress).collect(Collectors.toList());
+        }
+    }
+
+    protected TransportInterceptor getTransportInterceptor(DiscoveryNode localNode, ThreadPool threadPool) {
+        return NOOP_TRANSPORT_INTERCEPTOR;
+    }
+
+    protected ElectionStrategy getElectionStrategy() {
+        return ElectionStrategy.DEFAULT_INSTANCE;
+    }
+
+    public static final String NODE_ID_LOG_CONTEXT_KEY = "nodeId";
+
+    protected static String getNodeIdForLogContext(DiscoveryNode node) {
+        return "{" + node.getId() + "}{" + node.getEphemeralId() + "}";
+    }
+
+    public static Runnable onNodeLog(DiscoveryNode node, Runnable runnable) {
+        final String nodeId = getNodeIdForLogContext(node);
+        return new Runnable() {
+            @Override
+            public void run() {
+                try (CloseableThreadContext.Instance ignored = CloseableThreadContext.put(NODE_ID_LOG_CONTEXT_KEY, nodeId)) {
+                    runnable.run();
+                }
+            }
+
+            @Override
+            public String toString() {
+                return nodeId + ": " + runnable.toString();
+            }
+        };
+    }
+
+    static class AckCollector implements ClusterStatePublisher.AckListener {
+
+        private final Set<DiscoveryNode> ackedNodes = new HashSet<>();
+        private final List<DiscoveryNode> successfulNodes = new ArrayList<>();
+        private final List<DiscoveryNode> unsuccessfulNodes = new ArrayList<>();
+
+        @Override
+        public void onCommit(TimeValue commitTime) {
+            // TODO we only currently care about per-node acks
+        }
+
+        @Override
+        public void onNodeAck(DiscoveryNode node, Exception e) {
+            assertTrue("duplicate ack from " + node, ackedNodes.add(node));
+            if (e == null) {
+                successfulNodes.add(node);
+            } else {
+                unsuccessfulNodes.add(node);
+            }
+        }
+
+        boolean hasAckedSuccessfully(ClusterNode clusterNode) {
+            return successfulNodes.contains(clusterNode.localNode);
+        }
+
+        boolean hasAckedUnsuccessfully(ClusterNode clusterNode) {
+            return unsuccessfulNodes.contains(clusterNode.localNode);
+        }
+
+        boolean hasAcked(ClusterNode clusterNode) {
+            return ackedNodes.contains(clusterNode.localNode);
+        }
+
+        int getSuccessfulAckIndex(ClusterNode clusterNode) {
+            assert successfulNodes.contains(clusterNode.localNode) : "get index of " + clusterNode;
+            return successfulNodes.indexOf(clusterNode.localNode);
+        }
+    }
+
+    static class AckedFakeThreadPoolMasterService extends FakeThreadPoolMasterService {
+
+        AckCollector nextAckCollector = new AckCollector();
+
+        AckedFakeThreadPoolMasterService(String nodeName, String serviceName, Consumer<Runnable> onTaskAvailableToRun) {
+            super(nodeName, serviceName, onTaskAvailableToRun);
+        }
+
+        @Override
+        protected ClusterStatePublisher.AckListener wrapAckListener(ClusterStatePublisher.AckListener ackListener) {
+            final AckCollector ackCollector = nextAckCollector;
+            nextAckCollector = new AckCollector();
+            return new ClusterStatePublisher.AckListener() {
+                @Override
+                public void onCommit(TimeValue commitTime) {
+                    ackCollector.onCommit(commitTime);
+                    ackListener.onCommit(commitTime);
+                }
+
+                @Override
+                public void onNodeAck(DiscoveryNode node, Exception e) {
+                    ackCollector.onNodeAck(node, e);
+                    ackListener.onNodeAck(node, e);
+                }
+            };
+        }
+    }
+
+    static class DisruptableClusterApplierService extends ClusterApplierService {
+        private final String nodeName;
+        private final DeterministicTaskQueue deterministicTaskQueue;
+        ClusterStateApplyResponse clusterStateApplyResponse = ClusterStateApplyResponse.SUCCEED;
+
+        DisruptableClusterApplierService(String nodeName, Settings settings, ClusterSettings clusterSettings,
+                                         DeterministicTaskQueue deterministicTaskQueue, Function<Runnable, Runnable> runnableWrapper) {
+            super(nodeName, settings, clusterSettings, deterministicTaskQueue.getThreadPool(runnableWrapper));
+            this.nodeName = nodeName;
+            this.deterministicTaskQueue = deterministicTaskQueue;
+            addStateApplier(event -> {
+                switch (clusterStateApplyResponse) {
+                    case SUCCEED:
+                    case HANG:
+                        final ClusterState oldClusterState = event.previousState();
+                        final ClusterState newClusterState = event.state();
+                        assert oldClusterState.version() <= newClusterState.version() : "updating cluster state from version "
+                            + oldClusterState.version() + " to stale version " + newClusterState.version();
+                        break;
+                    case FAIL:
+                        throw new ElasticsearchException("simulated cluster state applier failure");
+                }
+            });
+        }
+
+        @Override
+        protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() {
+            return new MockSinglePrioritizingExecutor(nodeName, deterministicTaskQueue);
+        }
+
+        @Override
+        public void onNewClusterState(String source, Supplier<ClusterState> clusterStateSupplier, ClusterApplyListener listener) {
+            if (clusterStateApplyResponse == ClusterStateApplyResponse.HANG) {
+                if (randomBoolean()) {
+                    // apply cluster state, but don't notify listener
+                    super.onNewClusterState(source, clusterStateSupplier, (source1, e) -> {
+                        // ignore result
+                    });
+                }
+            } else {
+                super.onNewClusterState(source, clusterStateSupplier, listener);
+            }
+        }
+
+        @Override
+        protected void connectToNodesAndWait(ClusterState newClusterState) {
+            // don't do anything, and don't block
+        }
+    }
+
+    protected DiscoveryNode createDiscoveryNode(int nodeIndex, boolean masterEligible) {
+        final TransportAddress address = buildNewFakeTransportAddress();
+        return new DiscoveryNode("", "node" + nodeIndex,
+            UUIDs.randomBase64UUID(random()), // generated deterministically for repeatable tests
+            address.address().getHostString(), address.getAddress(), address, Collections.emptyMap(),
+            masterEligible ? DiscoveryNodeRole.BUILT_IN_ROLES : emptySet(), Version.CURRENT);
+    }
+
+    /**
+     * How to behave with a new cluster state
+     */
+    enum ClusterStateApplyResponse {
+        /**
+         * Apply the state (default)
+         */
+        SUCCEED,
+
+        /**
+         * Reject the state with an exception.
+         */
+        FAIL,
+
+        /**
+         * Never respond either way.
+         */
+        HANG,
+    }
+
+    public ClusterState setValue(ClusterState clusterState, int key, long value) {
+        return ClusterState.builder(clusterState).metaData(
+            MetaData.builder(clusterState.metaData())
+                .persistentSettings(Settings.builder()
+                    .put(clusterState.metaData().persistentSettings())
+                    .put("value_" + key, value)
+                    .build())
+                .build())
+            .build();
+    }
+
+    public long value(ClusterState clusterState) {
+        return value(clusterState, 0);
+    }
+
+    public long value(ClusterState clusterState, int key) {
+        return clusterState.metaData().persistentSettings().getAsLong("value_" + key, 0L);
+    }
+
+    public void assertStateEquals(ClusterState clusterState1, ClusterState clusterState2) {
+        assertEquals(clusterState1.version(), clusterState2.version());
+        assertEquals(clusterState1.term(), clusterState2.term());
+        assertEquals(keySet(clusterState1), keySet(clusterState2));
+        for (int key : keySet(clusterState1)) {
+            assertEquals(value(clusterState1, key), value(clusterState2, key));
+        }
+    }
+
+    public Set<Integer> keySet(ClusterState clusterState) {
+        return clusterState.metaData().persistentSettings().keySet().stream()
+            .filter(s -> s.startsWith("value_")).map(s -> Integer.valueOf(s.substring("value_".length()))).collect(Collectors.toSet());
+    }
+
+    /**
+     * Simple register model. Writes are modeled by providing an integer input. Reads are modeled by providing null as input.
+     * Responses that time out are modeled by returning null. Successful writes return the previous value of the register.
+     */
+    private final SequentialSpec spec = new LinearizabilityChecker.KeyedSpec() {
+        @Override
+        public Object getKey(Object value) {
+            return ((Tuple) value).v1();
+        }
+
+        @Override
+        public Object getValue(Object value) {
+            return ((Tuple) value).v2();
+        }
+
+        @Override
+        public Object initialState() {
+            return 0L;
+        }
+
+        @Override
+        public Optional<Object> nextState(Object currentState, Object input, Object output) {
+            // null input is read, non-null is write
+            if (input == null) {
+                // history is completed with null, simulating timeout, which assumes that read went through
+                if (output == null || currentState.equals(output)) {
+                    return Optional.of(currentState);
+                }
+                return Optional.empty();
+            } else {
+                if (output == null || currentState.equals(output)) {
+                    // history is completed with null, simulating timeout, which assumes that write went through
+                    return Optional.of(input);
+                }
+                return Optional.empty();
+            }
+        }
+    };
+
+    public void testRegisterSpecConsistency() {
+        assertThat(spec.initialState(), equalTo(0L));
+        assertThat(spec.nextState(7, 42, 7), equalTo(Optional.of(42))); // successful write 42 returns previous value 7
+        assertThat(spec.nextState(7, 42, null), equalTo(Optional.of(42))); // write 42 times out
+        assertThat(spec.nextState(7, null, 7), equalTo(Optional.of(7))); // successful read
+        assertThat(spec.nextState(7, null, null), equalTo(Optional.of(7))); // read times out
+        assertThat(spec.nextState(7, null, 42), equalTo(Optional.empty()));
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/CoordinationStateTestCluster.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/CoordinationStateTestCluster.java
new file mode 100644
index 0000000000000..582ecd1fcface
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/CoordinationStateTestCluster.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.coordination;
+
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static com.carrotsearch.randomizedtesting.RandomizedTest.rarely;
+import static java.util.stream.Collectors.toSet;
+import static org.apache.lucene.util.LuceneTestCase.random;
+import static org.elasticsearch.test.ESTestCase.randomFrom;
+import static org.elasticsearch.test.ESTestCase.randomIntBetween;
+import static org.elasticsearch.test.ESTestCase.randomLong;
+import static org.elasticsearch.test.ESTestCase.randomLongBetween;
+import static org.elasticsearch.test.ESTestCase.randomSubsetOf;
+import static org.hamcrest.Matchers.hasSize;
+import static org.junit.Assert.assertThat;
+
+public class CoordinationStateTestCluster {
+
+    public static ClusterState clusterState(long term, long version, DiscoveryNode localNode,
+                                            CoordinationMetaData.VotingConfiguration lastCommittedConfig,
+                                            CoordinationMetaData.VotingConfiguration lastAcceptedConfig, long value) {
+        return clusterState(term, version, DiscoveryNodes.builder().add(localNode).localNodeId(localNode.getId()).build(),
+            lastCommittedConfig, lastAcceptedConfig, value);
+    }
+
+    public static ClusterState clusterState(long term, long version, DiscoveryNodes discoveryNodes,
+                                            CoordinationMetaData.VotingConfiguration lastCommittedConfig,
+                                            CoordinationMetaData.VotingConfiguration lastAcceptedConfig, long value) {
+        return setValue(ClusterState.builder(ClusterName.DEFAULT)
+            .version(version)
+            .nodes(discoveryNodes)
+            .metaData(MetaData.builder()
+                .clusterUUID(UUIDs.randomBase64UUID(random())) // generate cluster UUID deterministically for repeatable tests
+                .coordinationMetaData(CoordinationMetaData.builder()
+                    .term(term)
+                    .lastCommittedConfiguration(lastCommittedConfig)
+                    .lastAcceptedConfiguration(lastAcceptedConfig)
+                    .build()))
+            .stateUUID(UUIDs.randomBase64UUID(random())) // generate cluster state UUID deterministically for repeatable tests
+            .build(), value);
+    }
+
+    public static ClusterState setValue(ClusterState clusterState, long value) {
+        return ClusterState.builder(clusterState).metaData(
+            MetaData.builder(clusterState.metaData())
+                .persistentSettings(Settings.builder()
+                    .put(clusterState.metaData().persistentSettings())
+                    .put("value", value)
+                    .build())
+                .build())
+            .build();
+    }
+
+    public static long value(ClusterState clusterState) {
+        return clusterState.metaData().persistentSettings().getAsLong("value", 0L);
+    }
+
+    static class ClusterNode {
+
+        final DiscoveryNode localNode;
+        final CoordinationState.PersistedState persistedState;
+        private final ElectionStrategy electionStrategy;
+        CoordinationState state;
+
+        ClusterNode(DiscoveryNode localNode, ElectionStrategy electionStrategy) {
+            this.localNode = localNode;
+            persistedState = new InMemoryPersistedState(0L,
+                clusterState(0L, 0L, localNode, CoordinationMetaData.VotingConfiguration.EMPTY_CONFIG,
+                    CoordinationMetaData.VotingConfiguration.EMPTY_CONFIG, 0L));
+            this.electionStrategy = electionStrategy;
+            state = new CoordinationState(localNode, persistedState, electionStrategy);
+        }
+
+        void reboot() {
+            state = new CoordinationState(localNode, persistedState, electionStrategy);
+        }
+
+        void setInitialState(CoordinationMetaData.VotingConfiguration initialConfig, long initialValue) {
+            final ClusterState.Builder builder = ClusterState.builder(state.getLastAcceptedState());
+            builder.metaData(MetaData.builder()
+                .coordinationMetaData(CoordinationMetaData.builder()
+                    .lastAcceptedConfiguration(initialConfig)
+                    .lastCommittedConfiguration(initialConfig)
+                    .build()));
+            state.setInitialState(setValue(builder.build(), initialValue));
+        }
+    }
+
+    final ElectionStrategy electionStrategy;
+    final List<Message> messages;
+    final List<ClusterNode> clusterNodes;
+    final CoordinationMetaData.VotingConfiguration initialConfiguration;
+    final long initialValue;
+
+    CoordinationStateTestCluster(List<DiscoveryNode> nodes, ElectionStrategy electionStrategy) {
+        this.electionStrategy = electionStrategy;
+        messages = new ArrayList<>();
+
+        clusterNodes = nodes.stream()
+            .map(node -> new ClusterNode(node, electionStrategy))
+            .collect(Collectors.toList());
+
+        initialConfiguration = randomVotingConfig();
+        initialValue = randomLong();
+    }
+
+    static class Message {
+        final DiscoveryNode sourceNode;
+        final DiscoveryNode targetNode;
+        final Object payload;
+
+        Message(DiscoveryNode sourceNode, DiscoveryNode targetNode, Object payload) {
+            this.sourceNode = sourceNode;
+            this.targetNode = targetNode;
+            this.payload = payload;
+        }
+    }
+
+    void reply(Message m, Object payload) {
+        messages.add(new Message(m.targetNode, m.sourceNode, payload));
+    }
+
+    void broadcast(DiscoveryNode sourceNode, Object payload) {
+        messages.addAll(clusterNodes.stream().map(cn -> new Message(sourceNode, cn.localNode, payload)).collect(Collectors.toList()));
+    }
+
+    Optional<ClusterNode> getNode(DiscoveryNode node) {
+        return clusterNodes.stream().filter(cn -> cn.localNode.equals(node)).findFirst();
+    }
+
+    CoordinationMetaData.VotingConfiguration randomVotingConfig() {
+        return new CoordinationMetaData.VotingConfiguration(
+            randomSubsetOf(randomIntBetween(1, clusterNodes.size()), clusterNodes).stream()
+                .map(cn -> cn.localNode.getId()).collect(toSet()));
+    }
+
+    void applyMessage(Message message) {
+        final Optional<ClusterNode> maybeNode = getNode(message.targetNode);
+        if (maybeNode.isPresent() == false) {
+            throw new CoordinationStateRejectedException("node not available");
+        } else {
+            final Object payload = message.payload;
+            if (payload instanceof StartJoinRequest) {
+                reply(message, maybeNode.get().state.handleStartJoin((StartJoinRequest) payload));
+            } else if (payload instanceof Join) {
+                maybeNode.get().state.handleJoin((Join) payload);
+            } else if (payload instanceof PublishRequest) {
+                reply(message, maybeNode.get().state.handlePublishRequest((PublishRequest) payload));
+            } else if (payload instanceof PublishResponse) {
+                maybeNode.get().state.handlePublishResponse(message.sourceNode, (PublishResponse) payload)
+                    .ifPresent(ac -> broadcast(message.targetNode, ac));
+            } else if (payload instanceof ApplyCommitRequest) {
+                maybeNode.get().state.handleCommit((ApplyCommitRequest) payload);
+            } else {
+                throw new AssertionError("unknown message type");
+            }
+        }
+    }
+
+    void runRandomly() {
+        final int iterations = 10000;
+        final long maxTerm = 4;
+        long nextTerm = 1;
+        for (int i = 0; i < iterations; i++) {
+            try {
+                if (rarely() && nextTerm < maxTerm) {
+                    final long term = rarely() ? randomLongBetween(0, maxTerm + 1) : nextTerm++;
+                    final StartJoinRequest startJoinRequest = new StartJoinRequest(randomFrom(clusterNodes).localNode, term);
+                    broadcast(startJoinRequest.getSourceNode(), startJoinRequest);
+                } else if (rarely()) {
+                    randomFrom(clusterNodes).setInitialState(initialConfiguration, initialValue);
+                } else if (rarely() && rarely()) {
+                    randomFrom(clusterNodes).reboot();
+                } else if (rarely()) {
+                    final List<ClusterNode> masterNodes = clusterNodes.stream().filter(cn -> cn.state.electionWon())
+                        .collect(Collectors.toList());
+                    if (masterNodes.isEmpty() == false) {
+                        final ClusterNode clusterNode = randomFrom(masterNodes);
+                        final long term = rarely() ? randomLongBetween(0, maxTerm + 1) : clusterNode.state.getCurrentTerm();
+                        final long version = rarely() ? randomIntBetween(0, 5) : clusterNode.state.getLastPublishedVersion() + 1;
+                        final CoordinationMetaData.VotingConfiguration acceptedConfig = rarely() ? randomVotingConfig() :
+                            clusterNode.state.getLastAcceptedConfiguration();
+                        final PublishRequest publishRequest = clusterNode.state.handleClientValue(
+                            clusterState(term, version, clusterNode.localNode, clusterNode.state.getLastCommittedConfiguration(),
+                                acceptedConfig, randomLong()));
+                        broadcast(clusterNode.localNode, publishRequest);
+                    }
+                } else if (messages.isEmpty() == false) {
+                    applyMessage(randomFrom(messages));
+                }
+
+                // check node invariants after each iteration
+                clusterNodes.forEach(cn -> cn.state.invariant());
+            } catch (CoordinationStateRejectedException e) {
+                // ignore
+            }
+        }
+
+        // check system invariants. It's sufficient to do this at the end as these invariants are monotonic.
+        invariant();
+    }
+
+    void invariant() {
+        // one master per term
+        messages.stream().filter(m -> m.payload instanceof PublishRequest)
+            .collect(Collectors.groupingBy(m -> ((PublishRequest) m.payload).getAcceptedState().term()))
+            .forEach((term, publishMessages) -> {
+                Set<DiscoveryNode> mastersForTerm = publishMessages.stream().collect(Collectors.groupingBy(m -> m.sourceNode)).keySet();
+                assertThat("Multiple masters " + mastersForTerm + " for term " + term, mastersForTerm, hasSize(1));
+            });
+
+        // unique cluster state per (term, version) pair
+        messages.stream().filter(m -> m.payload instanceof PublishRequest)
+            .map(m -> ((PublishRequest) m.payload).getAcceptedState())
+            .collect(Collectors.groupingBy(ClusterState::term))
+            .forEach((term, clusterStates) -> {
+                clusterStates.stream().collect(Collectors.groupingBy(ClusterState::version))
+                    .forEach((version, clusterStates1) -> {
+                        Set<String> clusterStateUUIDsForTermAndVersion = clusterStates1.stream().collect(Collectors.groupingBy(
+                            ClusterState::stateUUID
+                        )).keySet();
+                        assertThat("Multiple cluster states " + clusterStates1 + " for term " + term + " and version " + version,
+                            clusterStateUUIDsForTermAndVersion, hasSize(1));
+
+                        Set<Long> clusterStateValuesForTermAndVersion = clusterStates1.stream().collect(Collectors.groupingBy(
+                            CoordinationStateTestCluster::value
+                        )).keySet();
+
+                        assertThat("Multiple cluster states " + clusterStates1 + " for term " + term + " and version " + version,
+                            clusterStateValuesForTermAndVersion, hasSize(1));
+                    });
+            });
+    }
+}
diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/LinearizabilityChecker.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/LinearizabilityChecker.java
similarity index 100%
rename from server/src/test/java/org/elasticsearch/cluster/coordination/LinearizabilityChecker.java
rename to test/framework/src/main/java/org/elasticsearch/cluster/coordination/LinearizabilityChecker.java
diff --git a/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java b/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java
index 3a6161a9f7fa0..84689cf223d20 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java
@@ -19,9 +19,12 @@
 
 package org.elasticsearch.common.settings;
 
+import org.elasticsearch.common.hash.MessageDigests;
+
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -35,6 +38,7 @@ public class MockSecureSettings implements SecureSettings {
 
     private Map<String, SecureString> secureStrings = new HashMap<>();
     private Map<String, byte[]> files = new HashMap<>();
+    private Map<String, byte[]> sha256Digests = new HashMap<>();
     private Set<String> settingNames = new HashSet<>();
     private final AtomicBoolean closed = new AtomicBoolean(false);
 
@@ -44,6 +48,7 @@ public MockSecureSettings() {
     private MockSecureSettings(MockSecureSettings source) {
         secureStrings.putAll(source.secureStrings);
         files.putAll(source.files);
+        sha256Digests.putAll(source.sha256Digests);
         settingNames.addAll(source.settingNames);
     }
 
@@ -69,15 +74,22 @@ public InputStream getFile(String setting) {
         return new ByteArrayInputStream(files.get(setting));
     }
 
+    @Override
+    public byte[] getSHA256Digest(String setting) {
+        return sha256Digests.get(setting);
+    }
+
     public void setString(String setting, String value) {
         ensureOpen();
         secureStrings.put(setting, new SecureString(value.toCharArray()));
+        sha256Digests.put(setting, MessageDigests.sha256().digest(value.getBytes(StandardCharsets.UTF_8)));
         settingNames.add(setting);
     }
 
     public void setFile(String setting, byte[] value) {
         ensureOpen();
         files.put(setting, value);
+        sha256Digests.put(setting, MessageDigests.sha256().digest(value));
         settingNames.add(setting);
     }
 
@@ -90,6 +102,7 @@ public void merge(MockSecureSettings secureSettings) {
         }
         settingNames.addAll(secureSettings.settingNames);
         secureStrings.putAll(secureSettings.secureStrings);
+        sha256Digests.putAll(secureSettings.sha256Digests);
         files.putAll(secureSettings.files);
     }
 
diff --git a/server/src/test/java/org/elasticsearch/gateway/MockGatewayMetaState.java b/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java
similarity index 95%
rename from server/src/test/java/org/elasticsearch/gateway/MockGatewayMetaState.java
rename to test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java
index 7749a0edc37b8..317a9d1b7ba0f 100644
--- a/server/src/test/java/org/elasticsearch/gateway/MockGatewayMetaState.java
+++ b/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java
@@ -37,8 +37,6 @@
  * It's not always easy / convenient to construct these dependencies.
  * This class constructor takes far fewer dependencies and constructs usable {@link GatewayMetaState} with 2 restrictions:
  * no metadata upgrade will be performed and no cluster state updaters will be run. This is sufficient for most of the tests.
- * Metadata upgrade is tested in {@link  GatewayMetaStateTests} and different {@link ClusterStateUpdaters} in
- * {@link ClusterStateUpdatersTests}.
  */
 public class MockGatewayMetaState extends GatewayMetaState {
     private final DiscoveryNode localNode;
diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
index e25217eaccc9b..e68ae70921263 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
@@ -37,6 +37,7 @@
 import org.apache.lucene.index.MergePolicy;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.ReferenceManager;
@@ -410,7 +411,7 @@ protected Translog createTranslog(Path translogPath, LongSupplier primaryTermSup
         String translogUUID = Translog.createEmptyTranslog(translogPath, SequenceNumbers.NO_OPS_PERFORMED, shardId,
             primaryTermSupplier.getAsLong());
         return new Translog(translogConfig, translogUUID, createTranslogDeletionPolicy(INDEX_SETTINGS),
-            () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTermSupplier);
+            () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTermSupplier, seqNo -> {});
     }
 
     protected TranslogHandler createTranslogHandler(IndexSettings indexSettings) {
@@ -834,7 +835,7 @@ public List<Engine.Operation> generateHistoryOnReplica(int numOps, boolean allow
             final Engine.Operation.TYPE opType = randomFrom(Engine.Operation.TYPE.values());
             final boolean isNestedDoc = includeNestedDocs && opType == Engine.Operation.TYPE.INDEX && randomBoolean();
             final int nestedValues = between(0, 3);
-            final long startTime = threadPool.relativeTimeInMillis();
+            final long startTime = threadPool.relativeTimeInNanos();
             final int copies = allowDuplicate && rarely() ? between(2, 4) : 1;
             for (int copy = 0; copy < copies; copy++) {
                 final ParsedDocument doc = isNestedDoc ? nestedParsedDocFactory.apply(id, nestedValues) : createParsedDoc(id, null);
@@ -1111,10 +1112,8 @@ public static void assertMaxSeqNoInCommitUserData(Engine engine) throws Exceptio
         List<IndexCommit> commits = DirectoryReader.listCommits(engine.store.directory());
         for (IndexCommit commit : commits) {
             try (DirectoryReader reader = DirectoryReader.open(commit)) {
-                AtomicLong maxSeqNoFromDocs = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
-                Lucene.scanSeqNosInReader(reader, 0, Long.MAX_VALUE, n -> maxSeqNoFromDocs.set(Math.max(n, maxSeqNoFromDocs.get())));
                 assertThat(Long.parseLong(commit.getUserData().get(SequenceNumbers.MAX_SEQ_NO)),
-                    greaterThanOrEqualTo(maxSeqNoFromDocs.get()));
+                    greaterThanOrEqualTo(maxSeqNosInReader(reader)));
             }
         }
     }
@@ -1148,7 +1147,7 @@ public static Translog getTranslog(Engine engine) {
      * @throws InterruptedException if the thread was interrupted while blocking on the condition
      */
     public static void waitForOpsToComplete(InternalEngine engine, long seqNo) throws InterruptedException {
-        engine.getLocalCheckpointTracker().waitForOpsToComplete(seqNo);
+        engine.getLocalCheckpointTracker().waitForProcessedOpsToComplete(seqNo);
     }
 
     public static boolean hasSnapshottedCommits(Engine engine) {
@@ -1177,4 +1176,15 @@ public long getAsLong() {
             return get();
         }
     }
+
+    static long maxSeqNosInReader(DirectoryReader reader) throws IOException {
+        long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
+        for (LeafReaderContext leaf : reader.leaves()) {
+            final NumericDocValues seqNoDocValues = leaf.reader().getNumericDocValues(SeqNoFieldMapper.NAME);
+            while (seqNoDocValues.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+                maxSeqNo = SequenceNumbers.max(maxSeqNo, seqNoDocValues.longValue());
+            }
+        }
+        return maxSeqNo;
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java
index ced2a7bff78d7..8e8b4687844b2 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java
@@ -63,7 +63,7 @@ long appliedOperations() {
     public TranslogHandler(NamedXContentRegistry xContentRegistry, IndexSettings indexSettings) {
         Map<String, NamedAnalyzer> analyzers = new HashMap<>();
         analyzers.put(AnalysisRegistry.DEFAULT_ANALYZER_NAME, new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer()));
-        IndexAnalyzers indexAnalyzers = new IndexAnalyzers(indexSettings, analyzers, emptyMap(), emptyMap());
+        IndexAnalyzers indexAnalyzers = new IndexAnalyzers(analyzers, emptyMap(), emptyMap());
         SimilarityService similarityService = new SimilarityService(indexSettings, null, emptyMap());
         MapperRegistry mapperRegistry = new IndicesModule(emptyList()).getMapperRegistry();
         mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry,
@@ -113,6 +113,7 @@ public int run(Engine engine, Translog.Snapshot snapshot) throws IOException {
             opsRecovered++;
             appliedOperations.incrementAndGet();
         }
+        engine.syncTranslog();
         return opsRecovered;
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
index b11a0f84fb84a..b91197bfe0851 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
@@ -49,6 +49,7 @@
 import org.elasticsearch.action.support.replication.TransportWriteActionTestHelper;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.routing.AllocationId;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RecoverySource;
@@ -158,7 +159,7 @@ IndexRequest copyIndexRequest(IndexRequest inRequest) throws IOException {
 
     protected DiscoveryNode getDiscoveryNode(String id) {
         return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(),
-            Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT);
+            Collections.singleton(DiscoveryNodeRole.DATA_ROLE), Version.CURRENT);
     }
 
     protected class ReplicationGroup implements AutoCloseable, Iterable<IndexShard> {
@@ -551,7 +552,7 @@ public synchronized void removeRetentionLease(String id, ActionListener<Replicat
 
         public void executeRetentionLeasesSyncRequestOnReplica(RetentionLeaseSyncAction.Request request, IndexShard replica) {
             final PlainActionFuture<Releasable> acquirePermitFuture = new PlainActionFuture<>();
-            replica.acquireReplicaOperationPermit(getPrimary().getOperationPrimaryTerm(), getPrimary().getGlobalCheckpoint(),
+            replica.acquireReplicaOperationPermit(getPrimary().getOperationPrimaryTerm(), getPrimary().getLastKnownGlobalCheckpoint(),
                 getPrimary().getMaxSeqNoOfUpdatesOrDeletes(), acquirePermitFuture, ThreadPool.Names.SAME, request);
             try (Releasable ignored = acquirePermitFuture.actionGet()) {
                 replica.updateRetentionLeasesOnReplica(request.getRetentionLeases());
@@ -657,7 +658,12 @@ public long localCheckpoint() {
 
             @Override
             public long globalCheckpoint() {
-                return getPrimaryShard().getGlobalCheckpoint();
+                return getPrimaryShard().getLastSyncedGlobalCheckpoint();
+            }
+
+            @Override
+            public long computedGlobalCheckpoint() {
+                return getPrimaryShard().getLastKnownGlobalCheckpoint();
             }
 
             @Override
@@ -691,7 +697,8 @@ public void performOn(
                         try {
                             performOnReplica(request, replica);
                             releasable.close();
-                            delegatedListener.onResponse(new ReplicaResponse(replica.getLocalCheckpoint(), replica.getGlobalCheckpoint()));
+                            delegatedListener.onResponse(new ReplicaResponse(replica.getLocalCheckpoint(),
+                                replica.getLastKnownGlobalCheckpoint()));
                         } catch (final Exception e) {
                             Releasables.closeWhileHandlingException(releasable);
                             delegatedListener.onFailure(e);
@@ -754,7 +761,7 @@ protected void performOnPrimary(IndexShard primary, BulkShardRequest request, Ac
         @Override
         protected void performOnReplica(BulkShardRequest request, IndexShard replica) throws Exception {
             executeShardBulkOnReplica(request, replica, getPrimaryShard().getPendingPrimaryTerm(),
-                getPrimaryShard().getGlobalCheckpoint(), getPrimaryShard().getMaxSeqNoOfUpdatesOrDeletes());
+                getPrimaryShard().getLastKnownGlobalCheckpoint(), getPrimaryShard().getMaxSeqNoOfUpdatesOrDeletes());
         }
     }
 
@@ -825,7 +832,7 @@ void indexOnReplica(BulkShardRequest request, ReplicationGroup group, IndexShard
 
     void indexOnReplica(BulkShardRequest request, ReplicationGroup group, IndexShard replica, long term) throws Exception {
         executeShardBulkOnReplica(request, replica, term,
-            group.primary.getGlobalCheckpoint(), group.primary.getMaxSeqNoOfUpdatesOrDeletes());
+            group.primary.getLastKnownGlobalCheckpoint(), group.primary.getMaxSeqNoOfUpdatesOrDeletes());
     }
 
     /**
@@ -833,7 +840,7 @@ void indexOnReplica(BulkShardRequest request, ReplicationGroup group, IndexShard
      */
     void deleteOnReplica(BulkShardRequest request, ReplicationGroup group, IndexShard replica) throws Exception {
         executeShardBulkOnReplica(request, replica, group.primary.getPendingPrimaryTerm(),
-            group.primary.getGlobalCheckpoint(), group.primary.getMaxSeqNoOfUpdatesOrDeletes());
+            group.primary.getLastKnownGlobalCheckpoint(), group.primary.getMaxSeqNoOfUpdatesOrDeletes());
     }
 
     class GlobalCheckpointSync extends ReplicationAction<
@@ -882,7 +889,7 @@ protected void performOnPrimary(IndexShard primary, ResyncReplicationRequest req
         @Override
         protected void performOnReplica(ResyncReplicationRequest request, IndexShard replica) throws Exception {
             executeResyncOnReplica(replica, request, getPrimaryShard().getPendingPrimaryTerm(),
-                getPrimaryShard().getGlobalCheckpoint(), getPrimaryShard().getMaxSeqNoOfUpdatesOrDeletes());
+                getPrimaryShard().getLastKnownGlobalCheckpoint(), getPrimaryShard().getMaxSeqNoOfUpdatesOrDeletes());
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
index 2a2176f1c100d..32607fa607ef6 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.index.shard;
 
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexNotFoundException;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.Version;
@@ -27,6 +28,7 @@
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -86,7 +88,6 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -278,8 +279,9 @@ protected IndexShard newShard(ShardId shardId, boolean primary, IndexingOperatio
      *                (ready to recover from another shard)
      */
     protected IndexShard newShard(ShardId shardId, boolean primary, String nodeId, IndexMetaData indexMetaData,
-                                  @Nullable IndexSearcherWrapper searcherWrapper) throws IOException {
-        return newShard(shardId, primary, nodeId, indexMetaData, searcherWrapper, () -> {});
+             @Nullable CheckedFunction<DirectoryReader, DirectoryReader, IOException> readerWrapper)
+        throws IOException {
+        return newShard(shardId, primary, nodeId, indexMetaData, readerWrapper, () -> {});
     }
 
     /**
@@ -291,11 +293,12 @@ protected IndexShard newShard(ShardId shardId, boolean primary, String nodeId, I
      *                (ready to recover from another shard)
      */
     protected IndexShard newShard(ShardId shardId, boolean primary, String nodeId, IndexMetaData indexMetaData,
-                                  @Nullable IndexSearcherWrapper searcherWrapper, Runnable globalCheckpointSyncer) throws IOException {
+                                  @Nullable CheckedFunction<DirectoryReader, DirectoryReader, IOException> readerWrapper,
+                                  Runnable globalCheckpointSyncer) throws IOException {
         ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, nodeId, primary, ShardRoutingState.INITIALIZING,
             primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE);
         return newShard(
-            shardRouting, indexMetaData, searcherWrapper, new InternalEngineFactory(), globalCheckpointSyncer, RetentionLeaseSyncer.EMPTY);
+            shardRouting, indexMetaData, readerWrapper, new InternalEngineFactory(), globalCheckpointSyncer, RetentionLeaseSyncer.EMPTY);
     }
 
     /**
@@ -306,10 +309,11 @@ protected IndexShard newShard(ShardId shardId, boolean primary, String nodeId, I
      * @param indexMetaData indexMetaData for the shard, including any mapping
      * @param listeners     an optional set of listeners to add to the shard
      */
-    protected IndexShard newShard(
-            ShardRouting routing, IndexMetaData indexMetaData, EngineFactory engineFactory, IndexingOperationListener... listeners)
+    protected IndexShard newShard(ShardRouting routing, IndexMetaData indexMetaData,
+            @Nullable CheckedFunction<DirectoryReader, DirectoryReader, IOException> indexReaderWrapper,
+            EngineFactory engineFactory, IndexingOperationListener... listeners)
         throws IOException {
-        return newShard(routing, indexMetaData, null, engineFactory, () -> {}, RetentionLeaseSyncer.EMPTY, listeners);
+        return newShard(routing, indexMetaData, indexReaderWrapper, engineFactory, () -> {}, RetentionLeaseSyncer.EMPTY, listeners);
     }
 
     /**
@@ -317,22 +321,20 @@ protected IndexShard newShard(
      * current node id the shard is assigned to.
      * @param routing                shard routing to use
      * @param indexMetaData          indexMetaData for the shard, including any mapping
-     * @param indexSearcherWrapper   an optional wrapper to be used during searchers
+     * @param indexReaderWrapper     an optional wrapper to be used during search
      * @param globalCheckpointSyncer callback for syncing global checkpoints
      * @param listeners              an optional set of listeners to add to the shard
      */
     protected IndexShard newShard(ShardRouting routing, IndexMetaData indexMetaData,
-                                  @Nullable IndexSearcherWrapper indexSearcherWrapper,
-                                  @Nullable EngineFactory engineFactory,
-                                  Runnable globalCheckpointSyncer,
-                                  RetentionLeaseSyncer retentionLeaseSyncer,
-                                  IndexingOperationListener... listeners)
+             @Nullable CheckedFunction<DirectoryReader, DirectoryReader, IOException> indexReaderWrapper,
+             @Nullable EngineFactory engineFactory, Runnable globalCheckpointSyncer, RetentionLeaseSyncer retentionLeaseSyncer,
+             IndexingOperationListener... listeners)
         throws IOException {
         // add node id as name to settings for proper logging
         final ShardId shardId = routing.shardId();
         final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(createTempDir());
         ShardPath shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId);
-        return newShard(routing, shardPath, indexMetaData, null, indexSearcherWrapper, engineFactory, globalCheckpointSyncer,
+        return newShard(routing, shardPath, indexMetaData, null, indexReaderWrapper, engineFactory, globalCheckpointSyncer,
             retentionLeaseSyncer, EMPTY_EVENT_LISTENER, listeners);
     }
 
@@ -342,14 +344,14 @@ protected IndexShard newShard(ShardRouting routing, IndexMetaData indexMetaData,
      * @param shardPath                     path to use for shard data
      * @param indexMetaData                 indexMetaData for the shard, including any mapping
      * @param storeProvider                 an optional custom store provider to use. If null a default file based store will be created
-     * @param indexSearcherWrapper          an optional wrapper to be used during searchers
+     * @param indexReaderWrapper            an optional wrapper to be used during search
      * @param globalCheckpointSyncer        callback for syncing global checkpoints
      * @param indexEventListener            index event listener
      * @param listeners                     an optional set of listeners to add to the shard
      */
     protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMetaData indexMetaData,
                                   @Nullable CheckedFunction<IndexSettings, Store, IOException> storeProvider,
-                                  @Nullable IndexSearcherWrapper indexSearcherWrapper,
+                                  @Nullable CheckedFunction<DirectoryReader, DirectoryReader, IOException> indexReaderWrapper,
                                   @Nullable EngineFactory engineFactory,
                                   Runnable globalCheckpointSyncer, RetentionLeaseSyncer retentionLeaseSyncer,
                                   IndexEventListener indexEventListener, IndexingOperationListener... listeners) throws IOException {
@@ -382,7 +384,7 @@ protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMe
                     similarityService,
                     engineFactory,
                     indexEventListener,
-                    indexSearcherWrapper,
+                    indexReaderWrapper,
                     threadPool,
                     BigArrays.NON_RECYCLING_INSTANCE,
                     warmer,
@@ -562,7 +564,7 @@ protected void recoveryEmptyReplica(IndexShard replica, boolean startReplica) th
     }
 
     protected DiscoveryNode getFakeDiscoNode(String id) {
-        return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), EnumSet.allOf(DiscoveryNode.Role.class),
+        return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), DiscoveryNodeRole.BUILT_IN_ROLES,
             Version.CURRENT);
     }
 
@@ -753,12 +755,14 @@ protected Engine.IndexResult indexDoc(IndexShard shard, String type, String id,
                 result = shard.applyIndexOperationOnPrimary(Versions.MATCH_ANY, VersionType.INTERNAL, sourceToParse,
                     SequenceNumbers.UNASSIGNED_SEQ_NO, 0, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false);
             }
+            shard.sync(); // advance local checkpoint
             shard.updateLocalCheckpointForShard(shard.routingEntry().allocationId().getId(),
                 shard.getLocalCheckpoint());
         } else {
             final long seqNo = shard.seqNoStats().getMaxSeqNo() + 1;
             shard.advanceMaxSeqNoOfUpdatesOrDeletes(seqNo); // manually replicate max_seq_no_of_updates
             result = shard.applyIndexOperationOnReplica(seqNo, 0, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false, sourceToParse);
+            shard.sync(); // advance local checkpoint
             if (result.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) {
                 throw new TransportReplicationAction.RetryOnReplicaException(shard.shardId,
                     "Mappings are not available on the replica yet, triggered update: " + result.getRequiredMappingUpdate());
@@ -777,11 +781,14 @@ protected Engine.DeleteResult deleteDoc(IndexShard shard, String type, String id
         if (shard.routingEntry().primary()) {
             result = shard.applyDeleteOperationOnPrimary(
                 Versions.MATCH_ANY, type, id, VersionType.INTERNAL, SequenceNumbers.UNASSIGNED_SEQ_NO, 0);
-            shard.updateLocalCheckpointForShard(shard.routingEntry().allocationId().getId(), shard.getEngine().getLocalCheckpoint());
+            shard.sync(); // advance local checkpoint
+            shard.updateLocalCheckpointForShard(shard.routingEntry().allocationId().getId(),
+                shard.getLocalCheckpoint());
         } else {
             final long seqNo = shard.seqNoStats().getMaxSeqNo() + 1;
             shard.advanceMaxSeqNoOfUpdatesOrDeletes(seqNo); // manually replicate max_seq_no_of_updates
             result = shard.applyDeleteOperationOnReplica(seqNo, 0L, type, id);
+            shard.sync(); // advance local checkpoint
         }
         return result;
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java
index 2279b48c3c023..81df8dd27e892 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java
@@ -100,7 +100,7 @@ public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, Met
     @Override
     public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<IndexId> indices, long startTime, String failure,
                                          int totalShards, List<SnapshotShardFailure> shardFailures, long repositoryStateId,
-                                         boolean includeGlobalState) {
+                                         boolean includeGlobalState, Map<String, Object> userMetadata) {
         return null;
     }
 
diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/FakeThreadPoolMasterService.java b/test/framework/src/main/java/org/elasticsearch/indices/cluster/FakeThreadPoolMasterService.java
similarity index 100%
rename from server/src/test/java/org/elasticsearch/indices/cluster/FakeThreadPoolMasterService.java
rename to test/framework/src/main/java/org/elasticsearch/indices/cluster/FakeThreadPoolMasterService.java
diff --git a/test/framework/src/main/java/org/elasticsearch/indices/recovery/AsyncRecoveryTarget.java b/test/framework/src/main/java/org/elasticsearch/indices/recovery/AsyncRecoveryTarget.java
index d5a7ab8109e12..afd2aa4e85888 100644
--- a/test/framework/src/main/java/org/elasticsearch/indices/recovery/AsyncRecoveryTarget.java
+++ b/test/framework/src/main/java/org/elasticsearch/indices/recovery/AsyncRecoveryTarget.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.index.store.StoreFileMetaData;
 import org.elasticsearch.index.translog.Translog;
 
-import java.io.IOException;
 import java.util.List;
 import java.util.concurrent.Executor;
 
@@ -70,13 +69,15 @@ public void indexTranslogOperations(List<Translog.Operation> operations, int tot
 
     @Override
     public void receiveFileInfo(List<String> phase1FileNames, List<Long> phase1FileSizes, List<String> phase1ExistingFileNames,
-                                List<Long> phase1ExistingFileSizes, int totalTranslogOps) {
-        target.receiveFileInfo(phase1FileNames, phase1FileSizes, phase1ExistingFileNames, phase1ExistingFileSizes, totalTranslogOps);
+                                List<Long> phase1ExistingFileSizes, int totalTranslogOps, ActionListener<Void> listener) {
+        executor.execute(() -> target.receiveFileInfo(
+            phase1FileNames, phase1FileSizes, phase1ExistingFileNames, phase1ExistingFileSizes, totalTranslogOps, listener));
     }
 
     @Override
-    public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetaData) throws IOException {
-        target.cleanFiles(totalTranslogOps, globalCheckpoint, sourceMetaData);
+    public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetaData,
+                           ActionListener<Void> listener) {
+        executor.execute(() -> target.cleanFiles(totalTranslogOps, globalCheckpoint, sourceMetaData, listener));
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java
new file mode 100644
index 0000000000000..beab182176694
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/AbstractThirdPartyRepositoryTestCase.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.repositories;
+
+import org.elasticsearch.action.ActionRunnable;
+import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.blobstore.BlobMetaData;
+import org.elasticsearch.common.blobstore.BlobPath;
+import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
+import org.elasticsearch.common.settings.SecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
+import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil;
+import org.elasticsearch.snapshots.SnapshotState;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.ByteArrayInputStream;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executor;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.not;
+
+public abstract class AbstractThirdPartyRepositoryTestCase extends ESSingleNodeTestCase {
+
+    @Override
+    protected Settings nodeSettings() {
+        return Settings.builder()
+            .put(super.nodeSettings())
+            .setSecureSettings(credentials())
+            .build();
+    }
+
+    protected abstract SecureSettings credentials();
+
+    protected abstract void createRepository(String repoName);
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        createRepository("test-repo");
+        deleteAndAssertEmpty(getRepository().basePath());
+    }
+
+    private void deleteAndAssertEmpty(BlobPath path) throws Exception {
+        final BlobStoreRepository repo = getRepository();
+        final PlainActionFuture<Void> future = PlainActionFuture.newFuture();
+        repo.threadPool().generic().execute(new ActionRunnable<>(future) {
+            @Override
+            protected void doRun() throws Exception {
+                repo.blobStore().blobContainer(path).delete();
+                future.onResponse(null);
+            }
+        });
+        future.actionGet();
+        final BlobPath parent = path.parent();
+        if (parent == null) {
+            assertChildren(path, Collections.emptyList());
+        } else {
+            assertDeleted(parent, path.toArray()[path.toArray().length - 1]);
+        }
+    }
+
+    public void testCreateSnapshot() {
+        createIndex("test-idx-1");
+        createIndex("test-idx-2");
+        createIndex("test-idx-3");
+        ensureGreen();
+
+        logger.info("--> indexing some data");
+        for (int i = 0; i < 100; i++) {
+            client().prepareIndex("test-idx-1", "doc", Integer.toString(i)).setSource("foo", "bar" + i).get();
+            client().prepareIndex("test-idx-2", "doc", Integer.toString(i)).setSource("foo", "bar" + i).get();
+            client().prepareIndex("test-idx-3", "doc", Integer.toString(i)).setSource("foo", "bar" + i).get();
+        }
+        client().admin().indices().prepareRefresh().get();
+
+        final String snapshotName = "test-snap-" + System.currentTimeMillis();
+
+        logger.info("--> snapshot");
+        CreateSnapshotResponse createSnapshotResponse = client().admin()
+            .cluster()
+            .prepareCreateSnapshot("test-repo", snapshotName)
+            .setWaitForCompletion(true)
+            .setIndices("test-idx-*", "-test-idx-3")
+            .get();
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
+            equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
+
+        assertThat(client().admin()
+                .cluster()
+                .prepareGetSnapshots("test-repo")
+                .setSnapshots(snapshotName)
+                .get()
+                .getSnapshots("test-repo")
+                .get(0)
+                .state(),
+            equalTo(SnapshotState.SUCCESS));
+
+        assertTrue(client().admin()
+                .cluster()
+                .prepareDeleteSnapshot("test-repo", snapshotName)
+                .get()
+                .isAcknowledged());
+    }
+
+    public void testListChildren() throws Exception {
+        final BlobStoreRepository repo = getRepository();
+        final PlainActionFuture<Void> future = PlainActionFuture.newFuture();
+        final Executor genericExec = repo.threadPool().generic();
+        final int testBlobLen = randomIntBetween(1, 100);
+        genericExec.execute(new ActionRunnable<>(future) {
+            @Override
+            protected void doRun() throws Exception {
+                final BlobStore blobStore = repo.blobStore();
+                blobStore.blobContainer(repo.basePath().add("foo"))
+                    .writeBlob("nested-blob", new ByteArrayInputStream(randomByteArrayOfLength(testBlobLen)), testBlobLen, false);
+                blobStore.blobContainer(repo.basePath().add("foo").add("nested"))
+                    .writeBlob("bar", new ByteArrayInputStream(randomByteArrayOfLength(testBlobLen)), testBlobLen, false);
+                blobStore.blobContainer(repo.basePath().add("foo").add("nested2"))
+                    .writeBlob("blub", new ByteArrayInputStream(randomByteArrayOfLength(testBlobLen)), testBlobLen, false);
+                future.onResponse(null);
+            }
+        });
+        future.actionGet();
+        assertChildren(repo.basePath(), Collections.singleton("foo"));
+        assertBlobsByPrefix(repo.basePath(), "fo", Collections.emptyMap());
+        assertChildren(repo.basePath().add("foo"), List.of("nested", "nested2"));
+        assertBlobsByPrefix(repo.basePath().add("foo"), "nest",
+            Collections.singletonMap("nested-blob", new PlainBlobMetaData("nested-blob", testBlobLen)));
+        assertChildren(repo.basePath().add("foo").add("nested"), Collections.emptyList());
+        if (randomBoolean()) {
+            deleteAndAssertEmpty(repo.basePath());
+        } else {
+            deleteAndAssertEmpty(repo.basePath().add("foo"));
+        }
+    }
+
+    protected void assertBlobsByPrefix(BlobPath path, String prefix, Map<String, BlobMetaData> blobs) throws Exception {
+        final PlainActionFuture<Map<String, BlobMetaData>> future = PlainActionFuture.newFuture();
+        final BlobStoreRepository repository = getRepository();
+        repository.threadPool().generic().execute(new ActionRunnable<>(future) {
+            @Override
+            protected void doRun() throws Exception {
+                final BlobStore blobStore = repository.blobStore();
+                future.onResponse(blobStore.blobContainer(path).listBlobsByPrefix(prefix));
+            }
+        });
+        Map<String, BlobMetaData> foundBlobs = future.actionGet();
+        if (blobs.isEmpty()) {
+            assertThat(foundBlobs.keySet(), empty());
+        } else {
+            assertThat(foundBlobs.keySet(), containsInAnyOrder(blobs.keySet().toArray(Strings.EMPTY_ARRAY)));
+            for (Map.Entry<String, BlobMetaData> entry : foundBlobs.entrySet()) {
+                assertEquals(entry.getValue().length(), blobs.get(entry.getKey()).length());
+            }
+        }
+    }
+
+    public void testCleanup() throws Exception {
+        createRepository("test-repo");
+
+        createIndex("test-idx-1");
+        createIndex("test-idx-2");
+        createIndex("test-idx-3");
+        ensureGreen();
+
+        logger.info("--> indexing some data");
+        for (int i = 0; i < 100; i++) {
+            client().prepareIndex("test-idx-1", "doc", Integer.toString(i)).setSource("foo", "bar" + i).get();
+            client().prepareIndex("test-idx-2", "doc", Integer.toString(i)).setSource("foo", "bar" + i).get();
+            client().prepareIndex("test-idx-3", "doc", Integer.toString(i)).setSource("foo", "bar" + i).get();
+        }
+        client().admin().indices().prepareRefresh().get();
+
+        final String snapshotName = "test-snap-" + System.currentTimeMillis();
+
+        logger.info("--> snapshot");
+        CreateSnapshotResponse createSnapshotResponse = client().admin()
+            .cluster()
+            .prepareCreateSnapshot("test-repo", snapshotName)
+            .setWaitForCompletion(true)
+            .setIndices("test-idx-*", "-test-idx-3")
+            .get();
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
+            equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
+
+        assertThat(client().admin()
+                .cluster()
+                .prepareGetSnapshots("test-repo")
+                .setSnapshots(snapshotName)
+                .get()
+                .getSnapshots("test-repo")
+                .get(0)
+                .state(),
+            equalTo(SnapshotState.SUCCESS));
+
+        logger.info("--> creating a dangling index folder");
+        final BlobStoreRepository repo =
+            (BlobStoreRepository) getInstanceFromNode(RepositoriesService.class).repository("test-repo");
+        final PlainActionFuture<Void> future = PlainActionFuture.newFuture();
+        final Executor genericExec = repo.threadPool().executor(ThreadPool.Names.GENERIC);
+        genericExec.execute(new ActionRunnable<>(future) {
+            @Override
+            protected void doRun() throws Exception {
+                final BlobStore blobStore = repo.blobStore();
+                blobStore.blobContainer(BlobPath.cleanPath().add("indices").add("foo"))
+                    .writeBlob("bar", new ByteArrayInputStream(new byte[0]), 0, false);
+                future.onResponse(null);
+            }
+        });
+        future.actionGet();
+        assertTrue(assertCorruptionVisible(repo, genericExec));
+        logger.info("--> deleting a snapshot to trigger repository cleanup");
+        client().admin().cluster().deleteSnapshot(new DeleteSnapshotRequest("test-repo", snapshotName)).actionGet();
+
+        assertConsistentRepository(repo, genericExec);
+    }
+
+    protected boolean assertCorruptionVisible(BlobStoreRepository repo, Executor executor) throws Exception {
+        final PlainActionFuture<Boolean> future = PlainActionFuture.newFuture();
+        executor.execute(new ActionRunnable<>(future) {
+            @Override
+            protected void doRun() throws Exception {
+                final BlobStore blobStore = repo.blobStore();
+                future.onResponse(
+                    blobStore.blobContainer(BlobPath.cleanPath().add("indices")).children().containsKey("foo")
+                        && blobStore.blobContainer(BlobPath.cleanPath().add("indices").add("foo")).blobExists("bar")
+                );
+            }
+        });
+        return future.actionGet();
+    }
+
+    protected void assertConsistentRepository(BlobStoreRepository repo, Executor executor) throws Exception {
+        BlobStoreTestUtil.assertConsistency(repo, executor);
+    }
+
+    protected void assertDeleted(BlobPath path, String name) throws Exception {
+        assertThat(listChildren(path), not(contains(name)));
+    }
+
+    protected void assertChildren(BlobPath path, Collection<String> children) throws Exception {
+        listChildren(path);
+        final Set<String> foundChildren = listChildren(path);
+        if (children.isEmpty()) {
+            assertThat(foundChildren, empty());
+        } else {
+            assertThat(foundChildren, containsInAnyOrder(children.toArray(Strings.EMPTY_ARRAY)));
+        }
+    }
+
+    private Set<String> listChildren(BlobPath path) {
+        final PlainActionFuture<Set<String>> future = PlainActionFuture.newFuture();
+        final BlobStoreRepository repository = getRepository();
+        repository.threadPool().generic().execute(new ActionRunnable<>(future) {
+            @Override
+            protected void doRun() throws Exception {
+                final BlobStore blobStore = repository.blobStore();
+                future.onResponse(blobStore.blobContainer(path).children().keySet());
+            }
+        });
+        return future.actionGet();
+    }
+
+    private BlobStoreRepository getRepository() {
+        return (BlobStoreRepository) getInstanceFromNode(RepositoriesService.class).repository("test-repo");
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java
new file mode 100644
index 0000000000000..2b53b6fae9ac1
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.repositories.blobstore;
+
+import org.elasticsearch.action.ActionRunnable;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.blobstore.BlobContainer;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.repositories.IndexId;
+import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.repositories.RepositoryData;
+import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Executor;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+public final class BlobStoreTestUtil {
+
+    public static void assertRepoConsistency(InternalTestCluster testCluster, String repoName) {
+        final BlobStoreRepository repo =
+            (BlobStoreRepository) testCluster.getCurrentMasterNodeInstance(RepositoriesService.class).repository(repoName);
+        BlobStoreTestUtil.assertConsistency(repo, repo.threadPool().executor(ThreadPool.Names.GENERIC));
+    }
+
+    /**
+     * Assert that there are no unreferenced indices or unreferenced root-level metadata blobs in any repository.
+     * TODO: Expand the logic here to also check for unreferenced segment blobs and shard level metadata
+     * @param repository BlobStoreRepository to check
+     * @param executor Executor to run all repository calls on. This is needed since the production {@link BlobStoreRepository}
+     *                 implementations assert that all IO inducing calls happen on the generic or snapshot thread-pools and hence callers
+     *                 of this assertion must pass an executor on those when using such an implementation.
+     */
+    public static void assertConsistency(BlobStoreRepository repository, Executor executor) {
+        final PlainActionFuture<Void> listener = PlainActionFuture.newFuture();
+        executor.execute(new ActionRunnable<>(listener) {
+            @Override
+            protected void doRun() throws Exception {
+                final BlobContainer blobContainer = repository.blobContainer();
+                assertTrue(
+                    "Could not find index.latest blob for repo [" + repository + "]", blobContainer.blobExists("index.latest"));
+                final long latestGen;
+                try (DataInputStream inputStream = new DataInputStream(blobContainer.readBlob("index.latest"))) {
+                    latestGen = inputStream.readLong();
+                }
+                assertIndexGenerations(blobContainer, latestGen);
+                final RepositoryData repositoryData;
+                try (InputStream blob = blobContainer.readBlob("index-" + latestGen);
+                     XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY,
+                         LoggingDeprecationHandler.INSTANCE, blob)) {
+                    repositoryData = RepositoryData.snapshotsFromXContent(parser, latestGen);
+                }
+                assertIndexUUIDs(blobContainer, repositoryData);
+                assertSnapshotUUIDs(blobContainer, repositoryData);
+                listener.onResponse(null);
+            }
+        });
+        listener.actionGet(TimeValue.timeValueMinutes(1L));
+    }
+
+    private static void assertIndexGenerations(BlobContainer repoRoot, long latestGen) throws IOException {
+        final long[] indexGenerations = repoRoot.listBlobsByPrefix("index-").keySet().stream()
+            .map(s -> s.replace("index-", ""))
+            .mapToLong(Long::parseLong).sorted().toArray();
+        assertEquals(latestGen, indexGenerations[indexGenerations.length - 1]);
+        assertTrue(indexGenerations.length <= 2);
+    }
+
+    private static void assertIndexUUIDs(BlobContainer repoRoot, RepositoryData repositoryData) throws IOException {
+        final List<String> expectedIndexUUIDs =
+            repositoryData.getIndices().values().stream().map(IndexId::getId).collect(Collectors.toList());
+        final BlobContainer indicesContainer = repoRoot.children().get("indices");
+        final List<String> foundIndexUUIDs;
+        if (indicesContainer == null) {
+            foundIndexUUIDs = Collections.emptyList();
+        } else {
+            foundIndexUUIDs = indicesContainer.children().keySet().stream().filter(
+                s -> s.startsWith("extra") == false).collect(Collectors.toList());
+        }
+        assertThat(foundIndexUUIDs, containsInAnyOrder(expectedIndexUUIDs.toArray(Strings.EMPTY_ARRAY)));
+    }
+
+    private static void assertSnapshotUUIDs(BlobContainer repoRoot, RepositoryData repositoryData) throws IOException {
+        final List<String> expectedSnapshotUUIDs =
+            repositoryData.getSnapshotIds().stream().map(SnapshotId::getUUID).collect(Collectors.toList());
+        for (String prefix : new String[]{"snap-", "meta-"}) {
+                final Collection<String> foundSnapshotUUIDs = repoRoot.listBlobs().keySet().stream().filter(p -> p.startsWith(prefix))
+                    .map(p -> p.replace(prefix, "").replace(".dat", ""))
+                    .collect(Collectors.toSet());
+                assertThat(foundSnapshotUUIDs, containsInAnyOrder(expectedSnapshotUUIDs.toArray(Strings.EMPTY_ARRAY)));
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java
index 8187a46fa7425..947a2c9887c10 100644
--- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java
@@ -148,7 +148,7 @@ public void testSnapshotAndRestore() throws Exception {
         assertAcked(client().admin().cluster().prepareDeleteSnapshot(repoName, snapshotName).get());
 
         expectThrows(SnapshotMissingException.class, () ->
-            client().admin().cluster().prepareGetSnapshots(repoName).setSnapshots(snapshotName).get());
+            client().admin().cluster().prepareGetSnapshots(repoName).setSnapshots(snapshotName).get().getSnapshots(repoName));
 
         expectThrows(SnapshotMissingException.class, () ->
             client().admin().cluster().prepareDeleteSnapshot(repoName, snapshotName).get());
diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java
index df554ea42de28..6860bee1d0af7 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java
@@ -95,9 +95,6 @@ public static SearchRequest randomSearchRequest(Supplier<SearchSourceBuilder> ra
         if (randomBoolean()) {
             searchRequest.indicesOptions(IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
         }
-        if (randomBoolean()) {
-            searchRequest.types(generateRandomStringArray(10, 10, false, false));
-        }
         if (randomBoolean()) {
             searchRequest.preference(randomAlphaOfLengthBetween(3, 10));
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
index 597c2a5ac849b..9af05f3b41d7f 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -32,7 +32,6 @@
 import org.apache.lucene.search.QueryCachingPolicy;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Weight;
-import org.apache.lucene.search.XIndexSearcher;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.lease.Releasable;
@@ -47,7 +46,7 @@
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache.Listener;
 import org.elasticsearch.index.cache.query.DisabledQueryCache;
-import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.ContentPath;
@@ -58,7 +57,6 @@
 import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.mapper.ObjectMapper.Nested;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.query.support.NestedScope;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
@@ -83,9 +81,10 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -106,7 +105,7 @@ public abstract class AggregatorTestCase extends ESTestCase {
     private List<Releasable> releasables = new ArrayList<>();
     private static final String TYPE_NAME = "type";
 
-    protected AggregatorFactory<?> createAggregatorFactory(AggregationBuilder aggregationBuilder,
+    protected AggregatorFactory createAggregatorFactory(AggregationBuilder aggregationBuilder,
                                                            IndexSearcher indexSearcher,
                                                            MappedFieldType... fieldTypes) throws IOException {
         return createAggregatorFactory(aggregationBuilder, indexSearcher, createIndexSettings(),
@@ -114,7 +113,7 @@ protected AggregatorFactory<?> createAggregatorFactory(AggregationBuilder aggreg
     }
 
 
-    protected AggregatorFactory<?> createAggregatorFactory(AggregationBuilder aggregationBuilder,
+    protected AggregatorFactory createAggregatorFactory(AggregationBuilder aggregationBuilder,
                                                            IndexSearcher indexSearcher,
                                                            IndexSettings indexSettings,
                                                            MultiBucketConsumer bucketConsumer,
@@ -123,7 +122,7 @@ protected AggregatorFactory<?> createAggregatorFactory(AggregationBuilder aggreg
     }
 
     /** Create a factory for the given aggregation builder. */
-    protected AggregatorFactory<?> createAggregatorFactory(Query query,
+    protected AggregatorFactory createAggregatorFactory(Query query,
                                                            AggregationBuilder aggregationBuilder,
                                                            IndexSearcher indexSearcher,
                                                            IndexSettings indexSettings,
@@ -153,14 +152,15 @@ protected AggregatorFactory<?> createAggregatorFactory(Query query,
         when(searchContext.getForField(Mockito.any(MappedFieldType.class)))
             .thenAnswer(invocationOnMock -> ifds.getForField((MappedFieldType) invocationOnMock.getArguments()[0]));
 
-        SearchLookup searchLookup = new SearchLookup(mapperService, ifds::getForField, new String[]{TYPE_NAME});
+        SearchLookup searchLookup = new SearchLookup(mapperService, ifds::getForField);
         when(searchContext.lookup()).thenReturn(searchLookup);
 
-        QueryShardContext queryShardContext = queryShardContextMock(mapperService);
-        when(queryShardContext.getIndexSettings()).thenReturn(indexSettings);
+        QueryShardContext queryShardContext = queryShardContextMock(mapperService, indexSettings, circuitBreakerService);
+
         when(searchContext.getQueryShardContext()).thenReturn(queryShardContext);
         Map<String, MappedFieldType> fieldNameToType = new HashMap<>();
         fieldNameToType.putAll(Arrays.stream(fieldTypes)
+            .filter(Objects::nonNull)
             .collect(Collectors.toMap(MappedFieldType::name, Function.identity())));
         fieldNameToType.putAll(getFieldAliases(fieldTypes));
 
@@ -187,16 +187,11 @@ private void registerFieldTypes(QueryShardContext queryShardContext,
             String fieldName = entry.getKey();
             MappedFieldType fieldType = entry.getValue();
 
-            when(queryShardContext.fieldMapper(fieldName)).thenReturn(fieldType);
+            when(mapperService.fullName(fieldName)).thenReturn(fieldType);
             when(searchContext.smartNameFieldType(fieldName)).thenReturn(fieldType);
         }
 
-        for (MappedFieldType fieldType : new HashSet<>(fieldNameToType.values())) {
-            when(queryShardContext.getForField(fieldType)).then(invocation ->
-                fieldType.fielddataBuilder(mapperService.getIndexSettings().getIndex().getName())
-                    .build(mapperService.getIndexSettings(), fieldType,
-                        new IndexFieldDataCache.None(), circuitBreakerService, mapperService));
-        }
+
     }
 
     protected <A extends Aggregator> A createAggregator(AggregationBuilder aggregationBuilder,
@@ -243,7 +238,6 @@ protected <A extends Aggregator> A createAggregator(Query query,
     }
 
     protected SearchContext createSearchContext(IndexSearcher indexSearcher, IndexSettings indexSettings) {
-        Engine.Searcher searcher = new Engine.Searcher("aggregator_test", indexSearcher, () -> indexSearcher.getIndexReader().close());
         QueryCache queryCache = new DisabledQueryCache(indexSettings);
         QueryCachingPolicy queryCachingPolicy = new QueryCachingPolicy() {
             @Override
@@ -256,7 +250,8 @@ public boolean shouldCache(Query query) {
                 return false;
             }
         };
-        ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(searcher, queryCache, queryCachingPolicy);
+        ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(indexSearcher.getIndexReader(),
+            indexSearcher.getSimilarity(), queryCache, queryCachingPolicy);
 
         SearchContext searchContext = mock(SearchContext.class);
         when(searchContext.numberOfShards()).thenReturn(1);
@@ -302,12 +297,31 @@ protected MapperService mapperServiceMock() {
     /**
      * sub-tests that need a more complex mock can overwrite this
      */
-    protected QueryShardContext queryShardContextMock(MapperService mapperService) {
-        QueryShardContext queryShardContext = mock(QueryShardContext.class);
-        when(queryShardContext.getMapperService()).thenReturn(mapperService);
-        NestedScope nestedScope = new NestedScope();
-        when(queryShardContext.nestedScope()).thenReturn(nestedScope);
-        return queryShardContext;
+    protected QueryShardContext queryShardContextMock(MapperService mapperService, IndexSettings indexSettings,
+                                                      CircuitBreakerService circuitBreakerService) {
+
+        return new QueryShardContext(0, indexSettings, null, null,
+            getIndexFieldDataLookup(mapperService, circuitBreakerService),
+            mapperService, null, getMockScriptService(), xContentRegistry(),
+            writableRegistry(), null, null, System::currentTimeMillis, null);
+    }
+
+    /**
+     * Sub-tests that need a more complex index field data provider can override this
+     */
+    protected BiFunction<MappedFieldType, String, IndexFieldData<?>> getIndexFieldDataLookup(MapperService mapperService,
+                                                                                             CircuitBreakerService circuitBreakerService) {
+        return (fieldType, s) -> fieldType.fielddataBuilder(mapperService.getIndexSettings().getIndex().getName())
+            .build(mapperService.getIndexSettings(), fieldType,
+                new IndexFieldDataCache.None(), circuitBreakerService, mapperService);
+
+    }
+
+    /**
+     * Sub-tests that need scripting can override this method to provide a script service and pre-baked scripts
+     */
+    protected ScriptService getMockScriptService() {
+        return null;
     }
 
     protected <A extends InternalAggregation, C extends Aggregator> A search(IndexSearcher searcher,
@@ -397,7 +411,7 @@ protected <A extends InternalAggregation, C extends Aggregator> A searchAndReduc
                     new InternalAggregation.ReduceContext(root.context().bigArrays(), null,
                         reduceBucketConsumer, false);
                 A reduced = (A) aggs.get(0).doReduce(toReduce, context);
-                InternalAggregationTestCase.assertMultiBucketConsumer(reduced, reduceBucketConsumer);
+                doAssertReducedMultiBucketConsumer(reduced, reduceBucketConsumer);
                 aggs = new ArrayList<>(aggs.subList(r, toReduceSize));
                 aggs.add(reduced);
             }
@@ -413,12 +427,16 @@ protected <A extends InternalAggregation, C extends Aggregator> A searchAndReduc
                     internalAgg = (A) pipelineAggregator.reduce(internalAgg, context);
                 }
             }
-            InternalAggregationTestCase.assertMultiBucketConsumer(internalAgg, reduceBucketConsumer);
+            doAssertReducedMultiBucketConsumer(internalAgg, reduceBucketConsumer);
             return internalAgg;
         }
 
     }
 
+    protected void doAssertReducedMultiBucketConsumer(Aggregation agg, MultiBucketConsumerService.MultiBucketConsumer bucketConsumer) {
+        InternalAggregationTestCase.assertMultiBucketConsumer(agg, bucketConsumer);
+    }
+
     private static class ShardSearcher extends IndexSearcher {
         private final List<LeafReaderContext> ctx;
 
@@ -448,16 +466,8 @@ protected static DirectoryReader wrap(DirectoryReader directoryReader) throws IO
      */
     protected static IndexSearcher newIndexSearcher(IndexReader indexReader) {
         if (randomBoolean()) {
-            final IndexSearcher delegate = new IndexSearcher(indexReader);
-            final XIndexSearcher wrappedSearcher = new XIndexSearcher(delegate);
             // this executes basic query checks and asserts that weights are normalized only once etc.
-            return new AssertingIndexSearcher(random(), indexReader) {
-                @Override
-                protected void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
-                    // we cannot use the asserting searcher because the weight is created by the ContextIndexSearcher
-                    wrappedSearcher.search(leaves, weight, collector);
-                }
-             };
+            return new AssertingIndexSearcher(random(), indexReader);
         } else {
             return new IndexSearcher(indexReader);
         }
diff --git a/server/src/test/java/org/elasticsearch/action/support/broadcast/AbstractBroadcastResponseTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBroadcastResponseTestCase.java
similarity index 98%
rename from server/src/test/java/org/elasticsearch/action/support/broadcast/AbstractBroadcastResponseTestCase.java
rename to test/framework/src/main/java/org/elasticsearch/test/AbstractBroadcastResponseTestCase.java
index 5bf48fa589764..87084577baaef 100644
--- a/server/src/test/java/org/elasticsearch/action/support/broadcast/AbstractBroadcastResponseTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBroadcastResponseTestCase.java
@@ -17,10 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.action.support.broadcast;
+package org.elasticsearch.test;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
+import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.ToXContent;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
index 61298ce0e8418..72cfd4926fedd 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java
@@ -22,6 +22,7 @@
 import com.carrotsearch.randomizedtesting.RandomizedTest;
 import com.carrotsearch.randomizedtesting.SeedUtils;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.util.Accountable;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
@@ -350,7 +351,7 @@ private static class ServiceHolder implements Closeable {
             List<Setting<?>> additionalSettings = pluginsService.getPluginSettings();
             SettingsModule settingsModule =
                     new SettingsModule(nodeSettings, additionalSettings, pluginsService.getPluginSettingsFilter(), Collections.emptySet());
-            searchModule = new SearchModule(nodeSettings, false, pluginsService.filterPlugins(SearchPlugin.class));
+            searchModule = new SearchModule(nodeSettings, pluginsService.filterPlugins(SearchPlugin.class));
             IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class));
             List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
             entries.addAll(indicesModule.getNamedWriteables());
@@ -417,8 +418,9 @@ public void close() throws IOException {
         }
 
         QueryShardContext createShardContext(IndexReader reader) {
-            return new QueryShardContext(0, idxSettings, bitsetFilterCache, indexFieldDataService::getForField, mapperService,
-                similarityService, scriptService, xContentRegistry, namedWriteableRegistry, this.client, reader, () -> nowInMillis, null);
+            return new QueryShardContext(0, idxSettings, bitsetFilterCache, IndexSearcher::new, indexFieldDataService::getForField,
+                mapperService, similarityService, scriptService, xContentRegistry, namedWriteableRegistry, this.client, reader,
+                () -> nowInMillis, null);
         }
 
         ScriptModule createScriptModule(List<ScriptPlugin> scriptPlugins) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index 54f6df6ac0cca..29c3fc5a27b6a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -165,7 +165,7 @@ public void testUnknownField() throws IOException {
      * parse exception. Some specific objects do not cause any exception as they can hold arbitrary content; they can be
      * declared by overriding {@link #getObjectsHoldingArbitraryContent()}.
      */
-    public final void testUnknownObjectException() throws IOException {
+    public void testUnknownObjectException() throws IOException {
         Set<String> candidates = new HashSet<>();
         // Adds the valid query to the list of queries to modify and test
         candidates.add(createTestQueryBuilder().toString());
@@ -428,13 +428,6 @@ public void testToQuery() throws IOException {
              * we first rewrite the query with a private context, then reset the context and then build the actual lucene query*/
             QueryBuilder rewritten = rewriteQuery(firstQuery, new QueryShardContext(context));
             Query firstLuceneQuery = rewritten.toQuery(context);
-            if (isCacheable(firstQuery)) {
-                assertTrue("query was marked as not cacheable in the context but this test indicates it should be cacheable: "
-                        + firstQuery.toString(), context.isCacheable());
-            } else {
-                assertFalse("query was marked as cacheable in the context but this test indicates it should not be cacheable: "
-                        + firstQuery.toString(), context.isCacheable());
-            }
             assertNotNull("toQuery should not return null", firstLuceneQuery);
             assertLuceneQuery(firstQuery, firstLuceneQuery, searchContext);
             //remove after assertLuceneQuery since the assertLuceneQuery impl might access the context as well
@@ -471,17 +464,13 @@ public void testToQuery() throws IOException {
         }
     }
 
-    private QueryBuilder rewriteQuery(QB queryBuilder, QueryRewriteContext rewriteContext) throws IOException {
+    protected QueryBuilder rewriteQuery(QB queryBuilder, QueryRewriteContext rewriteContext) throws IOException {
         QueryBuilder rewritten = rewriteAndFetch(queryBuilder, rewriteContext);
         // extra safety to fail fast - serialize the rewritten version to ensure it's serializable.
         assertSerialization(rewritten);
         return rewritten;
     }
 
-    protected boolean isCacheable(QB queryBuilder) {
-        return true;
-    }
-
     /**
      * Few queries allow you to set the boost on the Java API, although the corresponding parser
      * doesn't parse it as it isn't supported. This method allows to disable boost related tests for those queries.
@@ -806,4 +795,17 @@ protected QueryBuilder rewriteAndFetch(QueryBuilder builder, QueryRewriteContext
     public boolean isTextField(String fieldName) {
         return fieldName.equals(STRING_FIELD_NAME) || fieldName.equals(STRING_ALIAS_FIELD_NAME);
     }
+
+    /**
+     * Check that a query is generally cacheable. Tests for query builders that are not always cacheable
+     * should overwrite this method and make sure the different cases are always tested
+     */
+    public void testCacheability() throws IOException {
+        QB queryBuilder = createTestQueryBuilder();
+        QueryShardContext context = createShardContext();
+        QueryBuilder rewriteQuery = rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(rewriteQuery.toQuery(context));
+        assertTrue("query should be cacheable: " + queryBuilder.toString(), context.isCacheable());
+    }
+
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java b/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java
index c2e97f35faee9..d46c09e12621d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java
@@ -27,8 +27,10 @@
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.bulk.BulkShardRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -141,7 +143,7 @@ public void run() {
                                     }
 
                                 }
-                                BulkRequestBuilder bulkRequest = client.prepareBulk();
+                                BulkRequestBuilder bulkRequest = client.prepareBulk().setTimeout(timeout);
                                 for (int i = 0; i < batchSize; i++) {
                                     id = idGenerator.incrementAndGet();
                                     if (useAutoGeneratedIDs) {
@@ -151,16 +153,21 @@ public void run() {
                                                 .setSource(generateSource(id, threadRandom)));
                                     }
                                 }
-                                BulkResponse bulkResponse = bulkRequest.get();
-                                for (BulkItemResponse bulkItemResponse : bulkResponse) {
-                                    if (bulkItemResponse.isFailed() == false) {
-                                        boolean add = ids.add(bulkItemResponse.getId());
-                                        assert add : "ID: " + bulkItemResponse.getId() + " already used";
-                                    } else {
-                                        failures.add(bulkItemResponse.getFailure().getCause());
+                                try {
+                                    BulkResponse bulkResponse = bulkRequest.get();
+                                    for (BulkItemResponse bulkItemResponse : bulkResponse) {
+                                        if (bulkItemResponse.isFailed() == false) {
+                                            boolean add = ids.add(bulkItemResponse.getId());
+                                            assert add : "ID: " + bulkItemResponse.getId() + " already used";
+                                        } else {
+                                            failures.add(bulkItemResponse.getFailure().getCause());
+                                        }
+                                    }
+                                } catch (Exception e) {
+                                    if (ignoreIndexingFailures == false) {
+                                        throw e;
                                     }
                                 }
-
                             } else {
 
                                 if (hasBudget.get() && !availableBudget.tryAcquire(250, TimeUnit.MILLISECONDS)) {
@@ -169,15 +176,27 @@ public void run() {
                                 }
                                 id = idGenerator.incrementAndGet();
                                 if (useAutoGeneratedIDs) {
-                                    IndexResponse indexResponse = client.prepareIndex(index, type)
-                                            .setSource(generateSource(id, threadRandom)).get();
-                                    boolean add = ids.add(indexResponse.getId());
-                                    assert add : "ID: " + indexResponse.getId() + " already used";
+                                    try {
+                                        IndexResponse indexResponse = client.prepareIndex(index, type)
+                                            .setTimeout(timeout).setSource(generateSource(id, threadRandom)).get();
+                                        boolean add = ids.add(indexResponse.getId());
+                                        assert add : "ID: " + indexResponse.getId() + " already used";
+                                    } catch (Exception e) {
+                                        if (ignoreIndexingFailures == false) {
+                                            throw e;
+                                        }
+                                    }
                                 } else {
-                                    IndexResponse indexResponse = client.prepareIndex(index, type, Long.toString(id))
-                                            .setSource(generateSource(id, threadRandom)).get();
-                                    boolean add = ids.add(indexResponse.getId());
-                                    assert add : "ID: " + indexResponse.getId() + " already used";
+                                    try {
+                                        IndexResponse indexResponse = client.prepareIndex(index, type, Long.toString(id))
+                                            .setTimeout(timeout).setSource(generateSource(id, threadRandom)).get();
+                                        boolean add = ids.add(indexResponse.getId());
+                                        assert add : "ID: " + indexResponse.getId() + " already used";
+                                    } catch (Exception e) {
+                                        if (ignoreIndexingFailures == false) {
+                                            throw e;
+                                        }
+                                    }
                                 }
                             }
                         }
@@ -217,6 +236,18 @@ private XContentBuilder generateSource(long id, Random random) throws IOExceptio
 
     }
 
+    private volatile TimeValue timeout = BulkShardRequest.DEFAULT_TIMEOUT;
+
+    public void setRequestTimeout(TimeValue timeout) {
+        this.timeout = timeout;
+    }
+
+    private volatile boolean ignoreIndexingFailures;
+
+    public void setIgnoreIndexingFailures(boolean ignoreIndexingFailures) {
+        this.ignoreIndexingFailures = ignoreIndexingFailures;
+    }
+
     private void setBudget(int numOfDocs) {
         logger.debug("updating budget to [{}]", numOfDocs);
         if (numOfDocs >= 0) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
index 5477d292cc2c6..4a34337cdc384 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
@@ -26,7 +26,9 @@
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.coordination.ClusterStatePublisher;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterApplier;
 import org.elasticsearch.cluster.service.ClusterApplier.ClusterApplyListener;
@@ -35,11 +37,9 @@
 import org.elasticsearch.cluster.service.MasterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.cluster.coordination.ClusterStatePublisher;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Collections;
-import java.util.EnumSet;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -123,7 +123,7 @@ public void onFailure(String source, Exception e) {
 
     public static ClusterService createClusterService(ThreadPool threadPool) {
         DiscoveryNode discoveryNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(),
-                EnumSet.allOf(DiscoveryNode.Role.class), Version.CURRENT);
+                DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT);
         return createClusterService(threadPool, discoveryNode);
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 0ab0afb6b0e7a..81be0e26cffce 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -38,7 +38,6 @@
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
@@ -1318,9 +1317,17 @@ protected ForceMergeResponse forceMerge() {
     /**
      * Returns <code>true</code> iff the given index exists otherwise <code>false</code>
      */
-    protected boolean indexExists(String index) {
-        IndicesExistsResponse actionGet = client().admin().indices().prepareExists(index).execute().actionGet();
-        return actionGet.isExists();
+    protected static boolean indexExists(String index) {
+        return indexExists(index, client());
+    }
+
+    /**
+     * Returns <code>true</code> iff the given index exists otherwise <code>false</code>
+     */
+    public static boolean indexExists(String index, Client client) {
+        GetIndexResponse getIndexResponse = client.admin().indices().prepareGetIndex().setIndices(index)
+            .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED).execute().actionGet();
+        return getIndexResponse.getIndices().length > 0;
     }
 
     /**
@@ -1608,9 +1615,10 @@ public enum Scope {
         boolean supportsDedicatedMasters() default true;
 
         /**
-         * The cluster automatically manages the bootstrap voting configuration. Set this to false to manage the setting manually.
+         * Indicates whether the cluster automatically manages cluster bootstrapping and the removal of any master-eligible nodes. If
+         * set to {@code false} then the tests must manage these processes explicitly.
          */
-        boolean autoMinMasterNodes() default true;
+        boolean autoManageMasterNodes() default true;
 
         /**
          * Returns the number of client nodes in the cluster. Default is {@link InternalTestCluster#DEFAULT_NUM_CLIENT_NODES}, a
@@ -1698,9 +1706,9 @@ private boolean getSupportsDedicatedMasters() {
         return annotation == null ? true : annotation.supportsDedicatedMasters();
     }
 
-    private boolean getAutoMinMasterNodes() {
+    private boolean getAutoManageMasterNodes() {
         ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class);
-        return annotation == null ? true : annotation.autoMinMasterNodes();
+        return annotation == null ? true : annotation.autoManageMasterNodes();
     }
 
     private int getNumDataNodes() {
@@ -1838,7 +1846,7 @@ protected TestCluster buildTestCluster(Scope scope, long seed) throws IOExceptio
             }
             mockPlugins = mocks;
         }
-        return new InternalTestCluster(seed, createTempDir(), supportsDedicatedMasters, getAutoMinMasterNodes(),
+        return new InternalTestCluster(seed, createTempDir(), supportsDedicatedMasters, getAutoManageMasterNodes(),
             minNumDataNodes, maxNumDataNodes,
             InternalTestCluster.clusterName(scope.name(), seed) + "-cluster", nodeConfigurationSource, getNumClientNodes(),
             nodePrefix, mockPlugins, getClientWrapper(), forbidPrivateIndexSettings());
@@ -1846,10 +1854,8 @@ protected TestCluster buildTestCluster(Scope scope, long seed) throws IOExceptio
 
     protected NodeConfigurationSource getNodeConfigSource() {
         Settings.Builder initialNodeSettings = Settings.builder();
-        Settings.Builder initialTransportClientSettings = Settings.builder();
         if (addMockTransportService()) {
-            initialNodeSettings.put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType());
-            initialTransportClientSettings.put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType());
+            initialNodeSettings.put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType());;
         }
         return new NodeConfigurationSource() {
             @Override
@@ -1885,6 +1891,13 @@ protected boolean addMockHttpTransport() {
         return true;
     }
 
+    /**
+     * Returns {@code true} if this test cluster can use a mock internal engine. Defaults to true.
+     */
+    protected boolean addMockInternalEngine() {
+        return true;
+    }
+
     /**
      * Returns a function that allows to wrap / filter all clients that are exposed by the test cluster. This is useful
      * for debugging or request / response pre and post processing. It also allows to intercept all calls done by the test
@@ -1907,7 +1920,7 @@ protected Collection<Class<? extends Plugin>> getMockPlugins() {
             if (randomBoolean()) {
                 mocks.add(NodeMocksPlugin.class);
             }
-            if (randomBoolean()) {
+            if (addMockInternalEngine() && randomBoolean()) {
                 mocks.add(MockEngineFactoryPlugin.class);
             }
             if (randomBoolean()) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 6b36f985c210b..66a27d10780d4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -52,6 +52,8 @@
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.CheckedBiFunction;
 import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.SuppressForbidden;
@@ -347,6 +349,16 @@ public final void before()  {
         }
     }
 
+    @BeforeClass
+    public static void setPossibleRoles() {
+        DiscoveryNode.setPossibleRoles(DiscoveryNodeRole.BUILT_IN_ROLES);
+    }
+
+    @AfterClass
+    public static void clearPossibleRoles() {
+        DiscoveryNode.setPossibleRoles(Set.of());
+    }
+
     /**
      * Whether or not we check after each test whether it has left warnings behind. That happens if any deprecated feature or syntax
      * was used by the test and the test didn't assert on it using {@link #assertWarnings(String...)}.
@@ -955,6 +967,11 @@ public NodeEnvironment newNodeEnvironment(Settings settings) throws IOException
         return new NodeEnvironment(build, TestEnvironment.newEnvironment(build));
     }
 
+    public Environment newEnvironment() {
+        Settings build = buildEnvSettings(Settings.EMPTY);
+        return TestEnvironment.newEnvironment(build);
+    }
+
     /** Return consistent index settings for the provided index version. */
     public static Settings.Builder settings(Version version) {
         Settings.Builder builder = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
index f9d72e38044fb..2b7704964c86c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalAggregationTestCase.java
@@ -172,7 +172,7 @@ public abstract class InternalAggregationTestCase<T extends InternalAggregation>
     };
 
     private final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(
-            new SearchModule(Settings.EMPTY, false, emptyList()).getNamedWriteables());
+            new SearchModule(Settings.EMPTY, emptyList()).getNamedWriteables());
 
     private final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(getNamedXContents());
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 695564690c4b1..ef894c3cfd3ed 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -44,7 +44,7 @@
 import org.elasticsearch.cluster.coordination.ClusterBootstrapService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNode.Role;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
@@ -137,7 +137,6 @@
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
-import static java.util.Collections.emptyList;
 import static org.apache.lucene.util.LuceneTestCase.TEST_NIGHTLY;
 import static org.apache.lucene.util.LuceneTestCase.rarely;
 import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING;
@@ -227,7 +226,7 @@ public final class InternalTestCluster extends TestCluster {
 
     private final ExecutorService executor;
 
-    private final boolean autoManageMinMasterNodes;
+    private final boolean autoManageMasterNodes;
 
     private final Collection<Class<? extends Plugin>> mockPlugins;
 
@@ -250,7 +249,7 @@ public InternalTestCluster(
             final long clusterSeed,
             final Path baseDir,
             final boolean randomlyAddDedicatedMasters,
-            final boolean autoManageMinMasterNodes,
+            final boolean autoManageMasterNodes,
             final int minNumDataNodes,
             final int maxNumDataNodes,
             final String clusterName,
@@ -263,7 +262,7 @@ public InternalTestCluster(
                 clusterSeed,
                 baseDir,
                 randomlyAddDedicatedMasters,
-                autoManageMinMasterNodes,
+                autoManageMasterNodes,
                 minNumDataNodes,
                 maxNumDataNodes,
                 clusterName,
@@ -279,7 +278,7 @@ public InternalTestCluster(
             final long clusterSeed,
             final Path baseDir,
             final boolean randomlyAddDedicatedMasters,
-            final boolean autoManageMinMasterNodes,
+            final boolean autoManageMasterNodes,
             final int minNumDataNodes,
             final int maxNumDataNodes,
             final String clusterName,
@@ -290,7 +289,7 @@ public InternalTestCluster(
             final Function<Client, Client> clientWrapper,
             final boolean forbidPrivateIndexSettings) {
         super(clusterSeed);
-        this.autoManageMinMasterNodes = autoManageMinMasterNodes;
+        this.autoManageMasterNodes = autoManageMasterNodes;
         this.clientWrapper = clientWrapper;
         this.forbidPrivateIndexSettings = forbidPrivateIndexSettings;
         this.baseDir = baseDir;
@@ -305,7 +304,7 @@ public InternalTestCluster(
 
         Random random = new Random(clusterSeed);
 
-        boolean useDedicatedMasterNodes = randomlyAddDedicatedMasters ? random.nextBoolean() : false;
+        boolean useDedicatedMasterNodes = randomlyAddDedicatedMasters && random.nextBoolean();
 
         this.numSharedDataNodes = RandomNumbers.randomIntBetween(random, minNumDataNodes, maxNumDataNodes);
         assert this.numSharedDataNodes >= 0;
@@ -345,10 +344,10 @@ public InternalTestCluster(
         }
 
         logger.info("Setup InternalTestCluster [{}] with seed [{}] using [{}] dedicated masters, " +
-                "[{}] (data) nodes and [{}] coord only nodes (min_master_nodes are [{}])",
+                "[{}] (data) nodes and [{}] coord only nodes (master nodes are [{}])",
             clusterName, SeedUtils.formatSeed(clusterSeed),
             numSharedDedicatedMasterNodes, numSharedDataNodes, numSharedCoordOnlyNodes,
-            autoManageMinMasterNodes ? "auto-managed" : "manual");
+            autoManageMasterNodes ? "auto-managed" : "manual");
         this.nodeConfigurationSource = nodeConfigurationSource;
         numDataPaths = random.nextInt(5) == 0 ? 2 + random.nextInt(3) : 1;
         Builder builder = Settings.builder();
@@ -394,12 +393,11 @@ public InternalTestCluster(
     /**
      * Sets {@link #bootstrapMasterNodeIndex} to the given value, see {@link #bootstrapMasterNodeWithSpecifiedIndex(List)}
      * for the description of how this field is used.
-     * It's only possible to change {@link #bootstrapMasterNodeIndex} value if autoManageMinMasterNodes is false.
+     * It's only possible to change {@link #bootstrapMasterNodeIndex} value if autoManageMasterNodes is false.
      */
     public void setBootstrapMasterNodeIndex(int bootstrapMasterNodeIndex) {
-        if (autoManageMinMasterNodes && bootstrapMasterNodeIndex != -1) {
-            throw new AssertionError("bootstrapMasterNodeIndex should be -1 if autoManageMinMasterNodes is true");
-        }
+        assert autoManageMasterNodes == false || bootstrapMasterNodeIndex == -1
+            : "bootstrapMasterNodeIndex should be -1 if autoManageMasterNodes is true, but was " + bootstrapMasterNodeIndex;
         this.bootstrapMasterNodeIndex = bootstrapMasterNodeIndex;
     }
 
@@ -531,7 +529,7 @@ private NodeAndClient getOrBuildRandomNode() {
         final Runnable onTransportServiceStarted = () -> {}; // do not create unicast host file for this one node.
 
         final int nodeId = nextNodeId.getAndIncrement();
-        final Settings settings = getNodeSettings(nodeId, random.nextLong(), Settings.EMPTY, 1);
+        final Settings settings = getNodeSettings(nodeId, random.nextLong(), Settings.EMPTY);
         final Settings nodeSettings = Settings.builder()
                 .putList(INITIAL_MASTER_NODES_SETTING.getKey(), Node.NODE_NAME_SETTING.get(settings))
                 .put(settings)
@@ -605,7 +603,7 @@ public synchronized void ensureAtMostNumDataNodes(int n) throws IOException {
         }
     }
 
-    private Settings getNodeSettings(final int nodeId, final long seed, final Settings extraSettings, final int defaultMinMasterNodes) {
+    private Settings getNodeSettings(final int nodeId, final long seed, final Settings extraSettings) {
         final Settings settings = getSettings(nodeId, seed, extraSettings);
 
         final String name = buildNodeName(nodeId, settings);
@@ -629,8 +627,8 @@ private Settings getNodeSettings(final int nodeId, final long seed, final Settin
         updatedSettings.put("node.name", name);
         updatedSettings.put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed);
 
-        if (autoManageMinMasterNodes) {
-            assertThat("automatically managing min master nodes require nodes to complete a join cycle when starting",
+        if (autoManageMasterNodes) {
+            assertThat("if master nodes are automatically managed then nodes must complete a join cycle when starting",
                 updatedSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()), nullValue());
         }
 
@@ -698,10 +696,10 @@ private String buildNodeName(int id, Settings settings) {
     private static String getRoleSuffix(Settings settings) {
         String suffix = "";
         if (Node.NODE_MASTER_SETTING.exists(settings) && Node.NODE_MASTER_SETTING.get(settings)) {
-            suffix = suffix + Role.MASTER.getAbbreviation();
+            suffix = suffix + DiscoveryNodeRole.MASTER_ROLE.roleNameAbbreviation();
         }
         if (Node.NODE_DATA_SETTING.exists(settings) && Node.NODE_DATA_SETTING.get(settings)) {
-            suffix = suffix + Role.DATA.getAbbreviation();
+            suffix = suffix + DiscoveryNodeRole.DATA_ROLE.roleNameAbbreviation();
         }
         if (Node.NODE_MASTER_SETTING.exists(settings) && Node.NODE_MASTER_SETTING.get(settings) == false &&
             Node.NODE_DATA_SETTING.exists(settings) && Node.NODE_DATA_SETTING.get(settings) == false
@@ -810,8 +808,6 @@ public synchronized void close() throws IOException {
         }
     }
 
-    private static final int REMOVED_MINIMUM_MASTER_NODES = Integer.MAX_VALUE;
-
     private final class NodeAndClient implements Closeable {
         private MockNode node;
         private final Settings originalNodeSettings;
@@ -889,18 +885,17 @@ void startNode() {
         /**
          * closes the node and prepares it to be restarted
          */
-        Settings closeForRestart(RestartCallback callback, int minMasterNodes) throws Exception {
+        Settings closeForRestart(RestartCallback callback) throws Exception {
             assert callback != null;
             close();
+            removeNode(this);
             Settings callbackSettings = callback.onNodeStopped(name);
             assert callbackSettings != null;
             Settings.Builder newSettings = Settings.builder();
-            newSettings.put(callbackSettings);
-            if (minMasterNodes >= 0) {
-                if (INITIAL_MASTER_NODES_SETTING.exists(callbackSettings) == false) {
-                    newSettings.putList(INITIAL_MASTER_NODES_SETTING.getKey());
-                }
+            if (autoManageMasterNodes) {
+                newSettings.putList(INITIAL_MASTER_NODES_SETTING.getKey());
             }
+            newSettings.put(callbackSettings);
             // delete data folders now, before we start other nodes that may claim it
             clearDataIfNeeded(callback);
             return newSettings.build();
@@ -1026,9 +1021,7 @@ private synchronized void reset(boolean wipeData) throws IOException {
 
         // start any missing node
         assert newSize == numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes;
-        final int numberOfMasterNodes = numSharedDedicatedMasterNodes > 0 ? numSharedDedicatedMasterNodes : numSharedDataNodes;
-        final int defaultMinMasterNodes = (numberOfMasterNodes / 2) + 1;
-        final List<NodeAndClient> toStartAndPublish = new ArrayList<>(); // we want to start nodes in one go due to min master nodes
+        final List<NodeAndClient> toStartAndPublish = new ArrayList<>(); // we want to start nodes in one go
         final Runnable onTransportServiceStarted = () -> rebuildUnicastHostFiles(toStartAndPublish);
 
         final List<Settings> settings = new ArrayList<>();
@@ -1037,7 +1030,7 @@ private synchronized void reset(boolean wipeData) throws IOException {
             final Settings.Builder extraSettings = Settings.builder();
             extraSettings.put(Node.NODE_MASTER_SETTING.getKey(), true);
             extraSettings.put(Node.NODE_DATA_SETTING.getKey(), false);
-            settings.add(getNodeSettings(i, sharedNodesSeeds[i], extraSettings.build(), defaultMinMasterNodes));
+            settings.add(getNodeSettings(i, sharedNodesSeeds[i], extraSettings.build()));
         }
         for (int i = numSharedDedicatedMasterNodes; i < numSharedDedicatedMasterNodes + numSharedDataNodes; i++) {
             final Settings.Builder extraSettings = Settings.builder();
@@ -1046,13 +1039,13 @@ private synchronized void reset(boolean wipeData) throws IOException {
                 extraSettings.put(Node.NODE_MASTER_SETTING.getKey(), false).build();
                 extraSettings.put(Node.NODE_DATA_SETTING.getKey(), true).build();
             }
-            settings.add(getNodeSettings(i, sharedNodesSeeds[i], extraSettings.build(), defaultMinMasterNodes));
+            settings.add(getNodeSettings(i, sharedNodesSeeds[i], extraSettings.build()));
         }
         for (int i = numSharedDedicatedMasterNodes + numSharedDataNodes;
              i < numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes; i++) {
             final Builder extraSettings = Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false)
                 .put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_INGEST_SETTING.getKey(), false);
-            settings.add(getNodeSettings(i, sharedNodesSeeds[i], extraSettings.build(), defaultMinMasterNodes));
+            settings.add(getNodeSettings(i, sharedNodesSeeds[i], extraSettings.build()));
         }
 
         int autoBootstrapMasterNodeIndex = -1;
@@ -1061,7 +1054,7 @@ private synchronized void reset(boolean wipeData) throws IOException {
                 .map(Node.NODE_NAME_SETTING::get)
                 .collect(Collectors.toList());
 
-        if (prevNodeCount == 0 && autoManageMinMasterNodes) {
+        if (prevNodeCount == 0 && autoManageMasterNodes) {
             if (numSharedDedicatedMasterNodes > 0) {
                 autoBootstrapMasterNodeIndex = RandomNumbers.randomIntBetween(random, 0, numSharedDedicatedMasterNodes - 1);
             } else if (numSharedDataNodes > 0) {
@@ -1084,7 +1077,7 @@ private synchronized void reset(boolean wipeData) throws IOException {
 
         nextNodeId.set(newSize);
         assert size() == newSize;
-        if (autoManageMinMasterNodes && newSize > 0) {
+        if (autoManageMasterNodes && newSize > 0) {
             validateClusterFormed();
         }
         logger.debug("Cluster is consistent again - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]",
@@ -1190,7 +1183,7 @@ private void assertNoPendingIndexOperations() throws Exception {
                     }
                 }
             }
-        });
+        }, 60, TimeUnit.SECONDS);
     }
 
     private void assertOpenTranslogReferences() throws Exception {
@@ -1209,7 +1202,7 @@ private void assertOpenTranslogReferences() throws Exception {
                     }
                 }
             }
-        });
+        }, 60, TimeUnit.SECONDS);
     }
 
     private void assertNoSnapshottedIndexCommit() throws Exception {
@@ -1301,7 +1294,7 @@ public void assertSeqNos() throws Exception {
                     }
                 }
             }
-        });
+        }, 30, TimeUnit.SECONDS);
     }
 
     /**
@@ -1469,7 +1462,7 @@ public InetSocketAddress[] httpAddresses() {
         for (HttpServerTransport httpServerTransport : getInstances(HttpServerTransport.class)) {
             addresses.add(httpServerTransport.boundAddress().publishAddress().address());
         }
-        return addresses.toArray(new InetSocketAddress[addresses.size()]);
+        return addresses.toArray(new InetSocketAddress[0]);
     }
 
     /**
@@ -1543,8 +1536,7 @@ private synchronized void startAndPublishNodesAndClients(List<NodeAndClient> nod
             }
             nodeAndClients.forEach(this::publishNode);
 
-            if (autoManageMinMasterNodes && currentMasters > 0 && newMasters > 0 &&
-                getMinMasterNodes(currentMasters + newMasters) > currentMasters) {
+            if (autoManageMasterNodes && newMasters > 0) {
                 // update once masters have joined
                 validateClusterFormed();
             }
@@ -1652,33 +1644,17 @@ private void restartNode(NodeAndClient nodeAndClient, RestartCallback callback)
             activeDisruptionScheme.removeFromNode(nodeAndClient.name, this);
         }
 
-        Set<String> excludedNodeIds = excludeMasters(Collections.singleton(nodeAndClient));
-
-        final Settings newSettings = nodeAndClient.closeForRestart(callback,
-                autoManageMinMasterNodes ? getMinMasterNodes(getMasterNodesCount()) : -1);
-
+        final Set<String> excludedNodeIds = excludeMasters(Collections.singleton(nodeAndClient));
+        final Settings newSettings = nodeAndClient.closeForRestart(callback);
         removeExclusions(excludedNodeIds);
 
-        boolean success = false;
-        try {
-            nodeAndClient.recreateNode(newSettings, () -> rebuildUnicastHostFiles(emptyList()));
-            nodeAndClient.startNode();
-            success = true;
-        } finally {
-            if (success == false) {
-                removeNode(nodeAndClient);
-            }
-        }
-
-        if (activeDisruptionScheme != null) {
-            activeDisruptionScheme.applyToNode(nodeAndClient.name, this);
-        }
+        nodeAndClient.recreateNode(newSettings, () -> rebuildUnicastHostFiles(Collections.singletonList(nodeAndClient)));
+        nodeAndClient.startNode();
+        publishNode(nodeAndClient);
 
         if (callback.validateClusterForming() || excludedNodeIds.isEmpty() == false) {
-            // we have to validate cluster size if updateMinMaster == true, because we need the
-            // second node to join in order to increment min_master_nodes back to 2.
-            // we also have to do via the node that was just restarted as it may be that the master didn't yet process
-            // the fact it left
+            // we have to validate cluster size to ensure that the restarted node has rejoined the cluster if it was master-eligible;
+            // we have to do this via the node that was just restarted as it may be that the master didn't yet process the fact that it left
             validateClusterFormed(nodeAndClient.name);
         }
     }
@@ -1694,7 +1670,7 @@ private NodeAndClient removeNode(NodeAndClient nodeAndClient) {
     private Set<String> excludeMasters(Collection<NodeAndClient> nodeAndClients) {
         assert Thread.holdsLock(this);
         final Set<String> excludedNodeIds = new HashSet<>();
-        if (autoManageMinMasterNodes && nodeAndClients.size() > 0) {
+        if (autoManageMasterNodes && nodeAndClients.size() > 0) {
 
             final long currentMasters = nodes.values().stream().filter(NodeAndClient::isMasterEligible).count();
             final long stoppingMasters = nodeAndClients.stream().filter(NodeAndClient::isMasterEligible).count();
@@ -1739,9 +1715,9 @@ private void removeExclusions(Set<String> excludedNodeIds) {
     public synchronized void fullRestart(RestartCallback callback) throws Exception {
         int numNodesRestarted = 0;
         final Settings[] newNodeSettings = new Settings[nextNodeId.get()];
-        Map<Set<Role>, List<NodeAndClient>> nodesByRoles = new HashMap<>();
+        Map<Set<DiscoveryNodeRole>, List<NodeAndClient>> nodesByRoles = new HashMap<>();
         Set[] rolesOrderedByOriginalStartupOrder = new Set[nextNodeId.get()];
-        final int minMasterNodes = autoManageMinMasterNodes ? getMinMasterNodes(getMasterNodesCount()) : -1;
+        final int nodeCount = nodes.size();
         for (NodeAndClient nodeAndClient : nodes.values()) {
             callback.doAfterNodes(numNodesRestarted++, nodeAndClient.nodeClient());
             logger.info("Stopping and resetting node [{}] ", nodeAndClient.name);
@@ -1749,13 +1725,15 @@ public synchronized void fullRestart(RestartCallback callback) throws Exception
                 activeDisruptionScheme.removeFromNode(nodeAndClient.name, this);
             }
             DiscoveryNode discoveryNode = getInstanceFromNode(ClusterService.class, nodeAndClient.node()).localNode();
-            final Settings newSettings = nodeAndClient.closeForRestart(callback, minMasterNodes);
+            final Settings newSettings = nodeAndClient.closeForRestart(callback);
             newNodeSettings[nodeAndClient.nodeAndClientId()] = newSettings;
             rolesOrderedByOriginalStartupOrder[nodeAndClient.nodeAndClientId()] = discoveryNode.getRoles();
             nodesByRoles.computeIfAbsent(discoveryNode.getRoles(), k -> new ArrayList<>()).add(nodeAndClient);
         }
 
-        assert nodesByRoles.values().stream().mapToInt(List::size).sum() == nodes.size();
+        callback.onAllNodesStopped();
+
+        assert nodesByRoles.values().stream().mapToInt(List::size).sum() == nodeCount;
 
         // randomize start up order, but making sure that:
         // 1) A data folder that was assigned to a data node will stay so
@@ -1934,25 +1912,18 @@ public List<String> startNodes(int numOfNodes, Settings settings) {
      */
     public synchronized List<String> startNodes(Settings... extraSettings) {
         final int newMasterCount = Math.toIntExact(Stream.of(extraSettings).filter(Node.NODE_MASTER_SETTING::get).count());
-        final int defaultMinMasterNodes;
-        if (autoManageMinMasterNodes) {
-            defaultMinMasterNodes = getMinMasterNodes(getMasterNodesCount() + newMasterCount);
-        } else {
-            defaultMinMasterNodes = -1;
-        }
         final List<NodeAndClient> nodes = new ArrayList<>();
         final int prevMasterCount = getMasterNodesCount();
-        int autoBootstrapMasterNodeIndex =
-                prevMasterCount == 0 && autoManageMinMasterNodes && newMasterCount > 0 && Arrays.stream(extraSettings)
-            .allMatch(s -> Node.NODE_MASTER_SETTING.get(s) == false
-                || ZEN2_DISCOVERY_TYPE.equals(DISCOVERY_TYPE_SETTING.get(s)))
+        int autoBootstrapMasterNodeIndex = autoManageMasterNodes && prevMasterCount == 0 && newMasterCount > 0
+            && Arrays.stream(extraSettings)
+                    .allMatch(s -> Node.NODE_MASTER_SETTING.get(s) == false || ZEN2_DISCOVERY_TYPE.equals(DISCOVERY_TYPE_SETTING.get(s)))
             ? RandomNumbers.randomIntBetween(random, 0, newMasterCount - 1) : -1;
 
         final int numOfNodes = extraSettings.length;
         final int firstNodeId = nextNodeId.getAndIncrement();
         final List<Settings> settings = new ArrayList<>();
         for (int i = 0; i < numOfNodes; i++) {
-            settings.add(getNodeSettings(firstNodeId + i, random.nextLong(), extraSettings[i], defaultMinMasterNodes));
+            settings.add(getNodeSettings(firstNodeId + i, random.nextLong(), extraSettings[i]));
         }
         nextNodeId.set(firstNodeId + numOfNodes);
 
@@ -1978,7 +1949,7 @@ public synchronized List<String> startNodes(Settings... extraSettings) {
             nodes.add(nodeAndClient);
         }
         startAndPublishNodesAndClients(nodes);
-        if (autoManageMinMasterNodes) {
+        if (autoManageMasterNodes) {
             validateClusterFormed();
         }
         return nodes.stream().map(NodeAndClient::getName).collect(Collectors.toList());
@@ -2004,11 +1975,6 @@ public List<String> startDataOnlyNodes(int numNodes) {
                 .put(Node.NODE_DATA_SETTING.getKey(), true).build());
     }
 
-    /** calculates a min master nodes value based on the given number of master nodes */
-    private static int getMinMasterNodes(int eligibleMasterNodes) {
-        return eligibleMasterNodes / 2 + 1;
-    }
-
     private int getMasterNodesCount() {
         return (int) nodes.values().stream().filter(n -> Node.NODE_MASTER_SETTING.get(n.node().settings())).count();
     }
@@ -2143,7 +2109,7 @@ synchronized String routingKeyForShard(Index index, int shard, Random random) {
                         greaterThan(shard));
                 OperationRouting operationRouting = clusterService.operationRouting();
                 while (true) {
-                    String routing = RandomStrings.randomAsciiOfLength(random, 10);
+                    String routing = RandomStrings.randomAsciiLettersOfLength(random, 10);
                     final int targetShard = operationRouting
                             .indexShards(clusterService.state(), index.getName(), null, routing)
                             .shardId().getId();
@@ -2210,6 +2176,9 @@ public Settings onNodeStopped(String nodeName) throws Exception {
             return Settings.EMPTY;
         }
 
+        public void onAllNodesStopped() throws Exception {
+        }
+
         /**
          * Executed for each node before the {@code n + 1} node is restarted. The given client is
          * an active client to the node that will be restarted next.
diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java b/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
index 58b722cf15274..77d1c315b6970 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
@@ -18,12 +18,17 @@
  */
 package org.elasticsearch.test;
 
+import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexService;
@@ -34,6 +39,9 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -72,8 +80,11 @@ public void onIndexModule(IndexModule module) {
         }
 
         @Override
-        public Collection<Module> createGuiceModules() {
-            return Collections.singleton(binder -> binder.bind(TestEventListener.class).toInstance(listener));
+        public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
+                                                   ResourceWatcherService resourceWatcherService, ScriptService scriptService,
+                                                   NamedXContentRegistry xContentRegistry, Environment environment,
+                                                   NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) {
+            return Collections.singletonList(listener);
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java b/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java
index c6a5d77faf5f3..e9c53ed896765 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java
@@ -117,7 +117,7 @@ public UnseenEventExpectation(String name, String logger, Level level, String me
 
         @Override
         public void assertMatched() {
-            assertThat("expected to see " + name + " but did not", saw, equalTo(false));
+            assertThat("expected not to see " + name + " but did", saw, equalTo(false));
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java b/test/framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java
index 60c69bbd6c652..35891245bf919 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java
@@ -37,11 +37,6 @@ public Settings nodeSettings(int nodeOrdinal) {
         public Path nodeConfigPath(int nodeOrdinal) {
             return null;
         }
-
-        @Override
-        public Settings transportClientSettings() {
-            return Settings.EMPTY;
-        }
     };
 
     /**
@@ -55,14 +50,4 @@ public Settings transportClientSettings() {
     public Collection<Class<? extends Plugin>> nodePlugins() {
         return Collections.emptyList();
     }
-
-    public Settings transportClientSettings() {
-        return Settings.EMPTY;
-    }
-
-    /** Returns plugins that should be loaded in the transport client */
-    public Collection<Class<? extends Plugin>> transportClientPlugins() {
-        return Collections.emptyList();
-    }
-
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java b/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java
index d95e1d32663e7..d034fa0c5e3e3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.test.client;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -51,7 +51,7 @@ public NoOpClient(String testName) {
 
     @Override
     protected <Request extends ActionRequest, Response extends ActionResponse>
-    void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+    void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
         listener.onResponse(null);
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java
index c523aa15e58ac..eb39b1c16d00c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java
@@ -87,8 +87,14 @@ public TransportService createTransportService(Settings settings, ThreadPool thr
 
     @Override
     public Releasable openConnection(DiscoveryNode node, ConnectionProfile profile, ActionListener<Connection> listener) {
-        final Optional<DisruptableMockTransport> matchingTransport = getDisruptableMockTransport(node.getAddress());
-        if (matchingTransport.isPresent()) {
+        final Optional<DisruptableMockTransport> optionalMatchingTransport = getDisruptableMockTransport(node.getAddress());
+        if (optionalMatchingTransport.isPresent()) {
+            final DisruptableMockTransport matchingTransport = optionalMatchingTransport.get();
+            final ConnectionStatus connectionStatus = getConnectionStatus(matchingTransport.getLocalNode());
+            if (connectionStatus != ConnectionStatus.CONNECTED) {
+                throw new ConnectTransportException(node, "node [" + node + "] is [" + connectionStatus + "] not [CONNECTED]");
+            }
+
             listener.onResponse(new CloseableConnection() {
                 @Override
                 public DiscoveryNode getNode() {
@@ -98,12 +104,12 @@ public DiscoveryNode getNode() {
                 @Override
                 public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
                     throws TransportException {
-                    onSendRequest(requestId, action, request, matchingTransport.get());
+                    onSendRequest(requestId, action, request, matchingTransport);
                 }
             });
             return () -> {};
         } else {
-            throw new ConnectTransportException(node, "node " + node + " does not exist");
+            throw new ConnectTransportException(node, "node [" + node + "] does not exist");
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java
index b3a6fe8490895..52b086db338f3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java
@@ -24,14 +24,9 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FilterDirectoryReader;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.AssertingIndexSearcher;
-import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.QueryCache;
 import org.apache.lucene.search.QueryCachingPolicy;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.search.XIndexSearcher;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.settings.Setting;
@@ -47,7 +42,6 @@
 import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.util.IdentityHashMap;
-import java.util.List;
 import java.util.Random;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -151,19 +145,8 @@ public AssertingIndexSearcher newSearcher(Engine.Searcher searcher) throws Engin
         if (reader instanceof DirectoryReader && mockContext.wrapReader) {
             wrappedReader = wrapReader((DirectoryReader) reader);
         }
-        final IndexSearcher delegate = new IndexSearcher(wrappedReader);
-        delegate.setSimilarity(searcher.searcher().getSimilarity());
-        delegate.setQueryCache(filterCache);
-        delegate.setQueryCachingPolicy(filterCachingPolicy);
-        final XIndexSearcher wrappedSearcher = new XIndexSearcher(delegate);
         // this executes basic query checks and asserts that weights are normalized only once etc.
-        final AssertingIndexSearcher assertingIndexSearcher = new AssertingIndexSearcher(mockContext.random, wrappedReader) {
-            @Override
-            protected void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
-                // we cannot use the asserting searcher because the weight is created by the ContextIndexSearcher
-                wrappedSearcher.search(leaves, weight, collector);
-            }
-        };
+        final AssertingIndexSearcher assertingIndexSearcher = new AssertingIndexSearcher(mockContext.random, wrappedReader);
         assertingIndexSearcher.setSimilarity(searcher.searcher().getSimilarity());
         assertingIndexSearcher.setQueryCache(filterCache);
         assertingIndexSearcher.setQueryCachingPolicy(filterCachingPolicy);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index fdb7b8984c3b0..a06696169de45 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -28,7 +28,6 @@
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
@@ -89,7 +88,6 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -452,20 +450,7 @@ public static void assertIndexTemplateExists(GetIndexTemplatesResponse templates
         assertThat(templateNames, hasItem(name));
     }
 
-    /**
-     * Assert that aliases are missing
-     */
-    public static void assertAliasesMissing(AliasesExistResponse aliasesExistResponse) {
-        assertFalse("Aliases shouldn't exist", aliasesExistResponse.exists());
-    }
-
-    /**
-     * Assert that aliases exist
-     */
-    public static void assertAliasesExist(AliasesExistResponse aliasesExistResponse) {
-        assertTrue("Aliases should exist", aliasesExistResponse.exists());
-    }
-
+    /*
     /*
      * matchers
      */
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/OptionalMatchers.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/OptionalMatchers.java
new file mode 100644
index 0000000000000..6481dcd352548
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/OptionalMatchers.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.hamcrest;
+
+import org.hamcrest.Description;
+import org.hamcrest.TypeSafeMatcher;
+
+import java.util.Optional;
+
+public class OptionalMatchers {
+
+    private static class IsEmptyMatcher extends TypeSafeMatcher<Optional<?>> {
+
+        @Override
+        protected boolean matchesSafely(final Optional<?> item) {
+            // noinspection OptionalAssignedToNull
+            return item != null && item.isEmpty();
+        }
+
+        @Override
+        public void describeTo(final Description description) {
+            description.appendText("expected empty optional");
+        }
+
+        @Override
+        protected void describeMismatchSafely(final Optional<?> item, final Description mismatchDescription) {
+            if (item == null) {
+                mismatchDescription.appendText("was null");
+            } else {
+                mismatchDescription.appendText("was ").appendText(item.toString());
+            }
+        }
+
+    }
+
+    public static IsEmptyMatcher isEmpty() {
+        return new IsEmptyMatcher();
+    }
+
+    private static class IsPresentMatcher extends TypeSafeMatcher<Optional<?>> {
+
+        @Override
+        protected boolean matchesSafely(final Optional<?> item) {
+            return item != null && item.isPresent();
+        }
+
+        @Override
+        public void describeTo(final Description description) {
+            description.appendText("expected non-empty optional");
+        }
+
+        @Override
+        protected void describeMismatchSafely(final Optional<?> item, final Description mismatchDescription) {
+            if (item == null) {
+                mismatchDescription.appendText("was null");
+            } else {
+                mismatchDescription.appendText("was empty");
+            }
+        }
+
+    }
+
+    public static IsPresentMatcher isPresent() {
+        return new IsPresentMatcher();
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
index 054b126b4ef4f..24e38bbec58d3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
@@ -84,8 +84,14 @@ public void testFailure(Failure failure) throws Exception {
         b.append(failure.getDescription().getClassName());
         final String methodName = failure.getDescription().getMethodName();
         if (methodName != null) {
-            b.append(".");
-            b.append(failure.getDescription().getMethodName());
+            // fallback to system property filter when tests contain "."
+            if (methodName.contains(".")) {
+                b.append("\" -Dtests.method=\"");
+                b.append(methodName);
+            } else {
+                b.append(".");
+                b.append(methodName);
+            }
         }
         b.append("\"");
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 87596beb451d1..969cebc0c60b5 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -540,7 +540,16 @@ private void wipeCluster() throws Exception {
                 // All other repo types we really don't have a chance of being able to iterate properly, sadly.
                 Request listRequest = new Request("GET", "/_snapshot/" + repoName + "/_all");
                 listRequest.addParameter("ignore_unavailable", "true");
-                List<?> snapshots = (List<?>) entityAsMap(adminClient.performRequest(listRequest)).get("snapshots");
+
+                Map<?, ?> response = entityAsMap(adminClient.performRequest(listRequest));
+                Map<?, ?> oneRepoResponse;
+                if (response.containsKey("responses")) {
+                    oneRepoResponse = ((Map<?,?>)((List<?>) response.get("responses")).get(0));
+                } else {
+                    oneRepoResponse = response;
+                }
+
+                List<?> snapshots = (List<?>) oneRepoResponse.get("snapshots");
                 for (Object snapshot : snapshots) {
                     Map<?, ?> snapshotInfo = (Map<?, ?>) snapshot;
                     String name = (String) snapshotInfo.get("snapshot");
@@ -935,6 +944,9 @@ private static boolean isXPackTemplate(String name) {
         if (name.startsWith(".watch") || name.startsWith(".triggered_watches")) {
             return true;
         }
+        if (name.startsWith(".data-frame-")) {
+            return true;
+        }
         if (name.startsWith(".ml-")) {
             return true;
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApi.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApi.java
index 72c94762a4314..421da469f3b3a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApi.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApi.java
@@ -24,6 +24,7 @@
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
@@ -39,6 +40,12 @@ public class ClientYamlSuiteRestApi {
     private Map<String, Boolean> pathParts = new HashMap<>();
     private Map<String, Boolean> params = new HashMap<>();
     private Body body = Body.NOT_SUPPORTED;
+    private Stability stability = Stability.UNKNOWN;
+
+    public enum Stability
+    {
+        UNKNOWN, EXPERIMENTAL, BETA, STABLE
+    }
 
     public enum Body {
         NOT_SUPPORTED, OPTIONAL, REQUIRED
@@ -139,6 +146,12 @@ public boolean isBodyRequired() {
         return body == Body.REQUIRED;
     }
 
+    public void setStability(String stability) {
+        this.stability = ClientYamlSuiteRestApi.Stability.valueOf(stability.toUpperCase(Locale.ROOT));
+    }
+
+    public Stability getStability() { return this.stability; }
+
     /**
      * Finds the best matching rest path given the current parameters and replaces
      * placeholders with their corresponding values received as arguments
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java
index 3c6216eed76ed..cd797ae4e0705 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParser.java
@@ -51,6 +51,19 @@ public ClientYamlSuiteRestApi parse(String location, XContentParser parser) thro
         while (parser.nextToken() != XContentParser.Token.END_OBJECT || level >= 0) {
 
             if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
+                if ("stability".equals(parser.currentName()))
+                {
+                    parser.nextToken();
+                    String stability = parser.textOrNull();
+                    try {
+                        restApi.setStability(stability);
+                    } catch (IllegalArgumentException ex)
+                    {
+                        throw new IllegalArgumentException("API [" + apiName + "] sets wrong state for stability ("
+                            + stability + ") [" + location + "]");
+                    }
+                }
+
                 if ("methods".equals(parser.currentName())) {
                     parser.nextToken();
                     while (parser.nextToken() == XContentParser.Token.VALUE_STRING) {
@@ -144,6 +157,10 @@ public ClientYamlSuiteRestApi parse(String location, XContentParser parser) thro
         assert parser.currentToken() == XContentParser.Token.END_OBJECT : "Expected [END_OBJECT] but was ["  + parser.currentToken() +"]";
         parser.nextToken();
 
+        if (restApi.getStability() == ClientYamlSuiteRestApi.Stability.UNKNOWN) {
+            throw new IllegalArgumentException("API [" + apiName + "] does not explicitly declare its stability in [" + location + "]");
+        }
+
         return restApi;
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index d7aa6e5903e33..7cd706b3564ce 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -41,6 +41,7 @@
 import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.RunOnce;
+import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
@@ -68,6 +69,7 @@
 import java.util.Map;
 import java.util.Queue;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingDeque;
@@ -281,8 +283,25 @@ public void addUnresponsiveRule(TransportAddress transportAddress) {
             return () -> {};
         });
 
-        transport().addSendBehavior(transportAddress, (connection, requestId, action, request, options) -> {
-            // don't send anything, the receiving node is unresponsive
+        transport().addSendBehavior(transportAddress, new StubbableTransport.SendRequestBehavior() {
+            private Set<Transport.Connection> toClose = ConcurrentHashMap.newKeySet();
+            @Override
+            public void sendRequest(Transport.Connection connection, long requestId, String action,
+                                    TransportRequest request, TransportRequestOptions options) {
+                // don't send anything, the receiving node is unresponsive
+                toClose.add(connection);
+            }
+
+            @Override
+            public void clearCallback() {
+                // close to simulate that tcp-ip eventually times out and closes connection (necessary to ensure transport eventually
+                // responds).
+                try {
+                    IOUtils.close(toClose);
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            }
         });
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java b/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java
index f40820cc8771d..03fcc89059622 100644
--- a/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java
+++ b/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java
@@ -24,12 +24,12 @@
 
 public class TestThreadPool extends ThreadPool {
 
-    public TestThreadPool(String name) {
-        this(name, Settings.EMPTY);
+    public TestThreadPool(String name, ExecutorBuilder<?>... customBuilders) {
+        this(name, Settings.EMPTY, customBuilders);
     }
 
-    public TestThreadPool(String name, Settings settings) {
-        super(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), name).put(settings).build());
+    public TestThreadPool(String name, Settings settings, ExecutorBuilder<?>... customBuilders) {
+        super(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), name).put(settings).build(), customBuilders);
     }
 
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java b/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java
deleted file mode 100644
index cb415798d22cd..0000000000000
--- a/test/framework/src/main/java/org/elasticsearch/transport/MockTransportClient.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.transport;
-
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.network.NetworkModule;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.transport.nio.MockNioTransportPlugin;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-
-@SuppressWarnings({"unchecked","varargs"})
-@Deprecated
-public class MockTransportClient extends TransportClient {
-    private static final Settings DEFAULT_SETTINGS = Settings.builder().put("transport.type.default",
-        MockNioTransportPlugin.MOCK_NIO_TRANSPORT_NAME).build();
-
-    public MockTransportClient(Settings settings, Class<? extends Plugin>... plugins) {
-        this(settings, Arrays.asList(plugins));
-    }
-
-    public MockTransportClient(Settings settings, Collection<Class<? extends Plugin>> plugins) {
-        this(settings, plugins, null);
-    }
-
-    public MockTransportClient(Settings settings, Collection<Class<? extends Plugin>> plugins, HostFailureListener listener) {
-        super(settings, DEFAULT_SETTINGS, addMockTransportIfMissing(settings, plugins), listener);
-    }
-
-    private static Collection<Class<? extends Plugin>> addMockTransportIfMissing(Settings settings,
-                                                                                 Collection<Class<? extends Plugin>> plugins) {
-        boolean settingExists = NetworkModule.TRANSPORT_TYPE_SETTING.exists(settings);
-        String transportType = NetworkModule.TRANSPORT_TYPE_SETTING.get(settings);
-        if (settingExists == false || MockNioTransportPlugin.MOCK_NIO_TRANSPORT_NAME.equals(transportType)) {
-            if (plugins.contains(MockNioTransportPlugin.class)) {
-                return plugins;
-            } else {
-                plugins = new ArrayList<>(plugins);
-                plugins.add(MockNioTransportPlugin.class);
-                return plugins;
-            }
-        }
-        return plugins;
-    }
-
-    public NamedWriteableRegistry getNamedWriteableRegistry() {
-        return namedWriteableRegistry;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java
index 9470b7548adfb..336e69dcd86f4 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java
@@ -84,7 +84,7 @@ public MockNioTransport(Settings settings, Version version, ThreadPool threadPoo
                             PageCacheRecycler pageCacheRecycler, NamedWriteableRegistry namedWriteableRegistry,
                             CircuitBreakerService circuitBreakerService) {
         super(settings, version, threadPool, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, networkService);
-        this.transportThreadWatchdog = new TransportThreadWatchdog(threadPool);
+        this.transportThreadWatchdog = new TransportThreadWatchdog(threadPool, settings);
     }
 
     @Override
@@ -322,21 +322,20 @@ public void sendMessage(BytesReference reference, ActionListener<Void> listener)
     }
 
     static final class TransportThreadWatchdog {
-
-        private static final long WARN_THRESHOLD = TimeUnit.MILLISECONDS.toNanos(150);
-
         // Only check every 2s to not flood the logs on a blocked thread.
         // We mostly care about long blocks and not random slowness anyway and in tests would randomly catch slow operations that block for
         // less than 2s eventually.
         private static final TimeValue CHECK_INTERVAL = TimeValue.timeValueSeconds(2);
 
+        private final long warnThreshold;
         private final ThreadPool threadPool;
         private final ConcurrentHashMap<Thread, Long> registry = new ConcurrentHashMap<>();
 
         private volatile boolean stopped;
 
-        TransportThreadWatchdog(ThreadPool threadPool) {
+        TransportThreadWatchdog(ThreadPool threadPool, Settings settings) {
             this.threadPool = threadPool;
+            warnThreshold = ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING.get(settings).nanos() + TimeValue.timeValueMillis(100L).nanos();
             threadPool.schedule(this::logLongRunningExecutions, CHECK_INTERVAL, ThreadPool.Names.GENERIC);
         }
 
@@ -353,7 +352,7 @@ public void unregister() {
 
         private void maybeLogElapsedTime(long startTime) {
             long elapsedTime = threadPool.relativeTimeInNanos() - startTime;
-            if (elapsedTime > WARN_THRESHOLD) {
+            if (elapsedTime > warnThreshold) {
                 logger.warn(
                     new ParameterizedMessage("Slow execution on network thread [{} milliseconds]",
                         TimeUnit.NANOSECONDS.toMillis(elapsedTime)),
@@ -363,11 +362,13 @@ private void maybeLogElapsedTime(long startTime) {
 
         private void logLongRunningExecutions() {
             for (Map.Entry<Thread, Long> entry : registry.entrySet()) {
-                final long elapsedTime = threadPool.relativeTimeInMillis() - entry.getValue();
-                if (elapsedTime > WARN_THRESHOLD) {
+                final long elapsedTimeInNanos = threadPool.relativeTimeInNanos() - entry.getValue();
+                if (elapsedTimeInNanos > warnThreshold) {
                     final Thread thread = entry.getKey();
-                    logger.warn("Slow execution on network thread [{}] [{} milliseconds]: \n{}", thread.getName(),
-                        TimeUnit.NANOSECONDS.toMillis(elapsedTime),
+                    logger.warn("Potentially blocked execution on network thread [{}] [{}] [{} milliseconds]: \n{}",
+                        thread.getName(),
+                        thread.getState(),
+                        TimeUnit.NANOSECONDS.toMillis(elapsedTimeInNanos),
                         Arrays.stream(thread.getStackTrace()).map(Object::toString).collect(Collectors.joining("\n")));
                 }
             }
diff --git a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
index a9948b8b64856..93ed0c79171b4 100644
--- a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
@@ -41,8 +41,8 @@ public class MockSearchServiceTests extends ESTestCase {
     public void testAssertNoInFlightContext() {
         final long nowInMillis = randomNonNegativeLong();
         SearchContext s = new TestSearchContext(new QueryShardContext(0,
-            new IndexSettings(EMPTY_INDEX_METADATA, Settings.EMPTY), null, null, null, null, null, xContentRegistry(),
-            writableRegistry(), null, null, () -> nowInMillis, null)) {
+            new IndexSettings(EMPTY_INDEX_METADATA, Settings.EMPTY), null, null, null, null, null, null,
+            xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null)) {
 
             @Override
             public SearchShardTarget shardTarget() {
diff --git a/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java
index cab910ad6c430..3e21f7c8b6bdf 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/VersionUtilsTests.java
@@ -190,10 +190,8 @@ public static class TestUnstableBranch {
         public static final Version V_5_3_1 = Version.fromString("5.3.1");
         public static final Version V_5_3_2 = Version.fromString("5.3.2");
         public static final Version V_5_4_0 = Version.fromString("5.4.0");
-        public static final Version V_6_0_0_alpha1 = Version.fromString("6.0.0-alpha1");
-        public static final Version V_6_0_0_alpha2 = Version.fromString("6.0.0-alpha2");
-        public static final Version V_6_0_0_beta1 = Version.fromString("6.0.0-beta1");
-        public static final Version CURRENT = V_6_0_0_beta1;
+        public static final Version V_6_0_0 = Version.fromString("6.0.0");
+        public static final Version CURRENT = V_6_0_0;
     }
 
     public void testResolveReleasedVersionsForUnstableBranch() {
@@ -204,23 +202,17 @@ public void testResolveReleasedVersionsForUnstableBranch() {
 
         assertThat(released, equalTo(Arrays.asList(
             TestUnstableBranch.V_5_3_0,
-            TestUnstableBranch.V_5_3_1,
-            TestUnstableBranch.V_6_0_0_alpha1,
-            TestUnstableBranch.V_6_0_0_alpha2)));
+            TestUnstableBranch.V_5_3_1)));
         assertThat(unreleased, equalTo(Arrays.asList(
             TestUnstableBranch.V_5_3_2,
             TestUnstableBranch.V_5_4_0,
-            TestUnstableBranch.V_6_0_0_beta1)));
+            TestUnstableBranch.V_6_0_0)));
     }
 
     public static class TestNewMajorRelease {
         public static final Version V_5_6_0 = Version.fromString("5.6.0");
         public static final Version V_5_6_1 = Version.fromString("5.6.1");
         public static final Version V_5_6_2 = Version.fromString("5.6.2");
-        public static final Version V_6_0_0_alpha1 = Version.fromString("6.0.0-alpha1");
-        public static final Version V_6_0_0_alpha2 = Version.fromString("6.0.0-alpha2");
-        public static final Version V_6_0_0_beta1 = Version.fromString("6.0.0-beta1");
-        public static final Version V_6_0_0_beta2 = Version.fromString("6.0.0-beta2");
         public static final Version V_6_0_0 = Version.fromString("6.0.0");
         public static final Version V_6_0_1 = Version.fromString("6.0.1");
         public static final Version CURRENT = V_6_0_1;
@@ -235,10 +227,6 @@ public void testResolveReleasedVersionsAtNewMajorRelease() {
         assertThat(released, equalTo(Arrays.asList(
             TestNewMajorRelease.V_5_6_0,
             TestNewMajorRelease.V_5_6_1,
-            TestNewMajorRelease.V_6_0_0_alpha1,
-            TestNewMajorRelease.V_6_0_0_alpha2,
-            TestNewMajorRelease.V_6_0_0_beta1,
-            TestNewMajorRelease.V_6_0_0_beta2,
             TestNewMajorRelease.V_6_0_0)));
         assertThat(unreleased, equalTo(Arrays.asList(
             TestNewMajorRelease.V_5_6_2,
@@ -249,10 +237,6 @@ public static class TestVersionBumpIn6x {
         public static final Version V_5_6_0 = Version.fromString("5.6.0");
         public static final Version V_5_6_1 = Version.fromString("5.6.1");
         public static final Version V_5_6_2 = Version.fromString("5.6.2");
-        public static final Version V_6_0_0_alpha1 = Version.fromString("6.0.0-alpha1");
-        public static final Version V_6_0_0_alpha2 = Version.fromString("6.0.0-alpha2");
-        public static final Version V_6_0_0_beta1 = Version.fromString("6.0.0-beta1");
-        public static final Version V_6_0_0_beta2 = Version.fromString("6.0.0-beta2");
         public static final Version V_6_0_0 = Version.fromString("6.0.0");
         public static final Version V_6_0_1 = Version.fromString("6.0.1");
         public static final Version V_6_1_0 = Version.fromString("6.1.0");
@@ -268,10 +252,6 @@ public void testResolveReleasedVersionsAtVersionBumpIn6x() {
         assertThat(released, equalTo(Arrays.asList(
             TestVersionBumpIn6x.V_5_6_0,
             TestVersionBumpIn6x.V_5_6_1,
-            TestVersionBumpIn6x.V_6_0_0_alpha1,
-            TestVersionBumpIn6x.V_6_0_0_alpha2,
-            TestVersionBumpIn6x.V_6_0_0_beta1,
-            TestVersionBumpIn6x.V_6_0_0_beta2,
             TestVersionBumpIn6x.V_6_0_0)));
         assertThat(unreleased, equalTo(Arrays.asList(
             TestVersionBumpIn6x.V_5_6_2,
@@ -283,10 +263,6 @@ public static class TestNewMinorBranchIn6x {
         public static final Version V_5_6_0 = Version.fromString("5.6.0");
         public static final Version V_5_6_1 = Version.fromString("5.6.1");
         public static final Version V_5_6_2 = Version.fromString("5.6.2");
-        public static final Version V_6_0_0_alpha1 = Version.fromString("6.0.0-alpha1");
-        public static final Version V_6_0_0_alpha2 = Version.fromString("6.0.0-alpha2");
-        public static final Version V_6_0_0_beta1 = Version.fromString("6.0.0-beta1");
-        public static final Version V_6_0_0_beta2 = Version.fromString("6.0.0-beta2");
         public static final Version V_6_0_0 = Version.fromString("6.0.0");
         public static final Version V_6_0_1 = Version.fromString("6.0.1");
         public static final Version V_6_1_0 = Version.fromString("6.1.0");
@@ -305,10 +281,6 @@ public void testResolveReleasedVersionsAtNewMinorBranchIn6x() {
         assertThat(released, equalTo(Arrays.asList(
             TestNewMinorBranchIn6x.V_5_6_0,
             TestNewMinorBranchIn6x.V_5_6_1,
-            TestNewMinorBranchIn6x.V_6_0_0_alpha1,
-            TestNewMinorBranchIn6x.V_6_0_0_alpha2,
-            TestNewMinorBranchIn6x.V_6_0_0_beta1,
-            TestNewMinorBranchIn6x.V_6_0_0_beta2,
             TestNewMinorBranchIn6x.V_6_0_0,
             TestNewMinorBranchIn6x.V_6_0_1,
             TestNewMinorBranchIn6x.V_6_1_0,
@@ -330,11 +302,6 @@ public void testGradleVersionsMatchVersionUtils() {
                 /* Java lists all versions from the 5.x series onwards, but we only want to consider
                  * ones that we're supposed to be compatible with. */
                 .filter(v -> v.onOrAfter(Version.CURRENT.minimumIndexCompatibilityVersion()))
-                /* Gradle will never include *released* alphas or betas because it will prefer
-                 * the unreleased branch head. Gradle is willing to use branch heads that are
-                 * beta or rc so that we have *something* to test against even though we
-                 * do not offer backwards compatibility for alphas, betas, or rcs. */
-                .filter(Version::isRelease)
                 .collect(toList());
 
         List<String> releasedIndexCompatible = released.stream()
@@ -347,14 +314,6 @@ public void testGradleVersionsMatchVersionUtils() {
                 /* Java lists all versions from the 5.x series onwards, but we only want to consider
                  * ones that we're supposed to be compatible with. */
                 .filter(v -> v.onOrAfter(Version.CURRENT.minimumIndexCompatibilityVersion()))
-                /* Note that gradle skips alphas because they don't have any backwards
-                 * compatibility guarantees but keeps the last beta and rc in a branch
-                 * on when there are only betas an RCs in that branch so that we have
-                 * *something* to test that branch against. There is no need to recreate
-                 * that logic here because allUnreleasedVersions already only contains
-                 * the heads of branches so it should be good enough to just keep all
-                 * the non-alphas.*/
-                .filter(v -> false == v.isAlpha())
                 .map(Object::toString)
                 .collect(toCollection(LinkedHashSet::new)));
         assertEquals(unreleasedIndexCompatible, indexCompatible.unreleased);
diff --git a/test/framework/src/test/java/org/elasticsearch/test/disruption/DisruptableMockTransportTests.java b/test/framework/src/test/java/org/elasticsearch/test/disruption/DisruptableMockTransportTests.java
index 4060b7f5cd843..90a47d09e6ca9 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/disruption/DisruptableMockTransportTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/disruption/DisruptableMockTransportTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.disruption.DisruptableMockTransport.ConnectionStatus;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
@@ -53,6 +54,7 @@
 
 import static org.elasticsearch.transport.TransportService.NOOP_TRANSPORT_INTERCEPTOR;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.endsWith;
 
 public class DisruptableMockTransportTests extends ESTestCase {
 
@@ -399,4 +401,27 @@ public void testUnavailableOnRequestOnlyReceivesExceptionalResponse() throws IOE
         deterministicTaskQueue.runAllRunnableTasks();
         assertTrue(responseHandlerCalled.get());
     }
+
+    public void testBrokenLinkFailsToConnect() {
+        service1.disconnectFromNode(node2);
+
+        disconnectedLinks.add(Tuple.tuple(node1, node2));
+        assertThat(expectThrows(ConnectTransportException.class, () -> service1.connectToNode(node2)).getMessage(),
+            endsWith("is [DISCONNECTED] not [CONNECTED]"));
+        disconnectedLinks.clear();
+
+        blackholedLinks.add(Tuple.tuple(node1, node2));
+        assertThat(expectThrows(ConnectTransportException.class, () -> service1.connectToNode(node2)).getMessage(),
+            endsWith("is [BLACK_HOLE] not [CONNECTED]"));
+        blackholedLinks.clear();
+
+        blackholedRequestLinks.add(Tuple.tuple(node1, node2));
+        assertThat(expectThrows(ConnectTransportException.class, () -> service1.connectToNode(node2)).getMessage(),
+            endsWith("is [BLACK_HOLE_REQUESTS_ONLY] not [CONNECTED]"));
+        blackholedRequestLinks.clear();
+
+        final DiscoveryNode node3 = new DiscoveryNode("node3", buildNewFakeTransportAddress(), Version.CURRENT);
+        assertThat(expectThrows(ConnectTransportException.class, () -> service1.connectToNode(node3)).getMessage(),
+            endsWith("does not exist"));
+    }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java b/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java
index a2f6b3ed654e6..7f01d96740d77 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/disruption/NetworkDisruptionIT.java
@@ -19,25 +19,37 @@
 
 package org.elasticsearch.test.disruption;
 
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.disruption.NetworkDisruption.TwoPartitions;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportResponse;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false)
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class NetworkDisruptionIT extends ESIntegTestCase {
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
@@ -102,4 +114,80 @@ public void testNetworkPartitionRemovalRestoresConnections() throws Exception {
         }
     }
 
+    public void testTransportRespondsEventually() throws InterruptedException {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        internalCluster().ensureAtLeastNumDataNodes(randomIntBetween(3, 5));
+        final NetworkDisruption.DisruptedLinks disruptedLinks;
+        if (randomBoolean()) {
+            disruptedLinks = TwoPartitions.random(random(), internalCluster().getNodeNames());
+        } else {
+            disruptedLinks = NetworkDisruption.Bridge.random(random(), internalCluster().getNodeNames());
+        }
+
+        NetworkDisruption networkDisruption = new NetworkDisruption(disruptedLinks, randomFrom(new NetworkDisruption.NetworkUnresponsive(),
+            new NetworkDisruption.NetworkDisconnect(), NetworkDisruption.NetworkDelay.random(random())));
+        internalCluster().setDisruptionScheme(networkDisruption);
+
+        networkDisruption.startDisrupting();
+
+        int requests = randomIntBetween(1, 200);
+        CountDownLatch latch = new CountDownLatch(requests);
+        for (int i = 0; i < requests - 1; ++i) {
+            sendRequest(
+                internalCluster().getInstance(TransportService.class), internalCluster().getInstance(TransportService.class),
+                latch);
+        }
+
+        // send a request that is guaranteed disrupted.
+        Tuple<TransportService, TransportService> disruptedPair = findDisruptedPair(disruptedLinks);
+        sendRequest(disruptedPair.v1(), disruptedPair.v2(), latch);
+
+        // give a bit of time to send something under disruption.
+        assertFalse(latch.await(500, TimeUnit.MILLISECONDS)
+            && networkDisruption.getNetworkLinkDisruptionType() instanceof NetworkDisruption.NetworkDisconnect == false);
+        networkDisruption.stopDisrupting();
+
+        latch.await(30, TimeUnit.SECONDS);
+        assertEquals("All requests must respond, requests: " + requests, 0, latch.getCount());
+    }
+
+    private Tuple<TransportService, TransportService> findDisruptedPair(NetworkDisruption.DisruptedLinks disruptedLinks) {
+        Optional<Tuple<TransportService, TransportService>> disruptedPair = disruptedLinks.nodes().stream()
+            .flatMap(n1 -> disruptedLinks.nodes().stream().map(n2 -> Tuple.tuple(n1, n2)))
+            .filter(pair -> disruptedLinks.disrupt(pair.v1(), pair.v2()))
+            .map(pair -> Tuple.tuple(internalCluster().getInstance(TransportService.class, pair.v1()),
+                internalCluster().getInstance(TransportService.class, pair.v2())))
+            .findFirst();
+        // since we have 3+ nodes, we are sure to find a disrupted pair, also for bridge disruptions.
+        assertTrue(disruptedPair.isPresent());
+        return disruptedPair.get();
+    }
+
+    private void sendRequest(TransportService source, TransportService target, CountDownLatch latch) {
+        source.sendRequest(target.getLocalNode(), ClusterHealthAction.NAME, new ClusterHealthRequest(),
+            new TransportResponseHandler<>() {
+                private AtomicBoolean responded = new AtomicBoolean();
+                @Override
+                public void handleResponse(TransportResponse response) {
+                    assertTrue(responded.compareAndSet(false, true));
+                    latch.countDown();
+                }
+
+                @Override
+                public void handleException(TransportException exp) {
+                    assertTrue(responded.compareAndSet(false, true));
+                    latch.countDown();
+                }
+
+                @Override
+                public String executor() {
+                    return ThreadPool.Names.SAME;
+                }
+
+                @Override
+                public TransportResponse read(StreamInput in) throws IOException {
+                    return ClusterHealthResponse.readResponseFrom(in);
+                }
+            });
+    }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserFailingTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserFailingTests.java
index 8d1502577502e..3fd343a17e990 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserFailingTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserFailingTests.java
@@ -34,6 +34,7 @@ public void testDuplicateMethods() throws Exception {
        parseAndExpectFailure("{\n" +
                "  \"ping\": {" +
                "    \"documentation\": \"http://www.elasticsearch.org/guide/\"," +
+               "    \"stability\": \"stable\",\n" +
                "    \"methods\": [\"PUT\", \"PUT\"]," +
                "    \"url\": {" +
                "      \"path\": \"/\"," +
@@ -54,6 +55,7 @@ public void testDuplicatePaths() throws Exception {
         parseAndExpectFailure("{\n" +
                 "  \"ping\": {" +
                 "    \"documentation\": \"http://www.elasticsearch.org/guide/\"," +
+                "    \"stability\": \"stable\",\n" +
                 "    \"methods\": [\"PUT\"]," +
                 "    \"url\": {" +
                 "      \"path\": \"/pingone\"," +
@@ -94,6 +96,7 @@ private void parseAndExpectFailure(String brokenJson, String location, String ex
     private static final String BROKEN_SPEC_PARAMS = "{\n" +
             "  \"ping\": {" +
             "    \"documentation\": \"http://www.elasticsearch.org/guide/\"," +
+            "    \"stability\": \"stable\",\n" +
             "    \"methods\": [\"HEAD\"]," +
             "    \"url\": {" +
             "      \"path\": \"/\"," +
@@ -113,6 +116,7 @@ private void parseAndExpectFailure(String brokenJson, String location, String ex
     private static final String BROKEN_SPEC_PARTS = "{\n" +
             "  \"ping\": {" +
             "    \"documentation\": \"http://www.elasticsearch.org/guide/\"," +
+            "    \"stability\": \"stable\",\n" +
             "    \"methods\": [\"HEAD\"]," +
             "    \"url\": {" +
             "      \"path\": \"/\"," +
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java
index 9ba393966a363..f2211f8feacfe 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java
@@ -95,6 +95,7 @@ public void testRequiredBodyWithoutUrlParts() throws Exception {
         String spec = "{\n" +
             "  \"count\": {\n" +
             "    \"documentation\": \"whatever\",\n" +
+            "    \"stability\": \"stable\",\n" +
             "    \"methods\": [ \"GET\", \"POST\" ],\n" +
             "    \"url\": {\n" +
             "      \"path\": \"/whatever\",\n" +
@@ -119,6 +120,7 @@ public void testRequiredBodyWithoutUrlParts() throws Exception {
     private static final String REST_SPEC_COUNT_API = "{\n" +
             "  \"count\": {\n" +
             "    \"documentation\": \"http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-count.html\",\n" +
+            "    \"stability\": \"stable\",\n" +
             "    \"methods\": [\"POST\", \"GET\"],\n" +
             "    \"url\": {\n" +
             "      \"path\": \"/_count\",\n" +
@@ -149,6 +151,7 @@ public void testRequiredBodyWithoutUrlParts() throws Exception {
     private static final String REST_SPEC_GET_TEMPLATE_API = "{\n" +
             "  \"indices.get_template\": {\n" +
             "    \"documentation\": \"http://www.elasticsearch.org/guide/reference/api/admin-indices-templates/\",\n" +
+            "    \"stability\": \"stable\",\n" +
             "    \"methods\": [\"GET\"],\n" +
             "    \"url\": {\n" +
             "      \"path\": \"/_template/{name}\",\n" +
@@ -170,6 +173,7 @@ public void testRequiredBodyWithoutUrlParts() throws Exception {
     private static final String REST_SPEC_INDEX_API = "{\n" +
             "  \"index\": {\n" +
             "    \"documentation\": \"http://elasticsearch.org/guide/reference/api/index_/\",\n" +
+            "    \"stability\": \"stable\",\n" +
             "    \"methods\": [\"POST\", \"PUT\"],\n" +
             "    \"url\": {\n" +
             "      \"path\": \"/{index}/{type}\",\n" +
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterIT.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterIT.java
index 160103e8f2421..4e708975ef98c 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterIT.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterIT.java
@@ -18,8 +18,11 @@
  */
 package org.elasticsearch.test.test;
 
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.test.InternalTestCluster;
 
 import java.io.IOException;
 
@@ -61,4 +64,26 @@ public void testStoppingNodesOneByOne() throws IOException {
 
         ensureGreen();
     }
+
+    public void testOperationsDuringRestart() throws Exception {
+        internalCluster().startMasterOnlyNode();
+        internalCluster().startDataOnlyNodes(2);
+        internalCluster().restartRandomDataNode(new InternalTestCluster.RestartCallback() {
+            @Override
+            public Settings onNodeStopped(String nodeName) throws Exception {
+                ensureGreen();
+                internalCluster().validateClusterFormed();
+                assertNotNull(internalCluster().getInstance(NodeClient.class));
+                internalCluster().restartRandomDataNode(new InternalTestCluster.RestartCallback() {
+                    @Override
+                    public Settings onNodeStopped(String nodeName) throws Exception {
+                        ensureGreen();
+                        internalCluster().validateClusterFormed();
+                        return super.onNodeStopped(nodeName);
+                    }
+                });
+                return super.onNodeStopped(nodeName);
+            }
+        });
+    }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
index 6fffd246ec245..f7e0aa526290a 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
@@ -21,6 +21,7 @@
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
@@ -53,9 +54,6 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.cluster.node.DiscoveryNode.Role.DATA;
-import static org.elasticsearch.cluster.node.DiscoveryNode.Role.INGEST;
-import static org.elasticsearch.cluster.node.DiscoveryNode.Role.MASTER;
 import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileNotExists;
@@ -148,7 +146,6 @@ public void testBeforeTest() throws Exception {
             bootstrapMasterNodeIndex = maxNumDataNodes == 0 ? -1 : randomIntBetween(0, maxNumDataNodes - 1);
         }
         final int numClientNodes = randomIntBetween(0, 2);
-        String transportClient = getTestTransportType();
         NodeConfigurationSource nodeConfigurationSource = new NodeConfigurationSource() {
             @Override
             public Settings nodeSettings(int nodeOrdinal) {
@@ -167,12 +164,6 @@ public Settings nodeSettings(int nodeOrdinal) {
             public Path nodeConfigPath(int nodeOrdinal) {
                 return null;
             }
-
-            @Override
-            public Settings transportClientSettings() {
-                return Settings.builder()
-                    .put(NetworkModule.TRANSPORT_TYPE_KEY, transportClient).build();
-            }
         };
 
         String nodePrefix = "foobar";
@@ -220,7 +211,6 @@ public void testDataFolderAssignmentAndCleaning() throws IOException, Interrupte
         final int maxNumDataNodes = 2;
         final int numClientNodes = randomIntBetween(0, 2);
         final String clusterName1 = "shared1";
-        String transportClient = getTestTransportType();
         NodeConfigurationSource nodeConfigurationSource = new NodeConfigurationSource() {
             @Override
             public Settings nodeSettings(int nodeOrdinal) {
@@ -235,12 +225,6 @@ public Settings nodeSettings(int nodeOrdinal) {
             public Path nodeConfigPath(int nodeOrdinal) {
                 return null;
             }
-
-            @Override
-            public Settings transportClientSettings() {
-                return Settings.builder()
-                    .put(NetworkModule.TRANSPORT_TYPE_KEY, transportClient).build();
-            }
         };
         String nodePrefix = "test";
         Path baseDir = createTempDir();
@@ -311,7 +295,6 @@ public void testDifferentRolesMaintainPathOnRestart() throws Exception {
         final Path baseDir = createTempDir();
         final int numNodes = 5;
 
-        String transportClient = getTestTransportType();
         InternalTestCluster cluster = new InternalTestCluster(randomLong(), baseDir, false,
                 false, 0, 0, "test", new NodeConfigurationSource() {
 
@@ -329,41 +312,32 @@ public Settings nodeSettings(int nodeOrdinal) {
             public Path nodeConfigPath(int nodeOrdinal) {
                 return null;
             }
-
-            @Override
-            public Settings transportClientSettings() {
-                return Settings.builder()
-                        .put(NetworkModule.TRANSPORT_TYPE_KEY, transportClient).build();
-            }
         }, 0, "", mockPlugins(), Function.identity());
         cluster.beforeTest(random());
-        List<DiscoveryNode.Role> roles = new ArrayList<>();
+        List<DiscoveryNodeRole> roles = new ArrayList<>();
         for (int i = 0; i < numNodes; i++) {
-            final DiscoveryNode.Role role = i == numNodes - 1 && roles.contains(MASTER) == false ?
-                MASTER : // last node and still no master
-                randomFrom(MASTER, DiscoveryNode.Role.DATA, DiscoveryNode.Role.INGEST);
+            final DiscoveryNodeRole role = i == numNodes - 1 && roles.contains(DiscoveryNodeRole.MASTER_ROLE) == false ?
+                    DiscoveryNodeRole.MASTER_ROLE : // last node and still no master
+                randomFrom(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.INGEST_ROLE);
             roles.add(role);
         }
 
-        cluster.setBootstrapMasterNodeIndex(randomIntBetween(0, (int) roles.stream().filter(role -> role.equals(MASTER)).count() - 1));
+        cluster.setBootstrapMasterNodeIndex(
+                randomIntBetween(0, (int) roles.stream().filter(role -> role.equals(DiscoveryNodeRole.MASTER_ROLE)).count() - 1));
 
         try {
-            Map<DiscoveryNode.Role, Set<String>> pathsPerRole = new HashMap<>();
+            Map<DiscoveryNodeRole, Set<String>> pathsPerRole = new HashMap<>();
             for (int i = 0; i < numNodes; i++) {
-                final DiscoveryNode.Role role = roles.get(i);
+                final DiscoveryNodeRole role = roles.get(i);
                 final String node;
-                switch (role) {
-                    case MASTER:
-                        node = cluster.startMasterOnlyNode();
-                        break;
-                    case DATA:
-                        node = cluster.startDataOnlyNode();
-                        break;
-                    case INGEST:
-                        node = cluster.startCoordinatingOnlyNode(Settings.EMPTY);
-                        break;
-                    default:
-                        throw new IllegalStateException("get your story straight");
+                if (role == DiscoveryNodeRole.MASTER_ROLE) {
+                    node = cluster.startMasterOnlyNode();
+                } else if (role == DiscoveryNodeRole.DATA_ROLE) {
+                    node = cluster.startDataOnlyNode();
+                } else if (role == DiscoveryNodeRole.INGEST_ROLE) {
+                    node = cluster.startCoordinatingOnlyNode(Settings.EMPTY);
+                } else {
+                    throw new IllegalStateException("get your story straight");
                 }
                 Set<String> rolePaths = pathsPerRole.computeIfAbsent(role, k -> new HashSet<>());
                 for (Path path : getNodePaths(cluster, node)) {
@@ -373,21 +347,21 @@ public Settings transportClientSettings() {
             cluster.validateClusterFormed();
             cluster.fullRestart();
 
-            Map<DiscoveryNode.Role, Set<String>> result = new HashMap<>();
+            Map<DiscoveryNodeRole, Set<String>> result = new HashMap<>();
             for (String name : cluster.getNodeNames()) {
                 DiscoveryNode node = cluster.getInstance(ClusterService.class, name).localNode();
                 List<String> paths = Arrays.stream(getNodePaths(cluster, name)).map(Path::toString).collect(Collectors.toList());
                 if (node.isMasterNode()) {
-                    result.computeIfAbsent(MASTER, k -> new HashSet<>()).addAll(paths);
+                    result.computeIfAbsent(DiscoveryNodeRole.MASTER_ROLE, k -> new HashSet<>()).addAll(paths);
                 } else if (node.isDataNode()) {
-                    result.computeIfAbsent(DATA, k -> new HashSet<>()).addAll(paths);
+                    result.computeIfAbsent(DiscoveryNodeRole.DATA_ROLE, k -> new HashSet<>()).addAll(paths);
                 } else {
-                    result.computeIfAbsent(INGEST, k -> new HashSet<>()).addAll(paths);
+                    result.computeIfAbsent(DiscoveryNodeRole.INGEST_ROLE, k -> new HashSet<>()).addAll(paths);
                 }
             }
 
             assertThat(result.size(), equalTo(pathsPerRole.size()));
-            for (DiscoveryNode.Role role : result.keySet()) {
+            for (DiscoveryNodeRole role : result.keySet()) {
                 assertThat("path are not the same for " + role, result.get(role), equalTo(pathsPerRole.get(role)));
             }
         } finally {
@@ -396,7 +370,6 @@ public Settings transportClientSettings() {
     }
 
     public void testTwoNodeCluster() throws Exception {
-        String transportClient = getTestTransportType();
         NodeConfigurationSource nodeConfigurationSource = new NodeConfigurationSource() {
             @Override
             public Settings nodeSettings(int nodeOrdinal) {
@@ -411,12 +384,6 @@ public Settings nodeSettings(int nodeOrdinal) {
             public Path nodeConfigPath(int nodeOrdinal) {
                 return null;
             }
-
-            @Override
-            public Settings transportClientSettings() {
-                return Settings.builder()
-                    .put(NetworkModule.TRANSPORT_TYPE_KEY, transportClient).build();
-            }
         };
         String nodePrefix = "test";
         Path baseDir = createTempDir();
diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/TestEventHandlerTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/TestEventHandlerTests.java
index 424d4922f024e..d8bb790b2a34b 100644
--- a/test/framework/src/test/java/org/elasticsearch/transport/nio/TestEventHandlerTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/TestEventHandlerTests.java
@@ -23,6 +23,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.nio.ServerChannelContext;
 import org.elasticsearch.nio.SocketChannelContext;
 import org.elasticsearch.test.ESTestCase;
@@ -57,7 +58,7 @@ public void tearDown() throws Exception {
 
     public void testLogOnElapsedTime() throws Exception {
         long start = System.nanoTime();
-        long end = start + TimeUnit.MILLISECONDS.toNanos(200);
+        long end = start + TimeUnit.MILLISECONDS.toNanos(400);
         AtomicBoolean isStart = new AtomicBoolean(true);
         LongSupplier timeSupplier = () -> {
             if (isStart.compareAndSet(true, false)) {
@@ -70,7 +71,7 @@ public void testLogOnElapsedTime() throws Exception {
         final ThreadPool threadPool = mock(ThreadPool.class);
         doAnswer(i -> timeSupplier.getAsLong()).when(threadPool).relativeTimeInNanos();
         TestEventHandler eventHandler =
-            new TestEventHandler((e) -> {}, () -> null, new MockNioTransport.TransportThreadWatchdog(threadPool));
+            new TestEventHandler(e -> {}, () -> null, new MockNioTransport.TransportThreadWatchdog(threadPool, Settings.EMPTY));
 
         ServerChannelContext serverChannelContext = mock(ServerChannelContext.class);
         SocketChannelContext socketChannelContext = mock(SocketChannelContext.class);
diff --git a/test/logger-usage/build.gradle b/test/logger-usage/build.gradle
index 80cb060167114..f949e2a162931 100644
--- a/test/logger-usage/build.gradle
+++ b/test/logger-usage/build.gradle
@@ -22,7 +22,7 @@ dependencies {
   compile 'org.ow2.asm:asm-tree:7.1'
   compile 'org.ow2.asm:asm-analysis:7.1'
   compile "org.apache.logging.log4j:log4j-api:${versions.log4j}"
-  testCompile "org.elasticsearch.test:framework:${version}"
+  testCompile project(":test:framework")
 }
 
 loggerUsageCheck.enabled = false
diff --git a/x-pack/build.gradle b/x-pack/build.gradle
index 9c90bbbbfc269..f9b13f07618ce 100644
--- a/x-pack/build.gradle
+++ b/x-pack/build.gradle
@@ -26,17 +26,3 @@ subprojects {
   project.ext.licenseFile = rootProject.file('licenses/ELASTIC-LICENSE.txt')
   project.ext.noticeFile = xpackRootProject.file('NOTICE.txt')
 }
-
-subprojects {
-  ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-ccr:${version}": xpackModule('ccr')]
-  ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-core:${version}": xpackModule('core')]
-  ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-deprecation:${version}": xpackModule('deprecation')]
-  ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-graph:${version}": xpackModule('graph')]
-  ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-ilm:${version}": xpackModule('ilm')]
-  ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-logstash:${version}": xpackModule('logstash')]
-  ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-ml:${version}": xpackModule('ml')]
-  ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-monitoring:${version}": xpackModule('monitoring')]
-  ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-security:${version}": xpackModule('security')]
-  ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-sql:${version}": xpackModule('sql')]
-  ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-watcher:${version}": xpackModule('watcher')]
-}
diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle
index 0a23bb9c9cf62..fe07450bbc10e 100644
--- a/x-pack/docs/build.gradle
+++ b/x-pack/docs/build.gradle
@@ -18,7 +18,6 @@ buildRestTests.expectedUnconvertedCandidates = [
 ]
 
 dependencies {
-    // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
     testCompile project(path: xpackModule('core'), configuration: 'default')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     if (isEclipse) {
diff --git a/x-pack/docs/en/rest-api/security.asciidoc b/x-pack/docs/en/rest-api/security.asciidoc
index abad1e38d77fd..059dbc1e74716 100644
--- a/x-pack/docs/en/rest-api/security.asciidoc
+++ b/x-pack/docs/en/rest-api/security.asciidoc
@@ -8,6 +8,7 @@ You can use the following APIs to perform security activities.
 * <<security-api-clear-cache>>
 * <<security-api-has-privileges>>
 * <<security-api-ssl>>
+* <<security-api-get-builtin-privileges>>
 
 [float]
 [[security-api-app-privileges]]
@@ -105,6 +106,7 @@ include::security/disable-users.asciidoc[]
 include::security/enable-users.asciidoc[]
 include::security/get-api-keys.asciidoc[]
 include::security/get-app-privileges.asciidoc[]
+include::security/get-builtin-privileges.asciidoc[]
 include::security/get-role-mappings.asciidoc[]
 include::security/get-roles.asciidoc[]
 include::security/get-tokens.asciidoc[]
diff --git a/x-pack/docs/en/rest-api/security/create-api-keys.asciidoc b/x-pack/docs/en/rest-api/security/create-api-keys.asciidoc
index 3b151d3d47d65..5fda7c5c3e488 100644
--- a/x-pack/docs/en/rest-api/security/create-api-keys.asciidoc
+++ b/x-pack/docs/en/rest-api/security/create-api-keys.asciidoc
@@ -37,17 +37,21 @@ The following parameters can be specified in the body of a POST or PUT request:
 `name`::
 (string) Specifies the name for this API key.
 
-`role_descriptors` (required)::
+`role_descriptors` (optional)::
 (array-of-role-descriptor) An array of role descriptors for this API key. This
-parameter is required but can be an empty array, which applies the permissions
-of the authenticated user. If you supply role descriptors, they must be a subset
-of the authenticated user's permissions. The structure of role descriptor is the
+parameter is optional. When it is not specified or is an empty array, then the API key will have
+the permissions of the authenticated user. If you supply role descriptors, they must
+be a subset of the authenticated user's permissions. The structure of role descriptor is the
 same as the request for create role API. For more details, see
 <<security-api-roles,role management APIs>>.
 
 `expiration`::
 (string) Optional expiration time for the API key. By default, API keys never expire.
 
+==== Authorization
+
+To use this API, you must have at least the `manage_api_key` cluster privilege.
+
 ==== Examples
 
 The following example creates an API key:
diff --git a/x-pack/docs/en/rest-api/security/get-api-keys.asciidoc b/x-pack/docs/en/rest-api/security/get-api-keys.asciidoc
index 0e5865c3cd9ae..4d7084e1ece26 100644
--- a/x-pack/docs/en/rest-api/security/get-api-keys.asciidoc
+++ b/x-pack/docs/en/rest-api/security/get-api-keys.asciidoc
@@ -39,6 +39,10 @@ or `name`.
 
 NOTE: While all parameters are optional, at least one of them is required.
 
+==== Authorization
+
+To use this API, you must have at least the `manage_api_key` cluster privilege.
+
 ==== Examples
 
 If you create an API key as follows:
diff --git a/x-pack/docs/en/rest-api/security/get-builtin-privileges.asciidoc b/x-pack/docs/en/rest-api/security/get-builtin-privileges.asciidoc
new file mode 100644
index 0000000000000..0ace996f96d71
--- /dev/null
+++ b/x-pack/docs/en/rest-api/security/get-builtin-privileges.asciidoc
@@ -0,0 +1,107 @@
+[role="xpack"]
+[[security-api-get-builtin-privileges]]
+=== Get builtin privileges API
+++++
+<titleabbrev>Get builtin privileges</titleabbrev>
+++++
+
+Retrieves the list of 
+{stack-ov}/security-privileges.html#privileges-list-cluster[cluster privileges] and
+{stack-ov}/security-privileges.html#privileges-list-indices[index privileges] that are
+available in this version of {es}.
+
+==== Request
+
+`GET /_security/privilege/_builtin` 
+
+
+==== Description
+
+This API retrieves the set of cluster and index privilege names that are available in the
+version of {es} that is being queried.
+
+To check whether a user has particular privileges, use the
+<<security-api-has-privileges,has privileges API>>.
+
+==== Results
+
+The response is an object with two fields:
+
+`cluster`:: (array of string) The list of
+ {stack-ov}/security-privileges.html#privileges-list-cluster[cluster privileges] that are
+ understood by this version of {es}
+
+`index`:: (array of string) The list of
+ {stack-ov}/security-privileges.html#privileges-list-indices[index privileges] that are
+ understood by this version of {es}
+
+==== Authorization
+
+To use this API, you must have - the `manage_security` cluster privilege
+(or a greater privilege such as `all`)
+
+==== Examples
+
+The following example retrieves the names of all builtin privileges:
+
+[source,js]
+--------------------------------------------------
+GET /_security/privilege/_builtin
+--------------------------------------------------
+// CONSOLE
+// TEST
+
+A successful call returns an object with "cluster" and "index" fields.
+
+[source,js]
+--------------------------------------------------
+{
+  "cluster" : [
+    "all",
+    "create_snapshot",
+    "manage",
+    "manage_api_key",
+    "manage_ccr",
+    "manage_data_frame_transforms",
+    "manage_ilm",
+    "manage_index_templates",
+    "manage_ingest_pipelines",
+    "manage_ml",
+    "manage_oidc",
+    "manage_pipeline",
+    "manage_rollup",
+    "manage_saml",
+    "manage_security",
+    "manage_token",
+    "manage_watcher",
+    "monitor",
+    "monitor_data_frame_transforms",
+    "monitor_ml",
+    "monitor_rollup",
+    "monitor_watcher",
+    "none",
+    "read_ccr",
+    "read_ilm",
+    "transport_client"
+  ],
+  "index" : [
+    "all",
+    "create",
+    "create_index",
+    "delete",
+    "delete_index",
+    "index",
+    "manage",
+    "manage_follow_index",
+    "manage_ilm",
+    "manage_leader_index",
+    "monitor",
+    "none",
+    "read",
+    "read_cross_cluster",
+    "view_index_metadata",
+    "write"
+  ]
+}
+--------------------------------------------------
+// TESTRESPONSE
diff --git a/x-pack/docs/en/rest-api/security/get-tokens.asciidoc b/x-pack/docs/en/rest-api/security/get-tokens.asciidoc
index cba3b638bc10c..44450dd43c806 100644
--- a/x-pack/docs/en/rest-api/security/get-tokens.asciidoc
+++ b/x-pack/docs/en/rest-api/security/get-tokens.asciidoc
@@ -41,14 +41,21 @@ The following parameters can be specified in the body of a POST request and
 pertain to creating a token:
 
 `grant_type`::
-(string) The type of grant. Supported grant types are: `password`,
-`client_credentials` and `refresh_token`.
+(string) The type of grant. Supported grant types are: `password`, `_kerberos`,
+`client_credentials` and `refresh_token`. The `_kerberos` grant type
+is supported internally and implements SPNEGO based Kerberos support. The `_kerberos`
+grant type may change from version to version.
 
 `password`::
 (string) The user's password. If you specify the `password` grant type, this 
 parameter is required. This parameter is not valid with any other supported
 grant type.
 
+`kerberos_ticket`::
+(string) base64 encoded kerberos ticket. If you specify the `_kerberos` grant type,
+this parameter is required. This parameter is not valid with any other supported
+grant type.
+
 `refresh_token`::
 (string) If you specify the `refresh_token` grant type, this parameter is 
 required. It contains the string that was returned when you created the token 
@@ -160,4 +167,34 @@ be used one time.
 }
 --------------------------------------------------
 // TESTRESPONSE[s/dGhpcyBpcyBub3QgYSByZWFsIHRva2VuIGJ1dCBpdCBpcyBvbmx5IHRlc3QgZGF0YS4gZG8gbm90IHRyeSB0byByZWFkIHRva2VuIQ==/$body.access_token/]
-// TESTRESPONSE[s/vLBPvmAB6KvwvJZr27cS/$body.refresh_token/]
\ No newline at end of file
+// TESTRESPONSE[s/vLBPvmAB6KvwvJZr27cS/$body.refresh_token/]
+
+The following example obtains a access token and refresh token using the `kerberos` grant type,
+which simply creates a token in exchange for the base64 encoded kerberos ticket:
+
+[source,js]
+--------------------------------------------------
+POST /_security/oauth2/token
+{
+  "grant_type" : "_kerberos",
+  "kerberos_ticket" : "YIIB6wYJKoZIhvcSAQICAQBuggHaMIIB1qADAgEFoQMCAQ6iBtaDcp4cdMODwOsIvmvdX//sye8NDJZ8Gstabor3MOGryBWyaJ1VxI4WBVZaSn1WnzE06Xy2"
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+The API will return a new token and refresh token if kerberos authentication is successful.
+Each refresh token may only be used one time. When the mutual authentication is requested in the Spnego GSS context,
+ a base64 encoded token will be returned by the server in the `kerberos_authentication_response_token`
+ for clients to consume and finalize the authentication.
+
+[source,js]
+--------------------------------------------------
+{
+  "access_token" : "dGhpcyBpcyBub3QgYSByZWFsIHRva2VuIGJ1dCBpdCBpcyBvbmx5IHRlc3QgZGF0YS4gZG8gbm90IHRyeSB0byByZWFkIHRva2VuIQ==",
+  "type" : "Bearer",
+  "expires_in" : 1200,
+  "refresh_token": "vLBPvmAB6KvwvJZr27cS"
+  "kerberos_authentication_response_token": "YIIB6wYJKoZIhvcSAQICAQBuggHaMIIB1qADAg"
+}
+--------------------------------------------------
+// NOTCONSOLE
\ No newline at end of file
diff --git a/x-pack/docs/en/rest-api/security/invalidate-api-keys.asciidoc b/x-pack/docs/en/rest-api/security/invalidate-api-keys.asciidoc
index 8e496fb58664f..5e6351683b752 100644
--- a/x-pack/docs/en/rest-api/security/invalidate-api-keys.asciidoc
+++ b/x-pack/docs/en/rest-api/security/invalidate-api-keys.asciidoc
@@ -31,14 +31,18 @@ pertain to invalidating api keys:
 
 `realm_name` (optional)::
 (string) The name of an authentication realm. This parameter cannot be used with
-either `api_key_id` or `api_key_name`.
+either `id` or `name`.
 
 `username` (optional)::
 (string) The username of a user. This parameter cannot be used with either
-`api_key_id` or `api_key_name`.
+`id` or `name`.
 
 NOTE: While all parameters are optional, at least one of them is required.
 
+==== Authorization
+
+To use this API, you must have at least the `manage_api_key` cluster privilege.
+
 ==== Examples
 
 If you create an API key as follows:
@@ -47,8 +51,7 @@ If you create an API key as follows:
 ------------------------------------------------------------
 POST /_security/api_key
 {
-  "name": "my-api-key",
-  "role_descriptors": {}
+  "name": "my-api-key"
 }
 ------------------------------------------------------------
 // CONSOLE
diff --git a/x-pack/docs/en/security/authentication/oidc-guide.asciidoc b/x-pack/docs/en/security/authentication/oidc-guide.asciidoc
index df5ce11c63c14..d7d68ada0e721 100644
--- a/x-pack/docs/en/security/authentication/oidc-guide.asciidoc
+++ b/x-pack/docs/en/security/authentication/oidc-guide.asciidoc
@@ -103,12 +103,13 @@ xpack.security.authc.realms.oidc.oidc1:
   rp.client_id: "the_client_id"
   rp.response_type: code
   rp.redirect_uri: "https://kibana.example.org:5601/api/security/v1/oidc"
+  op.issuer: "https://op.example.org"
   op.authorization_endpoint: "https://op.example.org/oauth2/v1/authorize"
   op.token_endpoint: "https://op.example.org/oauth2/v1/token"
+  op.jwkset_path: oidc/jwkset.json
   op.userinfo_endpoint: "https://op.example.org/oauth2/v1/userinfo"
   op.endsession_endpoint: "https://op.example.org/oauth2/v1/logout"
-  op.issuer: "https://op.example.org"
-  op.jwkset_path: oidc/jwkset.json
+  rp.post_logout_redirect_uri: "https://kibana.example.org:5601/logged_out"
   claims.principal: sub
   claims.groups: "http://example.info/claims/groups"
 -------------------------------------------------------------------------------------
@@ -146,6 +147,10 @@ rp.redirect_uri::
     _exactly_ the same as the one <<oidc-guide-op, configured with the OP upon registration>> and will
     typically be +$\{kibana-url}/api/security/v1/oidc+ where _$\{kibana-url}_ is the base URL for your {kib} instance
 
+op.issuer::
+    A verifiable Identifier for your OpenID Connect Provider. An Issuer Identifier is usually a case sensitive URL.
+    The value for this setting should be provided by your OpenID Connect Provider.
+
 op.authorization_endpoint::
     The URL for the Authorization Endpoint in the OP. This is where the user's browser
     will be redirected to start the authentication process. The value for this setting should be provided by your
@@ -156,6 +161,13 @@ op.token_endpoint::
     {es} will send a request to exchange the code for an ID Token, in the case where the Authorization Code
     flow is used. The value for this setting should be provided by your OpenID Connect Provider.
 
+op.jwkset_path::
+    The path to a file or a URL containing a JSON Web Key Set with the key material that the OpenID Connect
+    Provider uses for signing tokens and claims responses. If a path is set, it is resolved relative to the {es}
+    config directory.
+    {es} will automatically monitor this file for changes and will reload the configuration whenever
+    it is updated. Your OpenID Connect Provider should provide you with this file or a URL where it is available.
+
 op.userinfo_endpoint::
     (Optional) The URL for the UserInfo Endpoint in the OpenID Connect Provider. This is the endpoint of the OP that
     can be queried to get further user information, if required. The value for this setting should be provided by your
@@ -166,12 +178,11 @@ op.endsession_endpoint::
     browser will be redirected after local logout, if the realm is configured for RP initiated Single Logout and
     the OP supports it. The value for this setting should be provided by your OpenID Connect Provider.
 
-op.jwkset_path::
-    The path to a file containing a JSON Web Key Set with the key material that the OpenID Connect
-    Provider uses for signing tokens and claims responses. The path is resolved relative to the {es}
-    config directory.
-    {es} will automatically monitor this file for changes and will reload the configuration whenever
-    it is updated. Your OpenID Connect Provider should provide you with this file.
+rp.post_logout_redirect_uri::
+    (Optional) The Redirect URL where the OpenID Connect Provider should redirect the user after a
+    successful Single Logout (assuming `op.endsession_endpoint` above is also set). This should be set to a value that
+    will not trigger a new OpenID Connect Authentication, such as +$\{kibana-url}/logged_out+ where _$\{kibana-url}_ is
+    the base URL for your {kib} instance.
 
 claims.principal:: See <<oidc-claims-mapping>>.
 claims.groups:: See <<oidc-claims-mapping>>.
@@ -306,6 +317,7 @@ realm, as demonstrated in the realm configuration below:
 [source, yaml]
 -------------------------------------------------------------------------------------
 xpack.security.authc.realms.oidc.oidc1:
+  order: 2
   rp.client_id: "the_client_id"
   rp.response_type: code
   rp.redirect_uri: "https://kibana.example.org:5601/api/security/v1/oidc"
@@ -369,6 +381,30 @@ will trigger re-authentication of the user. For instance, when using OpenID Conn
 single sign-on to {kib}, this could be set to +$\{kibana-url}/logged_out+,  which will show a user-
 friendly message to the user.
 
+[[oidc-ssl-config]]
+==== OpenID Connect Realm SSL Configuration
+
+OpenID Connect depends on TLS to provide security properties such as encryption in transit and endpoint authentication. The RP
+is required to establish back-channel communication with the OP in order to exchange the code for an ID Token during the
+Authorization code grant flow and in order to get additional user information from the UserInfo endpoint. Furthermore, if
+you configure `op.jwks_path` as a URL, {es} will need to get the OP's signing keys from the file hosted there. As such, it is
+important that {es} can validate and trust the server certificate that the OP uses for TLS. Since the system truststore is
+used for the client context of outgoing https connections, if your OP is using a certificate from a trusted CA, no additional
+configuration is needed.
+
+However, if the issuer of your OP's certificate is not trusted by the JVM on which {es} is running (e.g it uses a organization CA), then you must configure
+{es} to trust that CA. Assuming that you have the CA certificate that has signed the certificate that the OP uses for TLS
+stored in the /oidc/company-ca.pem` file stored in the configuration directory of {es}, you need to set the following
+property in the realm configuration:
+
+[source, yaml]
+-------------------------------------------------------------------------------------
+xpack.security.authc.realms.oidc.oidc1:
+  order: 1
+  ...
+  ssl.certificate_authorities: ["/oidc/company-ca.pem"]
+-------------------------------------------------------------------------------------
+
 [[oidc-role-mapping]]
 === Configuring role mappings
 
diff --git a/x-pack/docs/en/security/ccs-clients-integrations/cross-cluster.asciidoc b/x-pack/docs/en/security/ccs-clients-integrations/cross-cluster.asciidoc
index b72afcb9b011c..6912da6416e74 100644
--- a/x-pack/docs/en/security/ccs-clients-integrations/cross-cluster.asciidoc
+++ b/x-pack/docs/en/security/ccs-clients-integrations/cross-cluster.asciidoc
@@ -52,10 +52,10 @@ PUT _cluster/settings
   "persistent": {
     "cluster": {
       "remote": {
-        "cluster_one": {
+        "one": {
           "seeds": [ "10.0.1.1:9300" ]
         },
-        "cluster_two": {
+        "two": {
           "seeds": [ "10.0.2.1:9300" ]
         }
       }
@@ -83,7 +83,7 @@ First, enable cluster `one` to perform cross cluster search on remote cluster
 PUT _cluster/settings
 {
   "persistent": {
-    "cluster.remote.cluster_two.seeds": [ "10.0.2.1:9300" ]
+    "cluster.remote.two.seeds": [ "10.0.2.1:9300" ]
   }
 }
 -----------------------------------------------------------
diff --git a/x-pack/docs/en/security/configuring-es.asciidoc b/x-pack/docs/en/security/configuring-es.asciidoc
index fdc49ef21e213..94a295aa5640f 100644
--- a/x-pack/docs/en/security/configuring-es.asciidoc
+++ b/x-pack/docs/en/security/configuring-es.asciidoc
@@ -148,9 +148,6 @@ include::{es-repo-dir}/security/securing-communications/configuring-tls-docker.a
 :edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/docs/reference/security/securing-communications/enabling-cipher-suites.asciidoc
 include::{es-repo-dir}/security/securing-communications/enabling-cipher-suites.asciidoc[]
 
-:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/docs/reference/security/securing-communications/separating-node-client-traffic.asciidoc
-include::{es-repo-dir}/security/securing-communications/separating-node-client-traffic.asciidoc[]
-
 :edit_url:
 include::authentication/configuring-active-directory-realm.asciidoc[]
 include::authentication/configuring-file-realm.asciidoc[]
diff --git a/x-pack/docs/en/security/securing-communications.asciidoc b/x-pack/docs/en/security/securing-communications.asciidoc
index 6672c0316493e..3bb9793477b98 100644
--- a/x-pack/docs/en/security/securing-communications.asciidoc
+++ b/x-pack/docs/en/security/securing-communications.asciidoc
@@ -24,8 +24,3 @@ include::{es-repo-dir}/security/securing-communications/setting-up-ssl.asciidoc[
 === Enabling cipher suites for stronger encryption
 
 See {ref}/ciphers.html[Enabling Cipher Suites for Stronger Encryption].
-
-[[separating-node-client-traffic]]
-=== Separating node-to-node and client traffic
-
-See {ref}/separating-node-client-traffic.html[Separating node-to-node and client traffic].
diff --git a/x-pack/docs/en/watcher/actions.asciidoc b/x-pack/docs/en/watcher/actions.asciidoc
index 34bdd2513a4d4..7f5cd3218b5d3 100644
--- a/x-pack/docs/en/watcher/actions.asciidoc
+++ b/x-pack/docs/en/watcher/actions.asciidoc
@@ -192,6 +192,49 @@ of a watch during its execution:
 image::images/action-throttling.jpg[align="center"]
 
 
+[[action-foreach]]
+=== Running an action for each element in an array
+
+You can use the `foreach` field in an action to trigger the configured action
+for every element within that array.
+
+In order to protect from long running watches, after one hundred runs with an
+foreach loop the execution is gracefully stopped.
+
+[source,js]
+--------------------------------------------------
+PUT _watcher/watch/log_event_watch
+{
+  "trigger" : {
+    "schedule" : { "interval" : "5m" }
+  },
+  "input" : {
+    "search" : {
+      "request" : {
+        "indices" : "log-events",
+        "body" : {
+          "query" : { "match" : { "status" : "error" } }
+        }
+      }
+    }
+  },
+  "condition" : {
+    "compare" : { "ctx.payload.hits.total" : { "gt" : 0 } }
+  },
+  "actions" : {
+    "log_hits" : {
+      "foreach" : "ctx.payload.hits.hits", <1>
+      "logging" : {
+        "text" : "Found id {{ctx.payload._id}} with field {{ctx.payload._source.my_field}}"
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+<1> The logging statement will be executed for each of the returned search hits.
+
 [[action-conditions]]
 === Adding conditions to actions
 
diff --git a/x-pack/docs/en/watcher/condition/array-compare.asciidoc b/x-pack/docs/en/watcher/condition/array-compare.asciidoc
index 0a7966f0b6fb0..e82eaf5384061 100644
--- a/x-pack/docs/en/watcher/condition/array-compare.asciidoc
+++ b/x-pack/docs/en/watcher/condition/array-compare.asciidoc
@@ -27,7 +27,7 @@ than or equal to 25:
       "ctx.payload.aggregations.top_tweeters.buckets" : { <1>
         "path": "doc_count", <2>
         "gte": { <3>
-          "value": 25, <4>
+          "value": 25 <4>
         }
       }
     }
diff --git a/x-pack/docs/en/watcher/how-watcher-works.asciidoc b/x-pack/docs/en/watcher/how-watcher-works.asciidoc
index 80aeb69a38d5b..5ecc5b41ec61c 100644
--- a/x-pack/docs/en/watcher/how-watcher-works.asciidoc
+++ b/x-pack/docs/en/watcher/how-watcher-works.asciidoc
@@ -149,14 +149,14 @@ primary and all replicas of this particular shard will reload.
 Because the watches are executed on the node, where the watch shards are, you can create
 dedicated watcher nodes by using shard allocation filtering.
 
-You could configure nodes with a dedicated `node.attr.watcher: true` property and
+You could configure nodes with a dedicated `node.attr.role: watcher` property and
 then configure the `.watches` index like this:
 
 [source,js]
 ------------------------
 PUT .watches/_settings
 {
-  "index.routing.allocation.include": "watcher"
+  "index.routing.allocation.include.role": "watcher"
 }
 ------------------------
 // CONSOLE
@@ -442,4 +442,4 @@ references the `email_notification_subject` template:
   }
 }
 ----------------------------------------------------------------------
-// NOTCONSOLE
\ No newline at end of file
+// NOTCONSOLE
diff --git a/x-pack/license-tools/build.gradle b/x-pack/license-tools/build.gradle
index 4bd17713a2fea..2a032dc0cd15a 100644
--- a/x-pack/license-tools/build.gradle
+++ b/x-pack/license-tools/build.gradle
@@ -1,9 +1,9 @@
 apply plugin: 'elasticsearch.build'
 
 dependencies {
-  compile "org.elasticsearch.plugin:x-pack-core:${version}"
-  compile "org.elasticsearch:elasticsearch:${version}"
-  testCompile "org.elasticsearch.test:framework:${version}"
+  compile project(':x-pack:plugin:core')
+  compile project(':server')
+  testCompile project(':test:framework')
 }
 
 project.forbiddenPatterns {
diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle
index fc5dc839ef05e..630b43ed71e58 100644
--- a/x-pack/plugin/build.gradle
+++ b/x-pack/plugin/build.gradle
@@ -4,6 +4,7 @@ import org.elasticsearch.gradle.test.NodeInfo
 
 import java.nio.charset.StandardCharsets
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -22,19 +23,16 @@ subprojects {
       // see the root Gradle file for additional logic regarding this configuration
       project.configurations.create('featureAwarePlugin')
       project.dependencies.add('featureAwarePlugin', project.configurations.compileClasspath)
-      project.dependencies.add(
-              'featureAwarePlugin',
-              "org.elasticsearch.xpack.test:feature-aware:${org.elasticsearch.gradle.VersionProperties.elasticsearch}")
+      project.dependencies.add('featureAwarePlugin', project(':x-pack:test:feature-aware'))
       project.dependencies.add('featureAwarePlugin', project.sourceSets.main.output.getClassesDirs())
 
-      final Task featureAwareTask = project.tasks.create("featureAwareCheck", LoggedExec) {
+      File successMarker = file("$buildDir/markers/featureAware")
+      task featureAwareCheck(type: LoggedExec) {
         description = "Runs FeatureAwareCheck on main classes."
         dependsOn project.configurations.featureAwarePlugin
-
-        final File successMarker = new File(project.buildDir, 'markers/featureAware')
         outputs.file(successMarker)
 
-        executable = new File(project.runtimeJavaHome, 'bin/java')
+        executable = "${project.runtimeJavaHome}/bin/java"
 
         // default to main class files if such a source set exists
         final List files = []
@@ -55,7 +53,7 @@ subprojects {
         }
       }
 
-      project.precommit.dependsOn featureAwareTask
+      project.precommit.dependsOn featureAwareCheck
     }
   }
 }
@@ -87,28 +85,8 @@ artifacts {
   testArtifacts testJar
 }
 
-integTestRunner {
-  /*
-   * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
-   * other if we allow them to set the number of available processors as it's set-once in Netty.
-   */
-  systemProperty 'es.set.netty.runtime.available.processors', 'false'
-
-
-  // TODO: fix this rest test to not depend on a hardcoded port!
-  def blacklist = ['getting_started/10_monitor_cluster_health/*']
-  boolean snapshot = "true".equals(System.getProperty("build.snapshot", "true"))
-  if (!snapshot) {
-    // these tests attempt to install basic/internal licenses signed against the dev/public.key
-    // Since there is no infrastructure in place (anytime soon) to generate licenses using the production
-    // private key, these tests are whitelisted in non-snapshot test runs
-    blacklist.addAll(['xpack/15_basic/*', 'license/20_put_license/*'])
-  }
-  systemProperty 'tests.rest.blacklist', blacklist.join(',')
-}
-
 // location for keys and certificates
-File keystoreDir = new File(project.buildDir, 'keystore')
+File keystoreDir = file("$buildDir/keystore")
 File nodeKey = file("$keystoreDir/testnode.pem")
 File nodeCert = file("$keystoreDir/testnode.crt")
 
@@ -127,11 +105,31 @@ task copyKeyCerts(type: Copy) {
 sourceSets.test.resources.srcDir(keystoreDir)
 processTestResources.dependsOn(copyKeyCerts)
 
-integTestCluster {
+integTest.runner {
+  /*
+   * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
+   * other if we allow them to set the number of available processors as it's set-once in Netty.
+   */
+  systemProperty 'es.set.netty.runtime.available.processors', 'false'
+
+
+  // TODO: fix this rest test to not depend on a hardcoded port!
+  def blacklist = ['getting_started/10_monitor_cluster_health/*']
+  boolean snapshot = Boolean.valueOf(System.getProperty("build.snapshot", "true"))
+  if (!snapshot) {
+    // these tests attempt to install basic/internal licenses signed against the dev/public.key
+    // Since there is no infrastructure in place (anytime soon) to generate licenses using the production
+    // private key, these tests are whitelisted in non-snapshot test runs
+    blacklist.addAll(['xpack/15_basic/*', 'license/20_put_license/*'])
+  }
+  systemProperty 'tests.rest.blacklist', blacklist.join(',')
   dependsOn copyKeyCerts
+}
+
+testClusters.integTest {
+  distribution = 'DEFAULT' // this is important since we use the reindex module in ML
   setting 'xpack.ml.enabled', 'true'
   setting 'xpack.security.enabled', 'true'
-  setting 'logger.org.elasticsearch.xpack.ml.datafeed', 'TRACE'
   // Integration tests are supposed to enable/disable exporters before/after each test
   setting 'xpack.monitoring.exporters._local.type', 'local'
   setting 'xpack.monitoring.exporters._local.enabled', 'false'
@@ -143,50 +141,10 @@ integTestCluster {
   setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
   setting 'xpack.security.audit.enabled', 'true'
   setting 'xpack.license.self_generated.type', 'trial'
-  keystoreSetting 'bootstrap.password', 'x-pack-test-password'
-  keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
-  distribution = 'default' // this is important since we use the reindex module in ML
-
-  setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'
+  keystore 'bootstrap.password', 'x-pack-test-password'
+  keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
 
+  user username: "x_pack_rest_user", password: "x-pack-test-password"
   extraConfigFile nodeKey.name, nodeKey
   extraConfigFile nodeCert.name, nodeCert
-
-  waitCondition = { NodeInfo node, AntBuilder ant ->
-      File tmpFile = new File(node.cwd, 'wait.success')
-
-      for (int i = 0; i < 10; i++) {
-        // we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned
-        HttpURLConnection httpURLConnection = null;
-        try {
-          httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}&wait_for_status=yellow").openConnection();
-          httpURLConnection.setRequestProperty("Authorization", "Basic " +
-                  Base64.getEncoder().encodeToString("x_pack_rest_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)));
-          httpURLConnection.setRequestMethod("GET");
-          httpURLConnection.connect();
-          if (httpURLConnection.getResponseCode() == 200) {
-            tmpFile.withWriter StandardCharsets.UTF_8.name(), {
-              it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name()))
-            }
-          }
-        } catch (Exception e) {
-          if (i == 9) {
-            logger.error("final attempt of calling cluster health failed", e)
-          } else {
-            logger.debug("failed to call cluster health", e)
-          }
-        } finally {
-          if (httpURLConnection != null) {
-            httpURLConnection.disconnect();
-          }
-        }
-
-        // did not start, so wait a bit before trying again
-        Thread.sleep(500L);
-      }
-      return tmpFile.exists()
-  }
-}
-if (integTestCluster.distribution.startsWith("oss-")) {
-  integTest.enabled = false
 }
diff --git a/x-pack/plugin/ccr/build.gradle b/x-pack/plugin/ccr/build.gradle
index 4b3ba9307da28..dfc3b85dfe111 100644
--- a/x-pack/plugin/ccr/build.gradle
+++ b/x-pack/plugin/ccr/build.gradle
@@ -48,7 +48,7 @@ gradle.projectsEvaluated {
 }
 
 dependencies {
-    compileOnly "org.elasticsearch:elasticsearch:${version}"
+    compileOnly project(":server")
 
     compileOnly project(path: xpackModule('core'), configuration: 'default')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
diff --git a/x-pack/plugin/ccr/qa/build.gradle b/x-pack/plugin/ccr/qa/build.gradle
index d3e95d997c3fb..6a39489515632 100644
--- a/x-pack/plugin/ccr/qa/build.gradle
+++ b/x-pack/plugin/ccr/qa/build.gradle
@@ -14,4 +14,5 @@ subprojects {
       include 'rest-api-spec/api/**'
     }
   }
+  
 }
diff --git a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle
index cbba093c5526b..a45d3e1b32607 100644
--- a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle
+++ b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle
@@ -1,5 +1,6 @@
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
@@ -11,56 +12,51 @@ dependencies {
     testCompile project(':x-pack:plugin:ccr:qa')
 }
 
-task leaderClusterTest(type: RestIntegTestTask) {
+task "leader-cluster"(type: RestIntegTestTask) {
     mustRunAfter(precommit)
+    runner {
+        systemProperty 'tests.target_cluster', 'leader'
+    }
 }
-
-leaderClusterTestCluster {
-    numNodes = 1
-    clusterName = 'leader-cluster'
+testClusters."leader-cluster" {
+    distribution = "DEFAULT"
     setting 'xpack.license.self_generated.type', 'trial'
 }
 
-leaderClusterTestRunner {
-    systemProperty 'tests.target_cluster', 'leader'
-}
-
+File policyFile = file("${buildDir}/tmp/java.policy")
 task writeJavaPolicy {
     doLast {
-        final File tmp = file("${buildDir}/tmp")
-        if (tmp.exists() == false && tmp.mkdirs() == false) {
+        if (policyFile.parentFile.exists() == false && policyFile.parentFile.mkdirs() == false) {
             throw new GradleException("failed to create temporary directory [${tmp}]")
         }
-        final File javaPolicy = file("${tmp}/java.policy")
-        javaPolicy.write(
+        policyFile.write(
                 [
                         "grant {",
-                        "  permission java.io.FilePermission \"${-> followClusterTest.getNodes().get(0).homeDir}/logs/${-> followClusterTest.getNodes().get(0).clusterName}_server.json\", \"read\";",
+                        "  permission java.io.FilePermission \"${-> testClusters."follow-cluster".getFirstNode().getServerLog()}\", \"read\";",
                         "};"
-                ].join("\n"))
+                ].join("\n")
+        )
     }
 }
 
-task followClusterTest(type: RestIntegTestTask) {}
-followClusterTest.dependsOn writeJavaPolicy
+task "follow-cluster"(type: RestIntegTestTask) {
+    dependsOn 'writeJavaPolicy', "leader-cluster"
+    useCluster testClusters."leader-cluster"
+    runner {
+        systemProperty 'java.security.policy', "file://${policyFile}"
+        systemProperty 'tests.target_cluster', 'follow'
+        nonInputProperties.systemProperty 'tests.leader_host', "${-> testClusters."leader-cluster".getAllHttpSocketURI().get(0)}"
+        nonInputProperties.systemProperty 'log', "${-> testClusters."follow-cluster".getFirstNode().getServerLog()}"
+    }
+}
 
-followClusterTestCluster {
-    dependsOn leaderClusterTestRunner
-    numNodes = 1
-    clusterName = 'follow-cluster'
+testClusters."follow-cluster" {
+    distribution = "DEFAULT"
     setting 'xpack.monitoring.collection.enabled', 'true'
     setting 'xpack.license.self_generated.type', 'trial'
-    setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
+    setting 'cluster.remote.leader_cluster.seeds', { "\"${testClusters."leader-cluster".getAllTransportPortURI().join(",")}\"" }
 }
 
-followClusterTestRunner {
-    systemProperty 'java.security.policy', "file://${buildDir}/tmp/java.policy"
-    systemProperty 'tests.target_cluster', 'follow'
-    nonInputProperties.systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
-    nonInputProperties.systemProperty 'log', "${-> followClusterTest.getNodes().get(0).homeDir}/logs/" +
-            "${-> followClusterTest.getNodes().get(0).clusterName}_server.json"
-    finalizedBy 'leaderClusterTestCluster#stop'
-}
 
-check.dependsOn followClusterTest
+check.dependsOn "follow-cluster"
 test.enabled = false // no unit tests for multi-cluster-search, only the rest integration test
diff --git a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java
index 8fb305ba06ee6..4e46de95adb03 100644
--- a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java
+++ b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java
@@ -104,7 +104,7 @@ private Matcher<JsonLogLine> autoFollowCoordinatorWarn() {
             protected Boolean featureValueOf(JsonLogLine actual) {
                 return actual.level().equals("WARN") &&
                     actual.component().equals("o.e.x.c.a.AutoFollowCoordinator") &&
-                    actual.nodeName().equals("node-0") &&
+                    actual.nodeName().startsWith("follow-cluster-0") &&
                     actual.message().contains("failure occurred while fetching cluster state for auto follow pattern [test_pattern]") &&
                     actual.stacktrace().contains("org.elasticsearch.ElasticsearchStatusException: can not fetch remote cluster state " +
                         "as the remote cluster [leader_cluster] is not licensed for [ccr]; the license mode [BASIC]" +
diff --git a/x-pack/plugin/ccr/qa/multi-cluster/build.gradle b/x-pack/plugin/ccr/qa/multi-cluster/build.gradle
index ba4d7ea2064a1..ae0d4247d7038 100644
--- a/x-pack/plugin/ccr/qa/multi-cluster/build.gradle
+++ b/x-pack/plugin/ccr/qa/multi-cluster/build.gradle
@@ -1,5 +1,6 @@
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
@@ -11,57 +12,57 @@ dependencies {
     testCompile project(':x-pack:plugin:ccr:qa')
 }
 
-task leaderClusterTest(type: RestIntegTestTask) {
+task "leader-cluster"(type: RestIntegTestTask) {
     mustRunAfter(precommit)
+    runner {
+        systemProperty 'tests.target_cluster', 'leader'
+    }
 }
 
-leaderClusterTestCluster {
-    numNodes = 1
-    clusterName = 'leader-cluster'
+testClusters."leader-cluster" {
+    distribution = "DEFAULT"
     setting 'xpack.license.self_generated.type', 'trial'
-    setting 'node.name', 'leader'
-}
-
-leaderClusterTestRunner {
-    systemProperty 'tests.target_cluster', 'leader'
 }
 
-task middleClusterTest(type: RestIntegTestTask) {}
 
-middleClusterTestCluster {
-    dependsOn leaderClusterTestRunner
-    numNodes = 1
-    clusterName = 'middle-cluster'
+task "middle-cluster"(type: RestIntegTestTask) {
+    dependsOn "leader-cluster"
+    useCluster testClusters."leader-cluster"
+    runner {
+        systemProperty 'tests.target_cluster', 'middle'
+        nonInputProperties.systemProperty 'tests.leader_host',
+                "${-> testClusters."leader-cluster".getAllHttpSocketURI().get(0)}"
+    }
+}
+testClusters."middle-cluster" {
+    distribution = "DEFAULT"
     setting 'xpack.license.self_generated.type', 'trial'
-    setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
-    setting 'node.name', 'middle'
+    setting 'cluster.remote.leader_cluster.seeds',
+            { "\"${testClusters."leader-cluster".getAllTransportPortURI().join(",")}\"" }
 }
 
-middleClusterTestRunner {
-    systemProperty 'tests.target_cluster', 'middle'
-    nonInputProperties.systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
+task 'follow-cluster'(type: RestIntegTestTask) {
+    dependsOn "leader-cluster", "middle-cluster"
+    useCluster testClusters."leader-cluster"
+    useCluster testClusters."middle-cluster"
+    runner {
+        systemProperty 'tests.target_cluster', 'follow'
+        nonInputProperties.systemProperty 'tests.leader_host',
+                "${-> testClusters."leader-cluster".getAllHttpSocketURI().get(0)}"
+        nonInputProperties.systemProperty 'tests.middle_host',
+                "${-> testClusters."middle-cluster".getAllHttpSocketURI().get(0)}"
+    }
 }
 
-task followClusterTest(type: RestIntegTestTask) {}
-
-followClusterTestCluster {
-    dependsOn middleClusterTestRunner
-    numNodes = 1
-    clusterName = 'follow-cluster'
+testClusters."follow-cluster" {
+    distribution = "DEFAULT"
     setting 'xpack.monitoring.collection.enabled', 'true'
     setting 'xpack.license.self_generated.type', 'trial'
-    setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
-    setting 'cluster.remote.middle_cluster.seeds', "\"${-> middleClusterTest.nodes.get(0).transportUri()}\""
-    setting 'node.name', 'follow'
-}
-
-followClusterTestRunner {
-    systemProperty 'tests.target_cluster', 'follow'
-    nonInputProperties.systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
-    nonInputProperties.systemProperty 'tests.middle_host', "${-> middleClusterTest.nodes.get(0).httpUri()}"
-    finalizedBy 'leaderClusterTestCluster#stop'
-    finalizedBy 'middleClusterTestCluster#stop'
+    setting 'cluster.remote.leader_cluster.seeds',
+            { "\"${testClusters."leader-cluster".getAllTransportPortURI().join(",")}\""}
+    setting 'cluster.remote.middle_cluster.seeds',
+            { "\"${testClusters."middle-cluster".getAllTransportPortURI().join(",")}\""}
 }
 
-check.dependsOn followClusterTest
+check.dependsOn "follow-cluster"
 test.enabled = false // no unit tests for multi-cluster-search, only the rest integration test
diff --git a/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle b/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle
index 2ef358b6d735d..fa0c02eee3206 100644
--- a/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle
+++ b/x-pack/plugin/ccr/qa/non-compliant-license/build.gradle
@@ -1,5 +1,6 @@
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
@@ -11,34 +12,31 @@ dependencies {
     testCompile project(':x-pack:plugin:ccr:qa:')
 }
 
-task leaderClusterTest(type: RestIntegTestTask) {
+task 'leader-cluster'(type: RestIntegTestTask) {
     mustRunAfter(precommit)
+    runner {
+        systemProperty 'tests.target_cluster', 'leader'
+    }
 }
-
-leaderClusterTestCluster {
-    numNodes = 1
-    clusterName = 'leader-cluster'
+testClusters.'leader-cluster' {
+    distribution = "DEFAULT"
 }
 
-leaderClusterTestRunner {
-    systemProperty 'tests.target_cluster', 'leader'
+task 'follow-cluster'(type: RestIntegTestTask) {
+    dependsOn 'leader-cluster'
+    useCluster testClusters.'leader-cluster'
+    runner {
+        systemProperty 'tests.target_cluster', 'follow'
+        nonInputProperties.systemProperty 'tests.leader_host',
+                { "${testClusters.'follow-cluster'.getAllHttpSocketURI().get(0)}" }
+    }
 }
-
-task followClusterTest(type: RestIntegTestTask) {}
-
-followClusterTestCluster {
-    dependsOn leaderClusterTestRunner
-    numNodes = 1
-    clusterName = 'follow-cluster'
+testClusters.'follow-cluster' {
+    distribution = "DEFAULT"
     setting 'xpack.license.self_generated.type', 'trial'
-    setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
-}
-
-followClusterTestRunner {
-    systemProperty 'tests.target_cluster', 'follow'
-    nonInputProperties.systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
-    finalizedBy 'leaderClusterTestCluster#stop'
+    setting 'cluster.remote.leader_cluster.seeds',
+            { "\"${testClusters.'leader-cluster'.getAllTransportPortURI().join(",")}\"" }
 }
 
-check.dependsOn followClusterTest
+check.dependsOn "follow-cluster"
 test.enabled = false
diff --git a/x-pack/plugin/ccr/qa/rest/build.gradle b/x-pack/plugin/ccr/qa/rest/build.gradle
index ba0c05e09791a..1e4ca1abcbce0 100644
--- a/x-pack/plugin/ccr/qa/rest/build.gradle
+++ b/x-pack/plugin/ccr/qa/rest/build.gradle
@@ -1,5 +1,6 @@
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
@@ -14,29 +15,18 @@ task restTest(type: RestIntegTestTask) {
     mustRunAfter(precommit)
 }
 
-restTestCluster {
-    distribution 'default'
+testClusters.restTest {
+    distribution = 'default'
     // Disable assertions in FollowingEngineAssertions, otherwise an AssertionError is thrown before
     // indexing a document directly in a follower index. In a rest test we like to test the exception
     // that is thrown in production when indexing a document directly in a follower index.
-    environment 'ES_JAVA_OPTS', '-da:org.elasticsearch.xpack.ccr.index.engine.FollowingEngineAssertions'
+    jvmArgs '-da:org.elasticsearch.xpack.ccr.index.engine.FollowingEngineAssertions'
     setting 'xpack.ml.enabled', 'false'
     setting 'xpack.monitoring.enabled', 'false'
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.license.self_generated.type', 'trial'
     // TODO: reduce the need for superuser here
-    setupCommand 'setup-ccr-user',
-            'bin/elasticsearch-users', 'useradd', 'ccr-user', '-p', 'ccr-user-password', '-r', 'superuser'
-    waitCondition = { node, ant ->
-        File tmpFile = new File(node.cwd, 'wait.success')
-        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-                dest: tmpFile.toString(),
-                username: 'ccr-user',
-                password: 'ccr-user-password',
-                ignoreerrors: true,
-                retries: 10)
-        return tmpFile.exists()
-    }
+    user username:'ccr-user', password: 'ccr-user-password', role: 'superuser'
 }
 
 check.dependsOn restTest
diff --git a/x-pack/plugin/ccr/qa/restart/build.gradle b/x-pack/plugin/ccr/qa/restart/build.gradle
index cace98d97b015..d29cb136e57b2 100644
--- a/x-pack/plugin/ccr/qa/restart/build.gradle
+++ b/x-pack/plugin/ccr/qa/restart/build.gradle
@@ -1,60 +1,55 @@
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
     testCompile project(':x-pack:plugin:ccr:qa')
 }
 
-task leaderClusterTest(type: RestIntegTestTask) {
+task 'leader-cluster'(type: RestIntegTestTask) {
     mustRunAfter(precommit)
+    runner {
+        systemProperty 'tests.target_cluster', 'leader'
+    }
 }
-
-leaderClusterTestCluster {
-    numNodes = 1
-    clusterName = 'leader-cluster'
+testClusters.'leader-cluster' {
+    distribution = "DEFAULT"
     setting 'xpack.license.self_generated.type', 'trial'
-    setting 'node.name', 'leader'
 }
 
-leaderClusterTestRunner {
-    systemProperty 'tests.target_cluster', 'leader'
+task 'follow-cluster'(type: RestIntegTestTask) {
+    dependsOn 'leader-cluster'
+    useCluster testClusters.'leader-cluster'
+    runner {
+        systemProperty 'tests.target_cluster', 'follow'
+        nonInputProperties.systemProperty 'tests.leader_host',
+                "${-> testClusters.'leader-cluster'.getAllHttpSocketURI().get(0)}"
+    }
 }
-
-task followClusterTest(type: RestIntegTestTask) {}
-
-followClusterTestCluster {
-    dependsOn leaderClusterTestRunner
-    numNodes = 1
-    clusterName = 'follow-cluster'
+testClusters.'follow-cluster' {
+    distribution = "DEFAULT"
     setting 'xpack.monitoring.collection.enabled', 'true'
     setting 'xpack.license.self_generated.type', 'trial'
-    setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
-    setting 'node.name', 'follow'
-}
-
-followClusterTestRunner {
-    systemProperty 'tests.target_cluster', 'follow'
-    nonInputProperties.systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
+    setting 'cluster.remote.leader_cluster.seeds',
+            { "\"${testClusters.'leader-cluster'.getAllTransportPortURI().get(0)}\"" }
+    nameCustomization = { 'follow'  }
 }
 
-task followClusterRestartTest(type: RestIntegTestTask) {}
-
-followClusterRestartTestCluster {
-    dependsOn followClusterTestRunner, 'followClusterTestCluster#stop'
-    numNodes = 1
-    clusterName = 'follow-cluster'
-    dataDir = { nodeNumber -> followClusterTest.nodes[0].dataDir }
-    setting 'xpack.monitoring.collection.enabled', 'true'
-    setting 'xpack.license.self_generated.type', 'trial'
-    setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
-    setting 'node.name', 'follow'
-}
+task followClusterRestartTest(type: Test) {
+    dependsOn tasks.'follow-cluster'
+    useCluster testClusters.'leader-cluster'
+    useCluster testClusters.'follow-cluster'
 
-followClusterRestartTestRunner {
+    maxParallelForks = 1
+    systemProperty 'tests.rest.load_packaged', 'false'
     systemProperty 'tests.target_cluster', 'follow-restart'
-    nonInputProperties.systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
-    finalizedBy 'leaderClusterTestCluster#stop'
+    doFirst {
+        testClusters.'follow-cluster'.restart()
+        nonInputProperties.systemProperty 'tests.leader_host', "${-> testClusters.'leader-cluster'.getAllHttpSocketURI().get(0)}"
+        nonInputProperties.systemProperty 'tests.rest.cluster', "${-> testClusters.'follow-cluster'.getAllHttpSocketURI().join(",")}"
+    }
+    outputs.doNotCacheIf "Caching of REST tests not implemented yet", { false }
 }
 
 check.dependsOn followClusterRestartTest
diff --git a/x-pack/plugin/ccr/qa/security/build.gradle b/x-pack/plugin/ccr/qa/security/build.gradle
index 872e99051018c..a24dc8cd99fdc 100644
--- a/x-pack/plugin/ccr/qa/security/build.gradle
+++ b/x-pack/plugin/ccr/qa/security/build.gradle
@@ -1,5 +1,6 @@
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
@@ -11,69 +12,44 @@ dependencies {
     testCompile project(':x-pack:plugin:ccr:qa')
 }
 
-task leaderClusterTest(type: RestIntegTestTask) {
+task 'leader-cluster'(type: RestIntegTestTask) {
     mustRunAfter(precommit)
+    runner {
+        systemProperty 'tests.target_cluster', 'leader'
+    }
 }
 
-leaderClusterTestCluster {
-    numNodes = 1
-    clusterName = 'leader-cluster'
+testClusters.'leader-cluster' {
+    distribution = 'Default'
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.monitoring.enabled', 'false'
-    extraConfigFile 'roles.yml', 'leader-roles.yml'
-    setupCommand 'setupTestAdmin',
-            'bin/elasticsearch-users', 'useradd', "test_admin", '-p', 'x-pack-test-password', '-r', "superuser"
-    setupCommand 'setupCcrUser',
-            'bin/elasticsearch-users', 'useradd', "test_ccr", '-p', 'x-pack-test-password', '-r', "ccruser"
-    waitCondition = { node, ant ->
-        File tmpFile = new File(node.cwd, 'wait.success')
-        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-                dest: tmpFile.toString(),
-                username: 'test_admin',
-                password: 'x-pack-test-password',
-                ignoreerrors: true,
-                retries: 10)
-        return tmpFile.exists()
-    }
+    extraConfigFile 'roles.yml', file('leader-roles.yml')
+    user username: "test_admin", role: "superuser"
+    user username: "test_ccr", role: "ccruser"
 }
 
-leaderClusterTestRunner {
-    systemProperty 'tests.target_cluster', 'leader'
+task 'follow-cluster'(type: RestIntegTestTask) {
+    dependsOn 'leader-cluster'
+    useCluster testClusters.'leader-cluster'
+    runner {
+        systemProperty 'tests.target_cluster', 'follow'
+        nonInputProperties.systemProperty 'tests.leader_host', "${-> testClusters.'leader-cluster'.getAllHttpSocketURI().get(0)}"
+    }
 }
 
-task followClusterTest(type: RestIntegTestTask) {}
-
-followClusterTestCluster {
-    dependsOn leaderClusterTestRunner
-    numNodes = 1
-    clusterName = 'follow-cluster'
-    setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
+testClusters.'follow-cluster' {
+    distribution = 'Default'
+    setting 'cluster.remote.leader_cluster.seeds', {
+        "\"${testClusters.'leader-cluster'.getAllTransportPortURI().join(",")}\""
+    }
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.monitoring.collection.enabled', 'true'
-    extraConfigFile 'roles.yml', 'follower-roles.yml'
-    setupCommand 'setupTestAdmin',
-            'bin/elasticsearch-users', 'useradd', "test_admin", '-p', 'x-pack-test-password', '-r', "superuser"
-    setupCommand 'setupCcrUser',
-            'bin/elasticsearch-users', 'useradd', "test_ccr", '-p', 'x-pack-test-password', '-r', "ccruser"
-    waitCondition = { node, ant ->
-        File tmpFile = new File(node.cwd, 'wait.success')
-        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-                dest: tmpFile.toString(),
-                username: 'test_admin',
-                password: 'x-pack-test-password',
-                ignoreerrors: true,
-                retries: 10)
-        return tmpFile.exists()
-    }
-}
-
-followClusterTestRunner {
-    systemProperty 'tests.target_cluster', 'follow'
-    nonInputProperties.systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
-    finalizedBy 'leaderClusterTestCluster#stop'
+    extraConfigFile 'roles.yml', file('follower-roles.yml')
+    user username: "test_admin", role: "superuser"
+    user username: "test_ccr", role: "ccruser"
 }
 
-check.dependsOn followClusterTest
+check.dependsOn 'follow-cluster'
 test.enabled = false // no unit tests for multi-cluster-search, only the rest integration test
diff --git a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java
index 33e7c2f2bf177..b555e5f441146 100644
--- a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java
+++ b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java
@@ -143,6 +143,7 @@ protected static void verifyCcrMonitoring(final String expectedLeaderIndex, fina
         int followerMaxSeqNo = 0;
         int followerMappingVersion = 0;
         int followerSettingsVersion = 0;
+        int followerAliasesVersion = 0;
 
         List<?> hits = (List<?>) XContentMapValues.extractValue("hits.hits", response);
         assertThat(hits.size(), greaterThanOrEqualTo(1));
@@ -164,11 +165,15 @@ protected static void verifyCcrMonitoring(final String expectedLeaderIndex, fina
             int foundFollowerSettingsVersion =
                     (int) XContentMapValues.extractValue("_source.ccr_stats.follower_settings_version", hit);
             followerSettingsVersion = Math.max(followerSettingsVersion, foundFollowerSettingsVersion);
+            int foundFollowerAliasesVersion =
+                    (int) XContentMapValues.extractValue("_source.ccr_stats.follower_aliases_version", hit);
+            followerAliasesVersion = Math.max(followerAliasesVersion, foundFollowerAliasesVersion);
         }
 
         assertThat(followerMaxSeqNo, greaterThan(0));
         assertThat(followerMappingVersion, greaterThan(0));
         assertThat(followerSettingsVersion, greaterThan(0));
+        assertThat(followerAliasesVersion, greaterThan(0));
     }
 
     protected static void verifyAutoFollowMonitoring() throws IOException {
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java
index c74e39f017a3c..b926bc986510b 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java
@@ -9,14 +9,15 @@
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.admin.indices.mapping.put.MappingRequestValidator;
+import org.elasticsearch.action.RequestValidators;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -85,8 +86,9 @@
 import org.elasticsearch.xpack.ccr.rest.RestResumeFollowAction;
 import org.elasticsearch.xpack.ccr.rest.RestUnfollowAction;
 import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
-import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
 import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus;
 import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
 import org.elasticsearch.xpack.core.ccr.action.DeleteAutoFollowPatternAction;
@@ -198,8 +200,10 @@ public List<PersistentTasksExecutor<?>> getPersistentTasksExecutor(ClusterServic
     }
 
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        var usageAction = new ActionHandler<>(XPackUsageFeatureAction.CCR, CCRUsageTransportAction.class);
+        var infoAction = new ActionHandler<>(XPackInfoFeatureAction.CCR, CCRInfoTransportAction.class);
         if (enabled == false) {
-            return emptyList();
+            return Arrays.asList(usageAction, infoAction);
         }
 
         return Arrays.asList(
@@ -230,7 +234,9 @@ public List<PersistentTasksExecutor<?>> getPersistentTasksExecutor(ClusterServic
                 new ActionHandler<>(PutAutoFollowPatternAction.INSTANCE, TransportPutAutoFollowPatternAction.class),
                 new ActionHandler<>(GetAutoFollowPatternAction.INSTANCE, TransportGetAutoFollowPatternAction.class),
                 // forget follower action
-                new ActionHandler<>(ForgetFollowerAction.INSTANCE, TransportForgetFollowerAction.class));
+                new ActionHandler<>(ForgetFollowerAction.INSTANCE, TransportForgetFollowerAction.class),
+                usageAction,
+                infoAction);
     }
 
     public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
@@ -336,16 +342,15 @@ public void onIndexModule(IndexModule indexModule) {
         }
     }
 
-    @Override
-    public Collection<Module> createGuiceModules() {
-        return Collections.singleton(b -> XPackPlugin.bindFeatureSet(b, CCRFeatureSet.class));
-    }
-
     protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); }
 
     @Override
-    public Collection<MappingRequestValidator> mappingRequestValidators() {
+    public Collection<RequestValidators.RequestValidator<PutMappingRequest>> mappingRequestValidators() {
         return Collections.singletonList(CcrRequests.CCR_PUT_MAPPING_REQUEST_VALIDATOR);
     }
 
+    @Override
+    public Collection<RequestValidators.RequestValidator<IndicesAliasesRequest>> indicesAliasesRequestValidators() {
+        return Collections.singletonList(CcrRequests.CCR_INDICES_ALIASES_REQUEST_VALIDATOR);
+    }
 }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java
index a0a81b1a51677..286d524f60b4a 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.ccr;
 
 import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -355,7 +355,7 @@ public static Client wrapClient(Client client, Map<String, String> headers) {
             return new FilterClient(client) {
                 @Override
                 protected <Request extends ActionRequest, Response extends ActionResponse>
-                void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+                void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
                     final Supplier<ThreadContext.StoredContext> supplier = threadContext.newRestorableContext(false);
                     try (ThreadContext.StoredContext ignore = stashWithHeaders(threadContext, filteredHeaders)) {
                         super.doExecute(action, request, new ContextPreservingActionListener<>(supplier, listener));
@@ -370,7 +370,7 @@ private static Client systemClient(Client client) {
         return new FilterClient(client) {
             @Override
             protected <Request extends ActionRequest, Response extends ActionResponse>
-            void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+            void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
                 final Supplier<ThreadContext.StoredContext> supplier = threadContext.newRestorableContext(false);
                 try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
                     threadContext.markAsSystemContext();
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CcrRequests.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CcrRequests.java
index 02ee7d1f138c9..8bb29ce3982c6 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CcrRequests.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CcrRequests.java
@@ -7,9 +7,10 @@
 
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.RequestValidators;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.action.admin.indices.mapping.put.MappingRequestValidator;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
@@ -22,6 +23,7 @@
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.Optional;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
@@ -85,21 +87,42 @@ public static void getIndexMetadata(Client client, Index index, long mappingVers
         ));
     }
 
-    public static final MappingRequestValidator CCR_PUT_MAPPING_REQUEST_VALIDATOR = (request, state, indices) -> {
-        if (request.origin() == null) {
-            return null; // a put-mapping-request on old versions does not have origin.
-        }
-        final List<Index> followingIndices = Arrays.stream(indices)
-            .filter(index -> {
-                final IndexMetaData indexMetaData = state.metaData().index(index);
-                return indexMetaData != null && CcrSettings.CCR_FOLLOWING_INDEX_SETTING.get(indexMetaData.getSettings());
-            }).collect(Collectors.toList());
-        if (followingIndices.isEmpty() == false && "ccr".equals(request.origin()) == false) {
-            final String errorMessage = "can't put mapping to the following indices "
-                + "[" + followingIndices.stream().map(Index::getName).collect(Collectors.joining(", ")) + "]; "
-                + "the mapping of the following indices are self-replicated from its leader indices";
-            return new ElasticsearchStatusException(errorMessage, RestStatus.FORBIDDEN);
-        }
-        return null;
-    };
+    public static final RequestValidators.RequestValidator<PutMappingRequest> CCR_PUT_MAPPING_REQUEST_VALIDATOR =
+            (request, state, indices) -> {
+                if (request.origin() == null) {
+                    return Optional.empty(); // a put-mapping-request on old versions does not have origin.
+                }
+                final List<Index> followingIndices = Arrays.stream(indices)
+                        .filter(index -> {
+                            final IndexMetaData indexMetaData = state.metaData().index(index);
+                            return indexMetaData != null && CcrSettings.CCR_FOLLOWING_INDEX_SETTING.get(indexMetaData.getSettings());
+                        }).collect(Collectors.toList());
+                if (followingIndices.isEmpty() == false && "ccr".equals(request.origin()) == false) {
+                    final String errorMessage = "can't put mapping to the following indices "
+                            + "[" + followingIndices.stream().map(Index::getName).collect(Collectors.joining(", ")) + "]; "
+                            + "the mapping of the following indices are self-replicated from its leader indices";
+                    return Optional.of(new ElasticsearchStatusException(errorMessage, RestStatus.FORBIDDEN));
+                }
+                return Optional.empty();
+            };
+
+    public static final RequestValidators.RequestValidator<IndicesAliasesRequest> CCR_INDICES_ALIASES_REQUEST_VALIDATOR =
+            (request, state, indices) -> {
+                if (request.origin() == null) {
+                    return Optional.empty(); // an indices aliases request on old versions does not have origin
+                }
+                final List<Index> followingIndices = Arrays.stream(indices)
+                        .filter(index -> {
+                            final IndexMetaData indexMetaData = state.metaData().index(index);
+                            return indexMetaData != null && CcrSettings.CCR_FOLLOWING_INDEX_SETTING.get(indexMetaData.getSettings());
+                        }).collect(Collectors.toList());
+                if (followingIndices.isEmpty() == false && "ccr".equals(request.origin()) == false) {
+                    final String errorMessage = "can't modify aliases on indices "
+                            + "[" + followingIndices.stream().map(Index::getName).collect(Collectors.joining(", ")) + "]; "
+                            + "aliases of following indices are self-replicated from their leader indices";
+                    return Optional.of(new ElasticsearchStatusException(errorMessage, RestStatus.FORBIDDEN));
+                }
+                return Optional.empty();
+            };
+
 }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
index 33b8a274431d2..16c22b7ab202b 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardChangesAction.java
@@ -7,7 +7,8 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
@@ -51,7 +52,7 @@
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
 
-public class ShardChangesAction extends Action<ShardChangesAction.Response> {
+public class ShardChangesAction extends ActionType<ShardChangesAction.Response> {
 
     public static final ShardChangesAction INSTANCE = new ShardChangesAction();
     public static final String NAME = "indices:data/read/xpack/ccr/shard_changes";
@@ -60,11 +61,6 @@ private ShardChangesAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
@@ -74,8 +70,8 @@ public static class Request extends SingleShardRequest<Request> {
 
         private long fromSeqNo;
         private int maxOperationCount;
-        private ShardId shardId;
-        private String expectedHistoryUUID;
+        private final ShardId shardId;
+        private final String expectedHistoryUUID;
         private TimeValue pollTimeout = TransportResumeFollowAction.DEFAULT_READ_POLL_TIMEOUT;
         private ByteSizeValue maxBatchSize = TransportResumeFollowAction.DEFAULT_MAX_READ_REQUEST_SIZE;
 
@@ -87,7 +83,17 @@ public Request(ShardId shardId, String expectedHistoryUUID) {
             this.expectedHistoryUUID = expectedHistoryUUID;
         }
 
-        Request() {
+        Request(StreamInput in) throws IOException {
+            super(in);
+            fromSeqNo = in.readVLong();
+            maxOperationCount = in.readVInt();
+            shardId = new ShardId(in);
+            expectedHistoryUUID = in.readString();
+            pollTimeout = in.readTimeValue();
+            maxBatchSize = new ByteSizeValue(in);
+
+            // Starting the clock in order to know how much time is spent on fetching operations:
+            relativeStartNanos = System.nanoTime();
         }
 
         public ShardId getShard() {
@@ -147,20 +153,6 @@ public ActionRequestValidationException validate() {
             return validationException;
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            fromSeqNo = in.readVLong();
-            maxOperationCount = in.readVInt();
-            shardId = new ShardId(in);
-            expectedHistoryUUID = in.readString();
-            pollTimeout = in.readTimeValue();
-            maxBatchSize = new ByteSizeValue(in);
-
-            // Starting the clock in order to know how much time is spent on fetching operations:
-            relativeStartNanos = System.nanoTime();
-        }
-
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
@@ -219,6 +211,12 @@ public long getSettingsVersion() {
             return settingsVersion;
         }
 
+        private long aliasesVersion;
+
+        public long getAliasesVersion() {
+            return aliasesVersion;
+        }
+
         private long globalCheckpoint;
 
         public long getGlobalCheckpoint() {
@@ -256,6 +254,11 @@ public long getTookInMillis() {
             super(in);
             mappingVersion = in.readVLong();
             settingsVersion = in.readVLong();
+            if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+                aliasesVersion = in.readVLong();
+            } else {
+                aliasesVersion = 0;
+            }
             globalCheckpoint = in.readZLong();
             maxSeqNo = in.readZLong();
             maxSeqNoOfUpdatesOrDeletes = in.readZLong();
@@ -264,16 +267,17 @@ public long getTookInMillis() {
         }
 
         Response(
-            final long mappingVersion,
-            final long settingsVersion,
-            final long globalCheckpoint,
-            final long maxSeqNo,
-            final long maxSeqNoOfUpdatesOrDeletes,
-            final Translog.Operation[] operations,
-            final long tookInMillis) {
-
+                final long mappingVersion,
+                final long settingsVersion,
+                final long aliasesVersion,
+                final long globalCheckpoint,
+                final long maxSeqNo,
+                final long maxSeqNoOfUpdatesOrDeletes,
+                final Translog.Operation[] operations,
+                final long tookInMillis) {
             this.mappingVersion = mappingVersion;
             this.settingsVersion = settingsVersion;
+            this.aliasesVersion = aliasesVersion;
             this.globalCheckpoint = globalCheckpoint;
             this.maxSeqNo = maxSeqNo;
             this.maxSeqNoOfUpdatesOrDeletes = maxSeqNoOfUpdatesOrDeletes;
@@ -291,6 +295,9 @@ public void writeTo(final StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeVLong(mappingVersion);
             out.writeVLong(settingsVersion);
+            if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+                out.writeVLong(aliasesVersion);
+            }
             out.writeZLong(globalCheckpoint);
             out.writeZLong(maxSeqNo);
             out.writeZLong(maxSeqNoOfUpdatesOrDeletes);
@@ -305,6 +312,7 @@ public boolean equals(final Object o) {
             final Response that = (Response) o;
             return mappingVersion == that.mappingVersion &&
                     settingsVersion == that.settingsVersion &&
+                    aliasesVersion == that.aliasesVersion &&
                     globalCheckpoint == that.globalCheckpoint &&
                     maxSeqNo == that.maxSeqNo &&
                     maxSeqNoOfUpdatesOrDeletes == that.maxSeqNoOfUpdatesOrDeletes &&
@@ -317,6 +325,7 @@ public int hashCode() {
             return Objects.hash(
                     mappingVersion,
                     settingsVersion,
+                    aliasesVersion,
                     globalCheckpoint,
                     maxSeqNo,
                     maxSeqNoOfUpdatesOrDeletes,
@@ -361,9 +370,11 @@ protected Response shardOperation(Request request, ShardId shardId) throws IOExc
             final IndexMetaData indexMetaData = indexService.getMetaData();
             final long mappingVersion = indexMetaData.getMappingVersion();
             final long settingsVersion = indexMetaData.getSettingsVersion();
+            final long aliasesVersion = indexMetaData.getAliasesVersion();
             return getResponse(
                     mappingVersion,
                     settingsVersion,
+                    aliasesVersion,
                     seqNoStats,
                     maxSeqNoOfUpdatesOrDeletes,
                     operations,
@@ -436,12 +447,14 @@ private void globalCheckpointAdvancementFailure(
 
                     final long mappingVersion = indexMetaData.getMappingVersion();
                     final long settingsVersion = indexMetaData.getSettingsVersion();
+                    final long aliasesVersion = indexMetaData.getAliasesVersion();
                     final SeqNoStats latestSeqNoStats = indexShard.seqNoStats();
                     final long maxSeqNoOfUpdatesOrDeletes = indexShard.getMaxSeqNoOfUpdatesOrDeletes();
                     listener.onResponse(
                             getResponse(
                                     mappingVersion,
                                     settingsVersion,
+                                    aliasesVersion,
                                     latestSeqNoStats,
                                     maxSeqNoOfUpdatesOrDeletes,
                                     EMPTY_OPERATIONS_ARRAY,
@@ -541,6 +554,7 @@ static Translog.Operation[] getOperations(
     static Response getResponse(
             final long mappingVersion,
             final long settingsVersion,
+            final long aliasesVersion,
             final SeqNoStats seqNoStats,
             final long maxSeqNoOfUpdates,
             final Translog.Operation[] operations,
@@ -550,6 +564,7 @@ static Response getResponse(
         return new Response(
                 mappingVersion,
                 settingsVersion,
+                aliasesVersion,
                 seqNoStats.getGlobalCheckpoint(),
                 seqNoStats.getMaxSeqNo(),
                 maxSeqNoOfUpdates,
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java
index 0ee86a6058c63..4ad0fb1dfd0d6 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java
@@ -78,6 +78,7 @@ public abstract class ShardFollowNodeTask extends AllocatedPersistentTask {
     private int numOutstandingWrites = 0;
     private long currentMappingVersion = 0;
     private long currentSettingsVersion = 0;
+    private long currentAliasesVersion = 0;
     private long totalReadRemoteExecTimeMillis = 0;
     private long totalReadTimeMillis = 0;
     private long successfulReadRequests = 0;
@@ -154,15 +155,27 @@ void start(
                 synchronized (ShardFollowNodeTask.this) {
                     currentSettingsVersion = leaderSettingsVersion;
                 }
+            });
+            updateAliases(leaderAliasesVersion -> {
+                synchronized (ShardFollowNodeTask.this) {
+                    currentAliasesVersion = leaderAliasesVersion;
+                }
+            });
+            synchronized (ShardFollowNodeTask.this) {
                 LOGGER.info(
-                        "{} following leader shard {}, follower global checkpoint=[{}], mapping version=[{}], settings version=[{}]",
+                        "{} following leader shard {}, " +
+                                "follower global checkpoint=[{}], " +
+                                "mapping version=[{}], " +
+                                "settings version=[{}], " +
+                                "aliases version=[{}]",
                         params.getFollowShardId(),
                         params.getLeaderShardId(),
                         followerGlobalCheckpoint,
-                        leaderMappingVersion,
-                        leaderSettingsVersion);
-                coordinateReads();
-            });
+                        currentMappingVersion,
+                        currentSettingsVersion,
+                        currentAliasesVersion);
+            }
+            coordinateReads();
         });
     }
 
@@ -306,12 +319,14 @@ void handleReadResponse(long from, long maxRequiredSeqNo, ShardChangesAction.Res
         // In order to process this read response (3), we need to check and potentially update the follow index's setting (1) and
         // check and potentially update the follow index's mappings (2).
 
-        // 3) handle read response:
+        // 4) handle read response:
         Runnable handleResponseTask = () -> innerHandleReadResponse(from, maxRequiredSeqNo, response);
-        // 2) update follow index mapping:
+        // 3) update follow index mapping:
         Runnable updateMappingsTask = () -> maybeUpdateMapping(response.getMappingVersion(), handleResponseTask);
-        // 1) update follow index settings:
-        maybeUpdateSettings(response.getSettingsVersion(), updateMappingsTask);
+        // 2) update follow index settings:
+        Runnable updateSettingsTask = () -> maybeUpdateSettings(response.getSettingsVersion(), updateMappingsTask);
+        // 1) update follow index aliases:
+        maybeUpdateAliases(response.getAliasesVersion(), updateSettingsTask);
     }
 
     void handleFallenBehindLeaderShard(Exception e, long from, int maxOperationCount, long maxRequiredSeqNo, AtomicInteger retryCounter) {
@@ -423,7 +438,7 @@ private synchronized void maybeUpdateMapping(long minimumRequiredMappingVersion,
 
     private synchronized void maybeUpdateSettings(final Long minimumRequiredSettingsVersion, Runnable task) {
         if (currentSettingsVersion >= minimumRequiredSettingsVersion) {
-            LOGGER.trace("{} settings version [{}] is higher or equal than minimum required mapping version [{}]",
+            LOGGER.trace("{} settings version [{}] is higher or equal than minimum required settings version [{}]",
                     params.getFollowShardId(), currentSettingsVersion, minimumRequiredSettingsVersion);
             task.run();
         } else {
@@ -436,6 +451,27 @@ private synchronized void maybeUpdateSettings(final Long minimumRequiredSettings
         }
     }
 
+    private synchronized void maybeUpdateAliases(final Long minimumRequiredAliasesVersion, final Runnable task) {
+        if (currentAliasesVersion >= minimumRequiredAliasesVersion) {
+            LOGGER.trace(
+                    "{} aliases version [{}] is higher or equal than minimum required aliases version [{}]",
+                    params.getFollowShardId(),
+                    currentAliasesVersion,
+                    minimumRequiredAliasesVersion);
+            task.run();
+        } else {
+            LOGGER.trace(
+                    "{} updating aliases, aliases version [{}] is lower than minimum required aliases version [{}]",
+                    params.getFollowShardId(),
+                    currentAliasesVersion,
+                    minimumRequiredAliasesVersion);
+            updateAliases(aliasesVersion -> {
+                currentAliasesVersion = aliasesVersion;
+                task.run();
+            });
+        }
+    }
+
     private void updateMapping(long minRequiredMappingVersion, LongConsumer handler) {
         updateMapping(minRequiredMappingVersion, handler, new AtomicInteger(0));
     }
@@ -453,6 +489,14 @@ private void updateSettings(final LongConsumer handler, final AtomicInteger retr
         innerUpdateSettings(handler, e -> handleFailure(e, retryCounter, () -> updateSettings(handler, retryCounter)));
     }
 
+    private void updateAliases(final LongConsumer handler) {
+        updateAliases(handler, new AtomicInteger());
+    }
+
+    private void updateAliases(final LongConsumer handler, final AtomicInteger retryCounter) {
+        innerUpdateAliases(handler, e -> handleFailure(e, retryCounter, () -> updateAliases(handler, retryCounter)));
+    }
+
     private void handleFailure(Exception e, AtomicInteger retryCounter, Runnable task) {
         assert e != null;
         if (shouldRetry(params.getRemoteCluster(), e)) {
@@ -511,6 +555,8 @@ static boolean shouldRetry(String remoteCluster, Exception e) {
 
     protected abstract void innerUpdateSettings(LongConsumer handler, Consumer<Exception> errorHandler);
 
+    protected abstract void innerUpdateAliases(LongConsumer handler, Consumer<Exception> errorHandler);
+
     protected abstract void innerSendBulkShardOperationsRequest(String followerHistoryUUID,
                                                                 List<Translog.Operation> operations,
                                                                 long leaderMaxSeqNoOfUpdatesOrDeletes,
@@ -566,6 +612,7 @@ public synchronized ShardFollowNodeTaskStatus getStatus() {
                 bufferSizeInBytes,
                 currentMappingVersion,
                 currentSettingsVersion,
+                currentAliasesVersion,
                 totalReadTimeMillis,
                 totalReadRemoteExecTimeMillis,
                 successfulReadRequests,
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTask.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTask.java
index 5d4564a2030c1..50b32679aee0c 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTask.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTask.java
@@ -183,6 +183,6 @@ public String toString() {
 
     @Override
     public Version getMinimalSupportedVersion() {
-        return Version.V_6_5_0;
+        return Version.CURRENT.minimumCompatibilityVersion();
     }
 }
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java
index 595303d0bce72..3a699dda7c338 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.admin.indices.close.CloseIndexResponse;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
@@ -26,12 +27,14 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.unit.TimeValue;
@@ -62,7 +65,9 @@
 import org.elasticsearch.xpack.ccr.action.bulk.BulkShardOperationsRequest;
 import org.elasticsearch.xpack.ccr.action.bulk.BulkShardOperationsResponse;
 
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -173,10 +178,13 @@ protected void innerUpdateSettings(final LongConsumer finalHandler, final Consum
                         finalHandler.accept(leaderIMD.getSettingsVersion());
                     } else {
                         // Figure out which settings have been updated:
-                        final Settings updatedSettings = settings.filter(
-                            s -> existingSettings.get(s) == null || existingSettings.get(s).equals(settings.get(s)) == false
-                        );
-
+                        final Settings updatedSettings = settings.filter(s -> {
+                            final Setting<?> indexSettings = indexScopedSettings.get(s);
+                            if (indexSettings == null || indexSettings.isPrivateIndex() || indexSettings.isInternalIndex()) {
+                                return false;
+                            }
+                            return existingSettings.get(s) == null || existingSettings.get(s).equals(settings.get(s)) == false;
+                        });
                         // Figure out whether the updated settings are all dynamic settings and
                         // if so just update the follower index's settings:
                         if (updatedSettings.keySet().stream().allMatch(indexScopedSettings::isDynamicSetting)) {
@@ -200,6 +208,125 @@ protected void innerUpdateSettings(final LongConsumer finalHandler, final Consum
                 }
             }
 
+            @Override
+            protected void innerUpdateAliases(final LongConsumer handler, final Consumer<Exception> errorHandler) {
+                /*
+                 * The strategy for updating the aliases is fairly simple. We look at the aliases that exist on the leader, and those that
+                 * exist on the follower. We partition these aliases into three sets: the aliases that exist on both the leader and the
+                 * follower, the aliases that are on the leader only, and the aliases that are on the follower only.
+                 *
+                 * For the aliases that are on the leader and the follower, we compare the aliases and add an action to overwrite the
+                 * follower view of the alias if the aliases are different. If the aliases are the same, we skip the alias. Note that the
+                 * meaning of equals here intentionally ignores the write index. There are two reasons for this. First, follower indices
+                 * do not receive direct writes so conceptually the write index is not useful. Additionally, there is a larger challenge.
+                 * Suppose that we did copy over the write index from the leader to the follower. On the leader, when the write index is
+                 * swapped from one index to another, this is done atomically. However, to do this on the follower, we would have to step
+                 * outside the shard follow tasks framework and have a separate framework for copying aliases over. This is because if we
+                 * try to manage the aliases including the write aliases with the shard follow tasks, we do not have a way to move the write
+                 * index atomically (since we have a single-index view here only) and therefore we can end up in situations where we would
+                 * try to assign the write index to two indices. Further, trying to do this outside the shard follow tasks framework has
+                 * problems too, since it could be that the new aliases arrive on the coordinator before the write index has even been
+                 * created on the local cluster. So there are race conditions either way. All of this put together means that we will simply
+                 * ignore the write index.
+                 *
+                 * For aliases that are on the leader but not the follower, we copy those aliases over to the follower.
+                 *
+                 * For aliases that are on the follower but not the leader, we remove those aliases from the follower.
+                 */
+                final var leaderIndex = params.getLeaderShardId().getIndex();
+                final var followerIndex = params.getFollowShardId().getIndex();
+
+                final var clusterStateRequest = CcrRequests.metaDataRequest(leaderIndex.getName());
+
+                final CheckedConsumer<ClusterStateResponse, Exception> onResponse = clusterStateResponse -> {
+                    final var leaderIndexMetaData = clusterStateResponse.getState().metaData().getIndexSafe(leaderIndex);
+                    final var followerIndexMetaData = clusterService.state().metaData().getIndexSafe(followerIndex);
+
+                    // partition the aliases into the three sets
+                    final var aliasesOnLeaderNotOnFollower = new HashSet<String>();
+                    final var aliasesInCommon = new HashSet<String>();
+                    final var aliasesOnFollowerNotOnLeader = new HashSet<String>();
+
+                    for (final var aliasName : leaderIndexMetaData.getAliases().keys()) {
+                        if (followerIndexMetaData.getAliases().containsKey(aliasName.value)) {
+                            aliasesInCommon.add(aliasName.value);
+                        } else {
+                            aliasesOnLeaderNotOnFollower.add(aliasName.value);
+                        }
+                    }
+
+                    for (final var aliasName : followerIndexMetaData.getAliases().keys()) {
+                        if (leaderIndexMetaData.getAliases().containsKey(aliasName.value)) {
+                            assert aliasesInCommon.contains(aliasName.value) : aliasName.value;
+                        } else {
+                            aliasesOnFollowerNotOnLeader.add(aliasName.value);
+                        }
+                    }
+
+                    final var aliasActions = new ArrayList<IndicesAliasesRequest.AliasActions>();
+
+                    // add the aliases the follower does not have
+                    for (final var aliasName : aliasesOnLeaderNotOnFollower) {
+                        final var alias = leaderIndexMetaData.getAliases().get(aliasName);
+                        // we intentionally override that the alias is not a write alias as follower indices do not receive direct writes
+                        aliasActions.add(IndicesAliasesRequest.AliasActions.add()
+                                .index(followerIndex.getName())
+                                .alias(alias.alias())
+                                .filter(alias.filter() == null ? null : alias.filter().toString())
+                                .indexRouting(alias.indexRouting())
+                                .searchRouting(alias.searchRouting())
+                                .writeIndex(false));
+                    }
+
+                    // update the aliases that are different (ignoring write aliases)
+                    for (final var aliasName : aliasesInCommon) {
+                        final var leaderAliasMetaData = leaderIndexMetaData.getAliases().get(aliasName);
+                        // we intentionally override that the alias is not a write alias as follower indices do not receive direct writes
+                        final var leaderAliasMetaDataWithoutWriteIndex = new AliasMetaData.Builder(aliasName)
+                                .filter(leaderAliasMetaData.filter())
+                                .indexRouting(leaderAliasMetaData.indexRouting())
+                                .searchRouting(leaderAliasMetaData.searchRouting())
+                                .writeIndex(false)
+                                .build();
+                        final var followerAliasMetaData = followerIndexMetaData.getAliases().get(aliasName);
+                        if (leaderAliasMetaDataWithoutWriteIndex.equals(followerAliasMetaData)) {
+                            // skip this alias, the leader and follower have the same modulo the write index
+                            continue;
+                        }
+                        // we intentionally override that the alias is not a write alias as follower indices do not receive direct writes
+                        aliasActions.add(IndicesAliasesRequest.AliasActions.add()
+                                .index(followerIndex.getName())
+                                .alias(leaderAliasMetaData.alias())
+                                .filter(leaderAliasMetaData.filter() == null ? null : leaderAliasMetaData.filter().toString())
+                                .indexRouting(leaderAliasMetaData.indexRouting())
+                                .searchRouting(leaderAliasMetaData.searchRouting())
+                                .writeIndex(false));
+                    }
+
+                    // remove aliases that the leader no longer has
+                    for (final var aliasName : aliasesOnFollowerNotOnLeader) {
+                        aliasActions.add(IndicesAliasesRequest.AliasActions.remove().index(followerIndex.getName()).alias(aliasName));
+                    }
+
+                    if (aliasActions.isEmpty()) {
+                        handler.accept(leaderIndexMetaData.getAliasesVersion());
+                    } else {
+                        final var request = new IndicesAliasesRequest();
+                        request.origin("ccr");
+                        aliasActions.forEach(request::addAliasAction);
+                        followerClient.admin().indices().aliases(
+                                request,
+                                ActionListener.wrap(r -> handler.accept(leaderIndexMetaData.getAliasesVersion()), errorHandler));
+                    }
+                };
+
+                try {
+                    remoteClient(params).admin().cluster().state(clusterStateRequest, ActionListener.wrap(onResponse, errorHandler));
+                } catch (final NoSuchRemoteClusterException e) {
+                    errorHandler.accept(e);
+                }
+            }
+
             private void closeIndexUpdateSettingsAndOpenIndex(String followIndex,
                                                               Settings updatedSettings,
                                                               Runnable handler,
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java
index 13214b71165aa..8f14f367bc608 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java
@@ -88,15 +88,16 @@ protected void doExecute(Task task, CcrStatsAction.Request request, ActionListen
 
     @Override
     protected void masterOperation(
-            CcrStatsAction.Request request,
-            ClusterState state,
-            ActionListener<CcrStatsAction.Response> listener
+        Task task, CcrStatsAction.Request request,
+        ClusterState state,
+        ActionListener<CcrStatsAction.Response> listener
     ) throws Exception {
         CheckedConsumer<FollowStatsAction.StatsResponses, Exception> handler = statsResponse -> {
             AutoFollowStats stats = autoFollowCoordinator.getStats();
             listener.onResponse(new CcrStatsAction.Response(stats, statsResponse));
         };
         FollowStatsAction.StatsRequest statsRequest = new FollowStatsAction.StatsRequest();
+        statsRequest.setParentTask(clusterService.localNode().getId(), task.getId());
         client.execute(FollowStatsAction.INSTANCE, statsRequest, ActionListener.wrap(handler, listener::onFailure));
     }
 
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java
index 3062aa46d05d6..3623017ac5d35 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternAction.java
@@ -18,12 +18,15 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern;
 import org.elasticsearch.xpack.core.ccr.action.DeleteAutoFollowPatternAction;
 
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -44,13 +47,18 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(DeleteAutoFollowPatternAction.Request request,
+    protected void masterOperation(Task task, DeleteAutoFollowPatternAction.Request request,
                                    ClusterState state,
                                    ActionListener<AcknowledgedResponse> listener) throws Exception {
         clusterService.submitStateUpdateTask("put-auto-follow-pattern-" + request.getName(),
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java
index cb96b8bb29851..64782cda63615 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.ccr.Ccr;
@@ -58,7 +59,7 @@ protected FollowInfoAction.Response read(StreamInput in) throws IOException {
     }
 
     @Override
-    protected void masterOperation(FollowInfoAction.Request request,
+    protected void masterOperation(Task task, FollowInfoAction.Request request,
                                    ClusterState state,
                                    ActionListener<FollowInfoAction.Response> listener) throws Exception {
 
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternAction.java
index f3fd4149ed3e9..6359815ced443 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternAction.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
@@ -57,7 +58,7 @@ protected GetAutoFollowPatternAction.Response read(StreamInput in) throws IOExce
     }
 
     @Override
-    protected void masterOperation(GetAutoFollowPatternAction.Request request,
+    protected void masterOperation(Task task, GetAutoFollowPatternAction.Request request,
                                    ClusterState state,
                                    ActionListener<GetAutoFollowPatternAction.Response> listener) throws Exception {
         Map<String, AutoFollowPattern> autoFollowPatterns = getAutoFollowPattern(state.metaData(), request.getName());
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java
index 2c420f83d6e15..12c0aec2bf3e7 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPauseFollowAction.java
@@ -16,12 +16,15 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.persistent.PersistentTasksService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -47,13 +50,18 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(PauseFollowAction.Request request,
+    protected void masterOperation(Task task, PauseFollowAction.Request request,
                                    ClusterState state,
                                    ActionListener<AcknowledgedResponse> listener) throws Exception {
         PersistentTasksCustomMetaData persistentTasksMetaData = state.metaData().custom(PersistentTasksCustomMetaData.TYPE);
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java
index a218ec2dcaa7c..ee616b423f23a 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java
@@ -21,7 +21,9 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.license.LicenseUtils;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.ccr.CcrLicenseChecker;
@@ -29,6 +31,7 @@
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern;
 import org.elasticsearch.xpack.core.ccr.action.PutAutoFollowPatternAction;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -63,13 +66,18 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(PutAutoFollowPatternAction.Request request,
+    protected void masterOperation(Task task, PutAutoFollowPatternAction.Request request,
                                    ClusterState state,
                                    ActionListener<AcknowledgedResponse> listener) throws Exception {
         if (ccrLicenseChecker.isCcrAllowed() == false) {
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java
index 2a95c2a3c7aab..37d33b5187c43 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.snapshots.RestoreInfo;
 import org.elasticsearch.snapshots.RestoreService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.ccr.CcrLicenseChecker;
@@ -94,9 +95,9 @@ protected PutFollowAction.Response read(StreamInput in) throws IOException {
 
     @Override
     protected void masterOperation(
-            final PutFollowAction.Request request,
-            final ClusterState state,
-            final ActionListener<PutFollowAction.Response> listener) {
+        Task task, final PutFollowAction.Request request,
+        final ClusterState state,
+        final ActionListener<PutFollowAction.Response> listener) {
         if (ccrLicenseChecker.isCcrAllowed() == false) {
             listener.onFailure(LicenseUtils.newComplianceException("ccr"));
             return;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
index ec71c93a33f30..5d9aa0cc8076e 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -41,6 +42,7 @@
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.persistent.PersistentTasksService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.ccr.Ccr;
@@ -100,9 +102,14 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
@@ -111,7 +118,7 @@ protected ClusterBlockException checkBlock(ResumeFollowAction.Request request, C
     }
 
     @Override
-    protected void masterOperation(final ResumeFollowAction.Request request,
+    protected void masterOperation(Task task, final ResumeFollowAction.Request request,
                                    ClusterState state,
                                    final ActionListener<AcknowledgedResponse> listener) throws Exception {
         if (ccrLicenseChecker.isCcrAllowed() == false) {
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java
index e6c889ca6e3c1..4e1337d884319 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportUnfollowAction.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.Index;
@@ -33,6 +34,7 @@
 import org.elasticsearch.index.seqno.RetentionLeaseNotFoundException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.ccr.Ccr;
@@ -40,6 +42,7 @@
 import org.elasticsearch.xpack.ccr.CcrSettings;
 import org.elasticsearch.xpack.core.ccr.action.UnfollowAction;
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
@@ -72,16 +75,21 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
     protected void masterOperation(
-            final UnfollowAction.Request request,
-            final ClusterState state,
-            final ActionListener<AcknowledgedResponse> listener) {
+        Task task, final UnfollowAction.Request request,
+        final ClusterState state,
+        final ActionListener<AcknowledgedResponse> listener) {
         clusterService.submitStateUpdateTask("unfollow_action", new ClusterStateUpdateTask() {
 
             @Override
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsAction.java
index a85e5c50e840d..02af69d9ac3ae 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.ccr.action.bulk;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class BulkShardOperationsAction extends Action<BulkShardOperationsResponse> {
+public class BulkShardOperationsAction extends StreamableResponseActionType<BulkShardOperationsResponse> {
 
     public static final BulkShardOperationsAction INSTANCE = new BulkShardOperationsAction();
     public static final String NAME = "indices:data/write/bulk_shard_operations[s]";
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java
index 45ffbf6998d90..5f8f1d5368a62 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java
@@ -144,10 +144,11 @@ public static CcrWritePrimaryResult shardOperationOnPrimary(
                     assert failure.getSeqNo() == targetOp.seqNo() : targetOp.seqNo() + " != " + failure.getSeqNo();
                     if (failure.getExistingPrimaryTerm().isPresent()) {
                         appliedOperations.add(rewriteOperationWithPrimaryTerm(sourceOp, failure.getExistingPrimaryTerm().getAsLong()));
-                    } else if (targetOp.seqNo() > primary.getGlobalCheckpoint()) {
-                        assert false : "can't find primary_term for existing op=" + targetOp + " gcp=" + primary.getGlobalCheckpoint();
+                    } else if (targetOp.seqNo() > primary.getLastKnownGlobalCheckpoint()) {
+                        assert false :
+                            "can't find primary_term for existing op=" + targetOp + " gcp=" + primary.getLastKnownGlobalCheckpoint();
                         throw new IllegalStateException("can't find primary_term for existing op=" + targetOp +
-                            " global_checkpoint=" + primary.getGlobalCheckpoint(), failure);
+                            " global_checkpoint=" + primary.getLastKnownGlobalCheckpoint(), failure);
                     }
                 } else {
                     assert false : "Only already-processed error should happen; op=[" + targetOp + "] error=[" + result.getFailure() + "]";
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java
index c4651e877fadd..df46535549b7c 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/ClearCcrRestoreSessionAction.java
@@ -6,7 +6,7 @@
 
 package org.elasticsearch.xpack.ccr.action.repositories;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.ActionFilters;
@@ -20,7 +20,7 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.ccr.repository.CcrRestoreSourceService;
 
-public class ClearCcrRestoreSessionAction extends Action<ClearCcrRestoreSessionAction.ClearCcrRestoreSessionResponse> {
+public class ClearCcrRestoreSessionAction extends ActionType<ClearCcrRestoreSessionAction.ClearCcrRestoreSessionResponse> {
 
     public static final ClearCcrRestoreSessionAction INSTANCE = new ClearCcrRestoreSessionAction();
     public static final String NAME = "internal:admin/ccr/restore/session/clear";
@@ -29,11 +29,6 @@ private ClearCcrRestoreSessionAction() {
         super(NAME);
     }
 
-    @Override
-    public ClearCcrRestoreSessionResponse newResponse() {
-        throw new UnsupportedOperationException();
-    }
-
     @Override
     public Writeable.Reader<ClearCcrRestoreSessionResponse> getResponseReader() {
         return ClearCcrRestoreSessionResponse::new;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/DeleteInternalCcrRepositoryAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/DeleteInternalCcrRepositoryAction.java
index 93d432fe93fed..0d2a28e686a56 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/DeleteInternalCcrRepositoryAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/DeleteInternalCcrRepositoryAction.java
@@ -6,7 +6,7 @@
 
 package org.elasticsearch.xpack.ccr.action.repositories;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.ActionFilters;
@@ -20,7 +20,7 @@
 
 import java.io.IOException;
 
-public class DeleteInternalCcrRepositoryAction extends Action<DeleteInternalCcrRepositoryAction.DeleteInternalCcrRepositoryResponse> {
+public class DeleteInternalCcrRepositoryAction extends ActionType<DeleteInternalCcrRepositoryAction.DeleteInternalCcrRepositoryResponse> {
 
     public static final DeleteInternalCcrRepositoryAction INSTANCE = new DeleteInternalCcrRepositoryAction();
     public static final String NAME = "internal:admin/ccr/internal_repository/delete";
@@ -29,11 +29,6 @@ private DeleteInternalCcrRepositoryAction() {
         super(NAME);
     }
 
-    @Override
-    public DeleteInternalCcrRepositoryResponse newResponse() {
-        throw new UnsupportedOperationException();
-    }
-
     @Override
     public Writeable.Reader<DeleteInternalCcrRepositoryResponse> getResponseReader() {
         return DeleteInternalCcrRepositoryResponse::new;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java
index 37dfc84f46a01..b22ca4dab7888 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java
@@ -6,7 +6,7 @@
 
 package org.elasticsearch.xpack.ccr.action.repositories;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.ActionFilters;
@@ -27,7 +27,7 @@
 
 import java.io.IOException;
 
-public class GetCcrRestoreFileChunkAction extends Action<GetCcrRestoreFileChunkAction.GetCcrRestoreFileChunkResponse> {
+public class GetCcrRestoreFileChunkAction extends ActionType<GetCcrRestoreFileChunkAction.GetCcrRestoreFileChunkResponse> {
 
     public static final GetCcrRestoreFileChunkAction INSTANCE = new GetCcrRestoreFileChunkAction();
     public static final String NAME = "internal:admin/ccr/restore/file_chunk/get";
@@ -36,11 +36,6 @@ private GetCcrRestoreFileChunkAction() {
         super(NAME);
     }
 
-    @Override
-    public GetCcrRestoreFileChunkResponse newResponse() {
-        throw new UnsupportedOperationException();
-    }
-
     @Override
     public Writeable.Reader<GetCcrRestoreFileChunkResponse> getResponseReader() {
         return GetCcrRestoreFileChunkResponse::new;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionAction.java
index 91ec057ac4eb6..231821dadfc55 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionAction.java
@@ -6,7 +6,7 @@
 
 package org.elasticsearch.xpack.ccr.action.repositories;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
@@ -30,7 +30,7 @@
 
 import java.io.IOException;
 
-public class PutCcrRestoreSessionAction extends Action<PutCcrRestoreSessionAction.PutCcrRestoreSessionResponse> {
+public class PutCcrRestoreSessionAction extends ActionType<PutCcrRestoreSessionAction.PutCcrRestoreSessionResponse> {
 
     public static final PutCcrRestoreSessionAction INSTANCE = new PutCcrRestoreSessionAction();
     public static final String NAME = "internal:admin/ccr/restore/session/put";
@@ -39,11 +39,6 @@ private PutCcrRestoreSessionAction() {
         super(NAME);
     }
 
-    @Override
-    public PutCcrRestoreSessionResponse newResponse() {
-        return new PutCcrRestoreSessionResponse();
-    }
-
     @Override
     public Writeable.Reader<PutCcrRestoreSessionAction.PutCcrRestoreSessionResponse> getResponseReader() {
         return PutCcrRestoreSessionAction.PutCcrRestoreSessionResponse::new;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionRequest.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionRequest.java
index e348df79223a7..15db362c43c24 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionRequest.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionRequest.java
@@ -16,10 +16,13 @@
 
 public class PutCcrRestoreSessionRequest extends SingleShardRequest<PutCcrRestoreSessionRequest> {
 
-    private String sessionUUID;
-    private ShardId shardId;
+    private final String sessionUUID;
+    private final ShardId shardId;
 
-    PutCcrRestoreSessionRequest() {
+    PutCcrRestoreSessionRequest(StreamInput in) throws IOException {
+        super(in);
+        sessionUUID = in.readString();
+        shardId = new ShardId(in);
     }
 
     public PutCcrRestoreSessionRequest(String sessionUUID, ShardId shardId) {
@@ -33,13 +36,6 @@ public ActionRequestValidationException validate() {
         return null;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        sessionUUID = in.readString();
-        shardId = new ShardId(in);
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutInternalCcrRepositoryAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutInternalCcrRepositoryAction.java
index 397137ffb494a..5f69097f6daa9 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutInternalCcrRepositoryAction.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutInternalCcrRepositoryAction.java
@@ -6,7 +6,7 @@
 
 package org.elasticsearch.xpack.ccr.action.repositories;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.ActionFilters;
@@ -20,7 +20,7 @@
 
 import java.io.IOException;
 
-public class PutInternalCcrRepositoryAction extends Action<PutInternalCcrRepositoryAction.PutInternalCcrRepositoryResponse> {
+public class PutInternalCcrRepositoryAction extends ActionType<PutInternalCcrRepositoryAction.PutInternalCcrRepositoryResponse> {
 
     public static final PutInternalCcrRepositoryAction INSTANCE = new PutInternalCcrRepositoryAction();
     public static final String NAME = "internal:admin/ccr/internal_repository/put";
@@ -29,11 +29,6 @@ private PutInternalCcrRepositoryAction() {
         super(NAME);
     }
 
-    @Override
-    public PutInternalCcrRepositoryResponse newResponse() {
-        throw new UnsupportedOperationException();
-    }
-
     @Override
     public Writeable.Reader<PutInternalCcrRepositoryResponse> getResponseReader() {
         return PutInternalCcrRepositoryResponse::new;
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngine.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngine.java
index 619e0a04baf9a..8d4f0b219bd2c 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngine.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngine.java
@@ -90,7 +90,7 @@ protected InternalEngine.IndexingStrategy indexingStrategyForOperation(final Ind
             } else {
                 return IndexingStrategy.processButSkipLucene(false, index.version());
             }
-        } else if (maxSeqNoOfUpdatesOrDeletes <= getLocalCheckpoint()) {
+        } else if (maxSeqNoOfUpdatesOrDeletes <= getProcessedLocalCheckpoint()) {
             assert maxSeqNoOfUpdatesOrDeletes < index.seqNo() : "seq_no[" + index.seqNo() + "] <= msu[" + maxSeqNoOfUpdatesOrDeletes + "]";
             numOfOptimizedIndexing.inc();
             return InternalEngine.IndexingStrategy.optimizedAppendOnly(index.version());
diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
index 3010f90b803e9..a6406df0fbe7a 100644
--- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
+++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
@@ -255,7 +255,8 @@ public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, Met
 
     @Override
     public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<IndexId> indices, long startTime, String failure, int totalShards,
-                                         List<SnapshotShardFailure> shardFailures, long repositoryStateId, boolean includeGlobalState) {
+                                         List<SnapshotShardFailure> shardFailures, long repositoryStateId, boolean includeGlobalState,
+                                         Map<String, Object> userMetadata) {
         throw new UnsupportedOperationException("Unsupported for repository of type: " + TYPE);
     }
 
@@ -490,10 +491,10 @@ protected void restoreFiles(List<FileInfo> filesToRecover, Store store) throws I
                     while (offset < fileLength && error.get() == null) {
                         final long requestSeqId = requestSeqIdTracker.generateSeqNo();
                         try {
-                            requestSeqIdTracker.waitForOpsToComplete(requestSeqId - ccrSettings.getMaxConcurrentFileChunks());
+                            requestSeqIdTracker.waitForProcessedOpsToComplete(requestSeqId - ccrSettings.getMaxConcurrentFileChunks());
 
                             if (error.get() != null) {
-                                requestSeqIdTracker.markSeqNoAsCompleted(requestSeqId);
+                                requestSeqIdTracker.markSeqNoAsProcessed(requestSeqId);
                                 break;
                             }
 
@@ -513,7 +514,7 @@ protected void restoreFiles(List<FileInfo> filesToRecover, Store store) throws I
                                         @Override
                                         public void onFailure(Exception e) {
                                             error.compareAndSet(null, Tuple.tuple(fileInfo.metadata(), e));
-                                            requestSeqIdTracker.markSeqNoAsCompleted(requestSeqId);
+                                            requestSeqIdTracker.markSeqNoAsProcessed(requestSeqId);
                                         }
 
                                         @Override
@@ -525,24 +526,24 @@ protected void doRun() throws Exception {
                                             throttleListener.accept(nanosPaused);
                                             final boolean lastChunk = r.getOffset() + actualChunkSize >= fileLength;
                                             multiFileWriter.writeFileChunk(fileInfo.metadata(), r.getOffset(), r.getChunk(), lastChunk);
-                                            requestSeqIdTracker.markSeqNoAsCompleted(requestSeqId);
+                                            requestSeqIdTracker.markSeqNoAsProcessed(requestSeqId);
                                         }
                                     }),
                                     e -> {
                                         error.compareAndSet(null, Tuple.tuple(fileInfo.metadata(), e));
-                                        requestSeqIdTracker.markSeqNoAsCompleted(requestSeqId);
+                                        requestSeqIdTracker.markSeqNoAsProcessed(requestSeqId);
                                     }
                                     ), timeout, ThreadPool.Names.GENERIC, GetCcrRestoreFileChunkAction.NAME);
                             remoteClient.execute(GetCcrRestoreFileChunkAction.INSTANCE, request, listener);
                         } catch (Exception e) {
                             error.compareAndSet(null, Tuple.tuple(fileInfo.metadata(), e));
-                            requestSeqIdTracker.markSeqNoAsCompleted(requestSeqId);
+                            requestSeqIdTracker.markSeqNoAsProcessed(requestSeqId);
                         }
                     }
                 }
 
                 try {
-                    requestSeqIdTracker.waitForOpsToComplete(requestSeqIdTracker.getMaxSeqNo());
+                    requestSeqIdTracker.waitForProcessedOpsToComplete(requestSeqIdTracker.getMaxSeqNo());
                 } catch (InterruptedException e) {
                     Thread.currentThread().interrupt();
                     throw new ElasticsearchException(e);
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
index fd7c8bd4086ef..ce1120a02373e 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
@@ -257,16 +257,6 @@ public Collection<Class<? extends Plugin>> nodePlugins() {
                         CcrIntegTestCase.this.nodePlugins().stream())
                         .collect(Collectors.toList());
             }
-
-            @Override
-            public Settings transportClientSettings() {
-                return super.transportClientSettings();
-            }
-
-            @Override
-            public Collection<Class<? extends Plugin>> transportClientPlugins() {
-                return Arrays.asList(LocalStateCcr.class, getTestTransportPlugin());
-            }
         };
     }
 
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
index 0bcb3daac6284..b971182e3d7cc 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
@@ -8,7 +8,6 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Strings;
@@ -18,14 +17,15 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xpack.CcrIntegTestCase;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
 import org.elasticsearch.xpack.core.ccr.AutoFollowStats;
 import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
 import org.elasticsearch.xpack.core.ccr.action.DeleteAutoFollowPatternAction;
 import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction;
-import org.elasticsearch.xpack.core.ccr.action.FollowParameters;
 import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.FollowerInfo;
+import org.elasticsearch.xpack.core.ccr.action.FollowParameters;
 import org.elasticsearch.xpack.core.ccr.action.PutAutoFollowPatternAction;
 
 import java.util.Arrays;
@@ -68,22 +68,17 @@ public void testAutoFollow() throws Exception {
 
         createLeaderIndex("logs-201901", leaderIndexSettings);
         assertBusy(() -> {
-            IndicesExistsRequest request = new IndicesExistsRequest("copy-logs-201901");
-            assertTrue(followerClient().admin().indices().exists(request).actionGet().isExists());
+            assertTrue(ESIntegTestCase.indexExists("copy-logs-201901", followerClient()));
         });
         createLeaderIndex("transactions-201901", leaderIndexSettings);
         assertBusy(() -> {
             AutoFollowStats autoFollowStats = getAutoFollowStats();
             assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(2L));
-
-            IndicesExistsRequest request = new IndicesExistsRequest("copy-transactions-201901");
-            assertTrue(followerClient().admin().indices().exists(request).actionGet().isExists());
+            assertTrue(ESIntegTestCase.indexExists("copy-transactions-201901", followerClient()));
         });
 
-        IndicesExistsRequest request = new IndicesExistsRequest("copy-metrics-201901");
-        assertFalse(followerClient().admin().indices().exists(request).actionGet().isExists());
-        request = new IndicesExistsRequest("copy-logs-201812");
-        assertFalse(followerClient().admin().indices().exists(request).actionGet().isExists());
+        assertFalse(ESIntegTestCase.indexExists("copy-metrics-201901", followerClient()));
+        assertFalse(ESIntegTestCase.indexExists("copy-logs-201812", followerClient()));
     }
 
     public void testCleanFollowedLeaderIndexUUIDs() throws Exception {
@@ -99,8 +94,7 @@ public void testCleanFollowedLeaderIndexUUIDs() throws Exception {
             AutoFollowStats autoFollowStats = getAutoFollowStats();
             assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(1L));
 
-            IndicesExistsRequest request = new IndicesExistsRequest("copy-logs-201901");
-            assertTrue(followerClient().admin().indices().exists(request).actionGet().isExists());
+            assertTrue(ESIntegTestCase.indexExists("copy-logs-201901", followerClient()));
 
             MetaData metaData = getFollowerCluster().clusterService().state().metaData();
             String leaderIndexUUID = metaData.index("copy-logs-201901")
@@ -329,8 +323,7 @@ public void testConflictingPatterns() throws Exception {
             assertThat(autoFollowStats.getNumberOfFailedFollowIndices(), equalTo(0L));
             assertThat(autoFollowStats.getNumberOfFailedRemoteClusterStateRequests(), equalTo(0L));
         });
-        IndicesExistsRequest request = new IndicesExistsRequest("copy-logs-201701");
-        assertTrue(followerClient().admin().indices().exists(request).actionGet().isExists());
+        assertTrue(ESIntegTestCase.indexExists("copy-logs-201701", followerClient()));
 
         createLeaderIndex("logs-201801", leaderIndexSettings);
         assertBusy(() -> {
@@ -351,8 +344,7 @@ public void testConflictingPatterns() throws Exception {
                 "matches with other patterns [my-pattern1]"));
         });
 
-        request = new IndicesExistsRequest("copy-logs-201801");
-        assertFalse(followerClient().admin().indices().exists(request).actionGet().isExists());
+        assertFalse(ESIntegTestCase.indexExists("copy-logs-201801", followerClient()));
     }
 
     public void testAutoFollowSoftDeletesDisabled() throws Exception {
@@ -373,8 +365,7 @@ public void testAutoFollowSoftDeletesDisabled() throws Exception {
             ElasticsearchException failure  = autoFollowStats.getRecentAutoFollowErrors().firstEntry().getValue().v2();
             assertThat(failure.getMessage(), equalTo("index [logs-20200101] cannot be followed, " +
                 "because soft deletes are not enabled"));
-            IndicesExistsRequest request = new IndicesExistsRequest("copy-logs-20200101");
-            assertFalse(followerClient().admin().indices().exists(request).actionGet().isExists());
+            assertFalse(ESIntegTestCase.indexExists("copy-logs-20200101", followerClient()));
         });
 
         // Soft deletes are enabled:
@@ -387,8 +378,7 @@ public void testAutoFollowSoftDeletesDisabled() throws Exception {
         assertBusy(() -> {
             AutoFollowStats autoFollowStats = getAutoFollowStats();
             assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(1L));
-            IndicesExistsRequest request = new IndicesExistsRequest("copy-logs-20200102");
-            assertTrue(followerClient().admin().indices().exists(request).actionGet().isExists());
+            assertTrue(ESIntegTestCase.indexExists("copy-logs-20200102", followerClient()));
         });
     }
 
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java
index 69e41ffddab43..4242c2fde04bc 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java
@@ -8,18 +8,17 @@
 
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
 
-public class CCRFeatureSetUsageTests extends AbstractWireSerializingTestCase<CCRFeatureSet.Usage> {
+public class CCRFeatureSetUsageTests extends AbstractWireSerializingTestCase<CCRInfoTransportAction.Usage> {
 
     @Override
-    protected CCRFeatureSet.Usage createTestInstance() {
-        return new CCRFeatureSet.Usage(randomBoolean(), randomBoolean(), randomIntBetween(0, Integer.MAX_VALUE),
+    protected CCRInfoTransportAction.Usage createTestInstance() {
+        return new CCRInfoTransportAction.Usage(randomBoolean(), randomBoolean(), randomIntBetween(0, Integer.MAX_VALUE),
             randomIntBetween(0, Integer.MAX_VALUE), randomNonNegativeLong());
     }
 
     @Override
-    protected Writeable.Reader<CCRFeatureSet.Usage> instanceReader() {
-        return CCRFeatureSet.Usage::new;
+    protected Writeable.Reader<CCRInfoTransportAction.Usage> instanceReader() {
+        return CCRInfoTransportAction.Usage::new;
     }
 }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java
similarity index 75%
rename from x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java
rename to x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java
index f2fb475816ec2..8c7310a0a1bf3 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.ccr;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -15,9 +16,9 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
-import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
 import org.junit.Before;
 import org.mockito.Mockito;
 
@@ -31,7 +32,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class CCRFeatureSetTests extends ESTestCase {
+public class CCRInfoTransportActionTests extends ESTestCase {
 
     private XPackLicenseState licenseState;
     private ClusterService clusterService;
@@ -43,38 +44,33 @@ public void init() {
     }
 
     public void testAvailable() {
-        CCRFeatureSet featureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService);
+        CCRInfoTransportAction featureSet = new CCRInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
 
         when(licenseState.isCcrAllowed()).thenReturn(false);
         assertThat(featureSet.available(), equalTo(false));
 
         when(licenseState.isCcrAllowed()).thenReturn(true);
         assertThat(featureSet.available(), equalTo(true));
-
-        featureSet = new CCRFeatureSet(Settings.EMPTY, null, clusterService);
-        assertThat(featureSet.available(), equalTo(false));
     }
 
     public void testEnabled() {
         Settings.Builder settings = Settings.builder().put("xpack.ccr.enabled", false);
-        CCRFeatureSet featureSet = new CCRFeatureSet(settings.build(), licenseState, clusterService);
+        CCRInfoTransportAction featureSet = new CCRInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState);
         assertThat(featureSet.enabled(), equalTo(false));
 
         settings = Settings.builder().put("xpack.ccr.enabled", true);
-        featureSet = new CCRFeatureSet(settings.build(), licenseState, clusterService);
+        featureSet = new CCRInfoTransportAction(mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState);
         assertThat(featureSet.enabled(), equalTo(true));
     }
 
     public void testName() {
-        CCRFeatureSet featureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService);
+        CCRInfoTransportAction featureSet = new CCRInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
         assertThat(featureSet.name(), equalTo("ccr"));
     }
 
-    public void testNativeCodeInfo() {
-        CCRFeatureSet featureSet = new CCRFeatureSet (Settings.EMPTY, licenseState, clusterService);
-        assertNull(featureSet.nativeCodeInfo());
-    }
-
     public void testUsageStats() throws Exception {
         MetaData.Builder metaData = MetaData.builder();
 
@@ -109,12 +105,13 @@ public void testUsageStats() throws Exception {
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metaData(metaData).build();
         Mockito.when(clusterService.state()).thenReturn(clusterState);
 
-        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
-        CCRFeatureSet ccrFeatureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService);
-        ccrFeatureSet.usage(future);
-        CCRFeatureSet.Usage ccrUsage = (CCRFeatureSet.Usage) future.get();
-        assertThat(ccrUsage.enabled(), equalTo(ccrFeatureSet.enabled()));
-        assertThat(ccrUsage.available(), equalTo(ccrFeatureSet.available()));
+        var usageAction = new CCRUsageTransportAction(mock(TransportService.class), null, null,
+            mock(ActionFilters.class), null, Settings.EMPTY, licenseState);
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, clusterState, future);
+        CCRInfoTransportAction.Usage ccrUsage = (CCRInfoTransportAction.Usage) future.get().getUsage();
+        assertThat(ccrUsage.enabled(), equalTo(true));
+        assertThat(ccrUsage.available(), equalTo(false));
 
         assertThat(ccrUsage.getNumberOfFollowerIndices(), equalTo(numFollowerIndices));
         if (numFollowerIndices != 0) {
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrAliasesIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrAliasesIT.java
new file mode 100644
index 0000000000000..31ce722b33c04
--- /dev/null
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrAliasesIT.java
@@ -0,0 +1,412 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.ccr;
+
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.common.CheckedRunnable;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException;
+import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.xpack.CcrIntegTestCase;
+import org.elasticsearch.xpack.ccr.action.ShardFollowTask;
+import org.elasticsearch.xpack.core.ccr.action.PutFollowAction;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutionException;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+
+public class CcrAliasesIT extends CcrIntegTestCase {
+
+    public void testAliasOnIndexCreation() throws Exception {
+        final String aliasName = randomAlphaOfLength(16);
+        final String aliases;
+        try (XContentBuilder builder = jsonBuilder()) {
+            builder.startObject();
+            {
+                builder.startObject("aliases");
+                {
+                    builder.startObject(aliasName);
+                    {
+
+                    }
+                    builder.endObject();
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            aliases = BytesReference.bytes(builder).utf8ToString();
+        }
+        assertAcked(leaderClient().admin().indices().prepareCreate("leader").setSource(aliases, XContentType.JSON));
+        final PutFollowAction.Request followRequest = putFollow("leader", "follower");
+        followerClient().execute(PutFollowAction.INSTANCE, followRequest).get();
+
+        ensureFollowerGreen(true, "follower");
+
+        // wait for the shard follow task to exist
+        assertBusy(() -> assertShardFollowTask(1));
+
+        assertAliasesExist("leader", "follower", aliasName);
+    }
+
+    public void testAddAlias() throws Exception {
+        runAddAliasTest(null);
+    }
+
+    public void testAddExplicitNotWriteAlias() throws Exception {
+        runAddAliasTest(false);
+    }
+
+    public void testWriteAliasIsIgnored() throws Exception {
+        runAddAliasTest(true);
+    }
+
+    private void runAddAliasTest(final Boolean isWriteAlias) throws Exception {
+        runAddAliasTest(isWriteAlias, aliasName -> {});
+    }
+
+    /**
+     * Runs an add alias test which adds a random alias to the leader exist, and then asserts that the alias is replicated to the follower.
+     * The specified post assertions gives the caller the opportunity to add additional assertions on the alias that is added. These
+     * assertions are executed after all other assertions that the alias exists.
+     *
+     * @param isWriteIndex   whether or not the leader index is the write index for the alias
+     * @param postAssertions the post assertions to execute
+     * @param <E>            the type of checked exception the post assertions callback can throw
+     * @throws Exception if a checked exception is thrown while executing the add alias test
+     */
+    private <E extends Exception> void runAddAliasTest(
+            final Boolean isWriteIndex,
+            final CheckedConsumer<String, E> postAssertions) throws Exception {
+        assertAcked(leaderClient().admin().indices().prepareCreate("leader"));
+        final PutFollowAction.Request followRequest = putFollow("leader", "follower");
+        // we set a low poll timeout so that shard changes requests are responded to quickly even without indexing
+        followRequest.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(100));
+        followerClient().execute(PutFollowAction.INSTANCE, followRequest).get();
+
+        ensureFollowerGreen(true, "follower");
+
+        assertBusy(() -> assertShardFollowTask(1));
+
+        final String aliasName = randomAlphaOfLength(16);
+        addRandomAlias("leader", aliasName, isWriteIndex);
+
+        assertAliasesExist("leader", "follower", aliasName);
+
+        postAssertions.accept(aliasName);
+    }
+
+    private void addRandomAlias(final String index, final String aliasName, final Boolean isWriteIndex) {
+        final IndicesAliasesRequest.AliasActions add = IndicesAliasesRequest.AliasActions.add();
+        add.index(index);
+        add.alias(aliasName);
+        add.writeIndex(isWriteIndex);
+        if (randomBoolean()) {
+            add.routing(randomAlphaOfLength(16));
+        } else {
+            if (randomBoolean()) {
+                add.indexRouting(randomAlphaOfLength(16));
+            }
+            if (randomBoolean()) {
+                add.searchRouting(randomAlphaOfLength(16));
+            }
+        }
+        if (randomBoolean()) {
+            add.filter(termQuery(randomAlphaOfLength(16), randomAlphaOfLength(16)));
+        }
+
+        assertAcked(leaderClient().admin().indices().prepareAliases().addAliasAction(add));
+    }
+
+    public void testAddMultipleAliasesAtOnce() throws Exception {
+        assertAcked(leaderClient().admin().indices().prepareCreate("leader"));
+        final PutFollowAction.Request followRequest = putFollow("leader", "follower");
+        // we set a low poll timeout so that shard changes requests are responded to quickly even without indexing
+        followRequest.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(100));
+        followerClient().execute(PutFollowAction.INSTANCE, followRequest).get();
+
+        ensureFollowerGreen(true, "follower");
+
+        assertBusy(() -> assertShardFollowTask(1));
+
+        final int numberOfAliases = randomIntBetween(2, 8);
+        final IndicesAliasesRequestBuilder builder = leaderClient().admin().indices().prepareAliases();
+        for (int i = 0; i < numberOfAliases; i++) {
+            builder.addAlias("leader", "alias_" + i);
+        }
+        assertAcked(builder);
+
+        final String[] aliases = new String[numberOfAliases];
+        for (int i = 0; i < numberOfAliases; i++) {
+            aliases[i] = "alias_" + i;
+        }
+        assertAliasesExist("leader", "follower", aliases);
+    }
+
+    public void testAddMultipleAliasesSequentially() throws Exception {
+        assertAcked(leaderClient().admin().indices().prepareCreate("leader"));
+        final PutFollowAction.Request followRequest = putFollow("leader", "follower");
+        // we set a low poll timeout so that shard changes requests are responded to quickly even without indexing
+        followRequest.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(100));
+        followerClient().execute(PutFollowAction.INSTANCE, followRequest).get();
+
+        ensureFollowerGreen(true, "follower");
+
+        assertBusy(() -> assertShardFollowTask(1));
+
+        final int numberOfAliases = randomIntBetween(2, 8);
+        for (int i = 0; i < numberOfAliases; i++) {
+            assertAcked(leaderClient().admin().indices().prepareAliases().addAlias("leader", "alias_" + i));
+
+            final String[] aliases = new String[i + 1];
+            for (int j = 0; j < i + 1; j++) {
+                aliases[j] = "alias_" + j;
+            }
+            assertAliasesExist("leader", "follower", aliases);
+        }
+    }
+
+    public void testUpdateExistingAlias() throws Exception {
+        runAddAliasTest(
+                null,
+                /*
+                 * After the alias is added (via runAddAliasTest) we modify the alias in place, and then assert that the modification is
+                 * eventually replicated.
+                 */
+                aliasName -> {
+                    assertAcked(leaderClient().admin()
+                            .indices()
+                            .prepareAliases()
+                            .addAlias("leader", aliasName, termQuery(randomAlphaOfLength(16), randomAlphaOfLength(16))));
+                    assertAliasesExist("leader", "follower", aliasName);
+                });
+    }
+
+    public void testRemoveExistingAlias() throws Exception {
+        runAddAliasTest(
+                false,
+                aliasName -> {
+                    removeAlias(aliasName);
+                    assertAliasExistence(aliasName, false);
+                }
+        );
+    }
+
+    private void removeAlias(final String aliasName) {
+        assertAcked(leaderClient().admin().indices().prepareAliases().removeAlias("leader", aliasName));
+    }
+
+    public void testStress() throws Exception {
+        assertAcked(leaderClient().admin().indices().prepareCreate("leader"));
+        final PutFollowAction.Request followRequest = putFollow("leader", "follower");
+        // we set a low poll timeout so that shard changes requests are responded to quickly even without indexing
+        followRequest.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(100));
+        followerClient().execute(PutFollowAction.INSTANCE, followRequest).get();
+
+        final int numberOfThreads = randomIntBetween(2, 4);
+        final int numberOfIterations = randomIntBetween(4, 32);
+        final CyclicBarrier barrier = new CyclicBarrier(numberOfThreads + 1);
+        final List<Thread> threads = new ArrayList<>(numberOfThreads);
+
+        for (int i = 0; i < numberOfThreads; i++) {
+            final Thread thread = new Thread(() -> {
+                try {
+                    barrier.await();
+                } catch (final BrokenBarrierException | InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+                for (int j = 0; j < numberOfIterations; j++) {
+                    final String action = randomFrom("create", "update", "delete");
+                    switch (action) {
+                        case "create":
+                            addRandomAlias("leader", randomAlphaOfLength(16), randomFrom(new Boolean[] { null, false, true }));
+                            break;
+                        case "update":
+                            try {
+                                final String[] aliases = getAliasesOnLeader();
+                                if (aliases.length == 0) {
+                                    continue;
+                                }
+                                final String alias = randomFrom(aliases);
+                                /*
+                                 * Add an alias with the same name, which acts as an update (although another thread could concurrently
+                                 * remove).
+                                 */
+                                addRandomAlias("leader", alias, randomFrom(new Boolean[] { null, false, true }));
+                            } catch (final Exception e) {
+                                throw new RuntimeException(e);
+                            }
+                            break;
+                        case "delete":
+                            try {
+                                final String[] aliases = getAliasesOnLeader();
+                                if (aliases.length == 0) {
+                                    continue;
+                                }
+                                final String alias = randomFrom(aliases);
+                                try {
+                                    removeAlias(alias);
+                                } catch (final AliasesNotFoundException e) {
+                                    // ignore, it could have been deleted by another thread
+                                    continue;
+                                }
+                            } catch (final Exception e) {
+                                throw new RuntimeException(e);
+                            }
+                            break;
+                        default:
+                            assert false : action;
+                    }
+                }
+                try {
+                    barrier.await();
+                } catch (final BrokenBarrierException | InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+            });
+            thread.start();
+            threads.add(thread);
+        }
+        barrier.await();
+
+        barrier.await();
+
+        for (final Thread thread : threads) {
+            thread.join();
+        }
+
+        assertAliasesExist("leader", "follower", getAliasesOnLeader());
+    }
+
+    private String[] getAliasesOnLeader() throws InterruptedException, ExecutionException {
+        final GetAliasesResponse response = leaderClient().admin().indices().getAliases(new GetAliasesRequest().indices("leader")).get();
+        return response.getAliases().get("leader").stream().map(AliasMetaData::alias).toArray(String[]::new);
+    }
+
+    private void assertAliasesExist(final String leaderIndex, final String followerIndex, final String... aliases) throws Exception {
+        assertAliasesExist(leaderIndex, followerIndex, (alias, aliasMetaData) -> {}, aliases);
+    }
+
+    private <E extends Exception> void assertAliasesExist(
+            final String leaderIndex,
+            final String followerIndex,
+            final CheckedBiConsumer<String, AliasMetaData, E> aliasMetaDataAssertion,
+            final String... aliases) throws Exception {
+        // we must check serially because aliases exist will return true if any but not necessarily all of the requested aliases exist
+        for (final String alias : aliases) {
+            assertAliasExistence(alias, true);
+        }
+
+        assertBusy(() -> {
+            final GetAliasesResponse followerResponse =
+                    followerClient().admin().indices().getAliases(new GetAliasesRequest().indices(followerIndex)).get();
+            assertThat(
+                    "expected follower to have [" + aliases.length + "] aliases, but was " + followerResponse.getAliases().toString(),
+                    followerResponse.getAliases().get(followerIndex),
+                    hasSize(aliases.length));
+            for (final String alias : aliases) {
+                final AliasMetaData followerAliasMetaData = getAliasMetaData(followerResponse, followerIndex, alias);
+
+                final GetAliasesResponse leaderResponse =
+                        leaderClient().admin().indices().getAliases(new GetAliasesRequest().indices(leaderIndex).aliases(alias)).get();
+                final AliasMetaData leaderAliasMetaData = getAliasMetaData(leaderResponse, leaderIndex, alias);
+
+                assertThat(
+                        "alias [" + alias + "] index routing did not replicate, but was " + followerAliasMetaData.toString(),
+                        followerAliasMetaData.indexRouting(), equalTo(leaderAliasMetaData.indexRouting()));
+                assertThat(
+                        "alias [" + alias + "] search routing did not replicate, but was " + followerAliasMetaData.toString(),
+                        followerAliasMetaData.searchRoutingValues(), equalTo(leaderAliasMetaData.searchRoutingValues()));
+                assertThat(
+                        "alias [" + alias + "] filtering did not replicate, but was " + followerAliasMetaData.toString(),
+                        followerAliasMetaData.filter(), equalTo(leaderAliasMetaData.filter()));
+                assertThat(
+                        "alias [" + alias + "] should not be a write index, but was " + followerAliasMetaData.toString(),
+                        followerAliasMetaData.writeIndex(),
+                        equalTo(false));
+                aliasMetaDataAssertion.accept(alias, followerAliasMetaData);
+            }
+        });
+    }
+
+    private void assertAliasExistence(final String alias, final boolean exists) throws Exception {
+        assertBusy(() -> {
+            // we must check serially because aliases exist will return true if any but not necessarily all of the requested aliases exist
+            final GetAliasesResponse response = followerClient().admin()
+                    .indices()
+                    .getAliases(new GetAliasesRequest().indices("follower").aliases(alias))
+                    .get();
+            if (exists) {
+                assertFalse("alias [" + alias + "] did not exist", response.getAliases().isEmpty());
+            } else {
+                assertTrue("alias [" + alias + "] exists", response.getAliases().isEmpty());
+            }
+        });
+    }
+
+    private AliasMetaData getAliasMetaData(final GetAliasesResponse response, final String index, final String alias) {
+        final Optional<AliasMetaData> maybeAliasMetaData =
+                response.getAliases().get(index).stream().filter(a -> a.getAlias().equals(alias)).findFirst();
+        assertTrue("alias [" + alias + "] did not exist", maybeAliasMetaData.isPresent());
+        return maybeAliasMetaData.get();
+    }
+
+    private CheckedRunnable<Exception> assertShardFollowTask(final int numberOfPrimaryShards) {
+        return () -> {
+            final ClusterState clusterState = followerClient().admin().cluster().prepareState().get().getState();
+            final PersistentTasksCustomMetaData taskMetadata = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
+            assertNotNull("task metadata for follower should exist", taskMetadata);
+
+            final ListTasksRequest listTasksRequest = new ListTasksRequest();
+            listTasksRequest.setDetailed(true);
+            listTasksRequest.setActions(ShardFollowTask.NAME + "[c]");
+            final ListTasksResponse listTasksResponse = followerClient().admin().cluster().listTasks(listTasksRequest).actionGet();
+            assertThat("expected no node failures", listTasksResponse.getNodeFailures().size(), equalTo(0));
+            assertThat("expected no task failures", listTasksResponse.getTaskFailures().size(), equalTo(0));
+
+            final List<TaskInfo> taskInfos = listTasksResponse.getTasks();
+            assertThat("expected a task for each shard", taskInfos.size(), equalTo(numberOfPrimaryShards));
+            final Collection<PersistentTasksCustomMetaData.PersistentTask<?>> shardFollowTasks =
+                    taskMetadata.findTasks(ShardFollowTask.NAME, Objects::nonNull);
+            for (final PersistentTasksCustomMetaData.PersistentTask<?> shardFollowTask : shardFollowTasks) {
+                TaskInfo taskInfo = null;
+                final String expectedId = "id=" + shardFollowTask.getId();
+                for (final TaskInfo info : taskInfos) {
+                    if (expectedId.equals(info.getDescription())) {
+                        taskInfo = info;
+                        break;
+                    }
+                }
+                assertNotNull("task info for shard follow task [" + expectedId + "] should exist", taskInfo);
+            }
+        };
+    }
+
+}
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java
index 9595485e2721c..2f0aed395a73c 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java
@@ -6,7 +6,6 @@
 
 package org.elasticsearch.xpack.ccr;
 
-import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
@@ -44,7 +43,6 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.snapshots.RestoreInfo;
 import org.elasticsearch.snapshots.RestoreService;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.RemoteTransportException;
@@ -88,7 +86,6 @@
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
 
-@TestLogging(value = "org.elasticsearch.xpack.ccr:trace,org.elasticsearch.indices.recovery:trace")
 public class CcrRetentionLeaseIT extends CcrIntegTestCase {
 
     public static final class RetentionLeaseRenewIntervalSettingPlugin extends Plugin {
@@ -191,6 +188,7 @@ public void testRetentionLeaseIsTakenAtTheStartOfRecovery() throws Exception {
             assertThat(stats.getShards(), arrayWithSize(numberOfShards * (1 + numberOfReplicas)));
             final List<ShardStats> shardsStats = getShardsStats(stats);
             for (int i = 0; i < numberOfShards * (1 + numberOfReplicas); i++) {
+                assertNotNull(shardsStats.get(i).getRetentionLeaseStats());
                 final RetentionLeases currentRetentionLeases = shardsStats.get(i).getRetentionLeaseStats().retentionLeases();
                 assertThat(Strings.toString(shardsStats.get(i)), currentRetentionLeases.leases(), hasSize(1));
                 final RetentionLease retentionLease =
@@ -224,9 +222,9 @@ public void testRetentionLeaseIsRenewedDuringRecovery() throws Exception {
 
         // block the recovery from completing; this ensures the background sync is still running
         final ClusterStateResponse followerClusterState = followerClient().admin().cluster().prepareState().clear().setNodes(true).get();
-        for (final ObjectCursor<DiscoveryNode> senderNode : followerClusterState.getState().nodes().getNodes().values()) {
+        for (final DiscoveryNode senderNode : followerClusterState.getState().nodes()) {
             final MockTransportService senderTransportService =
-                    (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.value.getName());
+                    (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.getName());
             senderTransportService.addSendBehavior(
                     (connection, requestId, action, request, options) -> {
                         if (ClearCcrRestoreSessionAction.NAME.equals(action)
@@ -248,9 +246,9 @@ public void testRetentionLeaseIsRenewedDuringRecovery() throws Exception {
             assertRetentionLeaseRenewal(numberOfShards, numberOfReplicas, followerIndex, leaderIndex);
             latch.countDown();
         } finally {
-            for (final ObjectCursor<DiscoveryNode> senderNode : followerClusterState.getState().nodes().getDataNodes().values()) {
+            for (final DiscoveryNode senderNode : followerClusterState.getState().nodes()) {
                 final MockTransportService senderTransportService =
-                        (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.value.getName());
+                        (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.getName());
                 senderTransportService.clearAllRules();
             }
         }
@@ -304,45 +302,47 @@ public void testRetentionLeasesAreNotBeingRenewedAfterRecoveryCompletes() throws
                 leaderClient().admin().cluster().prepareState().clear().setMetaData(true).setIndices(leaderIndex).get();
         final String leaderUUID = leaderIndexClusterState.getState().metaData().index(leaderIndex).getIndexUUID();
 
-        // sample the leases after recovery
-        final List<RetentionLeases> retentionLeases = new ArrayList<>();
+        /*
+         * We want to ensure that the background renewal is cancelled at the end of recovery. To do this, we will sleep a small multiple
+         * of the renew interval. If the renews are not cancelled, we expect that a renewal would have been sent while we were sleeping.
+         * After we wake up, it should be the case that the retention leases are the same (same timestamp) as that indicates that they were
+         * not renewed while we were sleeping.
+         */
         assertBusy(() -> {
-            retentionLeases.clear();
-            final IndicesStatsResponse stats =
+            // sample the leases after recovery
+            final List<RetentionLeases> retentionLeases = new ArrayList<>();
+            assertBusy(() -> {
+                retentionLeases.clear();
+                final IndicesStatsResponse stats =
                     leaderClient().admin().indices().stats(new IndicesStatsRequest().clear().indices(leaderIndex)).actionGet();
-            assertNotNull(stats.getShards());
-            assertThat(stats.getShards(), arrayWithSize(numberOfShards * (1 + numberOfReplicas)));
-            final List<ShardStats> shardsStats = getShardsStats(stats);
-            for (int i = 0; i < numberOfShards * (1 + numberOfReplicas); i++) {
-                final RetentionLeases currentRetentionLeases = shardsStats.get(i).getRetentionLeaseStats().retentionLeases();
-                assertThat(Strings.toString(shardsStats.get(i)), currentRetentionLeases.leases(), hasSize(1));
-                final ClusterStateResponse followerIndexClusterState =
+                assertNotNull(stats.getShards());
+                assertThat(stats.getShards(), arrayWithSize(numberOfShards * (1 + numberOfReplicas)));
+                final List<ShardStats> shardsStats = getShardsStats(stats);
+                for (int i = 0; i < numberOfShards * (1 + numberOfReplicas); i++) {
+                    assertNotNull(shardsStats.get(i).getRetentionLeaseStats());
+                    final RetentionLeases currentRetentionLeases = shardsStats.get(i).getRetentionLeaseStats().retentionLeases();
+                    assertThat(Strings.toString(shardsStats.get(i)), currentRetentionLeases.leases(), hasSize(1));
+                    final ClusterStateResponse followerIndexClusterState =
                         followerClient().admin().cluster().prepareState().clear().setMetaData(true).setIndices(followerIndex).get();
-                final String followerUUID = followerIndexClusterState.getState().metaData().index(followerIndex).getIndexUUID();
-                final RetentionLease retentionLease =
+                    final String followerUUID = followerIndexClusterState.getState().metaData().index(followerIndex).getIndexUUID();
+                    final RetentionLease retentionLease =
                         currentRetentionLeases.leases().iterator().next();
-                final String expectedRetentionLeaseId = retentionLeaseId(
+                    final String expectedRetentionLeaseId = retentionLeaseId(
                         getFollowerCluster().getClusterName(),
                         new Index(followerIndex, followerUUID),
                         getLeaderCluster().getClusterName(),
                         new Index(leaderIndex, leaderUUID));
-                assertThat(retentionLease.id(), equalTo(expectedRetentionLeaseId));
-                retentionLeases.add(currentRetentionLeases);
-            }
-        });
-
-        /*
-         * We want to ensure that the background renewal is cancelled at the end of recovery. To do this, we will sleep a small multiple
-         * of the renew interval. If the renews are not cancelled, we expect that a renewal would have been sent while we were sleeping.
-         * After we wake up, it should be the case that the retention leases are the same (same timestamp) as that indicates that they were
-         * not renewed while we were sleeping.
-         */
-        final TimeValue renewIntervalSetting = CcrRetentionLeases.RETENTION_LEASE_RENEW_INTERVAL_SETTING.get(followerClusterSettings());
-        final long renewEnd = System.nanoTime();
-        Thread.sleep(Math.max(0, randomIntBetween(2, 4) * renewIntervalSetting.millis() - TimeUnit.NANOSECONDS.toMillis(renewEnd - start)));
-
-        // now ensure that the retention leases are the same
-        assertBusy(() -> {
+                    assertThat(retentionLease.id(), equalTo(expectedRetentionLeaseId));
+                    retentionLeases.add(currentRetentionLeases);
+                }
+            });
+            // sleep a small multiple of the renew interval
+            final TimeValue renewIntervalSetting = CcrRetentionLeases.RETENTION_LEASE_RENEW_INTERVAL_SETTING.get(followerClusterSettings());
+            final long renewEnd = System.nanoTime();
+            Thread.sleep(
+                Math.max(0, randomIntBetween(2, 4) * renewIntervalSetting.millis() - TimeUnit.NANOSECONDS.toMillis(renewEnd - start)));
+
+            // now ensure that the retention leases are the same
             final IndicesStatsResponse stats =
                     leaderClient().admin().indices().stats(new IndicesStatsRequest().clear().indices(leaderIndex)).actionGet();
             assertNotNull(stats.getShards());
@@ -352,6 +352,7 @@ public void testRetentionLeasesAreNotBeingRenewedAfterRecoveryCompletes() throws
                 if (shardsStats.get(i).getShardRouting().primary() == false) {
                     continue;
                 }
+                assertNotNull(shardsStats.get(i).getRetentionLeaseStats());
                 final RetentionLeases currentRetentionLeases = shardsStats.get(i).getRetentionLeaseStats().retentionLeases();
                 assertThat(Strings.toString(shardsStats.get(i)), currentRetentionLeases.leases(), hasSize(1));
                 final ClusterStateResponse followerIndexClusterState =
@@ -405,9 +406,9 @@ public void testUnfollowRemovesRetentionLeases() throws Exception {
 
         final ClusterStateResponse followerClusterState = followerClient().admin().cluster().prepareState().clear().setNodes(true).get();
         try {
-            for (final ObjectCursor<DiscoveryNode> senderNode : followerClusterState.getState().nodes().getNodes().values()) {
+            for (final DiscoveryNode senderNode : followerClusterState.getState().nodes()) {
                 final MockTransportService senderTransportService =
-                        (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.value.getName());
+                        (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.getName());
                 senderTransportService.addSendBehavior(
                         (connection, requestId, action, request, options) -> {
                             if (RetentionLeaseActions.Remove.ACTION_NAME.equals(action)
@@ -456,9 +457,9 @@ public void testUnfollowRemovesRetentionLeases() throws Exception {
                 assertThat(Strings.toString(shardStats), shardStats.getRetentionLeaseStats().retentionLeases().leases(), empty());
             }
         } finally {
-            for (final ObjectCursor<DiscoveryNode> senderNode : followerClusterState.getState().nodes().getDataNodes().values()) {
+            for (final DiscoveryNode senderNode : followerClusterState.getState().nodes()) {
                 final MockTransportService senderTransportService =
-                        (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.value.getName());
+                        (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.getName());
                 senderTransportService.clearAllRules();
             }
         }
@@ -488,9 +489,9 @@ public void testUnfollowFailsToRemoveRetentionLeases() throws Exception {
 
         final ClusterStateResponse followerClusterState = followerClient().admin().cluster().prepareState().clear().setNodes(true).get();
         try {
-            for (final ObjectCursor<DiscoveryNode> senderNode : followerClusterState.getState().nodes().getNodes().values()) {
+            for (final DiscoveryNode senderNode : followerClusterState.getState().nodes()) {
                 final MockTransportService senderTransportService =
-                        (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.value.getName());
+                        (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.getName());
                 senderTransportService.addSendBehavior(
                         (connection, requestId, action, request, options) -> {
                             if (RetentionLeaseActions.Remove.ACTION_NAME.equals(action)
@@ -526,9 +527,9 @@ public void testUnfollowFailsToRemoveRetentionLeases() throws Exception {
                             getLeaderCluster().getClusterName(),
                             new Index(leaderIndex, leaderUUID))));
         } finally {
-            for (final ObjectCursor<DiscoveryNode> senderNode : followerClusterState.getState().nodes().getDataNodes().values()) {
+            for (final DiscoveryNode senderNode : followerClusterState.getState().nodes()) {
                 final MockTransportService senderTransportService =
-                        (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.value.getName());
+                        (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.getName());
                 senderTransportService.clearAllRules();
             }
         }
@@ -603,6 +604,7 @@ public void testRetentionLeaseAdvancesWhileFollowing() throws Exception {
             assertThat(stats.getShards(), arrayWithSize(numberOfShards * (1 + numberOfReplicas)));
             final List<ShardStats> shardsStats = getShardsStats(stats);
             for (int i = 0; i < numberOfShards * (1 + numberOfReplicas); i++) {
+                assertNotNull(shardsStats.get(i).getRetentionLeaseStats());
                 final RetentionLeases currentRetentionLeases = shardsStats.get(i).getRetentionLeaseStats().retentionLeases();
                 assertThat(Strings.toString(shardsStats.get(i)), currentRetentionLeases.leases(), hasSize(1));
                 final RetentionLease retentionLease =
@@ -655,46 +657,47 @@ public void testRetentionLeaseRenewalIsCancelledWhenFollowingIsPaused() throws E
         final ClusterStateResponse leaderIndexClusterState =
                 leaderClient().admin().cluster().prepareState().clear().setMetaData(true).setIndices(leaderIndex).get();
         final String leaderUUID = leaderIndexClusterState.getState().metaData().index(leaderIndex).getIndexUUID();
-
-        // sample the leases after pausing
-        final List<RetentionLeases> retentionLeases = new ArrayList<>();
+        /*
+         * We want to ensure that the background renewal is cancelled after pausing. To do this, we will sleep a small multiple of the renew
+         * interval. If the renews are not cancelled, we expect that a renewal would have been sent while we were sleeping. After we wake
+         * up, it should be the case that the retention leases are the same (same timestamp) as that indicates that they were not renewed
+         * while we were sleeping.
+         */
         assertBusy(() -> {
-            retentionLeases.clear();
-            final IndicesStatsResponse stats =
+            // sample the leases after pausing
+            final List<RetentionLeases> retentionLeases = new ArrayList<>();
+            assertBusy(() -> {
+                retentionLeases.clear();
+                final IndicesStatsResponse stats =
                     leaderClient().admin().indices().stats(new IndicesStatsRequest().clear().indices(leaderIndex)).actionGet();
-            assertNotNull(stats.getShards());
-            assertThat(stats.getShards(), arrayWithSize(numberOfShards * (1 + numberOfReplicas)));
-            final List<ShardStats> shardsStats = getShardsStats(stats);
-            for (int i = 0; i < numberOfShards * (1 + numberOfReplicas); i++) {
-                final RetentionLeases currentRetentionLeases = shardsStats.get(i).getRetentionLeaseStats().retentionLeases();
-                assertThat(Strings.toString(shardsStats.get(i)), currentRetentionLeases.leases(), hasSize(1));
-                final ClusterStateResponse followerIndexClusterState =
+                assertNotNull(stats.getShards());
+                assertThat(stats.getShards(), arrayWithSize(numberOfShards * (1 + numberOfReplicas)));
+                final List<ShardStats> shardsStats = getShardsStats(stats);
+                for (int i = 0; i < numberOfShards * (1 + numberOfReplicas); i++) {
+                    assertNotNull(shardsStats.get(i).getRetentionLeaseStats());
+                    final RetentionLeases currentRetentionLeases = shardsStats.get(i).getRetentionLeaseStats().retentionLeases();
+                    assertThat(Strings.toString(shardsStats.get(i)), currentRetentionLeases.leases(), hasSize(1));
+                    final ClusterStateResponse followerIndexClusterState =
                         followerClient().admin().cluster().prepareState().clear().setMetaData(true).setIndices(followerIndex).get();
-                final String followerUUID = followerIndexClusterState.getState().metaData().index(followerIndex).getIndexUUID();
-                final RetentionLease retentionLease =
+                    final String followerUUID = followerIndexClusterState.getState().metaData().index(followerIndex).getIndexUUID();
+                    final RetentionLease retentionLease =
                         currentRetentionLeases.leases().iterator().next();
-                final String expectedRetentionLeaseId = retentionLeaseId(
+                    final String expectedRetentionLeaseId = retentionLeaseId(
                         getFollowerCluster().getClusterName(),
                         new Index(followerIndex, followerUUID),
                         getLeaderCluster().getClusterName(),
                         new Index(leaderIndex, leaderUUID));
-                assertThat(retentionLease.id(), equalTo(expectedRetentionLeaseId));
-                retentionLeases.add(currentRetentionLeases);
-            }
-        });
-
-        /*
-         * We want to ensure that the background renewal is cancelled after pausing. To do this, we will sleep a small multiple of the renew
-         * interval. If the renews are not cancelled, we expect that a renewal would have been sent while we were sleeping. After we wake
-         * up, it should be the case that the retention leases are the same (same timestamp) as that indicates that they were not renewed
-         * while we were sleeping.
-         */
-        final TimeValue renewIntervalSetting = CcrRetentionLeases.RETENTION_LEASE_RENEW_INTERVAL_SETTING.get(followerClusterSettings());
-        final long renewEnd = System.nanoTime();
-        Thread.sleep(Math.max(0, randomIntBetween(2, 4) * renewIntervalSetting.millis() - TimeUnit.NANOSECONDS.toMillis(renewEnd - start)));
-
-        // now ensure that the retention leases are the same
-        assertBusy(() -> {
+                    assertThat(retentionLease.id(), equalTo(expectedRetentionLeaseId));
+                    retentionLeases.add(currentRetentionLeases);
+                }
+            });
+            // sleep a small multiple of the renew interval
+            final TimeValue renewIntervalSetting = CcrRetentionLeases.RETENTION_LEASE_RENEW_INTERVAL_SETTING.get(followerClusterSettings());
+            final long renewEnd = System.nanoTime();
+            Thread.sleep(
+                Math.max(0, randomIntBetween(2, 4) * renewIntervalSetting.millis() - TimeUnit.NANOSECONDS.toMillis(renewEnd - start)));
+
+            // now ensure that the retention leases are the same
             final IndicesStatsResponse stats =
                     leaderClient().admin().indices().stats(new IndicesStatsRequest().clear().indices(leaderIndex)).actionGet();
             assertNotNull(stats.getShards());
@@ -704,6 +707,7 @@ public void testRetentionLeaseRenewalIsCancelledWhenFollowingIsPaused() throws E
                 if (shardsStats.get(i).getShardRouting().primary() == false) {
                     continue;
                 }
+                assertNotNull(shardsStats.get(i).getRetentionLeaseStats());
                 final RetentionLeases currentRetentionLeases = shardsStats.get(i).getRetentionLeaseStats().retentionLeases();
                 assertThat(Strings.toString(shardsStats.get(i)), currentRetentionLeases.leases(), hasSize(1));
                 final ClusterStateResponse followerIndexClusterState =
@@ -766,35 +770,36 @@ public void testRetentionLeaseIsAddedIfItDisappearsWhileFollowing() throws Excep
         final CountDownLatch latch = new CountDownLatch(1);
 
         final ClusterStateResponse followerClusterState = followerClient().admin().cluster().prepareState().clear().setNodes(true).get();
-        for (final ObjectCursor<DiscoveryNode> senderNode : followerClusterState.getState().nodes().getNodes().values()) {
-            final MockTransportService senderTransportService =
-                    (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.value.getName());
-            senderTransportService.addSendBehavior(
+        try {
+            for (final DiscoveryNode senderNode : followerClusterState.getState().nodes()) {
+                final MockTransportService senderTransportService =
+                    (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.getName());
+                senderTransportService.addSendBehavior(
                     (connection, requestId, action, request, options) -> {
                         if (RetentionLeaseActions.Renew.ACTION_NAME.equals(action)
-                                || TransportActionProxy.getProxyAction(RetentionLeaseActions.Renew.ACTION_NAME).equals(action)) {
+                            || TransportActionProxy.getProxyAction(RetentionLeaseActions.Renew.ACTION_NAME).equals(action)) {
                             senderTransportService.clearAllRules();
                             final RetentionLeaseActions.RenewRequest renewRequest = (RetentionLeaseActions.RenewRequest) request;
                             final String primaryShardNodeId =
-                                    getLeaderCluster()
-                                            .clusterService()
-                                            .state()
-                                            .routingTable()
-                                            .index(leaderIndex)
-                                            .shard(renewRequest.getShardId().id())
-                                            .primaryShard()
-                                            .currentNodeId();
+                                getLeaderCluster()
+                                    .clusterService()
+                                    .state()
+                                    .routingTable()
+                                    .index(leaderIndex)
+                                    .shard(renewRequest.getShardId().id())
+                                    .primaryShard()
+                                    .currentNodeId();
                             final String primaryShardNodeName =
-                                    getLeaderCluster().clusterService().state().nodes().get(primaryShardNodeId).getName();
+                                getLeaderCluster().clusterService().state().nodes().get(primaryShardNodeId).getName();
                             final IndexShard primary =
-                                    getLeaderCluster()
-                                            .getInstance(IndicesService.class, primaryShardNodeName)
-                                            .getShardOrNull(renewRequest.getShardId());
+                                getLeaderCluster()
+                                    .getInstance(IndicesService.class, primaryShardNodeName)
+                                    .getShardOrNull(renewRequest.getShardId());
                             final CountDownLatch innerLatch = new CountDownLatch(1);
                             // this forces the background renewal from following to face a retention lease not found exception
                             primary.removeRetentionLease(
-                                    getRetentionLeaseId(followerIndex, leaderIndex),
-                                    ActionListener.wrap(r -> innerLatch.countDown(), e -> fail(e.toString())));
+                                getRetentionLeaseId(followerIndex, leaderIndex),
+                                ActionListener.wrap(r -> innerLatch.countDown(), e -> fail(e.toString())));
 
                             try {
                                 innerLatch.await();
@@ -807,11 +812,18 @@ public void testRetentionLeaseIsAddedIfItDisappearsWhileFollowing() throws Excep
                         }
                         connection.sendRequest(requestId, action, request, options);
                     });
-        }
+            }
 
-        latch.await();
+            latch.await();
 
-        assertRetentionLeaseRenewal(numberOfShards, numberOfReplicas, followerIndex, leaderIndex);
+            assertRetentionLeaseRenewal(numberOfShards, numberOfReplicas, followerIndex, leaderIndex);
+        } finally {
+            for (final DiscoveryNode senderNode : followerClusterState.getState().nodes()) {
+                final MockTransportService senderTransportService =
+                    (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.getName());
+                senderTransportService.clearAllRules();
+            }
+        }
     }
 
     /**
@@ -858,9 +870,9 @@ public void testPeriodicRenewalDoesNotAddRetentionLeaseAfterUnfollow() throws Ex
         final ClusterStateResponse followerClusterState = followerClient().admin().cluster().prepareState().clear().setNodes(true).get();
 
         try {
-            for (final ObjectCursor<DiscoveryNode> senderNode : followerClusterState.getState().nodes().getNodes().values()) {
+            for (final DiscoveryNode senderNode : followerClusterState.getState().nodes()) {
                 final MockTransportService senderTransportService =
-                        (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.value.getName());
+                        (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.getName());
                 senderTransportService.addSendBehavior(
                         (connection, requestId, action, request, options) -> {
                             if (RetentionLeaseActions.Renew.ACTION_NAME.equals(action)
@@ -911,12 +923,13 @@ public void onResponseReceived(
                     leaderClient().admin().indices().stats(new IndicesStatsRequest().clear().indices(leaderIndex)).actionGet();
             final List<ShardStats> afterUnfollowShardsStats = getShardsStats(afterUnfollowStats);
             for (final ShardStats shardStats : afterUnfollowShardsStats) {
+                assertNotNull(shardStats.getRetentionLeaseStats());
                 assertThat(Strings.toString(shardStats), shardStats.getRetentionLeaseStats().retentionLeases().leases(), empty());
             }
         } finally {
-            for (final ObjectCursor<DiscoveryNode> senderNode : followerClusterState.getState().nodes().getDataNodes().values()) {
+            for (final DiscoveryNode senderNode : followerClusterState.getState().nodes()) {
                 final MockTransportService senderTransportService =
-                        (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.value.getName());
+                    (MockTransportService) getFollowerCluster().getInstance(TransportService.class, senderNode.getName());
                 senderTransportService.clearAllRules();
             }
         }
@@ -961,6 +974,7 @@ public void testForgetFollower() throws Exception {
                 leaderClient().admin().indices().stats(new IndicesStatsRequest().clear().indices(leaderIndex)).actionGet();
         final List<ShardStats> afterForgetFollowerShardsStats = getShardsStats(afterForgetFollowerStats);
         for (final ShardStats shardStats : afterForgetFollowerShardsStats) {
+            assertNotNull(shardStats.getRetentionLeaseStats());
             assertThat(Strings.toString(shardStats), shardStats.getRetentionLeaseStats().retentionLeases().leases(), empty());
         }
     }
@@ -980,6 +994,7 @@ private void assertRetentionLeaseRenewal(
             assertThat(stats.getShards(), arrayWithSize(numberOfShards * (1 + numberOfReplicas)));
             final List<ShardStats> shardsStats = getShardsStats(stats);
             for (int i = 0; i < numberOfShards * (1 + numberOfReplicas); i++) {
+                assertNotNull(shardsStats.get(i).getRetentionLeaseStats());
                 final RetentionLeases currentRetentionLeases = shardsStats.get(i).getRetentionLeaseStats().retentionLeases();
                 assertThat(Strings.toString(shardsStats.get(i)), currentRetentionLeases.leases(), hasSize(1));
                 final RetentionLease retentionLease =
@@ -997,6 +1012,7 @@ private void assertRetentionLeaseRenewal(
             assertThat(stats.getShards(), arrayWithSize(numberOfShards * (1 + numberOfReplicas)));
             final List<ShardStats> shardsStats = getShardsStats(stats);
             for (int i = 0; i < numberOfShards * (1 + numberOfReplicas); i++) {
+                assertNotNull(shardsStats.get(i).getRetentionLeaseStats());
                 final RetentionLeases currentRetentionLeases = shardsStats.get(i).getRetentionLeaseStats().retentionLeases();
                 assertThat(Strings.toString(shardsStats.get(i)), currentRetentionLeases.leases(), hasSize(1));
                 final RetentionLease retentionLease =
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java
index c1d6a3aad337a..9642e0a641a2b 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java
@@ -27,11 +27,9 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.InternalTestCluster;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.CcrIntegTestCase;
 import org.elasticsearch.xpack.core.ccr.action.FollowStatsAction;
 import org.elasticsearch.xpack.core.ccr.action.PutFollowAction;
-import org.elasticsearch.xpack.core.ccr.client.CcrClient;
 import org.hamcrest.Matchers;
 
 import java.util.Locale;
@@ -45,8 +43,6 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 
-@TestLogging("org.elasticsearch.xpack.ccr:TRACE,org.elasticsearch.xpack.ccr.action.ShardChangesAction:DEBUG,"
-    + "org.elasticsearch.index.shard:TRACE")
 public class FollowerFailOverIT extends CcrIntegTestCase {
 
     @Override
@@ -215,6 +211,9 @@ public void testAddNewReplicasOnFollower() throws Exception {
                     if (rarely()) {
                         followerClient().admin().indices().prepareFlush("follower-index").get();
                     }
+                    if (rarely()) {
+                        followerClient().admin().indices().prepareForceMerge("follower-index").setMaxNumSegments(1).get();
+                    }
                     if (rarely()) {
                         followerClient().admin().indices().prepareRefresh("follower-index").get();
                     }
@@ -284,11 +283,11 @@ public void testReadRequestsReturnLatestMappingVersion() throws Exception {
             IndexResponse indexResp = leaderCluster.client().prepareIndex("leader-index", "doc", "1")
                 .setSource("{\"balance\": 100}", XContentType.JSON).setTimeout(TimeValue.ZERO).get();
             assertThat(indexResp.getResult(), equalTo(DocWriteResponse.Result.CREATED));
-            assertThat(indexShard.getGlobalCheckpoint(), equalTo(0L));
+            assertThat(indexShard.getLastKnownGlobalCheckpoint(), equalTo(0L));
             // Make sure at least one read-request which requires mapping sync is completed.
             assertBusy(() -> {
-                CcrClient ccrClient = new CcrClient(followerClient());
-                FollowStatsAction.StatsResponses responses = ccrClient.followStats(new FollowStatsAction.StatsRequest()).actionGet();
+                FollowStatsAction.StatsResponses responses =
+                    followerClient().execute(FollowStatsAction.INSTANCE, new FollowStatsAction.StatsRequest()).actionGet();
                 long bytesRead = responses.getStatsResponses().stream().mapToLong(r -> r.status().bytesRead()).sum();
                 assertThat(bytesRead, Matchers.greaterThan(0L));
             }, 60, TimeUnit.SECONDS);
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
index a4aeae12926f1..665d45ad25950 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
@@ -17,10 +17,11 @@
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
@@ -41,10 +42,13 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.health.ClusterIndexHealth;
 import org.elasticsearch.cluster.health.ClusterShardHealth;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -64,10 +68,12 @@
 import org.elasticsearch.index.seqno.RetentionLeaseActions;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.SnapshotRestoreException;
 import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.test.BackgroundIndexer;
+import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.transport.NoSuchRemoteClusterException;
 import org.elasticsearch.transport.RemoteClusterService;
 import org.elasticsearch.transport.TransportService;
@@ -84,6 +90,7 @@
 import org.elasticsearch.xpack.core.ccr.action.UnfollowAction;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -98,23 +105,30 @@
 import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.xpack.ccr.CcrRetentionLeases.retentionLeaseId;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class IndexFollowingIT extends CcrIntegTestCase {
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/41037")
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Stream.concat(super.nodePlugins().stream(), Stream.of(PrivateSettingPlugin.class)).collect(Collectors.toList());
+    }
+
     public void testFollowIndex() throws Exception {
         final int numberOfPrimaryShards = randomIntBetween(1, 3);
         int numberOfReplicas = between(0, 1);
@@ -412,6 +426,43 @@ public void testDoNotAllowPutMappingToFollower() throws Exception {
         assertAcked(followerClient().admin().indices().putMapping(putMappingRequest).actionGet());
     }
 
+    public void testDoNotAllowAddAliasToFollower() throws Exception {
+        final String leaderIndexSettings =
+                getIndexSettings(between(1, 2), between(0, 1), singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true"));
+        assertAcked(leaderClient().admin().indices().prepareCreate("leader").setSource(leaderIndexSettings, XContentType.JSON));
+        followerClient().execute(PutFollowAction.INSTANCE, putFollow("leader", "follower")).get();
+        final IndicesAliasesRequest request = new IndicesAliasesRequest()
+                .addAliasAction(IndicesAliasesRequest.AliasActions.add().index("follower").alias("follower_alias"));
+        final ElasticsearchStatusException e =
+                expectThrows(ElasticsearchStatusException.class, () -> followerClient().admin().indices().aliases(request).actionGet());
+        assertThat(
+                e,
+                hasToString(containsString("can't modify aliases on indices [follower]; "
+                        + "aliases of following indices are self-replicated from their leader indices")));
+        assertThat(e.status(), equalTo(RestStatus.FORBIDDEN));
+    }
+
+    public void testAddAliasAfterUnfollow() throws Exception {
+        final String leaderIndexSettings =
+                getIndexSettings(between(1, 2), between(0, 1), singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true"));
+        assertAcked(leaderClient().admin().indices().prepareCreate("leader").setSource(leaderIndexSettings, XContentType.JSON));
+        followerClient().execute(PutFollowAction.INSTANCE, putFollow("leader", "follower")).get();
+        pauseFollow("follower");
+        followerClient().admin().indices().close(new CloseIndexRequest("follower")).actionGet();
+        assertAcked(followerClient().execute(UnfollowAction.INSTANCE, new UnfollowAction.Request("follower")).actionGet());
+        followerClient().admin().indices().open(new OpenIndexRequest("follower")).actionGet();
+        final IndicesAliasesRequest request = new IndicesAliasesRequest()
+                .addAliasAction(IndicesAliasesRequest.AliasActions.add().index("follower").alias("follower_alias"));
+        assertAcked(followerClient().admin().indices().aliases(request).actionGet());
+        final GetAliasesResponse response =
+                followerClient().admin().indices().getAliases(new GetAliasesRequest("follower_alias")).actionGet();
+        assertThat(response.getAliases().keys().size(), equalTo(1));
+        assertThat(response.getAliases().keys().iterator().next().value, equalTo("follower"));
+        final List<AliasMetaData> aliasMetaData = response.getAliases().get("follower");
+        assertThat(aliasMetaData, hasSize(1));
+        assertThat(aliasMetaData.get(0).alias(), equalTo("follower_alias"));
+    }
+
     public void testFollowIndex_backlog() throws Exception {
         int numberOfShards = between(1, 5);
         String leaderIndexSettings = getIndexSettings(numberOfShards, between(0, 1),
@@ -782,9 +833,7 @@ public void testLeaderIndexRed() throws Exception {
                 () -> followerClient().execute(PutFollowAction.INSTANCE, followRequest).actionGet());
             assertThat(e.getMessage(), equalTo("no index stats available for the leader index"));
 
-            IndicesExistsResponse existsResponse = followerClient().admin().indices().exists(new IndicesExistsRequest("index2"))
-                .actionGet();
-            assertThat(existsResponse.isExists(), is(false));
+            assertThat(ESIntegTestCase.indexExists("index2", followerClient()), is(false));
         } finally {
             // Always unset allocation enable setting to avoid other assertions from failing too when this test fails:
             ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
@@ -970,6 +1019,46 @@ public void testUpdateAnalysisLeaderIndexSettings() throws Exception {
         assertThat(hasFollowIndexBeenClosedChecker.getAsBoolean(), is(true));
     }
 
+    public void testDoNotReplicatePrivateSettings() throws Exception {
+        assertAcked(leaderClient().admin().indices().prepareCreate("leader").setSource(
+            getIndexSettings(1, 0, singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")), XContentType.JSON));
+        ensureLeaderGreen("leader");
+        final PutFollowAction.Request followRequest = putFollow("leader", "follower");
+        followerClient().execute(PutFollowAction.INSTANCE, followRequest).get();
+        ClusterService clusterService = getLeaderCluster().getInstance(ClusterService.class, getLeaderCluster().getMasterName());
+        clusterService.submitStateUpdateTask("test", new ClusterStateUpdateTask() {
+            @Override
+            public ClusterState execute(ClusterState currentState) {
+                final IndexMetaData indexMetaData = currentState.metaData().index("leader");
+                Settings.Builder settings = Settings.builder()
+                    .put(indexMetaData.getSettings())
+                    .put("index.max_ngram_diff", 2);
+                if (randomBoolean()) {
+                    settings.put(PrivateSettingPlugin.INDEX_INTERNAL_SETTING.getKey(), "private-value");
+                }
+                if (randomBoolean()) {
+                    settings.put(PrivateSettingPlugin.INDEX_PRIVATE_SETTING.getKey(), "interval-value");
+                }
+                final MetaData.Builder metadata = MetaData.builder(currentState.metaData())
+                    .put(IndexMetaData.builder(indexMetaData)
+                        .settingsVersion(indexMetaData.getSettingsVersion() + 1)
+                        .settings(settings).build(), true);
+                return ClusterState.builder(currentState).metaData(metadata).build();
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                throw new AssertionError(e);
+            }
+        });
+        assertBusy(() -> {
+            GetSettingsResponse resp = followerClient().admin().indices().prepareGetSettings("follower").get();
+            assertThat(resp.getSetting("follower", "index.max_ngram_diff"), equalTo("2"));
+            assertThat(resp.getSetting("follower", PrivateSettingPlugin.INDEX_INTERNAL_SETTING.getKey()), nullValue());
+            assertThat(resp.getSetting("follower", PrivateSettingPlugin.INDEX_PRIVATE_SETTING.getKey()), nullValue());
+        });
+    }
+
     public void testMustCloseIndexAndPauseToRestartWithPutFollowing() throws Exception {
         final int numberOfPrimaryShards = randomIntBetween(1, 3);
         final String leaderIndexSettings = getIndexSettings(numberOfPrimaryShards, between(0, 1),
@@ -1337,4 +1426,15 @@ private String getIndexSettingsWithNestedMapping(final int numberOfShards, final
         return settings;
     }
 
+    public static class PrivateSettingPlugin extends Plugin {
+        static final Setting<String> INDEX_INTERNAL_SETTING =
+            Setting.simpleString("index.internal", Setting.Property.IndexScope, Setting.Property.InternalIndex);
+        static final Setting<String> INDEX_PRIVATE_SETTING =
+            Setting.simpleString("index.private", Setting.Property.IndexScope, Setting.Property.PrivateIndex);
+
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Arrays.asList(INDEX_INTERNAL_SETTING, INDEX_PRIVATE_SETTING);
+        }
+    }
 }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java
index 0df3f4ea47f43..739f510fb7a92 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xpack.CcrSingleNodeTestCase;
 import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
 import org.elasticsearch.xpack.core.ccr.action.FollowStatsAction;
@@ -89,7 +90,7 @@ public void testDoNotCreateFollowerIfLeaderDoesNotHaveSoftDeletes() throws Excep
         IllegalArgumentException error = expectThrows(IllegalArgumentException.class,
             () -> client().execute(PutFollowAction.INSTANCE, putFollowRequest).actionGet());
         assertThat(error.getMessage(), equalTo("leader index [leader-index] does not have soft deletes enabled"));
-        assertThat(client().admin().indices().prepareExists("follower-index").get().isExists(), equalTo(false));
+        assertThat(ESIntegTestCase.indexExists("follower-index", client()), equalTo(false));
     }
 
     public void testRemoveRemoteConnection() throws Exception {
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java
index 75e6a732c8210..7648d10aa0915 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java
@@ -813,8 +813,7 @@ public void testAutoFollowerSoftDeletesDisabled() {
         Client client = mock(Client.class);
         when(client.getRemoteClusterClient(anyString())).thenReturn(client);
 
-        ClusterState remoteState = randomBoolean() ? createRemoteClusterState("logs-20190101", false) :
-            createRemoteClusterState("logs-20190101", null);
+        ClusterState remoteState = createRemoteClusterState("logs-20190101", false);
 
         AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("logs-*"),
             null, null, null, null, null, null, null, null, null, null, null);
@@ -953,13 +952,9 @@ void cleanFollowedRemoteIndices(ClusterState remoteClusterState, List<String> pa
         assertThat(entries.get(0).getValue(), nullValue());
     }
 
-    private static ClusterState createRemoteClusterState(String indexName, Boolean enableSoftDeletes) {
+    private static ClusterState createRemoteClusterState(String indexName, boolean enableSoftDeletes) {
         Settings.Builder indexSettings;
-        if (enableSoftDeletes != null) {
-            indexSettings = settings(Version.CURRENT).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), enableSoftDeletes);
-        } else {
-            indexSettings = settings(Version.V_6_6_0);
-        }
+        indexSettings = settings(Version.CURRENT).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), enableSoftDeletes);
 
         IndexMetaData indexMetaData = IndexMetaData.builder(indexName)
             .settings(indexSettings)
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesActionTests.java
index e8b21f05c5c2d..9f6850fe20fc7 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesActionTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesActionTests.java
@@ -67,7 +67,7 @@ public void testGetOperations() throws Exception {
             int size = max - min + 1;
             final Translog.Operation[] operations = ShardChangesAction.getOperations(
                     indexShard,
-                    indexShard.getGlobalCheckpoint(),
+                    indexShard.getLastKnownGlobalCheckpoint(),
                     min,
                     size,
                     indexShard.getHistoryUUID(),
@@ -83,7 +83,7 @@ public void testGetOperations() throws Exception {
                     IllegalStateException.class,
                     () -> ShardChangesAction.getOperations(
                             indexShard,
-                            indexShard.getGlobalCheckpoint(),
+                            indexShard.getLastKnownGlobalCheckpoint(),
                             numWrites,
                             numWrites + 1,
                             indexShard.getHistoryUUID(),
@@ -92,18 +92,19 @@ public void testGetOperations() throws Exception {
                     Locale.ROOT,
                     "not exposing operations from [%d] greater than the global checkpoint [%d]",
                     numWrites,
-                    indexShard.getGlobalCheckpoint());
+                    indexShard.getLastKnownGlobalCheckpoint());
             assertThat(e, hasToString(containsString(message)));
         }
 
         // get operations for a range some operations do not exist:
-        Translog.Operation[] operations = ShardChangesAction.getOperations(indexShard, indexShard.getGlobalCheckpoint(),
+        Translog.Operation[] operations = ShardChangesAction.getOperations(indexShard, indexShard.getLastKnownGlobalCheckpoint(),
             numWrites  - 10, numWrites + 10, indexShard.getHistoryUUID(), new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES));
         assertThat(operations.length, equalTo(10));
 
         // Unexpected history UUID:
         Exception e = expectThrows(IllegalStateException.class, () -> ShardChangesAction.getOperations(indexShard,
-            indexShard.getGlobalCheckpoint(), 0, 10, "different-history-uuid", new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES)));
+            indexShard.getLastKnownGlobalCheckpoint(), 0, 10, "different-history-uuid",
+            new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES)));
         assertThat(e.getMessage(), equalTo("unexpected history uuid, expected [different-history-uuid], actual [" +
                 indexShard.getHistoryUUID() + "]"));
 
@@ -112,7 +113,7 @@ public void testGetOperations() throws Exception {
             final long fromSeqNo = randomLongBetween(Long.MIN_VALUE, -1);
             final int batchSize = randomIntBetween(0, Integer.MAX_VALUE);
             final IllegalArgumentException invalidRangeError = expectThrows(IllegalArgumentException.class,
-                () -> ShardChangesAction.getOperations(indexShard, indexShard.getGlobalCheckpoint(),
+                () -> ShardChangesAction.getOperations(indexShard, indexShard.getLastKnownGlobalCheckpoint(),
                     fromSeqNo, batchSize, indexShard.getHistoryUUID(), new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES)));
             assertThat(invalidRangeError.getMessage(),
                 equalTo("Invalid range; from_seqno [" + fromSeqNo + "], to_seqno [" + (fromSeqNo + batchSize - 1) + "]"));
@@ -125,7 +126,8 @@ public void testGetOperationsWhenShardNotStarted() throws Exception {
         ShardRouting shardRouting = TestShardRouting.newShardRouting("index", 0, "_node_id", true, ShardRoutingState.INITIALIZING);
         Mockito.when(indexShard.routingEntry()).thenReturn(shardRouting);
         expectThrows(IndexShardNotStartedException.class, () -> ShardChangesAction.getOperations(indexShard,
-            indexShard.getGlobalCheckpoint(), 0, 1, indexShard.getHistoryUUID(), new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES)));
+            indexShard.getLastKnownGlobalCheckpoint(), 0, 1, indexShard.getHistoryUUID(),
+            new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES)));
     }
 
     public void testGetOperationsExceedByteLimit() throws Exception {
@@ -142,7 +144,7 @@ public void testGetOperationsExceedByteLimit() throws Exception {
         }
 
         final IndexShard indexShard = indexService.getShard(0);
-        final Translog.Operation[] operations = ShardChangesAction.getOperations(indexShard, indexShard.getGlobalCheckpoint(),
+        final Translog.Operation[] operations = ShardChangesAction.getOperations(indexShard, indexShard.getLastKnownGlobalCheckpoint(),
             0, 12, indexShard.getHistoryUUID(), new ByteSizeValue(256, ByteSizeUnit.BYTES));
         assertThat(operations.length, equalTo(12));
         assertThat(operations[0].seqNo(), equalTo(0L));
@@ -172,7 +174,7 @@ public void testGetOperationsAlwaysReturnAtLeastOneOp() throws Exception {
         final IndexShard indexShard = indexService.getShard(0);
         final Translog.Operation[] operations =
             ShardChangesAction.getOperations(
-                    indexShard, indexShard.getGlobalCheckpoint(), 0, 1, indexShard.getHistoryUUID(), ByteSizeValue.ZERO);
+                    indexShard, indexShard.getLastKnownGlobalCheckpoint(), 0, 1, indexShard.getHistoryUUID(), ByteSizeValue.ZERO);
         assertThat(operations.length, equalTo(1));
         assertThat(operations[0].seqNo(), equalTo(0L));
     }
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesRequestTests.java
index 2ea2086990b32..5b2f06652b007 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesRequestTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesRequestTests.java
@@ -5,13 +5,14 @@
  */
 package org.elasticsearch.xpack.ccr.action;
 
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.nullValue;
 
-public class ShardChangesRequestTests extends AbstractStreamableTestCase<ShardChangesAction.Request> {
+public class ShardChangesRequestTests extends AbstractWireSerializingTestCase<ShardChangesAction.Request> {
 
     @Override
     protected ShardChangesAction.Request createTestInstance() {
@@ -23,8 +24,8 @@ protected ShardChangesAction.Request createTestInstance() {
     }
 
     @Override
-    protected ShardChangesAction.Request createBlankInstance() {
-        return new ShardChangesAction.Request();
+    protected Writeable.Reader<ShardChangesAction.Request> instanceReader() {
+        return ShardChangesAction.Request::new;
     }
 
     public void testValidate() {
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesResponseTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesResponseTests.java
index a5b28caf9dfb2..9c785ecd22c2f 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesResponseTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesResponseTests.java
@@ -15,6 +15,7 @@ public class ShardChangesResponseTests extends AbstractWireSerializingTestCase<S
     protected ShardChangesAction.Response createTestInstance() {
         final long mappingVersion = randomNonNegativeLong();
         final long settingsVersion = randomNonNegativeLong();
+        final long aliasesVersion = randomNonNegativeLong();
         final long leaderGlobalCheckpoint = randomNonNegativeLong();
         final long leaderMaxSeqNo = randomLongBetween(leaderGlobalCheckpoint, Long.MAX_VALUE);
         final long maxSeqNoOfUpdatesOrDeletes = randomLongBetween(-1, Long.MAX_VALUE);
@@ -26,6 +27,7 @@ protected ShardChangesAction.Response createTestInstance() {
         return new ShardChangesAction.Response(
             mappingVersion,
             settingsVersion,
+            aliasesVersion,
             leaderGlobalCheckpoint,
             leaderMaxSeqNo,
             maxSeqNoOfUpdatesOrDeletes,
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskRandomTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskRandomTests.java
index 46c7c51586c53..21d5d3547b57c 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskRandomTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskRandomTests.java
@@ -42,15 +42,19 @@
 public class ShardFollowNodeTaskRandomTests extends ESTestCase {
 
     public void testSingleReaderWriter() throws Exception {
-        TestRun testRun = createTestRun(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
-            randomIntBetween(1, 2048));
+        TestRun testRun = createTestRun(
+                randomNonNegativeLong(),
+                randomNonNegativeLong(),
+                randomNonNegativeLong(),
+                randomNonNegativeLong(),
+                randomIntBetween(1, 2048));
         ShardFollowNodeTask task = createShardFollowTask(1, testRun);
         startAndAssertAndStopTask(task, testRun);
     }
 
     public void testMultipleReaderWriter() throws Exception {
         int concurrency = randomIntBetween(2, 8);
-        TestRun testRun = createTestRun(0, 0, 0, between(1, 1024));
+        TestRun testRun = createTestRun(0, 0, 0, 0, between(1, 1024));
         ShardFollowNodeTask task = createShardFollowTask(concurrency, testRun);
         startAndAssertAndStopTask(task, testRun);
     }
@@ -110,6 +114,7 @@ private ShardFollowNodeTask createShardFollowTask(int concurrency, TestRun testR
 
             private volatile long mappingVersion = 0L;
             private volatile long settingsVersion = 0L;
+            private volatile long aliasesVersion = 0L;
             private final Map<Long, Integer> fromToSlot = new HashMap<>();
 
             @Override
@@ -122,6 +127,11 @@ protected void innerUpdateSettings(LongConsumer handler, Consumer<Exception> err
                 handler.accept(settingsVersion);
             }
 
+            @Override
+            protected void innerUpdateAliases(LongConsumer handler, Consumer<Exception> errorHandler) {
+                handler.accept(aliasesVersion);
+            }
+
             @Override
             protected void innerSendBulkShardOperationsRequest(
                 String followerHistoryUUID, List<Translog.Operation> operations,
@@ -129,13 +139,13 @@ protected void innerSendBulkShardOperationsRequest(
                 Consumer<BulkShardOperationsResponse> handler,
                 Consumer<Exception> errorHandler) {
                 for(Translog.Operation op : operations) {
-                    tracker.markSeqNoAsCompleted(op.seqNo());
+                    tracker.markSeqNoAsProcessed(op.seqNo());
                 }
                 receivedOperations.addAll(operations);
 
                 // Emulate network thread and avoid SO:
                 final BulkShardOperationsResponse response = new BulkShardOperationsResponse();
-                response.setGlobalCheckpoint(tracker.getCheckpoint());
+                response.setGlobalCheckpoint(tracker.getProcessedCheckpoint());
                 response.setMaxSeqNo(tracker.getMaxSeqNo());
                 threadPool.generic().execute(() -> handler.accept(response));
             }
@@ -170,10 +180,17 @@ protected void innerSendShardChangesRequest(long from, int maxOperationCount, Co
                         }
                     } else {
                         assert from >= testRun.finalExpectedGlobalCheckpoint;
-                        final long globalCheckpoint = tracker.getCheckpoint();
+                        final long globalCheckpoint = tracker.getProcessedCheckpoint();
                         final long maxSeqNo = tracker.getMaxSeqNo();
-                        handler.accept(new ShardChangesAction.Response(0L, 0L, globalCheckpoint, maxSeqNo, randomNonNegativeLong(),
-                            new Translog.Operation[0], 1L));
+                        handler.accept(new ShardChangesAction.Response(
+                                0L,
+                                0L,
+                                0L,
+                                globalCheckpoint,
+                                maxSeqNo,
+                                randomNonNegativeLong(),
+                                new Translog.Operation[0],
+                                1L));
                     }
                 };
                 threadPool.generic().execute(task);
@@ -233,10 +250,16 @@ private void tearDown() {
         };
     }
 
-    private static TestRun createTestRun(long startSeqNo, long startMappingVersion, long startSettingsVersion, int maxOperationCount) {
+    private static TestRun createTestRun(
+            final long startSeqNo,
+            final long startMappingVersion,
+            final long startSettingsVersion,
+            final long startAliasesVersion,
+            final int maxOperationCount) {
         long prevGlobalCheckpoint = startSeqNo;
         long mappingVersion = startMappingVersion;
         long settingsVersion = startSettingsVersion;
+        long aliasesVersion = startAliasesVersion;
         int numResponses = randomIntBetween(16, 256);
         Map<Long, List<TestResponse>> responses = new HashMap<>(numResponses);
         for (int i = 0; i < numResponses; i++) {
@@ -247,7 +270,9 @@ private static TestRun createTestRun(long startSeqNo, long startMappingVersion,
             if (sometimes()) {
                 settingsVersion++;
             }
-
+            if (sometimes()) {
+                aliasesVersion++;
+            }
             if (sometimes()) {
                 List<TestResponse> item = new ArrayList<>();
                 // Sometimes add a random retryable error
@@ -268,6 +293,7 @@ private static TestRun createTestRun(long startSeqNo, long startMappingVersion,
                     new ShardChangesAction.Response(
                         mappingVersion,
                         settingsVersion,
+                        aliasesVersion,
                         nextGlobalCheckPoint,
                         nextGlobalCheckPoint,
                         randomNonNegativeLong(),
@@ -293,6 +319,7 @@ private static TestRun createTestRun(long startSeqNo, long startMappingVersion,
                         ShardChangesAction.Response response = new ShardChangesAction.Response(
                             mappingVersion,
                             settingsVersion,
+                            aliasesVersion,
                             prevGlobalCheckpoint,
                             prevGlobalCheckpoint,
                             randomNonNegativeLong(),
@@ -312,6 +339,7 @@ private static TestRun createTestRun(long startSeqNo, long startMappingVersion,
                     ShardChangesAction.Response response = new ShardChangesAction.Response(
                         mappingVersion,
                         settingsVersion,
+                        aliasesVersion,
                         localLeaderGCP,
                         localLeaderGCP,
                         randomNonNegativeLong(),
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskStatusTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskStatusTests.java
index 9cac01d278e74..413960b69c834 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskStatusTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskStatusTests.java
@@ -59,6 +59,7 @@ protected ShardFollowNodeTaskStatus createTestInstance() {
                 randomNonNegativeLong(),
                 randomNonNegativeLong(),
                 randomNonNegativeLong(),
+                randomNonNegativeLong(),
                 randomReadExceptions(),
                 randomLong(),
                 randomBoolean() ? new ElasticsearchException("fatal error") : null);
@@ -80,6 +81,7 @@ protected void assertEqualInstances(final ShardFollowNodeTaskStatus expectedInst
         assertThat(newInstance.writeBufferOperationCount(), equalTo(expectedInstance.writeBufferOperationCount()));
         assertThat(newInstance.followerMappingVersion(), equalTo(expectedInstance.followerMappingVersion()));
         assertThat(newInstance.followerSettingsVersion(), equalTo(expectedInstance.followerSettingsVersion()));
+        assertThat(newInstance.followerAliasesVersion(), equalTo(expectedInstance.followerAliasesVersion()));
         assertThat(newInstance.totalReadTimeMillis(), equalTo(expectedInstance.totalReadTimeMillis()));
         assertThat(newInstance.successfulReadRequests(), equalTo(expectedInstance.successfulReadRequests()));
         assertThat(newInstance.failedReadRequests(), equalTo(expectedInstance.failedReadRequests()));
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskTests.java
index 09d00dc6a33ac..ef1dc43869ac4 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskTests.java
@@ -72,6 +72,8 @@ public class ShardFollowNodeTaskTests extends ESTestCase {
     private Queue<Long> mappingVersions;
     private Queue<Exception> settingsUpdateFailures;
     private Queue<Long> settingsVersions;
+    private Queue<Exception> aliasesUpdateFailures;
+    private Queue<Long> aliasesVersions;
     private Queue<Long> leaderGlobalCheckpoints;
     private Queue<Long> followerGlobalCheckpoints;
     private Queue<Long> maxSeqNos;
@@ -88,7 +90,7 @@ public void testCoordinateReads() {
         task.coordinateReads();
         assertThat(shardChangesRequests, contains(new long[]{0L, 8L})); // treat this a peak request
         shardChangesRequests.clear();
-        task.innerHandleReadResponse(0, 5L, generateShardChangesResponse(0, 5L, 0L, 0L, 60L));
+        task.innerHandleReadResponse(0, 5L, generateShardChangesResponse(0, 5L, 0L, 0L, 1L, 60L));
         assertThat(shardChangesRequests, contains(new long[][]{
             {6L, 8L}, {14L, 8L}, {22L, 8L}, {30L, 8L}, {38L, 8L}, {46L, 8L}, {54L, 7L}}
         ));
@@ -113,7 +115,7 @@ public void testMaxWriteBufferCount() {
 
         shardChangesRequests.clear();
         // Also invokes the coordinatesReads() method:
-        task.innerHandleReadResponse(0L, 63L, generateShardChangesResponse(0, 63, 0L, 0L, 128L));
+        task.innerHandleReadResponse(0L, 63L, generateShardChangesResponse(0, 63, 0L, 0L, 1L, 128L));
         assertThat(shardChangesRequests.size(), equalTo(0)); // no more reads, because write buffer count limit has been reached
 
         ShardFollowNodeTaskStatus status = task.getStatus();
@@ -139,7 +141,7 @@ public void testMaxWriteBufferSize() {
 
         shardChangesRequests.clear();
         // Also invokes the coordinatesReads() method:
-        task.innerHandleReadResponse(0L, 63L, generateShardChangesResponse(0, 63, 0L, 0L, 128L));
+        task.innerHandleReadResponse(0L, 63L, generateShardChangesResponse(0, 63, 0L, 0L, 1L, 128L));
         assertThat(shardChangesRequests.size(), equalTo(0)); // no more reads, because write buffer size limit has been reached
 
         ShardFollowNodeTaskStatus status = task.getStatus();
@@ -204,7 +206,7 @@ public void testTaskCancelledAfterReadLimitHasBeenReached() {
         task.markAsCompleted();
         shardChangesRequests.clear();
         // Also invokes the coordinatesReads() method:
-        task.innerHandleReadResponse(0L, 15L, generateShardChangesResponse(0, 15, 0L, 0L, 31L));
+        task.innerHandleReadResponse(0L, 15L, generateShardChangesResponse(0, 15, 0L, 0L, 1L, 31L));
         assertThat(shardChangesRequests.size(), equalTo(0)); // no more reads, because task has been cancelled
         assertThat(bulkShardOperationRequests.size(), equalTo(0)); // no more writes, because task has been cancelled
 
@@ -234,7 +236,7 @@ public void testTaskCancelledAfterWriteBufferLimitHasBeenReached() {
         task.markAsCompleted();
         shardChangesRequests.clear();
         // Also invokes the coordinatesReads() method:
-        task.innerHandleReadResponse(0L, 63L, generateShardChangesResponse(0, 63, 0L, 0L, 128L));
+        task.innerHandleReadResponse(0L, 63L, generateShardChangesResponse(0, 63, 0L, 0L, 1L, 128L));
         assertThat(shardChangesRequests.size(), equalTo(0)); // no more reads, because task has been cancelled
         assertThat(bulkShardOperationRequests.size(), equalTo(0)); // no more writes, because task has been cancelled
 
@@ -483,7 +485,7 @@ public void testHandleReadResponse() {
         startTask(task, 63, -1);
 
         task.coordinateReads();
-        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 0L, 63L);
+        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 0L, 1L, 63L);
         task.innerHandleReadResponse(0L, 63L, response);
 
         assertThat(bulkShardOperationRequests.size(), equalTo(1));
@@ -513,7 +515,7 @@ public void testReceiveLessThanRequested() {
         assertThat(shardChangesRequests.get(0)[1], equalTo(64L));
 
         shardChangesRequests.clear();
-        ShardChangesAction.Response response = generateShardChangesResponse(0, 20, 0L, 0L, 31L);
+        ShardChangesAction.Response response = generateShardChangesResponse(0, 20, 0L, 0L, 1L, 31L);
         task.innerHandleReadResponse(0L, 63L, response);
 
         assertThat(shardChangesRequests.size(), equalTo(1));
@@ -542,7 +544,7 @@ public void testCancelAndReceiveLessThanRequested() {
 
         shardChangesRequests.clear();
         task.markAsCompleted();
-        ShardChangesAction.Response response = generateShardChangesResponse(0, 31, 0L, 0L, 31L);
+        ShardChangesAction.Response response = generateShardChangesResponse(0, 31, 0L, 0L, 1L, 31L);
         task.innerHandleReadResponse(0L, 64L, response);
 
         assertThat(shardChangesRequests.size(), equalTo(0));
@@ -568,7 +570,7 @@ public void testReceiveNothingExpectedSomething() {
         assertThat(shardChangesRequests.get(0)[1], equalTo(64L));
 
         shardChangesRequests.clear();
-        task.innerHandleReadResponse(0L, 63L, new ShardChangesAction.Response(0, 0, 0, 0, 100, new Translog.Operation[0], 1L));
+        task.innerHandleReadResponse(0L, 63L, new ShardChangesAction.Response(0, 0, 0, 0, 0, 100, new Translog.Operation[0], 1L));
 
         assertThat(shardChangesRequests.size(), equalTo(1));
         assertThat(shardChangesRequests.get(0)[0], equalTo(0L));
@@ -591,7 +593,7 @@ public void testMappingUpdate() {
 
         mappingVersions.add(1L);
         task.coordinateReads();
-        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 1L, 0L, 63L);
+        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 1L, 0L, 0L, 63L);
         task.handleReadResponse(0L, 63L, response);
 
         assertThat(bulkShardOperationRequests.size(), equalTo(1));
@@ -620,7 +622,7 @@ public void testMappingUpdateRetryableError() {
         }
         mappingVersions.add(1L);
         task.coordinateReads();
-        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 1L, 0L, 63L);
+        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 1L, 0L, 0L, 63L);
         task.handleReadResponse(0L, 63L, response);
 
         assertThat(mappingUpdateFailures.size(), equalTo(0));
@@ -645,7 +647,7 @@ public void testMappingUpdateNonRetryableError() {
 
         mappingUpdateFailures.add(new RuntimeException());
         task.coordinateReads();
-        ShardChangesAction.Response response = generateShardChangesResponse(0, 64, 1L, 0L, 64L);
+        ShardChangesAction.Response response = generateShardChangesResponse(0, 64, 1L, 0L, 0L, 64L);
         task.handleReadResponse(0L, 64L, response);
 
         assertThat(bulkShardOperationRequests.size(), equalTo(0));
@@ -668,7 +670,7 @@ public void testSettingsUpdate() {
 
         settingsVersions.add(1L);
         task.coordinateReads();
-        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 1L, 63L);
+        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 1L, 0L, 63L);
         task.handleReadResponse(0L, 63L, response);
 
         assertThat(bulkShardOperationRequests.size(), equalTo(1));
@@ -677,6 +679,7 @@ public void testSettingsUpdate() {
         ShardFollowNodeTaskStatus status = task.getStatus();
         assertThat(status.followerMappingVersion(), equalTo(0L));
         assertThat(status.followerSettingsVersion(), equalTo(1L));
+        assertThat(status.followerAliasesVersion(), equalTo(0L));
         assertThat(status.outstandingReadRequests(), equalTo(1));
         assertThat(status.outstandingWriteRequests(), equalTo(1));
         assertThat(status.lastRequestedSeqNo(), equalTo(63L));
@@ -698,15 +701,16 @@ public void testSettingsUpdateRetryableError() {
         }
         settingsVersions.add(1L);
         task.coordinateReads();
-        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 1L, 63L);
+        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 1L, 0L, 63L);
         task.handleReadResponse(0L, 63L, response);
 
-        assertThat(mappingUpdateFailures.size(), equalTo(0));
+        assertThat(settingsUpdateFailures.size(), equalTo(0));
         assertThat(bulkShardOperationRequests.size(), equalTo(1));
         assertThat(task.isStopped(), equalTo(false));
         ShardFollowNodeTaskStatus status = task.getStatus();
         assertThat(status.followerMappingVersion(), equalTo(0L));
         assertThat(status.followerSettingsVersion(), equalTo(1L));
+        assertThat(status.followerAliasesVersion(), equalTo(0L));
         assertThat(status.outstandingReadRequests(), equalTo(1));
         assertThat(status.outstandingWriteRequests(), equalTo(1));
         assertThat(status.lastRequestedSeqNo(), equalTo(63L));
@@ -723,7 +727,7 @@ public void testSettingsUpdateNonRetryableError() {
 
         settingsUpdateFailures.add(new RuntimeException());
         task.coordinateReads();
-        ShardChangesAction.Response response = generateShardChangesResponse(0, 64, 0L, 1L, 64L);
+        ShardChangesAction.Response response = generateShardChangesResponse(0, 64, 0L, 1L, 0L, 64L);
         task.handleReadResponse(0L, 64L, response);
 
         assertThat(bulkShardOperationRequests.size(), equalTo(0));
@@ -731,6 +735,89 @@ public void testSettingsUpdateNonRetryableError() {
         ShardFollowNodeTaskStatus status = task.getStatus();
         assertThat(status.followerMappingVersion(), equalTo(0L));
         assertThat(status.followerSettingsVersion(), equalTo(0L));
+        assertThat(status.followerAliasesVersion(), equalTo(0L));
+        assertThat(status.outstandingReadRequests(), equalTo(1));
+        assertThat(status.outstandingWriteRequests(), equalTo(0));
+        assertThat(status.lastRequestedSeqNo(), equalTo(63L));
+        assertThat(status.leaderGlobalCheckpoint(), equalTo(63L));
+    }
+
+    public void testAliasUpdate() {
+        final ShardFollowTaskParams params = new ShardFollowTaskParams();
+        params.maxReadRequestOperationCount = 64;
+        params.maxOutstandingReadRequests = 1;
+        params.maxOutstandingWriteRequests = 1;
+        final ShardFollowNodeTask task = createShardFollowTask(params);
+        startTask(task, 63, -1);
+
+        aliasesVersions.add(1L);
+        task.coordinateReads();
+        final ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 0L, 1L, 63L);
+        task.handleReadResponse(0L, 63L, response);
+
+        assertThat(bulkShardOperationRequests.size(), equalTo(1));
+        assertThat(bulkShardOperationRequests.get(0), equalTo(Arrays.asList(response.getOperations())));
+
+        final ShardFollowNodeTaskStatus status = task.getStatus();
+        assertThat(status.followerMappingVersion(), equalTo(0L));
+        assertThat(status.followerSettingsVersion(), equalTo(0L));
+        assertThat(status.followerAliasesVersion(), equalTo(1L));
+        assertThat(status.outstandingReadRequests(), equalTo(1));
+        assertThat(status.outstandingWriteRequests(), equalTo(1));
+        assertThat(status.lastRequestedSeqNo(), equalTo(63L));
+        assertThat(status.leaderGlobalCheckpoint(), equalTo(63L));
+        assertThat(status.followerGlobalCheckpoint(), equalTo(-1L));
+    }
+
+    public void testAliasUpdateRetryableError() {
+        final ShardFollowTaskParams params = new ShardFollowTaskParams();
+        params.maxReadRequestOperationCount = 64;
+        params.maxOutstandingReadRequests = 1;
+        params.maxOutstandingWriteRequests = 1;
+        final ShardFollowNodeTask task = createShardFollowTask(params);
+        startTask(task, 63, -1);
+
+        int max = randomIntBetween(1, 30);
+        for (int i = 0; i < max; i++) {
+            aliasesUpdateFailures.add(new ConnectException());
+        }
+        aliasesVersions.add(1L);
+        task.coordinateReads();
+        final ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 0L, 1L, 63L);
+        task.handleReadResponse(0L, 63L, response);
+
+        assertThat(aliasesUpdateFailures.size(), equalTo(0));
+        assertThat(bulkShardOperationRequests.size(), equalTo(1));
+        assertThat(task.isStopped(), equalTo(false));
+        final ShardFollowNodeTaskStatus status = task.getStatus();
+        assertThat(status.followerMappingVersion(), equalTo(0L));
+        assertThat(status.followerSettingsVersion(), equalTo(0L));
+        assertThat(status.followerAliasesVersion(), equalTo(1L));
+        assertThat(status.outstandingReadRequests(), equalTo(1));
+        assertThat(status.outstandingWriteRequests(), equalTo(1));
+        assertThat(status.lastRequestedSeqNo(), equalTo(63L));
+        assertThat(status.leaderGlobalCheckpoint(), equalTo(63L));
+    }
+
+    public void testAliasUpdateNonRetryableError() {
+        final ShardFollowTaskParams params = new ShardFollowTaskParams();
+        params.maxReadRequestOperationCount = 64;
+        params.maxOutstandingReadRequests = 1;
+        params.maxOutstandingWriteRequests = 1;
+        final ShardFollowNodeTask task = createShardFollowTask(params);
+        startTask(task, 63, -1);
+
+        aliasesUpdateFailures.add(new RuntimeException());
+        task.coordinateReads();
+        final ShardChangesAction.Response response = generateShardChangesResponse(0, 64, 0L, 0L, 1L, 64L);
+        task.handleReadResponse(0L, 64L, response);
+
+        assertThat(bulkShardOperationRequests.size(), equalTo(0));
+        assertThat(task.isStopped(), equalTo(true));
+        final ShardFollowNodeTaskStatus status = task.getStatus();
+        assertThat(status.followerMappingVersion(), equalTo(0L));
+        assertThat(status.followerSettingsVersion(), equalTo(0L));
+        assertThat(status.followerAliasesVersion(), equalTo(0L));
         assertThat(status.outstandingReadRequests(), equalTo(1));
         assertThat(status.outstandingWriteRequests(), equalTo(0));
         assertThat(status.lastRequestedSeqNo(), equalTo(63L));
@@ -752,7 +839,7 @@ public void testCoordinateWrites() {
         assertThat(shardChangesRequests.get(0)[0], equalTo(0L));
         assertThat(shardChangesRequests.get(0)[1], equalTo(128L));
 
-        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 0L, 63L);
+        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 0L, 1L, 63L);
         // Also invokes coordinatesWrites()
         task.innerHandleReadResponse(0L, 63L, response);
 
@@ -772,7 +859,7 @@ public void testMaxOutstandingWrites() {
         params.maxWriteRequestOperationCount = 64;
         params.maxOutstandingWriteRequests = 2;
         ShardFollowNodeTask task = createShardFollowTask(params);
-        ShardChangesAction.Response response = generateShardChangesResponse(0, 256, 0L, 0L, 256L);
+        ShardChangesAction.Response response = generateShardChangesResponse(0, 256, 0L, 0L, 1L, 256L);
         // Also invokes coordinatesWrites()
         task.innerHandleReadResponse(0L, 64L, response);
 
@@ -785,7 +872,7 @@ public void testMaxOutstandingWrites() {
 
         params.maxOutstandingWriteRequests = 4; // change to 4 outstanding writers
         task = createShardFollowTask(params);
-        response = generateShardChangesResponse(0, 256, 0L, 0L, 256L);
+        response = generateShardChangesResponse(0, 256, 0L, 0L, 1L, 256L);
         // Also invokes coordinatesWrites()
         task.innerHandleReadResponse(0L, 64L, response);
 
@@ -804,7 +891,7 @@ public void testMaxWriteRequestCount() {
         params.maxWriteRequestOperationCount = 8;
         params.maxOutstandingWriteRequests = 32;
         ShardFollowNodeTask task = createShardFollowTask(params);
-        ShardChangesAction.Response response = generateShardChangesResponse(0, 256, 0L, 0L, 256L);
+        ShardChangesAction.Response response = generateShardChangesResponse(0, 256, 0L, 0L, 1L, 256L);
         // Also invokes coordinatesWrites()
         task.innerHandleReadResponse(0L, 64L, response);
 
@@ -835,7 +922,7 @@ public void testRetryableError() {
         for (int i = 0; i < max; i++) {
             writeFailures.add(new ShardNotFoundException(new ShardId("leader_index", "", 0)));
         }
-        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 0L, 63L);
+        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 0L, 1L, 63L);
         // Also invokes coordinatesWrites()
         task.innerHandleReadResponse(0L, 63L, response);
 
@@ -864,7 +951,7 @@ public void testNonRetryableError() {
         assertThat(shardChangesRequests.get(0)[1], equalTo(64L));
 
         writeFailures.add(new RuntimeException());
-        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 0L, 63L);
+        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 0L, 1L, 63L);
         // Also invokes coordinatesWrites()
         task.innerHandleReadResponse(0L, 63L, response);
 
@@ -891,7 +978,7 @@ public void testMaxWriteRequestSize() {
         assertThat(shardChangesRequests.get(0)[0], equalTo(0L));
         assertThat(shardChangesRequests.get(0)[1], equalTo(64L));
 
-        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 0L, 64L);
+        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 0L, 1L, 64L);
         // Also invokes coordinatesWrites()
         task.innerHandleReadResponse(0L, 64L, response);
 
@@ -914,7 +1001,7 @@ public void testHandleWriteResponse() {
 
         shardChangesRequests.clear();
         followerGlobalCheckpoints.add(63L);
-        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 0L, 63L);
+        ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 0L, 1L, 63L);
         // Also invokes coordinatesWrites()
         task.innerHandleReadResponse(0L, 63L, response);
 
@@ -1013,6 +1100,8 @@ private ShardFollowNodeTask createShardFollowTask(ShardFollowTaskParams params)
         mappingVersions = new LinkedList<>();
         settingsUpdateFailures = new LinkedList<>();
         settingsVersions = new LinkedList<>();
+        aliasesUpdateFailures = new LinkedList<>();
+        aliasesVersions = new LinkedList<>();
         leaderGlobalCheckpoints = new LinkedList<>();
         followerGlobalCheckpoints = new LinkedList<>();
         maxSeqNos = new LinkedList<>();
@@ -1048,6 +1137,20 @@ protected void innerUpdateSettings(LongConsumer handler, Consumer<Exception> err
                 }
             }
 
+            @Override
+            protected void innerUpdateAliases(final LongConsumer handler, final Consumer<Exception> errorHandler) {
+                final Exception failure = aliasesUpdateFailures.poll();
+                if (failure != null) {
+                    errorHandler.accept(failure);
+                    return;
+                }
+
+                final Long aliasesVersion = aliasesVersions.poll();
+                if (aliasesVersion != null) {
+                    handler.accept(aliasesVersion);
+                }
+            }
+
             @Override
             protected void innerSendBulkShardOperationsRequest(
                 String followerHistoryUUID, final List<Translog.Operation> operations,
@@ -1086,6 +1189,7 @@ protected void innerSendShardChangesRequest(long from, int requestBatchSize, Con
                     final ShardChangesAction.Response response = new ShardChangesAction.Response(
                         mappingVersions.poll(),
                         0L,
+                        0L,
                         leaderGlobalCheckpoints.poll(),
                         maxSeqNos.poll(),
                         randomNonNegativeLong(),
@@ -1153,6 +1257,7 @@ private static ShardChangesAction.Response generateShardChangesResponse(long fro
                                                                             long toSeqNo,
                                                                             long mappingVersion,
                                                                             long settingsVersion,
+                                                                            long aliasesVersion,
                                                                             long leaderGlobalCheckPoint) {
         List<Translog.Operation> ops = new ArrayList<>();
         for (long seqNo = fromSeqNo; seqNo <= toSeqNo; seqNo++) {
@@ -1163,6 +1268,7 @@ private static ShardChangesAction.Response generateShardChangesResponse(long fro
         return new ShardChangesAction.Response(
             mappingVersion,
             settingsVersion,
+            aliasesVersion,
             leaderGlobalCheckPoint,
             leaderGlobalCheckPoint,
             randomNonNegativeLong(),
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java
index abef313d0b017..f88b6542392c8 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowTaskReplicationTests.java
@@ -105,7 +105,8 @@ public void testSimpleCcrReplication() throws Exception {
                 leaderGroup.assertAllEqual(docCount);
                 Set<String> indexedDocIds = getShardDocUIDs(leaderGroup.getPrimary());
                 assertBusy(() -> {
-                    assertThat(followerGroup.getPrimary().getGlobalCheckpoint(), equalTo(leaderGroup.getPrimary().getGlobalCheckpoint()));
+                    assertThat(followerGroup.getPrimary().getLastKnownGlobalCheckpoint(),
+                        equalTo(leaderGroup.getPrimary().getLastKnownGlobalCheckpoint()));
                     followerGroup.assertAllEqual(indexedDocIds.size());
                 });
                 for (IndexShard shard : followerGroup) {
@@ -119,7 +120,8 @@ public void testSimpleCcrReplication() throws Exception {
                 }
                 leaderGroup.syncGlobalCheckpoint();
                 assertBusy(() -> {
-                    assertThat(followerGroup.getPrimary().getGlobalCheckpoint(), equalTo(leaderGroup.getPrimary().getGlobalCheckpoint()));
+                    assertThat(followerGroup.getPrimary().getLastKnownGlobalCheckpoint(),
+                        equalTo(leaderGroup.getPrimary().getLastKnownGlobalCheckpoint()));
                     followerGroup.assertAllEqual(indexedDocIds.size() - deleteDocIds.size());
                 });
                 shardFollowTask.markAsCompleted();
@@ -192,7 +194,8 @@ public void testChangeLeaderHistoryUUID() throws Exception {
                 leaderGroup.assertAllEqual(docCount);
                 Set<String> indexedDocIds = getShardDocUIDs(leaderGroup.getPrimary());
                 assertBusy(() -> {
-                    assertThat(followerGroup.getPrimary().getGlobalCheckpoint(), equalTo(leaderGroup.getPrimary().getGlobalCheckpoint()));
+                    assertThat(followerGroup.getPrimary().getLastKnownGlobalCheckpoint(),
+                        equalTo(leaderGroup.getPrimary().getLastKnownGlobalCheckpoint()));
                     followerGroup.assertAllEqual(indexedDocIds.size());
                 });
 
@@ -235,7 +238,8 @@ public void testChangeFollowerHistoryUUID() throws Exception {
                 leaderGroup.assertAllEqual(docCount);
                 Set<String> indexedDocIds = getShardDocUIDs(leaderGroup.getPrimary());
                 assertBusy(() -> {
-                    assertThat(followerGroup.getPrimary().getGlobalCheckpoint(), equalTo(leaderGroup.getPrimary().getGlobalCheckpoint()));
+                    assertThat(followerGroup.getPrimary().getLastKnownGlobalCheckpoint(),
+                        equalTo(leaderGroup.getPrimary().getLastKnownGlobalCheckpoint()));
                     followerGroup.assertAllEqual(indexedDocIds.size());
                 });
 
@@ -282,11 +286,12 @@ public void testRetryBulkShardOperations() throws Exception {
                 // Simulates some bulk requests are completed on the primary and replicated to some (but all) replicas of the follower
                 // but the primary of the follower crashed before these requests completed.
                 for (int numBulks = between(1, 5), i = 0; i < numBulks; i++) {
-                    long fromSeqNo = randomLongBetween(0, leadingPrimary.getGlobalCheckpoint());
-                    long toSeqNo = randomLongBetween(fromSeqNo, leadingPrimary.getGlobalCheckpoint());
+                    long fromSeqNo = randomLongBetween(0, leadingPrimary.getLastKnownGlobalCheckpoint());
+                    long toSeqNo = randomLongBetween(fromSeqNo, leadingPrimary.getLastKnownGlobalCheckpoint());
                     int numOps = Math.toIntExact(toSeqNo + 1 - fromSeqNo);
-                    Translog.Operation[] ops = ShardChangesAction.getOperations(leadingPrimary, leadingPrimary.getGlobalCheckpoint(),
-                        fromSeqNo, numOps, leadingPrimary.getHistoryUUID(), new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES));
+                    Translog.Operation[] ops = ShardChangesAction.getOperations(leadingPrimary,
+                        leadingPrimary.getLastKnownGlobalCheckpoint(), fromSeqNo, numOps, leadingPrimary.getHistoryUUID(),
+                        new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES));
 
                     IndexShard followingPrimary = followerGroup.getPrimary();
                     TransportWriteAction.WritePrimaryResult<BulkShardOperationsRequest, BulkShardOperationsResponse> primaryResult =
@@ -296,7 +301,7 @@ public void testRetryBulkShardOperations() throws Exception {
                     for (IndexShard replica : randomSubsetOf(followerGroup.getReplicas())) {
                         final PlainActionFuture<Releasable> permitFuture = new PlainActionFuture<>();
                         replica.acquireReplicaOperationPermit(followingPrimary.getOperationPrimaryTerm(),
-                            followingPrimary.getGlobalCheckpoint(), followingPrimary.getMaxSeqNoOfUpdatesOrDeletes(),
+                            followingPrimary.getLastKnownGlobalCheckpoint(), followingPrimary.getMaxSeqNoOfUpdatesOrDeletes(),
                             permitFuture, ThreadPool.Names.SAME, primaryResult);
                         try (Releasable ignored = permitFuture.get()) {
                             TransportBulkShardOperationsAction.shardOperationOnReplica(primaryResult.replicaRequest(), replica, logger);
@@ -308,13 +313,14 @@ public void testRetryBulkShardOperations() throws Exception {
                 ShardFollowNodeTask shardFollowTask = createShardFollowTask(leaderGroup, followerGroup);
                 SeqNoStats followerSeqNoStats = followerGroup.getPrimary().seqNoStats();
                 shardFollowTask.start(followerGroup.getPrimary().getHistoryUUID(),
-                    leadingPrimary.getGlobalCheckpoint(),
+                    leadingPrimary.getLastKnownGlobalCheckpoint(),
                     leadingPrimary.getMaxSeqNoOfUpdatesOrDeletes(),
                     followerSeqNoStats.getGlobalCheckpoint(),
                     followerSeqNoStats.getMaxSeqNo());
                 try {
                     assertBusy(() -> {
-                        assertThat(followerGroup.getPrimary().getGlobalCheckpoint(), equalTo(leadingPrimary.getGlobalCheckpoint()));
+                        assertThat(followerGroup.getPrimary().getLastKnownGlobalCheckpoint(),
+                            equalTo(leadingPrimary.getLastKnownGlobalCheckpoint()));
                         assertConsistentHistoryBetweenLeaderAndFollower(leaderGroup, followerGroup, true);
                     });
                 } finally {
@@ -380,9 +386,9 @@ public void testSimpleRemoteRecovery() throws Exception {
                 ShardFollowNodeTask followTask = createShardFollowTask(leader, follower);
                 followTask.start(
                     follower.getPrimary().getHistoryUUID(),
-                    leader.getPrimary().getGlobalCheckpoint(),
+                    leader.getPrimary().getLastKnownGlobalCheckpoint(),
                     leader.getPrimary().seqNoStats().getMaxSeqNo(),
-                    follower.getPrimary().getGlobalCheckpoint(),
+                    follower.getPrimary().getLastKnownGlobalCheckpoint(),
                     follower.getPrimary().seqNoStats().getMaxSeqNo()
                 );
                 leader.appendDocs(between(0, 100));
@@ -403,9 +409,9 @@ public void testRetentionLeaseManagement() throws Exception {
                 final ShardFollowNodeTask task = createShardFollowTask(leader, follower);
                 task.start(
                         follower.getPrimary().getHistoryUUID(),
-                        leader.getPrimary().getGlobalCheckpoint(),
+                        leader.getPrimary().getLastKnownGlobalCheckpoint(),
                         leader.getPrimary().seqNoStats().getMaxSeqNo(),
-                        follower.getPrimary().getGlobalCheckpoint(),
+                        follower.getPrimary().getLastKnownGlobalCheckpoint(),
                         follower.getPrimary().seqNoStats().getMaxSeqNo());
                 final Scheduler.Cancellable renewable = task.getRenewable();
                 assertNotNull(renewable);
@@ -514,6 +520,12 @@ protected void innerUpdateSettings(LongConsumer handler, Consumer<Exception> err
                 handler.accept(1L);
             }
 
+            @Override
+            protected void innerUpdateAliases(LongConsumer handler, Consumer<Exception> errorHandler) {
+                // no-op as alias updates are not tested here
+                handler.accept(1L);
+            }
+
             @Override
             protected void innerSendBulkShardOperationsRequest(
                 final String followerHistoryUUID,
@@ -544,14 +556,21 @@ protected void innerSendShardChangesRequest(long from, int maxOperationCount, Co
                             final SeqNoStats seqNoStats = indexShard.seqNoStats();
                             final long maxSeqNoOfUpdatesOrDeletes = indexShard.getMaxSeqNoOfUpdatesOrDeletes();
                             if (from > seqNoStats.getGlobalCheckpoint()) {
-                                handler.accept(ShardChangesAction.getResponse(1L, 1L, seqNoStats,
-                                    maxSeqNoOfUpdatesOrDeletes, ShardChangesAction.EMPTY_OPERATIONS_ARRAY, 1L));
+                                handler.accept(ShardChangesAction.getResponse(
+                                        1L,
+                                        1L,
+                                        1L,
+                                        seqNoStats,
+                                        maxSeqNoOfUpdatesOrDeletes,
+                                        ShardChangesAction.EMPTY_OPERATIONS_ARRAY,
+                                        1L));
                                 return;
                             }
                             Translog.Operation[] ops = ShardChangesAction.getOperations(indexShard, seqNoStats.getGlobalCheckpoint(), from,
                                 maxOperationCount, recordedLeaderIndexHistoryUUID, params.getMaxReadRequestSize());
                             // hard code mapping version; this is ok, as mapping updates are not tested here
                             final ShardChangesAction.Response response = new ShardChangesAction.Response(
+                                1L,
                                 1L,
                                 1L,
                                 seqNoStats.getGlobalCheckpoint(),
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/StatsResponsesTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/StatsResponsesTests.java
index 72ba0cd70672e..4e9aadf8d82a8 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/StatsResponsesTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/StatsResponsesTests.java
@@ -57,6 +57,7 @@ static FollowStatsAction.StatsResponses createStatsResponse() {
                 randomNonNegativeLong(),
                 randomNonNegativeLong(),
                 randomNonNegativeLong(),
+                randomNonNegativeLong(),
                 Collections.emptyNavigableMap(),
                 randomLong(),
                 randomBoolean() ? new ElasticsearchException("fatal error") : null);
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java
index 44f8583bb9b5a..6a4e4d4b6cf0b 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.xpack.ccr.Ccr;
 import org.elasticsearch.xpack.ccr.CcrSettings;
 import org.elasticsearch.xpack.ccr.IndexFollowingIT;
@@ -79,16 +78,6 @@ public void testValidation() throws IOException {
             Exception e = expectThrows(IllegalArgumentException.class, () -> validate(request, leaderIMD, followIMD, UUIDs, null));
             assertThat(e.getMessage(), equalTo("leader index [index1] does not have soft deletes enabled"));
         }
-        {
-            // should fail because leader index does not have soft deletes enabled (by default).
-            Version prevVersion = VersionUtils.randomVersionBetween(
-                random(), Version.V_6_5_0, VersionUtils.getPreviousVersion(Version.V_7_0_0));
-            IndexMetaData leaderIMD = IndexMetaData.builder("index1").settings(settings(prevVersion)).numberOfShards(1)
-                .numberOfReplicas(0).setRoutingNumShards(1).putMapping("_doc", "{\"properties\": {}}").build();
-            IndexMetaData followIMD = createIMD("index2", 5, Settings.EMPTY, customMetaData);
-            Exception e = expectThrows(IllegalArgumentException.class, () -> validate(request, leaderIMD, followIMD, UUIDs, null));
-            assertThat(e.getMessage(), equalTo("leader index [index1] does not have soft deletes enabled"));
-        }
         {
             // should fail because the follower index does not have soft deletes enabled
             IndexMetaData leaderIMD = createIMD("index1", 5, Settings.EMPTY, null);
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsTests.java
index 856b6da2f9d7e..43302a5177ed7 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/bulk/BulkShardOperationsTests.java
@@ -142,7 +142,7 @@ public void testPrimaryResultIncludeOnlyAppliedOperations() throws Exception {
             newPrimary.getHistoryUUID(), Stream.concat(secondBulk.stream(), existingOps.stream()).collect(Collectors.toList()),
             seqno, newPrimary, logger);
         final long newPrimaryTerm = newPrimary.getOperationPrimaryTerm();
-        final long globalCheckpoint = newPrimary.getGlobalCheckpoint();
+        final long globalCheckpoint = newPrimary.getLastKnownGlobalCheckpoint();
         final List<Translog.Operation> appliedOperations = Stream.concat(
             secondBulk.stream().map(op -> rewriteOperationWithPrimaryTerm(op, newPrimaryTerm)),
             existingOps.stream().filter(op -> op.seqNo() > globalCheckpoint).map(op -> rewriteOperationWithPrimaryTerm(op, oldPrimaryTerm))
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java
index 4a56d6370eb91..98bfa1b2068bb 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java
@@ -337,7 +337,7 @@ public void testBasicOptimization() throws Exception {
             for (int i = 0; i < numDocs; i++) {
                 leader.index(indexForPrimary(Integer.toString(i)));
             }
-            EngineTestCase.waitForOpsToComplete(follower, leader.getLocalCheckpoint());
+            EngineTestCase.waitForOpsToComplete(follower, leader.getProcessedLocalCheckpoint());
             assertThat(follower.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(-1L));
             assertThat(follower.getNumberOfOptimizedIndexing(), equalTo(numDocs));
             assertThat(getDocIds(follower, true), equalTo(getDocIds(leader, true)));
@@ -350,7 +350,7 @@ public void testBasicOptimization() throws Exception {
                     leader.delete(deleteForPrimary(Integer.toString(i)));
                 }
             }
-            EngineTestCase.waitForOpsToComplete(follower, leader.getLocalCheckpoint());
+            EngineTestCase.waitForOpsToComplete(follower, leader.getProcessedLocalCheckpoint());
             assertThat(follower.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(leader.getMaxSeqNoOfUpdatesOrDeletes()));
             assertThat(follower.getNumberOfOptimizedIndexing(), equalTo(numDocs));
             assertThat(getDocIds(follower, true), equalTo(getDocIds(leader, true)));
@@ -362,7 +362,7 @@ public void testBasicOptimization() throws Exception {
                 docIds.add(docId);
                 leader.index(indexForPrimary(docId));
             }
-            EngineTestCase.waitForOpsToComplete(follower, leader.getLocalCheckpoint());
+            EngineTestCase.waitForOpsToComplete(follower, leader.getProcessedLocalCheckpoint());
             assertThat(follower.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(leader.getMaxSeqNoOfUpdatesOrDeletes()));
             assertThat(follower.getNumberOfOptimizedIndexing(), equalTo(numDocs + moreDocs));
             assertThat(getDocIds(follower, true), equalTo(getDocIds(leader, true)));
@@ -378,7 +378,7 @@ public void testOptimizeAppendOnly() throws Exception {
         runFollowTest((leader, follower) -> {
             EngineTestCase.concurrentlyApplyOps(ops, leader);
             assertThat(follower.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(-1L));
-            EngineTestCase.waitForOpsToComplete(follower, leader.getLocalCheckpoint());
+            EngineTestCase.waitForOpsToComplete(follower, leader.getProcessedLocalCheckpoint());
             assertThat(follower.getNumberOfOptimizedIndexing(), equalTo((long) numOps));
         });
     }
@@ -396,13 +396,13 @@ public void testOptimizeMultipleVersions() throws Exception {
         Randomness.shuffle(ops);
         runFollowTest((leader, follower) -> {
             EngineTestCase.concurrentlyApplyOps(ops, leader);
-            EngineTestCase.waitForOpsToComplete(follower, leader.getLocalCheckpoint());
+            EngineTestCase.waitForOpsToComplete(follower, leader.getProcessedLocalCheckpoint());
             final List<Engine.Operation> appendOps = new ArrayList<>();
             for (int numAppends = scaledRandomIntBetween(0, 100), i = 0; i < numAppends; i++) {
                 appendOps.add(indexForPrimary("append-" + i));
             }
             EngineTestCase.concurrentlyApplyOps(appendOps, leader);
-            EngineTestCase.waitForOpsToComplete(follower, leader.getLocalCheckpoint());
+            EngineTestCase.waitForOpsToComplete(follower, leader.getProcessedLocalCheckpoint());
             assertThat(follower.getNumberOfOptimizedIndexing(), greaterThanOrEqualTo((long) appendOps.size()));
         });
     }
@@ -410,19 +410,19 @@ public void testOptimizeMultipleVersions() throws Exception {
     public void testOptimizeSingleDocSequentially() throws Exception {
         runFollowTest((leader, follower) -> {
             leader.index(indexForPrimary("id"));
-            EngineTestCase.waitForOpsToComplete(follower, leader.getLocalCheckpoint());
+            EngineTestCase.waitForOpsToComplete(follower, leader.getProcessedLocalCheckpoint());
             assertThat(follower.getNumberOfOptimizedIndexing(), equalTo(1L));
 
             leader.delete(deleteForPrimary("id"));
-            EngineTestCase.waitForOpsToComplete(follower, leader.getLocalCheckpoint());
+            EngineTestCase.waitForOpsToComplete(follower, leader.getProcessedLocalCheckpoint());
             assertThat(follower.getNumberOfOptimizedIndexing(), equalTo(1L));
 
             leader.index(indexForPrimary("id"));
-            EngineTestCase.waitForOpsToComplete(follower, leader.getLocalCheckpoint());
+            EngineTestCase.waitForOpsToComplete(follower, leader.getProcessedLocalCheckpoint());
             assertThat(follower.getNumberOfOptimizedIndexing(), equalTo(2L));
 
             leader.index(indexForPrimary("id"));
-            EngineTestCase.waitForOpsToComplete(follower, leader.getLocalCheckpoint());
+            EngineTestCase.waitForOpsToComplete(follower, leader.getProcessedLocalCheckpoint());
             assertThat(follower.getNumberOfOptimizedIndexing(), equalTo(2L));
         });
     }
@@ -432,20 +432,20 @@ public void testOptimizeSingleDocConcurrently() throws Exception {
         Randomness.shuffle(ops);
         runFollowTest((leader, follower) -> {
             EngineTestCase.concurrentlyApplyOps(ops, leader);
-            EngineTestCase.waitForOpsToComplete(follower, leader.getLocalCheckpoint());
+            EngineTestCase.waitForOpsToComplete(follower, leader.getProcessedLocalCheckpoint());
             assertThat(getDocIds(follower, true), equalTo(getDocIds(leader, true)));
             long numOptimized = follower.getNumberOfOptimizedIndexing();
 
             leader.delete(deleteForPrimary("id"));
-            EngineTestCase.waitForOpsToComplete(follower, leader.getLocalCheckpoint());
+            EngineTestCase.waitForOpsToComplete(follower, leader.getProcessedLocalCheckpoint());
             assertThat(follower.getNumberOfOptimizedIndexing(), equalTo(numOptimized));
 
             leader.index(indexForPrimary("id"));
-            EngineTestCase.waitForOpsToComplete(follower, leader.getLocalCheckpoint());
+            EngineTestCase.waitForOpsToComplete(follower, leader.getProcessedLocalCheckpoint());
             assertThat(follower.getNumberOfOptimizedIndexing(), equalTo(numOptimized + 1L));
 
             leader.index(indexForPrimary("id"));
-            EngineTestCase.waitForOpsToComplete(follower, leader.getLocalCheckpoint());
+            EngineTestCase.waitForOpsToComplete(follower, leader.getProcessedLocalCheckpoint());
             assertThat(follower.getNumberOfOptimizedIndexing(), equalTo(numOptimized + 1L));
         });
     }
@@ -454,7 +454,7 @@ private void runFollowTest(CheckedBiConsumer<InternalEngine, FollowingEngine, Ex
         final CheckedBiConsumer<InternalEngine, FollowingEngine, Exception> wrappedTask = (leader, follower) -> {
             Thread[] threads = new Thread[between(1, 8)];
             AtomicBoolean taskIsCompleted = new AtomicBoolean();
-            AtomicLong lastFetchedSeqNo = new AtomicLong(follower.getLocalCheckpoint());
+            AtomicLong lastFetchedSeqNo = new AtomicLong(follower.getProcessedLocalCheckpoint());
             CountDownLatch latch = new CountDownLatch(threads.length + 1);
             for (int i = 0; i < threads.length; i++) {
                 threads[i] = new Thread(() -> {
@@ -472,7 +472,7 @@ private void runFollowTest(CheckedBiConsumer<InternalEngine, FollowingEngine, Ex
                 latch.countDown();
                 latch.await();
                 task.accept(leader, follower);
-                EngineTestCase.waitForOpsToComplete(follower, leader.getLocalCheckpoint());
+                EngineTestCase.waitForOpsToComplete(follower, leader.getProcessedLocalCheckpoint());
             } finally {
                 taskIsCompleted.set(true);
                 for (Thread thread : threads) {
@@ -516,7 +516,7 @@ private void fetchOperations(AtomicBoolean stopped, AtomicLong lastFetchedSeqNo,
         final MapperService mapperService = EngineTestCase.createMapperService("test");
         final TranslogHandler translogHandler = new TranslogHandler(xContentRegistry(), follower.config().getIndexSettings());
         while (stopped.get() == false) {
-            final long checkpoint = leader.getLocalCheckpoint();
+            final long checkpoint = leader.getProcessedLocalCheckpoint();
             final long lastSeqNo = lastFetchedSeqNo.get();
             if (lastSeqNo < checkpoint) {
                 final long nextSeqNo = randomLongBetween(lastSeqNo + 1, checkpoint);
@@ -607,7 +607,7 @@ public void testProcessOnceOnPrimary() throws Exception {
                     }
                 }
                 // Primary should reject duplicates
-                globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), followingEngine.getLocalCheckpoint()));
+                globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), followingEngine.getProcessedLocalCheckpoint()));
                 final long newTerm = randomLongBetween(oldTerm + 1, Long.MAX_VALUE);
                 for (Engine.Operation op : operations) {
                     Engine.Result result = applyOperation(followingEngine, op, newTerm, Engine.Operation.Origin.PRIMARY);
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/FollowStatsMonitoringDocTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/FollowStatsMonitoringDocTests.java
index fd8904307db3e..12c251bb9d775 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/FollowStatsMonitoringDocTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/FollowStatsMonitoringDocTests.java
@@ -95,6 +95,7 @@ public void testToXContent() throws IOException {
         final long writeBufferSizeInBytes = randomNonNegativeLong();
         final long followerMappingVersion = randomNonNegativeLong();
         final long followerSettingsVersion = randomNonNegativeLong();
+        final long followerAliasesVersion = randomNonNegativeLong();
         final long totalReadTimeMillis = randomLongBetween(0, 4096);
         final long totalReadRemoteExecTimeMillis = randomLongBetween(0, 4096);
         final long successfulReadRequests = randomNonNegativeLong();
@@ -126,6 +127,7 @@ public void testToXContent() throws IOException {
                 writeBufferSizeInBytes,
                 followerMappingVersion,
                 followerSettingsVersion,
+                followerAliasesVersion,
                 totalReadTimeMillis,
                 totalReadRemoteExecTimeMillis,
                 successfulReadRequests,
@@ -173,6 +175,7 @@ public void testToXContent() throws IOException {
                                         + "\"write_buffer_size_in_bytes\":" + writeBufferSizeInBytes + ","
                                         + "\"follower_mapping_version\":" + followerMappingVersion + ","
                                         + "\"follower_settings_version\":" + followerSettingsVersion + ","
+                                        + "\"follower_aliases_version\":" + followerAliasesVersion + ","
                                         + "\"total_read_time_millis\":" + totalReadTimeMillis + ","
                                         + "\"total_read_remote_exec_time_millis\":" + totalReadRemoteExecTimeMillis + ","
                                         + "\"successful_read_requests\":" + successfulReadRequests + ","
@@ -218,6 +221,7 @@ public void testShardFollowNodeTaskStatusFieldsMapped() throws IOException {
             1,
             1,
             1,
+            1,
             100,
             50,
             10,
diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollectorTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollectorTests.java
index bb44fd59da5d2..2b85cd11e927f 100644
--- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollectorTests.java
+++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollectorTests.java
@@ -15,9 +15,8 @@
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ccr.AutoFollowStats;
 import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus;
-import org.elasticsearch.xpack.core.ccr.action.FollowStatsAction;
 import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
-import org.elasticsearch.xpack.core.ccr.client.CcrClient;
+import org.elasticsearch.xpack.core.ccr.action.FollowStatsAction;
 import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc;
 import org.elasticsearch.xpack.monitoring.BaseCollectorTestCase;
@@ -33,6 +32,7 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -127,7 +127,7 @@ public void testDoCollect() throws Exception {
         whenClusterStateWithUUID(clusterUuid);
 
         final MonitoringDoc.Node node = randomMonitoringNode(random());
-        final CcrClient client = mock(CcrClient.class);
+        final Client client = mock(Client.class);
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         final List<FollowStatsAction.StatsResponse> statuses = mockStatuses();
 
@@ -142,7 +142,7 @@ public void testDoCollect() throws Exception {
         final ActionFuture<CcrStatsAction.Response> future = (ActionFuture<CcrStatsAction.Response>) mock(ActionFuture.class);
         final CcrStatsAction.Response response = new CcrStatsAction.Response(autoFollowStats, statsResponse);
 
-        when(client.stats(any())).thenReturn(future);
+        when(client.execute(eq(CcrStatsAction.INSTANCE), any(CcrStatsAction.Request.class))).thenReturn(future);
         when(future.actionGet(timeout)).thenReturn(response);
 
         final StatsCollector collector = new StatsCollector(settings, clusterService, licenseState, client, threadContext);
diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle
index d805a491e093a..151b675fa1085 100644
--- a/x-pack/plugin/core/build.gradle
+++ b/x-pack/plugin/core/build.gradle
@@ -23,13 +23,14 @@ dependencyLicenses {
 }
 
 dependencies {
-    compileOnly "org.elasticsearch:elasticsearch:${version}"
+    compileOnly project(":server")
     compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
     compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
     compile "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
     compile "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
 
     compile "commons-logging:commons-logging:${versions.commonslogging}"
+    compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
     compile "commons-codec:commons-codec:${versions.commonscodec}"
 
     // security deps
@@ -48,7 +49,7 @@ dependencies {
     testCompile project(path: ':modules:parent-join', configuration: 'runtime')
     testCompile project(path: ':modules:lang-mustache', configuration: 'runtime')
     testCompile project(path: ':modules:analysis-common', configuration: 'runtime')
-    testCompile ("org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}")
+    testCompile project(":client:rest-high-level")
 
     if (isEclipse == false || project.path == ":x-pack:plugin:core-tests") {
       testCompile(project(':x-pack:license-tools')) {
@@ -102,7 +103,7 @@ forbiddenApisMain {
 if (isEclipse) {
     // in eclipse the project is under a fake root, we need to change around the source sets
     sourceSets {
-        if (project.path == ":libs:core") {
+        if (project.path == ":libs:elasticsearch-core") {
             main.java.srcDirs = ['java']
             main.resources.srcDirs = ['resources']
         } else {
@@ -145,7 +146,7 @@ task testJar(type: Jar) {
 }
 
 artifacts {
-    // normal es plugins do not publish the jar but we need to since users need it for Transport Clients and extensions
+    // normal es plugins do not publish the jar but we need to since users need it for extensions
     archives jar
     testArtifacts testJar
 }
diff --git a/x-pack/plugin/core/licenses/log4j-1.2-api-2.11.1.jar.sha1 b/x-pack/plugin/core/licenses/log4j-1.2-api-2.11.1.jar.sha1
new file mode 100644
index 0000000000000..575d75dbda8c5
--- /dev/null
+++ b/x-pack/plugin/core/licenses/log4j-1.2-api-2.11.1.jar.sha1
@@ -0,0 +1 @@
+3aba3398fe064a3eab4331f88161c7480e848418
\ No newline at end of file
diff --git a/x-pack/plugin/core/licenses/log4j-LICENSE.txt b/x-pack/plugin/core/licenses/log4j-LICENSE.txt
new file mode 100644
index 0000000000000..6279e5206de13
--- /dev/null
+++ b/x-pack/plugin/core/licenses/log4j-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 1999-2005 The Apache Software Foundation
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/x-pack/plugin/core/licenses/log4j-NOTICE.txt b/x-pack/plugin/core/licenses/log4j-NOTICE.txt
new file mode 100644
index 0000000000000..0375732360047
--- /dev/null
+++ b/x-pack/plugin/core/licenses/log4j-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache log4j
+Copyright 2007 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
\ No newline at end of file
diff --git a/x-pack/plugin/core/src/main/config/log4j2.properties b/x-pack/plugin/core/src/main/config/log4j2.properties
index 2b7e112eb14fc..c37faf84afbea 100644
--- a/x-pack/plugin/core/src/main/config/log4j2.properties
+++ b/x-pack/plugin/core/src/main/config/log4j2.properties
@@ -3,7 +3,8 @@ appender.audit_rolling.name = audit_rolling
 appender.audit_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_audit.json
 appender.audit_rolling.layout.type = PatternLayout
 appender.audit_rolling.layout.pattern = {\
-                "@timestamp":"%d{ISO8601}"\
+                "type":"audit", \
+                "timestamp":"%d{yyyy-MM-dd'T'HH:mm:ss,SSSZ}"\
                 %varsNotEmpty{, "node.name":"%enc{%map{node.name}}{JSON}"}\
                 %varsNotEmpty{, "node.id":"%enc{%map{node.id}}{JSON}"}\
                 %varsNotEmpty{, "host.name":"%enc{%map{host.name}}{JSON}"}\
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
index 50f1125b275f1..412d64803a015 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
@@ -78,7 +78,7 @@ public FrozenEngine(EngineConfig config) {
 
         boolean success = false;
         Directory directory = store.directory();
-        try (DirectoryReader reader = DirectoryReader.open(directory)) {
+        try (DirectoryReader reader = DirectoryReader.open(directory, OFF_HEAP_READER_ATTRIBUTES)) {
             canMatchReader = ElasticsearchDirectoryReader.wrap(new RewriteCachingDirectoryReader(directory, reader.leaves()),
                 config.getShardId());
             // we record the segment stats here - that's what the reader needs when it's open and it give the user
@@ -168,7 +168,7 @@ private synchronized DirectoryReader getOrOpenReader() throws IOException {
                 for (ReferenceManager.RefreshListener listeners : config ().getInternalRefreshListener()) {
                     listeners.beforeRefresh();
                 }
-                reader = DirectoryReader.open(engineConfig.getStore().directory());
+                reader = DirectoryReader.open(engineConfig.getStore().directory(), OFF_HEAP_READER_ATTRIBUTES);
                 processReader(reader);
                 reader = lastOpenedReader = wrapReader(reader, Function.identity());
                 reader.getReaderCacheHelper().addClosedListener(this::onReaderClosed);
@@ -272,8 +272,8 @@ public static class ReacquireEngineSearcherListener implements SearchOperationLi
 
         @Override
         public void validateSearchContext(SearchContext context, TransportRequest transportRequest) {
-            Searcher engineSearcher = context.searcher().getEngineSearcher();
-            LazyDirectoryReader lazyDirectoryReader = unwrapLazyReader(engineSearcher.getDirectoryReader());
+            DirectoryReader dirReader = context.searcher().getDirectoryReader();
+            LazyDirectoryReader lazyDirectoryReader = unwrapLazyReader(dirReader);
             if (lazyDirectoryReader != null) {
                 try {
                     lazyDirectoryReader.reset();
@@ -297,8 +297,8 @@ private void registerRelease(SearchContext context, LazyDirectoryReader lazyDire
 
         @Override
         public void onNewContext(SearchContext context) {
-            Searcher engineSearcher = context.searcher().getEngineSearcher();
-            LazyDirectoryReader lazyDirectoryReader = unwrapLazyReader(engineSearcher.getDirectoryReader());
+            DirectoryReader dirReader = context.searcher().getDirectoryReader();
+            LazyDirectoryReader lazyDirectoryReader = unwrapLazyReader(dirReader);
             if (lazyDirectoryReader != null) {
                 registerRelease(context, lazyDirectoryReader);
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseAction.java
index c0dbb02aed719..41356285935cc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/DeleteLicenseAction.java
@@ -5,10 +5,11 @@
  */
 package org.elasticsearch.license;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class DeleteLicenseAction extends Action<AcknowledgedResponse> {
+public class DeleteLicenseAction extends ActionType<AcknowledgedResponse> {
 
     public static final DeleteLicenseAction INSTANCE = new DeleteLicenseAction();
     public static final String NAME = "cluster:admin/xpack/license/delete";
@@ -18,7 +19,7 @@ private DeleteLicenseAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetBasicStatusAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetBasicStatusAction.java
index be97ff59172fd..e93320c6a434a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetBasicStatusAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetBasicStatusAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.license;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class GetBasicStatusAction extends Action<GetBasicStatusResponse> {
+public class GetBasicStatusAction extends StreamableResponseActionType<GetBasicStatusResponse> {
 
     public static final GetBasicStatusAction INSTANCE = new GetBasicStatusAction();
     public static final String NAME = "cluster:admin/xpack/license/basic_status";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetBasicStatusRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetBasicStatusRequestBuilder.java
index e56451c4a92ff..368bd4a9b735e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetBasicStatusRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetBasicStatusRequestBuilder.java
@@ -10,7 +10,7 @@
 
 class GetBasicStatusRequestBuilder extends ActionRequestBuilder<GetBasicStatusRequest, GetBasicStatusResponse> {
 
-    GetBasicStatusRequestBuilder(ElasticsearchClient client, GetBasicStatusAction action) {
-        super(client, action, new GetBasicStatusRequest());
+    GetBasicStatusRequestBuilder(ElasticsearchClient client) {
+        super(client, GetBasicStatusAction.INSTANCE, new GetBasicStatusRequest());
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetLicenseAction.java
index a6f19ea95b1e8..3584e89834c55 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetLicenseAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.license;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class GetLicenseAction extends Action<GetLicenseResponse> {
+public class GetLicenseAction extends StreamableResponseActionType<GetLicenseResponse> {
 
     public static final GetLicenseAction INSTANCE = new GetLicenseAction();
     public static final String NAME = "cluster:monitor/xpack/license/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetTrialStatusAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetTrialStatusAction.java
index 69c14e1b6dc83..be00612ef62a0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetTrialStatusAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetTrialStatusAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.license;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class GetTrialStatusAction extends Action<GetTrialStatusResponse> {
+public class GetTrialStatusAction extends StreamableResponseActionType<GetTrialStatusResponse> {
 
     public static final GetTrialStatusAction INSTANCE = new GetTrialStatusAction();
     public static final String NAME = "cluster:admin/xpack/license/trial_status";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetTrialStatusRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetTrialStatusRequestBuilder.java
index d220b476bc578..4102e98f45d68 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetTrialStatusRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetTrialStatusRequestBuilder.java
@@ -10,7 +10,7 @@
 
 class GetTrialStatusRequestBuilder extends ActionRequestBuilder<GetTrialStatusRequest, GetTrialStatusResponse> {
 
-    GetTrialStatusRequestBuilder(ElasticsearchClient client, GetTrialStatusAction action) {
-        super(client, action, new GetTrialStatusRequest());
+    GetTrialStatusRequestBuilder(ElasticsearchClient client) {
+        super(client, GetTrialStatusAction.INSTANCE, new GetTrialStatusRequest());
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseUtils.java
index 4c8a558682b13..c39b37373ea13 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseUtils.java
@@ -6,12 +6,9 @@
 package org.elasticsearch.license;
 
 import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.rest.RestStatus;
 
-import java.util.stream.StreamSupport;
-
 public class LicenseUtils {
 
     public static final String EXPIRED_FEATURE_METADATA = "es.license.expired.feature";
@@ -58,13 +55,6 @@ public static boolean signatureNeedsUpdate(License license, DiscoveryNodes curre
 
     public static int compatibleLicenseVersion(DiscoveryNodes currentNodes) {
         assert License.VERSION_CRYPTO_ALGORITHMS == License.VERSION_CURRENT : "update this method when adding a new version";
-
-        if (StreamSupport.stream(currentNodes.spliterator(), false)
-            .allMatch(node -> node.getVersion().onOrAfter(Version.V_6_4_0))) {
-            // License.VERSION_CRYPTO_ALGORITHMS was introduced in 6.4.0
-            return License.VERSION_CRYPTO_ALGORITHMS;
-        } else {
-            return License.VERSION_START_DATE;
-        }
+        return License.VERSION_CRYPTO_ALGORITHMS;
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java
index 80d1fb68f0e2a..4dcfa94f6e4d2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicensingClient.java
@@ -45,11 +45,11 @@ public void deleteLicense(DeleteLicenseRequest request, ActionListener<Acknowled
     }
 
     public PostStartTrialRequestBuilder preparePostStartTrial() {
-        return new PostStartTrialRequestBuilder(client, PostStartTrialAction.INSTANCE);
+        return new PostStartTrialRequestBuilder(client);
     }
 
     public GetTrialStatusRequestBuilder prepareGetStartTrial() {
-        return new GetTrialStatusRequestBuilder(client, GetTrialStatusAction.INSTANCE);
+        return new GetTrialStatusRequestBuilder(client);
     }
 
     public void postStartTrial(PostStartTrialRequest request, ActionListener<PostStartTrialResponse> listener) {
@@ -61,10 +61,10 @@ public void postStartBasic(PostStartBasicRequest request, ActionListener<PostSta
     }
 
     public PostStartBasicRequestBuilder preparePostStartBasic() {
-        return new PostStartBasicRequestBuilder(client, PostStartBasicAction.INSTANCE);
+        return new PostStartBasicRequestBuilder(client);
     }
 
     public GetBasicStatusRequestBuilder prepareGetStartBasic() {
-        return new GetBasicStatusRequestBuilder(client, GetBasicStatusAction.INSTANCE);
+        return new GetBasicStatusRequestBuilder(client);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicAction.java
index 864bfa2b7780f..3087a97b8aecd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicAction.java
@@ -5,9 +5,10 @@
  */
 package org.elasticsearch.license;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class PostStartBasicAction extends Action<PostStartBasicResponse> {
+public class PostStartBasicAction extends ActionType<PostStartBasicResponse> {
 
     public static final PostStartBasicAction INSTANCE = new PostStartBasicAction();
     public static final String NAME = "cluster:admin/xpack/license/start_basic";
@@ -17,7 +18,7 @@ private PostStartBasicAction() {
     }
 
     @Override
-    public PostStartBasicResponse newResponse() {
-        return new PostStartBasicResponse();
+    public Writeable.Reader<PostStartBasicResponse> getResponseReader() {
+        return PostStartBasicResponse::new;
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicRequestBuilder.java
index 09591bff2e8ba..aa9ab47ec9aaa 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicRequestBuilder.java
@@ -10,8 +10,8 @@
 
 class PostStartBasicRequestBuilder extends ActionRequestBuilder<PostStartBasicRequest, PostStartBasicResponse> {
 
-    PostStartBasicRequestBuilder(ElasticsearchClient client, PostStartBasicAction action) {
-        super(client, action, new PostStartBasicRequest());
+    PostStartBasicRequestBuilder(ElasticsearchClient client) {
+        super(client, PostStartBasicAction.INSTANCE, new PostStartBasicRequest());
     }
 
     public PostStartBasicRequestBuilder setAcknowledge(boolean acknowledge) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicResponse.java
index 40e910430e31c..84cbcdef102e5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartBasicResponse.java
@@ -26,8 +26,8 @@ public class PostStartBasicResponse extends AcknowledgedResponse implements Stat
     private static final ParseField ERROR_MESSAGE_FIELD = new ParseField("error_message");
     private static final ParseField MESSAGE_FIELD = new ParseField("message");
 
-    private Map<String, String[]> acknowledgeMessages;
-    private String acknowledgeMessage;
+    private final Map<String, String[]> acknowledgeMessages;
+    private final String acknowledgeMessage;
 
     public enum Status {
         GENERATED_BASIC(true, null, RestStatus.OK),
@@ -57,9 +57,24 @@ RestStatus getRestStatus() {
         }
     }
 
-    private Status status;
+    private final Status status;
 
-    public PostStartBasicResponse() {
+    public PostStartBasicResponse(StreamInput in) throws IOException {
+        super(in);
+        status = in.readEnum(Status.class);
+        acknowledgeMessage = in.readOptionalString();
+        int size = in.readVInt();
+        Map<String, String[]> acknowledgeMessages = new HashMap<>(size);
+        for (int i = 0; i < size; i++) {
+            String feature = in.readString();
+            int nMessages = in.readVInt();
+            String[] messages = new String[nMessages];
+            for (int j = 0; j < nMessages; j++) {
+                messages[j] = in.readString();
+            }
+            acknowledgeMessages.put(feature, messages);
+        }
+        this.acknowledgeMessages = acknowledgeMessages;
     }
 
     PostStartBasicResponse(Status status) {
@@ -77,25 +92,6 @@ public Status getStatus() {
         return status;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        status = in.readEnum(Status.class);
-        acknowledgeMessage = in.readOptionalString();
-        int size = in.readVInt();
-        Map<String, String[]> acknowledgeMessages = new HashMap<>(size);
-        for (int i = 0; i < size; i++) {
-            String feature = in.readString();
-            int nMessages = in.readVInt();
-            String[] messages = new String[nMessages];
-            for (int j = 0; j < nMessages; j++) {
-                messages[j] = in.readString();
-            }
-            acknowledgeMessages.put(feature, messages);
-        }
-        this.acknowledgeMessages = acknowledgeMessages;
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialAction.java
index 609fa42caabc2..bead8bc72797c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.license;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class PostStartTrialAction extends Action<PostStartTrialResponse> {
+public class PostStartTrialAction extends StreamableResponseActionType<PostStartTrialResponse> {
 
     public static final PostStartTrialAction INSTANCE = new PostStartTrialAction();
     public static final String NAME = "cluster:admin/xpack/license/start_trial";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialRequestBuilder.java
index 8e12c879f997d..93a6207d519d1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PostStartTrialRequestBuilder.java
@@ -10,8 +10,8 @@
 
 class PostStartTrialRequestBuilder extends ActionRequestBuilder<PostStartTrialRequest, PostStartTrialResponse> {
 
-    PostStartTrialRequestBuilder(ElasticsearchClient client, PostStartTrialAction action) {
-        super(client, action, new PostStartTrialRequest());
+    PostStartTrialRequestBuilder(ElasticsearchClient client) {
+        super(client, PostStartTrialAction.INSTANCE, new PostStartTrialRequest());
     }
 
     public PostStartTrialRequestBuilder setAcknowledge(boolean acknowledge) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseAction.java
index 497b203f4136d..120a2ae505d40 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/PutLicenseAction.java
@@ -5,10 +5,11 @@
  */
 package org.elasticsearch.license;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
 
-public class PutLicenseAction extends Action<PutLicenseResponse> {
+public class PutLicenseAction extends ActionType<PutLicenseResponse> {
 
     public static final PutLicenseAction INSTANCE = new PutLicenseAction();
     public static final String NAME = "cluster:admin/xpack/license/put";
@@ -18,7 +19,7 @@ private PutLicenseAction() {
     }
 
     @Override
-    public PutLicenseResponse newResponse() {
-        return new PutLicenseResponse();
+    public Writeable.Reader<PutLicenseResponse> getResponseReader() {
+        return PutLicenseResponse::new;
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLicenseAction.java
index 5383726adc318..56c1898ac793f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestDeleteLicenseAction.java
@@ -7,20 +7,20 @@
 package org.elasticsearch.license;
 
 import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xpack.core.XPackClient;
-import org.elasticsearch.xpack.core.rest.XPackRestHandler;
 
 import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 
-public class RestDeleteLicenseAction extends XPackRestHandler {
+public class RestDeleteLicenseAction extends BaseRestHandler {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestDeleteLicenseAction.class));
 
@@ -29,7 +29,7 @@ public class RestDeleteLicenseAction extends XPackRestHandler {
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
                 DELETE, "/_license", this,
-                DELETE, URI_BASE + "/license", deprecationLogger);
+                DELETE,  "/_xpack/license", deprecationLogger);
     }
 
     @Override
@@ -38,12 +38,12 @@ public String getName() {
     }
 
     @Override
-    public RestChannelConsumer doPrepareRequest(final RestRequest request, final XPackClient client) throws IOException {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         DeleteLicenseRequest deleteLicenseRequest = new DeleteLicenseRequest();
         deleteLicenseRequest.timeout(request.paramAsTime("timeout", deleteLicenseRequest.timeout()));
         deleteLicenseRequest.masterNodeTimeout(request.paramAsTime("master_timeout", deleteLicenseRequest.masterNodeTimeout()));
 
-        return channel -> client.es().admin().cluster().execute(DeleteLicenseAction.INSTANCE, deleteLicenseRequest,
+        return channel -> client.admin().cluster().execute(DeleteLicenseAction.INSTANCE, deleteLicenseRequest,
                 new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetBasicStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetBasicStatus.java
index 0195b350b050c..3e61c3978bcbc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetBasicStatus.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetBasicStatus.java
@@ -7,17 +7,17 @@
 package org.elasticsearch.license;
 
 import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xpack.core.XPackClient;
-import org.elasticsearch.xpack.core.rest.XPackRestHandler;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
-public class RestGetBasicStatus extends XPackRestHandler {
+public class RestGetBasicStatus extends BaseRestHandler {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetBasicStatus.class));
 
@@ -26,12 +26,12 @@ public class RestGetBasicStatus extends XPackRestHandler {
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
                 GET, "/_license/basic_status", this,
-                GET, URI_BASE + "/license/basic_status", deprecationLogger);
+                GET, "/_xpack/license/basic_status", deprecationLogger);
     }
 
     @Override
-    protected RestChannelConsumer doPrepareRequest(RestRequest request, XPackClient client) {
-        return channel -> client.licensing().prepareGetStartBasic().execute(new RestToXContentListener<>(channel));
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
+        return channel -> new GetBasicStatusRequestBuilder(client).execute(new RestToXContentListener<>(channel));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java
index 02809ae974cd7..4c1102208c840 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetLicenseAction.java
@@ -7,18 +7,18 @@
 package org.elasticsearch.license;
 
 import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
-import org.elasticsearch.xpack.core.XPackClient;
-import org.elasticsearch.xpack.core.rest.XPackRestHandler;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -28,7 +28,7 @@
 import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
-public class RestGetLicenseAction extends XPackRestHandler {
+public class RestGetLicenseAction extends BaseRestHandler {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetLicenseAction.class));
 
@@ -37,7 +37,7 @@ public class RestGetLicenseAction extends XPackRestHandler {
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
                 GET, "/_license", this,
-                GET,  URI_BASE + "/license", deprecationLogger);
+                GET, "/_xpack/license", deprecationLogger);
     }
 
     @Override
@@ -52,15 +52,15 @@ public String getName() {
      * The licenses are sorted by latest issue_date
      */
     @Override
-    public RestChannelConsumer doPrepareRequest(final RestRequest request, final XPackClient client) throws IOException {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final Map<String, String> overrideParams = new HashMap<>(2);
         overrideParams.put(License.REST_VIEW_MODE, "true");
         overrideParams.put(License.LICENSE_VERSION_MODE, String.valueOf(License.VERSION_CURRENT));
         final ToXContent.Params params = new ToXContent.DelegatingMapParams(overrideParams, request);
         GetLicenseRequest getLicenseRequest = new GetLicenseRequest();
         getLicenseRequest.local(request.paramAsBoolean("local", getLicenseRequest.local()));
-        return channel -> client.es().admin().cluster().execute(GetLicenseAction.INSTANCE, getLicenseRequest,
-                new RestBuilderListener<GetLicenseResponse>(channel) {
+        return channel -> client.admin().cluster().execute(GetLicenseAction.INSTANCE, getLicenseRequest,
+                new RestBuilderListener<>(channel) {
                     @Override
                     public RestResponse buildResponse(GetLicenseResponse response, XContentBuilder builder) throws Exception {
                         // Default to pretty printing, but allow ?pretty=false to disable
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetTrialStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetTrialStatus.java
index 20366328e5031..2ee79da977357 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetTrialStatus.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetTrialStatus.java
@@ -7,17 +7,17 @@
 package org.elasticsearch.license;
 
 import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xpack.core.XPackClient;
-import org.elasticsearch.xpack.core.rest.XPackRestHandler;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
-public class RestGetTrialStatus extends XPackRestHandler {
+public class RestGetTrialStatus extends BaseRestHandler {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetTrialStatus.class));
 
@@ -26,12 +26,12 @@ public class RestGetTrialStatus extends XPackRestHandler {
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
                 GET, "/_license/trial_status", this,
-                GET, URI_BASE + "/license/trial_status", deprecationLogger);
+                GET, "/_xpack/license/trial_status", deprecationLogger);
     }
 
     @Override
-    protected RestChannelConsumer doPrepareRequest(RestRequest request, XPackClient client) {
-        return channel -> client.licensing().prepareGetStartTrial().execute(new RestToXContentListener<>(channel));
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
+        return channel -> new GetTrialStatusRequestBuilder(client).execute(new RestToXContentListener<>(channel));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartBasicLicense.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartBasicLicense.java
index 79e8849669c8f..77f09383b407d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartBasicLicense.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartBasicLicense.java
@@ -7,19 +7,19 @@
 package org.elasticsearch.license;
 
 import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
-import org.elasticsearch.xpack.core.XPackClient;
-import org.elasticsearch.xpack.core.rest.XPackRestHandler;
 
 import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
-public class RestPostStartBasicLicense extends XPackRestHandler {
+public class RestPostStartBasicLicense extends BaseRestHandler {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestPostStartBasicLicense.class));
 
@@ -28,16 +28,16 @@ public class RestPostStartBasicLicense extends XPackRestHandler {
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
                 POST, "/_license/start_basic", this,
-                POST, URI_BASE + "/license/start_basic", deprecationLogger);
+                POST, "/_xpack/license/start_basic", deprecationLogger);
     }
 
     @Override
-    protected RestChannelConsumer doPrepareRequest(RestRequest request, XPackClient client) throws IOException {
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         PostStartBasicRequest startBasicRequest = new PostStartBasicRequest();
         startBasicRequest.acknowledge(request.paramAsBoolean("acknowledge", false));
         startBasicRequest.timeout(request.paramAsTime("timeout", startBasicRequest.timeout()));
         startBasicRequest.masterNodeTimeout(request.paramAsTime("master_timeout", startBasicRequest.masterNodeTimeout()));
-        return channel -> client.licensing().postStartBasic(startBasicRequest, new RestStatusToXContentListener<>(channel));
+        return channel -> client.execute(PostStartBasicAction.INSTANCE, startBasicRequest, new RestStatusToXContentListener<>(channel));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartTrialLicense.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartTrialLicense.java
index a263d0d82c26a..d2ce0776305da 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartTrialLicense.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPostStartTrialLicense.java
@@ -7,23 +7,23 @@
 package org.elasticsearch.license;
 
 import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
-import org.elasticsearch.xpack.core.XPackClient;
-import org.elasticsearch.xpack.core.rest.XPackRestHandler;
 
 import java.io.IOException;
 import java.util.Map;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
-public class RestPostStartTrialLicense extends XPackRestHandler {
+public class RestPostStartTrialLicense extends BaseRestHandler {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestPostStartTrialLicense.class));
 
@@ -32,16 +32,16 @@ public class RestPostStartTrialLicense extends XPackRestHandler {
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
                 POST, "/_license/start_trial", this,
-                POST, URI_BASE + "/license/start_trial", deprecationLogger);
+                POST, "/_xpack/license/start_trial", deprecationLogger);
     }
 
     @Override
-    protected RestChannelConsumer doPrepareRequest(RestRequest request, XPackClient client) throws IOException {
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         PostStartTrialRequest startTrialRequest = new PostStartTrialRequest();
         startTrialRequest.setType(request.param("type", "trial"));
         startTrialRequest.acknowledge(request.paramAsBoolean("acknowledge", false));
-        return channel -> client.licensing().postStartTrial(startTrialRequest,
-                new RestBuilderListener<PostStartTrialResponse>(channel) {
+        return channel -> client.execute(PostStartTrialAction.INSTANCE, startTrialRequest,
+                new RestBuilderListener<>(channel) {
                     @Override
                     public RestResponse buildResponse(PostStartTrialResponse response, XContentBuilder builder) throws Exception {
                         PostStartTrialResponse.Status status = response.getStatus();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPutLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPutLicenseAction.java
index 986dacb687738..698ec440a6cbe 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPutLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestPutLicenseAction.java
@@ -7,20 +7,20 @@
 package org.elasticsearch.license;
 
 import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xpack.core.XPackClient;
-import org.elasticsearch.xpack.core.rest.XPackRestHandler;
 
 import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 
-public class RestPutLicenseAction extends XPackRestHandler {
+public class RestPutLicenseAction extends BaseRestHandler {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestPutLicenseAction.class));
 
@@ -30,11 +30,11 @@ public class RestPutLicenseAction extends XPackRestHandler {
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
                 POST, "/_license", this,
-                POST, URI_BASE + "/license", deprecationLogger);
+                POST, "/_xpack/license", deprecationLogger);
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
                 PUT, "/_license", this,
-                PUT, URI_BASE + "/license", deprecationLogger);
+                PUT,  "/_xpack/license", deprecationLogger);
     }
 
     @Override
@@ -43,7 +43,7 @@ public String getName() {
     }
 
     @Override
-    public RestChannelConsumer doPrepareRequest(final RestRequest request, final XPackClient client) throws IOException {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         if (request.hasContent() == false) {
             throw new IllegalArgumentException("The license must be provided in the request body");
         }
@@ -58,8 +58,7 @@ public RestChannelConsumer doPrepareRequest(final RestRequest request, final XPa
                     "/_license/start_basic API to install a basic license that does not expire.");
         }
 
-        return channel -> client.es().admin().cluster().execute(PutLicenseAction.INSTANCE, putLicenseRequest,
-                new RestToXContentListener<>(channel));
+        return channel -> client.execute(PutLicenseAction.INSTANCE, putLicenseRequest, new RestToXContentListener<>(channel));
     }
 
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java
index 6e138edb53f00..4209a99d3330c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportDeleteLicenseAction.java
@@ -17,10 +17,14 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 public class TransportDeleteLicenseAction extends TransportMasterNodeAction<DeleteLicenseRequest, AcknowledgedResponse> {
 
     private final LicenseService licenseService;
@@ -39,9 +43,14 @@ protected String executor() {
         return ThreadPool.Names.MANAGEMENT;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
@@ -50,8 +59,8 @@ protected ClusterBlockException checkBlock(DeleteLicenseRequest request, Cluster
     }
 
     @Override
-    protected void masterOperation(final DeleteLicenseRequest request, ClusterState state, final ActionListener<AcknowledgedResponse>
-            listener) throws ElasticsearchException {
+    protected void masterOperation(Task task, final DeleteLicenseRequest request, ClusterState state,
+                                   final ActionListener<AcknowledgedResponse> listener) throws ElasticsearchException {
         licenseService.removeLicense(request, new ActionListener<ClusterStateUpdateResponse>() {
             @Override
             public void onResponse(ClusterStateUpdateResponse clusterStateUpdateResponse) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetBasicStatusAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetBasicStatusAction.java
index 7377daae40643..a22041c4a4125 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetBasicStatusAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetBasicStatusAction.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -38,7 +39,7 @@ protected GetBasicStatusResponse newResponse() {
     }
 
     @Override
-    protected void masterOperation(GetBasicStatusRequest request, ClusterState state,
+    protected void masterOperation(Task task, GetBasicStatusRequest request, ClusterState state,
                                    ActionListener<GetBasicStatusResponse> listener) throws Exception {
         LicensesMetaData licensesMetaData = state.metaData().custom(LicensesMetaData.TYPE);
         if (licensesMetaData == null) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetLicenseAction.java
index 676ae6543e765..cda8a57fdba2d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetLicenseAction.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.protocol.xpack.license.GetLicenseRequest;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -48,7 +49,7 @@ protected ClusterBlockException checkBlock(GetLicenseRequest request, ClusterSta
     }
 
     @Override
-    protected void masterOperation(final GetLicenseRequest request, ClusterState state,
+    protected void masterOperation(Task task, final GetLicenseRequest request, ClusterState state,
                                    final ActionListener<GetLicenseResponse> listener) throws ElasticsearchException {
         listener.onResponse(new GetLicenseResponse(licenseService.getLicense()));
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetTrialStatusAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetTrialStatusAction.java
index 45e04548bcaa1..f31e6977663c4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetTrialStatusAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetTrialStatusAction.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -37,7 +38,7 @@ protected GetTrialStatusResponse newResponse() {
     }
 
     @Override
-    protected void masterOperation(GetTrialStatusRequest request, ClusterState state,
+    protected void masterOperation(Task task, GetTrialStatusRequest request, ClusterState state,
                                    ActionListener<GetTrialStatusResponse> listener) throws Exception {
         LicensesMetaData licensesMetaData = state.metaData().custom(LicensesMetaData.TYPE);
         listener.onResponse(new GetTrialStatusResponse(licensesMetaData == null || licensesMetaData.isEligibleForTrial()));
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartBasicAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartBasicAction.java
index e03e6201e917c..5368936d2e9dc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartBasicAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartBasicAction.java
@@ -14,9 +14,13 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 public class TransportPostStartBasicAction extends TransportMasterNodeAction<PostStartBasicRequest, PostStartBasicResponse> {
 
     private final LicenseService licenseService;
@@ -37,11 +41,16 @@ protected String executor() {
 
     @Override
     protected PostStartBasicResponse newResponse() {
-        return new PostStartBasicResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected PostStartBasicResponse read(StreamInput in) throws IOException {
+        return new PostStartBasicResponse(in);
     }
 
     @Override
-    protected void masterOperation(PostStartBasicRequest request, ClusterState state,
+    protected void masterOperation(Task task, PostStartBasicRequest request, ClusterState state,
                                    ActionListener<PostStartBasicResponse> listener) throws Exception {
         licenseService.startBasicLicense(request, listener);
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java
index 7e29f61ec5e2c..5a181eea4b620 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPostStartTrialAction.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -41,7 +42,7 @@ protected PostStartTrialResponse newResponse() {
     }
 
     @Override
-    protected void masterOperation(PostStartTrialRequest request, ClusterState state,
+    protected void masterOperation(Task task, PostStartTrialRequest request, ClusterState state,
                                    ActionListener<PostStartTrialResponse> listener) throws Exception {
         licenseService.startTrialLicense(request, listener);
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPutLicenseAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPutLicenseAction.java
index 85338fc6e0179..a85a13427da93 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPutLicenseAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportPutLicenseAction.java
@@ -15,10 +15,14 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.protocol.xpack.license.PutLicenseResponse;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 public class TransportPutLicenseAction extends TransportMasterNodeAction<PutLicenseRequest, PutLicenseResponse> {
 
     private final LicenseService licenseService;
@@ -39,7 +43,12 @@ protected String executor() {
 
     @Override
     protected PutLicenseResponse newResponse() {
-        return new PutLicenseResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected PutLicenseResponse read(StreamInput in) throws IOException {
+        return new PutLicenseResponse(in);
     }
 
     @Override
@@ -48,8 +57,8 @@ protected ClusterBlockException checkBlock(PutLicenseRequest request, ClusterSta
     }
 
     @Override
-    protected void masterOperation(final PutLicenseRequest request, ClusterState state, final ActionListener<PutLicenseResponse>
-            listener) throws ElasticsearchException {
+    protected void masterOperation(Task task, final PutLicenseRequest request, ClusterState state, final ActionListener<PutLicenseResponse>
+        listener) throws ElasticsearchException {
         licenseService.registerLicense(request, listener);
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
index e206ed3db5149..859fc074e1796 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
@@ -454,12 +454,12 @@ public synchronized boolean isTokenServiceAllowed() {
     }
 
     /**
-     * @return whether the Elasticsearch {@code ApiKeyService} is allowed based on the license {@link OperationMode}
+     * @return whether the Elasticsearch {@code ApiKeyService} is allowed based on the current node/cluster state
      */
     public synchronized boolean isApiKeyServiceAllowed() {
         final OperationMode mode = status.mode;
         final boolean isSecurityCurrentlyEnabled = isSecurityEnabled(mode, isSecurityExplicitlyEnabled, isSecurityEnabled);
-        return isSecurityCurrentlyEnabled && (mode == OperationMode.GOLD || mode == OperationMode.PLATINUM || mode == OperationMode.TRIAL);
+        return isSecurityCurrentlyEnabled;
     }
 
     /**
@@ -608,6 +608,15 @@ public synchronized boolean isRollupAllowed() {
         return status.active;
     }
 
+    /**
+     * Voting only node functionality is always available as long as there is a valid license
+     *
+     * @return true if the license is active
+     */
+    public synchronized boolean isVotingOnlyAllowed() {
+        return status.active;
+    }
+
     /**
      * Logstash is allowed as long as there is an active license of type TRIAL, STANDARD, GOLD or PLATINUM
      * @return {@code true} as long as there is a valid license
@@ -687,6 +696,24 @@ public synchronized boolean isJdbcAllowed() {
         return licensed && localStatus.active;
     }
 
+    /**
+     * Determine if support for flattened object fields should be enabled.
+     * <p>
+     * Flattened fields are available for all license types except {@link OperationMode#MISSING}.
+     */
+    public synchronized boolean isFlattenedAllowed() {
+        return status.active;
+    }
+
+    /**
+     * Determine if Vectors support should be enabled.
+     * <p>
+     *  Vectors is available for all license types except {@link OperationMode#MISSING}
+     */
+    public synchronized boolean isVectorsAllowed() {
+        return status.active;
+    }
+
     /**
      * Determine if ODBC support should be enabled.
      * <p>
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java
index a9fb0edee3389..8ee6695010e94 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.protocol.xpack;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
@@ -331,42 +332,48 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
         public static class FeatureSet implements ToXContentObject, Writeable {
             private final String name;
-            @Nullable private final String description;
             private final boolean available;
             private final boolean enabled;
-            @Nullable private final Map<String, Object> nativeCodeInfo;
 
-            public FeatureSet(String name, @Nullable String description, boolean available, boolean enabled,
-                              @Nullable Map<String, Object> nativeCodeInfo) {
+            public FeatureSet(String name, boolean available, boolean enabled) {
                 this.name = name;
-                this.description = description;
                 this.available = available;
                 this.enabled = enabled;
-                this.nativeCodeInfo = nativeCodeInfo;
             }
 
             public FeatureSet(StreamInput in) throws IOException {
-                this(in.readString(), in.readOptionalString(), in.readBoolean(), in.readBoolean(), in.readMap());
+                this(in.readString(), readAvailable(in), in.readBoolean());
+                if (in.getVersion().before(Version.V_8_0_0)) {
+                    in.readMap(); // backcompat reading native code info, but no longer used here
+                }
+            }
+
+            // this is separated out so that the removed description can be read from the stream on construction
+            // TODO: remove this for 8.0
+            private static boolean readAvailable(StreamInput in) throws IOException {
+                if (in.getVersion().before(Version.V_7_3_0)) {
+                    in.readOptionalString();
+                }
+                return in.readBoolean();
             }
 
             @Override
             public void writeTo(StreamOutput out) throws IOException {
                 out.writeString(name);
-                out.writeOptionalString(description);
+                if (out.getVersion().before(Version.V_7_3_0)) {
+                    out.writeOptionalString(null);
+                }
                 out.writeBoolean(available);
                 out.writeBoolean(enabled);
-                out.writeMap(nativeCodeInfo);
+                if (out.getVersion().before(Version.V_8_0_0)) {
+                    out.writeMap(Collections.emptyMap());
+                }
             }
 
             public String name() {
                 return name;
             }
 
-            @Nullable
-            public String description() {
-                return description;
-            }
-
             public boolean available() {
                 return available;
             }
@@ -375,39 +382,26 @@ public boolean enabled() {
                 return enabled;
             }
 
-            @Nullable
-            public Map<String, Object> nativeCodeInfo() {
-                return nativeCodeInfo;
-            }
-
             @Override
             public boolean equals(Object other) {
                 if (other == null || other.getClass() != getClass()) return false;
                 if (this == other) return true;
                 FeatureSet rhs = (FeatureSet) other;
                 return Objects.equals(name, rhs.name)
-                        && Objects.equals(description, rhs.description)
                         && available == rhs.available
-                        && enabled == rhs.enabled
-                        && Objects.equals(nativeCodeInfo, rhs.nativeCodeInfo);
+                        && enabled == rhs.enabled;
             }
 
             @Override
             public int hashCode() {
-                return Objects.hash(name, description, available, enabled, nativeCodeInfo);
+                return Objects.hash(name, available, enabled);
             }
 
             @Override
             public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
                 builder.startObject();
-                if (description != null) {
-                    builder.field("description", description);
-                }
                 builder.field("available", available);
                 builder.field("enabled", enabled);
-                if (nativeCodeInfo != null) {
-                    builder.field("native_code_info", nativeCodeInfo);
-                }
                 return builder.endObject();
             }
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/license/PutLicenseResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/license/PutLicenseResponse.java
index 1bf4a78cf46ad..eb9eca4f4ddf3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/license/PutLicenseResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/license/PutLicenseResponse.java
@@ -20,11 +20,26 @@
 
 public class PutLicenseResponse extends AcknowledgedResponse {
 
-    private LicensesStatus status;
-    private Map<String, String[]> acknowledgeMessages;
-    private String acknowledgeHeader;
+    private final LicensesStatus status;
+    private final Map<String, String[]> acknowledgeMessages;
+    private final String acknowledgeHeader;
 
-    public PutLicenseResponse() {
+    public PutLicenseResponse(StreamInput in) throws IOException {
+        super(in);
+        status = LicensesStatus.fromId(in.readVInt());
+        acknowledgeHeader = in.readOptionalString();
+        int size = in.readVInt();
+        Map<String, String[]> acknowledgeMessages = new HashMap<>(size);
+        for (int i = 0; i < size; i++) {
+            String feature = in.readString();
+            int nMessages = in.readVInt();
+            String[] messages = new String[nMessages];
+            for (int j = 0; j < nMessages; j++) {
+                messages[j] = in.readString();
+            }
+            acknowledgeMessages.put(feature, messages);
+        }
+        this.acknowledgeMessages = acknowledgeMessages;
     }
 
     public PutLicenseResponse(boolean acknowledged, LicensesStatus status) {
@@ -51,25 +66,6 @@ public String acknowledgeHeader() {
         return acknowledgeHeader;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        status = LicensesStatus.fromId(in.readVInt());
-        acknowledgeHeader = in.readOptionalString();
-        int size = in.readVInt();
-        Map<String, String[]> acknowledgeMessages = new HashMap<>(size);
-        for (int i = 0; i < size; i++) {
-            String feature = in.readString();
-            int nMessages = in.readVInt();
-            String[] messages = new String[nMessages];
-            for (int j = 0; j < nMessages; j++) {
-                messages[j] = in.readString();
-            }
-            acknowledgeMessages.put(feature, messages);
-        }
-        this.acknowledgeMessages = acknowledgeMessages;
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java
index 102bcde9dc77b..c0cae8c5426e3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshot.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.snapshots;
 
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.blocktree.BlockTreeTermsReader;
 import org.apache.lucene.index.CheckIndex;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocValuesType;
@@ -80,7 +81,8 @@ public synchronized List<String> syncSnapshot(IndexCommit commit) throws IOExcep
         List<String> createdFiles = new ArrayList<>();
         String segmentFileName;
         try (Lock writeLock = targetDirectory.obtainLock(IndexWriter.WRITE_LOCK_NAME);
-             StandardDirectoryReader reader = (StandardDirectoryReader) DirectoryReader.open(commit)) {
+             StandardDirectoryReader reader = (StandardDirectoryReader) DirectoryReader.open(commit,
+                 Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name()))) {
             SegmentInfos segmentInfos = reader.getSegmentInfos().clone();
             DirectoryReader wrappedReader = wrapReader(reader);
             List<SegmentCommitInfo> newInfos = new ArrayList<>();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java
index bb5819e1bda43..f6c3124c9be9f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.snapshots;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.apache.lucene.codecs.blocktree.BlockTreeTermsReader;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.SegmentInfos;
@@ -37,6 +38,7 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.file.Path;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.function.Function;
@@ -135,7 +137,8 @@ protected void closeInternal() {
             final long maxDoc = segmentInfos.totalMaxDoc();
             tempStore.bootstrapNewHistory(maxDoc, maxDoc);
             store.incRef();
-            try (DirectoryReader reader = DirectoryReader.open(tempStore.directory())) {
+            try (DirectoryReader reader = DirectoryReader.open(tempStore.directory(),
+                Collections.singletonMap(BlockTreeTermsReader.FST_MODE_KEY, BlockTreeTermsReader.FSTLoadMode.OFF_HEAP.name()))) {
                 IndexCommit indexCommit = reader.getIndexCommit();
                 super.snapshotShard(tempStore, mapperService, snapshotId, indexId, indexCommit, snapshotStatus);
             } finally {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CCRFeatureSet.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/ccr/CCRInfoTransportAction.java
similarity index 65%
rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CCRFeatureSet.java
rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/ccr/CCRInfoTransportAction.java
index f6e9e76d448f7..9f6c3b1467e8d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CCRFeatureSet.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/ccr/CCRInfoTransportAction.java
@@ -3,39 +3,36 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.core.ccr;
+package org.elasticsearch.xpack.ccr;
 
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Nullable;
+import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
 
 import java.io.IOException;
-import java.time.Instant;
-import java.util.Map;
 import java.util.Objects;
 
-public class CCRFeatureSet implements XPackFeatureSet {
+public class CCRInfoTransportAction extends XPackInfoFeatureTransportAction {
 
     private final boolean enabled;
     private final XPackLicenseState licenseState;
-    private final ClusterService clusterService;
 
     @Inject
-    public CCRFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState, ClusterService clusterService) {
+    public CCRInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                  Settings settings, XPackLicenseState licenseState) {
+        super(XPackInfoFeatureAction.CCR.name(), transportService, actionFilters);
         this.enabled = XPackSettings.CCR_ENABLED_SETTING.get(settings);
         this.licenseState = licenseState;
-        this.clusterService = clusterService;
     }
 
     @Override
@@ -43,14 +40,9 @@ public String name() {
         return XPackField.CCR;
     }
 
-    @Override
-    public String description() {
-        return "Cross Cluster Replication";
-    }
-
     @Override
     public boolean available() {
-        return licenseState != null && licenseState.isCcrAllowed();
+        return licenseState.isCcrAllowed();
     }
 
     @Override
@@ -58,41 +50,6 @@ public boolean enabled() {
         return enabled;
     }
 
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return null;
-    }
-
-    @Override
-    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
-        MetaData metaData = clusterService.state().metaData();
-
-        int numberOfFollowerIndices = 0;
-        long lastFollowerIndexCreationDate = 0L;
-        for (IndexMetaData imd : metaData) {
-            if (imd.getCustomData("ccr") != null) {
-                numberOfFollowerIndices++;
-                if (lastFollowerIndexCreationDate < imd.getCreationDate()) {
-                    lastFollowerIndexCreationDate = imd.getCreationDate();
-                }
-            }
-        }
-        AutoFollowMetadata autoFollowMetadata = metaData.custom(AutoFollowMetadata.TYPE);
-        int numberOfAutoFollowPatterns = autoFollowMetadata != null ? autoFollowMetadata.getPatterns().size() : 0;
-
-        Long lastFollowTimeInMillis;
-        if (numberOfFollowerIndices == 0) {
-            // Otherwise we would return a value that makes no sense.
-            lastFollowTimeInMillis = null;
-        } else {
-            lastFollowTimeInMillis = Math.max(0, Instant.now().toEpochMilli() - lastFollowerIndexCreationDate);
-        }
-
-        Usage usage =
-            new Usage(available(), enabled(), numberOfFollowerIndices, numberOfAutoFollowPatterns, lastFollowTimeInMillis);
-        listener.onResponse(usage);
-    }
-
     public static class Usage extends XPackFeatureSet.Usage {
 
         private final int numberOfFollowerIndices;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/ccr/CCRUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/ccr/CCRUsageTransportAction.java
new file mode 100644
index 0000000000000..e8adbaec187ef
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/ccr/CCRUsageTransportAction.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ccr;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
+
+import java.time.Instant;
+
+public class CCRUsageTransportAction extends XPackUsageFeatureTransportAction {
+
+    private final Settings settings;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public CCRUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                   ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                   Settings settings, XPackLicenseState licenseState) {
+        super(XPackUsageFeatureAction.CCR.name(), transportService, clusterService,
+            threadPool, actionFilters, indexNameExpressionResolver);
+        this.settings = settings;
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
+        MetaData metaData = state.metaData();
+
+        int numberOfFollowerIndices = 0;
+        long lastFollowerIndexCreationDate = 0L;
+        for (IndexMetaData imd : metaData) {
+            if (imd.getCustomData("ccr") != null) {
+                numberOfFollowerIndices++;
+                if (lastFollowerIndexCreationDate < imd.getCreationDate()) {
+                    lastFollowerIndexCreationDate = imd.getCreationDate();
+                }
+            }
+        }
+        AutoFollowMetadata autoFollowMetadata = metaData.custom(AutoFollowMetadata.TYPE);
+        int numberOfAutoFollowPatterns = autoFollowMetadata != null ? autoFollowMetadata.getPatterns().size() : 0;
+
+        Long lastFollowTimeInMillis;
+        if (numberOfFollowerIndices == 0) {
+            // Otherwise we would return a value that makes no sense.
+            lastFollowTimeInMillis = null;
+        } else {
+            lastFollowTimeInMillis = Math.max(0, Instant.now().toEpochMilli() - lastFollowerIndexCreationDate);
+        }
+
+        CCRInfoTransportAction.Usage usage = new CCRInfoTransportAction.Usage(licenseState.isCcrAllowed(),
+            XPackSettings.CCR_ENABLED_SETTING.get(settings), numberOfFollowerIndices, numberOfAutoFollowPatterns, lastFollowTimeInMillis);
+        listener.onResponse(new XPackUsageFeatureResponse(usage));
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
index a0074f4231876..ec4fc7a87adcf 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
@@ -81,7 +81,7 @@ public static <Request extends ActionRequest, Response extends ActionResponse> v
      */
     public static <Request extends ActionRequest, Response extends ActionResponse,
             RequestBuilder extends ActionRequestBuilder<Request, Response>> void executeAsyncWithOrigin(
-        Client client, String origin, Action<Response> action, Request request,
+        Client client, String origin, ActionType<Response> action, Request request,
         ActionListener<Response> listener) {
         final ThreadContext threadContext = client.threadPool().getThreadContext();
         final Supplier<ThreadContext.StoredContext> supplier = threadContext.newRestorableContext(false);
@@ -139,7 +139,7 @@ public static <T extends ActionResponse> T executeWithHeaders(Map<String, String
      *            The listener to call when the action is complete
      */
     public static <Request extends ActionRequest, Response extends ActionResponse>
-    void executeWithHeadersAsync(Map<String, String> headers, String origin, Client client, Action<Response> action, Request request,
+    void executeWithHeadersAsync(Map<String, String> headers, String origin, Client client, ActionType<Response> action, Request request,
                                  ActionListener<Response> listener) {
 
         Map<String, String> filteredHeaders = headers.entrySet().stream().filter(e -> SECURITY_HEADER_FILTERS.contains(e.getKey()))
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/EmptyXPackFeatureSet.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/EmptyXPackFeatureSet.java
deleted file mode 100644
index 3a6cc899d6274..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/EmptyXPackFeatureSet.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core;
-
-import org.elasticsearch.action.ActionListener;
-
-import java.util.Collections;
-import java.util.Map;
-
-public class EmptyXPackFeatureSet implements XPackFeatureSet {
-    @Override
-    public String name() {
-        return "Empty XPackFeatureSet";
-    }
-
-    @Override
-    public String description() {
-        return "Core will not function without this empty featureset compliments of the way the TransportXPackInfoAction Guice works";
-    }
-
-    @Override
-    public boolean available() {
-        return false;
-    }
-
-    @Override
-    public boolean enabled() {
-        return false;
-    }
-
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return Collections.emptyMap();
-    }
-
-    @Override
-    public void usage(ActionListener<Usage> listener) {
-
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClient.java
deleted file mode 100644
index e6cd2ed176c9c..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClient.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.license.LicensingClient;
-import org.elasticsearch.protocol.xpack.XPackInfoRequest;
-import org.elasticsearch.protocol.xpack.XPackInfoResponse;
-import org.elasticsearch.xpack.core.action.TransportFreezeIndexAction.FreezeIndexAction;
-import org.elasticsearch.xpack.core.action.TransportFreezeIndexAction.FreezeRequest;
-import org.elasticsearch.xpack.core.action.TransportFreezeIndexAction.FreezeResponse;
-import org.elasticsearch.xpack.core.action.XPackInfoAction;
-import org.elasticsearch.xpack.core.action.XPackInfoRequestBuilder;
-import org.elasticsearch.xpack.core.ccr.client.CcrClient;
-import org.elasticsearch.xpack.core.indexlifecycle.client.ILMClient;
-import org.elasticsearch.xpack.core.ml.client.MachineLearningClient;
-import org.elasticsearch.xpack.core.monitoring.client.MonitoringClient;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
-
-import java.util.Collections;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.ExecutionException;
-
-import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER;
-import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
-
-public class XPackClient {
-
-    private final Client client;
-
-    private final CcrClient ccrClient;
-    private final LicensingClient licensingClient;
-    private final MonitoringClient monitoringClient;
-    private final SecurityClient securityClient;
-    private final WatcherClient watcherClient;
-    private final MachineLearningClient machineLearning;
-    private final ILMClient ilmClient;
-
-    public XPackClient(Client client) {
-        this.client = Objects.requireNonNull(client, "client");
-        this.ccrClient = new CcrClient(client);
-        this.licensingClient = new LicensingClient(client);
-        this.monitoringClient = new MonitoringClient(client);
-        this.securityClient = new SecurityClient(client);
-        this.watcherClient = new WatcherClient(client);
-        this.machineLearning = new MachineLearningClient(client);
-        this.ilmClient = new ILMClient(client);
-    }
-
-    public Client es() {
-        return client;
-    }
-
-    public CcrClient ccr() {
-        return ccrClient;
-    }
-
-    public LicensingClient licensing() {
-        return licensingClient;
-    }
-
-    public MonitoringClient monitoring() {
-        return monitoringClient;
-    }
-
-    public SecurityClient security() {
-        return securityClient;
-    }
-
-    public WatcherClient watcher() {
-        return watcherClient;
-    }
-
-    public MachineLearningClient machineLearning() {
-        return machineLearning;
-    }
-
-    public ILMClient ilmClient() {
-        return ilmClient;
-    }
-
-    public XPackClient withHeaders(Map<String, String> headers) {
-        return new XPackClient(client.filterWithHeader(headers));
-    }
-
-    /**
-     * Returns a client that will call xpack APIs on behalf of the given user.
-     *
-     * @param username The username of the user
-     * @param passwd    The password of the user. This char array can be cleared after calling this method.
-     */
-    public XPackClient withAuth(String username, char[] passwd) {
-        return withHeaders(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue(username, new SecureString(passwd))));
-    }
-
-    public XPackInfoRequestBuilder prepareInfo() {
-        return new XPackInfoRequestBuilder(client);
-    }
-
-    public void info(XPackInfoRequest request, ActionListener<XPackInfoResponse> listener) {
-        client.execute(XPackInfoAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Freezes or unfreeze one or more indices
-     */
-    public void freeze(FreezeRequest request, ActionListener<FreezeResponse> listener) {
-        client.execute(FreezeIndexAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Freeze or unfreeze one or more indices
-     */
-    public FreezeResponse freeze(FreezeRequest request)
-        throws ExecutionException, InterruptedException {
-        PlainActionFuture<FreezeResponse> future = new PlainActionFuture<>();
-        freeze(request, future);
-        return future.get();
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
index 6b457ae2fda9e..fec75af8f0f67 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.NamedDiff;
@@ -35,7 +35,7 @@
 import org.elasticsearch.xpack.core.action.XPackUsageAction;
 import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
-import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
+import org.elasticsearch.xpack.ccr.CCRInfoTransportAction;
 import org.elasticsearch.xpack.core.dataframe.DataFrameFeatureSetUsage;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.action.DeleteDataFrameTransformAction;
@@ -48,7 +48,10 @@
 import org.elasticsearch.xpack.core.dataframe.action.StopDataFrameTransformAction;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransform;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformState;
+import org.elasticsearch.xpack.core.dataframe.transforms.SyncConfig;
+import org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig;
 import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
+import org.elasticsearch.xpack.core.flattened.FlattenedFeatureSetUsage;
 import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage;
 import org.elasticsearch.xpack.core.graph.action.GraphExploreAction;
 import org.elasticsearch.xpack.core.indexlifecycle.AllocateAction;
@@ -79,12 +82,14 @@
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteCalendarAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteCalendarEventAction;
+import org.elasticsearch.xpack.core.ml.action.DeleteDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteExpiredDataAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteFilterAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteForecastAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteJobAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteModelSnapshotAction;
+import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
 import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
 import org.elasticsearch.xpack.core.ml.action.FindFileStructureAction;
 import org.elasticsearch.xpack.core.ml.action.FlushJobAction;
@@ -93,6 +98,8 @@
 import org.elasticsearch.xpack.core.ml.action.GetCalendarEventsAction;
 import org.elasticsearch.xpack.core.ml.action.GetCalendarsAction;
 import org.elasticsearch.xpack.core.ml.action.GetCategoriesAction;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction;
 import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.GetFiltersAction;
@@ -111,11 +118,13 @@
 import org.elasticsearch.xpack.core.ml.action.PostDataAction;
 import org.elasticsearch.xpack.core.ml.action.PreviewDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.PutCalendarAction;
+import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.PutFilterAction;
 import org.elasticsearch.xpack.core.ml.action.PutJobAction;
 import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction;
 import org.elasticsearch.xpack.core.ml.action.SetUpgradeModeAction;
+import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.UpdateCalendarJobAction;
@@ -127,6 +136,18 @@
 import org.elasticsearch.xpack.core.ml.action.ValidateDetectorAction;
 import org.elasticsearch.xpack.core.ml.action.ValidateJobConfigAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.DataFrameAnalysis;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetection;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.Evaluation;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.AucRoc;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.BinarySoftClassification;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.ConfusionMatrix;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.Precision;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.Recall;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.ScoreByThresholdResult;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.SoftClassificationMetric;
 import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
 import org.elasticsearch.xpack.core.monitoring.MonitoringFeatureSetUsage;
 import org.elasticsearch.xpack.core.rollup.RollupFeatureSetUsage;
@@ -174,6 +195,8 @@
 import org.elasticsearch.xpack.core.ssl.action.GetCertificateInfoAction;
 import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeAction;
 import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction;
+import org.elasticsearch.xpack.core.vectors.VectorsFeatureSetUsage;
+import org.elasticsearch.xpack.core.votingonly.VotingOnlyNodeFeatureSetUsage;
 import org.elasticsearch.xpack.core.watcher.WatcherFeatureSetUsage;
 import org.elasticsearch.xpack.core.watcher.WatcherMetaData;
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchAction;
@@ -224,7 +247,7 @@ public List<Setting<?>> getSettings() {
     }
 
     @Override
-    public List<Action<? extends ActionResponse>> getClientActions() {
+    public List<ActionType<? extends ActionResponse>> getClientActions() {
         return Arrays.asList(
                 // deprecation
                 DeprecationInfoAction.INSTANCE,
@@ -281,6 +304,12 @@ public List<Action<? extends ActionResponse>> getClientActions() {
                 PersistJobAction.INSTANCE,
                 FindFileStructureAction.INSTANCE,
                 SetUpgradeModeAction.INSTANCE,
+                PutDataFrameAnalyticsAction.INSTANCE,
+                GetDataFrameAnalyticsAction.INSTANCE,
+                GetDataFrameAnalyticsStatsAction.INSTANCE,
+                DeleteDataFrameAnalyticsAction.INSTANCE,
+                StartDataFrameAnalyticsAction.INSTANCE,
+                EvaluateDataFrameAction.INSTANCE,
                 // security
                 ClearRealmCacheAction.INSTANCE,
                 ClearRolesCacheAction.INSTANCE,
@@ -373,11 +402,30 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                         StartDatafeedAction.DatafeedParams::new),
                 new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.JOB_TASK_NAME,
                         OpenJobAction.JobParams::new),
+                new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME,
+                    StartDataFrameAnalyticsAction.TaskParams::new),
                 // ML - Task states
                 new NamedWriteableRegistry.Entry(PersistentTaskState.class, JobTaskState.NAME, JobTaskState::new),
                 new NamedWriteableRegistry.Entry(PersistentTaskState.class, DatafeedState.NAME, DatafeedState::fromStream),
+                new NamedWriteableRegistry.Entry(PersistentTaskState.class, DataFrameAnalyticsTaskState.NAME,
+                    DataFrameAnalyticsTaskState::new),
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.MACHINE_LEARNING,
                         MachineLearningFeatureSetUsage::new),
+                // ML - Data frame analytics
+                new NamedWriteableRegistry.Entry(DataFrameAnalysis.class, OutlierDetection.NAME.getPreferredName(), OutlierDetection::new),
+                // ML - Data frame evaluation
+                new NamedWriteableRegistry.Entry(Evaluation.class, BinarySoftClassification.NAME.getPreferredName(),
+                        BinarySoftClassification::new),
+                new NamedWriteableRegistry.Entry(SoftClassificationMetric.class, AucRoc.NAME.getPreferredName(), AucRoc::new),
+                new NamedWriteableRegistry.Entry(SoftClassificationMetric.class, Precision.NAME.getPreferredName(), Precision::new),
+                new NamedWriteableRegistry.Entry(SoftClassificationMetric.class, Recall.NAME.getPreferredName(), Recall::new),
+                new NamedWriteableRegistry.Entry(SoftClassificationMetric.class, ConfusionMatrix.NAME.getPreferredName(),
+                        ConfusionMatrix::new),
+                new NamedWriteableRegistry.Entry(EvaluationMetricResult.class, AucRoc.NAME.getPreferredName(), AucRoc.Result::new),
+                new NamedWriteableRegistry.Entry(EvaluationMetricResult.class, ScoreByThresholdResult.NAME, ScoreByThresholdResult::new),
+                new NamedWriteableRegistry.Entry(EvaluationMetricResult.class, ConfusionMatrix.NAME.getPreferredName(),
+                        ConfusionMatrix.Result::new),
+
                 // monitoring
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.MONITORING, MonitoringFeatureSetUsage::new),
                 // security
@@ -412,7 +460,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 new NamedWriteableRegistry.Entry(MetaData.Custom.class, AutoFollowMetadata.TYPE, AutoFollowMetadata::new),
                 new NamedWriteableRegistry.Entry(NamedDiff.class, AutoFollowMetadata.TYPE,
                     in -> AutoFollowMetadata.readDiffFrom(MetaData.Custom.class, AutoFollowMetadata.TYPE, in)),
-                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.CCR, CCRFeatureSet.Usage::new),
+                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.CCR, CCRInfoTransportAction.Usage::new),
                 // ILM
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX_LIFECYCLE,
                     IndexLifecycleFeatureSetUsage::new),
@@ -437,7 +485,13 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.DATA_FRAME, DataFrameFeatureSetUsage::new),
                 new NamedWriteableRegistry.Entry(PersistentTaskParams.class, DataFrameField.TASK_NAME, DataFrameTransform::new),
                 new NamedWriteableRegistry.Entry(Task.Status.class, DataFrameField.TASK_NAME, DataFrameTransformState::new),
-                new NamedWriteableRegistry.Entry(PersistentTaskState.class, DataFrameField.TASK_NAME, DataFrameTransformState::new)
+                new NamedWriteableRegistry.Entry(PersistentTaskState.class, DataFrameField.TASK_NAME, DataFrameTransformState::new),
+                new NamedWriteableRegistry.Entry(SyncConfig.class, DataFrameField.TIME_BASED_SYNC.getPreferredName(), TimeSyncConfig::new),
+                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.FLATTENED, FlattenedFeatureSetUsage::new),
+                // Vectors
+                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.VECTORS, VectorsFeatureSetUsage::new),
+                // Voting Only Node
+                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.VOTING_ONLY, VotingOnlyNodeFeatureSetUsage::new)
         );
     }
 
@@ -452,9 +506,13 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
                         StartDatafeedAction.DatafeedParams::fromXContent),
                 new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(MlTasks.JOB_TASK_NAME),
                         OpenJobAction.JobParams::fromXContent),
+                new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME),
+                        StartDataFrameAnalyticsAction.TaskParams::fromXContent),
                 // ML - Task states
                 new NamedXContentRegistry.Entry(PersistentTaskState.class, new ParseField(DatafeedState.NAME), DatafeedState::fromXContent),
                 new NamedXContentRegistry.Entry(PersistentTaskState.class, new ParseField(JobTaskState.NAME), JobTaskState::fromXContent),
+                new NamedXContentRegistry.Entry(PersistentTaskState.class, new ParseField(DataFrameAnalyticsTaskState.NAME),
+                    DataFrameAnalyticsTaskState::fromXContent),
                 // watcher
                 new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(WatcherMetaData.TYPE),
                         WatcherMetaData::fromXContent),
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackFeatureSet.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackFeatureSet.java
index 075625704d14c..1080f72285e89 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackFeatureSet.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackFeatureSet.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.core;
 
-import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -13,22 +12,15 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
-import java.util.Map;
 
 public interface XPackFeatureSet {
 
     String name();
 
-    String description();
-
     boolean available();
 
     boolean enabled();
 
-    Map<String, Object> nativeCodeInfo();
-
-    void usage(ActionListener<Usage> listener);
-
     abstract class Usage implements ToXContentObject, NamedWriteable {
 
         private static final String AVAILABLE_XFIELD = "available";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java
index a5baf4d4f9382..351606e321873 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java
@@ -37,6 +37,12 @@ public final class XPackField {
     public static final String CCR = "ccr";
     /** Name constant for the data frame feature. */
     public static final String DATA_FRAME = "data_frame";
+    /** Name constant for flattened fields. */
+    public static final String FLATTENED = "flattened";
+    /** Name constant for the vectors feature. */
+    public static final String VECTORS = "vectors";
+    /** Name constant for the voting-only-node feature. */
+    public static final String VOTING_ONLY = "voting_only";
 
     private XPackField() {}
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
index 696e3a2871fb3..a9d4e9f8b9bec 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
@@ -9,12 +9,12 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.SpecialPermission;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.ActionFilter;
+import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -23,7 +23,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.inject.Binder;
-import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.DeprecationLogger;
@@ -47,6 +46,9 @@
 import org.elasticsearch.plugins.EnginePlugin;
 import org.elasticsearch.plugins.ExtensiblePlugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
+import org.elasticsearch.protocol.xpack.XPackInfoRequest;
+import org.elasticsearch.protocol.xpack.XPackInfoResponse;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
@@ -54,13 +56,17 @@
 import org.elasticsearch.snapshots.SourceOnlySnapshotRepository;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.core.action.ReloadAnalyzerAction;
 import org.elasticsearch.xpack.core.action.TransportFreezeIndexAction;
+import org.elasticsearch.xpack.core.action.TransportReloadAnalyzersAction;
 import org.elasticsearch.xpack.core.action.TransportXPackInfoAction;
 import org.elasticsearch.xpack.core.action.TransportXPackUsageAction;
 import org.elasticsearch.xpack.core.action.XPackInfoAction;
 import org.elasticsearch.xpack.core.action.XPackUsageAction;
+import org.elasticsearch.xpack.core.action.XPackUsageResponse;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.rest.action.RestFreezeIndexAction;
+import org.elasticsearch.xpack.core.rest.action.RestReloadAnalyzersAction;
 import org.elasticsearch.xpack.core.rest.action.RestXPackInfoAction;
 import org.elasticsearch.xpack.core.rest.action.RestXPackUsageAction;
 import org.elasticsearch.xpack.core.security.authc.TokenMetaData;
@@ -217,15 +223,6 @@ public Settings additionalSettings() {
         return Settings.builder().put(super.additionalSettings()).put(xpackInstalledNodeAttrSetting, "true").build();
     }
 
-    @Override
-    public Collection<Module> createGuiceModules() {
-        ArrayList<Module> modules = new ArrayList<>();
-        //modules.add(b -> b.bind(Clock.class).toInstance(getClock()));
-        // used to get core up and running, we do not bind the actual feature set here
-        modules.add(b -> XPackPlugin.createFeatureSetMultiBinder(b, EmptyXPackFeatureSet.class));
-        return modules;
-    }
-
     @Override
     public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
                                                ResourceWatcherService resourceWatcherService, ScriptService scriptService,
@@ -250,17 +247,28 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> actions = new ArrayList<>();
-        actions.add(new ActionHandler<>(XPackInfoAction.INSTANCE, TransportXPackInfoAction.class));
-        actions.add(new ActionHandler<>(XPackUsageAction.INSTANCE, TransportXPackUsageAction.class));
+        actions.add(new ActionHandler<>(XPackInfoAction.INSTANCE, getInfoAction()));
+        actions.add(new ActionHandler<>(XPackUsageAction.INSTANCE, getUsageAction()));
         actions.add(new ActionHandler<>(TransportFreezeIndexAction.FreezeIndexAction.INSTANCE,
             TransportFreezeIndexAction.class));
         actions.addAll(licensing.getActions());
+        actions.add(new ActionHandler<>(ReloadAnalyzerAction.INSTANCE, TransportReloadAnalyzersAction.class));
         return actions;
     }
 
+    // overridable for tests
+    protected Class<? extends TransportAction<XPackUsageRequest, XPackUsageResponse>> getUsageAction() {
+        return TransportXPackUsageAction.class;
+    }
+
+    // overridable for tests
+    protected Class<? extends TransportAction<XPackInfoRequest, XPackInfoResponse>> getInfoAction() {
+        return TransportXPackInfoAction.class;
+    }
+
     @Override
-    public List<Action<? extends ActionResponse>> getClientActions() {
-        List<Action<? extends ActionResponse>> actions = new ArrayList<>();
+    public List<ActionType<? extends ActionResponse>> getClientActions() {
+        List<ActionType<? extends ActionResponse>> actions = new ArrayList<>();
         actions.addAll(licensing.getClientActions());
         actions.addAll(super.getClientActions());
         return actions;
@@ -281,6 +289,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
         handlers.add(new RestXPackInfoAction(settings, restController));
         handlers.add(new RestXPackUsageAction(settings, restController));
         handlers.add(new RestFreezeIndexAction(settings, restController));
+        handlers.add(new RestReloadAnalyzersAction(settings, restController));
         handlers.addAll(licensing.getRestHandlers(settings, restController, clusterSettings, indexScopedSettings, settingsFilter,
                 indexNameExpressionResolver, nodesInCluster));
         return handlers;
@@ -296,10 +305,6 @@ public static Multibinder<XPackFeatureSet> createFeatureSetMultiBinder(Binder bi
         return Multibinder.newSetBinder(binder, XPackFeatureSet.class);
     }
 
-    public static boolean transportClientMode(Settings settings) {
-        return TransportClient.CLIENT_TYPE.equals(settings.get(Client.CLIENT_TYPE_SETTING_S.getKey()));
-    }
-
     public static Path resolveConfigFile(Environment env, String name) {
         Path config =  env.configFile().resolve(name);
         if (Files.exists(config) == false) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
index 848aacd621111..ad11f103f33ea 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java
@@ -114,6 +114,13 @@ private XPackSettings() {
     /** Setting for enabling or disabling sql. Defaults to true. */
     public static final Setting<Boolean> SQL_ENABLED = Setting.boolSetting("xpack.sql.enabled", true, Setting.Property.NodeScope);
 
+    /** Setting for enabling or disabling flattened fields. Defaults to true. */
+    public static final Setting<Boolean> FLATTENED_ENABLED = Setting.boolSetting("xpack.flattened.enabled",
+        true, Setting.Property.NodeScope);
+
+    /** Setting for enabling or disabling vectors. Defaults to true. */
+    public static final Setting<Boolean> VECTORS_ENABLED = Setting.boolSetting("xpack.vectors.enabled", true, Setting.Property.NodeScope);
+
     /*
      * SSL settings. These are the settings that are specifically registered for SSL. Many are private as we do not explicitly use them
      * but instead parse based on a prefix (eg *.ssl.*)
@@ -204,6 +211,8 @@ public static List<Setting<?>> getAllSettings() {
         settings.add(PASSWORD_HASHING_ALGORITHM);
         settings.add(INDEX_LIFECYCLE_ENABLED);
         settings.add(DATA_FRAME_ENABLED);
+        settings.add(FLATTENED_ENABLED);
+        settings.add(VECTORS_ENABLED);
         return Collections.unmodifiableList(settings);
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/AbstractTransportGetResourcesAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/AbstractTransportGetResourcesAction.java
index 1aaf14013ed41..c414397dc539a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/AbstractTransportGetResourcesAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/AbstractTransportGetResourcesAction.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.action.util.ExpandedIdsMatcher;
 import org.elasticsearch.xpack.core.action.util.QueryPage;
@@ -70,7 +71,10 @@ protected AbstractTransportGetResourcesAction(String actionName, TransportServic
     protected void searchResources(AbstractGetResourcesRequest request, ActionListener<QueryPage<Resource>> listener) {
         String[] tokens = Strings.tokenizeToStringArray(request.getResourceId(), ",");
         SearchSourceBuilder sourceBuilder = new SearchSourceBuilder()
-            .sort(request.getResourceIdField())
+            .sort(SortBuilders.fieldSort(request.getResourceIdField())
+                // If there are no resources, there might be no mapping for the id field.
+                // This makes sure we don't get an error if that happens.
+                .unmappedType("long"))
             .query(buildQuery(tokens, request.getResourceIdField()));
         if (request.getPageParams() != null) {
             sourceBuilder.from(request.getPageParams().getFrom())
@@ -84,7 +88,7 @@ protected void searchResources(AbstractGetResourcesRequest request, ActionListen
                 indicesOptions.expandWildcardsOpen(),
                 indicesOptions.expandWildcardsClosed(),
                 indicesOptions))
-            .source(sourceBuilder);
+            .source(sourceBuilder.trackTotalHits(true));
 
         executeAsyncWithOrigin(client.threadPool().getThreadContext(),
             executionOrigin(),
@@ -94,6 +98,7 @@ protected void searchResources(AbstractGetResourcesRequest request, ActionListen
                 public void onResponse(SearchResponse response) {
                     List<Resource> docs = new ArrayList<>();
                     Set<String> foundResourceIds = new HashSet<>();
+                    long totalHitCount = response.getHits().getTotalHits().value;
                     for (SearchHit hit : response.getHits().getHits()) {
                         BytesReference docSource = hit.getSourceRef();
                         try (InputStream stream = docSource.streamInput();
@@ -111,7 +116,7 @@ public void onResponse(SearchResponse response) {
                     if (requiredMatches.hasUnmatchedIds()) {
                         listener.onFailure(notFoundException(requiredMatches.unmatchedIdsString()));
                     } else {
-                        listener.onResponse(new QueryPage<>(docs, docs.size(), getResultsField()));
+                        listener.onResponse(new QueryPage<>(docs, totalHitCount, getResultsField()));
                     }
                 }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/ReloadAnalyzerAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/ReloadAnalyzerAction.java
new file mode 100644
index 0000000000000..7427d7f642906
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/ReloadAnalyzerAction.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.action;
+
+import org.elasticsearch.action.StreamableResponseActionType;
+
+public class ReloadAnalyzerAction extends StreamableResponseActionType<ReloadAnalyzersResponse> {
+
+    public static final ReloadAnalyzerAction INSTANCE = new ReloadAnalyzerAction();
+    public static final String NAME = "indices:admin/reload_analyzers";
+
+    private ReloadAnalyzerAction() {
+        super(NAME);
+    }
+
+    @Override
+    public ReloadAnalyzersResponse newResponse() {
+        return new ReloadAnalyzersResponse();
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/ReloadAnalyzersRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/ReloadAnalyzersRequest.java
new file mode 100644
index 0000000000000..8721abd3403a7
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/ReloadAnalyzersRequest.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.action;
+
+import org.elasticsearch.action.support.broadcast.BroadcastRequest;
+
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * Request for reloading index search analyzers
+ */
+public class ReloadAnalyzersRequest extends BroadcastRequest<ReloadAnalyzersRequest> {
+
+    /**
+     * Constructs a new request for reloading index search analyzers for one or more indices
+     */
+    public ReloadAnalyzersRequest(String... indices) {
+        super(indices);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        ReloadAnalyzersRequest that = (ReloadAnalyzersRequest) o;
+        return Objects.equals(indicesOptions(), that.indicesOptions())
+                && Arrays.equals(indices, that.indices);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(indicesOptions(), Arrays.hashCode(indices));
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/ReloadAnalyzersResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/ReloadAnalyzersResponse.java
new file mode 100644
index 0000000000000..b996c98bb5843
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/ReloadAnalyzersResponse.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.action;
+
+import org.elasticsearch.action.support.DefaultShardOperationFailedException;
+import org.elasticsearch.action.support.broadcast.BroadcastResponse;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.action.TransportReloadAnalyzersAction.ReloadResult;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
+/**
+ * The response object that will be returned when reloading analyzers
+ */
+public class ReloadAnalyzersResponse extends BroadcastResponse  {
+
+    private final Map<String, ReloadDetails> reloadDetails;
+    private static final ParseField RELOAD_DETAILS_FIELD = new ParseField("reload_details");
+    private static final ParseField INDEX_FIELD = new ParseField("index");
+    private static final ParseField RELOADED_ANALYZERS_FIELD = new ParseField("reloaded_analyzers");
+    private static final ParseField RELOADED_NODE_IDS_FIELD = new ParseField("reloaded_node_ids");
+
+
+    public ReloadAnalyzersResponse() {
+        reloadDetails = Collections.emptyMap();
+    }
+
+    public ReloadAnalyzersResponse(int totalShards, int successfulShards, int failedShards,
+            List<DefaultShardOperationFailedException> shardFailures, Map<String, ReloadDetails> reloadedIndicesNodes) {
+        super(totalShards, successfulShards, failedShards, shardFailures);
+        this.reloadDetails = reloadedIndicesNodes;
+    }
+
+    public final Map<String, ReloadDetails> getReloadDetails() {
+        return this.reloadDetails;
+    }
+
+    /**
+     * Override in subclass to add custom fields following the common `_shards` field
+     */
+    @Override
+    protected void addCustomXContentFields(XContentBuilder builder, Params params) throws IOException {
+        builder.startArray(RELOAD_DETAILS_FIELD.getPreferredName());
+        for (Entry<String, ReloadDetails> indexDetails : reloadDetails.entrySet()) {
+            builder.startObject();
+            ReloadDetails value = indexDetails.getValue();
+            builder.field(INDEX_FIELD.getPreferredName(), value.getIndexName());
+            builder.field(RELOADED_ANALYZERS_FIELD.getPreferredName(), value.getReloadedAnalyzers());
+            builder.field(RELOADED_NODE_IDS_FIELD.getPreferredName(), value.getReloadedIndicesNodes());
+            builder.endObject();
+        }
+        builder.endArray();
+    }
+
+    @SuppressWarnings({ "unchecked" })
+    private static final ConstructingObjectParser<ReloadAnalyzersResponse, Void> PARSER = new ConstructingObjectParser<>("reload_analyzer",
+            true, arg -> {
+                BroadcastResponse response = (BroadcastResponse) arg[0];
+                List<ReloadDetails> results = (List<ReloadDetails>) arg[1];
+                Map<String, ReloadDetails> reloadedNodeIds = new HashMap<>();
+                for (ReloadDetails result : results) {
+                    reloadedNodeIds.put(result.getIndexName(), result);
+                }
+                return new ReloadAnalyzersResponse(response.getTotalShards(), response.getSuccessfulShards(), response.getFailedShards(),
+                        Arrays.asList(response.getShardFailures()), reloadedNodeIds);
+            });
+
+    @SuppressWarnings({ "unchecked" })
+    private static final ConstructingObjectParser<ReloadDetails, Void> ENTRY_PARSER = new ConstructingObjectParser<>(
+            "reload_analyzer.entry", true, arg -> {
+                return new ReloadDetails((String) arg[0], new HashSet<>((List<String>) arg[1]), new HashSet<>((List<String>) arg[2]));
+            });
+
+    static {
+        declareBroadcastFields(PARSER);
+        PARSER.declareObjectArray(constructorArg(), ENTRY_PARSER, RELOAD_DETAILS_FIELD);
+        ENTRY_PARSER.declareString(constructorArg(), INDEX_FIELD);
+        ENTRY_PARSER.declareStringArray(constructorArg(), RELOADED_ANALYZERS_FIELD);
+        ENTRY_PARSER.declareStringArray(constructorArg(), RELOADED_NODE_IDS_FIELD);
+    }
+
+    public static ReloadAnalyzersResponse fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public static class ReloadDetails {
+
+        private final String indexName;
+        private final Set<String> reloadedIndicesNodes;
+        private final Set<String> reloadedAnalyzers;
+
+        public ReloadDetails(String name, Set<String> reloadedIndicesNodes, Set<String> reloadedAnalyzers) {
+            this.indexName = name;
+            this.reloadedIndicesNodes = reloadedIndicesNodes;
+            this.reloadedAnalyzers = reloadedAnalyzers;
+        }
+
+        public String getIndexName() {
+            return indexName;
+        }
+
+        public Set<String> getReloadedIndicesNodes() {
+            return reloadedIndicesNodes;
+        }
+
+        public Set<String> getReloadedAnalyzers() {
+            return reloadedAnalyzers;
+        }
+
+        void merge(ReloadResult other) {
+            assert this.indexName == other.index;
+            this.reloadedAnalyzers.addAll(other.reloadedSearchAnalyzers);
+            this.reloadedIndicesNodes.add(other.nodeId);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportFreezeIndexAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportFreezeIndexAction.java
index 91b91ddd04f3c..228f5c9f2be30 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportFreezeIndexAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportFreezeIndexAction.java
@@ -7,7 +7,7 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
@@ -38,6 +38,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.index.Index;
@@ -86,7 +87,12 @@ protected void doExecute(Task task, FreezeRequest request, ActionListener<Freeze
 
     @Override
     protected FreezeResponse newResponse() {
-        return new FreezeResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected FreezeResponse read(StreamInput in) throws IOException {
+        return new FreezeResponse(in);
     }
 
     private Index[] resolveIndices(FreezeRequest request, ClusterState state) {
@@ -108,11 +114,6 @@ private Index[] resolveIndices(FreezeRequest request, ClusterState state) {
         return indices.toArray(Index.EMPTY_ARRAY);
     }
 
-    @Override
-    protected void masterOperation(FreezeRequest request, ClusterState state, ActionListener<FreezeResponse> listener) {
-        throw new UnsupportedOperationException("The task parameter is required");
-    }
-
     @Override
     protected void masterOperation(Task task, TransportFreezeIndexAction.FreezeRequest request, ClusterState state,
                                    ActionListener<TransportFreezeIndexAction.FreezeResponse> listener) throws Exception {
@@ -127,7 +128,7 @@ protected void masterOperation(Task task, TransportFreezeIndexAction.FreezeReque
             .masterNodeTimeout(request.masterNodeTimeout())
             .indices(concreteIndices);
 
-        indexStateService.closeIndices(closeRequest, new ActionListener<CloseIndexResponse>() {
+        indexStateService.closeIndices(closeRequest, new ActionListener<>() {
             @Override
             public void onResponse(final CloseIndexResponse response) {
                 if (response.isAcknowledged()) {
@@ -149,13 +150,13 @@ public void onFailure(final Exception t) {
     private void toggleFrozenSettings(final Index[] concreteIndices, final FreezeRequest request,
                                       final ActionListener<FreezeResponse> listener) {
         clusterService.submitStateUpdateTask("toggle-frozen-settings",
-            new AckedClusterStateUpdateTask<AcknowledgedResponse>(Priority.URGENT, request, new ActionListener<AcknowledgedResponse>() {
+            new AckedClusterStateUpdateTask<>(Priority.URGENT, request, new ActionListener<AcknowledgedResponse>() {
                 @Override
                 public void onResponse(AcknowledgedResponse acknowledgedResponse) {
                     OpenIndexClusterStateUpdateRequest updateRequest = new OpenIndexClusterStateUpdateRequest()
                         .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout())
                         .indices(concreteIndices).waitForActiveShards(request.waitForActiveShards());
-                    indexStateService.openIndex(updateRequest, new ActionListener<OpenIndexClusterStateUpdateResponse>() {
+                    indexStateService.openIndex(updateRequest, new ActionListener<>() {
                         @Override
                         public void onResponse(OpenIndexClusterStateUpdateResponse openIndexClusterStateUpdateResponse) {
                             listener.onResponse(new FreezeResponse(openIndexClusterStateUpdateResponse.isAcknowledged(),
@@ -217,8 +218,8 @@ protected ClusterBlockException checkBlock(FreezeRequest request, ClusterState s
     }
 
     public static class FreezeResponse extends OpenIndexResponse {
-        public FreezeResponse() {
-            super();
+        FreezeResponse(StreamInput in) throws IOException {
+            super(in);
         }
 
         public FreezeResponse(boolean acknowledged, boolean shardsAcknowledged) {
@@ -226,7 +227,7 @@ public FreezeResponse(boolean acknowledged, boolean shardsAcknowledged) {
         }
     }
 
-    public static class FreezeIndexAction extends Action<FreezeResponse> {
+    public static class FreezeIndexAction extends ActionType<FreezeResponse> {
 
         public static final FreezeIndexAction INSTANCE = new FreezeIndexAction();
         public static final String NAME = "indices:admin/freeze";
@@ -236,8 +237,8 @@ private FreezeIndexAction() {
         }
 
         @Override
-        public FreezeResponse newResponse() {
-            return new FreezeResponse();
+        public Writeable.Reader<FreezeResponse> getResponseReader() {
+            return FreezeResponse::new;
         }
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportReloadAnalyzersAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportReloadAnalyzersAction.java
new file mode 100644
index 0000000000000..4ae7b45db731f
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportReloadAnalyzersAction.java
@@ -0,0 +1,160 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.DefaultShardOperationFailedException;
+import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
+import org.elasticsearch.cluster.routing.PlainShardsIterator;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardsIterator;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.action.ReloadAnalyzersResponse.ReloadDetails;
+import org.elasticsearch.xpack.core.action.TransportReloadAnalyzersAction.ReloadResult;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * Indices clear cache action.
+ */
+public class TransportReloadAnalyzersAction
+        extends TransportBroadcastByNodeAction<ReloadAnalyzersRequest, ReloadAnalyzersResponse, ReloadResult> {
+
+    private static final Logger logger = LogManager.getLogger(TransportReloadAnalyzersAction.class);
+    private final IndicesService indicesService;
+
+    @Inject
+    public TransportReloadAnalyzersAction(ClusterService clusterService, TransportService transportService, IndicesService indicesService,
+            ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(ReloadAnalyzerAction.NAME, clusterService, transportService, actionFilters, indexNameExpressionResolver,
+                ReloadAnalyzersRequest::new, ThreadPool.Names.MANAGEMENT, false);
+        this.indicesService = indicesService;
+    }
+
+    @Override
+    protected ReloadResult readShardResult(StreamInput in) throws IOException {
+        ReloadResult reloadResult = new ReloadResult();
+        reloadResult.readFrom(in);
+        return reloadResult;
+    }
+
+    @Override
+    protected ReloadAnalyzersResponse newResponse(ReloadAnalyzersRequest request, int totalShards, int successfulShards, int failedShards,
+            List<ReloadResult> responses, List<DefaultShardOperationFailedException> shardFailures, ClusterState clusterState) {
+        Map<String, ReloadDetails> reloadedIndicesDetails = new HashMap<String, ReloadDetails>();
+        for (ReloadResult result : responses) {
+            if (reloadedIndicesDetails.containsKey(result.index)) {
+                reloadedIndicesDetails.get(result.index).merge(result);;
+            } else {
+                HashSet<String> nodeIds = new HashSet<String>();
+                nodeIds.add(result.nodeId);
+                ReloadDetails details = new ReloadDetails(result.index, nodeIds, new HashSet<String>(result.reloadedSearchAnalyzers));
+                reloadedIndicesDetails.put(result.index, details);
+            }
+        }
+        return new ReloadAnalyzersResponse(totalShards, successfulShards, failedShards, shardFailures, reloadedIndicesDetails);
+    }
+
+    @Override
+    protected ReloadAnalyzersRequest readRequestFrom(StreamInput in) throws IOException {
+        final ReloadAnalyzersRequest request = new ReloadAnalyzersRequest();
+        request.readFrom(in);
+        return request;
+    }
+
+    @Override
+    protected ReloadResult shardOperation(ReloadAnalyzersRequest request, ShardRouting shardRouting) throws IOException {
+        logger.info("reloading analyzers for index shard " + shardRouting);
+        IndexService indexService = indicesService.indexService(shardRouting.index());
+        List<String> reloadedSearchAnalyzers = indexService.mapperService().reloadSearchAnalyzers(indicesService.getAnalysis());
+        return new ReloadResult(shardRouting.index().getName(), shardRouting.currentNodeId(), reloadedSearchAnalyzers);
+    }
+
+    static final class ReloadResult implements Streamable {
+        String index;
+        String nodeId;
+        List<String> reloadedSearchAnalyzers;
+
+        private ReloadResult(String index, String nodeId, List<String> reloadedSearchAnalyzers) {
+            this.index = index;
+            this.nodeId = nodeId;
+            this.reloadedSearchAnalyzers = reloadedSearchAnalyzers;
+        }
+
+        private ReloadResult() {
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            this.index = in.readString();
+            this.nodeId = in.readString();
+            this.reloadedSearchAnalyzers = in.readStringList();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(index);
+            out.writeString(nodeId);
+            out.writeStringCollection(this.reloadedSearchAnalyzers);
+        }
+    }
+
+    /**
+     * The reload request should go to only one shard per node the index lives on
+     */
+    @Override
+    protected ShardsIterator shards(ClusterState clusterState, ReloadAnalyzersRequest request, String[] concreteIndices) {
+        RoutingTable routingTable = clusterState.routingTable();
+        List<ShardRouting> shards = new ArrayList<>();
+        for (String index : concreteIndices) {
+            Set<String> nodesCovered = new HashSet<>();
+            IndexRoutingTable indexRoutingTable = routingTable.index(index);
+            for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
+                for (ShardRouting shardRouting : indexShardRoutingTable) {
+                    if (nodesCovered.contains(shardRouting.currentNodeId()) == false) {
+                        shards.add(shardRouting);
+                        nodesCovered.add(shardRouting.currentNodeId());
+                    }
+                }
+            }
+        }
+        return new PlainShardsIterator(shards);
+    }
+
+    @Override
+    protected ClusterBlockException checkGlobalBlock(ClusterState state, ReloadAnalyzersRequest request) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+
+    @Override
+    protected ClusterBlockException checkRequestBlock(ClusterState state, ReloadAnalyzersRequest request, String[] concreteIndices) {
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, concreteIndices);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java
index 16f4541e0a7d9..d252cec4b6afd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java
@@ -8,33 +8,41 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse;
+import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse.LicenseInfo;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackBuild;
-import org.elasticsearch.xpack.core.XPackFeatureSet;
 
-import java.util.Set;
-import java.util.stream.Collectors;
+import java.util.HashSet;
+import java.util.List;
 
 public class TransportXPackInfoAction extends HandledTransportAction<XPackInfoRequest, XPackInfoResponse> {
 
     private final LicenseService licenseService;
-    private final Set<XPackFeatureSet> featureSets;
+    private final NodeClient client;
+    private final List<XPackInfoFeatureAction> infoActions;
 
     @Inject
     public TransportXPackInfoAction(TransportService transportService, ActionFilters actionFilters, LicenseService licenseService,
-                                    Set<XPackFeatureSet> featureSets) {
+                                    NodeClient client) {
         super(XPackInfoAction.NAME, transportService, actionFilters,
             XPackInfoRequest::new);
         this.licenseService = licenseService;
-        this.featureSets = featureSets;
+        this.client = client;
+        this.infoActions = infoActions();
+    }
+
+    // overrideable for tests
+    protected List<XPackInfoFeatureAction> infoActions() {
+        return XPackInfoFeatureAction.ALL;
     }
 
     @Override
@@ -55,13 +63,15 @@ protected void doExecute(Task task, XPackInfoRequest request, ActionListener<XPa
             }
         }
 
-        XPackInfoResponse.FeatureSetsInfo featureSetsInfo = null;
+        FeatureSetsInfo featureSetsInfo = null;
         if (request.getCategories().contains(XPackInfoRequest.Category.FEATURES)) {
-            Set<FeatureSet> featureSets = this.featureSets.stream().map(fs ->
-                    new FeatureSet(fs.name(), request.isVerbose() ? fs.description() : null, fs.available(), fs.enabled(),
-                            request.isVerbose() ? fs.nativeCodeInfo() : null))
-                    .collect(Collectors.toSet());
-            featureSetsInfo = new XPackInfoResponse.FeatureSetsInfo(featureSets);
+            var featureSets = new HashSet<FeatureSet>();
+            for (var infoAction : infoActions) {
+                // local actions are executed directly, not on a separate thread, so no thread safe collection is necessary
+                client.executeLocally(infoAction, request,
+                    ActionListener.wrap(response -> featureSets.add(response.getInfo()), listener::onFailure));
+            }
+            featureSetsInfo = new FeatureSetsInfo(featureSets);
         }
 
         listener.onResponse(new XPackInfoResponse(buildInfo, licenseInfo, featureSetsInfo));
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java
index dd1b73415b541..7379233df3c55 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackUsageAction.java
@@ -8,12 +8,14 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
@@ -23,22 +25,28 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 import java.util.function.BiConsumer;
 
 public class TransportXPackUsageAction extends TransportMasterNodeAction<XPackUsageRequest, XPackUsageResponse> {
 
-    private final List<XPackFeatureSet> featureSets;
+    private final NodeClient client;
+    private final List<XPackUsageFeatureAction> usageActions;
 
     @Inject
     public TransportXPackUsageAction(ThreadPool threadPool, TransportService transportService,
                                      ClusterService clusterService, ActionFilters actionFilters,
-                                     IndexNameExpressionResolver indexNameExpressionResolver, Set<XPackFeatureSet> featureSets) {
+                                     IndexNameExpressionResolver indexNameExpressionResolver, NodeClient client) {
         super(XPackUsageAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,
                 XPackUsageRequest::new);
-        this.featureSets = List.copyOf(featureSets);
+        this.client = client;
+        this.usageActions = usageActions();
+    }
+
+    // overrideable for tests
+    protected List<XPackUsageFeatureAction> usageActions() {
+        return XPackUsageFeatureAction.ALL;
     }
 
     @Override
@@ -52,8 +60,8 @@ protected XPackUsageResponse newResponse() {
     }
 
     @Override
-    protected void masterOperation(XPackUsageRequest request, ClusterState state, ActionListener<XPackUsageResponse> listener) {
-        final ActionListener<List<XPackFeatureSet.Usage>> usageActionListener = new ActionListener<List<Usage>>() {
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state, ActionListener<XPackUsageResponse> listener) {
+        final ActionListener<List<XPackFeatureSet.Usage>> usageActionListener = new ActionListener<>() {
             @Override
             public void onResponse(List<Usage> usages) {
                 listener.onResponse(new XPackUsageResponse(usages));
@@ -64,13 +72,13 @@ public void onFailure(Exception e) {
                 listener.onFailure(e);
             }
         };
-        final AtomicReferenceArray<Usage> featureSetUsages = new AtomicReferenceArray<>(featureSets.size());
+        final AtomicReferenceArray<Usage> featureSetUsages = new AtomicReferenceArray<>(usageActions.size());
         final AtomicInteger position = new AtomicInteger(0);
-        final BiConsumer<XPackFeatureSet, ActionListener<List<Usage>>> consumer = (featureSet, iteratingListener) -> {
-            featureSet.usage(new ActionListener<Usage>() {
+        final BiConsumer<XPackUsageFeatureAction, ActionListener<List<Usage>>> consumer = (featureUsageAction, iteratingListener) -> {
+            client.executeLocally(featureUsageAction, request, new ActionListener<>() {
                 @Override
-                public void onResponse(Usage usage) {
-                    featureSetUsages.set(position.getAndIncrement(), usage);
+                public void onResponse(XPackUsageFeatureResponse usageResponse) {
+                    featureSetUsages.set(position.getAndIncrement(), usageResponse.getUsage());
                     // the value sent back doesn't matter since our predicate keeps iterating
                     iteratingListener.onResponse(Collections.emptyList());
                 }
@@ -81,8 +89,8 @@ public void onFailure(Exception e) {
                 }
             });
         };
-        IteratingActionListener<List<XPackFeatureSet.Usage>, XPackFeatureSet> iteratingActionListener =
-                new IteratingActionListener<>(usageActionListener, consumer, featureSets,
+        IteratingActionListener<List<XPackFeatureSet.Usage>, XPackUsageFeatureAction> iteratingActionListener =
+                new IteratingActionListener<>(usageActionListener, consumer, usageActions,
                         threadPool.getThreadContext(), (ignore) -> {
                     final List<Usage> usageList = new ArrayList<>(featureSetUsages.length());
                     for (int i = 0; i < featureSetUsages.length(); i++) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoAction.java
index a47ce7a41c698..02acdadfe2d70 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoAction.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.core.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.protocol.xpack.XPackInfoResponse;
 
-public class XPackInfoAction extends Action<XPackInfoResponse> {
+public class XPackInfoAction extends StreamableResponseActionType<XPackInfoResponse> {
 
     public static final String NAME = "cluster:monitor/xpack/info";
     public static final XPackInfoAction INSTANCE = new XPackInfoAction();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java
new file mode 100644
index 0000000000000..25cf63f3d45f7
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.action;
+
+import org.elasticsearch.action.StreamableResponseActionType;
+import org.elasticsearch.xpack.core.XPackField;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * A base action for info about a feature plugin.
+ *
+ * This action is implemented by each feature plugin, bound to the public constants here. The
+ * {@link XPackInfoAction} implementation iterates over the {@link #ALL} list of actions to form
+ * the complete info result.
+ */
+public class XPackInfoFeatureAction extends StreamableResponseActionType<XPackInfoFeatureResponse> {
+
+    private static final String BASE_NAME = "cluster:monitor/xpack/info/";
+
+    public static final XPackInfoFeatureAction SECURITY = new XPackInfoFeatureAction(XPackField.SECURITY);
+    public static final XPackInfoFeatureAction MONITORING = new XPackInfoFeatureAction(XPackField.MONITORING);
+    public static final XPackInfoFeatureAction WATCHER = new XPackInfoFeatureAction(XPackField.WATCHER);
+    public static final XPackInfoFeatureAction GRAPH = new XPackInfoFeatureAction(XPackField.GRAPH);
+    public static final XPackInfoFeatureAction MACHINE_LEARNING = new XPackInfoFeatureAction(XPackField.MACHINE_LEARNING);
+    public static final XPackInfoFeatureAction LOGSTASH = new XPackInfoFeatureAction(XPackField.LOGSTASH);
+    public static final XPackInfoFeatureAction SQL = new XPackInfoFeatureAction(XPackField.SQL);
+    public static final XPackInfoFeatureAction ROLLUP = new XPackInfoFeatureAction(XPackField.ROLLUP);
+    public static final XPackInfoFeatureAction INDEX_LIFECYCLE = new XPackInfoFeatureAction(XPackField.INDEX_LIFECYCLE);
+    public static final XPackInfoFeatureAction CCR = new XPackInfoFeatureAction(XPackField.CCR);
+    public static final XPackInfoFeatureAction DATA_FRAME = new XPackInfoFeatureAction(XPackField.DATA_FRAME);
+    public static final XPackInfoFeatureAction FLATTENED = new XPackInfoFeatureAction(XPackField.FLATTENED);
+    public static final XPackInfoFeatureAction VECTORS = new XPackInfoFeatureAction(XPackField.VECTORS);
+    public static final XPackInfoFeatureAction VOTING_ONLY = new XPackInfoFeatureAction(XPackField.VOTING_ONLY);
+
+    public static final List<XPackInfoFeatureAction> ALL = Arrays.asList(
+        SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, CCR, DATA_FRAME, FLATTENED,
+        VECTORS, VOTING_ONLY
+    );
+
+    private XPackInfoFeatureAction(String name) {
+        super(BASE_NAME + name);
+    }
+
+    @Override
+    public XPackInfoFeatureResponse newResponse() {
+        return new XPackInfoFeatureResponse();
+    }
+
+    @Override
+    public String toString() {
+        return "ActionType [" + name() + "]";
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureResponse.java
new file mode 100644
index 0000000000000..b40a0778d81da
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureResponse.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet;
+
+import java.io.IOException;
+
+public class XPackInfoFeatureResponse extends ActionResponse {
+
+    private FeatureSet info;
+
+    public XPackInfoFeatureResponse() {
+        // empty, for readFrom
+    }
+
+    public XPackInfoFeatureResponse(FeatureSet info) {
+        this.info = info;
+    }
+
+    public FeatureSet getInfo() {
+        return info;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        info.writeTo(out);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        info = new FeatureSet(in);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureTransportAction.java
new file mode 100644
index 0000000000000..672128a2fd8be
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureTransportAction.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.protocol.xpack.XPackInfoRequest;
+import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+
+/**
+ * A base class to implement {@link XPackInfoFeatureAction} actions.
+ *
+ * Extend this class and implement the abstract methods, and register the appropriate
+ * {@link XPackInfoFeatureAction} to the subclass of this class.
+ */
+public abstract class XPackInfoFeatureTransportAction extends HandledTransportAction<XPackInfoRequest, XPackInfoFeatureResponse> {
+
+    public XPackInfoFeatureTransportAction(String name, TransportService transportService, ActionFilters actionFilters) {
+        super(name, transportService, actionFilters, XPackInfoRequest::new);
+    }
+
+    protected abstract String name();
+
+    protected abstract boolean available();
+
+    protected abstract boolean enabled();
+
+    @Override
+    protected void doExecute(Task task, XPackInfoRequest request, ActionListener<XPackInfoFeatureResponse> listener) {
+        listener.onResponse(new XPackInfoFeatureResponse(new FeatureSet(name(), available(), enabled())));
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageAction.java
index 40311a4e88457..96f9bb8310c08 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.core.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class XPackUsageAction extends Action<XPackUsageResponse> {
+public class XPackUsageAction extends StreamableResponseActionType<XPackUsageResponse> {
 
     public static final String NAME = "cluster:monitor/xpack/usage";
     public static final XPackUsageAction INSTANCE = new XPackUsageAction();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java
new file mode 100644
index 0000000000000..3c2c5b5b3e310
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.action;
+
+import org.elasticsearch.action.StreamableResponseActionType;
+import org.elasticsearch.xpack.core.XPackField;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * A base action for for usage of a feature plugin.
+ *
+ * This action is implemented by each feature plugin, bound to the public constants here. The
+ * {@link XPackUsageAction} implementationn iterates over the {@link #ALL} list of actions to form
+ * the complete usage result.
+ */
+public class XPackUsageFeatureAction extends StreamableResponseActionType<XPackUsageFeatureResponse> {
+
+    private static final String BASE_NAME = "cluster:monitor/xpack/usage/";
+
+    public static final XPackUsageFeatureAction SECURITY = new XPackUsageFeatureAction(XPackField.SECURITY);
+    public static final XPackUsageFeatureAction MONITORING = new XPackUsageFeatureAction(XPackField.MONITORING);
+    public static final XPackUsageFeatureAction WATCHER = new XPackUsageFeatureAction(XPackField.WATCHER);
+    public static final XPackUsageFeatureAction GRAPH = new XPackUsageFeatureAction(XPackField.GRAPH);
+    public static final XPackUsageFeatureAction MACHINE_LEARNING = new XPackUsageFeatureAction(XPackField.MACHINE_LEARNING);
+    public static final XPackUsageFeatureAction LOGSTASH = new XPackUsageFeatureAction(XPackField.LOGSTASH);
+    public static final XPackUsageFeatureAction SQL = new XPackUsageFeatureAction(XPackField.SQL);
+    public static final XPackUsageFeatureAction ROLLUP = new XPackUsageFeatureAction(XPackField.ROLLUP);
+    public static final XPackUsageFeatureAction INDEX_LIFECYCLE = new XPackUsageFeatureAction(XPackField.INDEX_LIFECYCLE);
+    public static final XPackUsageFeatureAction CCR = new XPackUsageFeatureAction(XPackField.CCR);
+    public static final XPackUsageFeatureAction DATA_FRAME = new XPackUsageFeatureAction(XPackField.DATA_FRAME);
+    public static final XPackUsageFeatureAction FLATTENED = new XPackUsageFeatureAction(XPackField.FLATTENED);
+    public static final XPackUsageFeatureAction VECTORS = new XPackUsageFeatureAction(XPackField.VECTORS);
+    public static final XPackUsageFeatureAction VOTING_ONLY = new XPackUsageFeatureAction(XPackField.VOTING_ONLY);
+
+    public static final List<XPackUsageFeatureAction> ALL = Arrays.asList(
+        SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, CCR, DATA_FRAME, FLATTENED,
+        VECTORS, VOTING_ONLY
+    );
+
+    private XPackUsageFeatureAction(String name) {
+        super(BASE_NAME + name);
+    }
+
+    @Override
+    public XPackUsageFeatureResponse newResponse() {
+        return new XPackUsageFeatureResponse();
+    }
+
+    @Override
+    public String toString() {
+        return "ActionType [" + name() + "]";
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureResponse.java
new file mode 100644
index 0000000000000..1a3d8c69e1da1
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureResponse.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+
+import java.io.IOException;
+
+public class XPackUsageFeatureResponse extends ActionResponse {
+
+    private XPackFeatureSet.Usage usage;
+
+    public XPackUsageFeatureResponse() {
+        // empty, for readFrom
+    }
+
+    public XPackUsageFeatureResponse(XPackFeatureSet.Usage usage) {
+        this.usage = usage;
+    }
+
+    public XPackFeatureSet.Usage getUsage() {
+        return usage;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeNamedWriteable(usage);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        usage = in.readNamedWriteable(XPackFeatureSet.Usage.class);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureTransportAction.java
new file mode 100644
index 0000000000000..baaae6f622199
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureTransportAction.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.action;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+public abstract class XPackUsageFeatureTransportAction extends TransportMasterNodeAction<XPackUsageRequest, XPackUsageFeatureResponse> {
+
+    public XPackUsageFeatureTransportAction(String name, TransportService transportService, ClusterService clusterService,
+                                            ThreadPool threadPool, ActionFilters actionFilters,
+                                            IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(name, transportService, clusterService, threadPool,
+              actionFilters, indexNameExpressionResolver, XPackUsageRequest::new);
+    }
+
+    @Override
+    protected String executor() {
+        return ThreadPool.Names.MANAGEMENT;
+    }
+
+    @Override
+    protected XPackUsageFeatureResponse newResponse() {
+        return new XPackUsageFeatureResponse();
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(XPackUsageRequest request, ClusterState state) {
+        return null;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java
index a8758ed6c2d5a..b06ba584178c0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java
@@ -120,7 +120,7 @@ public String getWriteableName() {
 
     @Override
     public Version getMinimalSupportedVersion() {
-        return Version.V_6_5_0.minimumCompatibilityVersion();
+        return Version.CURRENT.minimumCompatibilityVersion();
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowStats.java
index 6c605ede85e24..879c8d79cd548 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowStats.java
@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.core.ccr;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -19,7 +18,6 @@
 
 import java.io.IOException;
 import java.util.AbstractMap;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
@@ -118,12 +116,8 @@ public AutoFollowStats(StreamInput in) throws IOException {
         numberOfSuccessfulFollowIndices = in.readVLong();
         // note: the casts to the following Writeable.Reader<T> instances are needed by some IDEs (e.g. Eclipse 4.8) as a compiler help
         recentAutoFollowErrors = new TreeMap<>(in.readMap((Writeable.Reader<String>) StreamInput::readString,
-                (Writeable.Reader<Tuple<Long, ElasticsearchException>>) in1 -> new Tuple<>(in1.readZLong(), in1.readException())));
-        if (in.getVersion().onOrAfter(Version.V_6_6_0)) {
-            autoFollowedClusters = new TreeMap<>(in.readMap(StreamInput::readString, AutoFollowedCluster::new));
-        } else {
-            autoFollowedClusters = Collections.emptyNavigableMap();
-        }
+            (Writeable.Reader<Tuple<Long, ElasticsearchException>>) in1 -> new Tuple<>(in1.readZLong(), in1.readException())));
+        autoFollowedClusters = new TreeMap<>(in.readMap(StreamInput::readString, AutoFollowedCluster::new));
     }
 
     @Override
@@ -135,10 +129,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out1.writeZLong(value.v1());
             out1.writeException(value.v2());
         });
-
-        if (out.getVersion().onOrAfter(Version.V_6_6_0)) {
-            out.writeMap(autoFollowedClusters, StreamOutput::writeString, (out1, value) -> value.writeTo(out1));
-        }
+        out.writeMap(autoFollowedClusters, StreamOutput::writeString, (out1, value) -> value.writeTo(out1));
     }
 
     public long getNumberOfFailedFollowIndices() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/ShardFollowNodeTaskStatus.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/ShardFollowNodeTaskStatus.java
index 33a5d495c1631..2b09490477630 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/ShardFollowNodeTaskStatus.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/ShardFollowNodeTaskStatus.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.core.ccr;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
@@ -49,6 +50,7 @@ public class ShardFollowNodeTaskStatus implements Task.Status {
     private static final ParseField WRITE_BUFFER_SIZE_IN_BYTES_FIELD = new ParseField("write_buffer_size_in_bytes");
     private static final ParseField FOLLOWER_MAPPING_VERSION_FIELD = new ParseField("follower_mapping_version");
     private static final ParseField FOLLOWER_SETTINGS_VERSION_FIELD = new ParseField("follower_settings_version");
+    private static final ParseField FOLLOWER_ALIASES_VERSION_FIELD = new ParseField("follower_aliases_version");
     private static final ParseField TOTAL_READ_TIME_MILLIS_FIELD = new ParseField("total_read_time_millis");
     private static final ParseField TOTAL_READ_REMOTE_EXEC_TIME_MILLIS_FIELD = new ParseField("total_read_remote_exec_time_millis");
     private static final ParseField SUCCESSFUL_READ_REQUESTS_FIELD = new ParseField("successful_read_requests");
@@ -93,12 +95,13 @@ public class ShardFollowNodeTaskStatus implements Task.Status {
                             (long) args[22],
                             (long) args[23],
                             (long) args[24],
+                            (long) args[25],
                             new TreeMap<>(
-                                    ((List<Map.Entry<Long, Tuple<Integer, ElasticsearchException>>>) args[25])
+                                    ((List<Map.Entry<Long, Tuple<Integer, ElasticsearchException>>>) args[26])
                                             .stream()
                                             .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))),
-                            (long) args[26],
-                            (ElasticsearchException) args[27]));
+                            (long) args[27],
+                            (ElasticsearchException) args[28]));
 
     public static final String READ_EXCEPTIONS_ENTRY_PARSER_NAME = "shard-follow-node-task-status-read-exceptions-entry";
 
@@ -123,6 +126,7 @@ public class ShardFollowNodeTaskStatus implements Task.Status {
         STATUS_PARSER.declareLong(ConstructingObjectParser.constructorArg(), WRITE_BUFFER_SIZE_IN_BYTES_FIELD);
         STATUS_PARSER.declareLong(ConstructingObjectParser.constructorArg(), FOLLOWER_MAPPING_VERSION_FIELD);
         STATUS_PARSER.declareLong(ConstructingObjectParser.constructorArg(), FOLLOWER_SETTINGS_VERSION_FIELD);
+        STATUS_PARSER.declareLong(ConstructingObjectParser.constructorArg(), FOLLOWER_ALIASES_VERSION_FIELD);
         STATUS_PARSER.declareLong(ConstructingObjectParser.constructorArg(), TOTAL_READ_TIME_MILLIS_FIELD);
         STATUS_PARSER.declareLong(ConstructingObjectParser.constructorArg(), TOTAL_READ_REMOTE_EXEC_TIME_MILLIS_FIELD);
         STATUS_PARSER.declareLong(ConstructingObjectParser.constructorArg(), SUCCESSFUL_READ_REQUESTS_FIELD);
@@ -243,6 +247,12 @@ public long followerSettingsVersion() {
         return followerSettingsVersion;
     }
 
+    private final long followerAliasesVersion;
+
+    public long followerAliasesVersion() {
+        return followerAliasesVersion;
+    }
+
     private final long totalReadTimeMillis;
 
     public long totalReadTimeMillis() {
@@ -337,6 +347,7 @@ public ShardFollowNodeTaskStatus(
             final long writeBufferSizeInBytes,
             final long followerMappingVersion,
             final long followerSettingsVersion,
+            final long followerAliasesVersion,
             final long totalReadTimeMillis,
             final long totalReadRemoteExecTimeMillis,
             final long successfulReadRequests,
@@ -365,6 +376,7 @@ public ShardFollowNodeTaskStatus(
         this.writeBufferSizeInBytes = writeBufferSizeInBytes;
         this.followerMappingVersion = followerMappingVersion;
         this.followerSettingsVersion = followerSettingsVersion;
+        this.followerAliasesVersion = followerAliasesVersion;
         this.totalReadTimeMillis = totalReadTimeMillis;
         this.totalReadRemoteExecTimeMillis = totalReadRemoteExecTimeMillis;
         this.successfulReadRequests = successfulReadRequests;
@@ -396,6 +408,11 @@ public ShardFollowNodeTaskStatus(final StreamInput in) throws IOException {
         this.writeBufferSizeInBytes = in.readVLong();
         this.followerMappingVersion = in.readVLong();
         this.followerSettingsVersion = in.readVLong();
+        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+            this.followerAliasesVersion = in.readVLong();
+        } else {
+            this.followerAliasesVersion = 0L;
+        }
         this.totalReadTimeMillis = in.readVLong();
         this.totalReadRemoteExecTimeMillis = in.readVLong();
         this.successfulReadRequests = in.readVLong();
@@ -434,6 +451,9 @@ public void writeTo(final StreamOutput out) throws IOException {
         out.writeVLong(writeBufferSizeInBytes);
         out.writeVLong(followerMappingVersion);
         out.writeVLong(followerSettingsVersion);
+        if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+            out.writeVLong(followerAliasesVersion);
+        }
         out.writeVLong(totalReadTimeMillis);
         out.writeVLong(totalReadRemoteExecTimeMillis);
         out.writeVLong(successfulReadRequests);
@@ -484,6 +504,7 @@ public XContentBuilder toXContentFragment(final XContentBuilder builder, final P
                 new ByteSizeValue(writeBufferSizeInBytes));
         builder.field(FOLLOWER_MAPPING_VERSION_FIELD.getPreferredName(), followerMappingVersion);
         builder.field(FOLLOWER_SETTINGS_VERSION_FIELD.getPreferredName(), followerSettingsVersion);
+        builder.field(FOLLOWER_ALIASES_VERSION_FIELD.getPreferredName(), followerAliasesVersion);
         builder.humanReadableField(
                 TOTAL_READ_TIME_MILLIS_FIELD.getPreferredName(),
                 "total_read_time",
@@ -564,7 +585,8 @@ public boolean equals(final Object o) {
                 writeBufferOperationCount == that.writeBufferOperationCount &&
                 writeBufferSizeInBytes == that.writeBufferSizeInBytes &&
                 followerMappingVersion == that.followerMappingVersion &&
-                followerSettingsVersion== that.followerSettingsVersion &&
+                followerSettingsVersion == that.followerSettingsVersion &&
+                followerAliasesVersion == that.followerAliasesVersion &&
                 totalReadTimeMillis == that.totalReadTimeMillis &&
                 totalReadRemoteExecTimeMillis == that.totalReadRemoteExecTimeMillis &&
                 successfulReadRequests == that.successfulReadRequests &&
@@ -604,6 +626,7 @@ public int hashCode() {
                 writeBufferSizeInBytes,
                 followerMappingVersion,
                 followerSettingsVersion,
+                followerAliasesVersion,
                 totalReadTimeMillis,
                 totalReadRemoteExecTimeMillis,
                 successfulReadRequests,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
index 5b2033443dcf7..3ce2191759d30 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -20,7 +20,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class CcrStatsAction extends Action<CcrStatsAction.Response> {
+public class CcrStatsAction extends ActionType<CcrStatsAction.Response> {
 
     public static final String NAME = "cluster:monitor/ccr/stats";
     public static final CcrStatsAction INSTANCE = new CcrStatsAction();
@@ -29,11 +29,6 @@ private CcrStatsAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/DeleteAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/DeleteAutoFollowPatternAction.java
index 5bd767c5c50bb..2020192d90fd2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/DeleteAutoFollowPatternAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/DeleteAutoFollowPatternAction.java
@@ -5,19 +5,20 @@
  */
 package org.elasticsearch.xpack.core.ccr.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 
 import java.io.IOException;
 import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-public class DeleteAutoFollowPatternAction extends Action<AcknowledgedResponse> {
+public class DeleteAutoFollowPatternAction extends ActionType<AcknowledgedResponse> {
 
     public static final String NAME = "cluster:admin/xpack/ccr/auto_follow_pattern/delete";
     public static final DeleteAutoFollowPatternAction INSTANCE = new DeleteAutoFollowPatternAction();
@@ -27,8 +28,8 @@ private DeleteAutoFollowPatternAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends AcknowledgedRequest<Request> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java
index 122ae2b0a179d..90c0dc3ce65ce 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ccr.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
@@ -22,7 +22,7 @@
 import java.util.List;
 import java.util.Objects;
 
-public class FollowInfoAction extends Action<FollowInfoAction.Response> {
+public class FollowInfoAction extends ActionType<FollowInfoAction.Response> {
 
     public static final String NAME = "cluster:monitor/ccr/follow_info";
 
@@ -32,11 +32,6 @@ private FollowInfoAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowStatsAction.java
index 72353f405cfee..44d27777d9fa7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowStatsAction.java
@@ -6,7 +6,7 @@
 
 package org.elasticsearch.xpack.core.ccr.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.IndicesRequest;
@@ -29,7 +29,7 @@
 import java.util.Objects;
 import java.util.TreeMap;
 
-public class FollowStatsAction extends Action<FollowStatsAction.StatsResponses> {
+public class FollowStatsAction extends ActionType<FollowStatsAction.StatsResponses> {
 
     public static final String NAME = "cluster:monitor/ccr/follow_stats";
 
@@ -39,11 +39,6 @@ private FollowStatsAction() {
         super(NAME);
     }
 
-    @Override
-    public StatsResponses newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<StatsResponses> getResponseReader() {
         return StatsResponses::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ForgetFollowerAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ForgetFollowerAction.java
index d2a0b565496d6..a86ac8abf569f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ForgetFollowerAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ForgetFollowerAction.java
@@ -6,8 +6,8 @@
 
 package org.elasticsearch.xpack.core.ccr.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.broadcast.BroadcastRequest;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.common.ParseField;
@@ -19,7 +19,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class ForgetFollowerAction extends Action<BroadcastResponse> {
+public class ForgetFollowerAction extends StreamableResponseActionType<BroadcastResponse> {
 
     public static final String NAME = "indices:admin/xpack/ccr/forget_follower";
     public static final ForgetFollowerAction INSTANCE = new ForgetFollowerAction();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/GetAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/GetAutoFollowPatternAction.java
index cd37692da43a3..36ce0e3e2fe28 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/GetAutoFollowPatternAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/GetAutoFollowPatternAction.java
@@ -6,7 +6,7 @@
 
 package org.elasticsearch.xpack.core.ccr.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
@@ -21,7 +21,7 @@
 import java.util.Map;
 import java.util.Objects;
 
-public class GetAutoFollowPatternAction extends Action<GetAutoFollowPatternAction.Response> {
+public class GetAutoFollowPatternAction extends ActionType<GetAutoFollowPatternAction.Response> {
 
     public static final String NAME = "cluster:admin/xpack/ccr/auto_follow_pattern/get";
     public static final GetAutoFollowPatternAction INSTANCE = new GetAutoFollowPatternAction();
@@ -30,11 +30,6 @@ private GetAutoFollowPatternAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PauseFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PauseFollowAction.java
index c20cda9ab4a4a..85224d09001ae 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PauseFollowAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PauseFollowAction.java
@@ -6,17 +6,18 @@
 
 package org.elasticsearch.xpack.core.ccr.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 
 import java.io.IOException;
 import java.util.Objects;
 
-public class PauseFollowAction extends Action<AcknowledgedResponse> {
+public class PauseFollowAction extends ActionType<AcknowledgedResponse> {
 
     public static final PauseFollowAction INSTANCE = new PauseFollowAction();
     public static final String NAME = "cluster:admin/xpack/ccr/pause_follow";
@@ -26,8 +27,8 @@ private PauseFollowAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends MasterNodeRequest<Request> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java
index f26e8d7f82a51..3d6beb67f3de4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java
@@ -5,12 +5,13 @@
  */
 package org.elasticsearch.xpack.core.ccr.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -25,7 +26,7 @@
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern.REMOTE_CLUSTER_FIELD;
 
-public class PutAutoFollowPatternAction extends Action<AcknowledgedResponse> {
+public class PutAutoFollowPatternAction extends ActionType<AcknowledgedResponse> {
 
     public static final String NAME = "cluster:admin/xpack/ccr/auto_follow_pattern/put";
     public static final PutAutoFollowPatternAction INSTANCE = new PutAutoFollowPatternAction();
@@ -36,8 +37,8 @@ private PutAutoFollowPatternAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends AcknowledgedRequest<Request> implements ToXContentObject {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java
index 4d20e6d820de2..a9eef20362886 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java
@@ -6,7 +6,7 @@
 
 package org.elasticsearch.xpack.core.ccr.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.IndicesRequest;
@@ -27,7 +27,7 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-public final class PutFollowAction extends Action<PutFollowAction.Response> {
+public final class PutFollowAction extends ActionType<PutFollowAction.Response> {
 
     public static final PutFollowAction INSTANCE = new PutFollowAction();
     public static final String NAME = "indices:admin/xpack/ccr/put_follow";
@@ -36,11 +36,6 @@ private PutFollowAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java
index 547f04889a669..a55644facd2a9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java
@@ -6,12 +6,13 @@
 
 package org.elasticsearch.xpack.core.ccr.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -22,7 +23,7 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-public final class ResumeFollowAction extends Action<AcknowledgedResponse> {
+public final class ResumeFollowAction extends ActionType<AcknowledgedResponse> {
 
     public static final ResumeFollowAction INSTANCE = new ResumeFollowAction();
     public static final String NAME = "cluster:admin/xpack/ccr/resume_follow";
@@ -32,8 +33,8 @@ private ResumeFollowAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends MasterNodeRequest<Request> implements ToXContentObject {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/UnfollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/UnfollowAction.java
index 9bb440cc0895d..2b879d7210327 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/UnfollowAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/UnfollowAction.java
@@ -6,7 +6,7 @@
 
 package org.elasticsearch.xpack.core.ccr.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -14,12 +14,13 @@
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 
 import java.io.IOException;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-public class UnfollowAction extends Action<AcknowledgedResponse> {
+public class UnfollowAction extends ActionType<AcknowledgedResponse> {
 
     public static final UnfollowAction INSTANCE = new UnfollowAction();
     public static final String NAME = "indices:admin/xpack/ccr/unfollow";
@@ -29,8 +30,8 @@ private UnfollowAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends AcknowledgedRequest<Request> implements IndicesRequest {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/client/CcrClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/client/CcrClient.java
deleted file mode 100644
index 00b115131d1fb..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/client/CcrClient.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.core.ccr.client;
-
-import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.action.support.broadcast.BroadcastResponse;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
-import org.elasticsearch.xpack.core.ccr.action.DeleteAutoFollowPatternAction;
-import org.elasticsearch.xpack.core.ccr.action.FollowStatsAction;
-import org.elasticsearch.xpack.core.ccr.action.ForgetFollowerAction;
-import org.elasticsearch.xpack.core.ccr.action.GetAutoFollowPatternAction;
-import org.elasticsearch.xpack.core.ccr.action.PauseFollowAction;
-import org.elasticsearch.xpack.core.ccr.action.PutAutoFollowPatternAction;
-import org.elasticsearch.xpack.core.ccr.action.PutFollowAction;
-import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction;
-import org.elasticsearch.xpack.core.ccr.action.UnfollowAction;
-
-import java.util.Objects;
-
-public class CcrClient {
-
-    private final ElasticsearchClient client;
-
-    public CcrClient(final ElasticsearchClient client) {
-        this.client = Objects.requireNonNull(client, "client");
-    }
-
-    public void putFollow(
-            final PutFollowAction.Request request,
-            final ActionListener<PutFollowAction.Response> listener) {
-        client.execute(PutFollowAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<PutFollowAction.Response> putFollow(final PutFollowAction.Request request) {
-        final PlainActionFuture<PutFollowAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(PutFollowAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void resumeFollow(final ResumeFollowAction.Request request, final ActionListener<AcknowledgedResponse> listener) {
-        client.execute(ResumeFollowAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<AcknowledgedResponse> resumeFollow(final ResumeFollowAction.Request request) {
-        final PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        client.execute(ResumeFollowAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void followStats(
-            final FollowStatsAction.StatsRequest request,
-            final ActionListener<FollowStatsAction.StatsResponses> listener) {
-        client.execute(FollowStatsAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<FollowStatsAction.StatsResponses> followStats(final FollowStatsAction.StatsRequest request) {
-        final PlainActionFuture<FollowStatsAction.StatsResponses> listener = PlainActionFuture.newFuture();
-        client.execute(FollowStatsAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void stats(final CcrStatsAction.Request request,
-                      final ActionListener<CcrStatsAction.Response> listener) {
-        client.execute(CcrStatsAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<CcrStatsAction.Response> stats(final CcrStatsAction.Request request) {
-        final PlainActionFuture<CcrStatsAction.Response> listener = PlainActionFuture.newFuture();
-        stats(request, listener);
-        return listener;
-    }
-
-    public void pauseFollow(final PauseFollowAction.Request request, final ActionListener<AcknowledgedResponse> listener) {
-        client.execute(PauseFollowAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<AcknowledgedResponse> pauseFollow(final PauseFollowAction.Request request) {
-        final PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        client.execute(PauseFollowAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void unfollow(final UnfollowAction.Request request, final ActionListener<AcknowledgedResponse> listener) {
-        client.execute(UnfollowAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<AcknowledgedResponse> unfollow(final UnfollowAction.Request request) {
-        final PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        client.execute(UnfollowAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void forgetFollower(final ForgetFollowerAction.Request request, final ActionListener<BroadcastResponse> listener) {
-        client.execute(ForgetFollowerAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<BroadcastResponse> forgetFollower(final ForgetFollowerAction.Request request) {
-        final PlainActionFuture<BroadcastResponse> listener = PlainActionFuture.newFuture();
-        client.execute(ForgetFollowerAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void putAutoFollowPattern(
-            final PutAutoFollowPatternAction.Request request,
-            final ActionListener<AcknowledgedResponse> listener) {
-        client.execute(PutAutoFollowPatternAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<AcknowledgedResponse> putAutoFollowPattern(final PutAutoFollowPatternAction.Request request) {
-        final PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        client.execute(PutAutoFollowPatternAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void deleteAutoFollowPattern(
-            final DeleteAutoFollowPatternAction.Request request,
-            final ActionListener<AcknowledgedResponse> listener) {
-        client.execute(DeleteAutoFollowPatternAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<AcknowledgedResponse> deleteAutoFollowPattern(final DeleteAutoFollowPatternAction.Request request) {
-        final PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        client.execute(DeleteAutoFollowPatternAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void getAutoFollowPattern(
-        final GetAutoFollowPatternAction.Request request,
-        final ActionListener<GetAutoFollowPatternAction.Response> listener) {
-        client.execute(GetAutoFollowPatternAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<GetAutoFollowPatternAction.Response> getAutoFollowPattern(final GetAutoFollowPatternAction.Request request) {
-        final PlainActionFuture<GetAutoFollowPatternAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(GetAutoFollowPatternAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/time/TimeUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/time/TimeUtils.java
similarity index 87%
rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/time/TimeUtils.java
rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/time/TimeUtils.java
index ea0994dad717c..e345feb59b04e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/time/TimeUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/time/TimeUtils.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.core.ml.utils.time;
+package org.elasticsearch.xpack.core.common.time;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
@@ -12,6 +12,7 @@
 import org.elasticsearch.index.mapper.DateFieldMapper;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.Date;
 import java.util.concurrent.TimeUnit;
 
@@ -25,12 +26,22 @@ public static Date parseTimeField(XContentParser parser, String fieldName) throw
         if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
             return new Date(parser.longValue());
         } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
-            return new Date(TimeUtils.dateStringToEpoch(parser.text()));
+            return new Date(dateStringToEpoch(parser.text()));
         }
         throw new IllegalArgumentException(
                 "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]");
     }
 
+    public static Instant parseTimeFieldToInstant(XContentParser parser, String fieldName) throws IOException {
+        if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
+            return Instant.ofEpochMilli(parser.longValue());
+        } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
+            return Instant.ofEpochMilli(dateStringToEpoch(parser.text()));
+        }
+        throw new IllegalArgumentException(
+            "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]");
+    }
+
     /**
      * First tries to parse the date first as a Long and convert that to an
      * epoch time. If the long number has more than 10 digits it is considered a
@@ -112,8 +123,6 @@ private static void checkNonNegative(TimeValue timeValue, ParseField field) {
         }
     }
 
-
-
     /**
      * Check the given {@code timeValue} is a multiple of the {@code baseUnit}
      */
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java
index c61ed2ddde8be..2dedef82eb3cf 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java
@@ -28,7 +28,12 @@ public final class DataFrameField {
     public static final ParseField DESTINATION = new ParseField("dest");
     public static final ParseField FORCE = new ParseField("force");
     public static final ParseField MAX_PAGE_SEARCH_SIZE = new ParseField("max_page_search_size");
+    public static final ParseField FIELD = new ParseField("field");
+    public static final ParseField SYNC = new ParseField("sync");
+    public static final ParseField TIME_BASED_SYNC = new ParseField("time");
+    public static final ParseField DELAY = new ParseField("delay");
 
+    public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
     /**
      * Fields for checkpointing
      */
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameMessages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameMessages.java
index d31892692a5df..5d6cf54c44d14 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameMessages.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameMessages.java
@@ -38,7 +38,9 @@ public class DataFrameMessages {
     public static final String FAILED_TO_PARSE_TRANSFORM_CONFIGURATION =
             "Failed to parse transform configuration for data frame transform [{0}]";
     public static final String FAILED_TO_PARSE_TRANSFORM_STATISTICS_CONFIGURATION =
-        "Failed to parse transform statistics for data frame transform [{0}]";
+            "Failed to parse transform statistics for data frame transform [{0}]";
+    public static final String FAILED_TO_LOAD_TRANSFORM_CHECKPOINT =
+            "Failed to load data frame transform configuration for transform [{0}]";
     public static final String DATA_FRAME_TRANSFORM_CONFIGURATION_NO_TRANSFORM =
             "Data frame transform configuration must specify exactly 1 function";
     public static final String DATA_FRAME_TRANSFORM_CONFIGURATION_PIVOT_NO_GROUP_BY =
@@ -57,14 +59,18 @@ public class DataFrameMessages {
     public static final String LOG_DATA_FRAME_TRANSFORM_CONFIGURATION_BAD_AGGREGATION =
             "Failed to parse aggregation for data frame pivot transform";
     public static final String LOG_DATA_FRAME_TRANSFORM_PIVOT_REDUCE_PAGE_SIZE =
-            "Search returned with out of memory error, reducing number of buckets per search from [{0}] to [{1}]";
+            "Insufficient memory for search, reducing number of buckets per search from [{0}] to [{1}]";
     public static final String LOG_DATA_FRAME_TRANSFORM_PIVOT_LOW_PAGE_SIZE_FAILURE =
-            "Search returned with out of memory error after repeated page size reductions to [{0}], unable to continue pivot, "
+            "Insufficient memory for search after repeated page size reductions to [{0}], unable to continue pivot, "
             + "please simplify job or increase heap size on data nodes.";
 
     public static final String FAILED_TO_PARSE_TRANSFORM_CHECKPOINTS =
             "Failed to parse transform checkpoints for [{0}]";
 
+
+    public static final String ID_TOO_LONG = "The id cannot contain more than {0} characters.";
+    public static final String INVALID_ID = "Invalid {0}; ''{1}'' can contain lowercase alphanumeric (a-z and 0-9), hyphens or " +
+        "underscores; must start and end with alphanumeric";
     private DataFrameMessages() {
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameNamedXContentProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameNamedXContentProvider.java
new file mode 100644
index 0000000000000..9eacfc5ff1eae
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameNamedXContentProvider.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.dataframe;
+
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.plugins.spi.NamedXContentProvider;
+import org.elasticsearch.xpack.core.dataframe.transforms.SyncConfig;
+import org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class DataFrameNamedXContentProvider implements NamedXContentProvider {
+
+    @Override
+    public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
+        return Arrays.asList(
+                new NamedXContentRegistry.Entry(SyncConfig.class,
+                        DataFrameField.TIME_BASED_SYNC,
+                        TimeSyncConfig::parse));
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/DeleteDataFrameTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/DeleteDataFrameTransformAction.java
index 715fa0f5dc78b..5f6d7dccdf2d0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/DeleteDataFrameTransformAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/DeleteDataFrameTransformAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.dataframe.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
@@ -18,7 +18,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class DeleteDataFrameTransformAction extends Action<AcknowledgedResponse> {
+public class DeleteDataFrameTransformAction extends ActionType<AcknowledgedResponse> {
 
     public static final DeleteDataFrameTransformAction INSTANCE = new DeleteDataFrameTransformAction();
     public static final String NAME = "cluster:admin/data_frame/delete";
@@ -27,18 +27,9 @@ private DeleteDataFrameTransformAction() {
         super(NAME);
     }
 
-    @Override
-    public AcknowledgedResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
-        return in -> {
-            AcknowledgedResponse response = new AcknowledgedResponse();
-            response.readFrom(in);
-            return response;
-        };
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends MasterNodeRequest<Request> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsAction.java
index 7cee5dbcbf527..afeaaaf4b2d93 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsAction.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.core.dataframe.action;
 
 import org.apache.logging.log4j.LogManager;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -28,7 +28,7 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-public class GetDataFrameTransformsAction extends Action<GetDataFrameTransformsAction.Response>{
+public class GetDataFrameTransformsAction extends ActionType<GetDataFrameTransformsAction.Response> {
 
     public static final GetDataFrameTransformsAction INSTANCE = new GetDataFrameTransformsAction();
     public static final String NAME = "cluster:monitor/data_frame/get";
@@ -40,11 +40,6 @@ private GetDataFrameTransformsAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
@@ -91,8 +86,8 @@ public static class Response extends AbstractGetResourcesResponse<DataFrameTrans
         public static final String INVALID_TRANSFORMS_DEPRECATION_WARNING = "Found [{}] invalid transforms";
         private static final ParseField INVALID_TRANSFORMS = new ParseField("invalid_transforms");
 
-        public Response(List<DataFrameTransformConfig> transformConfigs) {
-            super(new QueryPage<>(transformConfigs, transformConfigs.size(), DataFrameField.TRANSFORMS));
+        public Response(List<DataFrameTransformConfig> transformConfigs, long count) {
+            super(new QueryPage<>(transformConfigs, count, DataFrameField.TRANSFORMS));
         }
 
         public Response() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsAction.java
index 78ce0ee58c961..8c4438e09db2c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsAction.java
@@ -7,7 +7,8 @@
 package org.elasticsearch.xpack.core.dataframe.action;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.tasks.BaseTasksRequest;
@@ -21,8 +22,10 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.xpack.core.action.util.PageParams;
+import org.elasticsearch.xpack.core.action.util.QueryPage;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats;
+import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -31,7 +34,7 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-public class GetDataFrameTransformsStatsAction extends Action<GetDataFrameTransformsStatsAction.Response> {
+public class GetDataFrameTransformsStatsAction extends ActionType<GetDataFrameTransformsStatsAction.Response> {
 
     public static final GetDataFrameTransformsStatsAction INSTANCE = new GetDataFrameTransformsStatsAction();
     public static final String NAME = "cluster:monitor/data_frame/stats/get";
@@ -39,11 +42,6 @@ public GetDataFrameTransformsStatsAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
@@ -52,6 +50,7 @@ public Writeable.Reader<Response> getResponseReader() {
     public static class Request extends BaseTasksRequest<Request> {
         private final String id;
         private PageParams pageParams = PageParams.defaultParams();
+        private boolean allowNoMatch = true;
 
         public static final int MAX_SIZE_RETURN = 1000;
         // used internally to expand the queried id expression
@@ -71,6 +70,9 @@ public Request(StreamInput in) throws IOException {
             id = in.readString();
             expandedIds = Collections.unmodifiableList(in.readStringList());
             pageParams = new PageParams(in);
+            if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+                allowNoMatch = in.readBoolean();
+            }
         }
 
         @Override
@@ -100,12 +102,23 @@ public final PageParams getPageParams() {
             return pageParams;
         }
 
+        public boolean isAllowNoMatch() {
+            return allowNoMatch;
+        }
+
+        public void setAllowNoMatch(boolean allowNoMatch) {
+            this.allowNoMatch = allowNoMatch;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeString(id);
             out.writeStringCollection(expandedIds);
             pageParams.writeTo(out);
+            if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+                out.writeBoolean(allowNoMatch);
+            }
         }
 
         @Override
@@ -120,7 +133,7 @@ public ActionRequestValidationException validate() {
 
         @Override
         public int hashCode() {
-            return Objects.hash(id, pageParams);
+            return Objects.hash(id, pageParams, allowNoMatch);
         }
 
         @Override
@@ -132,37 +145,59 @@ public boolean equals(Object obj) {
                 return false;
             }
             Request other = (Request) obj;
-            return Objects.equals(id, other.id) && Objects.equals(pageParams, other.pageParams);
+            return Objects.equals(id, other.id)
+                && Objects.equals(pageParams, other.pageParams)
+                && allowNoMatch == other.allowNoMatch;
         }
     }
 
     public static class Response extends BaseTasksResponse implements ToXContentObject {
-        private List<DataFrameTransformStateAndStats> transformsStateAndStats;
+        private final QueryPage<DataFrameTransformStateAndStats> transformsStateAndStats;
 
-        public Response(List<DataFrameTransformStateAndStats> transformsStateAndStats) {
-            super(Collections.emptyList(), Collections.emptyList());
-            this.transformsStateAndStats = transformsStateAndStats;
+        public Response(List<DataFrameTransformStateAndStats> transformStateAndStats, long count) {
+            this(new QueryPage<>(transformStateAndStats, count, DataFrameField.TRANSFORMS));
         }
 
-        public Response(List<DataFrameTransformStateAndStats> transformsStateAndStats, List<TaskOperationFailure> taskFailures,
-                List<? extends ElasticsearchException> nodeFailures) {
+        public Response(List<DataFrameTransformStateAndStats> transformStateAndStats,
+                        long count,
+                        List<TaskOperationFailure> taskFailures,
+                        List<? extends ElasticsearchException> nodeFailures) {
+            this(new QueryPage<>(transformStateAndStats, count, DataFrameField.TRANSFORMS), taskFailures, nodeFailures);
+        }
+
+        private Response(QueryPage<DataFrameTransformStateAndStats> transformsStateAndStats) {
+            this(transformsStateAndStats, Collections.emptyList(), Collections.emptyList());
+        }
+
+        private Response(QueryPage<DataFrameTransformStateAndStats> transformsStateAndStats,
+                         List<TaskOperationFailure> taskFailures,
+                         List<? extends ElasticsearchException> nodeFailures) {
             super(taskFailures, nodeFailures);
-            this.transformsStateAndStats = transformsStateAndStats;
+            this.transformsStateAndStats = ExceptionsHelper.requireNonNull(transformsStateAndStats, "transformsStateAndStats");
         }
 
         public Response(StreamInput in) throws IOException {
             super(in);
-            transformsStateAndStats = in.readList(DataFrameTransformStateAndStats::new);
+            if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+                transformsStateAndStats = new QueryPage<>(in, DataFrameTransformStateAndStats::new);
+            } else {
+                List<DataFrameTransformStateAndStats> stats = in.readList(DataFrameTransformStateAndStats::new);
+                transformsStateAndStats = new QueryPage<>(stats, stats.size(), DataFrameField.TRANSFORMS);
+            }
         }
 
         public List<DataFrameTransformStateAndStats> getTransformsStateAndStats() {
-            return transformsStateAndStats;
+            return transformsStateAndStats.results();
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            out.writeList(transformsStateAndStats);
+            if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+                transformsStateAndStats.writeTo(out);
+            } else {
+                out.writeList(transformsStateAndStats.results());
+            }
         }
 
         @Override
@@ -174,8 +209,7 @@ public void readFrom(StreamInput in) {
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             toXContentCommon(builder, params);
-            builder.field(DataFrameField.COUNT.getPreferredName(), transformsStateAndStats.size());
-            builder.field(DataFrameField.TRANSFORMS.getPreferredName(), transformsStateAndStats);
+            transformsStateAndStats.doXContentBody(builder, params);
             builder.endObject();
             return builder;
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformAction.java
index 90e0adba16269..904a45042712e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformAction.java
@@ -6,7 +6,8 @@
 
 package org.elasticsearch.xpack.core.dataframe.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
@@ -24,15 +25,18 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig;
+import org.elasticsearch.xpack.core.dataframe.transforms.DestConfig;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-public class PreviewDataFrameTransformAction extends Action<PreviewDataFrameTransformAction.Response> {
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+public class PreviewDataFrameTransformAction extends ActionType<PreviewDataFrameTransformAction.Response> {
 
     public static final PreviewDataFrameTransformAction INSTANCE = new PreviewDataFrameTransformAction();
     public static final String NAME = "cluster:admin/data_frame/preview";
@@ -41,11 +45,6 @@ private PreviewDataFrameTransformAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
@@ -66,8 +65,20 @@ public Request(StreamInput in) throws IOException {
 
         public static Request fromXContent(final XContentParser parser) throws IOException {
             Map<String, Object> content = parser.map();
-            // Destination and ID are not required for Preview, so we just supply our own
-            content.put(DataFrameField.DESTINATION.getPreferredName(), Collections.singletonMap("index", "unused-transform-preview-index"));
+            // dest.index and ID are not required for Preview, so we just supply our own
+            Map<String, String> tempDestination = new HashMap<>();
+            tempDestination.put(DestConfig.INDEX.getPreferredName(), "unused-transform-preview-index");
+            // Users can still provide just dest.pipeline to preview what their data would look like given the pipeline ID
+            Object providedDestination = content.get(DataFrameField.DESTINATION.getPreferredName());
+            if (providedDestination instanceof Map) {
+                @SuppressWarnings("unchecked")
+                Map<String, String> destMap = (Map<String, String>)providedDestination;
+                String pipeline = destMap.get(DestConfig.PIPELINE.getPreferredName());
+                if (pipeline != null) {
+                    tempDestination.put(DestConfig.PIPELINE.getPreferredName(), pipeline);
+                }
+            }
+            content.put(DataFrameField.DESTINATION.getPreferredName(), tempDestination);
             content.put(DataFrameField.ID.getPreferredName(), "transform-preview");
             try(XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().map(content);
                 XContentParser newParser = XContentType.JSON
@@ -81,9 +92,16 @@ public static Request fromXContent(final XContentParser parser) throws IOExcepti
 
         @Override
         public ActionRequestValidationException validate() {
-            return null;
+            ActionRequestValidationException validationException = null;
+            if(config.getPivotConfig() != null) {
+                for(String failure : config.getPivotConfig().aggFieldValidation()) {
+                    validationException = addValidationError(failure, validationException);
+                }
+            }
+            return validationException;
         }
 
+
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             return this.config.toXContent(builder, params);
@@ -120,11 +138,14 @@ public boolean equals(Object obj) {
     public static class Response extends ActionResponse implements ToXContentObject {
 
         private List<Map<String, Object>> docs;
+        private Map<String, Object> mappings;
         public static ParseField PREVIEW = new ParseField("preview");
+        public static ParseField MAPPINGS = new ParseField("mappings");
 
         static ObjectParser<Response, Void> PARSER = new ObjectParser<>("data_frame_transform_preview", Response::new);
         static {
             PARSER.declareObjectArray(Response::setDocs, (p, c) -> p.mapOrdered(), PREVIEW);
+            PARSER.declareObject(Response::setMappings, (p, c) -> p.mapOrdered(), MAPPINGS);
         }
         public Response() {}
 
@@ -134,6 +155,10 @@ public Response(StreamInput in) throws IOException {
             for (int i = 0; i < size; i++) {
                 this.docs.add(in.readMap());
             }
+            if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+                Map<String, Object> objectMap = in.readMap();
+                this.mappings = objectMap == null ? null : Map.copyOf(objectMap);
+            }
         }
 
         public Response(List<Map<String, Object>> docs) {
@@ -144,18 +169,56 @@ public void setDocs(List<Map<String, Object>> docs) {
             this.docs = new ArrayList<>(docs);
         }
 
+        public void setMappings(Map<String, Object> mappings) {
+            this.mappings = Map.copyOf(mappings);
+        }
+
+        /**
+         * This takes the a {@code Map<String, String>} of the type "fieldname: fieldtype" and transforms it into the
+         * typical mapping format.
+         *
+         * Example:
+         *
+         * input:
+         * {"field1.subField1": "long", "field2": "keyword"}
+         *
+         * output:
+         * {
+         *     "properties": {
+         *         "field1.subField1": {
+         *             "type": "long"
+         *         },
+         *         "field2": {
+         *             "type": "keyword"
+         *         }
+         *     }
+         * }
+         * @param mappings A Map of the form {"fieldName": "fieldType"}
+         */
+        public void setMappingsFromStringMap(Map<String, String> mappings) {
+            Map<String, Object> fieldMappings = new HashMap<>();
+            mappings.forEach((k, v) -> fieldMappings.put(k, Map.of("type", v)));
+            this.mappings = Map.of("properties", fieldMappings);
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeInt(docs.size());
             for (Map<String, Object> doc : docs) {
                 out.writeMapWithConsistentOrder(doc);
             }
+            if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+                out.writeMap(mappings);
+            }
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             builder.field(PREVIEW.getPreferredName(), docs);
+            if (mappings != null) {
+                builder.field(MAPPINGS.getPreferredName(), mappings);
+            }
             builder.endObject();
             return builder;
         }
@@ -171,12 +234,12 @@ public boolean equals(Object obj) {
             }
 
             Response other = (Response) obj;
-            return Objects.equals(other.docs, docs);
+            return Objects.equals(other.docs, docs) && Objects.equals(other.mappings, mappings);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hashCode(docs);
+            return Objects.hash(docs, mappings);
         }
 
         public static Response fromXContent(final XContentParser parser) throws IOException {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/PutDataFrameTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/PutDataFrameTransformAction.java
index 2608fb87761f9..00873d7630754 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/PutDataFrameTransformAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/PutDataFrameTransformAction.java
@@ -6,23 +6,30 @@
 
 package org.elasticsearch.xpack.core.dataframe.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.indices.InvalidIndexNameException;
+import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig;
+import org.elasticsearch.xpack.core.dataframe.utils.DataFrameStrings;
+import org.elasticsearch.xpack.core.dataframe.DataFrameMessages;
 
 import java.io.IOException;
+import java.util.Locale;
 import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.cluster.metadata.MetaDataCreateIndexService.validateIndexOrAliasName;
 
-public class PutDataFrameTransformAction extends Action<AcknowledgedResponse> {
+public class PutDataFrameTransformAction extends ActionType<AcknowledgedResponse> {
 
     public static final PutDataFrameTransformAction INSTANCE = new PutDataFrameTransformAction();
     public static final String NAME = "cluster:admin/data_frame/put";
@@ -32,8 +39,8 @@ private PutDataFrameTransformAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends AcknowledgedRequest<Request> implements ToXContentObject {
@@ -67,6 +74,25 @@ public ActionRequestValidationException validate() {
             for(String failure : config.getPivotConfig().aggFieldValidation()) {
                 validationException = addValidationError(failure, validationException);
             }
+            String destIndex = config.getDestination().getIndex();
+            try {
+                validateIndexOrAliasName(destIndex, InvalidIndexNameException::new);
+                if (!destIndex.toLowerCase(Locale.ROOT).equals(destIndex)) {
+                    validationException = addValidationError("dest.index [" + destIndex +"] must be lowercase", validationException);
+                }
+            } catch (InvalidIndexNameException ex) {
+                validationException = addValidationError(ex.getMessage(), validationException);
+            }
+            if (DataFrameStrings.isValidId(config.getId()) == false) {
+                validationException = addValidationError(
+                    DataFrameMessages.getMessage(DataFrameMessages.INVALID_ID, DataFrameField.ID.getPreferredName(), config.getId()),
+                    validationException);
+            }
+            if (DataFrameStrings.hasValidLengthForId(config.getId()) == false) {
+                validationException = addValidationError(
+                    DataFrameMessages.getMessage(DataFrameMessages.ID_TOO_LONG, DataFrameStrings.ID_LENGTH_LIMIT),
+                    validationException);
+            }
             return validationException;
         }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StartDataFrameTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StartDataFrameTransformAction.java
index 6216b4489db92..2854232c543ec 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StartDataFrameTransformAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StartDataFrameTransformAction.java
@@ -6,7 +6,7 @@
 
 package org.elasticsearch.xpack.core.dataframe.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.tasks.BaseTasksResponse;
@@ -22,7 +22,7 @@
 import java.util.Collections;
 import java.util.Objects;
 
-public class StartDataFrameTransformAction extends Action<StartDataFrameTransformAction.Response> {
+public class StartDataFrameTransformAction extends ActionType<StartDataFrameTransformAction.Response> {
 
     public static final StartDataFrameTransformAction INSTANCE = new StartDataFrameTransformAction();
     public static final String NAME = "cluster:admin/data_frame/start";
@@ -31,11 +31,6 @@ private StartDataFrameTransformAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
@@ -96,33 +91,33 @@ public boolean equals(Object obj) {
     }
 
     public static class Response extends BaseTasksResponse implements ToXContentObject {
-        private final boolean started;
+        private final boolean acknowledged;
 
         public Response(StreamInput in) throws IOException {
             super(in);
-            started = in.readBoolean();
+            acknowledged = in.readBoolean();
         }
 
-        public Response(boolean started) {
+        public Response(boolean acknowledged) {
             super(Collections.emptyList(), Collections.emptyList());
-            this.started = started;
+            this.acknowledged = acknowledged;
         }
 
-        public boolean isStarted() {
-            return started;
+        public boolean isAcknowledged() {
+            return acknowledged;
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            out.writeBoolean(started);
+            out.writeBoolean(acknowledged);
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             toXContentCommon(builder, params);
-            builder.field("started", started);
+            builder.field("acknowledged", acknowledged);
             builder.endObject();
             return builder;
         }
@@ -137,12 +132,12 @@ public boolean equals(Object obj) {
                 return false;
             }
             Response response = (Response) obj;
-            return started == response.started;
+            return acknowledged == response.acknowledged;
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(started);
+            return Objects.hash(acknowledged);
         }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StartDataFrameTransformTaskAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StartDataFrameTransformTaskAction.java
index fe453200fb2bc..6d6bf8e7db70c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StartDataFrameTransformTaskAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StartDataFrameTransformTaskAction.java
@@ -6,7 +6,7 @@
 
 package org.elasticsearch.xpack.core.dataframe.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.tasks.BaseTasksRequest;
 import org.elasticsearch.action.support.tasks.BaseTasksResponse;
@@ -23,7 +23,7 @@
 import java.util.Collections;
 import java.util.Objects;
 
-public class StartDataFrameTransformTaskAction extends Action<StartDataFrameTransformTaskAction.Response> {
+public class StartDataFrameTransformTaskAction extends ActionType<StartDataFrameTransformTaskAction.Response> {
 
     public static final StartDataFrameTransformTaskAction INSTANCE = new StartDataFrameTransformTaskAction();
     public static final String NAME = "cluster:admin/data_frame/start_task";
@@ -32,11 +32,6 @@ private StartDataFrameTransformTaskAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformAction.java
index 99699c3a48cb5..de62d59c5e0c1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformAction.java
@@ -6,7 +6,8 @@
 package org.elasticsearch.xpack.core.dataframe.action;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.tasks.BaseTasksRequest;
@@ -31,7 +32,7 @@
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
-public class StopDataFrameTransformAction extends Action<StopDataFrameTransformAction.Response> {
+public class StopDataFrameTransformAction extends ActionType<StopDataFrameTransformAction.Response> {
 
     public static final StopDataFrameTransformAction INSTANCE = new StopDataFrameTransformAction();
     public static final String NAME = "cluster:admin/data_frame/stop";
@@ -42,11 +43,6 @@ private StopDataFrameTransformAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
@@ -56,15 +52,17 @@ public static class Request extends BaseTasksRequest<Request> {
         private final String id;
         private final boolean waitForCompletion;
         private final boolean force;
+        private final boolean allowNoMatch;
         private Set<String> expandedIds;
 
-        public Request(String id, boolean waitForCompletion, boolean force, @Nullable TimeValue timeout) {
+        public Request(String id, boolean waitForCompletion, boolean force, @Nullable TimeValue timeout, boolean allowNoMatch) {
             this.id = ExceptionsHelper.requireNonNull(id, DataFrameField.ID.getPreferredName());
             this.waitForCompletion = waitForCompletion;
             this.force = force;
 
             // use the timeout value already present in BaseTasksRequest
             this.setTimeout(timeout == null ? DEFAULT_TIMEOUT : timeout);
+            this.allowNoMatch = allowNoMatch;
         }
 
         public Request(StreamInput in) throws IOException {
@@ -75,6 +73,11 @@ public Request(StreamInput in) throws IOException {
             if (in.readBoolean()) {
                 expandedIds = new HashSet<>(Arrays.asList(in.readStringArray()));
             }
+            if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+                this.allowNoMatch = in.readBoolean();
+            } else {
+                this.allowNoMatch = true;
+            }
         }
 
         public String getId() {
@@ -97,6 +100,10 @@ public void setExpandedIds(Set<String> expandedIds ) {
             this.expandedIds = expandedIds;
         }
 
+        public boolean isAllowNoMatch() {
+            return allowNoMatch;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
@@ -108,6 +115,9 @@ public void writeTo(StreamOutput out) throws IOException {
             if (hasExpandedIds) {
                 out.writeStringArray(expandedIds.toArray(new String[0]));
             }
+            if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+                out.writeBoolean(allowNoMatch);
+            }
         }
 
         @Override
@@ -118,7 +128,7 @@ public ActionRequestValidationException validate() {
         @Override
         public int hashCode() {
             // the base class does not implement hashCode, therefore we need to hash timeout ourselves
-            return Objects.hash(id, waitForCompletion, force, expandedIds, this.getTimeout());
+            return Objects.hash(id, waitForCompletion, force, expandedIds, this.getTimeout(), allowNoMatch);
         }
 
         @Override
@@ -140,7 +150,8 @@ public boolean equals(Object obj) {
             return Objects.equals(id, other.id) &&
                     Objects.equals(waitForCompletion, other.waitForCompletion) &&
                     Objects.equals(force, other.force) &&
-                    Objects.equals(expandedIds, other.expandedIds);
+                    Objects.equals(expandedIds, other.expandedIds) &&
+                    allowNoMatch == other.allowNoMatch;
         }
 
         @Override
@@ -158,40 +169,40 @@ public boolean match(Task task) {
 
     public static class Response extends BaseTasksResponse implements Writeable, ToXContentObject {
 
-        private final boolean stopped;
+        private final boolean acknowledged;
 
         public Response(StreamInput in) throws IOException {
             super(in);
-            stopped = in.readBoolean();
+            acknowledged = in.readBoolean();
         }
 
-        public Response(boolean stopped) {
+        public Response(boolean acknowledged) {
             super(Collections.emptyList(), Collections.emptyList());
-            this.stopped = stopped;
+            this.acknowledged = acknowledged;
         }
 
         public Response(List<TaskOperationFailure> taskFailures,
                         List<? extends ElasticsearchException> nodeFailures,
-                        boolean stopped) {
+                        boolean acknowledged) {
             super(taskFailures, nodeFailures);
-            this.stopped = stopped;
+            this.acknowledged = acknowledged;
         }
 
-        public boolean isStopped() {
-            return stopped;
+        public boolean isAcknowledged() {
+            return acknowledged;
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            out.writeBoolean(stopped);
+            out.writeBoolean(acknowledged);
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             toXContentCommon(builder, params);
-            builder.field("stopped", stopped);
+            builder.field("acknowledged", acknowledged);
             builder.endObject();
             return builder;
         }
@@ -203,12 +214,12 @@ public boolean equals(Object o) {
             if (o == null || getClass() != o.getClass())
                 return false;
             Response response = (Response) o;
-            return stopped == response.stopped;
+            return acknowledged == response.acknowledged;
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(stopped);
+            return Objects.hash(acknowledged);
         }
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/notifications/DataFrameAuditMessage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/notifications/DataFrameAuditMessage.java
index dd6aee25580df..e0ebd8e97d9a0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/notifications/DataFrameAuditMessage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/notifications/DataFrameAuditMessage.java
@@ -11,8 +11,8 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.common.notifications.AbstractAuditMessage;
 import org.elasticsearch.xpack.core.common.notifications.Level;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
-import org.elasticsearch.xpack.core.dataframe.utils.TimeUtils;
 
 import java.util.Date;
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransform.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransform.java
index 0e4780f5ad0a8..e620e4f859543 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransform.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransform.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
@@ -22,22 +23,35 @@
 public class DataFrameTransform extends AbstractDiffable<DataFrameTransform> implements XPackPlugin.XPackPersistentTaskParams {
 
     public static final String NAME = DataFrameField.TASK_NAME;
+    public static final ParseField VERSION = new ParseField(DataFrameField.VERSION);
 
     private final String transformId;
+    private final Version version;
 
     public static final ConstructingObjectParser<DataFrameTransform, Void> PARSER = new ConstructingObjectParser<>(NAME,
-            a -> new DataFrameTransform((String) a[0]));
+            a -> new DataFrameTransform((String) a[0], (String) a[1]));
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), DataFrameField.ID);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), VERSION);
     }
 
-    public DataFrameTransform(String transformId) {
+    private DataFrameTransform(String transformId, String version) {
+        this(transformId, version == null ? null : Version.fromString(version));
+    }
+
+    public DataFrameTransform(String transformId, Version version) {
         this.transformId = transformId;
+        this.version = version == null ? Version.V_7_2_0 : version;
     }
 
     public DataFrameTransform(StreamInput in) throws IOException {
         this.transformId  = in.readString();
+        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+            this.version = Version.readVersion(in);
+        } else {
+            this.version = Version.V_7_2_0;
+        }
     }
 
     @Override
@@ -53,12 +67,16 @@ public Version getMinimalSupportedVersion() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(transformId);
+        if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+            Version.writeVersion(version, out);
+        }
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(DataFrameField.ID.getPreferredName(), transformId);
+        builder.field(VERSION.getPreferredName(), version);
         builder.endObject();
         return builder;
     }
@@ -67,6 +85,10 @@ public String getId() {
         return transformId;
     }
 
+    public Version getVersion() {
+        return version;
+    }
+
     public static DataFrameTransform fromXContent(XContentParser parser) throws IOException {
         return PARSER.parse(parser, null);
     }
@@ -83,11 +105,11 @@ public boolean equals(Object other) {
 
         DataFrameTransform that = (DataFrameTransform) other;
 
-        return Objects.equals(this.transformId, that.transformId);
+        return Objects.equals(this.transformId, that.transformId) && Objects.equals(this.version, that.version);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(transformId);
+        return Objects.hash(transformId, version);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java
index ee35fe3d21ec7..e3ad50d9b889e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java
@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.core.dataframe.transforms;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
@@ -14,15 +15,19 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.DataFrameMessages;
 import org.elasticsearch.xpack.core.dataframe.transforms.pivot.PivotConfig;
 import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.Collections;
 import java.util.Map;
 import java.util.Objects;
@@ -42,6 +47,8 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
     public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot");
 
     public static final ParseField DESCRIPTION = new ParseField("description");
+    public static final ParseField VERSION = new ParseField("version");
+    public static final ParseField CREATE_TIME = new ParseField("create_time");
     private static final ConstructingObjectParser<DataFrameTransformConfig, String> STRICT_PARSER = createParser(false);
     private static final ConstructingObjectParser<DataFrameTransformConfig, String> LENIENT_PARSER = createParser(true);
     private static final int MAX_DESCRIPTION_LENGTH = 1_000;
@@ -49,13 +56,22 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
     private final String id;
     private final SourceConfig source;
     private final DestConfig dest;
+    private final SyncConfig syncConfig;
     private final String description;
     // headers store the user context from the creating user, which allows us to run the transform as this user
     // the header only contains name, groups and other context but no authorization keys
     private Map<String, String> headers;
+    private Version transformVersion;
+    private Instant createTime;
 
     private final PivotConfig pivotConfig;
 
+    private static void validateStrictParsingParams(Object arg, String parameterName) {
+        if (arg != null) {
+            throw new IllegalArgumentException("Found [" + parameterName + "], not allowed for strict parsing");
+        }
+    }
+
     private static ConstructingObjectParser<DataFrameTransformConfig, String> createParser(boolean lenient) {
         ConstructingObjectParser<DataFrameTransformConfig, String> parser = new ConstructingObjectParser<>(NAME, lenient,
                 (args, optionalId) -> {
@@ -72,46 +88,74 @@ private static ConstructingObjectParser<DataFrameTransformConfig, String> create
                     SourceConfig source = (SourceConfig) args[1];
                     DestConfig dest = (DestConfig) args[2];
 
-                    // ignored, only for internal storage: String docType = (String) args[3];
+                    SyncConfig syncConfig = (SyncConfig) args[3];
+                    // ignored, only for internal storage: String docType = (String) args[4];
 
-                    // on strict parsing do not allow injection of headers
-                    if (lenient == false && args[4] != null) {
-                        throw new IllegalArgumentException("Found [headers], not allowed for strict parsing");
+                    // on strict parsing do not allow injection of headers, transform version, or create time
+                    if (lenient == false) {
+                        validateStrictParsingParams(args[5], HEADERS.getPreferredName());
+                        validateStrictParsingParams(args[8], CREATE_TIME.getPreferredName());
+                        validateStrictParsingParams(args[9], VERSION.getPreferredName());
                     }
 
                     @SuppressWarnings("unchecked")
-                    Map<String, String> headers = (Map<String, String>) args[4];
-
-                    PivotConfig pivotConfig = (PivotConfig) args[5];
-                    String description = (String)args[6];
-                    return new DataFrameTransformConfig(id, source, dest, headers, pivotConfig, description);
+                    Map<String, String> headers = (Map<String, String>) args[5];
+
+                    PivotConfig pivotConfig = (PivotConfig) args[6];
+                    String description = (String)args[7];
+                    return new DataFrameTransformConfig(id,
+                        source,
+                        dest,
+                        syncConfig,
+                        headers,
+                        pivotConfig,
+                        description,
+                        (Instant)args[8],
+                        (String)args[9]);
                 });
 
         parser.declareString(optionalConstructorArg(), DataFrameField.ID);
         parser.declareObject(constructorArg(), (p, c) -> SourceConfig.fromXContent(p, lenient), DataFrameField.SOURCE);
         parser.declareObject(constructorArg(), (p, c) -> DestConfig.fromXContent(p, lenient), DataFrameField.DESTINATION);
 
+        parser.declareObject(optionalConstructorArg(), (p, c) -> parseSyncConfig(p, lenient), DataFrameField.SYNC);
+
         parser.declareString(optionalConstructorArg(), DataFrameField.INDEX_DOC_TYPE);
+
         parser.declareObject(optionalConstructorArg(), (p, c) -> p.mapStrings(), HEADERS);
         parser.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p, lenient), PIVOT_TRANSFORM);
         parser.declareString(optionalConstructorArg(), DESCRIPTION);
-
+        parser.declareField(optionalConstructorArg(),
+            p -> TimeUtils.parseTimeFieldToInstant(p, CREATE_TIME.getPreferredName()), CREATE_TIME, ObjectParser.ValueType.VALUE);
+        parser.declareString(optionalConstructorArg(), VERSION);
         return parser;
     }
 
+    private static SyncConfig parseSyncConfig(XContentParser parser, boolean ignoreUnknownFields) throws IOException {
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+        SyncConfig syncConfig = parser.namedObject(SyncConfig.class, parser.currentName(), ignoreUnknownFields);
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        return syncConfig;
+    }
+
     public static String documentId(String transformId) {
         return NAME + "-" + transformId;
     }
 
-    public DataFrameTransformConfig(final String id,
-                                    final SourceConfig source,
-                                    final DestConfig dest,
-                                    final Map<String, String> headers,
-                                    final PivotConfig pivotConfig,
-                                    final String description) {
+    DataFrameTransformConfig(final String id,
+                             final SourceConfig source,
+                             final DestConfig dest,
+                             final SyncConfig syncConfig,
+                             final Map<String, String> headers,
+                             final PivotConfig pivotConfig,
+                             final String description,
+                             final Instant createTime,
+                             final String version){
         this.id = ExceptionsHelper.requireNonNull(id, DataFrameField.ID.getPreferredName());
         this.source = ExceptionsHelper.requireNonNull(source, DataFrameField.SOURCE.getPreferredName());
         this.dest = ExceptionsHelper.requireNonNull(dest, DataFrameField.DESTINATION.getPreferredName());
+        this.syncConfig = syncConfig;
         this.setHeaders(headers == null ? Collections.emptyMap() : headers);
         this.pivotConfig = pivotConfig;
         this.description = description;
@@ -123,6 +167,18 @@ public DataFrameTransformConfig(final String id,
         if (this.description != null && this.description.length() > MAX_DESCRIPTION_LENGTH) {
             throw new IllegalArgumentException("[description] must be less than 1000 characters in length.");
         }
+        this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli());
+        this.transformVersion = version == null ? null : Version.fromString(version);
+    }
+
+    public DataFrameTransformConfig(final String id,
+                                    final SourceConfig source,
+                                    final DestConfig dest,
+                                    final SyncConfig syncConfig,
+                                    final Map<String, String> headers,
+                                    final PivotConfig pivotConfig,
+                                    final String description) {
+        this(id, source, dest, syncConfig, headers, pivotConfig, description, null, null);
     }
 
     public DataFrameTransformConfig(final StreamInput in) throws IOException {
@@ -132,6 +188,15 @@ public DataFrameTransformConfig(final StreamInput in) throws IOException {
         setHeaders(in.readMap(StreamInput::readString, StreamInput::readString));
         pivotConfig = in.readOptionalWriteable(PivotConfig::new);
         description = in.readOptionalString();
+        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+            syncConfig = in.readOptionalNamedWriteable(SyncConfig.class);
+            createTime = in.readOptionalInstant();
+            transformVersion = in.readBoolean() ? Version.readVersion(in) : null;
+        } else {
+            syncConfig = null;
+            createTime = null;
+            transformVersion = null;
+        }
     }
 
     public String getId() {
@@ -146,12 +211,36 @@ public DestConfig getDestination() {
         return dest;
     }
 
+    public SyncConfig getSyncConfig() {
+        return syncConfig;
+    }
+
     public Map<String, String> getHeaders() {
         return headers;
     }
 
-    public void setHeaders(Map<String, String> headers) {
+    public DataFrameTransformConfig setHeaders(Map<String, String> headers) {
         this.headers = headers;
+        return this;
+    }
+
+    public Version getVersion() {
+        return transformVersion;
+    }
+
+    public DataFrameTransformConfig setVersion(Version transformVersion) {
+        this.transformVersion = transformVersion;
+        return this;
+    }
+
+    public Instant getCreateTime() {
+        return createTime;
+    }
+
+    public DataFrameTransformConfig setCreateTime(Instant createTime) {
+        ExceptionsHelper.requireNonNull(createTime, CREATE_TIME.getPreferredName());
+        this.createTime = Instant.ofEpochMilli(createTime.toEpochMilli());
+        return this;
     }
 
     public PivotConfig getPivotConfig() {
@@ -168,6 +257,10 @@ public boolean isValid() {
             return false;
         }
 
+        if (syncConfig != null && syncConfig.isValid() == false) {
+            return false;
+        }
+
         return source.isValid() && dest.isValid();
     }
 
@@ -179,6 +272,16 @@ public void writeTo(final StreamOutput out) throws IOException {
         out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeString);
         out.writeOptionalWriteable(pivotConfig);
         out.writeOptionalString(description);
+        if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+            out.writeOptionalNamedWriteable(syncConfig);
+            out.writeOptionalInstant(createTime);
+            if (transformVersion != null) {
+                out.writeBoolean(true);
+                Version.writeVersion(transformVersion, out);
+            } else {
+                out.writeBoolean(false);
+            }
+        }
     }
 
     @Override
@@ -187,6 +290,11 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
         builder.field(DataFrameField.ID.getPreferredName(), id);
         builder.field(DataFrameField.SOURCE.getPreferredName(), source);
         builder.field(DataFrameField.DESTINATION.getPreferredName(), dest);
+        if (syncConfig != null) {
+            builder.startObject(DataFrameField.SYNC.getPreferredName());
+            builder.field(syncConfig.getWriteableName(), syncConfig);
+            builder.endObject();
+        }
         if (pivotConfig != null) {
             builder.field(PIVOT_TRANSFORM.getPreferredName(), pivotConfig);
         }
@@ -199,6 +307,12 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
         if (description != null) {
             builder.field(DESCRIPTION.getPreferredName(), description);
         }
+        if (transformVersion != null) {
+            builder.field(VERSION.getPreferredName(), transformVersion);
+        }
+        if (createTime != null) {
+            builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli());
+        }
         builder.endObject();
         return builder;
     }
@@ -218,14 +332,17 @@ public boolean equals(Object other) {
         return Objects.equals(this.id, that.id)
                 && Objects.equals(this.source, that.source)
                 && Objects.equals(this.dest, that.dest)
+                && Objects.equals(this.syncConfig, that.syncConfig)
                 && Objects.equals(this.headers, that.headers)
                 && Objects.equals(this.pivotConfig, that.pivotConfig)
-                && Objects.equals(this.description, that.description);
+                && Objects.equals(this.description, that.description)
+                && Objects.equals(this.createTime, that.createTime)
+                && Objects.equals(this.transformVersion, that.transformVersion);
     }
 
     @Override
     public int hashCode(){
-        return Objects.hash(id, source, dest, headers, pivotConfig, description);
+        return Objects.hash(id, source, dest, syncConfig, headers, pivotConfig, description, createTime, transformVersion);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformState.java
index d4480caa0b9a4..2c3ad36d6849a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformState.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformState.java
@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.core.dataframe.transforms;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
@@ -41,6 +42,8 @@ public class DataFrameTransformState implements Task.Status, PersistentTaskState
     private final Map<String, Object> currentPosition;
     @Nullable
     private final String reason;
+    @Nullable
+    private NodeAttributes node;
 
     public static final ParseField TASK_STATE = new ParseField("task_state");
     public static final ParseField INDEXER_STATE = new ParseField("indexer_state");
@@ -48,6 +51,7 @@ public class DataFrameTransformState implements Task.Status, PersistentTaskState
     public static final ParseField CHECKPOINT = new ParseField("checkpoint");
     public static final ParseField REASON = new ParseField("reason");
     public static final ParseField PROGRESS = new ParseField("progress");
+    public static final ParseField NODE = new ParseField("node");
 
     @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<DataFrameTransformState, Void> PARSER = new ConstructingObjectParser<>(NAME,
@@ -57,7 +61,8 @@ public class DataFrameTransformState implements Task.Status, PersistentTaskState
                 (Map<String, Object>) args[2],
                 (long) args[3],
                 (String) args[4],
-                (DataFrameTransformProgress) args[5]));
+                (DataFrameTransformProgress) args[5],
+                (NodeAttributes) args[6]));
 
     static {
         PARSER.declareField(constructorArg(), p -> DataFrameTransformTaskState.fromString(p.text()), TASK_STATE, ValueType.STRING);
@@ -66,6 +71,7 @@ public class DataFrameTransformState implements Task.Status, PersistentTaskState
         PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), CHECKPOINT);
         PARSER.declareString(optionalConstructorArg(), REASON);
         PARSER.declareField(optionalConstructorArg(), DataFrameTransformProgress.PARSER::apply, PROGRESS, ValueType.OBJECT);
+        PARSER.declareField(optionalConstructorArg(), NodeAttributes.PARSER::apply, NODE, ValueType.OBJECT);
     }
 
     public DataFrameTransformState(DataFrameTransformTaskState taskState,
@@ -73,13 +79,24 @@ public DataFrameTransformState(DataFrameTransformTaskState taskState,
                                    @Nullable Map<String, Object> position,
                                    long checkpoint,
                                    @Nullable String reason,
-                                   @Nullable DataFrameTransformProgress progress) {
+                                   @Nullable DataFrameTransformProgress progress,
+                                   @Nullable NodeAttributes node) {
         this.taskState = taskState;
         this.indexerState = indexerState;
         this.currentPosition = position == null ? null : Collections.unmodifiableMap(new LinkedHashMap<>(position));
         this.checkpoint = checkpoint;
         this.reason = reason;
         this.progress = progress;
+        this.node = node;
+    }
+
+    public DataFrameTransformState(DataFrameTransformTaskState taskState,
+                                   IndexerState indexerState,
+                                   @Nullable Map<String, Object> position,
+                                   long checkpoint,
+                                   @Nullable String reason,
+                                   @Nullable DataFrameTransformProgress progress) {
+        this(taskState, indexerState, position, checkpoint, reason, progress, null);
     }
 
     public DataFrameTransformState(StreamInput in) throws IOException {
@@ -90,6 +107,11 @@ public DataFrameTransformState(StreamInput in) throws IOException {
         checkpoint = in.readLong();
         reason = in.readOptionalString();
         progress = in.readOptionalWriteable(DataFrameTransformProgress::new);
+        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+            node = in.readOptionalWriteable(NodeAttributes::new);
+        } else {
+            node = null;
+        }
     }
 
     public DataFrameTransformTaskState getTaskState() {
@@ -125,6 +147,15 @@ public String getReason() {
         return reason;
     }
 
+    public NodeAttributes getNode() {
+        return node;
+    }
+
+    public DataFrameTransformState setNode(NodeAttributes node) {
+        this.node = node;
+        return this;
+    }
+
     public static DataFrameTransformState fromXContent(XContentParser parser) {
         try {
             return PARSER.parse(parser, null);
@@ -148,6 +179,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (progress != null) {
             builder.field(PROGRESS.getPreferredName(), progress);
         }
+        if (node != null) {
+            builder.field(NODE.getPreferredName(), node);
+        }
         builder.endObject();
         return builder;
     }
@@ -165,6 +199,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeLong(checkpoint);
         out.writeOptionalString(reason);
         out.writeOptionalWriteable(progress);
+        if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+            out.writeOptionalWriteable(node);
+        }
     }
 
     @Override
@@ -184,12 +221,13 @@ public boolean equals(Object other) {
             Objects.equals(this.currentPosition, that.currentPosition) &&
             this.checkpoint == that.checkpoint &&
             Objects.equals(this.reason, that.reason) &&
-            Objects.equals(this.progress, that.progress);
+            Objects.equals(this.progress, that.progress) &&
+            Objects.equals(this.node, that.node);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(taskState, indexerState, currentPosition, checkpoint, reason, progress);
+        return Objects.hash(taskState, indexerState, currentPosition, checkpoint, reason, progress, node);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DestConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DestConfig.java
index 29b2b8c5dc0e1..00d03066b6fff 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DestConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DestConfig.java
@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.core.dataframe.transforms;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -20,10 +21,12 @@
 import java.util.Objects;
 
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 public class DestConfig implements Writeable, ToXContentObject {
 
     public static final ParseField INDEX = new ParseField("index");
+    public static final ParseField PIPELINE = new ParseField("pipeline");
 
     public static final ConstructingObjectParser<DestConfig, Void> STRICT_PARSER = createParser(false);
     public static final ConstructingObjectParser<DestConfig, Void> LENIENT_PARSER = createParser(true);
@@ -31,25 +34,37 @@ public class DestConfig implements Writeable, ToXContentObject {
     private static ConstructingObjectParser<DestConfig, Void> createParser(boolean lenient) {
         ConstructingObjectParser<DestConfig, Void> parser = new ConstructingObjectParser<>("data_frame_config_dest",
             lenient,
-            args -> new DestConfig((String)args[0]));
+            args -> new DestConfig((String)args[0], (String) args[1]));
         parser.declareString(constructorArg(), INDEX);
+        parser.declareString(optionalConstructorArg(), PIPELINE);
         return parser;
     }
 
     private final String index;
+    private final String pipeline;
 
-    public DestConfig(String index) {
+    public DestConfig(String index, String pipeline) {
         this.index = ExceptionsHelper.requireNonNull(index, INDEX.getPreferredName());
+        this.pipeline = pipeline;
     }
 
     public DestConfig(final StreamInput in) throws IOException {
         index = in.readString();
+        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+            pipeline = in.readOptionalString();
+        } else {
+            pipeline = null;
+        }
     }
 
     public String getIndex() {
         return index;
     }
 
+    public String getPipeline() {
+        return pipeline;
+    }
+
     public boolean isValid() {
         return index.isEmpty() == false;
     }
@@ -57,12 +72,18 @@ public boolean isValid() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(index);
+        if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+            out.writeOptionalString(pipeline);
+        }
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(INDEX.getPreferredName(), index);
+        if (pipeline != null) {
+            builder.field(PIPELINE.getPreferredName(), pipeline);
+        }
         builder.endObject();
         return builder;
     }
@@ -77,12 +98,13 @@ public boolean equals(Object other) {
         }
 
         DestConfig that = (DestConfig) other;
-        return Objects.equals(index, that.index);
+        return Objects.equals(index, that.index) &&
+            Objects.equals(pipeline, that.pipeline);
     }
 
     @Override
     public int hashCode(){
-        return Objects.hash(index);
+        return Objects.hash(index, pipeline);
     }
 
     public static DestConfig fromXContent(final XContentParser parser, boolean lenient) throws IOException {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/NodeAttributes.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/NodeAttributes.java
new file mode 100644
index 0000000000000..76b40bd23778a
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/NodeAttributes.java
@@ -0,0 +1,171 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.dataframe.transforms;
+
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * A Pojo class containing an Elastic Node's attributes
+ */
+public class NodeAttributes implements ToXContentObject, Writeable {
+
+    public static final ParseField ID = new ParseField("id");
+    public static final ParseField NAME = new ParseField("name");
+    public static final ParseField EPHEMERAL_ID = new ParseField("ephemeral_id");
+    public static final ParseField TRANSPORT_ADDRESS = new ParseField("transport_address");
+    public static final ParseField ATTRIBUTES = new ParseField("attributes");
+
+    @SuppressWarnings("unchecked")
+    public static final ConstructingObjectParser<NodeAttributes, Void> PARSER =
+        new ConstructingObjectParser<>("node", true,
+            (a) -> {
+                int i = 0;
+                String id = (String) a[i++];
+                String name = (String) a[i++];
+                String ephemeralId = (String) a[i++];
+                String transportAddress = (String) a[i++];
+                Map<String, String> attributes = (Map<String, String>) a[i];
+                return new NodeAttributes(id, name, ephemeralId, transportAddress, attributes);
+            });
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), ID);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), EPHEMERAL_ID);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), TRANSPORT_ADDRESS);
+        PARSER.declareField(ConstructingObjectParser.constructorArg(),
+            (p, c) -> p.mapStrings(),
+            ATTRIBUTES,
+            ObjectParser.ValueType.OBJECT);
+    }
+
+    private final String id;
+    private final String name;
+    private final String ephemeralId;
+    private final String transportAddress;
+    private final Map<String, String> attributes;
+
+    public static NodeAttributes fromDiscoveryNode(DiscoveryNode node) {
+        return new NodeAttributes(node.getId(),
+            node.getName(),
+            node.getEphemeralId(),
+            node.getAddress().toString(),
+            // TODO add data_frame attributes when/if they are added
+            Collections.emptyMap());
+    }
+
+    public NodeAttributes(String id, String name, String ephemeralId, String transportAddress, Map<String, String> attributes) {
+        this.id = ExceptionsHelper.requireNonNull(id, ID.getPreferredName());
+        this.name = ExceptionsHelper.requireNonNull(name, NAME.getPreferredName());
+        this.ephemeralId = ExceptionsHelper.requireNonNull(ephemeralId, EPHEMERAL_ID.getPreferredName());
+        this.transportAddress = ExceptionsHelper.requireNonNull(transportAddress, TRANSPORT_ADDRESS.getPreferredName());
+        this.attributes = Collections.unmodifiableMap(ExceptionsHelper.requireNonNull(attributes, ATTRIBUTES.getPreferredName()));
+    }
+
+    public NodeAttributes(StreamInput in) throws IOException {
+        this.id = in.readString();
+        this.name = in.readString();
+        this.ephemeralId = in.readString();
+        this.transportAddress = in.readString();
+        this.attributes = Collections.unmodifiableMap(in.readMap(StreamInput::readString, StreamInput::readString));
+    }
+
+    /**
+     * The unique identifier of the node.
+     */
+    public String getId() {
+        return id;
+    }
+
+    /**
+     * The node name.
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * The ephemeral id of the node.
+     */
+    public String getEphemeralId() {
+        return ephemeralId;
+    }
+
+    /**
+     * The host and port where transport HTTP connections are accepted.
+     */
+    public String getTransportAddress() {
+        return transportAddress;
+    }
+
+    public Map<String, String> getAttributes() {
+        return attributes;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(ID.getPreferredName(), id);
+        builder.field(NAME.getPreferredName(), name);
+        builder.field(EPHEMERAL_ID.getPreferredName(), ephemeralId);
+        builder.field(TRANSPORT_ADDRESS.getPreferredName(), transportAddress);
+        builder.field(ATTRIBUTES.getPreferredName(), attributes);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, name, ephemeralId, transportAddress, attributes);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        NodeAttributes that = (NodeAttributes) other;
+        return Objects.equals(id, that.id) &&
+            Objects.equals(name, that.name) &&
+            Objects.equals(ephemeralId, that.ephemeralId) &&
+            Objects.equals(transportAddress, that.transportAddress) &&
+            Objects.equals(attributes, that.attributes);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(id);
+        out.writeString(name);
+        out.writeString(ephemeralId);
+        out.writeString(transportAddress);
+        out.writeMap(attributes, StreamOutput::writeString, StreamOutput::writeString);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/SyncConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/SyncConfig.java
new file mode 100644
index 0000000000000..19ff79ea7e0ee
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/SyncConfig.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.dataframe.transforms;
+
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.index.query.QueryBuilder;
+
+public interface SyncConfig extends ToXContentObject, NamedWriteable {
+
+    /**
+     * Validate configuration
+     *
+     * @return true if valid
+     */
+    boolean isValid();
+
+    QueryBuilder getRangeQuery(DataFrameTransformCheckpoint newCheckpoint);
+
+    QueryBuilder getRangeQuery(DataFrameTransformCheckpoint oldCheckpoint, DataFrameTransformCheckpoint newCheckpoint);
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/TimeSyncConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/TimeSyncConfig.java
new file mode 100644
index 0000000000000..0490394d90b26
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/TimeSyncConfig.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.dataframe.transforms;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.RangeQueryBuilder;
+import org.elasticsearch.xpack.core.dataframe.DataFrameField;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+public class TimeSyncConfig  implements SyncConfig {
+
+    private static final String NAME = "data_frame_transform_pivot_sync_time";
+
+    private final String field;
+    private final TimeValue delay;
+
+    private static final ConstructingObjectParser<TimeSyncConfig, Void> STRICT_PARSER = createParser(false);
+    private static final ConstructingObjectParser<TimeSyncConfig, Void> LENIENT_PARSER = createParser(true);
+
+    private static ConstructingObjectParser<TimeSyncConfig, Void> createParser(boolean lenient) {
+        ConstructingObjectParser<TimeSyncConfig, Void> parser = new ConstructingObjectParser<>(NAME, lenient,
+                args -> {
+                    String field = (String) args[0];
+                    TimeValue delay = args[1] != null ? (TimeValue) args[1] : TimeValue.ZERO;
+
+                    return new TimeSyncConfig(field, delay);
+                    });
+
+        parser.declareString(constructorArg(), DataFrameField.FIELD);
+        parser.declareField(optionalConstructorArg(),
+                (p, c) -> TimeValue.parseTimeValue(p.textOrNull(), DataFrameField.DELAY.getPreferredName()), DataFrameField.DELAY,
+                ObjectParser.ValueType.STRING_OR_NULL);
+
+                    return parser;
+                }
+
+    public TimeSyncConfig() {
+        this(null, null);
+    }
+
+    public TimeSyncConfig(final String field, final TimeValue delay) {
+        this.field = ExceptionsHelper.requireNonNull(field, DataFrameField.FIELD.getPreferredName());
+        this.delay = ExceptionsHelper.requireNonNull(delay, DataFrameField.DELAY.getPreferredName());
+    }
+
+    public TimeSyncConfig(StreamInput in) throws IOException {
+        this.field = in.readString();
+        this.delay = in.readTimeValue();
+    }
+
+    public String getField() {
+        return field;
+    }
+
+    public TimeValue getDelay() {
+        return delay;
+    }
+
+    @Override
+    public boolean isValid() {
+        return true;
+    }
+
+    @Override
+    public void writeTo(final StreamOutput out) throws IOException {
+        out.writeString(field);
+        out.writeTimeValue(delay);
+    }
+
+    @Override
+    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+        builder.startObject();
+        builder.field(DataFrameField.FIELD.getPreferredName(), field);
+        if (delay.duration() > 0) {
+            builder.field(DataFrameField.DELAY.getPreferredName(), delay.getStringRep());
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        final TimeSyncConfig that = (TimeSyncConfig) other;
+
+        return Objects.equals(this.field, that.field)
+                && Objects.equals(this.delay, that.delay);
+    }
+
+    @Override
+    public int hashCode(){
+        return Objects.hash(field, delay);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true, true);
+    }
+
+    public static TimeSyncConfig parse(final XContentParser parser) {
+        return LENIENT_PARSER.apply(parser, null);
+    }
+
+    public static TimeSyncConfig fromXContent(final XContentParser parser, boolean lenient) throws IOException {
+        return lenient ? LENIENT_PARSER.apply(parser, null) : STRICT_PARSER.apply(parser, null);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return DataFrameField.TIME_BASED_SYNC.getPreferredName();
+    }
+
+    @Override
+    public QueryBuilder getRangeQuery(DataFrameTransformCheckpoint newCheckpoint) {
+        return new RangeQueryBuilder(field).lt(newCheckpoint.getTimeUpperBound()).format("epoch_millis");
+    }
+
+    @Override
+    public QueryBuilder getRangeQuery(DataFrameTransformCheckpoint oldCheckpoint, DataFrameTransformCheckpoint newCheckpoint) {
+        return new RangeQueryBuilder(field).gte(oldCheckpoint.getTimeUpperBound()).lt(newCheckpoint.getTimeUpperBound())
+                .format("epoch_millis");
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/DateHistogramGroupSource.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/DateHistogramGroupSource.java
index a3861ef65f648..3b856f454c4f5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/DateHistogramGroupSource.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/DateHistogramGroupSource.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.dataframe.transforms.pivot;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -14,6 +15,7 @@
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 
@@ -21,6 +23,7 @@
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.util.Objects;
+import java.util.Set;
 
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
@@ -185,13 +188,11 @@ private void writeInterval(Interval interval, StreamOutput out) throws IOExcepti
 
     private static final String NAME = "data_frame_date_histogram_group";
     private static final ParseField TIME_ZONE = new ParseField("time_zone");
-    private static final ParseField FORMAT = new ParseField("format");
 
     private static final ConstructingObjectParser<DateHistogramGroupSource, Void> STRICT_PARSER = createParser(false);
     private static final ConstructingObjectParser<DateHistogramGroupSource, Void> LENIENT_PARSER = createParser(true);
 
     private final Interval interval;
-    private String format;
     private ZoneId timeZone;
 
     public DateHistogramGroupSource(String field, Interval interval) {
@@ -203,7 +204,10 @@ public DateHistogramGroupSource(StreamInput in) throws IOException {
         super(in);
         this.interval = readInterval(in);
         this.timeZone = in.readOptionalZoneId();
-        this.format = in.readOptionalString();
+        // Format was optional in 7.2.x, removed in 7.3+
+        if (in.getVersion().before(Version.V_7_3_0)) {
+            in.readOptionalString();
+        }
     }
 
     private static ConstructingObjectParser<DateHistogramGroupSource, Void> createParser(boolean lenient) {
@@ -240,7 +244,6 @@ private static ConstructingObjectParser<DateHistogramGroupSource, Void> createPa
             }
         }, TIME_ZONE, ObjectParser.ValueType.LONG);
 
-        parser.declareString(DateHistogramGroupSource::setFormat, FORMAT);
         return parser;
     }
 
@@ -257,14 +260,6 @@ public Interval getInterval() {
         return interval;
     }
 
-    public String getFormat() {
-        return format;
-    }
-
-    public void setFormat(String format) {
-        this.format = format;
-    }
-
     public ZoneId getTimeZone() {
         return timeZone;
     }
@@ -278,7 +273,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(field);
         writeInterval(interval, out);
         out.writeOptionalZoneId(timeZone);
-        out.writeOptionalString(format);
+        // Format was optional in 7.2.x, removed in 7.3+
+        if (out.getVersion().before(Version.V_7_3_0)) {
+            out.writeOptionalString(null);
+        }
     }
 
     @Override
@@ -291,9 +289,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (timeZone != null) {
             builder.field(TIME_ZONE.getPreferredName(), timeZone.toString());
         }
-        if (format != null) {
-            builder.field(FORMAT.getPreferredName(), format);
-        }
         builder.endObject();
         return builder;
     }
@@ -312,12 +307,22 @@ public boolean equals(Object other) {
 
         return Objects.equals(this.field, that.field) &&
             Objects.equals(interval, that.interval) &&
-            Objects.equals(timeZone, that.timeZone) &&
-            Objects.equals(format, that.format);
+            Objects.equals(timeZone, that.timeZone);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(field, interval, timeZone, format);
+        return Objects.hash(field, interval, timeZone);
+    }
+
+    @Override
+    public QueryBuilder getIncrementalBucketUpdateFilterQuery(Set<String> changedBuckets) {
+        // no need for an extra range filter as this is already done by checkpoints
+        return null;
+    }
+
+    @Override
+    public boolean supportsIncrementalBucketUpdate() {
+        return false;
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/HistogramGroupSource.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/HistogramGroupSource.java
index 737590a0cc197..372f4ad99b608 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/HistogramGroupSource.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/HistogramGroupSource.java
@@ -11,9 +11,11 @@
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryBuilder;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Set;
 
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
@@ -99,4 +101,15 @@ public boolean equals(Object other) {
     public int hashCode() {
         return Objects.hash(field, interval);
     }
+
+    @Override
+    public QueryBuilder getIncrementalBucketUpdateFilterQuery(Set<String> changedBuckets) {
+        // histograms are simple and cheap, so we skip this optimization
+        return null;
+    }
+
+    @Override
+    public boolean supportsIncrementalBucketUpdate() {
+        return false;
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/PivotConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/PivotConfig.java
index ab2f7d489ac9a..038299bfd8326 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/PivotConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/PivotConfig.java
@@ -100,12 +100,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    public void toCompositeAggXContent(XContentBuilder builder, Params params) throws IOException {
+    public void toCompositeAggXContent(XContentBuilder builder, boolean forChangeDetection) throws IOException {
         builder.startObject();
         builder.field(CompositeAggregationBuilder.SOURCES_FIELD_NAME.getPreferredName());
         builder.startArray();
 
         for (Entry<String, SingleGroupSource> groupBy : groups.getGroups().entrySet()) {
+            // some group source do not implement change detection or not makes no sense, skip those
+            if (forChangeDetection && groupBy.getValue().supportsIncrementalBucketUpdate() == false) {
+                continue;
+            }
             builder.startObject();
             builder.startObject(groupBy.getKey());
             builder.field(groupBy.getValue().getType().value(), groupBy.getValue());
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/SingleGroupSource.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/SingleGroupSource.java
index 0a4cf2579460e..ff1f9c3d54ac8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/SingleGroupSource.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/SingleGroupSource.java
@@ -14,10 +14,12 @@
 import org.elasticsearch.common.xcontent.AbstractObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
 
 import java.io.IOException;
 import java.util.Locale;
 import java.util.Objects;
+import java.util.Set;
 
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
@@ -94,6 +96,10 @@ public void writeTo(StreamOutput out) throws IOException {
 
     public abstract Type getType();
 
+    public abstract boolean supportsIncrementalBucketUpdate();
+
+    public abstract QueryBuilder getIncrementalBucketUpdateFilterQuery(Set<String> changedBuckets);
+
     public String getField() {
         return field;
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/TermsGroupSource.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/TermsGroupSource.java
index d4585a611b367..891b160da0762 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/TermsGroupSource.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/TermsGroupSource.java
@@ -9,8 +9,11 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.TermsQueryBuilder;
 
 import java.io.IOException;
+import java.util.Set;
 
 /*
  * A terms aggregation source for group_by
@@ -47,4 +50,14 @@ public Type getType() {
     public static TermsGroupSource fromXContent(final XContentParser parser, boolean lenient) throws IOException {
         return lenient ? LENIENT_PARSER.apply(parser, null) : STRICT_PARSER.apply(parser, null);
     }
+
+    @Override
+    public QueryBuilder getIncrementalBucketUpdateFilterQuery(Set<String> changedBuckets) {
+        return new TermsQueryBuilder(field, changedBuckets);
+    }
+
+    @Override
+    public boolean supportsIncrementalBucketUpdate() {
+        return true;
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/utils/DataFrameStrings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/utils/DataFrameStrings.java
new file mode 100644
index 0000000000000..22dd3526a24c1
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/utils/DataFrameStrings.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.dataframe.utils;
+
+import org.elasticsearch.cluster.metadata.MetaData;
+
+import java.util.regex.Pattern;
+
+/**
+ * Yet Another String utilities class.
+ */
+public final class DataFrameStrings {
+
+    /**
+     * Valid user id pattern.
+     * Matches a string that contains lowercase characters, digits, hyphens, underscores or dots.
+     * The string may start and end only in characters or digits.
+     * Note that '.' is allowed but not documented.
+     */
+    private static final Pattern VALID_ID_CHAR_PATTERN = Pattern.compile("[a-z0-9](?:[a-z0-9_\\-\\.]*[a-z0-9])?");
+
+    public static final int ID_LENGTH_LIMIT = 64;
+
+    private DataFrameStrings() {
+    }
+
+    public static boolean isValidId(String id) {
+        return id != null && VALID_ID_CHAR_PATTERN.matcher(id).matches() && !MetaData.ALL.equals(id);
+    }
+
+    /**
+     * Checks if the given {@code id} has a valid length.
+     * We keep IDs in a length shorter or equal than {@link #ID_LENGTH_LIMIT}
+     * in order to avoid unfriendly errors when storing docs with
+     * more than 512 bytes.
+     *
+     * @param id the id
+     * @return {@code true} if the id has a valid length
+     */
+    public static boolean hasValidLengthForId(String id) {
+        return id.length() <= ID_LENGTH_LIMIT;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/utils/TimeUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/utils/TimeUtils.java
deleted file mode 100644
index 4cf3859380cd8..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/utils/TimeUtils.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.dataframe.utils;
-
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.mapper.DateFieldMapper;
-
-import java.io.IOException;
-import java.util.Date;
-import java.util.concurrent.TimeUnit;
-
-public final class TimeUtils {
-
-    private TimeUtils() {
-        // Do nothing
-    }
-
-    public static Date parseTimeField(XContentParser parser, String fieldName) throws IOException {
-        if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
-            return new Date(parser.longValue());
-        } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
-            return new Date(TimeUtils.dateStringToEpoch(parser.text()));
-        }
-        throw new IllegalArgumentException(
-                "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]");
-    }
-
-    /**
-     * First tries to parse the date first as a Long and convert that to an
-     * epoch time. If the long number has more than 10 digits it is considered a
-     * time in milliseconds else if 10 or less digits it is in seconds. If that
-     * fails it tries to parse the string using
-     * {@link DateFieldMapper#DEFAULT_DATE_TIME_FORMATTER}
-     *
-     * If the date string cannot be parsed -1 is returned.
-     *
-     * @return The epoch time in milliseconds or -1 if the date cannot be
-     *         parsed.
-     */
-    public static long dateStringToEpoch(String date) {
-        try {
-            long epoch = Long.parseLong(date);
-            if (date.trim().length() <= 10) { // seconds
-                return epoch * 1000;
-            } else {
-                return epoch;
-            }
-        } catch (NumberFormatException nfe) {
-            // not a number
-        }
-
-        try {
-            return DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(date);
-        } catch (ElasticsearchParseException | IllegalArgumentException e) {
-        }
-        // Could not do the conversion
-        return -1;
-    }
-
-    /**
-     * Checks that the given {@code timeValue} is a non-negative multiple value of the {@code baseUnit}.
-     *
-     * <ul>
-     *   <li>400ms is valid for base unit of seconds</li>
-     *   <li>450ms is invalid for base unit of seconds but valid for base unit of milliseconds</li>
-     * </ul>
-     */
-    public static void checkNonNegativeMultiple(TimeValue timeValue, TimeUnit baseUnit, ParseField field) {
-        checkNonNegative(timeValue, field);
-        checkMultiple(timeValue, baseUnit, field);
-    }
-
-    /**
-     * Checks that the given {@code timeValue} is a positive multiple value of the {@code baseUnit}.
-     *
-     * <ul>
-     *   <li>400ms is valid for base unit of seconds</li>
-     *   <li>450ms is invalid for base unit of seconds but valid for base unit of milliseconds</li>
-     * </ul>
-     */
-    public static void checkPositiveMultiple(TimeValue timeValue, TimeUnit baseUnit, ParseField field) {
-        checkPositive(timeValue, field);
-        checkMultiple(timeValue, baseUnit, field);
-    }
-
-    /**
-     * Checks that the given {@code timeValue} is positive.
-     *
-     * <ul>
-     *   <li>1s is valid</li>
-     *   <li>-1s is invalid</li>
-     * </ul>
-     */
-    public static void checkPositive(TimeValue timeValue, ParseField field) {
-        long nanos = timeValue.getNanos();
-        if (nanos <= 0) {
-            throw new IllegalArgumentException(field.getPreferredName() + " cannot be less or equal than 0. Value = "
-                    + timeValue.toString());
-        }
-    }
-
-    private static void checkNonNegative(TimeValue timeValue, ParseField field) {
-        long nanos = timeValue.getNanos();
-        if (nanos < 0) {
-            throw new IllegalArgumentException(field.getPreferredName() + " cannot be less than 0. Value = " + timeValue.toString());
-        }
-    }
-
-
-
-    /**
-     * Check the given {@code timeValue} is a multiple of the {@code baseUnit}
-     */
-    public static void checkMultiple(TimeValue timeValue, TimeUnit baseUnit, ParseField field) {
-        long nanos = timeValue.getNanos();
-        TimeValue base = new TimeValue(1, baseUnit);
-        long baseNanos = base.getNanos();
-        if (nanos % baseNanos != 0) {
-            throw new IllegalArgumentException(field.getPreferredName() + " has to be a multiple of " + base.toString() + "; actual was '"
-                    + timeValue.toString() + "'");
-        }
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java
index 54d260e32532f..1409b13f0a044 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.core.deprecation;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
@@ -39,7 +39,7 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-public class DeprecationInfoAction extends Action<DeprecationInfoAction.Response> {
+public class DeprecationInfoAction extends StreamableResponseActionType<DeprecationInfoAction.Response> {
 
     public static final DeprecationInfoAction INSTANCE = new DeprecationInfoAction();
     public static final String NAME = "cluster:admin/xpack/deprecation/info";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckAction.java
index db0b0a0603de5..b650da55fc0b0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckAction.java
@@ -6,7 +6,8 @@
 
 package org.elasticsearch.xpack.core.deprecation;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder;
@@ -23,7 +24,7 @@
  * Runs deprecation checks on each node. Deprecation checks are performed locally so that filtered settings
  * can be accessed in the deprecation checks.
  */
-public class NodesDeprecationCheckAction extends Action<NodesDeprecationCheckResponse> {
+public class NodesDeprecationCheckAction extends StreamableResponseActionType<NodesDeprecationCheckResponse> {
     public static final NodesDeprecationCheckAction INSTANCE = new NodesDeprecationCheckAction();
     public static final String NAME = "cluster:admin/xpack/deprecation/nodes/info";
 
@@ -41,8 +42,7 @@ public static class NodeRequest extends BaseNodeRequest {
         NodesDeprecationCheckRequest request;
 
         public NodeRequest() {}
-        public NodeRequest(String nodeId, NodesDeprecationCheckRequest request) {
-            super(nodeId);
+        public NodeRequest(NodesDeprecationCheckRequest request) {
             this.request = request;
         }
 
@@ -113,7 +113,7 @@ public static class RequestBuilder extends NodesOperationRequestBuilder<NodesDep
         NodesDeprecationCheckResponse, RequestBuilder> {
 
         protected RequestBuilder(ElasticsearchClient client,
-                                 Action<NodesDeprecationCheckResponse> action,
+                                 ActionType<NodesDeprecationCheckResponse> action,
                                  NodesDeprecationCheckRequest request) {
             super(client, action, request);
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java
new file mode 100644
index 0000000000000..fe1a9bdadfdc1
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.flattened;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
+
+import java.io.IOException;
+
+public class FlattenedFeatureSetUsage extends XPackFeatureSet.Usage {
+
+    public FlattenedFeatureSetUsage(StreamInput input) throws IOException {
+        super(input);
+    }
+
+    public FlattenedFeatureSetUsage(boolean available, boolean enabled) {
+        super(XPackField.FLATTENED, available, enabled);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/graph/action/GraphExploreAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/graph/action/GraphExploreAction.java
index e4fd8d0435106..4b1cebedd664e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/graph/action/GraphExploreAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/graph/action/GraphExploreAction.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.core.graph.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.protocol.xpack.graph.GraphExploreResponse;
 
-public class GraphExploreAction extends Action<GraphExploreResponse> {
+public class GraphExploreAction extends StreamableResponseActionType<GraphExploreResponse> {
 
     public static final GraphExploreAction INSTANCE = new GraphExploreAction();
     public static final String NAME = "indices:data/read/xpack/graph/explore";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java
index 80b0378ae35ff..efe57f44e89dc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexer.java
@@ -6,7 +6,8 @@
 
 package org.elasticsearch.xpack.core.indexing;
 
-import org.apache.log4j.Logger;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
@@ -35,7 +36,7 @@
  * @param <JobPosition> Type that defines a job position to be defined by the implementation.
  */
 public abstract class AsyncTwoPhaseIndexer<JobPosition, JobStats extends IndexerJobStats> {
-    private static final Logger logger = Logger.getLogger(AsyncTwoPhaseIndexer.class.getName());
+    private static final Logger logger = LogManager.getLogger(AsyncTwoPhaseIndexer.class.getName());
 
     private final JobStats stats;
 
@@ -89,28 +90,21 @@ public synchronized IndexerState start() {
      * Sets the internal state to {@link IndexerState#STOPPING} if an async job is
      * running in the background, {@link #onStop()} will be called when the background job
      * detects that the indexer is stopped.
-     * If there is no job running when this function is called
-     * the state is set to {@link IndexerState#STOPPED} and {@link #onStop()} called directly.
+     * If there is no job running when this function is called the returned
+     * state is {@link IndexerState#STOPPED} and {@link #onStop()} will not be called.
      *
      * @return The new state for the indexer (STOPPED, STOPPING or ABORTING if the job was already aborted).
      */
     public synchronized IndexerState stop() {
-        AtomicBoolean wasStartedAndSetStopped = new AtomicBoolean(false);
-        IndexerState currentState = state.updateAndGet(previousState -> {
+        return state.updateAndGet(previousState -> {
             if (previousState == IndexerState.INDEXING) {
                 return IndexerState.STOPPING;
             } else if (previousState == IndexerState.STARTED) {
-                wasStartedAndSetStopped.set(true);
                 return IndexerState.STOPPED;
             } else {
                 return previousState;
             }
         });
-
-        if (wasStartedAndSetStopped.get()) {
-            onStop();
-        }
-        return currentState;
     }
 
     /**
@@ -287,20 +281,24 @@ private void finishWithIndexingFailure(Exception exc) {
     }
 
     private IndexerState finishAndSetState() {
-        return state.updateAndGet(prev -> {
+        AtomicBoolean callOnStop = new AtomicBoolean(false);
+        AtomicBoolean callOnAbort = new AtomicBoolean(false);
+        IndexerState updatedState = state.updateAndGet(prev -> {
+            callOnAbort.set(false);
+            callOnStop.set(false);
             switch (prev) {
             case INDEXING:
                 // ready for another job
                 return IndexerState.STARTED;
 
             case STOPPING:
+                callOnStop.set(true);
                 // must be started again
-                onStop();
                 return IndexerState.STOPPED;
 
             case ABORTING:
+                callOnAbort.set(true);
                 // abort and exit
-                onAbort();
                 return IndexerState.ABORTING; // This shouldn't matter, since onAbort() will kill the task first
 
             case STOPPED:
@@ -315,6 +313,14 @@ private IndexerState finishAndSetState() {
                 throw new IllegalStateException("Indexer job encountered an illegal state [" + prev + "]");
             }
         });
+
+        if (callOnStop.get()) {
+            onStop();
+        } else if (callOnAbort.get()) {
+            onAbort();
+        }
+
+        return updatedState;
     }
 
     private void onSearchResponse(SearchResponse searchResponse) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java
index 2ef9d242d9ef4..72134b0553b98 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.core.indexing;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -61,14 +60,12 @@ public IndexerJobStats(StreamInput in) throws IOException {
         this.numInputDocuments = in.readVLong();
         this.numOuputDocuments = in.readVLong();
         this.numInvocations = in.readVLong();
-        if (in.getVersion().onOrAfter(Version.V_6_6_0)) {
-            this.indexTime = in.readVLong();
-            this.searchTime = in.readVLong();
-            this.indexTotal = in.readVLong();
-            this.searchTotal = in.readVLong();
-            this.indexFailures = in.readVLong();
-            this.searchFailures = in.readVLong();
-        }
+        this.indexTime = in.readVLong();
+        this.searchTime = in.readVLong();
+        this.indexTotal = in.readVLong();
+        this.searchTotal = in.readVLong();
+        this.indexFailures = in.readVLong();
+        this.searchFailures = in.readVLong();
     }
 
     public long getNumPages() {
@@ -163,14 +160,12 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(numInputDocuments);
         out.writeVLong(numOuputDocuments);
         out.writeVLong(numInvocations);
-        if (out.getVersion().onOrAfter(Version.V_6_6_0)) {
-            out.writeVLong(indexTime);
-            out.writeVLong(searchTime);
-            out.writeVLong(indexTotal);
-            out.writeVLong(searchTotal);
-            out.writeVLong(indexFailures);
-            out.writeVLong(searchFailures);
-        }
+        out.writeVLong(indexTime);
+        out.writeVLong(searchTime);
+        out.writeVLong(indexTotal);
+        out.writeVLong(searchTotal);
+        out.writeVLong(indexFailures);
+        out.writeVLong(searchFailures);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleMetadata.java
index c0cafa8e9079e..161033601ee3f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleMetadata.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/IndexLifecycleMetadata.java
@@ -106,7 +106,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public Version getMinimalSupportedVersion() {
-        return Version.V_6_6_0;
+        return Version.CURRENT.minimumCompatibilityVersion();
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStep.java
index d37d8c0a18805..5db9eb9e90155 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStep.java
@@ -5,10 +5,15 @@
  */
 package org.elasticsearch.xpack.core.indexlifecycle;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.segments.IndexSegments;
 import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest;
+import org.elasticsearch.action.admin.indices.segments.ShardSegments;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
@@ -17,13 +22,17 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
 import java.util.Objects;
-import java.util.stream.StreamSupport;
+import java.util.stream.Collectors;
 
 /**
  * This {@link Step} evaluates whether force_merge was successful by checking the segment count.
  */
 public class SegmentCountStep extends AsyncWaitStep {
+
+    private static final Logger logger = LogManager.getLogger(SegmentCountStep.class);
     public static final String NAME = "segment-count";
 
     private final int maxNumSegments;
@@ -41,10 +50,19 @@ public int getMaxNumSegments() {
     public void evaluateCondition(IndexMetaData indexMetaData, Listener listener) {
         getClient().admin().indices().segments(new IndicesSegmentsRequest(indexMetaData.getIndex().getName()),
             ActionListener.wrap(response -> {
-                long numberShardsLeftToMerge =
-                    StreamSupport.stream(response.getIndices().get(indexMetaData.getIndex().getName()).spliterator(), false)
-                        .filter(iss -> Arrays.stream(iss.getShards()).anyMatch(p -> p.getSegments().size() > maxNumSegments)).count();
-                listener.onResponse(numberShardsLeftToMerge == 0, new Info(numberShardsLeftToMerge));
+                IndexSegments segments = response.getIndices().get(indexMetaData.getIndex().getName());
+                List<ShardSegments> unmergedShards = segments.getShards().values().stream()
+                    .flatMap(iss -> Arrays.stream(iss.getShards()))
+                    .filter(shardSegments -> shardSegments.getSegments().size() > maxNumSegments)
+                    .collect(Collectors.toList());
+                if (unmergedShards.size() > 0) {
+                    Map<ShardRouting, Integer> unmergedShardCounts = unmergedShards.stream()
+                        .collect(Collectors.toMap(ShardSegments::getShardRouting, ss -> ss.getSegments().size()));
+                    logger.info("[{}] best effort force merge to [{}] segments did not succeed for {} shards: {}",
+                        indexMetaData.getIndex().getName(), maxNumSegments, unmergedShards.size(), unmergedShardCounts);
+                }
+                // Force merging is best effort, so always return true that the condition has been met.
+                listener.onResponse(true, new Info(unmergedShards.size()));
             }, listener::onFailure));
     }
 
@@ -90,8 +108,12 @@ public long getNumberShardsLeftToMerge() {
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
-            builder.field(MESSAGE.getPreferredName(),
-                    "Waiting for [" + numberShardsLeftToMerge + "] shards " + "to forcemerge");
+            if (numberShardsLeftToMerge == 0) {
+                builder.field(MESSAGE.getPreferredName(), "all shards force merged successfully");
+            } else {
+                builder.field(MESSAGE.getPreferredName(),
+                    "[" + numberShardsLeftToMerge + "] shards did not successfully force merge");
+            }
             builder.field(SHARDS_TO_MERGE.getPreferredName(), numberShardsLeftToMerge);
             builder.endObject();
             return builder;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/SetSingleNodeAllocateStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/SetSingleNodeAllocateStep.java
index 7973289b968d5..5966e3a2981c5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/SetSingleNodeAllocateStep.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/SetSingleNodeAllocateStep.java
@@ -5,38 +5,50 @@
  */
 package org.elasticsearch.xpack.core.indexlifecycle;
 
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.shard.ShardId;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
 /**
  * Allocates all shards in a single index to one node.
  * For example, as preparation for shrinking that index.
  */
 public class SetSingleNodeAllocateStep extends AsyncActionStep {
+    private static final Logger logger = LogManager.getLogger(SetSingleNodeAllocateStep.class);
     public static final String NAME = "set-single-node-allocation";
 
-    private static final AllocationDeciders ALLOCATION_DECIDERS = new AllocationDeciders(Collections.singletonList(
-            new FilterAllocationDecider(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))));
+    // These allocation deciders were chosen because these are the conditions that can prevent
+    // allocation long-term, and that we can inspect in advance. Most other allocation deciders
+    // will either only delay relocation (e.g. ThrottlingAllocationDecider), or don't work very
+    // well when reallocating potentially many shards at once (e.g. DiskThresholdDecider)
+    private static final AllocationDeciders ALLOCATION_DECIDERS = new AllocationDeciders(List.of(
+        new FilterAllocationDecider(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
+        new NodeVersionAllocationDecider()
+    ));
 
     public SetSingleNodeAllocateStep(StepKey key, StepKey nextStepKey, Client client) {
         super(key, nextStepKey, client);
@@ -44,18 +56,24 @@ public SetSingleNodeAllocateStep(StepKey key, StepKey nextStepKey, Client client
 
     @Override
     public void performAction(IndexMetaData indexMetaData, ClusterState clusterState, ClusterStateObserver observer, Listener listener) {
-        RoutingAllocation allocation = new RoutingAllocation(ALLOCATION_DECIDERS, clusterState.getRoutingNodes(), clusterState, null,
+        final RoutingNodes routingNodes = clusterState.getRoutingNodes();
+        RoutingAllocation allocation = new RoutingAllocation(ALLOCATION_DECIDERS, routingNodes, clusterState, null,
                 System.nanoTime());
         List<String> validNodeIds = new ArrayList<>();
-        Optional<ShardRouting> anyShard = clusterState.getRoutingTable().allShards(indexMetaData.getIndex().getName()).stream().findAny();
-        if (anyShard.isPresent()) {
-            // Iterate through the nodes finding ones that are acceptable for the current allocation rules of the shard
-            for (RoutingNode node : clusterState.getRoutingNodes()) {
-                boolean canRemainOnCurrentNode = ALLOCATION_DECIDERS.canRemain(anyShard.get(), node, allocation)
-                        .type() == Decision.Type.YES;
-                if (canRemainOnCurrentNode) {
-                    DiscoveryNode discoveryNode = node.node();
-                    validNodeIds.add(discoveryNode.getId());
+        final Map<ShardId, List<ShardRouting>> routingsByShardId = clusterState.getRoutingTable()
+            .allShards(indexMetaData.getIndex().getName())
+            .stream()
+            .collect(Collectors.groupingBy(ShardRouting::shardId));
+
+
+        if (routingsByShardId.isEmpty() == false) {
+            for (RoutingNode node : routingNodes) {
+                boolean canAllocateOneCopyOfEachShard = routingsByShardId.values().stream() // For each shard
+                    .allMatch(shardRoutings -> shardRoutings.stream() // Can we allocate at least one shard copy to this node?
+                        .map(shardRouting -> ALLOCATION_DECIDERS.canAllocate(shardRouting, node, allocation).type())
+                        .anyMatch(Decision.Type.YES::equals));
+                if (canAllocateOneCopyOfEachShard) {
+                    validNodeIds.add(node.node().getId());
                 }
             }
             // Shuffle the list of nodes so the one we pick is random
@@ -70,6 +88,7 @@ public void performAction(IndexMetaData indexMetaData, ClusterState clusterState
                         ActionListener.wrap(response -> listener.onResponse(true), listener::onFailure));
             } else {
                 // No nodes currently match the allocation rules so just wait until there is one that does
+                logger.debug("could not find any nodes to allocate index [{}] onto prior to shrink");
                 listener.onResponse(false);
             }
         } else {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleAction.java
index ba962d635ee6e..f9f5893bcb0cc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleAction.java
@@ -6,19 +6,20 @@
 
 package org.elasticsearch.xpack.core.indexlifecycle.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 
 import java.io.IOException;
 import java.util.Objects;
 
-public class DeleteLifecycleAction extends Action<DeleteLifecycleAction.Response> {
+public class DeleteLifecycleAction extends ActionType<DeleteLifecycleAction.Response> {
     public static final DeleteLifecycleAction INSTANCE = new DeleteLifecycleAction();
     public static final String NAME = "cluster:admin/ilm/delete";
 
@@ -27,13 +28,14 @@ protected DeleteLifecycleAction() {
     }
 
     @Override
-    public Response newResponse() {
-        return new Response();
+    public Writeable.Reader<Response> getResponseReader() {
+        return Response::new;
     }
 
     public static class Response extends AcknowledgedResponse implements ToXContentObject {
 
-        public Response() {
+        public Response(StreamInput in) throws IOException {
+            super(in);
         }
 
         public Response(boolean acknowledged) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/ExplainLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/ExplainLifecycleAction.java
index 5acbbcb4967f3..bb53f42040cc3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/ExplainLifecycleAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/ExplainLifecycleAction.java
@@ -6,10 +6,10 @@
 
 package org.elasticsearch.xpack.core.indexlifecycle.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleResponse;
 
-public class ExplainLifecycleAction extends Action<ExplainLifecycleResponse> {
+public class ExplainLifecycleAction extends StreamableResponseActionType<ExplainLifecycleResponse> {
     public static final ExplainLifecycleAction INSTANCE = new ExplainLifecycleAction();
     public static final String NAME = "indices:admin/ilm/explain";
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleAction.java
index aaa295354a850..b34889f5f1351 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetLifecycleAction.java
@@ -6,9 +6,9 @@
 
 package org.elasticsearch.xpack.core.indexlifecycle.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -23,7 +23,7 @@
 import java.util.List;
 import java.util.Objects;
 
-public class GetLifecycleAction extends Action<GetLifecycleAction.Response> {
+public class GetLifecycleAction extends StreamableResponseActionType<GetLifecycleAction.Response> {
     public static final GetLifecycleAction INSTANCE = new GetLifecycleAction();
     public static final String NAME = "cluster:admin/ilm/get";
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetStatusAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetStatusAction.java
index 40765f0aa6650..eaa3e9a468faf 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetStatusAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/GetStatusAction.java
@@ -6,9 +6,9 @@
 
 package org.elasticsearch.xpack.core.indexlifecycle.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -20,7 +20,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class GetStatusAction extends Action<GetStatusAction.Response> {
+public class GetStatusAction extends StreamableResponseActionType<GetStatusAction.Response> {
     public static final GetStatusAction INSTANCE = new GetStatusAction();
     public static final String NAME = "cluster:admin/ilm/operation_mode/get";
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepAction.java
index e92502d4ef526..8e2f04c66fc24 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepAction.java
@@ -6,7 +6,7 @@
  */
 package org.elasticsearch.xpack.core.indexlifecycle.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -23,7 +24,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class MoveToStepAction extends Action<MoveToStepAction.Response> {
+public class MoveToStepAction extends ActionType<MoveToStepAction.Response> {
     public static final MoveToStepAction INSTANCE = new MoveToStepAction();
     public static final String NAME = "cluster:admin/ilm/_move/post";
 
@@ -32,13 +33,14 @@ protected MoveToStepAction() {
     }
 
     @Override
-    public Response newResponse() {
-        return new Response();
+    public Writeable.Reader<Response> getResponseReader() {
+        return Response::new;
     }
 
     public static class Response extends AcknowledgedResponse implements ToXContentObject {
 
-        public Response() {
+        public Response(StreamInput in) throws IOException {
+            super(in);
         }
 
         public Response(boolean acknowledged) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleAction.java
index 09e4d978b6094..df46356ac20e2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.indexlifecycle.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -22,7 +23,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class PutLifecycleAction extends Action<PutLifecycleAction.Response> {
+public class PutLifecycleAction extends ActionType<PutLifecycleAction.Response> {
     public static final PutLifecycleAction INSTANCE = new PutLifecycleAction();
     public static final String NAME = "cluster:admin/ilm/put";
 
@@ -31,13 +32,14 @@ protected PutLifecycleAction() {
     }
 
     @Override
-    public Response newResponse() {
-        return new Response();
+    public Writeable.Reader<Response> getResponseReader() {
+        return Response::new;
     }
 
     public static class Response extends AcknowledgedResponse implements ToXContentObject {
 
-        public Response() {
+        public Response(StreamInput in) throws IOException {
+            super(in);
         }
 
         public Response(boolean acknowledged) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RemoveIndexLifecyclePolicyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RemoveIndexLifecyclePolicyAction.java
index 0e530baa57f0c..53be7b7f79a44 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RemoveIndexLifecyclePolicyAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RemoveIndexLifecyclePolicyAction.java
@@ -6,10 +6,10 @@
 
 package org.elasticsearch.xpack.core.indexlifecycle.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.ParseField;
@@ -24,7 +24,7 @@
 import java.util.List;
 import java.util.Objects;
 
-public class RemoveIndexLifecyclePolicyAction extends Action<RemoveIndexLifecyclePolicyAction.Response> {
+public class RemoveIndexLifecyclePolicyAction extends StreamableResponseActionType<RemoveIndexLifecyclePolicyAction.Response> {
     public static final RemoveIndexLifecyclePolicyAction INSTANCE = new RemoveIndexLifecyclePolicyAction();
     public static final String NAME = "indices:admin/ilm/remove_policy";
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryAction.java
index 6af7f2de6f0b7..f9bcb3474fbec 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryAction.java
@@ -6,7 +6,7 @@
 
 package org.elasticsearch.xpack.core.indexlifecycle.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -15,13 +15,14 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Objects;
 
-public class RetryAction extends Action<RetryAction.Response> {
+public class RetryAction extends ActionType<RetryAction.Response> {
     public static final RetryAction INSTANCE = new RetryAction();
     public static final String NAME = "indices:admin/ilm/retry";
 
@@ -30,13 +31,14 @@ protected RetryAction() {
     }
 
     @Override
-    public Response newResponse() {
-        return new Response();
+    public Writeable.Reader<Response> getResponseReader() {
+        return Response::new;
     }
 
     public static class Response extends AcknowledgedResponse implements ToXContentObject {
 
-        public Response() {
+        public Response(StreamInput in) throws IOException {
+            super(in);
         }
 
         public Response(boolean acknowledged) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StartILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StartILMAction.java
index 5d2f066d60b15..2145c5cd53d36 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StartILMAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StartILMAction.java
@@ -6,10 +6,11 @@
 
 package org.elasticsearch.xpack.core.indexlifecycle.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class StartILMAction extends Action<AcknowledgedResponse> {
+public class StartILMAction extends ActionType<AcknowledgedResponse> {
     public static final StartILMAction INSTANCE = new StartILMAction();
     public static final String NAME = "cluster:admin/ilm/start";
 
@@ -18,8 +19,8 @@ protected StartILMAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StopILMAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StopILMAction.java
index 63adeabb30f90..09228322fce58 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StopILMAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/action/StopILMAction.java
@@ -6,10 +6,11 @@
 
 package org.elasticsearch.xpack.core.indexlifecycle.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
-public class StopILMAction extends Action<AcknowledgedResponse> {
+public class StopILMAction extends ActionType<AcknowledgedResponse> {
     public static final StopILMAction INSTANCE = new StopILMAction();
     public static final String NAME = "cluster:admin/ilm/stop";
 
@@ -18,8 +19,8 @@ protected StopILMAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/client/ILMClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/client/ILMClient.java
deleted file mode 100644
index 5e81d2e2aa7e5..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/client/ILMClient.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.core.indexlifecycle.client;
-
-import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleRequest;
-import org.elasticsearch.xpack.core.indexlifecycle.ExplainLifecycleResponse;
-import org.elasticsearch.xpack.core.indexlifecycle.StartILMRequest;
-import org.elasticsearch.xpack.core.indexlifecycle.StopILMRequest;
-import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction;
-import org.elasticsearch.xpack.core.indexlifecycle.action.ExplainLifecycleAction;
-import org.elasticsearch.xpack.core.indexlifecycle.action.GetLifecycleAction;
-import org.elasticsearch.xpack.core.indexlifecycle.action.GetStatusAction;
-import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction;
-import org.elasticsearch.xpack.core.indexlifecycle.action.RemoveIndexLifecyclePolicyAction;
-import org.elasticsearch.xpack.core.indexlifecycle.action.RetryAction;
-import org.elasticsearch.xpack.core.indexlifecycle.action.StartILMAction;
-import org.elasticsearch.xpack.core.indexlifecycle.action.StopILMAction;
-
-/**
- * A wrapper to elasticsearch clients that exposes all ILM related APIs
- */
-public class ILMClient {
-
-    private ElasticsearchClient client;
-
-    public ILMClient(ElasticsearchClient client) {
-        this.client = client;
-    }
-
-    /**
-     * Create or modify a lifecycle policy definition
-     */
-    public void putLifecyclePolicy(PutLifecycleAction.Request request, ActionListener<PutLifecycleAction.Response> listener) {
-        client.execute(PutLifecycleAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Create or modify a lifecycle policy definition
-     */
-    public ActionFuture<PutLifecycleAction.Response> putLifecyclePolicy(PutLifecycleAction.Request request) {
-        return client.execute(PutLifecycleAction.INSTANCE, request);
-    }
-
-    /**
-     * Get a lifecycle policy definition
-     */
-    public void getLifecyclePolicy(GetLifecycleAction.Request request, ActionListener<GetLifecycleAction.Response> listener) {
-        client.execute(GetLifecycleAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Get a lifecycle policy definition
-     */
-    public ActionFuture<GetLifecycleAction.Response> getLifecyclePolicy(GetLifecycleAction.Request request) {
-        return client.execute(GetLifecycleAction.INSTANCE, request);
-    }
-
-    /**
-     * Delete a lifecycle policy definition
-     */
-    public void deleteLifecyclePolicy(DeleteLifecycleAction.Request request, ActionListener<DeleteLifecycleAction.Response> listener) {
-        client.execute(DeleteLifecycleAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Delete a lifecycle policy definition
-     */
-    public ActionFuture<DeleteLifecycleAction.Response> deleteLifecyclePolicy(DeleteLifecycleAction.Request request) {
-        return client.execute(DeleteLifecycleAction.INSTANCE, request);
-    }
-
-    /**
-     * Explain the current lifecycle state for an index
-     */
-    public void explainLifecycle(ExplainLifecycleRequest request, ActionListener<ExplainLifecycleResponse> listener) {
-        client.execute(ExplainLifecycleAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Explain the current lifecycle state for an index
-     */
-    public ActionFuture<ExplainLifecycleResponse> explainLifecycle(ExplainLifecycleRequest request) {
-        return client.execute(ExplainLifecycleAction.INSTANCE, request);
-    }
-
-    /**
-     * Returns the current status of the ILM plugin
-     */
-    public void getStatus(GetStatusAction.Request request, ActionListener<GetStatusAction.Response> listener) {
-        client.execute(GetStatusAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Returns the current status of the ILM plugin
-     */
-    public ActionFuture<GetStatusAction.Response> getStatus(GetStatusAction.Request request) {
-        return client.execute(GetStatusAction.INSTANCE, request);
-    }
-
-    /**
-     * Removes index lifecycle management from an index
-     */
-    public void removeIndexLifecyclePolicy(RemoveIndexLifecyclePolicyAction.Request request,
-            ActionListener<RemoveIndexLifecyclePolicyAction.Response> listener) {
-        client.execute(RemoveIndexLifecyclePolicyAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Removes index lifecycle management from an index
-     */
-    public ActionFuture<RemoveIndexLifecyclePolicyAction.Response> removeIndexLifecyclePolicy(
-            RemoveIndexLifecyclePolicyAction.Request request) {
-        return client.execute(RemoveIndexLifecyclePolicyAction.INSTANCE, request);
-    }
-
-    /**
-     * Retries the policy for an index which is currently in ERROR
-     */
-    public void retryPolicy(RetryAction.Request request, ActionListener<RetryAction.Response> listener) {
-        client.execute(RetryAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Removes index lifecycle management from an index
-     */
-    public ActionFuture<RetryAction.Response> retryPolicy(RetryAction.Request request) {
-        return client.execute(RetryAction.INSTANCE, request);
-    }
-
-    /**
-     * Starts the ILM plugin
-     */
-    public void startILM(StartILMRequest request, ActionListener<AcknowledgedResponse> listener) {
-        client.execute(StartILMAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Starts the ILM plugin
-     */
-    public ActionFuture<AcknowledgedResponse> startILM(StartILMRequest request) {
-        return client.execute(StartILMAction.INSTANCE, request);
-    }
-
-    /**
-     * Stops the ILM plugin
-     */
-    public void stopILM(StopILMRequest request, ActionListener<AcknowledgedResponse> listener) {
-        client.execute(StopILMAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Stops the ILM plugin
-     */
-    public ActionFuture<AcknowledgedResponse> stopILM(StopILMRequest request) {
-        return client.execute(StopILMAction.INSTANCE, request);
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java
index 6b5ba086c6fe0..5c3da41df7349 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningField.java
@@ -13,7 +13,7 @@ public final class MachineLearningField {
     public static final Setting<Boolean> AUTODETECT_PROCESS =
             Setting.boolSetting("xpack.ml.autodetect_process", true, Setting.Property.NodeScope);
     public static final Setting<ByteSizeValue> MAX_MODEL_MEMORY_LIMIT =
-            Setting.memorySizeSetting("xpack.ml.max_model_memory_limit", new ByteSizeValue(0),
+            Setting.memorySizeSetting("xpack.ml.max_model_memory_limit", ByteSizeValue.ZERO,
                     Setting.Property.Dynamic, Setting.Property.NodeScope);
     public static final TimeValue STATE_PERSIST_RESTORE_TIMEOUT = TimeValue.timeValueMinutes(30);
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java
index cd32505a48e3e..9ac63f026b089 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java
@@ -11,6 +11,8 @@
 import org.elasticsearch.persistent.PersistentTasksClusterService;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
 import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
 
@@ -23,9 +25,11 @@ public final class MlTasks {
 
     public static final String JOB_TASK_NAME = "xpack/ml/job";
     public static final String DATAFEED_TASK_NAME = "xpack/ml/datafeed";
+    public static final String DATA_FRAME_ANALYTICS_TASK_NAME = "xpack/ml/data_frame/analytics";
 
     public static final String JOB_TASK_ID_PREFIX = "job-";
     public static final String DATAFEED_TASK_ID_PREFIX = "datafeed-";
+    public static final String DATA_FRAME_ANALYTICS_TASK_ID_PREFIX = "data_frame_analytics-";
 
     public static final PersistentTasksCustomMetaData.Assignment AWAITING_UPGRADE =
         new PersistentTasksCustomMetaData.Assignment(null,
@@ -50,6 +54,17 @@ public static String datafeedTaskId(String datafeedId) {
         return DATAFEED_TASK_ID_PREFIX + datafeedId;
     }
 
+    /**
+     * Namespaces the task ids for data frame analytics.
+     */
+    public static String dataFrameAnalyticsTaskId(String id) {
+        return DATA_FRAME_ANALYTICS_TASK_ID_PREFIX + id;
+    }
+
+    public static String dataFrameAnalyticsIdFromTaskId(String taskId) {
+        return taskId.replaceFirst(DATA_FRAME_ANALYTICS_TASK_ID_PREFIX, "");
+    }
+
     @Nullable
     public static PersistentTasksCustomMetaData.PersistentTask<?> getJobTask(String jobId, @Nullable PersistentTasksCustomMetaData tasks) {
         return tasks == null ? null : tasks.getTask(jobTaskId(jobId));
@@ -61,6 +76,12 @@ public static PersistentTasksCustomMetaData.PersistentTask<?> getDatafeedTask(St
         return tasks == null ? null : tasks.getTask(datafeedTaskId(datafeedId));
     }
 
+    @Nullable
+    public static PersistentTasksCustomMetaData.PersistentTask<?> getDataFrameAnalyticsTask(String analyticsId,
+                                                                                            @Nullable PersistentTasksCustomMetaData tasks) {
+        return tasks == null ? null : tasks.getTask(dataFrameAnalyticsTaskId(analyticsId));
+    }
+
     /**
      * Note that the return value of this method does NOT take node relocations into account.
      * Use {@link #getJobStateModifiedForReassignments} to return a value adjusted to the most
@@ -120,6 +141,16 @@ public static DatafeedState getDatafeedState(String datafeedId, @Nullable Persis
         }
     }
 
+    public static DataFrameAnalyticsState getDataFrameAnalyticsState(String analyticsId, @Nullable PersistentTasksCustomMetaData tasks) {
+        PersistentTasksCustomMetaData.PersistentTask<?> task = getDataFrameAnalyticsTask(analyticsId, tasks);
+        if (task != null && task.getState() != null) {
+            DataFrameAnalyticsTaskState taskState = (DataFrameAnalyticsTaskState) task.getState();
+            return taskState.getState();
+        } else {
+            return DataFrameAnalyticsState.STOPPED;
+        }
+    }
+
     /**
      * The job Ids of anomaly detector job tasks.
      * All anomaly detector jobs are returned regardless of the status of the
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java
index 95ec597bb9cd8..1ee538bad6ce5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/CloseJobAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.support.tasks.BaseTasksRequest;
 import org.elasticsearch.action.support.tasks.BaseTasksResponse;
@@ -26,7 +26,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class CloseJobAction extends Action<CloseJobAction.Response> {
+public class CloseJobAction extends ActionType<CloseJobAction.Response> {
 
     public static final CloseJobAction INSTANCE = new CloseJobAction();
     public static final String NAME = "cluster:admin/xpack/ml/job/close";
@@ -35,11 +35,6 @@ private CloseJobAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteCalendarAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteCalendarAction.java
index da0756157b0ba..044f13001e739 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteCalendarAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteCalendarAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
@@ -13,13 +13,14 @@
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.xpack.core.ml.calendars.Calendar;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Objects;
 
-public class DeleteCalendarAction extends Action<AcknowledgedResponse> {
+public class DeleteCalendarAction extends ActionType<AcknowledgedResponse> {
 
     public static final DeleteCalendarAction INSTANCE = new DeleteCalendarAction();
     public static final String NAME = "cluster:admin/xpack/ml/calendars/delete";
@@ -29,8 +30,8 @@ private DeleteCalendarAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends AcknowledgedRequest<Request> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteCalendarEventAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteCalendarEventAction.java
index 3e64a54293924..a39a0cec3c9ab 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteCalendarEventAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteCalendarEventAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
@@ -13,6 +13,7 @@
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.xpack.core.ml.calendars.Calendar;
 import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
@@ -20,7 +21,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class DeleteCalendarEventAction extends Action<AcknowledgedResponse> {
+public class DeleteCalendarEventAction extends ActionType<AcknowledgedResponse> {
 
     public static final DeleteCalendarEventAction INSTANCE = new DeleteCalendarEventAction();
     public static final String NAME = "cluster:admin/xpack/ml/calendars/events/delete";
@@ -30,8 +31,8 @@ private DeleteCalendarEventAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends AcknowledgedRequest<Request> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteDataFrameAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteDataFrameAnalyticsAction.java
new file mode 100644
index 0000000000000..7aea3fc4298e4
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteDataFrameAnalyticsAction.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class DeleteDataFrameAnalyticsAction extends ActionType<AcknowledgedResponse> {
+
+    public static final DeleteDataFrameAnalyticsAction INSTANCE = new DeleteDataFrameAnalyticsAction();
+    public static final String NAME = "cluster:admin/xpack/ml/data_frame/analytics/delete";
+
+    private DeleteDataFrameAnalyticsAction() {
+        super(NAME);
+    }
+
+    @Override
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
+    }
+
+    public static class Request extends AcknowledgedRequest<Request> implements ToXContentFragment {
+
+        private String id;
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            id = in.readString();
+        }
+
+        public Request() {}
+
+        public Request(String id) {
+            this.id = ExceptionsHelper.requireNonNull(id, DataFrameAnalyticsConfig.ID);
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field(DataFrameAnalyticsConfig.ID.getPreferredName(), id);
+            return builder;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            DeleteDataFrameAnalyticsAction.Request request = (DeleteDataFrameAnalyticsAction.Request) o;
+            return Objects.equals(id, request.id);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(id);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(id);
+        }
+    }
+
+    public static class RequestBuilder extends MasterNodeOperationRequestBuilder<Request, AcknowledgedResponse, RequestBuilder> {
+
+        protected RequestBuilder(ElasticsearchClient client, DeleteDataFrameAnalyticsAction action) {
+            super(client, action, new Request());
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteDatafeedAction.java
index 73cdbeef44259..34640a4b5a8e1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteDatafeedAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteDatafeedAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
@@ -22,7 +23,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class DeleteDatafeedAction extends Action<AcknowledgedResponse> {
+public class DeleteDatafeedAction extends ActionType<AcknowledgedResponse> {
 
     public static final DeleteDatafeedAction INSTANCE = new DeleteDatafeedAction();
     public static final String NAME = "cluster:admin/xpack/ml/datafeeds/delete";
@@ -32,8 +33,8 @@ private DeleteDatafeedAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends AcknowledgedRequest<Request> implements ToXContentFragment {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteExpiredDataAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteExpiredDataAction.java
index 271d8ad5fa33c..bde9494ff7d54 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteExpiredDataAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteExpiredDataAction.java
@@ -5,11 +5,11 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -20,7 +20,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class DeleteExpiredDataAction extends Action<DeleteExpiredDataAction.Response> {
+public class DeleteExpiredDataAction extends StreamableResponseActionType<DeleteExpiredDataAction.Response> {
 
     public static final DeleteExpiredDataAction INSTANCE = new DeleteExpiredDataAction();
     public static final String NAME = "cluster:admin/xpack/ml/delete_expired_data";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteFilterAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteFilterAction.java
index 3870ad79555ca..ff591b97d0d5a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteFilterAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteFilterAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -14,13 +14,14 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Objects;
 
 
-public class DeleteFilterAction extends Action<AcknowledgedResponse> {
+public class DeleteFilterAction extends ActionType<AcknowledgedResponse> {
 
     public static final DeleteFilterAction INSTANCE = new DeleteFilterAction();
     public static final String NAME = "cluster:admin/xpack/ml/filters/delete";
@@ -30,8 +31,8 @@ private DeleteFilterAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends AcknowledgedRequest<Request> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteForecastAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteForecastAction.java
index b9f981ae98026..98dcc43ac5a99 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteForecastAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteForecastAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
@@ -13,13 +13,14 @@
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
 
-public class DeleteForecastAction extends Action<AcknowledgedResponse> {
+public class DeleteForecastAction extends ActionType<AcknowledgedResponse> {
 
     public static final DeleteForecastAction INSTANCE = new DeleteForecastAction();
     public static final String NAME = "cluster:admin/xpack/ml/job/forecast/delete";
@@ -29,8 +30,8 @@ private DeleteForecastAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends AcknowledgedRequest<Request> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteJobAction.java
index 6b279e0852183..14a8df580b83b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteJobAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -13,6 +13,7 @@
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
@@ -23,7 +24,7 @@
 import java.util.Map;
 import java.util.Objects;
 
-public class DeleteJobAction extends Action<AcknowledgedResponse> {
+public class DeleteJobAction extends ActionType<AcknowledgedResponse> {
 
     public static final DeleteJobAction INSTANCE = new DeleteJobAction();
     public static final String NAME = "cluster:admin/xpack/ml/job/delete";
@@ -33,8 +34,8 @@ private DeleteJobAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends AcknowledgedRequest<Request> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteModelSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteModelSnapshotAction.java
index 4bc079b387c0a..c97739ac77ee8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteModelSnapshotAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteModelSnapshotAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
@@ -13,13 +13,14 @@
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshotField;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
 import java.io.IOException;
 
-public class DeleteModelSnapshotAction extends Action<AcknowledgedResponse> {
+public class DeleteModelSnapshotAction extends ActionType<AcknowledgedResponse> {
 
     public static final DeleteModelSnapshotAction INSTANCE = new DeleteModelSnapshotAction();
     public static final String NAME = "cluster:admin/xpack/ml/job/model_snapshots/delete";
@@ -29,8 +30,8 @@ private DeleteModelSnapshotAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends ActionRequest {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/EvaluateDataFrameAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/EvaluateDataFrameAction.java
new file mode 100644
index 0000000000000..3ce82cbfc8675
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/EvaluateDataFrameAction.java
@@ -0,0 +1,215 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.Evaluation;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class EvaluateDataFrameAction extends StreamableResponseActionType<EvaluateDataFrameAction.Response> {
+
+    public static final EvaluateDataFrameAction INSTANCE = new EvaluateDataFrameAction();
+    public static final String NAME = "cluster:monitor/xpack/ml/data_frame/evaluate";
+
+    private EvaluateDataFrameAction() {
+        super(NAME);
+    }
+
+    @Override
+    public Response newResponse() {
+        return new Response();
+    }
+
+    public static class Request extends ActionRequest implements ToXContentObject {
+
+        private static final ParseField INDEX = new ParseField("index");
+        private static final ParseField EVALUATION = new ParseField("evaluation");
+
+        private static final ConstructingObjectParser<Request, Void> PARSER = new ConstructingObjectParser<>(NAME,
+            a -> new Request((List<String>) a[0], (Evaluation) a[1]));
+
+        static {
+            PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INDEX);
+            PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> parseEvaluation(p), EVALUATION);
+        }
+
+        private static Evaluation parseEvaluation(XContentParser parser) throws IOException {
+            XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+            XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+            Evaluation evaluation = parser.namedObject(Evaluation.class, parser.currentName(), null);
+            XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            return evaluation;
+        }
+
+        public static Request parseRequest(XContentParser parser) {
+            return PARSER.apply(parser, null);
+        }
+
+        private String[] indices;
+        private Evaluation evaluation;
+
+        private Request(List<String> indices, Evaluation evaluation) {
+            setIndices(indices);
+            setEvaluation(evaluation);
+        }
+
+        public Request() {
+        }
+
+        public String[] getIndices() {
+            return indices;
+        }
+
+        public final void setIndices(List<String> indices) {
+            ExceptionsHelper.requireNonNull(indices, INDEX);
+            if (indices.isEmpty()) {
+                throw ExceptionsHelper.badRequestException("At least one index must be specified");
+            }
+            this.indices = indices.toArray(new String[indices.size()]);
+        }
+
+        public Evaluation getEvaluation() {
+            return evaluation;
+        }
+
+        public final void setEvaluation(Evaluation evaluation) {
+            this.evaluation = ExceptionsHelper.requireNonNull(evaluation, EVALUATION);
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            indices = in.readStringArray();
+            evaluation = in.readNamedWriteable(Evaluation.class);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeStringArray(indices);
+            out.writeNamedWriteable(evaluation);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.array(INDEX.getPreferredName(), indices);
+            builder.startObject(EVALUATION.getPreferredName());
+            builder.field(evaluation.getName(), evaluation);
+            builder.endObject();
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(Arrays.hashCode(indices), evaluation);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Request that = (Request) o;
+            return Arrays.equals(indices, that.indices) && Objects.equals(evaluation, that.evaluation);
+        }
+    }
+
+    static class RequestBuilder extends ActionRequestBuilder<Request, Response> {
+
+        RequestBuilder(ElasticsearchClient client) {
+            super(client, INSTANCE, new Request());
+        }
+    }
+
+    public static class Response extends ActionResponse implements ToXContentObject {
+
+        private String evaluationName;
+        private List<EvaluationMetricResult> metrics;
+
+        public Response() {
+        }
+
+        public Response(String evaluationName, List<EvaluationMetricResult> metrics) {
+            this.evaluationName = Objects.requireNonNull(evaluationName);
+            this.metrics = Objects.requireNonNull(metrics);
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            this.evaluationName = in.readString();
+            this.metrics = in.readNamedWriteableList(EvaluationMetricResult.class);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(evaluationName);
+            out.writeList(metrics);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.startObject(evaluationName);
+            for (EvaluationMetricResult metric : metrics) {
+                builder.field(metric.getName(), metric);
+            }
+            builder.endObject();
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(evaluationName, metrics);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            Response other = (Response) obj;
+            return Objects.equals(evaluationName, other.evaluationName) && Objects.equals(metrics, other.metrics);
+        }
+
+        @Override
+        public final String toString() {
+            return Strings.toString(this);
+        }
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FinalizeJobExecutionAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FinalizeJobExecutionAction.java
index 9121ce1b75d9f..e007ca42602a7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FinalizeJobExecutionAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FinalizeJobExecutionAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
@@ -13,10 +13,11 @@
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 
 import java.io.IOException;
 
-public class FinalizeJobExecutionAction extends Action<AcknowledgedResponse> {
+public class FinalizeJobExecutionAction extends ActionType<AcknowledgedResponse> {
 
     public static final FinalizeJobExecutionAction INSTANCE = new FinalizeJobExecutionAction();
     public static final String NAME = "cluster:internal/xpack/ml/job/finalize_job_execution";
@@ -26,8 +27,8 @@ private FinalizeJobExecutionAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends MasterNodeRequest<Request> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FindFileStructureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FindFileStructureAction.java
index f4eb7fbc4e53b..ecfcaa4af05f3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FindFileStructureAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FindFileStructureAction.java
@@ -5,11 +5,12 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -30,7 +31,7 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-public class FindFileStructureAction extends Action<FindFileStructureAction.Response> {
+public class FindFileStructureAction extends StreamableResponseActionType<FindFileStructureAction.Response> {
 
     public static final FindFileStructureAction INSTANCE = new FindFileStructureAction();
     public static final String NAME = "cluster:monitor/xpack/ml/findfilestructure";
@@ -113,6 +114,7 @@ public boolean equals(Object other) {
     public static class Request extends ActionRequest {
 
         public static final ParseField LINES_TO_SAMPLE = new ParseField("lines_to_sample");
+        public static final ParseField LINE_MERGE_SIZE_LIMIT = new ParseField("line_merge_size_limit");
         public static final ParseField TIMEOUT = new ParseField("timeout");
         public static final ParseField CHARSET = FileStructure.CHARSET;
         public static final ParseField FORMAT = FileStructure.FORMAT;
@@ -130,6 +132,7 @@ public static class Request extends ActionRequest {
             "[%s] may only be specified if [" + FORMAT.getPreferredName() + "] is [%s]";
 
         private Integer linesToSample;
+        private Integer lineMergeSizeLimit;
         private TimeValue timeout;
         private String charset;
         private FileStructure.Format format;
@@ -154,6 +157,14 @@ public void setLinesToSample(Integer linesToSample) {
             this.linesToSample = linesToSample;
         }
 
+        public Integer getLineMergeSizeLimit() {
+            return lineMergeSizeLimit;
+        }
+
+        public void setLineMergeSizeLimit(Integer lineMergeSizeLimit) {
+            this.lineMergeSizeLimit = lineMergeSizeLimit;
+        }
+
         public TimeValue getTimeout() {
             return timeout;
         }
@@ -291,6 +302,10 @@ public ActionRequestValidationException validate() {
                 validationException =
                     addValidationError("[" + LINES_TO_SAMPLE.getPreferredName() + "] must be positive if specified", validationException);
             }
+            if (lineMergeSizeLimit != null && lineMergeSizeLimit <= 0) {
+                validationException = addValidationError("[" + LINE_MERGE_SIZE_LIMIT.getPreferredName() + "] must be positive if specified",
+                    validationException);
+            }
             if (format != FileStructure.Format.DELIMITED) {
                 if (columnNames != null) {
                     validationException = addIncompatibleArgError(COLUMN_NAMES, FileStructure.Format.DELIMITED, validationException);
@@ -324,6 +339,9 @@ public ActionRequestValidationException validate() {
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             linesToSample = in.readOptionalVInt();
+            if (in.getVersion().onOrAfter(Version.CURRENT)) {
+                lineMergeSizeLimit = in.readOptionalVInt();
+            }
             timeout = in.readOptionalTimeValue();
             charset = in.readOptionalString();
             format = in.readBoolean() ? in.readEnum(FileStructure.Format.class) : null;
@@ -342,6 +360,9 @@ public void readFrom(StreamInput in) throws IOException {
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeOptionalVInt(linesToSample);
+            if (out.getVersion().onOrAfter(Version.CURRENT)) {
+                out.writeOptionalVInt(lineMergeSizeLimit);
+            }
             out.writeOptionalTimeValue(timeout);
             out.writeOptionalString(charset);
             if (format == null) {
@@ -378,8 +399,8 @@ public void writeTo(StreamOutput out) throws IOException {
 
         @Override
         public int hashCode() {
-            return Objects.hash(linesToSample, timeout, charset, format, columnNames, hasHeaderRow, delimiter, grokPattern, timestampFormat,
-                timestampField, sample);
+            return Objects.hash(linesToSample, lineMergeSizeLimit, timeout, charset, format, columnNames, hasHeaderRow, delimiter,
+                grokPattern, timestampFormat, timestampField, sample);
         }
 
         @Override
@@ -395,6 +416,7 @@ public boolean equals(Object other) {
 
             Request that = (Request) other;
             return Objects.equals(this.linesToSample, that.linesToSample) &&
+                Objects.equals(this.lineMergeSizeLimit, that.lineMergeSizeLimit) &&
                 Objects.equals(this.timeout, that.timeout) &&
                 Objects.equals(this.charset, that.charset) &&
                 Objects.equals(this.format, that.format) &&
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FlushJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FlushJobAction.java
index b73d2b502a49d..f549049efa7da 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FlushJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/FlushJobAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.support.tasks.BaseTasksResponse;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -25,7 +25,7 @@
 import java.util.Date;
 import java.util.Objects;
 
-public class FlushJobAction extends Action<FlushJobAction.Response> {
+public class FlushJobAction extends ActionType<FlushJobAction.Response> {
 
     public static final FlushJobAction INSTANCE = new FlushJobAction();
     public static final String NAME = "cluster:admin/xpack/ml/job/flush";
@@ -34,11 +34,6 @@ private FlushJobAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ForecastJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ForecastJobAction.java
index fb107579c6e66..0e4d9f1028310 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ForecastJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ForecastJobAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.support.tasks.BaseTasksResponse;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -24,7 +24,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class ForecastJobAction extends Action<ForecastJobAction.Response> {
+public class ForecastJobAction extends ActionType<ForecastJobAction.Response> {
 
     public static final ForecastJobAction INSTANCE = new ForecastJobAction();
     public static final String NAME = "cluster:admin/xpack/ml/job/forecast";
@@ -33,11 +33,6 @@ private ForecastJobAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetBucketsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetBucketsAction.java
index f0be0f74bfd00..860cdc9b79911 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetBucketsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetBucketsAction.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -28,7 +28,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class GetBucketsAction extends Action<GetBucketsAction.Response> {
+public class GetBucketsAction extends StreamableResponseActionType<GetBucketsAction.Response> {
 
     public static final GetBucketsAction INSTANCE = new GetBucketsAction();
     public static final String NAME = "cluster:monitor/xpack/ml/job/results/buckets/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetCalendarEventsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetCalendarEventsAction.java
index c2d25590513eb..1e4a4f60ae72c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetCalendarEventsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetCalendarEventsAction.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.ParseField;
@@ -29,7 +29,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class GetCalendarEventsAction extends Action<GetCalendarEventsAction.Response> {
+public class GetCalendarEventsAction extends StreamableResponseActionType<GetCalendarEventsAction.Response> {
     public static final GetCalendarEventsAction INSTANCE = new GetCalendarEventsAction();
     public static final String NAME = "cluster:monitor/xpack/ml/calendars/events/get";
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetCalendarsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetCalendarsAction.java
index 3165e7f29cae1..29e2fea89a83e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetCalendarsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetCalendarsAction.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -28,7 +28,7 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-public class GetCalendarsAction extends Action<GetCalendarsAction.Response> {
+public class GetCalendarsAction extends StreamableResponseActionType<GetCalendarsAction.Response> {
 
     public static final GetCalendarsAction INSTANCE = new GetCalendarsAction();
     public static final String NAME = "cluster:monitor/xpack/ml/calendars/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetCategoriesAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetCategoriesAction.java
index 3048f7b2bdda5..575a8c1b5aacf 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetCategoriesAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetCategoriesAction.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -29,7 +29,7 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-public class GetCategoriesAction extends Action<GetCategoriesAction.Response> {
+public class GetCategoriesAction extends StreamableResponseActionType<GetCategoriesAction.Response> {
 
     public static final GetCategoriesAction INSTANCE = new GetCategoriesAction();
     public static final String NAME = "cluster:monitor/xpack/ml/job/results/categories/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsAction.java
new file mode 100644
index 0000000000000..3873cdac96428
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsAction.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.StreamableResponseActionType;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.core.action.AbstractGetResourcesRequest;
+import org.elasticsearch.xpack.core.action.AbstractGetResourcesResponse;
+import org.elasticsearch.xpack.core.action.util.QueryPage;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+
+import java.io.IOException;
+import java.util.Collections;
+
+public class GetDataFrameAnalyticsAction extends StreamableResponseActionType<GetDataFrameAnalyticsAction.Response> {
+
+    public static final GetDataFrameAnalyticsAction INSTANCE = new GetDataFrameAnalyticsAction();
+    public static final String NAME = "cluster:monitor/xpack/ml/data_frame/analytics/get";
+
+    private GetDataFrameAnalyticsAction() {
+        super(NAME);
+    }
+
+    @Override
+    public Response newResponse() {
+        return new Response(new QueryPage<>(Collections.emptyList(), 0, Response.RESULTS_FIELD));
+    }
+
+    public static class Request extends AbstractGetResourcesRequest {
+
+        public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
+
+        public Request() {
+            setAllowNoResources(true);
+        }
+
+        public Request(String id) {
+            setResourceId(id);
+            setAllowNoResources(true);
+        }
+
+        public Request(StreamInput in) throws IOException {
+            readFrom(in);
+        }
+
+        @Override
+        public String getResourceIdField() {
+            return DataFrameAnalyticsConfig.ID.getPreferredName();
+        }
+    }
+
+    public static class Response extends AbstractGetResourcesResponse<DataFrameAnalyticsConfig> {
+
+        public static final ParseField RESULTS_FIELD = new ParseField("data_frame_analytics");
+
+        public Response() {}
+
+        public Response(QueryPage<DataFrameAnalyticsConfig> analytics) {
+            super(analytics);
+        }
+
+        @Override
+        protected Reader<DataFrameAnalyticsConfig> getReader() {
+            return DataFrameAnalyticsConfig::new;
+        }
+    }
+
+    public static class RequestBuilder extends ActionRequestBuilder<Request, Response> {
+
+        public RequestBuilder(ElasticsearchClient client) {
+            super(client, INSTANCE, new Request());
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
new file mode 100644
index 0000000000000..a0e70463c52e1
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java
@@ -0,0 +1,325 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.TaskOperationFailure;
+import org.elasticsearch.action.support.tasks.BaseTasksRequest;
+import org.elasticsearch.action.support.tasks.BaseTasksResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.xpack.core.action.util.PageParams;
+import org.elasticsearch.xpack.core.action.util.QueryPage;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class GetDataFrameAnalyticsStatsAction extends ActionType<GetDataFrameAnalyticsStatsAction.Response> {
+
+    public static final GetDataFrameAnalyticsStatsAction INSTANCE = new GetDataFrameAnalyticsStatsAction();
+    public static final String NAME = "cluster:monitor/xpack/ml/data_frame/analytics/stats/get";
+
+    private GetDataFrameAnalyticsStatsAction() {
+        super(NAME);
+    }
+
+    @Override
+    public Writeable.Reader<Response> getResponseReader() {
+        return Response::new;
+    }
+
+    public static class Request extends BaseTasksRequest<Request> {
+
+        public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
+
+        private String id;
+        private boolean allowNoMatch = true;
+        private PageParams pageParams = PageParams.defaultParams();
+
+        // Used internally to store the expanded IDs
+        private List<String> expandedIds = Collections.emptyList();
+
+        public Request(String id) {
+            this.id = ExceptionsHelper.requireNonNull(id, DataFrameAnalyticsConfig.ID.getPreferredName());
+            this.expandedIds = Collections.singletonList(id);
+        }
+
+        public Request() {}
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            id = in.readString();
+            allowNoMatch = in.readBoolean();
+            pageParams = in.readOptionalWriteable(PageParams::new);
+            expandedIds = in.readStringList();
+        }
+
+        public void setExpandedIds(List<String> expandedIds) {
+            this.expandedIds = Objects.requireNonNull(expandedIds);
+        }
+
+        public List<String> getExpandedIds() {
+            return expandedIds;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(id);
+            out.writeBoolean(allowNoMatch);
+            out.writeOptionalWriteable(pageParams);
+            out.writeStringCollection(expandedIds);
+        }
+
+        public void setId(String id) {
+            this.id = id;
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        public boolean isAllowNoMatch() {
+            return allowNoMatch;
+        }
+
+        public void setAllowNoMatch(boolean allowNoMatch) {
+            this.allowNoMatch = allowNoMatch;
+        }
+
+        public void setPageParams(PageParams pageParams) {
+            this.pageParams = pageParams;
+        }
+
+        public PageParams getPageParams() {
+            return pageParams;
+        }
+
+        @Override
+        public boolean match(Task task) {
+            return expandedIds.stream().anyMatch(expandedId -> StartDataFrameAnalyticsAction.TaskMatcher.match(task, expandedId));
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(id, allowNoMatch, pageParams);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            Request other = (Request) obj;
+            return Objects.equals(id, other.id) && allowNoMatch == other.allowNoMatch && Objects.equals(pageParams, other.pageParams);
+        }
+    }
+
+    public static class RequestBuilder extends ActionRequestBuilder<Request, Response> {
+
+        public RequestBuilder(ElasticsearchClient client, GetDataFrameAnalyticsStatsAction action) {
+            super(client, action, new Request());
+        }
+    }
+
+    public static class Response extends BaseTasksResponse implements ToXContentObject {
+
+        public static class Stats implements ToXContentObject, Writeable {
+
+            private final String id;
+            private final DataFrameAnalyticsState state;
+            @Nullable
+            private final String failureReason;
+            @Nullable
+            private final Integer progressPercentage;
+            @Nullable
+            private final DiscoveryNode node;
+            @Nullable
+            private final String assignmentExplanation;
+
+            public Stats(String id, DataFrameAnalyticsState state, @Nullable String failureReason, @Nullable Integer progressPercentage,
+                         @Nullable DiscoveryNode node, @Nullable String assignmentExplanation) {
+                this.id = Objects.requireNonNull(id);
+                this.state = Objects.requireNonNull(state);
+                this.failureReason = failureReason;
+                this.progressPercentage = progressPercentage;
+                this.node = node;
+                this.assignmentExplanation = assignmentExplanation;
+            }
+
+            public Stats(StreamInput in) throws IOException {
+                id = in.readString();
+                state = DataFrameAnalyticsState.fromStream(in);
+                failureReason = in.readOptionalString();
+                progressPercentage = in.readOptionalInt();
+                node = in.readOptionalWriteable(DiscoveryNode::new);
+                assignmentExplanation = in.readOptionalString();
+            }
+
+            public String getId() {
+                return id;
+            }
+
+            public DataFrameAnalyticsState getState() {
+                return state;
+            }
+
+            @Override
+            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                // TODO: Have callers wrap the content with an object as they choose rather than forcing it upon them
+                builder.startObject();
+                {
+                    toUnwrappedXContent(builder);
+                }
+                return builder.endObject();
+            }
+
+            public XContentBuilder toUnwrappedXContent(XContentBuilder builder) throws IOException {
+                builder.field(DataFrameAnalyticsConfig.ID.getPreferredName(), id);
+                builder.field("state", state.toString());
+                if (failureReason != null) {
+                    builder.field("failure_reason", failureReason);
+                }
+                if (progressPercentage != null) {
+                    builder.field("progress_percent", progressPercentage);
+                }
+                if (node != null) {
+                    builder.startObject("node");
+                    builder.field("id", node.getId());
+                    builder.field("name", node.getName());
+                    builder.field("ephemeral_id", node.getEphemeralId());
+                    builder.field("transport_address", node.getAddress().toString());
+
+                    builder.startObject("attributes");
+                    for (Map.Entry<String, String> entry : node.getAttributes().entrySet()) {
+                        builder.field(entry.getKey(), entry.getValue());
+                    }
+                    builder.endObject();
+                    builder.endObject();
+                }
+                if (assignmentExplanation != null) {
+                    builder.field("assignment_explanation", assignmentExplanation);
+                }
+                return builder;
+            }
+
+            @Override
+            public void writeTo(StreamOutput out) throws IOException {
+                out.writeString(id);
+                state.writeTo(out);
+                out.writeOptionalString(failureReason);
+                out.writeOptionalInt(progressPercentage);
+                out.writeOptionalWriteable(node);
+                out.writeOptionalString(assignmentExplanation);
+            }
+
+            @Override
+            public int hashCode() {
+                return Objects.hash(id, state, failureReason, progressPercentage, node, assignmentExplanation);
+            }
+
+            @Override
+            public boolean equals(Object obj) {
+                if (obj == null) {
+                    return false;
+                }
+                if (getClass() != obj.getClass()) {
+                    return false;
+                }
+                Stats other = (Stats) obj;
+                return Objects.equals(id, other.id)
+                        && Objects.equals(this.state, other.state)
+                        && Objects.equals(this.failureReason, other.failureReason)
+                        && Objects.equals(this.node, other.node)
+                        && Objects.equals(this.assignmentExplanation, other.assignmentExplanation);
+            }
+        }
+
+        private QueryPage<Stats> stats;
+
+        public Response(QueryPage<Stats> stats) {
+            this(Collections.emptyList(), Collections.emptyList(), stats);
+        }
+
+        public Response(List<TaskOperationFailure> taskFailures, List<? extends ElasticsearchException> nodeFailures,
+                        QueryPage<Stats> stats) {
+            super(taskFailures, nodeFailures);
+            this.stats = stats;
+        }
+
+        public Response(StreamInput in) throws IOException {
+            super(in);
+            stats = new QueryPage<>(in, Stats::new);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            stats.writeTo(out);
+        }
+
+        public QueryPage<Stats> getResponse() {
+            return stats;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            stats.doXContentBody(builder, params);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(stats);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            Response other = (Response) obj;
+            return Objects.equals(stats, other.stats);
+        }
+
+        @Override
+        public final String toString() {
+            return Strings.toString(this);
+        }
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsAction.java
index 950fa58af95c8..435434ab93d38 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsAction.java
@@ -5,8 +5,8 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -22,7 +22,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class GetDatafeedsAction extends Action<GetDatafeedsAction.Response> {
+public class GetDatafeedsAction extends StreamableResponseActionType<GetDatafeedsAction.Response> {
 
     public static final GetDatafeedsAction INSTANCE = new GetDatafeedsAction();
     public static final String NAME = "cluster:monitor/xpack/ml/datafeeds/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java
index 39055501444f5..9caf07a5bd8ff 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java
@@ -5,8 +5,8 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -28,7 +28,7 @@
 import java.util.Map;
 import java.util.Objects;
 
-public class GetDatafeedsStatsAction extends Action<GetDatafeedsStatsAction.Response> {
+public class GetDatafeedsStatsAction extends StreamableResponseActionType<GetDatafeedsStatsAction.Response> {
 
     public static final GetDatafeedsStatsAction INSTANCE = new GetDatafeedsStatsAction();
     public static final String NAME = "cluster:monitor/xpack/ml/datafeeds/stats/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetFiltersAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetFiltersAction.java
index d63dfc39d732b..d3ec3f2e32455 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetFiltersAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetFiltersAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.rest.RestStatus;
@@ -17,11 +17,10 @@
 import org.elasticsearch.xpack.core.action.util.QueryPage;
 import org.elasticsearch.xpack.core.ml.job.config.MlFilter;
 
-
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
 
-public class GetFiltersAction extends Action<GetFiltersAction.Response> {
+public class GetFiltersAction extends StreamableResponseActionType<GetFiltersAction.Response> {
 
     public static final GetFiltersAction INSTANCE = new GetFiltersAction();
     public static final String NAME = "cluster:admin/xpack/ml/filters/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetInfluencersAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetInfluencersAction.java
index a15336a97c09c..2042d581a2239 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetInfluencersAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetInfluencersAction.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -27,8 +27,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class GetInfluencersAction
-extends Action<GetInfluencersAction.Response> {
+public class GetInfluencersAction extends StreamableResponseActionType<GetInfluencersAction.Response> {
 
     public static final GetInfluencersAction INSTANCE = new GetInfluencersAction();
     public static final String NAME = "cluster:monitor/xpack/ml/job/results/influencers/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsAction.java
index 98b1eb7a118f9..5d92a356d551c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsAction.java
@@ -5,8 +5,8 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -22,7 +22,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class GetJobsAction extends Action<GetJobsAction.Response> {
+public class GetJobsAction extends StreamableResponseActionType<GetJobsAction.Response> {
 
     public static final GetJobsAction INSTANCE = new GetJobsAction();
     public static final String NAME = "cluster:monitor/xpack/ml/job/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java
index 17de9dfc3522c..558b166315e80 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java
@@ -6,8 +6,7 @@
 package org.elasticsearch.xpack.core.ml.action;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.TaskOperationFailure;
@@ -30,6 +29,7 @@
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 import org.elasticsearch.xpack.core.ml.stats.ForecastStats;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 
@@ -39,7 +39,9 @@
 import java.util.Map;
 import java.util.Objects;
 
-public class GetJobsStatsAction extends Action<GetJobsStatsAction.Response> {
+import static org.elasticsearch.Version.V_7_3_0;
+
+public class GetJobsStatsAction extends ActionType<GetJobsStatsAction.Response> {
 
     public static final GetJobsStatsAction INSTANCE = new GetJobsStatsAction();
     public static final String NAME = "cluster:monitor/xpack/ml/job/stats/get";
@@ -49,16 +51,12 @@ public class GetJobsStatsAction extends Action<GetJobsStatsAction.Response> {
     private static final String FORECASTS_STATS = "forecasts_stats";
     private static final String STATE = "state";
     private static final String NODE = "node";
+    private static final String TIMING_STATS = "timing_stats";
 
     private GetJobsStatsAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
@@ -151,22 +149,24 @@ public static class Response extends BaseTasksResponse implements ToXContentObje
 
         public static class JobStats implements ToXContentObject, Writeable {
             private final String jobId;
-            private DataCounts dataCounts;
+            private final DataCounts dataCounts;
             @Nullable
-            private ModelSizeStats modelSizeStats;
+            private final ModelSizeStats modelSizeStats;
             @Nullable
-            private ForecastStats forecastStats;
+            private final ForecastStats forecastStats;
             @Nullable
-            private TimeValue openTime;
-            private JobState state;
+            private final TimeValue openTime;
+            private final JobState state;
             @Nullable
-            private DiscoveryNode node;
+            private final DiscoveryNode node;
             @Nullable
-            private String assignmentExplanation;
+            private final String assignmentExplanation;
+            @Nullable
+            private final TimingStats timingStats;
 
             public JobStats(String jobId, DataCounts dataCounts, @Nullable ModelSizeStats modelSizeStats,
-                    @Nullable ForecastStats forecastStats, JobState state, @Nullable DiscoveryNode node,
-                    @Nullable String assignmentExplanation, @Nullable TimeValue opentime) {
+                            @Nullable ForecastStats forecastStats, JobState state, @Nullable DiscoveryNode node,
+                            @Nullable String assignmentExplanation, @Nullable TimeValue openTime, @Nullable TimingStats timingStats) {
                 this.jobId = Objects.requireNonNull(jobId);
                 this.dataCounts = Objects.requireNonNull(dataCounts);
                 this.modelSizeStats = modelSizeStats;
@@ -174,7 +174,8 @@ public JobStats(String jobId, DataCounts dataCounts, @Nullable ModelSizeStats mo
                 this.state = Objects.requireNonNull(state);
                 this.node = node;
                 this.assignmentExplanation = assignmentExplanation;
-                this.openTime = opentime;
+                this.openTime = openTime;
+                this.timingStats = timingStats;
             }
 
             public JobStats(StreamInput in) throws IOException {
@@ -185,8 +186,11 @@ public JobStats(StreamInput in) throws IOException {
                 node = in.readOptionalWriteable(DiscoveryNode::new);
                 assignmentExplanation = in.readOptionalString();
                 openTime = in.readOptionalTimeValue();
-                if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
-                    forecastStats = in.readOptionalWriteable(ForecastStats::new);
+                forecastStats = in.readOptionalWriteable(ForecastStats::new);
+                if (in.getVersion().onOrAfter(V_7_3_0)) {
+                    timingStats = in.readOptionalWriteable(TimingStats::new);
+                } else {
+                    timingStats = null;
                 }
             }
 
@@ -222,6 +226,10 @@ public TimeValue getOpenTime() {
                 return openTime;
             }
 
+            public TimingStats getTimingStats() {
+                return timingStats;
+            }
+
             @Override
             public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
                 // TODO: Have callers wrap the content with an object as they choose rather than forcing it upon them
@@ -263,6 +271,9 @@ public XContentBuilder toUnwrappedXContent(XContentBuilder builder) throws IOExc
                 if (openTime != null) {
                     builder.field("open_time", openTime.getStringRep());
                 }
+                if (timingStats != null) {
+                    builder.field(TIMING_STATS, timingStats);
+                }
                 return builder;
             }
 
@@ -275,14 +286,16 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeOptionalWriteable(node);
                 out.writeOptionalString(assignmentExplanation);
                 out.writeOptionalTimeValue(openTime);
-                if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
-                    out.writeOptionalWriteable(forecastStats);
+                out.writeOptionalWriteable(forecastStats);
+                if (out.getVersion().onOrAfter(V_7_3_0)) {
+                    out.writeOptionalWriteable(timingStats);
                 }
             }
 
             @Override
             public int hashCode() {
-                return Objects.hash(jobId, dataCounts, modelSizeStats, forecastStats, state, node, assignmentExplanation, openTime);
+                return Objects.hash(
+                    jobId, dataCounts, modelSizeStats, forecastStats, state, node, assignmentExplanation, openTime, timingStats);
             }
 
             @Override
@@ -294,14 +307,15 @@ public boolean equals(Object obj) {
                     return false;
                 }
                 JobStats other = (JobStats) obj;
-                return Objects.equals(jobId, other.jobId)
-                        && Objects.equals(this.dataCounts, other.dataCounts)
-                        && Objects.equals(this.modelSizeStats, other.modelSizeStats)
-                        && Objects.equals(this.forecastStats, other.forecastStats)
-                        && Objects.equals(this.state, other.state)
-                        && Objects.equals(this.node, other.node)
-                        && Objects.equals(this.assignmentExplanation, other.assignmentExplanation)
-                        && Objects.equals(this.openTime, other.openTime);
+                return Objects.equals(this.jobId, other.jobId)
+                    && Objects.equals(this.dataCounts, other.dataCounts)
+                    && Objects.equals(this.modelSizeStats, other.modelSizeStats)
+                    && Objects.equals(this.forecastStats, other.forecastStats)
+                    && Objects.equals(this.state, other.state)
+                    && Objects.equals(this.node, other.node)
+                    && Objects.equals(this.assignmentExplanation, other.assignmentExplanation)
+                    && Objects.equals(this.openTime, other.openTime)
+                    && Objects.equals(this.timingStats, other.timingStats);
             }
         }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetModelSnapshotsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetModelSnapshotsAction.java
index 1fc93b68e1af7..0b1bee54a3a0d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetModelSnapshotsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetModelSnapshotsAction.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
@@ -28,7 +28,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class GetModelSnapshotsAction extends Action<GetModelSnapshotsAction.Response> {
+public class GetModelSnapshotsAction extends StreamableResponseActionType<GetModelSnapshotsAction.Response> {
 
     public static final GetModelSnapshotsAction INSTANCE = new GetModelSnapshotsAction();
     public static final String NAME = "cluster:monitor/xpack/ml/job/model_snapshots/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java
index da1bc74da4859..4b657fdf26ec7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetOverallBucketsAction.java
@@ -6,10 +6,10 @@
 package org.elasticsearch.xpack.core.ml.action;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -47,7 +47,7 @@
  * the interval.
  * </p>
  */
-public class GetOverallBucketsAction extends Action<GetOverallBucketsAction.Response> {
+public class GetOverallBucketsAction extends StreamableResponseActionType<GetOverallBucketsAction.Response> {
 
     public static final GetOverallBucketsAction INSTANCE = new GetOverallBucketsAction();
     public static final String NAME = "cluster:monitor/xpack/ml/job/results/overall_buckets/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetRecordsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetRecordsAction.java
index 46eeca8c70065..c1af5da77265c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetRecordsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetRecordsAction.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -27,7 +27,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class GetRecordsAction extends Action<GetRecordsAction.Response> {
+public class GetRecordsAction extends StreamableResponseActionType<GetRecordsAction.Response> {
 
     public static final GetRecordsAction INSTANCE = new GetRecordsAction();
     public static final String NAME = "cluster:monitor/xpack/ml/job/results/records/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/IsolateDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/IsolateDatafeedAction.java
index 8f681472ee8b3..611052717b81d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/IsolateDatafeedAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/IsolateDatafeedAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.tasks.BaseTasksRequest;
@@ -35,7 +35,7 @@
  * task ensures the current datafeed task can complete inconsequentially while
  * the datafeed persistent task may be stopped or reassigned on another node.
  */
-public class IsolateDatafeedAction extends Action<IsolateDatafeedAction.Response> {
+public class IsolateDatafeedAction extends ActionType<IsolateDatafeedAction.Response> {
 
     public static final IsolateDatafeedAction INSTANCE = new IsolateDatafeedAction();
     public static final String NAME = "cluster:internal/xpack/ml/datafeed/isolate";
@@ -44,11 +44,6 @@ private IsolateDatafeedAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/KillProcessAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/KillProcessAction.java
index df54d693f75ff..410fc110d2416 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/KillProcessAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/KillProcessAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.support.tasks.BaseTasksResponse;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -16,7 +16,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class KillProcessAction extends Action<KillProcessAction.Response> {
+public class KillProcessAction extends ActionType<KillProcessAction.Response> {
 
     public static final KillProcessAction INSTANCE = new KillProcessAction();
     public static final String NAME = "cluster:internal/xpack/ml/job/kill/process";
@@ -25,11 +25,6 @@ private KillProcessAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/MlInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/MlInfoAction.java
index b0d635202c9fd..aebecf81fd0d8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/MlInfoAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/MlInfoAction.java
@@ -5,11 +5,11 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -21,7 +21,7 @@
 import java.util.Map;
 import java.util.Objects;
 
-public class MlInfoAction extends Action<MlInfoAction.Response> {
+public class MlInfoAction extends StreamableResponseActionType<MlInfoAction.Response> {
 
     public static final MlInfoAction INSTANCE = new MlInfoAction();
     public static final String NAME = "cluster:monitor/xpack/ml/info/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java
index 418ae16d3e632..4f50248bbbe0e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/OpenJobAction.java
@@ -6,7 +6,7 @@
 package org.elasticsearch.xpack.core.ml.action;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -34,7 +35,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class OpenJobAction extends Action<AcknowledgedResponse> {
+public class OpenJobAction extends ActionType<AcknowledgedResponse> {
 
     public static final OpenJobAction INSTANCE = new OpenJobAction();
     public static final String NAME = "cluster:admin/xpack/ml/job/open";
@@ -44,8 +45,8 @@ private OpenJobAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends MasterNodeRequest<Request> implements ToXContentObject {
@@ -170,9 +171,7 @@ public JobParams(String jobId) {
         public JobParams(StreamInput in) throws IOException {
             jobId = in.readString();
             timeout = TimeValue.timeValueMillis(in.readVLong());
-            if (in.getVersion().onOrAfter(Version.V_6_6_0)) {
-                job = in.readOptionalWriteable(Job::new);
-            }
+            job = in.readOptionalWriteable(Job::new);
         }
 
         public String getJobId() {
@@ -209,9 +208,7 @@ public String getWriteableName() {
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(jobId);
             out.writeVLong(timeout.millis());
-            if (out.getVersion().onOrAfter(Version.V_6_6_0)) {
-                out.writeOptionalWriteable(job);
-            }
+            out.writeOptionalWriteable(job);
         }
 
         @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PersistJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PersistJobAction.java
index 3de585efc4649..e401240853a7f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PersistJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PersistJobAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.support.tasks.BaseTasksResponse;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -16,7 +16,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class PersistJobAction extends Action<PersistJobAction.Response> {
+public class PersistJobAction extends ActionType<PersistJobAction.Response> {
 
     public static final PersistJobAction INSTANCE = new PersistJobAction();
     public static final String NAME = "cluster:admin/xpack/ml/job/persist";
@@ -25,11 +25,6 @@ private PersistJobAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostCalendarEventsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostCalendarEventsAction.java
index ae68b2fdb26ab..02acb708b9d3d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostCalendarEventsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostCalendarEventsAction.java
@@ -5,11 +5,11 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -29,7 +29,7 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 
-public class PostCalendarEventsAction extends Action<PostCalendarEventsAction.Response> {
+public class PostCalendarEventsAction extends StreamableResponseActionType<PostCalendarEventsAction.Response> {
     public static final PostCalendarEventsAction INSTANCE = new PostCalendarEventsAction();
     public static final String NAME = "cluster:admin/xpack/ml/calendars/events/post";
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostDataAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostDataAction.java
index 0393f2c463932..6fe9c17dca99a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostDataAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostDataAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.support.tasks.BaseTasksResponse;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -24,7 +24,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class PostDataAction extends Action<PostDataAction.Response> {
+public class PostDataAction extends ActionType<PostDataAction.Response> {
 
     public static final PostDataAction INSTANCE = new PostDataAction();
     public static final String NAME = "cluster:admin/xpack/ml/job/data/post";
@@ -33,11 +33,6 @@ private PostDataAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedAction.java
index af8a99b9828bb..1585be4b472ce 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedAction.java
@@ -5,11 +5,11 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -25,7 +25,7 @@
 import java.io.InputStream;
 import java.util.Objects;
 
-public class PreviewDatafeedAction extends Action<PreviewDatafeedAction.Response> {
+public class PreviewDatafeedAction extends StreamableResponseActionType<PreviewDatafeedAction.Response> {
 
     public static final PreviewDatafeedAction INSTANCE = new PreviewDatafeedAction();
     public static final String NAME = "cluster:admin/xpack/ml/datafeeds/preview";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutCalendarAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutCalendarAction.java
index 0314103a3006b..cc2c41890b053 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutCalendarAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutCalendarAction.java
@@ -5,11 +5,11 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -27,7 +27,7 @@
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
-public class PutCalendarAction extends Action<PutCalendarAction.Response>  {
+public class PutCalendarAction extends StreamableResponseActionType<PutCalendarAction.Response> {
     public static final PutCalendarAction INSTANCE = new PutCalendarAction();
     public static final String NAME = "cluster:admin/xpack/ml/calendars/put";
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsAction.java
new file mode 100644
index 0000000000000..aa291186899ff
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsAction.java
@@ -0,0 +1,153 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.job.messages.Messages;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class PutDataFrameAnalyticsAction extends StreamableResponseActionType<PutDataFrameAnalyticsAction.Response> {
+
+    public static final PutDataFrameAnalyticsAction INSTANCE = new PutDataFrameAnalyticsAction();
+    public static final String NAME = "cluster:admin/xpack/ml/data_frame/analytics/put";
+
+    private PutDataFrameAnalyticsAction() {
+        super(NAME);
+    }
+
+    @Override
+    public Response newResponse() {
+        return new Response();
+    }
+
+    public static class Request extends AcknowledgedRequest<Request> implements ToXContentObject {
+
+        public static Request parseRequest(String id, XContentParser parser) {
+            DataFrameAnalyticsConfig.Builder config = DataFrameAnalyticsConfig.STRICT_PARSER.apply(parser, null);
+            if (config.getId() == null) {
+                config.setId(id);
+            } else if (!Strings.isNullOrEmpty(id) && !id.equals(config.getId())) {
+                // If we have both URI and body ID, they must be identical
+                throw new IllegalArgumentException(Messages.getMessage(Messages.INCONSISTENT_ID, DataFrameAnalyticsConfig.ID,
+                    config.getId(), id));
+            }
+
+            return new PutDataFrameAnalyticsAction.Request(config.build());
+        }
+
+        private DataFrameAnalyticsConfig config;
+
+        public Request() {}
+
+        public Request(DataFrameAnalyticsConfig config) {
+            this.config = config;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            config = new DataFrameAnalyticsConfig(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            config.writeTo(out);
+        }
+
+        public DataFrameAnalyticsConfig getConfig() {
+            return config;
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            config.toXContent(builder, params);
+            return builder;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            PutDataFrameAnalyticsAction.Request request = (PutDataFrameAnalyticsAction.Request) o;
+            return Objects.equals(config, request.config);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(config);
+        }
+    }
+
+    public static class Response extends ActionResponse implements ToXContentObject {
+
+        private DataFrameAnalyticsConfig config;
+
+        public Response(DataFrameAnalyticsConfig config) {
+            this.config = config;
+        }
+
+        Response() {}
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            config = new DataFrameAnalyticsConfig(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            config.writeTo(out);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            config.toXContent(builder, params);
+            return builder;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Response response = (Response) o;
+            return Objects.equals(config, response.config);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(config);
+        }
+    }
+
+    public static class RequestBuilder extends MasterNodeOperationRequestBuilder<Request, Response, RequestBuilder> {
+
+        protected RequestBuilder(ElasticsearchClient client, PutDataFrameAnalyticsAction action) {
+            super(client, action, new Request());
+        }
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java
index 1ac325b864536..af9340692336a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -21,7 +21,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class PutDatafeedAction extends Action<PutDatafeedAction.Response> {
+public class PutDatafeedAction extends StreamableResponseActionType<PutDatafeedAction.Response> {
 
     public static final PutDatafeedAction INSTANCE = new PutDatafeedAction();
     public static final String NAME = "cluster:admin/xpack/ml/datafeeds/put";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutFilterAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutFilterAction.java
index 0ed5e8f22aadb..0b83488ed6447 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutFilterAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutFilterAction.java
@@ -5,11 +5,11 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -25,7 +25,7 @@
 import java.util.Objects;
 
 
-public class PutFilterAction extends Action<PutFilterAction.Response> {
+public class PutFilterAction extends StreamableResponseActionType<PutFilterAction.Response> {
 
     public static final PutFilterAction INSTANCE = new PutFilterAction();
     public static final String NAME = "cluster:admin/xpack/ml/filters/put";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutJobAction.java
index 2ae19c4f32250..5c9bfaa047d14 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutJobAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -24,7 +24,7 @@
 import java.util.List;
 import java.util.Objects;
 
-public class PutJobAction extends Action<PutJobAction.Response> {
+public class PutJobAction extends StreamableResponseActionType<PutJobAction.Response> {
 
     public static final PutJobAction INSTANCE = new PutJobAction();
     public static final String NAME = "cluster:admin/xpack/ml/job/put";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotAction.java
index cae1efb7e7a31..938d3a96ee0bc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RevertModelSnapshotAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -28,7 +28,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class RevertModelSnapshotAction extends Action<RevertModelSnapshotAction.Response> {
+public class RevertModelSnapshotAction extends StreamableResponseActionType<RevertModelSnapshotAction.Response> {
 
     public static final RevertModelSnapshotAction INSTANCE = new RevertModelSnapshotAction();
     public static final String NAME = "cluster:admin/xpack/ml/job/model_snapshots/revert";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/SetUpgradeModeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/SetUpgradeModeAction.java
index ac9ae2b084498..9592743554fe3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/SetUpgradeModeAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/SetUpgradeModeAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -21,7 +22,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class SetUpgradeModeAction extends Action<AcknowledgedResponse> {
+public class SetUpgradeModeAction extends ActionType<AcknowledgedResponse> {
 
     public static final SetUpgradeModeAction INSTANCE = new SetUpgradeModeAction();
     public static final String NAME = "cluster:admin/xpack/ml/upgrade_mode";
@@ -31,8 +32,8 @@ private SetUpgradeModeAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends AcknowledgedRequest<Request> implements ToXContentObject {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java
new file mode 100644
index 0000000000000..b0a45c3742188
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java
@@ -0,0 +1,245 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.core.ml.MlTasks;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.job.messages.Messages;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class StartDataFrameAnalyticsAction extends ActionType<AcknowledgedResponse> {
+
+    public static final StartDataFrameAnalyticsAction INSTANCE = new StartDataFrameAnalyticsAction();
+    public static final String NAME = "cluster:admin/xpack/ml/data_frame/analytics/start";
+
+    private StartDataFrameAnalyticsAction() {
+        super(NAME);
+    }
+
+    @Override
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
+    }
+
+    public static class Request extends MasterNodeRequest<Request> implements ToXContentObject {
+
+        public static final ParseField TIMEOUT = new ParseField("timeout");
+
+        private static final ObjectParser<Request, Void> PARSER = new ObjectParser<>(NAME, Request::new);
+
+        static {
+            PARSER.declareString((request, id) -> request.id = id, DataFrameAnalyticsConfig.ID);
+            PARSER.declareString((request, val) -> request.setTimeout(TimeValue.parseTimeValue(val, TIMEOUT.getPreferredName())), TIMEOUT);
+        }
+
+        public static Request parseRequest(String id, XContentParser parser) {
+            Request request = PARSER.apply(parser, null);
+            if (request.getId() == null) {
+                request.setId(id);
+            } else if (!Strings.isNullOrEmpty(id) && !id.equals(request.getId())) {
+                throw new IllegalArgumentException(Messages.getMessage(Messages.INCONSISTENT_ID, DataFrameAnalyticsConfig.ID,
+                    request.getId(), id));
+            }
+            return request;
+        }
+
+        private String id;
+        private TimeValue timeout = TimeValue.timeValueSeconds(20);
+
+        public Request(String id) {
+            setId(id);
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            id = in.readString();
+            timeout = in.readTimeValue();
+        }
+
+        public Request() {}
+
+        public final void setId(String id) {
+            this.id = ExceptionsHelper.requireNonNull(id, DataFrameAnalyticsConfig.ID);
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        public void setTimeout(TimeValue timeout) {
+            this.timeout = timeout;
+        }
+
+        public TimeValue getTimeout() {
+            return timeout;
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(id);
+            out.writeTimeValue(timeout);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            if (id != null) {
+                builder.field(DataFrameAnalyticsConfig.ID.getPreferredName(), id);
+            }
+            builder.field(TIMEOUT.getPreferredName(), timeout.getStringRep());
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(id, timeout);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) {
+                return true;
+            }
+            if (obj == null || obj.getClass() != getClass()) {
+                return false;
+            }
+            StartDataFrameAnalyticsAction.Request other = (StartDataFrameAnalyticsAction.Request) obj;
+            return Objects.equals(id, other.id) && Objects.equals(timeout, other.timeout);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+
+    static class RequestBuilder extends ActionRequestBuilder<Request, AcknowledgedResponse> {
+
+        RequestBuilder(ElasticsearchClient client, StartDataFrameAnalyticsAction action) {
+            super(client, action, new Request());
+        }
+    }
+
+    public static class TaskParams implements XPackPlugin.XPackPersistentTaskParams {
+
+        public static final Version VERSION_INTRODUCED = Version.V_7_3_0;
+
+        public static ConstructingObjectParser<TaskParams, Void> PARSER = new ConstructingObjectParser<>(
+            MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, true, a -> new TaskParams((String) a[0], (String) a[1]));
+
+        static {
+            PARSER.declareString(ConstructingObjectParser.constructorArg(), DataFrameAnalyticsConfig.ID);
+            PARSER.declareString(ConstructingObjectParser.constructorArg(), DataFrameAnalyticsConfig.VERSION);
+        }
+
+        public static TaskParams fromXContent(XContentParser parser) {
+            return PARSER.apply(parser, null);
+        }
+
+        private final String id;
+        private final Version version;
+
+        public TaskParams(String id, Version version) {
+            this.id = Objects.requireNonNull(id);
+            this.version = Objects.requireNonNull(version);
+        }
+
+        private TaskParams(String id, String version) {
+            this(id, Version.fromString(version));
+        }
+
+        public TaskParams(StreamInput in) throws IOException {
+            this.id = in.readString();
+            this.version = Version.readVersion(in);
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        @Override
+        public String getWriteableName() {
+            return MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            return VERSION_INTRODUCED;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(id);
+            Version.writeVersion(version, out);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(DataFrameAnalyticsConfig.ID.getPreferredName(), id);
+            builder.field(DataFrameAnalyticsConfig.VERSION.getPreferredName(), version);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(id, version);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (o == this) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            TaskParams other = (TaskParams) o;
+            return Objects.equals(id, other.id) && Objects.equals(version, other.version);
+        }
+    }
+
+    public interface TaskMatcher {
+
+        static boolean match(Task task, String expectedId) {
+            if (task instanceof TaskMatcher) {
+                if (MetaData.ALL.equals(expectedId)) {
+                    return true;
+                }
+                String expectedDescription = MlTasks.DATA_FRAME_ANALYTICS_TASK_ID_PREFIX + expectedId;
+                return expectedDescription.equals(task.getDescription());
+            }
+            return false;
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java
index cdfdf06bf85ac..9344985686965 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java
@@ -7,7 +7,7 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ValidateActions;
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.time.DateMathParser;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -38,7 +39,7 @@
 import java.util.Objects;
 import java.util.function.LongSupplier;
 
-public class StartDatafeedAction extends Action<AcknowledgedResponse> {
+public class StartDatafeedAction extends ActionType<AcknowledgedResponse> {
 
     public static final ParseField START_TIME = new ParseField("start");
     public static final ParseField END_TIME = new ParseField("end");
@@ -52,8 +53,8 @@ private StartDatafeedAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends MasterNodeRequest<Request> implements ToXContentObject {
@@ -195,10 +196,8 @@ public DatafeedParams(StreamInput in) throws IOException {
             startTime = in.readVLong();
             endTime = in.readOptionalLong();
             timeout = TimeValue.timeValueMillis(in.readVLong());
-            if (in.getVersion().onOrAfter(Version.V_6_6_0)) {
-                jobId = in.readOptionalString();
-                datafeedIndices = in.readStringList();
-            }
+            jobId = in.readOptionalString();
+            datafeedIndices = in.readStringList();
         }
 
         DatafeedParams() {
@@ -272,10 +271,8 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVLong(startTime);
             out.writeOptionalLong(endTime);
             out.writeVLong(timeout.millis());
-            if (out.getVersion().onOrAfter(Version.V_6_6_0)) {
-                out.writeOptionalString(jobId);
-                out.writeStringCollection(datafeedIndices);
-            }
+            out.writeOptionalString(jobId);
+            out.writeStringCollection(datafeedIndices);
         }
 
         @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsAction.java
new file mode 100644
index 0000000000000..3e671a10e5f0e
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsAction.java
@@ -0,0 +1,234 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.tasks.BaseTasksRequest;
+import org.elasticsearch.action.support.tasks.BaseTasksResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.job.messages.Messages;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
+public class StopDataFrameAnalyticsAction extends ActionType<StopDataFrameAnalyticsAction.Response> {
+
+    public static final StopDataFrameAnalyticsAction INSTANCE = new StopDataFrameAnalyticsAction();
+    public static final String NAME = "cluster:admin/xpack/ml/data_frame/analytics/stop";
+
+    private StopDataFrameAnalyticsAction() {
+        super(NAME);
+    }
+
+    @Override
+    public Writeable.Reader<Response> getResponseReader() {
+        return Response::new;
+    }
+
+    public static class Request extends BaseTasksRequest<Request> implements ToXContentObject {
+
+        public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
+        public static final ParseField FORCE = new ParseField("force");
+        public static final ParseField TIMEOUT = new ParseField("timeout");
+
+        private static final ObjectParser<Request, Void> PARSER = new ObjectParser<>(NAME, Request::new);
+
+        static {
+            PARSER.declareString((request, id) -> request.id = id, DataFrameAnalyticsConfig.ID);
+            PARSER.declareString((request, val) -> request.setTimeout(TimeValue.parseTimeValue(val, TIMEOUT.getPreferredName())), TIMEOUT);
+            PARSER.declareBoolean(Request::setAllowNoMatch, ALLOW_NO_MATCH);
+            PARSER.declareBoolean(Request::setForce, FORCE);
+        }
+
+        public static Request parseRequest(String id, XContentParser parser) {
+            Request request = PARSER.apply(parser, null);
+            if (request.getId() == null) {
+                request.setId(id);
+            } else if (!Strings.isNullOrEmpty(id) && !id.equals(request.getId())) {
+                throw new IllegalArgumentException(Messages.getMessage(Messages.INCONSISTENT_ID, DataFrameAnalyticsConfig.ID,
+                    request.getId(), id));
+            }
+            return request;
+        }
+
+        private String id;
+        private boolean allowNoMatch = true;
+        private boolean force;
+        private Set<String> expandedIds = Collections.emptySet();
+
+        public Request(String id) {
+            setId(id);
+        }
+
+        public Request(StreamInput in) throws IOException {
+            super(in);
+            id = in.readString();
+            allowNoMatch = in.readBoolean();
+            force = in.readBoolean();
+            expandedIds = new HashSet<>(Arrays.asList(in.readStringArray()));
+        }
+
+        public Request() {}
+
+        public final void setId(String id) {
+            this.id = ExceptionsHelper.requireNonNull(id, DataFrameAnalyticsConfig.ID);
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        public boolean allowNoMatch() {
+            return allowNoMatch;
+        }
+
+        public void setAllowNoMatch(boolean allowNoMatch) {
+            this.allowNoMatch = allowNoMatch;
+        }
+
+        public boolean isForce() {
+            return force;
+        }
+
+        public void setForce(boolean force) {
+            this.force = force;
+        }
+
+        @Nullable
+        public Set<String> getExpandedIds() {
+            return expandedIds;
+        }
+
+        public void setExpandedIds(Set<String> expandedIds) {
+            this.expandedIds = Objects.requireNonNull(expandedIds);
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(id);
+            out.writeBoolean(allowNoMatch);
+            out.writeBoolean(force);
+            out.writeStringArray(expandedIds.toArray(new String[0]));
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder
+                .startObject()
+                .field(DataFrameAnalyticsConfig.ID.getPreferredName(), id)
+                .field(ALLOW_NO_MATCH.getPreferredName(), allowNoMatch)
+                .field(FORCE.getPreferredName(), force)
+                .endObject();
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(id, getTimeout(), allowNoMatch, force, expandedIds);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) {
+                return true;
+            }
+            if (obj == null || obj.getClass() != getClass()) {
+                return false;
+            }
+            StopDataFrameAnalyticsAction.Request other = (StopDataFrameAnalyticsAction.Request) obj;
+            return Objects.equals(id, other.id)
+                && Objects.equals(getTimeout(), other.getTimeout())
+                && allowNoMatch == other.allowNoMatch
+                && force == other.force
+                && Objects.equals(expandedIds, other.expandedIds);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+
+    public static class Response extends BaseTasksResponse implements Writeable, ToXContentObject {
+
+        private final boolean stopped;
+
+        public Response(boolean stopped) {
+            super(null, null);
+            this.stopped = stopped;
+        }
+
+        public Response(StreamInput in) throws IOException {
+            super(in);
+            stopped = in.readBoolean();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeBoolean(stopped);
+        }
+
+        public boolean isStopped() {
+            return stopped;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            toXContentCommon(builder, params);
+            builder.field("stopped", stopped);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+            Response response = (Response) o;
+            return stopped == response.stopped;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(stopped);
+        }
+    }
+
+    static class RequestBuilder extends ActionRequestBuilder<Request, Response> {
+
+        RequestBuilder(ElasticsearchClient client, StopDataFrameAnalyticsAction action) {
+            super(client, action, new Request());
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java
index 0021040c69801..72e6b63b32aca 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StopDatafeedAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.tasks.BaseTasksRequest;
@@ -28,7 +28,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class StopDatafeedAction extends Action<StopDatafeedAction.Response> {
+public class StopDatafeedAction extends ActionType<StopDatafeedAction.Response> {
 
     public static final StopDatafeedAction INSTANCE = new StopDatafeedAction();
     public static final String NAME = "cluster:admin/xpack/ml/datafeed/stop";
@@ -38,11 +38,6 @@ private StopDatafeedAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateCalendarJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateCalendarJobAction.java
index e70a2e3189b01..02c3f7dad40da 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateCalendarJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateCalendarJobAction.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -18,7 +18,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class UpdateCalendarJobAction extends Action<PutCalendarAction.Response> {
+public class UpdateCalendarJobAction extends StreamableResponseActionType<PutCalendarAction.Response> {
     public static final UpdateCalendarJobAction INSTANCE = new UpdateCalendarJobAction();
     public static final String NAME = "cluster:admin/xpack/ml/calendars/jobs/update";
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedAction.java
index 6ba34efa839b5..93c96a8a9cfc6 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedAction.java
@@ -5,8 +5,8 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -20,7 +20,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class UpdateDatafeedAction extends Action<PutDatafeedAction.Response> {
+public class UpdateDatafeedAction extends StreamableResponseActionType<PutDatafeedAction.Response> {
 
     public static final UpdateDatafeedAction INSTANCE = new UpdateDatafeedAction();
     public static final String NAME = "cluster:admin/xpack/ml/datafeeds/update";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateFilterAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateFilterAction.java
index 57b3d3457d736..95696ec5ed80c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateFilterAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateFilterAction.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
@@ -32,7 +32,7 @@
 import java.util.TreeSet;
 
 
-public class UpdateFilterAction extends Action<PutFilterAction.Response> {
+public class UpdateFilterAction extends StreamableResponseActionType<PutFilterAction.Response> {
 
     public static final UpdateFilterAction INSTANCE = new UpdateFilterAction();
     public static final String NAME = "cluster:admin/xpack/ml/filters/update";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java
index 3e1b0ea6b3c55..80dcb4e6f7091 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java
@@ -5,8 +5,8 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -23,7 +23,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class UpdateJobAction extends Action<PutJobAction.Response> {
+public class UpdateJobAction extends StreamableResponseActionType<PutJobAction.Response> {
     public static final UpdateJobAction INSTANCE = new UpdateJobAction();
     public static final String NAME = "cluster:admin/xpack/ml/job/update";
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateModelSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateModelSnapshotAction.java
index 1414719693f2c..5592fc6022380 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateModelSnapshotAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateModelSnapshotAction.java
@@ -5,11 +5,11 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
@@ -29,7 +29,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class UpdateModelSnapshotAction extends Action<UpdateModelSnapshotAction.Response> {
+public class UpdateModelSnapshotAction extends StreamableResponseActionType<UpdateModelSnapshotAction.Response> {
 
     public static final UpdateModelSnapshotAction INSTANCE = new UpdateModelSnapshotAction();
     public static final String NAME = "cluster:admin/xpack/ml/job/model_snapshots/update";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateProcessAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateProcessAction.java
index 6a8e1703ad1f2..331344fdace64 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateProcessAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateProcessAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.support.tasks.BaseTasksResponse;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -23,7 +23,7 @@
 import java.util.List;
 import java.util.Objects;
 
-public class UpdateProcessAction extends Action<UpdateProcessAction.Response> {
+public class UpdateProcessAction extends ActionType<UpdateProcessAction.Response> {
 
     public static final UpdateProcessAction INSTANCE = new UpdateProcessAction();
     public static final String NAME = "cluster:internal/xpack/ml/job/update/process";
@@ -32,11 +32,6 @@ private UpdateProcessAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ValidateDetectorAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ValidateDetectorAction.java
index e0d6b8acd849c..6a33fa1fdbd6d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ValidateDetectorAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ValidateDetectorAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
@@ -13,6 +13,7 @@
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -21,7 +22,7 @@
 import java.io.IOException;
 import java.util.Objects;
 
-public class ValidateDetectorAction extends Action<AcknowledgedResponse> {
+public class ValidateDetectorAction extends ActionType<AcknowledgedResponse> {
 
     public static final ValidateDetectorAction INSTANCE = new ValidateDetectorAction();
     public static final String NAME = "cluster:admin/xpack/ml/job/validate/detector";
@@ -31,8 +32,8 @@ protected ValidateDetectorAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class RequestBuilder extends ActionRequestBuilder<Request, AcknowledgedResponse> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ValidateJobConfigAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ValidateJobConfigAction.java
index 99af65f4412e9..f6abae83eef2d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ValidateJobConfigAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/ValidateJobConfigAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.ml.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
@@ -13,6 +13,7 @@
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
@@ -22,7 +23,7 @@
 import java.util.List;
 import java.util.Objects;
 
-public class ValidateJobConfigAction extends Action<AcknowledgedResponse> {
+public class ValidateJobConfigAction extends ActionType<AcknowledgedResponse> {
 
     public static final ValidateJobConfigAction INSTANCE = new ValidateJobConfigAction();
     public static final String NAME = "cluster:admin/xpack/ml/job/validate";
@@ -32,8 +33,8 @@ protected ValidateJobConfigAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class RequestBuilder extends ActionRequestBuilder<Request, AcknowledgedResponse> {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/Annotation.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/Annotation.java
index 185808de00463..91c4053ed1568 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/Annotation.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/Annotation.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.io.IOException;
 import java.util.Date;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java
index afd10e0c17b01..03bb25d3652e5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java
@@ -23,7 +23,7 @@
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.Intervals;
 import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.io.IOException;
 import java.time.Instant;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/client/MachineLearningClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/client/MachineLearningClient.java
deleted file mode 100644
index af2f3ad08fd1a..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/client/MachineLearningClient.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.ml.client;
-
-import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
-import org.elasticsearch.xpack.core.ml.action.DeleteDatafeedAction;
-import org.elasticsearch.xpack.core.ml.action.DeleteFilterAction;
-import org.elasticsearch.xpack.core.ml.action.DeleteJobAction;
-import org.elasticsearch.xpack.core.ml.action.DeleteModelSnapshotAction;
-import org.elasticsearch.xpack.core.ml.action.FlushJobAction;
-import org.elasticsearch.xpack.core.ml.action.GetBucketsAction;
-import org.elasticsearch.xpack.core.ml.action.GetCategoriesAction;
-import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction;
-import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction;
-import org.elasticsearch.xpack.core.ml.action.GetFiltersAction;
-import org.elasticsearch.xpack.core.ml.action.GetInfluencersAction;
-import org.elasticsearch.xpack.core.ml.action.GetJobsAction;
-import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
-import org.elasticsearch.xpack.core.ml.action.GetModelSnapshotsAction;
-import org.elasticsearch.xpack.core.ml.action.GetRecordsAction;
-import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
-import org.elasticsearch.xpack.core.ml.action.PostDataAction;
-import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction;
-import org.elasticsearch.xpack.core.ml.action.PutFilterAction;
-import org.elasticsearch.xpack.core.ml.action.PutJobAction;
-import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction;
-import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
-import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction;
-import org.elasticsearch.xpack.core.ml.action.UpdateDatafeedAction;
-import org.elasticsearch.xpack.core.ml.action.UpdateJobAction;
-import org.elasticsearch.xpack.core.ml.action.UpdateModelSnapshotAction;
-import org.elasticsearch.xpack.core.ml.action.ValidateDetectorAction;
-import org.elasticsearch.xpack.core.ml.action.ValidateJobConfigAction;
-
-public class MachineLearningClient {
-
-    private final ElasticsearchClient client;
-
-    public MachineLearningClient(ElasticsearchClient client) {
-        this.client = client;
-    }
-
-    public void closeJob(CloseJobAction.Request request,
-            ActionListener<CloseJobAction.Response> listener) {
-        client.execute(CloseJobAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<CloseJobAction.Response> closeJob(CloseJobAction.Request request) {
-        PlainActionFuture<CloseJobAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(CloseJobAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void deleteDatafeed(DeleteDatafeedAction.Request request,
-            ActionListener<AcknowledgedResponse> listener) {
-        client.execute(DeleteDatafeedAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<AcknowledgedResponse> deleteDatafeed(
-            DeleteDatafeedAction.Request request) {
-        PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        client.execute(DeleteDatafeedAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void deleteFilter(DeleteFilterAction.Request request,
-            ActionListener<AcknowledgedResponse> listener) {
-        client.execute(DeleteFilterAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<AcknowledgedResponse> deleteFilter(
-            DeleteFilterAction.Request request) {
-        PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        client.execute(DeleteFilterAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void deleteJob(DeleteJobAction.Request request,
-            ActionListener<AcknowledgedResponse> listener) {
-        client.execute(DeleteJobAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<AcknowledgedResponse> deleteJob(DeleteJobAction.Request request) {
-        PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        client.execute(DeleteJobAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void deleteModelSnapshot(DeleteModelSnapshotAction.Request request,
-            ActionListener<AcknowledgedResponse> listener) {
-        client.execute(DeleteModelSnapshotAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<AcknowledgedResponse> deleteModelSnapshot(
-            DeleteModelSnapshotAction.Request request) {
-        PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        client.execute(DeleteModelSnapshotAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void flushJob(FlushJobAction.Request request,
-            ActionListener<FlushJobAction.Response> listener) {
-        client.execute(FlushJobAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<FlushJobAction.Response> flushJob(FlushJobAction.Request request) {
-        PlainActionFuture<FlushJobAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(FlushJobAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void getBuckets(GetBucketsAction.Request request,
-            ActionListener<GetBucketsAction.Response> listener) {
-        client.execute(GetBucketsAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<GetBucketsAction.Response> getBuckets(GetBucketsAction.Request request) {
-        PlainActionFuture<GetBucketsAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(GetBucketsAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void getCategories(GetCategoriesAction.Request request,
-            ActionListener<GetCategoriesAction.Response> listener) {
-        client.execute(GetCategoriesAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<GetCategoriesAction.Response> getCategories(
-            GetCategoriesAction.Request request) {
-        PlainActionFuture<GetCategoriesAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(GetCategoriesAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void getDatafeeds(GetDatafeedsAction.Request request,
-            ActionListener<GetDatafeedsAction.Response> listener) {
-        client.execute(GetDatafeedsAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<GetDatafeedsAction.Response> getDatafeeds(
-            GetDatafeedsAction.Request request) {
-        PlainActionFuture<GetDatafeedsAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(GetDatafeedsAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void getDatafeedsStats(GetDatafeedsStatsAction.Request request,
-            ActionListener<GetDatafeedsStatsAction.Response> listener) {
-        client.execute(GetDatafeedsStatsAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<GetDatafeedsStatsAction.Response> getDatafeedsStats(
-            GetDatafeedsStatsAction.Request request) {
-        PlainActionFuture<GetDatafeedsStatsAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(GetDatafeedsStatsAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void getFilters(GetFiltersAction.Request request,
-            ActionListener<GetFiltersAction.Response> listener) {
-        client.execute(GetFiltersAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<GetFiltersAction.Response> getFilters(GetFiltersAction.Request request) {
-        PlainActionFuture<GetFiltersAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(GetFiltersAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void getInfluencers(GetInfluencersAction.Request request,
-                               ActionListener<GetInfluencersAction.Response> listener) {
-        client.execute(GetInfluencersAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<GetInfluencersAction.Response> getInfluencers(
-            GetInfluencersAction.Request request) {
-        PlainActionFuture<GetInfluencersAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(GetInfluencersAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void getJobs(GetJobsAction.Request request,
-            ActionListener<GetJobsAction.Response> listener) {
-        client.execute(GetJobsAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<GetJobsAction.Response> getJobs(GetJobsAction.Request request) {
-        PlainActionFuture<GetJobsAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(GetJobsAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void getJobsStats(GetJobsStatsAction.Request request,
-            ActionListener<GetJobsStatsAction.Response> listener) {
-        client.execute(GetJobsStatsAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<GetJobsStatsAction.Response> getJobsStats(
-            GetJobsStatsAction.Request request) {
-        PlainActionFuture<GetJobsStatsAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(GetJobsStatsAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void getModelSnapshots(GetModelSnapshotsAction.Request request,
-            ActionListener<GetModelSnapshotsAction.Response> listener) {
-        client.execute(GetModelSnapshotsAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<GetModelSnapshotsAction.Response> getModelSnapshots(
-            GetModelSnapshotsAction.Request request) {
-        PlainActionFuture<GetModelSnapshotsAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(GetModelSnapshotsAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void getRecords(GetRecordsAction.Request request,
-            ActionListener<GetRecordsAction.Response> listener) {
-        client.execute(GetRecordsAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<GetRecordsAction.Response> getRecords(GetRecordsAction.Request request) {
-        PlainActionFuture<GetRecordsAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(GetRecordsAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void openJob(OpenJobAction.Request request,
-            ActionListener<AcknowledgedResponse> listener) {
-        client.execute(OpenJobAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<AcknowledgedResponse> openJob(OpenJobAction.Request request) {
-        PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        client.execute(OpenJobAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void postData(PostDataAction.Request request,
-            ActionListener<PostDataAction.Response> listener) {
-        client.execute(PostDataAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<PostDataAction.Response> postData(PostDataAction.Request request) {
-        PlainActionFuture<PostDataAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(PostDataAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void putDatafeed(PutDatafeedAction.Request request,
-            ActionListener<PutDatafeedAction.Response> listener) {
-        client.execute(PutDatafeedAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<PutDatafeedAction.Response> putDatafeed(PutDatafeedAction.Request request) {
-        PlainActionFuture<PutDatafeedAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(PutDatafeedAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void putFilter(PutFilterAction.Request request,
-            ActionListener<PutFilterAction.Response> listener) {
-        client.execute(PutFilterAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<PutFilterAction.Response> putFilter(PutFilterAction.Request request) {
-        PlainActionFuture<PutFilterAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(PutFilterAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void putJob(PutJobAction.Request request,
-            ActionListener<PutJobAction.Response> listener) {
-        client.execute(PutJobAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<PutJobAction.Response> putJob(PutJobAction.Request request) {
-        PlainActionFuture<PutJobAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(PutJobAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void revertModelSnapshot(RevertModelSnapshotAction.Request request,
-            ActionListener<RevertModelSnapshotAction.Response> listener) {
-        client.execute(RevertModelSnapshotAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<RevertModelSnapshotAction.Response> revertModelSnapshot(
-            RevertModelSnapshotAction.Request request) {
-        PlainActionFuture<RevertModelSnapshotAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(RevertModelSnapshotAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void startDatafeed(StartDatafeedAction.Request request,
-            ActionListener<AcknowledgedResponse> listener) {
-        client.execute(StartDatafeedAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<AcknowledgedResponse> startDatafeed(
-            StartDatafeedAction.Request request) {
-        PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        client.execute(StartDatafeedAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void stopDatafeed(StopDatafeedAction.Request request,
-            ActionListener<StopDatafeedAction.Response> listener) {
-        client.execute(StopDatafeedAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<StopDatafeedAction.Response> stopDatafeed(
-            StopDatafeedAction.Request request) {
-        PlainActionFuture<StopDatafeedAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(StopDatafeedAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void updateDatafeed(UpdateDatafeedAction.Request request,
-            ActionListener<PutDatafeedAction.Response> listener) {
-        client.execute(UpdateDatafeedAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<PutDatafeedAction.Response> updateDatafeed(
-            UpdateDatafeedAction.Request request) {
-        PlainActionFuture<PutDatafeedAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(UpdateDatafeedAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void updateJob(UpdateJobAction.Request request,
-            ActionListener<PutJobAction.Response> listener) {
-        client.execute(UpdateJobAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<PutJobAction.Response> updateJob(UpdateJobAction.Request request) {
-        PlainActionFuture<PutJobAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(UpdateJobAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void updateModelSnapshot(UpdateModelSnapshotAction.Request request,
-            ActionListener<UpdateModelSnapshotAction.Response> listener) {
-        client.execute(UpdateModelSnapshotAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<UpdateModelSnapshotAction.Response> updateModelSnapshot(
-            UpdateModelSnapshotAction.Request request) {
-        PlainActionFuture<UpdateModelSnapshotAction.Response> listener = PlainActionFuture.newFuture();
-        client.execute(UpdateModelSnapshotAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void validateDetector(ValidateDetectorAction.Request request,
-                                    ActionListener<AcknowledgedResponse> listener) {
-        client.execute(ValidateDetectorAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<AcknowledgedResponse> validateDetector(
-            ValidateDetectorAction.Request request) {
-        PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        client.execute(ValidateDetectorAction.INSTANCE, request, listener);
-        return listener;
-    }
-
-    public void validateJobConfig(ValidateJobConfigAction.Request request,
-                                 ActionListener<AcknowledgedResponse> listener) {
-        client.execute(ValidateJobConfigAction.INSTANCE, request, listener);
-    }
-
-    public ActionFuture<AcknowledgedResponse> validateJobConfig(
-            ValidateJobConfigAction.Request request) {
-        PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        client.execute(ValidateJobConfigAction.INSTANCE, request, listener);
-        return listener;
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java
index f08c4a9d7391d..9285256c76819 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java
@@ -8,7 +8,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
@@ -31,9 +30,10 @@
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.MlStrings;
+import org.elasticsearch.xpack.core.ml.utils.QueryProvider;
 import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
 import org.elasticsearch.xpack.core.ml.utils.XContentObjectTransformer;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -123,7 +123,7 @@ private static ObjectParser<Builder, Void> createParser(boolean ignoreUnknownFie
         parser.declareString((builder, val) ->
             builder.setFrequency(TimeValue.parseTimeValue(val, FREQUENCY.getPreferredName())), FREQUENCY);
         parser.declareObject(Builder::setQueryProvider,
-            (p, c) -> QueryProvider.fromXContent(p, ignoreUnknownFields),
+            (p, c) -> QueryProvider.fromXContent(p, ignoreUnknownFields, Messages.DATAFEED_CONFIG_QUERY_BAD_FORMAT),
             QUERY);
         parser.declareObject(Builder::setAggregationsSafe,
             (p, c) -> AggProvider.fromXContent(p, ignoreUnknownFields),
@@ -204,12 +204,6 @@ public DatafeedConfig(StreamInput in) throws IOException {
         } else {
             this.indices = null;
         }
-        // This consumes the list of types if there was one.
-        if (in.getVersion().before(Version.V_7_0_0)) {
-            if (in.readBoolean()) {
-                in.readStringList();
-            }
-        }
         // each of these writables are version aware
         this.queryProvider = QueryProvider.fromStream(in);
         // This reads a boolean from the stream, if true, it sends the stream to the `fromStream` method
@@ -223,11 +217,7 @@ public DatafeedConfig(StreamInput in) throws IOException {
         this.scrollSize = in.readOptionalVInt();
         this.chunkingConfig = in.readOptionalWriteable(ChunkingConfig::new);
         this.headers = Collections.unmodifiableMap(in.readMap(StreamInput::readString, StreamInput::readString));
-        if (in.getVersion().onOrAfter(Version.V_6_6_0)) {
-            delayedDataCheckConfig = in.readOptionalWriteable(DelayedDataCheckConfig::new);
-        } else {
-            delayedDataCheckConfig = DelayedDataCheckConfig.defaultDelayedDataCheckConfig();
-        }
+        delayedDataCheckConfig = in.readOptionalWriteable(DelayedDataCheckConfig::new);
     }
 
     /**
@@ -408,12 +398,6 @@ public void writeTo(StreamOutput out) throws IOException {
         } else {
             out.writeBoolean(false);
         }
-        // Write the now removed types to prior versions.
-        // An empty list is expected
-        if (out.getVersion().before(Version.V_7_0_0)) {
-            out.writeBoolean(true);
-            out.writeStringCollection(Collections.emptyList());
-        }
 
         // Each of these writables are version aware
         queryProvider.writeTo(out); // never null
@@ -429,9 +413,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalVInt(scrollSize);
         out.writeOptionalWriteable(chunkingConfig);
         out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeString);
-        if (out.getVersion().onOrAfter(Version.V_6_6_0)) {
-            out.writeOptionalWriteable(delayedDataCheckConfig);
-        }
+        out.writeOptionalWriteable(delayedDataCheckConfig);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java
index ccbb516197217..9f38d0323f244 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java
@@ -22,7 +22,9 @@
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.utils.QueryProvider;
 import org.elasticsearch.xpack.core.ml.utils.XContentObjectTransformer;
 
 import java.io.IOException;
@@ -53,7 +55,8 @@ public class DatafeedUpdate implements Writeable, ToXContentObject {
                 TimeValue.parseTimeValue(val, DatafeedConfig.QUERY_DELAY.getPreferredName())), DatafeedConfig.QUERY_DELAY);
         PARSER.declareString((builder, val) -> builder.setFrequency(
                 TimeValue.parseTimeValue(val, DatafeedConfig.FREQUENCY.getPreferredName())), DatafeedConfig.FREQUENCY);
-        PARSER.declareObject(Builder::setQuery, (p, c) -> QueryProvider.fromXContent(p, false), DatafeedConfig.QUERY);
+        PARSER.declareObject(Builder::setQuery, (p, c) -> QueryProvider.fromXContent(p, false, Messages.DATAFEED_CONFIG_QUERY_BAD_FORMAT),
+            DatafeedConfig.QUERY);
         PARSER.declareObject(Builder::setAggregationsSafe,
             (p, c) -> AggProvider.fromXContent(p, false),
             DatafeedConfig.AGGREGATIONS);
@@ -134,11 +137,7 @@ public DatafeedUpdate(StreamInput in) throws IOException {
         }
         this.scrollSize = in.readOptionalVInt();
         this.chunkingConfig = in.readOptionalWriteable(ChunkingConfig::new);
-        if (in.getVersion().onOrAfter(Version.V_6_6_0)) {
-            delayedDataCheckConfig = in.readOptionalWriteable(DelayedDataCheckConfig::new);
-        } else {
-            delayedDataCheckConfig = null;
-        }
+        delayedDataCheckConfig = in.readOptionalWriteable(DelayedDataCheckConfig::new);
     }
 
     /**
@@ -181,9 +180,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         out.writeOptionalVInt(scrollSize);
         out.writeOptionalWriteable(chunkingConfig);
-        if (out.getVersion().onOrAfter(Version.V_6_6_0)) {
-            out.writeOptionalWriteable(delayedDataCheckConfig);
-        }
+        out.writeOptionalWriteable(delayedDataCheckConfig);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DelayedDataCheckConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DelayedDataCheckConfig.java
index 9406b91d119c7..52de35af1fac3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DelayedDataCheckConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DelayedDataCheckConfig.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.io.IOException;
 import java.util.Objects;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java
new file mode 100644
index 0000000000000..99460a6883a56
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java
@@ -0,0 +1,385 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.DataFrameAnalysis;
+import org.elasticsearch.xpack.core.ml.job.messages.Messages;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING;
+import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.VALUE;
+
+public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable {
+
+    public static final String TYPE = "data_frame_analytics_config";
+
+    public static final ByteSizeValue DEFAULT_MODEL_MEMORY_LIMIT = new ByteSizeValue(1, ByteSizeUnit.GB);
+    public static final ByteSizeValue MIN_MODEL_MEMORY_LIMIT = new ByteSizeValue(1, ByteSizeUnit.MB);
+    public static final ByteSizeValue PROCESS_MEMORY_OVERHEAD = new ByteSizeValue(20, ByteSizeUnit.MB);
+
+    public static final ParseField ID = new ParseField("id");
+    public static final ParseField SOURCE = new ParseField("source");
+    public static final ParseField DEST = new ParseField("dest");
+    public static final ParseField ANALYSIS = new ParseField("analysis");
+    public static final ParseField CONFIG_TYPE = new ParseField("config_type");
+    public static final ParseField ANALYZED_FIELDS = new ParseField("analyzed_fields");
+    public static final ParseField MODEL_MEMORY_LIMIT = new ParseField("model_memory_limit");
+    public static final ParseField HEADERS = new ParseField("headers");
+    public static final ParseField CREATE_TIME = new ParseField("create_time");
+    public static final ParseField VERSION = new ParseField("version");
+
+    public static final ObjectParser<Builder, Void> STRICT_PARSER = createParser(false);
+    public static final ObjectParser<Builder, Void> LENIENT_PARSER = createParser(true);
+
+    public static ObjectParser<Builder, Void> createParser(boolean ignoreUnknownFields) {
+        ObjectParser<Builder, Void> parser = new ObjectParser<>(TYPE, ignoreUnknownFields, Builder::new);
+
+        parser.declareString((c, s) -> {}, CONFIG_TYPE);
+        parser.declareString(Builder::setId, ID);
+        parser.declareObject(Builder::setSource, DataFrameAnalyticsSource.createParser(ignoreUnknownFields), SOURCE);
+        parser.declareObject(Builder::setDest, DataFrameAnalyticsDest.createParser(ignoreUnknownFields), DEST);
+        parser.declareObject(Builder::setAnalysis, (p, c) -> parseAnalysis(p, ignoreUnknownFields), ANALYSIS);
+        parser.declareField(Builder::setAnalyzedFields,
+            (p, c) -> FetchSourceContext.fromXContent(p),
+            ANALYZED_FIELDS,
+            OBJECT_ARRAY_BOOLEAN_OR_STRING);
+        parser.declareField(Builder::setModelMemoryLimit,
+            (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MODEL_MEMORY_LIMIT.getPreferredName()), MODEL_MEMORY_LIMIT, VALUE);
+        if (ignoreUnknownFields) {
+            // Headers are not parsed by the strict (config) parser, so headers supplied in the _body_ of a REST request will be rejected.
+            // (For config, headers are explicitly transferred from the auth headers by code in the put data frame actions.)
+            parser.declareObject(Builder::setHeaders, (p, c) -> p.mapStrings(), HEADERS);
+            // Creation time is set automatically during PUT, so create_time supplied in the _body_ of a REST request will be rejected.
+            parser.declareField(Builder::setCreateTime,
+                p -> TimeUtils.parseTimeFieldToInstant(p, CREATE_TIME.getPreferredName()),
+                CREATE_TIME,
+                ObjectParser.ValueType.VALUE);
+            // Version is set automatically during PUT, so version supplied in the _body_ of a REST request will be rejected.
+            parser.declareField(Builder::setVersion, p -> {
+                if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                    return Version.fromString(p.text());
+                }
+                throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+            }, VERSION, ObjectParser.ValueType.STRING);
+        }
+        return parser;
+    }
+
+    private static DataFrameAnalysis parseAnalysis(XContentParser parser, boolean ignoreUnknownFields) throws IOException {
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
+        DataFrameAnalysis analysis = parser.namedObject(DataFrameAnalysis.class, parser.currentName(), ignoreUnknownFields);
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        return analysis;
+    }
+
+    private final String id;
+    private final DataFrameAnalyticsSource source;
+    private final DataFrameAnalyticsDest dest;
+    private final DataFrameAnalysis analysis;
+    private final FetchSourceContext analyzedFields;
+    /**
+     * This may be null up to the point of persistence, as the relationship with <code>xpack.ml.max_model_memory_limit</code>
+     * depends on whether the user explicitly set the value or if the default was requested.  <code>null</code> indicates
+     * the default was requested, which in turn means a default higher than the maximum is silently capped.
+     * A non-<code>null</code> value higher than <code>xpack.ml.max_model_memory_limit</code> will cause a
+     * validation error even if it is equal to the default value.  This behaviour matches what is done in
+     * {@link org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits}.
+     */
+    private final ByteSizeValue modelMemoryLimit;
+    private final Map<String, String> headers;
+    private final Instant createTime;
+    private final Version version;
+
+    public DataFrameAnalyticsConfig(String id, DataFrameAnalyticsSource source, DataFrameAnalyticsDest dest,
+                                    DataFrameAnalysis analysis, Map<String, String> headers, ByteSizeValue modelMemoryLimit,
+                                    FetchSourceContext analyzedFields, Instant createTime, Version version) {
+        this.id = ExceptionsHelper.requireNonNull(id, ID);
+        this.source = ExceptionsHelper.requireNonNull(source, SOURCE);
+        this.dest = ExceptionsHelper.requireNonNull(dest, DEST);
+        this.analysis = ExceptionsHelper.requireNonNull(analysis, ANALYSIS);
+        this.analyzedFields = analyzedFields;
+        this.modelMemoryLimit = modelMemoryLimit;
+        this.headers = Collections.unmodifiableMap(headers);
+        this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli());;
+        this.version = version;
+    }
+
+    public DataFrameAnalyticsConfig(StreamInput in) throws IOException {
+        this.id = in.readString();
+        this.source = new DataFrameAnalyticsSource(in);
+        this.dest = new DataFrameAnalyticsDest(in);
+        this.analysis = in.readNamedWriteable(DataFrameAnalysis.class);
+        this.analyzedFields = in.readOptionalWriteable(FetchSourceContext::new);
+        this.modelMemoryLimit = in.readOptionalWriteable(ByteSizeValue::new);
+        this.headers = Collections.unmodifiableMap(in.readMap(StreamInput::readString, StreamInput::readString));
+        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+            createTime = in.readOptionalInstant();
+            version = in.readBoolean() ? Version.readVersion(in) : null;
+        } else {
+            createTime = null;
+            version = null;
+        }
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public DataFrameAnalyticsSource getSource() {
+        return source;
+    }
+
+    public DataFrameAnalyticsDest getDest() {
+        return dest;
+    }
+
+    public DataFrameAnalysis getAnalysis() {
+        return analysis;
+    }
+
+    public FetchSourceContext getAnalyzedFields() {
+        return analyzedFields;
+    }
+
+    public ByteSizeValue getModelMemoryLimit() {
+        return modelMemoryLimit != null ? modelMemoryLimit : DEFAULT_MODEL_MEMORY_LIMIT;
+    }
+
+    public Map<String, String> getHeaders() {
+        return headers;
+    }
+
+    public Instant getCreateTime() {
+        return createTime;
+    }
+
+    public Version getVersion() {
+        return version;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(ID.getPreferredName(), id);
+        builder.field(SOURCE.getPreferredName(), source);
+        builder.field(DEST.getPreferredName(), dest);
+
+        builder.startObject(ANALYSIS.getPreferredName());
+        builder.field(analysis.getWriteableName(), analysis);
+        builder.endObject();
+
+        if (params.paramAsBoolean(ToXContentParams.INCLUDE_TYPE, false)) {
+            builder.field(CONFIG_TYPE.getPreferredName(), TYPE);
+        }
+        if (analyzedFields != null) {
+            builder.field(ANALYZED_FIELDS.getPreferredName(), analyzedFields);
+        }
+        builder.field(MODEL_MEMORY_LIMIT.getPreferredName(), getModelMemoryLimit().getStringRep());
+        if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
+            builder.field(HEADERS.getPreferredName(), headers);
+        }
+        if (createTime != null) {
+            builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli());
+        }
+        if (version != null) {
+            builder.field(VERSION.getPreferredName(), version);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(id);
+        source.writeTo(out);
+        dest.writeTo(out);
+        out.writeNamedWriteable(analysis);
+        out.writeOptionalWriteable(analyzedFields);
+        out.writeOptionalWriteable(modelMemoryLimit);
+        out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeString);
+        if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+            out.writeOptionalInstant(createTime);
+            if (version != null) {
+                out.writeBoolean(true);
+                Version.writeVersion(version, out);
+            } else {
+                out.writeBoolean(false);
+            }
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        DataFrameAnalyticsConfig other = (DataFrameAnalyticsConfig) o;
+        return Objects.equals(id, other.id)
+            && Objects.equals(source, other.source)
+            && Objects.equals(dest, other.dest)
+            && Objects.equals(analysis, other.analysis)
+            && Objects.equals(headers, other.headers)
+            && Objects.equals(getModelMemoryLimit(), other.getModelMemoryLimit())
+            && Objects.equals(analyzedFields, other.analyzedFields)
+            && Objects.equals(createTime, other.createTime)
+            && Objects.equals(version, other.version);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, source, dest, analysis, headers, getModelMemoryLimit(), analyzedFields, createTime, version);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    public static String documentId(String id) {
+        return TYPE + "-" + id;
+    }
+
+    public static class Builder {
+
+        private String id;
+        private DataFrameAnalyticsSource source;
+        private DataFrameAnalyticsDest dest;
+        private DataFrameAnalysis analysis;
+        private FetchSourceContext analyzedFields;
+        private ByteSizeValue modelMemoryLimit;
+        private ByteSizeValue maxModelMemoryLimit;
+        private Map<String, String> headers = Collections.emptyMap();
+        private Instant createTime;
+        private Version version;
+
+        public Builder() {}
+
+        public Builder(String id) {
+            setId(id);
+        }
+
+        public Builder(ByteSizeValue maxModelMemoryLimit) {
+            this.maxModelMemoryLimit = maxModelMemoryLimit;
+        }
+
+        public Builder(DataFrameAnalyticsConfig config) {
+            this(config, null);
+        }
+
+        public Builder(DataFrameAnalyticsConfig config, ByteSizeValue maxModelMemoryLimit) {
+            this.id = config.id;
+            this.source = new DataFrameAnalyticsSource(config.source);
+            this.dest = new DataFrameAnalyticsDest(config.dest);
+            this.analysis = config.analysis;
+            this.headers = new HashMap<>(config.headers);
+            this.modelMemoryLimit = config.modelMemoryLimit;
+            this.maxModelMemoryLimit = maxModelMemoryLimit;
+            if (config.analyzedFields != null) {
+                this.analyzedFields = new FetchSourceContext(true, config.analyzedFields.includes(), config.analyzedFields.excludes());
+            }
+            this.createTime = config.createTime;
+            this.version = config.version;
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        public Builder setId(String id) {
+            this.id = ExceptionsHelper.requireNonNull(id, ID);
+            return this;
+        }
+
+        public Builder setSource(DataFrameAnalyticsSource source) {
+            this.source = ExceptionsHelper.requireNonNull(source, SOURCE);
+            return this;
+        }
+
+        public Builder setDest(DataFrameAnalyticsDest dest) {
+            this.dest = ExceptionsHelper.requireNonNull(dest, DEST);
+            return this;
+        }
+
+        public Builder setAnalysis(DataFrameAnalysis analysis) {
+            this.analysis = ExceptionsHelper.requireNonNull(analysis, ANALYSIS);
+            return this;
+        }
+
+        public Builder setAnalyzedFields(FetchSourceContext fields) {
+            this.analyzedFields = fields;
+            return this;
+        }
+
+        public Builder setHeaders(Map<String, String> headers) {
+            this.headers = headers;
+            return this;
+        }
+
+        public Builder setModelMemoryLimit(ByteSizeValue modelMemoryLimit) {
+            if (modelMemoryLimit != null && modelMemoryLimit.compareTo(MIN_MODEL_MEMORY_LIMIT) < 0) {
+                throw new IllegalArgumentException("[" + MODEL_MEMORY_LIMIT.getPreferredName()
+                    + "] must be at least [" + MIN_MODEL_MEMORY_LIMIT.getStringRep() + "]");
+            }
+            this.modelMemoryLimit = modelMemoryLimit;
+            return this;
+        }
+
+        private void applyMaxModelMemoryLimit() {
+
+            boolean maxModelMemoryIsSet = maxModelMemoryLimit != null && maxModelMemoryLimit.getMb() > 0;
+
+            if (modelMemoryLimit == null) {
+                // Default is silently capped if higher than limit
+                if (maxModelMemoryIsSet && DEFAULT_MODEL_MEMORY_LIMIT.compareTo(maxModelMemoryLimit) > 0) {
+                    modelMemoryLimit = maxModelMemoryLimit;
+                }
+            } else if (maxModelMemoryIsSet && modelMemoryLimit.compareTo(maxModelMemoryLimit) > 0) {
+                // Explicit setting higher than limit is an error
+                throw ExceptionsHelper.badRequestException(Messages.getMessage(Messages.JOB_CONFIG_MODEL_MEMORY_LIMIT_GREATER_THAN_MAX,
+                    modelMemoryLimit, maxModelMemoryLimit));
+            }
+        }
+
+        public Builder setCreateTime(Instant createTime) {
+            this.createTime = createTime;
+            return this;
+        }
+
+        public Builder setVersion(Version version) {
+            this.version = version;
+            return this;
+        }
+
+        public DataFrameAnalyticsConfig build() {
+            applyMaxModelMemoryLimit();
+            return new DataFrameAnalyticsConfig(id, source, dest, analysis, headers, modelMemoryLimit, analyzedFields, createTime, version);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsDest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsDest.java
new file mode 100644
index 0000000000000..3bc435336f062
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsDest.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.indices.InvalidIndexNameException;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+
+import static org.elasticsearch.cluster.metadata.MetaDataCreateIndexService.validateIndexOrAliasName;
+
+public class DataFrameAnalyticsDest implements Writeable, ToXContentObject {
+
+    public static final ParseField INDEX = new ParseField("index");
+    public static final ParseField RESULTS_FIELD = new ParseField("results_field");
+
+    private static final String DEFAULT_RESULTS_FIELD = "ml";
+
+    public static ConstructingObjectParser<DataFrameAnalyticsDest, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<DataFrameAnalyticsDest, Void> parser = new ConstructingObjectParser<>("data_frame_analytics_dest",
+            ignoreUnknownFields, a -> new DataFrameAnalyticsDest((String) a[0], (String) a[1]));
+        parser.declareString(ConstructingObjectParser.constructorArg(), INDEX);
+        parser.declareString(ConstructingObjectParser.optionalConstructorArg(), RESULTS_FIELD);
+        return parser;
+    }
+
+    private final String index;
+    private final String resultsField;
+
+    public DataFrameAnalyticsDest(String index, @Nullable String resultsField) {
+        this.index = ExceptionsHelper.requireNonNull(index, INDEX);
+        if (index.isEmpty()) {
+            throw ExceptionsHelper.badRequestException("[{}] must be non-empty", INDEX);
+        }
+        this.resultsField = resultsField == null ? DEFAULT_RESULTS_FIELD : resultsField;
+    }
+
+    public DataFrameAnalyticsDest(StreamInput in) throws IOException {
+        index = in.readString();
+        resultsField = in.readString();
+    }
+
+    public DataFrameAnalyticsDest(DataFrameAnalyticsDest other) {
+        this.index = other.index;
+        this.resultsField = other.resultsField;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(index);
+        out.writeString(resultsField);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(INDEX.getPreferredName(), index);
+        builder.field(RESULTS_FIELD.getPreferredName(), resultsField);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        DataFrameAnalyticsDest other = (DataFrameAnalyticsDest) o;
+        return Objects.equals(index, other.index) && Objects.equals(resultsField, other.resultsField);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(index, resultsField);
+    }
+
+    public String getIndex() {
+        return index;
+    }
+
+    public String getResultsField() {
+        return resultsField;
+    }
+
+    public void validate() {
+        if (index != null) {
+            validateIndexOrAliasName(index, InvalidIndexNameException::new);
+            if (index.toLowerCase(Locale.ROOT).equals(index) == false) {
+                throw new InvalidIndexNameException(index, "dest.index must be lowercase");
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsSource.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsSource.java
new file mode 100644
index 0000000000000..e08c39a75208f
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsSource.java
@@ -0,0 +1,149 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.xpack.core.ml.job.messages.Messages;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.utils.QueryProvider;
+import org.elasticsearch.xpack.core.ml.utils.XContentObjectTransformer;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class DataFrameAnalyticsSource implements Writeable, ToXContentObject {
+
+    public static final ParseField INDEX = new ParseField("index");
+    public static final ParseField QUERY = new ParseField("query");
+
+    public static ConstructingObjectParser<DataFrameAnalyticsSource, Void> createParser(boolean ignoreUnknownFields) {
+        ConstructingObjectParser<DataFrameAnalyticsSource, Void> parser = new ConstructingObjectParser<>("data_frame_analytics_source",
+            ignoreUnknownFields, a -> new DataFrameAnalyticsSource(((List<String>) a[0]).toArray(new String[0]), (QueryProvider) a[1]));
+        parser.declareStringArray(ConstructingObjectParser.constructorArg(), INDEX);
+        parser.declareObject(ConstructingObjectParser.optionalConstructorArg(),
+            (p, c) -> QueryProvider.fromXContent(p, ignoreUnknownFields, Messages.DATA_FRAME_ANALYTICS_BAD_QUERY_FORMAT), QUERY);
+        return parser;
+    }
+
+    private final String[] index;
+    private final QueryProvider queryProvider;
+
+    public DataFrameAnalyticsSource(String[] index, @Nullable QueryProvider queryProvider) {
+        this.index = ExceptionsHelper.requireNonNull(index, INDEX);
+        if (index.length == 0) {
+            throw new IllegalArgumentException("source.index must specify at least one index");
+        }
+        if (Arrays.stream(index).anyMatch(Strings::isNullOrEmpty)) {
+            throw new IllegalArgumentException("source.index must contain non-null and non-empty strings");
+        }
+        this.queryProvider = queryProvider == null ? QueryProvider.defaultQuery() : queryProvider;
+    }
+
+    public DataFrameAnalyticsSource(StreamInput in) throws IOException {
+        index = in.readStringArray();
+        queryProvider = QueryProvider.fromStream(in);
+    }
+
+    public DataFrameAnalyticsSource(DataFrameAnalyticsSource other) {
+        this.index = Arrays.copyOf(other.index, other.index.length);
+        this.queryProvider = new QueryProvider(other.queryProvider);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeStringArray(index);
+        queryProvider.writeTo(out);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.array(INDEX.getPreferredName(), index);
+        builder.field(QUERY.getPreferredName(), queryProvider.getQuery());
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        DataFrameAnalyticsSource other = (DataFrameAnalyticsSource) o;
+        return Arrays.equals(index, other.index)
+            && Objects.equals(queryProvider, other.queryProvider);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(Arrays.asList(index), queryProvider);
+    }
+
+    public String[] getIndex() {
+        return index;
+    }
+
+    /**
+     * Get the fully parsed query from the semi-parsed stored {@code Map<String, Object>}
+     *
+     * @return Fully parsed query
+     */
+    public QueryBuilder getParsedQuery() {
+        Exception exception = queryProvider.getParsingException();
+        if (exception != null) {
+            if (exception instanceof RuntimeException) {
+                throw (RuntimeException) exception;
+            } else {
+                throw new ElasticsearchException(queryProvider.getParsingException());
+            }
+        }
+        return queryProvider.getParsedQuery();
+    }
+
+    Exception getQueryParsingException() {
+        return queryProvider.getParsingException();
+    }
+
+    /**
+     * Calls the parser and returns any gathered deprecations
+     *
+     * @param namedXContentRegistry XContent registry to transform the lazily parsed query
+     * @return The deprecations from parsing the query
+     */
+    public List<String> getQueryDeprecations(NamedXContentRegistry namedXContentRegistry) {
+        List<String> deprecations = new ArrayList<>();
+        try {
+            XContentObjectTransformer.queryBuilderTransformer(namedXContentRegistry).fromMap(queryProvider.getQuery(),
+                deprecations);
+        } catch (Exception exception) {
+            // Certain thrown exceptions wrap up the real Illegal argument making it hard to determine cause for the user
+            if (exception.getCause() instanceof IllegalArgumentException) {
+                exception = (Exception) exception.getCause();
+            }
+            throw ExceptionsHelper.badRequestException(Messages.DATA_FRAME_ANALYTICS_BAD_QUERY_FORMAT, exception);
+        }
+        return deprecations;
+    }
+
+    public Map<String, Object> getQuery() {
+        return queryProvider.getQuery();
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsState.java
new file mode 100644
index 0000000000000..8f0a481ed9030
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsState.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Locale;
+
+public enum DataFrameAnalyticsState implements Writeable {
+
+    STARTED, REINDEXING, ANALYZING, STOPPING, STOPPED, FAILED;
+
+    public static DataFrameAnalyticsState fromString(String name) {
+        return valueOf(name.trim().toUpperCase(Locale.ROOT));
+    }
+
+    public static DataFrameAnalyticsState fromStream(StreamInput in) throws IOException {
+        return in.readEnum(DataFrameAnalyticsState.class);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeEnum(this);
+    }
+
+    @Override
+    public String toString() {
+        return name().toLowerCase(Locale.ROOT);
+    }
+
+    /**
+     * @return {@code true} if state matches any of the given {@code candidates}
+     */
+    public boolean isAnyOf(DataFrameAnalyticsState... candidates) {
+        return Arrays.stream(candidates).anyMatch(candidate -> this == candidate);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java
new file mode 100644
index 0000000000000..188f3a2bdf2f6
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.persistent.PersistentTaskState;
+import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.xpack.core.ml.MlTasks;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class DataFrameAnalyticsTaskState implements PersistentTaskState {
+
+    public static final String NAME = MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME;
+
+    private static ParseField STATE = new ParseField("state");
+    private static ParseField ALLOCATION_ID = new ParseField("allocation_id");
+    private static ParseField REASON = new ParseField("reason");
+
+    private final DataFrameAnalyticsState state;
+    private final long allocationId;
+    private final String reason;
+
+    private static final ConstructingObjectParser<DataFrameAnalyticsTaskState, Void> PARSER =
+            new ConstructingObjectParser<>(NAME, true,
+                a -> new DataFrameAnalyticsTaskState((DataFrameAnalyticsState) a[0], (long) a[1], (String) a[2]));
+
+    static {
+        PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> {
+           if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+               return DataFrameAnalyticsState.fromString(p.text());
+           }
+           throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, STATE, ObjectParser.ValueType.STRING);
+        PARSER.declareLong(ConstructingObjectParser.constructorArg(), ALLOCATION_ID);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), REASON);
+    }
+
+    public static DataFrameAnalyticsTaskState fromXContent(XContentParser parser) {
+        try {
+            return PARSER.parse(parser, null);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public DataFrameAnalyticsTaskState(DataFrameAnalyticsState state, long allocationId, @Nullable String reason) {
+        this.state = Objects.requireNonNull(state);
+        this.allocationId = allocationId;
+        this.reason = reason;
+    }
+
+    public DataFrameAnalyticsTaskState(StreamInput in) throws IOException {
+        this.state = DataFrameAnalyticsState.fromStream(in);
+        this.allocationId = in.readLong();
+        this.reason = in.readOptionalString();
+    }
+
+    public DataFrameAnalyticsState getState() {
+        return state;
+    }
+
+    @Nullable
+    public String getReason() {
+        return reason;
+    }
+
+    public boolean isStatusStale(PersistentTasksCustomMetaData.PersistentTask<?> task) {
+        return allocationId != task.getAllocationId();
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        state.writeTo(out);
+        out.writeLong(allocationId);
+        out.writeOptionalString(reason);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(STATE.getPreferredName(), state.toString());
+        builder.field(ALLOCATION_ID.getPreferredName(), allocationId);
+        if (reason != null) {
+            builder.field(REASON.getPreferredName(), reason);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        DataFrameAnalyticsTaskState that = (DataFrameAnalyticsTaskState) o;
+        return allocationId == that.allocationId &&
+            state == that.state &&
+            Objects.equals(reason, that.reason);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(state, allocationId, reason);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java
new file mode 100644
index 0000000000000..f21533d917602
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java
@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.analyses;
+
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+
+import java.util.Map;
+
+public interface DataFrameAnalysis extends ToXContentObject, NamedWriteable {
+
+    Map<String, Object> getParams();
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/MlDataFrameAnalysisNamedXContentProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/MlDataFrameAnalysisNamedXContentProvider.java
new file mode 100644
index 0000000000000..a48a23e4a8393
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/MlDataFrameAnalysisNamedXContentProvider.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.analyses;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.plugins.spi.NamedXContentProvider;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class MlDataFrameAnalysisNamedXContentProvider implements NamedXContentProvider {
+
+    @Override
+    public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
+        List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
+
+        namedXContent.add(new NamedXContentRegistry.Entry(DataFrameAnalysis.class, OutlierDetection.NAME, (p, c) -> {
+            boolean ignoreUnknownFields = (boolean) c;
+            return OutlierDetection.fromXContent(p, ignoreUnknownFields);
+        }));
+
+        return namedXContent;
+    }
+
+    public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+        List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
+
+        namedWriteables.add(new NamedWriteableRegistry.Entry(DataFrameAnalysis.class, OutlierDetection.NAME.getPreferredName(),
+            OutlierDetection::new));
+
+        return namedWriteables;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java
new file mode 100644
index 0000000000000..e6891116ad68b
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java
@@ -0,0 +1,167 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.analyses;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+
+public class OutlierDetection implements DataFrameAnalysis {
+
+    public static final ParseField NAME = new ParseField("outlier_detection");
+
+    public static final ParseField N_NEIGHBORS = new ParseField("n_neighbors");
+    public static final ParseField METHOD = new ParseField("method");
+    public static final ParseField FEATURE_INFLUENCE_THRESHOLD = new ParseField("feature_influence_threshold");
+
+    private static final ConstructingObjectParser<OutlierDetection, Void> LENIENT_PARSER = createParser(true);
+    private static final ConstructingObjectParser<OutlierDetection, Void> STRICT_PARSER = createParser(false);
+
+    private static ConstructingObjectParser<OutlierDetection, Void> createParser(boolean lenient) {
+        ConstructingObjectParser<OutlierDetection, Void> parser = new ConstructingObjectParser<>(NAME.getPreferredName(), lenient,
+            a -> new OutlierDetection((Integer) a[0], (Method) a[1], (Double) a[2]));
+        parser.declareInt(ConstructingObjectParser.optionalConstructorArg(), N_NEIGHBORS);
+        parser.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return Method.fromString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, METHOD, ObjectParser.ValueType.STRING);
+        parser.declareDouble(ConstructingObjectParser.optionalConstructorArg(), FEATURE_INFLUENCE_THRESHOLD);
+        return parser;
+    }
+
+    public static OutlierDetection fromXContent(XContentParser parser, boolean ignoreUnknownFields) {
+        return ignoreUnknownFields ? LENIENT_PARSER.apply(parser, null) : STRICT_PARSER.apply(parser, null);
+    }
+
+    private final Integer nNeighbors;
+    private final Method method;
+    private final Double featureInfluenceThreshold;
+
+    /**
+     * Constructs the outlier detection configuration
+     * @param nNeighbors The number of neighbors. Leave unspecified for dynamic detection.
+     * @param method The method. Leave unspecified for a dynamic mixture of methods.
+     * @param featureInfluenceThreshold The min outlier score required to calculate feature influence. Defaults to 0.1.
+     */
+    public OutlierDetection(@Nullable Integer nNeighbors, @Nullable Method method, @Nullable Double featureInfluenceThreshold) {
+        if (nNeighbors != null && nNeighbors <= 0) {
+            throw ExceptionsHelper.badRequestException("[{}] must be a positive integer", N_NEIGHBORS.getPreferredName());
+        }
+
+        if (featureInfluenceThreshold != null && (featureInfluenceThreshold < 0.0 || featureInfluenceThreshold > 1.0)) {
+            throw ExceptionsHelper.badRequestException("[{}] must be in [0, 1]", FEATURE_INFLUENCE_THRESHOLD.getPreferredName());
+        }
+
+        this.nNeighbors = nNeighbors;
+        this.method = method;
+        this.featureInfluenceThreshold = featureInfluenceThreshold;
+    }
+
+    /**
+     * Constructs the default outlier detection configuration
+     */
+    public OutlierDetection() {
+        this(null, null, null);
+    }
+
+    public OutlierDetection(StreamInput in) throws IOException {
+        nNeighbors = in.readOptionalVInt();
+        method = in.readBoolean() ? in.readEnum(Method.class) : null;
+        featureInfluenceThreshold = in.readOptionalDouble();
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME.getPreferredName();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalVInt(nNeighbors);
+
+        if (method != null) {
+            out.writeBoolean(true);
+            out.writeEnum(method);
+        } else {
+            out.writeBoolean(false);
+        }
+
+        out.writeOptionalDouble(featureInfluenceThreshold);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (nNeighbors != null) {
+            builder.field(N_NEIGHBORS.getPreferredName(), nNeighbors);
+        }
+        if (method != null) {
+            builder.field(METHOD.getPreferredName(), method);
+        }
+        if (featureInfluenceThreshold != null) {
+            builder.field(FEATURE_INFLUENCE_THRESHOLD.getPreferredName(), featureInfluenceThreshold);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        OutlierDetection that = (OutlierDetection) o;
+        return Objects.equals(nNeighbors, that.nNeighbors)
+            && Objects.equals(method, that.method)
+            && Objects.equals(featureInfluenceThreshold, that.featureInfluenceThreshold);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(nNeighbors, method, featureInfluenceThreshold);
+    }
+
+    @Override
+    public Map<String, Object> getParams() {
+        Map<String, Object> params = new HashMap<>();
+        if (nNeighbors != null) {
+            params.put(N_NEIGHBORS.getPreferredName(), nNeighbors);
+        }
+        if (method != null) {
+            params.put(METHOD.getPreferredName(), method);
+        }
+        if (featureInfluenceThreshold != null) {
+            params.put(FEATURE_INFLUENCE_THRESHOLD.getPreferredName(), featureInfluenceThreshold);
+        }
+        return params;
+    }
+
+    public enum Method {
+        LOF, LDOF, DISTANCE_KTH_NN, DISTANCE_KNN;
+
+        public static Method fromString(String value) {
+            return Method.valueOf(value.toUpperCase(Locale.ROOT));
+        }
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/Evaluation.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/Evaluation.java
new file mode 100644
index 0000000000000..c01c19e33e865
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/Evaluation.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+
+import java.util.List;
+
+/**
+ * Defines an evaluation
+ */
+public interface Evaluation extends ToXContentObject, NamedWriteable {
+
+    /**
+     * Returns the evaluation name
+     */
+    String getName();
+
+    /**
+     * Builds the search required to collect data to compute the evaluation result
+     */
+    SearchSourceBuilder buildSearch();
+
+    /**
+     * Computes the evaluation result
+     * @param searchResponse The search response required to compute the result
+     * @param listener A listener of the results
+     */
+    void evaluate(SearchResponse searchResponse, ActionListener<List<EvaluationMetricResult>> listener);
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationMetricResult.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationMetricResult.java
new file mode 100644
index 0000000000000..36b8adf9d4ea3
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/EvaluationMetricResult.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation;
+
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+
+/**
+ * The result of an evaluation metric
+ */
+public interface EvaluationMetricResult extends ToXContentObject, NamedWriteable {
+
+    /**
+     * Returns the name of the metric
+     */
+    String getName();
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java
new file mode 100644
index 0000000000000..f4a6dba88e3b1
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/MlEvaluationNamedXContentProvider.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.plugins.spi.NamedXContentProvider;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.AucRoc;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.BinarySoftClassification;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.ConfusionMatrix;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.Precision;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.Recall;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.ScoreByThresholdResult;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.SoftClassificationMetric;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class MlEvaluationNamedXContentProvider implements NamedXContentProvider {
+
+    @Override
+    public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
+        List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
+
+        // Evaluations
+        namedXContent.add(new NamedXContentRegistry.Entry(Evaluation.class, BinarySoftClassification.NAME,
+            BinarySoftClassification::fromXContent));
+
+        // Soft classification metrics
+        namedXContent.add(new NamedXContentRegistry.Entry(SoftClassificationMetric.class, AucRoc.NAME, AucRoc::fromXContent));
+        namedXContent.add(new NamedXContentRegistry.Entry(SoftClassificationMetric.class, Precision.NAME, Precision::fromXContent));
+        namedXContent.add(new NamedXContentRegistry.Entry(SoftClassificationMetric.class, Recall.NAME, Recall::fromXContent));
+        namedXContent.add(new NamedXContentRegistry.Entry(SoftClassificationMetric.class, ConfusionMatrix.NAME,
+            ConfusionMatrix::fromXContent));
+
+        return namedXContent;
+    }
+
+    public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+        List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
+
+        // Evaluations
+        namedWriteables.add(new NamedWriteableRegistry.Entry(Evaluation.class, BinarySoftClassification.NAME.getPreferredName(),
+            BinarySoftClassification::new));
+
+        // Evaluation Metrics
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SoftClassificationMetric.class, AucRoc.NAME.getPreferredName(),
+            AucRoc::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SoftClassificationMetric.class, Precision.NAME.getPreferredName(),
+            Precision::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SoftClassificationMetric.class, Recall.NAME.getPreferredName(),
+            Recall::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SoftClassificationMetric.class, ConfusionMatrix.NAME.getPreferredName(),
+            ConfusionMatrix::new));
+
+        // Evaluation Metrics Results
+        namedWriteables.add(new NamedWriteableRegistry.Entry(EvaluationMetricResult.class, AucRoc.NAME.getPreferredName(),
+            AucRoc.Result::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(EvaluationMetricResult.class, ScoreByThresholdResult.NAME,
+            ScoreByThresholdResult::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(EvaluationMetricResult.class, ConfusionMatrix.NAME.getPreferredName(),
+            ConfusionMatrix.Result::new));
+
+        return namedWriteables;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AbstractConfusionMatrixMetric.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AbstractConfusionMatrixMetric.java
new file mode 100644
index 0000000000000..facdcceea194f
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AbstractConfusionMatrixMetric.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+abstract class AbstractConfusionMatrixMetric implements SoftClassificationMetric {
+
+    public static final ParseField AT = new ParseField("at");
+
+    protected final double[] thresholds;
+
+    protected AbstractConfusionMatrixMetric(double[] thresholds) {
+        this.thresholds = ExceptionsHelper.requireNonNull(thresholds, AT);
+        if (thresholds.length == 0) {
+            throw ExceptionsHelper.badRequestException("[" + getMetricName() + "." + AT.getPreferredName()
+                + "] must have at least one value");
+        }
+        for (double threshold : thresholds) {
+            if (threshold < 0 || threshold > 1.0) {
+                throw ExceptionsHelper.badRequestException("[" + getMetricName() + "." + AT.getPreferredName()
+                    + "] values must be in [0.0, 1.0]");
+            }
+        }
+    }
+
+    protected AbstractConfusionMatrixMetric(StreamInput in) throws IOException {
+        this.thresholds = in.readDoubleArray();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeDoubleArray(thresholds);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(AT.getPreferredName(), thresholds);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public final List<AggregationBuilder> aggs(String actualField, List<ClassInfo> classInfos) {
+        List<AggregationBuilder> aggs = new ArrayList<>();
+        for (double threshold : thresholds) {
+            aggs.addAll(aggsAt(actualField, classInfos, threshold));
+        }
+        return aggs;
+    }
+
+    protected abstract List<AggregationBuilder> aggsAt(String labelField, List<ClassInfo> classInfos, double threshold);
+
+    protected enum Condition {
+        TP, FP, TN, FN;
+    }
+
+    protected String aggName(ClassInfo classInfo, double threshold, Condition condition) {
+        return getMetricName() + "_" + classInfo.getName() + "_at_" + threshold + "_" + condition.name();
+    }
+
+    protected AggregationBuilder buildAgg(ClassInfo classInfo, double threshold, Condition condition) {
+        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
+        switch (condition) {
+            case TP:
+                boolQuery.must(classInfo.matchingQuery());
+                boolQuery.must(QueryBuilders.rangeQuery(classInfo.getProbabilityField()).gte(threshold));
+                break;
+            case FP:
+                boolQuery.mustNot(classInfo.matchingQuery());
+                boolQuery.must(QueryBuilders.rangeQuery(classInfo.getProbabilityField()).gte(threshold));
+                break;
+            case TN:
+                boolQuery.mustNot(classInfo.matchingQuery());
+                boolQuery.must(QueryBuilders.rangeQuery(classInfo.getProbabilityField()).lt(threshold));
+                break;
+            case FN:
+                boolQuery.must(classInfo.matchingQuery());
+                boolQuery.must(QueryBuilders.rangeQuery(classInfo.getProbabilityField()).lt(threshold));
+                break;
+            default:
+                throw new IllegalArgumentException("Unknown enum value: " + condition);
+        }
+        return AggregationBuilders.filter(aggName(classInfo, threshold, condition), boolQuery);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AucRoc.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AucRoc.java
new file mode 100644
index 0000000000000..228dac00bfb68
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AucRoc.java
@@ -0,0 +1,350 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.filter.Filter;
+import org.elasticsearch.search.aggregations.metrics.Percentiles;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.IntStream;
+
+/**
+ * Area under the curve (AUC) of the receiver operating characteristic (ROC).
+ * The ROC curve is a plot of the TPR (true positive rate) against
+ * the FPR (false positive rate) over a varying threshold.
+ *
+ * This particular implementation is making use of ES aggregations
+ * to calculate the curve. It then uses the trapezoidal rule to calculate
+ * the AUC.
+ *
+ * In particular, in order to calculate the ROC, we get percentiles of TP
+ * and FP against the predicted probability. We call those Rate-Threshold
+ * curves. We then scan ROC points from each Rate-Threshold curve against the
+ * other using interpolation. This gives us an approximation of the ROC curve
+ * that has the advantage of being efficient and resilient to some edge cases.
+ *
+ * When this is used for multi-class classification, it will calculate the ROC
+ * curve of each class versus the rest.
+ */
+public class AucRoc implements SoftClassificationMetric {
+
+    public static final ParseField NAME = new ParseField("auc_roc");
+
+    public static final ParseField INCLUDE_CURVE = new ParseField("include_curve");
+
+    public static final ConstructingObjectParser<AucRoc, Void> PARSER = new ConstructingObjectParser<>(NAME.getPreferredName(),
+        a -> new AucRoc((Boolean) a[0]));
+
+    static {
+        PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), INCLUDE_CURVE);
+    }
+
+    private static final String PERCENTILES = "percentiles";
+
+    public static AucRoc fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    private final boolean includeCurve;
+
+    public AucRoc(Boolean includeCurve) {
+        this.includeCurve = includeCurve == null ? false : includeCurve;
+    }
+
+    public AucRoc(StreamInput in) throws IOException {
+        this.includeCurve = in.readBoolean();
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME.getPreferredName();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeBoolean(includeCurve);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(INCLUDE_CURVE.getPreferredName(), includeCurve);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public String getMetricName() {
+        return NAME.getPreferredName();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AucRoc that = (AucRoc) o;
+        return Objects.equals(includeCurve, that.includeCurve);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(includeCurve);
+    }
+
+    @Override
+    public List<AggregationBuilder> aggs(String actualField, List<ClassInfo> classInfos) {
+        double[] percentiles = IntStream.range(1, 100).mapToDouble(v -> (double) v).toArray();
+        List<AggregationBuilder> aggs = new ArrayList<>();
+        for (ClassInfo classInfo : classInfos) {
+            AggregationBuilder percentilesForClassValueAgg = AggregationBuilders
+                .filter(evaluatedLabelAggName(classInfo), classInfo.matchingQuery())
+                .subAggregation(
+                    AggregationBuilders.percentiles(PERCENTILES).field(classInfo.getProbabilityField()).percentiles(percentiles));
+            AggregationBuilder percentilesForRestAgg = AggregationBuilders
+                .filter(restLabelsAggName(classInfo), QueryBuilders.boolQuery().mustNot(classInfo.matchingQuery()))
+                .subAggregation(
+                    AggregationBuilders.percentiles(PERCENTILES).field(classInfo.getProbabilityField()).percentiles(percentiles));
+            aggs.add(percentilesForClassValueAgg);
+            aggs.add(percentilesForRestAgg);
+        }
+        return aggs;
+    }
+
+    private String evaluatedLabelAggName(ClassInfo classInfo) {
+        return getMetricName() + "_" + classInfo.getName();
+    }
+
+    private String restLabelsAggName(ClassInfo classInfo) {
+        return getMetricName() + "_non_" + classInfo.getName();
+    }
+
+    @Override
+    public EvaluationMetricResult evaluate(ClassInfo classInfo, Aggregations aggs) {
+        Filter classAgg = aggs.get(evaluatedLabelAggName(classInfo));
+        Filter restAgg = aggs.get(restLabelsAggName(classInfo));
+        double[] tpPercentiles = percentilesArray(classAgg.getAggregations().get(PERCENTILES),
+            "[" + getMetricName() + "] requires at least one actual_field to have the value [" + classInfo.getName() + "]");
+        double[] fpPercentiles = percentilesArray(restAgg.getAggregations().get(PERCENTILES),
+            "[" + getMetricName() + "] requires at least one actual_field to have a different value than [" + classInfo.getName() + "]");
+        List<AucRocPoint> aucRocCurve = buildAucRocCurve(tpPercentiles, fpPercentiles);
+        double aucRocScore = calculateAucScore(aucRocCurve);
+        return new Result(aucRocScore, includeCurve ? aucRocCurve : Collections.emptyList());
+    }
+
+    private static double[] percentilesArray(Percentiles percentiles, String errorIfUndefined) {
+        double[] result = new double[99];
+        percentiles.forEach(percentile -> {
+            if (Double.isNaN(percentile.getValue())) {
+                throw ExceptionsHelper.badRequestException(errorIfUndefined);
+            }
+            result[((int) percentile.getPercent()) - 1] = percentile.getValue();
+        });
+        return result;
+    }
+
+    /**
+     * Visible for testing
+     */
+    static List<AucRocPoint> buildAucRocCurve(double[] tpPercentiles, double[] fpPercentiles) {
+        assert tpPercentiles.length == fpPercentiles.length;
+        assert tpPercentiles.length == 99;
+
+        List<AucRocPoint> aucRocCurve = new ArrayList<>();
+        aucRocCurve.add(new AucRocPoint(0.0, 0.0, 1.0));
+        aucRocCurve.add(new AucRocPoint(1.0, 1.0, 0.0));
+        RateThresholdCurve tpCurve = new RateThresholdCurve(tpPercentiles, true);
+        RateThresholdCurve fpCurve = new RateThresholdCurve(fpPercentiles, false);
+        aucRocCurve.addAll(tpCurve.scanPoints(fpCurve));
+        aucRocCurve.addAll(fpCurve.scanPoints(tpCurve));
+        Collections.sort(aucRocCurve);
+        return aucRocCurve;
+    }
+
+    /**
+     * Visible for testing
+     */
+    static double calculateAucScore(List<AucRocPoint> rocCurve) {
+        // Calculates AUC based on the trapezoid rule
+        double aucRoc = 0.0;
+        for (int i = 1; i < rocCurve.size(); i++) {
+            AucRocPoint left = rocCurve.get(i - 1);
+            AucRocPoint right = rocCurve.get(i);
+            aucRoc += (right.fpr - left.fpr) * (right.tpr + left.tpr) / 2;
+        }
+        return aucRoc;
+    }
+
+    private static class RateThresholdCurve {
+
+        private final double[] percentiles;
+        private final boolean isTp;
+
+        private RateThresholdCurve(double[] percentiles, boolean isTp) {
+            this.percentiles = percentiles;
+            this.isTp = isTp;
+        }
+
+        private double getRate(int index) {
+            return 1 - 0.01 * (index + 1);
+        }
+
+        private double getThreshold(int index) {
+            return percentiles[index];
+        }
+
+        private double interpolateRate(double threshold) {
+            int binarySearchResult = Arrays.binarySearch(percentiles, threshold);
+            if (binarySearchResult >= 0) {
+                return getRate(binarySearchResult);
+            } else {
+                int right = (binarySearchResult * -1) -1;
+                int left = right - 1;
+                if (right >= percentiles.length) {
+                    return 0.0;
+                } else if (left < 0) {
+                    return 1.0;
+                } else {
+                    double rightRate = getRate(right);
+                    double leftRate = getRate(left);
+                    return interpolate(threshold, percentiles[left], leftRate, percentiles[right], rightRate);
+                }
+            }
+        }
+
+        private List<AucRocPoint> scanPoints(RateThresholdCurve againstCurve) {
+            List<AucRocPoint> points = new ArrayList<>();
+            for (int index = 0; index < percentiles.length; index++) {
+                double rate = getRate(index);
+                double scannedThreshold = getThreshold(index);
+                double againstRate = againstCurve.interpolateRate(scannedThreshold);
+                AucRocPoint point;
+                if (isTp) {
+                    point = new AucRocPoint(rate, againstRate, scannedThreshold);
+                } else {
+                    point = new AucRocPoint(againstRate, rate, scannedThreshold);
+                }
+                points.add(point);
+            }
+            return points;
+        }
+    }
+
+    public static final class AucRocPoint implements Comparable<AucRocPoint>, ToXContentObject, Writeable {
+        double tpr;
+        double fpr;
+        double threshold;
+
+        private AucRocPoint(double tpr, double fpr, double threshold) {
+            this.tpr = tpr;
+            this.fpr = fpr;
+            this.threshold = threshold;
+        }
+
+        private AucRocPoint(StreamInput in) throws IOException {
+            this.tpr = in.readDouble();
+            this.fpr = in.readDouble();
+            this.threshold = in.readDouble();
+        }
+
+        @Override
+        public int compareTo(AucRocPoint o) {
+            return Comparator.comparingDouble((AucRocPoint p) -> p.threshold).reversed()
+                .thenComparing(p -> p.fpr)
+                .thenComparing(p -> p.tpr)
+                .compare(this, o);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeDouble(tpr);
+            out.writeDouble(fpr);
+            out.writeDouble(threshold);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field("tpr", tpr);
+            builder.field("fpr", fpr);
+            builder.field("threshold", threshold);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+
+    private static double interpolate(double x, double x1, double y1, double x2, double y2) {
+        return y1 + (x - x1) * (y2 - y1) / (x2 - x1);
+    }
+
+    public static class Result implements EvaluationMetricResult {
+
+        private final double score;
+        private final List<AucRocPoint> curve;
+
+        public Result(double score, List<AucRocPoint> curve) {
+            this.score = score;
+            this.curve = Objects.requireNonNull(curve);
+        }
+
+        public Result(StreamInput in) throws IOException {
+            this.score = in.readDouble();
+            this.curve = in.readList(AucRocPoint::new);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return NAME.getPreferredName();
+        }
+
+        @Override
+        public String getName() {
+            return NAME.getPreferredName();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeDouble(score);
+            out.writeList(curve);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field("score", score);
+            if (curve.isEmpty() == false) {
+                builder.field("curve", curve);
+            }
+            builder.endObject();
+            return builder;
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/BinarySoftClassification.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/BinarySoftClassification.java
new file mode 100644
index 0000000000000..f594e7598fc20
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/BinarySoftClassification.java
@@ -0,0 +1,212 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.Evaluation;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Evaluation of binary soft classification methods, e.g. outlier detection.
+ * This is useful to evaluate problems where a model outputs a probability of whether
+ * a data frame row belongs to one of two groups.
+ */
+public class BinarySoftClassification implements Evaluation {
+
+    public static final ParseField NAME = new ParseField("binary_soft_classification");
+
+    private static final ParseField ACTUAL_FIELD = new ParseField("actual_field");
+    private static final ParseField PREDICTED_PROBABILITY_FIELD = new ParseField("predicted_probability_field");
+    private static final ParseField METRICS = new ParseField("metrics");
+
+    public static final ConstructingObjectParser<BinarySoftClassification, Void> PARSER = new ConstructingObjectParser<>(
+        NAME.getPreferredName(), a -> new BinarySoftClassification((String) a[0], (String) a[1], (List<SoftClassificationMetric>) a[2]));
+
+    static {
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), ACTUAL_FIELD);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), PREDICTED_PROBABILITY_FIELD);
+        PARSER.declareNamedObjects(ConstructingObjectParser.optionalConstructorArg(),
+            (p, c, n) -> p.namedObject(SoftClassificationMetric.class, n, null), METRICS);
+    }
+
+    public static BinarySoftClassification fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    /**
+     * The field where the actual class is marked up.
+     * The value of this field is assumed to either be 1 or 0, or true or false.
+     */
+    private final String actualField;
+
+    /**
+     * The field of the predicted probability in [0.0, 1.0].
+     */
+    private final String predictedProbabilityField;
+
+    /**
+     * The list of metrics to calculate
+     */
+    private final List<SoftClassificationMetric> metrics;
+
+    public BinarySoftClassification(String actualField, String predictedProbabilityField,
+                                    @Nullable List<SoftClassificationMetric> metrics) {
+        this.actualField = ExceptionsHelper.requireNonNull(actualField, ACTUAL_FIELD);
+        this.predictedProbabilityField = ExceptionsHelper.requireNonNull(predictedProbabilityField, PREDICTED_PROBABILITY_FIELD);
+        this.metrics = initMetrics(metrics);
+    }
+
+    private static List<SoftClassificationMetric> initMetrics(@Nullable List<SoftClassificationMetric> parsedMetrics) {
+        List<SoftClassificationMetric> metrics = parsedMetrics == null ? defaultMetrics() : parsedMetrics;
+        if (metrics.isEmpty()) {
+            throw ExceptionsHelper.badRequestException("[{}] must have one or more metrics", NAME.getPreferredName());
+        }
+        Collections.sort(metrics, Comparator.comparing(SoftClassificationMetric::getMetricName));
+        return metrics;
+    }
+
+    private static List<SoftClassificationMetric> defaultMetrics() {
+        List<SoftClassificationMetric> defaultMetrics = new ArrayList<>(4);
+        defaultMetrics.add(new AucRoc(false));
+        defaultMetrics.add(new Precision(Arrays.asList(0.25, 0.5, 0.75)));
+        defaultMetrics.add(new Recall(Arrays.asList(0.25, 0.5, 0.75)));
+        defaultMetrics.add(new ConfusionMatrix(Arrays.asList(0.25, 0.5, 0.75)));
+        return defaultMetrics;
+    }
+
+    public BinarySoftClassification(StreamInput in) throws IOException {
+        this.actualField = in.readString();
+        this.predictedProbabilityField = in.readString();
+        this.metrics = in.readNamedWriteableList(SoftClassificationMetric.class);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME.getPreferredName();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(actualField);
+        out.writeString(predictedProbabilityField);
+        out.writeNamedWriteableList(metrics);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(ACTUAL_FIELD.getPreferredName(), actualField);
+        builder.field(PREDICTED_PROBABILITY_FIELD.getPreferredName(), predictedProbabilityField);
+
+        builder.startObject(METRICS.getPreferredName());
+        for (SoftClassificationMetric metric : metrics) {
+            builder.field(metric.getMetricName(), metric);
+        }
+        builder.endObject();
+
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        BinarySoftClassification that = (BinarySoftClassification) o;
+        return Objects.equals(actualField, that.actualField)
+            && Objects.equals(predictedProbabilityField, that.predictedProbabilityField)
+            && Objects.equals(metrics, that.metrics);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(actualField, predictedProbabilityField, metrics);
+    }
+
+    @Override
+    public String getName() {
+        return NAME.getPreferredName();
+    }
+
+    @Override
+    public SearchSourceBuilder buildSearch() {
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.size(0);
+        searchSourceBuilder.query(buildQuery());
+        for (SoftClassificationMetric metric : metrics) {
+            List<AggregationBuilder> aggs = metric.aggs(actualField, Collections.singletonList(new BinaryClassInfo()));
+            aggs.forEach(searchSourceBuilder::aggregation);
+        }
+        return searchSourceBuilder;
+    }
+
+    private QueryBuilder buildQuery() {
+        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
+        boolQuery.filter(QueryBuilders.existsQuery(actualField));
+        boolQuery.filter(QueryBuilders.existsQuery(predictedProbabilityField));
+        return boolQuery;
+    }
+
+    @Override
+    public void evaluate(SearchResponse searchResponse, ActionListener<List<EvaluationMetricResult>> listener) {
+        if (searchResponse.getHits().getTotalHits().value == 0) {
+            listener.onFailure(ExceptionsHelper.badRequestException("No documents found containing both [{}, {}] fields", actualField,
+                predictedProbabilityField));
+            return;
+        }
+
+        List<EvaluationMetricResult> results = new ArrayList<>();
+        Aggregations aggs = searchResponse.getAggregations();
+        BinaryClassInfo binaryClassInfo = new BinaryClassInfo();
+        for (SoftClassificationMetric metric : metrics) {
+            results.add(metric.evaluate(binaryClassInfo, aggs));
+        }
+        listener.onResponse(results);
+    }
+
+    private class BinaryClassInfo implements SoftClassificationMetric.ClassInfo {
+
+        private QueryBuilder matchingQuery = QueryBuilders.queryStringQuery(actualField + ": (1 OR true)");
+
+        @Override
+        public String getName() {
+            return String.valueOf(true);
+        }
+
+        @Override
+        public QueryBuilder matchingQuery() {
+            return matchingQuery;
+        }
+
+        @Override
+        public String getProbabilityField() {
+            return predictedProbabilityField;
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/ConfusionMatrix.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/ConfusionMatrix.java
new file mode 100644
index 0000000000000..54f245962d515
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/ConfusionMatrix.java
@@ -0,0 +1,163 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.filter.Filter;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class ConfusionMatrix extends AbstractConfusionMatrixMetric {
+
+    public static final ParseField NAME = new ParseField("confusion_matrix");
+
+    private static final ConstructingObjectParser<ConfusionMatrix, Void> PARSER = new ConstructingObjectParser<>(NAME.getPreferredName(),
+        a -> new ConfusionMatrix((List<Double>) a[0]));
+
+    static {
+        PARSER.declareDoubleArray(ConstructingObjectParser.constructorArg(), AT);
+    }
+
+    public static ConfusionMatrix fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public ConfusionMatrix(List<Double> at) {
+        super(at.stream().mapToDouble(Double::doubleValue).toArray());
+    }
+
+    public ConfusionMatrix(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME.getPreferredName();
+    }
+
+    @Override
+    public String getMetricName() {
+        return NAME.getPreferredName();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ConfusionMatrix that = (ConfusionMatrix) o;
+        return Arrays.equals(thresholds, that.thresholds);
+    }
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(thresholds);
+    }
+
+    @Override
+    protected List<AggregationBuilder> aggsAt(String labelField, List<ClassInfo> classInfos, double threshold) {
+        List<AggregationBuilder> aggs = new ArrayList<>();
+        for (ClassInfo classInfo : classInfos) {
+            aggs.add(buildAgg(classInfo, threshold, Condition.TP));
+            aggs.add(buildAgg(classInfo, threshold, Condition.FP));
+            aggs.add(buildAgg(classInfo, threshold, Condition.TN));
+            aggs.add(buildAgg(classInfo, threshold, Condition.FN));
+        }
+        return aggs;
+    }
+
+    @Override
+    public EvaluationMetricResult evaluate(ClassInfo classInfo, Aggregations aggs) {
+        long[] tp = new long[thresholds.length];
+        long[] fp = new long[thresholds.length];
+        long[] tn = new long[thresholds.length];
+        long[] fn = new long[thresholds.length];
+        for (int i = 0; i < thresholds.length; i++) {
+            Filter tpAgg = aggs.get(aggName(classInfo, thresholds[i], Condition.TP));
+            Filter fpAgg = aggs.get(aggName(classInfo, thresholds[i], Condition.FP));
+            Filter tnAgg = aggs.get(aggName(classInfo, thresholds[i], Condition.TN));
+            Filter fnAgg = aggs.get(aggName(classInfo, thresholds[i], Condition.FN));
+            tp[i] = tpAgg.getDocCount();
+            fp[i] = fpAgg.getDocCount();
+            tn[i] = tnAgg.getDocCount();
+            fn[i] = fnAgg.getDocCount();
+        }
+        return new Result(thresholds, tp, fp, tn, fn);
+    }
+
+    public static class Result implements EvaluationMetricResult {
+
+        private final double[] thresholds;
+        private final long[] tp;
+        private final long[] fp;
+        private final long[] tn;
+        private final long[] fn;
+
+        public Result(double[] thresholds, long[] tp, long[] fp, long[] tn, long[] fn) {
+            assert thresholds.length == tp.length;
+            assert thresholds.length == fp.length;
+            assert thresholds.length == tn.length;
+            assert thresholds.length == fn.length;
+            this.thresholds = thresholds;
+            this.tp = tp;
+            this.fp = fp;
+            this.tn = tn;
+            this.fn = fn;
+        }
+
+        public Result(StreamInput in) throws IOException {
+            this.thresholds = in.readDoubleArray();
+            this.tp = in.readLongArray();
+            this.fp = in.readLongArray();
+            this.tn = in.readLongArray();
+            this.fn = in.readLongArray();
+        }
+
+        @Override
+        public String getWriteableName() {
+            return NAME.getPreferredName();
+        }
+
+        @Override
+        public String getName() {
+            return NAME.getPreferredName();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeDoubleArray(thresholds);
+            out.writeLongArray(tp);
+            out.writeLongArray(fp);
+            out.writeLongArray(tn);
+            out.writeLongArray(fn);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            for (int i = 0; i < thresholds.length; i++) {
+                builder.startObject(String.valueOf(thresholds[i]));
+                builder.field("tp", tp[i]);
+                builder.field("fp", fp[i]);
+                builder.field("tn", tn[i]);
+                builder.field("fn", fn[i]);
+                builder.endObject();
+            }
+            builder.endObject();
+            return builder;
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/Precision.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/Precision.java
new file mode 100644
index 0000000000000..d38a52bb203e8
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/Precision.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.filter.Filter;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class Precision extends AbstractConfusionMatrixMetric {
+
+    public static final ParseField NAME = new ParseField("precision");
+
+    private static final ConstructingObjectParser<Precision, Void> PARSER = new ConstructingObjectParser<>(NAME.getPreferredName(),
+        a -> new Precision((List<Double>) a[0]));
+
+    static {
+        PARSER.declareDoubleArray(ConstructingObjectParser.constructorArg(), AT);
+    }
+
+    public static Precision fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public Precision(List<Double> at) {
+        super(at.stream().mapToDouble(Double::doubleValue).toArray());
+    }
+
+    public Precision(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME.getPreferredName();
+    }
+
+    @Override
+    public String getMetricName() {
+        return NAME.getPreferredName();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Precision that = (Precision) o;
+        return Arrays.equals(thresholds, that.thresholds);
+    }
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(thresholds);
+    }
+
+    @Override
+    protected List<AggregationBuilder> aggsAt(String labelField, List<ClassInfo> classInfos, double threshold) {
+        List<AggregationBuilder> aggs = new ArrayList<>();
+        for (ClassInfo classInfo : classInfos) {
+            aggs.add(buildAgg(classInfo, threshold, Condition.TP));
+            aggs.add(buildAgg(classInfo, threshold, Condition.FP));
+        }
+        return aggs;
+    }
+
+    @Override
+    public EvaluationMetricResult evaluate(ClassInfo classInfo, Aggregations aggs) {
+        double[] precisions = new double[thresholds.length];
+        for (int i = 0; i < precisions.length; i++) {
+            double threshold = thresholds[i];
+            Filter tpAgg = aggs.get(aggName(classInfo, threshold, Condition.TP));
+            Filter fpAgg = aggs.get(aggName(classInfo, threshold, Condition.FP));
+            long tp = tpAgg.getDocCount();
+            long fp = fpAgg.getDocCount();
+            precisions[i] = tp + fp == 0 ? 0.0 : (double) tp / (tp + fp);
+        }
+        return new ScoreByThresholdResult(NAME.getPreferredName(), thresholds, precisions);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/Recall.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/Recall.java
new file mode 100644
index 0000000000000..5c4ab57241d95
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/Recall.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.filter.Filter;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class Recall extends AbstractConfusionMatrixMetric {
+
+    public static final ParseField NAME = new ParseField("recall");
+
+    private static final ConstructingObjectParser<Recall, Void> PARSER = new ConstructingObjectParser<>(NAME.getPreferredName(),
+        a -> new Recall((List<Double>) a[0]));
+
+    static {
+        PARSER.declareDoubleArray(ConstructingObjectParser.constructorArg(), AT);
+    }
+
+    public static Recall fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
+    public Recall(List<Double> at) {
+        super(at.stream().mapToDouble(Double::doubleValue).toArray());
+    }
+
+    public Recall(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME.getPreferredName();
+    }
+
+    @Override
+    public String getMetricName() {
+        return NAME.getPreferredName();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Recall that = (Recall) o;
+        return Arrays.equals(thresholds, that.thresholds);
+    }
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(thresholds);
+    }
+
+    @Override
+    protected List<AggregationBuilder> aggsAt(String actualField, List<ClassInfo> classInfos, double threshold) {
+        List<AggregationBuilder> aggs = new ArrayList<>();
+        for (ClassInfo classInfo: classInfos) {
+            aggs.add(buildAgg(classInfo, threshold, Condition.TP));
+            aggs.add(buildAgg(classInfo, threshold, Condition.FN));
+        }
+        return aggs;
+    }
+
+    @Override
+    public EvaluationMetricResult evaluate(ClassInfo classInfo, Aggregations aggs) {
+        double[] recalls = new double[thresholds.length];
+        for (int i = 0; i < recalls.length; i++) {
+            double threshold = thresholds[i];
+            Filter tpAgg = aggs.get(aggName(classInfo, threshold, Condition.TP));
+            Filter fnAgg =aggs.get(aggName(classInfo, threshold, Condition.FN));
+            long tp = tpAgg.getDocCount();
+            long fn = fnAgg.getDocCount();
+            recalls[i] = tp + fn == 0 ? 0.0 : (double) tp / (tp + fn);
+        }
+        return new ScoreByThresholdResult(NAME.getPreferredName(), thresholds, recalls);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/ScoreByThresholdResult.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/ScoreByThresholdResult.java
new file mode 100644
index 0000000000000..bd6b6e7db25a1
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/ScoreByThresholdResult.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class ScoreByThresholdResult implements EvaluationMetricResult {
+
+    public static final String NAME = "score_by_threshold_result";
+
+    private final String name;
+    private final double[] thresholds;
+    private final double[] scores;
+
+    public ScoreByThresholdResult(String name, double[] thresholds, double[] scores) {
+        assert thresholds.length == scores.length;
+        this.name = Objects.requireNonNull(name);
+        this.thresholds = thresholds;
+        this.scores = scores;
+    }
+
+    public ScoreByThresholdResult(StreamInput in) throws IOException {
+        this.name = in.readString();
+        this.thresholds = in.readDoubleArray();
+        this.scores = in.readDoubleArray();
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name);
+        out.writeDoubleArray(thresholds);
+        out.writeDoubleArray(scores);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        for (int i = 0; i < thresholds.length; i++) {
+            builder.field(String.valueOf(thresholds[i]), scores[i]);
+        }
+        builder.endObject();
+        return builder;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/SoftClassificationMetric.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/SoftClassificationMetric.java
new file mode 100644
index 0000000000000..dfb256e9b52f2
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/SoftClassificationMetric.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+
+import java.util.List;
+
+public interface SoftClassificationMetric extends ToXContentObject, NamedWriteable {
+
+    /**
+     * The information of a specific class
+     */
+    interface ClassInfo {
+
+        /**
+         * Returns the class name
+         */
+        String getName();
+
+        /**
+         * Returns a query that matches documents of the class
+         */
+        QueryBuilder matchingQuery();
+
+        /**
+         * Returns the field that has the probability to be of the class
+         */
+        String getProbabilityField();
+    }
+
+    /**
+     * Returns the name of the metric (which may differ to the writeable name)
+     */
+    String getMetricName();
+
+    /**
+     * Builds the aggregation that collect required data to compute the metric
+     * @param actualField the field that stores the actual class
+     * @param classInfos the information of each class to compute the metric for
+     * @return the aggregations required to compute the metric
+     */
+    List<AggregationBuilder> aggs(String actualField, List<ClassInfo> classInfos);
+
+    /**
+     * Calculates the metric result for a given class
+     * @param classInfo the class to calculate the metric for
+     * @param aggs the aggregations
+     * @return the metric result
+     */
+    EvaluationMetricResult evaluate(ClassInfo classInfo, Aggregations aggs);
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/filestructurefinder/FieldStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/filestructurefinder/FieldStats.java
index b371ca739bb6b..95abf16aa1444 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/filestructurefinder/FieldStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/filestructurefinder/FieldStats.java
@@ -5,7 +5,9 @@
  */
 package org.elasticsearch.xpack.core.ml.filestructurefinder;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -27,12 +29,14 @@ public class FieldStats implements ToXContentObject, Writeable {
     static final ParseField MAX_VALUE = new ParseField("max_value");
     static final ParseField MEAN_VALUE = new ParseField("mean_value");
     static final ParseField MEDIAN_VALUE = new ParseField("median_value");
+    static final ParseField EARLIEST = new ParseField("earliest");
+    static final ParseField LATEST = new ParseField("latest");
     static final ParseField TOP_HITS = new ParseField("top_hits");
 
     @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<FieldStats, Void> PARSER = new ConstructingObjectParser<>("field_stats", false,
         a -> new FieldStats((long) a[0], (int) a[1], (Double) a[2], (Double) a[3], (Double) a[4], (Double) a[5],
-            (List<Map<String, Object>>) a[6]));
+            (String) a[6], (String) a[7], (List<Map<String, Object>>) a[8]));
 
     static {
         PARSER.declareLong(ConstructingObjectParser.constructorArg(), COUNT);
@@ -41,6 +45,8 @@ public class FieldStats implements ToXContentObject, Writeable {
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), MAX_VALUE);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), MEAN_VALUE);
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), MEDIAN_VALUE);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), EARLIEST);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), LATEST);
         PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.mapOrdered(), TOP_HITS);
     }
 
@@ -50,20 +56,33 @@ public class FieldStats implements ToXContentObject, Writeable {
     private final Double maxValue;
     private final Double meanValue;
     private final Double medianValue;
+    private final String earliestTimestamp;
+    private final String latestTimestamp;
     private final List<Map<String, Object>> topHits;
 
     public FieldStats(long count, int cardinality, List<Map<String, Object>> topHits) {
-        this(count, cardinality, null, null, null, null, topHits);
+        this(count, cardinality, null, null, null, null, null, null, topHits);
+    }
+
+    public FieldStats(long count, int cardinality, String earliestTimestamp, String latestTimestamp, List<Map<String, Object>> topHits) {
+        this(count, cardinality, null, null, null, null, earliestTimestamp, latestTimestamp, topHits);
     }
 
     public FieldStats(long count, int cardinality, Double minValue, Double maxValue, Double meanValue, Double medianValue,
                       List<Map<String, Object>> topHits) {
+        this(count, cardinality, minValue, maxValue, meanValue, medianValue, null, null, topHits);
+    }
+
+    FieldStats(long count, int cardinality, Double minValue, Double maxValue, Double meanValue, Double medianValue,
+               String earliestTimestamp, String latestTimestamp, List<Map<String, Object>> topHits) {
         this.count = count;
         this.cardinality = cardinality;
         this.minValue = minValue;
         this.maxValue = maxValue;
         this.meanValue = meanValue;
         this.medianValue = medianValue;
+        this.earliestTimestamp = earliestTimestamp;
+        this.latestTimestamp = latestTimestamp;
         this.topHits = (topHits == null) ? Collections.emptyList() : Collections.unmodifiableList(topHits);
     }
 
@@ -74,6 +93,13 @@ public FieldStats(StreamInput in) throws IOException {
         maxValue = in.readOptionalDouble();
         meanValue = in.readOptionalDouble();
         medianValue = in.readOptionalDouble();
+        if (in.getVersion().onOrAfter(Version.V_7_3_0)) {
+            earliestTimestamp = in.readOptionalString();
+            latestTimestamp = in.readOptionalString();
+        } else {
+            earliestTimestamp = null;
+            latestTimestamp = null;
+        }
         topHits = in.readList(StreamInput::readMap);
     }
 
@@ -85,6 +111,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalDouble(maxValue);
         out.writeOptionalDouble(meanValue);
         out.writeOptionalDouble(medianValue);
+        if (out.getVersion().onOrAfter(Version.V_7_3_0)) {
+            out.writeOptionalString(earliestTimestamp);
+            out.writeOptionalString(latestTimestamp);
+        }
         out.writeCollection(topHits, StreamOutput::writeMap);
     }
 
@@ -112,6 +142,14 @@ public Double getMedianValue() {
         return medianValue;
     }
 
+    public String getEarliestTimestamp() {
+        return earliestTimestamp;
+    }
+
+    public String getLatestTimestamp() {
+        return latestTimestamp;
+    }
+
     public List<Map<String, Object>> getTopHits() {
         return topHits;
     }
@@ -134,6 +172,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (medianValue != null) {
             builder.field(MEDIAN_VALUE.getPreferredName(), toIntegerIfInteger(medianValue));
         }
+        if (earliestTimestamp != null) {
+            builder.field(EARLIEST.getPreferredName(), earliestTimestamp);
+        }
+        if (latestTimestamp != null) {
+            builder.field(LATEST.getPreferredName(), latestTimestamp);
+        }
         if (topHits.isEmpty() == false) {
             builder.field(TOP_HITS.getPreferredName(), topHits);
         }
@@ -154,7 +198,7 @@ public static Number toIntegerIfInteger(double d) {
     @Override
     public int hashCode() {
 
-        return Objects.hash(count, cardinality, minValue, maxValue, meanValue, medianValue, topHits);
+        return Objects.hash(count, cardinality, minValue, maxValue, meanValue, medianValue, earliestTimestamp, latestTimestamp, topHits);
     }
 
     @Override
@@ -175,6 +219,13 @@ public boolean equals(Object other) {
             Objects.equals(this.maxValue, that.maxValue) &&
             Objects.equals(this.meanValue, that.meanValue) &&
             Objects.equals(this.medianValue, that.medianValue) &&
+            Objects.equals(this.earliestTimestamp, that.earliestTimestamp) &&
+            Objects.equals(this.latestTimestamp, that.latestTimestamp) &&
             Objects.equals(this.topHits, that.topHits);
     }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java
index 9e01cd21e2b90..07b823f27768c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java
@@ -17,7 +17,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.io.IOException;
 import java.util.ArrayList;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java
index e0b66e30f2496..da820b1919cae 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/CategorizationAnalyzerConfig.java
@@ -6,16 +6,13 @@
 package org.elasticsearch.xpack.core.ml.job.config;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.analysis.NameOrDefinition;
 import org.elasticsearch.rest.action.admin.indices.RestAnalyzeAction;
 
 import java.io.IOException;
@@ -176,87 +173,6 @@ public static CategorizationAnalyzerConfig buildDefaultCategorizationAnalyzer(Li
         return builder.build();
     }
 
-    /**
-     * Simple store of either a name of a built-in analyzer element or a custom definition.
-     */
-    public static class NameOrDefinition implements ToXContentFragment, Writeable {
-
-        // Exactly one of these two members is not null
-        public final String name;
-        public final Settings definition;
-
-        NameOrDefinition(String name) {
-            this.name = Objects.requireNonNull(name);
-            this.definition = null;
-        }
-
-        NameOrDefinition(ParseField field, Map<String, Object> definition) {
-            this.name = null;
-            Objects.requireNonNull(definition);
-            try {
-                XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-                builder.map(definition);
-                this.definition = Settings.builder().loadFromSource(Strings.toString(builder), builder.contentType()).build();
-            } catch (IOException e) {
-                throw new IllegalArgumentException("Failed to parse [" + definition + "] in [" + field.getPreferredName() + "]", e);
-            }
-        }
-
-        NameOrDefinition(StreamInput in) throws IOException {
-            name = in.readOptionalString();
-            if (in.readBoolean()) {
-                definition = Settings.readSettingsFromStream(in);
-            } else {
-                definition = null;
-            }
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeOptionalString(name);
-            boolean isNotNullDefinition = this.definition != null;
-            out.writeBoolean(isNotNullDefinition);
-            if (isNotNullDefinition) {
-                Settings.writeSettingsToStream(definition, out);
-            }
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            if (definition == null) {
-                builder.value(name);
-            } else {
-                builder.startObject();
-                definition.toXContent(builder, params);
-                builder.endObject();
-            }
-            return builder;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            NameOrDefinition that = (NameOrDefinition) o;
-            return Objects.equals(name, that.name) &&
-                    Objects.equals(definition, that.definition);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(name, definition);
-        }
-
-        @Override
-        public String toString() {
-            if (definition == null) {
-                return name;
-            } else {
-                return definition.toDelimitedString(';');
-            }
-        }
-    }
-
     private final String analyzer;
     private final List<NameOrDefinition> charFilters;
     private final NameOrDefinition tokenizer;
@@ -373,7 +289,7 @@ public Builder addCharFilter(String charFilter) {
         }
 
         public Builder addCharFilter(Map<String, Object> charFilter) {
-            this.charFilters.add(new NameOrDefinition(CHAR_FILTERS, charFilter));
+            this.charFilters.add(new NameOrDefinition(charFilter));
             return this;
         }
 
@@ -383,7 +299,7 @@ public Builder setTokenizer(String tokenizer) {
         }
 
         public Builder setTokenizer(Map<String, Object> tokenizer) {
-            this.tokenizer = new NameOrDefinition(TOKENIZER, tokenizer);
+            this.tokenizer = new NameOrDefinition(tokenizer);
             return this;
         }
 
@@ -393,7 +309,7 @@ public Builder addTokenFilter(String tokenFilter) {
         }
 
         public Builder addTokenFilter(Map<String, Object> tokenFilter) {
-            this.tokenFilters.add(new NameOrDefinition(TOKEN_FILTERS, tokenFilter));
+            this.tokenFilters.add(new NameOrDefinition(tokenFilter));
             return this;
         }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java
index 25cd0cffe7b39..8ee63f6c11ea2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.core.ml.job.config;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -26,8 +25,6 @@
 
 public class DetectionRule implements ToXContentObject, Writeable {
 
-    public static final Version VERSION_INTRODUCED = Version.V_6_4_0;
-
     public static final ParseField DETECTION_RULE_FIELD = new ParseField("detection_rule");
     public static final ParseField ACTIONS_FIELD = new ParseField("actions");
     public static final ParseField SCOPE_FIELD = new ParseField("scope");
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java
index b27149ef412a5..4903a1383bcdd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java
@@ -246,11 +246,7 @@ public void writeTo(StreamOutput out) throws IOException {
         } else {
             out.writeBoolean(false);
         }
-        if (out.getVersion().onOrAfter(DetectionRule.VERSION_INTRODUCED)) {
-            out.writeList(rules);
-        } else {
-            out.writeList(Collections.emptyList());
-        }
+        out.writeList(rules);
         out.writeInt(detectorIndex);
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java
index 1cb44f9625cb5..5a2b2314e8d53 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java
@@ -26,7 +26,7 @@
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.MlStrings;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.io.IOException;
 import java.util.ArrayList;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java
index b50b7d2fa5126..c1058e9c0639e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java
@@ -132,11 +132,7 @@ public JobUpdate(StreamInput in) throws IOException {
         } else {
             jobVersion = null;
         }
-        if (in.getVersion().onOrAfter(Version.V_6_6_0)) {
-            clearJobFinishTime = in.readOptionalBoolean();
-        } else {
-            clearJobFinishTime = null;
-        }
+        clearJobFinishTime = in.readOptionalBoolean();
         if (in.getVersion().onOrAfter(Version.V_7_0_0) && in.readBoolean()) {
             modelSnapshotMinVersion = Version.readVersion(in);
         } else {
@@ -172,9 +168,7 @@ public void writeTo(StreamOutput out) throws IOException {
         } else {
             out.writeBoolean(false);
         }
-        if (out.getVersion().onOrAfter(Version.V_6_6_0)) {
-            out.writeOptionalBoolean(clearJobFinishTime);
-        }
+        out.writeOptionalBoolean(clearJobFinishTime);
         if (out.getVersion().onOrAfter(Version.V_7_0_0)) {
             if (modelSnapshotMinVersion != null) {
                 out.writeBoolean(true);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/MlFilter.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/MlFilter.java
index f2be3315b4dc7..4c60a4795e050 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/MlFilter.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/MlFilter.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.core.ml.job.config;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -76,20 +75,14 @@ private MlFilter(String id, String description, SortedSet<String> items) {
 
     public MlFilter(StreamInput in) throws IOException {
         id = in.readString();
-        if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
-            description = in.readOptionalString();
-        } else {
-            description = null;
-        }
+        description = in.readOptionalString();
         items = new TreeSet<>(Arrays.asList(in.readStringArray()));
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(id);
-        if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
-            out.writeOptionalString(description);
-        }
+        out.writeOptionalString(description);
         out.writeStringArray(items.toArray(new String[items.size()]));
     }
 
@@ -201,4 +194,4 @@ public MlFilter build() {
             return new MlFilter(id, description, items);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
index 22eb0dc357bed..dfb95d2adac33 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java
@@ -50,6 +50,9 @@ public final class Messages {
             "Datafeed frequency [{0}] must be a multiple of the aggregation interval [{1}]";
     public static final String DATAFEED_ID_ALREADY_TAKEN = "A datafeed with id [{0}] already exists";
 
+    public static final String DATA_FRAME_ANALYTICS_BAD_QUERY_FORMAT = "Data Frame Analytics config query is not parsable";
+    public static final String DATA_FRAME_ANALYTICS_BAD_FIELD_FILTER = "No field [{0}] could be detected";
+
     public static final String FILTER_CANNOT_DELETE = "Cannot delete filter [{0}] currently used by jobs {1}";
     public static final String FILTER_CONTAINS_TOO_MANY_ITEMS = "Filter [{0}] contains too many items; up to [{1}] items are allowed";
     public static final String FILTER_NOT_FOUND = "No filter with id [{0}] exists";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java
index 986c352b9cd7b..0fc7770758ad3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java
@@ -26,6 +26,10 @@
 import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DelayedDataCheckConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsSource;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetection;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
 import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
@@ -39,6 +43,7 @@
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshotField;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 import org.elasticsearch.xpack.core.ml.job.results.AnomalyCause;
 import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord;
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
@@ -143,6 +148,7 @@ public static XContentBuilder configMapping() throws IOException {
 
         addJobConfigFields(builder);
         addDatafeedConfigFields(builder);
+        addDataFrameAnalyticsFields(builder);
 
         builder.endObject()
                .endObject()
@@ -385,6 +391,60 @@ public static void addDatafeedConfigFields(XContentBuilder builder) throws IOExc
         .endObject();
     }
 
+    /**
+     * {@link DataFrameAnalyticsConfig} mapping.
+     * Does not include mapping for CREATE_TIME as this mapping is added by {@link #addJobConfigFields} method.
+     */
+    public static void addDataFrameAnalyticsFields(XContentBuilder builder) throws IOException {
+        builder.startObject(DataFrameAnalyticsConfig.ID.getPreferredName())
+            .field(TYPE, KEYWORD)
+        .endObject()
+        .startObject(DataFrameAnalyticsConfig.SOURCE.getPreferredName())
+            .startObject(PROPERTIES)
+                .startObject(DataFrameAnalyticsSource.INDEX.getPreferredName())
+                    .field(TYPE, KEYWORD)
+                .endObject()
+                .startObject(DataFrameAnalyticsSource.QUERY.getPreferredName())
+                    .field(ENABLED, false)
+                .endObject()
+            .endObject()
+        .endObject()
+        .startObject(DataFrameAnalyticsConfig.DEST.getPreferredName())
+            .startObject(PROPERTIES)
+                .startObject(DataFrameAnalyticsDest.INDEX.getPreferredName())
+                    .field(TYPE, KEYWORD)
+                .endObject()
+                .startObject(DataFrameAnalyticsDest.RESULTS_FIELD.getPreferredName())
+                    .field(TYPE, KEYWORD)
+                .endObject()
+            .endObject()
+        .endObject()
+        .startObject(DataFrameAnalyticsConfig.ANALYZED_FIELDS.getPreferredName())
+            .field(ENABLED, false)
+        .endObject()
+        .startObject(DataFrameAnalyticsConfig.ANALYSIS.getPreferredName())
+            .startObject(PROPERTIES)
+                .startObject(OutlierDetection.NAME.getPreferredName())
+                    .startObject(PROPERTIES)
+                        .startObject(OutlierDetection.N_NEIGHBORS.getPreferredName())
+                            .field(TYPE, INTEGER)
+                        .endObject()
+                        .startObject(OutlierDetection.METHOD.getPreferredName())
+                            .field(TYPE, KEYWORD)
+                        .endObject()
+                        .startObject(OutlierDetection.FEATURE_INFLUENCE_THRESHOLD.getPreferredName())
+                            .field(TYPE, DOUBLE)
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject()
+        // re-used: CREATE_TIME
+        .startObject(DataFrameAnalyticsConfig.VERSION.getPreferredName())
+            .field(TYPE, KEYWORD)
+        .endObject();
+    }
+
     /**
      * Creates a default mapping which has a dynamic template that
      * treats all dynamically added fields as keywords. This is needed
@@ -449,6 +509,7 @@ public static XContentBuilder resultsMapping(String mappingType, Collection<Stri
         addResultsMapping(builder);
         addCategoryDefinitionMapping(builder);
         addDataCountsMapping(builder);
+        addTimingStatsExceptBucketCountMapping(builder);
         addModelSnapshotMapping(builder);
 
         addTermFields(builder, extraTermFields);
@@ -790,8 +851,6 @@ private static void addInfluencerFieldsToMapping(XContentBuilder builder) throws
 
     /**
      * {@link DataCounts} mapping.
-     * The type is disabled so {@link DataCounts} aren't searchable and
-     * the '_all' field is disabled
      *
      * @throws IOException On builder write error
      */
@@ -846,6 +905,29 @@ private static void addDataCountsMapping(XContentBuilder builder) throws IOExcep
         .endObject();
     }
 
+    /**
+     * {@link TimingStats} mapping.
+     * Does not include mapping for BUCKET_COUNT as this mapping is added by {@link #addDataCountsMapping} method.
+     *
+     * @throws IOException On builder write error
+     */
+    private static void addTimingStatsExceptBucketCountMapping(XContentBuilder builder) throws IOException {
+        builder
+            // re-used: BUCKET_COUNT
+            .startObject(TimingStats.MIN_BUCKET_PROCESSING_TIME_MS.getPreferredName())
+                .field(TYPE, DOUBLE)
+            .endObject()
+            .startObject(TimingStats.MAX_BUCKET_PROCESSING_TIME_MS.getPreferredName())
+                .field(TYPE, DOUBLE)
+            .endObject()
+            .startObject(TimingStats.AVG_BUCKET_PROCESSING_TIME_MS.getPreferredName())
+                .field(TYPE, DOUBLE)
+            .endObject()
+            .startObject(TimingStats.EXPONENTIAL_AVG_BUCKET_PROCESSING_TIME_MS.getPreferredName())
+                .field(TYPE, DOUBLE)
+            .endObject();
+    }
+
     /**
      * Create the Elasticsearch mapping for {@linkplain CategoryDefinition}.
      * The '_all' field is disabled as the document isn't meant to be searched.
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/output/FlushAcknowledgement.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/output/FlushAcknowledgement.java
index ff47cfe1ca85c..932e51a60f151 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/output/FlushAcknowledgement.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/output/FlushAcknowledgement.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.io.IOException;
 import java.util.Date;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCounts.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCounts.java
index 8d542ce25af93..13e6459ca37eb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCounts.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCounts.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.io.IOException;
 import java.util.Date;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java
index f02120433efc4..2e78ab39fe6e3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.results.Result;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.io.IOException;
 import java.util.Date;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java
index 02bef36c00ab0..268da28b1d07e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java
@@ -22,7 +22,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.io.IOException;
 import java.io.InputStream;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStats.java
new file mode 100644
index 0000000000000..0bcc4ea5f45d2
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStats.java
@@ -0,0 +1,222 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.job.process.autodetect.state;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.ml.job.config.Job;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * Stats that give more insight into timing of various operations performed as part of anomaly detection job.
+ */
+public class TimingStats implements ToXContentObject, Writeable {
+
+    public static final ParseField BUCKET_COUNT = new ParseField("bucket_count");
+    public static final ParseField MIN_BUCKET_PROCESSING_TIME_MS = new ParseField("minimum_bucket_processing_time_ms");
+    public static final ParseField MAX_BUCKET_PROCESSING_TIME_MS = new ParseField("maximum_bucket_processing_time_ms");
+    public static final ParseField AVG_BUCKET_PROCESSING_TIME_MS = new ParseField("average_bucket_processing_time_ms");
+    public static final ParseField EXPONENTIAL_AVG_BUCKET_PROCESSING_TIME_MS =
+        new ParseField("exponential_average_bucket_processing_time_ms");
+
+    public static final ParseField TYPE = new ParseField("timing_stats");
+
+    public static final ConstructingObjectParser<TimingStats, Void> PARSER =
+        new ConstructingObjectParser<>(
+            TYPE.getPreferredName(),
+            true,
+            args ->
+                new TimingStats((String) args[0], (long) args[1], (Double) args[2], (Double) args[3], (Double) args[4], (Double) args[5]));
+
+    static {
+        PARSER.declareString(constructorArg(), Job.ID);
+        PARSER.declareLong(constructorArg(), BUCKET_COUNT);
+        PARSER.declareDouble(optionalConstructorArg(), MIN_BUCKET_PROCESSING_TIME_MS);
+        PARSER.declareDouble(optionalConstructorArg(), MAX_BUCKET_PROCESSING_TIME_MS);
+        PARSER.declareDouble(optionalConstructorArg(), AVG_BUCKET_PROCESSING_TIME_MS);
+        PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_BUCKET_PROCESSING_TIME_MS);
+    }
+
+    public static String documentId(String jobId) {
+        return jobId + "_timing_stats";
+    }
+
+    private final String jobId;
+    private long bucketCount;
+    private Double minBucketProcessingTimeMs;
+    private Double maxBucketProcessingTimeMs;
+    private Double avgBucketProcessingTimeMs;
+    private Double exponentialAvgBucketProcessingTimeMs;
+
+    public TimingStats(
+            String jobId,
+            long bucketCount,
+            @Nullable Double minBucketProcessingTimeMs,
+            @Nullable Double maxBucketProcessingTimeMs,
+            @Nullable Double avgBucketProcessingTimeMs,
+            @Nullable Double exponentialAvgBucketProcessingTimeMs) {
+        this.jobId = jobId;
+        this.bucketCount = bucketCount;
+        this.minBucketProcessingTimeMs = minBucketProcessingTimeMs;
+        this.maxBucketProcessingTimeMs = maxBucketProcessingTimeMs;
+        this.avgBucketProcessingTimeMs = avgBucketProcessingTimeMs;
+        this.exponentialAvgBucketProcessingTimeMs = exponentialAvgBucketProcessingTimeMs;
+    }
+
+    public TimingStats(String jobId) {
+        this(jobId, 0, null, null, null, null);
+    }
+
+    public TimingStats(TimingStats lhs) {
+        this(
+            lhs.jobId,
+            lhs.bucketCount,
+            lhs.minBucketProcessingTimeMs,
+            lhs.maxBucketProcessingTimeMs,
+            lhs.avgBucketProcessingTimeMs,
+            lhs.exponentialAvgBucketProcessingTimeMs);
+    }
+
+    public TimingStats(StreamInput in) throws IOException {
+        this.jobId = in.readString();
+        this.bucketCount = in.readLong();
+        this.minBucketProcessingTimeMs = in.readOptionalDouble();
+        this.maxBucketProcessingTimeMs = in.readOptionalDouble();
+        this.avgBucketProcessingTimeMs = in.readOptionalDouble();
+        this.exponentialAvgBucketProcessingTimeMs = in.readOptionalDouble();
+    }
+
+    public String getJobId() {
+        return jobId;
+    }
+
+    public long getBucketCount() {
+        return bucketCount;
+    }
+
+    public Double getMinBucketProcessingTimeMs() {
+        return minBucketProcessingTimeMs;
+    }
+
+    public Double getMaxBucketProcessingTimeMs() {
+        return maxBucketProcessingTimeMs;
+    }
+
+    public Double getAvgBucketProcessingTimeMs() {
+        return avgBucketProcessingTimeMs;
+    }
+
+    public Double getExponentialAvgBucketProcessingTimeMs() {
+        return exponentialAvgBucketProcessingTimeMs;
+    }
+
+    /**
+     * Updates the statistics (min, max, avg) for the given data point (bucket processing time).
+     */
+    public void updateStats(double bucketProcessingTimeMs) {
+        if (bucketProcessingTimeMs < 0.0) {
+            throw new IllegalArgumentException("bucketProcessingTimeMs must be non-negative, was: " + bucketProcessingTimeMs);
+        }
+        if (minBucketProcessingTimeMs == null || bucketProcessingTimeMs < minBucketProcessingTimeMs) {
+            minBucketProcessingTimeMs = bucketProcessingTimeMs;
+        }
+        if (maxBucketProcessingTimeMs == null || bucketProcessingTimeMs > maxBucketProcessingTimeMs) {
+            maxBucketProcessingTimeMs = bucketProcessingTimeMs;
+        }
+        if (avgBucketProcessingTimeMs == null) {
+            avgBucketProcessingTimeMs = bucketProcessingTimeMs;
+        } else {
+            // Calculate the cumulative moving average (see https://en.wikipedia.org/wiki/Moving_average#Cumulative_moving_average) of
+            // bucket processing times.
+            avgBucketProcessingTimeMs = (bucketCount * avgBucketProcessingTimeMs + bucketProcessingTimeMs) / (bucketCount + 1);
+        }
+        if (exponentialAvgBucketProcessingTimeMs == null) {
+            exponentialAvgBucketProcessingTimeMs = bucketProcessingTimeMs;
+        } else {
+            // Calculate the exponential moving average (see https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average) of
+            // bucket processing times.
+            exponentialAvgBucketProcessingTimeMs = (1 - ALPHA) * exponentialAvgBucketProcessingTimeMs + ALPHA * bucketProcessingTimeMs;
+        }
+        bucketCount++;
+    }
+
+    /**
+     * Constant smoothing factor used for calculating exponential moving average. Represents the degree of weighting decrease.
+     */
+    private static double ALPHA = 0.01;
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(jobId);
+        out.writeLong(bucketCount);
+        out.writeOptionalDouble(minBucketProcessingTimeMs);
+        out.writeOptionalDouble(maxBucketProcessingTimeMs);
+        out.writeOptionalDouble(avgBucketProcessingTimeMs);
+        out.writeOptionalDouble(exponentialAvgBucketProcessingTimeMs);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(Job.ID.getPreferredName(), jobId);
+        builder.field(BUCKET_COUNT.getPreferredName(), bucketCount);
+        if (minBucketProcessingTimeMs != null) {
+            builder.field(MIN_BUCKET_PROCESSING_TIME_MS.getPreferredName(), minBucketProcessingTimeMs);
+        }
+        if (maxBucketProcessingTimeMs != null) {
+            builder.field(MAX_BUCKET_PROCESSING_TIME_MS.getPreferredName(), maxBucketProcessingTimeMs);
+        }
+        if (avgBucketProcessingTimeMs != null) {
+            builder.field(AVG_BUCKET_PROCESSING_TIME_MS.getPreferredName(), avgBucketProcessingTimeMs);
+        }
+        if (exponentialAvgBucketProcessingTimeMs != null) {
+            builder.field(EXPONENTIAL_AVG_BUCKET_PROCESSING_TIME_MS.getPreferredName(), exponentialAvgBucketProcessingTimeMs);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        TimingStats that = (TimingStats) o;
+        return Objects.equals(this.jobId, that.jobId)
+            && this.bucketCount == that.bucketCount
+            && Objects.equals(this.minBucketProcessingTimeMs, that.minBucketProcessingTimeMs)
+            && Objects.equals(this.maxBucketProcessingTimeMs, that.maxBucketProcessingTimeMs)
+            && Objects.equals(this.avgBucketProcessingTimeMs, that.avgBucketProcessingTimeMs)
+            && Objects.equals(this.exponentialAvgBucketProcessingTimeMs, that.exponentialAvgBucketProcessingTimeMs);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(
+            jobId,
+            bucketCount,
+            minBucketProcessingTimeMs,
+            maxBucketProcessingTimeMs,
+            avgBucketProcessingTimeMs,
+            exponentialAvgBucketProcessingTimeMs);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java
index 3c099e309243b..ab32373644d19 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java
@@ -17,8 +17,7 @@
 import org.elasticsearch.xpack.core.ml.job.config.Detector;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
-import org.elasticsearch.Version;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -160,9 +159,7 @@ public AnomalyRecord(StreamInput in) throws IOException {
         jobId = in.readString();
         detectorIndex = in.readInt();
         probability = in.readDouble();
-        if (in.getVersion().onOrAfter(Version.V_6_5_0)) {
-            multiBucketImpact = in.readOptionalDouble();
-        }
+        multiBucketImpact = in.readOptionalDouble();
         byFieldName = in.readOptionalString();
         byFieldValue = in.readOptionalString();
         correlatedByFieldValue = in.readOptionalString();
@@ -197,9 +194,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(jobId);
         out.writeInt(detectorIndex);
         out.writeDouble(probability);
-        if (out.getVersion().onOrAfter(Version.V_6_5_0)) {
-            out.writeOptionalDouble(multiBucketImpact);
-        }
+        out.writeOptionalDouble(multiBucketImpact);
         out.writeOptionalString(byFieldName);
         out.writeOptionalString(byFieldValue);
         out.writeOptionalString(correlatedByFieldValue);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java
index 8e04e001ed6cd..f4134033ff8dc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.core.ml.job.results;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -16,7 +15,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -134,10 +133,6 @@ public Bucket(StreamInput in) throws IOException {
         isInterim = in.readBoolean();
         bucketInfluencers = in.readList(BucketInfluencer::new);
         processingTimeMs = in.readLong();
-        // bwc for perPartitionNormalization
-        if (in.getVersion().before(Version.V_6_5_0)) {
-            in.readList(Bucket::readOldPerPartitionNormalization);
-        }
         scheduledEvents = in.readStringList();
         if (scheduledEvents.isEmpty()) {
             scheduledEvents = Collections.emptyList();
@@ -156,10 +151,6 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(isInterim);
         out.writeList(bucketInfluencers);
         out.writeLong(processingTimeMs);
-        // bwc for perPartitionNormalization
-        if (out.getVersion().before(Version.V_6_5_0)) {
-            out.writeList(Collections.emptyList());
-        }
         out.writeStringCollection(scheduledEvents);
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencer.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencer.java
index e6031b3b8df52..8cb06b6e0a22d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencer.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencer.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.io.IOException;
 import java.util.Date;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/CategoryDefinition.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/CategoryDefinition.java
index 576bed5dcea2f..4b25b456c15b4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/CategoryDefinition.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/CategoryDefinition.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.core.ml.job.results;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -78,9 +77,7 @@ public CategoryDefinition(StreamInput in) throws IOException {
         regex = in.readString();
         maxMatchingLength = in.readLong();
         examples = new TreeSet<>(in.readStringList());
-        if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
-            grokPattern = in.readOptionalString();
-        }
+        grokPattern = in.readOptionalString();
     }
 
     @Override
@@ -91,9 +88,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(regex);
         out.writeLong(maxMatchingLength);
         out.writeStringCollection(examples);
-        if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
-            out.writeOptionalString(grokPattern);
-        }
+        out.writeOptionalString(grokPattern);
     }
 
     public String getJobId() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Forecast.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Forecast.java
index 03a9b801167e7..5f4e3c829c3d5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Forecast.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Forecast.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.io.IOException;
 import java.util.Date;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influencer.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influencer.java
index d226058bf1db4..d17b375459bb4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influencer.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influencer.java
@@ -15,7 +15,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.io.IOException;
 import java.util.Date;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ModelPlot.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ModelPlot.java
index 60c8c30b94230..ab7235ca27ac4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ModelPlot.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ModelPlot.java
@@ -14,7 +14,7 @@
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.io.IOException;
 import java.util.Date;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
index 51bdc5ce594ad..2b3497c0aff09 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
@@ -9,6 +9,10 @@
 import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DelayedDataCheckConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsSource;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetection;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
 import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
@@ -23,6 +27,7 @@
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshotField;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 
 import java.util.Arrays;
 import java.util.HashSet;
@@ -174,6 +179,12 @@ public final class ReservedFieldNames {
             Result.TIMESTAMP.getPreferredName(),
             Result.IS_INTERIM.getPreferredName(),
 
+            TimingStats.BUCKET_COUNT.getPreferredName(),
+            TimingStats.MIN_BUCKET_PROCESSING_TIME_MS.getPreferredName(),
+            TimingStats.MAX_BUCKET_PROCESSING_TIME_MS.getPreferredName(),
+            TimingStats.AVG_BUCKET_PROCESSING_TIME_MS.getPreferredName(),
+            TimingStats.EXPONENTIAL_AVG_BUCKET_PROCESSING_TIME_MS.getPreferredName(),
+
             GetResult._ID,
             GetResult._INDEX,
             GetResult._TYPE
@@ -261,6 +272,22 @@ public final class ReservedFieldNames {
             ChunkingConfig.MODE_FIELD.getPreferredName(),
             ChunkingConfig.TIME_SPAN_FIELD.getPreferredName(),
 
+            DataFrameAnalyticsConfig.ID.getPreferredName(),
+            DataFrameAnalyticsConfig.SOURCE.getPreferredName(),
+            DataFrameAnalyticsConfig.DEST.getPreferredName(),
+            DataFrameAnalyticsConfig.ANALYSIS.getPreferredName(),
+            DataFrameAnalyticsConfig.ANALYZED_FIELDS.getPreferredName(),
+            DataFrameAnalyticsConfig.CREATE_TIME.getPreferredName(),
+            DataFrameAnalyticsConfig.VERSION.getPreferredName(),
+            DataFrameAnalyticsDest.INDEX.getPreferredName(),
+            DataFrameAnalyticsDest.RESULTS_FIELD.getPreferredName(),
+            DataFrameAnalyticsSource.INDEX.getPreferredName(),
+            DataFrameAnalyticsSource.QUERY.getPreferredName(),
+            OutlierDetection.NAME.getPreferredName(),
+            OutlierDetection.N_NEIGHBORS.getPreferredName(),
+            OutlierDetection.METHOD.getPreferredName(),
+            OutlierDetection.FEATURE_INFLUENCE_THRESHOLD.getPreferredName(),
+
             ElasticsearchMappings.CONFIG_TYPE,
 
             GetResult._ID,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/AuditMessage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/AuditMessage.java
index c3328bb3263fd..6daa4223afd73 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/AuditMessage.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/AuditMessage.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.xpack.core.common.notifications.AbstractAuditMessage;
 import org.elasticsearch.xpack.core.common.notifications.Level;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.util.Date;
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/process/writer/RecordWriter.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/process/writer/RecordWriter.java
index b66fd948a5a83..2d4c636172eca 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/process/writer/RecordWriter.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/process/writer/RecordWriter.java
@@ -10,7 +10,7 @@
 
 /**
  * Interface for classes that write arrays of strings to the
- * Ml analytics processes.
+ * Ml data frame analytics processes.
  */
 public interface RecordWriter {
     /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java
index 47c0d4f64f96f..320eace983590 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
@@ -34,6 +35,14 @@ public static ResourceAlreadyExistsException datafeedAlreadyExists(String datafe
         return new ResourceAlreadyExistsException(Messages.getMessage(Messages.DATAFEED_ID_ALREADY_TAKEN, datafeedId));
     }
 
+    public static ResourceNotFoundException missingDataFrameAnalytics(String id) {
+        return new ResourceNotFoundException("No known data frame analytics with id [{}]", id);
+    }
+
+    public static ResourceAlreadyExistsException dataFrameAnalyticsAlreadyExists(String id) {
+        return new ResourceAlreadyExistsException("A data frame analytics with id [{}] already exists", id);
+    }
+
     public static ElasticsearchException serverError(String msg) {
         return new ElasticsearchException(msg);
     }
@@ -86,4 +95,8 @@ public static <T> T requireNonNull(T obj, String paramName) {
         }
         return obj;
     }
+
+    public static <T> T requireNonNull(T obj, ParseField paramName) {
+        return requireNonNull(obj, paramName.getPreferredName());
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/QueryProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/QueryProvider.java
similarity index 81%
rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/QueryProvider.java
rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/QueryProvider.java
index 755c5a3526d01..3fe0ba70331a0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/QueryProvider.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/QueryProvider.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.core.ml.datafeed;
+package org.elasticsearch.xpack.core.ml.utils;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -17,9 +17,6 @@
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.xpack.core.ml.job.messages.Messages;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
-import org.elasticsearch.xpack.core.ml.utils.XContentObjectTransformer;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -27,22 +24,22 @@
 import java.util.Map;
 import java.util.Objects;
 
-class QueryProvider implements Writeable, ToXContentObject {
+public class QueryProvider implements Writeable, ToXContentObject {
 
-    private static final Logger logger = LogManager.getLogger(AggProvider.class);
+    private static final Logger logger = LogManager.getLogger(QueryProvider.class);
 
     private Exception parsingException;
     private QueryBuilder parsedQuery;
     private Map<String, Object> query;
 
-    static QueryProvider defaultQuery() {
+    public static QueryProvider defaultQuery() {
         return new QueryProvider(
             Collections.singletonMap(MatchAllQueryBuilder.NAME, Collections.emptyMap()),
             QueryBuilders.matchAllQuery(),
             null);
     }
 
-    static QueryProvider fromXContent(XContentParser parser, boolean lenient) throws IOException {
+    public static QueryProvider fromXContent(XContentParser parser, boolean lenient, String failureMessage) throws IOException {
         Map<String, Object> query = parser.mapOrdered();
         QueryBuilder parsedQuery = null;
         Exception exception = null;
@@ -54,15 +51,15 @@ static QueryProvider fromXContent(XContentParser parser, boolean lenient) throws
             }
             exception = ex;
             if (lenient) {
-                logger.warn(Messages.DATAFEED_CONFIG_QUERY_BAD_FORMAT, ex);
+                logger.warn(failureMessage, ex);
             } else {
-                throw ExceptionsHelper.badRequestException(Messages.DATAFEED_CONFIG_QUERY_BAD_FORMAT, ex);
+                throw ExceptionsHelper.badRequestException(failureMessage, ex);
             }
         }
         return new QueryProvider(query, parsedQuery, exception);
     }
 
-    static QueryProvider fromParsedQuery(QueryBuilder parsedQuery) throws IOException {
+    public static QueryProvider fromParsedQuery(QueryBuilder parsedQuery) throws IOException {
         return parsedQuery == null ?
             null :
             new QueryProvider(
@@ -71,7 +68,7 @@ static QueryProvider fromParsedQuery(QueryBuilder parsedQuery) throws IOExceptio
                 null);
     }
 
-    static QueryProvider fromStream(StreamInput in) throws IOException {
+    public static QueryProvider fromStream(StreamInput in) throws IOException {
         return new QueryProvider(in.readMap(), in.readOptionalNamedWriteable(QueryBuilder.class), in.readException());
     }
 
@@ -81,7 +78,7 @@ static QueryProvider fromStream(StreamInput in) throws IOException {
         this.parsingException = parsingException;
     }
 
-    QueryProvider(QueryProvider other) {
+    public QueryProvider(QueryProvider other) {
         this(other.query, other.parsedQuery, other.parsingException);
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkAction.java
index 49fb085191e4e..6426c9fb2ce7b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.core.monitoring.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class MonitoringBulkAction extends Action<MonitoringBulkResponse> {
+public class MonitoringBulkAction extends StreamableResponseActionType<MonitoringBulkResponse> {
 
     public static final MonitoringBulkAction INSTANCE = new MonitoringBulkAction();
     public static final String NAME = "cluster:admin/xpack/monitoring/bulk";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/client/MonitoringClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/client/MonitoringClient.java
deleted file mode 100644
index 2dba6e6a4664f..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/client/MonitoringClient.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.monitoring.client;
-
-import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkAction;
-import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkRequest;
-import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkRequestBuilder;
-import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkResponse;
-
-import java.util.Map;
-
-public class MonitoringClient {
-
-    private final Client client;
-
-    @Inject
-    public MonitoringClient(Client client) {
-        this.client = client;
-    }
-
-
-    /**
-     * Creates a request builder that bulk index monitoring documents.
-     *
-     * @return The request builder
-     */
-    public MonitoringBulkRequestBuilder prepareMonitoringBulk() {
-        return new MonitoringBulkRequestBuilder(client);
-    }
-
-    /**
-     * Executes a bulk of index operations that concern monitoring documents.
-     *
-     * @param request  The monitoring bulk request
-     * @param listener A listener to be notified with a result
-     */
-    public void bulk(MonitoringBulkRequest request, ActionListener<MonitoringBulkResponse> listener) {
-        client.execute(MonitoringBulkAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Executes a bulk of index operations that concern monitoring documents.
-     *
-     * @param request  The monitoring bulk request
-     */
-    public ActionFuture<MonitoringBulkResponse> bulk(MonitoringBulkRequest request) {
-        return client.execute(MonitoringBulkAction.INSTANCE, request);
-    }
-
-    public MonitoringClient filterWithHeader(Map<String, String> headers) {
-        return new MonitoringClient(client.filterWithHeader(headers));
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/XPackRestHandler.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/XPackRestHandler.java
deleted file mode 100644
index 5ac0969624bc8..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/XPackRestHandler.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.rest;
-
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.xpack.core.XPackClient;
-
-import java.io.IOException;
-
-public abstract class XPackRestHandler extends BaseRestHandler {
-
-    protected static String URI_BASE = "/_xpack";
-
-    public XPackRestHandler(Settings settings) {
-        super(settings);
-    }
-
-    @Override
-    public final RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        return doPrepareRequest(request, new XPackClient(client));
-    }
-
-    protected abstract RestChannelConsumer doPrepareRequest(RestRequest request, XPackClient client) throws IOException;
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestFreezeIndexAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestFreezeIndexAction.java
index 9604cdd8b3183..580acbae0dc87 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestFreezeIndexAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestFreezeIndexAction.java
@@ -7,16 +7,17 @@
 
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xpack.core.XPackClient;
 import org.elasticsearch.xpack.core.action.TransportFreezeIndexAction;
-import org.elasticsearch.xpack.core.rest.XPackRestHandler;
+import org.elasticsearch.xpack.core.action.TransportFreezeIndexAction.FreezeIndexAction;
 
-public final class RestFreezeIndexAction extends XPackRestHandler {
+public final class RestFreezeIndexAction extends BaseRestHandler {
     public RestFreezeIndexAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.POST, "/{index}/_freeze", this);
@@ -24,7 +25,7 @@ public RestFreezeIndexAction(Settings settings, RestController controller) {
     }
 
     @Override
-    protected RestChannelConsumer doPrepareRequest(RestRequest request, XPackClient client) {
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
         boolean freeze = request.path().endsWith("/_freeze");
         TransportFreezeIndexAction.FreezeRequest freezeRequest =
             new TransportFreezeIndexAction.FreezeRequest(Strings.splitStringByCommaToArray(request.param("index")));
@@ -36,7 +37,7 @@ protected RestChannelConsumer doPrepareRequest(RestRequest request, XPackClient
             freezeRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards));
         }
         freezeRequest.setFreeze(freeze);
-        return channel -> client.freeze(freezeRequest, new RestToXContentListener<>(channel));
+        return channel -> client.execute(FreezeIndexAction.INSTANCE, freezeRequest, new RestToXContentListener<>(channel));
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestReloadAnalyzersAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestReloadAnalyzersAction.java
new file mode 100644
index 0000000000000..3b379e8cebbb5
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestReloadAnalyzersAction.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.rest.action;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.action.ReloadAnalyzerAction;
+import org.elasticsearch.xpack.core.action.ReloadAnalyzersRequest;
+
+import java.io.IOException;
+public class RestReloadAnalyzersAction extends BaseRestHandler {
+
+    public RestReloadAnalyzersAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.GET, "/{index}/_reload_search_analyzers", this);
+        controller.registerHandler(RestRequest.Method.POST, "/{index}/_reload_search_analyzers", this);
+    }
+
+    @Override
+    public String getName() {
+        return "reload_search_analyzers_action";
+    }
+
+    @Override
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+        ReloadAnalyzersRequest reloadAnalyzersRequest = new ReloadAnalyzersRequest(
+                Strings.splitStringByCommaToArray(request.param("index")));
+        reloadAnalyzersRequest.indicesOptions(IndicesOptions.fromRequest(request, reloadAnalyzersRequest.indicesOptions()));
+        return channel -> client.execute(ReloadAnalyzerAction.INSTANCE, reloadAnalyzersRequest, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackInfoAction.java
index c057c04cc637d..2a41be1cb0cb1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackInfoAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackInfoAction.java
@@ -5,13 +5,14 @@
  */
 package org.elasticsearch.xpack.core.rest.action;
 
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xpack.core.XPackClient;
-import org.elasticsearch.xpack.core.rest.XPackRestHandler;
+import org.elasticsearch.xpack.core.action.XPackInfoRequestBuilder;
 
 import java.io.IOException;
 import java.util.EnumSet;
@@ -19,11 +20,11 @@
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 
-public class RestXPackInfoAction extends XPackRestHandler {
+public class RestXPackInfoAction extends BaseRestHandler {
     public RestXPackInfoAction(Settings settings, RestController controller) {
         super(settings);
-        controller.registerHandler(HEAD, URI_BASE, this);
-        controller.registerHandler(GET, URI_BASE, this);
+        controller.registerHandler(HEAD, "/_xpack", this);
+        controller.registerHandler(GET, "/_xpack", this);
     }
 
     @Override
@@ -32,7 +33,7 @@ public String getName() {
     }
 
     @Override
-    public RestChannelConsumer doPrepareRequest(RestRequest request, XPackClient client) throws IOException {
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
 
         // we piggyback verbosity on "human" output
         boolean verbose = request.paramAsBoolean("human", true);
@@ -40,7 +41,7 @@ public RestChannelConsumer doPrepareRequest(RestRequest request, XPackClient cli
         EnumSet<XPackInfoRequest.Category> categories = XPackInfoRequest.Category
                 .toSet(request.paramAsStringArray("categories", new String[] { "_all" }));
         return channel ->
-                client.prepareInfo()
+                new XPackInfoRequestBuilder(client)
                         .setVerbose(verbose)
                         .setCategories(categories)
                         .execute(new RestToXContentListener<>(channel));
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackUsageAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackUsageAction.java
index 0f09f17dbb066..44b04c3548f7b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackUsageAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackUsageAction.java
@@ -6,29 +6,29 @@
 package org.elasticsearch.xpack.core.rest.action;
 
 import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
-import org.elasticsearch.xpack.core.XPackClient;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder;
 import org.elasticsearch.xpack.core.action.XPackUsageResponse;
-import org.elasticsearch.xpack.core.rest.XPackRestHandler;
 
 import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestStatus.OK;
 
-public class RestXPackUsageAction extends XPackRestHandler {
+public class RestXPackUsageAction extends BaseRestHandler {
     public RestXPackUsageAction(Settings settings, RestController controller) {
         super(settings);
-        controller.registerHandler(GET, URI_BASE + "/usage", this);
+        controller.registerHandler(GET, "/_xpack/usage", this);
     }
 
     @Override
@@ -37,11 +37,11 @@ public String getName() {
     }
 
     @Override
-    public RestChannelConsumer doPrepareRequest(RestRequest request, XPackClient client) throws IOException {
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         final TimeValue masterTimeout = request.paramAsTime("master_timeout", MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT);
-        return channel -> new XPackUsageRequestBuilder(client.es())
+        return channel -> new XPackUsageRequestBuilder(client)
                 .setMasterNodeTimeout(masterTimeout)
-                .execute(new RestBuilderListener<XPackUsageResponse>(channel) {
+                .execute(new RestBuilderListener<>(channel) {
                     @Override
                     public RestResponse buildResponse(XPackUsageResponse response, XContentBuilder builder) throws Exception {
                         builder.startObject();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/DeleteRollupJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/DeleteRollupJobAction.java
index 92a1a07ded027..2a8b8eebb22e9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/DeleteRollupJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/DeleteRollupJobAction.java
@@ -6,7 +6,7 @@
 package org.elasticsearch.xpack.core.rollup.action;
 
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.FailedNodeException;
@@ -29,7 +29,7 @@
 import java.util.List;
 import java.util.Objects;
 
-public class DeleteRollupJobAction extends Action<DeleteRollupJobAction.Response> {
+public class DeleteRollupJobAction extends ActionType<DeleteRollupJobAction.Response> {
 
     public static final DeleteRollupJobAction INSTANCE = new DeleteRollupJobAction();
     public static final String NAME = "cluster:admin/xpack/rollup/delete";
@@ -38,11 +38,6 @@ private DeleteRollupJobAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java
index f544c21a15c9a..f0e6354e88fde 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupCapsAction.java
@@ -5,11 +5,11 @@
  */
 package org.elasticsearch.xpack.core.rollup.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.ParseField;
@@ -17,7 +17,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -28,7 +27,7 @@
 import java.util.Map;
 import java.util.Objects;
 
-public class GetRollupCapsAction extends Action<GetRollupCapsAction.Response> {
+public class GetRollupCapsAction extends StreamableResponseActionType<GetRollupCapsAction.Response> {
 
     public static final GetRollupCapsAction INSTANCE = new GetRollupCapsAction();
     public static final String NAME = "cluster:monitor/xpack/rollup/get/caps";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java
index 9dcd673c39fb2..c0f45d4c6f339 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupIndexCapsAction.java
@@ -6,12 +6,12 @@
 package org.elasticsearch.xpack.core.rollup.action;
 
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.ParseField;
@@ -19,7 +19,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -31,7 +30,7 @@
 import java.util.Map;
 import java.util.Objects;
 
-public class GetRollupIndexCapsAction extends Action<GetRollupIndexCapsAction.Response> {
+public class GetRollupIndexCapsAction extends StreamableResponseActionType<GetRollupIndexCapsAction.Response> {
 
     public static final GetRollupIndexCapsAction INSTANCE = new GetRollupIndexCapsAction();
     public static final String NAME = "indices:data/read/xpack/rollup/get/index/caps";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupJobsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupJobsAction.java
index 913e544e74190..f67e1b836e24f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupJobsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupJobsAction.java
@@ -6,7 +6,7 @@
 package org.elasticsearch.xpack.core.rollup.action;
 
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.FailedNodeException;
@@ -34,7 +34,7 @@
 import java.util.List;
 import java.util.Objects;
 
-public class GetRollupJobsAction extends Action<GetRollupJobsAction.Response> {
+public class GetRollupJobsAction extends ActionType<GetRollupJobsAction.Response> {
 
     public static final GetRollupJobsAction INSTANCE = new GetRollupJobsAction();
     public static final String NAME = "cluster:monitor/xpack/rollup/get";
@@ -47,11 +47,6 @@ private GetRollupJobsAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/PutRollupJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/PutRollupJobAction.java
index 6c56e7f6f1f4b..4dcdc2c22f88a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/PutRollupJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/PutRollupJobAction.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.rollup.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.fieldcaps.FieldCapabilities;
@@ -16,6 +16,7 @@
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -25,7 +26,7 @@
 import java.util.Map;
 import java.util.Objects;
 
-public class PutRollupJobAction extends Action<AcknowledgedResponse> {
+public class PutRollupJobAction extends ActionType<AcknowledgedResponse> {
 
     public static final PutRollupJobAction INSTANCE = new PutRollupJobAction();
     public static final String NAME = "cluster:admin/xpack/rollup/put";
@@ -35,8 +36,8 @@ private PutRollupJobAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 
     public static class Request extends AcknowledgedRequest<Request> implements IndicesRequest, ToXContentObject {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupSearchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupSearchAction.java
index c6eecca5e3d80..b64f2401718f4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupSearchAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/RollupSearchAction.java
@@ -5,14 +5,14 @@
  */
 package org.elasticsearch.xpack.core.rollup.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.Writeable;
 
-public class RollupSearchAction extends Action<SearchResponse> {
+public class RollupSearchAction extends ActionType<SearchResponse> {
 
     public static final RollupSearchAction INSTANCE = new RollupSearchAction();
     public static final String NAME = "indices:admin/xpack/rollup/search";
@@ -21,11 +21,6 @@ private RollupSearchAction() {
         super(NAME);
     }
 
-    @Override
-    public SearchResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<SearchResponse> getResponseReader() {
         return SearchResponse::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/StartRollupJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/StartRollupJobAction.java
index ca2a5cd8d7264..c844acd63e198 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/StartRollupJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/StartRollupJobAction.java
@@ -6,7 +6,7 @@
 package org.elasticsearch.xpack.core.rollup.action;
 
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.tasks.BaseTasksRequest;
@@ -24,7 +24,7 @@
 import java.util.Collections;
 import java.util.Objects;
 
-public class StartRollupJobAction extends Action<StartRollupJobAction.Response> {
+public class StartRollupJobAction extends ActionType<StartRollupJobAction.Response> {
 
     public static final StartRollupJobAction INSTANCE = new StartRollupJobAction();
     public static final String NAME = "cluster:admin/xpack/rollup/start";
@@ -33,11 +33,6 @@ private StartRollupJobAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/StopRollupJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/StopRollupJobAction.java
index 6fc079e0328c9..42a968548043a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/StopRollupJobAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/StopRollupJobAction.java
@@ -5,8 +5,7 @@
  */
 package org.elasticsearch.xpack.core.rollup.action;
 
-import org.elasticsearch.Version;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.tasks.BaseTasksRequest;
@@ -28,7 +27,7 @@
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 
-public class StopRollupJobAction extends Action<StopRollupJobAction.Response> {
+public class StopRollupJobAction extends ActionType<StopRollupJobAction.Response> {
 
     public static final StopRollupJobAction INSTANCE = new StopRollupJobAction();
     public static final String NAME = "cluster:admin/xpack/rollup/stop";
@@ -40,11 +39,6 @@ private StopRollupJobAction() {
         super(NAME);
     }
 
-    @Override
-    public Response newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<Response> getResponseReader() {
         return Response::new;
@@ -70,20 +64,16 @@ public Request() {}
         public Request(StreamInput in) throws IOException {
             super(in);
             id = in.readString();
-            if (in.getVersion().onOrAfter(Version.V_6_6_0)) {
-                waitForCompletion = in.readBoolean();
-                timeout = in.readTimeValue();
-            }
+            waitForCompletion = in.readBoolean();
+            timeout = in.readTimeValue();
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeString(id);
-            if (out.getVersion().onOrAfter(Version.V_6_6_0)) {
-                out.writeBoolean(waitForCompletion);
-                out.writeTimeValue(timeout);
-            }
+            out.writeBoolean(waitForCompletion);
+            out.writeTimeValue(timeout);
         }
 
         public String getId() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyAction.java
index 5d211ea70b522..b289819d49869 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyAction.java
@@ -6,13 +6,13 @@
 
 package org.elasticsearch.xpack.core.security.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
 /**
- * Action for the creation of an API key
+ * ActionType for the creation of an API key
  */
-public final class CreateApiKeyAction extends Action<CreateApiKeyResponse> {
+public final class CreateApiKeyAction extends ActionType<CreateApiKeyResponse> {
 
     public static final String NAME = "cluster:admin/xpack/security/api_key/create";
     public static final CreateApiKeyAction INSTANCE = new CreateApiKeyAction();
@@ -21,11 +21,6 @@ private CreateApiKeyAction() {
         super(NAME);
     }
 
-    @Override
-    public CreateApiKeyResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<CreateApiKeyResponse> getResponseReader() {
         return CreateApiKeyResponse::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java
index 28a872c2222dd..78ba943f00442 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java
@@ -43,13 +43,13 @@ public CreateApiKeyRequest() {}
      * @param roleDescriptors list of {@link RoleDescriptor}s
      * @param expiration to specify expiration for the API key
      */
-    public CreateApiKeyRequest(String name, List<RoleDescriptor> roleDescriptors, @Nullable TimeValue expiration) {
+    public CreateApiKeyRequest(String name, @Nullable List<RoleDescriptor> roleDescriptors, @Nullable TimeValue expiration) {
         if (Strings.hasText(name)) {
             this.name = name;
         } else {
             throw new IllegalArgumentException("name must not be null or empty");
         }
-        this.roleDescriptors = Objects.requireNonNull(roleDescriptors, "role descriptors may not be null");
+        this.roleDescriptors = (roleDescriptors == null) ? List.of() : List.copyOf(roleDescriptors);
         this.expiration = expiration;
     }
 
@@ -57,7 +57,7 @@ public CreateApiKeyRequest(StreamInput in) throws IOException {
         super(in);
         this.name = in.readString();
         this.expiration = in.readOptionalTimeValue();
-        this.roleDescriptors = Collections.unmodifiableList(in.readList(RoleDescriptor::new));
+        this.roleDescriptors = List.copyOf(in.readList(RoleDescriptor::new));
         this.refreshPolicy = WriteRequest.RefreshPolicy.readFrom(in);
     }
 
@@ -77,7 +77,7 @@ public TimeValue getExpiration() {
         return expiration;
     }
 
-    public void setExpiration(TimeValue expiration) {
+    public void setExpiration(@Nullable TimeValue expiration) {
         this.expiration = expiration;
     }
 
@@ -85,8 +85,8 @@ public List<RoleDescriptor> getRoleDescriptors() {
         return roleDescriptors;
     }
 
-    public void setRoleDescriptors(List<RoleDescriptor> roleDescriptors) {
-        this.roleDescriptors = Collections.unmodifiableList(Objects.requireNonNull(roleDescriptors, "role descriptors may not be null"));
+    public void setRoleDescriptors(@Nullable List<RoleDescriptor> roleDescriptors) {
+        this.roleDescriptors = (roleDescriptors == null) ? List.of() : List.copyOf(roleDescriptors);
     }
 
     public WriteRequest.RefreshPolicy getRefreshPolicy() {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java
index 1a711aa7d9a26..c31dbeb7e4859 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java
@@ -39,7 +39,7 @@ public final class CreateApiKeyRequestBuilder extends ActionRequestBuilder<Creat
 
     static {
         PARSER.declareString(constructorArg(), new ParseField("name"));
-        PARSER.declareNamedObjects(constructorArg(), (p, c, n) -> {
+        PARSER.declareNamedObjects(optionalConstructorArg(), (p, c, n) -> {
             p.nextToken();
             return RoleDescriptor.parse(n, p, false);
         }, new ParseField("role_descriptors"));
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyAction.java
index 2af331909a3af..b0ee860c71a64 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyAction.java
@@ -6,13 +6,13 @@
 
 package org.elasticsearch.xpack.core.security.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
 /**
- * Action for retrieving API key(s)
+ * ActionType for retrieving API key(s)
  */
-public final class GetApiKeyAction extends Action<GetApiKeyResponse> {
+public final class GetApiKeyAction extends ActionType<GetApiKeyResponse> {
 
     public static final String NAME = "cluster:admin/xpack/security/api_key/get";
     public static final GetApiKeyAction INSTANCE = new GetApiKeyAction();
@@ -21,11 +21,6 @@ private GetApiKeyAction() {
         super(NAME);
     }
 
-    @Override
-    public GetApiKeyResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<GetApiKeyResponse> getResponseReader() {
         return GetApiKeyResponse::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyAction.java
index 0f5c7e66e724c..5b99a4c035015 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyAction.java
@@ -6,13 +6,13 @@
 
 package org.elasticsearch.xpack.core.security.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
 /**
- * Action for invalidating API key
+ * ActionType for invalidating API key
  */
-public final class InvalidateApiKeyAction extends Action<InvalidateApiKeyResponse> {
+public final class InvalidateApiKeyAction extends ActionType<InvalidateApiKeyResponse> {
 
     public static final String NAME = "cluster:admin/xpack/security/api_key/invalidate";
     public static final InvalidateApiKeyAction INSTANCE = new InvalidateApiKeyAction();
@@ -21,11 +21,6 @@ private InvalidateApiKeyAction() {
         super(NAME);
     }
 
-    @Override
-    public InvalidateApiKeyResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<InvalidateApiKeyResponse> getResponseReader() {
         return InvalidateApiKeyResponse::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectAuthenticateAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectAuthenticateAction.java
index b27a71e202e55..f65d82730ef3e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectAuthenticateAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectAuthenticateAction.java
@@ -5,13 +5,13 @@
  */
 package org.elasticsearch.xpack.core.security.action.oidc;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
 /**
- * Action for initiating an authentication process using OpenID Connect
+ * ActionType for initiating an authentication process using OpenID Connect
  */
-public final class OpenIdConnectAuthenticateAction extends Action<OpenIdConnectAuthenticateResponse> {
+public final class OpenIdConnectAuthenticateAction extends ActionType<OpenIdConnectAuthenticateResponse> {
 
     public static final OpenIdConnectAuthenticateAction INSTANCE = new OpenIdConnectAuthenticateAction();
     public static final String NAME = "cluster:admin/xpack/security/oidc/authenticate";
@@ -20,11 +20,6 @@ private OpenIdConnectAuthenticateAction() {
         super(NAME);
     }
 
-    @Override
-    public OpenIdConnectAuthenticateResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<OpenIdConnectAuthenticateResponse> getResponseReader() {
         return OpenIdConnectAuthenticateResponse::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectLogoutAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectLogoutAction.java
index 482484a7dedee..b1ab044dc0183 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectLogoutAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectLogoutAction.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.core.security.action.oidc;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
-public class OpenIdConnectLogoutAction extends Action<OpenIdConnectLogoutResponse> {
+public class OpenIdConnectLogoutAction extends ActionType<OpenIdConnectLogoutResponse> {
 
     public static final OpenIdConnectLogoutAction INSTANCE = new OpenIdConnectLogoutAction();
     public static final String NAME = "cluster:admin/xpack/security/oidc/logout";
@@ -17,11 +17,6 @@ private OpenIdConnectLogoutAction() {
         super(NAME);
     }
 
-    @Override
-    public OpenIdConnectLogoutResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<OpenIdConnectLogoutResponse> getResponseReader() {
         return OpenIdConnectLogoutResponse::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectPrepareAuthenticationAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectPrepareAuthenticationAction.java
index 2aa82c7286cec..44d6979500842 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectPrepareAuthenticationAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/oidc/OpenIdConnectPrepareAuthenticationAction.java
@@ -5,10 +5,10 @@
  */
 package org.elasticsearch.xpack.core.security.action.oidc;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.common.io.stream.Writeable;
 
-public class OpenIdConnectPrepareAuthenticationAction extends Action<OpenIdConnectPrepareAuthenticationResponse> {
+public class OpenIdConnectPrepareAuthenticationAction extends ActionType<OpenIdConnectPrepareAuthenticationResponse> {
 
     public static final OpenIdConnectPrepareAuthenticationAction INSTANCE = new OpenIdConnectPrepareAuthenticationAction();
     public static final String NAME = "cluster:admin/xpack/security/oidc/prepare";
@@ -17,11 +17,6 @@ private OpenIdConnectPrepareAuthenticationAction() {
         super(NAME);
     }
 
-    @Override
-    public OpenIdConnectPrepareAuthenticationResponse newResponse() {
-        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
-    }
-
     @Override
     public Writeable.Reader<OpenIdConnectPrepareAuthenticationResponse> getResponseReader() {
         return OpenIdConnectPrepareAuthenticationResponse::new;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesAction.java
index a36d2fdec74c4..edf62cf307161 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.privilege;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for deleting application privileges.
+ * ActionType for deleting application privileges.
  */
-public final class DeletePrivilegesAction extends Action<DeletePrivilegesResponse> {
+public final class DeletePrivilegesAction extends StreamableResponseActionType<DeletePrivilegesResponse> {
 
     public static final DeletePrivilegesAction INSTANCE = new DeletePrivilegesAction();
     public static final String NAME = "cluster:admin/xpack/security/privilege/delete";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequestBuilder.java
index c1d364476ea3e..020d78bf3544d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequestBuilder.java
@@ -15,8 +15,8 @@
 public final class DeletePrivilegesRequestBuilder extends ActionRequestBuilder<DeletePrivilegesRequest, DeletePrivilegesResponse>
         implements WriteRequestBuilder<DeletePrivilegesRequestBuilder> {
 
-    public DeletePrivilegesRequestBuilder(ElasticsearchClient client, DeletePrivilegesAction action) {
-        super(client, action, new DeletePrivilegesRequest());
+    public DeletePrivilegesRequestBuilder(ElasticsearchClient client) {
+        super(client, DeletePrivilegesAction.INSTANCE, new DeletePrivilegesRequest());
     }
 
     public DeletePrivilegesRequestBuilder privileges(String[] privileges) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesAction.java
new file mode 100644
index 0000000000000..862fc449f6eba
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesAction.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.StreamableResponseActionType;
+
+/**
+ * ActionType for retrieving builtin privileges from security
+ */
+public final class GetBuiltinPrivilegesAction extends StreamableResponseActionType<GetBuiltinPrivilegesResponse> {
+
+    public static final GetBuiltinPrivilegesAction INSTANCE = new GetBuiltinPrivilegesAction();
+    public static final String NAME = "cluster:admin/xpack/security/privilege/builtin/get";
+
+    private GetBuiltinPrivilegesAction() {
+        super(NAME);
+    }
+
+    @Override
+    public GetBuiltinPrivilegesResponse newResponse() {
+        return new GetBuiltinPrivilegesResponse();
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesRequest.java
new file mode 100644
index 0000000000000..a4f4eb9c8dd72
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesRequest.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+
+/**
+ * Request to retrieve built-in (cluster/index) privileges.
+ */
+public final class GetBuiltinPrivilegesRequest extends ActionRequest {
+
+    public GetBuiltinPrivilegesRequest() {
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponse.java
new file mode 100644
index 0000000000000..e09f2fc0b1a9f
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponse.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Objects;
+
+/**
+ * Response containing one or more application privileges retrieved from the security index
+ */
+public final class GetBuiltinPrivilegesResponse extends ActionResponse {
+
+    private String[] clusterPrivileges;
+    private String[] indexPrivileges;
+
+    public GetBuiltinPrivilegesResponse(String[] clusterPrivileges, String[] indexPrivileges) {
+        this.clusterPrivileges = Objects.requireNonNull(clusterPrivileges, "Cluster privileges cannot be null");
+        this.indexPrivileges =  Objects.requireNonNull(indexPrivileges, "Index privileges cannot be null");
+    }
+
+    public GetBuiltinPrivilegesResponse(Collection<String> clusterPrivileges,
+                                        Collection<String> indexPrivileges) {
+        this(clusterPrivileges.toArray(Strings.EMPTY_ARRAY), indexPrivileges.toArray(Strings.EMPTY_ARRAY));
+    }
+
+    public GetBuiltinPrivilegesResponse() {
+        this(Collections.emptySet(), Collections.emptySet());
+    }
+
+    public String[] getClusterPrivileges() {
+        return clusterPrivileges;
+    }
+
+    public String[] getIndexPrivileges() {
+        return indexPrivileges;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        this.clusterPrivileges = in.readStringArray();
+        this.indexPrivileges = in.readStringArray();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeStringArray(clusterPrivileges);
+        out.writeStringArray(indexPrivileges);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesAction.java
index 0b8743228c523..b48bc474b4ebe 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.privilege;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for retrieving one or more application privileges from the security index
+ * ActionType for retrieving one or more application privileges from the security index
  */
-public final class GetPrivilegesAction extends Action<GetPrivilegesResponse> {
+public final class GetPrivilegesAction extends StreamableResponseActionType<GetPrivilegesResponse> {
 
     public static final GetPrivilegesAction INSTANCE = new GetPrivilegesAction();
     public static final String NAME = "cluster:admin/xpack/security/privilege/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequest.java
index 585fb372d3247..9285dae76696b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequest.java
@@ -74,5 +74,4 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(application);
         out.writeStringArray(privileges);
     }
-
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequestBuilder.java
index 305c8d1ff7946..5adfd5774fab8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequestBuilder.java
@@ -13,8 +13,8 @@
  */
 public final class GetPrivilegesRequestBuilder extends ActionRequestBuilder<GetPrivilegesRequest, GetPrivilegesResponse> {
 
-    public GetPrivilegesRequestBuilder(ElasticsearchClient client, GetPrivilegesAction action) {
-        super(client, action, new GetPrivilegesRequest());
+    public GetPrivilegesRequestBuilder(ElasticsearchClient client) {
+        super(client, GetPrivilegesAction.INSTANCE, new GetPrivilegesRequest());
     }
 
     public GetPrivilegesRequestBuilder privileges(String... privileges) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesResponse.java
index 664673aa97e41..07b1c5596dea7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesResponse.java
@@ -12,6 +12,7 @@
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Objects;
 
 /**
  * Response containing one or more application privileges retrieved from the security index
@@ -21,11 +22,15 @@ public final class GetPrivilegesResponse extends ActionResponse {
     private ApplicationPrivilegeDescriptor[] privileges;
 
     public GetPrivilegesResponse(ApplicationPrivilegeDescriptor... privileges) {
-        this.privileges = privileges;
+        this.privileges = Objects.requireNonNull(privileges, "Application privileges cannot be null");
     }
 
     public GetPrivilegesResponse(Collection<ApplicationPrivilegeDescriptor> privileges) {
-        this(privileges.toArray(new ApplicationPrivilegeDescriptor[privileges.size()]));
+        this(privileges.toArray(new ApplicationPrivilegeDescriptor[0]));
+    }
+
+    public GetPrivilegesResponse() {
+        this(new ApplicationPrivilegeDescriptor[0]);
     }
 
     public ApplicationPrivilegeDescriptor[] privileges() {
@@ -44,4 +49,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeArray(privileges);
     }
 
+    public boolean isEmpty() {
+        return privileges.length == 0;
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesAction.java
index 3743bec144f29..56d9b285cceec 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.privilege;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for putting (adding/updating) one or more application privileges.
+ * ActionType for putting (adding/updating) one or more application privileges.
  */
-public final class PutPrivilegesAction extends Action<PutPrivilegesResponse> {
+public final class PutPrivilegesAction extends StreamableResponseActionType<PutPrivilegesResponse> {
 
     public static final PutPrivilegesAction INSTANCE = new PutPrivilegesAction();
     public static final String NAME = "cluster:admin/xpack/security/privilege/put";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilder.java
index 562e22a1eb925..b3ff11bba0ebf 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilder.java
@@ -29,8 +29,8 @@
 public final class PutPrivilegesRequestBuilder extends ActionRequestBuilder<PutPrivilegesRequest, PutPrivilegesResponse>
     implements WriteRequestBuilder<PutPrivilegesRequestBuilder> {
 
-    public PutPrivilegesRequestBuilder(ElasticsearchClient client, PutPrivilegesAction action) {
-        super(client, action, new PutPrivilegesRequest());
+    public PutPrivilegesRequestBuilder(ElasticsearchClient client) {
+        super(client, PutPrivilegesAction.INSTANCE, new PutPrivilegesRequest());
     }
 
     ApplicationPrivilegeDescriptor parsePrivilege(XContentParser parser, String applicationName, String privilegeName) throws IOException {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/realm/ClearRealmCacheAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/realm/ClearRealmCacheAction.java
index 7c3cd58a7f467..902c9263beb40 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/realm/ClearRealmCacheAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/realm/ClearRealmCacheAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.core.security.action.realm;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class ClearRealmCacheAction extends Action<ClearRealmCacheResponse> {
+public class ClearRealmCacheAction extends StreamableResponseActionType<ClearRealmCacheResponse> {
 
     public static final ClearRealmCacheAction INSTANCE = new ClearRealmCacheAction();
     public static final String NAME = "cluster:admin/xpack/security/realm/cache/clear";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/realm/ClearRealmCacheRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/realm/ClearRealmCacheRequest.java
index 3d7436b96160e..db84a3026471d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/realm/ClearRealmCacheRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/realm/ClearRealmCacheRequest.java
@@ -89,8 +89,7 @@ public static class Node extends BaseNodeRequest {
         public Node() {
         }
 
-        public Node(ClearRealmCacheRequest request, String nodeId) {
-            super(nodeId);
+        public Node(ClearRealmCacheRequest request) {
             this.realms = request.realms;
             this.usernames = request.usernames;
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/ClearRolesCacheAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/ClearRolesCacheAction.java
index 096b5380181fb..10c9497bbbfa7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/ClearRolesCacheAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/ClearRolesCacheAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.role;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
  * The action for clearing the cache used by native roles that are stored in an index.
  */
-public class ClearRolesCacheAction extends Action<ClearRolesCacheResponse> {
+public class ClearRolesCacheAction extends StreamableResponseActionType<ClearRolesCacheResponse> {
 
     public static final ClearRolesCacheAction INSTANCE = new ClearRolesCacheAction();
     public static final String NAME = "cluster:admin/xpack/security/roles/cache/clear";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/ClearRolesCacheRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/ClearRolesCacheRequest.java
index bb0bb293a880b..515038ae73039 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/ClearRolesCacheRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/ClearRolesCacheRequest.java
@@ -54,8 +54,7 @@ public static class Node extends BaseNodeRequest {
         public Node() {
         }
 
-        public Node(ClearRolesCacheRequest request, String nodeId) {
-            super(nodeId);
+        public Node(ClearRolesCacheRequest request) {
             this.names = request.names();
         }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/DeleteRoleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/DeleteRoleAction.java
index 6130f107fb726..7024ed1eb42c8 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/DeleteRoleAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/DeleteRoleAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.role;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for deleting a role from the security index
+ * ActionType for deleting a role from the security index
  */
-public class DeleteRoleAction extends Action<DeleteRoleResponse> {
+public class DeleteRoleAction extends StreamableResponseActionType<DeleteRoleResponse> {
 
     public static final DeleteRoleAction INSTANCE = new DeleteRoleAction();
     public static final String NAME = "cluster:admin/xpack/security/role/delete";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/GetRolesAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/GetRolesAction.java
index 53126440afb9b..715da9af619e1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/GetRolesAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/GetRolesAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.role;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action to retrieve a role from the security index
+ * ActionType to retrieve a role from the security index
  */
-public class GetRolesAction extends Action<GetRolesResponse> {
+public class GetRolesAction extends StreamableResponseActionType<GetRolesResponse> {
 
     public static final GetRolesAction INSTANCE = new GetRolesAction();
     public static final String NAME = "cluster:admin/xpack/security/role/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleAction.java
index 8396625e262ff..029adb6643000 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.role;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for adding a role to the security index
+ * ActionType for adding a role to the security index
  */
-public class PutRoleAction extends Action<PutRoleResponse> {
+public class PutRoleAction extends StreamableResponseActionType<PutRoleResponse> {
 
     public static final PutRoleAction INSTANCE = new PutRoleAction();
     public static final String NAME = "cluster:admin/xpack/security/role/put";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/DeleteRoleMappingAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/DeleteRoleMappingAction.java
index 6057daf959531..7dc2680fe21be 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/DeleteRoleMappingAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/DeleteRoleMappingAction.java
@@ -5,13 +5,13 @@
  */
 package org.elasticsearch.xpack.core.security.action.rolemapping;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for deleting a role-mapping from the
+ * ActionType for deleting a role-mapping from the
  * org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore
  */
-public class DeleteRoleMappingAction extends Action<DeleteRoleMappingResponse> {
+public class DeleteRoleMappingAction extends StreamableResponseActionType<DeleteRoleMappingResponse> {
 
     public static final DeleteRoleMappingAction INSTANCE = new DeleteRoleMappingAction();
     public static final String NAME = "cluster:admin/xpack/security/role_mapping/delete";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/DeleteRoleMappingRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/DeleteRoleMappingRequestBuilder.java
index 2ca23d31f693a..608b034e4e259 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/DeleteRoleMappingRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/DeleteRoleMappingRequestBuilder.java
@@ -16,9 +16,8 @@
 public class DeleteRoleMappingRequestBuilder extends ActionRequestBuilder<DeleteRoleMappingRequest, DeleteRoleMappingResponse>
         implements WriteRequestBuilder<DeleteRoleMappingRequestBuilder> {
 
-    public DeleteRoleMappingRequestBuilder(ElasticsearchClient client,
-                                           DeleteRoleMappingAction action) {
-        super(client, action, new DeleteRoleMappingRequest());
+    public DeleteRoleMappingRequestBuilder(ElasticsearchClient client) {
+        super(client, DeleteRoleMappingAction.INSTANCE, new DeleteRoleMappingRequest());
     }
 
     public DeleteRoleMappingRequestBuilder name(String name) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/GetRoleMappingsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/GetRoleMappingsAction.java
index e1488bf70913e..2792348111946 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/GetRoleMappingsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/GetRoleMappingsAction.java
@@ -5,14 +5,14 @@
  */
 package org.elasticsearch.xpack.core.security.action.rolemapping;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action to retrieve one or more role-mappings from X-Pack security
+ * ActionType to retrieve one or more role-mappings from X-Pack security
  *
  * see org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore
  */
-public class GetRoleMappingsAction extends Action<GetRoleMappingsResponse> {
+public class GetRoleMappingsAction extends StreamableResponseActionType<GetRoleMappingsResponse> {
 
     public static final GetRoleMappingsAction INSTANCE = new GetRoleMappingsAction();
     public static final String NAME = "cluster:admin/xpack/security/role_mapping/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/GetRoleMappingsRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/GetRoleMappingsRequestBuilder.java
index db4f8b7d88746..23b94967fb1e1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/GetRoleMappingsRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/GetRoleMappingsRequestBuilder.java
@@ -15,8 +15,8 @@
  */
 public class GetRoleMappingsRequestBuilder extends ActionRequestBuilder<GetRoleMappingsRequest, GetRoleMappingsResponse> {
 
-    public GetRoleMappingsRequestBuilder(ElasticsearchClient client, GetRoleMappingsAction action) {
-        super(client, action, new GetRoleMappingsRequest());
+    public GetRoleMappingsRequestBuilder(ElasticsearchClient client) {
+        super(client, GetRoleMappingsAction.INSTANCE, new GetRoleMappingsRequest());
     }
 
     public GetRoleMappingsRequestBuilder names(String... names) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/PutRoleMappingAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/PutRoleMappingAction.java
index 9c3068adf127f..3bb3e2f51496d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/PutRoleMappingAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/PutRoleMappingAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.rolemapping;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for adding a role to the security index
+ * ActionType for adding a role to the security index
  */
-public class PutRoleMappingAction extends Action<PutRoleMappingResponse> {
+public class PutRoleMappingAction extends StreamableResponseActionType<PutRoleMappingResponse> {
 
     public static final PutRoleMappingAction INSTANCE = new PutRoleMappingAction();
     public static final String NAME = "cluster:admin/xpack/security/role_mapping/put";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/PutRoleMappingRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/PutRoleMappingRequestBuilder.java
index 14f722d169410..8e1b63c2ab8d9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/PutRoleMappingRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/PutRoleMappingRequestBuilder.java
@@ -26,8 +26,8 @@
 public class PutRoleMappingRequestBuilder extends ActionRequestBuilder<PutRoleMappingRequest, PutRoleMappingResponse> implements
         WriteRequestBuilder<PutRoleMappingRequestBuilder> {
 
-    public PutRoleMappingRequestBuilder(ElasticsearchClient client, PutRoleMappingAction action) {
-        super(client, action, new PutRoleMappingRequest());
+    public PutRoleMappingRequestBuilder(ElasticsearchClient client) {
+        super(client, PutRoleMappingAction.INSTANCE, new PutRoleMappingRequest());
     }
 
     /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlAuthenticateAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlAuthenticateAction.java
index fca733a3938a7..fe40b521a1fff 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlAuthenticateAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlAuthenticateAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.saml;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for authenticating using SAML assertions
+ * ActionType for authenticating using SAML assertions
  */
-public final class SamlAuthenticateAction extends Action<SamlAuthenticateResponse> {
+public final class SamlAuthenticateAction extends StreamableResponseActionType<SamlAuthenticateResponse> {
 
     public static final String NAME = "cluster:admin/xpack/security/saml/authenticate";
     public static final SamlAuthenticateAction INSTANCE = new SamlAuthenticateAction();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlInvalidateSessionAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlInvalidateSessionAction.java
index dc5aa09627564..e6d74e36c794d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlInvalidateSessionAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlInvalidateSessionAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.saml;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action to perform IdP-initiated logout for a SAML-SSO user
+ * ActionType to perform IdP-initiated logout for a SAML-SSO user
  */
-public final class SamlInvalidateSessionAction extends Action<SamlInvalidateSessionResponse> {
+public final class SamlInvalidateSessionAction extends StreamableResponseActionType<SamlInvalidateSessionResponse> {
 
     public static final String NAME = "cluster:admin/xpack/security/saml/invalidate";
     public static final SamlInvalidateSessionAction INSTANCE = new SamlInvalidateSessionAction();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlLogoutAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlLogoutAction.java
index 9ea3a29ca4ad9..2d97a29d190bf 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlLogoutAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlLogoutAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.saml;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for initiating a logout process for a SAML-SSO user
+ * ActionType for initiating a logout process for a SAML-SSO user
  */
-public final class SamlLogoutAction extends Action<SamlLogoutResponse> {
+public final class SamlLogoutAction extends StreamableResponseActionType<SamlLogoutResponse> {
 
     public static final String NAME = "cluster:admin/xpack/security/saml/logout";
     public static final SamlLogoutAction INSTANCE = new SamlLogoutAction();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlPrepareAuthenticationAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlPrepareAuthenticationAction.java
index 12ad23ca50199..11e5e38662767 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlPrepareAuthenticationAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/saml/SamlPrepareAuthenticationAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.saml;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for initiating an authentication process using SAML assertions
+ * ActionType for initiating an authentication process using SAML assertions
  */
-public final class SamlPrepareAuthenticationAction extends Action<SamlPrepareAuthenticationResponse> {
+public final class SamlPrepareAuthenticationAction extends StreamableResponseActionType<SamlPrepareAuthenticationResponse> {
 
     public static final String NAME = "cluster:admin/xpack/security/saml/prepare";
     public static final SamlPrepareAuthenticationAction INSTANCE = new SamlPrepareAuthenticationAction();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenAction.java
index 7b913f594e582..4ee84f6ccde3f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.token;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for creating a new token
+ * ActionType for creating a new token
  */
-public final class CreateTokenAction extends Action<CreateTokenResponse> {
+public final class CreateTokenAction extends StreamableResponseActionType<CreateTokenResponse> {
 
     public static final String NAME = "cluster:admin/xpack/security/token/create";
     public static final CreateTokenAction INSTANCE = new CreateTokenAction();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequest.java
index 3fdfaab060542..aeae2d8590774 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequest.java
@@ -5,22 +5,18 @@
  */
 package org.elasticsearch.xpack.core.security.action.token;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.CharArrays;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.Locale;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -35,6 +31,7 @@ public final class CreateTokenRequest extends ActionRequest {
 
     public enum GrantType {
         PASSWORD("password"),
+        KERBEROS("_kerberos"),
         REFRESH_TOKEN("refresh_token"),
         AUTHORIZATION_CODE("authorization_code"),
         CLIENT_CREDENTIALS("client_credentials");
@@ -62,21 +59,23 @@ public static GrantType fromString(String grantType) {
     }
 
     private static final Set<GrantType> SUPPORTED_GRANT_TYPES = Collections.unmodifiableSet(
-        EnumSet.of(GrantType.PASSWORD, GrantType.REFRESH_TOKEN, GrantType.CLIENT_CREDENTIALS));
+        EnumSet.of(GrantType.PASSWORD, GrantType.KERBEROS, GrantType.REFRESH_TOKEN, GrantType.CLIENT_CREDENTIALS));
 
     private String grantType;
     private String username;
     private SecureString password;
+    private SecureString kerberosTicket;
     private String scope;
     private String refreshToken;
 
     public CreateTokenRequest() {}
 
-    public CreateTokenRequest(String grantType, @Nullable String username, @Nullable SecureString password, @Nullable String scope,
-                              @Nullable String refreshToken) {
+    public CreateTokenRequest(String grantType, @Nullable String username, @Nullable SecureString password,
+                              @Nullable SecureString kerberosTicket, @Nullable String scope, @Nullable String refreshToken) {
         this.grantType = grantType;
         this.username = username;
         this.password = password;
+        this.kerberosTicket = kerberosTicket;
         this.scope = scope;
         this.refreshToken = refreshToken;
     }
@@ -88,43 +87,28 @@ public ActionRequestValidationException validate() {
         if (type != null) {
             switch (type) {
                 case PASSWORD:
-                    if (Strings.isNullOrEmpty(username)) {
-                        validationException = addValidationError("username is missing", validationException);
-                    }
-                    if (password == null || password.getChars() == null || password.getChars().length == 0) {
-                        validationException = addValidationError("password is missing", validationException);
-                    }
-                    if (refreshToken != null) {
-                        validationException =
-                            addValidationError("refresh_token is not supported with the password grant_type", validationException);
-                    }
+                    validationException = validateUnsupportedField(type, "kerberos_ticket", kerberosTicket, validationException);
+                    validationException = validateUnsupportedField(type, "refresh_token", refreshToken, validationException);
+                    validationException = validateRequiredField("username", username, validationException);
+                    validationException = validateRequiredField("password", password, validationException);
+                    break;
+                case KERBEROS:
+                    validationException = validateUnsupportedField(type, "username", username, validationException);
+                    validationException = validateUnsupportedField(type, "password", password, validationException);
+                    validationException = validateUnsupportedField(type, "refresh_token", refreshToken, validationException);
+                    validationException = validateRequiredField("kerberos_ticket", kerberosTicket, validationException);
                     break;
                 case REFRESH_TOKEN:
-                    if (username != null) {
-                        validationException =
-                            addValidationError("username is not supported with the refresh_token grant_type", validationException);
-                    }
-                    if (password != null) {
-                        validationException =
-                            addValidationError("password is not supported with the refresh_token grant_type", validationException);
-                    }
-                    if (refreshToken == null) {
-                        validationException = addValidationError("refresh_token is missing", validationException);
-                    }
+                    validationException = validateUnsupportedField(type, "username", username, validationException);
+                    validationException = validateUnsupportedField(type, "password", password, validationException);
+                    validationException = validateUnsupportedField(type, "kerberos_ticket", kerberosTicket, validationException);
+                    validationException = validateRequiredField("refresh_token", refreshToken, validationException);
                     break;
                 case CLIENT_CREDENTIALS:
-                    if (username != null) {
-                        validationException =
-                            addValidationError("username is not supported with the client_credentials grant_type", validationException);
-                    }
-                    if (password != null) {
-                        validationException =
-                            addValidationError("password is not supported with the client_credentials grant_type", validationException);
-                    }
-                    if (refreshToken != null) {
-                        validationException = addValidationError("refresh_token is not supported with the client_credentials grant_type",
-                            validationException);
-                    }
+                    validationException = validateUnsupportedField(type, "username", username, validationException);
+                    validationException = validateUnsupportedField(type, "password", password, validationException);
+                    validationException = validateUnsupportedField(type, "kerberos_ticket", kerberosTicket, validationException);
+                    validationException = validateUnsupportedField(type, "refresh_token", refreshToken, validationException);
                     break;
                 default:
                     validationException = addValidationError("grant_type only supports the values: [" +
@@ -139,6 +123,32 @@ public ActionRequestValidationException validate() {
         return validationException;
     }
 
+    private static ActionRequestValidationException validateRequiredField(String field, String fieldValue,
+                                                                          ActionRequestValidationException validationException) {
+        if (Strings.isNullOrEmpty(fieldValue)) {
+            validationException = addValidationError(String.format(Locale.ROOT, "%s is missing", field), validationException);
+        }
+        return validationException;
+    }
+
+    private static ActionRequestValidationException validateRequiredField(String field, SecureString fieldValue,
+                                                                          ActionRequestValidationException validationException) {
+        if (fieldValue == null || fieldValue.getChars() == null || fieldValue.length() == 0) {
+            validationException = addValidationError(String.format(Locale.ROOT, "%s is missing", field), validationException);
+        }
+        return validationException;
+    }
+
+    private static ActionRequestValidationException validateUnsupportedField(GrantType grantType, String field, Object fieldValue,
+                                                                               ActionRequestValidationException validationException) {
+        if (fieldValue != null) {
+            validationException = addValidationError(
+                    String.format(Locale.ROOT, "%s is not supported with the %s grant_type", field, grantType.getValue()),
+                    validationException);
+        }
+        return validationException;
+    }
+
     public void setGrantType(String grantType) {
         this.grantType = grantType;
     }
@@ -151,6 +161,10 @@ public void setPassword(@Nullable SecureString password) {
         this.password = password;
     }
 
+    public void setKerberosTicket(@Nullable SecureString kerberosTicket) {
+        this.kerberosTicket = kerberosTicket;
+    }
+
     public void setScope(@Nullable String scope) {
         this.scope = scope;
     }
@@ -173,6 +187,11 @@ public SecureString getPassword() {
         return password;
     }
 
+    @Nullable
+    public SecureString getKerberosTicket() {
+        return kerberosTicket;
+    }
+
     @Nullable
     public String getScope() {
         return scope;
@@ -186,25 +205,12 @@ public String getRefreshToken() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        if (out.getVersion().before(Version.V_6_5_0) && GrantType.CLIENT_CREDENTIALS.getValue().equals(grantType)) {
-            throw new IllegalArgumentException("a request with the client_credentials grant_type cannot be sent to version [" +
-                out.getVersion() + "]");
-        }
-
         out.writeString(grantType);
         out.writeOptionalString(username);
-        if (password == null) {
-            out.writeOptionalBytesReference(null);
-        } else {
-            final byte[] passwordBytes = CharArrays.toUtf8Bytes(password.getChars());
-            try {
-                out.writeOptionalBytesReference(new BytesArray(passwordBytes));
-            } finally {
-                Arrays.fill(passwordBytes, (byte) 0);
-            }
-        }
+        out.writeOptionalSecureString(password);
         out.writeOptionalString(refreshToken);
         out.writeOptionalString(scope);
+        out.writeOptionalSecureString(kerberosTicket);
     }
 
     @Override
@@ -212,18 +218,10 @@ public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         grantType = in.readString();
         username = in.readOptionalString();
-        BytesReference bytesRef = in.readOptionalBytesReference();
-        if (bytesRef != null) {
-            byte[] bytes = BytesReference.toBytes(bytesRef);
-            try {
-                password = new SecureString(CharArrays.utf8BytesToChars(bytes));
-            } finally {
-                Arrays.fill(bytes, (byte) 0);
-            }
-        } else {
-            password = null;
-        }
+        password = in.readOptionalSecureString();
         refreshToken = in.readOptionalString();
         scope = in.readOptionalString();
+        kerberosTicket = in.readOptionalSecureString();
     }
+
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequestBuilder.java
index 90790de7cd395..1977422266299 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequestBuilder.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.core.security.action.token;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Nullable;
@@ -17,7 +17,7 @@
 public final class CreateTokenRequestBuilder
         extends ActionRequestBuilder<CreateTokenRequest, CreateTokenResponse> {
 
-    public CreateTokenRequestBuilder(ElasticsearchClient client, Action<CreateTokenResponse> action) {
+    public CreateTokenRequestBuilder(ElasticsearchClient client, ActionType<CreateTokenResponse> action) {
         super(client, action, new CreateTokenRequest());
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenResponse.java
index 93ddc56459677..299d66d840c16 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenResponse.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenResponse.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.core.security.action.token;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -27,14 +26,17 @@ public final class CreateTokenResponse extends ActionResponse implements ToXCont
     private TimeValue expiresIn;
     private String scope;
     private String refreshToken;
+    private String kerberosAuthenticationResponseToken;
 
     CreateTokenResponse() {}
 
-    public CreateTokenResponse(String tokenString, TimeValue expiresIn, String scope, String refreshToken) {
+    public CreateTokenResponse(String tokenString, TimeValue expiresIn, String scope, String refreshToken,
+                               String kerberosAuthenticationResponseToken) {
         this.tokenString = Objects.requireNonNull(tokenString);
         this.expiresIn = Objects.requireNonNull(expiresIn);
         this.scope = scope;
         this.refreshToken = refreshToken;
+        this.kerberosAuthenticationResponseToken = kerberosAuthenticationResponseToken;
     }
 
     public String getTokenString() {
@@ -53,15 +55,18 @@ public String getRefreshToken() {
         return refreshToken;
     }
 
+    public String getKerberosAuthenticationResponseToken() {
+        return kerberosAuthenticationResponseToken;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeString(tokenString);
         out.writeTimeValue(expiresIn);
         out.writeOptionalString(scope);
-        if (out.getVersion().onOrAfter(Version.V_6_5_0)) {
-            out.writeOptionalString(refreshToken);
-        }
+        out.writeOptionalString(refreshToken);
+        out.writeOptionalString(kerberosAuthenticationResponseToken);
     }
 
     @Override
@@ -70,9 +75,8 @@ public void readFrom(StreamInput in) throws IOException {
         tokenString = in.readString();
         expiresIn = in.readTimeValue();
         scope = in.readOptionalString();
-        if (in.getVersion().onOrAfter(Version.V_6_5_0)) {
-            refreshToken = in.readOptionalString();
-        }
+        refreshToken = in.readOptionalString();
+        kerberosAuthenticationResponseToken = in.readOptionalString();
     }
 
     @Override
@@ -88,6 +92,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (scope != null) {
             builder.field("scope", scope);
         }
+        if (kerberosAuthenticationResponseToken != null) {
+            builder.field("kerberos_authentication_response_token", kerberosAuthenticationResponseToken);
+        }
         return builder.endObject();
     }
 
@@ -99,11 +106,12 @@ public boolean equals(Object o) {
         return Objects.equals(tokenString, that.tokenString) &&
             Objects.equals(expiresIn, that.expiresIn) &&
             Objects.equals(scope, that.scope) &&
-            Objects.equals(refreshToken, that.refreshToken);
+            Objects.equals(refreshToken, that.refreshToken) &&
+            Objects.equals(kerberosAuthenticationResponseToken,  that.kerberosAuthenticationResponseToken);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(tokenString, expiresIn, scope, refreshToken);
+        return Objects.hash(tokenString, expiresIn, scope, refreshToken, kerberosAuthenticationResponseToken);
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenAction.java
index 57bd5bd35dd0c..72082288c2cfd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.token;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for invalidating one or more tokens
+ * ActionType for invalidating one or more tokens
  */
-public final class InvalidateTokenAction extends Action<InvalidateTokenResponse> {
+public final class InvalidateTokenAction extends StreamableResponseActionType<InvalidateTokenResponse> {
 
     public static final String NAME = "cluster:admin/xpack/security/token/invalidate";
     public static final InvalidateTokenAction INSTANCE = new InvalidateTokenAction();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/RefreshTokenAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/RefreshTokenAction.java
index 3478af2ec00f2..82dc98d4192c2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/RefreshTokenAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/token/RefreshTokenAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.core.security.action.token;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public final class RefreshTokenAction extends Action<CreateTokenResponse> {
+public final class RefreshTokenAction extends StreamableResponseActionType<CreateTokenResponse> {
 
     public static final String NAME = "cluster:admin/xpack/security/token/refresh";
     public static final RefreshTokenAction INSTANCE = new RefreshTokenAction();
@@ -20,4 +20,4 @@ private RefreshTokenAction() {
     public CreateTokenResponse newResponse() {
         return new CreateTokenResponse();
     }
-}
+}
\ No newline at end of file
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/AuthenticateAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/AuthenticateAction.java
index 18cfe85c8cb0c..606a32803c455 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/AuthenticateAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/AuthenticateAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.core.security.action.user;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class AuthenticateAction extends Action<AuthenticateResponse> {
+public class AuthenticateAction extends StreamableResponseActionType<AuthenticateResponse> {
 
     public static final String NAME = "cluster:admin/xpack/security/user/authenticate";
     public static final AuthenticateAction INSTANCE = new AuthenticateAction();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordAction.java
index d01717a64eadc..981d9cb67ec83 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.core.security.action.user;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class ChangePasswordAction extends Action<ChangePasswordResponse> {
+public class ChangePasswordAction extends StreamableResponseActionType<ChangePasswordResponse> {
 
     public static final ChangePasswordAction INSTANCE = new ChangePasswordAction();
     public static final String NAME = "cluster:admin/xpack/security/user/change_password";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java
index d7538c2a556f0..632e5fb6b5473 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/ChangePasswordRequestBuilder.java
@@ -42,7 +42,7 @@ public ChangePasswordRequestBuilder username(String username) {
     }
 
     public static char[] validateAndHashPassword(SecureString password, Hasher hasher) {
-        Validation.Error error = Validation.Users.validatePassword(password.getChars());
+        Validation.Error error = Validation.Users.validatePassword(password);
         if (error != null) {
             ValidationException validationException = new ValidationException();
             validationException.addValidationError(error.toString());
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/DeleteUserAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/DeleteUserAction.java
index 78666759dc0a7..e9b693a7521b9 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/DeleteUserAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/DeleteUserAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.user;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for deleting a native user.
+ * ActionType for deleting a native user.
  */
-public class DeleteUserAction extends Action<DeleteUserResponse> {
+public class DeleteUserAction extends StreamableResponseActionType<DeleteUserResponse> {
 
     public static final DeleteUserAction INSTANCE = new DeleteUserAction();
     public static final String NAME = "cluster:admin/xpack/security/user/delete";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesAction.java
index 6d51d74d89962..ca5327dcc76f4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.user;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action that lists the set of privileges held by a user.
+ * ActionType that lists the set of privileges held by a user.
  */
-public final class GetUserPrivilegesAction extends Action<GetUserPrivilegesResponse> {
+public final class GetUserPrivilegesAction extends StreamableResponseActionType<GetUserPrivilegesResponse> {
 
     public static final GetUserPrivilegesAction INSTANCE = new GetUserPrivilegesAction();
     public static final String NAME = "cluster:admin/xpack/security/user/list_privileges";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersAction.java
index 49532049ba908..55479c220c100 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.user;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for retrieving a user from the security index
+ * ActionType for retrieving a user from the security index
  */
-public class GetUsersAction extends Action<GetUsersResponse> {
+public class GetUsersAction extends StreamableResponseActionType<GetUsersResponse> {
 
     public static final GetUsersAction INSTANCE = new GetUsersAction();
     public static final String NAME = "cluster:admin/xpack/security/user/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesAction.java
index 30bb44a2c1c33..8d05574d295ef 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesAction.java
@@ -5,14 +5,14 @@
  */
 package org.elasticsearch.xpack.core.security.action.user;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 
 /**
  * This action is testing whether a user has the specified
  * {@link RoleDescriptor.IndicesPrivileges privileges}
  */
-public class HasPrivilegesAction extends Action<HasPrivilegesResponse> {
+public class HasPrivilegesAction extends StreamableResponseActionType<HasPrivilegesResponse> {
 
     public static final HasPrivilegesAction INSTANCE = new HasPrivilegesAction();
     public static final String NAME = "cluster:admin/xpack/security/user/has_privileges";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserAction.java
index 20bbde2366b5e..7d4d66c4ea3cb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.user;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * Action for putting (adding/updating) a native user.
+ * ActionType for putting (adding/updating) a native user.
  */
-public class PutUserAction extends Action<PutUserResponse> {
+public class PutUserAction extends StreamableResponseActionType<PutUserResponse> {
 
     public static final PutUserAction INSTANCE = new PutUserAction();
     public static final String NAME = "cluster:admin/xpack/security/user/put";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java
index f667bff513b78..bee938f6c0216 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/PutUserRequestBuilder.java
@@ -25,7 +25,6 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.Arrays;
 import java.util.Map;
 import java.util.Objects;
 
@@ -50,7 +49,15 @@ public PutUserRequestBuilder roles(String... roles) {
         return this;
     }
 
+    /**
+     * @deprecated Use {@link #password(SecureString, Hasher)} instead.
+     */
+    @Deprecated
     public PutUserRequestBuilder password(char[] password, Hasher hasher) {
+        return password(new SecureString(password), hasher);
+    }
+
+    public PutUserRequestBuilder password(SecureString password, Hasher hasher) {
         if (password != null) {
             Validation.Error error = Validation.Users.validatePassword(password);
             if (error != null) {
@@ -59,7 +66,7 @@ public PutUserRequestBuilder password(char[] password, Hasher hasher) {
             if (request.passwordHash() != null) {
                 throw validationException("password_hash has already been set");
             }
-            request.passwordHash(hasher.hash(new SecureString(password)));
+            request.passwordHash(hasher.hash(password));
         } else {
             request.passwordHash(null);
         }
@@ -119,9 +126,9 @@ public PutUserRequestBuilder source(String username, BytesReference source, XCon
                 } else if (User.Fields.PASSWORD.match(currentFieldName, parser.getDeprecationHandler())) {
                     if (token == XContentParser.Token.VALUE_STRING) {
                         String password = parser.text();
-                        char[] passwordChars = password.toCharArray();
-                        password(passwordChars, hasher);
-                        Arrays.fill(passwordChars, (char) 0);
+                        try(SecureString securePassword = new SecureString(password.toCharArray())) {
+                            password(securePassword, hasher);
+                        }
                     } else {
                         throw new ElasticsearchParseException(
                                 "expected field [{}] to be of type string, but found [{}] instead", currentFieldName, token);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/SetEnabledAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/SetEnabledAction.java
index 0368cdf7d7dbc..6073ba131d1b1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/SetEnabledAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/SetEnabledAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.security.action.user;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
  * This action is for setting the enabled flag on a native or reserved user
  */
-public class SetEnabledAction extends Action<SetEnabledResponse> {
+public class SetEnabledAction extends StreamableResponseActionType<SetEnabledResponse> {
 
     public static final SetEnabledAction INSTANCE = new SetEnabledAction();
     public static final String NAME = "cluster:admin/xpack/security/user/set_enabled";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java
index 5413a38bd6288..1b5b65e60c258 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/Hasher.java
@@ -360,14 +360,14 @@ public boolean verify(SecureString text, char[] hash) {
         public char[] hash(SecureString text) {
             MessageDigest md = MessageDigests.sha256();
             md.update(CharArrays.toUtf8Bytes(text.getChars()));
-            return Base64.getEncoder().encodeToString(md.digest()).toCharArray();
+            return Base64.getUrlEncoder().withoutPadding().encodeToString(md.digest()).toCharArray();
         }
 
         @Override
         public boolean verify(SecureString text, char[] hash) {
             MessageDigest md = MessageDigests.sha256();
             md.update(CharArrays.toUtf8Bytes(text.getChars()));
-            return CharArrays.constantTimeEquals(Base64.getEncoder().encodeToString(md.digest()).toCharArray(), hash);
+            return CharArrays.constantTimeEquals(Base64.getUrlEncoder().withoutPadding().encodeToString(md.digest()).toCharArray(), hash);
         }
     },
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCache.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCache.java
new file mode 100644
index 0000000000000..6b47c3da2fb58
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCache.java
@@ -0,0 +1,206 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.accesscontrol;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReaderContext;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.ReaderUtil;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.BitSet;
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.cache.Cache;
+import org.elasticsearch.common.cache.CacheBuilder;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.set.Sets;
+
+import java.io.Closeable;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * This is a cache for {@link BitSet} instances that are used with the {@link DocumentSubsetReader}.
+ * It is bounded by memory size and access time.
+ *
+ * @see org.elasticsearch.index.cache.bitset.BitsetFilterCache
+ */
+public final class DocumentSubsetBitsetCache implements IndexReader.ClosedListener, Closeable, Accountable {
+
+    /**
+     * The TTL defaults to 1 week. We depend on the {@code max_bytes} setting to keep the cache to a sensible size, by evicting LRU
+     * entries, however there is benefit in reclaiming memory by expiring bitsets that have not be used for some period of time.
+     * Because {@link org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission.Group#query} can be templated, it is
+     * not uncommon for a query to only be used for a relatively short period of time (e.g. because a user's metadata changed, or because
+     * that user is an infrequent user of Elasticsearch). This access time expiry helps free up memory in those circumstances even if the
+     * cache is never filled.
+     */
+    static final Setting<TimeValue> CACHE_TTL_SETTING =
+        Setting.timeSetting("xpack.security.dls.bitset.cache.ttl", TimeValue.timeValueHours(24 * 7), Property.NodeScope);
+
+    static final Setting<ByteSizeValue> CACHE_SIZE_SETTING = Setting.byteSizeSetting("xpack.security.dls.bitset.cache.size",
+            new ByteSizeValue(50, ByteSizeUnit.MB), Property.NodeScope);
+
+    private static final BitSet NULL_MARKER = new FixedBitSet(0);
+
+    private final Logger logger;
+    private final Cache<BitsetCacheKey, BitSet> bitsetCache;
+    private final Map<IndexReader.CacheKey, Set<BitsetCacheKey>> keysByIndex;
+
+    public DocumentSubsetBitsetCache(Settings settings) {
+        this.logger = LogManager.getLogger(getClass());
+        final TimeValue ttl = CACHE_TTL_SETTING.get(settings);
+        final ByteSizeValue size = CACHE_SIZE_SETTING.get(settings);
+        this.bitsetCache = CacheBuilder.<BitsetCacheKey, BitSet>builder()
+            .setExpireAfterAccess(ttl)
+            .setMaximumWeight(size.getBytes())
+            .weigher((key, bitSet) -> bitSet == NULL_MARKER ? 0 : bitSet.ramBytesUsed()).build();
+        this.keysByIndex = new ConcurrentHashMap<>();
+    }
+
+    @Override
+    public void onClose(IndexReader.CacheKey ownerCoreCacheKey) {
+        final Set<BitsetCacheKey> keys = keysByIndex.remove(ownerCoreCacheKey);
+        if (keys != null) {
+            // Because this Set has been removed from the map, and the only update to the set is performed in a
+            // Map#compute call, it should not be possible to get a concurrent modification here.
+            keys.forEach(bitsetCache::invalidate);
+        }
+    }
+
+    @Override
+    public void close() {
+        clear("close");
+    }
+
+    public void clear(String reason) {
+        logger.debug("clearing all DLS bitsets because [{}]", reason);
+        // Due to the order here, it is possible than a new entry could be added _after_ the keysByIndex map is cleared
+        // but _before_ the cache is cleared. This would mean it sits orphaned in keysByIndex, but this is not a issue.
+        // When the index is closed, the key will be removed from the map, and there will not be a corresponding item
+        // in the cache, which will make the cache-invalidate a no-op.
+        // Since the entry is not in the cache, if #getBitSet is called, it will be loaded, and the new key will be added
+        // to the index without issue.
+        keysByIndex.clear();
+        bitsetCache.invalidateAll();
+    }
+
+    int entryCount() {
+        return this.bitsetCache.count();
+    }
+
+    @Override
+    public long ramBytesUsed() {
+        return this.bitsetCache.weight();
+    }
+
+    /**
+     * Obtain the {@link BitSet} for the given {@code query} in the given {@code context}.
+     * If there is a cached entry for that query and context, it will be returned.
+     * Otherwise a new BitSet will be created and stored in the cache.
+     * The returned BitSet may be null (e.g. if the query has no results).
+     */
+    @Nullable
+    public BitSet getBitSet(final Query query, final LeafReaderContext context) throws ExecutionException {
+        final IndexReader.CacheHelper coreCacheHelper = context.reader().getCoreCacheHelper();
+        if (coreCacheHelper == null) {
+            throw new IllegalArgumentException("Reader " + context.reader() + " does not support caching");
+        }
+        coreCacheHelper.addClosedListener(this);
+        final IndexReader.CacheKey indexKey = coreCacheHelper.getKey();
+        final BitsetCacheKey cacheKey = new BitsetCacheKey(indexKey, query);
+
+        final BitSet bitSet = bitsetCache.computeIfAbsent(cacheKey, ignore1 -> {
+            // This ensures all insertions into the set are guarded by ConcurrentHashMap's atomicity guarantees.
+            keysByIndex.compute(indexKey, (ignore2, set) -> {
+                if (set == null) {
+                    set = Sets.newConcurrentHashSet();
+                }
+                set.add(cacheKey);
+                return set;
+            });
+            final IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(context);
+            final IndexSearcher searcher = new IndexSearcher(topLevelContext);
+            searcher.setQueryCache(null);
+            final Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1f);
+            Scorer s = weight.scorer(context);
+            if (s == null) {
+                // A cache loader is not allowed to return null, return a marker object instead.
+                return NULL_MARKER;
+            } else {
+                return BitSet.of(s.iterator(), context.reader().maxDoc());
+            }
+        });
+        if (bitSet == NULL_MARKER) {
+            return null;
+        } else {
+            return bitSet;
+        }
+    }
+
+    public static List<Setting<?>> getSettings() {
+        return List.of(CACHE_TTL_SETTING, CACHE_SIZE_SETTING);
+    }
+
+    public Map<String, Object> usageStats() {
+        final ByteSizeValue ram = new ByteSizeValue(ramBytesUsed(), ByteSizeUnit.BYTES);
+        return Map.of(
+            "count", entryCount(),
+            "memory", ram.toString(),
+            "memory_in_bytes", ram.getBytes()
+        );
+    }
+
+    private class BitsetCacheKey {
+        final IndexReader.CacheKey index;
+        final Query query;
+
+        private BitsetCacheKey(IndexReader.CacheKey index, Query query) {
+            this.index = index;
+            this.query = query;
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            if (this == other) {
+                return true;
+            }
+            if (other == null || getClass() != other.getClass()) {
+                return false;
+            }
+            final BitsetCacheKey that = (BitsetCacheKey) other;
+            return Objects.equals(this.index, that.index) &&
+                Objects.equals(this.query, that.query);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(index, query);
+        }
+
+        @Override
+        public String toString() {
+            return getClass().getSimpleName() + "(" + index + "," + query + ")";
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReader.java
index c7d84b3c40fa9..1cda15c8e3c5e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReader.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReader.java
@@ -16,11 +16,11 @@
 import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.BitSetIterator;
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.CombinedBitSet;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.cache.Cache;
 import org.elasticsearch.common.cache.CacheBuilder;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
-import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -33,9 +33,9 @@
  */
 public final class DocumentSubsetReader extends FilterLeafReader {
 
-    public static DocumentSubsetDirectoryReader wrap(DirectoryReader in, BitsetFilterCache bitsetFilterCache,
+    public static DocumentSubsetDirectoryReader wrap(DirectoryReader in, DocumentSubsetBitsetCache bitsetCache,
             Query roleQuery) throws IOException {
-        return new DocumentSubsetDirectoryReader(in, bitsetFilterCache, roleQuery);
+        return new DocumentSubsetDirectoryReader(in, bitsetCache, roleQuery);
     }
 
     /**
@@ -109,21 +109,21 @@ private static int getNumDocs(LeafReader reader, Query roleQuery, BitSet roleQue
     public static final class DocumentSubsetDirectoryReader extends FilterDirectoryReader {
 
         private final Query roleQuery;
-        private final BitsetFilterCache bitsetFilterCache;
+        private final DocumentSubsetBitsetCache bitsetCache;
 
-        DocumentSubsetDirectoryReader(final DirectoryReader in, final BitsetFilterCache bitsetFilterCache, final Query roleQuery)
-                throws IOException {
+        DocumentSubsetDirectoryReader(final DirectoryReader in, final DocumentSubsetBitsetCache bitsetCache,
+                                      final Query roleQuery) throws IOException {
             super(in, new SubReaderWrapper() {
                 @Override
                 public LeafReader wrap(LeafReader reader) {
                     try {
-                        return new DocumentSubsetReader(reader, bitsetFilterCache, roleQuery);
+                        return new DocumentSubsetReader(reader, bitsetCache, roleQuery);
                     } catch (Exception e) {
                         throw ExceptionsHelper.convertToElastic(e);
                     }
                 }
             });
-            this.bitsetFilterCache = bitsetFilterCache;
+            this.bitsetCache = bitsetCache;
             this.roleQuery = roleQuery;
 
             verifyNoOtherDocumentSubsetDirectoryReaderIsWrapped(in);
@@ -131,7 +131,7 @@ public LeafReader wrap(LeafReader reader) {
 
         @Override
         protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
-            return new DocumentSubsetDirectoryReader(in, bitsetFilterCache, roleQuery);
+            return new DocumentSubsetDirectoryReader(in, bitsetCache, roleQuery);
         }
 
         private static void verifyNoOtherDocumentSubsetDirectoryReaderIsWrapped(DirectoryReader reader) {
@@ -155,9 +155,9 @@ public CacheHelper getReaderCacheHelper() {
     private final BitSet roleQueryBits;
     private final int numDocs;
 
-    private DocumentSubsetReader(final LeafReader in, BitsetFilterCache bitsetFilterCache, final Query roleQuery) throws Exception {
+    private DocumentSubsetReader(final LeafReader in, DocumentSubsetBitsetCache bitsetCache, final Query roleQuery) throws Exception {
         super(in);
-        this.roleQueryBits = bitsetFilterCache.getBitSetProducer(roleQuery).getBitSet(in.getContext());
+        this.roleQueryBits = bitsetCache.getBitSet(roleQuery, in.getContext());
         this.numDocs = getNumDocs(in, roleQuery, roleQueryBits);
     }
 
@@ -172,18 +172,7 @@ public Bits getLiveDocs() {
             return roleQueryBits;
         } else {
             // apply deletes when needed:
-            return new Bits() {
-
-                @Override
-                public boolean get(int index) {
-                    return roleQueryBits.get(index) && actualLiveDocs.get(index);
-                }
-
-                @Override
-                public int length() {
-                    return roleQueryBits.length();
-                }
-            };
+            return new CombinedBitSet(roleQueryBits, actualLiveDocs);
         }
     }
 
@@ -208,13 +197,4 @@ public CacheHelper getReaderCacheHelper() {
         // Not delegated since we change the live docs
         return null;
     }
-
-    BitSet getRoleQueryBits() {
-        return roleQueryBits;
-    }
-
-    Bits getWrappedLiveDocs() {
-        return in.getLiveDocs();
-    }
-
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java
new file mode 100644
index 0000000000000..ea8f005be0397
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.authz.accesscontrol;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.ConstantScoreQuery;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.logging.LoggerMessageFormat;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.shard.ShardUtils;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
+import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions;
+import org.elasticsearch.xpack.core.security.support.Exceptions;
+import org.elasticsearch.xpack.core.security.user.User;
+
+import java.io.IOException;
+import java.util.function.Function;
+
+/**
+ * An IndexReader wrapper implementation that is used for field and document level security.
+ * <p>
+ * Based on the {@link ThreadContext} this class will enable field and/or document level security.
+ * <p>
+ * Field level security is enabled by wrapping the original {@link DirectoryReader} in a {@link FieldSubsetReader}
+ * in the {@link #apply(DirectoryReader)} method.
+ * <p>
+ * Document level security is enabled by wrapping the original {@link DirectoryReader} in a {@link DocumentSubsetReader}
+ * instance.
+ */
+public class SecurityIndexReaderWrapper implements CheckedFunction<DirectoryReader, DirectoryReader, IOException> {
+    private static final Logger logger = LogManager.getLogger(SecurityIndexReaderWrapper.class);
+
+    private final Function<ShardId, QueryShardContext> queryShardContextProvider;
+    private final DocumentSubsetBitsetCache bitsetCache;
+    private final XPackLicenseState licenseState;
+    private final ThreadContext threadContext;
+    private final ScriptService scriptService;
+
+    public SecurityIndexReaderWrapper(Function<ShardId, QueryShardContext> queryShardContextProvider,
+                                      DocumentSubsetBitsetCache bitsetCache, ThreadContext threadContext, XPackLicenseState licenseState,
+                                      ScriptService scriptService) {
+        this.scriptService = scriptService;
+        this.queryShardContextProvider = queryShardContextProvider;
+        this.bitsetCache = bitsetCache;
+        this.threadContext = threadContext;
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    public DirectoryReader apply(final DirectoryReader reader) {
+        if (licenseState.isDocumentAndFieldLevelSecurityAllowed() == false) {
+            return reader;
+        }
+
+        try {
+            final IndicesAccessControl indicesAccessControl = getIndicesAccessControl();
+
+            ShardId shardId = ShardUtils.extractShardId(reader);
+            if (shardId == null) {
+                throw new IllegalStateException(LoggerMessageFormat.format("couldn't extract shardId from reader [{}]", reader));
+            }
+
+            final IndicesAccessControl.IndexAccessControl permissions = indicesAccessControl.getIndexPermissions(shardId.getIndexName());
+            // No permissions have been defined for an index, so don't intercept the index reader for access control
+            if (permissions == null) {
+                return reader;
+            }
+
+            DirectoryReader wrappedReader = reader;
+            DocumentPermissions documentPermissions = permissions.getDocumentPermissions();
+            if (documentPermissions != null && documentPermissions.hasDocumentLevelPermissions()) {
+                BooleanQuery filterQuery = documentPermissions.filter(getUser(), scriptService, shardId, queryShardContextProvider);
+                if (filterQuery != null) {
+                    wrappedReader = DocumentSubsetReader.wrap(wrappedReader, bitsetCache, new ConstantScoreQuery(filterQuery));
+                }
+            }
+
+            return permissions.getFieldPermissions().filter(wrappedReader);
+        } catch (IOException e) {
+            logger.error("Unable to apply field level security");
+            throw ExceptionsHelper.convertToElastic(e);
+        }
+    }
+
+    protected IndicesAccessControl getIndicesAccessControl() {
+        IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
+        if (indicesAccessControl == null) {
+            throw Exceptions.authorizationError("no indices permissions found");
+        }
+        return indicesAccessControl;
+    }
+
+    protected User getUser(){
+        Authentication authentication = Authentication.getAuthentication(threadContext);
+        return authentication.getUser();
+    }
+
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java
deleted file mode 100644
index 56383909d846e..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.security.authz.accesscontrol;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.BulkScorer;
-import org.apache.lucene.search.CollectionTerminatedException;
-import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.ConjunctionDISI;
-import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.LeafCollector;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.util.BitSet;
-import org.apache.lucene.util.BitSetIterator;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.SparseFixedBitSet;
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.common.logging.LoggerMessageFormat;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
-import org.elasticsearch.index.engine.EngineException;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.shard.IndexSearcherWrapper;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.shard.ShardUtils;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.xpack.core.security.authc.Authentication;
-import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
-import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetReader.DocumentSubsetDirectoryReader;
-import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions;
-import org.elasticsearch.xpack.core.security.support.Exceptions;
-import org.elasticsearch.xpack.core.security.user.User;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.function.Function;
-
-/**
- * An {@link IndexSearcherWrapper} implementation that is used for field and document level security.
- * <p>
- * Based on the {@link ThreadContext} this class will enable field and/or document level security.
- * <p>
- * Field level security is enabled by wrapping the original {@link DirectoryReader} in a {@link FieldSubsetReader}
- * in the {@link #wrap(DirectoryReader)} method.
- * <p>
- * Document level security is enabled by wrapping the original {@link DirectoryReader} in a {@link DocumentSubsetReader}
- * instance.
- */
-public class SecurityIndexSearcherWrapper extends IndexSearcherWrapper {
-    private static final Logger logger = LogManager.getLogger(SecurityIndexSearcherWrapper.class);
-
-    private final Function<ShardId, QueryShardContext> queryShardContextProvider;
-    private final BitsetFilterCache bitsetFilterCache;
-    private final XPackLicenseState licenseState;
-    private final ThreadContext threadContext;
-    private final ScriptService scriptService;
-
-    public SecurityIndexSearcherWrapper(Function<ShardId, QueryShardContext> queryShardContextProvider,
-                                        BitsetFilterCache bitsetFilterCache, ThreadContext threadContext, XPackLicenseState licenseState,
-                                        ScriptService scriptService) {
-        this.scriptService = scriptService;
-        this.queryShardContextProvider = queryShardContextProvider;
-        this.bitsetFilterCache = bitsetFilterCache;
-        this.threadContext = threadContext;
-        this.licenseState = licenseState;
-    }
-
-    @Override
-    protected DirectoryReader wrap(final DirectoryReader reader) {
-        if (licenseState.isDocumentAndFieldLevelSecurityAllowed() == false) {
-            return reader;
-        }
-
-        try {
-            final IndicesAccessControl indicesAccessControl = getIndicesAccessControl();
-
-            ShardId shardId = ShardUtils.extractShardId(reader);
-            if (shardId == null) {
-                throw new IllegalStateException(LoggerMessageFormat.format("couldn't extract shardId from reader [{}]", reader));
-            }
-
-            final IndicesAccessControl.IndexAccessControl permissions = indicesAccessControl.getIndexPermissions(shardId.getIndexName());
-            // No permissions have been defined for an index, so don't intercept the index reader for access control
-            if (permissions == null) {
-                return reader;
-            }
-
-            DirectoryReader wrappedReader = reader;
-            DocumentPermissions documentPermissions = permissions.getDocumentPermissions();
-            if (documentPermissions != null && documentPermissions.hasDocumentLevelPermissions()) {
-                BooleanQuery filterQuery = documentPermissions.filter(getUser(), scriptService, shardId, queryShardContextProvider);
-                if (filterQuery != null) {
-                    wrappedReader = DocumentSubsetReader.wrap(wrappedReader, bitsetFilterCache, new ConstantScoreQuery(filterQuery));
-                }
-            }
-
-            return permissions.getFieldPermissions().filter(wrappedReader);
-        } catch (IOException e) {
-            logger.error("Unable to apply field level security");
-            throw ExceptionsHelper.convertToElastic(e);
-        }
-    }
-
-    @Override
-    protected IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
-        if (licenseState.isDocumentAndFieldLevelSecurityAllowed() == false) {
-            return searcher;
-        }
-
-        final DirectoryReader directoryReader = (DirectoryReader) searcher.getIndexReader();
-        if (directoryReader instanceof DocumentSubsetDirectoryReader) {
-            // The reasons why we return a custom searcher:
-            // 1) in the case the role query is sparse then large part of the main query can be skipped
-            // 2) If the role query doesn't match with any docs in a segment, that a segment can be skipped
-            IndexSearcher indexSearcher = new IndexSearcherWrapper((DocumentSubsetDirectoryReader) directoryReader);
-            indexSearcher.setQueryCache(indexSearcher.getQueryCache());
-            indexSearcher.setQueryCachingPolicy(indexSearcher.getQueryCachingPolicy());
-            indexSearcher.setSimilarity(indexSearcher.getSimilarity());
-            return indexSearcher;
-        }
-        return searcher;
-    }
-
-    static class IndexSearcherWrapper extends IndexSearcher {
-
-        IndexSearcherWrapper(DocumentSubsetDirectoryReader r) {
-            super(r);
-        }
-
-        @Override
-        protected void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
-            for (LeafReaderContext ctx : leaves) { // search each subreader
-                final LeafCollector leafCollector;
-                try {
-                    leafCollector = collector.getLeafCollector(ctx);
-                } catch (CollectionTerminatedException e) {
-                    // there is no doc of interest in this reader context
-                    // continue with the following leaf
-                    continue;
-                }
-                // The reader is always of type DocumentSubsetReader when we get here:
-                DocumentSubsetReader reader = (DocumentSubsetReader) ctx.reader();
-
-                BitSet roleQueryBits = reader.getRoleQueryBits();
-                if (roleQueryBits == null) {
-                    // nothing matches with the role query, so skip this segment:
-                    continue;
-                }
-
-                // if the role query result set is sparse then we should use the SparseFixedBitSet for advancing:
-                if (roleQueryBits instanceof SparseFixedBitSet) {
-                    Scorer scorer = weight.scorer(ctx);
-                    if (scorer != null) {
-                        SparseFixedBitSet sparseFixedBitSet = (SparseFixedBitSet) roleQueryBits;
-                        Bits realLiveDocs = reader.getWrappedLiveDocs();
-                        try {
-                            intersectScorerAndRoleBits(scorer, sparseFixedBitSet, leafCollector, realLiveDocs);
-                        } catch (CollectionTerminatedException e) {
-                            // collection was terminated prematurely
-                            // continue with the following leaf
-                        }
-                    }
-                } else {
-                    BulkScorer bulkScorer = weight.bulkScorer(ctx);
-                    if (bulkScorer != null) {
-                        Bits liveDocs = reader.getLiveDocs();
-                        try {
-                            bulkScorer.score(leafCollector, liveDocs);
-                        } catch (CollectionTerminatedException e) {
-                            // collection was terminated prematurely
-                            // continue with the following leaf
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    static void intersectScorerAndRoleBits(Scorer scorer, SparseFixedBitSet roleBits, LeafCollector collector, Bits acceptDocs) throws
-            IOException {
-        // ConjunctionDISI uses the DocIdSetIterator#cost() to order the iterators, so if roleBits has the lowest cardinality it should
-        // be used first:
-        DocIdSetIterator iterator = ConjunctionDISI.intersectIterators(Arrays.asList(new BitSetIterator(roleBits,
-                roleBits.approximateCardinality()), scorer.iterator()));
-        for (int docId = iterator.nextDoc(); docId < DocIdSetIterator.NO_MORE_DOCS; docId = iterator.nextDoc()) {
-            if (acceptDocs == null || acceptDocs.get(docId)) {
-                collector.collect(docId);
-            }
-        }
-    }
-
-    protected IndicesAccessControl getIndicesAccessControl() {
-        IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY);
-        if (indicesAccessControl == null) {
-            throw Exceptions.authorizationError("no indices permissions found");
-        }
-        return indicesAccessControl;
-    }
-
-    protected User getUser(){
-        Authentication authentication = Authentication.getAuthentication(threadContext);
-        return authentication.getUser();
-    }
-
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilege.java
index c4460b000e6d8..15a6d8bae92db 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilege.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ApplicationPrivilege.java
@@ -24,7 +24,7 @@
  * An application privilege has an application name (e.g. {@code "my-app"}) that identifies an application (that exists
  * outside of elasticsearch), a privilege name (e.g. {@code "admin}) that is meaningful to that application, and one or
  * more "action patterns" (e.g {@code "admin/user/*", "admin/team/*"}).
- * Action patterns must contain at least one special character from ({@code /}, {@code :}, {@code *}) to distinguish them
+ * ActionType patterns must contain at least one special character from ({@code /}, {@code :}, {@code *}) to distinguish them
  * from privilege names.
  * The action patterns are entirely optional - many application will find that simple "privilege names" are sufficient, but
  * they allow applications to define high level abstract privileges that map to multiple low level capabilities.
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilege.java
index f7d03c2356e5b..cae2f223bf6f0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilege.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilege.java
@@ -39,6 +39,7 @@ public final class ClusterPrivilege extends Privilege {
             InvalidateTokenAction.NAME, RefreshTokenAction.NAME);
     private static final Automaton MANAGE_OIDC_AUTOMATON = patterns("cluster:admin/xpack/security/oidc/*");
     private static final Automaton MANAGE_TOKEN_AUTOMATON = patterns("cluster:admin/xpack/security/token/*");
+    private static final Automaton MANAGE_API_KEY_AUTOMATON = patterns("cluster:admin/xpack/security/api_key/*");
     private static final Automaton MONITOR_AUTOMATON = patterns("cluster:monitor/*");
     private static final Automaton MONITOR_ML_AUTOMATON = patterns("cluster:monitor/xpack/ml/*");
     private static final Automaton MONITOR_DATA_FRAME_AUTOMATON = patterns("cluster:monitor/data_frame/*");
@@ -84,6 +85,7 @@ public final class ClusterPrivilege extends Privilege {
     public static final ClusterPrivilege MANAGE_SECURITY =       new ClusterPrivilege("manage_security",     MANAGE_SECURITY_AUTOMATON);
     public static final ClusterPrivilege MANAGE_SAML =           new ClusterPrivilege("manage_saml",         MANAGE_SAML_AUTOMATON);
     public static final ClusterPrivilege MANAGE_OIDC =           new ClusterPrivilege("manage_oidc", MANAGE_OIDC_AUTOMATON);
+    public static final ClusterPrivilege MANAGE_API_KEY =        new ClusterPrivilege("manage_api_key", MANAGE_API_KEY_AUTOMATON);
     public static final ClusterPrivilege MANAGE_PIPELINE =       new ClusterPrivilege("manage_pipeline", "cluster:admin/ingest/pipeline/*");
     public static final ClusterPrivilege MANAGE_CCR =            new ClusterPrivilege("manage_ccr", MANAGE_CCR_AUTOMATON);
     public static final ClusterPrivilege READ_CCR =              new ClusterPrivilege("read_ccr", READ_CCR_AUTOMATON);
@@ -112,6 +114,7 @@ public final class ClusterPrivilege extends Privilege {
             entry("manage_security", MANAGE_SECURITY),
             entry("manage_saml", MANAGE_SAML),
             entry("manage_oidc", MANAGE_OIDC),
+            entry("manage_api_key", MANAGE_API_KEY),
             entry("manage_pipeline", MANAGE_PIPELINE),
             entry("manage_rollup", MANAGE_ROLLUP),
             entry("manage_ccr", MANAGE_CCR),
@@ -173,4 +176,8 @@ private static ClusterPrivilege resolve(Set<String> name) {
         }
         return new ClusterPrivilege(name, Automatons.unionAndMinimize(automata));
     }
+
+    public static Set<String> names() {
+        return Collections.unmodifiableSet(VALUES.keySet());
+    }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java
index c4092d8947f78..0315a65b1a78b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java
@@ -7,13 +7,10 @@
 
 import org.apache.lucene.util.automaton.Automaton;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsAction;
-import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistAction;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
 import org.elasticsearch.action.admin.indices.close.CloseIndexAction;
 import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
-import org.elasticsearch.action.admin.indices.exists.types.TypesExistsAction;
 import org.elasticsearch.action.admin.indices.get.GetIndexAction;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsAction;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
@@ -57,10 +54,9 @@ public final class IndexPrivilege extends Privilege {
             unionAndMinimize(Arrays.asList(MONITOR_AUTOMATON, patterns("indices:admin/*")));
     private static final Automaton CREATE_INDEX_AUTOMATON = patterns(CreateIndexAction.NAME);
     private static final Automaton DELETE_INDEX_AUTOMATON = patterns(DeleteIndexAction.NAME);
-    private static final Automaton VIEW_METADATA_AUTOMATON = patterns(GetAliasesAction.NAME, AliasesExistAction.NAME,
-            GetIndexAction.NAME, IndicesExistsAction.NAME, GetFieldMappingsAction.NAME + "*", GetMappingsAction.NAME,
-            ClusterSearchShardsAction.NAME, TypesExistsAction.NAME, ValidateQueryAction.NAME + "*", GetSettingsAction.NAME,
-            ExplainLifecycleAction.NAME);
+    private static final Automaton VIEW_METADATA_AUTOMATON = patterns(GetAliasesAction.NAME,
+            GetIndexAction.NAME, GetFieldMappingsAction.NAME + "*", GetMappingsAction.NAME,
+            ClusterSearchShardsAction.NAME, ValidateQueryAction.NAME + "*", GetSettingsAction.NAME, ExplainLifecycleAction.NAME);
     private static final Automaton MANAGE_FOLLOW_INDEX_AUTOMATON = patterns(PutFollowAction.NAME, UnfollowAction.NAME,
         CloseIndexAction.NAME + "*");
     private static final Automaton MANAGE_LEADER_INDEX_AUTOMATON = patterns(ForgetFollowerAction.NAME + "*");
@@ -160,4 +156,9 @@ private static IndexPrivilege resolve(Set<String> name) {
     static Map<String, IndexPrivilege> values() {
         return VALUES;
     }
+
+    public static Set<String> names() {
+        return Collections.unmodifiableSet(VALUES.keySet());
+    }
+
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
index 49d4159f13968..494f92e1a4137 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java
@@ -9,6 +9,7 @@
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesAction;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkAction;
+import org.elasticsearch.xpack.core.security.action.privilege.GetBuiltinPrivilegesAction;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
@@ -109,7 +110,8 @@ private static Map<String, RoleDescriptor> initializeReservedRoles() {
                         null))
                 .put(KibanaUser.ROLE_NAME, new RoleDescriptor(KibanaUser.ROLE_NAME,
                         new String[] {
-                            "monitor", "manage_index_templates", MonitoringBulkAction.NAME, "manage_saml", "manage_token", "manage_oidc"
+                            "monitor", "manage_index_templates", MonitoringBulkAction.NAME, "manage_saml", "manage_token", "manage_oidc",
+                            GetBuiltinPrivilegesAction.NAME
                         },
                         new RoleDescriptor.IndicesPrivileges[] {
                                 RoleDescriptor.IndicesPrivileges.builder()
@@ -180,14 +182,22 @@ private static Map<String, RoleDescriptor> initializeReservedRoles() {
                             RoleDescriptor.IndicesPrivileges.builder()
                                 .indices(".data-frame-notifications*")
                                 .privileges("view_index_metadata", "read").build()
-                        }, null, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, null))
+                        },
+                        new RoleDescriptor.ApplicationResourcePrivileges[] {
+                            RoleDescriptor.ApplicationResourcePrivileges.builder()
+                                .application("kibana-*").resources("*").privileges("reserved_ml").build()
+                        }, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, null))
                 .put("data_frame_transforms_user", new RoleDescriptor("data_frame_transforms_user",
                         new String[] { "monitor_data_frame_transforms" },
                         new RoleDescriptor.IndicesPrivileges[]{
                             RoleDescriptor.IndicesPrivileges.builder()
                                 .indices(".data-frame-notifications*")
                                 .privileges("view_index_metadata", "read").build()
-                        }, null, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, null))
+                        },
+                        new RoleDescriptor.ApplicationResourcePrivileges[] {
+                            RoleDescriptor.ApplicationResourcePrivileges.builder()
+                                .application("kibana-*").resources("*").privileges("reserved_ml").build()
+                        }, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, null))
                 .put("watcher_admin", new RoleDescriptor("watcher_admin", new String[] { "manage_watcher" },
                         new RoleDescriptor.IndicesPrivileges[] {
                                 RoleDescriptor.IndicesPrivileges.builder().indices(Watch.INDEX, TriggeredWatchStoreField.INDEX_NAME,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java
deleted file mode 100644
index 4619035d0daaf..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java
+++ /dev/null
@@ -1,392 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.security.client;
-
-import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.xpack.core.security.action.CreateApiKeyAction;
-import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
-import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
-import org.elasticsearch.xpack.core.security.action.GetApiKeyAction;
-import org.elasticsearch.xpack.core.security.action.GetApiKeyRequest;
-import org.elasticsearch.xpack.core.security.action.GetApiKeyResponse;
-import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyAction;
-import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyRequest;
-import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyResponse;
-import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction;
-import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesAction;
-import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesAction;
-import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheAction;
-import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequest;
-import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheResponse;
-import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheAction;
-import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheRequest;
-import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheResponse;
-import org.elasticsearch.xpack.core.security.action.role.DeleteRoleAction;
-import org.elasticsearch.xpack.core.security.action.role.DeleteRoleRequest;
-import org.elasticsearch.xpack.core.security.action.role.DeleteRoleRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.role.DeleteRoleResponse;
-import org.elasticsearch.xpack.core.security.action.role.GetRolesAction;
-import org.elasticsearch.xpack.core.security.action.role.GetRolesRequest;
-import org.elasticsearch.xpack.core.security.action.role.GetRolesRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.role.GetRolesResponse;
-import org.elasticsearch.xpack.core.security.action.role.PutRoleAction;
-import org.elasticsearch.xpack.core.security.action.role.PutRoleRequest;
-import org.elasticsearch.xpack.core.security.action.role.PutRoleRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.role.PutRoleResponse;
-import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction;
-import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction;
-import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequest;
-import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsResponse;
-import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingAction;
-import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.saml.SamlAuthenticateAction;
-import org.elasticsearch.xpack.core.security.action.saml.SamlAuthenticateRequest;
-import org.elasticsearch.xpack.core.security.action.saml.SamlAuthenticateRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.saml.SamlAuthenticateResponse;
-import org.elasticsearch.xpack.core.security.action.saml.SamlPrepareAuthenticationRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.token.CreateTokenAction;
-import org.elasticsearch.xpack.core.security.action.token.CreateTokenRequest;
-import org.elasticsearch.xpack.core.security.action.token.CreateTokenRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.token.CreateTokenResponse;
-import org.elasticsearch.xpack.core.security.action.token.InvalidateTokenAction;
-import org.elasticsearch.xpack.core.security.action.token.InvalidateTokenRequest;
-import org.elasticsearch.xpack.core.security.action.token.InvalidateTokenRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.token.InvalidateTokenResponse;
-import org.elasticsearch.xpack.core.security.action.token.RefreshTokenAction;
-import org.elasticsearch.xpack.core.security.action.user.ChangePasswordAction;
-import org.elasticsearch.xpack.core.security.action.user.ChangePasswordRequest;
-import org.elasticsearch.xpack.core.security.action.user.ChangePasswordRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.user.ChangePasswordResponse;
-import org.elasticsearch.xpack.core.security.action.user.DeleteUserAction;
-import org.elasticsearch.xpack.core.security.action.user.DeleteUserRequest;
-import org.elasticsearch.xpack.core.security.action.user.DeleteUserRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.user.DeleteUserResponse;
-import org.elasticsearch.xpack.core.security.action.user.GetUsersAction;
-import org.elasticsearch.xpack.core.security.action.user.GetUsersRequest;
-import org.elasticsearch.xpack.core.security.action.user.GetUsersRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.user.GetUsersResponse;
-import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction;
-import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
-import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
-import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesAction;
-import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesRequest;
-import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse;
-import org.elasticsearch.xpack.core.security.action.user.PutUserAction;
-import org.elasticsearch.xpack.core.security.action.user.PutUserRequest;
-import org.elasticsearch.xpack.core.security.action.user.PutUserRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.user.PutUserResponse;
-import org.elasticsearch.xpack.core.security.action.user.SetEnabledAction;
-import org.elasticsearch.xpack.core.security.action.user.SetEnabledRequest;
-import org.elasticsearch.xpack.core.security.action.user.SetEnabledRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.user.SetEnabledResponse;
-import org.elasticsearch.xpack.core.security.authc.support.Hasher;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * A wrapper to elasticsearch clients that exposes all Security related APIs
- */
-public class SecurityClient {
-
-    private final ElasticsearchClient client;
-
-    public SecurityClient(ElasticsearchClient client) {
-        this.client = client;
-    }
-
-    /****************
-     * authc things *
-     ****************/
-
-    /**
-     * Clears the realm caches. It's possible to clear all user entries from all realms in the cluster or alternatively
-     * select the realms (by their unique names) and/or users (by their usernames) that should be evicted.
-     */
-    public ClearRealmCacheRequestBuilder prepareClearRealmCache() {
-        return new ClearRealmCacheRequestBuilder(client);
-    }
-
-    /**
-     * Clears the realm caches. It's possible to clear all user entries from all realms in the cluster or alternatively
-     * select the realms (by their unique names) and/or users (by their usernames) that should be evicted.
-     */
-    public void clearRealmCache(ClearRealmCacheRequest request, ActionListener<ClearRealmCacheResponse> listener) {
-        client.execute(ClearRealmCacheAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Clears the realm caches. It's possible to clear all user entries from all realms in the cluster or alternatively
-     * select the realms (by their unique names) and/or users (by their usernames) that should be evicted.
-     */
-    public ActionFuture<ClearRealmCacheResponse> clearRealmCache(ClearRealmCacheRequest request) {
-        return client.execute(ClearRealmCacheAction.INSTANCE, request);
-    }
-
-    /****************
-     * authz things *
-     ****************/
-
-    /**
-     * Clears the roles cache. This API only works for the naitve roles that are stored in an elasticsearch index. It is
-     * possible to clear the cache of all roles or to specify the names of individual roles that should have their cache
-     * cleared.
-     */
-    public ClearRolesCacheRequestBuilder prepareClearRolesCache() {
-        return new ClearRolesCacheRequestBuilder(client);
-    }
-
-    /**
-     * Clears the roles cache. This API only works for the naitve roles that are stored in an elasticsearch index. It is
-     * possible to clear the cache of all roles or to specify the names of individual roles that should have their cache
-     * cleared.
-     */
-    public void clearRolesCache(ClearRolesCacheRequest request, ActionListener<ClearRolesCacheResponse> listener) {
-        client.execute(ClearRolesCacheAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Clears the roles cache. This API only works for the naitve roles that are stored in an elasticsearch index. It is
-     * possible to clear the cache of all roles or to specify the names of individual roles that should have their cache
-     * cleared.
-     */
-    public ActionFuture<ClearRolesCacheResponse> clearRolesCache(ClearRolesCacheRequest request) {
-        return client.execute(ClearRolesCacheAction.INSTANCE, request);
-    }
-
-    /**
-     * Permissions / Privileges
-     */
-    public HasPrivilegesRequestBuilder prepareHasPrivileges(String username) {
-        return new HasPrivilegesRequestBuilder(client).username(username);
-    }
-
-    public HasPrivilegesRequestBuilder prepareHasPrivileges(String username, BytesReference source, XContentType xContentType)
-            throws IOException {
-        return new HasPrivilegesRequestBuilder(client).source(username, source, xContentType);
-    }
-
-    public void hasPrivileges(HasPrivilegesRequest request, ActionListener<HasPrivilegesResponse> listener) {
-        client.execute(HasPrivilegesAction.INSTANCE, request, listener);
-    }
-
-    public GetUserPrivilegesRequestBuilder prepareGetUserPrivileges(String username) {
-        return new GetUserPrivilegesRequestBuilder(client).username(username);
-    }
-
-    public void listUserPrivileges(GetUserPrivilegesRequest request, ActionListener<GetUserPrivilegesResponse> listener) {
-        client.execute(GetUserPrivilegesAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * User Management
-     */
-
-    public GetUsersRequestBuilder prepareGetUsers(String... usernames) {
-        return new GetUsersRequestBuilder(client).usernames(usernames);
-    }
-
-    public void getUsers(GetUsersRequest request, ActionListener<GetUsersResponse> listener) {
-        client.execute(GetUsersAction.INSTANCE, request, listener);
-    }
-
-    public DeleteUserRequestBuilder prepareDeleteUser(String username) {
-        return new DeleteUserRequestBuilder(client).username(username);
-    }
-
-    public void deleteUser(DeleteUserRequest request, ActionListener<DeleteUserResponse> listener) {
-        client.execute(DeleteUserAction.INSTANCE, request, listener);
-    }
-
-    public PutUserRequestBuilder preparePutUser(String username, BytesReference source, XContentType xContentType, Hasher hasher)
-        throws IOException {
-        return new PutUserRequestBuilder(client).source(username, source, xContentType, hasher);
-    }
-
-    public PutUserRequestBuilder preparePutUser(String username, char[] password, Hasher hasher, String... roles) {
-        return new PutUserRequestBuilder(client).username(username).password(password, hasher).roles(roles);
-    }
-
-    public void putUser(PutUserRequest request, ActionListener<PutUserResponse> listener) {
-        client.execute(PutUserAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Populates the {@link ChangePasswordRequest} with the username and password. Note: the passed in char[] will be cleared by this
-     * method.
-     */
-    public ChangePasswordRequestBuilder prepareChangePassword(String username, char[] password, Hasher hasher) {
-        return new ChangePasswordRequestBuilder(client).username(username).password(password, hasher);
-    }
-
-    public ChangePasswordRequestBuilder prepareChangePassword(String username, BytesReference source, XContentType xContentType,
-                                                              Hasher hasher) throws IOException {
-        return new ChangePasswordRequestBuilder(client).username(username).source(source, xContentType, hasher);
-    }
-
-    public void changePassword(ChangePasswordRequest request, ActionListener<ChangePasswordResponse> listener) {
-        client.execute(ChangePasswordAction.INSTANCE, request, listener);
-    }
-
-    public SetEnabledRequestBuilder prepareSetEnabled(String username, boolean enabled) {
-        return new SetEnabledRequestBuilder(client).username(username).enabled(enabled);
-    }
-
-    public void setEnabled(SetEnabledRequest request, ActionListener<SetEnabledResponse> listener) {
-        client.execute(SetEnabledAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Role Management
-     */
-
-    public GetRolesRequestBuilder prepareGetRoles(String... names) {
-        return new GetRolesRequestBuilder(client).names(names);
-    }
-
-    public void getRoles(GetRolesRequest request, ActionListener<GetRolesResponse> listener) {
-        client.execute(GetRolesAction.INSTANCE, request, listener);
-    }
-
-    public DeleteRoleRequestBuilder prepareDeleteRole(String name) {
-        return new DeleteRoleRequestBuilder(client).name(name);
-    }
-
-    public void deleteRole(DeleteRoleRequest request, ActionListener<DeleteRoleResponse> listener) {
-        client.execute(DeleteRoleAction.INSTANCE, request, listener);
-    }
-
-    public PutRoleRequestBuilder preparePutRole(String name) {
-        return new PutRoleRequestBuilder(client).name(name);
-    }
-
-    public PutRoleRequestBuilder preparePutRole(String name, BytesReference source, XContentType xContentType) throws IOException {
-        return new PutRoleRequestBuilder(client).source(name, source, xContentType);
-    }
-
-    public void putRole(PutRoleRequest request, ActionListener<PutRoleResponse> listener) {
-        client.execute(PutRoleAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Role Mappings
-     */
-
-    public GetRoleMappingsRequestBuilder prepareGetRoleMappings(String... names) {
-        return new GetRoleMappingsRequestBuilder(client, GetRoleMappingsAction.INSTANCE)
-                .names(names);
-    }
-
-    public void getRoleMappings(GetRoleMappingsRequest request,
-                                ActionListener<GetRoleMappingsResponse> listener) {
-        client.execute(GetRoleMappingsAction.INSTANCE, request, listener);
-    }
-
-    public PutRoleMappingRequestBuilder preparePutRoleMapping(
-            String name, BytesReference content, XContentType xContentType) throws IOException {
-        return new PutRoleMappingRequestBuilder(client, PutRoleMappingAction.INSTANCE).source(name, content, xContentType);
-    }
-
-    public DeleteRoleMappingRequestBuilder prepareDeleteRoleMapping(String name) {
-        return new DeleteRoleMappingRequestBuilder(client, DeleteRoleMappingAction.INSTANCE)
-                .name(name);
-    }
-
-    /* -- Application Privileges -- */
-    public GetPrivilegesRequestBuilder prepareGetPrivileges(String applicationName, String[] privileges) {
-        return new GetPrivilegesRequestBuilder(client, GetPrivilegesAction.INSTANCE).application(applicationName).privileges(privileges);
-    }
-
-    public PutPrivilegesRequestBuilder preparePutPrivileges(BytesReference bytesReference, XContentType xContentType) throws IOException {
-        return new PutPrivilegesRequestBuilder(client, PutPrivilegesAction.INSTANCE).source(bytesReference, xContentType);
-    }
-
-    public DeletePrivilegesRequestBuilder prepareDeletePrivileges(String applicationName, String[] privileges) {
-        return new DeletePrivilegesRequestBuilder(client, DeletePrivilegesAction.INSTANCE)
-            .application(applicationName)
-            .privileges(privileges);
-    }
-
-    public CreateTokenRequestBuilder prepareCreateToken() {
-        return new CreateTokenRequestBuilder(client, CreateTokenAction.INSTANCE);
-    }
-
-    public void createToken(CreateTokenRequest request, ActionListener<CreateTokenResponse> listener) {
-        client.execute(CreateTokenAction.INSTANCE, request, listener);
-    }
-
-    public InvalidateTokenRequestBuilder prepareInvalidateToken(String token) {
-        return new InvalidateTokenRequestBuilder(client).setTokenString(token);
-    }
-
-    public InvalidateTokenRequestBuilder prepareInvalidateToken() {
-        return new InvalidateTokenRequestBuilder(client);
-    }
-
-    public void invalidateToken(InvalidateTokenRequest request, ActionListener<InvalidateTokenResponse> listener) {
-        client.execute(InvalidateTokenAction.INSTANCE, request, listener);
-    }
-
-    /* -- Api Keys -- */
-    public CreateApiKeyRequestBuilder prepareCreateApiKey() {
-        return new CreateApiKeyRequestBuilder(client);
-    }
-
-    public CreateApiKeyRequestBuilder prepareCreateApiKey(BytesReference bytesReference, XContentType xContentType) throws IOException {
-        return new CreateApiKeyRequestBuilder(client).source(bytesReference, xContentType);
-    }
-
-    public void createApiKey(CreateApiKeyRequest request, ActionListener<CreateApiKeyResponse> listener) {
-        client.execute(CreateApiKeyAction.INSTANCE, request, listener);
-    }
-
-    public void invalidateApiKey(InvalidateApiKeyRequest request, ActionListener<InvalidateApiKeyResponse> listener) {
-        client.execute(InvalidateApiKeyAction.INSTANCE, request, listener);
-    }
-
-    public void getApiKey(GetApiKeyRequest request, ActionListener<GetApiKeyResponse> listener) {
-        client.execute(GetApiKeyAction.INSTANCE, request, listener);
-    }
-
-    public SamlAuthenticateRequestBuilder prepareSamlAuthenticate(byte[] xmlContent, List<String> validIds) {
-        final SamlAuthenticateRequestBuilder builder = new SamlAuthenticateRequestBuilder(client);
-        builder.saml(xmlContent);
-        builder.validRequestIds(validIds);
-        return builder;
-    }
-
-    public void samlAuthenticate(SamlAuthenticateRequest request, ActionListener<SamlAuthenticateResponse> listener) {
-        client.execute(SamlAuthenticateAction.INSTANCE, request, listener);
-    }
-
-    public SamlPrepareAuthenticationRequestBuilder prepareSamlPrepareAuthentication() {
-        return new SamlPrepareAuthenticationRequestBuilder(client);
-    }
-
-    public CreateTokenRequestBuilder prepareRefreshToken(String refreshToken) {
-        return new CreateTokenRequestBuilder(client, RefreshTokenAction.INSTANCE)
-                .setRefreshToken(refreshToken)
-                .setGrantType("refresh_token");
-    }
-
-    public void refreshToken(CreateTokenRequest request, ActionListener<CreateTokenResponse> listener) {
-        client.execute(RefreshTokenAction.INSTANCE, request, listener);
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/CacheIteratorHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/CacheIteratorHelper.java
new file mode 100644
index 0000000000000..0dfdab26815a5
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/CacheIteratorHelper.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.support;
+
+import org.elasticsearch.common.cache.Cache;
+import org.elasticsearch.common.util.concurrent.ReleasableLock;
+
+import java.util.Iterator;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Predicate;
+
+/**
+ * A utility class to facilitate iterating over (and modifying) a {@link org.elasticsearch.common.cache.Cache}.
+ * The semantics of the cache are such that when iterating (with the potential to call {@link Iterator#remove()}), we must prevent any
+ * other modifications.
+ * This class provides the necessary methods to support this constraint in a clear manner.
+ */
+public class CacheIteratorHelper<K, V> {
+    private final Cache<K, V> cache;
+    private final ReleasableLock updateLock;
+    private final ReleasableLock iteratorLock;
+
+    public CacheIteratorHelper(Cache<K, V> cache) {
+        this.cache = cache;
+        final ReadWriteLock lock = new ReentrantReadWriteLock();
+        // the lock is used in an odd manner; when iterating over the cache we cannot have modifiers other than deletes using the
+        // iterator but when not iterating we can modify the cache without external locking. When making normal modifications to the cache
+        // the read lock is obtained so that we can allow concurrent modifications; however when we need to iterate over the keys or values
+        // of the cache the write lock must obtained to prevent any modifications.
+        updateLock = new ReleasableLock(lock.readLock());
+        iteratorLock = new ReleasableLock(lock.writeLock());
+    }
+
+    public ReleasableLock acquireUpdateLock() {
+        return updateLock.acquire();
+    }
+
+    private ReleasableLock acquireForIterator() {
+        return iteratorLock.acquire();
+    }
+
+    public void removeKeysIf(Predicate<K> removeIf) {
+        // the cache cannot be modified while doing this operation per the terms of the cache iterator
+        try (ReleasableLock ignored = this.acquireForIterator()) {
+            Iterator<K> iterator = cache.keys().iterator();
+            while (iterator.hasNext()) {
+                K key = iterator.next();
+                if (removeIf.test(key)) {
+                    iterator.remove();
+                }
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/MetadataUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/MetadataUtils.java
index a48f562c7af7b..72388a8fdbcb5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/MetadataUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/MetadataUtils.java
@@ -5,14 +5,13 @@
  */
 package org.elasticsearch.xpack.core.security.support;
 
-import java.util.Collections;
 import java.util.Map;
 
 public class MetadataUtils {
 
     public static final String RESERVED_PREFIX = "_";
     public static final String RESERVED_METADATA_KEY = RESERVED_PREFIX + "reserved";
-    public static final Map<String, Object> DEFAULT_RESERVED_METADATA = Collections.singletonMap(RESERVED_METADATA_KEY, true);
+    public static final Map<String, Object> DEFAULT_RESERVED_METADATA = Map.of(RESERVED_METADATA_KEY, true);
 
     private MetadataUtils() {
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Validation.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Validation.java
index 55caed434114a..5c2c03042912b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Validation.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Validation.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.security.support;
 
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.xpack.core.security.authc.esnative.ClientReservedRealm;
@@ -81,10 +82,10 @@ public static Error validateUsername(String username, boolean allowReserved, Set
             return null;
         }
 
-        public static Error validatePassword(char[] password) {
-            return password.length >= MIN_PASSWD_LENGTH ?
-                    null :
-                    new Error("passwords must be at least [" + MIN_PASSWD_LENGTH + "] characters long");
+        public static Error validatePassword(SecureString password) {
+            return password.length() >= MIN_PASSWD_LENGTH ?
+                null :
+                new Error("passwords must be at least [" + MIN_PASSWD_LENGTH + "] characters long");
         }
 
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/APMSystemUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/APMSystemUser.java
index a63c3b0dc8ca1..e4fa627fb7ab4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/APMSystemUser.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/APMSystemUser.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.core.security.user;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.xpack.core.security.support.MetadataUtils;
 
 /**
@@ -15,7 +14,6 @@ public class APMSystemUser extends User {
 
     public static final String NAME = UsernamesField.APM_NAME;
     public static final String ROLE_NAME = UsernamesField.APM_ROLE;
-    public static final Version DEFINED_SINCE = Version.V_6_5_0;
 
     public APMSystemUser(boolean enabled) {
         super(NAME, new String[]{ ROLE_NAME }, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, enabled);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/RemoteMonitoringUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/RemoteMonitoringUser.java
index ad51c575d72d9..edfa9a8f5b8ca 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/RemoteMonitoringUser.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/RemoteMonitoringUser.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.core.security.user;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.xpack.core.security.support.MetadataUtils;
 
 /**
@@ -17,8 +16,6 @@ public class RemoteMonitoringUser extends User {
     public static final String COLLECTION_ROLE_NAME = UsernamesField.REMOTE_MONITORING_COLLECTION_ROLE;
     public static final String INDEXING_ROLE_NAME = UsernamesField.REMOTE_MONITORING_INDEXING_ROLE;
 
-    public static final Version DEFINED_SINCE = Version.V_6_5_0;
-
     public RemoteMonitoringUser(boolean enabled) {
         super(NAME, new String[]{ COLLECTION_ROLE_NAME, INDEXING_ROLE_NAME }, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, enabled);
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/User.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/User.java
index e9bca56dee126..1123930bc7375 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/User.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/User.java
@@ -15,7 +15,6 @@
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.Map;
 
 /**
@@ -33,11 +32,11 @@ public class User implements ToXContentObject {
     @Nullable private final String email;
 
     public User(String username, String... roles) {
-        this(username, roles, null, null, null, true);
+        this(username, roles, null, null, Map.of(), true);
     }
 
     public User(String username, String[] roles, User authenticatedUser) {
-        this(username, roles, null, null, null, true, authenticatedUser);
+        this(username, roles, null, null, Map.of(), true, authenticatedUser);
     }
 
     public User(User user, User authenticatedUser) {
@@ -52,7 +51,7 @@ private User(String username, String[] roles, String fullName, String email, Map
                 User authenticatedUser) {
         this.username = username;
         this.roles = roles == null ? Strings.EMPTY_ARRAY : roles;
-        this.metadata = metadata != null ? Collections.unmodifiableMap(metadata) : Collections.emptyMap();
+        this.metadata = metadata == null ? Map.of() : metadata;
         this.fullName = fullName;
         this.email = email;
         this.enabled = enabled;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java
index 0a902aba22cb6..6526bd394c94c 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/CertParsingUtils.java
@@ -7,8 +7,6 @@
 package org.elasticsearch.xpack.core.ssl;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
@@ -21,7 +19,6 @@
 import javax.net.ssl.X509ExtendedTrustManager;
 import java.io.IOException;
 import java.io.InputStream;
-
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.Key;
@@ -51,16 +48,9 @@ public class CertParsingUtils {
     private CertParsingUtils() {
         throw new IllegalStateException("Utility class should not be instantiated");
     }
-    /**
-     * Resolves a path with or without an {@link Environment} as we may be running in a transport client where we do not have access to
-     * the environment
-     */
-    @SuppressForbidden(reason = "we don't have the environment to resolve files from when running in a transport client")
-    static Path resolvePath(String path, @Nullable Environment environment) {
-        if (environment != null) {
-            return environment.configFile().resolve(path);
-        }
-        return PathUtils.get(path).normalize();
+
+    static Path resolvePath(String path, Environment environment) {
+        return environment.configFile().resolve(path);
     }
 
     public static KeyStore readKeyStore(Path path, String type, char[] password)
@@ -82,7 +72,7 @@ public static KeyStore readKeyStore(Path path, String type, char[] password)
      */
     public static Certificate[] readCertificates(List<String> certPaths, @Nullable Environment environment)
             throws CertificateException, IOException {
-        final List<Path> resolvedPaths = certPaths.stream().map(p -> resolvePath(p, environment)).collect(Collectors.toList());
+        final List<Path> resolvedPaths = certPaths.stream().map(p -> environment.configFile().resolve(p)).collect(Collectors.toList());
         return readCertificates(resolvedPaths);
     }
 
@@ -264,9 +254,9 @@ static KeyStore trustStore(Certificate[] certificates)
      * @return a trust manager with the trust material from the store
      */
     public static X509ExtendedTrustManager trustManager(String trustStorePath, String trustStoreType, char[] trustStorePassword,
-                                                        String trustStoreAlgorithm, @Nullable Environment env)
+                                                        String trustStoreAlgorithm, Environment env)
             throws NoSuchAlgorithmException, KeyStoreException, IOException, CertificateException {
-        KeyStore trustStore = readKeyStore(resolvePath(trustStorePath, env), trustStoreType, trustStorePassword);
+        KeyStore trustStore = readKeyStore(env.configFile().resolve(trustStorePath), trustStoreType, trustStorePassword);
         return trustManager(trustStore, trustStoreAlgorithm);
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/TrustConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/TrustConfig.java
index f570e0b42ed44..3b9d9039a4ed3 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/TrustConfig.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/TrustConfig.java
@@ -38,15 +38,15 @@ abstract class TrustConfig {
 
     /**
      * Creates a {@link X509ExtendedTrustManager} based on the provided configuration
-     * @param environment the environment to resolve files against or null in the case of running in a transport client
+     * @param environment the environment to resolve files against
      */
-    abstract X509ExtendedTrustManager createTrustManager(@Nullable Environment environment);
+    abstract X509ExtendedTrustManager createTrustManager(Environment environment);
 
-    abstract Collection<CertificateInfo> certificates(@Nullable Environment environment) throws GeneralSecurityException, IOException;
+    abstract Collection<CertificateInfo> certificates(Environment environment) throws GeneralSecurityException, IOException;
 
     /**
      * Returns a list of files that should be monitored for changes
-     * @param environment the environment to resolve files against or null in the case of running in a transport client
+     * @param environment the environment to resolve files against
      */
     abstract List<Path> filesToMonitor(@Nullable Environment environment);
 
@@ -70,7 +70,7 @@ abstract class TrustConfig {
      * in any format that the Security Provider might support, or a cryptographic software or hardware token in the case
      * of a PKCS#11 Provider.
      *
-     * @param environment   the environment to resolve files against or null in the case of running in a transport client
+     * @param environment   the environment to resolve files against
      * @param storePath     the path to the {@link KeyStore} to load, or null if a PKCS11 token is configured as the keystore/truststore
      *                      of the JVM
      * @param storeType     the type of the {@link KeyStore}
@@ -81,7 +81,7 @@ abstract class TrustConfig {
      * @throws NoSuchAlgorithmException if the algorithm used to check the integrity of the keystore cannot be found
      * @throws IOException              if there is an I/O issue with the KeyStore data or the password is incorrect
      */
-    KeyStore getStore(@Nullable Environment environment, @Nullable String storePath, String storeType, SecureString storePassword)
+    KeyStore getStore(Environment environment, @Nullable String storePath, String storeType, SecureString storePassword)
         throws KeyStoreException, CertificateException, NoSuchAlgorithmException, IOException {
         if (null != storePath) {
             try (InputStream in = Files.newInputStream(CertParsingUtils.resolvePath(storePath, environment))) {
@@ -136,7 +136,7 @@ Collection<CertificateInfo> certificates(Environment environment) throws General
         }
 
         @Override
-        List<Path> filesToMonitor(@Nullable Environment environment) {
+        List<Path> filesToMonitor(Environment environment) {
             return trustConfigs.stream().flatMap((tc) -> tc.filesToMonitor(environment).stream()).collect(Collectors.toList());
         }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/action/GetCertificateInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/action/GetCertificateInfoAction.java
index 4e1a84773db7d..ee9a5aa1b77b5 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/action/GetCertificateInfoAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/action/GetCertificateInfoAction.java
@@ -5,11 +5,11 @@
  */
 package org.elasticsearch.xpack.core.ssl.action;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -25,7 +25,7 @@
  * Action to obtain information about X.509 (SSL/TLS) certificates that are being used by X-Pack.
  * The primary use case is for tracking the expiry dates of certificates.
  */
-public class GetCertificateInfoAction extends Action<GetCertificateInfoAction.Response> {
+public class GetCertificateInfoAction extends StreamableResponseActionType<GetCertificateInfoAction.Response> {
 
     public static final GetCertificateInfoAction INSTANCE = new GetCertificateInfoAction();
     public static final String NAME = "cluster:monitor/xpack/ssl/certificates/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java
index d0a086bd649f0..1d57df3b54199 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.core.XPackClient;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
@@ -222,7 +221,7 @@ public void onFailure(Exception e) {
                         creationCheck.set(false);
                         onPutPolicyFailure(policy, e);
                     }
-                }, (req, listener) -> new XPackClient(client).ilmClient().putLifecyclePolicy(req, listener));
+                }, (req, listener) -> client.execute(PutLifecycleAction.INSTANCE, req, listener));
         });
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java
index 893c91f056c57..6b25f7855f107 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java
@@ -16,17 +16,15 @@
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.compress.NotXContentException;
+import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.core.internal.io.Streams;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.Map;
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
@@ -73,12 +71,7 @@ public static String loadTemplate(String resource, String version, String versio
      * Loads a resource from the classpath and returns it as a {@link BytesReference}
      */
     public static BytesReference load(String name) throws IOException {
-        try (InputStream is = TemplateUtils.class.getResourceAsStream(name)) {
-            try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
-                Streams.copy(is, out);
-                return new BytesArray(out.toByteArray());
-            }
-        }
+        return Streams.readFully(TemplateUtils.class.getResourceAsStream(name));
     }
 
     /**
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/actions/IndexUpgradeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/actions/IndexUpgradeAction.java
index 89279f4ea31cf..2cc6c9431d608 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/actions/IndexUpgradeAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/actions/IndexUpgradeAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.core.upgrade.actions;
 
-import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
 import org.elasticsearch.action.support.master.MasterNodeReadRequest;
@@ -26,7 +26,7 @@
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.xpack.core.upgrade.IndexUpgradeServiceFields.UPGRADE_INDEX_OPTIONS;
 
-public class IndexUpgradeAction extends Action<BulkByScrollResponse> {
+public class IndexUpgradeAction extends StreamableResponseActionType<BulkByScrollResponse> {
 
     public static final IndexUpgradeAction INSTANCE = new IndexUpgradeAction();
     public static final String NAME = "cluster:admin/xpack/upgrade";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/actions/IndexUpgradeInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/actions/IndexUpgradeInfoAction.java
index 3044c953a3e09..523e594725b9d 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/actions/IndexUpgradeInfoAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/upgrade/actions/IndexUpgradeInfoAction.java
@@ -5,14 +5,14 @@
  */
 package org.elasticsearch.xpack.core.upgrade.actions;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoRequest;
 import org.elasticsearch.protocol.xpack.migration.IndexUpgradeInfoResponse;
 
-public class IndexUpgradeInfoAction extends Action<IndexUpgradeInfoResponse> {
+public class IndexUpgradeInfoAction extends StreamableResponseActionType<IndexUpgradeInfoResponse> {
 
     public static final IndexUpgradeInfoAction INSTANCE = new IndexUpgradeInfoAction();
     public static final String NAME = "cluster:admin/xpack/upgrade/info";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/vectors/VectorsFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/vectors/VectorsFeatureSetUsage.java
new file mode 100644
index 0000000000000..00e42d46fb551
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/vectors/VectorsFeatureSetUsage.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.vectors;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
+
+import java.io.IOException;
+
+public class VectorsFeatureSetUsage extends XPackFeatureSet.Usage {
+
+    public VectorsFeatureSetUsage(StreamInput input) throws IOException {
+        super(input);
+    }
+
+    public VectorsFeatureSetUsage(boolean available, boolean enabled) {
+        super(XPackField.VECTORS, available, enabled);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/votingonly/VotingOnlyNodeFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/votingonly/VotingOnlyNodeFeatureSetUsage.java
new file mode 100644
index 0000000000000..71e94e6d31860
--- /dev/null
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/votingonly/VotingOnlyNodeFeatureSetUsage.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.votingonly;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
+
+import java.io.IOException;
+
+public class VotingOnlyNodeFeatureSetUsage extends XPackFeatureSet.Usage {
+    public VotingOnlyNodeFeatureSetUsage(StreamInput input) throws IOException {
+        super(input);
+    }
+
+    public VotingOnlyNodeFeatureSetUsage(boolean available) {
+        super(XPackField.VOTING_ONLY, available, true);
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/actions/ActionWrapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/actions/ActionWrapper.java
index eec08453498ab..951f305edf5bc 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/actions/ActionWrapper.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/actions/ActionWrapper.java
@@ -7,13 +7,17 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.script.JodaCompatibleZonedDateTime;
 import org.elasticsearch.xpack.core.watcher.actions.throttler.ActionThrottler;
 import org.elasticsearch.xpack.core.watcher.actions.throttler.Throttler;
 import org.elasticsearch.xpack.core.watcher.actions.throttler.ThrottlerField;
@@ -30,12 +34,21 @@
 import java.time.Clock;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 
 public class ActionWrapper implements ToXContentObject {
 
+    private final int MAXIMUM_FOREACH_RUNS = 100;
+
     private String id;
     @Nullable
     private final ExecutableCondition condition;
@@ -43,16 +56,20 @@ public class ActionWrapper implements ToXContentObject {
     private final ExecutableTransform<Transform, Transform.Result> transform;
     private final ActionThrottler throttler;
     private final ExecutableAction<? extends Action> action;
+    @Nullable
+    private String path;
 
     public ActionWrapper(String id, ActionThrottler throttler,
                          @Nullable ExecutableCondition condition,
                          @Nullable ExecutableTransform<Transform, Transform.Result> transform,
-                         ExecutableAction<? extends Action> action) {
+                         ExecutableAction<? extends Action> action,
+                         @Nullable String path) {
         this.id = id;
         this.condition = condition;
         this.throttler = throttler;
         this.transform = transform;
         this.action = action;
+        this.path = path;
     }
 
     public String id() {
@@ -140,16 +157,90 @@ public ActionWrapperResult execute(WatchExecutionContext ctx) {
                 return new ActionWrapperResult(id, conditionResult, null, new Action.Result.FailureWithException(action.type(), e));
             }
         }
-        try {
-            Action.Result actionResult = action.execute(id, ctx, payload);
-            return new ActionWrapperResult(id, conditionResult, transformResult, actionResult);
-        } catch (Exception e) {
-            action.logger().error(
+        if (Strings.isEmpty(path)) {
+            try {
+                Action.Result actionResult = action.execute(id, ctx, payload);
+                return new ActionWrapperResult(id, conditionResult, transformResult, actionResult);
+            } catch (Exception e) {
+                action.logger().error(
+                        (Supplier<?>) () -> new ParameterizedMessage("failed to execute action [{}/{}]", ctx.watch().id(), id), e);
+                return new ActionWrapperResult(id, new Action.Result.FailureWithException(action.type(), e));
+            }
+        } else {
+            try {
+                List<Action.Result> results = new ArrayList<>();
+                Object object = ObjectPath.eval(path, toMap(ctx));
+                int runs = 0;
+                if (object instanceof Collection) {
+                    Collection collection = Collection.class.cast(object);
+                    if (collection.isEmpty()) {
+                        throw new ElasticsearchException("foreach object [{}] was an empty list, could not run any action", path);
+                    } else {
+                        for (Object o : collection) {
+                            if (runs >= MAXIMUM_FOREACH_RUNS) {
+                                break;
+                            }
+                            if (o instanceof Map) {
+                                results.add(action.execute(id, ctx, new Payload.Simple((Map<String, Object>) o)));
+                            } else {
+                                results.add(action.execute(id, ctx, new Payload.Simple("_value", o)));
+                            }
+                            runs++;
+                        }
+                    }
+                } else if (object == null) {
+                    throw new ElasticsearchException("specified foreach object was null: [{}]", path);
+                } else {
+                    throw new ElasticsearchException("specified foreach object was not a an array/collection: [{}]", path);
+                }
+
+                // check if we have mixed results, then set to partial failure
+                final Set<Action.Result.Status> statuses = results.stream().map(Action.Result::status).collect(Collectors.toSet());
+                Action.Result.Status status;
+                if (statuses.size() == 1) {
+                    status = statuses.iterator().next();
+                } else {
+                    status = Action.Result.Status.PARTIAL_FAILURE;
+                }
+
+                final int numberOfActionsExecuted = runs;
+                return new ActionWrapperResult(id, conditionResult, transformResult,
+                    new Action.Result(action.type(), status) {
+                        @Override
+                        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                            builder.field("number_of_actions_executed", numberOfActionsExecuted);
+                            builder.startArray(WatchField.FOREACH.getPreferredName());
+                            for (Action.Result result : results) {
+                                builder.startObject();
+                                result.toXContent(builder, params);
+                                builder.endObject();
+                            }
+                            builder.endArray();
+                            return builder;
+                        }
+                });
+            } catch (Exception e) {
+                action.logger().error(
                     (Supplier<?>) () -> new ParameterizedMessage("failed to execute action [{}/{}]", ctx.watch().id(), id), e);
-            return new ActionWrapperResult(id, new Action.Result.FailureWithException(action.type(), e));
+                return new ActionWrapperResult(id, new Action.Result.FailureWithException(action.type(), e));
+            }
         }
     }
 
+    private Map<String, Object> toMap(WatchExecutionContext ctx) {
+        Map<String, Object> model = new HashMap<>();
+        model.put("id", ctx.id().value());
+        model.put("watch_id", ctx.id().watchId());
+        model.put("execution_time", new JodaCompatibleZonedDateTime(ctx.executionTime().toInstant(), ZoneOffset.UTC));
+        model.put("trigger", ctx.triggerEvent().data());
+        model.put("metadata", ctx.watch().metadata());
+        model.put("vars", ctx.vars());
+        if (ctx.payload().data() != null) {
+            model.put("payload", ctx.payload().data());
+        }
+        return Map.of("ctx", model);
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -186,6 +277,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                     .field(transform.type(), transform, params)
                     .endObject();
         }
+        if (Strings.isEmpty(path) == false) {
+            builder.field(WatchField.FOREACH.getPreferredName(), path);
+        }
         builder.field(action.type(), action, params);
         return builder.endObject();
     }
@@ -198,6 +292,7 @@ static ActionWrapper parse(String watchId, String actionId, XContentParser parse
         ExecutableCondition condition = null;
         ExecutableTransform<Transform, Transform.Result> transform = null;
         TimeValue throttlePeriod = null;
+        String path = null;
         ExecutableAction<? extends Action> action = null;
 
         String currentFieldName = null;
@@ -208,6 +303,8 @@ static ActionWrapper parse(String watchId, String actionId, XContentParser parse
             } else {
                 if (WatchField.CONDITION.match(currentFieldName, parser.getDeprecationHandler())) {
                     condition = actionRegistry.getConditionRegistry().parseExecutable(watchId, parser);
+                } else if (WatchField.FOREACH.match(currentFieldName, parser.getDeprecationHandler())) {
+                    path = parser.text();
                 } else if (Transform.TRANSFORM.match(currentFieldName, parser.getDeprecationHandler())) {
                     transform = actionRegistry.getTransformRegistry().parse(watchId, parser);
                 } else if (ThrottlerField.THROTTLE_PERIOD.match(currentFieldName, parser.getDeprecationHandler())) {
@@ -235,7 +332,7 @@ static ActionWrapper parse(String watchId, String actionId, XContentParser parse
         }
 
         ActionThrottler throttler = new ActionThrottler(clock, throttlePeriod, licenseState);
-        return new ActionWrapper(actionId, throttler, condition, transform, action);
+        return new ActionWrapper(actionId, throttler, condition, transform, action, path);
     }
 
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatchSourceBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatchSourceBuilder.java
index 001a430ddb1e6..12eb6a12ee2ad 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatchSourceBuilder.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatchSourceBuilder.java
@@ -101,7 +101,7 @@ public WatchSourceBuilder addAction(String id, TimeValue throttlePeriod, Transfo
     }
 
     public WatchSourceBuilder addAction(String id, TimeValue throttlePeriod, Transform transform, Action action) {
-        actions.put(id, new TransformedAction(id, action, throttlePeriod, null, transform));
+        actions.put(id, new TransformedAction(id, action, throttlePeriod, null, transform, null));
         return this;
     }
 
@@ -111,7 +111,13 @@ public WatchSourceBuilder addAction(String id, TimeValue throttlePeriod, Conditi
     }
 
     public WatchSourceBuilder addAction(String id, TimeValue throttlePeriod, Condition condition, Transform transform, Action action) {
-        actions.put(id, new TransformedAction(id, action, throttlePeriod, condition, transform));
+        actions.put(id, new TransformedAction(id, action, throttlePeriod, condition, transform, null));
+        return this;
+    }
+
+    public WatchSourceBuilder addAction(String id, TimeValue throttlePeriod, Condition condition, Transform transform, String path,
+                                        Action action) {
+        actions.put(id, new TransformedAction(id, action, throttlePeriod, condition, transform, path));
         return this;
     }
 
@@ -186,16 +192,18 @@ public final BytesReference buildAsBytes(XContentType contentType) {
     static class TransformedAction implements ToXContentObject {
 
         private final Action action;
+        @Nullable private String path;
         @Nullable private final TimeValue throttlePeriod;
         @Nullable private final Condition condition;
         @Nullable private final Transform transform;
 
         TransformedAction(String id, Action action, @Nullable TimeValue throttlePeriod,
-                          @Nullable Condition condition, @Nullable Transform transform) {
+                          @Nullable Condition condition, @Nullable Transform transform, @Nullable String path) {
             this.throttlePeriod = throttlePeriod;
             this.condition = condition;
             this.transform = transform;
             this.action = action;
+            this.path = path;
         }
 
         @Override
@@ -215,6 +223,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                         .field(transform.type(), transform, params)
                         .endObject();
             }
+            if (path != null) {
+                builder.field("foreach", path);
+            }
             builder.field(action.type(), action, params);
             return builder.endObject();
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatcherClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatcherClient.java
deleted file mode 100644
index 39dea2298aa1b..0000000000000
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/client/WatcherClient.java
+++ /dev/null
@@ -1,326 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.watcher.client;
-
-import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
-import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
-import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
-import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
-import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchAction;
-import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchRequestBuilder;
-import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchResponse;
-import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchAction;
-import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchRequestBuilder;
-import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchResponse;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchAction;
-import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequestBuilder;
-import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchAction;
-import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
-import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
-import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchAction;
-import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequestBuilder;
-import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction;
-import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
-import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceAction;
-import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceRequestBuilder;
-import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsAction;
-import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsRequestBuilder;
-import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsResponse;
-
-import java.util.Map;
-
-public class WatcherClient {
-
-    private final Client client;
-
-    public WatcherClient(Client client) {
-        this.client = client;
-    }
-
-    /**
-     * Creates a request builder that gets an watch by id
-     *
-     * @param id the id of the watch
-     * @return The request builder
-     */
-    public GetWatchRequestBuilder prepareGetWatch(String id) {
-        return new GetWatchRequestBuilder(client, id);
-    }
-
-    /**
-     * Creates a request builder that gets an watch
-     *
-     * @return the request builder
-     */
-    public GetWatchRequestBuilder prepareGetWatch() {
-        return new GetWatchRequestBuilder(client);
-    }
-
-    /**
-     * Gets an watch from the watch index
-     *
-     * @param request The get watch request
-     * @param listener The listener for the get watch response containing the GetResponse for this watch
-     */
-    public void getWatch(GetWatchRequest request, ActionListener<GetWatchResponse> listener) {
-        client.execute(GetWatchAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Gets an watch from the watch index
-     *
-     * @param request The get watch request with the watch id
-     * @return The response containing the GetResponse for this watch
-     */
-    public ActionFuture<GetWatchResponse> getWatch(GetWatchRequest request) {
-        return client.execute(GetWatchAction.INSTANCE, request);
-    }
-
-    /**
-     * Creates a request builder to delete an watch by id
-     *
-     * @param id the id of the watch
-     * @return The request builder
-     */
-    public DeleteWatchRequestBuilder prepareDeleteWatch(String id) {
-        return new DeleteWatchRequestBuilder(client, id);
-    }
-
-    /**
-     * Creates a request builder that deletes an watch
-     *
-     * @return The request builder
-     */
-    public DeleteWatchRequestBuilder prepareDeleteWatch() {
-        return new DeleteWatchRequestBuilder(client);
-    }
-
-    /**
-     * Deletes an watch
-     *
-     * @param request The delete request with the watch id to be deleted
-     * @param listener The listener for the delete watch response containing the DeleteResponse for this action
-     */
-    public void deleteWatch(DeleteWatchRequest request, ActionListener<DeleteWatchResponse> listener) {
-        client.execute(DeleteWatchAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Deletes an watch
-     *
-     * @param request The delete request with the watch id to be deleted
-     * @return The response containing the DeleteResponse for this action
-     */
-    public ActionFuture<DeleteWatchResponse> deleteWatch(DeleteWatchRequest request) {
-        return client.execute(DeleteWatchAction.INSTANCE, request);
-    }
-
-    /**
-     * Creates a request builder to build a request to put an watch
-     *
-     * @param id The id of the watch to put
-     * @return The builder to create the watch
-     */
-    public PutWatchRequestBuilder preparePutWatch(String id) {
-        return new PutWatchRequestBuilder(client, id);
-    }
-
-    /**
-     * Creates a request builder to build a request to put a watch
-     *
-     * @return The builder
-     */
-    public PutWatchRequestBuilder preparePutWatch() {
-        return new PutWatchRequestBuilder(client);
-    }
-
-    /**
-     * Adds the given watch to the watcher
-     *
-     * @param request The request containing the watch to be added
-     * @param listener The listener for the response containing the IndexResponse for this watch
-     */
-    public void putWatch(PutWatchRequest request, ActionListener<PutWatchResponse> listener) {
-        client.execute(PutWatchAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Adds a new watch
-     *
-     * @param request The request containing the watch to be added
-     * @return The response containing the IndexResponse for this watch
-     */
-    public ActionFuture<PutWatchResponse> putWatch(PutWatchRequest request) {
-        return client.execute(PutWatchAction.INSTANCE, request);
-    }
-
-    /**
-     * Gets the watcher stats
-     *
-     * @param request The request for the watcher stats
-     * @return The response containing the StatsResponse for this action
-     */
-    public ActionFuture<WatcherStatsResponse> watcherStats(WatcherStatsRequest request) {
-        return client.execute(WatcherStatsAction.INSTANCE, request);
-    }
-
-    /**
-     * Creates a request builder to build a request to get the watcher stats
-     *
-     * @return The builder get the watcher stats
-     */
-    public WatcherStatsRequestBuilder prepareWatcherStats() {
-        return new WatcherStatsRequestBuilder(client);
-    }
-
-    /**
-     * Gets the watcher stats
-     *
-     * @param request The request for the watcher stats
-     * @param listener The listener for the response containing the WatcherStatsResponse
-     */
-    public void watcherStats(WatcherStatsRequest request, ActionListener<WatcherStatsResponse> listener) {
-        client.execute(WatcherStatsAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Creates a request builder to ack a watch by id
-     *
-     * @param id the id of the watch
-     * @return The request builder
-     */
-    public AckWatchRequestBuilder prepareAckWatch(String id) {
-        return new AckWatchRequestBuilder(client, id);
-    }
-
-    /**
-     * Ack a watch
-     *
-     * @param request The ack request with the watch id to be acked
-     * @param listener The listener for the ack watch response
-     */
-    public void ackWatch(AckWatchRequest request, ActionListener<AckWatchResponse> listener) {
-        client.execute(AckWatchAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Acks a watch
-     *
-     * @param request The ack request with the watch id to be acked
-     * @return The AckWatchResponse
-     */
-    public ActionFuture<AckWatchResponse> ackWatch(AckWatchRequest request) {
-        return client.execute(AckWatchAction.INSTANCE, request);
-    }
-
-    /**
-     * Creates a request builder to activate a watch by id
-     *
-     * @param id the id of the watch
-     * @param activate indicates whether to activate or deactivate the watch
-     * @return The request builder
-     */
-    public ActivateWatchRequestBuilder prepareActivateWatch(String id, boolean activate) {
-        return new ActivateWatchRequestBuilder(client, id, activate);
-    }
-
-    /**
-     * Activate a watch
-     *
-     * @param request The activate request with the watch id
-     * @param listener The listener for the activate watch response
-     */
-    public void activateWatch(ActivateWatchRequest request, ActionListener<ActivateWatchResponse> listener) {
-        client.execute(ActivateWatchAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Activates a watch
-     *
-     * @param request The de/activate request with the watch id.
-     */
-    public ActionFuture<ActivateWatchResponse> activateWatch(ActivateWatchRequest request) {
-        return client.execute(ActivateWatchAction.INSTANCE, request);
-    }
-
-
-    /**
-     * Prepare a watch service request.
-     */
-    public WatcherServiceRequestBuilder prepareWatchService() {
-        return new WatcherServiceRequestBuilder(client);
-    }
-
-    /**
-     * Perform an watcher service request to either start, stop or restart the service.
-     */
-    public void watcherService(WatcherServiceRequest request, ActionListener<AcknowledgedResponse> listener) {
-        client.execute(WatcherServiceAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Perform an watcher service request to either start, stop or restart the service.
-     */
-    public ActionFuture<AcknowledgedResponse> watcherService(WatcherServiceRequest request) {
-        return client.execute(WatcherServiceAction.INSTANCE, request);
-    }
-
-
-
-    /**
-     * Creates a request builder to execute a watch by id
-     *
-     * @param id the id of the watch
-     * @return The request builder
-     */
-    public ExecuteWatchRequestBuilder prepareExecuteWatch(String id) {
-        return new ExecuteWatchRequestBuilder(client, id);
-    }
-
-    /**
-     * Creates a request builder that executes a watch
-     *
-     * @return The request builder
-     */
-    public ExecuteWatchRequestBuilder prepareExecuteWatch() {
-        return new ExecuteWatchRequestBuilder(client);
-    }
-
-    /**
-     * executes a watch
-     *
-     * @param request The run request with the watch id to be executed
-     * @param listener The listener for the run watch response
-     */
-    public void executeWatch(ExecuteWatchRequest request, ActionListener<ExecuteWatchResponse> listener) {
-        client.execute(ExecuteWatchAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Executes an watch
-     *
-     * @param request The execute request with the watch id to be executed
-     * @return The AckWatchResponse
-     */
-    public ActionFuture<ExecuteWatchResponse> executeWatch(ExecuteWatchRequest request) {
-        return client.execute(ExecuteWatchAction.INSTANCE, request);
-    }
-
-    public WatcherClient filterWithHeader(Map<String, String> headers) {
-        return new WatcherClient(client.filterWithHeader(headers));
-    }
-}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java
index 4007b06ee7eca..821ea9a433524 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java
@@ -14,8 +14,9 @@ public final class WatcherIndexTemplateRegistryField {
     // version 7: add full exception stack traces for better debugging
     // version 8: fix slack attachment property not to be dynamic, causing field type issues
     // version 9: add a user field defining which user executed the watch
+    // version 10: add support for foreach path in actions
     // Note: if you change this, also inform the kibana team around the watcher-ui
-    public static final String INDEX_TEMPLATE_VERSION = "9";
+    public static final String INDEX_TEMPLATE_VERSION = "10";
     public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION;
     public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION;
     public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java
index 1d155a5f0c02d..20b0086c1e4e2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java
@@ -120,11 +120,6 @@ public Map<String, String> mapStrings() throws IOException {
         return parser.mapStrings();
     }
 
-    @Override
-    public Map<String, String> mapStringsOrdered() throws IOException {
-        return parser.mapStringsOrdered();
-    }
-
     @Override
     public <T> Map<String, T> map(
             Supplier<Map<String, T>> mapFactory, CheckedFunction<XContentParser, T, IOException> mapValueParser) throws IOException {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/ack/AckWatchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/ack/AckWatchAction.java
index 04ec95a369af2..a3c9bbb7351b0 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/ack/AckWatchAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/ack/AckWatchAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.watcher.transport.actions.ack;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
  * This action acks a watch in memory, and the index
  */
-public class AckWatchAction extends Action<AckWatchResponse> {
+public class AckWatchAction extends StreamableResponseActionType<AckWatchResponse> {
 
     public static final AckWatchAction INSTANCE = new AckWatchAction();
     public static final String NAME = "cluster:admin/xpack/watcher/watch/ack";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/activate/ActivateWatchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/activate/ActivateWatchAction.java
index 936a21711547e..4b57b95113cca 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/activate/ActivateWatchAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/activate/ActivateWatchAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.watcher.transport.actions.activate;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
  * This action acks a watch in memory, and the index
  */
-public class ActivateWatchAction extends Action<ActivateWatchResponse> {
+public class ActivateWatchAction extends StreamableResponseActionType<ActivateWatchResponse> {
 
     public static final ActivateWatchAction INSTANCE = new ActivateWatchAction();
     public static final String NAME = "cluster:admin/xpack/watcher/watch/activate";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchAction.java
index eb440ddc2510e..00619966fc8f2 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/delete/DeleteWatchAction.java
@@ -5,13 +5,13 @@
  */
 package org.elasticsearch.xpack.core.watcher.transport.actions.delete;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 
 /**
  * This action deletes an watch from in memory, the scheduler and the index
  */
-public class DeleteWatchAction extends Action<DeleteWatchResponse> {
+public class DeleteWatchAction extends StreamableResponseActionType<DeleteWatchResponse> {
 
     public static final DeleteWatchAction INSTANCE = new DeleteWatchAction();
     public static final String NAME = "cluster:admin/xpack/watcher/watch/delete";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchAction.java
index 924f170959426..2c5699b1132b1 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchAction.java
@@ -5,13 +5,13 @@
  */
 package org.elasticsearch.xpack.core.watcher.transport.actions.execute;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
  * This action executes a watch, either ignoring the schedule and condition or just the schedule and can execute a subset of the actions,
  * optionally persisting the history entry
  */
-public class ExecuteWatchAction extends Action<ExecuteWatchResponse> {
+public class ExecuteWatchAction extends StreamableResponseActionType<ExecuteWatchResponse> {
 
     public static final ExecuteWatchAction INSTANCE = new ExecuteWatchAction();
     public static final String NAME = "cluster:admin/xpack/watcher/watch/execute";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchAction.java
index 4df72a964b65f..bef78b8e0e96b 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/get/GetWatchAction.java
@@ -5,10 +5,12 @@
  */
 package org.elasticsearch.xpack.core.watcher.transport.actions.get;
 
+import org.elasticsearch.action.StreamableResponseActionType;
+
 /**
  * This action gets an watch by name
  */
-public class GetWatchAction extends org.elasticsearch.action.Action<GetWatchResponse> {
+public class GetWatchAction extends StreamableResponseActionType<GetWatchResponse> {
 
     public static final GetWatchAction INSTANCE = new GetWatchAction();
     public static final String NAME = "cluster:monitor/xpack/watcher/watch/get";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchAction.java
index 509116b018e2b..379c8ff672888 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/put/PutWatchAction.java
@@ -5,13 +5,13 @@
  */
 package org.elasticsearch.xpack.core.watcher.transport.actions.put;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 
 /**
  * This action puts an watch into the watch index and adds it to the scheduler
  */
-public class PutWatchAction extends Action<PutWatchResponse> {
+public class PutWatchAction extends StreamableResponseActionType<PutWatchResponse> {
 
     public static final PutWatchAction INSTANCE = new PutWatchAction();
     public static final String NAME = "cluster:admin/xpack/watcher/watch/put";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/service/WatcherServiceAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/service/WatcherServiceAction.java
index 30b0b54725b5f..80276946fd698 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/service/WatcherServiceAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/service/WatcherServiceAction.java
@@ -5,11 +5,12 @@
  */
 package org.elasticsearch.xpack.core.watcher.transport.actions.service;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.io.stream.Writeable;
 
 
-public class WatcherServiceAction extends Action<AcknowledgedResponse> {
+public class WatcherServiceAction extends ActionType<AcknowledgedResponse> {
 
     public static final WatcherServiceAction INSTANCE = new WatcherServiceAction();
     public static final String NAME = "cluster:admin/xpack/watcher/service";
@@ -19,7 +20,7 @@ private WatcherServiceAction() {
     }
 
     @Override
-    public AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    public Writeable.Reader<AcknowledgedResponse> getResponseReader() {
+        return AcknowledgedResponse::new;
     }
 }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/stats/WatcherStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/stats/WatcherStatsAction.java
index 59fcff090f59e..fbc55e0836c3e 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/stats/WatcherStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/stats/WatcherStatsAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.core.watcher.transport.actions.stats;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
- * This Action gets the stats for the watcher plugin
+ * This ActionType gets the stats for the watcher plugin
  */
-public class WatcherStatsAction extends Action<WatcherStatsResponse> {
+public class WatcherStatsAction extends StreamableResponseActionType<WatcherStatsResponse> {
 
     public static final WatcherStatsAction INSTANCE = new WatcherStatsAction();
     public static final String NAME = "cluster:monitor/xpack/watcher/stats/dist";
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/stats/WatcherStatsRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/stats/WatcherStatsRequest.java
index 4fdbd2ac407ce..38ebe8f9a742a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/stats/WatcherStatsRequest.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/stats/WatcherStatsRequest.java
@@ -83,8 +83,7 @@ public static class Node extends BaseNodeRequest {
 
         public Node() {}
 
-        public Node(WatcherStatsRequest request, String nodeId) {
-            super(nodeId);
+        public Node(WatcherStatsRequest request) {
             includeCurrentWatches = request.includeCurrentWatches();
             includeQueuedWatches = request.includeQueuedWatches();
             includeStats = request.includeStats();
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/WatchField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/WatchField.java
index 6f6a1955927d9..1bcb62447bf76 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/WatchField.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/WatchField.java
@@ -13,6 +13,7 @@ public final class WatchField {
     public static final ParseField CONDITION = new ParseField("condition");
     public static final ParseField ACTIONS = new ParseField("actions");
     public static final ParseField TRANSFORM = new ParseField("transform");
+    public static final ParseField FOREACH = new ParseField("foreach");
     public static final ParseField THROTTLE_PERIOD = new ParseField("throttle_period_in_millis");
     public static final ParseField THROTTLE_PERIOD_HUMAN = new ParseField("throttle_period");
     public static final ParseField METADATA = new ParseField("metadata");
diff --git a/x-pack/plugin/core/src/main/resources/monitoring-es.json b/x-pack/plugin/core/src/main/resources/monitoring-es.json
index b47ebe79129c1..ee965bd629c7f 100644
--- a/x-pack/plugin/core/src/main/resources/monitoring-es.json
+++ b/x-pack/plugin/core/src/main/resources/monitoring-es.json
@@ -974,6 +974,9 @@
             "follower_settings_version": {
               "type": "long"
             },
+            "follower_aliases_version": {
+              "type": "long"
+            },
             "total_read_time_millis": {
               "type": "long"
             },
diff --git a/x-pack/plugin/core/src/main/resources/watch-history.json b/x-pack/plugin/core/src/main/resources/watch-history.json
index 9c5919f13a171..8b6bc435d2e1d 100644
--- a/x-pack/plugin/core/src/main/resources/watch-history.json
+++ b/x-pack/plugin/core/src/main/resources/watch-history.json
@@ -264,6 +264,13 @@
                 "reason" : {
                   "type" : "keyword"
                 },
+                "number_of_actions_executed": {
+                  "type": "integer"
+                },
+                "foreach" : {
+                  "type": "object",
+                  "enabled" : false
+                },
                 "email": {
                   "type": "object",
                   "dynamic": true,
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java
index 17019b0ac18cf..b03631550befe 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.seqno.SequenceNumbers;
+import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
@@ -146,12 +147,17 @@ public void testCircuitBreakerAccounting() throws IOException {
                 null, listener, null, globalCheckpoint::get, new HierarchyCircuitBreakerService(defaultSettings.getSettings(),
                     new ClusterSettings(defaultSettings.getNodeSettings(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
             CircuitBreaker breaker = config.getCircuitBreakerService().getBreaker(CircuitBreaker.ACCOUNTING);
-            long expectedUse;
+            final int docs;
             try (InternalEngine engine = createEngine(config)) {
-                addDocuments(globalCheckpoint, engine);
+                docs = addDocuments(globalCheckpoint, engine);
                 engine.flush(false, true); // first flush to make sure we have a commit that we open in the frozen engine blow.
                 engine.refresh("test"); // pull the reader to account for RAM in the breaker.
+            }
+            final long expectedUse;
+            try (ReadOnlyEngine readOnlyEngine = new ReadOnlyEngine(config, null, null, true, i -> i)) {
                 expectedUse = breaker.getUsed();
+                DocsStats docsStats = readOnlyEngine.docStats();
+                assertEquals(docs, docsStats.getCount());
             }
             assertTrue(expectedUse > 0);
             assertEquals(0, breaker.getUsed());
@@ -182,7 +188,7 @@ private int addDocuments(AtomicLong globalCheckpoint, InternalEngine engine) thr
             if (rarely()) {
                 engine.flush();
             }
-            globalCheckpoint.set(engine.getLocalCheckpoint());
+            globalCheckpoint.set(engine.getProcessedLocalCheckpoint());
         }
         engine.syncTranslog();
         return numDocsAdded;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java
index 9231bad9a8dfe..f1009542d1325 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.index.engine;
 
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
@@ -23,6 +24,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -37,16 +39,15 @@
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.ShardSearchLocalRequest;
 import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.elasticsearch.xpack.core.XPackClient;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.action.TransportFreezeIndexAction;
+import org.elasticsearch.xpack.core.action.TransportFreezeIndexAction.FreezeIndexAction;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.util.Collection;
 import java.util.EnumSet;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -64,13 +65,12 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(XPackPlugin.class);
     }
 
-    public void testCloseFreezeAndOpen() throws ExecutionException, InterruptedException {
+    public void testCloseFreezeAndOpen() {
         createIndex("index", Settings.builder().put("index.number_of_shards", 2).build());
         client().prepareIndex("index", "_doc", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         client().prepareIndex("index", "_doc", "2").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         client().prepareIndex("index", "_doc", "3").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
-        XPackClient xPackClient = new XPackClient(client());
-        assertAcked(xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest("index")));
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest("index")).actionGet());
         expectThrows(ClusterBlockException.class, () -> client().prepareIndex("index", "_doc", "4").setSource("field", "value")
             .setRefreshPolicy(IMMEDIATE).get());
         IndicesService indexServices = getInstanceFromNode(IndicesService.class);
@@ -106,7 +106,7 @@ public void testCloseFreezeAndOpen() throws ExecutionException, InterruptedExcep
         } while (searchResponse.getHits().getHits().length > 0);
     }
 
-    public void testSearchAndGetAPIsAreThrottled() throws InterruptedException, IOException, ExecutionException {
+    public void testSearchAndGetAPIsAreThrottled() throws InterruptedException, IOException {
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_doc")
             .startObject("properties").startObject("field").field("type", "text").field("term_vector", "with_positions_offsets_payloads")
             .endObject().endObject()
@@ -115,8 +115,7 @@ public void testSearchAndGetAPIsAreThrottled() throws InterruptedException, IOEx
         for (int i = 0; i < 10; i++) {
             client().prepareIndex("index", "_doc", "" + i).setSource("field", "foo bar baz").get();
         }
-        XPackClient xPackClient = new XPackClient(client());
-        assertAcked(xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest("index")));
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest("index")).actionGet());
         int numRequests = randomIntBetween(20, 50);
         CountDownLatch latch = new CountDownLatch(numRequests);
         ActionListener listener = ActionListener.wrap(latch::countDown);
@@ -135,7 +134,7 @@ public void testSearchAndGetAPIsAreThrottled() throws InterruptedException, IOEx
                     numRefreshes += 3;
                     break;
                 case 2:
-                    client().prepareTermVectors("index", "_doc", "" + randomIntBetween(0, 9)).execute(listener);
+                   client().prepareTermVectors("index", "" + randomIntBetween(0, 9)).execute(listener);
                     break;
                 case 3:
                     client().prepareExplain("index", "_doc", "" + randomIntBetween(0, 9)).setQuery(new MatchAllQueryBuilder())
@@ -150,7 +149,7 @@ public void testSearchAndGetAPIsAreThrottled() throws InterruptedException, IOEx
         assertEquals(numRefreshes, index.getTotal().refresh.getTotal());
     }
 
-    public void testFreezeAndUnfreeze() throws InterruptedException, ExecutionException {
+    public void testFreezeAndUnfreeze()  {
         createIndex("index", Settings.builder().put("index.number_of_shards", 2).build());
         client().prepareIndex("index", "_doc", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         client().prepareIndex("index", "_doc", "2").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
@@ -159,8 +158,7 @@ public void testFreezeAndUnfreeze() throws InterruptedException, ExecutionExcept
             // sometimes close it
             assertAcked(client().admin().indices().prepareClose("index").get());
         }
-        XPackClient xPackClient = new XPackClient(client());
-        assertAcked(xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest("index")));
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest("index")).actionGet());
         {
             IndicesService indexServices = getInstanceFromNode(IndicesService.class);
             Index index = resolveIndex("index");
@@ -169,7 +167,8 @@ public void testFreezeAndUnfreeze() throws InterruptedException, ExecutionExcept
             IndexShard shard = indexService.getShard(0);
             assertEquals(0, shard.refreshStats().getTotal());
         }
-        assertAcked(xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest("index").setFreeze(false)));
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE,
+            new TransportFreezeIndexAction.FreezeRequest("index").setFreeze(false)).actionGet());
         {
             IndicesService indexServices = getInstanceFromNode(IndicesService.class);
             Index index = resolveIndex("index");
@@ -190,40 +189,37 @@ private void assertIndexFrozen(String idx) {
         assertTrue(FrozenEngine.INDEX_FROZEN.get(indexService.getIndexSettings().getSettings()));
     }
 
-    public void testDoubleFreeze() throws ExecutionException, InterruptedException {
+    public void testDoubleFreeze() {
         createIndex("test-idx", Settings.builder().put("index.number_of_shards", 2).build());
-        XPackClient xPackClient = new XPackClient(client());
-        assertAcked(xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest("test-idx")));
-        ExecutionException executionException = expectThrows(ExecutionException.class,
-            () -> xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest("test-idx")
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest("test-idx")).actionGet());
+        ResourceNotFoundException exception = expectThrows(ResourceNotFoundException.class,
+            () -> client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest("test-idx")
                 .indicesOptions(new IndicesOptions(EnumSet.noneOf(IndicesOptions.Option.class),
-                EnumSet.of(IndicesOptions.WildcardStates.OPEN)))));
-        assertEquals("no index found to freeze", executionException.getCause().getMessage());
+                EnumSet.of(IndicesOptions.WildcardStates.OPEN)))).actionGet());
+        assertEquals("no index found to freeze", exception.getMessage());
     }
 
-    public void testUnfreezeClosedIndices() throws ExecutionException, InterruptedException {
+    public void testUnfreezeClosedIndices() {
         createIndex("idx", Settings.builder().put("index.number_of_shards", 1).build());
         client().prepareIndex("idx", "_doc", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         createIndex("idx-closed", Settings.builder().put("index.number_of_shards", 1).build());
         client().prepareIndex("idx-closed", "_doc", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
-        XPackClient xPackClient = new XPackClient(client());
-        assertAcked(xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest("idx")));
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest("idx")).actionGet());
         assertAcked(client().admin().indices().prepareClose("idx-closed").get());
-        assertAcked(xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest("idx*").setFreeze(false)
-            .indicesOptions(IndicesOptions.strictExpand())));
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest("idx*").setFreeze(false)
+            .indicesOptions(IndicesOptions.strictExpand())).actionGet());
         ClusterStateResponse stateResponse = client().admin().cluster().prepareState().get();
         assertEquals(IndexMetaData.State.CLOSE, stateResponse.getState().getMetaData().index("idx-closed").getState());
         assertEquals(IndexMetaData.State.OPEN, stateResponse.getState().getMetaData().index("idx").getState());
         assertHitCount(client().prepareSearch().get(), 1L);
     }
 
-    public void testFreezePattern() throws ExecutionException, InterruptedException {
+    public void testFreezePattern() {
         createIndex("test-idx", Settings.builder().put("index.number_of_shards", 1).build());
         client().prepareIndex("test-idx", "_doc", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         createIndex("test-idx-1", Settings.builder().put("index.number_of_shards", 1).build());
         client().prepareIndex("test-idx-1", "_doc", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
-        XPackClient xPackClient = new XPackClient(client());
-        assertAcked(xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest("test-idx")));
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest("test-idx")).actionGet());
         assertIndexFrozen("test-idx");
 
         IndicesStatsResponse index = client().admin().indices().prepareStats("test-idx").clear().setRefresh(true).get();
@@ -232,7 +228,7 @@ public void testFreezePattern() throws ExecutionException, InterruptedException
         index = client().admin().indices().prepareStats("test-idx").clear().setRefresh(true).get();
         assertEquals(1, index.getTotal().refresh.getTotal());
 
-        assertAcked(xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest("test*")));
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest("test*")).actionGet());
         assertIndexFrozen("test-idx");
         assertIndexFrozen("test-idx-1");
         index = client().admin().indices().prepareStats("test-idx").clear().setRefresh(true).get();
@@ -241,7 +237,7 @@ public void testFreezePattern() throws ExecutionException, InterruptedException
         assertEquals(0, index.getTotal().refresh.getTotal());
     }
 
-    public void testCanMatch() throws ExecutionException, InterruptedException, IOException {
+    public void testCanMatch() throws IOException {
         createIndex("index");
         client().prepareIndex("index", "_doc", "1").setSource("field", "2010-01-05T02:00").setRefreshPolicy(IMMEDIATE).execute()
             .actionGet();
@@ -269,8 +265,7 @@ public void testCanMatch() throws ExecutionException, InterruptedException, IOEx
                 new AliasFilter(null, Strings.EMPTY_ARRAY), 1f, -1, null, null)));
         }
 
-        XPackClient xPackClient = new XPackClient(client());
-        assertAcked(xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest("index")));
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest("index")).actionGet());
         {
 
             IndicesService indexServices = getInstanceFromNode(IndicesService.class);
@@ -298,46 +293,44 @@ public void testCanMatch() throws ExecutionException, InterruptedException, IOEx
         }
     }
 
-    public void testWriteToFrozenIndex() throws ExecutionException, InterruptedException {
+    public void testWriteToFrozenIndex() {
         createIndex("idx", Settings.builder().put("index.number_of_shards", 1).build());
         client().prepareIndex("idx", "_doc", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
-        XPackClient xPackClient = new XPackClient(client());
-        assertAcked(xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest("idx")));
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest("idx")).actionGet());
         assertIndexFrozen("idx");
         expectThrows(ClusterBlockException.class, () ->
         client().prepareIndex("idx", "_doc", "2").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get());
     }
 
-    public void testIgnoreUnavailable() throws ExecutionException, InterruptedException {
+    public void testIgnoreUnavailable() {
         createIndex("idx", Settings.builder().put("index.number_of_shards", 1).build());
         createIndex("idx-close", Settings.builder().put("index.number_of_shards", 1).build());
         assertAcked(client().admin().indices().prepareClose("idx-close"));
-        XPackClient xPackClient = new XPackClient(client());
-        assertAcked(xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest("idx*", "not_available")
-            .indicesOptions(IndicesOptions.fromParameters(null, "true", null, null, IndicesOptions.strictExpandOpen()))));
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest("idx*", "not_available")
+            .indicesOptions(IndicesOptions.fromParameters(null, "true", null, null, IndicesOptions.strictExpandOpen()))).actionGet());
         assertIndexFrozen("idx");
         assertEquals(IndexMetaData.State.CLOSE,
             client().admin().cluster().prepareState().get().getState().metaData().index("idx-close").getState());
     }
 
-    public void testUnfreezeClosedIndex() throws ExecutionException, InterruptedException {
+    public void testUnfreezeClosedIndex() {
         createIndex("idx", Settings.builder().put("index.number_of_shards", 1).build());
-        XPackClient xPackClient = new XPackClient(client());
-        assertAcked(xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest("idx")));
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest("idx")).actionGet());
         assertAcked(client().admin().indices().prepareClose("idx"));
         assertEquals(IndexMetaData.State.CLOSE,
             client().admin().cluster().prepareState().get().getState().metaData().index("idx").getState());
-        expectThrows(ExecutionException.class,
-            () -> xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest("id*").setFreeze(false)
+        expectThrows(IndexNotFoundException.class,
+            () -> client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest("id*").setFreeze(false)
                 .indicesOptions(new IndicesOptions(EnumSet.noneOf(IndicesOptions.Option.class),
-                    EnumSet.of(IndicesOptions.WildcardStates.OPEN)))));
+                    EnumSet.of(IndicesOptions.WildcardStates.OPEN)))).actionGet());
         // we don't resolve to closed indices
-        assertAcked(xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest("idx").setFreeze(false)));
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE,
+            new TransportFreezeIndexAction.FreezeRequest("idx").setFreeze(false)).actionGet());
         assertEquals(IndexMetaData.State.OPEN,
             client().admin().cluster().prepareState().get().getState().metaData().index("idx").getState());
     }
 
-    public void testFreezeIndexIncreasesIndexSettingsVersion() throws ExecutionException, InterruptedException {
+    public void testFreezeIndexIncreasesIndexSettingsVersion() {
         final String index = "test";
         createIndex(index, Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build());
         client().prepareIndex(index, "_doc").setSource("field", "value").execute().actionGet();
@@ -345,8 +338,7 @@ public void testFreezeIndexIncreasesIndexSettingsVersion() throws ExecutionExcep
         final long settingsVersion = client().admin().cluster().prepareState().get()
             .getState().metaData().index(index).getSettingsVersion();
 
-        XPackClient xPackClient = new XPackClient(client());
-        assertAcked(xPackClient.freeze(new TransportFreezeIndexAction.FreezeRequest(index)));
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest(index)).actionGet());
         assertIndexFrozen(index);
         assertThat(client().admin().cluster().prepareState().get().getState().metaData().index(index).getSettingsVersion(),
             greaterThan(settingsVersion));
@@ -371,14 +363,14 @@ public void testFreezeEmptyIndexWithTranslogOps() throws Exception {
             final Index index = client().admin().cluster().prepareState().get().getState().metaData().index(indexName).getIndex();
             final IndexService indexService = indicesService.indexService(index);
             assertThat(indexService.hasShard(0), is(true));
-            assertThat(indexService.getShard(0).getGlobalCheckpoint(), greaterThanOrEqualTo(nbNoOps - 1L));
+            assertThat(indexService.getShard(0).getLastKnownGlobalCheckpoint(), greaterThanOrEqualTo(nbNoOps - 1L));
         });
 
-        assertAcked(new XPackClient(client()).freeze(new TransportFreezeIndexAction.FreezeRequest(indexName)));
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest(indexName)).actionGet());
         assertIndexFrozen(indexName);
     }
 
-    public void testRecoveryState() throws ExecutionException, InterruptedException {
+    public void testRecoveryState() {
         final String indexName = "index_recovery_state";
         createIndex(indexName, Settings.builder()
             .put("index.number_of_replicas", 0)
@@ -390,7 +382,7 @@ public void testRecoveryState() throws ExecutionException, InterruptedException
             assertThat(indexResponse.status(), is(RestStatus.CREATED));
         }
 
-        assertAcked(new XPackClient(client()).freeze(new TransportFreezeIndexAction.FreezeRequest(indexName)));
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest(indexName)).actionGet());
         assertIndexFrozen(indexName);
 
         final IndexMetaData indexMetaData = client().admin().cluster().prepareState().get().getState().metaData().index(indexName);
@@ -409,4 +401,39 @@ public void testRecoveryState() throws ExecutionException, InterruptedException
             assertThat(recoveryState.getTranslog().recoveredPercent(), equalTo(100.0f));
         }
     }
+
+    public void testTranslogStats()  {
+        final String indexName = "test";
+        createIndex(indexName, Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .build());
+
+        final int nbDocs = randomIntBetween(0, 50);
+        int uncommittedOps = 0;
+        for (long i = 0; i < nbDocs; i++) {
+            final IndexResponse indexResponse = client().prepareIndex(indexName, "_doc", Long.toString(i)).setSource("field", i).get();
+            assertThat(indexResponse.status(), is(RestStatus.CREATED));
+
+            if (rarely()) {
+                client().admin().indices().prepareFlush(indexName).get();
+                uncommittedOps = 0;
+            } else {
+                uncommittedOps += 1;
+            }
+        }
+
+        IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName).clear().setTranslog(true).get();
+        assertThat(stats.getIndex(indexName), notNullValue());
+        assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(nbDocs));
+        assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(uncommittedOps));
+
+        assertAcked(client().execute(FreezeIndexAction.INSTANCE, new TransportFreezeIndexAction.FreezeRequest(indexName)).actionGet());
+        assertIndexFrozen(indexName);
+
+        IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
+        stats = client().admin().indices().prepareStats(indexName).setIndicesOptions(indicesOptions).clear().setTranslog(true).get();
+        assertThat(stats.getIndex(indexName), notNullValue());
+        assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(nbDocs));
+        assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(0));
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java
index 7c599e95cc026..91f61bed185b2 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/XPackLicenseStateTests.java
@@ -107,6 +107,8 @@ public void testSecurityBasicWithoutExplicitSecurityEnabled() {
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(false));
         assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.NONE));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
+        assertThat(licenseState.isTokenServiceAllowed(), is(false));
+        assertThat(licenseState.isApiKeyServiceAllowed(), is(false));
 
         assertThat(licenseState.isSecurityAvailable(), is(true));
         assertThat(licenseState.isSecurityDisabledByLicenseDefaults(), is(true));
@@ -124,6 +126,8 @@ public void testSecurityBasicWithExplicitSecurityEnabled() {
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(false));
         assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.NATIVE));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
+        assertThat(licenseState.isTokenServiceAllowed(), is(false));
+        assertThat(licenseState.isApiKeyServiceAllowed(), is(true));
 
         assertThat(licenseState.isSecurityAvailable(), is(true));
         assertThat(licenseState.isSecurityDisabledByLicenseDefaults(), is(false));
@@ -140,6 +144,8 @@ public void testSecurityDefaultBasicExpired() {
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(false));
         assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.NONE));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
+        assertThat(licenseState.isTokenServiceAllowed(), is(false));
+        assertThat(licenseState.isApiKeyServiceAllowed(), is(false));
     }
 
     public void testSecurityEnabledBasicExpired() {
@@ -154,6 +160,8 @@ public void testSecurityEnabledBasicExpired() {
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(false));
         assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.NATIVE));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
+        assertThat(licenseState.isTokenServiceAllowed(), is(false));
+        assertThat(licenseState.isApiKeyServiceAllowed(), is(true));
     }
 
     public void testSecurityStandard() {
@@ -196,6 +204,8 @@ public void testSecurityGold() {
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(false));
         assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.DEFAULT));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
+        assertThat(licenseState.isTokenServiceAllowed(), is(true));
+        assertThat(licenseState.isApiKeyServiceAllowed(), is(true));
     }
 
     public void testSecurityGoldExpired() {
@@ -210,6 +220,8 @@ public void testSecurityGoldExpired() {
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(false));
         assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.DEFAULT));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
+        assertThat(licenseState.isTokenServiceAllowed(), is(true));
+        assertThat(licenseState.isApiKeyServiceAllowed(), is(true));
     }
 
     public void testSecurityPlatinum() {
@@ -224,6 +236,8 @@ public void testSecurityPlatinum() {
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(true));
         assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.ALL));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(true));
+        assertThat(licenseState.isTokenServiceAllowed(), is(true));
+        assertThat(licenseState.isApiKeyServiceAllowed(), is(true));
     }
 
     public void testSecurityPlatinumExpired() {
@@ -238,6 +252,8 @@ public void testSecurityPlatinumExpired() {
         assertThat(licenseState.isDocumentAndFieldLevelSecurityAllowed(), is(true));
         assertThat(licenseState.allowedRealmType(), is(XPackLicenseState.AllowedRealmType.ALL));
         assertThat(licenseState.isCustomRoleProvidersAllowed(), is(false));
+        assertThat(licenseState.isTokenServiceAllowed(), is(true));
+        assertThat(licenseState.isApiKeyServiceAllowed(), is(true));
     }
 
     public void testNewTrialDefaultsSecurityOff() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotIT.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotIT.java
index 81be978d33103..a54c57aceb3a6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotIT.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotIT.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.MockEngineFactoryPlugin;
 import org.elasticsearch.index.engine.EngineFactory;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -65,10 +64,8 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     @Override
-    protected Collection<Class<? extends Plugin>> getMockPlugins() {
-        Collection<Class<? extends Plugin>> classes = new ArrayList<>(super.getMockPlugins());
-        classes.remove(MockEngineFactoryPlugin.class);
-        return classes;
+    protected boolean addMockInternalEngine() {
+        return false;
     }
 
     public static final class MyPlugin extends Plugin implements RepositoryPlugin, EnginePlugin {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java
index 948503b33478c..758cd16391c89 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/snapshots/SourceOnlySnapshotShardTests.java
@@ -83,7 +83,7 @@ public void testSourceIncomplete() throws IOException {
             .primaryTerm(0, primaryTerm)
             .putMapping("_doc",
                 "{\"_source\":{\"enabled\": false}}").build();
-        IndexShard shard = newShard(shardRouting, metaData, new InternalEngineFactory());
+        IndexShard shard = newShard(shardRouting, metaData, null, new InternalEngineFactory());
         recoverShardFromStore(shard);
 
         for (int i = 0; i < 1; i++) {
@@ -278,7 +278,7 @@ public IndexShard reindex(DirectoryReader reader, MappingMetaData mapping) throw
             .settings(settings)
             .primaryTerm(0, primaryTerm);
         metaData.putMapping(mapping);
-        IndexShard targetShard = newShard(targetShardRouting, metaData.build(), new InternalEngineFactory());
+        IndexShard targetShard = newShard(targetShardRouting, metaData.build(), null, new InternalEngineFactory());
         boolean success = false;
         try {
             recoverShardFromStore(targetShard);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java
index 3f8b279e5016c..e334f72bc4a8a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java
@@ -7,17 +7,20 @@
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.admin.indices.mapping.put.MappingRequestValidator;
+import org.elasticsearch.action.RequestValidators;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.coordination.ElectionStrategy;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -74,10 +77,12 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
@@ -130,16 +135,6 @@ protected void setLicenseState(XPackLicenseState licenseState) {
         this.licenseState = licenseState;
     }
 
-    @Override
-    public Collection<Module> createGuiceModules() {
-        ArrayList<Module> modules = new ArrayList<>();
-        modules.addAll(super.createGuiceModules());
-        filterPlugins(Plugin.class).stream().forEach(p ->
-            modules.addAll(p.createGuiceModules())
-        );
-        return modules;
-    }
-
     @Override
     public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
                                                ResourceWatcherService resourceWatcherService, ScriptService scriptService,
@@ -331,6 +326,7 @@ public List<ExecutorBuilder<?>> getExecutorBuilders(final Settings settings) {
         filterPlugins(Plugin.class).stream().forEach(p -> builders.addAll(p.getExecutorBuilders(settings)));
         return builders;
     }
+
     @Override
     public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
         return templates -> {
@@ -341,6 +337,20 @@ public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDat
         };
     }
 
+    @Override
+    public Map<String, ElectionStrategy> getElectionStrategies() {
+        Map<String, ElectionStrategy> electionStrategies = new HashMap<>();
+        filterPlugins(DiscoveryPlugin.class).stream().forEach(p -> electionStrategies.putAll(p.getElectionStrategies()));
+        return electionStrategies;
+    }
+
+    @Override
+    public Set<DiscoveryNodeRole> getRoles() {
+        Set<DiscoveryNodeRole> roles = new HashSet<>();
+        filterPlugins(Plugin.class).stream().forEach(p -> roles.addAll(p.getRoles()));
+        return roles;
+    }
+
     @Override
     public Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> getTokenizers() {
         Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> tokenizers = new HashMap<>();
@@ -431,13 +441,24 @@ public Optional<EngineFactory> getEngineFactory(IndexSettings indexSettings) {
     }
 
     @Override
-    public Collection<MappingRequestValidator> mappingRequestValidators() {
-        return filterPlugins(ActionPlugin.class).stream().flatMap(p -> p.mappingRequestValidators().stream())
-            .collect(Collectors.toList());
+    public Collection<RequestValidators.RequestValidator<PutMappingRequest>> mappingRequestValidators() {
+        return filterPlugins(ActionPlugin.class)
+                .stream()
+                .flatMap(p -> p.mappingRequestValidators().stream())
+                .collect(Collectors.toList());
+    }
+
+    @Override
+    public Collection<RequestValidators.RequestValidator<IndicesAliasesRequest>> indicesAliasesRequestValidators() {
+        return filterPlugins(ActionPlugin.class)
+                .stream()
+                .flatMap(p -> p.indicesAliasesRequestValidators().stream())
+                .collect(Collectors.toList());
     }
 
     private <T> List<T> filterPlugins(Class<T> type) {
         return plugins.stream().filter(x -> type.isAssignableFrom(x.getClass())).map(p -> ((T)p))
                 .collect(Collectors.toList());
     }
+
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadAnalyzersResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadAnalyzersResponseTests.java
new file mode 100644
index 0000000000000..3a10700ca1d90
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadAnalyzersResponseTests.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.action;
+
+import org.elasticsearch.action.support.DefaultShardOperationFailedException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractBroadcastResponseTestCase;
+import org.elasticsearch.xpack.core.action.ReloadAnalyzersResponse.ReloadDetails;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class ReloadAnalyzersResponseTests extends AbstractBroadcastResponseTestCase<ReloadAnalyzersResponse> {
+
+    @Override
+    protected ReloadAnalyzersResponse createTestInstance(int totalShards, int successfulShards, int failedShards,
+                                                 List<DefaultShardOperationFailedException> failures) {
+        Map<String, ReloadDetails> reloadedIndicesDetails = new HashMap<>();
+        int randomIndices = randomIntBetween(0, 5);
+        for (int i = 0; i < randomIndices; i++) {
+            String name = randomAlphaOfLengthBetween(5, 10);
+            Set<String> reloadedIndicesNodes = new HashSet<>(Arrays.asList(generateRandomStringArray(5, 5, false, true)));
+            Set<String> reloadedAnalyzers = new HashSet<>(Arrays.asList(generateRandomStringArray(5, 5, false, true)));
+            reloadedIndicesDetails.put(name, new ReloadDetails(name, reloadedIndicesNodes, reloadedAnalyzers));
+        }
+        return new ReloadAnalyzersResponse(totalShards, successfulShards, failedShards, failures, reloadedIndicesDetails);
+    }
+
+    @Override
+    protected ReloadAnalyzersResponse doParseInstance(XContentParser parser) throws IOException {
+        return ReloadAnalyzersResponse.fromXContent(parser);
+    }
+
+    @Override
+    public void testToXContent() {
+        Map<String, ReloadDetails> reloadedIndicesNodes = Collections.singletonMap("index",
+                new ReloadDetails("index", Collections.singleton("nodeId"), Collections.singleton("my_analyzer")));
+        ReloadAnalyzersResponse response = new ReloadAnalyzersResponse(10, 5, 5, null, reloadedIndicesNodes);
+        String output = Strings.toString(response);
+        assertEquals(
+                "{\"_shards\":{\"total\":10,\"successful\":5,\"failed\":5},"
+                + "\"reload_details\":[{\"index\":\"index\",\"reloaded_analyzers\":[\"my_analyzer\"],\"reloaded_node_ids\":[\"nodeId\"]}]"
+                + "}",
+                output);
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadSynonymAnalyzerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadSynonymAnalyzerTests.java
new file mode 100644
index 0000000000000..590e4bb982eb0
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadSynonymAnalyzerTests.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.action;
+
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction.AnalyzeToken;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction.Response;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+
+public class ReloadSynonymAnalyzerTests extends ESSingleNodeTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Arrays.asList(LocalStateCompositeXPackPlugin.class, CommonAnalysisPlugin.class);
+    }
+
+    public void testSynonymsUpdateable() throws FileNotFoundException, IOException {
+        String synonymsFileName = "synonyms.txt";
+        Path configDir = node().getEnvironment().configFile();
+        if (Files.exists(configDir) == false) {
+            Files.createDirectory(configDir);
+        }
+        Path synonymsFile = configDir.resolve(synonymsFileName);
+        if (Files.exists(synonymsFile) == false) {
+            Files.createFile(synonymsFile);
+        }
+        try (PrintWriter out = new PrintWriter(
+                new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) {
+            out.println("foo, baz");
+        }
+
+        final String indexName = "test";
+        final String analyzerName = "my_synonym_analyzer";
+        assertAcked(client().admin().indices().prepareCreate(indexName).setSettings(Settings.builder()
+                .put("index.number_of_shards", 5)
+                .put("index.number_of_replicas", 0)
+                .put("analysis.analyzer." + analyzerName + ".tokenizer", "standard")
+                .putList("analysis.analyzer." + analyzerName + ".filter", "lowercase", "my_synonym_filter")
+                .put("analysis.filter.my_synonym_filter.type", "synonym")
+                .put("analysis.filter.my_synonym_filter.updateable", "true")
+                .put("analysis.filter.my_synonym_filter.synonyms_path", synonymsFileName))
+                .addMapping("_doc", "field", "type=text,analyzer=standard,search_analyzer=" + analyzerName));
+
+        client().prepareIndex(indexName, "_doc", "1").setSource("field", "Foo").get();
+        assertNoFailures(client().admin().indices().prepareRefresh(indexName).execute().actionGet());
+
+        SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("field", "baz")).get();
+        assertHitCount(response, 1L);
+        response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("field", "buzz")).get();
+        assertHitCount(response, 0L);
+        Response analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(analyzerName).get();
+        assertEquals(2, analyzeResponse.getTokens().size());
+        assertEquals("foo", analyzeResponse.getTokens().get(0).getTerm());
+        assertEquals("baz", analyzeResponse.getTokens().get(1).getTerm());
+
+        // now update synonyms file and trigger reloading
+        try (PrintWriter out = new PrintWriter(
+                new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) {
+            out.println("foo, baz, buzz");
+        }
+        ReloadAnalyzersResponse reloadResponse = client().execute(ReloadAnalyzerAction.INSTANCE, new ReloadAnalyzersRequest(indexName))
+                .actionGet();
+        assertNoFailures(reloadResponse);
+        Set<String> reloadedAnalyzers = reloadResponse.getReloadDetails().get(indexName).getReloadedAnalyzers();
+        assertEquals(1, reloadedAnalyzers.size());
+        assertTrue(reloadedAnalyzers.contains(analyzerName));
+
+        analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "Foo").setAnalyzer(analyzerName).get();
+        assertEquals(3, analyzeResponse.getTokens().size());
+        Set<String> tokens = new HashSet<>();
+        analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t));
+        assertTrue(tokens.contains("foo"));
+        assertTrue(tokens.contains("baz"));
+        assertTrue(tokens.contains("buzz"));
+
+        response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("field", "baz")).get();
+        assertHitCount(response, 1L);
+        response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("field", "buzz")).get();
+        assertHitCount(response, 1L);
+    }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java
index f6c5c4ce232df..a5f27f73ab3d7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java
@@ -6,8 +6,9 @@
 package org.elasticsearch.xpack.core.action;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.protocol.xpack.XPackInfoRequest;
@@ -16,16 +17,14 @@
 import org.elasticsearch.protocol.xpack.license.LicenseStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.XPackFeatureSet;
 
-import java.util.Collections;
+import java.util.ArrayList;
 import java.util.EnumSet;
-import java.util.HashSet;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
@@ -35,7 +34,10 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsNull.notNullValue;
 import static org.hamcrest.core.IsNull.nullValue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.stub;
 import static org.mockito.Mockito.when;
 
 public class TransportXPackInfoActionTests extends ESTestCase {
@@ -44,21 +46,27 @@ public void testDoExecute() throws Exception {
 
         LicenseService licenseService = mock(LicenseService.class);
 
-        final Set<XPackFeatureSet> featureSets = new HashSet<>();
+        NodeClient client = mock(NodeClient.class);
+        Map<XPackInfoFeatureAction, FeatureSet> featureSets = new HashMap<>();
         int featureSetCount = randomIntBetween(0, 5);
-        for (int i = 0; i < featureSetCount; i++) {
-            XPackFeatureSet fs = mock(XPackFeatureSet.class);
-            when(fs.name()).thenReturn(randomAlphaOfLength(5));
-            when(fs.description()).thenReturn(randomAlphaOfLength(10));
-            when(fs.available()).thenReturn(randomBoolean());
-            when(fs.enabled()).thenReturn(randomBoolean());
-            featureSets.add(fs);
+        for (XPackInfoFeatureAction infoAction : randomSubsetOf(featureSetCount, XPackInfoFeatureAction.ALL)) {
+            FeatureSet featureSet = new FeatureSet(randomAlphaOfLength(5), randomBoolean(), randomBoolean());
+            featureSets.put(infoAction, featureSet);
+            stub(client.executeLocally(eq(infoAction), any(ActionRequest.class), any(ActionListener.class))).toAnswer(answer -> {
+                @SuppressWarnings("unchecked")
+                var listener = (ActionListener<XPackInfoFeatureResponse>)answer.getArguments()[2];
+                listener.onResponse(new XPackInfoFeatureResponse(featureSet));
+                return null;
+            });
         }
 
-        TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null,
-            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet());
-        TransportXPackInfoAction action = new TransportXPackInfoAction(transportService, mock(ActionFilters.class),
-            licenseService, featureSets);
+        TransportXPackInfoAction action = new TransportXPackInfoAction(mock(TransportService.class), mock(ActionFilters.class),
+            licenseService, client) {
+            @Override
+            protected List<XPackInfoFeatureAction> infoActions() {
+                return new ArrayList<>(featureSets.keySet());
+            }
+        };
 
         License license = mock(License.class);
         long expiryDate = randomLong();
@@ -81,6 +89,7 @@ public void testDoExecute() throws Exception {
         for (int i = 0; i < maxCategoryCount; i++) {
             categories.add(randomFrom(XPackInfoRequest.Category.values()));
         }
+        categories.add(XPackInfoRequest.Category.FEATURES);
         request.setCategories(categories);
 
         final CountDownLatch latch = new CountDownLatch(1);
@@ -128,14 +137,9 @@ public void onFailure(Exception e) {
             assertThat(response.get().getFeatureSetsInfo(), notNullValue());
             Map<String, FeatureSet> features = response.get().getFeatureSetsInfo().getFeatureSets();
             assertThat(features.size(), is(featureSets.size()));
-            for (XPackFeatureSet fs : featureSets) {
+            for (FeatureSet fs : featureSets.values()) {
                 assertThat(features, hasKey(fs.name()));
                 assertThat(features.get(fs.name()).name(), equalTo(fs.name()));
-                if (!request.isVerbose()) {
-                    assertThat(features.get(fs.name()).description(), is(nullValue()));
-                } else {
-                    assertThat(features.get(fs.name()).description(), is(fs.description()));
-                }
                 assertThat(features.get(fs.name()).available(), equalTo(fs.available()));
                 assertThat(features.get(fs.name()).enabled(), equalTo(fs.enabled()));
             }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditMessageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditMessageTests.java
index 8fb425698376e..e87e2cb0d93dc 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditMessageTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/notifications/AbstractAuditMessageTests.java
@@ -10,7 +10,7 @@
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractXContentTestCase;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 import org.junit.Before;
 
 import java.util.Date;
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/time/TimeUtilsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/time/TimeUtilsTests.java
similarity index 64%
rename from x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/time/TimeUtilsTests.java
rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/time/TimeUtilsTests.java
index d33968a37cfa7..e122202b5fa6c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/time/TimeUtilsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/time/TimeUtilsTests.java
@@ -3,18 +3,61 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.ml.utils.time;
+package org.elasticsearch.xpack.core.common.time;
 
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
 
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Date;
 import java.util.concurrent.TimeUnit;
 
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
 public class TimeUtilsTests extends ESTestCase {
 
-    public void testdateStringToEpoch() {
+    public void testParseTimeField() throws IOException {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "123456789")) {
+            parser.nextToken();
+            Date date = TimeUtils.parseTimeField(parser, "my_time_field");
+            assertThat(date.getTime(), equalTo(123456789L));
+        }
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "\"2016-05-01T10:00:00.333-0030\"")) {
+            parser.nextToken();
+            Date date = TimeUtils.parseTimeField(parser, "my_time_field");
+            assertThat(date.getTime(), equalTo(1462098600333L));
+        }
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{}")) {
+            parser.nextToken();
+            Exception e = expectThrows(IllegalArgumentException.class, () -> TimeUtils.parseTimeField(parser, "my_time_field"));
+            assertThat(e.getMessage(), containsString("unexpected token [START_OBJECT] for [my_time_field]"));
+        }
+    }
+
+    public void testParseTimeFieldToInstant() throws IOException {
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "123456789")) {
+            parser.nextToken();
+            Instant instant = TimeUtils.parseTimeFieldToInstant(parser, "my_time_field");
+            assertThat(instant.toEpochMilli(), equalTo(123456789L));
+        }
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "\"2016-05-01T10:00:00.333-0030\"")) {
+            parser.nextToken();
+            Instant instant = TimeUtils.parseTimeFieldToInstant(parser, "my_time_field");
+            assertThat(instant.toEpochMilli(), equalTo(1462098600333L));
+        }
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{}")) {
+            parser.nextToken();
+            Exception e = expectThrows(IllegalArgumentException.class, () -> TimeUtils.parseTimeFieldToInstant(parser, "my_time_field"));
+            assertThat(e.getMessage(), containsString("unexpected token [START_OBJECT] for [my_time_field]"));
+        }
+    }
+
+    public void testDateStringToEpoch() {
         assertEquals(1462096800000L, TimeUtils.dateStringToEpoch("2016-05-01T10:00:00Z"));
         assertEquals(1462096800333L, TimeUtils.dateStringToEpoch("2016-05-01T10:00:00.333Z"));
         assertEquals(1462096800334L, TimeUtils.dateStringToEpoch("2016-05-01T10:00:00.334+00"));
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/AbstractSerializingDataFrameTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/AbstractSerializingDataFrameTestCase.java
index 8b633cdfc26d5..494b1c76c5de4 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/AbstractSerializingDataFrameTestCase.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/AbstractSerializingDataFrameTestCase.java
@@ -13,6 +13,10 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.xpack.core.dataframe.DataFrameField;
+import org.elasticsearch.xpack.core.dataframe.DataFrameNamedXContentProvider;
+import org.elasticsearch.xpack.core.dataframe.transforms.SyncConfig;
+import org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig;
 import org.junit.Before;
 
 import java.util.List;
@@ -27,10 +31,14 @@ public abstract class AbstractSerializingDataFrameTestCase<T extends ToXContent
 
     @Before
     public void registerNamedObjects() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
 
         List<NamedWriteableRegistry.Entry> namedWriteables = searchModule.getNamedWriteables();
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SyncConfig.class, DataFrameField.TIME_BASED_SYNC.getPreferredName(),
+                TimeSyncConfig::new));
+
         List<NamedXContentRegistry.Entry> namedXContents = searchModule.getNamedXContents();
+        namedXContents.addAll(new DataFrameNamedXContentProvider().getNamedXContentParsers());
 
         namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables);
         namedXContentRegistry = new NamedXContentRegistry(namedXContents);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/AbstractWireSerializingDataFrameTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/AbstractWireSerializingDataFrameTestCase.java
index 91a7ec54dd256..85a24da5a1a24 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/AbstractWireSerializingDataFrameTestCase.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/AbstractWireSerializingDataFrameTestCase.java
@@ -12,6 +12,10 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.dataframe.DataFrameField;
+import org.elasticsearch.xpack.core.dataframe.DataFrameNamedXContentProvider;
+import org.elasticsearch.xpack.core.dataframe.transforms.SyncConfig;
+import org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig;
 import org.junit.Before;
 
 import java.util.List;
@@ -27,10 +31,14 @@ public abstract class AbstractWireSerializingDataFrameTestCase<T extends Writeab
 
     @Before
     public void registerNamedObjects() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
 
         List<NamedWriteableRegistry.Entry> namedWriteables = searchModule.getNamedWriteables();
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SyncConfig.class, DataFrameField.TIME_BASED_SYNC.getPreferredName(),
+                TimeSyncConfig::new));
+
         List<NamedXContentRegistry.Entry> namedXContents = searchModule.getNamedXContents();
+        namedXContents.addAll(new DataFrameNamedXContentProvider().getNamedXContentParsers());
 
         namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables);
         namedXContentRegistry = new NamedXContentRegistry(namedXContents);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsActionResponseTests.java
index d6e32bd46c439..4a63ff787ffda 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsActionResponseTests.java
@@ -32,7 +32,7 @@ public void testInvalidTransforms() throws IOException {
         transforms.add(DataFrameTransformConfigTests.randomDataFrameTransformConfig());
         transforms.add(DataFrameTransformConfigTests.randomInvalidDataFrameTransformConfig());
 
-        Response r = new Response(transforms);
+        Response r = new Response(transforms, transforms.size());
         XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
         r.toXContent(builder, XContent.EMPTY_PARAMS);
         Map<String, Object> responseAsMap = createParser(builder).map();
@@ -52,7 +52,7 @@ public void testNoHeaderInResponse() throws IOException {
             transforms.add(DataFrameTransformConfigTests.randomDataFrameTransformConfig());
         }
 
-        Response r = new Response(transforms);
+        Response r = new Response(transforms, transforms.size());
         XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
         r.toXContent(builder, XContent.EMPTY_PARAMS);
         Map<String, Object> responseAsMap = createParser(builder).map();
@@ -76,7 +76,7 @@ protected Response createTestInstance() {
             configs.add(DataFrameTransformConfigTests.randomDataFrameTransformConfig());
         }
 
-        return new Response(configs);
+        return new Response(configs, randomNonNegativeLong());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsActionResponseTests.java
index bbe53b59a8ace..aa5c1a03ef2ea 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/GetDataFrameTransformsStatsActionResponseTests.java
@@ -32,7 +32,7 @@ protected Response createTestInstance() {
             taskFailures.add(new TaskOperationFailure("node1", randomLongBetween(1, 10), new Exception("error")));
             nodeFailures.add(new FailedNodeException("node1", "message", new Exception("error")));
         }
-        return new Response(stats, taskFailures, nodeFailures);
+        return new Response(stats, randomLongBetween(stats.size(), 10_000_000L), taskFailures, nodeFailures);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformActionRequestTests.java
index 0cfc659e50646..ea6f2a47f4692 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformActionRequestTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.core.dataframe.action.PreviewDataFrameTransformAction.Request;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig;
+import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfigTests;
 import org.elasticsearch.xpack.core.dataframe.transforms.DestConfig;
 import org.elasticsearch.xpack.core.dataframe.transforms.pivot.PivotConfigTests;
 
@@ -39,9 +40,14 @@ protected boolean supportsUnknownFields() {
 
     @Override
     protected Request createTestInstance() {
-        DataFrameTransformConfig config = new DataFrameTransformConfig("transform-preview", randomSourceConfig(),
-                new DestConfig("unused-transform-preview-index"),
-                null, PivotConfigTests.randomPivotConfig(), null);
+        DataFrameTransformConfig config = new DataFrameTransformConfig(
+                "transform-preview",
+                randomSourceConfig(),
+                new DestConfig("unused-transform-preview-index", null),
+                randomBoolean() ? DataFrameTransformConfigTests.randomSyncConfig() : null,
+                null,
+                PivotConfigTests.randomPivotConfig(),
+                null);
         return new Request(config);
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformsActionResponseTests.java
index 96744b4cd8ced..0f4fd00ab632f 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformsActionResponseTests.java
@@ -35,13 +35,28 @@ protected Response createTestInstance() {
         int size = randomIntBetween(0, 10);
         List<Map<String, Object>> data = new ArrayList<>(size);
         for (int i = 0; i < size; i++) {
-            Map<String, Object> datum = new HashMap<>();
-            Map<String, Object> entry = new HashMap<>();
-            entry.put("value1", randomIntBetween(1, 100));
-            datum.put(randomAlphaOfLength(10), entry);
-            data.add(datum);
+            data.add(Map.of(randomAlphaOfLength(10), Map.of("value1", randomIntBetween(1, 100))));
         }
-        return new Response(data);
+
+        Response response = new Response(data);
+        if (randomBoolean()) {
+            size = randomIntBetween(0, 10);
+            if (randomBoolean()) {
+                Map<String, Object> mappings = new HashMap<>(size);
+                for (int i = 0; i < size; i++) {
+                    mappings.put(randomAlphaOfLength(10), Map.of("type", randomAlphaOfLength(10)));
+                }
+                response.setMappings(mappings);
+            } else {
+                Map<String, String> mappings = new HashMap<>(size);
+                for (int i = 0; i < size; i++) {
+                    mappings.put(randomAlphaOfLength(10), randomAlphaOfLength(10));
+                }
+                response.setMappingsFromStringMap(mappings);
+            }
+        }
+
+        return response;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformActionRequestTests.java
index 81f03902980ac..cce889baa9675 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformActionRequestTests.java
@@ -24,7 +24,7 @@ public class StopDataFrameTransformActionRequestTests extends AbstractWireSerial
     @Override
     protected Request createTestInstance() {
         TimeValue timeout = randomBoolean() ? TimeValue.timeValueMinutes(randomIntBetween(1, 10)) : null;
-        Request request = new Request(randomAlphaOfLengthBetween(1, 10), randomBoolean(), randomBoolean(), timeout);
+        Request request = new Request(randomAlphaOfLengthBetween(1, 10), randomBoolean(), randomBoolean(), timeout, randomBoolean());
         if (randomBoolean()) {
             request.setExpandedIds(new HashSet<>(Arrays.asList(generateRandomStringArray(5, 6, false))));
         }
@@ -40,9 +40,10 @@ public void testSameButDifferentTimeout() {
         String id = randomAlphaOfLengthBetween(1, 10);
         boolean waitForCompletion = randomBoolean();
         boolean force = randomBoolean();
+        boolean allowNoMatch = randomBoolean();
 
-        Request r1 = new Request(id, waitForCompletion, force, TimeValue.timeValueSeconds(10));
-        Request r2 = new Request(id, waitForCompletion, force, TimeValue.timeValueSeconds(20));
+        Request r1 = new Request(id, waitForCompletion, force, TimeValue.timeValueSeconds(10), allowNoMatch);
+        Request r2 = new Request(id, waitForCompletion, force, TimeValue.timeValueSeconds(20), allowNoMatch);
 
         assertNotEquals(r1,r2);
         assertNotEquals(r1.hashCode(),r2.hashCode());
@@ -55,11 +56,11 @@ public void testMatch() {
                 DataFrameField.PERSISTENT_TASK_DESCRIPTION_PREFIX + dataFrameId,
                 TaskId.EMPTY_TASK_ID, Collections.emptyMap());
 
-        Request request = new Request("unrelated", false, false, null);
+        Request request = new Request("unrelated", false, false, null, false);
         request.setExpandedIds(Set.of("foo", "bar"));
         assertFalse(request.match(dataFrameTask));
 
-        Request matchingRequest = new Request(dataFrameId, false, false, null);
+        Request matchingRequest = new Request(dataFrameId, false, false, null, false);
         matchingRequest.setExpandedIds(Set.of(dataFrameId));
         assertTrue(matchingRequest.match(dataFrameTask));
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/AbstractSerializingDataFrameTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/AbstractSerializingDataFrameTestCase.java
index 2b64fadac051a..7dd501160ba00 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/AbstractSerializingDataFrameTestCase.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/AbstractSerializingDataFrameTestCase.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.search.aggregations.BaseAggregationBuilder;
 import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
+import org.elasticsearch.xpack.core.dataframe.DataFrameNamedXContentProvider;
 import org.junit.Before;
 
 import java.util.Collections;
@@ -41,19 +42,22 @@ public abstract class AbstractSerializingDataFrameTestCase<T extends ToXContent
     @Before
     public void registerAggregationNamedObjects() throws Exception {
         // register aggregations as NamedWriteable
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
 
         List<NamedWriteableRegistry.Entry> namedWriteables = searchModule.getNamedWriteables();
         namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MockDeprecatedQueryBuilder.NAME,
                 MockDeprecatedQueryBuilder::new));
         namedWriteables.add(new NamedWriteableRegistry.Entry(AggregationBuilder.class, MockDeprecatedAggregationBuilder.NAME,
                 MockDeprecatedAggregationBuilder::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SyncConfig.class, DataFrameField.TIME_BASED_SYNC.getPreferredName(),
+                TimeSyncConfig::new));
 
         List<NamedXContentRegistry.Entry> namedXContents = searchModule.getNamedXContents();
         namedXContents.add(new NamedXContentRegistry.Entry(QueryBuilder.class,
                 new ParseField(MockDeprecatedQueryBuilder.NAME), (p, c) -> MockDeprecatedQueryBuilder.fromXContent(p)));
         namedXContents.add(new NamedXContentRegistry.Entry(BaseAggregationBuilder.class,
                 new ParseField(MockDeprecatedAggregationBuilder.NAME), (p, c) -> MockDeprecatedAggregationBuilder.fromXContent(p)));
+        namedXContents.addAll(new DataFrameNamedXContentProvider().getNamedXContentParsers());
 
         namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables);
         namedXContentRegistry = new NamedXContentRegistry(namedXContents);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java
index a735b5a02acb8..dd5b5c9ff8841 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java
@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.core.dataframe.transforms;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
@@ -18,6 +19,7 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -41,24 +43,42 @@ public static DataFrameTransformConfig randomDataFrameTransformConfig() {
     }
 
     public static DataFrameTransformConfig randomDataFrameTransformConfigWithoutHeaders(String id) {
-        return new DataFrameTransformConfig(id, randomSourceConfig(), randomDestConfig(), null,
-                PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000));
+        return new DataFrameTransformConfig(id,
+            randomSourceConfig(),
+            randomDestConfig(),
+            randomBoolean() ? null : randomSyncConfig(),
+            null,
+            PivotConfigTests.randomPivotConfig(),
+            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000),
+            null,
+            null);
     }
 
     public static DataFrameTransformConfig randomDataFrameTransformConfig(String id) {
-        return new DataFrameTransformConfig(id, randomSourceConfig(), randomDestConfig(), randomHeaders(),
-                PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000));
+        return new DataFrameTransformConfig(id,
+            randomSourceConfig(),
+            randomDestConfig(),
+            randomBoolean() ? null : randomSyncConfig(),
+            randomHeaders(),
+            PivotConfigTests.randomPivotConfig(),
+            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000),
+            randomBoolean() ? null : Instant.now(),
+            randomBoolean() ? null : Version.CURRENT.toString());
     }
 
     public static DataFrameTransformConfig randomInvalidDataFrameTransformConfig() {
         if (randomBoolean()) {
-            return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomInvalidSourceConfig(),
-                    randomDestConfig(), randomHeaders(), PivotConfigTests.randomPivotConfig(),
-                randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100));
+            return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomInvalidSourceConfig(), randomDestConfig(),
+                    randomBoolean() ? randomSyncConfig() : null, randomHeaders(), PivotConfigTests.randomPivotConfig(),
+                    randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000));
         } // else
-        return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomSourceConfig(),
-                randomDestConfig(), randomHeaders(), PivotConfigTests.randomInvalidPivotConfig(),
-            randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100));
+        return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomSourceConfig(), randomDestConfig(),
+                randomBoolean() ? randomSyncConfig() : null, randomHeaders(), PivotConfigTests.randomInvalidPivotConfig(),
+                randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000));
+    }
+
+    public static SyncConfig randomSyncConfig() {
+        return TimeSyncConfigTests.randomTimeSyncConfig();
     }
 
     @Before
@@ -147,6 +167,48 @@ public void testPreventHeaderInjection() throws IOException {
                 () -> createDataFrameTransformConfigFromString(pivotTransform, "test_header_injection"));
     }
 
+    public void testPreventCreateTimeInjection() throws IOException {
+        String pivotTransform = "{"
+            + " \"create_time\" : " + Instant.now().toEpochMilli() + " },"
+            + " \"source\" : {\"index\":\"src\"},"
+            + " \"dest\" : {\"index\": \"dest\"},"
+            + " \"pivot\" : {"
+            + " \"group_by\": {"
+            + "   \"id\": {"
+            + "     \"terms\": {"
+            + "       \"field\": \"id\""
+            + "} } },"
+            + " \"aggs\": {"
+            + "   \"avg\": {"
+            + "     \"avg\": {"
+            + "       \"field\": \"points\""
+            + "} } } } }";
+
+        expectThrows(IllegalArgumentException.class,
+            () -> createDataFrameTransformConfigFromString(pivotTransform, "test_createTime_injection"));
+    }
+
+    public void testPreventVersionInjection() throws IOException {
+        String pivotTransform = "{"
+            + " \"version\" : \"7.3.0\","
+            + " \"source\" : {\"index\":\"src\"},"
+            + " \"dest\" : {\"index\": \"dest\"},"
+            + " \"pivot\" : {"
+            + " \"group_by\": {"
+            + "   \"id\": {"
+            + "     \"terms\": {"
+            + "       \"field\": \"id\""
+            + "} } },"
+            + " \"aggs\": {"
+            + "   \"avg\": {"
+            + "     \"avg\": {"
+            + "       \"field\": \"points\""
+            + "} } } } }";
+
+        expectThrows(IllegalArgumentException.class,
+            () -> createDataFrameTransformConfigFromString(pivotTransform, "test_createTime_injection"));
+    }
+
     public void testXContentForInternalStorage() throws IOException {
         DataFrameTransformConfig dataFrameTransformConfig = randomDataFrameTransformConfig();
 
@@ -167,11 +229,11 @@ public void testXContentForInternalStorage() throws IOException {
 
     public void testMaxLengthDescription() {
         IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> new DataFrameTransformConfig("id",
-            randomSourceConfig(), randomDestConfig(), null, PivotConfigTests.randomPivotConfig(), randomAlphaOfLength(1001)));
+            randomSourceConfig(), randomDestConfig(), null, null, PivotConfigTests.randomPivotConfig(), randomAlphaOfLength(1001)));
         assertThat(exception.getMessage(), equalTo("[description] must be less than 1000 characters in length."));
         String description = randomAlphaOfLength(1000);
         DataFrameTransformConfig config = new DataFrameTransformConfig("id",
-            randomSourceConfig(), randomDestConfig(), null, PivotConfigTests.randomPivotConfig(), description);
+            randomSourceConfig(), randomDestConfig(), null, null, PivotConfigTests.randomPivotConfig(), description);
         assertThat(description, equalTo(config.getDescription()));
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateTests.java
index c978978b0589b..9f4ac546c89a4 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateTests.java
@@ -17,6 +17,7 @@
 import java.util.function.Predicate;
 
 import static org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformProgressTests.randomDataFrameTransformProgress;
+import static org.elasticsearch.xpack.core.dataframe.transforms.NodeAttributeTests.randomNodeAttributes;
 
 public class DataFrameTransformStateTests extends AbstractSerializingTestCase<DataFrameTransformState> {
 
@@ -26,7 +27,8 @@ public static DataFrameTransformState randomDataFrameTransformState() {
             randomPosition(),
             randomLongBetween(0,10),
             randomBoolean() ? null : randomAlphaOfLength(10),
-            randomBoolean() ? null : randomDataFrameTransformProgress());
+            randomBoolean() ? null : randomDataFrameTransformProgress(),
+            randomBoolean() ? null : randomNodeAttributes());
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformTests.java
new file mode 100644
index 0000000000000..d7463a6df7139
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformTests.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.dataframe.transforms;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class DataFrameTransformTests extends AbstractSerializingDataFrameTestCase<DataFrameTransform> {
+
+    @Override
+    protected DataFrameTransform doParseInstance(XContentParser parser) throws IOException {
+        return DataFrameTransform.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected DataFrameTransform createTestInstance() {
+        return new DataFrameTransform(randomAlphaOfLength(10), randomBoolean() ? null : Version.CURRENT);
+    }
+
+    @Override
+    protected Reader<DataFrameTransform> instanceReader() {
+        return DataFrameTransform::new;
+    }
+
+    public void testBackwardsSerialization() throws IOException {
+        for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) {
+            DataFrameTransform transformTask = createTestInstance();
+            try (BytesStreamOutput output = new BytesStreamOutput()) {
+                output.setVersion(Version.V_7_2_0);
+                transformTask.writeTo(output);
+                try (StreamInput in = output.bytes().streamInput()) {
+                    in.setVersion(Version.V_7_2_0);
+                    // Since the old version does not have the version serialized, the version NOW is 7.2.0
+                    DataFrameTransform streamedTask = new DataFrameTransform(in);
+                    assertThat(streamedTask.getVersion(), equalTo(Version.V_7_2_0));
+                    assertThat(streamedTask.getId(), equalTo(transformTask.getId()));
+                }
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DestConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DestConfigTests.java
index b29fa46c34ede..094267ba4adea 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DestConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DestConfigTests.java
@@ -17,7 +17,8 @@ public class DestConfigTests extends AbstractSerializingDataFrameTestCase<DestCo
     private boolean lenient;
 
     public static DestConfig randomDestConfig() {
-        return new DestConfig(randomAlphaOfLength(10));
+        return new DestConfig(randomAlphaOfLength(10),
+            randomBoolean() ? null : randomAlphaOfLength(10));
     }
 
     @Before
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/MockDeprecatedAggregationBuilder.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/MockDeprecatedAggregationBuilder.java
index c89e04d4990af..fd7fe91db9889 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/MockDeprecatedAggregationBuilder.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/MockDeprecatedAggregationBuilder.java
@@ -65,8 +65,8 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
+    protected ValuesSourceAggregatorFactory<ValuesSource> innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource> config,
+            AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException {
         return null;
     }
 
@@ -75,16 +75,6 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params)
         return null;
     }
 
-    @Override
-    protected int innerHashCode() {
-        return 0;
-    }
-
-    @Override
-    protected boolean innerEquals(Object obj) {
-        return false;
-    }
-
     public static MockDeprecatedAggregationBuilder fromXContent(XContentParser p) {
         deprecationLogger.deprecatedAndMaybeLog("deprecated_mock", DEPRECATION_MESSAGE);
         return new MockDeprecatedAggregationBuilder();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/NodeAttributeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/NodeAttributeTests.java
new file mode 100644
index 0000000000000..fdc7692c412cb
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/NodeAttributeTests.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.dataframe.transforms;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.function.Predicate;
+
+
+public class NodeAttributeTests extends AbstractSerializingTestCase<NodeAttributes> {
+
+    public static NodeAttributes randomNodeAttributes() {
+        return new NodeAttributes(randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            randomBoolean() ? Collections.emptyMap() : Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10)));
+    }
+
+    @Override
+    protected NodeAttributes doParseInstance(XContentParser parser) throws IOException {
+        return NodeAttributes.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected NodeAttributes createTestInstance() {
+        return randomNodeAttributes();
+    }
+
+    @Override
+    protected Reader<NodeAttributes> instanceReader() {
+        return NodeAttributes::new;
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+
+    @Override
+    protected Predicate<String> getRandomFieldsExcludeFilter() {
+        return field -> field.equals("attributes");
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/TimeSyncConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/TimeSyncConfigTests.java
new file mode 100644
index 0000000000000..763e13e77aee0
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/TimeSyncConfigTests.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.dataframe.transforms;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig;
+
+import java.io.IOException;
+
+public class TimeSyncConfigTests extends AbstractSerializingTestCase<TimeSyncConfig> {
+
+    public static TimeSyncConfig randomTimeSyncConfig() {
+        return new TimeSyncConfig(randomAlphaOfLengthBetween(1, 10), new TimeValue(randomNonNegativeLong()));
+    }
+
+    @Override
+    protected TimeSyncConfig doParseInstance(XContentParser parser) throws IOException {
+        return TimeSyncConfig.fromXContent(parser, false);
+    }
+
+    @Override
+    protected TimeSyncConfig createTestInstance() {
+        return randomTimeSyncConfig();
+    }
+
+    @Override
+    protected Reader<TimeSyncConfig> instanceReader() {
+        return TimeSyncConfig::new;
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/DateHistogramGroupSourceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/DateHistogramGroupSourceTests.java
index 7ce0374331323..b28cf603030f5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/DateHistogramGroupSourceTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/pivot/DateHistogramGroupSourceTests.java
@@ -6,6 +6,9 @@
 
 package org.elasticsearch.xpack.core.dataframe.transforms.pivot;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
@@ -29,12 +32,22 @@ public static DateHistogramGroupSource randomDateHistogramGroupSource() {
         if (randomBoolean()) {
             dateHistogramGroupSource.setTimeZone(randomZone());
         }
-        if (randomBoolean()) {
-            dateHistogramGroupSource.setFormat(randomAlphaOfLength(10));
-        }
         return dateHistogramGroupSource;
     }
 
+    public void testBackwardsSerialization() throws IOException {
+        DateHistogramGroupSource groupSource = randomDateHistogramGroupSource();
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            output.setVersion(Version.V_7_2_0);
+            groupSource.writeTo(output);
+            try (StreamInput in = output.bytes().streamInput()) {
+                in.setVersion(Version.V_7_2_0);
+                DateHistogramGroupSource streamedGroupSource = new DateHistogramGroupSource(in);
+                assertEquals(groupSource, streamedGroupSource);
+            }
+        }
+    }
+
     @Override
     protected DateHistogramGroupSource doParseInstance(XContentParser parser) throws IOException {
         return DateHistogramGroupSource.fromXContent(parser, false);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/utils/DataFrameStringsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/utils/DataFrameStringsTests.java
new file mode 100644
index 0000000000000..e05de4d808464
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/utils/DataFrameStringsTests.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.dataframe.utils;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class DataFrameStringsTests extends ESTestCase {
+
+    public void testValidId() {
+        assertTrue(DataFrameStrings.isValidId("valid-_id"));
+    }
+
+    public void testValidId_givenUppercase() {
+        assertFalse(DataFrameStrings.isValidId("MiXedCase"));
+    }
+
+    public void testValidId_givenStartsWithUnderScore() {
+        assertFalse(DataFrameStrings.isValidId("_this_bit_is_ok"));
+    }
+
+    public void testKasValidLengthForId_givenTooLong() {
+        assertTrue(DataFrameStrings.hasValidLengthForId("#".repeat(DataFrameStrings.ID_LENGTH_LIMIT )));
+        assertFalse(DataFrameStrings.hasValidLengthForId("#".repeat(DataFrameStrings.ID_LENGTH_LIMIT + 1)));
+    }
+}
+
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java
index 4249d7c61d0ad..053e41d9b2a63 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java
@@ -235,10 +235,11 @@ public void testStateMachine() throws Exception {
             assertThat(indexer.getState(), equalTo(IndexerState.STARTED));
             assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis()));
             assertThat(indexer.getState(), equalTo(IndexerState.INDEXING));
+            assertTrue(awaitBusy(() -> indexer.getPosition() == 2));
             countDownLatch.countDown();
-
-            assertThat(indexer.getPosition(), equalTo(2));
             assertTrue(awaitBusy(() -> isFinished.get()));
+            assertThat(indexer.getPosition(), equalTo(3));
+
             assertFalse(isStopped.get());
             assertThat(indexer.getStep(), equalTo(6));
             assertThat(indexer.getStats().getNumInvocations(), equalTo(1L));
@@ -267,25 +268,6 @@ public void testStateMachineBrokenSearch() throws InterruptedException {
         }
     }
 
-    public void testStop_AfterIndexerIsFinished() throws InterruptedException {
-        AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
-        final ExecutorService executor = Executors.newFixedThreadPool(1);
-        try {
-            CountDownLatch countDownLatch = new CountDownLatch(1);
-            MockIndexer indexer = new MockIndexer(executor, state, 2, countDownLatch, false);
-            indexer.start();
-            assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis()));
-            countDownLatch.countDown();
-            assertTrue(awaitBusy(() -> isFinished.get()));
-
-            indexer.stop();
-            assertTrue(isStopped.get());
-            assertThat(indexer.getState(), equalTo(IndexerState.STOPPED));
-        } finally {
-            executor.shutdownNow();
-        }
-    }
-
     public void testStop_WhileIndexing() throws InterruptedException {
         AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
         final ExecutorService executor = Executors.newFixedThreadPool(1);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStepTests.java
index ae0551020fbd1..2a549c2a32812 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SegmentCountStepTests.java
@@ -138,7 +138,7 @@ public void onFailure(Exception e) {
         assertEquals(new SegmentCountStep.Info(0L), conditionInfo.get());
     }
 
-    public void testIsConditionFails() {
+    public void testIsConditionIsTrueEvenWhenMoreSegments() {
         int maxNumSegments = randomIntBetween(3, 10);
         Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
         Client client = Mockito.mock(Client.class);
@@ -191,8 +191,8 @@ public void onFailure(Exception e) {
             }
         });
 
-        assertFalse(conditionMetResult.get());
-        assertEquals(new SegmentCountStep.Info(1L), conditionInfo.get());
+        assertTrue(conditionMetResult.get());
+        assertEquals(new SegmentCountStep.Info(0L), conditionInfo.get());
     }
 
     public void testThrowsException() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SetSingleNodeAllocateStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SetSingleNodeAllocateStepTests.java
index 525744d68af10..d001e56e439a7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SetSingleNodeAllocateStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/SetSingleNodeAllocateStepTests.java
@@ -30,6 +30,7 @@
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.node.Node;
+import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.xpack.core.indexlifecycle.AsyncActionStep.Listener;
 import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
 import org.hamcrest.Matchers;
@@ -39,12 +40,18 @@
 import org.mockito.stubbing.Answer;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 public class SetSingleNodeAllocateStepTests extends AbstractStepTestCase<SetSingleNodeAllocateStep> {
 
@@ -96,13 +103,13 @@ public static void assertSettingsRequestContainsValueFrom(UpdateSettingsRequest
     }
 
     public void testPerformActionNoAttrs() throws IOException {
+        final int numNodes = randomIntBetween(1, 20);
         IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
-                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, numNodes - 1)).build();
         Index index = indexMetaData.getIndex();
         Set<String> validNodeIds = new HashSet<>();
         Settings validNodeSettings = Settings.EMPTY;
         DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
-        int numNodes = randomIntBetween(1, 20);
         for (int i = 0; i < numNodes; i++) {
             String nodeId = "node_id_" + i;
             String nodeName = "node_" + i;
@@ -118,6 +125,7 @@ public void testPerformActionNoAttrs() throws IOException {
 
     public void testPerformActionAttrsAllNodesValid() throws IOException {
         int numAttrs = randomIntBetween(1, 10);
+        final int numNodes = randomIntBetween(1, 20);
         String[][] validAttrs = new String[numAttrs][2];
         for (int i = 0; i < numAttrs; i++) {
             validAttrs[i] = new String[] { randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20) };
@@ -127,12 +135,11 @@ public void testPerformActionAttrsAllNodesValid() throws IOException {
             indexSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + attr[0], attr[1]);
         }
         IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(indexSettings)
-                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, numNodes - 1)).build();
         Index index = indexMetaData.getIndex();
         Set<String> validNodeIds = new HashSet<>();
         Settings validNodeSettings = Settings.EMPTY;
         DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
-        int numNodes = randomIntBetween(1, 20);
         for (int i = 0; i < numNodes; i++) {
             String nodeId = "node_id_" + i;
             String nodeName = "node_" + i;
@@ -148,18 +155,18 @@ public void testPerformActionAttrsAllNodesValid() throws IOException {
     }
 
     public void testPerformActionAttrsSomeNodesValid() throws IOException {
+        final int numNodes = randomIntBetween(1, 20);
         String[] validAttr = new String[] { "box_type", "valid" };
         String[] invalidAttr = new String[] { "box_type", "not_valid" };
         Settings.Builder indexSettings = settings(Version.CURRENT);
         indexSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + validAttr[0], validAttr[1]);
         IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(indexSettings)
-                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, numNodes - 1)).build();
         Index index = indexMetaData.getIndex();
         Set<String> validNodeIds = new HashSet<>();
         Settings validNodeSettings = Settings.builder().put(Node.NODE_ATTRIBUTES.getKey() + validAttr[0], validAttr[1]).build();
         Settings invalidNodeSettings = Settings.builder().put(Node.NODE_ATTRIBUTES.getKey() + invalidAttr[0], invalidAttr[1]).build();
         DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
-        int numNodes = randomIntBetween(1, 20);
         for (int i = 0; i < numNodes; i++) {
             String nodeId = "node_id_" + i;
             String nodeName = "node_" + i;
@@ -180,16 +187,16 @@ public void testPerformActionAttrsSomeNodesValid() throws IOException {
     }
 
     public void testPerformActionAttrsNoNodesValid() {
+        final int numNodes = randomIntBetween(1, 20);
         String[] validAttr = new String[] { "box_type", "valid" };
         String[] invalidAttr = new String[] { "box_type", "not_valid" };
         Settings.Builder indexSettings = settings(Version.CURRENT);
         indexSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + validAttr[0], validAttr[1]);
         IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(indexSettings)
-                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, numNodes - 1)).build();
         Index index = indexMetaData.getIndex();
         Settings invalidNodeSettings = Settings.builder().put(Node.NODE_ATTRIBUTES.getKey() + invalidAttr[0], invalidAttr[1]).build();
         DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
-        int numNodes = randomIntBetween(1, 20);
         for (int i = 0; i < numNodes; i++) {
             String nodeId = "node_id_" + i;
             String nodeName = "node_" + i;
@@ -203,39 +210,41 @@ public void testPerformActionAttrsNoNodesValid() {
     }
 
     public void testPerformActionAttrsRequestFails() {
+        final int numNodes = randomIntBetween(1, 20);
         int numAttrs = randomIntBetween(1, 10);
-        String[][] validAttrs = new String[numAttrs][2];
+        Map<String, String> validAttributes = new HashMap<>();
         for (int i = 0; i < numAttrs; i++) {
-            validAttrs[i] = new String[] { randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20) };
+            validAttributes.put(randomValueOtherThanMany(validAttributes::containsKey,
+                () -> randomAlphaOfLengthBetween(1,20)), randomAlphaOfLengthBetween(1,20));
         }
         Settings.Builder indexSettings = settings(Version.CURRENT);
-        for (String[] attr : validAttrs) {
-            indexSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + attr[0], attr[1]);
-        }
+        validAttributes.forEach((k, v) -> {
+            indexSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + k, v);
+
+        });
         IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(indexSettings)
-                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, numNodes - 1)).build();
         Index index = indexMetaData.getIndex();
         Set<String> validNodeIds = new HashSet<>();
         Settings validNodeSettings = Settings.EMPTY;
         DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
-        int numNodes = randomIntBetween(1, 20);
         for (int i = 0; i < numNodes; i++) {
             String nodeId = "node_id_" + i;
             String nodeName = "node_" + i;
             int nodePort = 9300 + i;
-            String[] nodeAttr = randomFrom(validAttrs);
+            Map.Entry<String, String> nodeAttr = randomFrom(validAttributes.entrySet());
             Settings nodeSettings = Settings.builder().put(validNodeSettings).put(Node.NODE_NAME_SETTING.getKey(), nodeName)
-                    .put(Node.NODE_ATTRIBUTES.getKey() + nodeAttr[0], nodeAttr[1]).build();
+                .put(Node.NODE_ATTRIBUTES.getKey() + nodeAttr.getKey(), nodeAttr.getValue()).build();
             nodes.add(DiscoveryNode.createLocal(nodeSettings, new TransportAddress(TransportAddress.META_ADDRESS, nodePort), nodeId));
             validNodeIds.add(nodeId);
         }
 
         ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder().fPut(index.getName(),
-                indexMetaData);
+            indexMetaData);
         IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
-                .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node_id_0", true, ShardRoutingState.STARTED));
+            .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node_id_0", true, ShardRoutingState.STARTED));
         ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metaData(MetaData.builder().indices(indices.build()))
-                .nodes(nodes).routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build();
+            .nodes(nodes).routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build();
 
         SetSingleNodeAllocateStep step = createRandomInstance();
         Exception exception = new RuntimeException();
@@ -253,8 +262,8 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
                 @SuppressWarnings("unchecked")
                 ActionListener<AcknowledgedResponse> listener = (ActionListener<AcknowledgedResponse>) invocation.getArguments()[1];
                 assertSettingsRequestContainsValueFrom(request,
-                        IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_id", validNodeIds, true,
-                        indexMetaData.getIndex().getName());
+                    IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_id", validNodeIds, true,
+                    indexMetaData.getIndex().getName());
                 listener.onFailure(exception);
                 return null;
             }
@@ -285,6 +294,7 @@ public void onFailure(Exception e) {
 
     public void testPerformActionAttrsNoShard() {
         int numAttrs = randomIntBetween(1, 10);
+        final int numNodes = randomIntBetween(1, 20);
         String[][] validAttrs = new String[numAttrs][2];
         for (int i = 0; i < numAttrs; i++) {
             validAttrs[i] = new String[] { randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20) };
@@ -294,12 +304,10 @@ public void testPerformActionAttrsNoShard() {
             indexSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + attr[0], attr[1]);
         }
         IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(indexSettings)
-                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+                .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, numNodes - 1)).build();
         Index index = indexMetaData.getIndex();
-        Set<String> validNodeIds = new HashSet<>();
         Settings validNodeSettings = Settings.EMPTY;
         DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
-        int numNodes = randomIntBetween(1, 20);
         for (int i = 0; i < numNodes; i++) {
             String nodeId = "node_id_" + i;
             String nodeName = "node_" + i;
@@ -308,7 +316,6 @@ public void testPerformActionAttrsNoShard() {
             Settings nodeSettings = Settings.builder().put(validNodeSettings).put(Node.NODE_NAME_SETTING.getKey(), nodeName)
                     .put(Node.NODE_ATTRIBUTES.getKey() + nodeAttr[0], nodeAttr[1]).build();
             nodes.add(DiscoveryNode.createLocal(nodeSettings, new TransportAddress(TransportAddress.META_ADDRESS, nodePort), nodeId));
-            validNodeIds.add(nodeId);
         }
 
         ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder().fPut(index.getName(),
@@ -340,14 +347,193 @@ public void onFailure(Exception e) {
         Mockito.verifyZeroInteractions(client);
     }
 
+    public void testPerformActionSomeShardsOnlyOnNewNodes() {
+        final Version oldVersion = VersionUtils.randomPreviousCompatibleVersion(random(), Version.CURRENT);
+        final int numNodes = randomIntBetween(2, 20); // Need at least 2 nodes to have some nodes on a new version
+        final int numNewNodes = randomIntBetween(1, numNodes - 1);
+        final int numOldNodes = numNodes - numNewNodes;
+
+        final int numberOfShards = randomIntBetween(1, 5);
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(oldVersion))
+            .numberOfShards(numberOfShards).numberOfReplicas(randomIntBetween(0, numNewNodes - 1)).build();
+        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
+
+        Set<String> newNodeIds = new HashSet<>();
+        for (int i = 0; i < numNewNodes; i++) {
+            String nodeId = "new_node_id_" + i;
+            String nodeName = "new_node_" + i;
+            int nodePort = 9300 + i;
+            Settings nodeSettings = Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), nodeName).build();
+            newNodeIds.add(nodeId);
+            nodes.add(new DiscoveryNode(
+                Node.NODE_NAME_SETTING.get(nodeSettings),
+                nodeId,
+                new TransportAddress(TransportAddress.META_ADDRESS, nodePort),
+                Node.NODE_ATTRIBUTES.getAsMap(nodeSettings),
+                DiscoveryNode.getRolesFromSettings(nodeSettings),
+                Version.CURRENT));
+        }
+
+        Set<String> oldNodeIds = new HashSet<>();
+        for (int i = 0; i < numOldNodes; i++) {
+            String nodeId = "old_node_id_" + i;
+            String nodeName = "old_node_" + i;
+            int nodePort = 9300 + numNewNodes + i;
+            Settings nodeSettings = Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), nodeName).build();
+            oldNodeIds.add(nodeId);
+            nodes.add(new DiscoveryNode(
+                Node.NODE_NAME_SETTING.get(nodeSettings),
+                nodeId,
+                new TransportAddress(TransportAddress.META_ADDRESS, nodePort),
+                Node.NODE_ATTRIBUTES.getAsMap(nodeSettings),
+                DiscoveryNode.getRolesFromSettings(nodeSettings),
+                oldVersion));
+        }
+
+        Set<String> nodeIds = new HashSet<>();
+        nodeIds.addAll(newNodeIds);
+        nodeIds.addAll(oldNodeIds);
+
+        DiscoveryNodes discoveryNodes = nodes.build();
+        IndexRoutingTable.Builder indexRoutingTable = createRoutingTableWithOneShardOnSubset(indexMetaData, newNodeIds, nodeIds);
+
+        // Since one shard is already on only new nodes, we should always pick a new node
+        assertNodeSelected(indexMetaData, indexMetaData.getIndex(), newNodeIds, discoveryNodes, indexRoutingTable.build());
+    }
+
+    public void testPerformActionSomeShardsOnlyOnNewNodesButNewNodesInvalidAttrs() {
+        final Version oldVersion = VersionUtils.randomPreviousCompatibleVersion(random(), Version.CURRENT);
+        final int numNodes = randomIntBetween(2, 20); // Need at least 2 nodes to have some nodes on a new version
+        final int numNewNodes = randomIntBetween(1, numNodes - 1);
+        final int numOldNodes = numNodes - numNewNodes;
+        final int numberOfShards = randomIntBetween(1, 5);
+        final String attribute = "box_type";
+        final String validAttr = "valid";
+        final String invalidAttr = "not_valid";
+        Settings.Builder indexSettings = settings(oldVersion);
+        indexSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + attribute, validAttr);
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(indexSettings)
+            .numberOfShards(numberOfShards).numberOfReplicas(randomIntBetween(0, numNewNodes - 1)).build();
+        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
+
+        Set<String> newNodeIds = new HashSet<>();
+        for (int i = 0; i < numNewNodes; i++) {
+            String nodeId = "new_node_id_" + i;
+            String nodeName = "new_node_" + i;
+            int nodePort = 9300 + i;
+            Settings nodeSettings = Settings.builder()
+                .put(Node.NODE_NAME_SETTING.getKey(), nodeName)
+                .put(Node.NODE_ATTRIBUTES.getKey() + attribute, invalidAttr).build();
+            newNodeIds.add(nodeId);
+            nodes.add(new DiscoveryNode(
+                Node.NODE_NAME_SETTING.get(nodeSettings),
+                nodeId,
+                new TransportAddress(TransportAddress.META_ADDRESS, nodePort),
+                Node.NODE_ATTRIBUTES.getAsMap(nodeSettings),
+                DiscoveryNode.getRolesFromSettings(nodeSettings),
+                Version.CURRENT));
+        }
+
+        Set<String> oldNodeIds = new HashSet<>();
+        for (int i = 0; i < numOldNodes; i++) {
+            String nodeId = "old_node_id_" + i;
+            String nodeName = "old_node_" + i;
+            int nodePort = 9300 + numNewNodes + i;
+            Settings nodeSettings = Settings.builder()
+                .put(Node.NODE_NAME_SETTING.getKey(), nodeName)
+                .put(Node.NODE_ATTRIBUTES.getKey() + attribute, validAttr).build();
+            oldNodeIds.add(nodeId);
+            nodes.add(new DiscoveryNode(
+                Node.NODE_NAME_SETTING.get(nodeSettings),
+                nodeId,
+                new TransportAddress(TransportAddress.META_ADDRESS, nodePort),
+                Node.NODE_ATTRIBUTES.getAsMap(nodeSettings),
+                DiscoveryNode.getRolesFromSettings(nodeSettings),
+                oldVersion));
+        }
+        Set<String> nodeIds = new HashSet<>();
+        nodeIds.addAll(newNodeIds);
+        nodeIds.addAll(oldNodeIds);
+
+        DiscoveryNodes discoveryNodes = nodes.build();
+        IndexRoutingTable.Builder indexRoutingTable = createRoutingTableWithOneShardOnSubset(indexMetaData, newNodeIds, nodeIds);
+
+        assertNoValidNode(indexMetaData, indexMetaData.getIndex(), discoveryNodes, indexRoutingTable.build());
+    }
+
+    public void testPerformActionNewShardsExistButWithInvalidAttributes() {
+        final Version oldVersion = VersionUtils.randomPreviousCompatibleVersion(random(), Version.CURRENT);
+        final int numNodes = randomIntBetween(2, 20); // Need at least 2 nodes to have some nodes on a new version
+        final int numNewNodes = randomIntBetween(1, numNodes - 1);
+        final int numOldNodes = numNodes - numNewNodes;
+        final int numberOfShards = randomIntBetween(1, 5);
+        final String attribute = "box_type";
+        final String validAttr = "valid";
+        final String invalidAttr = "not_valid";
+        Settings.Builder indexSettings = settings(oldVersion);
+        indexSettings.put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + attribute, validAttr);
+        IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(indexSettings)
+            .numberOfShards(numberOfShards).numberOfReplicas(randomIntBetween(0, numOldNodes - 1)).build();
+        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
+
+        Set<String> newNodeIds = new HashSet<>();
+        for (int i = 0; i < numNewNodes; i++) {
+            String nodeId = "new_node_id_" + i;
+            String nodeName = "new_node_" + i;
+            int nodePort = 9300 + i;
+            Settings nodeSettings = Settings.builder()
+                .put(Node.NODE_NAME_SETTING.getKey(), nodeName)
+                .put(Node.NODE_ATTRIBUTES.getKey() + attribute, invalidAttr).build();
+            newNodeIds.add(nodeId);
+            nodes.add(new DiscoveryNode(
+                Node.NODE_NAME_SETTING.get(nodeSettings),
+                nodeId,
+                new TransportAddress(TransportAddress.META_ADDRESS, nodePort),
+                Node.NODE_ATTRIBUTES.getAsMap(nodeSettings),
+                DiscoveryNode.getRolesFromSettings(nodeSettings),
+                Version.CURRENT));
+        }
+
+        Set<String> oldNodeIds = new HashSet<>();
+        for (int i = 0; i < numOldNodes; i++) {
+            String nodeId = "old_node_id_" + i;
+            String nodeName = "old_node_" + i;
+            int nodePort = 9300 + numNewNodes + i;
+            Settings nodeSettings = Settings.builder()
+                .put(Node.NODE_NAME_SETTING.getKey(), nodeName)
+                .put(Node.NODE_ATTRIBUTES.getKey() + attribute, validAttr).build();
+            oldNodeIds.add(nodeId);
+            nodes.add(new DiscoveryNode(
+                Node.NODE_NAME_SETTING.get(nodeSettings),
+                nodeId,
+                new TransportAddress(TransportAddress.META_ADDRESS, nodePort),
+                Node.NODE_ATTRIBUTES.getAsMap(nodeSettings),
+                DiscoveryNode.getRolesFromSettings(nodeSettings),
+                oldVersion));
+        }
+        Set<String> nodeIds = new HashSet<>();
+        nodeIds.addAll(newNodeIds);
+        nodeIds.addAll(oldNodeIds);
+
+        DiscoveryNodes discoveryNodes = nodes.build();
+        IndexRoutingTable.Builder indexRoutingTable = createRoutingTableWithOneShardOnSubset(indexMetaData, oldNodeIds, oldNodeIds);
+
+        assertNodeSelected(indexMetaData, indexMetaData.getIndex(), oldNodeIds, discoveryNodes, indexRoutingTable.build());
+    }
+
     private void assertNodeSelected(IndexMetaData indexMetaData, Index index,
                                     Set<String> validNodeIds, DiscoveryNodes.Builder nodes) throws IOException {
+        DiscoveryNodes discoveryNodes = nodes.build();
+        IndexRoutingTable.Builder indexRoutingTable = createRoutingTable(indexMetaData, index, discoveryNodes);
+        assertNodeSelected(indexMetaData, index, validNodeIds, discoveryNodes, indexRoutingTable.build());
+    }
+
+    private void assertNodeSelected(IndexMetaData indexMetaData, Index index, Set<String> validNodeIds, DiscoveryNodes nodes,
+                                    IndexRoutingTable indexRoutingTable) {
         ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder().fPut(index.getName(),
-                indexMetaData);
-        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
-                .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node_id_0", true, ShardRoutingState.STARTED));
+            indexMetaData);
         ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metaData(MetaData.builder().indices(indices.build()))
-                .nodes(nodes).routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build();
+            .nodes(nodes).routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build();
 
         SetSingleNodeAllocateStep step = createRandomInstance();
 
@@ -365,8 +551,8 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
                 @SuppressWarnings("unchecked")
                 ActionListener<AcknowledgedResponse> listener = (ActionListener<AcknowledgedResponse>) invocation.getArguments()[1];
                 assertSettingsRequestContainsValueFrom(request,
-                        IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_id", validNodeIds, true,
-                        indexMetaData.getIndex().getName());
+                    IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_id", validNodeIds, true,
+                    indexMetaData.getIndex().getName());
                 listener.onResponse(new AcknowledgedResponse(true));
                 return null;
             }
@@ -396,12 +582,18 @@ public void onFailure(Exception e) {
     }
 
     private void assertNoValidNode(IndexMetaData indexMetaData, Index index, DiscoveryNodes.Builder nodes) {
-        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder().fPut(index.getName(),
-                indexMetaData);
-        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index)
-                .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), "node_id_0", true, ShardRoutingState.STARTED));
+        DiscoveryNodes discoveryNodes = nodes.build();
+        IndexRoutingTable.Builder indexRoutingTable = createRoutingTable(indexMetaData, index, discoveryNodes);
+
+        assertNoValidNode(indexMetaData, index, discoveryNodes, indexRoutingTable.build());
+    }
+
+    private void assertNoValidNode(IndexMetaData indexMetaData, Index index, DiscoveryNodes nodes, IndexRoutingTable indexRoutingTable) {
+
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData>builder().fPut(index.getName(),
+            indexMetaData);
         ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metaData(MetaData.builder().indices(indices.build()))
-                .nodes(nodes).routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build();
+            .nodes(nodes).routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build();
 
         SetSingleNodeAllocateStep step = createRandomInstance();
 
@@ -425,4 +617,57 @@ public void onFailure(Exception e) {
         Mockito.verifyZeroInteractions(client);
     }
 
+    private IndexRoutingTable.Builder createRoutingTable(IndexMetaData indexMetaData, Index index, DiscoveryNodes discoveryNodes) {
+        assertThat(indexMetaData.getNumberOfReplicas(), lessThanOrEqualTo(discoveryNodes.getSize() - 1));
+        List<String> nodeIds = new ArrayList<>();
+        for (DiscoveryNode node : discoveryNodes) {
+            nodeIds.add(node.getId());
+        }
+
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index);
+        for (int primary = 0; primary < indexMetaData.getNumberOfShards(); primary++) {
+            Set<String> nodesThisShardCanBePutOn = new HashSet<>(nodeIds);
+            String currentNode = randomFrom(nodesThisShardCanBePutOn);
+            nodesThisShardCanBePutOn.remove(currentNode);
+            indexRoutingTable.addShard(TestShardRouting.newShardRouting(new ShardId(index, primary), currentNode,
+                true, ShardRoutingState.STARTED));
+            for (int replica = 0; replica < indexMetaData.getNumberOfReplicas(); replica++) {
+                assertThat("not enough nodes to allocate all initial shards", nodesThisShardCanBePutOn.size(), greaterThan(0));
+                String replicaNode = randomFrom(nodesThisShardCanBePutOn);
+                nodesThisShardCanBePutOn.remove(replicaNode);
+                indexRoutingTable.addShard(TestShardRouting.newShardRouting(new ShardId(index, primary), replicaNode,
+                    false, ShardRoutingState.STARTED));
+            }
+        }
+        return indexRoutingTable;
+    }
+
+    private IndexRoutingTable.Builder createRoutingTableWithOneShardOnSubset(IndexMetaData indexMetaData, Set<String> subset,
+                                                                             Set<String> allNodeIds) {
+        IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(indexMetaData.getIndex());
+        final int numberOfShards = indexMetaData.getNumberOfShards();
+        final int shardOnlyOnNewNodes = randomIntBetween(0, numberOfShards - 1);
+        for (int primary = 0; primary < indexMetaData.getNumberOfShards(); primary++) {
+            Set<String> nodesThisShardCanBePutOn;
+            if (primary == shardOnlyOnNewNodes) {
+                // This shard should only be allocated to new nodes
+                nodesThisShardCanBePutOn = new HashSet<>(subset);
+            } else {
+                nodesThisShardCanBePutOn = new HashSet<>(allNodeIds);
+            }
+            String currentNode = randomFrom(nodesThisShardCanBePutOn);
+            nodesThisShardCanBePutOn.remove(currentNode);
+            indexRoutingTable.addShard(TestShardRouting.newShardRouting(new ShardId(indexMetaData.getIndex(), primary), currentNode,
+                true, ShardRoutingState.STARTED));
+            for (int replica = 0; replica < indexMetaData.getNumberOfReplicas(); replica++) {
+                assertThat("not enough nodes to allocate all initial shards", nodesThisShardCanBePutOn.size(), greaterThan(0));
+                String replicaNode = randomFrom(nodesThisShardCanBePutOn);
+                nodesThisShardCanBePutOn.remove(replicaNode);
+                indexRoutingTable.addShard(TestShardRouting.newShardRouting(new ShardId(indexMetaData.getIndex(), primary), replicaNode,
+                    false, ShardRoutingState.STARTED));
+            }
+        }
+        return indexRoutingTable;
+    }
+
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepTests.java
index a0ee01a240347..fe873066dc594 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/WaitForFollowShardTasksStepTests.java
@@ -188,6 +188,7 @@ private static ShardFollowNodeTaskStatus createShardFollowTaskStatus(int shardId
             0,
             0,
             0,
+            0,
             Collections.emptyNavigableMap(),
             0,
             null
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleResponseTests.java
index b671f72cf99c4..977469401d947 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/DeleteLifecycleResponseTests.java
@@ -5,10 +5,11 @@
  */
 package org.elasticsearch.xpack.core.indexlifecycle.action;
 
-import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction.Response;
 
-public class DeleteLifecycleResponseTests extends AbstractStreamableTestCase<DeleteLifecycleAction.Response> {
+public class DeleteLifecycleResponseTests extends AbstractWireSerializingTestCase<Response> {
 
     @Override
     protected Response createTestInstance() {
@@ -16,8 +17,8 @@ protected Response createTestInstance() {
     }
 
     @Override
-    protected Response createBlankInstance() {
-        return new Response();
+    protected Writeable.Reader<Response> instanceReader() {
+        return Response::new;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepResponseTests.java
index 7f9bacd35ac25..3cb47a94a3ad9 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/MoveToStepResponseTests.java
@@ -6,10 +6,11 @@
  */
 package org.elasticsearch.xpack.core.indexlifecycle.action;
 
-import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.indexlifecycle.action.MoveToStepAction.Response;
 
-public class MoveToStepResponseTests extends AbstractStreamableTestCase<Response> {
+public class MoveToStepResponseTests extends AbstractWireSerializingTestCase<Response> {
 
     @Override
     protected Response createTestInstance() {
@@ -17,8 +18,8 @@ protected Response createTestInstance() {
     }
 
     @Override
-    protected Response createBlankInstance() {
-        return new Response();
+    protected Writeable.Reader<Response> instanceReader() {
+        return Response::new;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleResponseTests.java
index 0c9acb2aa5dd2..713a87459d776 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/PutLifecycleResponseTests.java
@@ -5,10 +5,11 @@
  */
 package org.elasticsearch.xpack.core.indexlifecycle.action;
 
-import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction.Response;
 
-public class PutLifecycleResponseTests extends AbstractStreamableTestCase<PutLifecycleAction.Response> {
+public class PutLifecycleResponseTests extends AbstractWireSerializingTestCase<Response> {
 
     @Override
     protected Response createTestInstance() {
@@ -16,8 +17,8 @@ protected Response createTestInstance() {
     }
 
     @Override
-    protected Response createBlankInstance() {
-        return new Response();
+    protected Writeable.Reader<Response> instanceReader() {
+        return Response::new;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryResponseTests.java
index 24e758f9503c1..aa05f816fb531 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/action/RetryResponseTests.java
@@ -6,10 +6,11 @@
  */
 package org.elasticsearch.xpack.core.indexlifecycle.action;
 
-import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.xpack.core.indexlifecycle.action.RetryAction.Response;
 
-public class RetryResponseTests extends AbstractStreamableTestCase<Response> {
+public class RetryResponseTests extends AbstractWireSerializingTestCase<Response> {
 
     @Override
     protected Response createTestInstance() {
@@ -17,8 +18,8 @@ protected Response createTestInstance() {
     }
 
     @Override
-    protected Response createBlankInstance() {
-        return new Response();
+    protected Writeable.Reader<Response> instanceReader() {
+        return Response::new;
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java
index 3afe76b8b171f..f2015b1a2bbb5 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/MlTasksTests.java
@@ -22,6 +22,7 @@
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
 
 public class MlTasksTests extends ESTestCase {
     public void testGetJobState() {
@@ -161,4 +162,10 @@ public void testUnallocatedDatafeedIds() {
         assertThat(MlTasks.unallocatedDatafeedIds(tasksBuilder.build(), nodes),
                 containsInAnyOrder("datafeed_without_assignment", "datafeed_without_node"));
     }
+
+    public void testDataFrameAnalyticsTaskIds() {
+        String taskId = MlTasks.dataFrameAnalyticsTaskId("foo");
+        assertThat(taskId, equalTo("data_frame_analytics-foo"));
+        assertThat(MlTasks.dataFrameAnalyticsIdFromTaskId(taskId), equalTo("foo"));
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/EvaluateDataFrameActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/EvaluateDataFrameActionRequestTests.java
new file mode 100644
index 0000000000000..e899b7e6642da
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/EvaluateDataFrameActionRequestTests.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction.Request;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification.BinarySoftClassificationTests;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class EvaluateDataFrameActionRequestTests extends AbstractStreamableXContentTestCase<Request> {
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(new MlEvaluationNamedXContentProvider().getNamedWriteables());
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(new MlEvaluationNamedXContentProvider().getNamedXContentParsers());
+    }
+
+    @Override
+    protected Request createTestInstance() {
+        Request request = new Request();
+        int indicesCount = randomIntBetween(1, 5);
+        List<String> indices = new ArrayList<>(indicesCount);
+        for (int i = 0; i < indicesCount; i++) {
+            indices.add(randomAlphaOfLength(10));
+        }
+        request.setIndices(indices);
+        request.setEvaluation(BinarySoftClassificationTests.createRandom());
+        return request;
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected Request createBlankInstance() {
+        return new Request();
+    }
+
+    @Override
+    protected Request doParseInstance(XContentParser parser) {
+        return Request.parseRequest(parser);
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/FindFileStructureActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/FindFileStructureActionRequestTests.java
index ba69e6f750c96..e72b054015885 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/FindFileStructureActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/FindFileStructureActionRequestTests.java
@@ -26,6 +26,10 @@ protected FindFileStructureAction.Request createTestInstance() {
             request.setLinesToSample(randomIntBetween(10, 2000));
         }
 
+        if (randomBoolean()) {
+            request.setLineMergeSizeLimit(randomIntBetween(1000, 20000));
+        }
+
         if (randomBoolean()) {
             request.setCharset(randomAlphaOfLength(10));
         }
@@ -85,6 +89,18 @@ public void testValidateLinesToSample() {
         assertThat(e.getMessage(), containsString(" [lines_to_sample] must be positive if specified"));
     }
 
+    public void testValidateLineMergeSizeLimit() {
+
+        FindFileStructureAction.Request request = new FindFileStructureAction.Request();
+        request.setLineMergeSizeLimit(randomIntBetween(-1, 0));
+        request.setSample(new BytesArray("foo\n"));
+
+        ActionRequestValidationException e = request.validate();
+        assertNotNull(e);
+        assertThat(e.getMessage(), startsWith("Validation Failed: "));
+        assertThat(e.getMessage(), containsString(" [line_merge_size_limit] must be positive if specified"));
+    }
+
     public void testValidateNonDelimited() {
 
         FindFileStructureAction.Request request = new FindFileStructureAction.Request();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsActionResponseTests.java
new file mode 100644
index 0000000000000..8a7b6717abd92
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsActionResponseTests.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.xpack.core.action.util.QueryPage;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction.Response;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfigTests;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.MlDataFrameAnalysisNamedXContentProvider;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class GetDataFrameAnalyticsActionResponseTests extends AbstractStreamableTestCase<Response> {
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
+        namedWriteables.addAll(new MlDataFrameAnalysisNamedXContentProvider().getNamedWriteables());
+        namedWriteables.addAll(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedWriteables());
+        return new NamedWriteableRegistry(namedWriteables);
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
+        namedXContent.addAll(new MlDataFrameAnalysisNamedXContentProvider().getNamedXContentParsers());
+        namedXContent.addAll(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents());
+        return new NamedXContentRegistry(namedXContent);
+    }
+
+    @Override
+    protected Response createTestInstance() {
+        int listSize = randomInt(10);
+        List<DataFrameAnalyticsConfig> analytics = new ArrayList<>(listSize);
+        for (int j = 0; j < listSize; j++) {
+            analytics.add(DataFrameAnalyticsConfigTests.createRandom(DataFrameAnalyticsConfigTests.randomValidId()));
+        }
+        return new Response(new QueryPage<>(analytics, analytics.size(), Response.RESULTS_FIELD));
+    }
+
+    @Override
+    protected Response createBlankInstance() {
+        return new Response();
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsRequestTests.java
new file mode 100644
index 0000000000000..438474076c910
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsRequestTests.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.action.util.PageParams;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction.Request;
+
+public class GetDataFrameAnalyticsRequestTests extends AbstractWireSerializingTestCase<Request> {
+
+    @Override
+    protected Request createTestInstance() {
+        Request request = new Request();
+        request.setResourceId(randomAlphaOfLength(20));
+        request.setPageParams(new PageParams(randomIntBetween(0, 100), randomIntBetween(0, 100)));
+        return request;
+    }
+
+    @Override
+    protected Writeable.Reader<Request> instanceReader() {
+        return Request::new;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java
new file mode 100644
index 0000000000000..5a88f2ea52eab
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.action.util.QueryPage;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction.Response;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfigTests;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class GetDataFrameAnalyticsStatsActionResponseTests extends AbstractWireSerializingTestCase<Response> {
+
+    @Override
+    protected Response createTestInstance() {
+        int listSize = randomInt(10);
+        List<Response.Stats> analytics = new ArrayList<>(listSize);
+        for (int j = 0; j < listSize; j++) {
+            Integer progressPercentage = randomBoolean() ? null : randomIntBetween(0, 100);
+            String failureReason = randomBoolean() ? null : randomAlphaOfLength(10);
+            Response.Stats stats = new Response.Stats(DataFrameAnalyticsConfigTests.randomValidId(),
+                randomFrom(DataFrameAnalyticsState.values()), failureReason, progressPercentage, null, randomAlphaOfLength(20));
+            analytics.add(stats);
+        }
+        return new Response(new QueryPage<>(analytics, analytics.size(), GetDataFrameAnalyticsAction.Response.RESULTS_FIELD));
+    }
+
+    @Override
+    protected Writeable.Reader<Response> instanceReader() {
+        return Response::new;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsRequestTests.java
new file mode 100644
index 0000000000000..918d04873ef2c
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsRequestTests.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.action.util.PageParams;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction.Request;
+
+public class GetDataFrameAnalyticsStatsRequestTests extends AbstractWireSerializingTestCase<Request> {
+
+    @Override
+    protected Request createTestInstance() {
+        Request request = new Request(randomAlphaOfLength(20));
+        request.setPageParams(new PageParams(randomIntBetween(0, 100), randomIntBetween(0, 100)));
+        return request;
+    }
+
+    @Override
+    protected Writeable.Reader<Request> instanceReader() {
+        return Request::new;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionResponseTests.java
index eb40d6894f1e0..a44e1e53435c4 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionResponseTests.java
@@ -14,18 +14,18 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.AbstractStreamableTestCase;
-import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction.Response;
 import org.elasticsearch.xpack.core.action.util.QueryPage;
+import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction.Response;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
 
 import java.io.IOException;
 import java.net.InetAddress;
 import java.util.ArrayList;
-import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
@@ -75,7 +75,7 @@ public void testDatafeedStatsToXContent() throws IOException {
         TransportAddress transportAddress = new TransportAddress(TransportAddress.META_ADDRESS, 9000);
 
         DiscoveryNode node = new DiscoveryNode("df-node-name", "df-node-id", transportAddress, attributes,
-                EnumSet.noneOf(DiscoveryNode.Role.class),
+                Set.of(),
                 Version.CURRENT);
 
         Response.DatafeedStats stats = new Response.DatafeedStats("df-id", DatafeedState.STARTED, node, null);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsActionResponseTests.java
index 383a4a7d62e70..33bc6228b1293 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsActionResponseTests.java
@@ -37,7 +37,7 @@ protected Response createBlankInstance() {
 
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedWriteableRegistry(searchModule.getNamedWriteables());
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobStatsActionResponseTests.java
index 78513130bf25f..c0122fbe19033 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobStatsActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetJobStatsActionResponseTests.java
@@ -18,6 +18,8 @@
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCountsTests;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStatsTests;
 import org.elasticsearch.xpack.core.ml.stats.ForecastStats;
 import org.elasticsearch.xpack.core.ml.stats.ForecastStatsTests;
 
@@ -38,35 +40,19 @@ protected Response createTestInstance() {
         List<Response.JobStats> jobStatsList = new ArrayList<>(listSize);
         for (int j = 0; j < listSize; j++) {
             String jobId = randomAlphaOfLength(10);
-
             DataCounts dataCounts = new DataCountsTests().createTestInstance();
-
-            ModelSizeStats sizeStats = null;
-            if (randomBoolean()) {
-                sizeStats = new ModelSizeStats.Builder("foo").build();
-            }
-
-            ForecastStats forecastStats = null;
-            if (randomBoolean()) {
-                forecastStats = new ForecastStatsTests().createTestInstance();
-            }
-
+            ModelSizeStats sizeStats = randomBoolean() ? null : new ModelSizeStats.Builder("foo").build();
+            ForecastStats forecastStats = randomBoolean() ? null : new ForecastStatsTests().createTestInstance();
             JobState jobState = randomFrom(EnumSet.allOf(JobState.class));
-
-            DiscoveryNode node = null;
-            if (randomBoolean()) {
-                node = new DiscoveryNode("_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), Version.CURRENT);
-            }
-            String explanation = null;
-            if (randomBoolean()) {
-                explanation = randomAlphaOfLength(3);
-            }
-            TimeValue openTime = null;
-            if (randomBoolean()) {
-                openTime = parseTimeValue(randomPositiveTimeValue(), "open_time-Test");
-            }
-            Response.JobStats jobStats = new Response.JobStats(jobId, dataCounts, sizeStats, forecastStats, jobState, node, explanation,
-                    openTime);
+            DiscoveryNode node =
+                randomBoolean()
+                    ? null
+                    : new DiscoveryNode("_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), Version.CURRENT);
+            String explanation = randomBoolean() ? null : randomAlphaOfLength(3);
+            TimeValue openTime = randomBoolean() ? null : parseTimeValue(randomPositiveTimeValue(), "open_time-Test");
+            TimingStats timingStats = randomBoolean() ? null : TimingStatsTests.createTestInstance("foo");
+            Response.JobStats jobStats =
+                new Response.JobStats(jobId, dataCounts, sizeStats, forecastStats, jobState, node, explanation, openTime, timingStats);
             jobStatsList.add(jobStats);
         }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionRequestTests.java
new file mode 100644
index 0000000000000..1e5416d5a5dce
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionRequestTests.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.AbstractStreamableXContentTestCase;
+import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction.Request;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfigTests;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.MlDataFrameAnalysisNamedXContentProvider;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class PutDataFrameAnalyticsActionRequestTests extends AbstractStreamableXContentTestCase<Request> {
+
+    private String id;
+
+    @Before
+    public void setUpId() {
+        id = DataFrameAnalyticsConfigTests.randomValidId();
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
+        namedWriteables.addAll(new MlDataFrameAnalysisNamedXContentProvider().getNamedWriteables());
+        namedWriteables.addAll(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedWriteables());
+        return new NamedWriteableRegistry(namedWriteables);
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
+        namedXContent.addAll(new MlDataFrameAnalysisNamedXContentProvider().getNamedXContentParsers());
+        namedXContent.addAll(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents());
+        return new NamedXContentRegistry(namedXContent);
+    }
+
+    @Override
+    protected Request createTestInstance() {
+        return new Request(DataFrameAnalyticsConfigTests.createRandom(id));
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+
+    @Override
+    protected Request createBlankInstance() {
+        return new Request();
+    }
+
+    @Override
+    protected Request doParseInstance(XContentParser parser) {
+        return Request.parseRequest(id, parser);
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionResponseTests.java
new file mode 100644
index 0000000000000..d323505828e42
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionResponseTests.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.AbstractStreamableTestCase;
+import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction.Response;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfigTests;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.MlDataFrameAnalysisNamedXContentProvider;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class PutDataFrameAnalyticsActionResponseTests extends AbstractStreamableTestCase<Response> {
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
+        namedWriteables.addAll(new MlDataFrameAnalysisNamedXContentProvider().getNamedWriteables());
+        namedWriteables.addAll(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedWriteables());
+        return new NamedWriteableRegistry(namedWriteables);
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
+        namedXContent.addAll(new MlDataFrameAnalysisNamedXContentProvider().getNamedXContentParsers());
+        namedXContent.addAll(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents());
+        return new NamedXContentRegistry(namedXContent);
+    }
+
+    @Override
+    protected Response createTestInstance() {
+        return new Response(DataFrameAnalyticsConfigTests.createRandom(DataFrameAnalyticsConfigTests.randomValidId()));
+    }
+
+    @Override
+    protected Response createBlankInstance() {
+        return new Response();
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionRequestTests.java
index d688ee9b5b90d..8ba2f1bb161ea 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionRequestTests.java
@@ -51,13 +51,13 @@ protected Request doParseInstance(XContentParser parser) {
 
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedWriteableRegistry(searchModule.getNamedWriteables());
     }
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionResponseTests.java
index ca650914fe146..619770e8990ab 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionResponseTests.java
@@ -33,7 +33,7 @@ protected Response createBlankInstance() {
 
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedWriteableRegistry(searchModule.getNamedWriteables());
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsActionTaskParamsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsActionTaskParamsTests.java
new file mode 100644
index 0000000000000..4af9a2a374c5d
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsActionTaskParamsTests.java
@@ -0,0 +1,37 @@
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+
+import java.io.IOException;
+
+public class StartDataFrameAnalyticsActionTaskParamsTests extends AbstractSerializingTestCase<StartDataFrameAnalyticsAction.TaskParams> {
+
+    @Override
+    protected StartDataFrameAnalyticsAction.TaskParams doParseInstance(XContentParser parser) throws IOException {
+        return StartDataFrameAnalyticsAction.TaskParams.fromXContent(parser);
+    }
+
+    @Override
+    protected StartDataFrameAnalyticsAction.TaskParams createTestInstance() {
+        return new StartDataFrameAnalyticsAction.TaskParams(randomAlphaOfLength(10), Version.CURRENT);
+    }
+
+    @Override
+    protected Writeable.Reader<StartDataFrameAnalyticsAction.TaskParams> instanceReader() {
+        return StartDataFrameAnalyticsAction.TaskParams::new;
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsRequestTests.java
new file mode 100644
index 0000000000000..a3db5833b820d
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsRequestTests.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction.Request;
+
+public class StartDataFrameAnalyticsRequestTests extends AbstractWireSerializingTestCase<Request> {
+
+    @Override
+    protected Request createTestInstance() {
+        Request request = new Request(randomAlphaOfLength(20));
+        if (randomBoolean()) {
+            request.setTimeout(TimeValue.timeValueMillis(randomNonNegativeLong()));
+        }
+        return request;
+    }
+
+    @Override
+    protected Writeable.Reader<Request> instanceReader() {
+        return Request::new;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsActionResponseTests.java
new file mode 100644
index 0000000000000..d06d8cb1a1860
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsActionResponseTests.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.ml.action.StopDataFrameAnalyticsAction.Response;
+
+public class StopDataFrameAnalyticsActionResponseTests extends AbstractWireSerializingTestCase<Response> {
+
+    @Override
+    protected Response createTestInstance() {
+        return new Response(randomBoolean());
+    }
+
+    @Override
+    protected Writeable.Reader<Response> instanceReader() {
+        return Response::new;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsRequestTests.java
new file mode 100644
index 0000000000000..f2942905fadf0
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StopDataFrameAnalyticsRequestTests.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.action;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.ml.action.StopDataFrameAnalyticsAction.Request;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class StopDataFrameAnalyticsRequestTests extends AbstractWireSerializingTestCase<Request> {
+
+    @Override
+    protected Request createTestInstance() {
+        Request request = new Request(randomAlphaOfLength(20));
+        if (randomBoolean()) {
+            request.setTimeout(TimeValue.timeValueMillis(randomNonNegativeLong()));
+        }
+        if (randomBoolean()) {
+            request.setAllowNoMatch(randomBoolean());
+        }
+        if (randomBoolean()) {
+            request.setForce(randomBoolean());
+        }
+        int expandedIdsCount = randomIntBetween(0, 10);
+        Set<String> expandedIds = new HashSet<>();
+        for (int i = 0; i < expandedIdsCount; i++) {
+            expandedIds.add(randomAlphaOfLength(20));
+        }
+        request.setExpandedIds(expandedIds);
+        return request;
+    }
+
+    @Override
+    protected Writeable.Reader<Request> instanceReader() {
+        return Request::new;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedActionRequestTests.java
index af883ee916462..3674a6f7c4b96 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedActionRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedActionRequestTests.java
@@ -49,13 +49,13 @@ protected Request doParseInstance(XContentParser parser) {
 
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedWriteableRegistry(searchModule.getNamedWriteables());
     }
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/AggProviderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/AggProviderTests.java
index d544584376f47..d6fc9ec9bc467 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/AggProviderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/AggProviderTests.java
@@ -31,13 +31,13 @@ public class AggProviderTests extends AbstractSerializingTestCase<AggProvider> {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
     @Override
     protected NamedWriteableRegistry writableRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedWriteableRegistry(searchModule.getNamedWriteables());
     }
 
@@ -69,7 +69,7 @@ public static AggProvider createRandomValidAggProvider(String name, String field
         Map<String, Object> agg = Collections.singletonMap(name,
             Collections.singletonMap("avg", Collections.singletonMap("field", field)));
         try {
-            SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+            SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
             AggregatorFactories.Builder aggs =
                 XContentObjectTransformer.aggregatorTransformer(new NamedXContentRegistry(searchModule.getNamedXContents()))
                     .fromMap(agg);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java
index 062504cbfdc3c..15ba85a8edbf1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java
@@ -46,6 +46,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig.Mode;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
+import org.elasticsearch.xpack.core.ml.utils.QueryProvider;
 import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
 
 import java.io.IOException;
@@ -57,7 +58,7 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.xpack.core.ml.datafeed.QueryProviderTests.createRandomValidQueryProvider;
+import static org.elasticsearch.xpack.core.ml.utils.QueryProviderTests.createRandomValidQueryProvider;
 import static org.elasticsearch.xpack.core.ml.job.messages.Messages.DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_GREATER_THAN_ZERO;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -153,13 +154,13 @@ private static DatafeedConfig.Builder createRandomizedDatafeedConfigBuilder(Stri
 
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedWriteableRegistry(searchModule.getNamedWriteables());
     }
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
@@ -717,7 +718,7 @@ public void testSerializationOfComplexAggs() throws IOException {
         assertEquals(aggBuilder, parsedDatafeedConfig.getParsedAggregations(xContentRegistry()));
         assertEquals(datafeedConfig.getQuery(), parsedDatafeedConfig.getQuery());
 
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
         try(BytesStreamOutput output = new BytesStreamOutput()) {
             datafeedConfig.writeTo(output);
@@ -758,7 +759,7 @@ public void testSerializationOfComplexAggsBetweenVersions() throws IOException {
                     .filter(QueryBuilders.termQuery(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)))));
         DatafeedConfig datafeedConfig = datafeedConfigBuilder.build();
 
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
 
         try (BytesStreamOutput output = new BytesStreamOutput()) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java
index 6aa7487147ca1..35bff34f93803 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java
@@ -38,6 +38,7 @@
 import org.elasticsearch.test.AbstractSerializingTestCase;
 import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig.Mode;
 import org.elasticsearch.xpack.core.ml.job.config.JobTests;
+import org.elasticsearch.xpack.core.ml.utils.QueryProvider;
 import org.elasticsearch.xpack.core.ml.utils.XContentObjectTransformer;
 
 import java.io.IOException;
@@ -47,7 +48,7 @@
 import java.util.List;
 
 import static org.elasticsearch.xpack.core.ml.datafeed.AggProviderTests.createRandomValidAggProvider;
-import static org.elasticsearch.xpack.core.ml.datafeed.QueryProviderTests.createRandomValidQueryProvider;
+import static org.elasticsearch.xpack.core.ml.utils.QueryProviderTests.createRandomValidQueryProvider;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
@@ -135,13 +136,13 @@ protected DatafeedUpdate doParseInstance(XContentParser parser) {
 
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedWriteableRegistry(searchModule.getNamedWriteables());
     }
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
@@ -317,7 +318,7 @@ public void testSerializationOfComplexAggsBetweenVersions() throws IOException {
                     .filter(QueryBuilders.termQuery(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)))));
         DatafeedUpdate datafeedUpdate = datafeedUpdateBuilder.build();
 
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
 
         try (BytesStreamOutput output = new BytesStreamOutput()) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java
new file mode 100644
index 0000000000000..5464181d17b1d
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java
@@ -0,0 +1,312 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe;
+
+import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParseException;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.MlDataFrameAnalysisNamedXContentProvider;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetectionTests;
+import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.startsWith;
+
+public class DataFrameAnalyticsConfigTests extends AbstractSerializingTestCase<DataFrameAnalyticsConfig> {
+
+    @Override
+    protected DataFrameAnalyticsConfig doParseInstance(XContentParser parser) throws IOException {
+        ObjectParser<DataFrameAnalyticsConfig.Builder, Void> dataFrameAnalyticsConfigParser =
+            lenient
+                ? DataFrameAnalyticsConfig.LENIENT_PARSER
+                : DataFrameAnalyticsConfig.STRICT_PARSER;
+        return dataFrameAnalyticsConfigParser.apply(parser, null).build();
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
+        namedWriteables.addAll(new MlDataFrameAnalysisNamedXContentProvider().getNamedWriteables());
+        namedWriteables.addAll(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedWriteables());
+        return new NamedWriteableRegistry(namedWriteables);
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
+        namedXContent.addAll(new MlDataFrameAnalysisNamedXContentProvider().getNamedXContentParsers());
+        namedXContent.addAll(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents());
+        return new NamedXContentRegistry(namedXContent);
+    }
+
+    @Override
+    protected DataFrameAnalyticsConfig createTestInstance() {
+        return createRandom(randomValidId(), lenient);
+    }
+
+    @Override
+    protected Writeable.Reader<DataFrameAnalyticsConfig> instanceReader() {
+        return DataFrameAnalyticsConfig::new;
+    }
+
+    public static DataFrameAnalyticsConfig createRandom(String id) {
+        return createRandom(id, false);
+    }
+
+    public static DataFrameAnalyticsConfig createRandom(String id, boolean withGeneratedFields) {
+        return createRandomBuilder(id, withGeneratedFields).build();
+    }
+
+    public static DataFrameAnalyticsConfig.Builder createRandomBuilder(String id) {
+        return createRandomBuilder(id, false);
+    }
+
+    public static DataFrameAnalyticsConfig.Builder createRandomBuilder(String id, boolean withGeneratedFields) {
+        DataFrameAnalyticsSource source = DataFrameAnalyticsSourceTests.createRandom();
+        DataFrameAnalyticsDest dest = DataFrameAnalyticsDestTests.createRandom();
+        DataFrameAnalyticsConfig.Builder builder = new DataFrameAnalyticsConfig.Builder()
+            .setId(id)
+            .setAnalysis(OutlierDetectionTests.createRandom())
+            .setSource(source)
+            .setDest(dest);
+        if (randomBoolean()) {
+            builder.setAnalyzedFields(new FetchSourceContext(true,
+                generateRandomStringArray(10, 10, false, false),
+                generateRandomStringArray(10, 10, false, false)));
+        }
+        if (randomBoolean()) {
+            builder.setModelMemoryLimit(new ByteSizeValue(randomIntBetween(1, 16), randomFrom(ByteSizeUnit.MB, ByteSizeUnit.GB)));
+        }
+        if (withGeneratedFields) {
+            if (randomBoolean()) {
+                builder.setCreateTime(Instant.now());
+            }
+            if (randomBoolean()) {
+                builder.setVersion(Version.CURRENT);
+            }
+        }
+        return builder;
+    }
+
+    public static String randomValidId() {
+        CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz".toCharArray());
+        return generator.ofCodePointsLength(random(), 10, 10);
+    }
+
+    private static final String ANACHRONISTIC_QUERY_DATA_FRAME_ANALYTICS = "{\n" +
+        "    \"id\": \"old-data-frame\",\n" +
+        //query:match:type stopped being supported in 6.x
+        "    \"source\": {\"index\":\"my-index\", \"query\": {\"match\" : {\"query\":\"fieldName\", \"type\": \"phrase\"}}},\n" +
+        "    \"dest\": {\"index\":\"dest-index\"},\n" +
+        "    \"analysis\": {\"outlier_detection\": {\"n_neighbors\": 10}}\n" +
+        "}";
+
+    private static final String MODERN_QUERY_DATA_FRAME_ANALYTICS = "{\n" +
+        "    \"id\": \"data-frame\",\n" +
+        // match_all if parsed, adds default values in the options
+        "    \"source\": {\"index\":\"my-index\", \"query\": {\"match_all\" : {}}},\n" +
+        "    \"dest\": {\"index\":\"dest-index\"},\n" +
+        "    \"analysis\": {\"outlier_detection\": {\"n_neighbors\": 10}}\n" +
+        "}";
+
+    private boolean lenient;
+
+    @Before
+    public void chooseStrictOrLenient() {
+        lenient = randomBoolean();
+    }
+
+    public void testQueryConfigStoresUserInputOnly() throws IOException {
+        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
+            .createParser(xContentRegistry(),
+                DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                MODERN_QUERY_DATA_FRAME_ANALYTICS)) {
+
+            DataFrameAnalyticsConfig config = DataFrameAnalyticsConfig.LENIENT_PARSER.apply(parser, null).build();
+            assertThat(config.getSource().getQuery(), equalTo(Collections.singletonMap(MatchAllQueryBuilder.NAME, Collections.emptyMap())));
+        }
+
+        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
+            .createParser(xContentRegistry(),
+                DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                MODERN_QUERY_DATA_FRAME_ANALYTICS)) {
+
+            DataFrameAnalyticsConfig config = DataFrameAnalyticsConfig.STRICT_PARSER.apply(parser, null).build();
+            assertThat(config.getSource().getQuery(), equalTo(Collections.singletonMap(MatchAllQueryBuilder.NAME, Collections.emptyMap())));
+        }
+    }
+
+    public void testPastQueryConfigParse() throws IOException {
+        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
+            .createParser(xContentRegistry(),
+                DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                ANACHRONISTIC_QUERY_DATA_FRAME_ANALYTICS)) {
+
+            DataFrameAnalyticsConfig config = DataFrameAnalyticsConfig.LENIENT_PARSER.apply(parser, null).build();
+            ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> config.getSource().getParsedQuery());
+            assertEquals("[match] query doesn't support multiple fields, found [query] and [type]", e.getMessage());
+        }
+
+        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
+            .createParser(xContentRegistry(),
+                DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                ANACHRONISTIC_QUERY_DATA_FRAME_ANALYTICS)) {
+
+            XContentParseException e = expectThrows(XContentParseException.class,
+                () -> DataFrameAnalyticsConfig.STRICT_PARSER.apply(parser, null).build());
+            assertThat(e.getMessage(), containsString("[data_frame_analytics_config] failed to parse field [source]"));
+        }
+    }
+
+    public void testToXContentForInternalStorage() throws IOException {
+        DataFrameAnalyticsConfig.Builder builder = createRandomBuilder("foo");
+
+        // headers are only persisted to cluster state
+        Map<String, String> headers = new HashMap<>();
+        headers.put("header-name", "header-value");
+        builder.setHeaders(headers);
+        DataFrameAnalyticsConfig config = builder.build();
+
+        ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true"));
+
+        BytesReference forClusterstateXContent = XContentHelper.toXContent(config, XContentType.JSON, params, false);
+        XContentParser parser = XContentFactory.xContent(XContentType.JSON)
+            .createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, forClusterstateXContent.streamInput());
+
+        DataFrameAnalyticsConfig parsedConfig = DataFrameAnalyticsConfig.LENIENT_PARSER.apply(parser, null).build();
+        assertThat(parsedConfig.getHeaders(), hasEntry("header-name", "header-value"));
+
+        // headers are not written without the FOR_INTERNAL_STORAGE param
+        BytesReference nonClusterstateXContent = XContentHelper.toXContent(config, XContentType.JSON, ToXContent.EMPTY_PARAMS, false);
+        parser = XContentFactory.xContent(XContentType.JSON)
+            .createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, nonClusterstateXContent.streamInput());
+
+        parsedConfig = DataFrameAnalyticsConfig.LENIENT_PARSER.apply(parser, null).build();
+        assertThat(parsedConfig.getHeaders().entrySet(), hasSize(0));
+    }
+
+    public void testInvalidModelMemoryLimits() {
+
+        DataFrameAnalyticsConfig.Builder builder = new DataFrameAnalyticsConfig.Builder();
+
+        // All these are different ways of specifying a limit that is lower than the minimum
+        assertTooSmall(expectThrows(IllegalArgumentException.class,
+            () -> builder.setModelMemoryLimit(new ByteSizeValue(1048575, ByteSizeUnit.BYTES))));
+        assertTooSmall(expectThrows(IllegalArgumentException.class,
+            () -> builder.setModelMemoryLimit(new ByteSizeValue(0, ByteSizeUnit.BYTES))));
+        assertTooSmall(expectThrows(IllegalArgumentException.class,
+            () -> builder.setModelMemoryLimit(new ByteSizeValue(-1, ByteSizeUnit.BYTES))));
+        assertTooSmall(expectThrows(IllegalArgumentException.class,
+            () -> builder.setModelMemoryLimit(new ByteSizeValue(1023, ByteSizeUnit.KB))));
+        assertTooSmall(expectThrows(IllegalArgumentException.class,
+            () -> builder.setModelMemoryLimit(new ByteSizeValue(0, ByteSizeUnit.KB))));
+        assertTooSmall(expectThrows(IllegalArgumentException.class,
+            () -> builder.setModelMemoryLimit(new ByteSizeValue(0, ByteSizeUnit.MB))));
+    }
+
+    public void testNoMemoryCapping() {
+
+        DataFrameAnalyticsConfig uncapped = createRandom("foo");
+
+        ByteSizeValue unlimited = randomBoolean() ? null : ByteSizeValue.ZERO;
+        assertThat(uncapped.getModelMemoryLimit(),
+            equalTo(new DataFrameAnalyticsConfig.Builder(uncapped, unlimited).build().getModelMemoryLimit()));
+    }
+
+    public void testMemoryCapping() {
+
+        DataFrameAnalyticsConfig defaultLimitConfig = createRandomBuilder("foo").setModelMemoryLimit(null).build();
+
+        ByteSizeValue maxLimit = new ByteSizeValue(randomIntBetween(500, 1000), ByteSizeUnit.MB);
+        if (maxLimit.compareTo(defaultLimitConfig.getModelMemoryLimit()) < 0) {
+            assertThat(maxLimit,
+                equalTo(new DataFrameAnalyticsConfig.Builder(defaultLimitConfig, maxLimit).build().getModelMemoryLimit()));
+        } else {
+            assertThat(defaultLimitConfig.getModelMemoryLimit(),
+                equalTo(new DataFrameAnalyticsConfig.Builder(defaultLimitConfig, maxLimit).build().getModelMemoryLimit()));
+        }
+    }
+
+    public void testExplicitModelMemoryLimitTooHigh() {
+
+        ByteSizeValue configuredLimit = new ByteSizeValue(randomIntBetween(5, 10), ByteSizeUnit.GB);
+        DataFrameAnalyticsConfig explicitLimitConfig = createRandomBuilder("foo").setModelMemoryLimit(configuredLimit).build();
+
+        ByteSizeValue maxLimit = new ByteSizeValue(randomIntBetween(500, 1000), ByteSizeUnit.MB);
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
+            () -> new DataFrameAnalyticsConfig.Builder(explicitLimitConfig, maxLimit).build());
+        assertThat(e.getMessage(), startsWith("model_memory_limit"));
+        assertThat(e.getMessage(), containsString("must be less than the value of the xpack.ml.max_model_memory_limit setting"));
+    }
+
+    public void testPreventCreateTimeInjection() throws IOException {
+        String json = "{"
+            + " \"create_time\" : 123456789 },"
+            + " \"source\" : {\"index\":\"src\"},"
+            + " \"dest\" : {\"index\": \"dest\"},"
+            + "}";
+
+        try (XContentParser parser =
+                 XContentFactory.xContent(XContentType.JSON).createParser(
+                     xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json)) {
+            Exception e = expectThrows(IllegalArgumentException.class, () -> DataFrameAnalyticsConfig.STRICT_PARSER.apply(parser, null));
+            assertThat(e.getMessage(), containsString("unknown field [create_time], parser not found"));
+        }
+    }
+
+    public void testPreventVersionInjection() throws IOException {
+        String json = "{"
+            + " \"version\" : \"7.3.0\","
+            + " \"source\" : {\"index\":\"src\"},"
+            + " \"dest\" : {\"index\": \"dest\"},"
+            + "}";
+
+        try (XContentParser parser =
+                 XContentFactory.xContent(XContentType.JSON).createParser(
+                     xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, json)) {
+            Exception e = expectThrows(IllegalArgumentException.class, () -> DataFrameAnalyticsConfig.STRICT_PARSER.apply(parser, null));
+            assertThat(e.getMessage(), containsString("unknown field [version], parser not found"));
+        }
+    }
+
+    public void assertTooSmall(IllegalArgumentException e) {
+        assertThat(e.getMessage(), is("[model_memory_limit] must be at least [1mb]"));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsDestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsDestTests.java
new file mode 100644
index 0000000000000..bf8ce4c8a99b0
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsDestTests.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.indices.InvalidIndexNameException;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class DataFrameAnalyticsDestTests extends AbstractSerializingTestCase<DataFrameAnalyticsDest> {
+
+    @Override
+    protected DataFrameAnalyticsDest doParseInstance(XContentParser parser) throws IOException {
+        return DataFrameAnalyticsDest.createParser(false).apply(parser, null);
+    }
+
+    @Override
+    protected DataFrameAnalyticsDest createTestInstance() {
+        return createRandom();
+    }
+
+    public static DataFrameAnalyticsDest createRandom() {
+        String index = randomAlphaOfLength(10);
+        String resultsField = randomBoolean() ? null : randomAlphaOfLength(10);
+        return new DataFrameAnalyticsDest(index, resultsField);
+    }
+
+    @Override
+    protected Writeable.Reader<DataFrameAnalyticsDest> instanceReader() {
+        return DataFrameAnalyticsDest::new;
+    }
+
+    public void testValidate_GivenIndexWithFunkyChars() {
+        expectThrows(InvalidIndexNameException.class, () -> new DataFrameAnalyticsDest("<script>foo", null).validate());
+    }
+
+    public void testValidate_GivenIndexWithUppercaseChars() {
+        InvalidIndexNameException e = expectThrows(InvalidIndexNameException.class,
+            () -> new DataFrameAnalyticsDest("Foo", null).validate());
+        assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        assertThat(e.getMessage(), equalTo("Invalid index name [Foo], dest.index must be lowercase"));
+    }
+
+    public void testValidate_GivenValidIndexName() {
+        new DataFrameAnalyticsDest("foo_bar_42", null).validate();
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsSourceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsSourceTests.java
new file mode 100644
index 0000000000000..36c4774baa465
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsSourceTests.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.xpack.core.ml.utils.QueryProvider;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collections;
+
+public class DataFrameAnalyticsSourceTests extends AbstractSerializingTestCase<DataFrameAnalyticsSource> {
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
+        return new NamedWriteableRegistry(searchModule.getNamedWriteables());
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
+        return new NamedXContentRegistry(searchModule.getNamedXContents());
+    }
+
+    @Override
+    protected DataFrameAnalyticsSource doParseInstance(XContentParser parser) throws IOException {
+        return DataFrameAnalyticsSource.createParser(false).apply(parser, null);
+    }
+
+    @Override
+    protected DataFrameAnalyticsSource createTestInstance() {
+        return createRandom();
+    }
+
+    public static DataFrameAnalyticsSource createRandom() {
+        String[] index = generateRandomStringArray(10, 10, false, false);
+        QueryProvider queryProvider = null;
+        if (randomBoolean()) {
+            try {
+                queryProvider = QueryProvider.fromParsedQuery(QueryBuilders.termQuery(randomAlphaOfLength(10), randomAlphaOfLength(10)));
+            } catch (IOException e) {
+                // Should never happen
+                throw new UncheckedIOException(e);
+            }
+        }
+        return new DataFrameAnalyticsSource(index, queryProvider);
+    }
+
+    @Override
+    protected Writeable.Reader<DataFrameAnalyticsSource> instanceReader() {
+        return DataFrameAnalyticsSource::new;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsStateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsStateTests.java
new file mode 100644
index 0000000000000..1b0800c91e3f6
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsStateTests.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe;
+
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class DataFrameAnalyticsStateTests extends ESTestCase {
+
+    public void testFromString() {
+        assertThat(DataFrameAnalyticsState.fromString("started"), equalTo(DataFrameAnalyticsState.STARTED));
+        assertThat(DataFrameAnalyticsState.fromString("reindexing"), equalTo(DataFrameAnalyticsState.REINDEXING));
+        assertThat(DataFrameAnalyticsState.fromString("analyzing"), equalTo(DataFrameAnalyticsState.ANALYZING));
+        assertThat(DataFrameAnalyticsState.fromString("stopping"), equalTo(DataFrameAnalyticsState.STOPPING));
+        assertThat(DataFrameAnalyticsState.fromString("stopped"), equalTo(DataFrameAnalyticsState.STOPPED));
+        assertThat(DataFrameAnalyticsState.fromString("failed"), equalTo(DataFrameAnalyticsState.FAILED));
+    }
+
+    public void testToString() {
+        assertThat(DataFrameAnalyticsState.STARTED.toString(), equalTo("started"));
+        assertThat(DataFrameAnalyticsState.REINDEXING.toString(), equalTo("reindexing"));
+        assertThat(DataFrameAnalyticsState.ANALYZING.toString(), equalTo("analyzing"));
+        assertThat(DataFrameAnalyticsState.STOPPING.toString(), equalTo("stopping"));
+        assertThat(DataFrameAnalyticsState.STOPPED.toString(), equalTo("stopped"));
+        assertThat(DataFrameAnalyticsState.FAILED.toString(), equalTo("failed"));
+    }
+
+    public void testIsAnyOf() {
+        assertThat(DataFrameAnalyticsState.STARTED.isAnyOf(), is(false));
+        assertThat(DataFrameAnalyticsState.STARTED.isAnyOf(DataFrameAnalyticsState.STARTED), is(true));
+        assertThat(DataFrameAnalyticsState.STARTED.isAnyOf(DataFrameAnalyticsState.ANALYZING, DataFrameAnalyticsState.STOPPED), is(false));
+        assertThat(DataFrameAnalyticsState.STARTED.isAnyOf(DataFrameAnalyticsState.STARTED, DataFrameAnalyticsState.STOPPED), is(true));
+        assertThat(DataFrameAnalyticsState.ANALYZING.isAnyOf(DataFrameAnalyticsState.STARTED, DataFrameAnalyticsState.STOPPED), is(false));
+        assertThat(DataFrameAnalyticsState.ANALYZING.isAnyOf(DataFrameAnalyticsState.ANALYZING, DataFrameAnalyticsState.FAILED), is(true));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskStateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskStateTests.java
new file mode 100644
index 0000000000000..f6d2d421b3d9a
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskStateTests.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+
+import java.io.IOException;
+
+public class DataFrameAnalyticsTaskStateTests extends AbstractSerializingTestCase<DataFrameAnalyticsTaskState> {
+
+    @Override
+    protected DataFrameAnalyticsTaskState createTestInstance() {
+        return new DataFrameAnalyticsTaskState(randomFrom(DataFrameAnalyticsState.values()), randomLong(), randomAlphaOfLength(10));
+    }
+
+    @Override
+    protected Writeable.Reader<DataFrameAnalyticsTaskState> instanceReader() {
+        return DataFrameAnalyticsTaskState::new;
+    }
+
+    @Override
+    protected DataFrameAnalyticsTaskState doParseInstance(XContentParser parser) throws IOException {
+        return DataFrameAnalyticsTaskState.fromXContent(parser);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return true;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetectionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetectionTests.java
new file mode 100644
index 0000000000000..db6ed7d34aa7d
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetectionTests.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.analyses;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class OutlierDetectionTests extends AbstractSerializingTestCase<OutlierDetection> {
+
+    @Override
+    protected OutlierDetection doParseInstance(XContentParser parser) throws IOException {
+        return OutlierDetection.fromXContent(parser, false);
+    }
+
+    @Override
+    protected OutlierDetection createTestInstance() {
+        return createRandom();
+    }
+
+    public static OutlierDetection createRandom() {
+        Integer numberNeighbors = randomBoolean() ? null : randomIntBetween(1, 20);
+        OutlierDetection.Method method = randomBoolean() ? null : randomFrom(OutlierDetection.Method.values());
+        Double minScoreToWriteFeatureInfluence = randomBoolean() ? null : randomDoubleBetween(0.0, 1.0, true);
+        return new OutlierDetection(numberNeighbors, method, minScoreToWriteFeatureInfluence);
+    }
+
+    @Override
+    protected Writeable.Reader<OutlierDetection> instanceReader() {
+        return OutlierDetection::new;
+    }
+
+    public void testGetParams_GivenDefaults() {
+        OutlierDetection outlierDetection = new OutlierDetection();
+        assertThat(outlierDetection.getParams().isEmpty(), is(true));
+    }
+
+    public void testGetParams_GivenExplicitValues() {
+        OutlierDetection outlierDetection = new OutlierDetection(42, OutlierDetection.Method.LDOF, 0.42);
+
+        Map<String, Object> params = outlierDetection.getParams();
+
+        assertThat(params.size(), equalTo(3));
+        assertThat(params.get(OutlierDetection.N_NEIGHBORS.getPreferredName()), equalTo(42));
+        assertThat(params.get(OutlierDetection.METHOD.getPreferredName()), equalTo(OutlierDetection.Method.LDOF));
+        assertThat((Double) params.get(OutlierDetection.FEATURE_INFLUENCE_THRESHOLD.getPreferredName()),
+            is(closeTo(0.42, 1E-9)));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AucRocTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AucRocTests.java
new file mode 100644
index 0000000000000..6f8ca9339715d
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/AucRocTests.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class AucRocTests extends AbstractSerializingTestCase<AucRoc> {
+
+    @Override
+    protected AucRoc doParseInstance(XContentParser parser) throws IOException {
+        return AucRoc.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected AucRoc createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected Writeable.Reader<AucRoc> instanceReader() {
+        return AucRoc::new;
+    }
+
+    public static AucRoc createRandom() {
+        return new AucRoc(randomBoolean() ? randomBoolean() : null);
+    }
+
+    public void testCalculateAucScore_GivenZeroPercentiles() {
+        double[] tpPercentiles = zeroPercentiles();
+        double[] fpPercentiles = zeroPercentiles();
+
+        List<AucRoc.AucRocPoint> curve = AucRoc.buildAucRocCurve(tpPercentiles, fpPercentiles);
+        double aucRocScore = AucRoc.calculateAucScore(curve);
+
+        assertThat(aucRocScore, closeTo(0.5, 0.01));
+    }
+
+    public void testCalculateAucScore_GivenRandomTpPercentilesAndZeroFpPercentiles() {
+        double[] tpPercentiles = randomPercentiles();
+        double[] fpPercentiles = zeroPercentiles();
+
+        List<AucRoc.AucRocPoint> curve = AucRoc.buildAucRocCurve(tpPercentiles, fpPercentiles);
+        double aucRocScore = AucRoc.calculateAucScore(curve);
+
+        assertThat(aucRocScore, closeTo(1.0, 0.1));
+    }
+
+    public void testCalculateAucScore_GivenZeroTpPercentilesAndRandomFpPercentiles() {
+        double[] tpPercentiles = zeroPercentiles();
+        double[] fpPercentiles = randomPercentiles();
+
+        List<AucRoc.AucRocPoint> curve = AucRoc.buildAucRocCurve(tpPercentiles, fpPercentiles);
+        double aucRocScore = AucRoc.calculateAucScore(curve);
+
+        assertThat(aucRocScore, closeTo(0.0, 0.1));
+    }
+
+    public void testCalculateAucScore_GivenRandomPercentiles() {
+        for (int i = 0; i < 20; i++) {
+            double[] tpPercentiles = randomPercentiles();
+            double[] fpPercentiles = randomPercentiles();
+
+            List<AucRoc.AucRocPoint> curve = AucRoc.buildAucRocCurve(tpPercentiles, fpPercentiles);
+            double aucRocScore = AucRoc.calculateAucScore(curve);
+
+            List<AucRoc.AucRocPoint> inverseCurve = AucRoc.buildAucRocCurve(fpPercentiles, tpPercentiles);
+            double inverseAucRocScore = AucRoc.calculateAucScore(inverseCurve);
+
+            assertThat(aucRocScore, greaterThanOrEqualTo(0.0));
+            assertThat(aucRocScore, lessThanOrEqualTo(1.0));
+            assertThat(inverseAucRocScore, greaterThanOrEqualTo(0.0));
+            assertThat(inverseAucRocScore, lessThanOrEqualTo(1.0));
+            assertThat(aucRocScore + inverseAucRocScore, closeTo(1.0, 0.05));
+        }
+    }
+
+    public void testCalculateAucScore_GivenPrecalculated() {
+        double[] tpPercentiles = new double[99];
+        double[] fpPercentiles = new double[99];
+
+        double[] tpSimplified = new double[] { 0.3, 0.6, 0.5 , 0.8 };
+        double[] fpSimplified = new double[] { 0.1, 0.3, 0.5 , 0.5 };
+
+        for (int i = 0; i < tpPercentiles.length; i++) {
+            int simplifiedIndex = i / 25;
+            tpPercentiles[i] = tpSimplified[simplifiedIndex];
+            fpPercentiles[i] = fpSimplified[simplifiedIndex];
+        }
+
+        List<AucRoc.AucRocPoint> curve = AucRoc.buildAucRocCurve(tpPercentiles, fpPercentiles);
+        double aucRocScore = AucRoc.calculateAucScore(curve);
+
+        List<AucRoc.AucRocPoint> inverseCurve = AucRoc.buildAucRocCurve(fpPercentiles, tpPercentiles);
+        double inverseAucRocScore = AucRoc.calculateAucScore(inverseCurve);
+
+        assertThat(aucRocScore, closeTo(0.8, 0.05));
+        assertThat(inverseAucRocScore, closeTo(0.2, 0.05));
+    }
+
+    public static double[] zeroPercentiles() {
+        double[] percentiles = new double[99];
+        Arrays.fill(percentiles, 0.0);
+        return percentiles;
+    }
+
+    public static double[] randomPercentiles() {
+        double[] percentiles = new double[99];
+        for (int i = 0; i < percentiles.length; i++) {
+            percentiles[i] = randomDouble();
+        }
+        Arrays.sort(percentiles);
+        return percentiles;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/BinarySoftClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/BinarySoftClassificationTests.java
new file mode 100644
index 0000000000000..4f17df3536731
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/BinarySoftClassificationTests.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class BinarySoftClassificationTests extends AbstractSerializingTestCase<BinarySoftClassification> {
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(new MlEvaluationNamedXContentProvider().getNamedWriteables());
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(new MlEvaluationNamedXContentProvider().getNamedXContentParsers());
+    }
+
+    public static BinarySoftClassification createRandom() {
+        List<SoftClassificationMetric> metrics = new ArrayList<>();
+        if (randomBoolean()) {
+            metrics.add(AucRocTests.createRandom());
+        }
+        if (randomBoolean()) {
+            metrics.add(PrecisionTests.createRandom());
+        }
+        if (randomBoolean()) {
+            metrics.add(RecallTests.createRandom());
+        }
+        if (randomBoolean()) {
+            metrics.add(ConfusionMatrixTests.createRandom());
+        }
+        if (metrics.isEmpty()) {
+            // not a good day to play in the lottery; let's add them all
+            metrics.add(AucRocTests.createRandom());
+            metrics.add(PrecisionTests.createRandom());
+            metrics.add(RecallTests.createRandom());
+            metrics.add(ConfusionMatrixTests.createRandom());
+        }
+        return new BinarySoftClassification(randomAlphaOfLength(10), randomAlphaOfLength(10), metrics);
+    }
+
+    @Override
+    protected BinarySoftClassification doParseInstance(XContentParser parser) throws IOException {
+        return BinarySoftClassification.fromXContent(parser);
+    }
+
+    @Override
+    protected BinarySoftClassification createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected Writeable.Reader<BinarySoftClassification> instanceReader() {
+        return BinarySoftClassification::new;
+    }
+
+    public void testConstructor_GivenEmptyMetrics() {
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
+            () -> new BinarySoftClassification("foo", "bar", Collections.emptyList()));
+        assertThat(e.getMessage(), equalTo("[binary_soft_classification] must have one or more metrics"));
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/ConfusionMatrixTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/ConfusionMatrixTests.java
new file mode 100644
index 0000000000000..41f78051af420
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/ConfusionMatrixTests.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.filter.Filter;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ConfusionMatrixTests extends AbstractSerializingTestCase<ConfusionMatrix> {
+
+    @Override
+    protected ConfusionMatrix doParseInstance(XContentParser parser) throws IOException {
+        return ConfusionMatrix.fromXContent(parser);
+    }
+
+    @Override
+    protected ConfusionMatrix createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected Writeable.Reader<ConfusionMatrix> instanceReader() {
+        return ConfusionMatrix::new;
+    }
+
+    public static ConfusionMatrix createRandom() {
+        int thresholdsSize = randomIntBetween(1, 3);
+        List<Double> thresholds = new ArrayList<>(thresholdsSize);
+        for (int i = 0; i < thresholdsSize; i++) {
+            thresholds.add(randomDouble());
+        }
+        return new ConfusionMatrix(thresholds);
+    }
+
+    public void testEvaluate() {
+        SoftClassificationMetric.ClassInfo classInfo = mock(SoftClassificationMetric.ClassInfo.class);
+        when(classInfo.getName()).thenReturn("foo");
+
+        Aggregations aggs = new Aggregations(Arrays.asList(
+            createFilterAgg("confusion_matrix_foo_at_0.25_TP", 1L),
+            createFilterAgg("confusion_matrix_foo_at_0.25_FP", 2L),
+            createFilterAgg("confusion_matrix_foo_at_0.25_TN", 3L),
+            createFilterAgg("confusion_matrix_foo_at_0.25_FN", 4L),
+            createFilterAgg("confusion_matrix_foo_at_0.5_TP", 5L),
+            createFilterAgg("confusion_matrix_foo_at_0.5_FP", 6L),
+            createFilterAgg("confusion_matrix_foo_at_0.5_TN", 7L),
+            createFilterAgg("confusion_matrix_foo_at_0.5_FN", 8L)
+        ));
+
+        ConfusionMatrix confusionMatrix = new ConfusionMatrix(Arrays.asList(0.25, 0.5));
+        EvaluationMetricResult result = confusionMatrix.evaluate(classInfo, aggs);
+
+        String expected = "{\"0.25\":{\"tp\":1,\"fp\":2,\"tn\":3,\"fn\":4},\"0.5\":{\"tp\":5,\"fp\":6,\"tn\":7,\"fn\":8}}";
+        assertThat(Strings.toString(result), equalTo(expected));
+    }
+
+    private static Filter createFilterAgg(String name, long docCount) {
+        Filter agg = mock(Filter.class);
+        when(agg.getName()).thenReturn(name);
+        when(agg.getDocCount()).thenReturn(docCount);
+        return agg;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/PrecisionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/PrecisionTests.java
new file mode 100644
index 0000000000000..c12156c39373e
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/PrecisionTests.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.filter.Filter;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class PrecisionTests extends AbstractSerializingTestCase<Precision> {
+
+    @Override
+    protected Precision doParseInstance(XContentParser parser) throws IOException {
+        return Precision.fromXContent(parser);
+    }
+
+    @Override
+    protected Precision createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected Writeable.Reader<Precision> instanceReader() {
+        return Precision::new;
+    }
+
+    public static Precision createRandom() {
+        int thresholdsSize = randomIntBetween(1, 3);
+        List<Double> thresholds = new ArrayList<>(thresholdsSize);
+        for (int i = 0; i < thresholdsSize; i++) {
+            thresholds.add(randomDouble());
+        }
+        return new Precision(thresholds);
+    }
+
+    public void testEvaluate() {
+        SoftClassificationMetric.ClassInfo classInfo = mock(SoftClassificationMetric.ClassInfo.class);
+        when(classInfo.getName()).thenReturn("foo");
+
+        Aggregations aggs = new Aggregations(Arrays.asList(
+            createFilterAgg("precision_foo_at_0.25_TP", 1L),
+            createFilterAgg("precision_foo_at_0.25_FP", 4L),
+            createFilterAgg("precision_foo_at_0.5_TP", 3L),
+            createFilterAgg("precision_foo_at_0.5_FP", 1L),
+            createFilterAgg("precision_foo_at_0.75_TP", 5L),
+            createFilterAgg("precision_foo_at_0.75_FP", 0L)
+        ));
+
+        Precision precision = new Precision(Arrays.asList(0.25, 0.5, 0.75));
+        EvaluationMetricResult result = precision.evaluate(classInfo, aggs);
+
+        String expected = "{\"0.25\":0.2,\"0.5\":0.75,\"0.75\":1.0}";
+        assertThat(Strings.toString(result), equalTo(expected));
+    }
+
+    public void testEvaluate_GivenZeroTpAndFp() {
+        SoftClassificationMetric.ClassInfo classInfo = mock(SoftClassificationMetric.ClassInfo.class);
+        when(classInfo.getName()).thenReturn("foo");
+
+        Aggregations aggs = new Aggregations(Arrays.asList(
+            createFilterAgg("precision_foo_at_1.0_TP", 0L),
+            createFilterAgg("precision_foo_at_1.0_FP", 0L)
+        ));
+
+        Precision precision = new Precision(Arrays.asList(1.0));
+        EvaluationMetricResult result = precision.evaluate(classInfo, aggs);
+
+        String expected = "{\"1.0\":0.0}";
+        assertThat(Strings.toString(result), equalTo(expected));
+    }
+
+    private static Filter createFilterAgg(String name, long docCount) {
+        Filter agg = mock(Filter.class);
+        when(agg.getName()).thenReturn(name);
+        when(agg.getDocCount()).thenReturn(docCount);
+        return agg;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/RecallTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/RecallTests.java
new file mode 100644
index 0000000000000..fc85b44f151d4
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/evaluation/softclassification/RecallTests.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.dataframe.evaluation.softclassification;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.bucket.filter.Filter;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class RecallTests extends AbstractSerializingTestCase<Recall> {
+
+    @Override
+    protected Recall doParseInstance(XContentParser parser) throws IOException {
+        return Recall.fromXContent(parser);
+    }
+
+    @Override
+    protected Recall createTestInstance() {
+        return createRandom();
+    }
+
+    @Override
+    protected Writeable.Reader<Recall> instanceReader() {
+        return Recall::new;
+    }
+
+    public static Recall createRandom() {
+        int thresholdsSize = randomIntBetween(1, 3);
+        List<Double> thresholds = new ArrayList<>(thresholdsSize);
+        for (int i = 0; i < thresholdsSize; i++) {
+            thresholds.add(randomDouble());
+        }
+        return new Recall(thresholds);
+    }
+
+    public void testEvaluate() {
+        SoftClassificationMetric.ClassInfo classInfo = mock(SoftClassificationMetric.ClassInfo.class);
+        when(classInfo.getName()).thenReturn("foo");
+
+        Aggregations aggs = new Aggregations(Arrays.asList(
+            createFilterAgg("recall_foo_at_0.25_TP", 1L),
+            createFilterAgg("recall_foo_at_0.25_FN", 4L),
+            createFilterAgg("recall_foo_at_0.5_TP", 3L),
+            createFilterAgg("recall_foo_at_0.5_FN", 1L),
+            createFilterAgg("recall_foo_at_0.75_TP", 5L),
+            createFilterAgg("recall_foo_at_0.75_FN", 0L)
+        ));
+
+        Recall recall = new Recall(Arrays.asList(0.25, 0.5, 0.75));
+        EvaluationMetricResult result = recall.evaluate(classInfo, aggs);
+
+        String expected = "{\"0.25\":0.2,\"0.5\":0.75,\"0.75\":1.0}";
+        assertThat(Strings.toString(result), equalTo(expected));
+    }
+
+    public void testEvaluate_GivenZeroTpAndFp() {
+        SoftClassificationMetric.ClassInfo classInfo = mock(SoftClassificationMetric.ClassInfo.class);
+        when(classInfo.getName()).thenReturn("foo");
+
+        Aggregations aggs = new Aggregations(Arrays.asList(
+            createFilterAgg("recall_foo_at_1.0_TP", 0L),
+            createFilterAgg("recall_foo_at_1.0_FN", 0L)
+        ));
+
+        Recall recall = new Recall(Arrays.asList(1.0));
+        EvaluationMetricResult result = recall.evaluate(classInfo, aggs);
+
+        String expected = "{\"1.0\":0.0}";
+        assertThat(Strings.toString(result), equalTo(expected));
+    }
+
+    private static Filter createFilterAgg(String name, long docCount) {
+        Filter agg = mock(Filter.class);
+        when(agg.getName()).thenReturn(name);
+        when(agg.getDocCount()).thenReturn(docCount);
+        return agg;
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/filestructurefinder/FieldStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/filestructurefinder/FieldStatsTests.java
index ec46d25edd409..889eae19387fc 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/filestructurefinder/FieldStatsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/filestructurefinder/FieldStatsTests.java
@@ -30,6 +30,8 @@ static FieldStats createTestFieldStats() {
         Double maxValue = null;
         Double meanValue = null;
         Double medianValue = null;
+        String earliestTimestamp = null;
+        String latestTimestamp = null;
         boolean isMetric = randomBoolean();
         if (isMetric) {
             if (randomBoolean()) {
@@ -41,6 +43,12 @@ static FieldStats createTestFieldStats() {
             }
             meanValue = randomDouble();
             medianValue = randomDouble();
+        } else {
+            boolean isDate = randomBoolean();
+            if (isDate) {
+                earliestTimestamp = randomAlphaOfLength(20);
+                latestTimestamp = randomAlphaOfLength(20);
+            }
         }
 
         List<Map<String, Object>> topHits = new ArrayList<>();
@@ -55,7 +63,7 @@ static FieldStats createTestFieldStats() {
             topHits.add(topHit);
         }
 
-        return new FieldStats(count, cardinality, minValue, maxValue, meanValue, medianValue, topHits);
+        return new FieldStats(count, cardinality, minValue, maxValue, meanValue, medianValue, earliestTimestamp, latestTimestamp, topHits);
     }
 
     @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java
index 46d7c5b9e43da..00edb6bb6df2c 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java
@@ -29,6 +29,7 @@ public MlRestTestStateCleaner(Logger logger, RestClient adminClient) {
     public void clearMlMetadata() throws IOException {
         deleteAllDatafeeds();
         deleteAllJobs();
+        deleteAllDataFrameAnalytics();
         // indices will be deleted by the ESRestTestCase class
     }
 
@@ -91,4 +92,20 @@ private void deleteAllJobs() throws IOException {
             adminClient.performRequest(new Request("DELETE", "/_ml/anomaly_detectors/" + jobId));
         }
     }
+
+    private void deleteAllDataFrameAnalytics() throws IOException {
+        final Request analyticsRequest = new Request("GET", "/_ml/data_frame/analytics?size=10000");
+        analyticsRequest.addParameter("filter_path", "data_frame_analytics");
+        final Response analyticsResponse = adminClient.performRequest(analyticsRequest);
+        List<Map<String, Object>> analytics = (List<Map<String, Object>>) XContentMapValues.extractValue(
+            "data_frame_analytics", ESRestTestCase.entityAsMap(analyticsResponse));
+        if (analytics == null) {
+            return;
+        }
+
+        for (Map<String, Object> config : analytics) {
+            String id = (String) config.get("id");
+            adminClient.performRequest(new Request("DELETE", "/_ml/data_frame/analytics/" + id));
+        }
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java
index 42e328e3591e0..b3133ef7f8c17 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord;
 import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition;
 import org.elasticsearch.xpack.core.ml.job.results.ReservedFieldNames;
@@ -80,6 +81,7 @@ public void testResultsMapppingReservedFields() throws Exception {
         overridden.add(ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName());
         overridden.add(ModelSnapshot.TYPE.getPreferredName());
         overridden.add(Quantiles.TYPE.getPreferredName());
+        overridden.add(TimingStats.TYPE.getPreferredName());
 
         Set<String> expected = collectResultsDocFieldNames();
         expected.removeAll(overridden);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStatsTests.java
new file mode 100644
index 0000000000000..99e29d01bd724
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStatsTests.java
@@ -0,0 +1,149 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.ml.job.process.autodetect.state;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+import org.hamcrest.CustomTypeSafeMatcher;
+import org.hamcrest.Matcher;
+
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class TimingStatsTests extends AbstractSerializingTestCase<TimingStats> {
+
+    private static final String JOB_ID = "my-job-id";
+
+    public static TimingStats createTestInstance(String jobId) {
+        return new TimingStats(
+            jobId,
+            randomLong(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble(),
+            randomBoolean() ? null : randomDouble());
+    }
+
+    @Override
+    public TimingStats createTestInstance() {
+        return createTestInstance(randomAlphaOfLength(10));
+    }
+
+    @Override
+    protected Writeable.Reader<TimingStats> instanceReader() {
+        return TimingStats::new;
+    }
+
+    @Override
+    protected TimingStats doParseInstance(XContentParser parser) {
+        return TimingStats.PARSER.apply(parser, null);
+    }
+
+    public void testEquals() {
+        TimingStats stats1 = new TimingStats(JOB_ID, 7, 1.0, 2.0, 1.23, 7.89);
+        TimingStats stats2 = new TimingStats(JOB_ID, 7, 1.0, 2.0, 1.23, 7.89);
+        TimingStats stats3 = new TimingStats(JOB_ID, 7, 1.0, 3.0, 1.23, 7.89);
+
+        assertTrue(stats1.equals(stats1));
+        assertTrue(stats1.equals(stats2));
+        assertFalse(stats2.equals(stats3));
+    }
+
+    public void testHashCode() {
+        TimingStats stats1 = new TimingStats(JOB_ID, 7, 1.0, 2.0, 1.23, 7.89);
+        TimingStats stats2 = new TimingStats(JOB_ID, 7, 1.0, 2.0, 1.23, 7.89);
+        TimingStats stats3 = new TimingStats(JOB_ID, 7, 1.0, 3.0, 1.23, 7.89);
+
+        assertEquals(stats1.hashCode(), stats1.hashCode());
+        assertEquals(stats1.hashCode(), stats2.hashCode());
+        assertNotEquals(stats2.hashCode(), stats3.hashCode());
+    }
+
+    public void testDefaultConstructor() {
+        TimingStats stats = new TimingStats(JOB_ID);
+
+        assertThat(stats.getJobId(), equalTo(JOB_ID));
+        assertThat(stats.getBucketCount(), equalTo(0L));
+        assertThat(stats.getMinBucketProcessingTimeMs(), nullValue());
+        assertThat(stats.getMaxBucketProcessingTimeMs(), nullValue());
+        assertThat(stats.getAvgBucketProcessingTimeMs(), nullValue());
+        assertThat(stats.getExponentialAvgBucketProcessingTimeMs(), nullValue());
+    }
+
+    public void testConstructor() {
+        TimingStats stats = new TimingStats(JOB_ID, 7, 1.0, 2.0, 1.23, 7.89);
+
+        assertThat(stats.getJobId(), equalTo(JOB_ID));
+        assertThat(stats.getBucketCount(), equalTo(7L));
+        assertThat(stats.getMinBucketProcessingTimeMs(), equalTo(1.0));
+        assertThat(stats.getMaxBucketProcessingTimeMs(), equalTo(2.0));
+        assertThat(stats.getAvgBucketProcessingTimeMs(), equalTo(1.23));
+        assertThat(stats.getExponentialAvgBucketProcessingTimeMs(), equalTo(7.89));
+    }
+
+    public void testCopyConstructor() {
+        TimingStats stats1 = new TimingStats(JOB_ID, 7, 1.0, 2.0, 1.23, 7.89);
+        TimingStats stats2 = new TimingStats(stats1);
+
+        assertThat(stats2.getJobId(), equalTo(JOB_ID));
+        assertThat(stats2.getBucketCount(), equalTo(7L));
+        assertThat(stats2.getMinBucketProcessingTimeMs(), equalTo(1.0));
+        assertThat(stats2.getMaxBucketProcessingTimeMs(), equalTo(2.0));
+        assertThat(stats2.getAvgBucketProcessingTimeMs(), equalTo(1.23));
+        assertThat(stats2.getExponentialAvgBucketProcessingTimeMs(), equalTo(7.89));
+        assertEquals(stats1, stats2);
+        assertEquals(stats1.hashCode(), stats2.hashCode());
+    }
+
+    public void testUpdateStats() {
+        TimingStats stats = new TimingStats(JOB_ID);
+
+        stats.updateStats(3);
+        assertThat(stats, areCloseTo(new TimingStats(JOB_ID, 1, 3.0, 3.0, 3.0, 3.0), 1e-9));
+
+        stats.updateStats(2);
+        assertThat(stats, areCloseTo(new TimingStats(JOB_ID, 2, 2.0, 3.0, 2.5, 2.99), 1e-9));
+
+        stats.updateStats(4);
+        assertThat(stats, areCloseTo(new TimingStats(JOB_ID, 3, 2.0, 4.0, 3.0, 3.0001), 1e-9));
+
+        stats.updateStats(1);
+        assertThat(stats, areCloseTo(new TimingStats(JOB_ID, 4, 1.0, 4.0, 2.5, 2.980099), 1e-9));
+
+        stats.updateStats(5);
+        assertThat(stats, areCloseTo(new TimingStats(JOB_ID, 5, 1.0, 5.0, 3.0, 3.00029801), 1e-9));
+    }
+
+    public void testDocumentId() {
+        assertThat(TimingStats.documentId("my-job-id"), equalTo("my-job-id_timing_stats"));
+    }
+
+    /**
+     * Creates a matcher of {@link TimingStats}s that matches when an examined stats are equal
+     * to the specified <code>operand</code>, within a range of +/- <code>error</code>.
+     *
+     * @param operand
+     *     the expected value of matching stats
+     * @param error
+     *     the delta (+/-) within which matches will be allowed
+     */
+    private static Matcher<TimingStats> areCloseTo(TimingStats operand, double error) {
+        return new CustomTypeSafeMatcher<>("TimingStats close to " + operand) {
+            @Override
+            protected boolean matchesSafely(TimingStats item) {
+                return equalTo(operand.getJobId()).matches(item.getJobId())
+                    && equalTo(operand.getBucketCount()).matches(item.getBucketCount())
+                    && closeTo(operand.getMinBucketProcessingTimeMs(), error).matches(item.getMinBucketProcessingTimeMs())
+                    && closeTo(operand.getMaxBucketProcessingTimeMs(), error).matches(item.getMaxBucketProcessingTimeMs())
+                    && closeTo(operand.getAvgBucketProcessingTimeMs(), error).matches(item.getAvgBucketProcessingTimeMs())
+                    && closeTo(operand.getExponentialAvgBucketProcessingTimeMs(), error)
+                        .matches(item.getExponentialAvgBucketProcessingTimeMs());
+            }
+        };
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/QueryProviderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/QueryProviderTests.java
similarity index 92%
rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/QueryProviderTests.java
rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/QueryProviderTests.java
index 8d113aba33579..9a9331e9543a1 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/QueryProviderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/QueryProviderTests.java
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.core.ml.datafeed;
+package org.elasticsearch.xpack.core.ml.utils;
 
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -21,7 +21,7 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.AbstractSerializingTestCase;
-import org.elasticsearch.xpack.core.ml.utils.XContentObjectTransformer;
+import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -34,13 +34,13 @@ public class QueryProviderTests extends AbstractSerializingTestCase<QueryProvide
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
     @Override
     protected NamedWriteableRegistry writableRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedWriteableRegistry(searchModule.getNamedWriteables());
     }
 
@@ -61,7 +61,7 @@ protected Writeable.Reader<QueryProvider> instanceReader() {
 
     @Override
     protected QueryProvider doParseInstance(XContentParser parser) throws IOException {
-        return QueryProvider.fromXContent(parser, false);
+        return QueryProvider.fromXContent(parser, false, Messages.DATAFEED_CONFIG_QUERY_BAD_FORMAT);
     }
 
     public static QueryProvider createRandomValidQueryProvider() {
@@ -84,7 +84,7 @@ public void testEmptyQueryMap() throws IOException {
         XContentParser parser = XContentFactory.xContent(XContentType.JSON)
             .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, "{}");
         ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
-            () -> QueryProvider.fromXContent(parser, false));
+            () -> QueryProvider.fromXContent(parser, false, Messages.DATAFEED_CONFIG_QUERY_BAD_FORMAT));
         assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
         assertThat(e.getMessage(), equalTo("Datafeed query is not parsable"));
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/XContentObjectTransformerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/XContentObjectTransformerTests.java
index 2e3a6e056ae3f..d6ebc9a2e150b 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/XContentObjectTransformerTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/XContentObjectTransformerTests.java
@@ -41,7 +41,7 @@ public class XContentObjectTransformerTests extends ESTestCase {
 
     @Override
     public NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rest/action/ReloadSynonymAnalyzerIT.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rest/action/ReloadSynonymAnalyzerIT.java
new file mode 100644
index 0000000000000..f86e0f1c46cc2
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rest/action/ReloadSynonymAnalyzerIT.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.rest.action;
+
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction.AnalyzeToken;
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction.Response;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.ReloadAnalyzerAction;
+import org.elasticsearch.xpack.core.action.ReloadAnalyzersRequest;
+import org.elasticsearch.xpack.core.action.ReloadAnalyzersResponse;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+
+public class ReloadSynonymAnalyzerIT extends ESIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(XPackSettings.SECURITY_ENABLED.getKey(), false).build();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(LocalStateCompositeXPackPlugin.class, CommonAnalysisPlugin.class);
+    }
+
+    /**
+     * This test needs to write to the config directory, this is difficult in an external cluster so we overwrite this to force running with
+     * {@link InternalTestCluster}
+     */
+    @Override
+    protected boolean ignoreExternalCluster() {
+        return true;
+    }
+
+    public void testSynonymsUpdateable() throws FileNotFoundException, IOException, InterruptedException {
+        Path config = internalCluster().getInstance(Environment.class).configFile();
+        String synonymsFileName = "synonyms.txt";
+        Path synonymsFile = config.resolve(synonymsFileName);
+        Files.createFile(synonymsFile);
+        assertTrue(Files.exists(synonymsFile));
+        try (PrintWriter out = new PrintWriter(
+                new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.CREATE), StandardCharsets.UTF_8))) {
+            out.println("foo, baz");
+        }
+        assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder()
+                .put("index.number_of_shards", cluster().numDataNodes() * 2)
+                .put("index.number_of_replicas", 1)
+                .put("analysis.analyzer.my_synonym_analyzer.tokenizer", "standard")
+                .put("analysis.analyzer.my_synonym_analyzer.filter", "my_synonym_filter")
+                .put("analysis.filter.my_synonym_filter.type", "synonym")
+                .put("analysis.filter.my_synonym_filter.updateable", "true")
+                .put("analysis.filter.my_synonym_filter.synonyms_path", synonymsFileName))
+                .addMapping("_doc", "field", "type=text,analyzer=standard,search_analyzer=my_synonym_analyzer"));
+
+        client().prepareIndex("test", "_doc", "1").setSource("field", "foo").get();
+        assertNoFailures(client().admin().indices().prepareRefresh("test").execute().actionGet());
+
+        SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")).get();
+        assertHitCount(response, 1L);
+        response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "buzz")).get();
+        assertHitCount(response, 0L);
+        Response analyzeResponse = client().admin().indices().prepareAnalyze("test", "foo").setAnalyzer("my_synonym_analyzer").get();
+        assertEquals(2, analyzeResponse.getTokens().size());
+        assertEquals("foo", analyzeResponse.getTokens().get(0).getTerm());
+        assertEquals("baz", analyzeResponse.getTokens().get(1).getTerm());
+
+        // now update synonyms file several times and trigger reloading
+        for (int i = 0; i < 10; i++) {
+            String testTerm = randomAlphaOfLength(10);
+            try (PrintWriter out = new PrintWriter(
+                    new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) {
+                out.println("foo, baz, " + testTerm);
+            }
+            ReloadAnalyzersResponse reloadResponse = client().execute(ReloadAnalyzerAction.INSTANCE, new ReloadAnalyzersRequest("test"))
+                    .actionGet();
+            assertNoFailures(reloadResponse);
+            assertEquals(cluster().numDataNodes(), reloadResponse.getSuccessfulShards());
+            assertTrue(reloadResponse.getReloadDetails().containsKey("test"));
+            assertEquals("test", reloadResponse.getReloadDetails().get("test").getIndexName());
+            assertEquals(Collections.singleton("my_synonym_analyzer"),
+                    reloadResponse.getReloadDetails().get("test").getReloadedAnalyzers());
+
+            analyzeResponse = client().admin().indices().prepareAnalyze("test", "foo").setAnalyzer("my_synonym_analyzer").get();
+            assertEquals(3, analyzeResponse.getTokens().size());
+            Set<String> tokens = new HashSet<>();
+            analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t));
+            assertTrue(tokens.contains("foo"));
+            assertTrue(tokens.contains("baz"));
+            assertTrue(tokens.contains(testTerm));
+
+            response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")).get();
+            assertHitCount(response, 1L);
+            response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", testTerm)).get();
+            assertHitCount(response, 1L);
+        }
+    }
+}
\ No newline at end of file
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilderTests.java
index fb4f87089e8e7..181ae27f0ae9a 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilderTests.java
@@ -59,4 +59,22 @@ public void testParserAndCreateApiRequestBuilder() throws IOException {
             assertThat(request.getExpiration(), equalTo(TimeValue.parseTimeValue("1d", "expiration")));
         }
     }
+
+    public void testParserAndCreateApiRequestBuilderWithNullOrEmptyRoleDescriptors() throws IOException {
+        boolean withExpiration = randomBoolean();
+        boolean noRoleDescriptorsField = randomBoolean();
+        final String json = "{ \"name\" : \"my-api-key\""
+                + ((withExpiration) ? ", \"expiration\": \"1d\"" : "")
+                + ((noRoleDescriptorsField) ? "" : ", \"role_descriptors\": {}")
+                + "}";
+        final BytesArray source = new BytesArray(json);
+        final NodeClient mockClient = mock(NodeClient.class);
+        final CreateApiKeyRequest request = new CreateApiKeyRequestBuilder(mockClient).source(source, XContentType.JSON).request();
+        final List<RoleDescriptor> actualRoleDescriptors = request.getRoleDescriptors();
+        assertThat(request.getName(), equalTo("my-api-key"));
+        assertThat(actualRoleDescriptors.size(), is(0));
+        if (withExpiration) {
+            assertThat(request.getExpiration(), equalTo(TimeValue.parseTimeValue("1d", "expiration")));
+        }
+    }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java
index 654d56b42130e..1bb5398529f99 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java
@@ -82,10 +82,16 @@ public void testSerialization() throws IOException {
         final TimeValue expiration = randomBoolean() ? null :
             TimeValue.parseTimeValue(randomTimeValue(), "test serialization of create api key");
         final WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values());
-        final int numDescriptors = randomIntBetween(0, 4);
-        final List<RoleDescriptor> descriptorList = new ArrayList<>();
-        for (int i = 0; i < numDescriptors; i++) {
-            descriptorList.add(new RoleDescriptor("role_" + i, new String[] { "all" }, null, null));
+        boolean nullOrEmptyRoleDescriptors = randomBoolean();
+        final List<RoleDescriptor> descriptorList;
+        if (nullOrEmptyRoleDescriptors) {
+            descriptorList = randomBoolean() ? null : List.of();
+        } else {
+            final int numDescriptors = randomIntBetween(1, 4);
+            descriptorList = new ArrayList<>();
+            for (int i = 0; i < numDescriptors; i++) {
+                descriptorList.add(new RoleDescriptor("role_" + i, new String[] { "all" }, null, null));
+            }
         }
 
         final CreateApiKeyRequest request = new CreateApiKeyRequest();
@@ -95,9 +101,7 @@ public void testSerialization() throws IOException {
         if (refreshPolicy != request.getRefreshPolicy() || randomBoolean()) {
             request.setRefreshPolicy(refreshPolicy);
         }
-        if (descriptorList.isEmpty() == false || randomBoolean()) {
-            request.setRoleDescriptors(descriptorList);
-        }
+        request.setRoleDescriptors(descriptorList);
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             request.writeTo(out);
@@ -106,7 +110,11 @@ public void testSerialization() throws IOException {
                 assertEquals(name, serialized.getName());
                 assertEquals(expiration, serialized.getExpiration());
                 assertEquals(refreshPolicy, serialized.getRefreshPolicy());
-                assertEquals(descriptorList, serialized.getRoleDescriptors());
+                if (nullOrEmptyRoleDescriptors) {
+                    assertThat(serialized.getRoleDescriptors().isEmpty(), is(true));
+                } else {
+                    assertEquals(descriptorList, serialized.getRoleDescriptors());
+                }
             }
         }
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponseTests.java
new file mode 100644
index 0000000000000..f52b093d62b63
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetBuiltinPrivilegesResponseTests.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+
+public class GetBuiltinPrivilegesResponseTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        final String[] cluster = generateRandomStringArray(8, randomIntBetween(3, 8), false, true);
+        final String[] index = generateRandomStringArray(8, randomIntBetween(3, 8), false, true);
+        final GetBuiltinPrivilegesResponse original = new GetBuiltinPrivilegesResponse(cluster, index);
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        original.writeTo(out);
+
+        final GetBuiltinPrivilegesResponse copy = new GetBuiltinPrivilegesResponse();
+        copy.readFrom(out.bytes().streamInput());
+
+        assertThat(copy.getClusterPrivileges(), Matchers.equalTo(cluster));
+        assertThat(copy.getIndexPrivileges(), Matchers.equalTo(index));
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequestTests.java
index 4d67b82dfd4c0..db867f6775722 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesRequestTests.java
@@ -45,7 +45,8 @@ public void testValidation() {
         assertThat(request("my_app", "read", "write").validate(), nullValue());
         final ActionRequestValidationException exception = request("my_app", ((String[]) null)).validate();
         assertThat(exception, notNullValue());
-        assertThat(exception.validationErrors(), containsInAnyOrder("privileges cannot be null"));
+        assertThat(exception.validationErrors(),
+            containsInAnyOrder("privileges cannot be null"));
     }
 
     private GetPrivilegesRequest request(String application, String... privileges) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesResponseTests.java
index 50c247967a780..fe75dfbd52433 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/GetPrivilegesResponseTests.java
@@ -19,15 +19,7 @@
 public class GetPrivilegesResponseTests extends ESTestCase {
 
     public void testSerialization() throws IOException {
-        ApplicationPrivilegeDescriptor[] privileges = randomArray(6, ApplicationPrivilegeDescriptor[]::new, () ->
-            new ApplicationPrivilegeDescriptor(
-                randomAlphaOfLengthBetween(3, 8).toLowerCase(Locale.ROOT),
-                randomAlphaOfLengthBetween(3, 8).toLowerCase(Locale.ROOT),
-                Sets.newHashSet(randomArray(3, String[]::new, () -> randomAlphaOfLength(3).toLowerCase(Locale.ROOT) + "/*")),
-                Collections.emptyMap()
-            )
-        );
-        final GetPrivilegesResponse original = new GetPrivilegesResponse(privileges);
+        final GetPrivilegesResponse original = randomResponse();
 
         final BytesStreamOutput out = new BytesStreamOutput();
         original.writeTo(out);
@@ -38,4 +30,16 @@ public void testSerialization() throws IOException {
         assertThat(copy.privileges(), Matchers.equalTo(original.privileges()));
     }
 
+    private static GetPrivilegesResponse randomResponse() {
+        ApplicationPrivilegeDescriptor[] application = randomArray(6, ApplicationPrivilegeDescriptor[]::new, () ->
+            new ApplicationPrivilegeDescriptor(
+                randomAlphaOfLengthBetween(3, 8).toLowerCase(Locale.ROOT),
+                randomAlphaOfLengthBetween(3, 8).toLowerCase(Locale.ROOT),
+                Sets.newHashSet(randomArray(3, String[]::new, () -> randomAlphaOfLength(3).toLowerCase(Locale.ROOT) + "/*")),
+                Collections.emptyMap()
+            )
+        );
+        return new GetPrivilegesResponse(application);
+    }
+
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequestTests.java
index 2d8782f0111e6..54681e97fc531 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequestTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenRequestTests.java
@@ -6,8 +6,15 @@
 package org.elasticsearch.xpack.core.security.action.token;
 
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.action.token.CreateTokenRequest.GrantType;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.hasItem;
@@ -19,7 +26,7 @@ public void testRequestValidation() {
         ActionRequestValidationException ve = request.validate();
         assertNotNull(ve);
         assertEquals(1, ve.validationErrors().size());
-        assertThat(ve.validationErrors().get(0), containsString("[password, refresh_token, client_credentials]"));
+        assertThat(ve.validationErrors().get(0), containsString("[password, _kerberos, refresh_token, client_credentials]"));
         assertThat(ve.validationErrors().get(0), containsString("grant_type"));
 
         request.setGrantType("password");
@@ -49,20 +56,24 @@ public void testRequestValidation() {
         assertNull(ve);
 
         request.setRefreshToken(randomAlphaOfLengthBetween(1, 10));
+        request.setKerberosTicket(new SecureString(randomAlphaOfLengthBetween(1, 256).toCharArray()));
         ve = request.validate();
         assertNotNull(ve);
-        assertEquals(1, ve.validationErrors().size());
-        assertThat(ve.validationErrors().get(0), containsString("refresh_token is not supported"));
+        assertEquals(2, ve.validationErrors().size());
+        assertThat(ve.validationErrors(), hasItem(containsString("kerberos_ticket is not supported")));
+        assertThat(ve.validationErrors(), hasItem(containsString("refresh_token is not supported")));
 
         request.setGrantType("refresh_token");
         ve = request.validate();
         assertNotNull(ve);
-        assertEquals(2, ve.validationErrors().size());
+        assertEquals(3, ve.validationErrors().size());
         assertThat(ve.validationErrors(), hasItem(containsString("username is not supported")));
         assertThat(ve.validationErrors(), hasItem(containsString("password is not supported")));
+        assertThat(ve.validationErrors(), hasItem(containsString("kerberos_ticket is not supported")));
 
         request.setUsername(null);
         request.setPassword(null);
+        request.setKerberosTicket(null);
         ve = request.validate();
         assertNull(ve);
 
@@ -78,12 +89,59 @@ public void testRequestValidation() {
 
         request.setUsername(randomAlphaOfLengthBetween(1, 32));
         request.setPassword(new SecureString(randomAlphaOfLengthBetween(1, 32).toCharArray()));
+        request.setKerberosTicket(new SecureString(randomAlphaOfLengthBetween(1, 256).toCharArray()));
         request.setRefreshToken(randomAlphaOfLengthBetween(1, 32));
         ve = request.validate();
         assertNotNull(ve);
+        assertEquals(4, ve.validationErrors().size());
+        assertThat(ve.validationErrors(), hasItem(containsString("username is not supported")));
+        assertThat(ve.validationErrors(), hasItem(containsString("password is not supported")));
+        assertThat(ve.validationErrors(), hasItem(containsString("refresh_token is not supported")));
+        assertThat(ve.validationErrors(), hasItem(containsString("kerberos_ticket is not supported")));
+
+        request.setGrantType("_kerberos");
+        ve = request.validate();
+        assertNotNull(ve);
         assertEquals(3, ve.validationErrors().size());
         assertThat(ve.validationErrors(), hasItem(containsString("username is not supported")));
         assertThat(ve.validationErrors(), hasItem(containsString("password is not supported")));
         assertThat(ve.validationErrors(), hasItem(containsString("refresh_token is not supported")));
+
+        request.setUsername(null);
+        request.setPassword(null);
+        request.setRefreshToken(null);
+        ve = request.validate();
+        assertNull(ve);
+    }
+
+    public void testSerialization() throws IOException {
+        final String grantType = randomFrom(Arrays.stream(GrantType.values()).map(gt -> gt.getValue()).collect(Collectors.toList()));
+        final String username = randomBoolean() ? randomAlphaOfLength(5) : null;
+        final String scope = randomBoolean() ? randomAlphaOfLength(5) : null;
+        final SecureString password = randomBoolean() ? new SecureString(new char[] { 'p', 'a', 's', 's' }) : null;
+        final SecureString kerberosTicket = randomBoolean() ? new SecureString(new char[] { 'k', 'e', 'r', 'b' }) : null;
+        final String refreshToken = randomBoolean() ? randomAlphaOfLength(5) : null;
+        final CreateTokenRequest request = new CreateTokenRequest(grantType, username, password, kerberosTicket, scope, refreshToken);
+
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            request.writeTo(out);
+            try (StreamInput in = out.bytes().streamInput()) {
+                final CreateTokenRequest serialized = new CreateTokenRequest();
+                serialized.readFrom(in);
+                assertEquals(grantType, serialized.getGrantType());
+                if (scope != null) {
+                    assertEquals(scope, serialized.getScope());
+                }
+                if (password != null) {
+                    assertEquals(password, serialized.getPassword());
+                }
+                if (kerberosTicket != null) {
+                    assertEquals(kerberosTicket, serialized.getKerberosTicket());
+                }
+                if (refreshToken != null) {
+                    assertEquals(refreshToken, serialized.getRefreshToken());
+                }
+            }
+        }
     }
 }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenResponseTests.java
index ed73ef0562b43..e89357698c708 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenResponseTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/CreateTokenResponseTests.java
@@ -14,7 +14,7 @@ public class CreateTokenResponseTests extends ESTestCase {
 
     public void testSerialization() throws Exception {
         CreateTokenResponse response = new CreateTokenResponse(randomAlphaOfLengthBetween(1, 10), TimeValue.timeValueMinutes(20L),
-            randomBoolean() ? null : "FULL", randomAlphaOfLengthBetween(1, 10));
+            randomBoolean() ? null : "FULL", randomAlphaOfLengthBetween(1, 10), randomBoolean() ? null :randomAlphaOfLengthBetween(1, 10));
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             response.writeTo(output);
             try (StreamInput input = output.bytes().streamInput()) {
@@ -25,7 +25,7 @@ public void testSerialization() throws Exception {
         }
 
         response = new CreateTokenResponse(randomAlphaOfLengthBetween(1, 10), TimeValue.timeValueMinutes(20L),
-            randomBoolean() ? null : "FULL", null);
+            randomBoolean() ? null : "FULL", null, null);
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             response.writeTo(output);
             try (StreamInput input = output.bytes().streamInput()) {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java
new file mode 100644
index 0000000000000..df2c63f357a60
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java
@@ -0,0 +1,247 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authz.accesscontrol;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BitSet;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.hamcrest.Matchers;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DocumentSubsetBitsetCacheTests extends ESTestCase {
+
+    public void testSameBitSetIsReturnedForIdenticalQuery() throws Exception {
+        final DocumentSubsetBitsetCache cache = new DocumentSubsetBitsetCache(Settings.EMPTY);
+        runTestOnIndex((shardContext, leafContext) -> {
+            final Query query1 = QueryBuilders.termQuery("field-1", "value-1").toQuery(shardContext);
+            final BitSet bitSet1 = cache.getBitSet(query1, leafContext);
+            assertThat(bitSet1, notNullValue());
+
+            final Query query2 = QueryBuilders.termQuery("field-1", "value-1").toQuery(shardContext);
+            final BitSet bitSet2 = cache.getBitSet(query2, leafContext);
+            assertThat(bitSet2, notNullValue());
+
+            assertThat(bitSet2, Matchers.sameInstance(bitSet1));
+        });
+    }
+
+    public void testNullBitSetIsReturnedForNonMatchingQuery() throws Exception {
+        final DocumentSubsetBitsetCache cache = new DocumentSubsetBitsetCache(Settings.EMPTY);
+        runTestOnIndex((shardContext, leafContext) -> {
+            final Query query = QueryBuilders.termQuery("does-not-exist", "any-value").toQuery(shardContext);
+            final BitSet bitSet = cache.getBitSet(query, leafContext);
+            assertThat(bitSet, nullValue());
+        });
+    }
+
+    public void testNullEntriesAreNotCountedInMemoryUsage() throws Exception {
+        final DocumentSubsetBitsetCache cache = new DocumentSubsetBitsetCache(Settings.EMPTY);
+        assertThat(cache.ramBytesUsed(), equalTo(0L));
+
+        runTestOnIndex((shardContext, leafContext) -> {
+            for (int i = 1; i <= randomIntBetween(3, 6); i++) {
+                final Query query = QueryBuilders.termQuery("dne-" + i, "dne- " + i).toQuery(shardContext);
+                final BitSet bitSet = cache.getBitSet(query, leafContext);
+                assertThat(bitSet, nullValue());
+                assertThat(cache.ramBytesUsed(), equalTo(0L));
+                assertThat(cache.entryCount(), equalTo(i));
+            }
+        });
+    }
+
+    public void testCacheRespectsMemoryLimit() throws Exception {
+        // This value is based on the internal implementation details of lucene's FixedBitSet
+        // If the implementation changes, this can be safely updated to match the new ram usage for a single bitset
+        final long expectedBytesPerBitSet = 56;
+
+        // Enough to hold exactly 2 bit-sets in the cache
+        final long maxCacheBytes = expectedBytesPerBitSet * 2;
+        final Settings settings = Settings.builder()
+            .put(DocumentSubsetBitsetCache.CACHE_SIZE_SETTING.getKey(), maxCacheBytes + "b")
+            .build();
+        final DocumentSubsetBitsetCache cache = new DocumentSubsetBitsetCache(settings);
+        assertThat(cache.entryCount(), equalTo(0));
+        assertThat(cache.ramBytesUsed(), equalTo(0L));
+
+        runTestOnIndex((shardContext, leafContext) -> {
+            Query previousQuery = null;
+            BitSet previousBitSet = null;
+            for (int i = 1; i <= 5; i++) {
+                final TermQueryBuilder queryBuilder = QueryBuilders.termQuery("field-" + i, "value-" + i);
+                final Query query = queryBuilder.toQuery(shardContext);
+                final BitSet bitSet = cache.getBitSet(query, leafContext);
+                assertThat(bitSet, notNullValue());
+                assertThat(bitSet.ramBytesUsed(), equalTo(expectedBytesPerBitSet));
+
+                // The first time through we have 1 entry, after that we have 2
+                final int expectedCount = i == 1 ? 1 : 2;
+                assertThat(cache.entryCount(), equalTo(expectedCount));
+                assertThat(cache.ramBytesUsed(), equalTo(expectedCount * expectedBytesPerBitSet));
+
+                // Older queries should get evicted, but the query from last iteration should still be cached
+                if (previousQuery != null) {
+                    assertThat(cache.getBitSet(previousQuery, leafContext), sameInstance(previousBitSet));
+                    assertThat(cache.entryCount(), equalTo(expectedCount));
+                    assertThat(cache.ramBytesUsed(), equalTo(expectedCount * expectedBytesPerBitSet));
+                }
+                previousQuery = query;
+                previousBitSet = bitSet;
+
+                assertThat(cache.getBitSet(queryBuilder.toQuery(shardContext), leafContext), sameInstance(bitSet));
+                assertThat(cache.entryCount(), equalTo(expectedCount));
+                assertThat(cache.ramBytesUsed(), equalTo(expectedCount * expectedBytesPerBitSet));
+            }
+
+            assertThat(cache.entryCount(), equalTo(2));
+            assertThat(cache.ramBytesUsed(), equalTo(2 * expectedBytesPerBitSet));
+
+            cache.clear("testing");
+
+            assertThat(cache.entryCount(), equalTo(0));
+            assertThat(cache.ramBytesUsed(), equalTo(0L));
+        });
+    }
+
+    public void testCacheRespectsAccessTimeExpiry() throws Exception {
+        final Settings settings = Settings.builder()
+            .put(DocumentSubsetBitsetCache.CACHE_TTL_SETTING.getKey(), "10ms")
+            .build();
+        final DocumentSubsetBitsetCache cache = new DocumentSubsetBitsetCache(settings);
+        assertThat(cache.entryCount(), equalTo(0));
+        assertThat(cache.ramBytesUsed(), equalTo(0L));
+
+        runTestOnIndex((shardContext, leafContext) -> {
+            final Query query1 = QueryBuilders.termQuery("field-1", "value-1").toQuery(shardContext);
+            final BitSet bitSet1 = cache.getBitSet(query1, leafContext);
+            assertThat(bitSet1, notNullValue());
+
+            final Query query2 = QueryBuilders.termQuery("field-2", "value-2").toQuery(shardContext);
+            assertBusy(() -> {
+                // Force the cache to perform eviction
+                final BitSet bitSet2 = cache.getBitSet(query2, leafContext);
+                assertThat(bitSet2, notNullValue());
+
+                // Loop until the cache has less than 2 items, which mean that something we evicted
+                assertThat(cache.entryCount(), Matchers.lessThan(2));
+
+            }, 100, TimeUnit.MILLISECONDS);
+
+            // Check that the original bitset is no longer in the cache (a new instance is returned)
+            assertThat(cache.getBitSet(query1, leafContext), not(sameInstance(bitSet1)));
+        });
+    }
+
+    public void testCacheIsPerIndex() throws Exception {
+        final DocumentSubsetBitsetCache cache = new DocumentSubsetBitsetCache(Settings.EMPTY);
+        assertThat(cache.entryCount(), equalTo(0));
+        assertThat(cache.ramBytesUsed(), equalTo(0L));
+
+        final int iterations = randomIntBetween(3, 10);
+        AtomicInteger counter = new AtomicInteger(0);
+
+        final CheckedBiConsumer<QueryShardContext, LeafReaderContext, Exception> consumer = new CheckedBiConsumer<>() {
+            @Override
+            public void accept(QueryShardContext shardContext, LeafReaderContext leafContext) throws Exception {
+                final int count = counter.incrementAndGet();
+                final Query query = QueryBuilders.termQuery("field-1", "value-1").toQuery(shardContext);
+                final BitSet bitSet = cache.getBitSet(query, leafContext);
+
+                assertThat(bitSet, notNullValue());
+                assertThat(cache.entryCount(), equalTo(count));
+
+                if (count < iterations) {
+                    // Need to do this nested, or else the cache will be cleared when the index reader is closed
+                    runTestOnIndex(this);
+                }
+            }
+        };
+        runTestOnIndex(consumer);
+    }
+
+    public void testCacheClearEntriesWhenIndexIsClosed() throws Exception {
+        final DocumentSubsetBitsetCache cache = new DocumentSubsetBitsetCache(Settings.EMPTY);
+        assertThat(cache.entryCount(), equalTo(0));
+        assertThat(cache.ramBytesUsed(), equalTo(0L));
+
+        for (int i = 1; i <= randomIntBetween(2, 5); i++) {
+            runTestOnIndex((shardContext, leafContext) -> {
+                for (int j = 1; j <= randomIntBetween(2, 10); j++) {
+                    final Query query = QueryBuilders.termQuery("field-" + j, "value-1").toQuery(shardContext);
+                    final BitSet bitSet = cache.getBitSet(query, leafContext);
+                    assertThat(bitSet, notNullValue());
+                }
+                assertThat(cache.entryCount(), not(equalTo(0)));
+                assertThat(cache.ramBytesUsed(), not(equalTo(0L)));
+            });
+            assertThat(cache.entryCount(), equalTo(0));
+            assertThat(cache.ramBytesUsed(), equalTo(0L));
+        }
+    }
+
+    private void runTestOnIndex(CheckedBiConsumer<QueryShardContext, LeafReaderContext, Exception> body) throws Exception {
+        final ShardId shardId = new ShardId("idx_" + randomAlphaOfLengthBetween(2, 8), randomAlphaOfLength(12), 0);
+        final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.getIndex(), Settings.EMPTY);
+        final MapperService mapperService = mock(MapperService.class);
+        final long nowInMillis = randomNonNegativeLong();
+
+        final Client client = mock(Client.class);
+        when(client.settings()).thenReturn(Settings.EMPTY);
+
+        final IndexWriterConfig writerConfig = new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(NoMergePolicy.INSTANCE);
+        try (Directory directory = newDirectory();
+             IndexWriter iw = new IndexWriter(directory, writerConfig)) {
+            for (int i = 1; i <= 100; i++) {
+                Document document = new Document();
+                for (int j = 1; j <= 10; j++) {
+                    document.add(new StringField("field-" + j, "value-" + i, Field.Store.NO));
+                }
+                iw.addDocument(document);
+            }
+            iw.commit();
+
+            try (DirectoryReader directoryReader = DirectoryReader.open(directory)) {
+                final LeafReaderContext leaf = directoryReader.leaves().get(0);
+
+                final QueryShardContext context = new QueryShardContext(shardId.id(), indexSettings, null, null, null, mapperService,
+                    null, null, xContentRegistry(), writableRegistry(), client, leaf.reader(), () -> nowInMillis, null);
+
+                body.accept(context, leaf);
+            }
+        }
+    }
+
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java
index bd6ac12ee3c1b..c84c0027302e6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetReaderTests.java
@@ -21,18 +21,13 @@
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.Bits;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.apache.lucene.util.TestUtil;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetReader;
 import org.junit.After;
 import org.junit.Before;
 
@@ -45,7 +40,7 @@ public class DocumentSubsetReaderTests extends ESTestCase {
 
     private Directory directory;
     private DirectoryReader directoryReader;
-    private BitsetFilterCache bitsetFilterCache;
+    private DocumentSubsetBitsetCache bitsetCache;
 
     @Before
     public void setUpDirectory() {
@@ -55,18 +50,7 @@ public void setUpDirectory() {
         assertTrue(DocumentSubsetReader.NUM_DOCS_CACHE.toString(),
                 DocumentSubsetReader.NUM_DOCS_CACHE.isEmpty());
         directory = newDirectory();
-        IndexSettings settings = IndexSettingsModule.newIndexSettings("_index", Settings.EMPTY);
-        bitsetFilterCache = new BitsetFilterCache(settings, new BitsetFilterCache.Listener() {
-            @Override
-            public void onCache(ShardId shardId, Accountable accountable) {
-
-            }
-
-            @Override
-            public void onRemoval(ShardId shardId, Accountable accountable) {
-
-            }
-        });
+        bitsetCache = new DocumentSubsetBitsetCache(Settings.EMPTY);
     }
 
     @After
@@ -77,7 +61,7 @@ public void cleanDirectory() throws Exception {
         assertTrue(DocumentSubsetReader.NUM_DOCS_CACHE.toString(),
                 DocumentSubsetReader.NUM_DOCS_CACHE.isEmpty());
         directory.close();
-        bitsetFilterCache.close();
+        bitsetCache.close();
     }
 
     public void testSearch() throws Exception {
@@ -104,14 +88,14 @@ public void testSearch() throws Exception {
         iw.close();
         openDirectoryReader();
 
-        IndexSearcher indexSearcher = new IndexSearcher(DocumentSubsetReader.wrap(directoryReader, bitsetFilterCache,
+        IndexSearcher indexSearcher = new IndexSearcher(DocumentSubsetReader.wrap(directoryReader, bitsetCache,
                 new TermQuery(new Term("field", "value1"))));
         assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(1));
         TopDocs result = indexSearcher.search(new MatchAllDocsQuery(), 1);
         assertThat(result.totalHits.value, equalTo(1L));
         assertThat(result.scoreDocs[0].doc, equalTo(0));
 
-        indexSearcher = new IndexSearcher(DocumentSubsetReader.wrap(directoryReader, bitsetFilterCache,
+        indexSearcher = new IndexSearcher(DocumentSubsetReader.wrap(directoryReader, bitsetCache,
                 new TermQuery(new Term("field", "value2"))));
         assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(1));
         result = indexSearcher.search(new MatchAllDocsQuery(), 1);
@@ -119,13 +103,13 @@ public void testSearch() throws Exception {
         assertThat(result.scoreDocs[0].doc, equalTo(1));
 
         // this doc has been marked as deleted:
-        indexSearcher = new IndexSearcher(DocumentSubsetReader.wrap(directoryReader, bitsetFilterCache,
+        indexSearcher = new IndexSearcher(DocumentSubsetReader.wrap(directoryReader, bitsetCache,
                 new TermQuery(new Term("field", "value3"))));
         assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(0));
         result = indexSearcher.search(new MatchAllDocsQuery(), 1);
         assertThat(result.totalHits.value, equalTo(0L));
 
-        indexSearcher = new IndexSearcher(DocumentSubsetReader.wrap(directoryReader, bitsetFilterCache,
+        indexSearcher = new IndexSearcher(DocumentSubsetReader.wrap(directoryReader, bitsetCache,
                 new TermQuery(new Term("field", "value4"))));
         assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(1));
         result = indexSearcher.search(new MatchAllDocsQuery(), 1);
@@ -154,7 +138,7 @@ public void testLiveDocs() throws Exception {
 
         for (int i = 0; i < numDocs; i++) {
             Query roleQuery = new TermQuery(new Term("field", "value" + i));
-            DirectoryReader wrappedReader = DocumentSubsetReader.wrap(directoryReader, bitsetFilterCache, roleQuery);
+            DirectoryReader wrappedReader = DocumentSubsetReader.wrap(directoryReader, bitsetCache, roleQuery);
 
             LeafReader leafReader = wrappedReader.leaves().get(0).reader();
             assertThat(leafReader.hasDeletions(), is(true));
@@ -176,26 +160,16 @@ public void testWrapTwice() throws Exception {
         IndexWriterConfig iwc = new IndexWriterConfig(null);
         IndexWriter iw = new IndexWriter(dir, iwc);
         iw.close();
-        IndexSettings settings = IndexSettingsModule.newIndexSettings("_index", Settings.EMPTY);
-        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new BitsetFilterCache.Listener() {
-            @Override
-            public void onCache(ShardId shardId, Accountable accountable) {
-            }
-
-            @Override
-            public void onRemoval(ShardId shardId, Accountable accountable) {
-            }
-        });
-        DirectoryReader directoryReader = DocumentSubsetReader.wrap(DirectoryReader.open(dir), bitsetFilterCache, new MatchAllDocsQuery());
+        DirectoryReader directoryReader = DocumentSubsetReader.wrap(DirectoryReader.open(dir), bitsetCache, new MatchAllDocsQuery());
         try {
-            DocumentSubsetReader.wrap(directoryReader, bitsetFilterCache, new MatchAllDocsQuery());
+            DocumentSubsetReader.wrap(directoryReader, bitsetCache, new MatchAllDocsQuery());
             fail("shouldn't be able to wrap DocumentSubsetDirectoryReader twice");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("Can't wrap [class org.elasticsearch.xpack.core.security.authz.accesscontrol" +
                     ".DocumentSubsetReader$DocumentSubsetDirectoryReader] twice"));
         }
 
-        bitsetFilterCache.close();
+        bitsetCache.close();
         directoryReader.close();
         dir.close();
     }
@@ -219,7 +193,7 @@ public void testCoreCacheKey() throws Exception {
 
         // open reader
         DirectoryReader ir = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(iw), new ShardId("_index", "_na_", 0));
-        ir = DocumentSubsetReader.wrap(ir, bitsetFilterCache, new MatchAllDocsQuery());
+        ir = DocumentSubsetReader.wrap(ir, bitsetCache, new MatchAllDocsQuery());
         assertEquals(2, ir.numDocs());
         assertEquals(1, ir.leaves().size());
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java
similarity index 84%
rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperIntegrationTests.java
rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java
index 5eccd4090e8bf..3be46a031a0b2 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperIntegrationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java
@@ -21,7 +21,6 @@
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TotalHitCountCollector;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.Accountable;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -30,7 +29,6 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -39,6 +37,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.test.AbstractBuilderTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
@@ -62,7 +61,7 @@
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.when;
 
-public class SecurityIndexSearcherWrapperIntegrationTests extends AbstractBuilderTestCase {
+public class SecurityIndexReaderWrapperIntegrationTests extends AbstractBuilderTestCase {
 
     public void testDLS() throws Exception {
         ShardId shardId = new ShardId("_index", "_na_", 0);
@@ -83,24 +82,14 @@ public void testDLS() throws Exception {
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         final long nowInMillis = randomNonNegativeLong();
-        QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, null, null, mapperService, null,
-                null, xContentRegistry(), writableRegistry(), client, null, () -> nowInMillis, null);
+        QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, null, null, null, mapperService,
+                null, null, xContentRegistry(), writableRegistry(), client, null, () -> nowInMillis, null);
         QueryShardContext queryShardContext = spy(realQueryShardContext);
-        IndexSettings settings = IndexSettingsModule.newIndexSettings("_index", Settings.EMPTY);
-        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new BitsetFilterCache.Listener() {
-            @Override
-            public void onCache(ShardId shardId, Accountable accountable) {
-            }
-
-            @Override
-            public void onRemoval(ShardId shardId, Accountable accountable) {
-
-            }
-        });
+        DocumentSubsetBitsetCache bitsetCache = new DocumentSubsetBitsetCache(Settings.EMPTY);
         XPackLicenseState licenseState = mock(XPackLicenseState.class);
         when(licenseState.isDocumentAndFieldLevelSecurityAllowed()).thenReturn(true);
-        SecurityIndexSearcherWrapper wrapper = new SecurityIndexSearcherWrapper(s -> queryShardContext,
-                bitsetFilterCache, threadContext, licenseState, scriptService) {
+        SecurityIndexReaderWrapper wrapper = new SecurityIndexReaderWrapper(s -> queryShardContext,
+                bitsetCache, threadContext, licenseState, scriptService) {
 
             @Override
             protected IndicesAccessControl getIndicesAccessControl() {
@@ -156,8 +145,9 @@ protected IndicesAccessControl getIndicesAccessControl() {
             ParsedQuery parsedQuery = new ParsedQuery(new TermQuery(new Term("field", values[i])));
             doReturn(new TermQueryBuilder("field", values[i])).when(queryShardContext).parseInnerQueryBuilder(any(XContentParser.class));
             when(queryShardContext.toQuery(new TermsQueryBuilder("field", values[i]))).thenReturn(parsedQuery);
-            DirectoryReader wrappedDirectoryReader = wrapper.wrap(directoryReader);
-            IndexSearcher indexSearcher = wrapper.wrap(new IndexSearcher(wrappedDirectoryReader));
+            DirectoryReader wrappedDirectoryReader = wrapper.apply(directoryReader);
+            IndexSearcher indexSearcher = new ContextIndexSearcher(wrappedDirectoryReader,
+                IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
 
             int expectedHitCount = valuesHitCount[i];
             logger.info("Going to verify hit count with query [{}] with expected total hits [{}]", parsedQuery.query(), expectedHitCount);
@@ -167,7 +157,7 @@ protected IndicesAccessControl getIndicesAccessControl() {
             assertThat(wrappedDirectoryReader.numDocs(), equalTo(expectedHitCount));
         }
 
-        bitsetFilterCache.close();
+        bitsetCache.close();
         directoryReader.close();
         directory.close();
     }
@@ -206,24 +196,15 @@ public void testDLSWithLimitedPermissions() throws Exception {
         Client client = mock(Client.class);
         when(client.settings()).thenReturn(Settings.EMPTY);
         final long nowInMillis = randomNonNegativeLong();
-        QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, null, null, mapperService, null,
-                null, xContentRegistry(), writableRegistry(), client, null, () -> nowInMillis, null);
+        QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, null, null, null, mapperService,
+                null, null, xContentRegistry(), writableRegistry(), client, null, () -> nowInMillis, null);
         QueryShardContext queryShardContext = spy(realQueryShardContext);
-        IndexSettings settings = IndexSettingsModule.newIndexSettings("_index", Settings.EMPTY);
-        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new BitsetFilterCache.Listener() {
-            @Override
-            public void onCache(ShardId shardId, Accountable accountable) {
-            }
-
-            @Override
-            public void onRemoval(ShardId shardId, Accountable accountable) {
-            }
-        });
+        DocumentSubsetBitsetCache bitsetCache = new DocumentSubsetBitsetCache(Settings.EMPTY);
 
         XPackLicenseState licenseState = mock(XPackLicenseState.class);
         when(licenseState.isDocumentAndFieldLevelSecurityAllowed()).thenReturn(true);
-        SecurityIndexSearcherWrapper wrapper = new SecurityIndexSearcherWrapper(s -> queryShardContext,
-                bitsetFilterCache, threadContext, licenseState, scriptService) {
+        SecurityIndexReaderWrapper wrapper = new SecurityIndexReaderWrapper(s -> queryShardContext,
+                bitsetCache, threadContext, licenseState, scriptService) {
 
             @Override
             protected IndicesAccessControl getIndicesAccessControl() {
@@ -259,8 +240,9 @@ protected IndicesAccessControl getIndicesAccessControl() {
         iw.close();
 
         DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory), shardId);
-        DirectoryReader wrappedDirectoryReader = wrapper.wrap(directoryReader);
-        IndexSearcher indexSearcher = wrapper.wrap(new IndexSearcher(wrappedDirectoryReader));
+        DirectoryReader wrappedDirectoryReader = wrapper.apply(directoryReader);
+        IndexSearcher indexSearcher = new ContextIndexSearcher(wrappedDirectoryReader,
+            IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy());
 
         ScoreDoc[] hits = indexSearcher.search(new MatchAllDocsQuery(), 1000).scoreDocs;
         Set<Integer> actualDocIds = new HashSet<>();
@@ -278,7 +260,7 @@ protected IndicesAccessControl getIndicesAccessControl() {
             }
         }
 
-        bitsetFilterCache.close();
+        bitsetCache.close();
         directoryReader.close();
         directory.close();
     }
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperUnitTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperUnitTests.java
new file mode 100644
index 0000000000000..0535c8aa4e211
--- /dev/null
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperUnitTests.java
@@ -0,0 +1,225 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.authz.accesscontrol;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.SeqNoFieldMapper;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions;
+import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
+import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class SecurityIndexReaderWrapperUnitTests extends ESTestCase {
+
+    private static final Set<String> META_FIELDS;
+    static {
+        final Set<String> metaFields = new HashSet<>(Arrays.asList(MapperService.getAllMetaFields()));
+        metaFields.add(SourceFieldMapper.NAME);
+        metaFields.add(FieldNamesFieldMapper.NAME);
+        metaFields.add(SeqNoFieldMapper.NAME);
+        META_FIELDS = Collections.unmodifiableSet(metaFields);
+    }
+
+    private ThreadContext threadContext;
+    private ScriptService scriptService;
+    private SecurityIndexReaderWrapper securityIndexReaderWrapper;
+    private ElasticsearchDirectoryReader esIn;
+    private XPackLicenseState licenseState;
+
+    @Before
+    public void setup() throws Exception {
+        Index index = new Index("_index", "testUUID");
+        scriptService = mock(ScriptService.class);
+
+        ShardId shardId = new ShardId(index, 0);
+        licenseState = mock(XPackLicenseState.class);
+        when(licenseState.isDocumentAndFieldLevelSecurityAllowed()).thenReturn(true);
+        threadContext = new ThreadContext(Settings.EMPTY);
+        IndexShard indexShard = mock(IndexShard.class);
+        when(indexShard.shardId()).thenReturn(shardId);
+
+        Directory directory = new MMapDirectory(createTempDir());
+        IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig());
+        writer.close();
+
+        DirectoryReader in = DirectoryReader.open(directory); // unfortunately DirectoryReader isn't mock friendly
+        esIn = ElasticsearchDirectoryReader.wrap(in, shardId);
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        esIn.close();
+    }
+
+    public void testDefaultMetaFields() throws Exception {
+        securityIndexReaderWrapper =
+                new SecurityIndexReaderWrapper(null, null, threadContext, licenseState, scriptService) {
+            @Override
+            protected IndicesAccessControl getIndicesAccessControl() {
+                IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true,
+                        new FieldPermissions(fieldPermissionDef(new String[]{}, null)), DocumentPermissions.allowAll());
+                return new IndicesAccessControl(true, singletonMap("_index", indexAccessControl));
+            }
+        };
+
+        FieldSubsetReader.FieldSubsetDirectoryReader result =
+                (FieldSubsetReader.FieldSubsetDirectoryReader) securityIndexReaderWrapper.apply(esIn);
+        assertThat(result.getFilter().run("_uid"), is(true));
+        assertThat(result.getFilter().run("_id"), is(true));
+        assertThat(result.getFilter().run("_version"), is(true));
+        assertThat(result.getFilter().run("_type"), is(true));
+        assertThat(result.getFilter().run("_source"), is(true));
+        assertThat(result.getFilter().run("_routing"), is(true));
+        assertThat(result.getFilter().run("_timestamp"), is(true));
+        assertThat(result.getFilter().run("_ttl"), is(true));
+        assertThat(result.getFilter().run("_size"), is(true));
+        assertThat(result.getFilter().run("_index"), is(true));
+        assertThat(result.getFilter().run("_field_names"), is(true));
+        assertThat(result.getFilter().run("_seq_no"), is(true));
+        assertThat(result.getFilter().run("_some_random_meta_field"), is(true));
+        assertThat(result.getFilter().run("some_random_regular_field"), is(false));
+    }
+
+    public void testWrapReaderWhenFeatureDisabled() throws Exception {
+        when(licenseState.isDocumentAndFieldLevelSecurityAllowed()).thenReturn(false);
+        securityIndexReaderWrapper =
+                new SecurityIndexReaderWrapper(null, null, threadContext, licenseState, scriptService);
+        DirectoryReader reader = securityIndexReaderWrapper.apply(esIn);
+        assertThat(reader, sameInstance(esIn));
+    }
+
+    public void testWildcards() throws Exception {
+        Set<String> expected = new HashSet<>(META_FIELDS);
+        expected.add("field1_a");
+        expected.add("field1_b");
+        expected.add("field1_c");
+        assertResolved(new FieldPermissions(fieldPermissionDef(new String[] {"field1*"}, null)), expected, "field", "field2");
+    }
+
+    public void testDotNotion() throws Exception {
+        Set<String> expected = new HashSet<>(META_FIELDS);
+        expected.add("foo.bar");
+        assertResolved(new FieldPermissions(fieldPermissionDef(new String[] {"foo.bar"}, null)), expected, "foo", "foo.baz", "bar.foo");
+
+        expected = new HashSet<>(META_FIELDS);
+        expected.add("foo.bar");
+        assertResolved(new FieldPermissions(fieldPermissionDef(new String[] {"foo.*"}, null)), expected, "foo", "bar");
+    }
+
+    private void assertResolved(FieldPermissions permissions, Set<String> expected, String... fieldsToTest) {
+        for (String field : expected) {
+            assertThat(field, permissions.grantsAccessTo(field), is(true));
+        }
+        for (String field : fieldsToTest) {
+            assertThat(field, permissions.grantsAccessTo(field), is(expected.contains(field)));
+        }
+    }
+
+    public void testFieldPermissionsWithFieldExceptions() throws Exception {
+        securityIndexReaderWrapper =
+                new SecurityIndexReaderWrapper(null, null, threadContext, licenseState, null);
+        String[] grantedFields = new String[]{};
+        String[] deniedFields;
+        Set<String> expected = new HashSet<>(META_FIELDS);
+        // Presence of fields in a role with an empty array implies access to no fields except the meta fields
+        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, randomBoolean() ? null : new String[]{})),
+                expected, "foo", "bar");
+
+        // make sure meta fields cannot be denied access to
+        deniedFields = META_FIELDS.toArray(new String[0]);
+        assertResolved(new FieldPermissions(fieldPermissionDef(null, deniedFields)),
+                new HashSet<>(Arrays.asList("foo", "bar", "_some_plugin_meta_field")));
+
+        // check we can add all fields with *
+        grantedFields = new String[]{"*"};
+        expected = new HashSet<>(META_FIELDS);
+        expected.add("foo");
+        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, randomBoolean() ? null : new String[]{})), expected);
+
+        // same with null
+        grantedFields = null;
+        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, randomBoolean() ? null : new String[]{})), expected);
+
+        // check we remove only excluded fields
+        grantedFields = new String[]{"*"};
+        deniedFields = new String[]{"xfield"};
+        expected = new HashSet<>(META_FIELDS);
+        expected.add("foo");
+        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "xfield");
+
+        // same with null
+        grantedFields = null;
+        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "xfield");
+
+        // some other checks
+        grantedFields = new String[]{"field*"};
+        deniedFields = new String[]{"field1", "field2"};
+        expected = new HashSet<>(META_FIELDS);
+        expected.add("field3");
+        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field1", "field2");
+
+        grantedFields = new String[]{"field1", "field2"};
+        deniedFields = new String[]{"field2"};
+        expected = new HashSet<>(META_FIELDS);
+        expected.add("field1");
+        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field1", "field2");
+
+        grantedFields = new String[]{"field*"};
+        deniedFields = new String[]{"field2"};
+        expected = new HashSet<>(META_FIELDS);
+        expected.add("field1");
+        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field2");
+
+        deniedFields = new String[]{"field*"};
+        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)),
+                META_FIELDS, "field1", "field2");
+
+        // empty array for allowed fields always means no field is allowed
+        grantedFields = new String[]{};
+        deniedFields = new String[]{};
+        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)),
+                META_FIELDS, "field1", "field2");
+
+        // make sure all field can be explicitly allowed
+        grantedFields = new String[]{"*"};
+        deniedFields = randomBoolean() ? null : new String[]{};
+        expected = new HashSet<>(META_FIELDS);
+        expected.add("field1");
+        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected);
+    }
+
+    private static FieldPermissionsDefinition fieldPermissionDef(String[] granted, String[] denied) {
+        return new FieldPermissionsDefinition(granted, denied);
+    }
+}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java
deleted file mode 100644
index 7900eaba4c848..0000000000000
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java
+++ /dev/null
@@ -1,539 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.core.security.authz.accesscontrol;
-
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.NoMergePolicy;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.BulkScorer;
-import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.LeafCollector;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MMapDirectory;
-import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.BitSet;
-import org.apache.lucene.util.FixedBitSet;
-import org.apache.lucene.util.SparseFixedBitSet;
-import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.core.internal.io.IOUtils;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
-import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.SeqNoFieldMapper;
-import org.elasticsearch.index.mapper.SourceFieldMapper;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.aggregations.LeafBucketCollector;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetReader.DocumentSubsetDirectoryReader;
-import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions;
-import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
-import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
-import org.junit.After;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.IdentityHashMap;
-import java.util.Set;
-
-import static java.util.Collections.singletonMap;
-import static org.elasticsearch.xpack.core.security.authz.accesscontrol.SecurityIndexSearcherWrapper.intersectScorerAndRoleBits;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.sameInstance;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase {
-
-    private static final Set<String> META_FIELDS;
-    static {
-        final Set<String> metaFields = new HashSet<>(Arrays.asList(MapperService.getAllMetaFields()));
-        metaFields.add(SourceFieldMapper.NAME);
-        metaFields.add(FieldNamesFieldMapper.NAME);
-        metaFields.add(SeqNoFieldMapper.NAME);
-        META_FIELDS = Collections.unmodifiableSet(metaFields);
-    }
-
-    private ThreadContext threadContext;
-    private ScriptService scriptService;
-    private SecurityIndexSearcherWrapper securityIndexSearcherWrapper;
-    private ElasticsearchDirectoryReader esIn;
-    private XPackLicenseState licenseState;
-    private IndexSettings indexSettings;
-
-    @Before
-    public void setup() throws Exception {
-        Index index = new Index("_index", "testUUID");
-        scriptService = mock(ScriptService.class);
-        indexSettings = IndexSettingsModule.newIndexSettings(index, Settings.EMPTY);
-
-        ShardId shardId = new ShardId(index, 0);
-        licenseState = mock(XPackLicenseState.class);
-        when(licenseState.isDocumentAndFieldLevelSecurityAllowed()).thenReturn(true);
-        threadContext = new ThreadContext(Settings.EMPTY);
-        IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.shardId()).thenReturn(shardId);
-
-        Directory directory = new MMapDirectory(createTempDir());
-        IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig());
-        writer.close();
-
-        DirectoryReader in = DirectoryReader.open(directory); // unfortunately DirectoryReader isn't mock friendly
-        esIn = ElasticsearchDirectoryReader.wrap(in, shardId);
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        super.tearDown();
-        esIn.close();
-    }
-
-    public void testDefaultMetaFields() throws Exception {
-        securityIndexSearcherWrapper =
-                new SecurityIndexSearcherWrapper(null, null, threadContext, licenseState, scriptService) {
-            @Override
-            protected IndicesAccessControl getIndicesAccessControl() {
-                IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true,
-                        new FieldPermissions(fieldPermissionDef(new String[]{}, null)), DocumentPermissions.allowAll());
-                return new IndicesAccessControl(true, singletonMap("_index", indexAccessControl));
-            }
-        };
-
-        FieldSubsetReader.FieldSubsetDirectoryReader result =
-                (FieldSubsetReader.FieldSubsetDirectoryReader) securityIndexSearcherWrapper.wrap(esIn);
-        assertThat(result.getFilter().run("_uid"), is(true));
-        assertThat(result.getFilter().run("_id"), is(true));
-        assertThat(result.getFilter().run("_version"), is(true));
-        assertThat(result.getFilter().run("_type"), is(true));
-        assertThat(result.getFilter().run("_source"), is(true));
-        assertThat(result.getFilter().run("_routing"), is(true));
-        assertThat(result.getFilter().run("_timestamp"), is(true));
-        assertThat(result.getFilter().run("_ttl"), is(true));
-        assertThat(result.getFilter().run("_size"), is(true));
-        assertThat(result.getFilter().run("_index"), is(true));
-        assertThat(result.getFilter().run("_field_names"), is(true));
-        assertThat(result.getFilter().run("_seq_no"), is(true));
-        assertThat(result.getFilter().run("_some_random_meta_field"), is(true));
-        assertThat(result.getFilter().run("some_random_regular_field"), is(false));
-    }
-
-    public void testWrapReaderWhenFeatureDisabled() throws Exception {
-        when(licenseState.isDocumentAndFieldLevelSecurityAllowed()).thenReturn(false);
-        securityIndexSearcherWrapper =
-                new SecurityIndexSearcherWrapper(null, null, threadContext, licenseState, scriptService);
-        DirectoryReader reader = securityIndexSearcherWrapper.wrap(esIn);
-        assertThat(reader, sameInstance(esIn));
-    }
-
-    public void testWrapSearcherWhenFeatureDisabled() throws Exception {
-        securityIndexSearcherWrapper =
-                new SecurityIndexSearcherWrapper(null, null, threadContext, licenseState, scriptService);
-        IndexSearcher indexSearcher = new IndexSearcher(esIn);
-        IndexSearcher result = securityIndexSearcherWrapper.wrap(indexSearcher);
-        assertThat(result, sameInstance(indexSearcher));
-    }
-
-    public void testWildcards() throws Exception {
-        Set<String> expected = new HashSet<>(META_FIELDS);
-        expected.add("field1_a");
-        expected.add("field1_b");
-        expected.add("field1_c");
-        assertResolved(new FieldPermissions(fieldPermissionDef(new String[] {"field1*"}, null)), expected, "field", "field2");
-    }
-
-    public void testDotNotion() throws Exception {
-        Set<String> expected = new HashSet<>(META_FIELDS);
-        expected.add("foo.bar");
-        assertResolved(new FieldPermissions(fieldPermissionDef(new String[] {"foo.bar"}, null)), expected, "foo", "foo.baz", "bar.foo");
-
-        expected = new HashSet<>(META_FIELDS);
-        expected.add("foo.bar");
-        assertResolved(new FieldPermissions(fieldPermissionDef(new String[] {"foo.*"}, null)), expected, "foo", "bar");
-    }
-
-    public void testDelegateSimilarity() throws Exception {
-        IndexSettings settings = IndexSettingsModule.newIndexSettings("_index", Settings.EMPTY);
-        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new BitsetFilterCache.Listener() {
-            @Override
-            public void onCache(ShardId shardId, Accountable accountable) {
-            }
-
-            @Override
-            public void onRemoval(ShardId shardId, Accountable accountable) {
-
-            }
-        });
-        DirectoryReader directoryReader = DocumentSubsetReader.wrap(esIn, bitsetFilterCache, new MatchAllDocsQuery());
-        IndexSearcher indexSearcher = new IndexSearcher(directoryReader);
-        securityIndexSearcherWrapper =
-                new SecurityIndexSearcherWrapper(null, null, threadContext, licenseState, scriptService);
-        IndexSearcher result = securityIndexSearcherWrapper.wrap(indexSearcher);
-        assertThat(result, not(sameInstance(indexSearcher)));
-        assertThat(result.getSimilarity(), sameInstance(indexSearcher.getSimilarity()));
-        bitsetFilterCache.close();
-    }
-
-    public void testIntersectScorerAndRoleBits() throws Exception {
-        securityIndexSearcherWrapper =
-                new SecurityIndexSearcherWrapper(null, null, threadContext, licenseState, scriptService);
-        final Directory directory = newDirectory();
-        IndexWriter iw = new IndexWriter(
-                directory,
-                new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(NoMergePolicy.INSTANCE)
-        );
-
-        Document document = new Document();
-        document.add(new StringField("field1", "value1", Field.Store.NO));
-        document.add(new StringField("field2", "value1", Field.Store.NO));
-        iw.addDocument(document);
-
-        document = new Document();
-        document.add(new StringField("field1", "value2", Field.Store.NO));
-        document.add(new StringField("field2", "value1", Field.Store.NO));
-        iw.addDocument(document);
-
-        document = new Document();
-        document.add(new StringField("field1", "value3", Field.Store.NO));
-        document.add(new StringField("field2", "value1", Field.Store.NO));
-        iw.addDocument(document);
-
-        document = new Document();
-        document.add(new StringField("field1", "value4", Field.Store.NO));
-        document.add(new StringField("field2", "value1", Field.Store.NO));
-        iw.addDocument(document);
-
-        iw.commit();
-        iw.deleteDocuments(new Term("field1", "value3"));
-        iw.close();
-        DirectoryReader directoryReader = DirectoryReader.open(directory);
-        IndexSearcher searcher = new IndexSearcher(directoryReader);
-        Weight weight = searcher.createWeight(new TermQuery(new Term("field2", "value1")),
-                org.apache.lucene.search.ScoreMode.COMPLETE_NO_SCORES, 1f);
-
-        LeafReaderContext leaf = directoryReader.leaves().get(0);
-
-        SparseFixedBitSet sparseFixedBitSet = query(leaf, "field1", "value1");
-        LeafCollector leafCollector = new LeafBucketCollector() {
-            @Override
-            public void collect(int doc, long bucket) throws IOException {
-                assertThat(doc, equalTo(0));
-            }
-        };
-        intersectScorerAndRoleBits(weight.scorer(leaf), sparseFixedBitSet, leafCollector, leaf.reader().getLiveDocs());
-
-        sparseFixedBitSet = query(leaf, "field1", "value2");
-        leafCollector = new LeafBucketCollector() {
-            @Override
-            public void collect(int doc, long bucket) throws IOException {
-                assertThat(doc, equalTo(1));
-            }
-        };
-        intersectScorerAndRoleBits(weight.scorer(leaf), sparseFixedBitSet, leafCollector, leaf.reader().getLiveDocs());
-
-
-        sparseFixedBitSet = query(leaf, "field1", "value3");
-        leafCollector = new LeafBucketCollector() {
-            @Override
-            public void collect(int doc, long bucket) throws IOException {
-                fail("docId [" + doc + "] should have been deleted");
-            }
-        };
-        intersectScorerAndRoleBits(weight.scorer(leaf), sparseFixedBitSet, leafCollector, leaf.reader().getLiveDocs());
-
-        sparseFixedBitSet = query(leaf, "field1", "value4");
-        leafCollector = new LeafBucketCollector() {
-            @Override
-            public void collect(int doc, long bucket) throws IOException {
-                assertThat(doc, equalTo(3));
-            }
-        };
-        intersectScorerAndRoleBits(weight.scorer(leaf), sparseFixedBitSet, leafCollector, leaf.reader().getLiveDocs());
-
-        directoryReader.close();
-        directory.close();
-    }
-
-    private void assertResolved(FieldPermissions permissions, Set<String> expected, String... fieldsToTest) {
-        for (String field : expected) {
-            assertThat(field, permissions.grantsAccessTo(field), is(true));
-        }
-        for (String field : fieldsToTest) {
-            assertThat(field, permissions.grantsAccessTo(field), is(expected.contains(field)));
-        }
-    }
-
-    public void testFieldPermissionsWithFieldExceptions() throws Exception {
-        securityIndexSearcherWrapper =
-                new SecurityIndexSearcherWrapper(null, null, threadContext, licenseState, null);
-        String[] grantedFields = new String[]{};
-        String[] deniedFields;
-        Set<String> expected = new HashSet<>(META_FIELDS);
-        // Presence of fields in a role with an empty array implies access to no fields except the meta fields
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, randomBoolean() ? null : new String[]{})),
-                expected, "foo", "bar");
-
-        // make sure meta fields cannot be denied access to
-        deniedFields = META_FIELDS.toArray(new String[0]);
-        assertResolved(new FieldPermissions(fieldPermissionDef(null, deniedFields)),
-                new HashSet<>(Arrays.asList("foo", "bar", "_some_plugin_meta_field")));
-
-        // check we can add all fields with *
-        grantedFields = new String[]{"*"};
-        expected = new HashSet<>(META_FIELDS);
-        expected.add("foo");
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, randomBoolean() ? null : new String[]{})), expected);
-
-        // same with null
-        grantedFields = null;
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, randomBoolean() ? null : new String[]{})), expected);
-
-        // check we remove only excluded fields
-        grantedFields = new String[]{"*"};
-        deniedFields = new String[]{"xfield"};
-        expected = new HashSet<>(META_FIELDS);
-        expected.add("foo");
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "xfield");
-
-        // same with null
-        grantedFields = null;
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "xfield");
-
-        // some other checks
-        grantedFields = new String[]{"field*"};
-        deniedFields = new String[]{"field1", "field2"};
-        expected = new HashSet<>(META_FIELDS);
-        expected.add("field3");
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field1", "field2");
-
-        grantedFields = new String[]{"field1", "field2"};
-        deniedFields = new String[]{"field2"};
-        expected = new HashSet<>(META_FIELDS);
-        expected.add("field1");
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field1", "field2");
-
-        grantedFields = new String[]{"field*"};
-        deniedFields = new String[]{"field2"};
-        expected = new HashSet<>(META_FIELDS);
-        expected.add("field1");
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected, "field2");
-
-        deniedFields = new String[]{"field*"};
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)),
-                META_FIELDS, "field1", "field2");
-
-        // empty array for allowed fields always means no field is allowed
-        grantedFields = new String[]{};
-        deniedFields = new String[]{};
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)),
-                META_FIELDS, "field1", "field2");
-
-        // make sure all field can be explicitly allowed
-        grantedFields = new String[]{"*"};
-        deniedFields = randomBoolean() ? null : new String[]{};
-        expected = new HashSet<>(META_FIELDS);
-        expected.add("field1");
-        assertResolved(new FieldPermissions(fieldPermissionDef(grantedFields, deniedFields)), expected);
-    }
-
-    private SparseFixedBitSet query(LeafReaderContext leaf, String field, String value) throws IOException {
-        SparseFixedBitSet sparseFixedBitSet = new SparseFixedBitSet(leaf.reader().maxDoc());
-        TermsEnum tenum = leaf.reader().terms(field).iterator();
-        while (tenum.next().utf8ToString().equals(value) == false) {
-        }
-        PostingsEnum penum = tenum.postings(null);
-        sparseFixedBitSet.or(penum);
-        return sparseFixedBitSet;
-    }
-
-    public void testIndexSearcherWrapperSparseNoDeletions() throws IOException {
-        doTestIndexSearcherWrapper(true, false);
-    }
-
-    public void testIndexSearcherWrapperDenseNoDeletions() throws IOException {
-        doTestIndexSearcherWrapper(false, false);
-    }
-
-    public void testIndexSearcherWrapperSparseWithDeletions() throws IOException {
-        doTestIndexSearcherWrapper(true, true);
-    }
-
-    public void testIndexSearcherWrapperDenseWithDeletions() throws IOException {
-        doTestIndexSearcherWrapper(false, true);
-    }
-
-    static class CreateScorerOnceWeight extends Weight {
-
-        private final Weight weight;
-        private final Set<Object> seenLeaves = Collections.newSetFromMap(new IdentityHashMap<>());
-
-        protected CreateScorerOnceWeight(Weight weight) {
-            super(weight.getQuery());
-            this.weight = weight;
-        }
-
-        @Override
-        public void extractTerms(Set<Term> terms) {
-            weight.extractTerms(terms);
-        }
-
-        @Override
-        public Explanation explain(LeafReaderContext context, int doc) throws IOException {
-            return weight.explain(context, doc);
-        }
-
-        @Override
-        public Scorer scorer(LeafReaderContext context) throws IOException {
-            assertTrue(seenLeaves.add(context.reader().getCoreCacheHelper().getKey()));
-            return weight.scorer(context);
-        }
-
-        @Override
-        public BulkScorer bulkScorer(LeafReaderContext context)
-                throws IOException {
-            assertTrue(seenLeaves.add(context.reader().getCoreCacheHelper().getKey()));
-            return weight.bulkScorer(context);
-        }
-
-        @Override
-        public boolean isCacheable(LeafReaderContext ctx) {
-            return true;
-        }
-    }
-
-    static class CreateScorerOnceQuery extends Query {
-
-        private final Query query;
-
-        CreateScorerOnceQuery(Query query) {
-            this.query = query;
-        }
-
-        @Override
-        public String toString(String field) {
-            return query.toString(field);
-        }
-
-        @Override
-        public Query rewrite(IndexReader reader) throws IOException {
-            Query queryRewritten = query.rewrite(reader);
-            if (query != queryRewritten) {
-                return new CreateScorerOnceQuery(queryRewritten);
-            }
-            return super.rewrite(reader);
-        }
-
-        @Override
-        public Weight createWeight(IndexSearcher searcher, org.apache.lucene.search.ScoreMode scoreMode, float boost) throws IOException {
-            return new CreateScorerOnceWeight(query.createWeight(searcher, scoreMode, boost));
-        }
-
-        @Override
-        public boolean equals(Object obj) {
-            return sameClassAs(obj) && query.equals(((CreateScorerOnceQuery) obj).query);
-        }
-
-        @Override
-        public int hashCode() {
-            return 31 * classHash() + query.hashCode();
-        }
-    }
-
-    public void doTestIndexSearcherWrapper(boolean sparse, boolean deletions) throws IOException {
-        Directory dir = newDirectory();
-        IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(null));
-        Document doc = new Document();
-        StringField allowedField = new StringField("allowed", "yes", Store.NO);
-        doc.add(allowedField);
-        StringField fooField = new StringField("foo", "bar", Store.NO);
-        doc.add(fooField);
-        StringField deleteField = new StringField("delete", "no", Store.NO);
-        doc.add(deleteField);
-        w.addDocument(doc);
-        if (deletions) {
-            // add a document that matches foo:bar but will be deleted
-            deleteField.setStringValue("yes");
-            w.addDocument(doc);
-            deleteField.setStringValue("no");
-        }
-        allowedField.setStringValue("no");
-        w.addDocument(doc);
-        if (sparse) {
-            for (int i = 0; i < 1000; ++i) {
-                w.addDocument(doc);
-            }
-            w.forceMerge(1);
-        }
-        w.deleteDocuments(new Term("delete", "yes"));
-
-        IndexSettings settings = IndexSettingsModule.newIndexSettings("_index", Settings.EMPTY);
-        BitsetFilterCache.Listener listener = new BitsetFilterCache.Listener() {
-            @Override
-            public void onCache(ShardId shardId, Accountable accountable) {
-
-            }
-
-            @Override
-            public void onRemoval(ShardId shardId, Accountable accountable) {
-
-            }
-        };
-        DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(w), new ShardId(indexSettings.getIndex(), 0));
-        BitsetFilterCache cache = new BitsetFilterCache(settings, listener);
-        Query roleQuery = new TermQuery(new Term("allowed", "yes"));
-        BitSet bitSet = cache.getBitSetProducer(roleQuery).getBitSet(reader.leaves().get(0));
-        if (sparse) {
-            assertThat(bitSet, instanceOf(SparseFixedBitSet.class));
-        } else {
-            assertThat(bitSet, instanceOf(FixedBitSet.class));
-        }
-
-        DocumentSubsetDirectoryReader filteredReader = DocumentSubsetReader.wrap(reader, cache, roleQuery);
-        IndexSearcher searcher = new SecurityIndexSearcherWrapper.IndexSearcherWrapper(filteredReader);
-
-        // Searching a non-existing term will trigger a null scorer
-        assertEquals(0, searcher.count(new TermQuery(new Term("non_existing_field", "non_existing_value"))));
-
-        assertEquals(1, searcher.count(new TermQuery(new Term("foo", "bar"))));
-
-        // make sure scorers are created only once, see #1725
-        assertEquals(1, searcher.count(new CreateScorerOnceQuery(new MatchAllDocsQuery())));
-        IOUtils.close(reader, w, dir);
-    }
-
-    private static FieldPermissionsDefinition fieldPermissionDef(String[] granted, String[] denied) {
-        return new FieldPermissionsDefinition(granted, denied);
-    }
-}
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
index 78f9623f4fbb8..4433b9d3750e7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
@@ -113,6 +113,7 @@
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkAction;
 import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.privilege.GetBuiltinPrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesRequest;
 import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesAction;
@@ -317,6 +318,8 @@ public void testKibanaSystemRole() {
         assertThat(kibanaRole.cluster().check(PutPrivilegesAction.NAME, putKibanaPrivileges), is(true));
         assertThat(kibanaRole.cluster().check(PutPrivilegesAction.NAME, putSwiftypePrivileges), is(false));
 
+        assertThat(kibanaRole.cluster().check(GetBuiltinPrivilegesAction.NAME, request), is(true));
+
         // Everything else
         assertThat(kibanaRole.runAs().check(randomAlphaOfLengthBetween(1, 12)), is(false));
 
@@ -1096,6 +1099,18 @@ public void testDataFrameTransformsAdminRole() {
         assertNoAccessAllowed(role, ".data-frame-internal-1"); // internal use only
 
         assertNoAccessAllowed(role, RestrictedIndicesNames.RESTRICTED_NAMES);
+
+        final String kibanaApplicationWithRandomIndex = "kibana-" + randomFrom(randomAlphaOfLengthBetween(8, 24), ".kibana");
+        assertThat(role.application().grants(
+            new ApplicationPrivilege(kibanaApplicationWithRandomIndex, "app-foo", "foo"), "*"), is(false));
+        assertThat(role.application().grants(
+            new ApplicationPrivilege(kibanaApplicationWithRandomIndex, "app-reserved_ml", "reserved_ml"), "*"), is(true));
+
+        final String otherApplication = "logstash-" + randomAlphaOfLengthBetween(8, 24);
+        assertThat(role.application().grants(
+            new ApplicationPrivilege(otherApplication, "app-foo", "foo"), "*"), is(false));
+        assertThat(role.application().grants(
+            new ApplicationPrivilege(otherApplication, "app-reserved_ml", "reserved_ml"), "*"), is(false));
     }
 
     public void testDataFrameTransformsUserRole() {
@@ -1120,6 +1135,18 @@ public void testDataFrameTransformsUserRole() {
         assertNoAccessAllowed(role, ".data-frame-internal-1");
 
         assertNoAccessAllowed(role, RestrictedIndicesNames.RESTRICTED_NAMES);
+
+        final String kibanaApplicationWithRandomIndex = "kibana-" + randomFrom(randomAlphaOfLengthBetween(8, 24), ".kibana");
+        assertThat(role.application().grants(
+            new ApplicationPrivilege(kibanaApplicationWithRandomIndex, "app-foo", "foo"), "*"), is(false));
+        assertThat(role.application().grants(
+            new ApplicationPrivilege(kibanaApplicationWithRandomIndex, "app-reserved_ml", "reserved_ml"), "*"), is(true));
+
+        final String otherApplication = "logstash-" + randomAlphaOfLengthBetween(8, 24);
+        assertThat(role.application().grants(
+            new ApplicationPrivilege(otherApplication, "app-foo", "foo"), "*"), is(false));
+        assertThat(role.application().grants(
+            new ApplicationPrivilege(otherApplication, "app-reserved_ml", "reserved_ml"), "*"), is(false));
     }
 
     public void testWatcherAdminRole() {
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluatorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluatorTests.java
index 086c3a406b62f..630c32a0d15fc 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluatorTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluatorTests.java
@@ -44,7 +44,7 @@ public void setup() throws Exception {
 
     public void testTemplating() throws Exception {
         User user = new User("_username", new String[] { "role1", "role2" }, "_full_name", "_email",
-                Collections.singletonMap("key", "value"), true);
+                Map.of("key", "value"), true);
 
         TemplateScript.Factory compiledTemplate = templateParams -> new TemplateScript(templateParams) {
             @Override
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/ValidationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/ValidationTests.java
index 458280677a490..b6226ec5c18ac 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/ValidationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/support/ValidationTests.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.security.support;
 
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
@@ -59,12 +60,12 @@ public void testUsernameInvalidWhitespace() throws Exception {
     }
 
     public void testUsersValidatePassword() throws Exception {
-        String passwd = randomAlphaOfLength(randomIntBetween(0, 20));
+        SecureString passwd = new SecureString(randomAlphaOfLength(randomIntBetween(0, 20)).toCharArray());
         logger.info("{}[{}]", passwd, passwd.length());
         if (passwd.length() >= 6) {
-            assertThat(Users.validatePassword(passwd.toCharArray()), nullValue());
+            assertThat(Users.validatePassword(passwd), nullValue());
         } else {
-            assertThat(Users.validatePassword(passwd.toCharArray()), notNullValue());
+            assertThat(Users.validatePassword(passwd), notNullValue());
         }
     }
 
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/UserTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/UserTests.java
index 02813f53b8c7d..8ded0af4823a6 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/UserTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/UserTests.java
@@ -7,7 +7,7 @@
 
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.Collections;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.is;
 
@@ -16,7 +16,7 @@ public class UserTests extends ESTestCase {
     public void testUserToString() {
         User user = new User("u1", "r1");
         assertThat(user.toString(), is("User[username=u1,roles=[r1],fullName=null,email=null,metadata={}]"));
-        user = new User("u1", new String[] { "r1", "r2" }, "user1", "user1@domain.com", Collections.singletonMap("key", "val"), true);
+        user = new User("u1", new String[] { "r1", "r2" }, "user1", "user1@domain.com", Map.of("key", "val"), true);
         assertThat(user.toString(), is("User[username=u1,roles=[r1,r2],fullName=user1,email=user1@domain.com,metadata={key=val}]"));
         user = new User("u1", new String[] {"r1"}, new User("u2", "r2", "r3"));
         assertThat(user.toString(), is("User[username=u1,roles=[r1],fullName=null,email=null,metadata={}," +
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/CertParsingUtilsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/CertParsingUtilsTests.java
index 49ce0fdb80e70..4a85ad16c3858 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/CertParsingUtilsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/CertParsingUtilsTests.java
@@ -20,10 +20,10 @@
 import java.util.Collections;
 import java.util.List;
 
-import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
 
 public class CertParsingUtilsTests extends ESTestCase {
     public void testReadKeysCorrectly() throws Exception {
@@ -79,7 +79,7 @@ public void testReadEllipticCurveCertificateAndKey() throws Exception {
         verifyPrime256v1ECKey(keyNoSpecPath);
 
         Path certPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/prime256v1-cert.pem");
-        Certificate[] certs = CertParsingUtils.readCertificates(Collections.singletonList(certPath.toString()), null);
+        Certificate[] certs = CertParsingUtils.readCertificates(Collections.singletonList(certPath.toString()), newEnvironment());
         assertEquals(1, certs.length);
         Certificate cert = certs[0];
         assertNotNull(cert);
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
index 83b1d80f563a5..02ea0298b9bd3 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java
@@ -32,7 +32,6 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.http.MockResponse;
 import org.elasticsearch.test.http.MockWebServer;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
@@ -76,7 +75,6 @@
 /**
  * Unit tests for the reloading of SSL configuration
  */
-@TestLogging("org.elasticsearch.watcher:TRACE")
 public class SSLConfigurationReloaderTests extends ESTestCase {
 
     private ThreadPool threadPool;
@@ -114,7 +112,7 @@ public void testReloadingKeyStore() throws Exception {
             .put("xpack.security.transport.ssl.keystore.path", keystorePath)
             .setSecureSettings(secureSettings)
             .build();
-        final Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings);
+        final Environment env = TestEnvironment.newEnvironment(settings);
         //Load HTTPClient only once. Client uses the same store as a truststore
         try (CloseableHttpClient client = getSSLClient(keystorePath, "testnode")) {
             final Consumer<SSLContext> keyMaterialPreChecks = (context) -> {
@@ -172,8 +170,7 @@ public void testPEMKeyConfigReloading() throws Exception {
             .putList("xpack.security.transport.ssl.certificate_authorities", certPath.toString())
             .setSecureSettings(secureSettings)
             .build();
-        final Environment env = randomBoolean() ? null :
-            TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
+        final Environment env = newEnvironment();
         // Load HTTPClient once. Client uses a keystore containing testnode key/cert as a truststore
         try (CloseableHttpClient client = getSSLClient(Collections.singletonList(certPath))) {
             final Consumer<SSLContext> keyMaterialPreChecks = (context) -> {
@@ -229,7 +226,7 @@ public void testReloadingTrustStore() throws Exception {
             .put("path.home", createTempDir())
             .setSecureSettings(secureSettings)
             .build();
-        Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings);
+        Environment env = TestEnvironment.newEnvironment(settings);
         // Create the MockWebServer once for both pre and post checks
         try (MockWebServer server = getSslServer(trustStorePath, "testnode")) {
             final Consumer<SSLContext> trustMaterialPreChecks = (context) -> {
@@ -279,7 +276,7 @@ public void testReloadingPEMTrustConfig() throws Exception {
             .putList("xpack.security.transport.ssl.certificate_authorities", serverCertPath.toString())
             .put("path.home", createTempDir())
             .build();
-        Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings);
+        Environment env = TestEnvironment.newEnvironment(settings);
         // Create the MockWebServer once for both pre and post checks
         try (MockWebServer server = getSslServer(serverKeyPath, serverCertPath, "testnode")) {
             final Consumer<SSLContext> trustMaterialPreChecks = (context) -> {
@@ -328,7 +325,7 @@ public void testReloadingKeyStoreException() throws Exception {
             .setSecureSettings(secureSettings)
             .put("path.home", createTempDir())
             .build();
-        Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings);
+        Environment env = TestEnvironment.newEnvironment(settings);
         final SSLService sslService = new SSLService(settings, env);
         final SSLConfiguration config = sslService.getSSLConfiguration("xpack.security.transport.ssl.");
         final AtomicReference<Exception> exceptionRef = new AtomicReference<>();
@@ -380,7 +377,7 @@ public void testReloadingPEMKeyConfigException() throws Exception {
             .put("path.home", createTempDir())
             .setSecureSettings(secureSettings)
             .build();
-        Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings);
+        Environment env = TestEnvironment.newEnvironment(settings);
         final SSLService sslService = new SSLService(settings, env);
         final SSLConfiguration config = sslService.getSSLConfiguration("xpack.security.transport.ssl.");
         final AtomicReference<Exception> exceptionRef = new AtomicReference<>();
@@ -426,7 +423,7 @@ public void testTrustStoreReloadException() throws Exception {
             .put("path.home", createTempDir())
             .setSecureSettings(secureSettings)
             .build();
-        Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings);
+        Environment env = TestEnvironment.newEnvironment(settings);
         final SSLService sslService = new SSLService(settings, env);
         final SSLConfiguration config = sslService.getSSLConfiguration("xpack.security.transport.ssl.");
         final AtomicReference<Exception> exceptionRef = new AtomicReference<>();
@@ -469,7 +466,7 @@ public void testPEMTrustReloadException() throws Exception {
             .putList("xpack.security.transport.ssl.certificate_authorities", clientCertPath.toString())
             .put("path.home", createTempDir())
             .build();
-        Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings);
+        Environment env = TestEnvironment.newEnvironment(settings);
         final SSLService sslService = new SSLService(settings, env);
         final SSLConfiguration config = sslService.sslConfiguration(settings.getByPrefix("xpack.security.transport.ssl."));
         final AtomicReference<Exception> exceptionRef = new AtomicReference<>();
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationTests.java
index b19044566b2fd..a3ef742dadfe7 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationTests.java
@@ -245,8 +245,7 @@ public void testThatSettingsWithDifferentTruststoresHaveDifferentHashCode() {
     }
 
     public void testPEMFile() {
-        Environment env = randomBoolean() ? null :
-                TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
+        Environment env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
         MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString("secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
@@ -265,8 +264,7 @@ public void testPEMFile() {
     }
 
     public void testPEMFileBackcompat() {
-        Environment env = randomBoolean() ? null :
-                TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
+        Environment env = newEnvironment();
         Settings settings = Settings.builder()
             .put("key",
                 getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem"))
@@ -286,8 +284,7 @@ public void testPEMFileBackcompat() {
     }
 
     public void testPEMKeyAndTrustFiles() {
-        Environment env = randomBoolean() ? null :
-                TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
+        Environment env = newEnvironment();
         MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString("secure_key_passphrase", "testnode");
         Settings settings = Settings.builder()
@@ -311,8 +308,7 @@ public void testPEMKeyAndTrustFiles() {
     }
 
     public void testPEMKeyAndTrustFilesBackcompat() {
-        Environment env = randomBoolean() ? null :
-                TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
+        Environment env = newEnvironment();
         Settings settings = Settings.builder()
             .put("key", getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem"))
             .put("key_passphrase", "testnode")
@@ -335,9 +331,10 @@ public void testPEMKeyAndTrustFilesBackcompat() {
     }
 
     private void assertCombiningTrustConfigContainsCorrectIssuers(SSLConfiguration sslConfiguration) {
-        X509Certificate[] trustConfAcceptedIssuers = sslConfiguration.trustConfig().createTrustManager(null).getAcceptedIssuers();
-        X509Certificate[] keyConfAcceptedIssuers = sslConfiguration.keyConfig().createTrustManager(null).getAcceptedIssuers();
-        X509Certificate[] defaultAcceptedIssuers = new DefaultJDKTrustConfig(null).createTrustManager(null)
+        Environment env = newEnvironment();
+        X509Certificate[] trustConfAcceptedIssuers = sslConfiguration.trustConfig().createTrustManager(env).getAcceptedIssuers();
+        X509Certificate[] keyConfAcceptedIssuers = sslConfiguration.keyConfig().createTrustManager(env).getAcceptedIssuers();
+        X509Certificate[] defaultAcceptedIssuers = new DefaultJDKTrustConfig(null).createTrustManager(env)
             .getAcceptedIssuers();
         assertEquals(keyConfAcceptedIssuers.length + defaultAcceptedIssuers.length, trustConfAcceptedIssuers.length);
         assertThat(Arrays.asList(keyConfAcceptedIssuers), everyItem(isIn(trustConfAcceptedIssuers)));
diff --git a/x-pack/plugin/data-frame/build.gradle b/x-pack/plugin/data-frame/build.gradle
index 1e939b2ceb949..d822e3a61896d 100644
--- a/x-pack/plugin/data-frame/build.gradle
+++ b/x-pack/plugin/data-frame/build.gradle
@@ -9,7 +9,7 @@ esplugin {
 }
 
 dependencies {
-    compileOnly "org.elasticsearch:elasticsearch:${version}"
+    compileOnly project(":server")
 
     compileOnly project(path: xpackModule('core'), configuration: 'default')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/build.gradle b/x-pack/plugin/data-frame/qa/multi-node-tests/build.gradle
index 0517f71b2dd3e..9fcf079668780 100644
--- a/x-pack/plugin/data-frame/qa/multi-node-tests/build.gradle
+++ b/x-pack/plugin/data-frame/qa/multi-node-tests/build.gradle
@@ -1,3 +1,4 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -25,8 +26,10 @@ task copyKeyCerts(type: Copy) {
 sourceSets.test.resources.srcDir(keystoreDir)
 processTestResources.dependsOn(copyKeyCerts)
 
-integTestCluster {
-  dependsOn copyKeyCerts
+integTest.dependsOn copyKeyCerts
+
+testClusters.integTest {
+  distribution = 'DEFAULT'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'xpack.monitoring.enabled', 'false'
@@ -37,22 +40,11 @@ integTestCluster {
   setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
   setting 'xpack.security.audit.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
-  keystoreSetting 'bootstrap.password', 'x-pack-test-password'
-  keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
-  setupCommand 'setupDummyUser',
-          'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'
+  keystore 'bootstrap.password', 'x-pack-test-password'
+  keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
+  numberOfNodes = 3
 
-  numNodes = 3
   extraConfigFile nodeKey.name, nodeKey
   extraConfigFile nodeCert.name, nodeCert
-  waitCondition = { node, ant ->
-      File tmpFile = new File(node.cwd, 'wait.success')
-      ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-              dest: tmpFile.toString(),
-              username: 'x_pack_rest_user',
-              password: 'x-pack-test-password',
-              ignoreerrors: true,
-              retries: 10)
-      return tmpFile.exists()
-  }
+  user username: "x_pack_rest_user", password: "x-pack-test-password"
 }
diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java
index 1320b5ad231e8..a1d8aca86ba16 100644
--- a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java
+++ b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java
@@ -15,6 +15,8 @@
 import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.client.core.AcknowledgedResponse;
 import org.elasticsearch.client.dataframe.DeleteDataFrameTransformRequest;
+import org.elasticsearch.client.dataframe.GetDataFrameTransformRequest;
+import org.elasticsearch.client.dataframe.GetDataFrameTransformResponse;
 import org.elasticsearch.client.dataframe.GetDataFrameTransformStatsRequest;
 import org.elasticsearch.client.dataframe.GetDataFrameTransformStatsResponse;
 import org.elasticsearch.client.dataframe.PutDataFrameTransformRequest;
@@ -57,6 +59,7 @@
 import java.util.Base64;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -65,8 +68,6 @@
 
 abstract class DataFrameIntegTestCase extends ESRestTestCase {
 
-    protected static final String REVIEWS_INDEX_NAME = "data_frame_reviews";
-
     private Map<String, DataFrameTransformConfig> transformConfigs = new HashMap<>();
 
     protected void cleanUp() throws IOException {
@@ -120,6 +121,11 @@ protected GetDataFrameTransformStatsResponse getDataFrameTransformStats(String i
         return restClient.dataFrame().getDataFrameTransformStats(new GetDataFrameTransformStatsRequest(id), RequestOptions.DEFAULT);
     }
 
+    protected GetDataFrameTransformResponse getDataFrameTransform(String id) throws IOException {
+        RestHighLevelClient restClient = new TestRestHighLevelClient();
+        return restClient.dataFrame().getDataFrameTransform(new GetDataFrameTransformRequest(id), RequestOptions.DEFAULT);
+    }
+
     protected void waitUntilCheckpoint(String id, long checkpoint) throws Exception {
         waitUntilCheckpoint(id, checkpoint, TimeValue.timeValueSeconds(30));
     }
@@ -137,25 +143,21 @@ protected void waitUntilCheckpoint(String id, long checkpoint, TimeValue waitTim
 
     protected DateHistogramGroupSource createDateHistogramGroupSourceWithFixedInterval(String field,
                                                                                        DateHistogramInterval interval,
-                                                                                       ZoneId zone,
-                                                                                       String format) {
+                                                                                       ZoneId zone) {
         DateHistogramGroupSource.Builder builder = DateHistogramGroupSource.builder()
             .setField(field)
             .setInterval(new DateHistogramGroupSource.FixedInterval(interval))
-            .setTimeZone(zone)
-            .setFormat(format);
+            .setTimeZone(zone);
         return builder.build();
     }
 
     protected DateHistogramGroupSource createDateHistogramGroupSourceWithCalendarInterval(String field,
                                                                                           DateHistogramInterval interval,
-                                                                                          ZoneId zone,
-                                                                                          String format) {
+                                                                                          ZoneId zone) {
         DateHistogramGroupSource.Builder builder = DateHistogramGroupSource.builder()
             .setField(field)
             .setInterval(new DateHistogramGroupSource.CalendarInterval(interval))
-            .setTimeZone(zone)
-            .setFormat(format);
+            .setTimeZone(zone);
         return builder.build();
     }
 
@@ -198,23 +200,36 @@ protected DataFrameTransformConfig createTransformConfig(String id,
         return createTransformConfig(id, groups, aggregations, destinationIndex, QueryBuilders.matchAllQuery(), sourceIndices);
     }
 
+    protected DataFrameTransformConfig.Builder createTransformConfigBuilder(String id,
+                                                                            Map<String, SingleGroupSource> groups,
+                                                                            AggregatorFactories.Builder aggregations,
+                                                                            String destinationIndex,
+                                                                            QueryBuilder queryBuilder,
+                                                                            String... sourceIndices) throws Exception {
+        return DataFrameTransformConfig.builder()
+            .setId(id)
+            .setSource(SourceConfig.builder().setIndex(sourceIndices).setQueryConfig(createQueryConfig(queryBuilder)).build())
+            .setDest(DestConfig.builder().setIndex(destinationIndex).build())
+            .setPivotConfig(createPivotConfig(groups, aggregations))
+            .setDescription("Test data frame transform config id: " + id);
+    }
+
     protected DataFrameTransformConfig createTransformConfig(String id,
                                                              Map<String, SingleGroupSource> groups,
                                                              AggregatorFactories.Builder aggregations,
                                                              String destinationIndex,
                                                              QueryBuilder queryBuilder,
                                                              String... sourceIndices) throws Exception {
-        return DataFrameTransformConfig.builder()
-            .setId(id)
-            .setSource(SourceConfig.builder().setIndex(sourceIndices).setQueryConfig(createQueryConfig(queryBuilder)).build())
-            .setDest(new DestConfig(destinationIndex))
-            .setPivotConfig(createPivotConfig(groups, aggregations))
-            .setDescription("Test data frame transform config id: " + id)
-            .build();
+        return createTransformConfigBuilder(id, groups, aggregations, destinationIndex, queryBuilder, sourceIndices).build();
+    }
+
+    protected void bulkIndexDocs(BulkRequest request) throws Exception {
+        RestHighLevelClient restClient = new TestRestHighLevelClient();
+        BulkResponse response = restClient.bulk(request, RequestOptions.DEFAULT);
+        assertThat(response.buildFailureMessage(), response.hasFailures(), is(false));
     }
 
-    protected void createReviewsIndex() throws Exception {
-        final int numDocs = 1000;
+    protected void createReviewsIndex(String indexName, int numDocs) throws Exception {
         RestHighLevelClient restClient = new TestRestHighLevelClient();
 
         // create mapping
@@ -241,12 +256,12 @@ protected void createReviewsIndex() throws Exception {
             }
             builder.endObject();
             CreateIndexResponse response =
-                restClient.indices().create(new CreateIndexRequest(REVIEWS_INDEX_NAME).mapping(builder), RequestOptions.DEFAULT);
+                restClient.indices().create(new CreateIndexRequest(indexName).mapping(builder), RequestOptions.DEFAULT);
             assertThat(response.isAcknowledged(), is(true));
         }
 
         // create index
-        BulkRequest bulk = new BulkRequest(REVIEWS_INDEX_NAME);
+        BulkRequest bulk = new BulkRequest(indexName);
         int day = 10;
         for (int i = 0; i < numDocs; i++) {
             long user = i % 28;
@@ -256,7 +271,7 @@ protected void createReviewsIndex() throws Exception {
             int min = 10 + (i % 49);
             int sec = 10 + (i % 49);
 
-            String date_string = "2017-01-" + day + "T" + hour + ":" + min + ":" + sec + "Z";
+            String date_string = "2017-01-" + (day < 10 ? "0" + day : day) + "T" + hour + ":" + min + ":" + sec + "Z";
 
             StringBuilder sourceBuilder = new StringBuilder();
             sourceBuilder.append("{\"user_id\":\"")
@@ -277,13 +292,13 @@ protected void createReviewsIndex() throws Exception {
             if (i % 50 == 0) {
                 BulkResponse response = restClient.bulk(bulk, RequestOptions.DEFAULT);
                 assertThat(response.buildFailureMessage(), response.hasFailures(), is(false));
-                bulk = new BulkRequest(REVIEWS_INDEX_NAME);
-                day += 1;
+                bulk = new BulkRequest(indexName);
+                day = (day + 1) % 28;
             }
         }
         BulkResponse response = restClient.bulk(bulk, RequestOptions.DEFAULT);
         assertThat(response.buildFailureMessage(), response.hasFailures(), is(false));
-        restClient.indices().refresh(new RefreshRequest(REVIEWS_INDEX_NAME), RequestOptions.DEFAULT);
+        restClient.indices().refresh(new RefreshRequest(indexName), RequestOptions.DEFAULT);
     }
 
     protected Map<String, Object> toLazy(ToXContent parsedObject) throws Exception {
@@ -311,7 +326,7 @@ private void waitForPendingTasks() {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
@@ -324,9 +339,11 @@ protected Settings restClientSettings() {
             .build();
     }
 
-    private class TestRestHighLevelClient extends RestHighLevelClient {
+    private static class TestRestHighLevelClient extends RestHighLevelClient {
+        private static final List<NamedXContentRegistry.Entry> X_CONTENT_ENTRIES =
+            new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents();
         TestRestHighLevelClient() {
-            super(client(), restClient -> {}, Collections.emptyList());
+            super(client(), restClient -> {}, X_CONTENT_ENTRIES);
         }
     }
 }
diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java
index 486ea5e5d7403..456ba91ef4381 100644
--- a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java
+++ b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformIT.java
@@ -6,22 +6,32 @@
 
 package org.elasticsearch.xpack.dataframe.integration;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.core.IndexerState;
 import org.elasticsearch.client.dataframe.transforms.DataFrameTransformConfig;
-import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStats;
+import org.elasticsearch.client.dataframe.transforms.DataFrameTransformTaskState;
+import org.elasticsearch.client.dataframe.transforms.TimeSyncConfig;
 import org.elasticsearch.client.dataframe.transforms.pivot.SingleGroupSource;
 import org.elasticsearch.client.dataframe.transforms.pivot.TermsGroupSource;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.junit.After;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.HashMap;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 
 public class DataFrameTransformIT extends DataFrameIntegTestCase {
 
@@ -30,12 +40,12 @@ public void cleanTransforms() throws IOException {
         cleanUp();
     }
 
-    @AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/42344")
     public void testDataFrameTransformCrud() throws Exception {
-        createReviewsIndex();
+        String indexName = "basic-crud-reviews";
+        createReviewsIndex(indexName, 100);
 
         Map<String, SingleGroupSource> groups = new HashMap<>();
-        groups.put("by-day", createDateHistogramGroupSourceWithCalendarInterval("timestamp", DateHistogramInterval.DAY, null, null));
+        groups.put("by-day", createDateHistogramGroupSourceWithCalendarInterval("timestamp", DateHistogramInterval.DAY, null));
         groups.put("by-user", TermsGroupSource.builder().setField("user_id").build());
         groups.put("by-business", TermsGroupSource.builder().setField("business_id").build());
 
@@ -47,17 +57,103 @@ public void testDataFrameTransformCrud() throws Exception {
             groups,
             aggs,
             "reviews-by-user-business-day",
-            REVIEWS_INDEX_NAME);
+            indexName);
 
         assertTrue(putDataFrameTransform(config, RequestOptions.DEFAULT).isAcknowledged());
-        assertTrue(startDataFrameTransform(config.getId(), RequestOptions.DEFAULT).isStarted());
+        assertTrue(startDataFrameTransform(config.getId(), RequestOptions.DEFAULT).isAcknowledged());
 
         waitUntilCheckpoint(config.getId(), 1L);
 
-        DataFrameTransformStateAndStats stats = getDataFrameTransformStats(config.getId()).getTransformsStateAndStats().get(0);
-
-        assertThat(stats.getTransformState().getIndexerState(), equalTo(IndexerState.STARTED));
+        // It will eventually be stopped
+        assertBusy(() ->
+            assertThat(getDataFrameTransformStats(config.getId()).getTransformsStateAndStats().get(0).getTransformState().getIndexerState(),
+                equalTo(IndexerState.STOPPED)));
+        stopDataFrameTransform(config.getId());
+
+        DataFrameTransformConfig storedConfig = getDataFrameTransform(config.getId()).getTransformConfigurations().get(0);
+        assertThat(storedConfig.getVersion(), equalTo(Version.CURRENT));
+        Instant now = Instant.now();
+        assertTrue("[create_time] is not before current time", storedConfig.getCreateTime().isBefore(now));
+        deleteDataFrameTransform(config.getId());
     }
 
+    public void testContinuousDataFrameTransformCrud() throws Exception {
+        String indexName = "continuous-crud-reviews";
+        createReviewsIndex(indexName, 100);
+
+        Map<String, SingleGroupSource> groups = new HashMap<>();
+        groups.put("by-day", createDateHistogramGroupSourceWithCalendarInterval("timestamp", DateHistogramInterval.DAY, null));
+        groups.put("by-user", TermsGroupSource.builder().setField("user_id").build());
+        groups.put("by-business", TermsGroupSource.builder().setField("business_id").build());
+
+        AggregatorFactories.Builder aggs = AggregatorFactories.builder()
+            .addAggregator(AggregationBuilders.avg("review_score").field("stars"))
+            .addAggregator(AggregationBuilders.max("timestamp").field("timestamp"));
 
+        DataFrameTransformConfig config = createTransformConfigBuilder("data-frame-transform-crud",
+            groups,
+            aggs,
+            "reviews-by-user-business-day",
+            QueryBuilders.matchAllQuery(),
+            indexName)
+            .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)))
+            .build();
+
+        assertTrue(putDataFrameTransform(config, RequestOptions.DEFAULT).isAcknowledged());
+        assertTrue(startDataFrameTransform(config.getId(), RequestOptions.DEFAULT).isAcknowledged());
+
+        waitUntilCheckpoint(config.getId(), 1L);
+        assertThat(getDataFrameTransformStats(config.getId()).getTransformsStateAndStats().get(0).getTransformState().getTaskState(),
+                equalTo(DataFrameTransformTaskState.STARTED));
+
+        long docsIndexed = getDataFrameTransformStats(config.getId())
+            .getTransformsStateAndStats()
+            .get(0)
+            .getTransformStats()
+            .getNumDocuments();
+
+        DataFrameTransformConfig storedConfig = getDataFrameTransform(config.getId()).getTransformConfigurations().get(0);
+        assertThat(storedConfig.getVersion(), equalTo(Version.CURRENT));
+        Instant now = Instant.now();
+        assertTrue("[create_time] is not before current time", storedConfig.getCreateTime().isBefore(now));
+
+        // index some more docs
+        long timeStamp = Instant.now().toEpochMilli() - 1_000;
+        long user = 42;
+        BulkRequest bulk = new BulkRequest(indexName);
+        for (int i = 0; i < 25; i++) {
+            int stars = (i + 20) % 5;
+            long business = (i + 100) % 50;
+
+            StringBuilder sourceBuilder = new StringBuilder();
+            sourceBuilder.append("{\"user_id\":\"")
+                .append("user_")
+                .append(user)
+                .append("\",\"count\":")
+                .append(i)
+                .append(",\"business_id\":\"")
+                .append("business_")
+                .append(business)
+                .append("\",\"stars\":")
+                .append(stars)
+                .append(",\"timestamp\":")
+                .append(timeStamp)
+                .append("}");
+            bulk.add(new IndexRequest().source(sourceBuilder.toString(), XContentType.JSON));
+        }
+        bulk.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+        bulkIndexDocs(bulk);
+
+        waitUntilCheckpoint(config.getId(), 2L);
+
+        // Assert that we wrote the new docs
+        assertThat(getDataFrameTransformStats(config.getId())
+            .getTransformsStateAndStats()
+            .get(0)
+            .getTransformStats()
+            .getNumDocuments(), greaterThan(docsIndexed));
+
+        stopDataFrameTransform(config.getId());
+        deleteDataFrameTransform(config.getId());
+    }
 }
diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/build.gradle b/x-pack/plugin/data-frame/qa/single-node-tests/build.gradle
index 3fbd779f36239..a33e73ba4c318 100644
--- a/x-pack/plugin/data-frame/qa/single-node-tests/build.gradle
+++ b/x-pack/plugin/data-frame/qa/single-node-tests/build.gradle
@@ -1,3 +1,4 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -10,21 +11,9 @@ dependencies {
   testCompile project(path: xpackModule('data-frame'), configuration: 'runtime')
 }
 
-integTestCluster {
+testClusters.integTest {
+  distribution = 'DEFAULT'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.license.self_generated.type', 'trial'
-
-  setupCommand 'setupDummyUser',
-          'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'
-
-  waitCondition = { node, ant ->
-      File tmpFile = new File(node.cwd, 'wait.success')
-      ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-              dest: tmpFile.toString(),
-              username: 'x_pack_rest_user',
-              password: 'x-pack-test-password',
-              ignoreerrors: true,
-              retries: 10)
-      return tmpFile.exists()
-  }
-}
+  user username: "x_pack_rest_user", password: "x-pack-test-password"
+}
\ No newline at end of file
diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameAuditorIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameAuditorIT.java
index 7dc79c1ae8fbe..0877fe22c78f7 100644
--- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameAuditorIT.java
+++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameAuditorIT.java
@@ -6,7 +6,6 @@
 
 package org.elasticsearch.xpack.dataframe.integration;
 
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.xpack.dataframe.persistence.DataFrameInternalIndex;
 import org.junit.Before;
@@ -23,7 +22,6 @@
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.is;
 
-@LuceneTestCase.AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/42344")
 public class DataFrameAuditorIT extends DataFrameRestTestCase {
 
     private static final String TEST_USER_NAME = "df_admin_plus_data";
@@ -55,18 +53,18 @@ public void createIndexes() throws IOException {
 
     @SuppressWarnings("unchecked")
     public void testAuditorWritesAudits() throws Exception {
-        String transformId = "simplePivotForAudit";
+        String transformId = "simple_pivot_for_audit";
         String dataFrameIndex = "pivot_reviews_user_id_above_20";
         setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, dataFrameIndex);
         String query = "\"match\": {\"user_id\": \"user_26\"}";
 
-        createPivotReviewsTransform(transformId, dataFrameIndex, query, BASIC_AUTH_VALUE_DATA_FRAME_ADMIN_WITH_SOME_DATA_ACCESS);
+        createPivotReviewsTransform(transformId, dataFrameIndex, query, null, BASIC_AUTH_VALUE_DATA_FRAME_ADMIN_WITH_SOME_DATA_ACCESS);
 
         startAndWaitForTransform(transformId, dataFrameIndex, BASIC_AUTH_VALUE_DATA_FRAME_ADMIN_WITH_SOME_DATA_ACCESS);
 
         // Make sure we wrote to the audit
         final Request request = new Request("GET", DataFrameInternalIndex.AUDIT_INDEX + "/_search");
-        request.setJsonEntity("{\"query\":{\"term\":{\"transform_id\":\"simplePivotForAudit\"}}}");
+        request.setJsonEntity("{\"query\":{\"term\":{\"transform_id\":\"simple_pivot_for_audit\"}}}");
         assertBusy(() -> {
             assertTrue(indexExists(DataFrameInternalIndex.AUDIT_INDEX));
         });
diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameConfigurationIndexIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameConfigurationIndexIT.java
index d7e12cf2bee4d..681599331c8af 100644
--- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameConfigurationIndexIT.java
+++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameConfigurationIndexIT.java
@@ -8,7 +8,6 @@
 
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
@@ -23,7 +22,6 @@
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 
-@LuceneTestCase.AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/42344")
 public class DataFrameConfigurationIndexIT extends DataFrameRestTestCase {
 
     /**
diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameGetAndGetStatsIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameGetAndGetStatsIT.java
index 9bac6ca0b4049..a45fee6d9666f 100644
--- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameGetAndGetStatsIT.java
+++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameGetAndGetStatsIT.java
@@ -6,7 +6,6 @@
 
 package org.elasticsearch.xpack.dataframe.integration;
 
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
@@ -14,15 +13,16 @@
 import org.junit.Before;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
-@LuceneTestCase.AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/42344")
 public class DataFrameGetAndGetStatsIT extends DataFrameRestTestCase {
 
     private static final String TEST_USER_NAME = "df_user";
@@ -63,9 +63,13 @@ public void clearOutTransforms() throws Exception {
     public void testGetAndGetStats() throws Exception {
         createPivotReviewsTransform("pivot_1", "pivot_reviews_1", null);
         createPivotReviewsTransform("pivot_2", "pivot_reviews_2", null);
+        createContinuousPivotReviewsTransform("pivot_continuous", "pivot_reviews_continuous", null);
 
         startAndWaitForTransform("pivot_1", "pivot_reviews_1");
         startAndWaitForTransform("pivot_2", "pivot_reviews_2");
+        startAndWaitForContinuousTransform("pivot_continuous", "pivot_reviews_continuous", null);
+        stopDataFrameTransform("pivot_1", false);
+        stopDataFrameTransform("pivot_2", false);
 
         // Alternate testing between admin and lowly user, as both should be able to get the configs and stats
         String authHeader = randomFrom(BASIC_AUTH_VALUE_DATA_FRAME_USER, BASIC_AUTH_VALUE_DATA_FRAME_ADMIN);
@@ -73,19 +77,19 @@ public void testGetAndGetStats() throws Exception {
         // check all the different ways to retrieve all stats
         Request getRequest = createRequestWithAuth("GET", DATAFRAME_ENDPOINT + "_stats", authHeader);
         Map<String, Object> stats = entityAsMap(client().performRequest(getRequest));
-        assertEquals(2, XContentMapValues.extractValue("count", stats));
+        assertEquals(3, XContentMapValues.extractValue("count", stats));
         getRequest = createRequestWithAuth("GET", DATAFRAME_ENDPOINT + "_all/_stats", authHeader);
         stats = entityAsMap(client().performRequest(getRequest));
-        assertEquals(2, XContentMapValues.extractValue("count", stats));
+        assertEquals(3, XContentMapValues.extractValue("count", stats));
         getRequest = createRequestWithAuth("GET", DATAFRAME_ENDPOINT + "*/_stats", authHeader);
         stats = entityAsMap(client().performRequest(getRequest));
-        assertEquals(2, XContentMapValues.extractValue("count", stats));
+        assertEquals(3, XContentMapValues.extractValue("count", stats));
         getRequest = createRequestWithAuth("GET", DATAFRAME_ENDPOINT + "pivot_1,pivot_2/_stats", authHeader);
         stats = entityAsMap(client().performRequest(getRequest));
         assertEquals(2, XContentMapValues.extractValue("count", stats));
         getRequest = createRequestWithAuth("GET", DATAFRAME_ENDPOINT + "pivot_*/_stats", authHeader);
         stats = entityAsMap(client().performRequest(getRequest));
-        assertEquals(2, XContentMapValues.extractValue("count", stats));
+        assertEquals(3, XContentMapValues.extractValue("count", stats));
 
         List<Map<String, Object>> transformsStats = (List<Map<String, Object>>)XContentMapValues.extractValue("transforms", stats);
         // Verify that both transforms have valid stats
@@ -108,26 +112,39 @@ public void testGetAndGetStats() throws Exception {
         transformsStats = (List<Map<String, Object>>)XContentMapValues.extractValue("transforms", stats);
         assertEquals(1, transformsStats.size());
         Map<String, Object> state = (Map<String, Object>) XContentMapValues.extractValue("state", transformsStats.get(0));
+        assertEquals("stopped", XContentMapValues.extractValue("task_state", state));
+        assertEquals(null, XContentMapValues.extractValue("current_position", state));
+        assertEquals(1, XContentMapValues.extractValue("checkpoint", state));
+
+        // only continuous
+        getRequest = createRequestWithAuth("GET", DATAFRAME_ENDPOINT + "pivot_continuous/_stats", authHeader);
+        stats = entityAsMap(client().performRequest(getRequest));
+        assertEquals(1, XContentMapValues.extractValue("count", stats));
+
+        transformsStats = (List<Map<String, Object>>)XContentMapValues.extractValue("transforms", stats);
         assertEquals(1, transformsStats.size());
+        state = (Map<String, Object>) XContentMapValues.extractValue("state", transformsStats.get(0));
         assertEquals("started", XContentMapValues.extractValue("task_state", state));
-        assertEquals(null, XContentMapValues.extractValue("current_position", state));
         assertEquals(1, XContentMapValues.extractValue("checkpoint", state));
 
+
         // check all the different ways to retrieve all transforms
         getRequest = createRequestWithAuth("GET", DATAFRAME_ENDPOINT, authHeader);
         Map<String, Object> transforms = entityAsMap(client().performRequest(getRequest));
-        assertEquals(2, XContentMapValues.extractValue("count", transforms));
+        assertEquals(3, XContentMapValues.extractValue("count", transforms));
         getRequest = createRequestWithAuth("GET", DATAFRAME_ENDPOINT + "_all", authHeader);
         transforms = entityAsMap(client().performRequest(getRequest));
-        assertEquals(2, XContentMapValues.extractValue("count", transforms));
+        assertEquals(3, XContentMapValues.extractValue("count", transforms));
         getRequest = createRequestWithAuth("GET", DATAFRAME_ENDPOINT + "*", authHeader);
         transforms = entityAsMap(client().performRequest(getRequest));
-        assertEquals(2, XContentMapValues.extractValue("count", transforms));
+        assertEquals(3, XContentMapValues.extractValue("count", transforms));
 
         // only pivot_1
         getRequest = createRequestWithAuth("GET", DATAFRAME_ENDPOINT + "pivot_1", authHeader);
         transforms = entityAsMap(client().performRequest(getRequest));
         assertEquals(1, XContentMapValues.extractValue("count", transforms));
+
+        stopDataFrameTransform("pivot_continuous", false);
     }
 
     @SuppressWarnings("unchecked")
@@ -162,7 +179,7 @@ public void testGetPersistedStatsWithoutTask() throws Exception {
 
     @SuppressWarnings("unchecked")
     public void testGetProgressStatsWithPivotQuery() throws Exception {
-        String transformId = "simpleStatsPivotWithQuery";
+        String transformId = "simple_stats_pivot_with_query";
         String dataFrameIndex = "pivot_stats_reviews_user_id_above_20";
         String query = "\"match\": {\"user_id\": \"user_26\"}";
         createPivotReviewsTransform(transformId, dataFrameIndex, query);
@@ -171,7 +188,7 @@ public void testGetProgressStatsWithPivotQuery() throws Exception {
         // Alternate testing between admin and lowly user, as both should be able to get the configs and stats
         String authHeader = randomFrom(BASIC_AUTH_VALUE_DATA_FRAME_USER, BASIC_AUTH_VALUE_DATA_FRAME_ADMIN);
 
-        Request getRequest = createRequestWithAuth("GET", DATAFRAME_ENDPOINT + "simpleStatsPivotWithQuery/_stats", authHeader);
+        Request getRequest = createRequestWithAuth("GET", DATAFRAME_ENDPOINT + transformId + "/_stats", authHeader);
         Map<String, Object> stats = entityAsMap(client().performRequest(getRequest));
         assertEquals(1, XContentMapValues.extractValue("count", stats));
         List<Map<String, Object>> transformsStats = (List<Map<String, Object>>)XContentMapValues.extractValue("transforms", stats);
@@ -187,4 +204,86 @@ public void testGetProgressStatsWithPivotQuery() throws Exception {
             assertThat("percent_complete is not 100.0", progress.get("percent_complete"), equalTo(100.0));
         }
     }
+
+    @SuppressWarnings("unchecked")
+    public void testGetProgressResetWithContinuous() throws Exception {
+        String transformId = "pivot_progress_continuous";
+        String transformDest = transformId + "_idx";
+        String transformSrc = "reviews_cont_pivot_test";
+        createReviewsIndex(transformSrc);
+        final Request createDataframeTransformRequest = createRequestWithAuth("PUT", DATAFRAME_ENDPOINT + transformId, null);
+        String config = "{ \"dest\": {\"index\":\"" + transformDest + "\"},"
+            + " \"source\": {\"index\":\"" + transformSrc + "\"},"
+            + " \"sync\": {\"time\":{\"field\": \"timestamp\", \"delay\": \"1s\"}},"
+            + " \"pivot\": {"
+            + "   \"group_by\": {"
+            + "     \"reviewer\": {"
+            + "       \"terms\": {"
+            + "         \"field\": \"user_id\""
+            + " } } },"
+            + "   \"aggregations\": {"
+            + "     \"avg_rating\": {"
+            + "       \"avg\": {"
+            + "         \"field\": \"stars\""
+            + " } } } }"
+            + "}";
+
+        createDataframeTransformRequest.setJsonEntity(config);
+
+        Map<String, Object> createDataframeTransformResponse = entityAsMap(client().performRequest(createDataframeTransformRequest));
+        assertThat(createDataframeTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+        startAndWaitForContinuousTransform(transformId, transformDest, null);
+
+        Request getRequest = createRequestWithAuth("GET", DATAFRAME_ENDPOINT + transformId + "/_stats", null);
+        Map<String, Object> stats = entityAsMap(client().performRequest(getRequest));
+        List<Map<String, Object>> transformsStats = (List<Map<String, Object>>)XContentMapValues.extractValue("transforms", stats);
+        assertEquals(1, transformsStats.size());
+        // Verify that the transform's progress
+        for (Map<String, Object> transformStats : transformsStats) {
+            Map<String, Object> progress = (Map<String, Object>)XContentMapValues.extractValue("state.progress", transformStats);
+            assertThat("total_docs is not 1000", progress.get("total_docs"), equalTo(1000));
+            assertThat("docs_remaining is not 0", progress.get("docs_remaining"), equalTo(0));
+            assertThat("percent_complete is not 100.0", progress.get("percent_complete"), equalTo(100.0));
+        }
+
+        // add more docs to verify total_docs gets updated with continuous
+        int numDocs = 10;
+        final StringBuilder bulk = new StringBuilder();
+        long now = Instant.now().toEpochMilli() - 1_000;
+        for (int i = 0; i < numDocs; i++) {
+            bulk.append("{\"index\":{\"_index\":\"" + transformSrc + "\"}}\n")
+                .append("{\"user_id\":\"")
+                .append("user_")
+                // Doing only new users so that there is a deterministic number of docs for progress
+                .append(randomFrom(42, 47, 113))
+                .append("\",\"business_id\":\"")
+                .append("business_")
+                .append(10)
+                .append("\",\"stars\":")
+                .append(5)
+                .append(",\"timestamp\":")
+                .append(now)
+                .append("}\n");
+        }
+        bulk.append("\r\n");
+        final Request bulkRequest = new Request("POST", "/_bulk");
+        bulkRequest.addParameter("refresh", "true");
+        bulkRequest.setJsonEntity(bulk.toString());
+        client().performRequest(bulkRequest);
+
+        waitForDataFrameCheckpoint(transformId, 2L);
+
+        assertBusy(() -> {
+            Map<String, Object> statsResponse = entityAsMap(client().performRequest(getRequest));
+            List<Map<String, Object>> contStats = (List<Map<String, Object>>)XContentMapValues.extractValue("transforms", statsResponse);
+            assertEquals(1, contStats.size());
+            // add more docs to verify total_docs is the number of new docs added to the index
+            for (Map<String, Object> transformStats : contStats) {
+                Map<String, Object> progress = (Map<String, Object>)XContentMapValues.extractValue("state.progress", transformStats);
+                assertThat("total_docs is not 10", progress.get("total_docs"), equalTo(numDocs));
+                assertThat("docs_remaining is not 0", progress.get("docs_remaining"), equalTo(0));
+                assertThat("percent_complete is not 100.0", progress.get("percent_complete"), equalTo(100.0));
+            }
+        }, 60, TimeUnit.SECONDS);
+    }
 }
diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameMetaDataIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameMetaDataIT.java
index 5b95d1daead53..26a957ea055c2 100644
--- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameMetaDataIT.java
+++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameMetaDataIT.java
@@ -6,7 +6,6 @@
 
 package org.elasticsearch.xpack.dataframe.integration;
 
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
@@ -16,7 +15,6 @@
 import java.io.IOException;
 import java.util.Map;
 
-@LuceneTestCase.AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/42344")
 public class DataFrameMetaDataIT extends DataFrameRestTestCase {
 
     private boolean indicesCreated = false;
diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java
index a0bec6ec13c34..f2cd95ed1a9c7 100644
--- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java
+++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFramePivotRestIT.java
@@ -6,12 +6,12 @@
 
 package org.elasticsearch.xpack.dataframe.integration;
 
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
@@ -21,8 +21,9 @@
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThan;
 
-@LuceneTestCase.AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/42344")
 public class DataFramePivotRestIT extends DataFrameRestTestCase {
 
     private static final String TEST_USER_NAME = "df_admin_plus_data";
@@ -53,11 +54,11 @@ public void createIndexes() throws IOException {
     }
 
     public void testSimplePivot() throws Exception {
-        String transformId = "simplePivot";
+        String transformId = "simple-pivot";
         String dataFrameIndex = "pivot_reviews";
         setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, dataFrameIndex);
 
-        createPivotReviewsTransform(transformId, dataFrameIndex, null, BASIC_AUTH_VALUE_DATA_FRAME_ADMIN_WITH_SOME_DATA_ACCESS);
+        createPivotReviewsTransform(transformId, dataFrameIndex, null, null, BASIC_AUTH_VALUE_DATA_FRAME_ADMIN_WITH_SOME_DATA_ACCESS);
 
         startAndWaitForTransform(transformId, dataFrameIndex, BASIC_AUTH_VALUE_DATA_FRAME_ADMIN_WITH_SOME_DATA_ACCESS);
 
@@ -74,12 +75,12 @@ public void testSimplePivot() throws Exception {
     }
 
     public void testSimplePivotWithQuery() throws Exception {
-        String transformId = "simplePivotWithQuery";
+        String transformId = "simple_pivot_with_query";
         String dataFrameIndex = "pivot_reviews_user_id_above_20";
         setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, dataFrameIndex);
         String query = "\"match\": {\"user_id\": \"user_26\"}";
 
-        createPivotReviewsTransform(transformId, dataFrameIndex, query, BASIC_AUTH_VALUE_DATA_FRAME_ADMIN_WITH_SOME_DATA_ACCESS);
+        createPivotReviewsTransform(transformId, dataFrameIndex, query, null, BASIC_AUTH_VALUE_DATA_FRAME_ADMIN_WITH_SOME_DATA_ACCESS);
 
         startAndWaitForTransform(transformId, dataFrameIndex, BASIC_AUTH_VALUE_DATA_FRAME_ADMIN_WITH_SOME_DATA_ACCESS);
 
@@ -89,8 +90,162 @@ public void testSimplePivotWithQuery() throws Exception {
         assertOnePivotValue(dataFrameIndex + "/_search?q=reviewer:user_26", 3.918918918);
     }
 
+    public void testPivotWithPipeline() throws Exception {
+        String transformId = "simple_pivot_with_pipeline";
+        String dataFrameIndex = "pivot_with_pipeline";
+        String pipelineId = "my-pivot-pipeline";
+        int pipelineValue = 42;
+        Request pipelineRequest = new Request("PUT", "/_ingest/pipeline/" + pipelineId);
+        pipelineRequest.setJsonEntity("{\n" +
+            "  \"description\" : \"my pivot pipeline\",\n" +
+            "  \"processors\" : [\n" +
+            "    {\n" +
+            "      \"set\" : {\n" +
+            "        \"field\": \"pipeline_field\",\n" +
+            "        \"value\": " + pipelineValue +
+            "      }\n" +
+            "    }\n" +
+            "  ]\n" +
+            "}");
+        client().performRequest(pipelineRequest);
+
+        setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, dataFrameIndex);
+        createPivotReviewsTransform(transformId, dataFrameIndex, null, pipelineId, BASIC_AUTH_VALUE_DATA_FRAME_ADMIN_WITH_SOME_DATA_ACCESS);
+
+        startAndWaitForTransform(transformId, dataFrameIndex, BASIC_AUTH_VALUE_DATA_FRAME_ADMIN_WITH_SOME_DATA_ACCESS);
+
+        // we expect 27 documents as there shall be 27 user_id's
+        Map<String, Object> indexStats = getAsMap(dataFrameIndex + "/_stats");
+        assertEquals(27, XContentMapValues.extractValue("_all.total.docs.count", indexStats));
+
+        // get and check some users
+        assertOnePivotValue(dataFrameIndex + "/_search?q=reviewer:user_0", 3.776978417);
+        assertOnePivotValue(dataFrameIndex + "/_search?q=reviewer:user_5", 3.72);
+        assertOnePivotValue(dataFrameIndex + "/_search?q=reviewer:user_11", 3.846153846);
+        assertOnePivotValue(dataFrameIndex + "/_search?q=reviewer:user_20", 3.769230769);
+        assertOnePivotValue(dataFrameIndex + "/_search?q=reviewer:user_26", 3.918918918);
+
+        Map<String, Object> searchResult = getAsMap(dataFrameIndex + "/_search?q=reviewer:user_0");
+        Integer actual = (Integer) ((List<?>) XContentMapValues.extractValue("hits.hits._source.pipeline_field", searchResult)).get(0);
+        assertThat(actual, equalTo(pipelineValue));
+    }
+
+    public void testContinuousPivot() throws Exception {
+        String indexName = "continuous_reviews";
+        createReviewsIndex(indexName);
+        String transformId = "simple_continuous_pivot";
+        String dataFrameIndex = "pivot_reviews_continuous";
+        setupDataAccessRole(DATA_ACCESS_ROLE, indexName, dataFrameIndex);
+        final Request createDataframeTransformRequest = createRequestWithAuth("PUT", DATAFRAME_ENDPOINT + transformId,
+            BASIC_AUTH_VALUE_DATA_FRAME_ADMIN_WITH_SOME_DATA_ACCESS);
+        String config = "{"
+            + " \"source\": {\"index\":\"" + indexName + "\"},"
+            + " \"dest\": {\"index\":\"" + dataFrameIndex + "\"},"
+            + " \"sync\": {\"time\": {\"field\": \"timestamp\", \"delay\": \"1s\"}},"
+            + " \"pivot\": {"
+            + "   \"group_by\": {"
+            + "     \"reviewer\": {"
+            + "       \"terms\": {"
+            + "         \"field\": \"user_id\""
+            + " } } },"
+            + "   \"aggregations\": {"
+            + "     \"avg_rating\": {"
+            + "       \"avg\": {"
+            + "         \"field\": \"stars\""
+            + " } } } }"
+            + "}";
+        createDataframeTransformRequest.setJsonEntity(config);
+        Map<String, Object> createDataframeTransformResponse = entityAsMap(client().performRequest(createDataframeTransformRequest));
+        assertThat(createDataframeTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+
+        startAndWaitForContinuousTransform(transformId, dataFrameIndex, null);
+        assertTrue(indexExists(dataFrameIndex));
+        // get and check some users
+        assertOnePivotValue(dataFrameIndex + "/_search?q=reviewer:user_0", 3.776978417);
+        assertOnePivotValue(dataFrameIndex + "/_search?q=reviewer:user_5", 3.72);
+        assertOnePivotValue(dataFrameIndex + "/_search?q=reviewer:user_11", 3.846153846);
+        assertOnePivotValue(dataFrameIndex + "/_search?q=reviewer:user_20", 3.769230769);
+        assertOnePivotValue(dataFrameIndex + "/_search?q=reviewer:user_26", 3.918918918);
+
+        Map<String, Object> indexStats = getAsMap(dataFrameIndex + "/_stats");
+        assertEquals(27, XContentMapValues.extractValue("_all.total.docs.count", indexStats));
+        final StringBuilder bulk = new StringBuilder();
+        long user = 42;
+        long user26 = 26;
+
+        long dateStamp = Instant.now().toEpochMilli() - 1_000;
+        for (int i = 0; i < 25; i++) {
+            bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
+            int stars = (i * 32) % 5;
+            long business = (stars * user) % 13;
+            String location = (user + 10) + "," + (user + 15);
+
+            bulk.append("{\"user_id\":\"")
+                .append("user_")
+                .append(user)
+                .append("\",\"business_id\":\"")
+                .append("business_")
+                .append(business)
+                .append("\",\"stars\":")
+                .append(stars)
+                .append(",\"location\":\"")
+                .append(location)
+                .append("\",\"timestamp\":")
+                .append(dateStamp)
+                .append("}\n");
+
+            stars = 5;
+            business = 11;
+            bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
+            bulk.append("{\"user_id\":\"")
+                .append("user_")
+                .append(user26)
+                .append("\",\"business_id\":\"")
+                .append("business_")
+                .append(business)
+                .append("\",\"stars\":")
+                .append(stars)
+                .append(",\"location\":\"")
+                .append(location)
+                .append("\",\"timestamp\":")
+                .append(dateStamp)
+                .append("}\n");
+        }
+        bulk.append("\r\n");
+
+        final Request bulkRequest = new Request("POST", "/_bulk");
+        bulkRequest.addParameter("refresh", "true");
+        bulkRequest.setJsonEntity(bulk.toString());
+        client().performRequest(bulkRequest);
+
+        waitForDataFrameCheckpoint(transformId, 2);
+
+        stopDataFrameTransform(transformId, false);
+        refreshIndex(dataFrameIndex);
+
+        // assert that other users are unchanged
+        assertOnePivotValue(dataFrameIndex + "/_search?q=reviewer:user_0", 3.776978417);
+        assertOnePivotValue(dataFrameIndex + "/_search?q=reviewer:user_5", 3.72);
+        assertOnePivotValue(dataFrameIndex + "/_search?q=reviewer:user_11", 3.846153846);
+        assertOnePivotValue(dataFrameIndex + "/_search?q=reviewer:user_20", 3.769230769);
+
+
+        Map<String, Object> user26searchResult = getAsMap(dataFrameIndex + "/_search?q=reviewer:user_26");
+        assertEquals(1, XContentMapValues.extractValue("hits.total.value", user26searchResult));
+        double actual = (Double) ((List<?>) XContentMapValues.extractValue("hits.hits._source.avg_rating", user26searchResult))
+            .get(0);
+        assertThat(actual, greaterThan(3.92));
+
+        Map<String, Object> user42searchResult = getAsMap(dataFrameIndex + "/_search?q=reviewer:user_42");
+        assertEquals(1, XContentMapValues.extractValue("hits.total.value", user42searchResult));
+        actual = (Double) ((List<?>) XContentMapValues.extractValue("hits.hits._source.avg_rating", user42searchResult))
+            .get(0);
+        assertThat(actual, greaterThan(0.0));
+        assertThat(actual, lessThan(5.0));
+    }
+
     public void testHistogramPivot() throws Exception {
-        String transformId = "simpleHistogramPivot";
+        String transformId = "simple_histogram_pivot";
         String dataFrameIndex = "pivot_reviews_via_histogram";
         setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, dataFrameIndex);
 
@@ -128,7 +283,7 @@ public void testHistogramPivot() throws Exception {
     }
 
     public void testBiggerPivot() throws Exception {
-        String transformId = "biggerPivot";
+        String transformId = "bigger_pivot";
         String dataFrameIndex = "bigger_pivot_reviews";
         setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, dataFrameIndex);
 
@@ -140,38 +295,38 @@ public void testBiggerPivot() throws Exception {
             + " \"dest\": {\"index\":\"" + dataFrameIndex + "\"},";
 
         config += " \"pivot\": {"
-                + "   \"group_by\": {"
-                + "     \"reviewer\": {"
-                + "       \"terms\": {"
-                + "         \"field\": \"user_id\""
-                + " } } },"
-                + "   \"aggregations\": {"
-                + "     \"avg_rating\": {"
-                + "       \"avg\": {"
-                + "         \"field\": \"stars\""
-                + " } },"
-                + "     \"sum_rating\": {"
-                + "       \"sum\": {"
-                + "         \"field\": \"stars\""
-                + " } },"
-                + "     \"cardinality_business\": {"
-                + "       \"cardinality\": {"
-                + "         \"field\": \"business_id\""
-                + " } },"
-                + "     \"min_rating\": {"
-                + "       \"min\": {"
-                + "         \"field\": \"stars\""
-                + " } },"
-                + "     \"max_rating\": {"
-                + "       \"max\": {"
-                + "         \"field\": \"stars\""
-                + " } },"
-                + "     \"count\": {"
-                + "       \"value_count\": {"
-                + "         \"field\": \"business_id\""
-                + " } }"
-                + " } }"
-                + "}";
+            + "   \"group_by\": {"
+            + "     \"reviewer\": {"
+            + "       \"terms\": {"
+            + "         \"field\": \"user_id\""
+            + " } } },"
+            + "   \"aggregations\": {"
+            + "     \"avg_rating\": {"
+            + "       \"avg\": {"
+            + "         \"field\": \"stars\""
+            + " } },"
+            + "     \"sum_rating\": {"
+            + "       \"sum\": {"
+            + "         \"field\": \"stars\""
+            + " } },"
+            + "     \"cardinality_business\": {"
+            + "       \"cardinality\": {"
+            + "         \"field\": \"business_id\""
+            + " } },"
+            + "     \"min_rating\": {"
+            + "       \"min\": {"
+            + "         \"field\": \"stars\""
+            + " } },"
+            + "     \"max_rating\": {"
+            + "       \"max\": {"
+            + "         \"field\": \"stars\""
+            + " } },"
+            + "     \"count\": {"
+            + "       \"value_count\": {"
+            + "         \"field\": \"business_id\""
+            + " } }"
+            + " } }"
+            + "}";
 
         createDataframeTransformRequest.setJsonEntity(config);
         Map<String, Object> createDataframeTransformResponse = entityAsMap(client().performRequest(createDataframeTransformRequest));
@@ -203,7 +358,7 @@ public void testBiggerPivot() throws Exception {
     }
 
     public void testDateHistogramPivot() throws Exception {
-        String transformId = "simpleDateHistogramPivot";
+        String transformId = "simple_date_histogram_pivot";
         String dataFrameIndex = "pivot_reviews_via_date_histogram";
         setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, dataFrameIndex);
 
@@ -218,7 +373,7 @@ public void testDateHistogramPivot() throws Exception {
             + "   \"group_by\": {"
             + "     \"by_hr\": {"
             + "       \"date_histogram\": {"
-            + "         \"fixed_interval\": \"1h\",\"field\":\"timestamp\",\"format\":\"yyyy-MM-dd_HH\""
+            + "         \"fixed_interval\": \"1h\",\"field\":\"timestamp\""
             + " } } },"
             + "   \"aggregations\": {"
             + "     \"avg_rating\": {"
@@ -251,10 +406,60 @@ public void testPreviewTransform() throws Exception {
 
         config += " \"pivot\": {"
             + "   \"group_by\": {"
-            + "     \"reviewer\": {\"terms\": { \"field\": \"user_id\" }},"
-            + "     \"by_day\": {\"date_histogram\": {\"fixed_interval\": \"1d\",\"field\":\"timestamp\",\"format\":\"yyyy-MM-dd\"}}},"
+            + "     \"user.id\": {\"terms\": { \"field\": \"user_id\" }},"
+            + "     \"by_day\": {\"date_histogram\": {\"fixed_interval\": \"1d\",\"field\":\"timestamp\"}}},"
             + "   \"aggregations\": {"
-            + "     \"avg_rating\": {"
+            + "     \"user.avg_rating\": {"
+            + "       \"avg\": {"
+            + "         \"field\": \"stars\""
+            + " } } } }"
+            + "}";
+        createPreviewRequest.setJsonEntity(config);
+
+        Map<String, Object> previewDataframeResponse = entityAsMap(client().performRequest(createPreviewRequest));
+        List<Map<String, Object>> preview = (List<Map<String, Object>>) previewDataframeResponse.get("preview");
+        // preview is limited to 100
+        assertThat(preview.size(), equalTo(100));
+        Set<String> expectedTopLevelFields = new HashSet<>(Arrays.asList("user", "by_day"));
+        Set<String> expectedNestedFields = new HashSet<>(Arrays.asList("id", "avg_rating"));
+        preview.forEach(p -> {
+            Set<String> keys = p.keySet();
+            assertThat(keys, equalTo(expectedTopLevelFields));
+            Map<String, Object> nestedObj = (Map<String, Object>) p.get("user");
+            keys = nestedObj.keySet();
+            assertThat(keys, equalTo(expectedNestedFields));
+        });
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testPreviewTransformWithPipeline() throws Exception {
+        String pipelineId = "my-preview-pivot-pipeline";
+        int pipelineValue = 42;
+        Request pipelineRequest = new Request("PUT", "/_ingest/pipeline/" + pipelineId);
+        pipelineRequest.setJsonEntity("{\n" +
+            "  \"description\" : \"my pivot preview pipeline\",\n" +
+            "  \"processors\" : [\n" +
+            "    {\n" +
+            "      \"set\" : {\n" +
+            "        \"field\": \"pipeline_field\",\n" +
+            "        \"value\": " + pipelineValue +
+            "      }\n" +
+            "    }\n" +
+            "  ]\n" +
+            "}");
+        client().performRequest(pipelineRequest);
+
+        setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME);
+        final Request createPreviewRequest = createRequestWithAuth("POST", DATAFRAME_ENDPOINT + "_preview", null);
+
+        String config = "{ \"source\": {\"index\":\"" + REVIEWS_INDEX_NAME + "\"} ,"
+            + "\"dest\": {\"pipeline\": \"" + pipelineId + "\"},"
+            + " \"pivot\": {"
+            + "   \"group_by\": {"
+            + "     \"user.id\": {\"terms\": { \"field\": \"user_id\" }},"
+            + "     \"by_day\": {\"date_histogram\": {\"fixed_interval\": \"1d\",\"field\":\"timestamp\"}}},"
+            + "   \"aggregations\": {"
+            + "     \"user.avg_rating\": {"
             + "       \"avg\": {"
             + "         \"field\": \"stars\""
             + " } } } }"
@@ -265,15 +470,20 @@ public void testPreviewTransform() throws Exception {
         List<Map<String, Object>> preview = (List<Map<String, Object>>)previewDataframeResponse.get("preview");
         // preview is limited to 100
         assertThat(preview.size(), equalTo(100));
-        Set<String> expectedFields = new HashSet<>(Arrays.asList("reviewer", "by_day", "avg_rating"));
+        Set<String> expectedTopLevelFields = new HashSet<>(Arrays.asList("user", "by_day", "pipeline_field"));
+        Set<String> expectedNestedFields = new HashSet<>(Arrays.asList("id", "avg_rating"));
         preview.forEach(p -> {
             Set<String> keys = p.keySet();
-            assertThat(keys, equalTo(expectedFields));
+            assertThat(keys, equalTo(expectedTopLevelFields));
+            assertThat(p.get("pipeline_field"), equalTo(pipelineValue));
+            Map<String, Object> nestedObj = (Map<String, Object>)p.get("user");
+            keys = nestedObj.keySet();
+            assertThat(keys, equalTo(expectedNestedFields));
         });
     }
 
     public void testPivotWithMaxOnDateField() throws Exception {
-        String transformId = "simpleDateHistogramPivotWithMaxTime";
+        String transformId = "simple_date_histogram_pivot_with_max_time";
         String dataFrameIndex = "pivot_reviews_via_date_histogram_with_max_time";
         setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, dataFrameIndex);
 
@@ -287,7 +497,7 @@ public void testPivotWithMaxOnDateField() throws Exception {
         config +="    \"pivot\": { \n" +
             "        \"group_by\": {\n" +
             "            \"by_day\": {\"date_histogram\": {\n" +
-            "                \"fixed_interval\": \"1d\",\"field\":\"timestamp\",\"format\":\"yyyy-MM-dd\"\n" +
+            "                \"fixed_interval\": \"1d\",\"field\":\"timestamp\"\n" +
             "            }}\n" +
             "        },\n" +
             "    \n" +
@@ -320,7 +530,7 @@ public void testPivotWithMaxOnDateField() throws Exception {
     }
 
     public void testPivotWithScriptedMetricAgg() throws Exception {
-        String transformId = "scriptedMetricPivot";
+        String transformId = "scripted_metric_pivot";
         String dataFrameIndex = "scripted_metric_pivot_reviews";
         setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, dataFrameIndex);
 
@@ -373,7 +583,7 @@ public void testPivotWithScriptedMetricAgg() throws Exception {
     }
 
     public void testPivotWithBucketScriptAgg() throws Exception {
-        String transformId = "bucketScriptPivot";
+        String transformId = "bucket_script_pivot";
         String dataFrameIndex = "bucket_script_pivot_reviews";
         setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, dataFrameIndex);
 
@@ -424,7 +634,7 @@ public void testPivotWithBucketScriptAgg() throws Exception {
     }
 
     public void testPivotWithGeoCentroidAgg() throws Exception {
-        String transformId = "geoCentroidPivot";
+        String transformId = "geo_centroid_pivot";
         String dataFrameIndex = "geo_centroid_pivot_reviews";
         setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, dataFrameIndex);
 
@@ -473,6 +683,45 @@ public void testPivotWithGeoCentroidAgg() throws Exception {
         assertEquals((4 + 15), Double.valueOf(latlon[1]), 0.000001);
     }
 
+    public void testPivotWithWeightedAvgAgg() throws Exception {
+        String transformId = "weighted_avg_agg_transform";
+        String dataFrameIndex = "weighted_avg_pivot_reviews";
+        setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, dataFrameIndex);
+
+        final Request createDataframeTransformRequest = createRequestWithAuth("PUT", DATAFRAME_ENDPOINT + transformId,
+            BASIC_AUTH_VALUE_DATA_FRAME_ADMIN_WITH_SOME_DATA_ACCESS);
+
+        String config = "{"
+            + " \"source\": {\"index\":\"" + REVIEWS_INDEX_NAME + "\"},"
+            + " \"dest\": {\"index\":\"" + dataFrameIndex + "\"},";
+
+        config += " \"pivot\": {"
+            + "   \"group_by\": {"
+            + "     \"reviewer\": {"
+            + "       \"terms\": {"
+            + "         \"field\": \"user_id\""
+            + " } } },"
+            + "   \"aggregations\": {"
+            + "     \"avg_rating\": {"
+            + "       \"weighted_avg\": {"
+            + "         \"value\": {\"field\": \"stars\"},"
+            + "         \"weight\": {\"field\": \"stars\"}"
+            + "} } } }"
+            + "}";
+
+        createDataframeTransformRequest.setJsonEntity(config);
+        Map<String, Object> createDataframeTransformResponse = entityAsMap(client().performRequest(createDataframeTransformRequest));
+        assertThat(createDataframeTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+
+        startAndWaitForTransform(transformId, dataFrameIndex, BASIC_AUTH_VALUE_DATA_FRAME_ADMIN_WITH_SOME_DATA_ACCESS);
+        assertTrue(indexExists(dataFrameIndex));
+
+        Map<String, Object> searchResult = getAsMap(dataFrameIndex + "/_search?q=reviewer:user_4");
+        assertEquals(1, XContentMapValues.extractValue("hits.total.value", searchResult));
+        Number actual = (Number) ((List<?>) XContentMapValues.extractValue("hits.hits._source.avg_rating", searchResult)).get(0);
+        assertEquals(4.47169811, actual.doubleValue(), 0.000001);
+    }
+
     private void assertOnePivotValue(String query, double expected) throws IOException {
         Map<String, Object> searchResult = getAsMap(query);
 
diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameRestTestCase.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameRestTestCase.java
index 23bff163031ce..c8d7bf28842e3 100644
--- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameRestTestCase.java
+++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameRestTestCase.java
@@ -51,10 +51,7 @@ protected Settings restClientSettings() {
         return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE_SUPER_USER).build();
     }
 
-    /**
-     * Create a simple dataset for testing with reviewers, ratings and businesses
-     */
-    protected void createReviewsIndex() throws IOException {
+    protected void createReviewsIndex(String indexName) throws IOException {
         int[] distributionTable = {5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 3, 3, 2, 1, 1, 1};
 
         final int numDocs = 1000;
@@ -65,27 +62,27 @@ protected void createReviewsIndex() throws IOException {
             {
                 builder.startObject("mappings")
                     .startObject("properties")
-                      .startObject("timestamp")
-                         .field("type", "date")
-                      .endObject()
-                      .startObject("user_id")
-                        .field("type", "keyword")
-                      .endObject()
-                      .startObject("business_id")
-                        .field("type", "keyword")
-                      .endObject()
-                      .startObject("stars")
-                        .field("type", "integer")
-                      .endObject()
-                      .startObject("location")
-                        .field("type", "geo_point")
-                      .endObject()
+                    .startObject("timestamp")
+                    .field("type", "date")
+                    .endObject()
+                    .startObject("user_id")
+                    .field("type", "keyword")
+                    .endObject()
+                    .startObject("business_id")
+                    .field("type", "keyword")
                     .endObject()
-                  .endObject();
+                    .startObject("stars")
+                    .field("type", "integer")
+                    .endObject()
+                    .startObject("location")
+                    .field("type", "geo_point")
+                    .endObject()
+                    .endObject()
+                    .endObject();
             }
             builder.endObject();
             final StringEntity entity = new StringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON);
-            Request req = new Request("PUT", REVIEWS_INDEX_NAME);
+            Request req = new Request("PUT", indexName);
             req.setEntity(entity);
             client().performRequest(req);
         }
@@ -96,7 +93,7 @@ protected void createReviewsIndex() throws IOException {
         int hour = 10;
         int min = 10;
         for (int i = 0; i < numDocs; i++) {
-            bulk.append("{\"index\":{\"_index\":\"" + REVIEWS_INDEX_NAME + "\"}}\n");
+            bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n");
             long user = Math.round(Math.pow(i * 31 % 1000, distributionTable[i % distributionTable.length]) % 27);
             int stars = distributionTable[(i * 33) % distributionTable.length];
             long business = Math.round(Math.pow(user * stars, distributionTable[i % distributionTable.length]) % 13);
@@ -142,17 +139,60 @@ protected void createReviewsIndex() throws IOException {
         bulkRequest.setJsonEntity(bulk.toString());
         client().performRequest(bulkRequest);
     }
+    /**
+     * Create a simple dataset for testing with reviewers, ratings and businesses
+     */
+    protected void createReviewsIndex() throws IOException {
+        createReviewsIndex(REVIEWS_INDEX_NAME);
+    }
 
     protected void createPivotReviewsTransform(String transformId, String dataFrameIndex, String query) throws IOException {
         createPivotReviewsTransform(transformId, dataFrameIndex, query, null);
     }
 
-    protected void createPivotReviewsTransform(String transformId, String dataFrameIndex, String query, String authHeader)
+    protected void createPivotReviewsTransform(String transformId, String dataFrameIndex, String query, String pipeline)
+        throws IOException {
+        createPivotReviewsTransform(transformId, dataFrameIndex, query, pipeline, null);
+    }
+
+    protected void createContinuousPivotReviewsTransform(String transformId, String dataFrameIndex, String authHeader) throws IOException {
+
+        final Request createDataframeTransformRequest = createRequestWithAuth("PUT", DATAFRAME_ENDPOINT + transformId, authHeader);
+
+        String config = "{ \"dest\": {\"index\":\"" + dataFrameIndex + "\"},"
+            + " \"source\": {\"index\":\"" + REVIEWS_INDEX_NAME + "\"},"
+            + " \"sync\": {\"time\":{\"field\": \"timestamp\", \"delay\": \"15m\"}},"
+            + " \"pivot\": {"
+            + "   \"group_by\": {"
+            + "     \"reviewer\": {"
+            + "       \"terms\": {"
+            + "         \"field\": \"user_id\""
+            + " } } },"
+            + "   \"aggregations\": {"
+            + "     \"avg_rating\": {"
+            + "       \"avg\": {"
+            + "         \"field\": \"stars\""
+            + " } } } }"
+            + "}";
+
+        createDataframeTransformRequest.setJsonEntity(config);
+
+        Map<String, Object> createDataframeTransformResponse = entityAsMap(client().performRequest(createDataframeTransformRequest));
+        assertThat(createDataframeTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
+    }
+
+
+    protected void createPivotReviewsTransform(String transformId, String dataFrameIndex, String query, String pipeline, String authHeader)
         throws IOException {
         final Request createDataframeTransformRequest = createRequestWithAuth("PUT", DATAFRAME_ENDPOINT + transformId, authHeader);
 
-        String config = "{"
-            + " \"dest\": {\"index\":\"" + dataFrameIndex + "\"},";
+        String config = "{";
+
+        if (pipeline != null) {
+            config += " \"dest\": {\"index\":\"" + dataFrameIndex + "\", \"pipeline\":\"" + pipeline + "\"},";
+        } else {
+            config += " \"dest\": {\"index\":\"" + dataFrameIndex + "\"},";
+        }
 
         if (query != null) {
             config += " \"source\": {\"index\":\"" + REVIEWS_INDEX_NAME + "\", \"query\":{" + query + "}},";
@@ -191,7 +231,7 @@ protected void startDataframeTransform(String transformId, boolean force, String
             startTransformRequest.setOptions(expectWarnings(warnings));
         }
         Map<String, Object> startTransformResponse = entityAsMap(client().performRequest(startTransformRequest));
-        assertThat(startTransformResponse.get("started"), equalTo(Boolean.TRUE));
+        assertThat(startTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
     }
 
     protected void stopDataFrameTransform(String transformId, boolean force) throws Exception {
@@ -200,7 +240,7 @@ protected void stopDataFrameTransform(String transformId, boolean force) throws
         stopTransformRequest.addParameter(DataFrameField.FORCE.getPreferredName(), Boolean.toString(force));
         stopTransformRequest.addParameter(DataFrameField.WAIT_FOR_COMPLETION.getPreferredName(), Boolean.toString(true));
         Map<String, Object> stopTransformResponse = entityAsMap(client().performRequest(stopTransformRequest));
-        assertThat(stopTransformResponse.get("stopped"), equalTo(Boolean.TRUE));
+        assertThat(stopTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
     }
 
     protected void startAndWaitForTransform(String transformId, String dataFrameIndex) throws Exception {
@@ -218,6 +258,26 @@ protected void startAndWaitForTransform(String transformId, String dataFrameInde
         assertTrue(indexExists(dataFrameIndex));
         // wait until the dataframe has been created and all data is available
         waitForDataFrameCheckpoint(transformId);
+
+        waitForDataFrameStopped(transformId);
+        refreshIndex(dataFrameIndex);
+    }
+
+    protected void startAndWaitForContinuousTransform(String transformId,
+                                                      String dataFrameIndex,
+                                                      String authHeader) throws Exception {
+        startAndWaitForContinuousTransform(transformId, dataFrameIndex, authHeader, 1L);
+    }
+
+    protected void startAndWaitForContinuousTransform(String transformId,
+                                                      String dataFrameIndex,
+                                                      String authHeader,
+                                                      long checkpoint) throws Exception {
+        // start the transform
+        startDataframeTransform(transformId, false, authHeader, new String[0]);
+        assertTrue(indexExists(dataFrameIndex));
+        // wait until the dataframe has been created and all data is available
+        waitForDataFrameCheckpoint(transformId, checkpoint);
         refreshIndex(dataFrameIndex);
     }
 
@@ -233,11 +293,19 @@ protected Request createRequestWithAuth(final String method, final String endpoi
         return request;
     }
 
-    void waitForDataFrameCheckpoint(String transformId) throws Exception {
+    void waitForDataFrameStopped(String transformId) throws Exception {
         assertBusy(() -> {
-            long checkpoint = getDataFrameCheckpoint(transformId);
-            assertEquals(1, checkpoint);
-        }, 30, TimeUnit.SECONDS);
+            assertEquals("stopped", getDataFrameTaskState(transformId));
+            assertEquals("stopped", getDataFrameIndexerState(transformId));
+        }, 15, TimeUnit.SECONDS);
+    }
+
+    void waitForDataFrameCheckpoint(String transformId) throws Exception {
+        waitForDataFrameCheckpoint(transformId, 1L);
+    }
+
+    void waitForDataFrameCheckpoint(String transformId, long checkpoint) throws Exception {
+        assertBusy(() -> assertEquals(checkpoint, getDataFrameCheckpoint(transformId)), 30, TimeUnit.SECONDS);
     }
 
     void refreshIndex(String index) throws IOException {
diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTaskFailedStateIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTaskFailedStateIT.java
index 7b63644dd34ad..96aeeda8755f4 100644
--- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTaskFailedStateIT.java
+++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTaskFailedStateIT.java
@@ -6,7 +6,6 @@
 
 package org.elasticsearch.xpack.dataframe.integration;
 
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.rest.RestStatus;
@@ -20,7 +19,6 @@
 import static org.hamcrest.CoreMatchers.nullValue;
 import static org.hamcrest.Matchers.equalTo;
 
-@LuceneTestCase.AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/42344")
 public class DataFrameTaskFailedStateIT extends DataFrameRestTestCase {
 
     public void testDummy() {
diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java
index fea225ced3bd9..002126e5b60e8 100644
--- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java
+++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java
@@ -6,7 +6,6 @@
 
 package org.elasticsearch.xpack.dataframe.integration;
 
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
@@ -44,7 +43,6 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
-@LuceneTestCase.AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/42344")
 public class DataFrameTransformProgressIT extends ESRestTestCase {
     protected void createReviewsIndex() throws Exception {
         final int numDocs = 1000;
@@ -121,7 +119,7 @@ protected void createReviewsIndex() throws Exception {
     public void testGetProgress() throws Exception {
         createReviewsIndex();
         SourceConfig sourceConfig = new SourceConfig(REVIEWS_INDEX_NAME);
-        DestConfig destConfig = new DestConfig("unnecessary");
+        DestConfig destConfig = new DestConfig("unnecessary", null);
         GroupConfig histgramGroupConfig = new GroupConfig(Collections.emptyMap(),
             Collections.singletonMap("every_50", new HistogramGroupSource("count", 50.0)));
         AggregatorFactories.Builder aggs = new AggregatorFactories.Builder();
@@ -132,11 +130,14 @@ public void testGetProgress() throws Exception {
             sourceConfig,
             destConfig,
             null,
+            null,
             pivotConfig,
             null);
 
         final RestHighLevelClient restClient = new TestRestHighLevelClient();
-        SearchResponse response = restClient.search(TransformProgressGatherer.getSearchRequest(config), RequestOptions.DEFAULT);
+        SearchResponse response = restClient.search(
+            TransformProgressGatherer.getSearchRequest(config, config.getSource().getQueryConfig().getQuery()),
+            RequestOptions.DEFAULT);
 
         DataFrameTransformProgress progress =
             TransformProgressGatherer.searchResponseToDataFrameTransformProgressFunction().apply(response);
@@ -153,16 +154,37 @@ public void testGetProgress() throws Exception {
             sourceConfig,
             destConfig,
             null,
+            null,
             pivotConfig,
             null);
 
-        response = restClient.search(TransformProgressGatherer.getSearchRequest(config), RequestOptions.DEFAULT);
+        response = restClient.search(TransformProgressGatherer.getSearchRequest(config, config.getSource().getQueryConfig().getQuery()),
+            RequestOptions.DEFAULT);
         progress = TransformProgressGatherer.searchResponseToDataFrameTransformProgressFunction().apply(response);
 
         assertThat(progress.getTotalDocs(), equalTo(35L));
         assertThat(progress.getRemainingDocs(), equalTo(35L));
         assertThat(progress.getPercentComplete(), equalTo(0.0));
 
+        histgramGroupConfig = new GroupConfig(Collections.emptyMap(),
+            Collections.singletonMap("every_50", new HistogramGroupSource("missing_field", 50.0)));
+        pivotConfig = new PivotConfig(histgramGroupConfig, aggregationConfig, null);
+        config = new DataFrameTransformConfig("get_progress_transform",
+            sourceConfig,
+            destConfig,
+            null,
+            null,
+            pivotConfig,
+            null);
+
+        response = restClient.search(TransformProgressGatherer.getSearchRequest(config, config.getSource().getQueryConfig().getQuery()),
+            RequestOptions.DEFAULT);
+        progress = TransformProgressGatherer.searchResponseToDataFrameTransformProgressFunction().apply(response);
+
+        assertThat(progress.getTotalDocs(), equalTo(0L));
+        assertThat(progress.getRemainingDocs(), equalTo(0L));
+        assertThat(progress.getPercentComplete(), equalTo(100.0));
+
         deleteIndex(REVIEWS_INDEX_NAME);
     }
 
diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameUsageIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameUsageIT.java
index f98fa6a271365..9295fd908233d 100644
--- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameUsageIT.java
+++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameUsageIT.java
@@ -6,11 +6,11 @@
 
 package org.elasticsearch.xpack.dataframe.integration;
 
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.xpack.core.dataframe.DataFrameField;
+
+import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats;
 import org.elasticsearch.xpack.dataframe.persistence.DataFrameInternalIndex;
 import org.junit.Before;
@@ -19,30 +19,16 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.xpack.core.dataframe.DataFrameField.INDEX_DOC_TYPE;
-import static org.elasticsearch.xpack.dataframe.DataFrameFeatureSet.PROVIDED_STATS;
+import static org.elasticsearch.xpack.dataframe.DataFrameInfoTransportAction.PROVIDED_STATS;
 
-@LuceneTestCase.AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/42344")
 public class DataFrameUsageIT extends DataFrameRestTestCase {
-    private boolean indicesCreated = false;
-
-    // preserve indices in order to reuse source indices in several test cases
-    @Override
-    protected boolean preserveIndicesUponCompletion() {
-        return true;
-    }
 
     @Before
     public void createIndexes() throws IOException {
-
-        // it's not possible to run it as @BeforeClass as clients aren't initialized then, so we need this little hack
-        if (indicesCreated) {
-            return;
-        }
-
         createReviewsIndex();
-        indicesCreated = true;
     }
 
     public void testUsage() throws Exception {
@@ -57,30 +43,28 @@ public void testUsage() throws Exception {
 
         // create transforms
         createPivotReviewsTransform("test_usage", "pivot_reviews", null);
-        createPivotReviewsTransform("test_usage_no_task", "pivot_reviews_no_task", null);
-        createPivotReviewsTransform("test_usage_no_stats_or_task", "pivot_reviews_no_stats_or_task", null);
+        createPivotReviewsTransform("test_usage_no_stats", "pivot_reviews_no_stats", null);
+        createContinuousPivotReviewsTransform("test_usage_continuous", "pivot_reviews_continuous", null);
         usageResponse = client().performRequest(new Request("GET", "_xpack/usage"));
         usageAsMap = entityAsMap(usageResponse);
         assertEquals(3, XContentMapValues.extractValue("data_frame.transforms._all", usageAsMap));
         assertEquals(3, XContentMapValues.extractValue("data_frame.transforms.stopped", usageAsMap));
 
-        startAndWaitForTransform("test_usage_no_task", "pivot_reviews_no_task");
-        stopDataFrameTransform("test_usage_no_task", false);
-        // Remove the task, we should still have the transform and its stat doc
-        client().performRequest(new Request("POST", "_tasks/_cancel?actions="+ DataFrameField.TASK_NAME+"*"));
-
         startAndWaitForTransform("test_usage", "pivot_reviews");
+        stopDataFrameTransform("test_usage", false);
 
         Request statsExistsRequest = new Request("GET",
             DataFrameInternalIndex.INDEX_NAME+"/_search?q=" +
                 INDEX_DOC_TYPE.getPreferredName() + ":" +
                 DataFrameTransformStateAndStats.NAME);
-        // Verify that we have our two stats documents
+        // Verify that we have one stat document
         assertBusy(() -> {
             Map<String, Object> hasStatsMap = entityAsMap(client().performRequest(statsExistsRequest));
-            assertEquals(2, XContentMapValues.extractValue("hits.total.value", hasStatsMap));
+            assertEquals(1, XContentMapValues.extractValue("hits.total.value", hasStatsMap));
         });
 
+        startAndWaitForContinuousTransform("test_usage_continuous", "pivot_reviews_continuous", null);
+
         Request getRequest = new Request("GET", DATAFRAME_ENDPOINT + "test_usage/_stats");
         Map<String, Object> stats = entityAsMap(client().performRequest(getRequest));
         Map<String, Integer> expectedStats = new HashMap<>();
@@ -92,26 +76,36 @@ public void testUsage() throws Exception {
             expectedStats.put(statName, statistic);
         }
 
-        getRequest = new Request("GET", DATAFRAME_ENDPOINT + "test_usage_no_task/_stats");
-        stats = entityAsMap(client().performRequest(getRequest));
-        for(String statName : PROVIDED_STATS) {
-            @SuppressWarnings("unchecked")
-            List<Integer> specificStatistic = ((List<Integer>)XContentMapValues.extractValue("transforms.stats." + statName, stats));
-            assertNotNull(specificStatistic);
-            Integer statistic = (specificStatistic).get(0);
-            expectedStats.merge(statName, statistic, Integer::sum);
-        }
+        // Simply because we wait for continuous to reach checkpoint 1, does not mean that the statistics are written yet.
+        // Since we search against the indices for the statistics, we need to ensure they are written, so we will wait for that
+        // to be the case.
+        assertBusy(() -> {
+            Response response = client().performRequest(new Request("GET", "_xpack/usage"));
+            Map<String, Object> statsMap = entityAsMap(response);
+            // we should see some stats
+            assertEquals(3, XContentMapValues.extractValue("data_frame.transforms._all", statsMap));
+            assertEquals(2, XContentMapValues.extractValue("data_frame.transforms.stopped", statsMap));
+            assertEquals(1, XContentMapValues.extractValue("data_frame.transforms.started", statsMap));
+            for(String statName : PROVIDED_STATS) {
+                if (statName.equals(DataFrameIndexerTransformStats.INDEX_TIME_IN_MS.getPreferredName())
+                    ||statName.equals(DataFrameIndexerTransformStats.SEARCH_TIME_IN_MS.getPreferredName())) {
+                    continue;
+                }
+                assertEquals("Incorrect stat " +  statName,
+                    expectedStats.get(statName) * 2,
+                    XContentMapValues.extractValue("data_frame.stats." + statName, statsMap));
+            }
+            // Refresh the index so that statistics are searchable
+            refreshIndex(DataFrameInternalIndex.INDEX_TEMPLATE_NAME);
+        }, 60, TimeUnit.SECONDS);
+
+
+        stopDataFrameTransform("test_usage_continuous", false);
 
         usageResponse = client().performRequest(new Request("GET", "_xpack/usage"));
-
         usageAsMap = entityAsMap(usageResponse);
-        // we should see some stats
+
         assertEquals(3, XContentMapValues.extractValue("data_frame.transforms._all", usageAsMap));
-        assertEquals(1, XContentMapValues.extractValue("data_frame.transforms.started", usageAsMap));
-        assertEquals(2, XContentMapValues.extractValue("data_frame.transforms.stopped", usageAsMap));
-        for(String statName : PROVIDED_STATS) {
-            assertEquals("Incorrect stat " +  statName,
-                    expectedStats.get(statName), XContentMapValues.extractValue("data_frame.stats." + statName, usageAsMap));
-        }
+        assertEquals(3, XContentMapValues.extractValue("data_frame.transforms.stopped", usageAsMap));
     }
 }
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrame.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrame.java
index e8206311c012b..f83a546f12510 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrame.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrame.java
@@ -16,7 +16,6 @@
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -24,6 +23,7 @@
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry.Entry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.license.XPackLicenseState;
@@ -40,6 +40,9 @@
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.dataframe.DataFrameNamedXContentProvider;
 import org.elasticsearch.xpack.core.dataframe.action.DeleteDataFrameTransformAction;
 import org.elasticsearch.xpack.core.dataframe.action.GetDataFrameTransformsAction;
 import org.elasticsearch.xpack.core.dataframe.action.GetDataFrameTransformsStatsAction;
@@ -72,7 +75,6 @@
 
 import java.io.IOException;
 import java.time.Clock;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -102,13 +104,6 @@ public DataFrame(Settings settings) {
         this.enabled = XPackSettings.DATA_FRAME_ENABLED.get(settings);
     }
 
-    @Override
-    public Collection<Module> createGuiceModules() {
-        List<Module> modules = new ArrayList<>();
-        modules.add(b -> XPackPlugin.bindFeatureSet(b, DataFrameFeatureSet.class));
-        return modules;
-    }
-
     protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); }
 
     @Override
@@ -133,8 +128,10 @@ public List<RestHandler> getRestHandlers(final Settings settings, final RestCont
 
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
-        if (!enabled) {
-            return emptyList();
+        var usageAction = new ActionHandler<>(XPackUsageFeatureAction.DATA_FRAME, DataFrameUsageTransportAction.class);
+        var infoAction = new ActionHandler<>(XPackInfoFeatureAction.DATA_FRAME, DataFrameInfoTransportAction.class);
+        if (enabled == false) {
+            return Arrays.asList(usageAction, infoAction);
         }
 
         return Arrays.asList(
@@ -145,8 +142,9 @@ public List<RestHandler> getRestHandlers(final Settings settings, final RestCont
                 new ActionHandler<>(DeleteDataFrameTransformAction.INSTANCE, TransportDeleteDataFrameTransformAction.class),
                 new ActionHandler<>(GetDataFrameTransformsAction.INSTANCE, TransportGetDataFrameTransformsAction.class),
                 new ActionHandler<>(GetDataFrameTransformsStatsAction.INSTANCE, TransportGetDataFrameTransformsStatsAction.class),
-                new ActionHandler<>(PreviewDataFrameTransformAction.INSTANCE, TransportPreviewDataFrameTransformAction.class)
-                );
+                new ActionHandler<>(PreviewDataFrameTransformAction.INSTANCE, TransportPreviewDataFrameTransformAction.class),
+                usageAction,
+                infoAction);
     }
 
     @Override
@@ -217,4 +215,9 @@ public void close() {
             schedulerEngine.get().stop();
         }
     }
+
+    @Override
+    public List<Entry> getNamedXContent() {
+        return new DataFrameNamedXContentProvider().getNamedXContentParsers();
+    }
 }
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameFeatureSet.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameInfoTransportAction.java
similarity index 54%
rename from x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameFeatureSet.java
rename to x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameInfoTransportAction.java
index 82b8a6060e44e..4988d0e71df97 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameFeatureSet.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameInfoTransportAction.java
@@ -10,53 +10,40 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
-import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.dataframe.DataFrameFeatureSetUsage;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats;
-import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransform;
-import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig;
-import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformState;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats;
-import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformTaskState;
 import org.elasticsearch.xpack.dataframe.persistence.DataFrameInternalIndex;
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.Objects;
 
 
-public class DataFrameFeatureSet implements XPackFeatureSet {
+public class DataFrameInfoTransportAction extends XPackInfoFeatureTransportAction {
 
     private final boolean enabled;
-    private final Client client;
     private final XPackLicenseState licenseState;
-    private final ClusterService clusterService;
 
-    private static final Logger logger = LogManager.getLogger(DataFrameFeatureSet.class);
+    private static final Logger logger = LogManager.getLogger(DataFrameInfoTransportAction.class);
 
     public static final String[] PROVIDED_STATS = new String[] {
         DataFrameIndexerTransformStats.NUM_PAGES.getPreferredName(),
@@ -72,10 +59,10 @@ public class DataFrameFeatureSet implements XPackFeatureSet {
     };
 
     @Inject
-    public DataFrameFeatureSet(Settings settings, ClusterService clusterService, Client client, @Nullable XPackLicenseState licenseState) {
+    public DataFrameInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                        Settings settings, XPackLicenseState licenseState) {
+        super(XPackInfoFeatureAction.DATA_FRAME.name(), transportService, actionFilters);
         this.enabled = XPackSettings.DATA_FRAME_ENABLED.get(settings);
-        this.client = Objects.requireNonNull(client);
-        this.clusterService = Objects.requireNonNull(clusterService);
         this.licenseState = licenseState;
     }
 
@@ -84,14 +71,9 @@ public String name() {
         return XPackField.DATA_FRAME;
     }
 
-    @Override
-    public String description() {
-        return "Data Frame for the Elastic Stack";
-    }
-
     @Override
     public boolean available() {
-        return licenseState != null && licenseState.isDataFrameAllowed();
+        return licenseState.isDataFrameAllowed();
     }
 
     @Override
@@ -99,79 +81,6 @@ public boolean enabled() {
         return enabled;
     }
 
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return null;
-    }
-
-    @Override
-    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
-        if (enabled == false) {
-            listener.onResponse(new DataFrameFeatureSetUsage(available(),
-                enabled(),
-                Collections.emptyMap(),
-                DataFrameIndexerTransformStats.withDefaultTransformId()));
-            return;
-        }
-
-        PersistentTasksCustomMetaData taskMetadata = PersistentTasksCustomMetaData.getPersistentTasksCustomMetaData(clusterService.state());
-        Collection<PersistentTasksCustomMetaData.PersistentTask<?>> dataFrameTasks = taskMetadata == null ?
-            Collections.emptyList() :
-            taskMetadata.findTasks(DataFrameTransform.NAME, (t) -> true);
-        final int taskCount = dataFrameTasks.size();
-        final Map<String, Long> transformsCountByState = new HashMap<>();
-        for(PersistentTasksCustomMetaData.PersistentTask<?> dataFrameTask : dataFrameTasks) {
-            DataFrameTransformState state = (DataFrameTransformState)dataFrameTask.getState();
-            transformsCountByState.merge(state.getTaskState().value(), 1L, Long::sum);
-        }
-
-        ActionListener<DataFrameIndexerTransformStats> totalStatsListener = ActionListener.wrap(
-            statSummations -> listener.onResponse(new DataFrameFeatureSetUsage(available(),
-                enabled(),
-                transformsCountByState,
-                statSummations)),
-            listener::onFailure
-        );
-
-        ActionListener<SearchResponse> totalTransformCountListener = ActionListener.wrap(
-            transformCountSuccess -> {
-                if (transformCountSuccess.getShardFailures().length > 0) {
-                    logger.error("total transform count search returned shard failures: {}",
-                        Arrays.toString(transformCountSuccess.getShardFailures()));
-                }
-                long totalTransforms = transformCountSuccess.getHits().getTotalHits().value;
-                if (totalTransforms == 0) {
-                    listener.onResponse(new DataFrameFeatureSetUsage(available(),
-                        enabled(),
-                        transformsCountByState,
-                        DataFrameIndexerTransformStats.withDefaultTransformId()));
-                    return;
-                }
-                transformsCountByState.merge(DataFrameTransformTaskState.STOPPED.value(), totalTransforms - taskCount, Long::sum);
-                getStatisticSummations(client, totalStatsListener);
-            },
-            transformCountFailure -> {
-               if (transformCountFailure instanceof ResourceNotFoundException) {
-                   getStatisticSummations(client, totalStatsListener);
-               } else {
-                   listener.onFailure(transformCountFailure);
-               }
-            }
-        );
-
-        SearchRequest totalTransformCount = client.prepareSearch(DataFrameInternalIndex.INDEX_NAME)
-            .setTrackTotalHits(true)
-            .setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.boolQuery()
-                .filter(QueryBuilders.termQuery(DataFrameField.INDEX_DOC_TYPE.getPreferredName(), DataFrameTransformConfig.NAME))))
-            .request();
-
-        ClientHelper.executeAsyncWithOrigin(client.threadPool().getThreadContext(),
-            ClientHelper.DATA_FRAME_ORIGIN,
-            totalTransformCount,
-            totalTransformCountListener,
-            client::search);
-    }
-
     static DataFrameIndexerTransformStats parseSearchAggs(SearchResponse searchResponse) {
         List<Long> statisticsList = new ArrayList<>(PROVIDED_STATS.length);
 
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameUsageTransportAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameUsageTransportAction.java
new file mode 100644
index 0000000000000..01e729e71bfcf
--- /dev/null
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrameUsageTransportAction.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.dataframe;
+
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.dataframe.DataFrameFeatureSetUsage;
+import org.elasticsearch.xpack.core.dataframe.DataFrameField;
+import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats;
+import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransform;
+import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig;
+import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformState;
+import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformTaskState;
+import org.elasticsearch.xpack.dataframe.persistence.DataFrameInternalIndex;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class DataFrameUsageTransportAction extends XPackUsageFeatureTransportAction {
+
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+    private final Client client;
+
+    @Inject
+    public DataFrameUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                         ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                         Settings settings, XPackLicenseState licenseState, Client client) {
+        super(XPackUsageFeatureAction.DATA_FRAME.name(), transportService, clusterService,
+            threadPool, actionFilters, indexNameExpressionResolver);
+        this.enabled = XPackSettings.DATA_FRAME_ENABLED.get(settings);
+        this.licenseState = licenseState;
+        this.client = client;
+    }
+
+    @Override
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
+        boolean available = licenseState.isDataFrameAllowed();
+        if (enabled == false) {
+            var usage = new DataFrameFeatureSetUsage(available, enabled, Collections.emptyMap(),
+                DataFrameIndexerTransformStats.withDefaultTransformId());
+            listener.onResponse(new XPackUsageFeatureResponse(usage));
+            return;
+        }
+
+        PersistentTasksCustomMetaData taskMetadata = PersistentTasksCustomMetaData.getPersistentTasksCustomMetaData(state);
+        Collection<PersistentTasksCustomMetaData.PersistentTask<?>> dataFrameTasks = taskMetadata == null ?
+            Collections.emptyList() :
+            taskMetadata.findTasks(DataFrameTransform.NAME, (t) -> true);
+        final int taskCount = dataFrameTasks.size();
+        final Map<String, Long> transformsCountByState = new HashMap<>();
+        for(PersistentTasksCustomMetaData.PersistentTask<?> dataFrameTask : dataFrameTasks) {
+            DataFrameTransformState transformState = (DataFrameTransformState)dataFrameTask.getState();
+            transformsCountByState.merge(transformState.getTaskState().value(), 1L, Long::sum);
+        }
+
+        ActionListener<DataFrameIndexerTransformStats> totalStatsListener = ActionListener.wrap(
+            statSummations -> {
+                var usage = new DataFrameFeatureSetUsage(available, enabled, transformsCountByState, statSummations);
+                listener.onResponse(new XPackUsageFeatureResponse(usage));
+            },
+            listener::onFailure
+        );
+
+        ActionListener<SearchResponse> totalTransformCountListener = ActionListener.wrap(
+            transformCountSuccess -> {
+                if (transformCountSuccess.getShardFailures().length > 0) {
+                    logger.error("total transform count search returned shard failures: {}",
+                        Arrays.toString(transformCountSuccess.getShardFailures()));
+                }
+                long totalTransforms = transformCountSuccess.getHits().getTotalHits().value;
+                if (totalTransforms == 0) {
+                    var usage = new DataFrameFeatureSetUsage(available, enabled, transformsCountByState,
+                        DataFrameIndexerTransformStats.withDefaultTransformId());
+                    listener.onResponse(new XPackUsageFeatureResponse(usage));
+                    return;
+                }
+                transformsCountByState.merge(DataFrameTransformTaskState.STOPPED.value(), totalTransforms - taskCount, Long::sum);
+                DataFrameInfoTransportAction.getStatisticSummations(client, totalStatsListener);
+            },
+            transformCountFailure -> {
+                if (transformCountFailure instanceof ResourceNotFoundException) {
+                    DataFrameInfoTransportAction.getStatisticSummations(client, totalStatsListener);
+                } else {
+                    listener.onFailure(transformCountFailure);
+                }
+            }
+        );
+
+        SearchRequest totalTransformCount = client.prepareSearch(DataFrameInternalIndex.INDEX_NAME)
+            .setTrackTotalHits(true)
+            .setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.boolQuery()
+                .filter(QueryBuilders.termQuery(DataFrameField.INDEX_DOC_TYPE.getPreferredName(), DataFrameTransformConfig.NAME))))
+            .request();
+
+        ClientHelper.executeAsyncWithOrigin(client.threadPool().getThreadContext(),
+            ClientHelper.DATA_FRAME_ORIGIN,
+            totalTransformCount,
+            totalTransformCountListener,
+            client::search);
+    }
+}
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportDeleteDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportDeleteDataFrameTransformAction.java
index ac40334dfb443..3c6a9dda611c6 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportDeleteDataFrameTransformAction.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportDeleteDataFrameTransformAction.java
@@ -19,10 +19,12 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.dataframe.action.DeleteDataFrameTransformAction;
 import org.elasticsearch.xpack.core.dataframe.action.DeleteDataFrameTransformAction.Request;
+import org.elasticsearch.xpack.dataframe.notifications.DataFrameAuditor;
 import org.elasticsearch.xpack.dataframe.persistence.DataFrameTransformsConfigManager;
 
 import java.io.IOException;
@@ -30,14 +32,16 @@
 public class TransportDeleteDataFrameTransformAction extends TransportMasterNodeAction<Request, AcknowledgedResponse> {
 
     private final DataFrameTransformsConfigManager transformsConfigManager;
+    private final DataFrameAuditor auditor;
 
     @Inject
     public TransportDeleteDataFrameTransformAction(TransportService transportService, ActionFilters actionFilters, ThreadPool threadPool,
                                                    ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver,
-                                                   DataFrameTransformsConfigManager transformsConfigManager) {
+                                                   DataFrameTransformsConfigManager transformsConfigManager, DataFrameAuditor auditor) {
         super(DeleteDataFrameTransformAction.NAME, transportService, clusterService, threadPool, actionFilters,
                 Request::new, indexNameExpressionResolver);
         this.transformsConfigManager = transformsConfigManager;
+        this.auditor = auditor;
     }
 
     @Override
@@ -46,18 +50,18 @@ protected String executor() {
     }
 
     @Override
-    protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
     }
 
-    protected AcknowledgedResponse read(StreamInput in) throws IOException {
-        AcknowledgedResponse response = new AcknowledgedResponse();
-        response.readFrom(in);
-        return response;
+    @Override
+    protected AcknowledgedResponse newResponse() {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(Request request, ClusterState state, ActionListener<AcknowledgedResponse> listener) throws Exception {
+    protected void masterOperation(Task task, Request request, ClusterState state,
+                                   ActionListener<AcknowledgedResponse> listener) throws Exception {
         PersistentTasksCustomMetaData pTasksMeta = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
         if (pTasksMeta != null && pTasksMeta.getTask(request.getId()) != null) {
             listener.onFailure(new ElasticsearchStatusException("Cannot delete data frame [" + request.getId() +
@@ -65,7 +69,10 @@ protected void masterOperation(Request request, ClusterState state, ActionListen
         } else {
             // Task is not running, delete the configuration document
             transformsConfigManager.deleteTransform(request.getId(), ActionListener.wrap(
-                    r -> listener.onResponse(new AcknowledgedResponse(r)),
+                    r -> {
+                        auditor.info(request.getId(), "Deleted data frame transform.");
+                        listener.onResponse(new AcknowledgedResponse(r));
+                    },
                     listener::onFailure));
         }
     }
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsAction.java
index 00bc15b1db6c5..7d117b2bfb438 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsAction.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsAction.java
@@ -44,7 +44,7 @@ public TransportGetDataFrameTransformsAction(TransportService transportService,
     @Override
     protected void doExecute(Task task, Request request, ActionListener<Response> listener) {
         searchResources(request, ActionListener.wrap(
-            r -> listener.onResponse(new Response(r.results())),
+            r -> listener.onResponse(new Response(r.results(), r.count())),
             listener::onFailure
         ));
     }
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java
index df2d09a875d19..99b5704c7fd53 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -26,7 +27,11 @@
 import org.elasticsearch.xpack.core.dataframe.action.GetDataFrameTransformsStatsAction.Request;
 import org.elasticsearch.xpack.core.dataframe.action.GetDataFrameTransformsStatsAction.Response;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointingInfo;
+import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformState;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats;
+import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformTaskState;
+import org.elasticsearch.xpack.core.dataframe.transforms.NodeAttributes;
+import org.elasticsearch.xpack.core.indexing.IndexerState;
 import org.elasticsearch.xpack.dataframe.checkpoint.DataFrameTransformsCheckpointService;
 import org.elasticsearch.xpack.dataframe.persistence.DataFrameTransformsConfigManager;
 import org.elasticsearch.xpack.dataframe.transforms.DataFrameTransformTask;
@@ -70,7 +75,7 @@ protected Response newResponse(Request request, List<Response> tasks, List<TaskO
             .collect(Collectors.toList());
         List<ElasticsearchException> allFailedNodeExceptions = new ArrayList<>(failedNodeExceptions);
         allFailedNodeExceptions.addAll(tasks.stream().flatMap(r -> r.getNodeFailures().stream()).collect(Collectors.toList()));
-        return new Response(responses, taskOperationFailures, allFailedNodeExceptions);
+        return new Response(responses, responses.size(), taskOperationFailures, allFailedNodeExceptions);
     }
 
     @Override
@@ -80,36 +85,57 @@ protected void taskOperation(Request request, DataFrameTransformTask task, Actio
         String nodeId = state.nodes().getLocalNode().getId();
         if (task.isCancelled() == false) {
             transformsCheckpointService.getCheckpointStats(task.getTransformId(), task.getCheckpoint(), task.getInProgressCheckpoint(),
-                    ActionListener.wrap(checkpointStats -> {
-                        listener.onResponse(new Response(Collections.singletonList(
-                        new DataFrameTransformStateAndStats(task.getTransformId(), task.getState(), task.getStats(), checkpointStats))));
-            }, e -> {
-                    listener.onResponse(new Response(
-                        Collections.singletonList(new DataFrameTransformStateAndStats(task.getTransformId(), task.getState(),
-                                task.getStats(), DataFrameTransformCheckpointingInfo.EMPTY)),
+                ActionListener.wrap(checkpointStats -> listener.onResponse(new Response(
+                        Collections.singletonList(new DataFrameTransformStateAndStats(task.getTransformId(),
+                            task.getState(),
+                            task.getStats(),
+                            checkpointStats)),
+                        1L)),
+                    e -> listener.onResponse(new Response(
+                        Collections.singletonList(new DataFrameTransformStateAndStats(task.getTransformId(),
+                            task.getState(),
+                            task.getStats(),
+                            DataFrameTransformCheckpointingInfo.EMPTY)),
+                        1L,
                         Collections.emptyList(),
-                        Collections.singletonList(new FailedNodeException(nodeId, "Failed to retrieve checkpointing info", e))));
-            }));
+                        Collections.singletonList(new FailedNodeException(nodeId, "Failed to retrieve checkpointing info", e))))
+                ));
         } else {
-            listener.onResponse(new Response(Collections.emptyList()));
+            listener.onResponse(new Response(Collections.emptyList(), 0L));
         }
     }
 
     @Override
     protected void doExecute(Task task, Request request, ActionListener<Response> finalListener) {
-        dataFrameTransformsConfigManager.expandTransformIds(request.getId(), request.getPageParams(), ActionListener.wrap(
-            ids -> {
-                request.setExpandedIds(ids);
-                request.setNodes(DataFrameNodes.dataFrameTaskNodes(ids, clusterService.state()));
+        dataFrameTransformsConfigManager.expandTransformIds(request.getId(),
+            request.getPageParams(),
+            request.isAllowNoMatch(),
+            ActionListener.wrap(hitsAndIds -> {
+                request.setExpandedIds(hitsAndIds.v2());
+                final ClusterState state = clusterService.state();
+                request.setNodes(DataFrameNodes.dataFrameTaskNodes(hitsAndIds.v2(), state));
                 super.doExecute(task, request, ActionListener.wrap(
-                    response -> collectStatsForTransformsWithoutTasks(request, response, finalListener),
+                    response -> {
+                        PersistentTasksCustomMetaData tasksInProgress = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
+                        if (tasksInProgress != null) {
+                            // Mutates underlying state object with the assigned node attributes
+                            response.getTransformsStateAndStats().forEach(dtsas -> setNodeAttributes(dtsas, tasksInProgress, state));
+                        }
+                        collectStatsForTransformsWithoutTasks(request, response, ActionListener.wrap(
+                            finalResponse -> finalListener.onResponse(new Response(finalResponse.getTransformsStateAndStats(),
+                                hitsAndIds.v1(),
+                                finalResponse.getTaskFailures(),
+                                finalResponse.getNodeFailures())),
+                            finalListener::onFailure
+                        ));
+                    },
                     finalListener::onFailure
                 ));
             },
             e -> {
                 // If the index to search, or the individual config is not there, just return empty
                 if (e instanceof ResourceNotFoundException) {
-                    finalListener.onResponse(new Response(Collections.emptyList()));
+                    finalListener.onResponse(new Response(Collections.emptyList(), 0L));
                 } else {
                     finalListener.onFailure(e);
                 }
@@ -117,6 +143,16 @@ protected void doExecute(Task task, Request request, ActionListener<Response> fi
         ));
     }
 
+    private static void setNodeAttributes(DataFrameTransformStateAndStats dataFrameTransformStateAndStats,
+                                                                     PersistentTasksCustomMetaData persistentTasksCustomMetaData,
+                                                                     ClusterState state) {
+        var pTask = persistentTasksCustomMetaData.getTask(dataFrameTransformStateAndStats.getTransformId());
+        if (pTask != null) {
+            dataFrameTransformStateAndStats.getTransformState()
+                .setNode(NodeAttributes.fromDiscoveryNode(state.nodes().get(pTask.getExecutorNode())));
+        }
+    }
+
     private void collectStatsForTransformsWithoutTasks(Request request,
                                                        Response response,
                                                        ActionListener<Response> listener) {
@@ -136,7 +172,21 @@ private void collectStatsForTransformsWithoutTasks(Request request,
         ActionListener<List<DataFrameTransformStateAndStats>> searchStatsListener = ActionListener.wrap(
             stats -> {
                 List<DataFrameTransformStateAndStats> allStateAndStats = response.getTransformsStateAndStats();
-                allStateAndStats.addAll(stats);
+                // If the persistent task does NOT exist, it is STOPPED
+                // There is a potential race condition where the saved document does not actually have a STOPPED state
+                //    as the task is cancelled before we persist state.
+                stats.forEach(stat ->
+                    allStateAndStats.add(new DataFrameTransformStateAndStats(
+                        stat.getId(),
+                        new DataFrameTransformState(DataFrameTransformTaskState.STOPPED,
+                            IndexerState.STOPPED,
+                            stat.getTransformState().getPosition(),
+                            stat.getTransformState().getCheckpoint(),
+                            stat.getTransformState().getReason(),
+                            stat.getTransformState().getProgress()),
+                        stat.getTransformStats(),
+                        stat.getCheckpointingInfo()))
+                );
                 transformsWithoutTasks.removeAll(
                         stats.stream().map(DataFrameTransformStateAndStats::getId).collect(Collectors.toSet()));
 
@@ -148,7 +198,10 @@ private void collectStatsForTransformsWithoutTasks(Request request,
                 // it can easily become arbitrarily ordered based on which transforms don't have a task or stats docs
                 allStateAndStats.sort(Comparator.comparing(DataFrameTransformStateAndStats::getId));
 
-                listener.onResponse(new Response(allStateAndStats, response.getTaskFailures(), response.getNodeFailures()));
+                listener.onResponse(new Response(allStateAndStats,
+                    allStateAndStats.size(),
+                    response.getTaskFailures(),
+                    response.getNodeFailures()));
             },
             e -> {
                 if (e instanceof IndexNotFoundException) {
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java
index f4b93cc6ac412..efc9534cb2e33 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPreviewDataFrameTransformAction.java
@@ -6,8 +6,13 @@
 
 package org.elasticsearch.xpack.dataframe.action;
 
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ingest.SimulatePipelineAction;
+import org.elasticsearch.action.ingest.SimulatePipelineRequest;
+import org.elasticsearch.action.ingest.SimulatePipelineResponse;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
@@ -16,7 +21,14 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestStatus;
@@ -26,22 +38,28 @@
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.DataFrameMessages;
 import org.elasticsearch.xpack.core.dataframe.action.PreviewDataFrameTransformAction;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig;
 import org.elasticsearch.xpack.core.dataframe.transforms.SourceConfig;
+import org.elasticsearch.xpack.dataframe.transforms.pivot.AggregationResultUtils;
 import org.elasticsearch.xpack.dataframe.transforms.pivot.Pivot;
 
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.dataframe.transforms.DataFrameIndexer.COMPOSITE_AGGREGATION_NAME;
 
 public class TransportPreviewDataFrameTransformAction extends
     HandledTransportAction<PreviewDataFrameTransformAction.Request, PreviewDataFrameTransformAction.Response> {
 
+    private static final Logger logger = LogManager.getLogger(TransportPreviewDataFrameTransformAction.class);
     private static final int NUMBER_OF_PREVIEW_BUCKETS = 100;
     private final XPackLicenseState licenseState;
     private final Client client;
@@ -86,15 +104,36 @@ protected void doExecute(Task task,
 
         Pivot pivot = new Pivot(config.getPivotConfig());
 
-        getPreview(pivot, config.getSource(), ActionListener.wrap(
-            previewResponse -> listener.onResponse(new PreviewDataFrameTransformAction.Response(previewResponse)),
-            listener::onFailure
-        ));
+        getPreview(pivot, config.getSource(), config.getDestination().getPipeline(), config.getDestination().getIndex(), listener);
     }
 
-    private void getPreview(Pivot pivot, SourceConfig source, ActionListener<List<Map<String, Object>>> listener) {
+    @SuppressWarnings("unchecked")
+    private void getPreview(Pivot pivot,
+                            SourceConfig source,
+                            String pipeline,
+                            String dest,
+                            ActionListener<PreviewDataFrameTransformAction.Response> listener) {
+        final PreviewDataFrameTransformAction.Response previewResponse = new PreviewDataFrameTransformAction.Response();
+        ActionListener<SimulatePipelineResponse> pipelineResponseActionListener = ActionListener.wrap(
+            simulatePipelineResponse -> {
+                List<Map<String, Object>> response = new ArrayList<>(simulatePipelineResponse.getResults().size());
+                for(var simulateDocumentResult : simulatePipelineResponse.getResults()) {
+                    try(XContentBuilder xContentBuilder = XContentFactory.jsonBuilder()) {
+                        XContentBuilder content = simulateDocumentResult.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
+                        Map<String, Object> tempMap = XContentHelper.convertToMap(BytesReference.bytes(content),
+                            true,
+                            XContentType.JSON).v2();
+                        response.add((Map<String, Object>)XContentMapValues.extractValue("doc._source", tempMap));
+                    }
+                }
+                previewResponse.setDocs(response);
+                listener.onResponse(previewResponse);
+            },
+            listener::onFailure
+        );
         pivot.deduceMappings(client, source, ActionListener.wrap(
             deducedMappings -> {
+                previewResponse.setMappingsFromStringMap(deducedMappings);
                 ClientHelper.executeWithHeadersAsync(threadPool.getThreadContext().getHeaders(),
                     ClientHelper.DATA_FRAME_ORIGIN,
                     client,
@@ -102,14 +141,45 @@ private void getPreview(Pivot pivot, SourceConfig source, ActionListener<List<Ma
                     pivot.buildSearchRequest(source, null, NUMBER_OF_PREVIEW_BUCKETS),
                     ActionListener.wrap(
                         r -> {
-                            final CompositeAggregation agg = r.getAggregations().get(COMPOSITE_AGGREGATION_NAME);
-                            DataFrameIndexerTransformStats stats = DataFrameIndexerTransformStats.withDefaultTransformId();
-                            // remove all internal fields
-                            List<Map<String, Object>> results = pivot.extractResults(agg, deducedMappings, stats)
-                                    .peek(record -> {
-                                        record.keySet().removeIf(k -> k.startsWith("_"));
-                                    }).collect(Collectors.toList());
-                            listener.onResponse(results);
+                            try {
+                                final CompositeAggregation agg = r.getAggregations().get(COMPOSITE_AGGREGATION_NAME);
+                                DataFrameIndexerTransformStats stats = DataFrameIndexerTransformStats.withDefaultTransformId();
+                                // remove all internal fields
+
+                                if (pipeline == null) {
+                                    List<Map<String, Object>> results = pivot.extractResults(agg, deducedMappings, stats)
+                                        .peek(doc -> doc.keySet().removeIf(k -> k.startsWith("_")))
+                                        .collect(Collectors.toList());
+                                    previewResponse.setDocs(results);
+                                    listener.onResponse(previewResponse);
+                                } else {
+                                    List<Map<String, Object>> results = pivot.extractResults(agg, deducedMappings, stats)
+                                        .map(doc -> {
+                                            Map<String, Object> src = new HashMap<>();
+                                            String id = (String) doc.get(DataFrameField.DOCUMENT_ID_FIELD);
+                                            doc.keySet().removeIf(k -> k.startsWith("_"));
+                                            src.put("_source", doc);
+                                            src.put("_id", id);
+                                            src.put("_index", dest);
+                                            return src;
+                                        }).collect(Collectors.toList());
+                                    try (XContentBuilder builder = jsonBuilder()) {
+                                        builder.startObject();
+                                        builder.field("docs", results);
+                                        builder.endObject();
+                                        var pipelineRequest = new SimulatePipelineRequest(BytesReference.bytes(builder), XContentType.JSON);
+                                        pipelineRequest.setId(pipeline);
+                                        ClientHelper.executeAsyncWithOrigin(client,
+                                            ClientHelper.DATA_FRAME_ORIGIN,
+                                            SimulatePipelineAction.INSTANCE,
+                                            pipelineRequest,
+                                            pipelineResponseActionListener);
+                                    }
+                                }
+                            } catch (AggregationResultUtils.AggregationExtractionException extractionException) {
+                                listener.onFailure(
+                                        new ElasticsearchStatusException(extractionException.getMessage(), RestStatus.BAD_REQUEST));
+                            }
                         },
                         listener::onFailure
                     ));
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java
index 997739b2407a7..fa35719be8ae1 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportPutDataFrameTransformAction.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -21,14 +22,14 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
@@ -46,10 +47,12 @@
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
 import org.elasticsearch.xpack.core.security.support.Exceptions;
+import org.elasticsearch.xpack.dataframe.notifications.DataFrameAuditor;
 import org.elasticsearch.xpack.dataframe.persistence.DataFrameTransformsConfigManager;
 import org.elasticsearch.xpack.dataframe.transforms.pivot.Pivot;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
@@ -65,12 +68,14 @@ public class TransportPutDataFrameTransformAction
     private final Client client;
     private final DataFrameTransformsConfigManager dataFrameTransformsConfigManager;
     private final SecurityContext securityContext;
+    private final DataFrameAuditor auditor;
 
     @Inject
     public TransportPutDataFrameTransformAction(Settings settings, TransportService transportService, ThreadPool threadPool,
                                                 ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
                                                 ClusterService clusterService, XPackLicenseState licenseState,
-                                                DataFrameTransformsConfigManager dataFrameTransformsConfigManager, Client client) {
+                                                DataFrameTransformsConfigManager dataFrameTransformsConfigManager, Client client,
+                                                DataFrameAuditor auditor) {
         super(PutDataFrameTransformAction.NAME, transportService, clusterService, threadPool, actionFilters,
                 PutDataFrameTransformAction.Request::new, indexNameExpressionResolver);
         this.licenseState = licenseState;
@@ -78,6 +83,7 @@ public TransportPutDataFrameTransformAction(Settings settings, TransportService
         this.dataFrameTransformsConfigManager = dataFrameTransformsConfigManager;
         this.securityContext = XPackSettings.SECURITY_ENABLED.get(settings) ?
             new SecurityContext(settings, threadPool.getThreadContext()) : null;
+        this.auditor = auditor;
     }
 
     @Override
@@ -85,13 +91,18 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(Request request, ClusterState clusterState, ActionListener<AcknowledgedResponse> listener)
+    protected void masterOperation(Task task, Request request, ClusterState clusterState, ActionListener<AcknowledgedResponse> listener)
             throws Exception {
 
         if (!licenseState.isDataFrameAllowed()) {
@@ -106,8 +117,10 @@ protected void masterOperation(Request request, ClusterState clusterState, Actio
                     .filter(e -> ClientHelper.SECURITY_HEADER_FILTERS.contains(e.getKey()))
                     .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
 
-        DataFrameTransformConfig config = request.getConfig();
-        config.setHeaders(filteredHeaders);
+        DataFrameTransformConfig config = request.getConfig()
+            .setHeaders(filteredHeaders)
+            .setCreateTime(Instant.now())
+            .setVersion(Version.CURRENT);
 
         String transformId = config.getId();
         // quick check whether a transform has already been created under that name
@@ -149,7 +162,7 @@ protected void masterOperation(Request request, ClusterState clusterState, Actio
         final String[] concreteDest =
             indexNameExpressionResolver.concreteIndexNames(clusterState, IndicesOptions.lenientExpandOpen(), destIndex);
 
-        if (concreteDest.length > 1 || Regex.isSimpleMatchPattern(destIndex)) {
+        if (concreteDest.length > 1) {
             listener.onFailure(new ElasticsearchStatusException(
                 DataFrameMessages.getMessage(DataFrameMessages.REST_PUT_DATA_FRAME_DEST_SINGLE_INDEX, destIndex),
                 RestStatus.BAD_REQUEST
@@ -169,10 +182,9 @@ protected void masterOperation(Request request, ClusterState clusterState, Actio
         // Early check to verify that the user can create the destination index and can read from the source
         if (licenseState.isAuthAllowed()) {
             final String username = securityContext.getUser().principal();
-            RoleDescriptor.IndicesPrivileges sourceIndexPrivileges = RoleDescriptor.IndicesPrivileges.builder()
-                .indices(config.getSource().getIndex())
-                .privileges("read")
-                .build();
+            List<String> srcPrivileges = new ArrayList<>(2);
+            srcPrivileges.add("read");
+
             List<String> destPrivileges = new ArrayList<>(3);
             destPrivileges.add("read");
             destPrivileges.add("index");
@@ -180,12 +192,19 @@ protected void masterOperation(Request request, ClusterState clusterState, Actio
             // We should check that the creating user has the privileges to create the index.
             if (concreteDest.length == 0) {
                 destPrivileges.add("create_index");
+                // We need to read the source indices mapping to deduce the destination mapping
+                srcPrivileges.add("view_index_metadata");
             }
             RoleDescriptor.IndicesPrivileges destIndexPrivileges = RoleDescriptor.IndicesPrivileges.builder()
                 .indices(destIndex)
                 .privileges(destPrivileges)
                 .build();
 
+            RoleDescriptor.IndicesPrivileges sourceIndexPrivileges = RoleDescriptor.IndicesPrivileges.builder()
+                .indices(config.getSource().getIndex())
+                .privileges(srcPrivileges)
+                .build();
+
             HasPrivilegesRequest privRequest = new HasPrivilegesRequest();
             privRequest.applicationPrivileges(new RoleDescriptor.ApplicationResourcePrivileges[0]);
             privRequest.username(username);
@@ -213,17 +232,16 @@ private void handlePrivsResponse(String username,
         if (privilegesResponse.isCompleteMatch()) {
             putDataFrame(config, listener);
         } else {
-            XContentBuilder builder = JsonXContent.contentBuilder();
-            builder.startObject();
-            for (ResourcePrivileges index : privilegesResponse.getIndexPrivileges()) {
-                builder.field(index.getResource());
-                builder.map(index.getPrivileges());
-            }
-            builder.endObject();
+            List<String> indices = privilegesResponse.getIndexPrivileges()
+                .stream()
+                .map(ResourcePrivileges::getResource)
+                .collect(Collectors.toList());
 
-            listener.onFailure(Exceptions.authorizationError("Cannot create data frame transform [{}]" +
-                    " because user {} lacks permissions on the indices: {}",
-                config.getId(), username, Strings.toString(builder)));
+            listener.onFailure(Exceptions.authorizationError(
+                "Cannot create data frame transform [{}] because user {} lacks all the required permissions for indices: {}",
+                config.getId(),
+                username,
+                indices));
         }
     }
 
@@ -234,7 +252,10 @@ private void putDataFrame(DataFrameTransformConfig config, ActionListener<Acknow
 
         // <5> Return the listener, or clean up destination index on failure.
         ActionListener<Boolean> putTransformConfigurationListener = ActionListener.wrap(
-            putTransformConfigurationResult -> listener.onResponse(new AcknowledgedResponse(true)),
+            putTransformConfigurationResult -> {
+                auditor.info(config.getId(), "Created data frame transform.");
+                listener.onResponse(new AcknowledgedResponse(true));
+            },
             listener::onFailure
         );
 
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportStartDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportStartDataFrameTransformAction.java
index 39c0c74bbd504..4b2a11a7d0258 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportStartDataFrameTransformAction.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportStartDataFrameTransformAction.java
@@ -6,10 +6,11 @@
 
 package org.elasticsearch.xpack.dataframe.action;
 
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.support.ActionFilters;
@@ -29,6 +30,7 @@
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
@@ -46,8 +48,10 @@
 import org.elasticsearch.xpack.dataframe.transforms.pivot.Pivot;
 
 import java.io.IOException;
+import java.time.Clock;
 import java.util.Collection;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 
@@ -93,19 +97,21 @@ protected StartDataFrameTransformAction.Response read(StreamInput in) throws IOE
     }
 
     @Override
-    protected void masterOperation(StartDataFrameTransformAction.Request request,
+    protected void masterOperation(Task ignoredTask, StartDataFrameTransformAction.Request request,
                                    ClusterState state,
                                    ActionListener<StartDataFrameTransformAction.Response> listener) throws Exception {
         if (!licenseState.isDataFrameAllowed()) {
             listener.onFailure(LicenseUtils.newComplianceException(XPackField.DATA_FRAME));
             return;
         }
-        final DataFrameTransform transformTask = createDataFrameTransform(request.getId(), threadPool);
+        final AtomicReference<DataFrameTransform> transformTaskHolder = new AtomicReference<>();
 
-        // <3> Wait for the allocated task's state to STARTED
+        // <4> Wait for the allocated task's state to STARTED
         ActionListener<PersistentTasksCustomMetaData.PersistentTask<DataFrameTransform>> newPersistentTaskActionListener =
             ActionListener.wrap(
                 task -> {
+                    DataFrameTransform transformTask = transformTaskHolder.get();
+                    assert transformTask != null;
                     waitForDataFrameTaskStarted(task.getId(),
                         transformTask,
                         request.timeout(),
@@ -119,6 +125,8 @@ protected void masterOperation(StartDataFrameTransformAction.Request request,
         // <3> Create the task in cluster state so that it will start executing on the node
         ActionListener<Void> createOrGetIndexListener = ActionListener.wrap(
             unused -> {
+                DataFrameTransform transformTask = transformTaskHolder.get();
+                assert transformTask != null;
                 PersistentTasksCustomMetaData.PersistentTask<DataFrameTransform> existingTask =
                     getExistingTask(transformTask.getId(), state);
                 if (existingTask == null) {
@@ -177,6 +185,8 @@ protected void masterOperation(StartDataFrameTransformAction.Request request,
                     ));
                     return;
                 }
+
+                transformTaskHolder.set(createDataFrameTransform(config.getId(), config.getVersion()));
                 final String destinationIndex = config.getDestination().getIndex();
                 String[] dest = indexNameExpressionResolver.concreteIndexNames(state,
                     IndicesOptions.lenientExpandOpen(),
@@ -184,11 +194,10 @@ protected void masterOperation(StartDataFrameTransformAction.Request request,
 
                 if(dest.length == 0) {
                     auditor.info(request.getId(),
-                        "Could not find destination index [" +  destinationIndex + "]." +
-                            " Creating index with deduced mappings.");
+                        "Creating destination index [" +  destinationIndex + "] with deduced mappings.");
                     createDestinationIndex(config, createOrGetIndexListener);
                 } else {
-                    auditor.info(request.getId(), "Destination index [" + destinationIndex + "] already exists.");
+                    auditor.info(request.getId(), "Using existing destination index [" + destinationIndex + "].");
                     ClientHelper.executeAsyncWithOrigin(client.threadPool().getThreadContext(),
                         ClientHelper.DATA_FRAME_ORIGIN,
                         client.admin()
@@ -227,7 +236,9 @@ private void createDestinationIndex(final DataFrameTransformConfig config, final
         final Pivot pivot = new Pivot(config.getPivotConfig());
 
         ActionListener<Map<String, String>> deduceMappingsListener = ActionListener.wrap(
-            mappings -> DataframeIndex.createDestinationIndex(client,
+            mappings -> DataframeIndex.createDestinationIndex(
+                client,
+                Clock.systemUTC(),
                 config,
                 mappings,
                 ActionListener.wrap(r -> listener.onResponse(null), listener::onFailure)),
@@ -244,8 +255,8 @@ protected ClusterBlockException checkBlock(StartDataFrameTransformAction.Request
         return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
     }
 
-    private static DataFrameTransform createDataFrameTransform(String transformId, ThreadPool threadPool) {
-        return new DataFrameTransform(transformId);
+    private static DataFrameTransform createDataFrameTransform(String transformId, Version transformVersion) {
+        return new DataFrameTransform(transformId, transformVersion);
     }
 
     @SuppressWarnings("unchecked")
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportStopDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportStopDataFrameTransformAction.java
index 26f5259c69dc8..35a9d19658345 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportStopDataFrameTransformAction.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportStopDataFrameTransformAction.java
@@ -5,6 +5,8 @@
  */
 package org.elasticsearch.xpack.dataframe.action;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
@@ -12,6 +14,7 @@
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -26,6 +29,7 @@
 import org.elasticsearch.xpack.core.action.util.PageParams;
 import org.elasticsearch.xpack.core.dataframe.action.StopDataFrameTransformAction;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformTaskState;
+import org.elasticsearch.xpack.dataframe.persistence.DataFrameInternalIndex;
 import org.elasticsearch.xpack.dataframe.persistence.DataFrameTransformsConfigManager;
 import org.elasticsearch.xpack.dataframe.transforms.DataFrameTransformTask;
 
@@ -38,20 +42,25 @@ public class TransportStopDataFrameTransformAction extends
         TransportTasksAction<DataFrameTransformTask, StopDataFrameTransformAction.Request,
         StopDataFrameTransformAction.Response, StopDataFrameTransformAction.Response> {
 
+    private static final Logger logger = LogManager.getLogger(TransportStopDataFrameTransformAction.class);
+
     private final ThreadPool threadPool;
     private final DataFrameTransformsConfigManager dataFrameTransformsConfigManager;
     private final PersistentTasksService persistentTasksService;
+    private final Client client;
 
     @Inject
     public TransportStopDataFrameTransformAction(TransportService transportService, ActionFilters actionFilters,
                                                  ClusterService clusterService, ThreadPool threadPool,
                                                  PersistentTasksService persistentTasksService,
-                                                 DataFrameTransformsConfigManager dataFrameTransformsConfigManager) {
+                                                 DataFrameTransformsConfigManager dataFrameTransformsConfigManager,
+                                                 Client client) {
         super(StopDataFrameTransformAction.NAME, clusterService, transportService, actionFilters, StopDataFrameTransformAction.Request::new,
                 StopDataFrameTransformAction.Response::new, StopDataFrameTransformAction.Response::new, ThreadPool.Names.SAME);
         this.threadPool = threadPool;
         this.dataFrameTransformsConfigManager = dataFrameTransformsConfigManager;
         this.persistentTasksService = persistentTasksService;
+        this.client = client;
     }
 
     @Override
@@ -75,13 +84,15 @@ protected void doExecute(Task task, StopDataFrameTransformAction.Request request
                 finalListener = listener;
             }
 
-            dataFrameTransformsConfigManager.expandTransformIds(request.getId(), new PageParams(0, 10_000), ActionListener.wrap(
-                    expandedIds -> {
-                        request.setExpandedIds(new HashSet<>(expandedIds));
-                        request.setNodes(DataFrameNodes.dataFrameTaskNodes(expandedIds, clusterService.state()));
-                        super.doExecute(task, request, finalListener);
-                    },
-                    listener::onFailure
+            dataFrameTransformsConfigManager.expandTransformIds(request.getId(),
+                new PageParams(0, 10_000),
+                request.isAllowNoMatch(),
+                ActionListener.wrap(hitsAndIds -> {
+                    request.setExpandedIds(new HashSet<>(hitsAndIds.v2()));
+                    request.setNodes(DataFrameNodes.dataFrameTaskNodes(hitsAndIds.v2(), clusterService.state()));
+                    super.doExecute(task, request, finalListener);
+                },
+                listener::onFailure
             ));
         }
     }
@@ -124,20 +135,34 @@ protected StopDataFrameTransformAction.Response newResponse(StopDataFrameTransfo
         }
 
         // if tasks is empty allMatch is 'vacuously satisfied'
-        boolean allStopped = tasks.stream().allMatch(StopDataFrameTransformAction.Response::isStopped);
-        return new StopDataFrameTransformAction.Response(allStopped);
+        boolean allAcknowledged = tasks.stream().allMatch(StopDataFrameTransformAction.Response::isAcknowledged);
+        return new StopDataFrameTransformAction.Response(allAcknowledged);
     }
 
     private ActionListener<StopDataFrameTransformAction.Response>
     waitForStopListener(StopDataFrameTransformAction.Request request,
                         ActionListener<StopDataFrameTransformAction.Response> listener) {
 
+        ActionListener<StopDataFrameTransformAction.Response> onStopListener = ActionListener.wrap(
+            waitResponse ->
+                client.admin()
+                    .indices()
+                    .prepareRefresh(DataFrameInternalIndex.INDEX_NAME)
+                    .execute(ActionListener.wrap(
+                        r -> listener.onResponse(waitResponse),
+                        e -> {
+                            logger.info("Failed to refresh internal index after delete", e);
+                            listener.onResponse(waitResponse);
+                        })
+                    ),
+            listener::onFailure
+        );
         return ActionListener.wrap(
                 response -> {
                     // Wait until the persistent task is stopped
                     // Switch over to Generic threadpool so we don't block the network thread
                     threadPool.generic().execute(() ->
-                        waitForDataFrameStopped(request.getExpandedIds(), request.getTimeout(), listener));
+                        waitForDataFrameStopped(request.getExpandedIds(), request.getTimeout(), onStopListener));
                 },
                 listener::onFailure
         );
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointService.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointService.java
index 6fc2e334f9255..d028d3248c8f6 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointService.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformsCheckpointService.java
@@ -15,11 +15,15 @@
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpoint;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointStats;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointingInfo;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig;
+import org.elasticsearch.xpack.core.dataframe.transforms.SyncConfig;
+import org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig;
 import org.elasticsearch.xpack.dataframe.persistence.DataFrameTransformsConfigManager;
 
 import java.util.Arrays;
@@ -84,8 +88,8 @@ public void getCheckpoint(DataFrameTransformConfig transformConfig, long checkpo
             ActionListener<DataFrameTransformCheckpoint> listener) {
         long timestamp = System.currentTimeMillis();
 
-        // placeholder for time based synchronization
-        long timeUpperBound = 0;
+        // for time based synchronization
+        long timeUpperBound = getTimeStampForTimeBasedSynchronization(transformConfig.getSyncConfig(), timestamp);
 
         // 1st get index to see the indexes the user has access to
         GetIndexRequest getIndexRequest = new GetIndexRequest()
@@ -205,12 +209,31 @@ public void getCheckpointStats(
         );
     }
 
+    private long getTimeStampForTimeBasedSynchronization(SyncConfig syncConfig, long timestamp) {
+        if (syncConfig instanceof TimeSyncConfig) {
+            TimeSyncConfig timeSyncConfig = (TimeSyncConfig) syncConfig;
+            return timestamp - timeSyncConfig.getDelay().millis();
+        }
+
+        return 0L;
+    }
+
     static Map<String, long[]> extractIndexCheckPoints(ShardStats[] shards, Set<String> userIndices) {
         Map<String, TreeMap<Integer, Long>> checkpointsByIndex = new TreeMap<>();
 
         for (ShardStats shard : shards) {
             String indexName = shard.getShardRouting().getIndexName();
             if (userIndices.contains(indexName)) {
+                SeqNoStats seqNoStats = shard.getSeqNoStats();
+                // SeqNoStats could be `null`. This indicates that an `AlreadyClosed` exception was thrown somewhere down the stack
+                // Indicates that the index COULD be closed, or at least that the shard is not fully recovered yet.
+                if (seqNoStats == null) {
+                    logger.warn("failure gathering checkpoint information for index [{}] as seq_no_stats were null. Shard Stats [{}]",
+                        indexName,
+                        Strings.toString(shard));
+                    throw new CheckpointException(
+                        "Unable to gather checkpoint information for index [" + indexName + "]. seq_no_stats are missing.");
+                }
                 if (checkpointsByIndex.containsKey(indexName)) {
                     // we have already seen this index, just check/add shards
                     TreeMap<Integer, Long> checkpoints = checkpointsByIndex.get(indexName);
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManager.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManager.java
index ab893545a0d50..5d3a664054a4b 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManager.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManager.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -196,24 +197,29 @@ public void getTransformConfiguration(String transformId, ActionListener<DataFra
      * @param pageParams             The paging params
      * @param foundIdsListener       The listener on signal on success or failure
      */
-    public void expandTransformIds(String transformIdsExpression, PageParams pageParams, ActionListener<List<String>> foundIdsListener) {
+    public void expandTransformIds(String transformIdsExpression,
+                                   PageParams pageParams,
+                                   boolean allowNoMatch,
+                                   ActionListener<Tuple<Long, List<String>>> foundIdsListener) {
         String[] idTokens = ExpandedIdsMatcher.tokenizeExpression(transformIdsExpression);
         QueryBuilder queryBuilder = buildQueryFromTokenizedIds(idTokens, DataFrameTransformConfig.NAME);
 
         SearchRequest request = client.prepareSearch(DataFrameInternalIndex.INDEX_NAME)
             .addSort(DataFrameField.ID.getPreferredName(), SortOrder.ASC)
             .setFrom(pageParams.getFrom())
+            .setTrackTotalHits(true)
             .setSize(pageParams.getSize())
             .setQuery(queryBuilder)
             // We only care about the `id` field, small optimization
             .setFetchSource(DataFrameField.ID.getPreferredName(), "")
             .request();
 
-        final ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(idTokens, true);
+        final ExpandedIdsMatcher requiredMatches = new ExpandedIdsMatcher(idTokens, allowNoMatch);
 
         executeAsyncWithOrigin(client.threadPool().getThreadContext(), DATA_FRAME_ORIGIN, request,
             ActionListener.<SearchResponse>wrap(
                 searchResponse -> {
+                    long totalHits = searchResponse.getHits().getTotalHits().value;
                     List<String> ids = new ArrayList<>(searchResponse.getHits().getHits().length);
                     for (SearchHit hit : searchResponse.getHits().getHits()) {
                         BytesReference source = hit.getSourceRef();
@@ -235,7 +241,7 @@ public void expandTransformIds(String transformIdsExpression, PageParams pagePar
                                     requiredMatches.unmatchedIdsString())));
                         return;
                     }
-                    foundIdsListener.onResponse(ids);
+                    foundIdsListener.onResponse(new Tuple<>(totalHits, ids));
                 },
                 foundIdsListener::onFailure
             ), client::search);
@@ -249,8 +255,7 @@ public void expandTransformIds(String transformIdsExpression, PageParams pagePar
      */
     public void deleteTransform(String transformId, ActionListener<Boolean> listener) {
         DeleteByQueryRequest request = new DeleteByQueryRequest()
-                .setAbortOnVersionConflict(false) //since these documents are not updated, a conflict just means it was deleted previously
-                .setSlices(5);
+                .setAbortOnVersionConflict(false); //since these documents are not updated, a conflict just means it was deleted previously
 
         request.indices(DataFrameInternalIndex.INDEX_NAME);
         QueryBuilder query = QueryBuilders.termQuery(DataFrameField.ID.getPreferredName(), transformId);
@@ -336,6 +341,7 @@ public void getTransformStats(Collection<String> transformIds, ActionListener<Li
         SearchRequest searchRequest = client.prepareSearch(DataFrameInternalIndex.INDEX_NAME)
                 .addSort(DataFrameField.ID.getPreferredName(), SortOrder.ASC)
                 .setQuery(builder)
+                .setSize(Math.min(transformIds.size(), 10_000))
                 .request();
 
         executeAsyncWithOrigin(client.threadPool().getThreadContext(), DATA_FRAME_ORIGIN, searchRequest,
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java
index 1c4c9ee0158a5..005be73e8cee2 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java
@@ -13,15 +13,15 @@
 import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.DataFrameMessages;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig;
-import org.elasticsearch.xpack.core.dataframe.transforms.pivot.DateHistogramGroupSource;
-import org.elasticsearch.xpack.core.dataframe.transforms.pivot.SingleGroupSource;
 
 import java.io.IOException;
+import java.time.Clock;
 import java.util.Map;
 import java.util.Map.Entry;
 
@@ -33,24 +33,26 @@ public final class DataframeIndex {
 
     private static final String PROPERTIES = "properties";
     private static final String TYPE = "type";
-    private static final String FORMAT = "format";
     private static final String META = "_meta";
-    private static final String DEFAULT_TIME_FORMAT = "strict_date_optional_time||epoch_millis";
 
     private DataframeIndex() {
     }
 
-    public static void createDestinationIndex(Client client, DataFrameTransformConfig transformConfig, Map<String, String> mappings,
-            final ActionListener<Boolean> listener) {
+    public static void createDestinationIndex(Client client,
+                                              Clock clock,
+                                              DataFrameTransformConfig transformConfig,
+                                              Map<String, String> mappings,
+                                              ActionListener<Boolean> listener) {
         CreateIndexRequest request = new CreateIndexRequest(transformConfig.getDestination().getIndex());
 
         // TODO: revisit number of shards, number of replicas
         request.settings(Settings.builder() // <1>
-                .put("index.number_of_shards", 1).put("index.number_of_replicas", 0));
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1"));
 
-        request.mapping(SINGLE_MAPPING_NAME, createMappingXContent(mappings,
-            transformConfig.getPivotConfig().getGroupConfig().getGroups(),
-            transformConfig.getId()));
+        request.mapping(
+            SINGLE_MAPPING_NAME,
+            createMappingXContent(mappings, transformConfig.getId(), clock));
 
         client.execute(CreateIndexAction.INSTANCE, request, ActionListener.wrap(createIndexResponse -> {
             listener.onResponse(true);
@@ -63,30 +65,13 @@ public static void createDestinationIndex(Client client, DataFrameTransformConfi
     }
 
     private static XContentBuilder createMappingXContent(Map<String, String> mappings,
-                                                         Map<String, SingleGroupSource> groupSources,
-                                                         String id) {
+                                                         String id,
+                                                         Clock clock) {
         try {
             XContentBuilder builder = jsonBuilder().startObject();
             builder.startObject(SINGLE_MAPPING_NAME);
-            addMetaData(builder, id);
-            builder.startObject(PROPERTIES);
-            for (Entry<String, String> field : mappings.entrySet()) {
-                String fieldName = field.getKey();
-                String fieldType = field.getValue();
-
-                builder.startObject(fieldName);
-                builder.field(TYPE, fieldType);
-
-                SingleGroupSource groupSource = groupSources.get(fieldName);
-                if (groupSource instanceof DateHistogramGroupSource) {
-                    String format = ((DateHistogramGroupSource) groupSource).getFormat();
-                    if (format != null) {
-                        builder.field(FORMAT, DEFAULT_TIME_FORMAT + "||" + format);
-                    }
-                }
-                builder.endObject();
-            }
-            builder.endObject(); // properties
+            addProperties(builder, mappings);
+            addMetaData(builder, id, clock);
             builder.endObject(); // _doc type
             return builder.endObject();
         } catch (IOException e) {
@@ -94,17 +79,32 @@ private static XContentBuilder createMappingXContent(Map<String, String> mapping
         }
     }
 
-    private static XContentBuilder addMetaData(XContentBuilder builder, String id) throws IOException {
-        builder.startObject(META);
-        builder.field(DataFrameField.CREATED_BY, DataFrameField.DATA_FRAME_SIGNATURE);
-        builder.startObject(DataFrameField.META_FIELDNAME);
-        builder.field(DataFrameField.CREATION_DATE_MILLIS, System.currentTimeMillis());
-        builder.startObject(DataFrameField.VERSION);
-        builder.field(DataFrameField.CREATED, Version.CURRENT);
-        builder.endObject();
-        builder.field(DataFrameField.TRANSFORM, id);
-        builder.endObject(); // META_FIELDNAME
-        builder.endObject(); // META
+    private static XContentBuilder addProperties(XContentBuilder builder,
+                                                 Map<String, String> mappings) throws IOException {
+        builder.startObject(PROPERTIES);
+        for (Entry<String, String> field : mappings.entrySet()) {
+            String fieldName = field.getKey();
+            String fieldType = field.getValue();
+
+            builder.startObject(fieldName);
+            builder.field(TYPE, fieldType);
+
+            builder.endObject();
+        }
+        builder.endObject(); // PROPERTIES
         return builder;
     }
+
+    private static XContentBuilder addMetaData(XContentBuilder builder, String id, Clock clock) throws IOException {
+        return builder.startObject(META)
+            .field(DataFrameField.CREATED_BY, DataFrameField.DATA_FRAME_SIGNATURE)
+            .startObject(DataFrameField.META_FIELDNAME)
+                .field(DataFrameField.CREATION_DATE_MILLIS, clock.millis())
+                .startObject(DataFrameField.VERSION)
+                    .field(DataFrameField.CREATED, Version.CURRENT)
+                .endObject()
+                .field(DataFrameField.TRANSFORM, id)
+            .endObject() // META_FIELDNAME
+        .endObject(); // META
+    }
 }
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestGetDataFrameTransformsAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestGetDataFrameTransformsAction.java
index d079b2fd72cc9..95f6ec79fb18d 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestGetDataFrameTransformsAction.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestGetDataFrameTransformsAction.java
@@ -16,6 +16,8 @@
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.action.GetDataFrameTransformsAction;
 
+import static org.elasticsearch.xpack.core.dataframe.DataFrameField.ALLOW_NO_MATCH;
+
 public class RestGetDataFrameTransformsAction extends BaseRestHandler {
 
     public RestGetDataFrameTransformsAction(Settings settings, RestController controller) {
@@ -30,6 +32,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
 
         String id = restRequest.param(DataFrameField.ID.getPreferredName());
         request.setResourceId(id);
+        request.setAllowNoResources(restRequest.paramAsBoolean(ALLOW_NO_MATCH.getPreferredName(), true));
         if (restRequest.hasParam(PageParams.FROM.getPreferredName()) || restRequest.hasParam(PageParams.SIZE.getPreferredName())) {
             request.setPageParams(
                 new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), PageParams.DEFAULT_FROM),
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestGetDataFrameTransformsStatsAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestGetDataFrameTransformsStatsAction.java
index f2d14f8106958..f6d8dd40e5c6d 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestGetDataFrameTransformsStatsAction.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestGetDataFrameTransformsStatsAction.java
@@ -15,6 +15,8 @@
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.action.GetDataFrameTransformsStatsAction;
 
+import static org.elasticsearch.xpack.core.dataframe.DataFrameField.ALLOW_NO_MATCH;
+
 public class RestGetDataFrameTransformsStatsAction extends BaseRestHandler {
 
     public RestGetDataFrameTransformsStatsAction(Settings settings, RestController controller) {
@@ -27,6 +29,7 @@ public RestGetDataFrameTransformsStatsAction(Settings settings, RestController c
     protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
         String id = restRequest.param(DataFrameField.ID.getPreferredName());
         GetDataFrameTransformsStatsAction.Request request = new GetDataFrameTransformsStatsAction.Request(id);
+        request.setAllowNoMatch(restRequest.paramAsBoolean(ALLOW_NO_MATCH.getPreferredName(), true));
         if (restRequest.hasParam(PageParams.FROM.getPreferredName()) || restRequest.hasParam(PageParams.SIZE.getPreferredName())) {
             request.setPageParams(
                 new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), PageParams.DEFAULT_FROM),
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestStopDataFrameTransformAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestStopDataFrameTransformAction.java
index d34478b9ba941..e06cd7df45377 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestStopDataFrameTransformAction.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/rest/action/RestStopDataFrameTransformAction.java
@@ -30,8 +30,14 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
                 StopDataFrameTransformAction.DEFAULT_TIMEOUT);
         boolean waitForCompletion = restRequest.paramAsBoolean(DataFrameField.WAIT_FOR_COMPLETION.getPreferredName(), false);
         boolean force = restRequest.paramAsBoolean(DataFrameField.FORCE.getPreferredName(), false);
+        boolean allowNoMatch = restRequest.paramAsBoolean(DataFrameField.ALLOW_NO_MATCH.getPreferredName(), false);
 
-        StopDataFrameTransformAction.Request request = new StopDataFrameTransformAction.Request(id, waitForCompletion, force, timeout);
+
+        StopDataFrameTransformAction.Request request = new StopDataFrameTransformAction.Request(id,
+            waitForCompletion,
+            force,
+            timeout,
+            allowNoMatch);
 
         return channel -> client.execute(StopDataFrameTransformAction.INSTANCE, request,
                 new BaseTasksResponseToXContentListener<>(channel));
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameIndexer.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameIndexer.java
index f2fc71da7f059..1516b415d2a7d 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameIndexer.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameIndexer.java
@@ -16,10 +16,15 @@
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation;
+import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.DataFrameMessages;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats;
+import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpoint;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformProgress;
 import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
@@ -34,6 +39,7 @@
 import java.util.Collections;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
@@ -55,6 +61,8 @@ public abstract class DataFrameIndexer extends AsyncTwoPhaseIndexer<Map<String,
 
     private Pivot pivot;
     private int pageSize = 0;
+    protected volatile DataFrameTransformCheckpoint inProgressOrLastCheckpoint;
+    private volatile Map<String, Set<String>> changedBuckets;
 
     public DataFrameIndexer(Executor executor,
                             DataFrameAuditor auditor,
@@ -63,12 +71,14 @@ public DataFrameIndexer(Executor executor,
                             AtomicReference<IndexerState> initialState,
                             Map<String, Object> initialPosition,
                             DataFrameIndexerTransformStats jobStats,
-                            DataFrameTransformProgress transformProgress) {
+                            DataFrameTransformProgress transformProgress,
+                            DataFrameTransformCheckpoint inProgressOrLastCheckpoint) {
         super(executor, initialState, initialPosition, jobStats);
         this.auditor = Objects.requireNonNull(auditor);
         this.transformConfig = ExceptionsHelper.requireNonNull(transformConfig, "transformConfig");
         this.fieldMappings = ExceptionsHelper.requireNonNull(fieldMappings, "fieldMappings");
         this.progress = transformProgress;
+        this.inProgressOrLastCheckpoint = inProgressOrLastCheckpoint;
     }
 
     protected abstract void failIndexer(String message);
@@ -81,6 +91,10 @@ public DataFrameTransformConfig getConfig() {
         return transformConfig;
     }
 
+    public boolean isContinuous() {
+        return getConfig().getSyncConfig() != null;
+    }
+
     public Map<String, String> getFieldMappings() {
         return fieldMappings;
     }
@@ -92,7 +106,7 @@ public DataFrameTransformProgress getProgress() {
     /**
      * Request a checkpoint
      */
-    protected abstract void createCheckpoint(ActionListener<Void> listener);
+    protected abstract void createCheckpoint(ActionListener<DataFrameTransformCheckpoint> listener);
 
     @Override
     protected void onStart(long now, ActionListener<Void> listener) {
@@ -103,13 +117,7 @@ protected void onStart(long now, ActionListener<Void> listener) {
             if (pageSize == 0) {
                 pageSize = pivot.getInitialPageSize();
             }
-
-            // if run for the 1st time, create checkpoint
-            if (initialRun()) {
-                createCheckpoint(listener);
-            } else {
-                listener.onResponse(null);
-            }
+            listener.onResponse(null);
         } catch (Exception e) {
             listener.onFailure(e);
         }
@@ -121,8 +129,10 @@ protected boolean initialRun() {
 
     @Override
     protected void onFinish(ActionListener<Void> listener) {
-        // reset the page size, so we do not memorize a low page size forever, the pagesize will be re-calculated on start
-        pageSize = 0;
+        // reset the page size, so we do not memorize a low page size forever
+        pageSize = pivot.getInitialPageSize();
+        // reset the changed bucket to free memory
+        changedBuckets = null;
     }
 
     @Override
@@ -179,13 +189,49 @@ private Stream<IndexRequest> processBucketsToIndexRequests(CompositeAggregation
             }
 
             IndexRequest request = new IndexRequest(indexName).source(builder).id(id);
+            if (transformConfig.getDestination().getPipeline() != null) {
+                request.setPipeline(transformConfig.getDestination().getPipeline());
+            }
             return request;
         });
     }
 
+    protected QueryBuilder buildFilterQuery() {
+        QueryBuilder pivotQueryBuilder = getConfig().getSource().getQueryConfig().getQuery();
+
+        DataFrameTransformConfig config = getConfig();
+        if (config.getSyncConfig() != null) {
+            if (inProgressOrLastCheckpoint == null) {
+                throw new RuntimeException("in progress checkpoint not found");
+            }
+
+            BoolQueryBuilder filteredQuery = new BoolQueryBuilder()
+                .filter(pivotQueryBuilder)
+                .filter(config.getSyncConfig().getRangeQuery(inProgressOrLastCheckpoint));
+
+            if (changedBuckets != null && changedBuckets.isEmpty() == false) {
+                QueryBuilder pivotFilter = pivot.filterBuckets(changedBuckets);
+                if (pivotFilter != null) {
+                    filteredQuery.filter(pivotFilter);
+                }
+            }
+
+            logger.trace("running filtered query: {}", filteredQuery);
+            return filteredQuery;
+        } else {
+            return pivotQueryBuilder;
+        }
+    }
+
     @Override
     protected SearchRequest buildSearchRequest() {
-        return pivot.buildSearchRequest(getConfig().getSource(), getPosition(), pageSize);
+        SearchRequest searchRequest = new SearchRequest(getConfig().getSource().getIndex());
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder()
+            .aggregation(pivot.buildAggregation(getPosition(), pageSize))
+            .size(0)
+            .query(buildFilterQuery());
+        searchRequest.source(sourceBuilder);
+        return searchRequest;
     }
 
     /**
@@ -226,6 +272,84 @@ protected boolean handleCircuitBreakingException(Exception e) {
         return true;
     }
 
+    protected void getChangedBuckets(DataFrameTransformCheckpoint oldCheckpoint,
+                                     DataFrameTransformCheckpoint newCheckpoint,
+                                     ActionListener<Map<String, Set<String>>> listener) {
+
+        ActionListener<Map<String, Set<String>>> wrappedListener = ActionListener.wrap(
+            r -> {
+                this.inProgressOrLastCheckpoint = newCheckpoint;
+                this.changedBuckets = r;
+                listener.onResponse(r);
+            },
+            listener::onFailure
+        );
+        // initialize the map of changed buckets, the map might be empty if source do not require/implement
+        // changed bucket detection
+        Map<String, Set<String>> keys = pivot.initialIncrementalBucketUpdateMap();
+        if (keys.isEmpty()) {
+            logger.trace("This data frame does not implement changed bucket detection, returning");
+            wrappedListener.onResponse(null);
+            return;
+        }
+
+        SearchRequest searchRequest = new SearchRequest(getConfig().getSource().getIndex());
+        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
+
+        // we do not need the sub-aggs
+        CompositeAggregationBuilder changesAgg = pivot.buildIncrementalBucketUpdateAggregation(pageSize);
+        sourceBuilder.aggregation(changesAgg);
+        sourceBuilder.size(0);
+
+        QueryBuilder pivotQueryBuilder = getConfig().getSource().getQueryConfig().getQuery();
+
+        DataFrameTransformConfig config = getConfig();
+        if (config.getSyncConfig() != null) {
+            BoolQueryBuilder filteredQuery = new BoolQueryBuilder().
+                    filter(pivotQueryBuilder).
+                    filter(config.getSyncConfig().getRangeQuery(oldCheckpoint, newCheckpoint));
+
+            logger.trace("Gathering changes using query {}", filteredQuery);
+            sourceBuilder.query(filteredQuery);
+        } else {
+            logger.trace("No sync configured");
+            wrappedListener.onResponse(null);
+            return;
+        }
+
+        searchRequest.source(sourceBuilder);
+        searchRequest.allowPartialSearchResults(false);
+
+        collectChangedBuckets(searchRequest, changesAgg, keys, ActionListener.wrap(wrappedListener::onResponse, e -> {
+            // fall back if bucket collection failed
+            logger.error("Failed to retrieve changed buckets, fall back to complete retrieval", e);
+            wrappedListener.onResponse(null);
+        }));
+    }
+
+    void collectChangedBuckets(SearchRequest searchRequest, CompositeAggregationBuilder changesAgg, Map<String, Set<String>> keys,
+            ActionListener<Map<String, Set<String>>> finalListener) {
+
+        // re-using the existing search hook
+        doNextSearch(searchRequest, ActionListener.wrap(searchResponse -> {
+            final CompositeAggregation agg = searchResponse.getAggregations().get(COMPOSITE_AGGREGATION_NAME);
+
+            agg.getBuckets().stream().forEach(bucket -> {
+                bucket.getKey().forEach((k, v) -> {
+                    keys.get(k).add(v.toString());
+                });
+            });
+
+            if (agg.getBuckets().isEmpty()) {
+                finalListener.onResponse(keys);
+            } else {
+                // adjust the after key
+                changesAgg.aggregateAfter(agg.afterKey());
+                collectChangedBuckets(searchRequest, changesAgg, keys, finalListener);
+            }
+        }, finalListener::onFailure));
+    }
+
     /**
      * Inspect exception for circuit breaking exception and return the first one it can find.
      *
@@ -251,4 +375,6 @@ private static CircuitBreakingException getCircuitBreakingException(Exception e)
 
         return null;
     }
+
+    protected abstract boolean sourceHasChanged();
 }
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutor.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutor.java
index 9ed8da61d8feb..ded3a0324e721 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutor.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutor.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.persistent.AllocatedPersistentTask;
@@ -27,10 +28,10 @@
 import org.elasticsearch.xpack.core.dataframe.DataFrameMessages;
 import org.elasticsearch.xpack.core.dataframe.action.StartDataFrameTransformTaskAction;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransform;
+import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpoint;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformState;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats;
-import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformTaskState;
 import org.elasticsearch.xpack.core.indexing.IndexerState;
 import org.elasticsearch.xpack.core.scheduler.SchedulerEngine;
 import org.elasticsearch.xpack.dataframe.DataFrame;
@@ -84,7 +85,10 @@ public PersistentTasksCustomMetaData.Assignment getAssignment(DataFrameTransform
             logger.debug(reason);
             return new PersistentTasksCustomMetaData.Assignment(null, reason);
         }
-        return super.getAssignment(params, clusterState);
+        DiscoveryNode discoveryNode = selectLeastLoadedNode(clusterState, (node) ->
+            node.isDataNode() && node.getVersion().onOrAfter(params.getVersion())
+        );
+        return discoveryNode == null ? NO_NODE_FOUND : new PersistentTasksCustomMetaData.Assignment(discoveryNode.getId(), "");
     }
 
     static List<String> verifyIndicesPrimaryShardsAreActive(ClusterState clusterState) {
@@ -112,12 +116,6 @@ protected void nodeOperation(AllocatedPersistentTask task, @Nullable DataFrameTr
             new DataFrameTransformTask.ClientDataFrameIndexerBuilder(transformId)
                 .setAuditor(auditor)
                 .setClient(client)
-                .setIndexerState(currentIndexerState(transformPTaskState))
-                // If the transform persistent task state is `null` that means this is a "first run".
-                // If we have state then the task has relocated from another node in which case this
-                // state is preferred
-                .setInitialPosition(transformPTaskState == null ? null : transformPTaskState.getPosition())
-                .setProgress(transformPTaskState == null ? null : transformPTaskState.getProgress())
                 .setTransformsCheckpointService(dataFrameTransformsCheckpointService)
                 .setTransformsConfigManager(transformsConfigManager);
 
@@ -128,33 +126,60 @@ protected void nodeOperation(AllocatedPersistentTask task, @Nullable DataFrameTr
 
         Long previousCheckpoint = transformPTaskState != null ? transformPTaskState.getCheckpoint() : null;
 
-        // <3> Set the previous stats (if they exist), initialize the indexer, start the task (If it is STOPPED)
+        // <4> Set the previous stats (if they exist), initialize the indexer, start the task (If it is STOPPED)
         // Since we don't create the task until `_start` is called, if we see that the task state is stopped, attempt to start
         // Schedule execution regardless
         ActionListener<DataFrameTransformStateAndStats> transformStatsActionListener = ActionListener.wrap(
             stateAndStats -> {
-                indexerBuilder.setInitialStats(stateAndStats.getTransformStats());
-                if (transformPTaskState == null) { // prefer the persistent task state
-                    indexerBuilder.setInitialPosition(stateAndStats.getTransformState().getPosition());
-                    indexerBuilder.setProgress(stateAndStats.getTransformState().getProgress());
-                }
+                logger.trace("[{}] initializing state and stats: [{}]", transformId, stateAndStats.toString());
+                indexerBuilder.setInitialStats(stateAndStats.getTransformStats())
+                    .setInitialPosition(stateAndStats.getTransformState().getPosition())
+                    .setProgress(stateAndStats.getTransformState().getProgress())
+                    .setIndexerState(currentIndexerState(stateAndStats.getTransformState()));
+                logger.debug("[{}] Loading existing state: [{}], position [{}]",
+                    transformId,
+                    stateAndStats.getTransformState(),
+                    stateAndStats.getTransformState().getPosition());
 
-                final Long checkpoint = previousCheckpoint != null ? previousCheckpoint : stateAndStats.getTransformState().getCheckpoint();
+                final Long checkpoint = stateAndStats.getTransformState().getCheckpoint();
                 startTask(buildTask, indexerBuilder, checkpoint, startTaskListener);
             },
             error -> {
                 if (error instanceof ResourceNotFoundException == false) {
-                    logger.error("Unable to load previously persisted statistics for transform [" + params.getId() + "]", error);
+                    logger.warn("Unable to load previously persisted statistics for transform [" + params.getId() + "]", error);
                 }
                 startTask(buildTask, indexerBuilder, previousCheckpoint, startTaskListener);
             }
         );
 
+        // <3> set the in progress checkpoint for the indexer, get the in progress checkpoint
+        ActionListener<DataFrameTransformCheckpoint> getTransformCheckpointListener = ActionListener.wrap(
+            cp -> {
+                indexerBuilder.setInProgressOrLastCheckpoint(cp);
+                transformsConfigManager.getTransformStats(transformId, transformStatsActionListener);
+            },
+            error -> {
+                String msg = DataFrameMessages.getMessage(DataFrameMessages.FAILED_TO_LOAD_TRANSFORM_CHECKPOINT, transformId);
+                logger.error(msg, error);
+                markAsFailed(buildTask, msg);
+            }
+        );
+
         // <2> set fieldmappings for the indexer, get the previous stats (if they exist)
         ActionListener<Map<String, String>> getFieldMappingsListener = ActionListener.wrap(
             fieldMappings -> {
                 indexerBuilder.setFieldMappings(fieldMappings);
-                transformsConfigManager.getTransformStats(transformId, transformStatsActionListener);
+
+                long inProgressCheckpoint = transformPTaskState == null ? 0L :
+                    Math.max(transformPTaskState.getCheckpoint(), transformPTaskState.getInProgressCheckpoint());
+
+                logger.debug("Restore in progress or last checkpoint: {}", inProgressCheckpoint);
+
+                if (inProgressCheckpoint == 0) {
+                    getTransformCheckpointListener.onResponse(DataFrameTransformCheckpoint.EMPTY);
+                } else {
+                    transformsConfigManager.getTransformCheckpoint(transformId, inProgressCheckpoint, getTransformCheckpointListener);
+                }
             },
             error -> {
                 String msg = DataFrameMessages.getMessage(DataFrameMessages.DATA_FRAME_UNABLE_TO_GATHER_FIELD_MAPPINGS,
@@ -223,18 +248,8 @@ private void startTask(DataFrameTransformTask buildTask,
                            DataFrameTransformTask.ClientDataFrameIndexerBuilder indexerBuilder,
                            Long previousCheckpoint,
                            ActionListener<StartDataFrameTransformTaskAction.Response> listener) {
-        // If we are stopped, and it is an initial run, this means we have never been started,
-        // attempt to start the task
-
         buildTask.initializeIndexer(indexerBuilder);
-        // TODO isInitialRun is false after relocation??
-        if (buildTask.getState().getTaskState().equals(DataFrameTransformTaskState.STOPPED) && buildTask.isInitialRun()) {
-            logger.info("Data frame transform [{}] created.", buildTask.getTransformId());
-            buildTask.start(previousCheckpoint, listener);
-        } else {
-            logger.debug("No need to start task. Its current state is: {}", buildTask.getState().getIndexerState());
-            listener.onResponse(new StartDataFrameTransformTaskAction.Response(true));
-        }
+        buildTask.start(previousCheckpoint, listener);
     }
 
     @Override
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformTask.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformTask.java
index 926f233c454d1..a30b726b3d289 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformTask.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformTask.java
@@ -11,7 +11,9 @@
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.action.bulk.BulkAction;
+import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.search.SearchAction;
@@ -29,6 +31,7 @@
 import org.elasticsearch.xpack.core.dataframe.action.StartDataFrameTransformTaskAction.Response;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransform;
+import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpoint;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpointingInfo;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformProgress;
@@ -46,6 +49,9 @@
 
 import java.util.Arrays;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
@@ -53,6 +59,8 @@
 
 public class DataFrameTransformTask extends AllocatedPersistentTask implements SchedulerEngine.Listener {
 
+    // interval the scheduler sends an event
+    private static final int SCHEDULER_NEXT_MILLISECONDS = 10000;
     private static final Logger logger = LogManager.getLogger(DataFrameTransformTask.class);
     // TODO consider moving to dynamic cluster setting
     private static final int MAX_CONTINUOUS_FAILURES = 10;
@@ -66,7 +74,7 @@ public class DataFrameTransformTask extends AllocatedPersistentTask implements S
     private final Map<String, Object> initialPosition;
     private final IndexerState initialIndexerState;
 
-    private final SetOnce<DataFrameIndexer> indexer = new SetOnce<>();
+    private final SetOnce<ClientDataFrameIndexer> indexer = new SetOnce<>();
 
     private final AtomicReference<DataFrameTransformTaskState> taskState;
     private final AtomicReference<String> stateReason;
@@ -87,12 +95,10 @@ public DataFrameTransformTask(long id, String type, String action, TaskId parent
         String initialReason = null;
         long initialGeneration = 0;
         Map<String, Object> initialPosition = null;
-        logger.trace("[{}] init, got state: [{}]", transform.getId(), state != null);
         if (state != null) {
             initialTaskState = state.getTaskState();
             initialReason = state.getReason();
             final IndexerState existingState = state.getIndexerState();
-            logger.info("[{}] Loading existing state: [{}], position [{}]", transform.getId(), existingState, state.getPosition());
             if (existingState.equals(IndexerState.INDEXING)) {
                 // reset to started as no indexer is running
                 initialState = IndexerState.STARTED;
@@ -125,7 +131,7 @@ public Status getStatus() {
         return getState();
     }
 
-    private DataFrameIndexer getIndexer() {
+    private ClientDataFrameIndexer getIndexer() {
         return indexer.get();
     }
 
@@ -174,13 +180,8 @@ public long getInProgressCheckpoint() {
         }
     }
 
-    public boolean isStopped() {
-        IndexerState currentState = getIndexer() == null ? initialIndexerState : getIndexer().getState();
-        return currentState.equals(IndexerState.STOPPED);
-    }
-
-    boolean isInitialRun() {
-        return getIndexer() != null && getIndexer().initialRun();
+    public void setTaskStateStopped() {
+        taskState.set(DataFrameTransformTaskState.STOPPED);
     }
 
     /**
@@ -216,9 +217,14 @@ public synchronized void start(Long startingCheckpoint, ActionListener<Response>
             getIndexer().getProgress());
 
         logger.info("Updating state for data frame transform [{}] to [{}]", transform.getId(), state.toString());
+        // Even though the indexer information is persisted to an index, we still need DataFrameTransformTaskState in the clusterstate
+        // This keeps track of STARTED, FAILED, STOPPED
+        // This is because a FAILED state can occur because we cannot read the config from the internal index, which would imply that
+        //   we could not read the previous state information from said index.
         persistStateToClusterState(state, ActionListener.wrap(
             task -> {
-                auditor.info(transform.getId(), "Updated state to [" + state.getTaskState() + "]");
+                auditor.info(transform.getId(),
+                    "Updated data frame transform state to [" + state.getTaskState() + "].");
                 long now = System.currentTimeMillis();
                 // kick off the indexer
                 triggered(new Event(schedulerJobName(), now, now));
@@ -235,15 +241,21 @@ public synchronized void start(Long startingCheckpoint, ActionListener<Response>
 
     public synchronized void stop() {
         if (getIndexer() == null) {
+            // If there is no indexer the task has not been triggered
+            // but it still needs to be stopped and removed
+            shutdown();
             return;
         }
-        // taskState is initialized as STOPPED and is updated in tandem with the indexerState
-        // Consequently, if it is STOPPED, we consider the whole task STOPPED.
-        if (taskState.get() == DataFrameTransformTaskState.STOPPED) {
+
+        if (getIndexer().getState() == IndexerState.STOPPED) {
             return;
         }
 
-        getIndexer().stop();
+        IndexerState state = getIndexer().stop();
+        if (state == IndexerState.STOPPED) {
+            getIndexer().onStop();
+            getIndexer().doSaveState(state, getIndexer().getPosition(), () -> {});
+        }
     }
 
     @Override
@@ -252,9 +264,20 @@ public synchronized void triggered(Event event) {
             logger.warn("Data frame task [{}] triggered with an unintialized indexer", getTransformId());
             return;
         }
-        //  for now no rerun, so only trigger if checkpoint == 0
-        if (currentCheckpoint.get() == 0 && event.getJobName().equals(schedulerJobName())) {
-            logger.debug("Data frame indexer [{}] schedule has triggered, state: [{}]", event.getJobName(), getIndexer().getState());
+
+        // Ignore if event is not for this job
+        if (event.getJobName().equals(schedulerJobName()) == false)  {
+            return;
+        }
+
+        logger.debug("Data frame indexer [{}] schedule has triggered, state: [{}]", event.getJobName(), getIndexer().getState());
+
+        // if it runs for the 1st time we just do it, if not we check for changes
+        if (currentCheckpoint.get() == 0 ) {
+            logger.debug("Trigger initial run");
+            getIndexer().maybeTriggerAsyncJob(System.currentTimeMillis());
+        } else if (getIndexer().isContinuous() && getIndexer().sourceHasChanged()) {
+            logger.debug("Source has changed, triggering new indexer run");
             getIndexer().maybeTriggerAsyncJob(System.currentTimeMillis());
         }
     }
@@ -297,10 +320,13 @@ void persistStateToClusterState(DataFrameTransformState state,
     synchronized void markAsFailed(String reason, ActionListener<Void> listener) {
         taskState.set(DataFrameTransformTaskState.FAILED);
         stateReason.set(reason);
+        auditor.error(transform.getId(), reason);
+        // Even though the indexer information is persisted to an index, we still need DataFrameTransformTaskState in the clusterstate
+        // This keeps track of STARTED, FAILED, STOPPED
+        // This is because a FAILED state can occur because we cannot read the config from the internal index, which would imply that
+        //   we could not read the previous state information from said index.
         persistStateToClusterState(getState(), ActionListener.wrap(
-            r -> {
-                listener.onResponse(null);
-            },
+            r -> listener.onResponse(null),
             listener::onFailure
         ));
     }
@@ -337,7 +363,7 @@ private String schedulerJobName() {
 
     private SchedulerEngine.Schedule next() {
         return (startTime, now) -> {
-            return now + 1000; // to be fixed, hardcode something
+            return now + SCHEDULER_NEXT_MILLISECONDS;
         };
     }
 
@@ -357,6 +383,7 @@ static class ClientDataFrameIndexerBuilder {
         private IndexerState indexerState = IndexerState.STOPPED;
         private Map<String, Object> initialPosition;
         private DataFrameTransformProgress progress;
+        private DataFrameTransformCheckpoint inProgressOrLastCheckpoint;
 
         ClientDataFrameIndexerBuilder(String transformId) {
             this.transformId = transformId;
@@ -375,6 +402,7 @@ ClientDataFrameIndexer build(DataFrameTransformTask parentTask) {
                 this.transformConfig,
                 this.fieldMappings,
                 this.progress,
+                this.inProgressOrLastCheckpoint,
                 parentTask);
         }
 
@@ -436,15 +464,24 @@ ClientDataFrameIndexerBuilder setProgress(DataFrameTransformProgress progress) {
             this.progress = progress;
             return this;
         }
+
+        ClientDataFrameIndexerBuilder setInProgressOrLastCheckpoint(DataFrameTransformCheckpoint inProgressOrLastCheckpoint) {
+            this.inProgressOrLastCheckpoint = inProgressOrLastCheckpoint;
+            return this;
+        }
     }
 
     static class ClientDataFrameIndexer extends DataFrameIndexer {
+
+        private static final int ON_FINISH_AUDIT_FREQUENCY = 1000;
+
         private final Client client;
         private final DataFrameTransformsConfigManager transformsConfigManager;
         private final DataFrameTransformsCheckpointService transformsCheckpointService;
         private final String transformId;
         private final DataFrameTransformTask transformTask;
         private final AtomicInteger failureCount;
+        private volatile boolean auditBulkFailures = true;
         // Keeps track of the last exception that was written to our audit, keeps us from spamming the audit index
         private volatile String lastAuditedExceptionMessage = null;
 
@@ -459,6 +496,7 @@ static class ClientDataFrameIndexer extends DataFrameIndexer {
                                DataFrameTransformConfig transformConfig,
                                Map<String, String> fieldMappings,
                                DataFrameTransformProgress transformProgress,
+                               DataFrameTransformCheckpoint inProgressOrLastCheckpoint,
                                DataFrameTransformTask parentTask) {
             super(ExceptionsHelper.requireNonNull(parentTask, "parentTask")
                     .threadPool
@@ -469,7 +507,8 @@ static class ClientDataFrameIndexer extends DataFrameIndexer {
                 ExceptionsHelper.requireNonNull(initialState, "initialState"),
                 initialPosition,
                 initialStats == null ? new DataFrameIndexerTransformStats(transformId) : initialStats,
-                transformProgress);
+                transformProgress,
+                inProgressOrLastCheckpoint);
             this.transformId = ExceptionsHelper.requireNonNull(transformId, "transformId");
             this.transformsConfigManager = ExceptionsHelper.requireNonNull(transformsConfigManager, "transformsConfigManager");
             this.transformsCheckpointService = ExceptionsHelper.requireNonNull(transformsCheckpointService,
@@ -481,23 +520,39 @@ static class ClientDataFrameIndexer extends DataFrameIndexer {
 
         @Override
         protected void onStart(long now, ActionListener<Void> listener) {
-            // Reset our failure count as we are starting again
-            failureCount.set(0);
             // On each run, we need to get the total number of docs and reset the count of processed docs
             // Since multiple checkpoints can be executed in the task while it is running on the same node, we need to gather
             // the progress here, and not in the executor.
             if (initialRun()) {
-                TransformProgressGatherer.getInitialProgress(this.client, getConfig(), ActionListener.wrap(
-                    newProgress -> {
-                        progress = newProgress;
-                        super.onStart(now, listener);
+                ActionListener<Map<String, Set<String>>> changedBucketsListener = ActionListener.wrap(
+                    r -> {
+                        TransformProgressGatherer.getInitialProgress(this.client, buildFilterQuery(), getConfig(), ActionListener.wrap(
+                            newProgress -> {
+                                logger.trace("[{}] reset the progress from [{}] to [{}]", transformId, progress, newProgress);
+                                progress = newProgress;
+                                super.onStart(now, listener);
+                            },
+                            failure -> {
+                                progress = null;
+                                logger.warn("Unable to load progress information for task [" + transformId + "]", failure);
+                                super.onStart(now, listener);
+                            }
+                        ));
                     },
-                    failure -> {
-                        progress = null;
-                        logger.warn("Unable to load progress information for task [" + transformId + "]", failure);
-                        super.onStart(now, listener);
+                    listener::onFailure
+                );
+
+                createCheckpoint(ActionListener.wrap(cp -> {
+                    DataFrameTransformCheckpoint oldCheckpoint = inProgressOrLastCheckpoint;
+                    if (oldCheckpoint.isEmpty()) {
+                        // this is the 1st run, accept the new in progress checkpoint and go on
+                        inProgressOrLastCheckpoint = cp;
+                        changedBucketsListener.onResponse(null);
+                    } else {
+                        logger.debug ("Getting changes from {} to {}", oldCheckpoint.getTimeUpperBound(), cp.getTimeUpperBound());
+                        getChangedBuckets(oldCheckpoint, cp, changedBucketsListener);
                     }
-                ));
+                }, listener::onFailure));
             } else {
                 super.onStart(now, listener);
             }
@@ -515,6 +570,13 @@ public synchronized boolean maybeTriggerAsyncJob(long now) {
                 return false;
             }
 
+            // ignore trigger if indexer is running, prevents log spam in A2P indexer
+            IndexerState indexerState = getState();
+            if (IndexerState.INDEXING.equals(indexerState) || IndexerState.STOPPING.equals(indexerState)) {
+                logger.debug("Indexer for transform [{}] has state [{}], ignoring trigger", getJobId(), indexerState);
+                return false;
+            }
+
             return super.maybeTriggerAsyncJob(now);
         }
 
@@ -526,8 +588,28 @@ protected void doNextSearch(SearchRequest request, ActionListener<SearchResponse
 
         @Override
         protected void doNextBulk(BulkRequest request, ActionListener<BulkResponse> nextPhase) {
-            ClientHelper.executeWithHeadersAsync(transformConfig.getHeaders(), ClientHelper.DATA_FRAME_ORIGIN, client, BulkAction.INSTANCE,
-                    request, nextPhase);
+            ClientHelper.executeWithHeadersAsync(transformConfig.getHeaders(),
+                ClientHelper.DATA_FRAME_ORIGIN,
+                client,
+                BulkAction.INSTANCE,
+                request,
+                ActionListener.wrap(bulkResponse -> {
+                    if (bulkResponse.hasFailures() && auditBulkFailures) {
+                        int failureCount = 0;
+                        for(BulkItemResponse item : bulkResponse.getItems()) {
+                            if (item.isFailed()) {
+                                failureCount++;
+                            }
+                        }
+                        auditor.warning(transformId,
+                            "Experienced at least [" +
+                                failureCount +
+                                "] bulk index failures. See the logs of the node running the transform for details. " +
+                                bulkResponse.buildFailureMessage());
+                        auditBulkFailures = false;
+                    }
+                    nextPhase.onResponse(bulkResponse);
+                }, nextPhase::onFailure));
         }
 
         @Override
@@ -537,11 +619,30 @@ protected void doSaveState(IndexerState indexerState, Map<String, Object> positi
                 next.run();
                 return;
             }
+            // If we are `STOPPED` on a `doSaveState` call, that indicates we transitioned to `STOPPED` from `STOPPING`
+            // OR we called `doSaveState` manually as the indexer was not actively running.
+            // Since we save the state to an index, we should make sure that our task state is in parity with the indexer state
+            if (indexerState.equals(IndexerState.STOPPED)) {
+                transformTask.setTaskStateStopped();
+            }
+
+            DataFrameTransformTaskState taskState = transformTask.taskState.get();
+
+            if (indexerState.equals(IndexerState.STARTED)
+                && transformTask.currentCheckpoint.get() == 1
+                && this.isContinuous() == false) {
+                // set both to stopped so they are persisted as such
+                taskState = DataFrameTransformTaskState.STOPPED;
+                indexerState = IndexerState.STOPPED;
+
+                auditor.info(transformConfig.getId(), "Data frame finished indexing all data, initiating stop");
+                logger.info("Data frame [{}] finished indexing all data, initiating stop", transformConfig.getId());
+            }
 
             final DataFrameTransformState state = new DataFrameTransformState(
-                transformTask.taskState.get(),
+                taskState,
                 indexerState,
-                getPosition(),
+                position,
                 transformTask.currentCheckpoint.get(),
                 transformTask.stateReason.get(),
                 getProgress());
@@ -549,28 +650,28 @@ protected void doSaveState(IndexerState indexerState, Map<String, Object> positi
 
             // Persisting stats when we call `doSaveState` should be ok as we only call it on a state transition and
             // only every-so-often when doing the bulk indexing calls.  See AsyncTwoPhaseIndexer#onBulkResponse for current periodicity
-            ActionListener<PersistentTasksCustomMetaData.PersistentTask<?>> updateClusterStateListener = ActionListener.wrap(
-                task -> {
-                    transformsConfigManager.putOrUpdateTransformStats(
-                            new DataFrameTransformStateAndStats(transformId, state, getStats(),
-                                    DataFrameTransformCheckpointingInfo.EMPTY), // TODO should this be null
-                            ActionListener.wrap(
-                                    r -> {
-                                        next.run();
-                                    },
-                                    statsExc -> {
-                                        logger.error("Updating stats of transform [" + transformConfig.getId() + "] failed", statsExc);
-                                        next.run();
-                                    }
-                            ));
-                },
-                exc -> {
-                    logger.error("Updating persistent state of transform [" + transformConfig.getId() + "] failed", exc);
-                    next.run();
-                }
-            );
-
-            transformTask.persistStateToClusterState(state, updateClusterStateListener);
+            transformsConfigManager.putOrUpdateTransformStats(
+                    new DataFrameTransformStateAndStats(transformId, state, getStats(),
+                            DataFrameTransformCheckpointingInfo.EMPTY), // TODO should this be null
+                    ActionListener.wrap(
+                            r -> {
+                                // for auto stop shutdown the task
+                                if (state.getTaskState().equals(DataFrameTransformTaskState.STOPPED)) {
+                                    transformTask.shutdown();
+                                }
+                                next.run();
+                            },
+                            statsExc -> {
+                                logger.error("Updating stats of transform [" + transformConfig.getId() + "] failed", statsExc);
+                                auditor.warning(getJobId(),
+                                    "Failure updating stats of transform: " + statsExc.getMessage());
+                                // for auto stop shutdown the task
+                                if (state.getTaskState().equals(DataFrameTransformTaskState.STOPPED)) {
+                                    transformTask.shutdown();
+                                }
+                                next.run();
+                            }
+                    ));
         }
 
         @Override
@@ -595,10 +696,16 @@ protected void onFailure(Exception exc) {
         protected void onFinish(ActionListener<Void> listener) {
             try {
                 super.onFinish(listener);
-                long checkpoint = transformTask.currentCheckpoint.incrementAndGet();
-                auditor.info(transformTask.getTransformId(), "Finished indexing for data frame transform checkpoint [" + checkpoint + "]");
-                logger.info(
+                long checkpoint = transformTask.currentCheckpoint.getAndIncrement();
+                // Reset our failure count as we have finished and may start again with a new checkpoint
+                failureCount.set(0);
+                if (checkpoint % ON_FINISH_AUDIT_FREQUENCY == 0) {
+                    auditor.info(transformTask.getTransformId(),
+                        "Finished indexing for data frame transform checkpoint [" + checkpoint + "].");
+                }
+                logger.debug(
                     "Finished indexing for data frame transform [" + transformTask.getTransformId() + "] checkpoint [" + checkpoint + "]");
+                auditBulkFailures = true;
                 listener.onResponse(null);
             } catch (Exception e) {
                 listener.onFailure(e);
@@ -607,37 +714,25 @@ protected void onFinish(ActionListener<Void> listener) {
 
         @Override
         protected void onStop() {
-            auditor.info(transformConfig.getId(), "Indexer has stopped");
+            auditor.info(transformConfig.getId(), "Data frame transform has stopped.");
             logger.info("Data frame transform [{}] indexer has stopped", transformConfig.getId());
-            transformsConfigManager.putOrUpdateTransformStats(
-                    new DataFrameTransformStateAndStats(transformId, transformTask.getState(), getStats(),
-                            DataFrameTransformCheckpointingInfo.EMPTY), // TODO should this be null
-                    ActionListener.wrap(
-                            r -> {
-                                transformTask.shutdown();
-                            },
-                            statsExc -> {
-                                transformTask.shutdown();
-                                logger.error("Updating saving stats of transform [" + transformConfig.getId() + "] failed", statsExc);
-                            }
-                    ));
         }
 
         @Override
         protected void onAbort() {
-            auditor.info(transformConfig.getId(), "Received abort request, stopping indexer");
+            auditor.info(transformConfig.getId(), "Received abort request, stopping data frame transform.");
             logger.info("Data frame transform [" + transformConfig.getId() + "] received abort request, stopping indexer");
             transformTask.shutdown();
         }
 
         @Override
-        protected void createCheckpoint(ActionListener<Void> listener) {
+        protected void createCheckpoint(ActionListener<DataFrameTransformCheckpoint> listener) {
             transformsCheckpointService.getCheckpoint(transformConfig,
                 transformTask.currentCheckpoint.get() + 1,
                 ActionListener.wrap(
                     checkpoint -> transformsConfigManager.putTransformCheckpoint(checkpoint,
                         ActionListener.wrap(
-                            putCheckPointResponse -> listener.onResponse(null),
+                            putCheckPointResponse -> listener.onResponse(checkpoint),
                             createCheckpointException ->
                                 listener.onFailure(new RuntimeException("Failed to create checkpoint", createCheckpointException))
                     )),
@@ -646,6 +741,42 @@ protected void createCheckpoint(ActionListener<Void> listener) {
             ));
         }
 
+        @Override
+        public boolean sourceHasChanged() {
+            if (getState() == IndexerState.INDEXING) {
+                logger.trace("Indexer is still running, ignore");
+                return false;
+            }
+
+            CountDownLatch latch = new CountDownLatch(1);
+
+            SetOnce<Boolean> changed = new SetOnce<>();
+            transformsCheckpointService.getCheckpoint(transformConfig, new LatchedActionListener<>(ActionListener.wrap(
+                    cp -> {
+                        long behind = DataFrameTransformCheckpoint.getBehind(inProgressOrLastCheckpoint, cp);
+                        if (behind > 0) {
+                            logger.debug("Detected changes, dest is {} operations behind the source", behind);
+                            changed.set(true);
+                        } else {
+                            changed.set(false);
+                        }
+                    }, e -> {
+                        changed.set(false);
+                        logger.error("failure in update check", e);
+                    }), latch));
+
+            try {
+                if (latch.await(5, TimeUnit.SECONDS)) {
+                    logger.trace("Change detected:" + changed.get());
+                    return changed.get();
+                }
+            } catch (InterruptedException e) {
+                logger.error("Failed to check for update", e);
+            }
+
+            return false;
+        }
+
         private boolean isIrrecoverableFailure(Exception e) {
             return e instanceof IndexNotFoundException || e instanceof AggregationResultUtils.AggregationExtractionException;
         }
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/TransformProgressGatherer.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/TransformProgressGatherer.java
index 18a341e217294..6850f8a5e9420 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/TransformProgressGatherer.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/TransformProgressGatherer.java
@@ -11,6 +11,9 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig;
@@ -26,13 +29,16 @@ public final class TransformProgressGatherer {
     /**
      * This gathers the total docs given the config and search
      *
-     * TODO: Support checkpointing logic to restrict the query
-     * @param progressListener The listener to alert on completion
+     * @param client ES Client to make queries
+     * @param filterQuery The adapted filter that can optionally take into account checkpoint information
+     * @param config The transform config containing headers, source, pivot, etc. information
+     * @param progressListener The listener to notify when progress object has been created
      */
     public static void getInitialProgress(Client client,
+                                          QueryBuilder filterQuery,
                                           DataFrameTransformConfig config,
                                           ActionListener<DataFrameTransformProgress> progressListener) {
-        SearchRequest request = getSearchRequest(config);
+        SearchRequest request = getSearchRequest(config, filterQuery);
 
         ActionListener<SearchResponse> searchResponseActionListener = ActionListener.wrap(
             searchResponse -> progressListener.onResponse(searchResponseToDataFrameTransformProgressFunction().apply(searchResponse)),
@@ -46,13 +52,23 @@ public static void getInitialProgress(Client client,
             searchResponseActionListener);
     }
 
-    public static SearchRequest getSearchRequest(DataFrameTransformConfig config) {
+    public static SearchRequest getSearchRequest(DataFrameTransformConfig config, QueryBuilder filteredQuery) {
         SearchRequest request = new SearchRequest(config.getSource().getIndex());
         request.allowPartialSearchResults(false);
+        BoolQueryBuilder existsClauses = QueryBuilders.boolQuery();
+        config.getPivotConfig()
+            .getGroupConfig()
+            .getGroups()
+            .values()
+            // TODO change once we allow missing_buckets
+            .forEach(src -> existsClauses.must(QueryBuilders.existsQuery(src.getField())));
+
         request.source(new SearchSourceBuilder()
             .size(0)
             .trackTotalHits(true)
-            .query(config.getSource().getQueryConfig().getQuery()));
+            .query(QueryBuilders.boolQuery()
+                .filter(filteredQuery)
+                .filter(existsClauses)));
         return request;
     }
 
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/pivot/AggregationResultUtils.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/pivot/AggregationResultUtils.java
index f8857591b2322..0a5c25e77cb11 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/pivot/AggregationResultUtils.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/pivot/AggregationResultUtils.java
@@ -6,15 +6,13 @@
 
 package org.elasticsearch.xpack.dataframe.transforms.pivot;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.Numbers;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation;
 import org.elasticsearch.search.aggregations.metrics.GeoCentroid;
-import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation.SingleValue;
 import org.elasticsearch.search.aggregations.metrics.ScriptedMetric;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
@@ -23,6 +21,7 @@
 import org.elasticsearch.xpack.dataframe.transforms.IDGenerator;
 
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -32,7 +31,15 @@
 import static org.elasticsearch.xpack.dataframe.transforms.pivot.SchemaUtil.isNumericType;
 
 public final class AggregationResultUtils {
-    private static final Logger logger = LogManager.getLogger(AggregationResultUtils.class);
+
+    private static final Map<String, AggValueExtractor> TYPE_VALUE_EXTRACTOR_MAP;
+    static {
+        Map<String, AggValueExtractor> tempMap = new HashMap<>();
+        tempMap.put(SingleValue.class.getName(), new SingleValueAggExtractor());
+        tempMap.put(ScriptedMetric.class.getName(), new ScriptedMetricAggExtractor());
+        tempMap.put(GeoCentroid.class.getName(), new GeoCentroidAggExtractor());
+        TYPE_VALUE_EXTRACTOR_MAP = Collections.unmodifiableMap(tempMap);
+    }
 
     /**
      * Extracts aggregation results from a composite aggregation and puts it into a map.
@@ -61,7 +68,7 @@ public static Stream<Map<String, Object>> extractCompositeAggregationResults(Com
             groups.getGroups().keySet().forEach(destinationFieldName -> {
                 Object value = bucket.getKey().get(destinationFieldName);
                 idGen.add(destinationFieldName, value);
-                document.put(destinationFieldName, value);
+                updateDocument(document, destinationFieldName, value);
             });
 
             List<String> aggNames = aggregationBuilders.stream().map(AggregationBuilder::getName).collect(Collectors.toList());
@@ -73,27 +80,8 @@ public static Stream<Map<String, Object>> extractCompositeAggregationResults(Com
                 // TODO: support other aggregation types
                 Aggregation aggResult = bucket.getAggregations().get(aggName);
 
-                if (aggResult instanceof NumericMetricsAggregation.SingleValue) {
-                    NumericMetricsAggregation.SingleValue aggResultSingleValue = (SingleValue) aggResult;
-                    // If the type is numeric or if the formatted string is the same as simply making the value a string,
-                    //    gather the `value` type, otherwise utilize `getValueAsString` so we don't lose formatted outputs.
-                    if (isNumericType(fieldType) ||
-                        (aggResultSingleValue.getValueAsString().equals(String.valueOf(aggResultSingleValue.value())))) {
-                        updateDocument(document, aggName, aggResultSingleValue.value());
-                    } else {
-                        updateDocument(document, aggName, aggResultSingleValue.getValueAsString());
-                    }
-                } else if (aggResult instanceof ScriptedMetric) {
-                    updateDocument(document, aggName, ((ScriptedMetric) aggResult).aggregation());
-                } else if (aggResult instanceof GeoCentroid) {
-                    updateDocument(document, aggName, ((GeoCentroid) aggResult).centroid().toString());
-                } else {
-                    // Execution should never reach this point!
-                    // Creating transforms with unsupported aggregations shall not be possible
-                    throw new AggregationExtractionException("unsupported aggregation [{}] with name [{}]",
-                        aggResult.getType(),
-                        aggResult.getName());
-                }
+                AggValueExtractor extractor = getExtractor(aggResult);
+                updateDocument(document, aggName, extractor.value(aggResult, fieldType));
             }
 
             document.put(DataFrameField.DOCUMENT_ID_FIELD, idGen.getID());
@@ -102,6 +90,23 @@ public static Stream<Map<String, Object>> extractCompositeAggregationResults(Com
         });
     }
 
+    static AggValueExtractor getExtractor(Aggregation aggregation) {
+        if (aggregation instanceof SingleValue) {
+            return TYPE_VALUE_EXTRACTOR_MAP.get(SingleValue.class.getName());
+        } else if (aggregation instanceof ScriptedMetric) {
+            return TYPE_VALUE_EXTRACTOR_MAP.get(ScriptedMetric.class.getName());
+        } else if (aggregation instanceof GeoCentroid) {
+            return TYPE_VALUE_EXTRACTOR_MAP.get(GeoCentroid.class.getName());
+        } else {
+            // Execution should never reach this point!
+            // Creating transforms with unsupported aggregations shall not be possible
+            throw new AggregationExtractionException("unsupported aggregation [{}] with name [{}]",
+                aggregation.getType(),
+                aggregation.getName());
+        }
+    }
+
+
     @SuppressWarnings("unchecked")
     static void updateDocument(Map<String, Object> document, String fieldName, Object value) {
         String[] fieldTokens = fieldName.split("\\.");
@@ -147,4 +152,44 @@ public static class AggregationExtractionException extends ElasticsearchExceptio
             super(msg, args);
         }
     }
+
+    private interface AggValueExtractor {
+        Object value(Aggregation aggregation, String fieldType);
+    }
+
+    private static class SingleValueAggExtractor implements AggValueExtractor {
+        @Override
+        public Object value(Aggregation agg, String fieldType) {
+            SingleValue aggregation = (SingleValue)agg;
+            // If the double is invalid, this indicates sparse data
+            if (Numbers.isValidDouble(aggregation.value()) == false) {
+                return null;
+            }
+            // If the type is numeric or if the formatted string is the same as simply making the value a string,
+            //    gather the `value` type, otherwise utilize `getValueAsString` so we don't lose formatted outputs.
+            if (isNumericType(fieldType) ||
+                aggregation.getValueAsString().equals(String.valueOf(aggregation.value()))){
+                return aggregation.value();
+            } else {
+                return aggregation.getValueAsString();
+            }
+        }
+    }
+
+    private static class ScriptedMetricAggExtractor implements AggValueExtractor {
+        @Override
+        public Object value(Aggregation agg, String fieldType) {
+            ScriptedMetric aggregation = (ScriptedMetric)agg;
+            return aggregation.aggregation();
+        }
+    }
+
+    private static class GeoCentroidAggExtractor implements AggValueExtractor {
+        @Override
+        public Object value(Aggregation agg, String fieldType) {
+            GeoCentroid aggregation = (GeoCentroid)agg;
+            // if the account is `0` iff there is no contained centroid
+            return aggregation.count() > 0 ? aggregation.centroid().toString() : null;
+        }
+    }
 }
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/pivot/Aggregations.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/pivot/Aggregations.java
index 615c9b2e8d2e6..24cac4d58e6a4 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/pivot/Aggregations.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/pivot/Aggregations.java
@@ -34,9 +34,10 @@ enum AggregationType {
         VALUE_COUNT("value_count", "long"),
         MAX("max", SOURCE),
         MIN("min", SOURCE),
-        SUM("sum", SOURCE),
+        SUM("sum", "double"),
         GEO_CENTROID("geo_centroid", "geo_point"),
         SCRIPTED_METRIC("scripted_metric", DYNAMIC),
+        WEIGHTED_AVG("weighted_avg", DYNAMIC),
         BUCKET_SCRIPT("bucket_script", DYNAMIC);
 
         private final String aggregationType;
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/pivot/Pivot.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/pivot/Pivot.java
index 8205f2576da68..0c0104603f9c4 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/pivot/Pivot.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/pivot/Pivot.java
@@ -6,6 +6,8 @@
 
 package org.elasticsearch.xpack.dataframe.transforms.pivot;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchRequest;
@@ -13,9 +15,9 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -28,10 +30,15 @@
 import org.elasticsearch.xpack.core.dataframe.transforms.SourceConfig;
 import org.elasticsearch.xpack.core.dataframe.transforms.pivot.GroupConfig;
 import org.elasticsearch.xpack.core.dataframe.transforms.pivot.PivotConfig;
+import org.elasticsearch.xpack.core.dataframe.transforms.pivot.SingleGroupSource;
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -41,6 +48,7 @@ public class Pivot {
     public static final int TEST_QUERY_PAGE_SIZE = 50;
 
     private static final String COMPOSITE_AGGREGATION_NAME = "_data_frame";
+    private static final Logger logger = LogManager.getLogger(Pivot.class);
 
     private final PivotConfig config;
 
@@ -107,6 +115,26 @@ public AggregationBuilder buildAggregation(Map<String, Object> position, int pag
         return cachedCompositeAggregation;
     }
 
+    public CompositeAggregationBuilder buildIncrementalBucketUpdateAggregation(int pageSize) {
+
+        CompositeAggregationBuilder compositeAgg = createCompositeAggregationSources(config, true);
+        compositeAgg.size(pageSize);
+
+        return compositeAgg;
+    }
+
+    public Map<String, Set<String>> initialIncrementalBucketUpdateMap() {
+
+        Map<String, Set<String>> changedBuckets = new HashMap<>();
+        for(Entry<String, SingleGroupSource> entry: config.getGroupConfig().getGroups().entrySet()) {
+            if (entry.getValue().supportsIncrementalBucketUpdate()) {
+                changedBuckets.put(entry.getKey(), new HashSet<>());
+            }
+        }
+
+        return changedBuckets;
+    }
+
     public Stream<Map<String, Object>> extractResults(CompositeAggregation agg,
                                                       Map<String, String> fieldTypeMap,
                                                       DataFrameIndexerTransformStats dataFrameIndexerTransformStats) {
@@ -141,17 +169,66 @@ private void runTestQuery(Client client, SourceConfig sourceConfig, final Action
         }));
     }
 
+    public QueryBuilder filterBuckets(Map<String, Set<String>> changedBuckets) {
+
+        if (changedBuckets == null || changedBuckets.isEmpty()) {
+            return null;
+        }
+
+        if (config.getGroupConfig().getGroups().size() == 1) {
+            Entry<String, SingleGroupSource> entry = config.getGroupConfig().getGroups().entrySet().iterator().next();
+            // it should not be possible to get into this code path
+            assert (entry.getValue().supportsIncrementalBucketUpdate());
+
+            logger.trace("filter by bucket: " + entry.getKey() + "/" + entry.getValue().getField());
+            if (changedBuckets.containsKey(entry.getKey())) {
+                return entry.getValue().getIncrementalBucketUpdateFilterQuery(changedBuckets.get(entry.getKey()));
+            } else {
+                // should never happen
+                throw new RuntimeException("Could not find bucket value for key " + entry.getKey());
+            }
+        }
+
+        // else: more than 1 group by, need to nest it
+        BoolQueryBuilder filteredQuery = new BoolQueryBuilder();
+        for (Entry<String, SingleGroupSource> entry : config.getGroupConfig().getGroups().entrySet()) {
+            if (entry.getValue().supportsIncrementalBucketUpdate() == false) {
+                continue;
+            }
+
+            if (changedBuckets.containsKey(entry.getKey())) {
+                QueryBuilder sourceQueryFilter = entry.getValue().getIncrementalBucketUpdateFilterQuery(changedBuckets.get(entry.getKey()));
+                // the source might not define an filter optimization
+                if (sourceQueryFilter != null) {
+                    filteredQuery.filter(sourceQueryFilter);
+                }
+            } else {
+                // should never happen
+                throw new RuntimeException("Could not find bucket value for key " + entry.getKey());
+            }
+
+        }
+
+        return filteredQuery;
+    }
+
     private static CompositeAggregationBuilder createCompositeAggregation(PivotConfig config) {
+        final CompositeAggregationBuilder compositeAggregation = createCompositeAggregationSources(config, false);
+
+        config.getAggregationConfig().getAggregatorFactories().forEach(agg -> compositeAggregation.subAggregation(agg));
+        config.getAggregationConfig().getPipelineAggregatorFactories().forEach(agg -> compositeAggregation.subAggregation(agg));
+
+        return compositeAggregation;
+    }
+
+    private static CompositeAggregationBuilder createCompositeAggregationSources(PivotConfig config, boolean forChangeDetection) {
         CompositeAggregationBuilder compositeAggregation;
 
         try (XContentBuilder builder = jsonBuilder()) {
-            // write configuration for composite aggs into builder
-            config.toCompositeAggXContent(builder, ToXContentObject.EMPTY_PARAMS);
+            config.toCompositeAggXContent(builder, forChangeDetection);
             XContentParser parser = builder.generator().contentType().xContent().createParser(NamedXContentRegistry.EMPTY,
                     LoggingDeprecationHandler.INSTANCE, BytesReference.bytes(builder).streamInput());
             compositeAggregation = CompositeAggregationBuilder.parse(COMPOSITE_AGGREGATION_NAME, parser);
-            config.getAggregationConfig().getAggregatorFactories().forEach(agg -> compositeAggregation.subAggregation(agg));
-            config.getAggregationConfig().getPipelineAggregatorFactories().forEach(agg -> compositeAggregation.subAggregation(agg));
         } catch (IOException e) {
             throw new RuntimeException(DataFrameMessages.DATA_FRAME_TRANSFORM_PIVOT_FAILED_TO_CREATE_COMPOSITE_AGGREGATION, e);
         }
diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/pivot/SchemaUtil.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/pivot/SchemaUtil.java
index 304f35b8c4c8a..4ac77c38f7d5f 100644
--- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/pivot/SchemaUtil.java
+++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/transforms/pivot/SchemaUtil.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.ScriptedMetricAggregationBuilder;
+import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.dataframe.transforms.pivot.PivotConfig;
@@ -77,7 +78,7 @@ public static void deduceMappings(final Client client,
                 ValuesSourceAggregationBuilder<?, ?> valueSourceAggregation = (ValuesSourceAggregationBuilder<?, ?>) agg;
                 aggregationSourceFieldNames.put(valueSourceAggregation.getName(), valueSourceAggregation.field());
                 aggregationTypes.put(valueSourceAggregation.getName(), valueSourceAggregation.getType());
-            } else if(agg instanceof ScriptedMetricAggregationBuilder) {
+            } else if(agg instanceof ScriptedMetricAggregationBuilder || agg instanceof MultiValuesSourceAggregationBuilder) {
                 aggregationTypes.put(agg.getName(), agg.getType());
             } else {
                 // execution should not reach this point
diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/DataFrameFeatureSetTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/DataFrameInfoTransportActionTests.java
similarity index 74%
rename from x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/DataFrameFeatureSetTests.java
rename to x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/DataFrameInfoTransportActionTests.java
index f70cbb8b277c2..8695b774a35d8 100644
--- a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/DataFrameFeatureSetTests.java
+++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/DataFrameInfoTransportActionTests.java
@@ -7,9 +7,10 @@
 package org.elasticsearch.xpack.dataframe;
 
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -21,8 +22,9 @@
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.XPackFeatureSet.Usage;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats;
 import org.junit.Before;
 
@@ -33,13 +35,13 @@
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 
-import static org.elasticsearch.xpack.dataframe.DataFrameFeatureSet.PROVIDED_STATS;
+import static org.elasticsearch.xpack.dataframe.DataFrameInfoTransportAction.PROVIDED_STATS;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.core.Is.is;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class DataFrameFeatureSetTests extends ESTestCase {
+public class DataFrameInfoTransportActionTests extends ESTestCase {
     private XPackLicenseState licenseState;
 
     @Before
@@ -48,10 +50,8 @@ public void init() {
     }
 
     public void testAvailable() {
-        DataFrameFeatureSet featureSet = new DataFrameFeatureSet(Settings.EMPTY,
-            mock(ClusterService.class),
-            mock(Client.class),
-            licenseState);
+        DataFrameInfoTransportAction featureSet = new DataFrameInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
         boolean available = randomBoolean();
         when(licenseState.isDataFrameAllowed()).thenReturn(available);
         assertThat(featureSet.available(), is(available));
@@ -61,18 +61,14 @@ public void testEnabledSetting() {
         boolean enabled = randomBoolean();
         Settings.Builder settings = Settings.builder();
         settings.put("xpack.data_frame.enabled", enabled);
-        DataFrameFeatureSet featureSet = new DataFrameFeatureSet(settings.build(),
-            mock(ClusterService.class),
-            mock(Client.class),
-            licenseState);
+        DataFrameInfoTransportAction featureSet = new DataFrameInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState);
         assertThat(featureSet.enabled(), is(enabled));
     }
 
     public void testEnabledDefault() {
-        DataFrameFeatureSet featureSet = new DataFrameFeatureSet(Settings.EMPTY,
-            mock(ClusterService.class),
-            mock(Client.class),
-            licenseState);
+        DataFrameInfoTransportAction featureSet = new DataFrameInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
         assertTrue(featureSet.enabled());
     }
 
@@ -81,7 +77,8 @@ public void testParseSearchAggs() {
         SearchResponse withEmptyAggs = mock(SearchResponse.class);
         when(withEmptyAggs.getAggregations()).thenReturn(emptyAggs);
 
-        assertThat(DataFrameFeatureSet.parseSearchAggs(withEmptyAggs), equalTo(DataFrameIndexerTransformStats.withDefaultTransformId()));
+        assertThat(DataFrameInfoTransportAction.parseSearchAggs(withEmptyAggs),
+            equalTo(DataFrameIndexerTransformStats.withDefaultTransformId()));
 
         DataFrameIndexerTransformStats expectedStats = new DataFrameIndexerTransformStats("_all",
             1,  // numPages
@@ -104,7 +101,7 @@ public void testParseSearchAggs() {
         SearchResponse withAggs = mock(SearchResponse.class);
         when(withAggs.getAggregations()).thenReturn(aggregations);
 
-        assertThat(DataFrameFeatureSet.parseSearchAggs(withAggs), equalTo(expectedStats));
+        assertThat(DataFrameInfoTransportAction.parseSearchAggs(withAggs), equalTo(expectedStats));
     }
 
     private static Aggregation buildAgg(String name, double value) {
@@ -118,13 +115,11 @@ public void testUsageDisabled() throws IOException, InterruptedException, Execut
         when(licenseState.isDataFrameAllowed()).thenReturn(true);
         Settings.Builder settings = Settings.builder();
         settings.put("xpack.data_frame.enabled", false);
-        DataFrameFeatureSet featureSet = new DataFrameFeatureSet(settings.build(),
-            mock(ClusterService.class),
-            mock(Client.class),
-            licenseState);
-        PlainActionFuture<Usage> future = new PlainActionFuture<>();
-        featureSet.usage(future);
-        XPackFeatureSet.Usage usage = future.get();
+        var usageAction = new DataFrameUsageTransportAction(mock(TransportService.class), null, null,
+            mock(ActionFilters.class), null, settings.build(), licenseState, mock(Client.class));
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, mock(ClusterState.class), future);
+        XPackFeatureSet.Usage usage = future.get().getUsage();
 
         assertFalse(usage.enabled());
         try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/DataFrameNodesTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/DataFrameNodesTests.java
index ba549aa7e8bb4..276d3af39ef13 100644
--- a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/DataFrameNodesTests.java
+++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/action/DataFrameNodesTests.java
@@ -31,10 +31,10 @@ public void testDataframeNodes() {
 
         PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
         tasksBuilder.addTask(dataFrameIdFoo,
-                DataFrameField.TASK_NAME, new DataFrameTransform(dataFrameIdFoo),
+                DataFrameField.TASK_NAME, new DataFrameTransform(dataFrameIdFoo, Version.CURRENT),
                 new PersistentTasksCustomMetaData.Assignment("node-1", "test assignment"));
         tasksBuilder.addTask(dataFrameIdBar,
-                DataFrameField.TASK_NAME, new DataFrameTransform(dataFrameIdBar),
+                DataFrameField.TASK_NAME, new DataFrameTransform(dataFrameIdBar, Version.CURRENT),
                 new PersistentTasksCustomMetaData.Assignment("node-2", "test assignment"));
         tasksBuilder.addTask("test-task1", "testTasks", new PersistentTaskParams() {
                 @Override
diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformCheckpointServiceNodeTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformCheckpointServiceNodeTests.java
index f9c2d4fd95b22..166ec2de3febf 100644
--- a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformCheckpointServiceNodeTests.java
+++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/checkpoint/DataFrameTransformCheckpointServiceNodeTests.java
@@ -6,7 +6,7 @@
 
 package org.elasticsearch.xpack.dataframe.checkpoint;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -88,8 +88,8 @@ public void setShardStats(ShardStats[] shardStats) {
 
         @SuppressWarnings("unchecked")
         @Override
-        protected <Request extends ActionRequest, Response extends ActionResponse> void doExecute(Action<Response> action, Request request,
-                ActionListener<Response> listener) {
+        protected <Request extends ActionRequest, Response extends ActionResponse>
+        void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
 
             if (request instanceof GetIndexRequest) {
                 // for this test we only need the indices
diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManagerTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManagerTests.java
index 9c7af3efa5333..e7485e02e5b91 100644
--- a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManagerTests.java
+++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/persistence/DataFrameTransformsConfigManagerTests.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.xpack.core.action.util.PageParams;
 import org.elasticsearch.xpack.core.dataframe.DataFrameMessages;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpoint;
@@ -158,8 +159,9 @@ public void testExpandIds() throws Exception {
         assertAsync(listener ->
                 transformsConfigManager.expandTransformIds(transformConfig1.getId(),
                     PageParams.defaultParams(),
+                    true,
                     listener),
-            Collections.singletonList("transform1_expand"),
+            new Tuple<>(1L, Collections.singletonList("transform1_expand")),
             null,
             null);
 
@@ -167,8 +169,9 @@ public void testExpandIds() throws Exception {
         assertAsync(listener ->
                 transformsConfigManager.expandTransformIds("transform1_expand,transform2_expand",
                     PageParams.defaultParams(),
+                    true,
                     listener),
-            Arrays.asList("transform1_expand", "transform2_expand"),
+            new Tuple<>(2L, Arrays.asList("transform1_expand", "transform2_expand")),
             null,
             null);
 
@@ -176,8 +179,9 @@ public void testExpandIds() throws Exception {
         assertAsync(listener ->
                 transformsConfigManager.expandTransformIds("transform1*,transform2_expand,transform3_expand",
                     PageParams.defaultParams(),
+                    true,
                     listener),
-            Arrays.asList("transform1_expand", "transform2_expand", "transform3_expand"),
+            new Tuple<>(3L, Arrays.asList("transform1_expand", "transform2_expand", "transform3_expand")),
             null,
             null);
 
@@ -185,8 +189,9 @@ public void testExpandIds() throws Exception {
         assertAsync(listener ->
                 transformsConfigManager.expandTransformIds("_all",
                     PageParams.defaultParams(),
+                    true,
                     listener),
-            Arrays.asList("transform1_expand", "transform2_expand", "transform3_expand"),
+            new Tuple<>(3L, Arrays.asList("transform1_expand", "transform2_expand", "transform3_expand")),
             null,
             null);
 
@@ -194,8 +199,9 @@ public void testExpandIds() throws Exception {
         assertAsync(listener ->
                 transformsConfigManager.expandTransformIds("_all",
                     new PageParams(0, 1),
+                    true,
                     listener),
-            Collections.singletonList("transform1_expand"),
+            new Tuple<>(3L, Collections.singletonList("transform1_expand")),
             null,
             null);
 
@@ -203,8 +209,9 @@ public void testExpandIds() throws Exception {
         assertAsync(listener ->
                 transformsConfigManager.expandTransformIds("_all",
                     new PageParams(1, 2),
+                    true,
                     listener),
-            Arrays.asList("transform2_expand", "transform3_expand"),
+            new Tuple<>(3L, Arrays.asList("transform2_expand", "transform3_expand")),
             null,
             null);
 
@@ -212,8 +219,9 @@ public void testExpandIds() throws Exception {
         assertAsync(listener ->
                 transformsConfigManager.expandTransformIds("unknown,unknown2",
                     new PageParams(1, 2),
+                    true,
                     listener),
-            (List<String>)null,
+            (Tuple<Long, List<String>>)null,
             null,
             e -> {
                 assertThat(e, instanceOf(ResourceNotFoundException.class));
@@ -221,6 +229,20 @@ public void testExpandIds() throws Exception {
                     equalTo(DataFrameMessages.getMessage(DataFrameMessages.REST_DATA_FRAME_UNKNOWN_TRANSFORM, "unknown,unknown2")));
             });
 
+        // expand 1 id implicitly that does not exist
+        assertAsync(listener ->
+                transformsConfigManager.expandTransformIds("unknown*",
+                    new PageParams(1, 2),
+                    false,
+                    listener),
+            (Tuple<Long, List<String>>)null,
+            null,
+            e -> {
+                assertThat(e, instanceOf(ResourceNotFoundException.class));
+                assertThat(e.getMessage(),
+                    equalTo(DataFrameMessages.getMessage(DataFrameMessages.REST_DATA_FRAME_UNKNOWN_TRANSFORM, "unknown*")));
+            });
+
     }
 
     public void testStateAndStats() throws InterruptedException {
@@ -239,7 +261,7 @@ public void testStateAndStats() throws InterruptedException {
     }
 
     public void testGetStateAndStatsMultiple() throws InterruptedException {
-        int numStats = randomInt(5);
+        int numStats = randomIntBetween(10, 15);
         List<DataFrameTransformStateAndStats> expectedStats = new ArrayList<>();
         for (int i=0; i<numStats; i++) {
             DataFrameTransformStateAndStats stat =
diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndexTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndexTests.java
new file mode 100644
index 0000000000000..2ff7756c82f60
--- /dev/null
+++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndexTests.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.dataframe.persistence;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfigTests;
+import org.mockito.ArgumentCaptor;
+
+import java.io.IOException;
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+public class DataframeIndexTests extends ESTestCase {
+
+    private static final String TRANSFORM_ID = "some-random-transform-id";
+    private static final int CURRENT_TIME_MILLIS = 123456789;
+    private static final String CREATED_BY = "data-frame-transform";
+
+    private Client client = mock(Client.class);
+    private Clock clock = Clock.fixed(Instant.ofEpochMilli(CURRENT_TIME_MILLIS), ZoneId.systemDefault());
+
+    public void testCreateDestinationIndex() throws IOException {
+        doAnswer(
+            invocationOnMock -> {
+                @SuppressWarnings("unchecked")
+                ActionListener<CreateIndexResponse> listener = (ActionListener<CreateIndexResponse>) invocationOnMock.getArguments()[2];
+                listener.onResponse(null);
+                return null;
+            })
+            .when(client).execute(any(), any(), any());
+
+        DataframeIndex.createDestinationIndex(
+            client,
+            clock,
+            DataFrameTransformConfigTests.randomDataFrameTransformConfig(TRANSFORM_ID),
+            new HashMap<>(),
+            ActionListener.wrap(
+                value -> assertTrue(value),
+                e -> fail(e.getMessage())));
+
+        ArgumentCaptor<CreateIndexRequest> createIndexRequestCaptor = ArgumentCaptor.forClass(CreateIndexRequest.class);
+        verify(client).execute(eq(CreateIndexAction.INSTANCE), createIndexRequestCaptor.capture(), any());
+        verifyNoMoreInteractions(client);
+
+        CreateIndexRequest createIndexRequest = createIndexRequestCaptor.getValue();
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, createIndexRequest.mappings().get("_doc"))) {
+            Map<String, Object> map = parser.map();
+            assertThat(extractValue("_doc._meta._data_frame.transform", map), equalTo(TRANSFORM_ID));
+            assertThat(extractValue("_doc._meta._data_frame.creation_date_in_millis", map), equalTo(CURRENT_TIME_MILLIS));
+            assertThat(extractValue("_doc._meta.created_by", map), equalTo(CREATED_BY));
+        }
+    }
+}
diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameIndexerTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameIndexerTests.java
index 43198c6edfcf3..e3e9ff81eb653 100644
--- a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameIndexerTests.java
+++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameIndexerTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats;
+import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformCheckpoint;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig;
 import org.elasticsearch.xpack.core.dataframe.transforms.pivot.AggregationConfigTests;
 import org.elasticsearch.xpack.core.dataframe.transforms.pivot.GroupConfigTests;
@@ -73,7 +74,7 @@ class MockedDataFrameIndexer extends DataFrameIndexer {
                 Function<BulkRequest, BulkResponse> bulkFunction,
                 Consumer<Exception> failureConsumer) {
             super(executor, auditor, transformConfig, fieldMappings, initialState, initialPosition, jobStats,
-                    /* DataFrameTransformProgress */ null);
+                    /* DataFrameTransformProgress */ null, DataFrameTransformCheckpoint.EMPTY);
             this.searchFunction = searchFunction;
             this.bulkFunction = bulkFunction;
             this.failureConsumer = failureConsumer;
@@ -84,8 +85,8 @@ public CountDownLatch newLatch(int count) {
         }
 
         @Override
-        protected void createCheckpoint(ActionListener<Void> listener) {
-            listener.onResponse(null);
+        protected void createCheckpoint(ActionListener<DataFrameTransformCheckpoint> listener) {
+            listener.onResponse(DataFrameTransformCheckpoint.EMPTY);
         }
 
         @Override
@@ -163,6 +164,11 @@ protected void failIndexer(String message) {
             fail("failIndexer should not be called, received error: " + message);
         }
 
+        @Override
+        protected boolean sourceHasChanged() {
+            return false;
+        }
+
     }
 
     @Before
@@ -179,6 +185,7 @@ public void testPageSizeAdapt() throws InterruptedException {
             randomSourceConfig(),
             randomDestConfig(),
             null,
+            null,
             new PivotConfig(GroupConfigTests.randomGroupConfig(), AggregationConfigTests.randomAggregationConfig(), pageSize),
             randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000));
         AtomicReference<IndexerState> state = new AtomicReference<>(IndexerState.STOPPED);
@@ -227,4 +234,5 @@ public void testPageSizeAdapt() throws InterruptedException {
             executor.shutdownNow();
         }
     }
+
 }
diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutorTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutorTests.java
index 9e29fef1b5a03..49cdabfdf0601 100644
--- a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutorTests.java
+++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/DataFrameTransformPersistentTasksExecutorTests.java
@@ -7,10 +7,14 @@
 package org.elasticsearch.xpack.dataframe.transforms;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RecoverySource;
@@ -20,14 +24,93 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransform;
+import org.elasticsearch.xpack.core.scheduler.SchedulerEngine;
+import org.elasticsearch.xpack.dataframe.checkpoint.DataFrameTransformsCheckpointService;
+import org.elasticsearch.xpack.dataframe.notifications.DataFrameAuditor;
 import org.elasticsearch.xpack.dataframe.persistence.DataFrameInternalIndex;
+import org.elasticsearch.xpack.dataframe.persistence.DataFrameTransformsConfigManager;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
 
 public class DataFrameTransformPersistentTasksExecutorTests extends ESTestCase {
 
+    public void testNodeVersionAssignment() {
+        MetaData.Builder metaData = MetaData.builder();
+        RoutingTable.Builder routingTable = RoutingTable.builder();
+        addIndices(metaData, routingTable);
+        PersistentTasksCustomMetaData.Builder pTasksBuilder = PersistentTasksCustomMetaData.builder()
+            .addTask("data-frame-task-1",
+                DataFrameTransform.NAME,
+                new DataFrameTransform("data-frame-task-1", Version.CURRENT),
+                new PersistentTasksCustomMetaData.Assignment("current-data-node-with-1-tasks", ""))
+            .addTask("data-frame-task-2",
+                DataFrameTransform.NAME,
+                new DataFrameTransform("data-frame-task-2", Version.CURRENT),
+                new PersistentTasksCustomMetaData.Assignment("current-data-node-with-2-tasks", ""))
+            .addTask("data-frame-task-3",
+                DataFrameTransform.NAME,
+                new DataFrameTransform("data-frame-task-3", Version.CURRENT),
+                new PersistentTasksCustomMetaData.Assignment("current-data-node-with-2-tasks", ""));
+
+        PersistentTasksCustomMetaData pTasks = pTasksBuilder.build();
+
+        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, pTasks);
+
+        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder()
+            .add(new DiscoveryNode("past-data-node-1",
+                buildNewFakeTransportAddress(),
+                Collections.emptyMap(),
+                Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE),
+                Version.V_7_2_0))
+            .add(new DiscoveryNode("current-data-node-with-2-tasks",
+                buildNewFakeTransportAddress(),
+                Collections.emptyMap(),
+                Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE),
+                Version.CURRENT))
+            .add(new DiscoveryNode("non-data-node-1",
+                buildNewFakeTransportAddress(),
+                Collections.emptyMap(),
+                Set.of(DiscoveryNodeRole.MASTER_ROLE),
+                Version.CURRENT))
+            .add(new DiscoveryNode("current-data-node-with-1-tasks",
+                buildNewFakeTransportAddress(),
+                Collections.emptyMap(),
+                Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE),
+                Version.CURRENT));
+
+        ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"))
+            .nodes(nodes);
+        csBuilder.routingTable(routingTable.build());
+        csBuilder.metaData(metaData);
+
+        ClusterState cs = csBuilder.build();
+        Client client = mock(Client.class);
+        DataFrameTransformsConfigManager transformsConfigManager = new DataFrameTransformsConfigManager(client, xContentRegistry());
+        DataFrameTransformsCheckpointService dataFrameTransformsCheckpointService = new DataFrameTransformsCheckpointService(client,
+            transformsConfigManager);
+
+        DataFrameTransformPersistentTasksExecutor executor = new DataFrameTransformPersistentTasksExecutor(client,
+            transformsConfigManager,
+            dataFrameTransformsCheckpointService, mock(SchedulerEngine.class),
+            new DataFrameAuditor(client, ""),
+            mock(ThreadPool.class));
+
+        assertThat(executor.getAssignment(new DataFrameTransform("new-task-id", Version.CURRENT), cs).getExecutorNode(),
+            equalTo("current-data-node-with-1-tasks"));
+        assertThat(executor.getAssignment(new DataFrameTransform("new-old-task-id", Version.V_7_2_0), cs).getExecutorNode(),
+            equalTo("past-data-node-1"));
+    }
+
     public void testVerifyIndicesPrimaryShardsAreActive() {
         MetaData.Builder metaData = MetaData.builder();
         RoutingTable.Builder routingTable = RoutingTable.builder();
diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/pivot/AggregationResultUtilsTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/pivot/AggregationResultUtilsTests.java
index 1a835c9d19b59..34a4b0d80b4e4 100644
--- a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/pivot/AggregationResultUtilsTests.java
+++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/pivot/AggregationResultUtilsTests.java
@@ -135,8 +135,8 @@ aggTypedName, asMap(
                                   KEY, asMap(
                                           targetField, "ID3"),
                                   aggTypedName, asMap(
-                                          "value", 12.55),
-                                  DOC_COUNT, 9)
+                                          "value", Double.NaN),
+                                  DOC_COUNT, 0)
                     ));
 
         List<Map<String, Object>> expected = asList(
@@ -150,14 +150,14 @@ aggTypedName, asMap(
                         ),
                 asMap(
                         targetField, "ID3",
-                        aggName, 12.55
+                        aggName, null
                         )
                 );
         Map<String, String> fieldTypeMap = asStringMap(
             targetField, "keyword",
             aggName, "double"
         );
-        executeTest(groupBy, aggregationBuilders, Collections.emptyList(), input, fieldTypeMap, expected, 20);
+        executeTest(groupBy, aggregationBuilders, Collections.emptyList(), input, fieldTypeMap, expected, 11);
     }
 
     public void testExtractCompositeAggregationResultsMultipleGroups() throws IOException {
@@ -212,8 +212,8 @@ KEY, asMap(
                                           targetField2, "ID2_2"
                                           ),
                                   aggTypedName, asMap(
-                                          "value", 12.55),
-                                  DOC_COUNT, 4)
+                                          "value", Double.NaN),
+                                  DOC_COUNT, 0)
                     ));
 
         List<Map<String, Object>> expected = asList(
@@ -235,7 +235,7 @@ aggTypedName, asMap(
                 asMap(
                         targetField, "ID3",
                         targetField2, "ID2_2",
-                        aggName, 12.55
+                        aggName, null
                         )
                 );
         Map<String, String> fieldTypeMap = asStringMap(
@@ -243,7 +243,7 @@ aggTypedName, asMap(
             targetField, "keyword",
             targetField2, "keyword"
         );
-        executeTest(groupBy, aggregationBuilders, Collections.emptyList(), input, fieldTypeMap, expected, 10);
+        executeTest(groupBy, aggregationBuilders, Collections.emptyList(), input, fieldTypeMap, expected, 6);
     }
 
     public void testExtractCompositeAggregationResultsMultiAggregations() throws IOException {
@@ -287,7 +287,7 @@ KEY, asMap(
                                   aggTypedName, asMap(
                                           "value", 12.55),
                                   aggTypedName2, asMap(
-                                          "value", -2.44),
+                                          "value", Double.NaN),
                                   DOC_COUNT, 1)
                     ));
 
@@ -305,7 +305,7 @@ aggTypedName2, asMap(
                 asMap(
                         targetField, "ID3",
                         aggName, 12.55,
-                        aggName2, -2.44
+                        aggName2, null
                         )
                 );
         Map<String, String> fieldTypeMap = asStringMap(
@@ -383,8 +383,8 @@ KEY, asMap(
                     aggTypedName, asMap(
                         "value", 12.55),
                     aggTypedName2, asMap(
-                        "value", -100.44,
-                        "value_as_string", "-100.44F"),
+                        "value", Double.NaN,
+                        "value_as_string", "NaN"),
                     DOC_COUNT, 4)
             ));
 
@@ -411,7 +411,7 @@ aggTypedName2, asMap(
                 targetField, "ID3",
                 targetField2, "ID2_2",
                 aggName, 12.55,
-                aggName2, "-100.44F"
+                aggName2, null
             )
         );
         Map<String, String> fieldTypeMap = asStringMap(
@@ -476,8 +476,8 @@ KEY, asMap(
                         targetField2, "ID2_2"
                     ),
                     aggTypedName, asMap(
-                        "value", asMap("field", 12.0)),
-                    DOC_COUNT, 4)
+                        "value", null),
+                    DOC_COUNT, 0)
             ));
 
         List<Map<String, Object>> expected = asList(
@@ -499,14 +499,14 @@ aggName, asMap("field", 2.13)
             asMap(
                 targetField, "ID3",
                 targetField2, "ID2_2",
-                aggName, asMap("field", 12.0)
+                aggName, null
             )
         );
         Map<String, String> fieldTypeMap = asStringMap(
             targetField, "keyword",
             targetField2, "keyword"
         );
-        executeTest(groupBy, aggregationBuilders, Collections.emptyList(), input, fieldTypeMap, expected, 10);
+        executeTest(groupBy, aggregationBuilders, Collections.emptyList(), input, fieldTypeMap, expected, 6);
     }
 
     public void testExtractCompositeAggregationResultsWithPipelineAggregation() throws IOException {
@@ -576,7 +576,7 @@ KEY, asMap(
                     aggTypedName, asMap(
                         "value", 12.0),
                     pipelineAggTypedName, asMap(
-                        "value", 12.0),
+                        "value", Double.NaN),
                     DOC_COUNT, 4)
             ));
 
@@ -603,7 +603,7 @@ pipelineAggTypedName, asMap(
                 targetField, "ID3",
                 targetField2, "ID2_2",
                 aggName, 12.0,
-                pipelineAggName, 12.0
+                pipelineAggName, null
             )
         );
         Map<String, String> fieldTypeMap = asStringMap(
diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/pivot/AggregationsTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/pivot/AggregationsTests.java
index 8443699430a2a..99d858a611d7a 100644
--- a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/pivot/AggregationsTests.java
+++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/pivot/AggregationsTests.java
@@ -34,9 +34,9 @@ public void testResolveTargetMapping() {
         assertEquals("half_float", Aggregations.resolveTargetMapping("min", "half_float"));
 
         // sum
-        assertEquals("int", Aggregations.resolveTargetMapping("sum", "int"));
         assertEquals("double", Aggregations.resolveTargetMapping("sum", "double"));
-        assertEquals("half_float", Aggregations.resolveTargetMapping("sum", "half_float"));
+        assertEquals("double", Aggregations.resolveTargetMapping("sum", "half_float"));
+        assertEquals("double", Aggregations.resolveTargetMapping("sum", null));
 
         // geo_centroid
         assertEquals("geo_point", Aggregations.resolveTargetMapping("geo_centroid", "geo_point"));
@@ -49,5 +49,9 @@ public void testResolveTargetMapping() {
         // bucket_script
         assertEquals("_dynamic", Aggregations.resolveTargetMapping("bucket_script", null));
         assertEquals("_dynamic", Aggregations.resolveTargetMapping("bucket_script", "int"));
+
+        // weighted_avg
+        assertEquals("_dynamic", Aggregations.resolveTargetMapping("weighted_avg", null));
+        assertEquals("_dynamic", Aggregations.resolveTargetMapping("weighted_avg", "double"));
     }
 }
diff --git a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/pivot/PivotTests.java b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/pivot/PivotTests.java
index 20ea84502ed82..a9e3f118b6371 100644
--- a/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/pivot/PivotTests.java
+++ b/x-pack/plugin/data-frame/src/test/java/org/elasticsearch/xpack/dataframe/transforms/pivot/PivotTests.java
@@ -7,7 +7,7 @@
 package org.elasticsearch.xpack.dataframe.transforms.pivot;
 
 import org.apache.lucene.search.TotalHits;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -61,7 +61,7 @@ public class PivotTests extends ESTestCase {
     @Before
     public void registerAggregationNamedObjects() throws Exception {
         // register aggregations as NamedWriteable
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         namedXContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
@@ -154,8 +154,8 @@ private class MyMockClient extends NoOpClient {
 
         @SuppressWarnings("unchecked")
         @Override
-        protected <Request extends ActionRequest, Response extends ActionResponse> void doExecute(Action<Response> action, Request request,
-                ActionListener<Response> listener) {
+        protected <Request extends ActionRequest, Response extends ActionResponse>
+        void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
 
             if (request instanceof SearchRequest) {
                 SearchRequest searchRequest = (SearchRequest) request;
@@ -215,6 +215,16 @@ private AggregationConfig getAggregationConfig(String agg) throws IOException {
                 "\"buckets_path\":{\"param_1\":\"other_bucket\"}," +
                 "\"script\":\"return params.param_1\"}}}");
         }
+        if (agg.equals(AggregationType.WEIGHTED_AVG.getName())) {
+            return parseAggregations("{\n" +
+                "\"pivot_weighted_avg\": {\n" +
+                "  \"weighted_avg\": {\n" +
+                "   \"value\": {\"field\": \"values\"},\n" +
+                "   \"weight\": {\"field\": \"weights\"}\n" +
+                "  }\n" +
+                "}\n" +
+                "}");
+        }
         return parseAggregations("{\n" + "  \"pivot_" + agg + "\": {\n" + "    \"" + agg + "\": {\n" + "      \"field\": \"values\"\n"
                 + "    }\n" + "  }" + "}");
     }
diff --git a/x-pack/plugin/deprecation/build.gradle b/x-pack/plugin/deprecation/build.gradle
index 62d2a891929a4..bbf235131d772 100644
--- a/x-pack/plugin/deprecation/build.gradle
+++ b/x-pack/plugin/deprecation/build.gradle
@@ -10,7 +10,7 @@ esplugin {
 archivesBaseName = 'x-pack-deprecation'
 
 dependencies {
-    compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+    compileOnly project(":x-pack:plugin:core")
 }
 
 integTest.enabled = false
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java
index 6556a8ad0eff5..6b55006983d0c 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
@@ -82,7 +83,7 @@ protected ClusterBlockException checkBlock(DeprecationInfoAction.Request request
     }
 
     @Override
-    protected final void masterOperation(final DeprecationInfoAction.Request request, ClusterState state,
+    protected final void masterOperation(Task task, final DeprecationInfoAction.Request request, ClusterState state,
                                          final ActionListener<DeprecationInfoAction.Response> listener) {
         if (licenseState.isDeprecationAllowed()) {
 
diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java
index a315559a2f9a7..eeb3c7e8c8d75 100644
--- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java
+++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
@@ -52,8 +53,8 @@ protected NodesDeprecationCheckResponse newResponse(NodesDeprecationCheckRequest
     }
 
     @Override
-    protected NodesDeprecationCheckAction.NodeRequest newNodeRequest(String nodeId, NodesDeprecationCheckRequest request) {
-        return new NodesDeprecationCheckAction.NodeRequest(nodeId, request);
+    protected NodesDeprecationCheckAction.NodeRequest newNodeRequest(NodesDeprecationCheckRequest request) {
+        return new NodesDeprecationCheckAction.NodeRequest(request);
     }
 
     @Override
@@ -62,7 +63,7 @@ protected NodesDeprecationCheckAction.NodeResponse newNodeResponse() {
     }
 
     @Override
-    protected NodesDeprecationCheckAction.NodeResponse nodeOperation(NodesDeprecationCheckAction.NodeRequest request) {
+    protected NodesDeprecationCheckAction.NodeResponse nodeOperation(NodesDeprecationCheckAction.NodeRequest request, Task task) {
         List<DeprecationIssue> issues = DeprecationInfoAction.filterChecks(DeprecationChecks.NODE_SETTINGS_CHECKS,
             (c) -> c.apply(settings, pluginsService.info()));
 
diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java
index 8d98cc2adbebd..c3ca275ad350d 100644
--- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java
+++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/MlDeprecationChecksTests.java
@@ -19,7 +19,7 @@ public class MlDeprecationChecksTests extends ESTestCase {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/x-pack/plugin/graph/build.gradle b/x-pack/plugin/graph/build.gradle
index 286f8ac0a8917..a164189dd6a06 100644
--- a/x-pack/plugin/graph/build.gradle
+++ b/x-pack/plugin/graph/build.gradle
@@ -10,7 +10,6 @@ esplugin {
 archivesBaseName = 'x-pack-graph'
 
 dependencies {
-    // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
     compileOnly project(path: xpackModule('core'), configuration: 'default')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     if (isEclipse) {
diff --git a/x-pack/plugin/graph/qa/build.gradle b/x-pack/plugin/graph/qa/build.gradle
index 35bd236df5c23..25b21503bedd8 100644
--- a/x-pack/plugin/graph/qa/build.gradle
+++ b/x-pack/plugin/graph/qa/build.gradle
@@ -1,17 +1,17 @@
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
-subprojects {
-    // HACK: please fix this
-    // we want to add the rest api specs for xpack to qa tests, but we
-    // need to wait until after the project is evaluated to only apply
-    // to those that rest tests. this used to be done automatically
-    // when xpack was a plugin, but now there is no place with xpack as a module.
-    // instead, we should package these and make them easy to use for rest tests,
-    // but currently, they must be copied into the resources of the test runner.
-    project.tasks.withType(RestIntegTestTask) {
-        File xpackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources')
-        project.copyRestSpec.from(xpackResources) {
-            include 'rest-api-spec/api/**'
-        }
-    }
+subprojects {	
+    // HACK: please fix this	
+    // we want to add the rest api specs for xpack to qa tests, but we	
+    // need to wait until after the project is evaluated to only apply	
+    // to those that rest tests. this used to be done automatically	
+    // when xpack was a plugin, but now there is no place with xpack as a module.	
+    // instead, we should package these and make them easy to use for rest tests,	
+    // but currently, they must be copied into the resources of the test runner.	
+    project.tasks.withType(RestIntegTestTask) {	
+        File xpackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources')	
+        project.copyRestSpec.from(xpackResources) {	
+            include 'rest-api-spec/api/**'	
+        }	
+    }	
 }
diff --git a/x-pack/plugin/graph/qa/with-security/build.gradle b/x-pack/plugin/graph/qa/with-security/build.gradle
index f0f819b46d478..1a979d83355c9 100644
--- a/x-pack/plugin/graph/qa/with-security/build.gradle
+++ b/x-pack/plugin/graph/qa/with-security/build.gradle
@@ -1,8 +1,9 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile "org.elasticsearch.plugin:x-pack-core:${version}"
+  testCompile project(":x-pack:plugin:core")
 }
 
 // bring in graph rest test suite
@@ -12,25 +13,15 @@ task copyGraphRestTests(type: Copy) {
   include 'rest-api-spec/test/graph/**'
 }
 
-integTestCluster {
-  dependsOn copyGraphRestTests
-  extraConfigFile 'roles.yml', 'roles.yml'
-  setupCommand 'setupTestAdminUser',
-               'bin/elasticsearch-users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser'
-  setupCommand 'setupGraphExplorerUser',
-               'bin/elasticsearch-users', 'useradd', 'graph_explorer', '-p', 'x-pack-test-password', '-r', 'graph_explorer'
-  setupCommand 'setupPowerlessUser',
-               'bin/elasticsearch-users', 'useradd', 'no_graph_explorer', '-p', 'x-pack-test-password', '-r', 'no_graph_explorer'
-  setting 'xpack.license.self_generated.type', 'trial'
+integTest.dependsOn copyGraphRestTests
+
+testClusters.integTest {
+  distribution = "DEFAULT"
   setting 'xpack.security.enabled', 'true'
-  waitCondition = { node, ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-            dest: tmpFile.toString(),
-            username: 'test_admin',
-            password: 'x-pack-test-password',
-            ignoreerrors: true,
-            retries: 10)
-    return tmpFile.exists()
-  }
+  setting 'xpack.license.self_generated.type', 'trial'
+
+  extraConfigFile 'roles.yml', file('roles.yml')
+  user username: 'test_admin', password: 'x-pack-test-password'
+  user username: 'graph_explorer', password: 'x-pack-test-password', role: 'graph_explorer'
+  user username: 'no_graph_explorer', password: 'x-pack-test-password', role: 'no_graph_explorer'
 }
diff --git a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/Graph.java b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/Graph.java
index eba19eec8079c..7281a7bfde67b 100644
--- a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/Graph.java
+++ b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/Graph.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -18,14 +17,14 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
-import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.graph.action.GraphExploreAction;
 import org.elasticsearch.xpack.graph.action.TransportGraphExploreAction;
 import org.elasticsearch.xpack.graph.rest.action.RestGraphAction;
 
-import java.util.Collection;
-import java.util.Collections;
+import java.util.Arrays;
 import java.util.List;
 import java.util.function.Supplier;
 
@@ -34,26 +33,23 @@
 
 public class Graph extends Plugin implements ActionPlugin {
 
-    public static final String NAME = "graph";
     protected final boolean enabled;
 
-
     public Graph(Settings settings) {
         this.enabled = XPackSettings.GRAPH_ENABLED.get(settings);
     }
 
-    public Collection<Module> createGuiceModules() {
-        return Collections.singletonList(b -> {
-            XPackPlugin.bindFeatureSet(b, GraphFeatureSet.class);
-        });
-    }
-
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        var usageAction = new ActionHandler<>(XPackUsageFeatureAction.GRAPH, GraphUsageTransportAction.class);
+        var infoAction = new ActionHandler<>(XPackInfoFeatureAction.GRAPH, GraphInfoTransportAction.class);
         if (false == enabled) {
-            return emptyList();
+            return Arrays.asList(usageAction, infoAction);
         }
-        return singletonList(new ActionHandler<>(GraphExploreAction.INSTANCE, TransportGraphExploreAction.class));
+        return Arrays.asList(
+            new ActionHandler<>(GraphExploreAction.INSTANCE, TransportGraphExploreAction.class),
+            usageAction,
+            infoAction);
     }
 
     @Override
diff --git a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphFeatureSet.java b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphInfoTransportAction.java
similarity index 57%
rename from x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphFeatureSet.java
rename to x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphInfoTransportAction.java
index 4986950cc3ce6..d8c181e5196d5 100644
--- a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphFeatureSet.java
+++ b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphInfoTransportAction.java
@@ -5,25 +5,25 @@
  */
 package org.elasticsearch.xpack.graph;
 
-import java.util.Map;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.common.Nullable;
+import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
-import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
 
-public class GraphFeatureSet implements XPackFeatureSet {
+public class GraphInfoTransportAction extends XPackInfoFeatureTransportAction {
 
     private final boolean enabled;
     private final XPackLicenseState licenseState;
 
     @Inject
-    public GraphFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState) {
+    public GraphInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                    Settings settings, XPackLicenseState licenseState) {
+        super(XPackInfoFeatureAction.GRAPH.name(), transportService, actionFilters);
         this.enabled = XPackSettings.GRAPH_ENABLED.get(settings);
         this.licenseState = licenseState;
     }
@@ -33,11 +33,6 @@ public String name() {
         return XPackField.GRAPH;
     }
 
-    @Override
-    public String description() {
-        return "Graph Data Exploration for the Elastic Stack";
-    }
-
     @Override
     public boolean available() {
         return licenseState != null && licenseState.isGraphAllowed();
@@ -48,14 +43,4 @@ public boolean enabled() {
         return enabled;
     }
 
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return null;
-    }
-
-    @Override
-    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
-        listener.onResponse(new GraphFeatureSetUsage(available(), enabled()));
-    }
-
 }
diff --git a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphUsageTransportAction.java b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphUsageTransportAction.java
new file mode 100644
index 0000000000000..15a0b44909288
--- /dev/null
+++ b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/GraphUsageTransportAction.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.graph;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage;
+
+public class GraphUsageTransportAction extends XPackUsageFeatureTransportAction {
+
+    private final Settings settings;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public GraphUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                     ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                     Settings settings, XPackLicenseState licenseState) {
+        super(XPackUsageFeatureAction.GRAPH.name(), transportService, clusterService,
+              threadPool, actionFilters, indexNameExpressionResolver);
+        this.settings = settings;
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
+        GraphFeatureSetUsage usage =
+            new GraphFeatureSetUsage(licenseState.isGraphAllowed(), XPackSettings.GRAPH_ENABLED.get(settings));
+        listener.onResponse(new XPackUsageFeatureResponse(usage));
+    }
+}
diff --git a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/action/TransportGraphExploreAction.java b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/action/TransportGraphExploreAction.java
index d05c075d26b65..e6c7dd59efd9e 100644
--- a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/action/TransportGraphExploreAction.java
+++ b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/action/TransportGraphExploreAction.java
@@ -181,7 +181,7 @@ synchronized void expand() {
             currentHopNumber++;
             Hop currentHop = request.getHop(currentHopNumber);
 
-            final SearchRequest searchRequest = new SearchRequest(request.indices()).types(request.types()).indicesOptions(
+            final SearchRequest searchRequest = new SearchRequest(request.indices()).indicesOptions(
                     request.indicesOptions());
             if (request.routing() != null) {
                 searchRequest.routing(request.routing());
@@ -568,7 +568,7 @@ private void addBigOrClause(Map<String, Set<Vertex>> lastHopFindings, BoolQueryB
         public synchronized void start() {
             try {
 
-                final SearchRequest searchRequest = new SearchRequest(request.indices()).types(request.types()).indicesOptions(
+                final SearchRequest searchRequest = new SearchRequest(request.indices()).indicesOptions(
                         request.indicesOptions());
                 if (request.routing() != null) {
                     searchRequest.routing(request.routing());
diff --git a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java
index 130d6deed567f..80d4d638a0a09 100644
--- a/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java
+++ b/x-pack/plugin/graph/src/main/java/org/elasticsearch/xpack/graph/rest/action/RestGraphAction.java
@@ -9,6 +9,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationLogger;
@@ -19,11 +20,10 @@
 import org.elasticsearch.protocol.xpack.graph.GraphExploreRequest.TermBoost;
 import org.elasticsearch.protocol.xpack.graph.Hop;
 import org.elasticsearch.protocol.xpack.graph.VertexRequest;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xpack.core.XPackClient;
-import org.elasticsearch.xpack.core.rest.XPackRestHandler;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -38,7 +38,7 @@
 /**
  * @see GraphExploreRequest
  */
-public class RestGraphAction extends XPackRestHandler {
+public class RestGraphAction extends BaseRestHandler {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGraphAction.class));
     public static final String TYPES_DEPRECATION_MESSAGE = "[types removal]" +
@@ -68,19 +68,19 @@ public RestGraphAction(Settings settings, RestController controller) {
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
                 GET, "/{index}/_graph/explore", this,
-                GET, "/{index}" + URI_BASE + "/graph/_explore", deprecationLogger);
+                GET, "/{index}/_xpack/graph/_explore", deprecationLogger);
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
                 POST, "/{index}/_graph/explore", this,
-                POST, "/{index}" + URI_BASE + "/graph/_explore", deprecationLogger);
+                POST, "/{index}/_xpack/graph/_explore", deprecationLogger);
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
                 GET, "/{index}/{type}/_graph/explore", this,
-                GET, "/{index}/{type}" + URI_BASE + "/graph/_explore", deprecationLogger);
+                GET, "/{index}/{type}/_xpack/graph/_explore", deprecationLogger);
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
                 POST, "/{index}/{type}/_graph/explore", this,
-                POST, "/{index}/{type}" + URI_BASE + "/graph/_explore", deprecationLogger);
+                POST, "/{index}/{type}/_xpack/graph/_explore", deprecationLogger);
     }
 
     @Override
@@ -89,7 +89,7 @@ public String getName() {
     }
 
     @Override
-    public RestChannelConsumer doPrepareRequest(final RestRequest request, final XPackClient client) throws IOException {
+    public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         GraphExploreRequest graphRequest = new GraphExploreRequest(Strings.splitStringByCommaToArray(request.param("index")));
         graphRequest.indicesOptions(IndicesOptions.fromRequest(request, graphRequest.indicesOptions()));
         graphRequest.routing(request.param("routing"));
@@ -117,7 +117,7 @@ public RestChannelConsumer doPrepareRequest(final RestRequest request, final XPa
             deprecationLogger.deprecatedAndMaybeLog("graph_with_types", TYPES_DEPRECATION_MESSAGE);
             graphRequest.types(Strings.splitStringByCommaToArray(request.param("type")));
         }
-        return channel -> client.es().execute(INSTANCE, graphRequest, new RestToXContentListener<>(channel));
+        return channel -> client.execute(INSTANCE, graphRequest, new RestToXContentListener<>(channel));
     }
 
     private void parseHop(XContentParser parser, Hop currentHop, GraphExploreRequest graphRequest) throws IOException {
diff --git a/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphFeatureSetTests.java b/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java
similarity index 62%
rename from x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphFeatureSetTests.java
rename to x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java
index 6b3992c60e4d1..9e99a9679333f 100644
--- a/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphFeatureSetTests.java
+++ b/x-pack/plugin/graph/src/test/java/org/elasticsearch/xpack/graph/GraphInfoTransportActionTests.java
@@ -5,12 +5,15 @@
  */
 package org.elasticsearch.xpack.graph;
 
+import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage;
 import org.junit.Before;
 
@@ -18,7 +21,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class GraphFeatureSetTests extends ESTestCase {
+public class GraphInfoTransportActionTests extends ESTestCase {
 
     private XPackLicenseState licenseState;
 
@@ -28,13 +31,17 @@ public void init() throws Exception {
     }
 
     public void testAvailable() throws Exception {
-        GraphFeatureSet featureSet = new GraphFeatureSet(Settings.EMPTY, licenseState);
+        GraphInfoTransportAction featureSet = new GraphInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
         boolean available = randomBoolean();
         when(licenseState.isGraphAllowed()).thenReturn(available);
         assertThat(featureSet.available(), is(available));
-        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
-        featureSet.usage(future);
-        XPackFeatureSet.Usage usage = future.get();
+
+        var usageAction = new GraphUsageTransportAction(mock(TransportService.class), null, null,
+            mock(ActionFilters.class), null, Settings.EMPTY, licenseState);
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, null, future);
+        XPackFeatureSet.Usage usage = future.get().getUsage();
         assertThat(usage.available(), is(available));
 
         BytesStreamOutput out = new BytesStreamOutput();
@@ -53,11 +60,15 @@ public void testEnabled() throws Exception {
         } else {
             settings.put("xpack.graph.enabled", enabled);
         }
-        GraphFeatureSet featureSet = new GraphFeatureSet(settings.build(), licenseState);
+        GraphInfoTransportAction featureSet = new GraphInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState);
         assertThat(featureSet.enabled(), is(enabled));
-        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
-        featureSet.usage(future);
-        XPackFeatureSet.Usage usage = future.get();
+
+        GraphUsageTransportAction usageAction = new GraphUsageTransportAction(mock(TransportService.class),
+            null, null, mock(ActionFilters.class), null, settings.build(), licenseState);
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, null, future);
+        XPackFeatureSet.Usage usage = future.get().getUsage();
         assertThat(usage.enabled(), is(enabled));
 
         BytesStreamOutput out = new BytesStreamOutput();
diff --git a/x-pack/plugin/ilm/build.gradle b/x-pack/plugin/ilm/build.gradle
index 2b80ddc04207b..4e0c44ef8e4ab 100644
--- a/x-pack/plugin/ilm/build.gradle
+++ b/x-pack/plugin/ilm/build.gradle
@@ -13,7 +13,6 @@ esplugin {
 archivesBaseName = 'x-pack-ilm'
 
 dependencies {
-    // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
     compileOnly project(path: xpackModule('core'), configuration: 'default')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     if (isEclipse) {
diff --git a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle
index 76dbf676d738f..86b82208ca4c5 100644
--- a/x-pack/plugin/ilm/qa/multi-cluster/build.gradle
+++ b/x-pack/plugin/ilm/qa/multi-cluster/build.gradle
@@ -1,5 +1,6 @@
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
@@ -8,13 +9,20 @@ dependencies {
     testCompile project(':x-pack:plugin:ilm')
 }
 
-task leaderClusterTest(type: RestIntegTestTask) {
+File repoDir = file("$buildDir/testclusters/repo")
+
+task 'leader-cluster'(type: RestIntegTestTask) {
     mustRunAfter(precommit)
+    runner {
+        systemProperty 'tests.target_cluster', 'leader'
+        /* To support taking index snapshots, we have to set path.repo setting */
+        systemProperty 'tests.path.repo', repoDir.absolutePath
+    }
 }
 
-leaderClusterTestCluster {
-    numNodes = 1
-    clusterName = 'leader-cluster'
+testClusters.'leader-cluster' {
+    distribution = "DEFAULT"
+    setting 'path.repo', repoDir.absolutePath
     setting 'xpack.ilm.enabled', 'true'
     setting 'xpack.ccr.enabled', 'true'
     setting 'xpack.security.enabled', 'false'
@@ -25,18 +33,23 @@ leaderClusterTestCluster {
     setting 'indices.lifecycle.poll_interval', '1000ms'
 }
 
-leaderClusterTestRunner {
-    systemProperty 'tests.target_cluster', 'leader'
-    /* To support taking index snapshots, we have to set path.repo setting */
-    systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo")
+task 'follow-cluster'(type: RestIntegTestTask) {
+    dependsOn 'leader-cluster'
+    useCluster testClusters.'leader-cluster'
+    runner {
+        systemProperty 'tests.target_cluster', 'follow'
+        nonInputProperties.systemProperty 'tests.leader_host',
+                "${-> testClusters."leader-cluster".getAllHttpSocketURI().get(0)}"
+        nonInputProperties.systemProperty 'tests.leader_remote_cluster_seed',
+                "${-> testClusters.'leader-cluster'.getAllTransportPortURI().get(0)}"
+        /* To support taking index snapshots, we have to set path.repo setting */
+        systemProperty 'tests.path.repo', repoDir.absolutePath
+    }
 }
 
-task followClusterTest(type: RestIntegTestTask) {}
-
-followClusterTestCluster {
-    dependsOn leaderClusterTestRunner
-    numNodes = 1
-    clusterName = 'follow-cluster'
+testClusters.'follow-cluster' {
+    distribution = "DEFAULT"
+    setting 'path.repo', repoDir.absolutePath
     setting 'xpack.ilm.enabled', 'true'
     setting 'xpack.ccr.enabled', 'true'
     setting 'xpack.security.enabled', 'false'
@@ -45,17 +58,9 @@ followClusterTestCluster {
     setting 'xpack.ml.enabled', 'false'
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'indices.lifecycle.poll_interval', '1000ms'
-    setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
-}
-
-followClusterTestRunner {
-    systemProperty 'tests.target_cluster', 'follow'
-    nonInputProperties.systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
-    nonInputProperties.systemProperty 'tests.leader_remote_cluster_seed', "${-> leaderClusterTest.nodes.get(0).transportUri()}"
-    /* To support taking index snapshots, we have to set path.repo setting */
-    systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo")
-    finalizedBy 'leaderClusterTestCluster#stop'
+    setting 'cluster.remote.leader_cluster.seeds',
+            { "\"${testClusters.'leader-cluster'.getAllTransportPortURI().get(0)}\"" }
 }
 
-check.dependsOn followClusterTest
+check.dependsOn 'follow-cluster'
 test.enabled = false // no unit tests for this module, only the rest integration test
diff --git a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java
index b185a425934eb..14e70446ab120 100644
--- a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java
+++ b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java
@@ -11,7 +11,6 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
-import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
@@ -66,8 +65,8 @@ public void testBasicCCRAndILMIntegration() throws Exception {
             putILMPolicy(policyName, "50GB", null, TimeValue.timeValueHours(7*24));
             followIndex(indexName, indexName);
             ensureGreen(indexName);
-            // Aliases are not copied from leader index, so we need to add that for the rollover action in follower cluster:
-            client().performRequest(new Request("PUT", "/" + indexName + "/_alias/logs"));
+
+            assertBusy(() -> assertOK(client().performRequest(new Request("HEAD", "/" + indexName + "/_alias/logs"))));
 
             try (RestClient leaderClient = buildLeaderClient()) {
                 index(leaderClient, indexName, "1");
@@ -171,9 +170,6 @@ public void testCCRUnfollowDuringSnapshot() throws Exception {
                 // assert that snapshot succeeded
                 assertThat(getSnapshotState("snapshot"), equalTo("SUCCESS"));
                 assertOK(client().performRequest(new Request("DELETE", "/_snapshot/repo/snapshot")));
-                ResponseException e = expectThrows(ResponseException.class,
-                    () -> client().performRequest(new Request("GET", "/_snapshot/repo/snapshot")));
-                assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404));
             }
         } else {
             fail("unexpected target cluster [" + targetCluster + "]");
@@ -226,8 +222,8 @@ public void testCcrAndIlmWithRollover() throws Exception {
                 // Check that it got replicated to the follower
                 assertBusy(() -> assertTrue(indexExists(indexName)));
 
-                // Aliases are not copied from leader index, so we need to add that for the rollover action in follower cluster:
-                client().performRequest(new Request("PUT", "/" + indexName + "/_alias/" + alias));
+                // check that the alias was replicated
+                assertBusy(() -> assertOK(client().performRequest(new Request("HEAD", "/" + indexName + "/_alias/" + alias))));
 
                 index(leaderClient, indexName, "1");
                 assertDocumentExists(leaderClient, indexName, "1");
@@ -252,7 +248,6 @@ public void testCcrAndIlmWithRollover() throws Exception {
                     // And the old index should have a write block and indexing complete set
                     assertThat(getIndexSetting(leaderClient, indexName, "index.blocks.write"), equalTo("true"));
                     assertThat(getIndexSetting(leaderClient, indexName, "index.lifecycle.indexing_complete"), equalTo("true"));
-
                 });
 
                 assertBusy(() -> {
@@ -266,6 +261,8 @@ public void testCcrAndIlmWithRollover() throws Exception {
                     assertThat(getIndexSetting(client(), indexName, "index.xpack.ccr.following_index"), nullValue());
                     // The next index should have been created on the follower as well
                     indexExists(nextIndexName);
+                    // and the alias should be on the next index
+                    assertOK(client().performRequest(new Request("HEAD", "/" + nextIndexName + "/_alias/" + alias)));
                 });
 
                 assertBusy(() -> {
@@ -281,6 +278,74 @@ public void testCcrAndIlmWithRollover() throws Exception {
         }
     }
 
+    public void testAliasReplicatedOnShrink() throws Exception {
+        final String indexName = "shrink-alias-test";
+        final String shrunkenIndexName = "shrink-" + indexName;
+        final String policyName = "shrink-test-policy";
+
+        final int numberOfAliases = randomIntBetween(0, 4);
+
+        if ("leader".equals(targetCluster)) {
+            Settings indexSettings = Settings.builder()
+                    .put("index.soft_deletes.enabled", true)
+                    .put("index.number_of_shards", 3)
+                    .put("index.number_of_replicas", 0)
+                    .put("index.lifecycle.name", policyName) // this policy won't exist on the leader, that's fine
+                    .build();
+            final StringBuilder aliases = new StringBuilder();
+            boolean first = true;
+            for (int i = 0; i < numberOfAliases; i++) {
+                if (first == false) {
+                    aliases.append(",");
+                }
+                final Boolean isWriteIndex = randomFrom(new Boolean[] { null, false, true });
+                if (isWriteIndex == null) {
+                    aliases.append("\"alias_").append(i).append("\":{}");
+                } else {
+                    aliases.append("\"alias_").append(i).append("\":{\"is_write_index\":").append(isWriteIndex).append("}");
+                }
+                first = false;
+            }
+            createIndex(indexName, indexSettings, "", aliases.toString());
+            ensureGreen(indexName);
+        } else if ("follow".equals(targetCluster)) {
+            // Create a policy with just a Shrink action on the follower
+            putShrinkOnlyPolicy(client(), policyName);
+
+            // Follow the index
+            followIndex(indexName, indexName);
+            // Make sure it actually took
+            assertBusy(() -> assertTrue(indexExists(indexName)));
+            // This should now be in the "warm" phase waiting for the index to be ready to unfollow
+            assertBusy(() -> assertILMPolicy(client(), indexName, policyName, "warm", "unfollow", "wait-for-indexing-complete"));
+
+            // Set the indexing_complete flag on the leader so the index will actually unfollow
+            try (RestClient leaderClient = buildLeaderClient()) {
+                updateIndexSettings(leaderClient, indexName, Settings.builder()
+                        .put("index.lifecycle.indexing_complete", true)
+                        .build()
+                );
+            }
+
+            // Wait for the setting to get replicated
+            assertBusy(() -> assertThat(getIndexSetting(client(), indexName, "index.lifecycle.indexing_complete"), equalTo("true")));
+
+            // Wait for the index to continue with its lifecycle and be shrunk
+            assertBusy(() -> assertTrue(indexExists(shrunkenIndexName)));
+
+            // assert the aliases were replicated
+            assertBusy(() -> {
+                for (int i = 0; i < numberOfAliases; i++) {
+                    assertOK(client().performRequest(new Request("HEAD", "/" + shrunkenIndexName + "/_alias/alias_" + i)));
+                }
+            });
+            assertBusy(() -> assertOK(client().performRequest(new Request("HEAD", "/" + shrunkenIndexName + "/_alias/" + indexName))));
+
+            // Wait for the index to complete its policy
+            assertBusy(() -> assertILMPolicy(client(), shrunkenIndexName, policyName, "completed", "completed", "completed"));
+        }
+    }
+
     public void testUnfollowInjectedBeforeShrink() throws Exception {
         final String indexName = "shrink-test";
         final String shrunkenIndexName = "shrink-" + indexName;
@@ -706,13 +771,16 @@ public static void updatePolicy(String indexName, String policy) throws IOExcept
         assertOK(client().performRequest(changePolicyRequest));
     }
 
+    @SuppressWarnings("unchecked")
     private String getSnapshotState(String snapshot) throws IOException {
         Response response = client().performRequest(new Request("GET", "/_snapshot/repo/" + snapshot));
         Map<String, Object> responseMap;
         try (InputStream is = response.getEntity().getContent()) {
             responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
         }
-        @SuppressWarnings("unchecked") Map<String, Object> snapResponse = ((List<Map<String, Object>>) responseMap.get("snapshots")).get(0);
+
+        Map<String, Object> repoResponse = ((List<Map<String, Object>>) responseMap.get("responses")).get(0);
+        Map<String, Object> snapResponse = ((List<Map<String, Object>>) repoResponse.get("snapshots")).get(0);
         assertThat(snapResponse.get("snapshot"), equalTo(snapshot));
         return (String) snapResponse.get("state");
     }
diff --git a/x-pack/plugin/ilm/qa/multi-node/build.gradle b/x-pack/plugin/ilm/qa/multi-node/build.gradle
index 5f033626932df..1a72ee5cf6f17 100644
--- a/x-pack/plugin/ilm/qa/multi-node/build.gradle
+++ b/x-pack/plugin/ilm/qa/multi-node/build.gradle
@@ -1,3 +1,6 @@
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -5,15 +8,18 @@ dependencies {
     testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
 }
 
-integTestRunner {
+File repoDir = file("$buildDir/testclusters/repo")
+
+integTest.runner {
     /* To support taking index snapshots, we have to set path.repo setting */
-    systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo")
+    systemProperty 'tests.path.repo', repoDir
 }
 
-integTestCluster {
-    numNodes = 4
-    clusterName = 'ilm'
+testClusters.integTest {
+    distribution = 'DEFAULT'
+    numberOfNodes = 4
     
+    setting 'path.repo', repoDir.absolutePath
     setting 'xpack.ilm.enabled', 'true'
     setting 'xpack.security.enabled', 'false'
     setting 'xpack.watcher.enabled', 'false'
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/ChangePolicyforIndexIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/ChangePolicyforIndexIT.java
index 84596c423b339..1b93fed4fe334 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/ChangePolicyforIndexIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/ChangePolicyforIndexIT.java
@@ -28,6 +28,7 @@
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -44,10 +45,10 @@ public class ChangePolicyforIndexIT extends ESRestTestCase {
      * to rollover and a warm phase with an impossible allocation action. The
      * second policy has a rollover action requiring 1000 document and a warm
      * phase that moves the index to known nodes that will succeed. An index is
-     * created with the fiorst policy set and the test ensures the policy is in
+     * created with the first policy set and the test ensures the policy is in
      * the rollover step. It then changes the policy for the index to the second
      * policy. It indexes a single document and checks that the index moves past
-     * the hot phase and through the warm phasee (proving the hot phase
+     * the hot phase and through the warm phase (proving the hot phase
      * definition from the first policy was used) and then checks the allocation
      * settings from the second policy are set ont he index (proving the second
      * policy was used for the warm phase)
@@ -63,7 +64,7 @@ public void testChangePolicyForIndex() throws Exception {
         Map<String, Phase> phases2 = new HashMap<>();
         phases2.put("hot", new Phase("hot", TimeValue.ZERO, singletonMap(RolloverAction.NAME, new RolloverAction(null, null, 1000L))));
         phases2.put("warm", new Phase("warm", TimeValue.ZERO,
-                singletonMap(AllocateAction.NAME, new AllocateAction(1, singletonMap("_name", "node-1,node-2"), null, null))));
+                singletonMap(AllocateAction.NAME, new AllocateAction(1, singletonMap("_name", "integTest-1,integTest-2"), null, null))));
         LifecyclePolicy lifecyclePolicy2 = new LifecyclePolicy("policy_1", phases2);
         // PUT policy_1 and policy_2
         XContentBuilder builder1 = jsonBuilder();
@@ -81,7 +82,7 @@ public void testChangePolicyForIndex() throws Exception {
 
         // create the test-index index and set the policy to policy_1
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4)
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put("index.routing.allocation.include._name", "node-0")
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put("index.routing.allocation.include._name", "integTest-0")
                 .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias").put(LifecycleSettings.LIFECYCLE_NAME, "policy_1").build();
         Request createIndexRequest = new Request("PUT", "/" + indexName);
         createIndexRequest.setJsonEntity(
@@ -112,9 +113,9 @@ public void testChangePolicyForIndex() throws Exception {
         assertOK(client().performRequest(request));
 
         // Check the index goes to the warm phase and completes
-        assertBusy(() -> assertStep(indexName, TerminalPolicyStep.KEY));
+        assertBusy(() -> assertStep(indexName, TerminalPolicyStep.KEY), 30, TimeUnit.SECONDS);
 
-        // Check index is allocated on node-1 and node-2 as per policy_2
+        // Check index is allocated on integTest-1 and integTest-2 as per policy_2
         Request getSettingsRequest = new Request("GET", "/" + indexName + "/_settings");
         Response getSettingsResponse = client().performRequest(getSettingsRequest);
         assertOK(getSettingsResponse);
@@ -127,7 +128,7 @@ public void testChangePolicyForIndex() throws Exception {
         @SuppressWarnings("unchecked")
         String includesAllocation = (String) ((Map<String, Object>) ((Map<String, Object>) routingSettings.get("allocation"))
                 .get("include")).get("_name");
-        assertEquals("node-1,node-2", includesAllocation);
+        assertEquals("integTest-1,integTest-2", includesAllocation);
     }
 
     private void assertStep(String indexName, StepKey expectedStep) throws IOException {
diff --git a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java
index b6b317e0c67ef..757869640ebad 100644
--- a/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java
+++ b/x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java
@@ -86,7 +86,7 @@ public void testFullPolicy() throws Exception {
         String secondIndex = index + "-000002";
         createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put("index.routing.allocation.include._name", "node-0")
+            .put("index.routing.allocation.include._name", "integTest-0")
             .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias"));
 
         // create policy
@@ -105,7 +105,7 @@ public void testFullPolicy() throws Exception {
         // asserts that rollover was called
         assertBusy(() -> assertTrue(indexExists(secondIndex)));
         // asserts that shrink deleted the original index
-        assertBusy(() -> assertFalse(indexExists(originalIndex)), 20, TimeUnit.SECONDS);
+        assertBusy(() -> assertFalse(indexExists(originalIndex)), 60, TimeUnit.SECONDS);
         // asserts that the delete phase completed for the managed shrunken index
         assertBusy(() -> assertFalse(indexExists(shrunkenOriginalIndex)));
     }
@@ -114,7 +114,7 @@ public void testMoveToAllocateStep() throws Exception {
         String originalIndex = index + "-000001";
         createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put("index.routing.allocation.include._name", "node-0")
+            .put("index.routing.allocation.include._name", "integTest-0")
             .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias"));
 
         // create policy
@@ -148,7 +148,7 @@ public void testMoveToRolloverStep() throws Exception {
         String secondIndex = index + "-000002";
         createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put("index.routing.allocation.include._name", "node-0")
+            .put("index.routing.allocation.include._name", "integTest-0")
             .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias"));
 
         createFullPolicy(TimeValue.timeValueHours(10));
@@ -313,7 +313,7 @@ public void testRolloverAlreadyExists() throws Exception {
     public void testAllocateOnlyAllocation() throws Exception {
         createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
-        String allocateNodeName = "node-" + randomFrom(0, 1);
+        String allocateNodeName = "integTest-" + randomFrom(0, 1);
         AllocateAction allocateAction = new AllocateAction(null, null, null, singletonMap("_name", allocateNodeName));
         createNewSingletonPolicy(randomFrom("warm", "cold"), allocateAction);
         updatePolicy(index, policy);
@@ -393,9 +393,6 @@ public void testDeleteDuringSnapshot() throws Exception {
         // assert that snapshot is still in progress and clean up
         assertThat(getSnapshotState("snapshot"), equalTo("SUCCESS"));
         assertOK(client().performRequest(new Request("DELETE", "/_snapshot/repo/snapshot")));
-        ResponseException e = expectThrows(ResponseException.class,
-            () -> client().performRequest(new Request("GET", "/_snapshot/repo/snapshot")));
-        assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404));
     }
 
     public void testReadOnly() throws Exception {
@@ -509,7 +506,7 @@ public void testShrinkDuringSnapshot() throws Exception {
             .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
             // required so the shrink doesn't wait on SetSingleNodeAllocateStep
-            .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_name", "node-0"));
+            .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_name", "integTest-0"));
         // index document so snapshot actually does something
         indexDocument();
         // start snapshot
@@ -533,9 +530,6 @@ public void testShrinkDuringSnapshot() throws Exception {
         // assert that snapshot succeeded
         assertThat(getSnapshotState("snapshot"), equalTo("SUCCESS"));
         assertOK(client().performRequest(new Request("DELETE", "/_snapshot/repo/snapshot")));
-        ResponseException e = expectThrows(ResponseException.class,
-            () -> client().performRequest(new Request("GET", "/_snapshot/repo/snapshot")));
-        assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404));
     }
 
     public void testFreezeAction() throws Exception {
@@ -591,9 +585,6 @@ public void testFreezeDuringSnapshot() throws Exception {
         // assert that snapshot is still in progress and clean up
         assertThat(getSnapshotState("snapshot"), equalTo("SUCCESS"));
         assertOK(client().performRequest(new Request("DELETE", "/_snapshot/repo/snapshot")));
-        ResponseException e = expectThrows(ResponseException.class,
-            () -> client().performRequest(new Request("GET", "/_snapshot/repo/snapshot")));
-        assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404));
     }
 
     public void testSetPriority() throws Exception {
@@ -846,11 +837,11 @@ private void createFullPolicy(TimeValue hotTime) throws IOException {
         Map<String, LifecycleAction> warmActions = new HashMap<>();
         warmActions.put(SetPriorityAction.NAME, new SetPriorityAction(50));
         warmActions.put(ForceMergeAction.NAME, new ForceMergeAction(1));
-        warmActions.put(AllocateAction.NAME, new AllocateAction(1, singletonMap("_name", "node-1,node-2"), null, null));
+        warmActions.put(AllocateAction.NAME, new AllocateAction(1, singletonMap("_name", "integTest-1,integTest-2"), null, null));
         warmActions.put(ShrinkAction.NAME, new ShrinkAction(1));
         Map<String, LifecycleAction> coldActions = new HashMap<>();
         coldActions.put(SetPriorityAction.NAME, new SetPriorityAction(0));
-        coldActions.put(AllocateAction.NAME, new AllocateAction(0, singletonMap("_name", "node-3"), null, null));
+        coldActions.put(AllocateAction.NAME, new AllocateAction(0, singletonMap("_name", "integTest-3"), null, null));
         Map<String, Phase> phases = new HashMap<>();
         phases.put("hot", new Phase("hot", hotTime, hotActions));
         phases.put("warm", new Phase("warm", TimeValue.ZERO, warmActions));
@@ -977,13 +968,16 @@ private void indexDocument() throws IOException {
         logger.info(response.getStatusLine());
     }
 
+    @SuppressWarnings("unchecked")
     private String getSnapshotState(String snapshot) throws IOException {
         Response response = client().performRequest(new Request("GET", "/_snapshot/repo/" + snapshot));
         Map<String, Object> responseMap;
         try (InputStream is = response.getEntity().getContent()) {
             responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
         }
-        Map<String, Object> snapResponse = ((List<Map<String, Object>>) responseMap.get("snapshots")).get(0);
+
+        Map<String, Object> repoResponse = ((List<Map<String, Object>>) responseMap.get("responses")).get(0);
+        Map<String, Object> snapResponse = ((List<Map<String, Object>>) repoResponse.get("snapshots")).get(0);
         assertThat(snapResponse.get("snapshot"), equalTo(snapshot));
         return (String) snapResponse.get("state");
     }
diff --git a/x-pack/plugin/ilm/qa/rest/build.gradle b/x-pack/plugin/ilm/qa/rest/build.gradle
index 75ba234e55569..8fbe1fbdff07e 100644
--- a/x-pack/plugin/ilm/qa/rest/build.gradle
+++ b/x-pack/plugin/ilm/qa/rest/build.gradle
@@ -1,5 +1,6 @@
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
@@ -10,38 +11,25 @@ dependencies {
     testCompile project(path: xpackModule('ilm'), configuration: 'runtime')
 }
 
-task restTest(type: RestIntegTestTask) {
-    mustRunAfter(precommit)
-}
-
 def clusterCredentials = [username: System.getProperty('tests.rest.cluster.username', 'test_admin'),
                           password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')]
 
-
-restTestRunner {
-    systemProperty 'tests.rest.cluster.username', clusterCredentials.username
-    systemProperty 'tests.rest.cluster.password', clusterCredentials.password
+task restTest(type: RestIntegTestTask) {
+    mustRunAfter(precommit)
+    runner {
+        systemProperty 'tests.rest.cluster.username', clusterCredentials.username
+        systemProperty 'tests.rest.cluster.password', clusterCredentials.password
+    }
 }
 
-restTestCluster {
-    distribution 'default'
+testClusters.restTest {
+    distribution = 'DEFAULT'
     setting 'xpack.ilm.enabled', 'true'
     setting 'xpack.ml.enabled', 'false'
     setting 'xpack.monitoring.enabled', 'false'
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.license.self_generated.type', 'trial'
-    setupCommand 'setup-admin-user',
-            'bin/elasticsearch-users', 'useradd', clusterCredentials.username, '-p', clusterCredentials.password, '-r', 'superuser'
-    waitCondition = { node, ant ->
-        File tmpFile = new File(node.cwd, 'wait.success')
-        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-                dest: tmpFile.toString(),
-                username: clusterCredentials.username,
-                password: clusterCredentials.password,
-                ignoreerrors: true,
-                retries: 10)
-        return tmpFile.exists()
-    }
+    user clusterCredentials
 }
 
 check.dependsOn restTest
diff --git a/x-pack/plugin/ilm/qa/with-security/build.gradle b/x-pack/plugin/ilm/qa/with-security/build.gradle
index f1b972012e7f8..84685c3da7c88 100644
--- a/x-pack/plugin/ilm/qa/with-security/build.gradle
+++ b/x-pack/plugin/ilm/qa/with-security/build.gradle
@@ -1,3 +1,4 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -8,36 +9,22 @@ dependencies {
 def clusterCredentials = [username: System.getProperty('tests.rest.cluster.username', 'test_admin'),
                           password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')]
 
-integTestRunner {
-  systemProperty 'tests.rest.cluster.username', clusterCredentials.username
-  systemProperty 'tests.rest.cluster.password', clusterCredentials.password
+integTest {
+  runner {
+    systemProperty 'tests.rest.cluster.username', clusterCredentials.username
+    systemProperty 'tests.rest.cluster.password', clusterCredentials.password
+  }
 }
 
-integTestCluster {
+testClusters.integTest {
+  distribution = "DEFAULT"
   setting 'xpack.ilm.enabled', 'true'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.watcher.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
-  extraConfigFile 'roles.yml', 'roles.yml'
-  setupCommand 'setupIlmUser',
-          'bin/elasticsearch-users',
-          'useradd', "test_ilm",
-          '-p', 'x-pack-test-password', '-r', "ilm"
-  setupCommand 'setupDummyUser',
-               'bin/elasticsearch-users',
-               'useradd', clusterCredentials.username,
-               '-p', clusterCredentials.password,
-               '-r', 'superuser'
-  waitCondition = { node, ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-            dest: tmpFile.toString(),
-            username: clusterCredentials.username,
-            password: clusterCredentials.password,
-            ignoreerrors: true,
-            retries: 10)
-    return tmpFile.exists()
-  }
+  extraConfigFile 'roles.yml', file('roles.yml')
+  user clusterCredentials
+  user username: "test_ilm", password: "x-pack-test-password", role: "ilm"
 }
diff --git a/x-pack/plugin/ilm/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java b/x-pack/plugin/ilm/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java
index 8e74426e0b390..d8c50c470664e 100644
--- a/x-pack/plugin/ilm/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java
+++ b/x-pack/plugin/ilm/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.core.indexlifecycle.DeleteAction;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
@@ -137,7 +138,7 @@ public void testCanViewExplainOnUnmanagedIndex() throws Exception {
      * Tests when the user is limited by alias of an index is able to write to index
      * which was rolled over by an ILM policy.
      */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/41440")
+    @TestLogging("org.elasticsearch:DEBUG")
     public void testWhenUserLimitedByOnlyAliasOfIndexCanWriteToIndexWhichWasRolledoverByILMPolicy()
             throws IOException, InterruptedException {
         /*
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java
index 0b6c6001ca0ef..2a03a1c834fac 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycle.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -32,8 +31,9 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
-import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.indexlifecycle.AllocateAction;
 import org.elasticsearch.xpack.core.indexlifecycle.DeleteAction;
 import org.elasticsearch.xpack.core.indexlifecycle.ForceMergeAction;
@@ -42,9 +42,9 @@
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleType;
-import org.elasticsearch.xpack.core.indexlifecycle.SetPriorityAction;
 import org.elasticsearch.xpack.core.indexlifecycle.ReadOnlyAction;
 import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction;
+import org.elasticsearch.xpack.core.indexlifecycle.SetPriorityAction;
 import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction;
 import org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType;
 import org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction;
@@ -80,7 +80,6 @@
 import org.elasticsearch.xpack.indexlifecycle.action.TransportStopILMAction;
 
 import java.time.Clock;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -104,12 +103,6 @@ protected Clock getClock() {
         return Clock.systemUTC();
     }
 
-    public Collection<Module> createGuiceModules() {
-        List<Module> modules = new ArrayList<>();
-        modules.add(b -> XPackPlugin.bindFeatureSet(b, IndexLifecycleFeatureSet.class));
-        return modules;
-    }
-
     @Override
     public List<Setting<?>> getSettings() {
         return Arrays.asList(
@@ -182,8 +175,12 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
 
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        var usageAction =
+            new ActionHandler<>(XPackUsageFeatureAction.INDEX_LIFECYCLE, IndexLifecycleUsageTransportAction.class);
+        var infoAction =
+            new ActionHandler<>(XPackInfoFeatureAction.INDEX_LIFECYCLE, IndexLifecycleInfoTransportAction.class);
         if (enabled == false) {
-            return emptyList();
+            return Arrays.asList(usageAction, infoAction);
         }
         return Arrays.asList(
                 new ActionHandler<>(PutLifecycleAction.INSTANCE, TransportPutLifecycleAction.class),
@@ -195,7 +192,9 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
                 new ActionHandler<>(RetryAction.INSTANCE, TransportRetryAction.class),
                 new ActionHandler<>(StartILMAction.INSTANCE, TransportStartILMAction.class),
                 new ActionHandler<>(StopILMAction.INSTANCE, TransportStopILMAction.class),
-                new ActionHandler<>(GetStatusAction.INSTANCE, TransportGetStatusAction.class));
+                new ActionHandler<>(GetStatusAction.INSTANCE, TransportGetStatusAction.class),
+                usageAction,
+                infoAction);
     }
 
     @Override
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSet.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSet.java
deleted file mode 100644
index 2469621316889..0000000000000
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSet.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.indexlifecycle;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.XPackField;
-import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage;
-import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage.PhaseStats;
-import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage.PolicyStats;
-import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
-import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-public class IndexLifecycleFeatureSet implements XPackFeatureSet {
-
-    private final boolean enabled;
-    private final XPackLicenseState licenseState;
-    private ClusterService clusterService;
-
-    @Inject
-    public IndexLifecycleFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState, ClusterService clusterService) {
-        this.clusterService = clusterService;
-        this.enabled = XPackSettings.INDEX_LIFECYCLE_ENABLED.get(settings);
-        this.licenseState = licenseState;
-    }
-
-    @Override
-    public String name() {
-        return XPackField.INDEX_LIFECYCLE;
-    }
-
-    @Override
-    public String description() {
-        return "Index lifecycle management for the Elastic Stack";
-    }
-
-    @Override
-    public boolean available() {
-        return licenseState != null && licenseState.isIndexLifecycleAllowed();
-    }
-
-    @Override
-    public boolean enabled() {
-        return enabled;
-    }
-
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return null;
-    }
-
-    @Override
-    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
-        MetaData metaData = clusterService.state().metaData();
-        IndexLifecycleMetadata lifecycleMetadata = metaData.custom(IndexLifecycleMetadata.TYPE);
-        if (enabled() && lifecycleMetadata != null) {
-            Map<String, Integer> policyUsage = new HashMap<>();
-            metaData.indices().forEach(entry -> {
-                String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(entry.value.getSettings());
-                Integer indicesManaged = policyUsage.get(policyName);
-                if (indicesManaged == null) {
-                    indicesManaged = 1;
-                } else {
-                    indicesManaged = indicesManaged + 1;
-                }
-                policyUsage.put(policyName, indicesManaged);
-            });
-            List<PolicyStats> policyStats = lifecycleMetadata.getPolicies().values().stream().map(policy -> {
-                Map<String, PhaseStats> phaseStats = policy.getPhases().values().stream().map(phase -> {
-                    String[] actionNames = phase.getActions().keySet().toArray(new String[phase.getActions().size()]);
-                    return new Tuple<String, PhaseStats>(phase.getName(), new PhaseStats(phase.getMinimumAge(), actionNames));
-                }).collect(Collectors.toMap(Tuple::v1, Tuple::v2));
-                return new PolicyStats(phaseStats, policyUsage.getOrDefault(policy.getName(), 0));
-            }).collect(Collectors.toList());
-            listener.onResponse(new IndexLifecycleFeatureSetUsage(available(), enabled(), policyStats));
-        } else {
-            listener.onResponse(new IndexLifecycleFeatureSetUsage(available(), enabled()));
-        }
-    }
-
-}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInfoTransportAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInfoTransportAction.java
new file mode 100644
index 0000000000000..65fa28458e759
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInfoTransportAction.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
+
+public class IndexLifecycleInfoTransportAction extends XPackInfoFeatureTransportAction {
+
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public IndexLifecycleInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                             Settings settings, XPackLicenseState licenseState) {
+        super(XPackInfoFeatureAction.INDEX_LIFECYCLE.name(), transportService, actionFilters);
+        this.enabled = XPackSettings.INDEX_LIFECYCLE_ENABLED.get(settings);
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    public String name() {
+        return XPackField.INDEX_LIFECYCLE;
+    }
+
+    @Override
+    public boolean available() {
+        return licenseState.isIndexLifecycleAllowed();
+    }
+
+    @Override
+    public boolean enabled() {
+        return enabled;
+    }
+
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java
index 2d09d0184f489..a159b9f965e2a 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleService.java
@@ -29,7 +29,7 @@
 import org.elasticsearch.xpack.core.indexlifecycle.LifecyclePolicy;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
 import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
-import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction;
+import org.elasticsearch.xpack.core.indexlifecycle.ShrinkStep;
 import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
 import org.elasticsearch.xpack.core.scheduler.SchedulerEngine;
 
@@ -45,7 +45,7 @@
 public class IndexLifecycleService
     implements ClusterStateListener, ClusterStateApplier, SchedulerEngine.Listener, Closeable, LocalNodeMasterListener {
     private static final Logger logger = LogManager.getLogger(IndexLifecycleService.class);
-    private static final Set<String> IGNORE_ACTIONS_MAINTENANCE_REQUESTED = Collections.singleton(ShrinkAction.NAME);
+    private static final Set<String> IGNORE_STEPS_MAINTENANCE_REQUESTED = Collections.singleton(ShrinkStep.NAME);
     private volatile boolean isMaster = false;
     private volatile TimeValue pollInterval;
 
@@ -115,15 +115,15 @@ public void onMaster() {
                     StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
 
                     if (OperationMode.STOPPING == currentMode) {
-                        if (stepKey != null && IGNORE_ACTIONS_MAINTENANCE_REQUESTED.contains(stepKey.getAction())) {
-                            logger.info("waiting to stop ILM because index [{}] with policy [{}] is currently in action [{}]",
-                                idxMeta.getIndex().getName(), policyName, stepKey.getAction());
+                        if (stepKey != null && IGNORE_STEPS_MAINTENANCE_REQUESTED.contains(stepKey.getName())) {
+                            logger.info("waiting to stop ILM because index [{}] with policy [{}] is currently in step [{}]",
+                                idxMeta.getIndex().getName(), policyName, stepKey.getName());
                             lifecycleRunner.maybeRunAsyncAction(clusterState, idxMeta, policyName, stepKey);
-                            // ILM is trying to stop, but this index is in a Shrink action (or other dangerous action) so we can't stop
+                            // ILM is trying to stop, but this index is in a Shrink step (or other dangerous step) so we can't stop
                             safeToStop = false;
                         } else {
-                            logger.info("skipping policy execution for index [{}] with policy [{}] because ILM is stopping",
-                                idxMeta.getIndex().getName(), policyName);
+                            logger.info("skipping policy execution of step [{}] for index [{}] with policy [{}] because ILM is stopping",
+                                stepKey == null ? "n/a" : stepKey.getName(), idxMeta.getIndex().getName(), policyName);
                         }
                     } else {
                         lifecycleRunner.maybeRunAsyncAction(clusterState, idxMeta, policyName, stepKey);
@@ -249,19 +249,19 @@ void triggerPolicies(ClusterState clusterState, boolean fromClusterStateChange)
                 StepKey stepKey = IndexLifecycleRunner.getCurrentStepKey(lifecycleState);
 
                 if (OperationMode.STOPPING == currentMode) {
-                    if (stepKey != null && IGNORE_ACTIONS_MAINTENANCE_REQUESTED.contains(stepKey.getAction())) {
-                        logger.info("waiting to stop ILM because index [{}] with policy [{}] is currently in action [{}]",
-                            idxMeta.getIndex().getName(), policyName, stepKey.getAction());
+                    if (stepKey != null && IGNORE_STEPS_MAINTENANCE_REQUESTED.contains(stepKey.getName())) {
+                        logger.info("waiting to stop ILM because index [{}] with policy [{}] is currently in step [{}]",
+                            idxMeta.getIndex().getName(), policyName, stepKey.getName());
                         if (fromClusterStateChange) {
                             lifecycleRunner.runPolicyAfterStateChange(policyName, idxMeta);
                         } else {
                             lifecycleRunner.runPeriodicStep(policyName, idxMeta);
                         }
-                        // ILM is trying to stop, but this index is in a Shrink action (or other dangerous action) so we can't stop
+                        // ILM is trying to stop, but this index is in a Shrink step (or other dangerous step) so we can't stop
                         safeToStop = false;
                     } else {
-                        logger.info("skipping policy execution for index [{}] with policy [{}] because ILM is stopping",
-                            idxMeta.getIndex().getName(), policyName);
+                        logger.info("skipping policy execution of step [{}] for index [{}] with policy [{}] because ILM is stopping",
+                            stepKey == null ? "n/a" : stepKey.getName(), idxMeta.getIndex().getName(), policyName);
                     }
                 } else {
                     if (fromClusterStateChange) {
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleUsageTransportAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleUsageTransportAction.java
new file mode 100644
index 0000000000000..91fd744d9aabb
--- /dev/null
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleUsageTransportAction.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.indexlifecycle;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage;
+import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
+import org.elasticsearch.xpack.core.indexlifecycle.LifecycleSettings;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class IndexLifecycleUsageTransportAction extends XPackUsageFeatureTransportAction {
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public IndexLifecycleUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                              ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                              Settings settings, XPackLicenseState licenseState) {
+        super(XPackUsageFeatureAction.INDEX_LIFECYCLE.name(), transportService, clusterService, threadPool, actionFilters,
+            indexNameExpressionResolver);
+        this.enabled = XPackSettings.INDEX_LIFECYCLE_ENABLED.get(settings);
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
+        boolean available = licenseState.isIndexLifecycleAllowed();
+        MetaData metaData = state.metaData();
+        IndexLifecycleMetadata lifecycleMetadata = metaData.custom(IndexLifecycleMetadata.TYPE);
+        final IndexLifecycleFeatureSetUsage usage;
+        if (enabled && lifecycleMetadata != null) {
+            Map<String, Integer> policyUsage = new HashMap<>();
+            metaData.indices().forEach(entry -> {
+                String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(entry.value.getSettings());
+                Integer indicesManaged = policyUsage.get(policyName);
+                if (indicesManaged == null) {
+                    indicesManaged = 1;
+                } else {
+                    indicesManaged = indicesManaged + 1;
+                }
+                policyUsage.put(policyName, indicesManaged);
+            });
+            List<IndexLifecycleFeatureSetUsage.PolicyStats> policyStats = lifecycleMetadata.getPolicies().values().stream().map(policy -> {
+                Map<String, IndexLifecycleFeatureSetUsage.PhaseStats> phaseStats = policy.getPhases().values().stream().map(phase -> {
+                    String[] actionNames = phase.getActions().keySet().toArray(new String[phase.getActions().size()]);
+                    return new Tuple<>(phase.getName(), new IndexLifecycleFeatureSetUsage.PhaseStats(phase.getMinimumAge(), actionNames));
+                }).collect(Collectors.toMap(Tuple::v1, Tuple::v2));
+                return new IndexLifecycleFeatureSetUsage.PolicyStats(phaseStats, policyUsage.getOrDefault(policy.getName(), 0));
+            }).collect(Collectors.toList());
+            usage = new IndexLifecycleFeatureSetUsage(available, enabled, policyStats);
+        } else {
+            usage = new IndexLifecycleFeatureSetUsage(available, enabled);
+        }
+        listener.onResponse(new XPackUsageFeatureResponse(usage));
+    }
+}
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/LifecyclePolicySecurityClient.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/LifecyclePolicySecurityClient.java
index b923700d25f68..e116223304e4e 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/LifecyclePolicySecurityClient.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/LifecyclePolicySecurityClient.java
@@ -6,7 +6,7 @@
 
 package org.elasticsearch.xpack.indexlifecycle;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -46,8 +46,8 @@ public void close() {
     }
 
     @Override
-    protected <Request extends ActionRequest, Response extends ActionResponse> void doExecute(Action<Response> action, Request request,
-            ActionListener<Response> listener) {
+    protected <Request extends ActionRequest, Response extends ActionResponse> void doExecute(ActionType<Response> action, Request request,
+                                                                                              ActionListener<Response> listener) {
         ClientHelper.executeWithHeadersAsync(headers, origin, client, action, request, listener);
     }
 
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportDeleteLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportDeleteLifecycleAction.java
index 13232649e0df5..c35e3495c05a5 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportDeleteLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportDeleteLifecycleAction.java
@@ -20,6 +20,8 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
@@ -28,6 +30,7 @@
 import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction.Request;
 import org.elasticsearch.xpack.core.indexlifecycle.action.DeleteLifecycleAction.Response;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.SortedMap;
 import java.util.Spliterator;
@@ -53,11 +56,16 @@ protected String executor() {
 
     @Override
     protected Response newResponse() {
-        return new Response();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) throws Exception {
+    protected Response read(StreamInput in) throws IOException {
+        return new Response(in);
+    }
+
+    @Override
+    protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<Response> listener) throws Exception {
         clusterService.submitStateUpdateTask("delete-lifecycle-" + request.getPolicyName(),
                 new AckedClusterStateUpdateTask<Response>(request, listener) {
                     @Override
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportGetLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportGetLifecycleAction.java
index 4c3657b76cba3..53fcba9cdd6d0 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportGetLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportGetLifecycleAction.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
@@ -50,7 +51,7 @@ protected Response newResponse() {
     }
 
     @Override
-    protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) {
+    protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<Response> listener) {
         IndexLifecycleMetadata metadata = clusterService.state().metaData().custom(IndexLifecycleMetadata.TYPE);
         if (metadata == null) {
             if (request.getPolicyNames().length == 0) {
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportGetStatusAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportGetStatusAction.java
index 4a76120545087..d0a9cb0de42e3 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportGetStatusAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportGetStatusAction.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
@@ -43,7 +44,7 @@ protected Response newResponse() {
     }
 
     @Override
-    protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) {
+    protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<Response> listener) {
         IndexLifecycleMetadata metadata = state.metaData().custom(IndexLifecycleMetadata.TYPE);
         final Response response;
         if (metadata == null) {
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportMoveToStepAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportMoveToStepAction.java
index e5cb15f6c3b24..782baa2828b63 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportMoveToStepAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportMoveToStepAction.java
@@ -19,6 +19,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.indexlifecycle.action.MoveToStepAction;
@@ -26,6 +28,8 @@
 import org.elasticsearch.xpack.core.indexlifecycle.action.MoveToStepAction.Response;
 import org.elasticsearch.xpack.indexlifecycle.IndexLifecycleService;
 
+import java.io.IOException;
+
 public class TransportMoveToStepAction extends TransportMasterNodeAction<Request, Response> {
     private static final Logger logger = LogManager.getLogger(TransportMoveToStepAction.class);
 
@@ -46,11 +50,16 @@ protected String executor() {
 
     @Override
     protected Response newResponse() {
-        return new Response();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected Response read(StreamInput in) throws IOException {
+        return new Response(in);
     }
 
     @Override
-    protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) {
+    protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<Response> listener) {
         IndexMetaData indexMetaData = state.metaData().index(request.getIndex());
         if (indexMetaData == null) {
             listener.onFailure(new IllegalArgumentException("index [" + request.getIndex() + "] does not exist"));
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportPutLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportPutLifecycleAction.java
index d94de3e201a46..69c2aa0e827f8 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportPutLifecycleAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportPutLifecycleAction.java
@@ -17,6 +17,8 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ClientHelper;
@@ -27,6 +29,7 @@
 import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction.Request;
 import org.elasticsearch.xpack.core.indexlifecycle.action.PutLifecycleAction.Response;
 
+import java.io.IOException;
 import java.time.Instant;
 import java.util.Map;
 import java.util.SortedMap;
@@ -53,11 +56,16 @@ protected String executor() {
 
     @Override
     protected Response newResponse() {
-        return new Response();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) {
+    protected Response read(StreamInput in) throws IOException {
+        return new Response(in);
+    }
+
+    @Override
+    protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<Response> listener) {
         // headers from the thread context stored by the AuthenticationService to be shared between the
         // REST layer and the Transport layer here must be accessed within this thread and not in the
         // cluster state thread in the ClusterStateUpdateTask below since that thread does not share the
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportRemoveIndexLifecyclePolicyAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportRemoveIndexLifecyclePolicyAction.java
index 4e608b511d3e9..03a4a7e514009 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportRemoveIndexLifecyclePolicyAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportRemoveIndexLifecyclePolicyAction.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.indexlifecycle.action.RemoveIndexLifecyclePolicyAction;
@@ -53,7 +54,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state)
     }
 
     @Override
-    protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) throws Exception {
+    protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<Response> listener) throws Exception {
         final Index[] indices = indexNameExpressionResolver.concreteIndices(state, request.indicesOptions(), request.indices());
         clusterService.submitStateUpdateTask("remove-lifecycle-for-index",
                 new AckedClusterStateUpdateTask<Response>(request, listener) {
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportRetryAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportRetryAction.java
index c9fabe8225714..5d90b388473fd 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportRetryAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportRetryAction.java
@@ -17,6 +17,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.indexlifecycle.LifecycleExecutionState;
@@ -26,6 +28,8 @@
 import org.elasticsearch.xpack.core.indexlifecycle.action.RetryAction.Response;
 import org.elasticsearch.xpack.indexlifecycle.IndexLifecycleService;
 
+import java.io.IOException;
+
 public class TransportRetryAction extends TransportMasterNodeAction<Request, Response> {
 
     IndexLifecycleService indexLifecycleService;
@@ -46,11 +50,16 @@ protected String executor() {
 
     @Override
     protected Response newResponse() {
-        return new Response();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected Response read(StreamInput in) throws IOException {
+        return new Response(in);
     }
 
     @Override
-    protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) {
+    protected void masterOperation(Task task, Request request, ClusterState state, ActionListener<Response> listener) {
         clusterService.submitStateUpdateTask("ilm-re-run",
             new AckedClusterStateUpdateTask<Response>(request, listener) {
                 @Override
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportStartILMAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportStartILMAction.java
index aab9ef90e03d5..b4345566a572a 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportStartILMAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportStartILMAction.java
@@ -17,6 +17,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
@@ -24,6 +26,8 @@
 import org.elasticsearch.xpack.core.indexlifecycle.action.StartILMAction;
 import org.elasticsearch.xpack.indexlifecycle.OperationModeUpdateTask;
 
+import java.io.IOException;
+
 public class TransportStartILMAction extends TransportMasterNodeAction<StartILMRequest, AcknowledgedResponse> {
 
     @Inject
@@ -38,13 +42,18 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(StartILMRequest request, ClusterState state, ActionListener<AcknowledgedResponse> listener) {
+    protected void masterOperation(Task task, StartILMRequest request, ClusterState state, ActionListener<AcknowledgedResponse> listener) {
         clusterService.submitStateUpdateTask("ilm_operation_mode_update",
                 new AckedClusterStateUpdateTask<AcknowledgedResponse>(request, listener) {
                 @Override
diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportStopILMAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportStopILMAction.java
index f27373353ecda..2ffa519c9c183 100644
--- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportStopILMAction.java
+++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/indexlifecycle/action/TransportStopILMAction.java
@@ -17,6 +17,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
@@ -24,6 +26,8 @@
 import org.elasticsearch.xpack.core.indexlifecycle.action.StopILMAction;
 import org.elasticsearch.xpack.indexlifecycle.OperationModeUpdateTask;
 
+import java.io.IOException;
+
 public class TransportStopILMAction extends TransportMasterNodeAction<StopILMRequest, AcknowledgedResponse> {
 
     @Inject
@@ -38,13 +42,18 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(StopILMRequest request, ClusterState state, ActionListener<AcknowledgedResponse> listener) {
+    protected void masterOperation(Task task, StopILMRequest request, ClusterState state, ActionListener<AcknowledgedResponse> listener) {
         clusterService.submitStateUpdateTask("ilm_operation_mode_update",
                 new AckedClusterStateUpdateTask<AcknowledgedResponse>(request, listener) {
                 @Override
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSetTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInfoTransportActionTests.java
similarity index 79%
rename from x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSetTests.java
rename to x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInfoTransportActionTests.java
index d83a41b4e60bc..7265c5d6ff6be 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleFeatureSetTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInfoTransportActionTests.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.indexlifecycle;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
@@ -16,6 +17,8 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleFeatureSetUsage.PolicyStats;
 import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata;
@@ -39,7 +42,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class IndexLifecycleFeatureSetTests extends ESTestCase {
+public class IndexLifecycleInfoTransportActionTests extends ESTestCase {
 
     private XPackLicenseState licenseState;
     private ClusterService clusterService;
@@ -51,38 +54,34 @@ public void init() throws Exception {
     }
 
     public void testAvailable() {
-        IndexLifecycleFeatureSet featureSet = new IndexLifecycleFeatureSet(Settings.EMPTY, licenseState, clusterService);
+        IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
 
         when(licenseState.isIndexLifecycleAllowed()).thenReturn(false);
         assertThat(featureSet.available(), equalTo(false));
 
         when(licenseState.isIndexLifecycleAllowed()).thenReturn(true);
         assertThat(featureSet.available(), equalTo(true));
-
-        featureSet = new IndexLifecycleFeatureSet(Settings.EMPTY, null, clusterService);
-        assertThat(featureSet.available(), equalTo(false));
     }
 
     public void testEnabled() {
         Settings.Builder settings = Settings.builder().put("xpack.ilm.enabled", false);
-        IndexLifecycleFeatureSet featureSet = new IndexLifecycleFeatureSet(settings.build(), licenseState, clusterService);
+        IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState);
         assertThat(featureSet.enabled(), equalTo(false));
 
         settings = Settings.builder().put("xpack.ilm.enabled", true);
-        featureSet = new IndexLifecycleFeatureSet(settings.build(), licenseState, clusterService);
+        featureSet = new IndexLifecycleInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState);
         assertThat(featureSet.enabled(), equalTo(true));
     }
 
     public void testName() {
-        IndexLifecycleFeatureSet featureSet = new IndexLifecycleFeatureSet(Settings.EMPTY, licenseState, clusterService);
+        IndexLifecycleInfoTransportAction featureSet = new IndexLifecycleInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
         assertThat(featureSet.name(), equalTo("ilm"));
     }
 
-    public void testNativeCodeInfo() {
-        IndexLifecycleFeatureSet featureSet = new IndexLifecycleFeatureSet(Settings.EMPTY, licenseState, clusterService);
-        assertNull(featureSet.nativeCodeInfo());
-    }
-
     public void testUsageStats() throws Exception {
         Map<String, String> indexPolicies = new HashMap<>();
         List<LifecyclePolicy> policies = new ArrayList<>();
@@ -110,12 +109,13 @@ public void testUsageStats() throws Exception {
         ClusterState clusterState = buildClusterState(policies, indexPolicies);
         Mockito.when(clusterService.state()).thenReturn(clusterState);
 
-        PlainActionFuture<IndexLifecycleFeatureSet.Usage> future = new PlainActionFuture<>();
-        IndexLifecycleFeatureSet ilmFeatureSet = new IndexLifecycleFeatureSet(Settings.EMPTY, licenseState, clusterService);
-        ilmFeatureSet.usage(future);
-        IndexLifecycleFeatureSetUsage ilmUsage = (IndexLifecycleFeatureSetUsage) future.get();
-        assertThat(ilmUsage.enabled(), equalTo(ilmFeatureSet.enabled()));
-        assertThat(ilmUsage.available(), equalTo(ilmFeatureSet.available()));
+        var usageAction = new IndexLifecycleUsageTransportAction(mock(TransportService.class), null, null,
+            mock(ActionFilters.class), null, Settings.EMPTY, licenseState);
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, clusterState, future);
+        IndexLifecycleFeatureSetUsage ilmUsage = (IndexLifecycleFeatureSetUsage) future.get().getUsage();
+        assertThat(ilmUsage.enabled(), equalTo(true));
+        assertThat(ilmUsage.available(), equalTo(false));
 
         List<PolicyStats> policyStatsList = ilmUsage.getPolicyStats();
         assertThat(policyStatsList.size(), equalTo(policies.size()));
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationTests.java
index cc23579547ed0..6d6727174bc72 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleInitialisationTests.java
@@ -168,7 +168,7 @@ public void testSingleNodeCluster() throws Exception {
         RoutingNode routingNodeEntry1 = clusterState.getRoutingNodes().node(node1);
         assertThat(routingNodeEntry1.numberOfShardsWithState(STARTED), equalTo(1));
         assertBusy(() -> {
-            assertEquals(true, client().admin().indices().prepareExists("test").get().isExists());
+            assertTrue(indexExists("test"));
         });
         IndexLifecycleService indexLifecycleService = internalCluster().getInstance(IndexLifecycleService.class, server_1);
         assertThat(indexLifecycleService.getScheduler().jobCount(), equalTo(1));
@@ -267,9 +267,7 @@ public void testMasterDedicatedDataDedicated() throws Exception {
         RoutingNode routingNodeEntry1 = clusterState.getRoutingNodes().node(node2);
         assertThat(routingNodeEntry1.numberOfShardsWithState(STARTED), equalTo(1));
 
-        assertBusy(() -> {
-            assertEquals(true, client().admin().indices().prepareExists("test").get().isExists());
-        });
+        assertBusy(() -> assertTrue(indexExists("test")));
         assertBusy(() -> {
             LifecycleExecutionState lifecycleState = LifecycleExecutionState.fromIndexMetadata(client().admin().cluster()
                 .prepareState().execute().actionGet().getState().getMetaData().index("test"));
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleMetadataTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleMetadataTests.java
index 2444cbf99fd52..65f8399a6bd76 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleMetadataTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleMetadataTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractDiffableSerializationTestCase;
+import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.xpack.core.indexlifecycle.AllocateAction;
 import org.elasticsearch.xpack.core.indexlifecycle.DeleteAction;
 import org.elasticsearch.xpack.core.indexlifecycle.ForceMergeAction;
@@ -137,7 +138,8 @@ protected Reader<Diff<Custom>> diffReader() {
     }
 
     public void testMinimumSupportedVersion() {
-        assertEquals(Version.V_6_6_0, createTestInstance().getMinimalSupportedVersion());
+        Version min = createTestInstance().getMinimalSupportedVersion();
+        assertTrue(min.onOrBefore(VersionUtils.randomCompatibleVersion(random(), Version.CURRENT)));
     }
 
     public void testcontext() {
diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleServiceTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleServiceTests.java
index 67affe9f74ce0..3757c1cd5fb4f 100644
--- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleServiceTests.java
+++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/IndexLifecycleServiceTests.java
@@ -37,6 +37,7 @@
 import org.elasticsearch.xpack.core.indexlifecycle.OperationMode;
 import org.elasticsearch.xpack.core.indexlifecycle.Phase;
 import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction;
+import org.elasticsearch.xpack.core.indexlifecycle.ShrinkStep;
 import org.elasticsearch.xpack.core.indexlifecycle.Step;
 import org.elasticsearch.xpack.core.scheduler.SchedulerEngine;
 import org.junit.After;
@@ -58,6 +59,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -148,7 +150,7 @@ public void testStoppedModeSkip() {
     }
 
     public void testRequestedStopOnShrink() {
-        Step.StepKey mockShrinkStep = new Step.StepKey(randomAlphaOfLength(4), ShrinkAction.NAME, randomAlphaOfLength(5));
+        Step.StepKey mockShrinkStep = new Step.StepKey(randomAlphaOfLength(4), ShrinkAction.NAME, ShrinkStep.NAME);
         String policyName = randomAlphaOfLengthBetween(1, 20);
         IndexLifecycleRunnerTests.MockClusterStateActionStep mockStep =
             new IndexLifecycleRunnerTests.MockClusterStateActionStep(mockShrinkStep, randomStepKey());
@@ -180,14 +182,67 @@ public void testRequestedStopOnShrink() {
             .build();
 
         ClusterChangedEvent event = new ClusterChangedEvent("_source", currentState, ClusterState.EMPTY_STATE);
-        SetOnce<Boolean> executedShrink = new SetOnce<>();
+        SetOnce<Boolean> changedOperationMode = new SetOnce<>();
         doAnswer(invocationOnMock -> {
-            executedShrink.set(true);
+            changedOperationMode.set(true);
             return null;
-        }).when(clusterService).submitStateUpdateTask(anyString(), any(ExecuteStepsUpdateTask.class));
+        }).when(clusterService).submitStateUpdateTask(eq("ilm_operation_mode_update"), any(OperationModeUpdateTask.class));
+        indexLifecycleService.applyClusterState(event);
+        indexLifecycleService.triggerPolicies(currentState, true);
+        assertNull(changedOperationMode.get());
+    }
+
+    public void testRequestedStopInShrinkActionButNotShrinkStep() {
+        // test all the shrink action steps that ILM can be stopped during (basically all of them minus the actual shrink)
+        ShrinkAction action = new ShrinkAction(1);
+        action.toSteps(mock(Client.class), "warm", randomStepKey()).stream()
+            .map(sk -> sk.getKey().getName())
+            .filter(name -> name.equals(ShrinkStep.NAME) == false)
+            .forEach(this::verifyCanStopWithStep);
+    }
+
+    // Check that ILM can stop when in the shrink action on the provided step
+    private void verifyCanStopWithStep(String stoppableStep) {
+        Step.StepKey mockShrinkStep = new Step.StepKey(randomAlphaOfLength(4), ShrinkAction.NAME, stoppableStep);
+        String policyName = randomAlphaOfLengthBetween(1, 20);
+        IndexLifecycleRunnerTests.MockClusterStateActionStep mockStep =
+            new IndexLifecycleRunnerTests.MockClusterStateActionStep(mockShrinkStep, randomStepKey());
+        MockAction mockAction = new MockAction(Collections.singletonList(mockStep));
+        Phase phase = new Phase("phase", TimeValue.ZERO, Collections.singletonMap("action", mockAction));
+        LifecyclePolicy policy = newTestLifecyclePolicy(policyName, Collections.singletonMap(phase.getName(), phase));
+        SortedMap<String, LifecyclePolicyMetadata> policyMap = new TreeMap<>();
+        policyMap.put(policyName, new LifecyclePolicyMetadata(policy, Collections.emptyMap(),
+            randomNonNegativeLong(), randomNonNegativeLong()));
+        Index index = new Index(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20));
+        LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder();
+        lifecycleState.setPhase(mockShrinkStep.getPhase());
+        lifecycleState.setAction(mockShrinkStep.getAction());
+        lifecycleState.setStep(mockShrinkStep.getName());
+        IndexMetaData indexMetadata = IndexMetaData.builder(index.getName())
+            .settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME_SETTING.getKey(), policyName))
+            .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.build().asMap())
+            .numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
+        ImmutableOpenMap.Builder<String, IndexMetaData> indices = ImmutableOpenMap.<String, IndexMetaData> builder()
+            .fPut(index.getName(), indexMetadata);
+        MetaData metaData = MetaData.builder()
+            .putCustom(IndexLifecycleMetadata.TYPE, new IndexLifecycleMetadata(policyMap, OperationMode.STOPPING))
+            .indices(indices.build())
+            .persistentSettings(settings(Version.CURRENT).build())
+            .build();
+        ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(metaData)
+            .nodes(DiscoveryNodes.builder().localNodeId(nodeId).masterNodeId(nodeId).add(masterNode).build())
+            .build();
+
+        ClusterChangedEvent event = new ClusterChangedEvent("_source", currentState, ClusterState.EMPTY_STATE);
+        SetOnce<Boolean> changedOperationMode = new SetOnce<>();
+        doAnswer(invocationOnMock -> {
+            changedOperationMode.set(true);
+            return null;
+        }).when(clusterService).submitStateUpdateTask(eq("ilm_operation_mode_update"), any(OperationModeUpdateTask.class));
         indexLifecycleService.applyClusterState(event);
         indexLifecycleService.triggerPolicies(currentState, true);
-        assertTrue(executedShrink.get());
+        assertTrue(changedOperationMode.get());
     }
 
     public void testRequestedStopOnSafeAction() {
@@ -236,7 +291,7 @@ public void testRequestedStopOnSafeAction() {
             assertThat(task.getOperationMode(), equalTo(OperationMode.STOPPED));
             moveToMaintenance.set(true);
             return null;
-        }).when(clusterService).submitStateUpdateTask(anyString(), any(OperationModeUpdateTask.class));
+        }).when(clusterService).submitStateUpdateTask(eq("ilm_operation_mode_update"), any(OperationModeUpdateTask.class));
 
         indexLifecycleService.applyClusterState(event);
         indexLifecycleService.triggerPolicies(currentState, randomBoolean());
diff --git a/x-pack/plugin/logstash/build.gradle b/x-pack/plugin/logstash/build.gradle
index b799dbd4ceb95..35699ba9b96a5 100644
--- a/x-pack/plugin/logstash/build.gradle
+++ b/x-pack/plugin/logstash/build.gradle
@@ -10,7 +10,6 @@ esplugin {
 archivesBaseName = 'x-pack-logstash'
 
 dependencies {
-    // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
     compileOnly project(path: xpackModule('core'), configuration: 'default')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     if (isEclipse) {
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
index 9de1c2f56d1fd..7ea6e6d2e4daa 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
@@ -7,25 +7,26 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
-import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.template.TemplateUtils;
 
-import java.util.ArrayList;
-import java.util.Collection;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.function.UnaryOperator;
 import java.util.regex.Pattern;
 
 /**
- * This class activates/deactivates the logstash modules depending if we're running a node client or transport client
+ * This class supplies the logstash featureset and templates
  */
 public class Logstash extends Plugin implements ActionPlugin {
 
@@ -41,16 +42,11 @@ public Logstash(Settings settings) {
         this.enabled = XPackSettings.LOGSTASH_ENABLED.get(settings);
     }
 
-    boolean isEnabled() {
-      return enabled;
-    }
-
-    public Collection<Module> createGuiceModules() {
-        List<Module> modules = new ArrayList<>();
-        modules.add(b -> {
-            XPackPlugin.bindFeatureSet(b, LogstashFeatureSet.class);
-        });
-        return modules;
+    @Override
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        return Arrays.asList(
+            new ActionHandler<>(XPackUsageFeatureAction.LOGSTASH, LogstashUsageTransportAction.class),
+            new ActionHandler<>(XPackInfoFeatureAction.LOGSTASH, LogstashInfoTransportAction.class));
     }
 
     public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashFeatureSet.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportAction.java
similarity index 53%
rename from x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashFeatureSet.java
rename to x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportAction.java
index 001ab713350b0..49a48daf5ea9c 100644
--- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashFeatureSet.java
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportAction.java
@@ -5,26 +5,25 @@
  */
 package org.elasticsearch.xpack.logstash;
 
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.common.Nullable;
+import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.logstash.LogstashFeatureSetUsage;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
 
-
-import java.util.Map;
-
-public class LogstashFeatureSet implements XPackFeatureSet {
+public class LogstashInfoTransportAction extends XPackInfoFeatureTransportAction {
 
     private final boolean enabled;
     private final XPackLicenseState licenseState;
 
     @Inject
-    public LogstashFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState) {
+    public LogstashInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                       Settings settings, XPackLicenseState licenseState) {
+        super(XPackInfoFeatureAction.LOGSTASH.name(), transportService, actionFilters);
         this.enabled = XPackSettings.LOGSTASH_ENABLED.get(settings);
         this.licenseState = licenseState;
     }
@@ -34,14 +33,9 @@ public String name() {
         return XPackField.LOGSTASH;
     }
 
-    @Override
-    public String description() {
-        return "Logstash management component for X-Pack";
-    }
-
     @Override
     public boolean available() {
-        return licenseState != null && licenseState.isLogstashAllowed();
+        return licenseState.isLogstashAllowed();
     }
 
     @Override
@@ -49,14 +43,4 @@ public boolean enabled() {
         return enabled;
     }
 
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-      return null;
-    }
-
-    @Override
-    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
-        listener.onResponse(new LogstashFeatureSetUsage(available(), enabled()));
-    }
-
 }
diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashUsageTransportAction.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashUsageTransportAction.java
new file mode 100644
index 0000000000000..55f096ec2e8e9
--- /dev/null
+++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/LogstashUsageTransportAction.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.logstash;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.logstash.LogstashFeatureSetUsage;
+
+public class LogstashUsageTransportAction extends XPackUsageFeatureTransportAction {
+
+    private final Settings settings;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public LogstashUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                        ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                        Settings settings, XPackLicenseState licenseState) {
+        super(XPackUsageFeatureAction.LOGSTASH.name(), transportService, clusterService,
+            threadPool, actionFilters, indexNameExpressionResolver);
+        this.settings = settings;
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
+        boolean available = licenseState.isLogstashAllowed();
+        LogstashFeatureSetUsage usage =
+            new LogstashFeatureSetUsage(available, XPackSettings.LOGSTASH_ENABLED.get(settings));
+        listener.onResponse(new XPackUsageFeatureResponse(usage));
+    }
+}
diff --git a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashFeatureSetTests.java b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportActionTests.java
similarity index 55%
rename from x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashFeatureSetTests.java
rename to x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportActionTests.java
index 2878b674b4847..42115fbf74c07 100644
--- a/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashFeatureSetTests.java
+++ b/x-pack/plugin/logstash/src/test/java/org/elasticsearch/xpack/logstash/LogstashInfoTransportActionTests.java
@@ -5,19 +5,22 @@
  */
 package org.elasticsearch.xpack.logstash;
 
+import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.logstash.LogstashFeatureSetUsage;
 
-import static org.mockito.Mockito.mock;
 import static org.hamcrest.core.Is.is;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class LogstashFeatureSetTests extends ESTestCase {
+public class LogstashInfoTransportActionTests extends ESTestCase {
 
     public void testEnabledSetting() throws Exception {
         boolean enabled = randomBoolean();
@@ -25,12 +28,14 @@ public void testEnabledSetting() throws Exception {
             .put("path.home", createTempDir())
             .put("xpack.logstash.enabled", enabled)
             .build();
-        LogstashFeatureSet featureSet = new LogstashFeatureSet(settings, null);
+        LogstashInfoTransportAction featureSet = new LogstashInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), settings, null);
         assertThat(featureSet.enabled(), is(enabled));
 
-        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
-        featureSet.usage(future);
-        XPackFeatureSet.Usage usage = future.get();
+        LogstashUsageTransportAction usageAction = newUsageAction(settings, false);
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, null, future);
+        XPackFeatureSet.Usage usage = future.get().getUsage();
 
         BytesStreamOutput out = new BytesStreamOutput();
         usage.writeTo(out);
@@ -40,20 +45,23 @@ public void testEnabledSetting() throws Exception {
 
     public void testEnabledDefault() throws Exception {
         Settings settings = Settings.builder().put("path.home", createTempDir()).build();
-        LogstashFeatureSet featureSet = new LogstashFeatureSet(settings, null);
+        LogstashInfoTransportAction featureSet = new LogstashInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), settings, null);
         assertThat(featureSet.enabled(), is(true));
     }
 
     public void testAvailable() throws Exception {
         final XPackLicenseState licenseState = mock(XPackLicenseState.class);
-        LogstashFeatureSet featureSet = new LogstashFeatureSet(Settings.EMPTY, licenseState);
+        LogstashInfoTransportAction featureSet = new LogstashInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
         boolean available = randomBoolean();
         when(licenseState.isLogstashAllowed()).thenReturn(available);
         assertThat(featureSet.available(), is(available));
 
-        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
-        featureSet.usage(future);
-        XPackFeatureSet.Usage usage = future.get();
+        var usageAction = newUsageAction(Settings.EMPTY, available);
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, null, future);
+        XPackFeatureSet.Usage usage = future.get().getUsage();
         assertThat(usage.available(), is(available));
 
         BytesStreamOutput out = new BytesStreamOutput();
@@ -61,4 +69,11 @@ public void testAvailable() throws Exception {
         XPackFeatureSet.Usage serializedUsage = new LogstashFeatureSetUsage(out.bytes().streamInput());
         assertThat(serializedUsage.available(), is(available));
     }
+
+    private LogstashUsageTransportAction newUsageAction(Settings settings, boolean available) {
+        XPackLicenseState licenseState = mock(XPackLicenseState.class);
+        when(licenseState.isLogstashAllowed()).thenReturn(available);
+        return new LogstashUsageTransportAction(mock(TransportService.class), null,
+            null, mock(ActionFilters.class), null, settings, licenseState);
+    }
 }
diff --git a/x-pack/plugin/mapper-flattened/build.gradle b/x-pack/plugin/mapper-flattened/build.gradle
new file mode 100644
index 0000000000000..c9e1c408304ea
--- /dev/null
+++ b/x-pack/plugin/mapper-flattened/build.gradle
@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+evaluationDependsOn(xpackModule('core'))
+
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+    name 'flattened'
+    description 'Module for the flattened field type, which allows JSON objects to be flattened into a single field.'
+    classname 'org.elasticsearch.xpack.flattened.FlattenedMapperPlugin'
+    extendedPlugins = ['x-pack-core']
+}
+archivesBaseName = 'x-pack-flattened'
+
+dependencies {
+    compileOnly project(path: xpackModule('core'), configuration: 'default')
+    testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+    if (isEclipse) {
+        testCompile project(path: xpackModule('core-tests'), configuration: 'testArtifacts')
+    }
+}
+
+integTest.enabled = false
diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/FlattenedInfoTransportAction.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/FlattenedInfoTransportAction.java
new file mode 100644
index 0000000000000..abdfd8c4b85c9
--- /dev/null
+++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/FlattenedInfoTransportAction.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.flattened;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
+
+public class FlattenedInfoTransportAction extends XPackInfoFeatureTransportAction {
+
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public FlattenedInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                        Settings settings, XPackLicenseState licenseState) {
+        super(XPackInfoFeatureAction.FLATTENED.name(), transportService, actionFilters);
+        this.enabled = XPackSettings.FLATTENED_ENABLED.get(settings);
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    public String name() {
+        return XPackField.FLATTENED;
+    }
+
+    @Override
+    public boolean available() {
+        return licenseState.isFlattenedAllowed();
+    }
+
+    @Override
+    public boolean enabled() {
+        return enabled;
+    }
+
+}
diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/FlattenedMapperPlugin.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/FlattenedMapperPlugin.java
new file mode 100644
index 0000000000000..1dd407f70d3a2
--- /dev/null
+++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/FlattenedMapperPlugin.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.flattened;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.flattened.mapper.FlatObjectFieldMapper;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+
+public class FlattenedMapperPlugin extends Plugin implements MapperPlugin, ActionPlugin {
+
+    protected final boolean enabled;
+
+    public FlattenedMapperPlugin(Settings settings) {
+        this.enabled = XPackSettings.FLATTENED_ENABLED.get(settings);
+    }
+
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        if (enabled == false) {
+            return emptyMap();
+        }
+        return singletonMap(FlatObjectFieldMapper.CONTENT_TYPE, new FlatObjectFieldMapper.TypeParser());
+    }
+
+    @Override
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        return Arrays.asList(
+            new ActionHandler<>(XPackUsageFeatureAction.FLATTENED, FlattenedUsageTransportAction.class),
+            new ActionHandler<>(XPackInfoFeatureAction.FLATTENED, FlattenedInfoTransportAction.class));
+    }
+}
diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/FlattenedUsageTransportAction.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/FlattenedUsageTransportAction.java
new file mode 100644
index 0000000000000..c928e3c5f5dcc
--- /dev/null
+++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/FlattenedUsageTransportAction.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.flattened;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.flattened.FlattenedFeatureSetUsage;
+
+public class FlattenedUsageTransportAction extends XPackUsageFeatureTransportAction {
+
+    private final Settings settings;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public FlattenedUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                         ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                         Settings settings, XPackLicenseState licenseState) {
+        super(XPackUsageFeatureAction.FLATTENED.name(), transportService, clusterService,
+            threadPool, actionFilters, indexNameExpressionResolver);
+        this.settings = settings;
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
+        FlattenedFeatureSetUsage usage =
+            new FlattenedFeatureSetUsage(licenseState.isFlattenedAllowed(), XPackSettings.FLATTENED_ENABLED.get(settings));
+        listener.onResponse(new XPackUsageFeatureResponse(usage));
+    }
+}
diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java
new file mode 100644
index 0000000000000..eef76f0807bfb
--- /dev/null
+++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java
@@ -0,0 +1,616 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.flattened.mapper;
+
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.OrdinalMap;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
+import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData;
+import org.elasticsearch.index.mapper.DynamicKeyFieldMapper;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.StringFieldType;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.search.MultiValueMode;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.index.mapper.TypeParsers.parseField;
+
+/**
+ * A field mapper that accepts a JSON object and flattens it into a single field. This data type
+ * can be a useful alternative to an 'object' mapping when the object has a large, unknown set
+ * of keys.
+ *
+ * Currently the mapper extracts all leaf values of the JSON object, converts them to their text
+ * representations, and indexes each one as a keyword. It creates both a 'keyed' version of the token
+ * to allow searches on particular key-value pairs, as well as a 'root' token without the key
+ *
+ * As an example, given a flat object field called 'flat_object' and the following input
+ *
+ * {
+ *   "flat_object": {
+ *     "key1": "some value",
+ *     "key2": {
+ *       "key3": true
+ *     }
+ *   }
+ * }
+ *
+ * the mapper will produce untokenized string fields with the name "flat_object" and values
+ * "some value" and "true", as well as string fields called "flat_object._keyed" with values
+ * "key\0some value" and "key2.key3\0true". Note that \0 is used as a reserved separator
+ *  character (see {@link FlatObjectFieldParser#SEPARATOR}).
+ */
+public final class FlatObjectFieldMapper extends DynamicKeyFieldMapper {
+
+    public static final String CONTENT_TYPE = "flattened";
+    private static final String KEYED_FIELD_SUFFIX = "._keyed";
+
+    private static class Defaults {
+        public static final MappedFieldType FIELD_TYPE = new RootFlatObjectFieldType();
+
+        static {
+            FIELD_TYPE.setTokenized(false);
+            FIELD_TYPE.setStored(false);
+            FIELD_TYPE.setHasDocValues(true);
+            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
+            FIELD_TYPE.setOmitNorms(true);
+            FIELD_TYPE.freeze();
+        }
+
+        public static final int DEPTH_LIMIT = 20;
+        public static final int IGNORE_ABOVE = Integer.MAX_VALUE;
+    }
+
+    public static class Builder extends FieldMapper.Builder<Builder, FlatObjectFieldMapper> {
+        private int depthLimit = Defaults.DEPTH_LIMIT;
+        private int ignoreAbove = Defaults.IGNORE_ABOVE;
+
+        public Builder(String name) {
+            super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
+            builder = this;
+        }
+
+        @Override
+        public RootFlatObjectFieldType fieldType() {
+            return (RootFlatObjectFieldType) super.fieldType();
+        }
+
+        @Override
+        public Builder indexOptions(IndexOptions indexOptions) {
+            if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) > 0) {
+                throw new IllegalArgumentException("The [" + CONTENT_TYPE
+                    + "] field does not support positions, got [index_options]="
+                    + indexOptionToString(indexOptions));
+            }
+            return super.indexOptions(indexOptions);
+        }
+
+        public Builder depthLimit(int depthLimit) {
+            if (depthLimit < 0) {
+                throw new IllegalArgumentException("[depth_limit] must be positive, got " + depthLimit);
+            }
+            this.depthLimit = depthLimit;
+            return this;
+        }
+
+        public Builder eagerGlobalOrdinals(boolean eagerGlobalOrdinals) {
+            fieldType().setEagerGlobalOrdinals(eagerGlobalOrdinals);
+            return builder;
+        }
+
+        public Builder ignoreAbove(int ignoreAbove) {
+            if (ignoreAbove < 0) {
+                throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove);
+            }
+            this.ignoreAbove = ignoreAbove;
+            return this;
+        }
+
+        public Builder splitQueriesOnWhitespace(boolean splitQueriesOnWhitespace) {
+            fieldType().setSplitQueriesOnWhitespace(splitQueriesOnWhitespace);
+            return builder;
+        }
+
+        @Override
+        public Builder addMultiField(Mapper.Builder<?, ?> mapperBuilder) {
+            throw new UnsupportedOperationException("[fields] is not supported for [" + CONTENT_TYPE + "] fields.");
+        }
+
+        @Override
+        public Builder copyTo(CopyTo copyTo) {
+            throw new UnsupportedOperationException("[copy_to] is not supported for [" + CONTENT_TYPE + "] fields.");
+        }
+
+        @Override
+        public Builder store(boolean store) {
+            throw new UnsupportedOperationException("[store] is not supported for [" + CONTENT_TYPE + "] fields.");
+        }
+
+        @Override
+        public FlatObjectFieldMapper build(BuilderContext context) {
+            setupFieldType(context);
+            if (fieldType().splitQueriesOnWhitespace()) {
+                NamedAnalyzer whitespaceAnalyzer = new NamedAnalyzer("whitespace", AnalyzerScope.INDEX, new WhitespaceAnalyzer());
+                fieldType().setSearchAnalyzer(whitespaceAnalyzer);
+            }
+            return new FlatObjectFieldMapper(name, fieldType, defaultFieldType,
+                ignoreAbove, depthLimit, context.indexSettings());
+        }
+    }
+
+    public static class TypeParser implements Mapper.TypeParser {
+        @Override
+        public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+            Builder builder = new Builder(name);
+            parseField(builder, name, node, parserContext);
+            for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
+                Map.Entry<String, Object> entry = iterator.next();
+                String propName = entry.getKey();
+                Object propNode = entry.getValue();
+                if (propName.equals("depth_limit")) {
+                    builder.depthLimit(XContentMapValues.nodeIntegerValue(propNode, -1));
+                    iterator.remove();
+                } else if (propName.equals("eager_global_ordinals")) {
+                    builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode, "eager_global_ordinals"));
+                    iterator.remove();
+                } else if (propName.equals("ignore_above")) {
+                    builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
+                    iterator.remove();
+                } else if (propName.equals("null_value")) {
+                    if (propNode == null) {
+                        throw new MapperParsingException("Property [null_value] cannot be null.");
+                    }
+                    builder.nullValue(propNode.toString());
+                    iterator.remove();
+                } else if (propName.equals("split_queries_on_whitespace")) {
+                    builder.splitQueriesOnWhitespace
+                        (XContentMapValues.nodeBooleanValue(propNode, "split_queries_on_whitespace"));
+                    iterator.remove();
+                }
+            }
+            return builder;
+        }
+    }
+
+    /**
+     * A field type that represents the values under a particular JSON key, used
+     * when searching under a specific key as in 'my_flat_object.key: some_value'.
+     */
+    public static final class KeyedFlatObjectFieldType extends StringFieldType {
+        private final String key;
+        private boolean splitQueriesOnWhitespace;
+
+        public KeyedFlatObjectFieldType(String key) {
+            setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
+            setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
+            this.key = key;
+        }
+
+        public KeyedFlatObjectFieldType clone() {
+            return new KeyedFlatObjectFieldType(this);
+        }
+
+        private KeyedFlatObjectFieldType(KeyedFlatObjectFieldType ref) {
+            super(ref);
+            this.key = ref.key;
+            this.splitQueriesOnWhitespace = ref.splitQueriesOnWhitespace;
+        }
+
+        private KeyedFlatObjectFieldType(String name, String key, RootFlatObjectFieldType ref) {
+            super(ref);
+            setName(name);
+            this.key = key;
+            this.splitQueriesOnWhitespace = ref.splitQueriesOnWhitespace;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            if (!super.equals(o)) return false;
+            KeyedFlatObjectFieldType that = (KeyedFlatObjectFieldType) o;
+            return splitQueriesOnWhitespace == that.splitQueriesOnWhitespace;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(super.hashCode(), splitQueriesOnWhitespace);
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        public String key() {
+            return key;
+        }
+
+        public boolean splitQueriesOnWhitespace() {
+            return splitQueriesOnWhitespace;
+        }
+
+        public void setSplitQueriesOnWhitespace(boolean splitQueriesOnWhitespace) {
+            checkIfFrozen();
+            this.splitQueriesOnWhitespace = splitQueriesOnWhitespace;
+        }
+
+        @Override
+        public Query existsQuery(QueryShardContext context) {
+            Term term = new Term(name(), FlatObjectFieldParser.createKeyedValue(key, ""));
+            return new PrefixQuery(term);
+        }
+
+        @Override
+        public Query rangeQuery(Object lowerTerm,
+                                Object upperTerm,
+                                boolean includeLower,
+                                boolean includeUpper,
+                                QueryShardContext context) {
+
+            // We require range queries to specify both bounds because an unbounded query could incorrectly match
+            // values from other keys. For example, a query on the 'first' key with only a lower bound would become
+            // ("first\0value", null), which would also match the value "second\0value" belonging to the key 'second'.
+            if (lowerTerm == null || upperTerm == null) {
+                throw new IllegalArgumentException("[range] queries on keyed [" + CONTENT_TYPE +
+                    "] fields must include both an upper and a lower bound.");
+            }
+
+            return super.rangeQuery(lowerTerm, upperTerm,
+                includeLower, includeUpper, context);
+        }
+
+        @Override
+        public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions,
+                                boolean transpositions) {
+            throw new UnsupportedOperationException("[fuzzy] queries are not currently supported on keyed " +
+                "[" + CONTENT_TYPE + "] fields.");
+        }
+
+        @Override
+        public Query regexpQuery(String value, int flags, int maxDeterminizedStates,
+                                 MultiTermQuery.RewriteMethod method, QueryShardContext context) {
+            throw new UnsupportedOperationException("[regexp] queries are not currently supported on keyed " +
+                "[" + CONTENT_TYPE + "] fields.");
+        }
+
+        @Override
+        public Query wildcardQuery(String value,
+                                   MultiTermQuery.RewriteMethod method,
+                                   QueryShardContext context) {
+            throw new UnsupportedOperationException("[wildcard] queries are not currently supported on keyed " +
+                "[" + CONTENT_TYPE + "] fields.");
+        }
+
+        @Override
+        public BytesRef indexedValueForSearch(Object value) {
+            if (value == null) {
+                return null;
+            }
+
+            String stringValue = value instanceof BytesRef
+                ? ((BytesRef) value).utf8ToString()
+                : value.toString();
+            String keyedValue = FlatObjectFieldParser.createKeyedValue(key, stringValue);
+            return new BytesRef(keyedValue);
+        }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
+            failIfNoDocValues();
+            return new KeyedFlatObjectFieldData.Builder(key);
+        }
+    }
+
+    /**
+     * A field data implementation that gives access to the values associated with
+     * a particular JSON key.
+     *
+     * This class wraps the field data that is built directly on the keyed flat object field,
+     * and filters out values whose prefix doesn't match the requested key. Loading and caching
+     * is fully delegated to the wrapped field data, so that different {@link KeyedFlatObjectFieldData}
+     * for the same flat object field share the same global ordinals.
+     *
+     * Because of the code-level complexity it would introduce, it is currently not possible
+     * to retrieve the underlying global ordinals map through {@link #getOrdinalMap()}.
+     */
+    public static class KeyedFlatObjectFieldData implements IndexOrdinalsFieldData {
+        private final String key;
+        private final IndexOrdinalsFieldData delegate;
+
+        private KeyedFlatObjectFieldData(String key, IndexOrdinalsFieldData delegate) {
+            this.delegate = delegate;
+            this.key = key;
+        }
+
+        public String getKey() {
+            return key;
+        }
+
+        @Override
+        public String getFieldName() {
+            return delegate.getFieldName();
+        }
+
+        @Override
+        public SortField sortField(Object missingValue,
+                                   MultiValueMode sortMode,
+                                   XFieldComparatorSource.Nested nested,
+                                   boolean reverse) {
+            XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
+            return new SortField(getFieldName(), source, reverse);
+        }
+
+        @Override
+        public void clear() {
+            delegate.clear();
+        }
+
+        @Override
+        public AtomicOrdinalsFieldData load(LeafReaderContext context) {
+            AtomicOrdinalsFieldData fieldData = delegate.load(context);
+            return new KeyedFlatObjectAtomicFieldData(key, fieldData);
+        }
+
+        @Override
+        public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
+            AtomicOrdinalsFieldData fieldData = delegate.loadDirect(context);
+            return new KeyedFlatObjectAtomicFieldData(key, fieldData);
+        }
+
+        @Override
+        public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
+            IndexOrdinalsFieldData fieldData = delegate.loadGlobal(indexReader);
+            return new KeyedFlatObjectFieldData(key, fieldData);
+        }
+
+        @Override
+        public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
+            IndexOrdinalsFieldData fieldData = delegate.localGlobalDirect(indexReader);
+            return new KeyedFlatObjectFieldData(key, fieldData);
+        }
+
+        @Override
+        public OrdinalMap getOrdinalMap() {
+            throw new UnsupportedOperationException("The field data for the flat object field ["
+                + delegate.getFieldName() + "] does not allow access to the underlying ordinal map.");
+        }
+
+        @Override
+        public boolean supportsGlobalOrdinalsMapping() {
+            return false;
+        }
+
+        @Override
+        public Index index() {
+            return delegate.index();
+        }
+
+        public static class Builder implements IndexFieldData.Builder {
+            private final String key;
+
+            Builder(String key) {
+                this.key = key;
+            }
+
+            @Override
+            public IndexFieldData<?> build(IndexSettings indexSettings,
+                                           MappedFieldType fieldType,
+                                           IndexFieldDataCache cache,
+                                           CircuitBreakerService breakerService,
+                                           MapperService mapperService) {
+                String fieldName = fieldType.name();
+                IndexOrdinalsFieldData delegate = new SortedSetDVOrdinalsIndexFieldData(indexSettings,
+                    cache, fieldName, breakerService, AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION);
+                return new KeyedFlatObjectFieldData(key, delegate);
+            }
+        }
+    }
+
+    /**
+     * A field type that represents all 'root' values. This field type is used in
+     * searches on the flat object field itself, e.g. 'my_flat_object: some_value'.
+     */
+    public static final class RootFlatObjectFieldType extends StringFieldType {
+        private boolean splitQueriesOnWhitespace;
+
+        public RootFlatObjectFieldType() {
+            setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
+            setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
+        }
+
+        private RootFlatObjectFieldType(RootFlatObjectFieldType ref) {
+            super(ref);
+            this.splitQueriesOnWhitespace = ref.splitQueriesOnWhitespace;
+        }
+
+        public RootFlatObjectFieldType clone() {
+            return new RootFlatObjectFieldType(this);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            if (!super.equals(o)) return false;
+            RootFlatObjectFieldType that = (RootFlatObjectFieldType) o;
+            return splitQueriesOnWhitespace == that.splitQueriesOnWhitespace;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(super.hashCode(), splitQueriesOnWhitespace);
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        public boolean splitQueriesOnWhitespace() {
+            return splitQueriesOnWhitespace;
+        }
+
+        public void setSplitQueriesOnWhitespace(boolean splitQueriesOnWhitespace) {
+            checkIfFrozen();
+            this.splitQueriesOnWhitespace = splitQueriesOnWhitespace;
+        }
+
+        @Override
+        public Object valueForDisplay(Object value) {
+            if (value == null) {
+                return null;
+            }
+            BytesRef binaryValue = (BytesRef) value;
+            return binaryValue.utf8ToString();
+        }
+
+        @Override
+        public Query existsQuery(QueryShardContext context) {
+            if (hasDocValues()) {
+                return new DocValuesFieldExistsQuery(name());
+            } else {
+                return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));
+            }
+        }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
+            failIfNoDocValues();
+            return new DocValuesIndexFieldData.Builder();
+        }
+    }
+
+    private final FlatObjectFieldParser fieldParser;
+    private int depthLimit;
+    private int ignoreAbove;
+
+    private FlatObjectFieldMapper(String simpleName,
+                                  MappedFieldType fieldType,
+                                  MappedFieldType defaultFieldType,
+                                  int ignoreAbove,
+                                  int depthLimit,
+                                  Settings indexSettings) {
+        super(simpleName, fieldType, defaultFieldType, indexSettings, CopyTo.empty());
+        assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0;
+
+        this.depthLimit = depthLimit;
+        this.ignoreAbove = ignoreAbove;
+        this.fieldParser = new FlatObjectFieldParser(fieldType.name(), keyedFieldName(),
+            fieldType, depthLimit, ignoreAbove);
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+
+    @Override
+    protected void doMerge(Mapper mergeWith) {
+        super.doMerge(mergeWith);
+        this.ignoreAbove = ((FlatObjectFieldMapper) mergeWith).ignoreAbove;
+    }
+
+    @Override
+    protected FlatObjectFieldMapper clone() {
+        return (FlatObjectFieldMapper) super.clone();
+    }
+
+    @Override
+    public RootFlatObjectFieldType fieldType() {
+        return (RootFlatObjectFieldType) super.fieldType();
+    }
+
+    @Override
+    public KeyedFlatObjectFieldType keyedFieldType(String key) {
+        return new KeyedFlatObjectFieldType(keyedFieldName(), key, fieldType());
+    }
+
+    public String keyedFieldName() {
+        return fieldType.name() + KEYED_FIELD_SUFFIX;
+    }
+
+    @Override
+    protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
+        if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) {
+            return;
+        }
+
+        if (fieldType.indexOptions() == IndexOptions.NONE && !fieldType.hasDocValues()) {
+            context.parser().skipChildren();
+            return;
+        }
+
+        XContentParser xContentParser = context.parser();
+        fields.addAll(fieldParser.parse(xContentParser));
+
+        if (!fieldType.hasDocValues()) {
+            createFieldNamesField(context, fields);
+        }
+    }
+
+    @Override
+    protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
+        super.doXContentBody(builder, includeDefaults, params);
+
+        if (includeDefaults || depthLimit != Defaults.DEPTH_LIMIT) {
+            builder.field("depth_limit", depthLimit);
+        }
+
+        if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) {
+            builder.field("ignore_above", ignoreAbove);
+        }
+
+        if (includeDefaults || fieldType().nullValue() != null) {
+            builder.field("null_value", fieldType().nullValue());
+        }
+
+        if (includeDefaults || fieldType().splitQueriesOnWhitespace()) {
+            builder.field("split_queries_on_whitespace", fieldType().splitQueriesOnWhitespace());
+        }
+    }
+}
diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldParser.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldParser.java
new file mode 100644
index 0000000000000..a855440b2be71
--- /dev/null
+++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldParser.java
@@ -0,0 +1,167 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.flattened.mapper;
+
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.index.mapper.ContentPath;
+import org.elasticsearch.index.mapper.MappedFieldType;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A helper class for {@link FlatObjectFieldMapper} parses a JSON object
+ * and produces a pair of indexable fields for each leaf value.
+ */
+class FlatObjectFieldParser {
+    static final String SEPARATOR = "\0";
+    private static final byte SEPARATOR_BYTE = '\0';
+
+    private final String rootFieldName;
+    private final String keyedFieldName;
+
+    private final MappedFieldType fieldType;
+    private final int depthLimit;
+    private final int ignoreAbove;
+
+    FlatObjectFieldParser(String rootFieldName,
+                          String keyedFieldName,
+                          MappedFieldType fieldType,
+                          int depthLimit,
+                          int ignoreAbove) {
+        this.rootFieldName = rootFieldName;
+        this.keyedFieldName = keyedFieldName;
+        this.fieldType = fieldType;
+        this.depthLimit = depthLimit;
+        this.ignoreAbove = ignoreAbove;
+    }
+
+    public List<IndexableField> parse(XContentParser parser) throws IOException {
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT,
+            parser.currentToken(),
+            parser::getTokenLocation);
+
+        ContentPath path = new ContentPath();
+        List<IndexableField> fields = new ArrayList<>();
+
+        parseObject(parser, path, fields);
+        return fields;
+    }
+
+    private void parseObject(XContentParser parser,
+                             ContentPath path,
+                             List<IndexableField> fields) throws IOException {
+        String currentName = null;
+        while (true) {
+            XContentParser.Token token = parser.nextToken();
+            if (token == XContentParser.Token.END_OBJECT) {
+                return;
+            }
+
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentName = parser.currentName();
+            } else {
+                assert currentName != null;
+                parseFieldValue(token, parser, path, currentName, fields);
+            }
+        }
+    }
+
+    private void parseArray(XContentParser parser,
+                            ContentPath path,
+                            String currentName,
+                            List<IndexableField> fields) throws IOException {
+        while (true) {
+            XContentParser.Token token = parser.nextToken();
+            if (token == XContentParser.Token.END_ARRAY) {
+                return;
+            }
+            parseFieldValue(token, parser, path, currentName, fields);
+        }
+    }
+
+    private void parseFieldValue(XContentParser.Token token,
+                                 XContentParser parser,
+                                 ContentPath path,
+                                 String currentName,
+                                 List<IndexableField> fields) throws IOException {
+        if (token == XContentParser.Token.START_OBJECT) {
+            path.add(currentName);
+            validateDepthLimit(path);
+            parseObject(parser, path, fields);
+            path.remove();
+        } else if (token == XContentParser.Token.START_ARRAY) {
+            parseArray(parser, path, currentName, fields);
+        } else if (token.isValue()) {
+            String value = parser.text();
+            addField(path, currentName, value, fields);
+        } else if (token == XContentParser.Token.VALUE_NULL) {
+            if (fieldType.nullValueAsString() != null) {
+                addField(path, currentName, fieldType.nullValueAsString(), fields);
+            }
+        } else {
+            // Note that we throw an exception here just to be safe. We don't actually expect to reach
+            // this case, since XContentParser verifies that the input is well-formed as it parses.
+            throw new IllegalArgumentException("Encountered unexpected token [" + token.toString() + "].");
+        }
+    }
+
+    private void addField(ContentPath path,
+                          String currentName,
+                          String value,
+                          List<IndexableField> fields) {
+        if (value.length() > ignoreAbove) {
+            return;
+        }
+
+        String key = path.pathAsText(currentName);
+        if (key.contains(SEPARATOR)) {
+            throw new IllegalArgumentException("Keys in [flattened] fields cannot contain the reserved character \\0."
+                + " Offending key: [" + key + "].");
+        }
+        String keyedValue = createKeyedValue(key, value);
+
+        if (fieldType.indexOptions() != IndexOptions.NONE) {
+            fields.add(new StringField(rootFieldName, new BytesRef(value), Field.Store.NO));
+            fields.add(new StringField(keyedFieldName, new BytesRef(keyedValue), Field.Store.NO));
+        }
+
+        if (fieldType.hasDocValues()) {
+            fields.add(new SortedSetDocValuesField(rootFieldName, new BytesRef(value)));
+            fields.add(new SortedSetDocValuesField(keyedFieldName, new BytesRef(keyedValue)));
+        }
+    }
+
+    private void validateDepthLimit(ContentPath path) {
+        if (path.length() + 1 > depthLimit) {
+            throw new IllegalArgumentException("The provided [flattened] field [" + rootFieldName +"]" +
+                " exceeds the maximum depth limit of [" + depthLimit + "].");
+        }
+    }
+
+    static String createKeyedValue(String key, String value) {
+        return key + SEPARATOR + value;
+    }
+
+    static BytesRef extractKey(BytesRef keyedValue) {
+        int length;
+        for (length = 0; length < keyedValue.length; length++){
+            if (keyedValue.bytes[keyedValue.offset + length] == SEPARATOR_BYTE) {
+                break;
+            }
+        }
+        return new BytesRef(keyedValue.bytes, keyedValue.offset, length);
+    }
+}
diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectAtomicFieldData.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectAtomicFieldData.java
new file mode 100644
index 0000000000000..7f5a5235095b8
--- /dev/null
+++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectAtomicFieldData.java
@@ -0,0 +1,251 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.flattened.mapper;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues;
+import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.FieldData;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+
+/**
+ * The atomic field data implementation for {@link FlatObjectFieldMapper.KeyedFlatObjectFieldType}.
+ *
+ * This class wraps the field data that is built directly on the keyed flat object field,
+ * and filters out values whose prefix doesn't match the requested key.
+ *
+ * In order to support all usage patterns, the delegate's ordinal values are shifted
+ * to range from 0 to the number of total values.
+ */
+public class KeyedFlatObjectAtomicFieldData implements AtomicOrdinalsFieldData {
+
+    private final String key;
+    private final AtomicOrdinalsFieldData delegate;
+
+    KeyedFlatObjectAtomicFieldData(String key,
+                                   AtomicOrdinalsFieldData delegate) {
+        this.key = key;
+        this.delegate = delegate;
+    }
+
+    @Override
+    public long ramBytesUsed() {
+        return delegate.ramBytesUsed();
+    }
+
+    @Override
+    public Collection<Accountable> getChildResources() {
+        return delegate.getChildResources();
+    }
+
+    @Override
+    public SortedSetDocValues getOrdinalsValues() {
+        BytesRef keyBytes = new BytesRef(key);
+        SortedSetDocValues values = delegate.getOrdinalsValues();
+
+        long minOrd, maxOrd;
+        try {
+            minOrd = findMinOrd(keyBytes, values);
+            if (minOrd < 0) {
+                return DocValues.emptySortedSet();
+            }
+            maxOrd = findMaxOrd(keyBytes, values);
+            assert maxOrd >= 0;
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+
+        return new KeyedJsonDocValues(keyBytes, values, minOrd, maxOrd);
+    }
+
+    @Override
+    public void close() {
+        delegate.close();
+    }
+
+    @Override
+    public ScriptDocValues<?> getScriptValues() {
+        return AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION
+            .apply(getOrdinalsValues());
+    }
+
+    @Override
+    public SortedBinaryDocValues getBytesValues() {
+        return FieldData.toString(getOrdinalsValues());
+    }
+
+    /**
+     * Performs a binary search to find the first term with 'key' as a prefix.
+     */
+    static long findMinOrd(BytesRef key, SortedSetDocValues delegate) throws IOException {
+        long low = 0;
+        long high = delegate.getValueCount() - 1;
+
+        long result = -1;
+        while (low <= high) {
+            long mid = (low + high) >>> 1;
+            final BytesRef term = delegate.lookupOrd(mid);
+            int cmp = compare(key, term);
+
+            if (cmp == 0) {
+                result = mid;
+                high = mid - 1;
+            } else if (cmp < 0) {
+                high = mid - 1;
+            } else {
+                low = mid + 1;
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Performs a binary search to find the last term with 'key' as a prefix.
+     */
+    static long findMaxOrd(BytesRef key, SortedSetDocValues delegate) throws IOException {
+        long low = 0;
+        long high = delegate.getValueCount() - 1;
+
+        long result = -1;
+        while (low <= high) {
+            long mid = (low + high) >>> 1;
+            final BytesRef term = delegate.lookupOrd(mid);
+            int cmp = compare(key, term);
+
+            if (cmp == 0) {
+                result = mid;
+                low = mid + 1;
+            } else if (cmp < 0) {
+                high = mid - 1;
+            } else {
+                low = mid + 1;
+            }
+        }
+        return result;
+    }
+
+    private static int compare(BytesRef key, BytesRef term) {
+        BytesRef extractedKey = FlatObjectFieldParser.extractKey(term);
+        return key.compareTo(extractedKey);
+    }
+
+    private static class KeyedJsonDocValues extends AbstractSortedSetDocValues {
+
+        private final BytesRef key;
+        private final SortedSetDocValues delegate;
+
+        /**
+         * The first and last ordinals whose term has 'key' as a prefix. These
+         * values must be non-negative (there is at least one matching term).
+         */
+        private final long minOrd;
+        private final long maxOrd;
+
+        /**
+         * We cache the first ordinal in a document to avoid unnecessary iterations
+         * through the delegate doc values. If no ordinal is cached for the current
+         * document, this value will be -1.
+         */
+        private long cachedNextOrd;
+
+        private KeyedJsonDocValues(BytesRef key,
+                                   SortedSetDocValues delegate,
+                                   long minOrd,
+                                   long maxOrd) {
+            assert minOrd >= 0 && maxOrd >= 0;
+            this.key = key;
+            this.delegate = delegate;
+            this.minOrd = minOrd;
+            this.maxOrd = maxOrd;
+            this.cachedNextOrd = -1;
+        }
+
+        @Override
+        public long getValueCount() {
+            return maxOrd - minOrd + 1;
+        }
+
+        /**
+         * Returns the (un-prefixed) term value for the requested ordinal.
+         *
+         * Note that this method can only be called on ordinals returned from {@link #nextOrd()}.
+         */
+        @Override
+        public BytesRef lookupOrd(long ord) throws IOException {
+            long delegateOrd = unmapOrd(ord);
+            BytesRef keyedValue = delegate.lookupOrd(delegateOrd);
+
+            int prefixLength = key.length + 1;
+            int valueLength = keyedValue.length - prefixLength;
+            return new BytesRef(keyedValue.bytes, prefixLength, valueLength);
+        }
+
+        @Override
+        public long nextOrd() throws IOException {
+            if (cachedNextOrd >= 0) {
+                long nextOrd = cachedNextOrd;
+                cachedNextOrd = -1;
+                return mapOrd(nextOrd);
+            }
+
+            long ord = delegate.nextOrd();
+            if (ord != NO_MORE_ORDS && ord <= maxOrd) {
+                assert ord >= minOrd;
+                return mapOrd(ord);
+            } else {
+                return NO_MORE_ORDS;
+            }
+        }
+
+        @Override
+        public boolean advanceExact(int target) throws IOException {
+            if (delegate.advanceExact(target)) {
+                while (true) {
+                    long ord = delegate.nextOrd();
+                    if (ord == NO_MORE_ORDS || ord > maxOrd) {
+                        break;
+                    }
+
+                    if (ord >= minOrd) {
+                        cachedNextOrd = ord;
+                        return true;
+                    }
+                }
+            }
+
+            cachedNextOrd = -1;
+            return false;
+        }
+
+        /**
+         * Maps an ordinal from the delegate doc values into the filtered ordinal space. The
+         * ordinal is shifted to lie in the range [0, (maxOrd - minOrd)].
+         */
+        private long mapOrd(long ord) {
+            assert minOrd <= ord && ord <= maxOrd;
+            return ord - minOrd;
+        }
+
+        /**
+         * Given a filtered ordinal in the range [0, (maxOrd - minOrd)], maps it into the
+         * delegate ordinal space.
+         */
+        private long unmapOrd(long ord) {
+            assert 0 <= ord && ord <= maxOrd - minOrd;
+            return ord + minOrd;
+        }
+    }
+}
diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java
new file mode 100644
index 0000000000000..86e79cd556c4f
--- /dev/null
+++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java
@@ -0,0 +1,188 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+import org.elasticsearch.search.lookup.LeafDocLookup;
+import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.flattened.mapper.FlatObjectFieldMapper;
+import org.elasticsearch.xpack.flattened.mapper.FlatObjectFieldMapper.KeyedFlatObjectFieldType;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.function.Function;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class FlatObjectFieldLookupTests extends ESTestCase {
+
+    public void testFieldTypeLookup() {
+        String fieldName = "object1.object2.field";
+        FlatObjectFieldMapper mapper = createFlatObjectMapper(fieldName);
+
+        FieldTypeLookup lookup = new FieldTypeLookup()
+            .copyAndAddAll("type", singletonList(mapper), emptyList());
+        assertEquals(mapper.fieldType(), lookup.get(fieldName));
+
+        String objectKey = "key1.key2";
+        String searchFieldName = fieldName + "." + objectKey;
+
+        MappedFieldType searchFieldType = lookup.get(searchFieldName);
+        assertEquals(mapper.keyedFieldName(), searchFieldType.name());
+        assertThat(searchFieldType, instanceOf(KeyedFlatObjectFieldType.class));
+
+        FlatObjectFieldMapper.KeyedFlatObjectFieldType keyedFieldType = (KeyedFlatObjectFieldType) searchFieldType;
+        assertEquals(objectKey, keyedFieldType.key());
+    }
+
+    public void testFieldTypeLookupWithAlias() {
+        String fieldName = "object1.object2.field";
+        FlatObjectFieldMapper mapper = createFlatObjectMapper(fieldName);
+
+        String aliasName = "alias";
+        FieldAliasMapper alias = new FieldAliasMapper(aliasName, aliasName, fieldName);
+
+        FieldTypeLookup lookup = new FieldTypeLookup()
+            .copyAndAddAll("type", singletonList(mapper), singletonList(alias));
+        assertEquals(mapper.fieldType(), lookup.get(aliasName));
+
+        String objectKey = "key1.key2";
+        String searchFieldName = aliasName + "." + objectKey;
+
+        MappedFieldType searchFieldType = lookup.get(searchFieldName);
+        assertEquals(mapper.keyedFieldName(), searchFieldType.name());
+        assertThat(searchFieldType, instanceOf(KeyedFlatObjectFieldType.class));
+
+        KeyedFlatObjectFieldType keyedFieldType = (KeyedFlatObjectFieldType) searchFieldType;
+        assertEquals(objectKey, keyedFieldType.key());
+    }
+
+    public void testFieldTypeLookupWithMultipleFields() {
+        String field1 = "object1.object2.field";
+        String field2 = "object1.field";
+        String field3 = "object2.field";
+
+        FlatObjectFieldMapper mapper1 = createFlatObjectMapper(field1);
+        FlatObjectFieldMapper mapper2 = createFlatObjectMapper(field2);
+        FlatObjectFieldMapper mapper3 = createFlatObjectMapper(field3);
+
+        FieldTypeLookup lookup = new FieldTypeLookup()
+            .copyAndAddAll("type", Arrays.asList(mapper1, mapper2), emptyList());
+        assertNotNull(lookup.get(field1 + ".some.key"));
+        assertNotNull(lookup.get(field2 + ".some.key"));
+
+        lookup = lookup.copyAndAddAll("type", singletonList(mapper3), emptyList());
+        assertNotNull(lookup.get(field1 + ".some.key"));
+        assertNotNull(lookup.get(field2 + ".some.key"));
+        assertNotNull(lookup.get(field3 + ".some.key"));
+    }
+
+    public void testMaxDynamicKeyDepth() {
+        FieldTypeLookup lookup = new FieldTypeLookup();
+        assertEquals(0, lookup.maxKeyedLookupDepth());
+
+        // Add a flattened object field.
+        String flatObjectName = "object1.object2.field";
+        FlatObjectFieldMapper flatObjectField = createFlatObjectMapper(flatObjectName);
+        lookup = lookup.copyAndAddAll("type", singletonList(flatObjectField), emptyList());
+        assertEquals(3, lookup.maxKeyedLookupDepth());
+
+        // Add a short alias to that field.
+        String aliasName = "alias";
+        FieldAliasMapper alias = new FieldAliasMapper(aliasName, aliasName, flatObjectName);
+        lookup = lookup.copyAndAddAll("type", emptyList(), singletonList(alias));
+        assertEquals(3, lookup.maxKeyedLookupDepth());
+
+        // Add a longer alias to that field.
+        String longAliasName = "object1.object2.object3.alias";
+        FieldAliasMapper longAlias = new FieldAliasMapper(longAliasName, longAliasName, flatObjectName);
+        lookup = lookup.copyAndAddAll("type", emptyList(), singletonList(longAlias));
+        assertEquals(4, lookup.maxKeyedLookupDepth());
+
+        // Update the long alias to refer to a non-flattened object field.
+        String fieldName = "field";
+        MockFieldMapper field = new MockFieldMapper(fieldName);
+        longAlias = new FieldAliasMapper(longAliasName, longAliasName, fieldName);
+        lookup = lookup.copyAndAddAll("type", singletonList(field), singletonList(longAlias));
+        assertEquals(3, lookup.maxKeyedLookupDepth());
+    }
+
+    public void testFieldLookupIterator() {
+        MockFieldMapper mapper = new MockFieldMapper("foo");
+        FlatObjectFieldMapper flatObjectMapper = createFlatObjectMapper("object1.object2.field");
+
+        FieldTypeLookup lookup = new FieldTypeLookup()
+            .copyAndAddAll("type", Arrays.asList(mapper, flatObjectMapper), emptyList());
+
+        Set<String> fieldNames = new HashSet<>();
+        for (MappedFieldType fieldType : lookup) {
+            fieldNames.add(fieldType.name());
+        }
+
+        assertThat(fieldNames, containsInAnyOrder(
+            mapper.name(), flatObjectMapper.name(), flatObjectMapper.keyedFieldName()));
+    }
+
+    private FlatObjectFieldMapper createFlatObjectMapper(String fieldName) {
+        Settings settings = Settings.builder()
+            .put("index.version.created", Version.CURRENT)
+            .build();
+        Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
+        return new FlatObjectFieldMapper.Builder(fieldName).build(context);
+    }
+
+    public void testScriptDocValuesLookup() {
+        MapperService mapperService = mock(MapperService.class);
+
+        ScriptDocValues<?> docValues1 = mock(ScriptDocValues.class);
+        IndexFieldData<?> fieldData1 = createFieldData(docValues1);
+
+        ScriptDocValues<?> docValues2 = mock(ScriptDocValues.class);
+        IndexFieldData<?> fieldData2 = createFieldData(docValues2);
+
+        KeyedFlatObjectFieldType fieldType1 = new KeyedFlatObjectFieldType("key1");
+        when(mapperService.fullName("json.key1")).thenReturn(fieldType1);
+
+        KeyedFlatObjectFieldType fieldType2 = new KeyedFlatObjectFieldType( "key2");
+        when(mapperService.fullName("json.key2")).thenReturn(fieldType2);
+
+        Function<MappedFieldType, IndexFieldData<?>> fieldDataSupplier = fieldType -> {
+            KeyedFlatObjectFieldType keyedFieldType = (KeyedFlatObjectFieldType) fieldType;
+            return keyedFieldType.key().equals("key1") ? fieldData1 : fieldData2;
+        };
+
+        SearchLookup searchLookup = new SearchLookup(mapperService, fieldDataSupplier);
+        LeafDocLookup docLookup = searchLookup.doc().getLeafDocLookup(null);
+
+        assertEquals(docValues1, docLookup.get("json.key1"));
+        assertEquals(docValues2, docLookup.get("json.key2"));
+    }
+
+    private IndexFieldData<?> createFieldData(ScriptDocValues<?> scriptDocValues) {
+        AtomicFieldData atomicFieldData = mock(AtomicFieldData.class);
+        doReturn(scriptDocValues).when(atomicFieldData).getScriptValues();
+
+        IndexFieldData<?> fieldData = mock(IndexFieldData.class);
+        when(fieldData.getFieldName()).thenReturn("field");
+        doReturn(atomicFieldData).when(fieldData).load(anyObject());
+
+        return fieldData;
+    }
+}
diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java
new file mode 100644
index 0000000000000..71cdfa0693835
--- /dev/null
+++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java
@@ -0,0 +1,457 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.flattened.mapper;
+
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.DocumentMapperParser;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.flattened.FlattenedMapperPlugin;
+import org.elasticsearch.xpack.flattened.mapper.FlatObjectFieldMapper.KeyedFlatObjectFieldType;
+import org.elasticsearch.xpack.flattened.mapper.FlatObjectFieldMapper.RootFlatObjectFieldType;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
+import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents;
+import static org.hamcrest.Matchers.equalTo;
+
+public class FlatObjectFieldMapperTests extends ESSingleNodeTestCase {
+    private IndexService indexService;
+    private DocumentMapperParser parser;
+
+    @Before
+    public void setup() {
+        indexService = createIndex("test");
+        parser = indexService.mapperService().documentMapperParser();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(FlattenedMapperPlugin.class, XPackPlugin.class);
+    }
+
+    public void testDefaults() throws Exception {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+            .startObject("type")
+                .startObject("properties")
+                    .startObject("field")
+                        .field("type", "flattened")
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject());
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+            .startObject("field")
+                .field("key", "value")
+            .endObject()
+        .endObject());
+
+        ParsedDocument parsedDoc = mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON));
+
+        // Check the root fields.
+        IndexableField[] fields = parsedDoc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+
+        assertEquals("field", fields[0].name());
+        assertEquals(new BytesRef("value"), fields[0].binaryValue());
+        assertFalse(fields[0].fieldType().stored());
+        assertTrue(fields[0].fieldType().omitNorms());
+        assertEquals(DocValuesType.NONE, fields[0].fieldType().docValuesType());
+
+        assertEquals("field", fields[1].name());
+        assertEquals(new BytesRef("value"), fields[1].binaryValue());
+        assertEquals(DocValuesType.SORTED_SET, fields[1].fieldType().docValuesType());
+
+        // Check the keyed fields.
+        IndexableField[] keyedFields = parsedDoc.rootDoc().getFields("field._keyed");
+        assertEquals(2, keyedFields.length);
+
+        assertEquals("field._keyed", keyedFields[0].name());
+        assertEquals(new BytesRef("key\0value"), keyedFields[0].binaryValue());
+        assertFalse(keyedFields[0].fieldType().stored());
+        assertTrue(keyedFields[0].fieldType().omitNorms());
+        assertEquals(DocValuesType.NONE, keyedFields[0].fieldType().docValuesType());
+
+        assertEquals("field._keyed", keyedFields[1].name());
+        assertEquals(new BytesRef("key\0value"), keyedFields[1].binaryValue());
+        assertEquals(DocValuesType.SORTED_SET, keyedFields[1].fieldType().docValuesType());
+
+        // Check that there is no 'field names' field.
+        IndexableField[] fieldNamesFields = parsedDoc.rootDoc().getFields(FieldNamesFieldMapper.NAME);
+        assertEquals(0, fieldNamesFields.length);
+    }
+
+    public void testDisableIndex() throws Exception {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+            .startObject("type")
+                .startObject("properties")
+                    .startObject("field")
+                        .field("type", "flattened")
+                        .field("index", false)
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject());
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+            .startObject("field")
+                .field("key", "value")
+            .endObject()
+        .endObject());
+
+        ParsedDocument parsedDoc = mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON));
+
+        IndexableField[] fields = parsedDoc.rootDoc().getFields("field");
+        assertEquals(1, fields.length);
+        assertEquals(DocValuesType.SORTED_SET, fields[0].fieldType().docValuesType());
+
+        IndexableField[] keyedFields = parsedDoc.rootDoc().getFields("field._keyed");
+        assertEquals(1, keyedFields.length);
+        assertEquals(DocValuesType.SORTED_SET, keyedFields[0].fieldType().docValuesType());
+    }
+
+    public void testDisableDocValues() throws Exception {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+            .startObject("type")
+                .startObject("properties")
+                    .startObject("field")
+                        .field("type", "flattened")
+                        .field("doc_values", false)
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject());
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+            .startObject("field")
+            .field("key", "value")
+            .endObject()
+            .endObject());
+
+        ParsedDocument parsedDoc = mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON));
+
+        IndexableField[] fields = parsedDoc.rootDoc().getFields("field");
+        assertEquals(1, fields.length);
+        assertEquals(DocValuesType.NONE, fields[0].fieldType().docValuesType());
+
+        IndexableField[] keyedFields = parsedDoc.rootDoc().getFields("field._keyed");
+        assertEquals(1, keyedFields.length);
+        assertEquals(DocValuesType.NONE, keyedFields[0].fieldType().docValuesType());
+
+        IndexableField[] fieldNamesFields = parsedDoc.rootDoc().getFields(FieldNamesFieldMapper.NAME);
+        assertEquals(1, fieldNamesFields.length);
+        assertEquals("field", fieldNamesFields[0].stringValue());
+    }
+
+    public void testEnableStore() throws Exception {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+            .startObject("type")
+                .startObject("properties")
+                    .startObject("field")
+                        .field("type", "flattened")
+                        .field("store", true)
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject());
+
+        expectThrows(UnsupportedOperationException.class, () ->
+            parser.parse("type", new CompressedXContent(mapping)));
+    }
+
+    public void testIndexOptions() throws IOException {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+            .startObject("type")
+                .startObject("properties")
+                    .startObject("field")
+                        .field("type", "flattened")
+                        .field("index_options", "freqs")
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject());
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        for (String indexOptions : Arrays.asList("positions", "offsets")) {
+            String invalidMapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", "flattened")
+                            .field("index_options", indexOptions)
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject());
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                    () -> parser.parse("type", new CompressedXContent(invalidMapping)));
+            assertEquals("The [flattened] field does not support positions, got [index_options]=" + indexOptions, e.getMessage());
+        }
+    }
+
+    public void testNullField() throws Exception {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+            .startObject("type")
+                .startObject("properties")
+                    .startObject("field")
+                        .field("type", "flattened")
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject());
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+            .nullField("field")
+        .endObject());
+
+        ParsedDocument parsedDoc = mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON));
+        IndexableField[] fields = parsedDoc.rootDoc().getFields("field");
+        assertEquals(0, fields.length);
+    }
+
+    public void testMalformedJson() throws Exception {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+            .startObject("type")
+                .startObject("properties")
+                    .startObject("field")
+                        .field("type", "flattened")
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject());
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        BytesReference doc1 = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+            .field("field", "not a JSON object")
+        .endObject());
+
+        expectThrows(MapperParsingException.class, () -> mapper.parse(
+            new SourceToParse("test", "type", "1", doc1, XContentType.JSON)));
+
+        BytesReference doc2 = new BytesArray("{ \"field\": { \"key\": \"value\" ");
+        expectThrows(MapperParsingException.class, () -> mapper.parse(
+            new SourceToParse("test", "type", "1", doc2, XContentType.JSON)));
+    }
+
+    public void testFieldMultiplicity() throws Exception {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+            .startObject("type")
+                .startObject("properties")
+                    .startObject("field")
+                        .field("type", "flattened")
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject());
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+            .startArray("field")
+                .startObject()
+                    .field("key1", "value")
+                .endObject()
+                .startObject()
+                    .field("key2", true)
+                    .field("key3", false)
+                .endObject()
+            .endArray()
+        .endObject());
+
+        ParsedDocument parsedDoc = mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON));
+
+        IndexableField[] fields = parsedDoc.rootDoc().getFields("field");
+        assertEquals(6, fields.length);
+        assertEquals(new BytesRef("value"), fields[0].binaryValue());
+        assertEquals(new BytesRef("true"), fields[2].binaryValue());
+        assertEquals(new BytesRef("false"), fields[4].binaryValue());
+
+        IndexableField[] keyedFields = parsedDoc.rootDoc().getFields("field._keyed");
+        assertEquals(6, keyedFields.length);
+        assertEquals(new BytesRef("key1\0value"), keyedFields[0].binaryValue());
+        assertEquals(new BytesRef("key2\0true"), keyedFields[2].binaryValue());
+        assertEquals(new BytesRef("key3\0false"), keyedFields[4].binaryValue());
+    }
+
+    public void testDepthLimit() throws IOException {
+         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+            .startObject("type")
+                .startObject("properties")
+                    .startObject("field")
+                        .field("type", "flattened")
+                        .field("depth_limit", 2)
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject());
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+            .startObject("field")
+                .startObject("key1")
+                    .startObject("key2")
+                        .field("key3", "value")
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject());
+
+        expectThrows(MapperParsingException.class, () ->
+            mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON)));
+    }
+
+    public void testEagerGlobalOrdinals() throws IOException {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+            .startObject("type")
+                .startObject("properties")
+                    .startObject("field")
+                        .field("type", "flattened")
+                        .field("eager_global_ordinals", true)
+                    .endObject().endObject()
+                .endObject().endObject());
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        FieldMapper fieldMapper = (FieldMapper) mapper.mappers().getMapper("field");
+        assertTrue(fieldMapper.fieldType().eagerGlobalOrdinals());
+    }
+
+    public void testIgnoreAbove() throws IOException {
+         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+            .startObject("type")
+                .startObject("properties")
+                    .startObject("field")
+                        .field("type", "flattened")
+                        .field("ignore_above", 10)
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject());
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+            .startArray("field")
+                .startObject()
+                    .field("key", "a longer than usual value")
+                .endObject()
+            .endArray()
+        .endObject());
+
+        ParsedDocument parsedDoc = mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON));
+        IndexableField[] fields = parsedDoc.rootDoc().getFields("field");
+        assertEquals(0, fields.length);
+    }
+
+    public void testNullValues() throws Exception {
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+            .startObject("type")
+                .startObject("properties")
+                    .startObject("field")
+                        .field("type", "flattened")
+                    .endObject()
+                    .startObject("other_field")
+                        .field("type", "flattened")
+                        .field("null_value", "placeholder")
+                    .endObject()
+                .endObject()
+            .endObject()
+        .endObject());
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+            .startObject("field")
+                .nullField("key")
+            .endObject()
+            .startObject("other_field")
+                .nullField("key")
+            .endObject()
+        .endObject());
+        ParsedDocument parsedDoc = mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON));
+
+        IndexableField[] fields = parsedDoc.rootDoc().getFields("field");
+        assertEquals(0, fields.length);
+
+        IndexableField[] otherFields = parsedDoc.rootDoc().getFields("other_field");
+        assertEquals(2, otherFields.length);
+        assertEquals(new BytesRef("placeholder"), otherFields[0].binaryValue());
+        assertEquals(new BytesRef("placeholder"), otherFields[1].binaryValue());
+
+        IndexableField[] prefixedOtherFields = parsedDoc.rootDoc().getFields("other_field._keyed");
+        assertEquals(2, prefixedOtherFields.length);
+        assertEquals(new BytesRef("key\0placeholder"), prefixedOtherFields[0].binaryValue());
+        assertEquals(new BytesRef("key\0placeholder"), prefixedOtherFields[1].binaryValue());
+    }
+
+    public void testSplitQueriesOnWhitespace() throws IOException {
+        MapperService mapperService = indexService.mapperService();
+        String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
+            .startObject("type")
+                .startObject("properties")
+                    .startObject("field")
+                        .field("type", "flattened")
+                        .field("split_queries_on_whitespace", true)
+                    .endObject()
+                .endObject()
+            .endObject().endObject());
+        mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
+
+        RootFlatObjectFieldType rootFieldType = (RootFlatObjectFieldType) mapperService.fullName("field");
+        assertThat(rootFieldType.searchAnalyzer().name(), equalTo("whitespace"));
+        assertTokenStreamContents(rootFieldType.searchAnalyzer().analyzer().tokenStream("", "Hello World"),
+            new String[] {"Hello", "World"});
+
+        KeyedFlatObjectFieldType keyedFieldType = (KeyedFlatObjectFieldType) mapperService.fullName("field.key");
+        assertThat(keyedFieldType.searchAnalyzer().name(), equalTo("whitespace"));
+        assertTokenStreamContents(keyedFieldType.searchAnalyzer().analyzer().tokenStream("", "Hello World"),
+            new String[] {"Hello", "World"});
+    }
+}
diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldParserTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldParserTests.java
new file mode 100644
index 0000000000000..ff4de0d85cee1
--- /dev/null
+++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldParserTests.java
@@ -0,0 +1,318 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.flattened.mapper;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MockFieldMapper.FakeFieldType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.XContentTestUtils;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.List;
+
+public class FlatObjectFieldParserTests extends ESTestCase {
+    private FlatObjectFieldParser parser;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        parser = new FlatObjectFieldParser("field", "field._keyed",
+            new FakeFieldType(),
+            Integer.MAX_VALUE,
+            Integer.MAX_VALUE);
+    }
+
+    public void testTextValues() throws Exception {
+        String input = "{ \"key1\": \"value1\", \"key2\": \"value2\" }";
+        XContentParser xContentParser = createXContentParser(input);
+
+        List<IndexableField> fields = parser.parse(xContentParser);
+        assertEquals(4, fields.size());
+
+        IndexableField field1 = fields.get(0);
+        assertEquals("field", field1.name());
+        assertEquals(new BytesRef("value1"), field1.binaryValue());
+
+        IndexableField keyedField1 = fields.get(1);
+        assertEquals("field._keyed", keyedField1.name());
+        assertEquals(new BytesRef("key1\0value1"), keyedField1.binaryValue());
+
+        IndexableField field2 = fields.get(2);
+        assertEquals("field", field2.name());
+        assertEquals(new BytesRef("value2"), field2.binaryValue());
+
+        IndexableField keyedField2 = fields.get(3);
+        assertEquals("field._keyed", keyedField2.name());
+        assertEquals(new BytesRef("key2\0value2"), keyedField2.binaryValue());
+    }
+
+    public void testNumericValues() throws Exception {
+        String input = "{ \"key\": 2.718 }";
+        XContentParser xContentParser = createXContentParser(input);
+
+        List<IndexableField> fields = parser.parse(xContentParser);
+        assertEquals(2, fields.size());
+
+        IndexableField field = fields.get(0);
+        assertEquals("field", field.name());
+        assertEquals(new BytesRef("2.718"), field.binaryValue());
+
+        IndexableField keyedField = fields.get(1);
+        assertEquals("field._keyed", keyedField.name());
+        assertEquals(new BytesRef("key" + '\0' + "2.718"), keyedField.binaryValue());
+    }
+
+    public void testBooleanValues() throws Exception {
+        String input = "{ \"key\": false }";
+        XContentParser xContentParser = createXContentParser(input);
+
+        List<IndexableField> fields = parser.parse(xContentParser);
+        assertEquals(2, fields.size());
+
+        IndexableField field = fields.get(0);
+        assertEquals("field", field.name());
+        assertEquals(new BytesRef("false"), field.binaryValue());
+
+        IndexableField keyedField = fields.get(1);
+        assertEquals("field._keyed", keyedField.name());
+        assertEquals(new BytesRef("key\0false"), keyedField.binaryValue());
+    }
+
+    public void testBasicArrays() throws Exception {
+        String input = "{ \"key\": [true, false] }";
+        XContentParser xContentParser = createXContentParser(input);
+
+        List<IndexableField> fields = parser.parse(xContentParser);
+        assertEquals(4, fields.size());
+
+        IndexableField field1 = fields.get(0);
+        assertEquals("field", field1.name());
+        assertEquals(new BytesRef("true"), field1.binaryValue());
+
+        IndexableField keyedField1 = fields.get(1);
+        assertEquals("field._keyed", keyedField1.name());
+        assertEquals(new BytesRef("key\0true"), keyedField1.binaryValue());
+
+        IndexableField field2 = fields.get(2);
+        assertEquals("field", field2.name());
+        assertEquals(new BytesRef("false"), field2.binaryValue());
+
+        IndexableField keyedField2 = fields.get(3);
+        assertEquals("field._keyed", keyedField2.name());
+        assertEquals(new BytesRef("key\0false"), keyedField2.binaryValue());
+    }
+
+    public void testArrayOfArrays() throws Exception {
+        String input = "{ \"key\": [[true, \"value\"], 3] }";
+        XContentParser xContentParser = createXContentParser(input);
+
+        List<IndexableField> fields = parser.parse(xContentParser);
+        assertEquals(6, fields.size());
+
+        IndexableField field1 = fields.get(0);
+        assertEquals("field", field1.name());
+        assertEquals(new BytesRef("true"), field1.binaryValue());
+
+        IndexableField keyedField1 = fields.get(1);
+        assertEquals("field._keyed", keyedField1.name());
+        assertEquals(new BytesRef("key\0true"), keyedField1.binaryValue());
+
+        IndexableField field2 = fields.get(2);
+        assertEquals("field", field2.name());
+        assertEquals(new BytesRef("value"), field2.binaryValue());
+
+        IndexableField keyedField2 = fields.get(3);
+        assertEquals("field._keyed", keyedField2.name());
+        assertEquals(new BytesRef("key\0value"), keyedField2.binaryValue());
+
+        IndexableField field3 = fields.get(4);
+        assertEquals("field", field3.name());
+        assertEquals(new BytesRef("3"), field3.binaryValue());
+
+        IndexableField keyedField3 = fields.get(5);
+        assertEquals("field._keyed", keyedField3.name());
+        assertEquals(new BytesRef("key" + "\0" + "3"), keyedField3.binaryValue());
+    }
+
+    public void testArraysOfObjects() throws Exception {
+        String input = "{ \"key1\": [{ \"key2\": true }, false], \"key4\": \"other\" }";
+        XContentParser xContentParser = createXContentParser(input);
+
+        List<IndexableField> fields = parser.parse(xContentParser);
+        assertEquals(6, fields.size());
+
+        IndexableField field1 = fields.get(0);
+        assertEquals("field", field1.name());
+        assertEquals(new BytesRef("true"), field1.binaryValue());
+
+        IndexableField keyedField1 = fields.get(1);
+        assertEquals("field._keyed", keyedField1.name());
+        assertEquals(new BytesRef("key1.key2\0true"), keyedField1.binaryValue());
+
+        IndexableField field2 = fields.get(2);
+        assertEquals("field", field2.name());
+        assertEquals(new BytesRef("false"), field2.binaryValue());
+
+        IndexableField keyedField2 = fields.get(3);
+        assertEquals("field._keyed", keyedField2.name());
+        assertEquals(new BytesRef("key1\0false"), keyedField2.binaryValue());
+
+        IndexableField field3 = fields.get(4);
+        assertEquals("field", field3.name());
+        assertEquals(new BytesRef("other"), field3.binaryValue());
+
+        IndexableField keyedField3 = fields.get(5);
+        assertEquals("field._keyed", keyedField3.name());
+        assertEquals(new BytesRef("key4\0other"), keyedField3.binaryValue());
+    }
+
+    public void testNestedObjects() throws Exception {
+        String input = "{ \"parent1\": { \"key\" : \"value\" }," +
+            "\"parent2\": { \"key\" : \"value\" }}";
+        XContentParser xContentParser = createXContentParser(input);
+
+        List<IndexableField> fields = parser.parse(xContentParser);
+        assertEquals(4, fields.size());
+
+        IndexableField field1 = fields.get(0);
+        assertEquals("field", field1.name());
+        assertEquals(new BytesRef("value"), field1.binaryValue());
+
+        IndexableField keyedField1 = fields.get(1);
+        assertEquals("field._keyed", keyedField1.name());
+        assertEquals(new BytesRef("parent1.key\0value"), keyedField1.binaryValue());
+
+        IndexableField field2 = fields.get(2);
+        assertEquals("field", field2.name());
+        assertEquals(new BytesRef("value"), field2.binaryValue());
+
+        IndexableField keyedField2 = fields.get(3);
+        assertEquals("field._keyed", keyedField2.name());
+        assertEquals(new BytesRef("parent2.key\0value"), keyedField2.binaryValue());
+    }
+
+    public void testDepthLimit() throws Exception {
+        String input = "{ \"parent1\": { \"key\" : \"value\" }," +
+            "\"parent2\": [{ \"key\" : { \"key\" : \"value\" }}]}";
+        XContentParser xContentParser = createXContentParser(input);
+        FlatObjectFieldParser configuredParser = new FlatObjectFieldParser("field", "field._keyed",
+            new FakeFieldType(), 2, Integer.MAX_VALUE);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> configuredParser.parse(xContentParser));
+        assertEquals("The provided [flattened] field [field] exceeds the maximum depth limit of [2].", e.getMessage());
+    }
+
+    public void testDepthLimitBoundary() throws Exception {
+        String input = "{ \"parent1\": { \"key\" : \"value\" }," +
+            "\"parent2\": [{ \"key\" : { \"key\" : \"value\" }}]}";
+        XContentParser xContentParser = createXContentParser(input);
+        FlatObjectFieldParser configuredParser = new FlatObjectFieldParser("field", "field._keyed",
+            new FakeFieldType(), 3, Integer.MAX_VALUE);
+
+        List<IndexableField> fields = configuredParser.parse(xContentParser);
+        assertEquals(4, fields.size());
+    }
+
+    public void testIgnoreAbove() throws Exception {
+        String input = "{ \"key\": \"a longer field than usual\" }";
+        XContentParser xContentParser = createXContentParser(input);
+        FlatObjectFieldParser configuredParser = new FlatObjectFieldParser("field", "field._keyed",
+            new FakeFieldType(), Integer.MAX_VALUE, 10);
+
+        List<IndexableField> fields = configuredParser.parse(xContentParser);
+        assertEquals(0, fields.size());
+    }
+
+    public void testNullValues() throws Exception {
+        String input = "{ \"key\": null}";
+        XContentParser xContentParser = createXContentParser(input);
+
+        List<IndexableField> fields = parser.parse(xContentParser);
+        assertEquals(0, fields.size());
+
+        xContentParser = createXContentParser(input);
+
+        MappedFieldType fieldType = new FakeFieldType();
+        fieldType.setNullValue("placeholder");
+        FlatObjectFieldParser configuredParser = new FlatObjectFieldParser("field", "field._keyed",
+            fieldType, Integer.MAX_VALUE, Integer.MAX_VALUE);
+
+        fields = configuredParser.parse(xContentParser);
+        assertEquals(2, fields.size());
+
+        IndexableField field = fields.get(0);
+        assertEquals("field", field.name());
+        assertEquals(new BytesRef("placeholder"), field.binaryValue());
+
+        IndexableField keyedField = fields.get(1);
+        assertEquals("field._keyed", keyedField.name());
+        assertEquals(new BytesRef("key\0placeholder"), keyedField.binaryValue());
+    }
+
+    public void testMalformedJson() throws Exception {
+        String input = "{ \"key\": [true, false }";
+        XContentParser xContentParser = createXContentParser(input);
+
+        expectThrows(JsonParseException.class, () -> parser.parse(xContentParser));
+    }
+
+    public void testEmptyObject() throws Exception {
+        String input = "{}";
+        XContentParser xContentParser = createXContentParser(input);
+
+        List<IndexableField> fields = parser.parse(xContentParser);
+        assertEquals(0, fields.size());
+    }
+
+    public void testRandomFields() throws Exception {
+        BytesReference input = BytesReference.bytes(
+            XContentBuilder.builder(JsonXContent.jsonXContent)
+                .startObject()
+                    .startObject("object")
+                        .field("key", "value")
+                    .endObject()
+                    .startArray("array")
+                        .value(2.718)
+                    .endArray()
+                .endObject());
+
+        input = XContentTestUtils.insertRandomFields(XContentType.JSON, input, null, random());
+        XContentParser xContentParser = createXContentParser(input.utf8ToString());
+
+        List<IndexableField> fields = parser.parse(xContentParser);
+        assertTrue(fields.size() > 4);
+    }
+
+    public void testReservedCharacters() throws Exception {
+        BytesReference input = BytesReference.bytes(
+            XContentBuilder.builder(JsonXContent.jsonXContent)
+                .startObject()
+                    .field("k\0y", "value")
+                .endObject());
+        XContentParser xContentParser = createXContentParser(input.utf8ToString());
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parser.parse(xContentParser));
+        assertEquals("Keys in [flattened] fields cannot contain the reserved character \\0. Offending key: [k\0y].",
+            e.getMessage());
+    }
+
+    private XContentParser createXContentParser(String input) throws IOException {
+        XContentParser xContentParser = createParser(JsonXContent.jsonXContent, input);
+        xContentParser.nextToken();
+        return xContentParser;
+    }
+}
diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectIndexFieldDataTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectIndexFieldDataTests.java
new file mode 100644
index 0000000000000..97ca7d4d88ad3
--- /dev/null
+++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectIndexFieldDataTests.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.flattened.mapper;
+
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.mapper.ContentPath;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.flattened.FlattenedMapperPlugin;
+import org.elasticsearch.xpack.flattened.mapper.FlatObjectFieldMapper.KeyedFlatObjectFieldData;
+import org.elasticsearch.xpack.flattened.mapper.FlatObjectFieldMapper.KeyedFlatObjectFieldType;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class FlatObjectIndexFieldDataTests extends ESSingleNodeTestCase  {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(FlattenedMapperPlugin.class, XPackPlugin.class);
+    }
+
+    public void testGlobalFieldDataCaching() throws IOException {
+        // Set up the index service.
+        IndexService indexService = createIndex("test");
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        IndexFieldDataService ifdService = new IndexFieldDataService(indexService.getIndexSettings(),
+            indicesService.getIndicesFieldDataCache(),
+            indicesService.getCircuitBreakerService(),
+            indexService.mapperService());
+
+        Mapper.BuilderContext ctx = new Mapper.BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1));
+        FlatObjectFieldMapper fieldMapper = new FlatObjectFieldMapper.Builder("json").build(ctx);
+
+        AtomicInteger onCacheCalled = new AtomicInteger();
+        ifdService.setListener(new IndexFieldDataCache.Listener() {
+            @Override
+            public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) {
+                assertEquals(fieldMapper.keyedFieldName(), fieldName);
+                onCacheCalled.incrementAndGet();
+            }
+        });
+
+        // Add some documents.
+        Directory directory = LuceneTestCase.newDirectory();
+        IndexWriterConfig config = new IndexWriterConfig(new KeywordAnalyzer());
+        IndexWriter writer = new IndexWriter(directory, config);
+
+        Document doc = new Document();
+        doc.add(new SortedSetDocValuesField("json._keyed", new BytesRef("some_key\0some_value")));
+        writer.addDocument(doc);
+        writer.commit();
+        writer.addDocument(doc);
+        DirectoryReader reader = ElasticsearchDirectoryReader.wrap(
+            DirectoryReader.open(writer),
+            new ShardId("test", "_na_", 1));
+
+        // Load global field data for subfield 'key'.
+        KeyedFlatObjectFieldType fieldType1 = fieldMapper.keyedFieldType("key");
+        IndexFieldData<?> ifd1 = ifdService.getForField(fieldType1);
+        assertTrue(ifd1 instanceof KeyedFlatObjectFieldData);
+
+        KeyedFlatObjectFieldData fieldData1 = (KeyedFlatObjectFieldData) ifd1;
+        assertEquals("key", fieldData1.getKey());
+        fieldData1.loadGlobal(reader);
+        assertEquals(1, onCacheCalled.get());
+
+        // Load global field data for the subfield 'other_key'.
+        KeyedFlatObjectFieldType fieldType2 = fieldMapper.keyedFieldType("other_key");
+        IndexFieldData<?> ifd2 = ifdService.getForField(fieldType2);
+        assertTrue(ifd2 instanceof KeyedFlatObjectFieldData);
+
+        KeyedFlatObjectFieldData fieldData2 = (KeyedFlatObjectFieldData) ifd2;
+        assertEquals("other_key", fieldData2.getKey());
+        fieldData2.loadGlobal(reader);
+        assertEquals(1, onCacheCalled.get());
+
+        ifdService.clear();
+        reader.close();
+        writer.close();
+        directory.close();
+    }
+}
diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectSearchTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectSearchTests.java
new file mode 100644
index 0000000000000..373d1d16d8407
--- /dev/null
+++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectSearchTests.java
@@ -0,0 +1,512 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.flattened.mapper;
+
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.common.document.DocumentField;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory;
+import org.elasticsearch.search.aggregations.metrics.Cardinality;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.flattened.FlattenedMapperPlugin;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
+import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.cardinality;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertOrderedSearchHits;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.CoreMatchers.startsWith;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class FlatObjectSearchTests extends ESSingleNodeTestCase {
+
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(FlattenedMapperPlugin.class, XPackPlugin.class);
+    }
+
+    @Before
+    public void setUpIndex() throws IOException {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject()
+            .startObject("_doc")
+                .startObject("properties")
+                    .startObject("flat_object")
+                        .field("type", "flattened")
+                        .field("split_queries_on_whitespace", true)
+                    .endObject()
+                    .startObject("headers")
+                        .field("type", "flattened")
+                        .field("split_queries_on_whitespace", true)
+                    .endObject()
+                    .startObject("labels")
+                        .field("type", "flattened")
+                    .endObject()
+                .endObject()
+           .endObject()
+        .endObject();
+        createIndex("test", Settings.EMPTY, "_doc", mapping);
+    }
+
+    public void testMatchQuery() throws Exception {
+        client().prepareIndex("test", "_doc", "1")
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+            .setSource(XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("headers")
+                        .field("content-type", "application/json")
+                        .field("origin", "https://www.elastic.co")
+                    .endObject()
+            .endObject())
+            .get();
+
+        SearchResponse searchResponse = client().prepareSearch()
+            .setQuery(matchQuery("headers", "application/json"))
+            .get();
+        assertHitCount(searchResponse, 1L);
+
+        // Check that queries are split on whitespace.
+        searchResponse = client().prepareSearch()
+            .setQuery(matchQuery("headers.content-type", "application/json text/plain"))
+            .get();
+        assertHitCount(searchResponse, 1L);
+
+        searchResponse = client().prepareSearch()
+            .setQuery(matchQuery("headers.origin", "application/json"))
+            .get();
+        assertHitCount(searchResponse, 0L);
+    }
+
+    public void testMultiMatchQuery() throws Exception {
+        client().prepareIndex("test", "_doc", "1")
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+            .setSource(XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("headers")
+                        .field("content-type", "application/json")
+                        .field("origin", "https://www.elastic.co")
+                    .endObject()
+            .endObject())
+            .get();
+
+        SearchResponse searchResponse = client().prepareSearch()
+            .setQuery(multiMatchQuery("application/json", "headers"))
+            .get();
+        assertHitCount(searchResponse, 1L);
+
+        searchResponse = client().prepareSearch()
+            .setQuery(multiMatchQuery("application/json text/plain", "headers.content-type"))
+            .get();
+        assertHitCount(searchResponse, 1L);
+
+        searchResponse = client().prepareSearch()
+            .setQuery(multiMatchQuery("application/json", "headers.origin"))
+            .get();
+        assertHitCount(searchResponse, 0L);
+
+        searchResponse = client().prepareSearch()
+            .setQuery(multiMatchQuery("application/json", "headers.origin", "headers.contentType"))
+            .get();
+        assertHitCount(searchResponse, 0L);
+    }
+
+    public void testQueryStringQuery() throws Exception {
+        client().prepareIndex("test", "_doc", "1")
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+            .setSource(XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("flat_object")
+                        .field("field1", "value")
+                        .field("field2", "2.718")
+                    .endObject()
+                .endObject())
+            .get();
+
+        SearchResponse response = client().prepareSearch("test")
+            .setQuery(queryStringQuery("flat_object.field1:value"))
+            .get();
+        assertSearchResponse(response);
+        assertHitCount(response, 1);
+
+        response = client().prepareSearch("test")
+            .setQuery(queryStringQuery("flat_object.field1:value AND flat_object:2.718"))
+            .get();
+        assertSearchResponse(response);
+        assertHitCount(response, 1);
+
+        response = client().prepareSearch("test")
+            .setQuery(queryStringQuery("2.718").field("flat_object.field2"))
+            .get();
+        assertSearchResponse(response);
+        assertHitCount(response, 1);
+    }
+
+    public void testSimpleQueryStringQuery() throws Exception {
+        client().prepareIndex("test", "_doc", "1")
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+            .setSource(XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("flat_object")
+                        .field("field1", "value")
+                        .field("field2", "2.718")
+                    .endObject()
+                .endObject())
+            .get();
+
+        SearchResponse response = client().prepareSearch("test")
+            .setQuery(simpleQueryStringQuery("value").field("flat_object.field1"))
+            .get();
+        assertSearchResponse(response);
+        assertHitCount(response, 1);
+
+        response = client().prepareSearch("test")
+            .setQuery(simpleQueryStringQuery("+value +2.718").field("flat_object"))
+            .get();
+        assertSearchResponse(response);
+        assertHitCount(response, 1);
+
+        response = client().prepareSearch("test")
+            .setQuery(simpleQueryStringQuery("+value +3.141").field("flat_object"))
+            .get();
+        assertSearchResponse(response);
+        assertHitCount(response, 0);
+    }
+
+    public void testExists() throws Exception {
+        client().prepareIndex("test", "_doc", "1")
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+            .setSource(XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("headers")
+                        .field("content-type", "application/json")
+                    .endObject()
+                .endObject())
+            .get();
+
+        SearchResponse searchResponse = client().prepareSearch()
+            .setQuery(existsQuery("headers"))
+            .get();
+        assertHitCount(searchResponse, 1L);
+
+        searchResponse = client().prepareSearch()
+            .setQuery(existsQuery("headers.content-type"))
+            .get();
+        assertHitCount(searchResponse, 1L);
+
+        searchResponse = client().prepareSearch()
+            .setQuery(existsQuery("headers.nonexistent"))
+            .get();
+        assertHitCount(searchResponse, 0L);
+    }
+
+    public void testCardinalityAggregation() throws IOException {
+        int numDocs = randomIntBetween(2, 100);
+        int precisionThreshold = randomIntBetween(0, 1 << randomInt(20));
+
+        BulkRequestBuilder bulkRequest = client().prepareBulk("test", "_doc")
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE);
+
+        // Add a random number of documents containing a flat object field, plus
+        // a small number of dummy documents.
+        for (int i = 0; i < numDocs; ++i) {
+            bulkRequest.add(client().prepareIndex()
+                .setSource(XContentFactory.jsonBuilder().startObject()
+                    .startObject("flat_object")
+                        .field("first", i)
+                        .field("second", i / 2)
+                    .endObject()
+                .endObject()));
+        }
+
+        for (int i = 0; i < 10; i++) {
+            bulkRequest.add(client().prepareIndex("test", "_doc")
+                .setSource("other_field", "1"));
+        }
+
+        BulkResponse bulkResponse = bulkRequest.get();
+        assertNoFailures(bulkResponse);
+
+        // Test the root flat object field.
+        SearchResponse response = client().prepareSearch("test")
+            .addAggregation(cardinality("cardinality")
+                .precisionThreshold(precisionThreshold)
+                .field("flat_object"))
+            .get();
+
+        assertSearchResponse(response);
+        Cardinality count = response.getAggregations().get("cardinality");
+        assertCardinality(count, numDocs, precisionThreshold);
+
+        // Test two keyed flat object fields.
+        SearchResponse firstResponse = client().prepareSearch("test")
+            .addAggregation(cardinality("cardinality")
+                .precisionThreshold(precisionThreshold)
+                .field("flat_object.first"))
+            .get();
+        assertSearchResponse(firstResponse);
+
+        Cardinality firstCount = firstResponse.getAggregations().get("cardinality");
+        assertCardinality(firstCount, numDocs, precisionThreshold);
+
+        SearchResponse secondResponse = client().prepareSearch("test")
+            .addAggregation(cardinality("cardinality")
+                .precisionThreshold(precisionThreshold)
+                .field("flat_object.second"))
+            .get();
+        assertSearchResponse(secondResponse);
+
+        Cardinality secondCount = secondResponse.getAggregations().get("cardinality");
+        assertCardinality(secondCount, (numDocs + 1) / 2, precisionThreshold);
+    }
+
+    private void assertCardinality(Cardinality count, long value, int precisionThreshold) {
+        if (value <= precisionThreshold) {
+            // linear counting should be picked, and should be accurate
+            assertEquals(value, count.getValue());
+        } else {
+            // error is not bound, so let's just make sure it is > 0
+            assertThat(count.getValue(), greaterThan(0L));
+        }
+    }
+
+    public void testTermsAggregation() throws IOException {
+        BulkRequestBuilder bulkRequest = client().prepareBulk("test", "_doc")
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE);
+        for (int i = 0; i < 5; i++) {
+            bulkRequest.add(client().prepareIndex()
+                .setSource(XContentFactory.jsonBuilder().startObject()
+                        .startObject("labels")
+                            .field("priority", "urgent")
+                            .field("release", "v1.2." + i)
+                        .endObject()
+                    .endObject()));
+        }
+
+        BulkResponse bulkResponse = bulkRequest.get();
+        assertNoFailures(bulkResponse);
+
+        // Aggregate on the root 'labels' field.
+        TermsAggregationBuilder builder = createTermsAgg("labels");
+        SearchResponse response = client().prepareSearch("test")
+            .addAggregation(builder)
+            .get();
+        assertSearchResponse(response);
+
+        Terms terms = response.getAggregations().get("terms");
+        assertThat(terms, notNullValue());
+        assertThat(terms.getName(), equalTo("terms"));
+        assertThat(terms.getBuckets().size(), equalTo(6));
+
+        Terms.Bucket bucket1 = terms.getBuckets().get(0);
+        assertEquals("urgent", bucket1.getKey());
+        assertEquals(5, bucket1.getDocCount());
+
+        Terms.Bucket bucket2 = terms.getBuckets().get(1);
+        assertThat(bucket2.getKeyAsString(), startsWith("v1.2."));
+        assertEquals(1, bucket2.getDocCount());
+
+        // Aggregate on the 'priority' subfield.
+        TermsAggregationBuilder priorityAgg = createTermsAgg("labels.priority");
+        SearchResponse priorityResponse = client().prepareSearch("test")
+            .addAggregation(priorityAgg)
+            .get();
+        assertSearchResponse(priorityResponse);
+
+        Terms priorityTerms = priorityResponse.getAggregations().get("terms");
+        assertThat(priorityTerms, notNullValue());
+        assertThat(priorityTerms.getName(), equalTo("terms"));
+        assertThat(priorityTerms.getBuckets().size(), equalTo(1));
+
+        Terms.Bucket priorityBucket = priorityTerms.getBuckets().get(0);
+        assertEquals("urgent", priorityBucket.getKey());
+        assertEquals(5, priorityBucket.getDocCount());
+
+        // Aggregate on the 'release' subfield.
+        TermsAggregationBuilder releaseAgg = createTermsAgg("labels.release");
+        SearchResponse releaseResponse = client().prepareSearch("test")
+            .addAggregation(releaseAgg)
+            .get();
+        assertSearchResponse(releaseResponse);
+
+        Terms releaseTerms = releaseResponse.getAggregations().get("terms");
+        assertThat(releaseTerms, notNullValue());
+        assertThat(releaseTerms.getName(), equalTo("terms"));
+        assertThat(releaseTerms.getBuckets().size(), equalTo(5));
+
+        for (Terms.Bucket bucket : releaseTerms.getBuckets()) {
+            assertThat(bucket.getKeyAsString(), startsWith("v1.2."));
+            assertEquals(1, bucket.getDocCount());
+        }
+
+        // Aggregate on the 'priority' subfield with a min_doc_count of 0.
+        TermsAggregationBuilder minDocCountAgg = createTermsAgg("labels.priority")
+            .minDocCount(0);
+        SearchResponse minDocCountResponse = client().prepareSearch("test")
+            .addAggregation(minDocCountAgg)
+            .get();
+        assertSearchResponse(minDocCountResponse);
+
+        Terms minDocCountTerms = minDocCountResponse.getAggregations().get("terms");
+        assertThat(minDocCountTerms, notNullValue());
+        assertThat(minDocCountTerms.getName(), equalTo("terms"));
+        assertThat(minDocCountTerms.getBuckets().size(), equalTo(1));
+    }
+
+    private TermsAggregationBuilder createTermsAgg(String field) {
+        TermsAggregatorFactory.ExecutionMode executionMode = randomFrom(
+            TermsAggregatorFactory.ExecutionMode.values());
+        Aggregator.SubAggCollectionMode collectionMode = randomFrom(
+            Aggregator.SubAggCollectionMode.values());
+
+        return terms("terms")
+            .field(field)
+            .collectMode(collectionMode)
+            .executionHint(executionMode.toString());
+    }
+
+
+    public void testLoadDocValuesFields() throws Exception {
+        client().prepareIndex("test", "_doc", "1")
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+            .setSource(XContentFactory.jsonBuilder()
+                .startObject()
+                .startObject("flat_object")
+                .field("key", "value")
+                .field("other_key", "other_value")
+                .endObject()
+                .endObject())
+            .get();
+
+        SearchResponse response = client().prepareSearch("test")
+            .addDocValueField("flat_object")
+            .addDocValueField("flat_object.key")
+            .get();
+        assertSearchResponse(response);
+        assertHitCount(response, 1);
+
+        Map<String, DocumentField> fields = response.getHits().getAt(0).getFields();
+
+        DocumentField field = fields.get("flat_object");
+        assertEquals("flat_object", field.getName());
+        assertEquals(Arrays.asList("other_value", "value"), field.getValues());
+
+        DocumentField keyedField = fields.get("flat_object.key");
+        assertEquals("flat_object.key", keyedField.getName());
+        assertEquals("value", keyedField.getValue());
+    }
+
+    public void testFieldSort() throws Exception {
+        client().prepareIndex("test", "_doc", "1")
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+            .setSource(XContentFactory.jsonBuilder()
+                .startObject()
+                .startObject("flat_object")
+                .field("key", "A")
+                .field("other_key", "D")
+                .endObject()
+                .endObject())
+            .get();
+
+        client().prepareIndex("test", "_doc", "2")
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+            .setSource(XContentFactory.jsonBuilder()
+                .startObject()
+                .startObject("flat_object")
+                .field("key", "B")
+                .field("other_key", "C")
+                .endObject()
+                .endObject())
+            .get();
+
+        client().prepareIndex("test", "_doc", "3")
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+            .setSource(XContentFactory.jsonBuilder()
+                .startObject()
+                .startObject("flat_object")
+                .field("other_key", "E")
+                .endObject()
+                .endObject())
+            .get();
+
+        SearchResponse response = client().prepareSearch("test")
+            .addSort("flat_object", SortOrder.DESC)
+            .get();
+        assertSearchResponse(response);
+        assertHitCount(response, 3);
+        assertOrderedSearchHits(response, "3", "1", "2");
+
+        response = client().prepareSearch("test")
+            .addSort("flat_object.key", SortOrder.DESC)
+            .get();
+        assertSearchResponse(response);
+        assertHitCount(response, 3);
+        assertOrderedSearchHits(response, "2", "1", "3");
+
+        response = client().prepareSearch("test")
+            .addSort(new FieldSortBuilder("flat_object.key").order(SortOrder.DESC).missing("Z"))
+            .get();
+        assertSearchResponse(response);
+        assertHitCount(response, 3);
+        assertOrderedSearchHits(response, "3", "2", "1");
+    }
+
+    public void testSourceFiltering() {
+        Map<String, Object> headers = new HashMap<>();
+        headers.put("content-type", "application/json");
+        headers.put("origin", "https://www.elastic.co");
+        Map<String, Object> source = Collections.singletonMap("headers", headers);
+
+        client().prepareIndex("test", "_doc", "1")
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+            .setSource(source)
+            .get();
+
+        SearchResponse response = client().prepareSearch("test").setFetchSource(true).get();
+        assertThat(response.getHits().getAt(0).getSourceAsMap(), equalTo(source));
+
+        // Check 'include' filtering.
+        response = client().prepareSearch("test").setFetchSource("headers", null).get();
+        assertThat(response.getHits().getAt(0).getSourceAsMap(), equalTo(source));
+
+        response = client().prepareSearch("test").setFetchSource("headers.content-type", null).get();
+        Map<String, Object> filteredSource = Collections.singletonMap("headers",
+            Collections.singletonMap("content-type", "application/json"));
+        assertThat(response.getHits().getAt(0).getSourceAsMap(), equalTo(filteredSource));
+
+        // Check 'exclude' filtering.
+        response = client().prepareSearch("test").setFetchSource(null, "headers.content-type").get();
+        filteredSource = Collections.singletonMap("headers",
+            Collections.singletonMap("origin", "https://www.elastic.co"));
+        assertThat(response.getHits().getAt(0).getSourceAsMap(), equalTo(filteredSource));
+    }
+}
diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectAtomicFieldDataTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectAtomicFieldDataTests.java
new file mode 100644
index 0000000000000..bb6c195395d4e
--- /dev/null
+++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectAtomicFieldDataTests.java
@@ -0,0 +1,204 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.flattened.mapper;
+
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues;
+import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
+import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+
+import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
+
+public class KeyedFlatObjectAtomicFieldDataTests extends ESTestCase {
+    private AtomicOrdinalsFieldData delegate;
+
+    @Before
+    public void setUpDelegate() {
+        BytesRef[] allTerms = new BytesRef[60];
+        long[] documentOrds = new long[50];
+        int index = 0;
+
+        for (int ord = 0; ord < allTerms.length; ord++) {
+            String key;
+            if (ord < 20) {
+                key = "apple";
+            } else if (ord < 30) {
+                key = "avocado";
+            } else if (ord < 40) {
+                key = "banana";
+            } else if (ord < 41) {
+                key = "cantaloupe";
+            } else {
+                key = "cucumber";
+            }
+
+            allTerms[ord] = prefixedValue(key, "value" + ord);
+
+            // Do not include the term 'avocado' in the mock document.
+            if (key.equals("avocado") == false) {
+                documentOrds[index++] = ord;
+            }
+        }
+
+        delegate = new MockAtomicOrdinalsFieldData(allTerms, documentOrds);
+    }
+
+    private BytesRef prefixedValue(String key, String value) {
+        String term = FlatObjectFieldParser.createKeyedValue(key, value);
+        return new BytesRef(term);
+    }
+
+    public void testFindOrdinalBounds() throws IOException {
+        testFindOrdinalBounds("apple", delegate, 0, 19);
+        testFindOrdinalBounds("avocado", delegate, 20, 29);
+        testFindOrdinalBounds("banana", delegate, 30, 39);
+        testFindOrdinalBounds("berry", delegate, -1, -1);
+        testFindOrdinalBounds("cantaloupe", delegate, 40, 40);
+        testFindOrdinalBounds("cucumber", delegate, 41, 59);
+
+        AtomicOrdinalsFieldData emptyDelegate = new MockAtomicOrdinalsFieldData(new BytesRef[0], new long[0]);
+        testFindOrdinalBounds("apple", emptyDelegate, -1, -1);
+
+        BytesRef[] terms = new BytesRef[] { prefixedValue("prefix", "value") };
+        AtomicOrdinalsFieldData singleValueDelegate = new MockAtomicOrdinalsFieldData(terms, new long[0]);
+        testFindOrdinalBounds("prefix", singleValueDelegate, 0, 0);
+        testFindOrdinalBounds("prefix1", singleValueDelegate, -1, -1);
+
+        terms = new BytesRef[] { prefixedValue("prefix", "value"),
+            prefixedValue("prefix1", "value"),
+            prefixedValue("prefix1", "value1"),
+            prefixedValue("prefix2", "value"),
+            prefixedValue("prefix3", "value")};
+        AtomicOrdinalsFieldData oddLengthDelegate = new MockAtomicOrdinalsFieldData(terms, new long[0]);
+        testFindOrdinalBounds("prefix", oddLengthDelegate, 0, 0);
+        testFindOrdinalBounds("prefix1", oddLengthDelegate, 1, 2);
+        testFindOrdinalBounds("prefix2", oddLengthDelegate, 3, 3);
+        testFindOrdinalBounds("prefix3", oddLengthDelegate, 4, 4);
+    }
+
+    public void testFindOrdinalBounds(String key,
+                                      AtomicOrdinalsFieldData delegate,
+                                      long expectedMinOrd,
+                                      long expectedMacOrd) throws IOException {
+        BytesRef bytesKey = new BytesRef(key);
+
+        long actualMinOrd = KeyedFlatObjectAtomicFieldData.findMinOrd(bytesKey, delegate.getOrdinalsValues());
+        assertEquals(expectedMinOrd,  actualMinOrd);
+
+        long actualMaxOrd = KeyedFlatObjectAtomicFieldData.findMaxOrd(bytesKey, delegate.getOrdinalsValues());
+        assertEquals(expectedMacOrd, actualMaxOrd);
+    }
+
+    public void testAdvanceExact() throws IOException {
+        AtomicOrdinalsFieldData avocadoFieldData = new KeyedFlatObjectAtomicFieldData("avocado", delegate);
+        assertFalse(avocadoFieldData.getOrdinalsValues().advanceExact(0));
+
+        AtomicOrdinalsFieldData bananaFieldData = new KeyedFlatObjectAtomicFieldData("banana", delegate);
+        assertTrue(bananaFieldData.getOrdinalsValues().advanceExact(0));
+
+        AtomicOrdinalsFieldData nonexistentFieldData = new KeyedFlatObjectAtomicFieldData("berry", delegate);
+        assertFalse(nonexistentFieldData.getOrdinalsValues().advanceExact(0));
+    }
+
+    public void testNextOrd() throws IOException {
+        AtomicOrdinalsFieldData fieldData = new KeyedFlatObjectAtomicFieldData("banana", delegate);
+        SortedSetDocValues docValues = fieldData.getOrdinalsValues();
+        docValues.advanceExact(0);
+
+        int retrievedOrds = 0;
+        for (long ord = docValues.nextOrd(); ord != NO_MORE_ORDS; ord = docValues.nextOrd()) {
+            assertTrue(0 <= ord && ord < 10);
+            retrievedOrds++;
+
+            BytesRef expectedValue = new BytesRef("value" + (ord + 30));
+            BytesRef actualValue = docValues.lookupOrd(ord);
+            assertEquals(expectedValue, actualValue);
+        }
+
+        assertEquals(10, retrievedOrds);
+    }
+
+    public void testLookupOrd() throws IOException {
+        AtomicOrdinalsFieldData appleFieldData = new KeyedFlatObjectAtomicFieldData("apple", delegate);
+        SortedSetDocValues appleDocValues = appleFieldData.getOrdinalsValues();
+        assertEquals(new BytesRef("value0"), appleDocValues.lookupOrd(0));
+
+        AtomicOrdinalsFieldData cantaloupeFieldData = new KeyedFlatObjectAtomicFieldData("cantaloupe", delegate);
+        SortedSetDocValues cantaloupeDocValues = cantaloupeFieldData.getOrdinalsValues();
+        assertEquals(new BytesRef("value40"), cantaloupeDocValues.lookupOrd(0));
+
+        AtomicOrdinalsFieldData cucumberFieldData = new KeyedFlatObjectAtomicFieldData("cucumber", delegate);
+        SortedSetDocValues cucumberDocValues = cucumberFieldData.getOrdinalsValues();
+        assertEquals(new BytesRef("value41"), cucumberDocValues.lookupOrd(0));
+    }
+
+    private static class MockAtomicOrdinalsFieldData extends AbstractAtomicOrdinalsFieldData {
+        private final SortedSetDocValues docValues;
+
+        MockAtomicOrdinalsFieldData(BytesRef[] allTerms,
+                                    long[] documentOrds) {
+            super(AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION);
+            this.docValues = new MockSortedSetDocValues(allTerms, documentOrds);
+        }
+
+        @Override
+        public SortedSetDocValues getOrdinalsValues() {
+            return docValues;
+        }
+
+        @Override
+        public long ramBytesUsed() {
+            return 0;
+        }
+
+        @Override
+        public void close() {
+            // Nothing to do.
+        }
+    }
+
+    private static class MockSortedSetDocValues extends AbstractSortedSetDocValues {
+        private final BytesRef[] allTerms;
+        private final long[] documentOrds;
+        private int index;
+
+        MockSortedSetDocValues(BytesRef[] allTerms,
+                               long[] documentOrds) {
+            this.allTerms = allTerms;
+            this.documentOrds = documentOrds;
+        }
+
+        @Override
+        public boolean advanceExact(int docID) {
+            index = 0;
+            return true;
+        }
+
+        @Override
+        public long nextOrd() {
+            if (index == documentOrds.length) {
+                return NO_MORE_ORDS;
+            }
+            return documentOrds[index++];
+        }
+
+        @Override
+        public BytesRef lookupOrd(long ord) {
+            return allTerms[(int) ord];
+        }
+
+        @Override
+        public long getValueCount() {
+            return allTerms.length;
+        }
+    }
+}
diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectFieldTypeTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectFieldTypeTests.java
new file mode 100644
index 0000000000000..46901035c8a96
--- /dev/null
+++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectFieldTypeTests.java
@@ -0,0 +1,155 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.flattened.mapper;
+
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermInSetQuery;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.index.mapper.FieldTypeTestCase;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.xpack.flattened.mapper.FlatObjectFieldMapper.KeyedFlatObjectFieldType;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class KeyedFlatObjectFieldTypeTests extends FieldTypeTestCase {
+
+    @Before
+    public void setupProperties() {
+        addModifier(new Modifier("split_queries_on_whitespace", true) {
+            @Override
+            public void modify(MappedFieldType type) {
+                KeyedFlatObjectFieldType ft = (KeyedFlatObjectFieldType) type;
+                ft.setSplitQueriesOnWhitespace(!ft.splitQueriesOnWhitespace());
+            }
+        });
+    }
+
+    @Override
+    protected KeyedFlatObjectFieldType createDefaultFieldType() {
+        return new KeyedFlatObjectFieldType("key");
+    }
+
+    public void testIndexedValueForSearch() {
+        KeyedFlatObjectFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        BytesRef keywordValue = ft.indexedValueForSearch("value");
+        assertEquals(new BytesRef("key\0value"), keywordValue);
+
+        BytesRef doubleValue = ft.indexedValueForSearch(2.718);
+        assertEquals(new BytesRef("key\0" + "2.718"), doubleValue);
+
+        BytesRef booleanValue = ft.indexedValueForSearch(true);
+        assertEquals(new BytesRef("key\0true"), booleanValue);
+    }
+
+    public void testTermQuery() {
+        KeyedFlatObjectFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        Query expected = new TermQuery(new Term("field", "key\0value"));
+        assertEquals(expected, ft.termQuery("value", null));
+
+        ft.setIndexOptions(IndexOptions.NONE);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> ft.termQuery("field", null));
+        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
+    }
+
+    public void testTermsQuery() {
+        KeyedFlatObjectFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        Query expected = new TermInSetQuery("field",
+            new BytesRef("key\0value1"),
+            new BytesRef("key\0value2"));
+
+        List<String> terms = new ArrayList<>();
+        terms.add("value1");
+        terms.add("value2");
+        Query actual = ft.termsQuery(terms, null);
+
+        assertEquals(expected, actual);
+    }
+
+    public void testExistsQuery() {
+        KeyedFlatObjectFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        Query expected = new PrefixQuery(new Term("field", "key\0"));
+        assertEquals(expected, ft.existsQuery(null));
+    }
+
+    public void testPrefixQuery() {
+        KeyedFlatObjectFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        Query expected = new PrefixQuery(new Term("field", "key\0val"));
+        assertEquals(expected, ft.prefixQuery("val", MultiTermQuery.CONSTANT_SCORE_REWRITE, null));
+    }
+
+    public void testFuzzyQuery() {
+        KeyedFlatObjectFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class,
+            () -> ft.fuzzyQuery("valuee", Fuzziness.fromEdits(2), 1, 50, true));
+        assertEquals("[fuzzy] queries are not currently supported on keyed [flattened] fields.", e.getMessage());
+    }
+
+    public void testRangeQuery() {
+        KeyedFlatObjectFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        TermRangeQuery expected = new TermRangeQuery("field",
+            new BytesRef("key\0lower"),
+            new BytesRef("key\0upper"), false, false);
+        assertEquals(expected, ft.rangeQuery("lower", "upper", false, false, null));
+
+        expected = new TermRangeQuery("field",
+            new BytesRef("key\0lower"),
+            new BytesRef("key\0upper"), true, true);
+        assertEquals(expected, ft.rangeQuery("lower", "upper", true, true, null));
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            ft.rangeQuery("lower", null, false, false, null));
+        assertEquals("[range] queries on keyed [flattened] fields must include both an upper and a lower bound.",
+            e.getMessage());
+
+        e = expectThrows(IllegalArgumentException.class, () ->
+            ft.rangeQuery(null, "upper", false, false, null));
+        assertEquals("[range] queries on keyed [flattened] fields must include both an upper and a lower bound.",
+            e.getMessage());
+    }
+
+    public void testRegexpQuery() {
+        KeyedFlatObjectFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class,
+            () -> ft.regexpQuery("valu*", 0, 10, null, null));
+        assertEquals("[regexp] queries are not currently supported on keyed [flattened] fields.", e.getMessage());
+    }
+
+    public void testWildcardQuery() {
+        KeyedFlatObjectFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class,
+            () -> ft.wildcardQuery("valu*", null, null));
+        assertEquals("[wildcard] queries are not currently supported on keyed [flattened] fields.", e.getMessage());
+    }
+}
diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/RootFlatObjectFieldTypeTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/RootFlatObjectFieldTypeTests.java
new file mode 100644
index 0000000000000..be297663c6e74
--- /dev/null
+++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/RootFlatObjectFieldTypeTests.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.flattened.mapper;
+
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
+import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.RegexpQuery;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
+import org.elasticsearch.index.mapper.FieldTypeTestCase;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.xpack.flattened.mapper.FlatObjectFieldMapper.RootFlatObjectFieldType;
+import org.junit.Before;
+
+public class RootFlatObjectFieldTypeTests extends FieldTypeTestCase {
+
+    @Before
+    public void setupProperties() {
+        addModifier(new Modifier("split_queries_on_whitespace", true) {
+            @Override
+            public void modify(MappedFieldType type) {
+                RootFlatObjectFieldType ft = (RootFlatObjectFieldType) type;
+                ft.setSplitQueriesOnWhitespace(!ft.splitQueriesOnWhitespace());
+            }
+        });
+    }
+
+    @Override
+    protected RootFlatObjectFieldType createDefaultFieldType() {
+        return new RootFlatObjectFieldType();
+    }
+
+    public void testValueForDisplay() {
+        RootFlatObjectFieldType ft = createDefaultFieldType();
+
+        String fieldValue = "{ \"key\": \"value\" }";
+        BytesRef storedValue = new BytesRef(fieldValue);
+        assertEquals(fieldValue, ft.valueForDisplay(storedValue));
+    }
+
+    public void testTermQuery() {
+        RootFlatObjectFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        Query expected = new TermQuery(new Term("field", "value"));
+        assertEquals(expected, ft.termQuery("value", null));
+
+        ft.setIndexOptions(IndexOptions.NONE);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> ft.termQuery("field", null));
+        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
+    }
+
+    public void testExistsQuery() {
+        RootFlatObjectFieldType ft = new RootFlatObjectFieldType();
+        ft.setName("field");
+        assertEquals(
+            new TermQuery(new Term(FieldNamesFieldMapper.NAME, new BytesRef("field"))),
+            ft.existsQuery(null));
+
+        ft.setHasDocValues(true);
+        assertEquals(new DocValuesFieldExistsQuery("field"), ft.existsQuery(null));
+    }
+
+    public void testFuzzyQuery() {
+        RootFlatObjectFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        Query expected = new FuzzyQuery(new Term("field", "value"), 2, 1, 50, true);
+        Query actual = ft.fuzzyQuery("value", Fuzziness.fromEdits(2), 1, 50, true);
+        assertEquals(expected, actual);
+    }
+
+    public void testRangeQuery() {
+        RootFlatObjectFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        TermRangeQuery expected = new TermRangeQuery("field",
+            new BytesRef("lower"),
+            new BytesRef("upper"), false, false);
+        assertEquals(expected, ft.rangeQuery("lower", "upper", false, false, null));
+
+        expected = new TermRangeQuery("field",
+            new BytesRef("lower"),
+            new BytesRef("upper"), true, true);
+        assertEquals(expected, ft.rangeQuery("lower", "upper", true, true, null));
+    }
+
+    public void testRegexpQuery() {
+        RootFlatObjectFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        Query expected = new RegexpQuery(new Term("field", "val.*"));
+        Query actual = ft.regexpQuery("val.*", 0, 10, null, null);
+        assertEquals(expected, actual);
+    }
+
+    public void testWildcardQuery() {
+        RootFlatObjectFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        Query expected = new WildcardQuery(new Term("field", new BytesRef("valu*")));
+        assertEquals(expected, ft.wildcardQuery("valu*", null, null));
+    }
+}
diff --git a/x-pack/plugin/ml/build.gradle b/x-pack/plugin/ml/build.gradle
index c09ced75bdf06..ec35e7fb8849f 100644
--- a/x-pack/plugin/ml/build.gradle
+++ b/x-pack/plugin/ml/build.gradle
@@ -31,6 +31,7 @@ configurations {
                 substitute module("org.elasticsearch.ml:ml-cpp") with project(":ml-cpp")
             }
         }
+        resolutionStrategy.cacheChangingModulesFor 2, 'hours'
     }
 }
 
@@ -49,9 +50,8 @@ compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,
 compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked"
 
 dependencies {
-    // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
+    compileOnly project(':modules:lang-painless:spi')
     compileOnly project(path: xpackModule('core'), configuration: 'default')
-    compileOnly "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${versions.elasticsearch}"
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     if (isEclipse) {
       testCompile project(path: xpackModule('core-tests'), configuration: 'testArtifacts')
@@ -60,10 +60,12 @@ dependencies {
     testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
 
     // ml deps
-    compile project(':libs:grok')
+    compile project(':libs:elasticsearch-grok')
     compile "com.ibm.icu:icu4j:${versions.icu4j}"
     compile "net.sf.supercsv:super-csv:${versions.supercsv}"
-    nativeBundle "org.elasticsearch.ml:ml-cpp:${project.version}@zip"
+    nativeBundle("org.elasticsearch.ml:ml-cpp:${project.version}@zip") {
+      changing = true
+    }
     testCompile 'org.ini4j:ini4j:0.5.2'
 }
 
@@ -75,7 +77,7 @@ task testJar(type: Jar) {
     from sourceSets.test.output
 }
 artifacts {
-    // normal es plugins do not publish the jar but we need to since users need it for Transport Clients and extensions
+    // normal es plugins do not publish the jar but we need to since users need it for extensions
     archives jar
     testArtifacts testJar
 }
diff --git a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle
index b47016c134459..9bbf34e544152 100644
--- a/x-pack/plugin/ml/qa/basic-multi-node/build.gradle
+++ b/x-pack/plugin/ml/qa/basic-multi-node/build.gradle
@@ -1,17 +1,19 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-    testCompile "org.elasticsearch.plugin:x-pack-core:${version}"
+    testCompile project(":x-pack:plugin:core")
     testCompile project(path: xpackModule('ml'), configuration: 'runtime')
 }
 
-integTestCluster {
+testClusters.integTest {
+    distribution = 'DEFAULT'
+    numberOfNodes = 3
     setting 'xpack.ilm.enabled', 'false'
     setting 'xpack.security.enabled', 'false'
     setting 'xpack.monitoring.enabled', 'false'
     setting 'xpack.watcher.enabled', 'false'
     setting 'xpack.ml.enabled', 'true'
     setting 'xpack.license.self_generated.type', 'trial'
-    numNodes = 3
 }
diff --git a/x-pack/plugin/ml/qa/disabled/build.gradle b/x-pack/plugin/ml/qa/disabled/build.gradle
index 2aa5d47acef0d..3e7f8b5ca7f50 100644
--- a/x-pack/plugin/ml/qa/disabled/build.gradle
+++ b/x-pack/plugin/ml/qa/disabled/build.gradle
@@ -1,14 +1,15 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-    testCompile "org.elasticsearch.plugin:x-pack-core:${version}"
+    testCompile project(":x-pack:plugin:core")
     testCompile project(path: xpackModule('ml'), configuration: 'runtime')
 }
 
-integTestCluster {
+testClusters.integTest {
+    distribution = 'DEFAULT'
     setting 'xpack.ilm.enabled', 'false'
     setting 'xpack.security.enabled', 'false'
     setting 'xpack.ml.enabled', 'false'
-    numNodes = 1
 }
diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
index 93dc8c4fec131..6077b8ab099f6 100644
--- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle
+++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
@@ -1,8 +1,8 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
   testCompile project(path: xpackModule('core'), configuration: 'default')
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
   if (isEclipse) {
@@ -18,7 +18,10 @@ task copyMlRestTests(type: Copy) {
   include 'rest-api-spec/test/ml/**'
 }
 
-integTestRunner {
+integTest.runner {
+  dependsOn copyMlRestTests
+}
+integTest.runner  {
   systemProperty 'tests.rest.blacklist', [
     // Remove this test because it doesn't call an ML endpoint and we don't want
     // to grant extra permissions to the users used in this test suite
@@ -38,6 +41,51 @@ integTestRunner {
     'ml/datafeeds_crud/Test put datafeed with invalid query',
     'ml/datafeeds_crud/Test put datafeed with security headers in the body',
     'ml/datafeeds_crud/Test update datafeed with missing id',
+    'ml/data_frame_analytics_crud/Test put config with security headers in the body',
+    'ml/data_frame_analytics_crud/Test put config with create_time in the body',
+    'ml/data_frame_analytics_crud/Test put config with version in the body',
+    'ml/data_frame_analytics_crud/Test put config with inconsistent body/param ids',
+    'ml/data_frame_analytics_crud/Test put config with invalid id',
+    'ml/data_frame_analytics_crud/Test put config with invalid dest index name',
+    'ml/data_frame_analytics_crud/Test put config with pattern dest index name',
+    'ml/data_frame_analytics_crud/Test put config with missing concrete source index',
+    'ml/data_frame_analytics_crud/Test put config with missing wildcard source index',
+    'ml/data_frame_analytics_crud/Test put config with dest index same as source index',
+    'ml/data_frame_analytics_crud/Test put config with dest index matching multiple indices',
+    'ml/data_frame_analytics_crud/Test put config with dest index included in source via alias',
+    'ml/data_frame_analytics_crud/Test put config with unknown top level field',
+    'ml/data_frame_analytics_crud/Test put config with unknown field in outlier detection analysis',
+    'ml/data_frame_analytics_crud/Test put config given missing source',
+    'ml/data_frame_analytics_crud/Test put config given source with empty index array',
+    'ml/data_frame_analytics_crud/Test put config given source with empty string in index array',
+    'ml/data_frame_analytics_crud/Test put config given source without index',
+    'ml/data_frame_analytics_crud/Test put config given missing dest',
+    'ml/data_frame_analytics_crud/Test put config given dest with empty index',
+    'ml/data_frame_analytics_crud/Test put config given dest without index',
+    'ml/data_frame_analytics_crud/Test put config given missing analysis',
+    'ml/data_frame_analytics_crud/Test put config given empty analysis',
+    'ml/data_frame_analytics_crud/Test get given missing analytics',
+    'ml/data_frame_analytics_crud/Test get given missing analytics and allow_no_match is false',
+    'ml/data_frame_analytics_crud/Test get given expression without matches and allow_no_match is false',
+    'ml/data_frame_analytics_crud/Test get stats given missing analytics',
+    'ml/data_frame_analytics_crud/Test get stats given missing analytics and allow_no_match is false',
+    'ml/data_frame_analytics_crud/Test get stats given expression without matches and allow_no_match is false',
+    'ml/data_frame_analytics_crud/Test delete given missing config',
+    'ml/data_frame_analytics_crud/Test max model memory limit',
+    'ml/evaluate_data_frame/Test given missing index',
+    'ml/evaluate_data_frame/Test given index does not exist',
+    'ml/evaluate_data_frame/Test given missing evaluation',
+    'ml/evaluate_data_frame/Test binary_soft_classifition auc_roc given actual_field is always true',
+    'ml/evaluate_data_frame/Test binary_soft_classifition auc_roc given actual_field is always false',
+    'ml/evaluate_data_frame/Test binary_soft_classification given evaluation with emtpy metrics',
+    'ml/evaluate_data_frame/Test binary_soft_classification given missing actual_field',
+    'ml/evaluate_data_frame/Test binary_soft_classification given missing predicted_probability_field',
+    'ml/evaluate_data_frame/Test binary_soft_classification given precision with threshold less than zero',
+    'ml/evaluate_data_frame/Test binary_soft_classification given recall with threshold less than zero',
+    'ml/evaluate_data_frame/Test binary_soft_classification given confusion_matrix with threshold less than zero',
+    'ml/evaluate_data_frame/Test binary_soft_classification given precision with empty thresholds',
+    'ml/evaluate_data_frame/Test binary_soft_classification given recall with empty thresholds',
+    'ml/evaluate_data_frame/Test binary_soft_classification given confusion_matrix with empty thresholds',
     'ml/delete_job_force/Test cannot force delete a non-existent job',
     'ml/delete_model_snapshot/Test delete snapshot missing snapshotId',
     'ml/delete_model_snapshot/Test delete snapshot missing job_id',
@@ -48,6 +96,7 @@ integTestRunner {
     'ml/filter_crud/Test invalid param combinations',
     'ml/filter_crud/Test non-existing filter',
     'ml/filter_crud/Test update filter given remove item is not present',
+    'ml/filter_crud/Test get all filter given index exists but no mapping for filter_id',
     'ml/get_datafeed_stats/Test get datafeed stats given missing datafeed_id',
     'ml/get_datafeeds/Test get datafeed given missing datafeed_id',
     'ml/jobs_crud/Test cannot create job with existing categorizer state document',
@@ -88,9 +137,17 @@ integTestRunner {
     'ml/post_data/Test POST data with invalid parameters',
     'ml/preview_datafeed/Test preview missing datafeed',
     'ml/revert_model_snapshot/Test revert model with invalid snapshotId',
+    'ml/start_data_frame_analytics/Test start given missing config',
+    'ml/start_data_frame_analytics/Test start given missing source index',
+    'ml/start_data_frame_analytics/Test start given source index has no compatible fields',
+    'ml/start_data_frame_analytics/Test start with inconsistent body/param ids',
     'ml/start_stop_datafeed/Test start datafeed job, but not open',
     'ml/start_stop_datafeed/Test start non existing datafeed',
     'ml/start_stop_datafeed/Test stop non existing datafeed',
+    'ml/stop_data_frame_analytics/Test stop given missing config and allow_no_match is true',
+    'ml/stop_data_frame_analytics/Test stop given missing config and allow_no_match is false',
+    'ml/stop_data_frame_analytics/Test stop with expression that does not match and allow_no_match is false',
+    'ml/stop_data_frame_analytics/Test stop with inconsistent body/param ids',
     'ml/update_model_snapshot/Test without description',
     'ml/validate/Test invalid job config',
     'ml/validate/Test job config is invalid because model snapshot id set',
@@ -106,27 +163,14 @@ integTestRunner {
   ].join(',')
 }
 
-integTestCluster {
-  dependsOn copyMlRestTests
-  extraConfigFile 'roles.yml', 'roles.yml'
-  setupCommand 'setupTestAdminUser',
-               'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'
-  setupCommand 'setupMlAdminUser',
-               'bin/elasticsearch-users', 'useradd', 'ml_admin', '-p', 'x-pack-test-password', '-r', 'minimal,machine_learning_admin'
-  setupCommand 'setupMlUserUser',
-               'bin/elasticsearch-users', 'useradd', 'ml_user', '-p', 'x-pack-test-password', '-r', 'minimal,machine_learning_user'
-  setupCommand 'setupPowerlessUser',
-               'bin/elasticsearch-users', 'useradd', 'no_ml', '-p', 'x-pack-test-password', '-r', 'minimal'
+
+testClusters.integTest  {
+  distribution = 'DEFAULT'
+  extraConfigFile 'roles.yml', file('roles.yml')
+  user username: "x_pack_rest_user", password: "x-pack-test-password"
+  user username: "ml_admin", password: "x-pack-test-password", role: "minimal,machine_learning_admin"
+  user username: "ml_user", password: "x-pack-test-password", role: "minimal,machine_learning_user"
+  user username: "no_ml", password: "x-pack-test-password", role: "minimal"
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'xpack.security.enabled', 'true'
-  waitCondition = { node, ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-            dest: tmpFile.toString(),
-            username: 'x_pack_rest_user',
-            password: 'x-pack-test-password',
-            ignoreerrors: true,
-            retries: 10)
-    return tmpFile.exists()
-  }
 }
diff --git a/x-pack/plugin/ml/qa/ml-with-security/roles.yml b/x-pack/plugin/ml/qa/ml-with-security/roles.yml
index e47fe40a120cd..8533b81c07377 100644
--- a/x-pack/plugin/ml/qa/ml-with-security/roles.yml
+++ b/x-pack/plugin/ml/qa/ml-with-security/roles.yml
@@ -11,7 +11,7 @@ minimal:
       privileges:
         - indices:admin/create
         - indices:admin/refresh
-        - indices:data/read/field_caps
-        - indices:data/read/search
+        - read
+        - index
         - indices:data/write/bulk
         - indices:data/write/index
diff --git a/x-pack/plugin/ml/qa/ml-with-security/src/test/java/org/elasticsearch/smoketest/MlWithSecurityUserRoleIT.java b/x-pack/plugin/ml/qa/ml-with-security/src/test/java/org/elasticsearch/smoketest/MlWithSecurityUserRoleIT.java
index 67b72a648db60..803281257d25a 100644
--- a/x-pack/plugin/ml/qa/ml-with-security/src/test/java/org/elasticsearch/smoketest/MlWithSecurityUserRoleIT.java
+++ b/x-pack/plugin/ml/qa/ml-with-security/src/test/java/org/elasticsearch/smoketest/MlWithSecurityUserRoleIT.java
@@ -6,18 +6,30 @@
 package org.elasticsearch.smoketest;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
-
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.section.DoSection;
 import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Pattern;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.either;
 
 public class MlWithSecurityUserRoleIT extends MlWithSecurityIT {
 
+    /**
+     * These are actions that require the monitor role and/or access to the relevant source index.
+     * ml_user should have both of these in the tests.
+     */
+    private static final List<Pattern> ALLOWED_ACTION_PATTERNS = Arrays.asList(
+        Pattern.compile("ml\\.get_.*"),
+        Pattern.compile("ml\\.find_file_structure"),
+        Pattern.compile("ml\\.evaluate_data_frame")
+    );
+
     private final ClientYamlTestCandidate testCandidate;
 
     public MlWithSecurityUserRoleIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
@@ -30,14 +42,12 @@ public void test() throws IOException {
         try {
             super.test();
 
-            // We should have got here if and only if the only ML endpoints in the test were GETs
-            // or the find_file_structure API, which is also available to the machine_learning_user
-            // role
+            // We should have got here if and only if the only ML endpoints in the test were in the allowed list
             for (ExecutableSection section : testCandidate.getTestSection().getExecutableSections()) {
                 if (section instanceof DoSection) {
-                    if (((DoSection) section).getApiCallSection().getApi().startsWith("ml.") &&
-                            ((DoSection) section).getApiCallSection().getApi().startsWith("ml.get_") == false &&
-                            ((DoSection) section).getApiCallSection().getApi().equals("ml.find_file_structure") == false) {
+                    String apiName = ((DoSection) section).getApiCallSection().getApi();
+
+                    if (((DoSection) section).getApiCallSection().getApi().startsWith("ml.") && isAllowed(apiName) == false) {
                         fail("should have failed because of missing role");
                     }
                 }
@@ -50,6 +60,15 @@ public void test() throws IOException {
         }
     }
 
+    private static boolean isAllowed(String apiName) {
+        for (Pattern pattern : ALLOWED_ACTION_PATTERNS) {
+            if (pattern.matcher(apiName).find()) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     @Override
     protected String[] getCredentials() {
         return new String[]{"ml_user", "x-pack-test-password"};
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle
index e6fd8412c948b..82b8e5cd7e101 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle
@@ -1,8 +1,8 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-    // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
     testCompile project(path: xpackModule('core'), configuration: 'default')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     if (isEclipse) {
@@ -12,14 +12,6 @@ dependencies {
     testCompile project(path: xpackModule('ml'), configuration: 'testArtifacts')
 }
 
-integTestRunner {
-    /*
-     * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
-     * other if we allow them to set the number of available processors as it's set-once in Netty.
-     */
-    systemProperty 'es.set.netty.runtime.available.processors', 'false'
-}
-
 // location for keys and certificates
 File keystoreDir = new File(project.buildDir, 'keystore')
 File nodeKey = file("$keystoreDir/testnode.pem")
@@ -35,12 +27,24 @@ task copyKeyCerts(type: Copy) {
 sourceSets.test.resources.srcDir(keystoreDir)
 processTestResources.dependsOn(copyKeyCerts)
 
-integTestCluster {
+integTest {
     dependsOn copyKeyCerts
+    runner {
+        /*
+         * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
+         * other if we allow them to set the number of available processors as it's set-once in Netty.
+         */
+        systemProperty 'es.set.netty.runtime.available.processors', 'false'
+    }
+}
+
+testClusters.integTest {
+    numberOfNodes = 3
+    distribution = 'DEFAULT'
+
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.ml.enabled', 'true'
     setting 'xpack.watcher.enabled', 'false'
-    setting 'logger.org.elasticsearch.xpack.ml.datafeed', 'TRACE'
     setting 'xpack.monitoring.enabled', 'false'
     setting 'xpack.security.authc.token.enabled', 'true'
     setting 'xpack.security.transport.ssl.enabled', 'true'
@@ -51,25 +55,11 @@ integTestCluster {
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'xpack.ml.min_disk_space_off_heap', '200mb'
 
-    keystoreSetting 'bootstrap.password', 'x-pack-test-password'
-    keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
-
-    numNodes = 3
+    keystore 'bootstrap.password', 'x-pack-test-password'
+    keystore 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
 
-    setupCommand 'setupDummyUser',
-            'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'
+    user username: "x_pack_rest_user", password: "x-pack-test-password"
 
     extraConfigFile nodeKey.name, nodeKey
     extraConfigFile nodeCert.name, nodeCert
-
-    waitCondition = { node, ant ->
-        File tmpFile = new File(node.cwd, 'wait.success')
-        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-                dest: tmpFile.toString(),
-                username: 'x_pack_rest_user',
-                password: 'x-pack-test-password',
-                ignoreerrors: true,
-                retries: 10)
-        return tmpFile.exists()
-    }
 }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java
index 98d089e544b4e..a01d23fca62e0 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java
@@ -38,6 +38,7 @@ public void cleanUpTest() {
         cleanUp();
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/43013")
     public void testTooManyPartitions() throws Exception {
         assumeFalse("AwaitsFix(bugUrl = \"https://github.com/elastic/elasticsearch/issues/32033\")", Constants.WINDOWS);
         Detector.Builder detector = new Detector.Builder("count", null);
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java
index 426b58f686419..114f1ecd0aa83 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java
@@ -321,6 +321,71 @@ public void testLookbackOnlyWithNestedFields() throws Exception {
         assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0"));
     }
 
+    public void testLookbackWithGeo() throws Exception {
+        String jobId = "test-lookback-only-with-geo";
+        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
+        createJobRequest.setJsonEntity("{\n"
+            + "  \"description\": \"lat_long with geo_point\",\n"
+            + "  \"analysis_config\": {\n"
+            + "    \"bucket_span\": \"15m\",\n"
+            + "    \"detectors\": [\n"
+            + "      {\n"
+            + "        \"function\": \"lat_long\",\n"
+            + "        \"field_name\": \"location\"\n"
+            + "      }\n"
+            + "    ]\n"
+            + "  },"
+            + "  \"data_description\": {\"time_field\": \"time\"}\n"
+            + "}");
+        client().performRequest(createJobRequest);
+        String datafeedId = jobId + "-datafeed";
+        new DatafeedBuilder(datafeedId, jobId, "geo-data").build();
+
+        StringBuilder bulk = new StringBuilder();
+
+        Request createGeoData = new Request("PUT", "/geo-data");
+        createGeoData.setJsonEntity("{"
+            + "  \"mappings\": {"
+            + "    \"properties\": {"
+            + "      \"time\": { \"type\":\"date\"},"
+            + "      \"location\": { \"type\":\"geo_point\"}"
+            + "    }"
+            + "  }"
+            + "}");
+        client().performRequest(createGeoData);
+
+        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 1}}\n");
+        bulk.append("{\"time\":\"2016-06-01T00:00:00Z\",\"location\":{\"lat\":38.897676,\"lon\":-77.03653}}\n");
+        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 2}}\n");
+        bulk.append("{\"time\":\"2016-06-01T00:05:00Z\",\"location\":{\"lat\":38.897676,\"lon\":-77.03653}}\n");
+        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 3}}\n");
+        bulk.append("{\"time\":\"2016-06-01T00:10:00Z\",\"location\":{\"lat\":38.897676,\"lon\":-77.03653}}\n");
+        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 4}}\n");
+        bulk.append("{\"time\":\"2016-06-01T00:15:00Z\",\"location\":{\"lat\":38.897676,\"lon\":-77.03653}}\n");
+        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 5}}\n");
+        bulk.append("{\"time\":\"2016-06-01T00:20:00Z\",\"location\":{\"lat\":38.897676,\"lon\":-77.03653}}\n");
+        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 6}}\n");
+        bulk.append("{\"time\":\"2016-06-01T00:25:00Z\",\"location\":{\"lat\":38.897676,\"lon\":-77.03653}}\n");
+        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 7}}\n");
+        bulk.append("{\"time\":\"2016-06-01T00:30:00Z\",\"location\":{\"lat\":38.897676,\"lon\":-77.03653}}\n");
+        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 8}}\n");
+        bulk.append("{\"time\":\"2016-06-01T00:40:00Z\",\"location\":{\"lat\":90.0,\"lon\":-77.03653}}\n");
+        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 9}}\n");
+        bulk.append("{\"time\":\"2016-06-01T00:41:00Z\",\"location\":{\"lat\":38.897676,\"lon\":-77.03653}}\n");
+        bulkIndex(bulk.toString());
+
+        openJob(client(), jobId);
+
+        startDatafeedAndWaitUntilStopped(datafeedId);
+        waitUntilJobIsClosed(jobId);
+        Response jobStatsResponse = client().performRequest(
+            new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
+        String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
+        assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":9"));
+        assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":9"));
+        assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0"));
+    }
+
     public void testLookbackOnlyGivenEmptyIndex() throws Exception {
         new LookbackOnlyTestHelper("test-lookback-only-given-empty-index", "airline-data-empty")
                 .setShouldSucceedInput(false).setShouldSucceedProcessing(false).execute();
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
index eb89c15e146fe..d32608a47eb44 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
@@ -91,7 +91,6 @@ public void testDeleteExpiredDataGivenNothingToDelete() throws Exception {
         client().execute(DeleteExpiredDataAction.INSTANCE, new DeleteExpiredDataAction.Request()).get();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/ml-cpp/pulls/468")
     public void testDeleteExpiredData() throws Exception {
         // Index some unused state documents (more than 10K to test scrolling works)
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java
new file mode 100644
index 0000000000000..87e723db04896
--- /dev/null
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.integration;
+
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.xpack.core.ml.action.DeleteDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction;
+import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.action.StopDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Base class of ML integration tests that use a native data_frame_analytics process
+ */
+abstract class MlNativeDataFrameAnalyticsIntegTestCase extends MlNativeIntegTestCase {
+
+    private List<DataFrameAnalyticsConfig> analytics = new ArrayList<>();
+
+    @Override
+    protected void cleanUpResources() {
+        cleanUpAnalytics();
+    }
+
+    private void cleanUpAnalytics() {
+        for (DataFrameAnalyticsConfig config : analytics) {
+            try {
+                deleteAnalytics(config.getId());
+            } catch (Exception e) {
+                // ignore
+            }
+        }
+    }
+
+    protected void registerAnalytics(DataFrameAnalyticsConfig config) {
+        if (analytics.add(config) == false) {
+            throw new IllegalArgumentException("analytics config [" + config.getId() + "] is already registered");
+        }
+    }
+
+    protected PutDataFrameAnalyticsAction.Response putAnalytics(DataFrameAnalyticsConfig config) {
+        PutDataFrameAnalyticsAction.Request request = new PutDataFrameAnalyticsAction.Request(config);
+        return client().execute(PutDataFrameAnalyticsAction.INSTANCE, request).actionGet();
+    }
+
+    protected AcknowledgedResponse deleteAnalytics(String id) {
+        DeleteDataFrameAnalyticsAction.Request request = new DeleteDataFrameAnalyticsAction.Request(id);
+        return client().execute(DeleteDataFrameAnalyticsAction.INSTANCE, request).actionGet();
+    }
+
+    protected AcknowledgedResponse startAnalytics(String id) {
+        StartDataFrameAnalyticsAction.Request request = new StartDataFrameAnalyticsAction.Request(id);
+        return client().execute(StartDataFrameAnalyticsAction.INSTANCE, request).actionGet();
+    }
+
+    protected StopDataFrameAnalyticsAction.Response stopAnalytics(String id) {
+        StopDataFrameAnalyticsAction.Request request = new StopDataFrameAnalyticsAction.Request(id);
+        return client().execute(StopDataFrameAnalyticsAction.INSTANCE, request).actionGet();
+    }
+
+    protected void waitUntilAnalyticsIsStopped(String id) throws Exception {
+        waitUntilAnalyticsIsStopped(id, TimeValue.timeValueSeconds(30));
+    }
+
+    protected void waitUntilAnalyticsIsStopped(String id, TimeValue waitTime) throws Exception {
+        assertBusy(() -> assertThat(getAnalyticsStats(id).get(0).getState(), equalTo(DataFrameAnalyticsState.STOPPED)),
+                waitTime.getMillis(), TimeUnit.MILLISECONDS);
+    }
+
+    protected List<DataFrameAnalyticsConfig> getAnalytics(String id) {
+        GetDataFrameAnalyticsAction.Request request = new GetDataFrameAnalyticsAction.Request(id);
+        return client().execute(GetDataFrameAnalyticsAction.INSTANCE, request).actionGet().getResources().results();
+    }
+
+    protected List<GetDataFrameAnalyticsStatsAction.Response.Stats> getAnalyticsStats(String id) {
+        GetDataFrameAnalyticsStatsAction.Request request = new GetDataFrameAnalyticsStatsAction.Request(id);
+        GetDataFrameAnalyticsStatsAction.Response response = client().execute(GetDataFrameAnalyticsStatsAction.INSTANCE, request)
+            .actionGet();
+        return response.getResponse().results();
+    }
+
+    protected List<String> generateData(long timestamp, TimeValue bucketSpan, int bucketCount,
+                                      Function<Integer, Integer> timeToCountFunction) throws IOException {
+        List<String> data = new ArrayList<>();
+        long now = timestamp;
+        for (int bucketIndex = 0; bucketIndex < bucketCount; bucketIndex++) {
+            for (int count = 0; count < timeToCountFunction.apply(bucketIndex); count++) {
+                Map<String, Object> record = new HashMap<>();
+                record.put("time", now);
+                data.add(createJsonRecord(record));
+            }
+            now += bucketSpan.getMillis();
+        }
+        return data;
+    }
+
+    protected static String createJsonRecord(Map<String, Object> keyValueMap) throws IOException {
+        return Strings.toString(JsonXContent.contentBuilder().map(keyValueMap)) + "\n";
+    }
+}
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java
index 128bcce67994b..b26387d0b3d72 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java
@@ -31,8 +31,10 @@
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
+import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
 import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
 import org.elasticsearch.xpack.core.security.SecurityField;
 import org.elasticsearch.xpack.core.security.authc.TokenMetaData;
@@ -63,7 +65,7 @@ abstract class MlNativeIntegTestCase extends ESIntegTestCase {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
@@ -79,10 +81,7 @@ protected Function<Client, Client> getClientWrapper() {
         // we need to wrap node clients because we do not specify a user for nodes and all requests will use the system
         // user. This is ok for internal n2n stuff but the test framework does other things like wiping indices, repositories, etc
         // that the system user cannot do. so we wrap the node client with a user that can do these things since the client() calls
-        // are randomized to return both node clients and transport clients
-        // transport clients do not need to be wrapped since we specify the xpack.security.user setting that sets the default user to be
-        // used for the transport client. If we did not set a default user then the transport client would not even be allowed
-        // to connect
+        // return a node client
         return client -> client.filterWithHeader(headers);
     }
     @Override
@@ -147,14 +146,18 @@ private void waitForPendingTasks() {
     protected void ensureClusterStateConsistency() throws IOException {
         if (cluster() != null && cluster().size() > 0) {
             List<NamedWriteableRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedWriteables());
-            entries.addAll(new SearchModule(Settings.EMPTY, true, Collections.emptyList()).getNamedWriteables());
+            entries.addAll(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedWriteables());
             entries.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new));
             entries.add(new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.DATAFEED_TASK_NAME,
                     StartDatafeedAction.DatafeedParams::new));
+            entries.add(new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME,
+                StartDataFrameAnalyticsAction.TaskParams::new));
             entries.add(new NamedWriteableRegistry.Entry(PersistentTaskParams.class, MlTasks.JOB_TASK_NAME,
                     OpenJobAction.JobParams::new));
             entries.add(new NamedWriteableRegistry.Entry(PersistentTaskState.class, JobTaskState.NAME, JobTaskState::new));
             entries.add(new NamedWriteableRegistry.Entry(PersistentTaskState.class, DatafeedState.NAME, DatafeedState::fromStream));
+            entries.add(new NamedWriteableRegistry.Entry(PersistentTaskState.class, DataFrameAnalyticsTaskState.NAME,
+                DataFrameAnalyticsTaskState::new));
             entries.add(new NamedWriteableRegistry.Entry(ClusterState.Custom.class, TokenMetaData.TYPE, TokenMetaData::new));
             final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries);
             ClusterState masterClusterState = client().admin().cluster().prepareState().all().get().getState();
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/PersistJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/PersistJobIT.java
index a68fa2fe02a8d..b9666306911c7 100644
--- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/PersistJobIT.java
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/PersistJobIT.java
@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.stream.Collectors;
 
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
@@ -178,13 +179,12 @@ public void testPersistJobOnGracefulShutdown_givenNoDataAndNoTimeAdvance() throw
         closeJob(jobId);
 
         // Check that state has not been persisted
-        SearchResponse stateDocsResponse = client().prepareSearch(AnomalyDetectorsIndex.jobStateIndexPattern())
-            .setFetchSource(false)
-            .setTrackTotalHits(true)
-            .setSize(10000)
-            .get();
+        SearchResponse stateDocsResponse = client().prepareSearch(AnomalyDetectorsIndex.jobStateIndexPattern()).get();
+        assertThat(Arrays.asList(stateDocsResponse.getHits().getHits()), empty());
 
-        assertThat(stateDocsResponse.getHits().getTotalHits().value, equalTo(0L));
+        // Check that results have not been persisted
+        SearchResponse resultsDocsResponse = client().prepareSearch(AnomalyDetectorsIndex.jobResultsAliasedName(jobId)).get();
+        assertThat(Arrays.asList(resultsDocsResponse.getHits().getHits()), empty());
 
         deleteJob(jobId);
     }
diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java
new file mode 100644
index 0000000000000..c7295ce24db56
--- /dev/null
+++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RunDataFrameAnalyticsIT.java
@@ -0,0 +1,386 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.integration;
+
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsSource;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetection;
+import org.junit.After;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class RunDataFrameAnalyticsIT extends MlNativeDataFrameAnalyticsIntegTestCase {
+
+    @After
+    public void cleanup() {
+        cleanUp();
+    }
+
+    public void testOutlierDetectionWithFewDocuments() throws Exception {
+        String sourceIndex = "test-outlier-detection-with-few-docs";
+
+        client().admin().indices().prepareCreate(sourceIndex)
+            .addMapping("_doc", "numeric_1", "type=double", "numeric_2", "type=float", "categorical_1", "type=keyword")
+            .get();
+
+        BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
+        bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+
+        for (int i = 0; i < 5; i++) {
+            IndexRequest indexRequest = new IndexRequest(sourceIndex);
+
+            // We insert one odd value out of 5 for one feature
+            String docId = i == 0 ? "outlier" : "normal" + i;
+            indexRequest.id(docId);
+            indexRequest.source("numeric_1", i == 0 ? 100.0 : 1.0, "numeric_2", 1.0, "categorical_1", "foo_" + i);
+            bulkRequestBuilder.add(indexRequest);
+        }
+        BulkResponse bulkResponse = bulkRequestBuilder.get();
+        if (bulkResponse.hasFailures()) {
+            fail("Failed to index data: " + bulkResponse.buildFailureMessage());
+        }
+
+        String id = "test_outlier_detection_with_few_docs";
+        DataFrameAnalyticsConfig config = buildOutlierDetectionAnalytics(id, new String[] {sourceIndex}, sourceIndex + "-results", null);
+        registerAnalytics(config);
+        putAnalytics(config);
+
+        assertState(id, DataFrameAnalyticsState.STOPPED);
+
+        startAnalytics(id);
+        waitUntilAnalyticsIsStopped(id);
+
+        SearchResponse sourceData = client().prepareSearch(sourceIndex).get();
+        double scoreOfOutlier = 0.0;
+        double scoreOfNonOutlier = -1.0;
+        for (SearchHit hit : sourceData.getHits()) {
+            GetResponse destDocGetResponse = client().prepareGet().setIndex(config.getDest().getIndex()).setId(hit.getId()).get();
+            assertThat(destDocGetResponse.isExists(), is(true));
+            Map<String, Object> sourceDoc = hit.getSourceAsMap();
+            Map<String, Object> destDoc = destDocGetResponse.getSource();
+            for (String field : sourceDoc.keySet()) {
+                assertThat(destDoc.containsKey(field), is(true));
+                assertThat(destDoc.get(field), equalTo(sourceDoc.get(field)));
+            }
+            assertThat(destDoc.containsKey("ml"), is(true));
+
+            @SuppressWarnings("unchecked")
+            Map<String, Object> resultsObject = (Map<String, Object>) destDoc.get("ml");
+
+            assertThat(resultsObject.containsKey("outlier_score"), is(true));
+            double outlierScore = (double) resultsObject.get("outlier_score");
+            assertThat(outlierScore, allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(100.0)));
+            if (hit.getId().equals("outlier")) {
+                scoreOfOutlier = outlierScore;
+            } else {
+                if (scoreOfNonOutlier < 0) {
+                    scoreOfNonOutlier = outlierScore;
+                } else {
+                    assertThat(outlierScore, equalTo(scoreOfNonOutlier));
+                }
+            }
+        }
+        assertThat(scoreOfOutlier, is(greaterThan(scoreOfNonOutlier)));
+    }
+
+    public void testOutlierDetectionWithEnoughDocumentsToScroll() throws Exception {
+        String sourceIndex = "test-outlier-detection-with-enough-docs-to-scroll";
+
+        client().admin().indices().prepareCreate(sourceIndex)
+            .addMapping("_doc", "numeric_1", "type=double", "numeric_2", "type=float", "categorical_1", "type=keyword")
+            .get();
+
+        BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
+        bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+
+        int docCount = randomIntBetween(1024, 2048);
+        for (int i = 0; i < docCount; i++) {
+            IndexRequest indexRequest = new IndexRequest(sourceIndex);
+            indexRequest.source("numeric_1", randomDouble(), "numeric_2", randomFloat(), "categorical_1", randomAlphaOfLength(10));
+            bulkRequestBuilder.add(indexRequest);
+        }
+        BulkResponse bulkResponse = bulkRequestBuilder.get();
+        if (bulkResponse.hasFailures()) {
+            fail("Failed to index data: " + bulkResponse.buildFailureMessage());
+        }
+
+        String id = "test_outlier_detection_with_enough_docs_to_scroll";
+        DataFrameAnalyticsConfig config = buildOutlierDetectionAnalytics(
+                id, new String[] {sourceIndex}, sourceIndex + "-results", "custom_ml");
+        registerAnalytics(config);
+        putAnalytics(config);
+
+        assertState(id, DataFrameAnalyticsState.STOPPED);
+
+        startAnalytics(id);
+        waitUntilAnalyticsIsStopped(id);
+
+        // Check we've got all docs
+        SearchResponse searchResponse = client().prepareSearch(config.getDest().getIndex()).setTrackTotalHits(true).get();
+        assertThat(searchResponse.getHits().getTotalHits().value, equalTo((long) docCount));
+
+        // Check they all have an outlier_score
+        searchResponse = client().prepareSearch(config.getDest().getIndex())
+            .setTrackTotalHits(true)
+            .setQuery(QueryBuilders.existsQuery("custom_ml.outlier_score")).get();
+        assertThat(searchResponse.getHits().getTotalHits().value, equalTo((long) docCount));
+    }
+
+    public void testOutlierDetectionWithMoreFieldsThanDocValueFieldLimit() throws Exception {
+        String sourceIndex = "test-outlier-detection-with-more-fields-than-docvalue-limit";
+
+        client().admin().indices().prepareCreate(sourceIndex).get();
+
+        GetSettingsRequest getSettingsRequest = new GetSettingsRequest();
+        getSettingsRequest.indices(sourceIndex);
+        getSettingsRequest.names(IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING.getKey());
+        getSettingsRequest.includeDefaults(true);
+
+        GetSettingsResponse docValueLimitSetting = client().admin().indices().getSettings(getSettingsRequest).actionGet();
+        int docValueLimit = IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING.get(
+            docValueLimitSetting.getIndexToSettings().values().iterator().next().value);
+
+        BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
+        bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+
+        for (int i = 0; i < 100; i++) {
+
+            StringBuilder source = new StringBuilder("{");
+            for (int fieldCount = 0; fieldCount < docValueLimit + 1; fieldCount++) {
+                source.append("\"field_").append(fieldCount).append("\":").append(randomDouble());
+                if (fieldCount < docValueLimit) {
+                    source.append(",");
+                }
+            }
+            source.append("}");
+
+            IndexRequest indexRequest = new IndexRequest(sourceIndex);
+            indexRequest.source(source.toString(), XContentType.JSON);
+            bulkRequestBuilder.add(indexRequest);
+        }
+        BulkResponse bulkResponse = bulkRequestBuilder.get();
+        if (bulkResponse.hasFailures()) {
+            fail("Failed to index data: " + bulkResponse.buildFailureMessage());
+        }
+
+        String id = "test_outlier_detection_with_more_fields_than_docvalue_limit";
+        DataFrameAnalyticsConfig config = buildOutlierDetectionAnalytics(id, new String[] {sourceIndex}, sourceIndex + "-results", null);
+        registerAnalytics(config);
+        putAnalytics(config);
+
+        assertState(id, DataFrameAnalyticsState.STOPPED);
+
+        startAnalytics(id);
+        waitUntilAnalyticsIsStopped(id);
+
+        SearchResponse sourceData = client().prepareSearch(sourceIndex).get();
+        for (SearchHit hit : sourceData.getHits()) {
+            GetResponse destDocGetResponse = client().prepareGet().setIndex(config.getDest().getIndex()).setId(hit.getId()).get();
+            assertThat(destDocGetResponse.isExists(), is(true));
+            Map<String, Object> sourceDoc = hit.getSourceAsMap();
+            Map<String, Object> destDoc = destDocGetResponse.getSource();
+            for (String field : sourceDoc.keySet()) {
+                assertThat(destDoc.containsKey(field), is(true));
+                assertThat(destDoc.get(field), equalTo(sourceDoc.get(field)));
+            }
+            assertThat(destDoc.containsKey("ml"), is(true));
+
+            @SuppressWarnings("unchecked")
+            Map<String, Object> resultsObject = (Map<String, Object>) destDoc.get("ml");
+
+            assertThat(resultsObject.containsKey("outlier_score"), is(true));
+            double outlierScore = (double) resultsObject.get("outlier_score");
+            assertThat(outlierScore, allOf(greaterThanOrEqualTo(0.0), lessThanOrEqualTo(100.0)));
+        }
+    }
+
+    public void testStopOutlierDetectionWithEnoughDocumentsToScroll() {
+        String sourceIndex = "test-stop-outlier-detection-with-enough-docs-to-scroll";
+
+        client().admin().indices().prepareCreate(sourceIndex)
+            .addMapping("_doc", "numeric_1", "type=double", "numeric_2", "type=float", "categorical_1", "type=keyword")
+            .get();
+
+        BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
+        bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+
+        int docCount = randomIntBetween(1024, 2048);
+        for (int i = 0; i < docCount; i++) {
+            IndexRequest indexRequest = new IndexRequest(sourceIndex);
+            indexRequest.source("numeric_1", randomDouble(), "numeric_2", randomFloat(), "categorical_1", randomAlphaOfLength(10));
+            bulkRequestBuilder.add(indexRequest);
+        }
+        BulkResponse bulkResponse = bulkRequestBuilder.get();
+        if (bulkResponse.hasFailures()) {
+            fail("Failed to index data: " + bulkResponse.buildFailureMessage());
+        }
+
+        String id = "test_stop_outlier_detection_with_enough_docs_to_scroll";
+        DataFrameAnalyticsConfig config = buildOutlierDetectionAnalytics(
+                id, new String[] {sourceIndex}, sourceIndex + "-results", "custom_ml");
+        registerAnalytics(config);
+        putAnalytics(config);
+
+        assertState(id, DataFrameAnalyticsState.STOPPED);
+        startAnalytics(id);
+        assertState(id, DataFrameAnalyticsState.STARTED);
+
+        assertThat(stopAnalytics(id).isStopped(), is(true));
+        assertState(id, DataFrameAnalyticsState.STOPPED);
+        if (indexExists(config.getDest().getIndex()) == false) {
+            // We stopped before we even created the destination index
+            return;
+        }
+
+        SearchResponse searchResponse = client().prepareSearch(config.getDest().getIndex()).setTrackTotalHits(true).get();
+        if (searchResponse.getHits().getTotalHits().value == docCount) {
+            searchResponse = client().prepareSearch(config.getDest().getIndex())
+                .setTrackTotalHits(true)
+                .setQuery(QueryBuilders.existsQuery("custom_ml.outlier_score")).get();
+            logger.debug("We stopped during analysis: [{}] < [{}]", searchResponse.getHits().getTotalHits().value, docCount);
+            assertThat(searchResponse.getHits().getTotalHits().value, lessThan((long) docCount));
+        } else {
+            logger.debug("We stopped during reindexing: [{}] < [{}]", searchResponse.getHits().getTotalHits().value, docCount);
+        }
+    }
+
+    public void testOutlierDetectionWithMultipleSourceIndices() throws Exception {
+        String sourceIndex1 = "test-outlier-detection-with-multiple-source-indices-1";
+        String sourceIndex2 = "test-outlier-detection-with-multiple-source-indices-2";
+        String destIndex = "test-outlier-detection-with-multiple-source-indices-results";
+        String[] sourceIndex = new String[] { sourceIndex1, sourceIndex2 };
+
+        client().admin().indices().prepareCreate(sourceIndex1)
+            .addMapping("_doc", "numeric_1", "type=double", "numeric_2", "type=float", "categorical_1", "type=keyword")
+            .get();
+
+        client().admin().indices().prepareCreate(sourceIndex2)
+            .addMapping("_doc", "numeric_1", "type=double", "numeric_2", "type=float", "categorical_1", "type=keyword")
+            .get();
+
+        BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
+        bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+
+        for (String index : sourceIndex) {
+            for (int i = 0; i < 5; i++) {
+                IndexRequest indexRequest = new IndexRequest(index);
+                indexRequest.source("numeric_1", randomDouble(), "numeric_2", randomFloat(), "categorical_1", "foo_" + i);
+                bulkRequestBuilder.add(indexRequest);
+            }
+        }
+        BulkResponse bulkResponse = bulkRequestBuilder.get();
+        if (bulkResponse.hasFailures()) {
+            fail("Failed to index data: " + bulkResponse.buildFailureMessage());
+        }
+
+        String id = "test_outlier_detection_with_multiple_source_indices";
+        DataFrameAnalyticsConfig config = buildOutlierDetectionAnalytics(id, sourceIndex, destIndex, null);
+        registerAnalytics(config);
+        putAnalytics(config);
+
+        assertState(id, DataFrameAnalyticsState.STOPPED);
+
+        startAnalytics(id);
+        waitUntilAnalyticsIsStopped(id);
+
+        // Check we've got all docs
+        SearchResponse searchResponse = client().prepareSearch(config.getDest().getIndex()).setTrackTotalHits(true).get();
+        assertThat(searchResponse.getHits().getTotalHits().value, equalTo((long) bulkRequestBuilder.numberOfActions()));
+
+        // Check they all have an outlier_score
+        searchResponse = client().prepareSearch(config.getDest().getIndex())
+            .setTrackTotalHits(true)
+            .setQuery(QueryBuilders.existsQuery("ml.outlier_score")).get();
+        assertThat(searchResponse.getHits().getTotalHits().value, equalTo((long) bulkRequestBuilder.numberOfActions()));
+    }
+
+    public void testOutlierDetectionWithPreExistingDestIndex() throws Exception {
+        String sourceIndex = "test-outlier-detection-with-pre-existing-dest-index";
+        String destIndex = "test-outlier-detection-with-pre-existing-dest-index-results";
+
+        client().admin().indices().prepareCreate(sourceIndex)
+            .addMapping("_doc", "numeric_1", "type=double", "numeric_2", "type=float", "categorical_1", "type=keyword")
+            .get();
+
+        client().admin().indices().prepareCreate(destIndex)
+            .addMapping("_doc", "numeric_1", "type=double", "numeric_2", "type=float", "categorical_1", "type=keyword")
+            .get();
+
+        BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
+        bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+
+        for (int i = 0; i < 5; i++) {
+            IndexRequest indexRequest = new IndexRequest(sourceIndex);
+            indexRequest.source("numeric_1", randomDouble(), "numeric_2", randomFloat(), "categorical_1", "foo_" + i);
+            bulkRequestBuilder.add(indexRequest);
+        }
+        BulkResponse bulkResponse = bulkRequestBuilder.get();
+        if (bulkResponse.hasFailures()) {
+            fail("Failed to index data: " + bulkResponse.buildFailureMessage());
+        }
+
+        String id = "test_outlier_detection_with_pre_existing_dest_index";
+        DataFrameAnalyticsConfig config = buildOutlierDetectionAnalytics(id, new String[] {sourceIndex}, destIndex, null);
+        registerAnalytics(config);
+        putAnalytics(config);
+
+        assertState(id, DataFrameAnalyticsState.STOPPED);
+
+        startAnalytics(id);
+        waitUntilAnalyticsIsStopped(id);
+
+        // Check we've got all docs
+        SearchResponse searchResponse = client().prepareSearch(config.getDest().getIndex()).setTrackTotalHits(true).get();
+        assertThat(searchResponse.getHits().getTotalHits().value, equalTo((long) bulkRequestBuilder.numberOfActions()));
+
+        // Check they all have an outlier_score
+        searchResponse = client().prepareSearch(config.getDest().getIndex())
+            .setTrackTotalHits(true)
+            .setQuery(QueryBuilders.existsQuery("ml.outlier_score")).get();
+        assertThat(searchResponse.getHits().getTotalHits().value, equalTo((long) bulkRequestBuilder.numberOfActions()));
+    }
+
+    private static DataFrameAnalyticsConfig buildOutlierDetectionAnalytics(String id, String[] sourceIndex, String destIndex,
+                                                                           @Nullable String resultsField) {
+        DataFrameAnalyticsConfig.Builder configBuilder = new DataFrameAnalyticsConfig.Builder(id);
+        configBuilder.setSource(new DataFrameAnalyticsSource(sourceIndex, null));
+        configBuilder.setDest(new DataFrameAnalyticsDest(destIndex, resultsField));
+        configBuilder.setAnalysis(new OutlierDetection());
+        return configBuilder.build();
+    }
+
+    private void assertState(String id, DataFrameAnalyticsState state) {
+        List<GetDataFrameAnalyticsStatsAction.Response.Stats> stats = getAnalyticsStats(id);
+        assertThat(stats.size(), equalTo(1));
+        assertThat(stats.get(0).getId(), equalTo(id));
+        assertThat(stats.get(0).getState(), equalTo(state));
+    }
+}
diff --git a/x-pack/plugin/ml/qa/no-bootstrap-tests/build.gradle b/x-pack/plugin/ml/qa/no-bootstrap-tests/build.gradle
index 9eac3fdd37a80..e1131a4fa8af7 100644
--- a/x-pack/plugin/ml/qa/no-bootstrap-tests/build.gradle
+++ b/x-pack/plugin/ml/qa/no-bootstrap-tests/build.gradle
@@ -1,6 +1,6 @@
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
-    testCompile "org.elasticsearch.plugin:x-pack-core:${version}"
+    testCompile project(":x-pack:plugin:core")
     testCompile project(path: xpackModule('ml'), configuration: 'runtime')
-}
+}
\ No newline at end of file
diff --git a/x-pack/plugin/ml/qa/single-node-tests/build.gradle b/x-pack/plugin/ml/qa/single-node-tests/build.gradle
index f856c3d4c5ff4..036b46cb0ca99 100644
--- a/x-pack/plugin/ml/qa/single-node-tests/build.gradle
+++ b/x-pack/plugin/ml/qa/single-node-tests/build.gradle
@@ -1,12 +1,14 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile "org.elasticsearch.plugin:x-pack-core:${version}"
+  testCompile project(":x-pack:plugin:core")
   testCompile project(path: xpackModule('ml'), configuration: 'runtime')
 }
 
-integTestCluster {
+testClusters.integTest {
+  distribution = 'DEFAULT'
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
index 42e945ffec8fd..3976a0e578f16 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -22,7 +23,6 @@
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -58,17 +58,21 @@
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.MlMetaIndex;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteCalendarAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteCalendarEventAction;
+import org.elasticsearch.xpack.core.ml.action.DeleteDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteExpiredDataAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteFilterAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteForecastAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteJobAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteModelSnapshotAction;
+import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
 import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
 import org.elasticsearch.xpack.core.ml.action.FindFileStructureAction;
 import org.elasticsearch.xpack.core.ml.action.FlushJobAction;
@@ -77,6 +81,8 @@
 import org.elasticsearch.xpack.core.ml.action.GetCalendarEventsAction;
 import org.elasticsearch.xpack.core.ml.action.GetCalendarsAction;
 import org.elasticsearch.xpack.core.ml.action.GetCategoriesAction;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction;
 import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.GetFiltersAction;
@@ -95,12 +101,15 @@
 import org.elasticsearch.xpack.core.ml.action.PostDataAction;
 import org.elasticsearch.xpack.core.ml.action.PreviewDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.PutCalendarAction;
+import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.PutFilterAction;
 import org.elasticsearch.xpack.core.ml.action.PutJobAction;
 import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction;
 import org.elasticsearch.xpack.core.ml.action.SetUpgradeModeAction;
+import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
+import org.elasticsearch.xpack.core.ml.action.StopDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.UpdateCalendarJobAction;
 import org.elasticsearch.xpack.core.ml.action.UpdateDatafeedAction;
@@ -110,6 +119,8 @@
 import org.elasticsearch.xpack.core.ml.action.UpdateProcessAction;
 import org.elasticsearch.xpack.core.ml.action.ValidateDetectorAction;
 import org.elasticsearch.xpack.core.ml.action.ValidateJobConfigAction;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.MlDataFrameAnalysisNamedXContentProvider;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
 import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
@@ -118,12 +129,14 @@
 import org.elasticsearch.xpack.ml.action.TransportCloseJobAction;
 import org.elasticsearch.xpack.ml.action.TransportDeleteCalendarAction;
 import org.elasticsearch.xpack.ml.action.TransportDeleteCalendarEventAction;
+import org.elasticsearch.xpack.ml.action.TransportDeleteDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.ml.action.TransportDeleteDatafeedAction;
 import org.elasticsearch.xpack.ml.action.TransportDeleteExpiredDataAction;
 import org.elasticsearch.xpack.ml.action.TransportDeleteFilterAction;
 import org.elasticsearch.xpack.ml.action.TransportDeleteForecastAction;
 import org.elasticsearch.xpack.ml.action.TransportDeleteJobAction;
 import org.elasticsearch.xpack.ml.action.TransportDeleteModelSnapshotAction;
+import org.elasticsearch.xpack.ml.action.TransportEvaluateDataFrameAction;
 import org.elasticsearch.xpack.ml.action.TransportFinalizeJobExecutionAction;
 import org.elasticsearch.xpack.ml.action.TransportFindFileStructureAction;
 import org.elasticsearch.xpack.ml.action.TransportFlushJobAction;
@@ -132,6 +145,8 @@
 import org.elasticsearch.xpack.ml.action.TransportGetCalendarEventsAction;
 import org.elasticsearch.xpack.ml.action.TransportGetCalendarsAction;
 import org.elasticsearch.xpack.ml.action.TransportGetCategoriesAction;
+import org.elasticsearch.xpack.ml.action.TransportGetDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.ml.action.TransportGetDataFrameAnalyticsStatsAction;
 import org.elasticsearch.xpack.ml.action.TransportGetDatafeedsAction;
 import org.elasticsearch.xpack.ml.action.TransportGetDatafeedsStatsAction;
 import org.elasticsearch.xpack.ml.action.TransportGetFiltersAction;
@@ -150,12 +165,15 @@
 import org.elasticsearch.xpack.ml.action.TransportPostDataAction;
 import org.elasticsearch.xpack.ml.action.TransportPreviewDatafeedAction;
 import org.elasticsearch.xpack.ml.action.TransportPutCalendarAction;
+import org.elasticsearch.xpack.ml.action.TransportPutDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.ml.action.TransportPutDatafeedAction;
 import org.elasticsearch.xpack.ml.action.TransportPutFilterAction;
 import org.elasticsearch.xpack.ml.action.TransportPutJobAction;
 import org.elasticsearch.xpack.ml.action.TransportRevertModelSnapshotAction;
 import org.elasticsearch.xpack.ml.action.TransportSetUpgradeModeAction;
+import org.elasticsearch.xpack.ml.action.TransportStartDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.ml.action.TransportStartDatafeedAction;
+import org.elasticsearch.xpack.ml.action.TransportStopDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.ml.action.TransportStopDatafeedAction;
 import org.elasticsearch.xpack.ml.action.TransportUpdateCalendarJobAction;
 import org.elasticsearch.xpack.ml.action.TransportUpdateDatafeedAction;
@@ -168,6 +186,11 @@
 import org.elasticsearch.xpack.ml.datafeed.DatafeedJobBuilder;
 import org.elasticsearch.xpack.ml.datafeed.DatafeedManager;
 import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider;
+import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsManager;
+import org.elasticsearch.xpack.ml.dataframe.persistence.DataFrameAnalyticsConfigProvider;
+import org.elasticsearch.xpack.ml.dataframe.process.AnalyticsProcessFactory;
+import org.elasticsearch.xpack.ml.dataframe.process.AnalyticsProcessManager;
+import org.elasticsearch.xpack.ml.dataframe.process.NativeAnalyticsProcessFactory;
 import org.elasticsearch.xpack.ml.job.JobManager;
 import org.elasticsearch.xpack.ml.job.JobManagerHolder;
 import org.elasticsearch.xpack.ml.job.UpdateJobProcessNotifier;
@@ -187,9 +210,11 @@
 import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerFactory;
 import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerProcessFactory;
 import org.elasticsearch.xpack.ml.notifications.Auditor;
+import org.elasticsearch.xpack.ml.process.DummyController;
+import org.elasticsearch.xpack.ml.process.MlController;
+import org.elasticsearch.xpack.ml.process.MlControllerHolder;
 import org.elasticsearch.xpack.ml.process.MlMemoryTracker;
 import org.elasticsearch.xpack.ml.process.NativeController;
-import org.elasticsearch.xpack.ml.process.NativeControllerHolder;
 import org.elasticsearch.xpack.ml.process.NativeStorageProvider;
 import org.elasticsearch.xpack.ml.rest.RestDeleteExpiredDataAction;
 import org.elasticsearch.xpack.ml.rest.RestFindFileStructureAction;
@@ -211,6 +236,13 @@
 import org.elasticsearch.xpack.ml.rest.datafeeds.RestStartDatafeedAction;
 import org.elasticsearch.xpack.ml.rest.datafeeds.RestStopDatafeedAction;
 import org.elasticsearch.xpack.ml.rest.datafeeds.RestUpdateDatafeedAction;
+import org.elasticsearch.xpack.ml.rest.dataframe.RestDeleteDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.ml.rest.dataframe.RestEvaluateDataFrameAction;
+import org.elasticsearch.xpack.ml.rest.dataframe.RestGetDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.ml.rest.dataframe.RestGetDataFrameAnalyticsStatsAction;
+import org.elasticsearch.xpack.ml.rest.dataframe.RestPutDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.ml.rest.dataframe.RestStartDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.ml.rest.dataframe.RestStopDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.ml.rest.filter.RestDeleteFilterAction;
 import org.elasticsearch.xpack.ml.rest.filter.RestGetFiltersAction;
 import org.elasticsearch.xpack.ml.rest.filter.RestPutFilterAction;
@@ -296,6 +328,10 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu
     public static final Setting<Integer> MAX_OPEN_JOBS_PER_NODE =
             Setting.intSetting("xpack.ml.max_open_jobs", 20, 1, MAX_MAX_OPEN_JOBS_PER_NODE, Property.Dynamic, Property.NodeScope);
 
+    public static final Setting<TimeValue> PROCESS_CONNECT_TIMEOUT =
+        Setting.timeSetting("xpack.ml.process_connect_timeout", TimeValue.timeValueSeconds(10),
+            TimeValue.timeValueSeconds(5), Setting.Property.Dynamic, Setting.Property.NodeScope);
+
     // Undocumented setting for integration test purposes
     public static final Setting<ByteSizeValue> MIN_DISK_SPACE_OFF_HEAP =
         Setting.byteSizeSetting("xpack.ml.min_disk_space_off_heap", new ByteSizeValue(5, ByteSizeUnit.GB), Setting.Property.NodeScope);
@@ -308,6 +344,7 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu
 
     private final SetOnce<AutodetectProcessManager> autodetectProcessManager = new SetOnce<>();
     private final SetOnce<DatafeedManager> datafeedManager = new SetOnce<>();
+    private final SetOnce<DataFrameAnalyticsManager> dataFrameAnalyticsManager = new SetOnce<>();
     private final SetOnce<MlMemoryTracker> memoryTracker = new SetOnce<>();
 
     public MachineLearning(Settings settings, Path configPath) {
@@ -330,6 +367,7 @@ public static boolean isMlNode(DiscoveryNode node) {
     public List<Setting<?>> getSettings() {
         return List.of(
                 MachineLearningField.AUTODETECT_PROCESS,
+                PROCESS_CONNECT_TIMEOUT,
                 ML_ENABLED,
                 CONCURRENT_JOB_ALLOCATIONS,
                 MachineLearningField.MAX_MODEL_MEMORY_LIMIT,
@@ -431,32 +469,39 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
 
         NativeStorageProvider nativeStorageProvider = new NativeStorageProvider(environment, MIN_DISK_SPACE_OFF_HEAP.get(settings));
 
+        MlController mlController;
         AutodetectProcessFactory autodetectProcessFactory;
         NormalizerProcessFactory normalizerProcessFactory;
-        if (MachineLearningField.AUTODETECT_PROCESS.get(settings) && MachineLearningFeatureSet.isRunningOnMlPlatform(true)) {
+        AnalyticsProcessFactory analyticsProcessFactory;
+        if (MachineLearningField.AUTODETECT_PROCESS.get(settings)) {
             try {
-                NativeController nativeController = NativeControllerHolder.getNativeController(clusterService.getNodeName(), environment);
-                if (nativeController == null) {
-                    // This will only only happen when path.home is not set, which is disallowed in production
-                    throw new ElasticsearchException("Failed to create native process controller for Machine Learning");
-                }
+                NativeController nativeController = NativeController.makeNativeController(clusterService.getNodeName(), environment);
                 autodetectProcessFactory = new NativeAutodetectProcessFactory(
                     environment,
                     settings,
                     nativeController,
                     client,
                     clusterService);
-                normalizerProcessFactory = new NativeNormalizerProcessFactory(environment, nativeController);
+                normalizerProcessFactory = new NativeNormalizerProcessFactory(environment, nativeController, clusterService);
+                analyticsProcessFactory = new NativeAnalyticsProcessFactory(environment, nativeController, clusterService);
+                mlController = nativeController;
             } catch (IOException e) {
-                // This also should not happen in production, as the MachineLearningFeatureSet should have
-                // hit the same error first and brought down the node with a friendlier error message
-                throw new ElasticsearchException("Failed to create native process factories for Machine Learning", e);
+                // The low level cause of failure from the named pipe helper's perspective is almost never the real root cause, so
+                // only log this at the lowest level of detail.  It's almost always "file not found" on a named pipe we expect to be
+                // able to connect to, but the thing we really need to know is what stopped the native process creating the named pipe.
+                logger.trace("Failed to connect to ML native controller", e);
+                throw new ElasticsearchException("Failure running machine learning native code. This could be due to running "
+                    + "on an unsupported OS or distribution, missing OS libraries, or a problem with the temp directory. To "
+                    + "bypass this problem by running Elasticsearch without machine learning functionality set ["
+                    + XPackSettings.MACHINE_LEARNING_ENABLED.getKey() + ": false].");
             }
         } else {
+            mlController = new DummyController();
             autodetectProcessFactory = (job, autodetectParams, executorService, onProcessCrash) ->
                     new BlackHoleAutodetectProcess(job.getId());
             // factor of 1.0 makes renormalization a no-op
             normalizerProcessFactory = (jobId, quantilesState, bucketSpan, executorService) -> new MultiplyingNormalizerProcess(1.0);
+            analyticsProcessFactory = (jobId, analyticsProcessConfig, executorService, onProcessCrash) -> null;
         }
         NormalizerFactory normalizerFactory = new NormalizerFactory(normalizerProcessFactory,
                 threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME));
@@ -469,10 +514,21 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
         DatafeedManager datafeedManager = new DatafeedManager(threadPool, client, clusterService, datafeedJobBuilder,
                 System::currentTimeMillis, auditor, autodetectProcessManager);
         this.datafeedManager.set(datafeedManager);
-        MlMemoryTracker memoryTracker = new MlMemoryTracker(settings, clusterService, threadPool, jobManager, jobResultsProvider);
+
+        // Data frame analytics components
+        AnalyticsProcessManager analyticsProcessManager = new AnalyticsProcessManager(client, threadPool, analyticsProcessFactory);
+        DataFrameAnalyticsConfigProvider dataFrameAnalyticsConfigProvider = new DataFrameAnalyticsConfigProvider(client);
+        assert client instanceof NodeClient;
+        DataFrameAnalyticsManager dataFrameAnalyticsManager = new DataFrameAnalyticsManager((NodeClient) client,
+            dataFrameAnalyticsConfigProvider, analyticsProcessManager);
+        this.dataFrameAnalyticsManager.set(dataFrameAnalyticsManager);
+
+        // Components shared by anomaly detection and data frame analytics
+        MlMemoryTracker memoryTracker = new MlMemoryTracker(settings, clusterService, threadPool, jobManager, jobResultsProvider,
+            dataFrameAnalyticsConfigProvider);
         this.memoryTracker.set(memoryTracker);
-        MlLifeCycleService mlLifeCycleService = new MlLifeCycleService(environment, clusterService, datafeedManager,
-                autodetectProcessManager, memoryTracker);
+        MlLifeCycleService mlLifeCycleService = new MlLifeCycleService(clusterService, datafeedManager, mlController,
+            autodetectProcessManager, memoryTracker);
 
         // this object registers as a license state listener, and is never removed, so there's no need to retain another reference to it
         final InvalidLicenseEnforcer enforcer =
@@ -484,6 +540,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
 
         return Arrays.asList(
                 mlLifeCycleService,
+                new MlControllerHolder(mlController),
                 jobResultsProvider,
                 jobConfigProvider,
                 datafeedConfigProvider,
@@ -496,6 +553,8 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
                 auditor,
                 new MlAssignmentNotifier(settings, auditor, threadPool, client, clusterService),
                 memoryTracker,
+                analyticsProcessManager,
+                dataFrameAnalyticsConfigProvider,
                 nativeStorageProvider
         );
     }
@@ -511,18 +570,12 @@ public List<PersistentTasksExecutor<?>> getPersistentTasksExecutor(ClusterServic
         return Arrays.asList(
                 new TransportOpenJobAction.OpenJobPersistentTasksExecutor(settings, clusterService, autodetectProcessManager.get(),
                     memoryTracker.get(), client),
-                new TransportStartDatafeedAction.StartDatafeedPersistentTasksExecutor(datafeedManager.get())
+                new TransportStartDatafeedAction.StartDatafeedPersistentTasksExecutor(datafeedManager.get()),
+                new TransportStartDataFrameAnalyticsAction.TaskExecutor(settings, client, clusterService, dataFrameAnalyticsManager.get(),
+                    memoryTracker.get())
         );
     }
 
-    public Collection<Module> createGuiceModules() {
-        List<Module> modules = new ArrayList<>();
-        modules.add(b -> {
-            XPackPlugin.bindFeatureSet(b, MachineLearningFeatureSet.class);
-        });
-        return modules;
-    }
-
     @Override
     public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
                                              IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
@@ -577,14 +630,25 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
             new RestGetCalendarEventsAction(settings, restController),
             new RestPostCalendarEventAction(settings, restController),
             new RestFindFileStructureAction(settings, restController),
-            new RestSetUpgradeModeAction(settings, restController)
+            new RestSetUpgradeModeAction(settings, restController),
+            new RestGetDataFrameAnalyticsAction(settings, restController),
+            new RestGetDataFrameAnalyticsStatsAction(settings, restController),
+            new RestPutDataFrameAnalyticsAction(settings, restController),
+            new RestDeleteDataFrameAnalyticsAction(settings, restController),
+            new RestStartDataFrameAnalyticsAction(settings, restController),
+            new RestStopDataFrameAnalyticsAction(settings, restController),
+            new RestEvaluateDataFrameAction(settings, restController)
         );
     }
 
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        var usageAction =
+            new ActionHandler<>(XPackUsageFeatureAction.MACHINE_LEARNING, MachineLearningUsageTransportAction.class);
+        var infoAction =
+            new ActionHandler<>(XPackInfoFeatureAction.MACHINE_LEARNING, MachineLearningInfoTransportAction.class);
         if (false == enabled) {
-            return emptyList();
+            return Arrays.asList(usageAction, infoAction);
         }
         return Arrays.asList(
                 new ActionHandler<>(GetJobsAction.INSTANCE, TransportGetJobsAction.class),
@@ -636,8 +700,16 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
                 new ActionHandler<>(PostCalendarEventsAction.INSTANCE, TransportPostCalendarEventsAction.class),
                 new ActionHandler<>(PersistJobAction.INSTANCE, TransportPersistJobAction.class),
                 new ActionHandler<>(FindFileStructureAction.INSTANCE, TransportFindFileStructureAction.class),
-                new ActionHandler<>(SetUpgradeModeAction.INSTANCE, TransportSetUpgradeModeAction.class)
-        );
+                new ActionHandler<>(SetUpgradeModeAction.INSTANCE, TransportSetUpgradeModeAction.class),
+                new ActionHandler<>(GetDataFrameAnalyticsAction.INSTANCE, TransportGetDataFrameAnalyticsAction.class),
+                new ActionHandler<>(GetDataFrameAnalyticsStatsAction.INSTANCE, TransportGetDataFrameAnalyticsStatsAction.class),
+                new ActionHandler<>(PutDataFrameAnalyticsAction.INSTANCE, TransportPutDataFrameAnalyticsAction.class),
+                new ActionHandler<>(DeleteDataFrameAnalyticsAction.INSTANCE, TransportDeleteDataFrameAnalyticsAction.class),
+                new ActionHandler<>(StartDataFrameAnalyticsAction.INSTANCE, TransportStartDataFrameAnalyticsAction.class),
+                new ActionHandler<>(StopDataFrameAnalyticsAction.INSTANCE, TransportStopDataFrameAnalyticsAction.class),
+                new ActionHandler<>(EvaluateDataFrameAction.INSTANCE, TransportEvaluateDataFrameAction.class),
+                usageAction,
+                infoAction);
     }
 
     @Override
@@ -797,12 +869,20 @@ static long machineMemoryFromStats(OsStats stats) {
             if (containerLimitStr != null) {
                 BigInteger containerLimit = new BigInteger(containerLimitStr);
                 if ((containerLimit.compareTo(BigInteger.valueOf(mem)) < 0 && containerLimit.compareTo(BigInteger.ZERO) > 0)
-                        // mem < 0 means the value couldn't be obtained for some reason
-                        || (mem < 0 && containerLimit.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) < 0)) {
+                        // mem <= 0 means the value couldn't be obtained for some reason
+                        || (mem <= 0 && containerLimit.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) < 0)) {
                     mem = containerLimit.longValue();
                 }
             }
         }
         return mem;
     }
+
+    @Override
+    public List<NamedXContentRegistry.Entry> getNamedXContent() {
+        List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
+        namedXContent.addAll(new MlEvaluationNamedXContentProvider().getNamedXContentParsers());
+        namedXContent.addAll(new MlDataFrameAnalysisNamedXContentProvider().getNamedXContentParsers());
+        return namedXContent;
+    }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java
deleted file mode 100644
index 2a5b4369cdf9a..0000000000000
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java
+++ /dev/null
@@ -1,303 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.ml;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.Counter;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.plugins.Platforms;
-import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.XPackField;
-import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.ml.MachineLearningFeatureSetUsage;
-import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction;
-import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
-import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
-import org.elasticsearch.xpack.core.ml.job.config.Job;
-import org.elasticsearch.xpack.core.ml.job.config.JobState;
-import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
-import org.elasticsearch.xpack.core.ml.stats.ForecastStats;
-import org.elasticsearch.xpack.core.ml.stats.StatsAccumulator;
-import org.elasticsearch.xpack.ml.job.JobManagerHolder;
-import org.elasticsearch.xpack.ml.process.NativeController;
-import org.elasticsearch.xpack.ml.process.NativeControllerHolder;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
-
-public class MachineLearningFeatureSet implements XPackFeatureSet {
-
-    /**
-     * List of platforms for which the native processes are available
-     */
-    private static final List<String> mlPlatforms =
-            Arrays.asList("darwin-x86_64", "linux-x86_64", "windows-x86_64");
-
-    private final boolean enabled;
-    private final XPackLicenseState licenseState;
-    private final ClusterService clusterService;
-    private final Client client;
-    private final JobManagerHolder jobManagerHolder;
-    private final Map<String, Object> nativeCodeInfo;
-
-    @Inject
-    public MachineLearningFeatureSet(Environment environment, ClusterService clusterService, Client client,
-                                     @Nullable XPackLicenseState licenseState, JobManagerHolder jobManagerHolder) {
-        this.enabled = XPackSettings.MACHINE_LEARNING_ENABLED.get(environment.settings());
-        this.clusterService = Objects.requireNonNull(clusterService);
-        this.client = Objects.requireNonNull(client);
-        this.licenseState = licenseState;
-        this.jobManagerHolder = jobManagerHolder;
-        Map<String, Object> nativeCodeInfo = NativeController.UNKNOWN_NATIVE_CODE_INFO;
-        // Don't try to get the native code version if ML is disabled - it causes too much controversy
-        // if ML has been disabled because of some OS incompatibility.  Also don't try to get the native
-        // code version in the transport client - the controller process won't be running.
-        if (enabled) {
-            try {
-                if (isRunningOnMlPlatform(true)) {
-                    NativeController nativeController = NativeControllerHolder.getNativeController(clusterService.getNodeName(),
-                        environment);
-                    if (nativeController != null) {
-                        nativeCodeInfo = nativeController.getNativeCodeInfo();
-                    }
-                }
-            } catch (IOException | TimeoutException e) {
-                LogManager.getLogger(MachineLearningFeatureSet.class).error("Cannot get native code info for Machine Learning", e);
-                throw new ElasticsearchException("Cannot communicate with Machine Learning native code");
-            }
-        }
-        this.nativeCodeInfo = nativeCodeInfo;
-    }
-
-    static boolean isRunningOnMlPlatform(boolean fatalIfNot) {
-        return isRunningOnMlPlatform(Constants.OS_NAME, Constants.OS_ARCH, fatalIfNot);
-    }
-
-    static boolean isRunningOnMlPlatform(String osName, String osArch, boolean fatalIfNot) {
-        String platformName = Platforms.platformName(osName, osArch);
-        if (mlPlatforms.contains(platformName)) {
-            return true;
-        }
-        if (fatalIfNot) {
-            throw new ElasticsearchException("X-Pack is not supported and Machine Learning is not available for [" + platformName
-                    + "]; you can use the other X-Pack features (unsupported) by setting xpack.ml.enabled: false in elasticsearch.yml");
-        }
-        return false;
-    }
-
-    @Override
-    public String name() {
-        return XPackField.MACHINE_LEARNING;
-    }
-
-    @Override
-    public String description() {
-        return "Machine Learning for the Elastic Stack";
-    }
-
-    @Override
-    public boolean available() {
-        return licenseState != null && licenseState.isMachineLearningAllowed();
-    }
-
-    @Override
-    public boolean enabled() {
-        return enabled;
-    }
-
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return nativeCodeInfo;
-    }
-
-    @Override
-    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
-        ClusterState state = clusterService.state();
-        new Retriever(client, jobManagerHolder, available(), enabled(), mlNodeCount(state)).execute(listener);
-    }
-
-    private int mlNodeCount(final ClusterState clusterState) {
-        if (enabled == false) {
-            return 0;
-        }
-
-        int mlNodeCount = 0;
-        for (DiscoveryNode node : clusterState.getNodes()) {
-            if (MachineLearning.isMlNode(node)) {
-                ++mlNodeCount;
-            }
-        }
-        return mlNodeCount;
-    }
-
-    public static class Retriever {
-
-        private final Client client;
-        private final JobManagerHolder jobManagerHolder;
-        private final boolean available;
-        private final boolean enabled;
-        private Map<String, Object> jobsUsage;
-        private Map<String, Object> datafeedsUsage;
-        private int nodeCount;
-
-        public Retriever(Client client, JobManagerHolder jobManagerHolder, boolean available, boolean enabled, int nodeCount) {
-            this.client = Objects.requireNonNull(client);
-            this.jobManagerHolder = jobManagerHolder;
-            this.available = available;
-            this.enabled = enabled;
-            this.jobsUsage = new LinkedHashMap<>();
-            this.datafeedsUsage = new LinkedHashMap<>();
-            this.nodeCount = nodeCount;
-        }
-
-        public void execute(ActionListener<Usage> listener) {
-            // empty holder means either ML disabled or transport client mode
-            if (jobManagerHolder.isEmpty()) {
-                listener.onResponse(
-                    new MachineLearningFeatureSetUsage(available, enabled, Collections.emptyMap(), Collections.emptyMap(), 0));
-                return;
-            }
-
-            // Step 2. Extract usage from datafeeds stats and return usage response
-            ActionListener<GetDatafeedsStatsAction.Response> datafeedStatsListener =
-                    ActionListener.wrap(response -> {
-                                addDatafeedsUsage(response);
-                                listener.onResponse(new MachineLearningFeatureSetUsage(
-                                        available, enabled, jobsUsage, datafeedsUsage, nodeCount));
-                            },
-                            listener::onFailure
-                    );
-
-            // Step 1. Extract usage from jobs stats and then request stats for all datafeeds
-            GetJobsStatsAction.Request jobStatsRequest = new GetJobsStatsAction.Request(MetaData.ALL);
-            ActionListener<GetJobsStatsAction.Response> jobStatsListener = ActionListener.wrap(
-                    response -> {
-                        jobManagerHolder.getJobManager().expandJobs(MetaData.ALL, true, ActionListener.wrap(jobs -> {
-                            addJobsUsage(response, jobs.results());
-                            GetDatafeedsStatsAction.Request datafeedStatsRequest = new GetDatafeedsStatsAction.Request(
-                                    GetDatafeedsStatsAction.ALL);
-                            client.execute(GetDatafeedsStatsAction.INSTANCE, datafeedStatsRequest, datafeedStatsListener);
-                        }, listener::onFailure));
-                    }, listener::onFailure);
-
-            // Step 0. Kick off the chain of callbacks by requesting jobs stats
-            client.execute(GetJobsStatsAction.INSTANCE, jobStatsRequest, jobStatsListener);
-        }
-
-        private void addJobsUsage(GetJobsStatsAction.Response response, List<Job> jobs) {
-            StatsAccumulator allJobsDetectorsStats = new StatsAccumulator();
-            StatsAccumulator allJobsModelSizeStats = new StatsAccumulator();
-            ForecastStats allJobsForecastStats = new ForecastStats();
-
-            Map<JobState, Counter> jobCountByState = new HashMap<>();
-            Map<JobState, StatsAccumulator> detectorStatsByState = new HashMap<>();
-            Map<JobState, StatsAccumulator> modelSizeStatsByState = new HashMap<>();
-            Map<JobState, ForecastStats> forecastStatsByState = new HashMap<>();
-            Map<JobState, Map<String, Long>> createdByByState = new HashMap<>();
-
-            List<GetJobsStatsAction.Response.JobStats> jobsStats = response.getResponse().results();
-            Map<String, Job> jobMap = jobs.stream().collect(Collectors.toMap(Job::getId, item -> item));
-            Map<String, Long> allJobsCreatedBy = jobs.stream().map(this::jobCreatedBy)
-                    .collect(Collectors.groupingBy(item -> item, Collectors.counting()));;
-            for (GetJobsStatsAction.Response.JobStats jobStats : jobsStats) {
-                ModelSizeStats modelSizeStats = jobStats.getModelSizeStats();
-                Job job = jobMap.get(jobStats.getJobId());
-                int detectorsCount = job.getAnalysisConfig().getDetectors().size();
-                double modelSize = modelSizeStats == null ? 0.0
-                        : jobStats.getModelSizeStats().getModelBytes();
-
-                allJobsForecastStats.merge(jobStats.getForecastStats());
-                allJobsDetectorsStats.add(detectorsCount);
-                allJobsModelSizeStats.add(modelSize);
-
-                JobState jobState = jobStats.getState();
-                jobCountByState.computeIfAbsent(jobState, js -> Counter.newCounter()).addAndGet(1);
-                detectorStatsByState.computeIfAbsent(jobState,
-                        js -> new StatsAccumulator()).add(detectorsCount);
-                modelSizeStatsByState.computeIfAbsent(jobState,
-                        js -> new StatsAccumulator()).add(modelSize);
-                forecastStatsByState.merge(jobState, jobStats.getForecastStats(), (f1, f2) -> f1.merge(f2));
-                createdByByState.computeIfAbsent(jobState, js -> new HashMap<>())
-                        .compute(jobCreatedBy(job), (k, v) -> (v == null) ? 1L : (v + 1));
-            }
-
-            jobsUsage.put(MachineLearningFeatureSetUsage.ALL, createJobUsageEntry(jobs.size(), allJobsDetectorsStats,
-                    allJobsModelSizeStats, allJobsForecastStats, allJobsCreatedBy));
-            for (JobState jobState : jobCountByState.keySet()) {
-                jobsUsage.put(jobState.name().toLowerCase(Locale.ROOT), createJobUsageEntry(
-                        jobCountByState.get(jobState).get(),
-                        detectorStatsByState.get(jobState),
-                        modelSizeStatsByState.get(jobState),
-                        forecastStatsByState.get(jobState),
-                        createdByByState.get(jobState)));
-            }
-        }
-
-        private String jobCreatedBy(Job job) {
-            Map<String, Object> customSettings = job.getCustomSettings();
-            if (customSettings == null || customSettings.containsKey(MachineLearningFeatureSetUsage.CREATED_BY) == false) {
-                return "unknown";
-            }
-            // Replace non-alpha-numeric characters with underscores because
-            // the values from custom settings become keys in the usage data
-            return customSettings.get(MachineLearningFeatureSetUsage.CREATED_BY).toString().replaceAll("\\W", "_");
-        }
-
-        private Map<String, Object> createJobUsageEntry(long count, StatsAccumulator detectorStats,
-                                                        StatsAccumulator modelSizeStats,
-                                                        ForecastStats forecastStats, Map<String, Long> createdBy) {
-            Map<String, Object> usage = new HashMap<>();
-            usage.put(MachineLearningFeatureSetUsage.COUNT, count);
-            usage.put(MachineLearningFeatureSetUsage.DETECTORS, detectorStats.asMap());
-            usage.put(MachineLearningFeatureSetUsage.MODEL_SIZE, modelSizeStats.asMap());
-            usage.put(MachineLearningFeatureSetUsage.FORECASTS, forecastStats.asMap());
-            usage.put(MachineLearningFeatureSetUsage.CREATED_BY, createdBy);
-            return usage;
-        }
-
-        private void addDatafeedsUsage(GetDatafeedsStatsAction.Response response) {
-            Map<DatafeedState, Counter> datafeedCountByState = new HashMap<>();
-
-            List<GetDatafeedsStatsAction.Response.DatafeedStats> datafeedsStats = response.getResponse().results();
-            for (GetDatafeedsStatsAction.Response.DatafeedStats datafeedStats : datafeedsStats) {
-                datafeedCountByState.computeIfAbsent(datafeedStats.getDatafeedState(),
-                        ds -> Counter.newCounter()).addAndGet(1);
-            }
-
-            datafeedsUsage.put(MachineLearningFeatureSetUsage.ALL, createDatafeedUsageEntry(response.getResponse().count()));
-            for (DatafeedState datafeedState : datafeedCountByState.keySet()) {
-                datafeedsUsage.put(datafeedState.name().toLowerCase(Locale.ROOT),
-                        createDatafeedUsageEntry(datafeedCountByState.get(datafeedState).get()));
-            }
-        }
-
-        private Map<String, Object> createDatafeedUsageEntry(long count) {
-            Map<String, Object> usage = new HashMap<>();
-            usage.put(MachineLearningFeatureSetUsage.COUNT, count);
-            return usage;
-        }
-    }
-}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportAction.java
new file mode 100644
index 0000000000000..07fdc476cc307
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportAction.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
+
+public class MachineLearningInfoTransportAction extends XPackInfoFeatureTransportAction {
+
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public MachineLearningInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                              Settings settings, XPackLicenseState licenseState) {
+        super(XPackInfoFeatureAction.MACHINE_LEARNING.name(), transportService, actionFilters);
+        this.enabled = XPackSettings.MACHINE_LEARNING_ENABLED.get(settings);
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    public String name() {
+        return XPackField.MACHINE_LEARNING;
+    }
+
+    @Override
+    public boolean available() {
+        return licenseState.isMachineLearningAllowed();
+    }
+
+    @Override
+    public boolean enabled() {
+        return enabled;
+    }
+
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
new file mode 100644
index 0000000000000..be9c8d4c4fd6d
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningUsageTransportAction.java
@@ -0,0 +1,209 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml;
+
+import org.apache.lucene.util.Counter;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.ml.MachineLearningFeatureSetUsage;
+import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction;
+import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
+import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
+import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.core.ml.job.config.JobState;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
+import org.elasticsearch.xpack.core.ml.stats.ForecastStats;
+import org.elasticsearch.xpack.core.ml.stats.StatsAccumulator;
+import org.elasticsearch.xpack.ml.job.JobManagerHolder;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class MachineLearningUsageTransportAction extends XPackUsageFeatureTransportAction {
+
+    private final Client client;
+    private final XPackLicenseState licenseState;
+    private final JobManagerHolder jobManagerHolder;
+    private final boolean enabled;
+
+    @Inject
+    public MachineLearningUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                               Environment environment, Client client,
+                                               XPackLicenseState licenseState, JobManagerHolder jobManagerHolder) {
+        super(XPackUsageFeatureAction.MACHINE_LEARNING.name(), transportService, clusterService,
+              threadPool, actionFilters, indexNameExpressionResolver);
+        this.client = client;
+        this.licenseState = licenseState;
+        this.jobManagerHolder = jobManagerHolder;
+        this.enabled = XPackSettings.MACHINE_LEARNING_ENABLED.get(environment.settings());
+    }
+
+    @Override
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
+        if (enabled == false) {
+            MachineLearningFeatureSetUsage usage = new MachineLearningFeatureSetUsage(licenseState.isMachineLearningAllowed(), enabled,
+                Collections.emptyMap(), Collections.emptyMap(), 0);
+            listener.onResponse(new XPackUsageFeatureResponse(usage));
+            return;
+        }
+
+        Map<String, Object> jobsUsage = new LinkedHashMap<>();
+        Map<String, Object> datafeedsUsage = new LinkedHashMap<>();
+        int nodeCount = mlNodeCount(state);
+
+        // Step 2. Extract usage from datafeeds stats and return usage response
+        ActionListener<GetDatafeedsStatsAction.Response> datafeedStatsListener =
+            ActionListener.wrap(response -> {
+                addDatafeedsUsage(response, datafeedsUsage);
+                MachineLearningFeatureSetUsage usage = new MachineLearningFeatureSetUsage(licenseState.isMachineLearningAllowed(),
+                    enabled, jobsUsage, datafeedsUsage, nodeCount);
+                listener.onResponse(new XPackUsageFeatureResponse(usage));
+            },
+            listener::onFailure);
+
+        // Step 1. Extract usage from jobs stats and then request stats for all datafeeds
+        GetJobsStatsAction.Request jobStatsRequest = new GetJobsStatsAction.Request(MetaData.ALL);
+        ActionListener<GetJobsStatsAction.Response> jobStatsListener = ActionListener.wrap(
+            response -> {
+                jobManagerHolder.getJobManager().expandJobs(MetaData.ALL, true, ActionListener.wrap(jobs -> {
+                    addJobsUsage(response, jobs.results(), jobsUsage);
+                    GetDatafeedsStatsAction.Request datafeedStatsRequest = new GetDatafeedsStatsAction.Request(
+                        GetDatafeedsStatsAction.ALL);
+                    client.execute(GetDatafeedsStatsAction.INSTANCE, datafeedStatsRequest, datafeedStatsListener);
+                }, listener::onFailure));
+            }, listener::onFailure);
+
+        // Step 0. Kick off the chain of callbacks by requesting jobs stats
+        client.execute(GetJobsStatsAction.INSTANCE, jobStatsRequest, jobStatsListener);
+    }
+
+    private void addJobsUsage(GetJobsStatsAction.Response response, List<Job> jobs, Map<String, Object> jobsUsage) {
+        StatsAccumulator allJobsDetectorsStats = new StatsAccumulator();
+        StatsAccumulator allJobsModelSizeStats = new StatsAccumulator();
+        ForecastStats allJobsForecastStats = new ForecastStats();
+
+        Map<JobState, Counter> jobCountByState = new HashMap<>();
+        Map<JobState, StatsAccumulator> detectorStatsByState = new HashMap<>();
+        Map<JobState, StatsAccumulator> modelSizeStatsByState = new HashMap<>();
+        Map<JobState, ForecastStats> forecastStatsByState = new HashMap<>();
+        Map<JobState, Map<String, Long>> createdByByState = new HashMap<>();
+
+        List<GetJobsStatsAction.Response.JobStats> jobsStats = response.getResponse().results();
+        Map<String, Job> jobMap = jobs.stream().collect(Collectors.toMap(Job::getId, item -> item));
+        Map<String, Long> allJobsCreatedBy = jobs.stream().map(this::jobCreatedBy)
+            .collect(Collectors.groupingBy(item -> item, Collectors.counting()));;
+        for (GetJobsStatsAction.Response.JobStats jobStats : jobsStats) {
+            ModelSizeStats modelSizeStats = jobStats.getModelSizeStats();
+            Job job = jobMap.get(jobStats.getJobId());
+            int detectorsCount = job.getAnalysisConfig().getDetectors().size();
+            double modelSize = modelSizeStats == null ? 0.0
+                : jobStats.getModelSizeStats().getModelBytes();
+
+            allJobsForecastStats.merge(jobStats.getForecastStats());
+            allJobsDetectorsStats.add(detectorsCount);
+            allJobsModelSizeStats.add(modelSize);
+
+            JobState jobState = jobStats.getState();
+            jobCountByState.computeIfAbsent(jobState, js -> Counter.newCounter()).addAndGet(1);
+            detectorStatsByState.computeIfAbsent(jobState,
+                js -> new StatsAccumulator()).add(detectorsCount);
+            modelSizeStatsByState.computeIfAbsent(jobState,
+                js -> new StatsAccumulator()).add(modelSize);
+            forecastStatsByState.merge(jobState, jobStats.getForecastStats(), (f1, f2) -> f1.merge(f2));
+            createdByByState.computeIfAbsent(jobState, js -> new HashMap<>())
+                .compute(jobCreatedBy(job), (k, v) -> (v == null) ? 1L : (v + 1));
+        }
+
+        jobsUsage.put(MachineLearningFeatureSetUsage.ALL, createJobUsageEntry(jobs.size(), allJobsDetectorsStats,
+            allJobsModelSizeStats, allJobsForecastStats, allJobsCreatedBy));
+        for (JobState jobState : jobCountByState.keySet()) {
+            jobsUsage.put(jobState.name().toLowerCase(Locale.ROOT), createJobUsageEntry(
+                jobCountByState.get(jobState).get(),
+                detectorStatsByState.get(jobState),
+                modelSizeStatsByState.get(jobState),
+                forecastStatsByState.get(jobState),
+                createdByByState.get(jobState)));
+        }
+    }
+
+    private String jobCreatedBy(Job job) {
+        Map<String, Object> customSettings = job.getCustomSettings();
+        if (customSettings == null || customSettings.containsKey(MachineLearningFeatureSetUsage.CREATED_BY) == false) {
+            return "unknown";
+        }
+        // Replace non-alpha-numeric characters with underscores because
+        // the values from custom settings become keys in the usage data
+        return customSettings.get(MachineLearningFeatureSetUsage.CREATED_BY).toString().replaceAll("\\W", "_");
+    }
+
+    private Map<String, Object> createJobUsageEntry(long count, StatsAccumulator detectorStats,
+                                                    StatsAccumulator modelSizeStats,
+                                                    ForecastStats forecastStats, Map<String, Long> createdBy) {
+        Map<String, Object> usage = new HashMap<>();
+        usage.put(MachineLearningFeatureSetUsage.COUNT, count);
+        usage.put(MachineLearningFeatureSetUsage.DETECTORS, detectorStats.asMap());
+        usage.put(MachineLearningFeatureSetUsage.MODEL_SIZE, modelSizeStats.asMap());
+        usage.put(MachineLearningFeatureSetUsage.FORECASTS, forecastStats.asMap());
+        usage.put(MachineLearningFeatureSetUsage.CREATED_BY, createdBy);
+        return usage;
+    }
+
+    private void addDatafeedsUsage(GetDatafeedsStatsAction.Response response, Map<String, Object> datafeedsUsage) {
+        Map<DatafeedState, Counter> datafeedCountByState = new HashMap<>();
+
+        List<GetDatafeedsStatsAction.Response.DatafeedStats> datafeedsStats = response.getResponse().results();
+        for (GetDatafeedsStatsAction.Response.DatafeedStats datafeedStats : datafeedsStats) {
+            datafeedCountByState.computeIfAbsent(datafeedStats.getDatafeedState(),
+                ds -> Counter.newCounter()).addAndGet(1);
+        }
+
+        datafeedsUsage.put(MachineLearningFeatureSetUsage.ALL, createDatafeedUsageEntry(response.getResponse().count()));
+        for (DatafeedState datafeedState : datafeedCountByState.keySet()) {
+            datafeedsUsage.put(datafeedState.name().toLowerCase(Locale.ROOT),
+                createDatafeedUsageEntry(datafeedCountByState.get(datafeedState).get()));
+        }
+    }
+
+    private Map<String, Object> createDatafeedUsageEntry(long count) {
+        Map<String, Object> usage = new HashMap<>();
+        usage.put(MachineLearningFeatureSetUsage.COUNT, count);
+        return usage;
+    }
+
+    private static int mlNodeCount(final ClusterState clusterState) {
+        int mlNodeCount = 0;
+        for (DiscoveryNode node : clusterState.getNodes()) {
+            if (MachineLearning.isMlNode(node)) {
+                ++mlNodeCount;
+            }
+        }
+        return mlNodeCount;
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheck.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheck.java
index daa143ec01977..1c1a28dc8943e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheck.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheck.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.ml;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -23,8 +22,6 @@
  */
 public class MlConfigMigrationEligibilityCheck {
 
-    private static final Version MIN_NODE_VERSION = Version.V_6_6_0;
-
     public static final Setting<Boolean> ENABLE_CONFIG_MIGRATION = Setting.boolSetting(
         "xpack.ml.enable_config_migration", true, Setting.Property.Dynamic, Setting.Property.NodeScope);
 
@@ -43,7 +40,6 @@ private void setConfigMigrationEnabled(boolean configMigrationEnabled) {
     /**
      * Can migration start? Returns:
      *     False if config migration is disabled via the setting {@link #ENABLE_CONFIG_MIGRATION}
-     *     False if the min node version of the cluster is before {@link #MIN_NODE_VERSION}
      *     False if the .ml-config index shards are not active
      *     True otherwise
      * @param clusterState The cluster state
@@ -53,12 +49,6 @@ public boolean canStartMigration(ClusterState clusterState) {
         if (isConfigMigrationEnabled == false) {
             return false;
         }
-
-        Version minNodeVersion = clusterState.nodes().getMinNodeVersion();
-        if (minNodeVersion.before(MIN_NODE_VERSION)) {
-            return false;
-        }
-
         return mlConfigIndexIsAllocated(clusterState);
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlLifeCycleService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlLifeCycleService.java
index 7309afa6b3ab4..f54edcc789889 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlLifeCycleService.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlLifeCycleService.java
@@ -7,30 +7,27 @@
 
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.LifecycleListener;
-import org.elasticsearch.env.Environment;
 import org.elasticsearch.xpack.ml.datafeed.DatafeedManager;
+import org.elasticsearch.xpack.ml.process.MlController;
 import org.elasticsearch.xpack.ml.process.MlMemoryTracker;
-import org.elasticsearch.xpack.ml.process.NativeController;
-import org.elasticsearch.xpack.ml.process.NativeControllerHolder;
 import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager;
 
 import java.io.IOException;
+import java.util.Objects;
 
 public class MlLifeCycleService {
 
-    private final Environment environment;
-    private final ClusterService clusterService;
     private final DatafeedManager datafeedManager;
+    private final MlController mlController;
     private final AutodetectProcessManager autodetectProcessManager;
     private final MlMemoryTracker memoryTracker;
 
-    public MlLifeCycleService(Environment environment, ClusterService clusterService, DatafeedManager datafeedManager,
-                              AutodetectProcessManager autodetectProcessManager, MlMemoryTracker memoryTracker) {
-        this.environment = environment;
-        this.clusterService = clusterService;
-        this.datafeedManager = datafeedManager;
-        this.autodetectProcessManager = autodetectProcessManager;
-        this.memoryTracker = memoryTracker;
+    MlLifeCycleService(ClusterService clusterService, DatafeedManager datafeedManager, MlController mlController,
+                       AutodetectProcessManager autodetectProcessManager, MlMemoryTracker memoryTracker) {
+        this.datafeedManager = Objects.requireNonNull(datafeedManager);
+        this.mlController = Objects.requireNonNull(mlController);
+        this.autodetectProcessManager = Objects.requireNonNull(autodetectProcessManager);
+        this.memoryTracker = Objects.requireNonNull(memoryTracker);
         clusterService.addLifecycleListener(new LifecycleListener() {
             @Override
             public void beforeStop() {
@@ -41,27 +38,16 @@ public void beforeStop() {
 
     public synchronized void stop() {
         try {
-            if (MachineLearningFeatureSet.isRunningOnMlPlatform(false)) {
-                // This prevents datafeeds from sending data to autodetect processes WITHOUT stopping the
-                // datafeeds, so they get reallocated.  We have to do this first, otherwise the datafeeds
-                // could fail if they send data to a dead autodetect process.
-                if (datafeedManager != null) {
-                    datafeedManager.isolateAllDatafeedsOnThisNodeBeforeShutdown();
-                }
-                NativeController nativeController = NativeControllerHolder.getNativeController(clusterService.getNodeName(), environment);
-                if (nativeController != null) {
-                    // This kills autodetect processes WITHOUT closing the jobs, so they get reallocated.
-                    if (autodetectProcessManager != null) {
-                        autodetectProcessManager.killAllProcessesOnThisNode();
-                    }
-                    nativeController.stop();
-                }
-            }
+            // This prevents datafeeds from sending data to autodetect processes WITHOUT stopping the
+            // datafeeds, so they get reassigned.  We have to do this first, otherwise the datafeeds
+            // could fail if they send data to a dead autodetect process.
+            datafeedManager.isolateAllDatafeedsOnThisNodeBeforeShutdown();
+            // This kills autodetect processes WITHOUT closing the jobs, so they get reassigned.
+            autodetectProcessManager.killAllProcessesOnThisNode();
+            mlController.stop();
         } catch (IOException e) {
             // We're stopping anyway, so don't let this complicate the shutdown sequence
         }
-        if (memoryTracker != null) {
-            memoryTracker.stop();
-        }
+        memoryTracker.stop();
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarAction.java
index ffc0548554724..d4cdb48dd5afe 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteCalendarAction.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
 import org.elasticsearch.tasks.Task;
@@ -77,7 +78,7 @@ protected void doExecute(Task task, DeleteCalendarAction.Request request, Action
 
     private DeleteByQueryRequest buildDeleteByQuery(String calendarId) {
         DeleteByQueryRequest request = new DeleteByQueryRequest(MlMetaIndex.INDEX_NAME);
-        request.setSlices(5);
+        request.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES);
         request.setRefresh(true);
 
         QueryBuilder query = QueryBuilders.termsQuery(Calendar.ID.getPreferredName(), calendarId);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java
new file mode 100644
index 0000000000000..dcc72bc8c75aa
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.delete.DeleteAction;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.ml.MlTasks;
+import org.elasticsearch.xpack.core.ml.action.DeleteDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.ml.process.MlMemoryTracker;
+
+import java.io.IOException;
+
+import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
+import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
+
+/**
+ * The action is a master node action to ensure it reads an up-to-date cluster
+ * state in order to determine whether there is a persistent task for the analytics
+ * to delete.
+ */
+public class TransportDeleteDataFrameAnalyticsAction
+    extends TransportMasterNodeAction<DeleteDataFrameAnalyticsAction.Request, AcknowledgedResponse> {
+
+    private final Client client;
+    private final MlMemoryTracker memoryTracker;
+
+    @Inject
+    public TransportDeleteDataFrameAnalyticsAction(TransportService transportService, ClusterService clusterService,
+                                                   ThreadPool threadPool, ActionFilters actionFilters,
+                                                   IndexNameExpressionResolver indexNameExpressionResolver, Client client,
+                                                   MlMemoryTracker memoryTracker) {
+        super(DeleteDataFrameAnalyticsAction.NAME, transportService, clusterService, threadPool, actionFilters,
+            DeleteDataFrameAnalyticsAction.Request::new, indexNameExpressionResolver);
+        this.client = client;
+        this.memoryTracker = memoryTracker;
+    }
+
+    @Override
+    protected String executor() {
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected AcknowledgedResponse newResponse() {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
+    @Override
+    protected void masterOperation(Task task, DeleteDataFrameAnalyticsAction.Request request, ClusterState state,
+                                   ActionListener<AcknowledgedResponse> listener) {
+        String id = request.getId();
+        PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
+        DataFrameAnalyticsState taskState = MlTasks.getDataFrameAnalyticsState(id, tasks);
+        if (taskState != DataFrameAnalyticsState.STOPPED) {
+            listener.onFailure(ExceptionsHelper.conflictStatusException("Cannot delete data frame analytics [{}] while its status is [{}]",
+                id, taskState));
+            return;
+        }
+
+        // We clean up the memory tracker on delete because there is no stop; the task stops by itself
+        memoryTracker.removeDataFrameAnalyticsJob(id);
+
+        DeleteRequest deleteRequest = new DeleteRequest(AnomalyDetectorsIndex.configIndexName());
+        deleteRequest.id(DataFrameAnalyticsConfig.documentId(id));
+        deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+        deleteRequest.setParentTask(clusterService.localNode().getId(), task.getId());
+        executeAsyncWithOrigin(client, ML_ORIGIN, DeleteAction.INSTANCE, deleteRequest, ActionListener.wrap(
+            deleteResponse -> {
+                if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) {
+                    listener.onFailure(ExceptionsHelper.missingDataFrameAnalytics(id));
+                    return;
+                }
+                assert deleteResponse.getResult() == DocWriteResponse.Result.DELETED;
+                listener.onResponse(new AcknowledgedResponse(true));
+            },
+            listener::onFailure
+        ));
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(DeleteDataFrameAnalyticsAction.Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java
index 97e437ec4caf2..d731f1b3b08a2 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDatafeedAction.java
@@ -17,10 +17,12 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.persistent.PersistentTasksService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.MlTasks;
@@ -32,6 +34,8 @@
 import org.elasticsearch.xpack.ml.MlConfigMigrationEligibilityCheck;
 import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider;
 
+import java.io.IOException;
+
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
 import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
 
@@ -63,13 +67,18 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(DeleteDatafeedAction.Request request, ClusterState state,
+    protected void masterOperation(Task task, DeleteDatafeedAction.Request request, ClusterState state,
                                    ActionListener<AcknowledgedResponse> listener) {
 
         if (migrationEligibilityCheck.datafeedIsEligibleForMigration(request.getDatafeedId(), state)) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteForecastAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteForecastAction.java
index 2575e71444447..1e5659bd23398 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteForecastAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteForecastAction.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
@@ -197,8 +198,8 @@ private static Set<ForecastRequestStats> parseForecastsFromSearch(SearchResponse
     private DeleteByQueryRequest buildDeleteByQuery(String jobId, List<String> forecastsToDelete) {
         DeleteByQueryRequest request = new DeleteByQueryRequest()
             .setAbortOnVersionConflict(false) //since these documents are not updated, a conflict just means it was deleted previously
-            .setSize(MAX_FORECAST_TO_SEARCH)
-            .setSlices(5);
+            .setMaxDocs(MAX_FORECAST_TO_SEARCH)
+            .setSlices(AbstractBulkByScrollRequest.AUTO_SLICES);
 
         request.indices(AnomalyDetectorsIndex.jobResultsAliasedName(jobId));
         BoolQueryBuilder innerBoolQuery = QueryBuilders.boolQuery();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java
index 3f8321fa4b1b8..b40f49523c751 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteJobAction.java
@@ -32,12 +32,14 @@
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
 import org.elasticsearch.index.query.IdsQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
@@ -72,6 +74,7 @@
 import org.elasticsearch.xpack.ml.process.MlMemoryTracker;
 import org.elasticsearch.xpack.ml.utils.MlIndicesUtils;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -132,18 +135,18 @@ protected String executor() {
     }
 
     @Override
-    protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
     }
 
     @Override
-    protected ClusterBlockException checkBlock(DeleteJobAction.Request request, ClusterState state) {
-        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    protected AcknowledgedResponse newResponse() {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(DeleteJobAction.Request request, ClusterState state, ActionListener<AcknowledgedResponse> listener) {
-        throw new UnsupportedOperationException("the Task parameter is required");
+    protected ClusterBlockException checkBlock(DeleteJobAction.Request request, ClusterState state) {
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
     }
 
     @Override
@@ -231,7 +234,7 @@ private void normalDeleteJob(ParentTaskAssigningClient parentTaskClient, DeleteJ
         String jobId = request.getJobId();
 
         // We clean up the memory tracker on delete rather than close as close is not a master node action
-        memoryTracker.removeJob(jobId);
+        memoryTracker.removeAnomalyDetectorJob(jobId);
 
         // Step 4. When the job has been removed from the cluster state, return a response
         // -------
@@ -305,7 +308,7 @@ private void deleteJobDocuments(ParentTaskAssigningClient parentTaskClient, Stri
                                 new ConstantScoreQueryBuilder(new TermQueryBuilder(Job.ID.getPreferredName(), jobId));
                         request.setQuery(query);
                         request.setIndicesOptions(MlIndicesUtils.addIgnoreUnavailable(IndicesOptions.lenientExpandOpen()));
-                        request.setSlices(5);
+                        request.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES);
                         request.setAbortOnVersionConflict(false);
                         request.setRefresh(true);
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEvaluateDataFrameAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEvaluateDataFrameAction.java
new file mode 100644
index 0000000000000..bb7365cd53809
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportEvaluateDataFrameAction.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.Evaluation;
+import org.elasticsearch.xpack.core.ml.dataframe.evaluation.EvaluationMetricResult;
+
+import java.util.List;
+
+public class TransportEvaluateDataFrameAction extends HandledTransportAction<EvaluateDataFrameAction.Request,
+    EvaluateDataFrameAction.Response> {
+
+    private final ThreadPool threadPool;
+    private final Client client;
+
+    @Inject
+    public TransportEvaluateDataFrameAction(TransportService transportService, ActionFilters actionFilters, ThreadPool threadPool,
+                                            Client client) {
+        super(EvaluateDataFrameAction.NAME, transportService, actionFilters, EvaluateDataFrameAction.Request::new);
+        this.threadPool = threadPool;
+        this.client = client;
+    }
+
+    @Override
+    protected void doExecute(Task task, EvaluateDataFrameAction.Request request,
+                             ActionListener<EvaluateDataFrameAction.Response> listener) {
+        Evaluation evaluation = request.getEvaluation();
+        SearchRequest searchRequest = new SearchRequest(request.getIndices());
+        searchRequest.source(evaluation.buildSearch());
+
+        ActionListener<List<EvaluationMetricResult>> resultsListener = ActionListener.wrap(
+            results -> listener.onResponse(new EvaluateDataFrameAction.Response(evaluation.getName(), results)),
+            listener::onFailure
+        );
+
+        client.execute(SearchAction.INSTANCE, searchRequest, ActionListener.wrap(
+            searchResponse -> threadPool.generic().execute(() -> {
+                try {
+                    evaluation.evaluate(searchResponse, resultsListener);
+                } catch (Exception e) {
+                    listener.onFailure(e);
+                };
+            }),
+            listener::onFailure
+        ));
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java
index 14491d7ca9c61..24b8998763ec8 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionAction.java
@@ -19,6 +19,8 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
@@ -27,6 +29,7 @@
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.utils.VoidChainTaskExecutor;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.Date;
 import java.util.Map;
@@ -53,13 +56,18 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(FinalizeJobExecutionAction.Request request, ClusterState state,
+    protected void masterOperation(Task task, FinalizeJobExecutionAction.Request request, ClusterState state,
                                    ActionListener<AcknowledgedResponse> listener) {
         String jobIdString = String.join(",", request.getJobIds());
         logger.debug("finalizing jobs [{}]", jobIdString);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFindFileStructureAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFindFileStructureAction.java
index 41b9cab23e1ed..63c292a9bebfd 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFindFileStructureAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportFindFileStructureAction.java
@@ -18,6 +18,8 @@
 import org.elasticsearch.xpack.ml.filestructurefinder.FileStructureFinderManager;
 import org.elasticsearch.xpack.ml.filestructurefinder.FileStructureOverrides;
 
+import java.io.InputStream;
+
 public class TransportFindFileStructureAction
     extends HandledTransportAction<FindFileStructureAction.Request, FindFileStructureAction.Response> {
 
@@ -48,9 +50,11 @@ private FindFileStructureAction.Response buildFileStructureResponse(FindFileStru
 
         FileStructureFinderManager structureFinderManager = new FileStructureFinderManager(threadPool.scheduler());
 
-        FileStructureFinder fileStructureFinder = structureFinderManager.findFileStructure(request.getLinesToSample(),
-            request.getSample().streamInput(), new FileStructureOverrides(request), request.getTimeout());
+        try (InputStream sampleStream = request.getSample().streamInput()) {
+            FileStructureFinder fileStructureFinder = structureFinderManager.findFileStructure(request.getLinesToSample(),
+                request.getLineMergeSizeLimit(), sampleStream, new FileStructureOverrides(request), request.getTimeout());
 
-        return new FindFileStructureAction.Response(fileStructureFinder.getStructure());
+            return new FindFileStructureAction.Response(fileStructureFinder.getStructure());
+        }
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java
new file mode 100644
index 0000000000000..fe1d3ac36f138
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.action;
+
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.action.AbstractTransportGetResourcesAction;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+
+import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
+
+public class TransportGetDataFrameAnalyticsAction extends AbstractTransportGetResourcesAction<DataFrameAnalyticsConfig,
+        GetDataFrameAnalyticsAction.Request, GetDataFrameAnalyticsAction.Response> {
+
+    @Inject
+    public TransportGetDataFrameAnalyticsAction(TransportService transportService, ActionFilters actionFilters, Client client,
+                                                NamedXContentRegistry xContentRegistry) {
+        super(GetDataFrameAnalyticsAction.NAME, transportService, actionFilters, GetDataFrameAnalyticsAction.Request::new, client,
+            xContentRegistry);
+    }
+
+    @Override
+    protected ParseField getResultsField() {
+        return GetDataFrameAnalyticsAction.Response.RESULTS_FIELD;
+    }
+
+    @Override
+    protected String[] getIndices() {
+        return new String[] { AnomalyDetectorsIndex.configIndexName() };
+    }
+
+    @Override
+    protected DataFrameAnalyticsConfig parse(XContentParser parser) {
+        return DataFrameAnalyticsConfig.LENIENT_PARSER.apply(parser, null).build();
+    }
+
+    @Override
+    protected ResourceNotFoundException notFoundException(String resourceId) {
+        return ExceptionsHelper.missingDataFrameAnalytics(resourceId);
+    }
+
+    @Override
+    protected void doExecute(Task task, GetDataFrameAnalyticsAction.Request request,
+                             ActionListener<GetDataFrameAnalyticsAction.Response> listener) {
+        searchResources(request, ActionListener.wrap(
+            queryPage -> listener.onResponse(new GetDataFrameAnalyticsAction.Response(queryPage)),
+            listener::onFailure
+        ));
+    }
+
+    @Nullable
+    protected QueryBuilder additionalQuery() {
+        return QueryBuilders.termQuery(DataFrameAnalyticsConfig.CONFIG_TYPE.getPreferredName(), DataFrameAnalyticsConfig.TYPE);
+    }
+
+    @Override
+    protected String executionOrigin() {
+        return ML_ORIGIN;
+    }
+
+    @Override
+    protected String extractIdFromResource(DataFrameAnalyticsConfig config) {
+        return config.getId();
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
new file mode 100644
index 0000000000000..8d139ba9b6ca4
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java
@@ -0,0 +1,196 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.action.TaskOperationFailure;
+import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.tasks.TransportTasksAction;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.index.reindex.BulkByScrollTask;
+import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskResult;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.action.util.QueryPage;
+import org.elasticsearch.xpack.core.ml.MlTasks;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction.Response.Stats;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
+import org.elasticsearch.xpack.ml.action.TransportStartDataFrameAnalyticsAction.DataFrameAnalyticsTask;
+import org.elasticsearch.xpack.ml.dataframe.process.AnalyticsProcessManager;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
+import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
+
+public class TransportGetDataFrameAnalyticsStatsAction
+    extends TransportTasksAction<DataFrameAnalyticsTask, GetDataFrameAnalyticsStatsAction.Request,
+        GetDataFrameAnalyticsStatsAction.Response, QueryPage<Stats>> {
+
+    private static final Logger LOGGER = LogManager.getLogger(TransportGetDataFrameAnalyticsStatsAction.class);
+
+    private final Client client;
+    private final AnalyticsProcessManager analyticsProcessManager;
+
+    @Inject
+    public TransportGetDataFrameAnalyticsStatsAction(TransportService transportService, ClusterService clusterService, Client client,
+                                                     ActionFilters actionFilters, AnalyticsProcessManager analyticsProcessManager) {
+        super(GetDataFrameAnalyticsStatsAction.NAME, clusterService, transportService, actionFilters,
+            GetDataFrameAnalyticsStatsAction.Request::new, GetDataFrameAnalyticsStatsAction.Response::new,
+            in -> new QueryPage<>(in, GetDataFrameAnalyticsStatsAction.Response.Stats::new), ThreadPool.Names.MANAGEMENT);
+        this.client = client;
+        this.analyticsProcessManager = analyticsProcessManager;
+    }
+
+    @Override
+    protected GetDataFrameAnalyticsStatsAction.Response newResponse(GetDataFrameAnalyticsStatsAction.Request request,
+                                                                    List<QueryPage<Stats>> tasks,
+                                                                    List<TaskOperationFailure> taskFailures,
+                                                                    List<FailedNodeException> nodeFailures) {
+        List<Stats> stats = new ArrayList<>();
+        for (QueryPage<Stats> task : tasks) {
+            stats.addAll(task.results());
+        }
+        Collections.sort(stats, Comparator.comparing(Stats::getId));
+        return new GetDataFrameAnalyticsStatsAction.Response(taskFailures, nodeFailures, new QueryPage<>(stats, stats.size(),
+            GetDataFrameAnalyticsAction.Response.RESULTS_FIELD));
+    }
+
+    @Override
+    protected void taskOperation(GetDataFrameAnalyticsStatsAction.Request request, DataFrameAnalyticsTask task,
+                                 ActionListener<QueryPage<Stats>> listener) {
+        LOGGER.debug("Get stats for running task [{}]", task.getParams().getId());
+
+        ActionListener<Integer> progressListener = ActionListener.wrap(
+            progress -> {
+                Stats stats = buildStats(task.getParams().getId(), progress);
+                listener.onResponse(new QueryPage<>(Collections.singletonList(stats), 1,
+                    GetDataFrameAnalyticsAction.Response.RESULTS_FIELD));
+            }, listener::onFailure
+        );
+
+        ClusterState clusterState = clusterService.state();
+        PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
+        DataFrameAnalyticsState analyticsState = MlTasks.getDataFrameAnalyticsState(task.getParams().getId(), tasks);
+
+        // For a running task we report the progress associated with its current state
+        if (analyticsState == DataFrameAnalyticsState.REINDEXING) {
+            getReindexTaskProgress(task, progressListener);
+        } else {
+            progressListener.onResponse(analyticsProcessManager.getProgressPercent(task.getAllocationId()));
+        }
+    }
+
+    private void getReindexTaskProgress(DataFrameAnalyticsTask task, ActionListener<Integer> listener) {
+        TaskId reindexTaskId = new TaskId(clusterService.localNode().getId(), task.getReindexingTaskId());
+        GetTaskRequest getTaskRequest = new GetTaskRequest();
+        getTaskRequest.setTaskId(reindexTaskId);
+        client.admin().cluster().getTask(getTaskRequest, ActionListener.wrap(
+            taskResponse -> {
+                TaskResult taskResult = taskResponse.getTask();
+                BulkByScrollTask.Status taskStatus = (BulkByScrollTask.Status) taskResult.getTask().getStatus();
+                int progress =  taskStatus.getTotal() == 0 ? 100 : (int) (taskStatus.getCreated() * 100.0 / taskStatus.getTotal());
+                listener.onResponse(progress);
+            },
+            error -> {
+                if (error instanceof ResourceNotFoundException) {
+                    // The task has either not started yet or has finished, thus it is better to respond null and not show progress at all
+                    listener.onResponse(null);
+                } else {
+                    listener.onFailure(error);
+                }
+            }
+        ));
+    }
+
+    @Override
+    protected void doExecute(Task task, GetDataFrameAnalyticsStatsAction.Request request,
+                             ActionListener<GetDataFrameAnalyticsStatsAction.Response> listener) {
+        LOGGER.debug("Get stats for data frame analytics [{}]", request.getId());
+
+        ActionListener<GetDataFrameAnalyticsAction.Response> getResponseListener = ActionListener.wrap(
+            response -> {
+                List<String> expandedIds = response.getResources().results().stream().map(DataFrameAnalyticsConfig::getId)
+                    .collect(Collectors.toList());
+                request.setExpandedIds(expandedIds);
+                ActionListener<GetDataFrameAnalyticsStatsAction.Response> runningTasksStatsListener = ActionListener.wrap(
+                    runningTasksStatsResponse -> gatherStatsForStoppedTasks(request.getExpandedIds(), runningTasksStatsResponse, listener),
+                    listener::onFailure
+                );
+                super.doExecute(task, request, runningTasksStatsListener);
+            },
+            listener::onFailure
+        );
+
+        GetDataFrameAnalyticsAction.Request getRequest = new GetDataFrameAnalyticsAction.Request();
+        getRequest.setResourceId(request.getId());
+        getRequest.setAllowNoResources(request.isAllowNoMatch());
+        getRequest.setPageParams(request.getPageParams());
+        executeAsyncWithOrigin(client, ML_ORIGIN, GetDataFrameAnalyticsAction.INSTANCE, getRequest, getResponseListener);
+    }
+
+    void gatherStatsForStoppedTasks(List<String> expandedIds, GetDataFrameAnalyticsStatsAction.Response runningTasksResponse,
+                                    ActionListener<GetDataFrameAnalyticsStatsAction.Response> listener) {
+        List<String> stoppedTasksIds = determineStoppedTasksIds(expandedIds, runningTasksResponse.getResponse().results());
+        List<Stats> stoppedTasksStats = stoppedTasksIds.stream().map(this::buildStatsForStoppedTask).collect(Collectors.toList());
+        List<Stats> allTasksStats = new ArrayList<>(runningTasksResponse.getResponse().results());
+        allTasksStats.addAll(stoppedTasksStats);
+        Collections.sort(allTasksStats, Comparator.comparing(Stats::getId));
+        listener.onResponse(new GetDataFrameAnalyticsStatsAction.Response(new QueryPage<>(
+            allTasksStats, allTasksStats.size(), GetDataFrameAnalyticsAction.Response.RESULTS_FIELD)));
+    }
+
+    static List<String> determineStoppedTasksIds(List<String> expandedIds, List<Stats> runningTasksStats) {
+        Set<String> startedTasksIds = runningTasksStats.stream().map(Stats::getId).collect(Collectors.toSet());
+        return expandedIds.stream().filter(id -> startedTasksIds.contains(id) == false).collect(Collectors.toList());
+    }
+
+    private GetDataFrameAnalyticsStatsAction.Response.Stats buildStatsForStoppedTask(String concreteAnalyticsId) {
+        return buildStats(concreteAnalyticsId, null);
+    }
+
+    private GetDataFrameAnalyticsStatsAction.Response.Stats buildStats(String concreteAnalyticsId, @Nullable Integer progressPercent) {
+        ClusterState clusterState = clusterService.state();
+        PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
+        PersistentTasksCustomMetaData.PersistentTask<?> analyticsTask = MlTasks.getDataFrameAnalyticsTask(concreteAnalyticsId, tasks);
+        DataFrameAnalyticsState analyticsState = MlTasks.getDataFrameAnalyticsState(concreteAnalyticsId, tasks);
+        String failureReason = null;
+        if (analyticsState == DataFrameAnalyticsState.FAILED) {
+            DataFrameAnalyticsTaskState taskState = (DataFrameAnalyticsTaskState) analyticsTask.getState();
+            failureReason = taskState.getReason();
+        }
+        DiscoveryNode node = null;
+        String assignmentExplanation = null;
+        if (analyticsTask != null) {
+            node = clusterState.nodes().get(analyticsTask.getExecutorNode());
+            assignmentExplanation = analyticsTask.getAssignment().getExplanation();
+        }
+        return new GetDataFrameAnalyticsStatsAction.Response.Stats(
+            concreteAnalyticsId, analyticsState, failureReason, progressPercent, node, assignmentExplanation);
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java
index 72ebc044364fe..89f14f717064e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsAction.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
@@ -59,7 +60,7 @@ protected GetDatafeedsAction.Response newResponse() {
     }
 
     @Override
-    protected void masterOperation(GetDatafeedsAction.Request request, ClusterState state,
+    protected void masterOperation(Task task, GetDatafeedsAction.Request request, ClusterState state,
                                    ActionListener<GetDatafeedsAction.Response> listener) {
         logger.debug("Get datafeed '{}'", request.getDatafeedId());
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java
index 6b32d414516a0..6c90b15dab13b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDatafeedsStatsAction.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.MlTasks;
@@ -54,7 +55,7 @@ protected GetDatafeedsStatsAction.Response newResponse() {
     }
 
     @Override
-    protected void masterOperation(GetDatafeedsStatsAction.Request request, ClusterState state,
+    protected void masterOperation(Task task, GetDatafeedsStatsAction.Request request, ClusterState state,
                                    ActionListener<GetDatafeedsStatsAction.Response> listener) throws Exception {
         logger.debug("Get stats for datafeed '{}'", request.getDatafeedId());
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java
index 5687198f6ec69..ba81bfac88277 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsAction.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.GetJobsAction;
@@ -44,7 +45,7 @@ protected GetJobsAction.Response newResponse() {
     }
 
     @Override
-    protected void masterOperation(GetJobsAction.Request request, ClusterState state,
+    protected void masterOperation(Task task, GetJobsAction.Request request, ClusterState state,
                                    ActionListener<GetJobsAction.Response> listener) {
         logger.debug("Get job '{}'", request.getJobId());
         jobManager.expandJobs(request.getJobId(), request.allowNoJobs(), ActionListener.wrap(
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java
index eb9d4c9239d22..2ad70e996b51f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsAction.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.TriConsumer;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.unit.TimeValue;
@@ -29,6 +30,7 @@
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 import org.elasticsearch.xpack.core.ml.stats.ForecastStats;
 import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider;
 import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
@@ -42,7 +44,6 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
@@ -103,16 +104,19 @@ protected void taskOperation(GetJobsStatsAction.Request request, TransportOpenJo
         String jobId = task.getJobId();
         ClusterState state = clusterService.state();
         PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
-        Optional<Tuple<DataCounts, ModelSizeStats>> stats = processManager.getStatistics(task);
+        Optional<Tuple<DataCounts, Tuple<ModelSizeStats, TimingStats>>> stats = processManager.getStatistics(task);
         if (stats.isPresent()) {
+            DataCounts dataCounts = stats.get().v1();
+            ModelSizeStats modelSizeStats = stats.get().v2().v1();
+            TimingStats timingStats = stats.get().v2().v2();
             PersistentTasksCustomMetaData.PersistentTask<?> pTask = MlTasks.getJobTask(jobId, tasks);
             DiscoveryNode node = state.nodes().get(pTask.getExecutorNode());
             JobState jobState = MlTasks.getJobState(jobId, tasks);
             String assignmentExplanation = pTask.getAssignment().getExplanation();
             TimeValue openTime = durationToTimeValue(processManager.jobOpenTime(task));
             gatherForecastStats(jobId, forecastStats -> {
-                JobStats jobStats = new JobStats(jobId, stats.get().v1(),
-                        stats.get().v2(), forecastStats, jobState, node, assignmentExplanation, openTime);
+                JobStats jobStats = new JobStats(
+                    jobId, dataCounts, modelSizeStats, forecastStats, jobState, node, assignmentExplanation, openTime, timingStats);
                 listener.onResponse(new QueryPage<>(Collections.singletonList(jobStats), 1, Job.RESULTS_FIELD));
             }, listener::onFailure);
 
@@ -138,7 +142,7 @@ void gatherStatsForClosedJobs(GetJobsStatsAction.Request request, GetJobsStatsAc
             int slot = i;
             String jobId = closedJobIds.get(i);
             gatherForecastStats(jobId, forecastStats -> {
-                gatherDataCountsAndModelSizeStats(jobId, (dataCounts, modelSizeStats) -> {
+                gatherDataCountsModelSizeStatsAndTimingStats(jobId, (dataCounts, modelSizeStats, timingStats) -> {
                     JobState jobState = MlTasks.getJobState(jobId, tasks);
                     PersistentTasksCustomMetaData.PersistentTask<?> pTask = MlTasks.getJobTask(jobId, tasks);
                     String assignmentExplanation = null;
@@ -146,7 +150,7 @@ void gatherStatsForClosedJobs(GetJobsStatsAction.Request request, GetJobsStatsAc
                         assignmentExplanation = pTask.getAssignment().getExplanation();
                     }
                     jobStats.set(slot, new JobStats(jobId, dataCounts, modelSizeStats, forecastStats, jobState,
-                            null, assignmentExplanation, null));
+                            null, assignmentExplanation, null, timingStats));
                     if (counter.decrementAndGet() == 0) {
                         List<JobStats> results = response.getResponse().results();
                         results.addAll(jobStats.asList());
@@ -163,11 +167,13 @@ void gatherForecastStats(String jobId, Consumer<ForecastStats> handler, Consumer
         jobResultsProvider.getForecastStats(jobId, handler, errorHandler);
     }
 
-    void gatherDataCountsAndModelSizeStats(String jobId, BiConsumer<DataCounts, ModelSizeStats> handler,
-                                                   Consumer<Exception> errorHandler) {
+    void gatherDataCountsModelSizeStatsAndTimingStats(
+            String jobId, TriConsumer<DataCounts, ModelSizeStats, TimingStats> handler, Consumer<Exception> errorHandler) {
         jobResultsProvider.dataCounts(jobId, dataCounts -> {
             jobResultsProvider.modelSizeStats(jobId, modelSizeStats -> {
-                handler.accept(dataCounts, modelSizeStats);
+                jobResultsProvider.timingStats(jobId, timingStats -> {
+                    handler.apply(dataCounts, modelSizeStats, timingStats);
+                }, errorHandler);
             }, errorHandler);
         }, errorHandler);
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java
index 48fbf9566fd1e..58bb42a5e3e79 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java
@@ -20,19 +20,30 @@
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.ml.process.MlControllerHolder;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.TimeoutException;
 import java.util.function.Supplier;
 
 public class TransportMlInfoAction extends HandledTransportAction<MlInfoAction.Request, MlInfoAction.Response> {
 
     private final ClusterService clusterService;
+    private final Map<String, Object> nativeCodeInfo;
 
     @Inject
-    public TransportMlInfoAction(TransportService transportService, ActionFilters actionFilters, ClusterService clusterService) {
+    public TransportMlInfoAction(TransportService transportService, ActionFilters actionFilters,
+                                 ClusterService clusterService, MlControllerHolder mlControllerHolder) {
         super(MlInfoAction.NAME, transportService, actionFilters, (Supplier<MlInfoAction.Request>) MlInfoAction.Request::new);
+
         this.clusterService = clusterService;
+
+        try {
+            nativeCodeInfo = mlControllerHolder.getMlController().getNativeCodeInfo();
+        } catch (TimeoutException e) {
+            throw new RuntimeException("Could not get native code info from native controller", e);
+        }
     }
 
     @Override
@@ -40,6 +51,7 @@ protected void doExecute(Task task, MlInfoAction.Request request, ActionListener
         Map<String, Object> info = new HashMap<>();
         info.put("defaults", defaults());
         info.put("limits", limits());
+        info.put("native_code", nativeCodeInfo);
         info.put(MlMetadata.UPGRADE_MODE.getPreferredName(), upgradeMode());
         listener.onResponse(new MlInfoAction.Response(info));
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
index d2aea878b0f50..1be58fa6998ff 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ResourceAlreadyExistsException;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -24,8 +23,8 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.license.LicenseUtils;
@@ -36,6 +35,7 @@
 import org.elasticsearch.persistent.PersistentTasksExecutor;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -45,7 +45,6 @@
 import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
 import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
-import org.elasticsearch.xpack.core.ml.job.config.DetectionRule;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
 import org.elasticsearch.xpack.core.ml.job.config.JobTaskState;
@@ -54,16 +53,16 @@
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.MlConfigMigrationEligibilityCheck;
+import org.elasticsearch.xpack.ml.job.JobNodeSelector;
 import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider;
 import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager;
 import org.elasticsearch.xpack.ml.process.MlMemoryTracker;
 
+import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
+import java.util.Objects;
 import java.util.function.Predicate;
 
 import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
@@ -82,9 +81,6 @@ To ensure that a subsequent close job call will see that same task status (and s
 */
 public class TransportOpenJobAction extends TransportMasterNodeAction<OpenJobAction.Request, AcknowledgedResponse> {
 
-    private static final PersistentTasksCustomMetaData.Assignment AWAITING_LAZY_ASSIGNMENT =
-        new PersistentTasksCustomMetaData.Assignment(null, "persistent task is awaiting node assignment.");
-
     static final PersistentTasksCustomMetaData.Assignment AWAITING_MIGRATION =
             new PersistentTasksCustomMetaData.Assignment(null, "job cannot be assigned until it has been migrated.");
 
@@ -130,218 +126,6 @@ static void validate(String jobId, Job job) {
         }
     }
 
-    static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String jobId, Job job,
-                                                                            ClusterState clusterState,
-                                                                            int dynamicMaxOpenJobs,
-                                                                            int maxConcurrentJobAllocations,
-                                                                            int maxMachineMemoryPercent,
-                                                                            MlMemoryTracker memoryTracker,
-                                                                            boolean isMemoryTrackerRecentlyRefreshed,
-                                                                            Logger logger) {
-        // TODO: remove in 8.0.0
-        boolean allNodesHaveDynamicMaxWorkers = clusterState.getNodes().getMinNodeVersion().onOrAfter(Version.V_7_2_0);
-
-        // Try to allocate jobs according to memory usage, but if that's not possible (maybe due to a mixed version cluster or maybe
-        // because of some weird OS problem) then fall back to the old mechanism of only considering numbers of assigned jobs
-        boolean allocateByMemory = isMemoryTrackerRecentlyRefreshed;
-        if (isMemoryTrackerRecentlyRefreshed == false) {
-            logger.warn("Falling back to allocating job [{}] by job counts because a memory requirement refresh could not be scheduled",
-                jobId);
-        }
-
-        List<String> reasons = new LinkedList<>();
-        long maxAvailableCount = Long.MIN_VALUE;
-        long maxAvailableMemory = Long.MIN_VALUE;
-        DiscoveryNode minLoadedNodeByCount = null;
-        DiscoveryNode minLoadedNodeByMemory = null;
-        PersistentTasksCustomMetaData persistentTasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
-        for (DiscoveryNode node : clusterState.getNodes()) {
-            if (MachineLearning.isMlNode(node) == false) {
-                String reason = "Not opening job [" + jobId + "] on node [" + nodeNameOrId(node)
-                        + "], because this node isn't a ml node.";
-                logger.trace(reason);
-                reasons.add(reason);
-                continue;
-            }
-
-            if (nodeSupportsModelSnapshotVersion(node, job) == false) {
-                String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndVersion(node)
-                        + "], because the job's model snapshot requires a node of version ["
-                        + job.getModelSnapshotMinVersion() + "] or higher";
-                logger.trace(reason);
-                reasons.add(reason);
-                continue;
-            }
-
-            Set<String> compatibleJobTypes = Job.getCompatibleJobTypes(node.getVersion());
-            if (compatibleJobTypes.contains(job.getJobType()) == false) {
-                String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndVersion(node) +
-                        "], because this node does not support jobs of type [" + job.getJobType() + "]";
-                logger.trace(reason);
-                reasons.add(reason);
-                continue;
-            }
-
-            if (jobHasRules(job) && node.getVersion().before(DetectionRule.VERSION_INTRODUCED)) {
-                String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndVersion(node) + "], because jobs using " +
-                        "custom_rules require a node of version [" + DetectionRule.VERSION_INTRODUCED + "] or higher";
-                logger.trace(reason);
-                reasons.add(reason);
-                continue;
-            }
-
-            long numberOfAssignedJobs = 0;
-            int numberOfAllocatingJobs = 0;
-            long assignedJobMemory = 0;
-            if (persistentTasks != null) {
-                // find all the job tasks assigned to this node
-                Collection<PersistentTasksCustomMetaData.PersistentTask<?>> assignedTasks = persistentTasks.findTasks(
-                        MlTasks.JOB_TASK_NAME, task -> node.getId().equals(task.getExecutorNode()));
-                for (PersistentTasksCustomMetaData.PersistentTask<?> assignedTask : assignedTasks) {
-                    JobState jobState = MlTasks.getJobStateModifiedForReassignments(assignedTask);
-                    if (jobState.isAnyOf(JobState.CLOSED, JobState.FAILED) == false) {
-                        // Don't count CLOSED or FAILED jobs, as they don't consume native memory
-                        ++numberOfAssignedJobs;
-                        if (jobState == JobState.OPENING) {
-                            ++numberOfAllocatingJobs;
-                        }
-                        OpenJobAction.JobParams params = (OpenJobAction.JobParams) assignedTask.getParams();
-                        Long jobMemoryRequirement = memoryTracker.getJobMemoryRequirement(params.getJobId());
-                        if (jobMemoryRequirement == null) {
-                            allocateByMemory = false;
-                            logger.debug("Falling back to allocating job [{}] by job counts because " +
-                                    "the memory requirement for job [{}] was not available", jobId, params.getJobId());
-                        } else {
-                            assignedJobMemory += jobMemoryRequirement;
-                        }
-                    }
-                }
-            }
-            if (numberOfAllocatingJobs >= maxConcurrentJobAllocations) {
-                String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndMlAttributes(node)
-                        + "], because node exceeds [" + numberOfAllocatingJobs +
-                        "] the maximum number of jobs [" + maxConcurrentJobAllocations + "] in opening state";
-                logger.trace(reason);
-                reasons.add(reason);
-                continue;
-            }
-
-            Map<String, String> nodeAttributes = node.getAttributes();
-            int maxNumberOfOpenJobs = dynamicMaxOpenJobs;
-            // TODO: remove this in 8.0.0
-            if (allNodesHaveDynamicMaxWorkers == false) {
-                String maxNumberOfOpenJobsStr = nodeAttributes.get(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR);
-                try {
-                    maxNumberOfOpenJobs = Integer.parseInt(maxNumberOfOpenJobsStr);
-                } catch (NumberFormatException e) {
-                    String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndMlAttributes(node) + "], because " +
-                        MachineLearning.MAX_OPEN_JOBS_NODE_ATTR + " attribute [" + maxNumberOfOpenJobsStr + "] is not an integer";
-                    logger.trace(reason);
-                    reasons.add(reason);
-                    continue;
-                }
-            }
-            long availableCount = maxNumberOfOpenJobs - numberOfAssignedJobs;
-            if (availableCount == 0) {
-                String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndMlAttributes(node)
-                        + "], because this node is full. Number of opened jobs [" + numberOfAssignedJobs
-                        + "], " + MAX_OPEN_JOBS_PER_NODE.getKey() + " [" + maxNumberOfOpenJobs + "]";
-                logger.trace(reason);
-                reasons.add(reason);
-                continue;
-            }
-
-            if (maxAvailableCount < availableCount) {
-                maxAvailableCount = availableCount;
-                minLoadedNodeByCount = node;
-            }
-
-            String machineMemoryStr = nodeAttributes.get(MachineLearning.MACHINE_MEMORY_NODE_ATTR);
-            long machineMemory;
-            try {
-                machineMemory = Long.parseLong(machineMemoryStr);
-            } catch (NumberFormatException e) {
-                String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndMlAttributes(node) + "], because " +
-                    MachineLearning.MACHINE_MEMORY_NODE_ATTR + " attribute [" + machineMemoryStr + "] is not a long";
-                logger.trace(reason);
-                reasons.add(reason);
-                continue;
-            }
-
-            if (allocateByMemory) {
-                if (machineMemory > 0) {
-                    long maxMlMemory = machineMemory * maxMachineMemoryPercent / 100;
-                    Long estimatedMemoryFootprint = memoryTracker.getJobMemoryRequirement(jobId);
-                    if (estimatedMemoryFootprint != null) {
-                        long availableMemory = maxMlMemory - assignedJobMemory;
-                        if (estimatedMemoryFootprint > availableMemory) {
-                            String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndMlAttributes(node) +
-                                "], because this node has insufficient available memory. Available memory for ML [" + maxMlMemory +
-                                "], memory required by existing jobs [" + assignedJobMemory +
-                                "], estimated memory required for this job [" + estimatedMemoryFootprint + "]";
-                            logger.trace(reason);
-                            reasons.add(reason);
-                            continue;
-                        }
-
-                        if (maxAvailableMemory < availableMemory) {
-                            maxAvailableMemory = availableMemory;
-                            minLoadedNodeByMemory = node;
-                        }
-                    } else {
-                        // If we cannot get the job memory requirement,
-                        // fall back to simply allocating by job count
-                        allocateByMemory = false;
-                        logger.debug("Falling back to allocating job [{}] by job counts because its memory requirement was not available",
-                            jobId);
-                    }
-                } else {
-                    // If we cannot get the available memory on any machine in
-                    // the cluster, fall back to simply allocating by job count
-                    allocateByMemory = false;
-                    logger.debug("Falling back to allocating job [{}] by job counts because machine memory was not available for node [{}]",
-                        jobId, nodeNameAndMlAttributes(node));
-                }
-            }
-        }
-        DiscoveryNode minLoadedNode = allocateByMemory ? minLoadedNodeByMemory : minLoadedNodeByCount;
-        if (minLoadedNode != null) {
-            logger.debug("selected node [{}] for job [{}]", minLoadedNode, jobId);
-            return new PersistentTasksCustomMetaData.Assignment(minLoadedNode.getId(), "");
-        } else {
-            String explanation = String.join("|", reasons);
-            logger.debug("no node selected for job [{}], reasons [{}]", jobId, explanation);
-            return new PersistentTasksCustomMetaData.Assignment(null, explanation);
-        }
-    }
-
-    static String nodeNameOrId(DiscoveryNode node) {
-        String nodeNameOrID = node.getName();
-        if (Strings.isNullOrEmpty(nodeNameOrID)) {
-            nodeNameOrID = node.getId();
-        }
-        return nodeNameOrID;
-    }
-
-    static String nodeNameAndVersion(DiscoveryNode node) {
-        String nodeNameOrID = nodeNameOrId(node);
-        StringBuilder builder = new StringBuilder("{").append(nodeNameOrID).append('}');
-        builder.append('{').append("version=").append(node.getVersion()).append('}');
-        return builder.toString();
-    }
-
-    static String nodeNameAndMlAttributes(DiscoveryNode node) {
-        String nodeNameOrID = nodeNameOrId(node);
-
-        StringBuilder builder = new StringBuilder("{").append(nodeNameOrID).append('}');
-        for (Map.Entry<String, String> entry : node.getAttributes().entrySet()) {
-            if (entry.getKey().startsWith("ml.") || entry.getKey().equals("node.ml")) {
-                builder.append('{').append(entry).append('}');
-            }
-        }
-        return builder.toString();
-    }
-
     static String[] indicesOfInterest(String resultsIndex) {
         if (resultsIndex == null) {
             return new String[]{AnomalyDetectorsIndex.jobStateIndexPattern(), MlMetaIndex.INDEX_NAME};
@@ -381,6 +165,24 @@ private static boolean jobHasRules(Job job) {
         return job.getAnalysisConfig().getDetectors().stream().anyMatch(d -> d.getRules().isEmpty() == false);
     }
 
+    public static String nodeFilter(DiscoveryNode node, Job job) {
+
+        String jobId = job.getId();
+
+        if (TransportOpenJobAction.nodeSupportsModelSnapshotVersion(node, job) == false) {
+            return "Not opening job [" + jobId + "] on node [" + JobNodeSelector.nodeNameAndVersion(node)
+                + "], because the job's model snapshot requires a node of version ["
+                + job.getModelSnapshotMinVersion() + "] or higher";
+        }
+
+        if (Job.getCompatibleJobTypes(node.getVersion()).contains(job.getJobType()) == false) {
+            return "Not opening job [" + jobId + "] on node [" + JobNodeSelector.nodeNameAndVersion(node) +
+                "], because this node does not support jobs of type [" + job.getJobType() + "]";
+        }
+
+        return null;
+    }
+
     @Override
     protected String executor() {
         // This api doesn't do heavy or blocking operations (just delegates PersistentTasksService),
@@ -388,9 +190,14 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
@@ -402,7 +209,8 @@ protected ClusterBlockException checkBlock(OpenJobAction.Request request, Cluste
     }
 
     @Override
-    protected void masterOperation(OpenJobAction.Request request, ClusterState state, ActionListener<AcknowledgedResponse> listener) {
+    protected void masterOperation(Task task, OpenJobAction.Request request, ClusterState state,
+                                   ActionListener<AcknowledgedResponse> listener) {
         if (migrationEligibilityCheck.jobIsEligibleForMigration(request.getJobParams().getJobId(), state)) {
             listener.onFailure(ExceptionsHelper.configHasNotBeenMigrated("open job", request.getJobParams().getJobId()));
             return;
@@ -450,7 +258,8 @@ public void onFailure(Exception e) {
 
             // Tell the job tracker to refresh the memory requirement for this job and all other jobs that have persistent tasks
             ActionListener<Boolean> getJobHandler = ActionListener.wrap(
-                response -> memoryTracker.refreshJobMemoryAndAllOthers(jobParams.getJobId(), memoryRequirementRefreshListener),
+                response -> memoryTracker.refreshAnomalyDetectorJobMemoryAndAllOthers(jobParams.getJobId(),
+                    memoryRequirementRefreshListener),
                 listener::onFailure
             );
 
@@ -540,7 +349,6 @@ public static class OpenJobPersistentTasksExecutor extends PersistentTasksExecut
         private final AutodetectProcessManager autodetectProcessManager;
         private final MlMemoryTracker memoryTracker;
         private final Client client;
-        private final ClusterService clusterService;
 
         private volatile int maxConcurrentJobAllocations;
         private volatile int maxMachineMemoryPercent;
@@ -552,14 +360,13 @@ public OpenJobPersistentTasksExecutor(Settings settings, ClusterService clusterS
                                               AutodetectProcessManager autodetectProcessManager, MlMemoryTracker memoryTracker,
                                               Client client) {
             super(MlTasks.JOB_TASK_NAME, MachineLearning.UTILITY_THREAD_POOL_NAME);
-            this.autodetectProcessManager = autodetectProcessManager;
-            this.memoryTracker = memoryTracker;
-            this.client = client;
+            this.autodetectProcessManager = Objects.requireNonNull(autodetectProcessManager);
+            this.memoryTracker = Objects.requireNonNull(memoryTracker);
+            this.client = Objects.requireNonNull(client);
             this.maxConcurrentJobAllocations = MachineLearning.CONCURRENT_JOB_ALLOCATIONS.get(settings);
             this.maxMachineMemoryPercent = MachineLearning.MAX_MACHINE_MEMORY_PERCENT.get(settings);
             this.maxLazyMLNodes = MachineLearning.MAX_LAZY_ML_NODES.get(settings);
             this.maxOpenJobs = MAX_OPEN_JOBS_PER_NODE.get(settings);
-            this.clusterService = clusterService;
             clusterService.getClusterSettings()
                     .addSettingsUpdateConsumer(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, this::setMaxConcurrentJobAllocations);
             clusterService.getClusterSettings()
@@ -603,28 +410,11 @@ public PersistentTasksCustomMetaData.Assignment getAssignment(OpenJobAction.JobP
                 }
             }
 
-            PersistentTasksCustomMetaData.Assignment assignment = selectLeastLoadedMlNode(jobId,
-                params.getJob(),
-                clusterState,
-                maxOpenJobs,
-                maxConcurrentJobAllocations,
-                maxMachineMemoryPercent,
-                memoryTracker,
-                isMemoryTrackerRecentlyRefreshed,
-                logger);
-            if (assignment.getExecutorNode() == null) {
-                int numMlNodes = 0;
-                for (DiscoveryNode node : clusterState.getNodes()) {
-                    if (MachineLearning.isMlNode(node)) {
-                        numMlNodes++;
-                    }
-                }
-
-                if (numMlNodes < maxLazyMLNodes) { // Means we have lazy nodes left to allocate
-                    assignment = AWAITING_LAZY_ASSIGNMENT;
-                }
-            }
-            return assignment;
+            Job job = params.getJob();
+            JobNodeSelector jobNodeSelector = new JobNodeSelector(clusterState, jobId, MlTasks.JOB_TASK_NAME, memoryTracker,
+                maxLazyMLNodes, node -> nodeFilter(node, job));
+            return jobNodeSelector.selectNode(
+                maxOpenJobs, maxConcurrentJobAllocations, maxMachineMemoryPercent, isMemoryTrackerRecentlyRefreshed);
         }
 
         @Override
@@ -639,7 +429,7 @@ public void validate(OpenJobAction.JobParams params, ClusterState clusterState)
                 throw makeCurrentlyBeingUpgradedException(logger, params.getJobId(), assignment.getExplanation());
             }
 
-            if (assignment.getExecutorNode() == null && assignment.equals(AWAITING_LAZY_ASSIGNMENT) == false) {
+            if (assignment.getExecutorNode() == null && assignment.equals(JobNodeSelector.AWAITING_LAZY_ASSIGNMENT) == false) {
                 throw makeNoSuitableNodesException(logger, params.getJobId(), assignment.getExplanation());
             }
         }
@@ -755,7 +545,7 @@ public boolean test(PersistentTasksCustomMetaData.PersistentTask<?> persistentTa
                 PersistentTasksCustomMetaData.Assignment assignment = persistentTask.getAssignment();
 
                 // This means we are awaiting a new node to be spun up, ok to return back to the user to await node creation
-                if (assignment != null && assignment.equals(AWAITING_LAZY_ASSIGNMENT)) {
+                if (assignment != null && assignment.equals(JobNodeSelector.AWAITING_LAZY_ASSIGNMENT)) {
                     return true;
                 }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostDataAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostDataAction.java
index 2206f265a7cdc..28589683ffc79 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostDataAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostDataAction.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.TimeRange;
 
+import java.io.InputStream;
 import java.util.Optional;
 
 public class TransportPostDataAction extends TransportJobTaskAction<PostDataAction.Request, PostDataAction.Response> {
@@ -37,8 +38,8 @@ protected void taskOperation(PostDataAction.Request request, TransportOpenJobAct
                                  ActionListener<PostDataAction.Response> listener) {
         TimeRange timeRange = TimeRange.builder().startTime(request.getResetStart()).endTime(request.getResetEnd()).build();
         DataLoadParams params = new DataLoadParams(timeRange, Optional.ofNullable(request.getDataDescription()));
-        try {
-            processManager.processData(task, analysisRegistry, request.getContent().streamInput(), request.getXContentType(),
+        try (InputStream contentStream = request.getContent().streamInput()) {
+            processManager.processData(task, analysisRegistry, contentStream, request.getXContentType(),
                     params, (dataCounts, e) -> {
                 if (dataCounts != null) {
                     listener.onResponse(new PostDataAction.Response(dataCounts));
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java
new file mode 100644
index 0000000000000..d8f5dbb469f5f
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.action;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.license.LicenseUtils;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.ml.MachineLearningField;
+import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.job.messages.Messages;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.utils.MlStrings;
+import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
+import org.elasticsearch.xpack.core.security.support.Exceptions;
+import org.elasticsearch.xpack.ml.dataframe.SourceDestValidator;
+import org.elasticsearch.xpack.ml.dataframe.persistence.DataFrameAnalyticsConfigProvider;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+public class TransportPutDataFrameAnalyticsAction
+    extends HandledTransportAction<PutDataFrameAnalyticsAction.Request, PutDataFrameAnalyticsAction.Response> {
+
+    private final XPackLicenseState licenseState;
+    private final DataFrameAnalyticsConfigProvider configProvider;
+    private final ThreadPool threadPool;
+    private final SecurityContext securityContext;
+    private final Client client;
+    private final ClusterService clusterService;
+    private final IndexNameExpressionResolver indexNameExpressionResolver;
+
+    private volatile ByteSizeValue maxModelMemoryLimit;
+
+    @Inject
+    public TransportPutDataFrameAnalyticsAction(Settings settings, TransportService transportService, ActionFilters actionFilters,
+                                                XPackLicenseState licenseState, Client client, ThreadPool threadPool,
+                                                ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver,
+                                                DataFrameAnalyticsConfigProvider configProvider) {
+        super(PutDataFrameAnalyticsAction.NAME, transportService, actionFilters,
+            (Supplier<PutDataFrameAnalyticsAction.Request>) PutDataFrameAnalyticsAction.Request::new);
+        this.licenseState = licenseState;
+        this.configProvider = configProvider;
+        this.threadPool = threadPool;
+        this.securityContext = XPackSettings.SECURITY_ENABLED.get(settings) ?
+            new SecurityContext(settings, threadPool.getThreadContext()) : null;
+        this.client = client;
+        this.clusterService = clusterService;
+        this.indexNameExpressionResolver = Objects.requireNonNull(indexNameExpressionResolver);
+
+        maxModelMemoryLimit = MachineLearningField.MAX_MODEL_MEMORY_LIMIT.get(settings);
+        clusterService.getClusterSettings()
+            .addSettingsUpdateConsumer(MachineLearningField.MAX_MODEL_MEMORY_LIMIT, this::setMaxModelMemoryLimit);
+    }
+
+    private void setMaxModelMemoryLimit(ByteSizeValue maxModelMemoryLimit) {
+        this.maxModelMemoryLimit = maxModelMemoryLimit;
+    }
+
+    @Override
+    protected void doExecute(Task task, PutDataFrameAnalyticsAction.Request request,
+                             ActionListener<PutDataFrameAnalyticsAction.Response> listener) {
+        if (licenseState.isMachineLearningAllowed() == false) {
+            listener.onFailure(LicenseUtils.newComplianceException(XPackField.MACHINE_LEARNING));
+            return;
+        }
+        validateConfig(request.getConfig());
+        DataFrameAnalyticsConfig memoryCappedConfig =
+            new DataFrameAnalyticsConfig.Builder(request.getConfig(), maxModelMemoryLimit)
+                .setCreateTime(Instant.now())
+                .setVersion(Version.CURRENT)
+                .build();
+        if (licenseState.isAuthAllowed()) {
+            final String username = securityContext.getUser().principal();
+            RoleDescriptor.IndicesPrivileges sourceIndexPrivileges = RoleDescriptor.IndicesPrivileges.builder()
+                .indices(memoryCappedConfig.getSource().getIndex())
+                .privileges("read")
+                .build();
+            RoleDescriptor.IndicesPrivileges destIndexPrivileges = RoleDescriptor.IndicesPrivileges.builder()
+                .indices(memoryCappedConfig.getDest().getIndex())
+                .privileges("read", "index", "create_index")
+                .build();
+
+            HasPrivilegesRequest privRequest = new HasPrivilegesRequest();
+            privRequest.applicationPrivileges(new RoleDescriptor.ApplicationResourcePrivileges[0]);
+            privRequest.username(username);
+            privRequest.clusterPrivileges(Strings.EMPTY_ARRAY);
+            privRequest.indexPrivileges(sourceIndexPrivileges, destIndexPrivileges);
+
+            ActionListener<HasPrivilegesResponse> privResponseListener = ActionListener.wrap(
+                r -> handlePrivsResponse(username, memoryCappedConfig, r, listener),
+                listener::onFailure);
+
+            client.execute(HasPrivilegesAction.INSTANCE, privRequest, privResponseListener);
+        } else {
+            configProvider.put(memoryCappedConfig, threadPool.getThreadContext().getHeaders(), ActionListener.wrap(
+                indexResponse -> listener.onResponse(new PutDataFrameAnalyticsAction.Response(memoryCappedConfig)),
+                listener::onFailure
+            ));
+        }
+    }
+
+    private void handlePrivsResponse(String username, DataFrameAnalyticsConfig memoryCappedConfig,
+                                     HasPrivilegesResponse response,
+                                     ActionListener<PutDataFrameAnalyticsAction.Response> listener) throws IOException {
+        if (response.isCompleteMatch()) {
+            configProvider.put(memoryCappedConfig, threadPool.getThreadContext().getHeaders(), ActionListener.wrap(
+                indexResponse -> listener.onResponse(new PutDataFrameAnalyticsAction.Response(memoryCappedConfig)),
+                listener::onFailure
+            ));
+        } else {
+            XContentBuilder builder = JsonXContent.contentBuilder();
+            builder.startObject();
+            for (ResourcePrivileges index : response.getIndexPrivileges()) {
+                builder.field(index.getResource());
+                builder.map(index.getPrivileges());
+            }
+            builder.endObject();
+
+            listener.onFailure(Exceptions.authorizationError("Cannot create data frame analytics [{}]" +
+                    " because user {} lacks permissions on the indices: {}",
+                    memoryCappedConfig.getId(), username, Strings.toString(builder)));
+        }
+    }
+
+    private void validateConfig(DataFrameAnalyticsConfig config) {
+        if (MlStrings.isValidId(config.getId()) == false) {
+            throw ExceptionsHelper.badRequestException(Messages.getMessage(Messages.INVALID_ID, DataFrameAnalyticsConfig.ID,
+                config.getId()));
+        }
+        if (!MlStrings.hasValidLengthForId(config.getId())) {
+            throw ExceptionsHelper.badRequestException("id [{}] is too long; must not contain more than {} characters", config.getId(),
+                MlStrings.ID_LENGTH_LIMIT);
+        }
+        config.getDest().validate();
+        new SourceDestValidator(clusterService.state(), indexNameExpressionResolver).check(config);
+
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java
index 993fe548cc952..906ecda3ccaea 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java
@@ -92,7 +92,7 @@ protected PutDatafeedAction.Response newResponse() {
     }
 
     @Override
-    protected void masterOperation(PutDatafeedAction.Request request, ClusterState state,
+    protected void masterOperation(Task task, PutDatafeedAction.Request request, ClusterState state,
                                    ActionListener<PutDatafeedAction.Response> listener) {
         // If security is enabled only create the datafeed if the user requesting creation has
         // permission to read the indices the datafeed is going to read from
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutJobAction.java
index c1244649bffca..5e17a818b98f6 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutJobAction.java
@@ -53,7 +53,7 @@ protected PutJobAction.Response newResponse() {
     }
 
     @Override
-    protected void masterOperation(PutJobAction.Request request, ClusterState state,
+    protected void masterOperation(Task task, PutJobAction.Request request, ClusterState state,
                                    ActionListener<PutJobAction.Response> listener) throws Exception {
         jobManager.putJob(request, analysisRegistry, state, listener);
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java
index ab2fb1368345a..5a02cb165d9a0 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.MlTasks;
@@ -70,7 +71,7 @@ protected RevertModelSnapshotAction.Response newResponse() {
     }
 
     @Override
-    protected void masterOperation(RevertModelSnapshotAction.Request request, ClusterState state,
+    protected void masterOperation(Task task, RevertModelSnapshotAction.Request request, ClusterState state,
                                    ActionListener<RevertModelSnapshotAction.Response> listener) {
         if (migrationEligibilityCheck.jobIsEligibleForMigration(request.getJobId(), state)) {
             listener.onFailure(ExceptionsHelper.configHasNotBeenMigrated("revert model snapshot", request.getJobId()));
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java
index 44fc5f48a5942..5d5bf5efb8d02 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java
@@ -24,11 +24,13 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.persistent.PersistentTasksClusterService;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
@@ -37,6 +39,7 @@
 import org.elasticsearch.xpack.core.ml.action.SetUpgradeModeAction;
 import org.elasticsearch.xpack.ml.utils.TypedChainTaskExecutor;
 
+import java.io.IOException;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Set;
@@ -77,14 +80,19 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(SetUpgradeModeAction.Request request, ClusterState state, ActionListener<AcknowledgedResponse> listener)
-        throws Exception {
+    protected void masterOperation(Task task, SetUpgradeModeAction.Request request, ClusterState state,
+                                   ActionListener<AcknowledgedResponse> listener) throws Exception {
 
         // Don't want folks spamming this endpoint while it is in progress, only allow one request to be handled at a time
         if (isRunning.compareAndSet(false, true) == false) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
new file mode 100644
index 0000000000000..1d878adbc1d11
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java
@@ -0,0 +1,490 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
+import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.license.LicenseUtils;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.persistent.AllocatedPersistentTask;
+import org.elasticsearch.persistent.PersistentTaskParams;
+import org.elasticsearch.persistent.PersistentTaskState;
+import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.persistent.PersistentTasksExecutor;
+import org.elasticsearch.persistent.PersistentTasksService;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.ml.MlMetadata;
+import org.elasticsearch.xpack.core.ml.MlTasks;
+import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.ml.MachineLearning;
+import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsManager;
+import org.elasticsearch.xpack.ml.dataframe.MappingsMerger;
+import org.elasticsearch.xpack.ml.dataframe.SourceDestValidator;
+import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractorFactory;
+import org.elasticsearch.xpack.ml.dataframe.persistence.DataFrameAnalyticsConfigProvider;
+import org.elasticsearch.xpack.ml.job.JobNodeSelector;
+import org.elasticsearch.xpack.ml.process.MlMemoryTracker;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.xpack.core.ml.MlTasks.AWAITING_UPGRADE;
+import static org.elasticsearch.xpack.ml.MachineLearning.MAX_OPEN_JOBS_PER_NODE;
+
+/**
+ * Starts the persistent task for running data frame analytics.
+ *
+ * TODO Add to the upgrade mode action
+ */
+public class TransportStartDataFrameAnalyticsAction
+    extends TransportMasterNodeAction<StartDataFrameAnalyticsAction.Request, AcknowledgedResponse> {
+
+    private static final Logger LOGGER = LogManager.getLogger(TransportStartDataFrameAnalyticsAction.class);
+
+    private final XPackLicenseState licenseState;
+    private final Client client;
+    private final PersistentTasksService persistentTasksService;
+    private final DataFrameAnalyticsConfigProvider configProvider;
+    private final MlMemoryTracker memoryTracker;
+
+    @Inject
+    public TransportStartDataFrameAnalyticsAction(TransportService transportService, Client client, ClusterService clusterService,
+                                                  ThreadPool threadPool, ActionFilters actionFilters, XPackLicenseState licenseState,
+                                                  IndexNameExpressionResolver indexNameExpressionResolver,
+                                                  PersistentTasksService persistentTasksService,
+                                                  DataFrameAnalyticsConfigProvider configProvider, MlMemoryTracker memoryTracker) {
+        super(StartDataFrameAnalyticsAction.NAME, transportService, clusterService, threadPool, actionFilters,
+                StartDataFrameAnalyticsAction.Request::new, indexNameExpressionResolver);
+        this.licenseState = licenseState;
+        this.client = client;
+        this.persistentTasksService = persistentTasksService;
+        this.configProvider = configProvider;
+        this.memoryTracker = memoryTracker;
+    }
+
+    @Override
+    protected String executor() {
+        // This api doesn't do heavy or blocking operations (just delegates PersistentTasksService),
+        // so we can do this on the network thread
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected AcknowledgedResponse newResponse() {
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
+    }
+
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(StartDataFrameAnalyticsAction.Request request, ClusterState state) {
+        // We only delegate here to PersistentTasksService, but if there is a metadata writeblock,
+        // then delegating to PersistentTasksService doesn't make a whole lot of sense,
+        // because PersistentTasksService will then fail.
+        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+    }
+
+    @Override
+    protected void masterOperation(Task task, StartDataFrameAnalyticsAction.Request request, ClusterState state,
+                                   ActionListener<AcknowledgedResponse> listener) {
+        if (licenseState.isMachineLearningAllowed() == false) {
+            listener.onFailure(LicenseUtils.newComplianceException(XPackField.MACHINE_LEARNING));
+            return;
+        }
+
+        // Wait for analytics to be started
+        ActionListener<PersistentTasksCustomMetaData.PersistentTask<StartDataFrameAnalyticsAction.TaskParams>> waitForAnalyticsToStart =
+            new ActionListener<PersistentTasksCustomMetaData.PersistentTask<StartDataFrameAnalyticsAction.TaskParams>>() {
+                @Override
+                public void onResponse(PersistentTasksCustomMetaData.PersistentTask<StartDataFrameAnalyticsAction.TaskParams> task) {
+                    waitForAnalyticsStarted(task, request.getTimeout(), listener);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    if (e instanceof ResourceAlreadyExistsException) {
+                        e = new ElasticsearchStatusException("Cannot open data frame analytics [" + request.getId() +
+                            "] because it has already been opened", RestStatus.CONFLICT, e);
+                    }
+                    listener.onFailure(e);
+                }
+            };
+
+        AtomicReference<DataFrameAnalyticsConfig> configHolder = new AtomicReference<>();
+
+        // Start persistent task
+        ActionListener<Void> memoryRequirementRefreshListener = ActionListener.wrap(
+            aVoid -> {
+                StartDataFrameAnalyticsAction.TaskParams taskParams = new StartDataFrameAnalyticsAction.TaskParams(
+                    request.getId(), configHolder.get().getVersion());
+                persistentTasksService.sendStartRequest(MlTasks.dataFrameAnalyticsTaskId(request.getId()),
+                    MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, taskParams, waitForAnalyticsToStart);
+            },
+            listener::onFailure
+        );
+
+        // Tell the job tracker to refresh the memory requirement for this job and all other jobs that have persistent tasks
+        ActionListener<DataFrameAnalyticsConfig> configListener = ActionListener.wrap(
+            config -> {
+                configHolder.set(config);
+                memoryTracker.addDataFrameAnalyticsJobMemoryAndRefreshAllOthers(
+                    request.getId(), config.getModelMemoryLimit().getBytes(), memoryRequirementRefreshListener);
+            },
+            listener::onFailure
+        );
+
+        // Get config
+        getConfigAndValidate(request.getId(), configListener);
+    }
+
+    private void getConfigAndValidate(String id, ActionListener<DataFrameAnalyticsConfig> finalListener) {
+        // Validate mappings can be merged
+        ActionListener<DataFrameAnalyticsConfig> firstValidationListener = ActionListener.wrap(
+            config -> MappingsMerger.mergeMappings(client, config.getHeaders(), config.getSource().getIndex(), ActionListener.wrap(
+                    mappings -> finalListener.onResponse(config), finalListener::onFailure)),
+            finalListener::onFailure
+        );
+
+        // Validate source and dest; check data extraction is possible
+        ActionListener<DataFrameAnalyticsConfig> getConfigListener = ActionListener.wrap(
+            config -> {
+                new SourceDestValidator(clusterService.state(), indexNameExpressionResolver).check(config);
+                DataFrameDataExtractorFactory.validateConfigAndSourceIndex(client, config, firstValidationListener);
+            },
+            finalListener::onFailure
+        );
+
+        // First, get the config
+        configProvider.get(id, getConfigListener);
+    }
+
+    private void waitForAnalyticsStarted(PersistentTasksCustomMetaData.PersistentTask<StartDataFrameAnalyticsAction.TaskParams> task,
+                                         TimeValue timeout, ActionListener<AcknowledgedResponse> listener) {
+        AnalyticsPredicate predicate = new AnalyticsPredicate();
+        persistentTasksService.waitForPersistentTaskCondition(task.getId(), predicate, timeout,
+
+            new PersistentTasksService.WaitForPersistentTaskListener<PersistentTaskParams>() {
+
+                @Override
+                public void onResponse(PersistentTasksCustomMetaData.PersistentTask<PersistentTaskParams> persistentTask) {
+                    if (predicate.exception != null) {
+                        // We want to return to the caller without leaving an unassigned persistent task, to match
+                        // what would have happened if the error had been detected in the "fast fail" validation
+                        cancelAnalyticsStart(task, predicate.exception, listener);
+                    } else {
+                        listener.onResponse(new AcknowledgedResponse(true));
+                    }
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    listener.onFailure(e);
+                }
+
+                @Override
+                public void onTimeout(TimeValue timeout) {
+                    listener.onFailure(new ElasticsearchException("Starting data frame analytics [" + task.getParams().getId()
+                        + "] timed out after [" + timeout + "]"));
+                }
+        });
+    }
+
+    /**
+     * Important: the methods of this class must NOT throw exceptions.  If they did then the callers
+     * of endpoints waiting for a condition tested by this predicate would never get a response.
+     */
+    private class AnalyticsPredicate implements Predicate<PersistentTasksCustomMetaData.PersistentTask<?>> {
+
+        private volatile Exception exception;
+
+        @Override
+        public boolean test(PersistentTasksCustomMetaData.PersistentTask<?> persistentTask) {
+            if (persistentTask == null) {
+                return false;
+            }
+
+            PersistentTasksCustomMetaData.Assignment assignment = persistentTask.getAssignment();
+
+            // This means we are awaiting a new node to be spun up, ok to return back to the user to await node creation
+            if (assignment != null && assignment.equals(JobNodeSelector.AWAITING_LAZY_ASSIGNMENT)) {
+                return true;
+            }
+
+            if (assignment != null && assignment.equals(PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT) == false &&
+                assignment.isAssigned() == false) {
+                // Assignment has failed despite passing our "fast fail" validation
+                exception = new ElasticsearchStatusException("Could not start data frame analytics task, allocation explanation [" +
+                    assignment.getExplanation() + "]", RestStatus.TOO_MANY_REQUESTS);
+                return true;
+            }
+            DataFrameAnalyticsTaskState taskState = (DataFrameAnalyticsTaskState) persistentTask.getState();
+            DataFrameAnalyticsState analyticsState = taskState == null ? DataFrameAnalyticsState.STOPPED : taskState.getState();
+            switch (analyticsState) {
+                case STARTED:
+                case REINDEXING:
+                case ANALYZING:
+                    return true;
+                case STOPPING:
+                    exception = ExceptionsHelper.conflictStatusException("the task has been stopped while waiting to be started");
+                    return true;
+                case STOPPED:
+                    return false;
+                case FAILED:
+                default:
+                    exception = ExceptionsHelper.serverError("Unexpected task state [" + analyticsState + "] while waiting to be started");
+                    return true;
+            }
+        }
+    }
+
+    private void cancelAnalyticsStart(
+        PersistentTasksCustomMetaData.PersistentTask<StartDataFrameAnalyticsAction.TaskParams> persistentTask, Exception exception,
+        ActionListener<AcknowledgedResponse> listener) {
+        persistentTasksService.sendRemoveRequest(persistentTask.getId(),
+            new ActionListener<PersistentTasksCustomMetaData.PersistentTask<?>>() {
+                @Override
+                public void onResponse(PersistentTasksCustomMetaData.PersistentTask<?> task) {
+                    // We succeeded in cancelling the persistent task, but the
+                    // problem that caused us to cancel it is the overall result
+                    listener.onFailure(exception);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    LOGGER.error("[" + persistentTask.getParams().getId() + "] Failed to cancel persistent task that could " +
+                        "not be assigned due to [" + exception.getMessage() + "]", e);
+                    listener.onFailure(exception);
+                }
+            }
+        );
+    }
+
+    public static class DataFrameAnalyticsTask extends AllocatedPersistentTask implements StartDataFrameAnalyticsAction.TaskMatcher {
+
+        private final Client client;
+        private final ClusterService clusterService;
+        private final DataFrameAnalyticsManager analyticsManager;
+        private final StartDataFrameAnalyticsAction.TaskParams taskParams;
+        @Nullable
+        private volatile Long reindexingTaskId;
+        private volatile boolean isStopping;
+
+        public DataFrameAnalyticsTask(long id, String type, String action, TaskId parentTask, Map<String, String> headers,
+                                      Client client, ClusterService clusterService, DataFrameAnalyticsManager analyticsManager,
+                                      StartDataFrameAnalyticsAction.TaskParams taskParams) {
+            super(id, type, action, MlTasks.DATA_FRAME_ANALYTICS_TASK_ID_PREFIX + taskParams.getId(), parentTask, headers);
+            this.client = Objects.requireNonNull(client);
+            this.clusterService = Objects.requireNonNull(clusterService);
+            this.analyticsManager = Objects.requireNonNull(analyticsManager);
+            this.taskParams = Objects.requireNonNull(taskParams);
+        }
+
+        public StartDataFrameAnalyticsAction.TaskParams getParams() {
+            return taskParams;
+        }
+
+        public void setReindexingTaskId(Long reindexingTaskId) {
+            this.reindexingTaskId = reindexingTaskId;
+        }
+
+        @Nullable
+        public Long getReindexingTaskId() {
+            return reindexingTaskId;
+        }
+
+        public boolean isStopping() {
+            return isStopping;
+        }
+
+        @Override
+        protected void onCancelled() {
+            stop(getReasonCancelled(), TimeValue.ZERO);
+        }
+
+        public void stop(String reason, TimeValue timeout) {
+            isStopping = true;
+            if (reindexingTaskId != null) {
+                cancelReindexingTask(reason, timeout);
+            }
+            analyticsManager.stop(this);
+        }
+
+        private void cancelReindexingTask(String reason, TimeValue timeout) {
+            TaskId reindexTaskId = new TaskId(clusterService.localNode().getId(), reindexingTaskId);
+            LOGGER.debug("[{}] Cancelling reindex task [{}]", taskParams.getId(), reindexTaskId);
+
+            CancelTasksRequest cancelReindex = new CancelTasksRequest();
+            cancelReindex.setTaskId(reindexTaskId);
+            cancelReindex.setReason(reason);
+            cancelReindex.setTimeout(timeout);
+            CancelTasksResponse cancelReindexResponse = client.admin().cluster().cancelTasks(cancelReindex).actionGet();
+            Throwable firstError = null;
+            if (cancelReindexResponse.getNodeFailures().isEmpty() == false) {
+                firstError = cancelReindexResponse.getNodeFailures().get(0).getRootCause();
+            }
+            if (cancelReindexResponse.getTaskFailures().isEmpty() == false) {
+                firstError = cancelReindexResponse.getTaskFailures().get(0).getCause();
+            }
+            // There is a chance that the task is finished by the time we cancel it in which case we'll get
+            // a ResourceNotFoundException which we can ignore.
+            if (firstError != null && firstError instanceof ResourceNotFoundException == false) {
+                throw ExceptionsHelper.serverError("[" + taskParams.getId() + "] Error cancelling reindex task", firstError);
+            } else {
+                LOGGER.debug("[{}] Reindex task was successfully cancelled", taskParams.getId());
+            }
+        }
+    }
+
+    public static class TaskExecutor extends PersistentTasksExecutor<StartDataFrameAnalyticsAction.TaskParams> {
+
+        private final Client client;
+        private final ClusterService clusterService;
+        private final DataFrameAnalyticsManager manager;
+        private final MlMemoryTracker memoryTracker;
+
+        private volatile int maxMachineMemoryPercent;
+        private volatile int maxLazyMLNodes;
+        private volatile int maxOpenJobs;
+
+        public TaskExecutor(Settings settings, Client client, ClusterService clusterService, DataFrameAnalyticsManager manager,
+                            MlMemoryTracker memoryTracker) {
+            super(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, MachineLearning.UTILITY_THREAD_POOL_NAME);
+            this.client = Objects.requireNonNull(client);
+            this.clusterService = Objects.requireNonNull(clusterService);
+            this.manager = Objects.requireNonNull(manager);
+            this.memoryTracker = Objects.requireNonNull(memoryTracker);
+            this.maxMachineMemoryPercent = MachineLearning.MAX_MACHINE_MEMORY_PERCENT.get(settings);
+            this.maxLazyMLNodes = MachineLearning.MAX_LAZY_ML_NODES.get(settings);
+            this.maxOpenJobs = MAX_OPEN_JOBS_PER_NODE.get(settings);
+            clusterService.getClusterSettings()
+                .addSettingsUpdateConsumer(MachineLearning.MAX_MACHINE_MEMORY_PERCENT, this::setMaxMachineMemoryPercent);
+            clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.MAX_LAZY_ML_NODES, this::setMaxLazyMLNodes);
+            clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_OPEN_JOBS_PER_NODE, this::setMaxOpenJobs);
+        }
+
+        @Override
+        protected AllocatedPersistentTask createTask(
+            long id, String type, String action, TaskId parentTaskId,
+            PersistentTasksCustomMetaData.PersistentTask<StartDataFrameAnalyticsAction.TaskParams> persistentTask,
+            Map<String, String> headers) {
+            return new DataFrameAnalyticsTask(id, type, action, parentTaskId, headers, client, clusterService, manager,
+                persistentTask.getParams());
+        }
+
+        @Override
+        public PersistentTasksCustomMetaData.Assignment getAssignment(StartDataFrameAnalyticsAction.TaskParams params,
+                                                                      ClusterState clusterState) {
+
+            // If we are waiting for an upgrade to complete, we should not assign to a node
+            if (MlMetadata.getMlMetadata(clusterState).isUpgradeMode()) {
+                return AWAITING_UPGRADE;
+            }
+
+            String id = params.getId();
+
+            boolean isMemoryTrackerRecentlyRefreshed = memoryTracker.isRecentlyRefreshed();
+            if (isMemoryTrackerRecentlyRefreshed == false) {
+                boolean scheduledRefresh = memoryTracker.asyncRefresh();
+                if (scheduledRefresh) {
+                    String reason = "Not opening job [" + id + "] because job memory requirements are stale - refresh requested";
+                    LOGGER.debug(reason);
+                    return new PersistentTasksCustomMetaData.Assignment(null, reason);
+                }
+            }
+
+            JobNodeSelector jobNodeSelector = new JobNodeSelector(clusterState, id, MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, memoryTracker,
+                maxLazyMLNodes, node -> nodeFilter(node, id));
+            // Pass an effectively infinite value for max concurrent opening jobs, because data frame analytics jobs do
+            // not have an "opening" state so would never be rejected for causing too many jobs in the "opening" state
+            return jobNodeSelector.selectNode(
+                maxOpenJobs, Integer.MAX_VALUE, maxMachineMemoryPercent, isMemoryTrackerRecentlyRefreshed);
+        }
+
+        @Override
+        protected void nodeOperation(AllocatedPersistentTask task, StartDataFrameAnalyticsAction.TaskParams params,
+                                     PersistentTaskState state) {
+            LOGGER.info("[{}] Starting data frame analytics", params.getId());
+            DataFrameAnalyticsTaskState analyticsTaskState = (DataFrameAnalyticsTaskState) state;
+
+            // If we are "stopping" there is nothing to do
+            // If we are "failed" then we should leave the task as is; for recovery it must be force stopped.
+            if (analyticsTaskState != null && analyticsTaskState.getState().isAnyOf(
+                    DataFrameAnalyticsState.STOPPING, DataFrameAnalyticsState.FAILED)) {
+                return;
+            }
+
+            if (analyticsTaskState == null) {
+                DataFrameAnalyticsTaskState startedState = new DataFrameAnalyticsTaskState(DataFrameAnalyticsState.STARTED,
+                    task.getAllocationId(), null);
+                task.updatePersistentTaskState(startedState, ActionListener.wrap(
+                    response -> manager.execute((DataFrameAnalyticsTask) task, DataFrameAnalyticsState.STARTED),
+                    task::markAsFailed));
+            } else {
+                manager.execute((DataFrameAnalyticsTask)task, analyticsTaskState.getState());
+            }
+        }
+
+        public static String nodeFilter(DiscoveryNode node, String id) {
+
+            if (node.getVersion().before(StartDataFrameAnalyticsAction.TaskParams.VERSION_INTRODUCED)) {
+                return "Not opening job [" + id + "] on node [" + JobNodeSelector.nodeNameAndVersion(node)
+                    + "], because the data frame analytics requires a node of version ["
+                    + StartDataFrameAnalyticsAction.TaskParams.VERSION_INTRODUCED + "] or higher";
+            }
+
+            return null;
+        }
+
+        void setMaxMachineMemoryPercent(int maxMachineMemoryPercent) {
+            this.maxMachineMemoryPercent = maxMachineMemoryPercent;
+        }
+
+        void setMaxLazyMLNodes(int maxLazyMLNodes) {
+            this.maxLazyMLNodes = maxLazyMLNodes;
+        }
+
+        void setMaxOpenJobs(int maxOpenJobs) {
+            this.maxOpenJobs = maxOpenJobs;
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
index 1a0d1b84c87d5..bbb7eb3c9a969 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java
@@ -21,6 +21,7 @@
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -33,6 +34,7 @@
 import org.elasticsearch.persistent.PersistentTasksExecutor;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -54,6 +56,7 @@
 import org.elasticsearch.xpack.ml.job.persistence.JobConfigProvider;
 import org.elasticsearch.xpack.ml.notifications.Auditor;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
@@ -133,13 +136,18 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(StartDatafeedAction.Request request, ClusterState state,
+    protected void masterOperation(Task task, StartDatafeedAction.Request request, ClusterState state,
                                    ActionListener<AcknowledgedResponse> listener) {
         StartDatafeedAction.DatafeedParams params = request.getParams();
         if (licenseState.isMachineLearningAllowed() == false) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java
new file mode 100644
index 0000000000000..b02d89c17766c
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsAction.java
@@ -0,0 +1,270 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.action;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionListenerResponseHandler;
+import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.action.TaskOperationFailure;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.tasks.TransportTasksAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.discovery.MasterNotDiscoveredException;
+import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.persistent.PersistentTasksService;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.ml.MlTasks;
+import org.elasticsearch.xpack.core.ml.action.StopDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.ml.MachineLearning;
+import org.elasticsearch.xpack.ml.dataframe.persistence.DataFrameAnalyticsConfigProvider;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Stops the persistent task for running data frame analytics.
+ *
+ * TODO Add to the upgrade mode action
+ */
+public class TransportStopDataFrameAnalyticsAction
+    extends TransportTasksAction<TransportStartDataFrameAnalyticsAction.DataFrameAnalyticsTask, StopDataFrameAnalyticsAction.Request,
+        StopDataFrameAnalyticsAction.Response, StopDataFrameAnalyticsAction.Response> {
+
+    private static final Logger logger = LogManager.getLogger(TransportStopDataFrameAnalyticsAction.class);
+
+    private final ThreadPool threadPool;
+    private final PersistentTasksService persistentTasksService;
+    private final DataFrameAnalyticsConfigProvider configProvider;
+
+    @Inject
+    public TransportStopDataFrameAnalyticsAction(TransportService transportService, ActionFilters actionFilters,
+                                                 ClusterService clusterService, ThreadPool threadPool,
+                                                 PersistentTasksService persistentTasksService,
+                                                 DataFrameAnalyticsConfigProvider configProvider) {
+        super(StopDataFrameAnalyticsAction.NAME, clusterService, transportService, actionFilters, StopDataFrameAnalyticsAction.Request::new,
+            StopDataFrameAnalyticsAction.Response::new, StopDataFrameAnalyticsAction.Response::new, ThreadPool.Names.SAME);
+        this.threadPool = threadPool;
+        this.persistentTasksService = persistentTasksService;
+        this.configProvider = configProvider;
+    }
+
+    @Override
+    protected void doExecute(Task task, StopDataFrameAnalyticsAction.Request request,
+                             ActionListener<StopDataFrameAnalyticsAction.Response> listener) {
+        ClusterState state = clusterService.state();
+        DiscoveryNodes nodes = state.nodes();
+        if (nodes.isLocalNodeElectedMaster() == false) {
+            redirectToMasterNode(nodes.getMasterNode(), request, listener);
+            return;
+        }
+
+        logger.debug("Received request to stop data frame analytics [{}]", request.getId());
+
+        ActionListener<Set<String>> expandedIdsListener = ActionListener.wrap(
+            expandedIds -> {
+                logger.debug("Resolved data frame analytics to stop: {}", expandedIds);
+                if (expandedIds.isEmpty()) {
+                    listener.onResponse(new StopDataFrameAnalyticsAction.Response(true));
+                    return;
+                }
+
+                PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
+                Set<String> analyticsToStop = findAnalyticsToStop(tasks, expandedIds, request.isForce());
+                request.setExpandedIds(analyticsToStop);
+                request.setNodes(findAllocatedNodesAndRemoveUnassignedTasks(analyticsToStop, tasks));
+
+                ActionListener<StopDataFrameAnalyticsAction.Response> finalListener = ActionListener.wrap(
+                    r -> waitForTaskRemoved(expandedIds, request, r, listener),
+                    listener::onFailure
+                );
+
+                super.doExecute(task, request, finalListener);
+            },
+            listener::onFailure
+        );
+
+        expandIds(state, request, expandedIdsListener);
+    }
+
+    /** Visible for testing */
+    static Set<String> findAnalyticsToStop(PersistentTasksCustomMetaData tasks, Set<String> ids, boolean force) {
+        Set<String> startedAnalytics = new HashSet<>();
+        Set<String> stoppingAnalytics = new HashSet<>();
+        Set<String> failedAnalytics = new HashSet<>();
+        sortAnalyticsByTaskState(ids, tasks, startedAnalytics, stoppingAnalytics, failedAnalytics);
+
+        if (force == false && failedAnalytics.isEmpty() == false) {
+            ElasticsearchStatusException e = failedAnalytics.size() == 1 ? ExceptionsHelper.conflictStatusException(
+                "cannot close data frame analytics [{}] because it failed, use force stop instead", failedAnalytics.iterator().next()) :
+                ExceptionsHelper.conflictStatusException("one or more data frame analytics are in failed state, " +
+                    "use force stop instead");
+            throw e;
+        }
+
+        startedAnalytics.addAll(failedAnalytics);
+        return startedAnalytics;
+    }
+
+    private static void sortAnalyticsByTaskState(Set<String> analyticsIds, PersistentTasksCustomMetaData tasks,
+                                                 Set<String> startedAnalytics, Set<String> stoppingAnalytics,
+                                                 Set<String> failedAnalytics) {
+        for (String analyticsId : analyticsIds) {
+            switch (MlTasks.getDataFrameAnalyticsState(analyticsId, tasks)) {
+                case STARTED:
+                case REINDEXING:
+                case ANALYZING:
+                    startedAnalytics.add(analyticsId);
+                    break;
+                case STOPPING:
+                    stoppingAnalytics.add(analyticsId);
+                    break;
+                case STOPPED:
+                    break;
+                case FAILED:
+                    failedAnalytics.add(analyticsId);
+                    break;
+                default:
+                    break;
+            }
+        }
+    }
+
+    private void expandIds(ClusterState clusterState, StopDataFrameAnalyticsAction.Request request,
+                           ActionListener<Set<String>> expandedIdsListener) {
+        ActionListener<List<DataFrameAnalyticsConfig>> configsListener = ActionListener.wrap(
+            configs -> {
+                Set<String> matchingIds = configs.stream().map(DataFrameAnalyticsConfig::getId).collect(Collectors.toSet());
+                PersistentTasksCustomMetaData tasksMetaData = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
+                Set<String> startedIds = tasksMetaData == null ? Collections.emptySet() : tasksMetaData.tasks().stream()
+                    .filter(t -> t.getId().startsWith(MlTasks.DATA_FRAME_ANALYTICS_TASK_ID_PREFIX))
+                    .map(t -> t.getId().replaceFirst(MlTasks.DATA_FRAME_ANALYTICS_TASK_ID_PREFIX, ""))
+                    .collect(Collectors.toSet());
+                startedIds.retainAll(matchingIds);
+                expandedIdsListener.onResponse(startedIds);
+            },
+            expandedIdsListener::onFailure
+        );
+
+        configProvider.getMultiple(request.getId(), request.allowNoMatch(), configsListener);
+    }
+
+    private String[] findAllocatedNodesAndRemoveUnassignedTasks(Set<String> analyticsIds, PersistentTasksCustomMetaData tasks) {
+        List<String> nodes = new ArrayList<>();
+        for (String analyticsId : analyticsIds) {
+            PersistentTasksCustomMetaData.PersistentTask<?> task = MlTasks.getDataFrameAnalyticsTask(analyticsId, tasks);
+            if (task == null) {
+                // This should not be possible; we filtered started analytics thus the task should exist
+                String msg = "Requested data frame analytics [" + analyticsId + "] be stopped but the task could not be found";
+                assert task != null : msg;
+            } else if (task.isAssigned()) {
+                nodes.add(task.getExecutorNode());
+            } else {
+                // This means the task has not been assigned to a node yet so
+                // we can stop it by removing its persistent task.
+                // The listener is a no-op as we're already going to wait for the task to be removed.
+                persistentTasksService.sendRemoveRequest(task.getId(), ActionListener.wrap(r -> {}, e -> {}));
+            }
+        }
+        return nodes.toArray(new String[0]);
+    }
+
+    private void redirectToMasterNode(DiscoveryNode masterNode, StopDataFrameAnalyticsAction.Request request,
+                                      ActionListener<StopDataFrameAnalyticsAction.Response> listener) {
+        if (masterNode == null) {
+            listener.onFailure(new MasterNotDiscoveredException("no known master node"));
+        } else {
+            transportService.sendRequest(masterNode, actionName, request,
+                new ActionListenerResponseHandler<>(listener, StopDataFrameAnalyticsAction.Response::new));
+        }
+    }
+
+    @Override
+    protected StopDataFrameAnalyticsAction.Response newResponse(StopDataFrameAnalyticsAction.Request request,
+                                                                List<StopDataFrameAnalyticsAction.Response> tasks,
+                                                                List<TaskOperationFailure> taskOperationFailures,
+                                                                List<FailedNodeException> failedNodeExceptions) {
+        if (request.getExpandedIds().size() != tasks.size()) {
+            if (taskOperationFailures.isEmpty() == false) {
+                throw org.elasticsearch.ExceptionsHelper.convertToElastic(taskOperationFailures.get(0).getCause());
+            } else if (failedNodeExceptions.isEmpty() == false) {
+                throw org.elasticsearch.ExceptionsHelper.convertToElastic(failedNodeExceptions.get(0));
+            } else {
+                // This can happen when the actual task in the node no longer exists,
+                // which means the data frame analytic(s) have already been closed.
+                return new StopDataFrameAnalyticsAction.Response(true);
+            }
+        }
+        return new StopDataFrameAnalyticsAction.Response(tasks.stream().allMatch(StopDataFrameAnalyticsAction.Response::isStopped));
+    }
+
+    @Override
+    protected void taskOperation(StopDataFrameAnalyticsAction.Request request,
+                                 TransportStartDataFrameAnalyticsAction.DataFrameAnalyticsTask task,
+                                 ActionListener<StopDataFrameAnalyticsAction.Response> listener) {
+        DataFrameAnalyticsTaskState stoppingState =
+            new DataFrameAnalyticsTaskState(DataFrameAnalyticsState.STOPPING, task.getAllocationId(), null);
+        task.updatePersistentTaskState(stoppingState, ActionListener.wrap(pTask -> {
+                threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(new AbstractRunnable() {
+                    @Override
+                    public void onFailure(Exception e) {
+                        listener.onFailure(e);
+                    }
+
+                    @Override
+                    protected void doRun() {
+                        logger.info("[{}] Stopping task with force [{}]", task.getParams().getId(), request.isForce());
+                        task.stop("stop_data_frame_analytics (api)", request.getTimeout());
+                        listener.onResponse(new StopDataFrameAnalyticsAction.Response(true));
+                    }
+                });
+            },
+            e -> {
+                if (e instanceof ResourceNotFoundException) {
+                    // the task has disappeared so must have stopped
+                    listener.onResponse(new StopDataFrameAnalyticsAction.Response(true));
+                } else {
+                    listener.onFailure(e);
+                }
+            }));
+    }
+
+    void waitForTaskRemoved(Set<String> analyticsIds, StopDataFrameAnalyticsAction.Request request,
+                                StopDataFrameAnalyticsAction.Response response,
+                                ActionListener<StopDataFrameAnalyticsAction.Response> listener) {
+        persistentTasksService.waitForPersistentTasksCondition(persistentTasks ->
+                filterPersistentTasks(persistentTasks, analyticsIds).isEmpty(),
+            request.getTimeout(), ActionListener.wrap(
+                booleanResponse -> listener.onResponse(response),
+                listener::onFailure
+            ));
+    }
+
+    private static Collection<PersistentTasksCustomMetaData.PersistentTask<?>> filterPersistentTasks(
+            PersistentTasksCustomMetaData persistentTasks, Set<String> analyticsIds) {
+        return persistentTasks.findTasks(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME,
+            t -> analyticsIds.contains(MlTasks.dataFrameAnalyticsIdFromTaskId(t.getId())));
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java
index b7029eff79cd8..164c297251bbc 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDatafeedAction.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.MlTasks;
@@ -64,7 +65,7 @@ protected PutDatafeedAction.Response newResponse() {
     }
 
     @Override
-    protected void masterOperation(UpdateDatafeedAction.Request request, ClusterState state,
+    protected void masterOperation(Task task, UpdateDatafeedAction.Request request, ClusterState state,
                                    ActionListener<PutDatafeedAction.Response> listener) throws Exception {
 
         if (migrationEligibilityCheck.datafeedIsEligibleForMigration(request.getUpdate().getId(), state)) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateJobAction.java
index 3e9abf3ae4179..5505669d9ea4e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateJobAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateJobAction.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.PutJobAction;
@@ -44,7 +45,8 @@ protected PutJobAction.Response newResponse() {
     }
 
     @Override
-    protected void masterOperation(UpdateJobAction.Request request, ClusterState state, ActionListener<PutJobAction.Response> listener) {
+    protected void masterOperation(Task task, UpdateJobAction.Request request, ClusterState state,
+                                   ActionListener<PutJobAction.Response> listener) {
         jobManager.updateJob(request, listener);
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java
index db6e094714701..4a9e4fd41d9c2 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java
@@ -11,7 +11,7 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -19,7 +19,6 @@
 import org.elasticsearch.common.xcontent.XContentElasticsearchExtension;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.core.internal.io.Streams;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.core.ml.action.FlushJobAction;
@@ -38,7 +37,6 @@
 import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory;
 import org.elasticsearch.xpack.ml.notifications.Auditor;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Date;
@@ -407,9 +405,7 @@ private DataCounts postData(InputStream inputStream, XContentType xContentType)
             throws IOException {
         PostDataAction.Request request = new PostDataAction.Request(jobId);
         request.setDataDescription(dataDescription);
-        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-        Streams.copy(inputStream, outputStream);
-        request.setContent(new BytesArray(outputStream.toByteArray()), xContentType);
+        request.setContent(Streams.readFully(inputStream), xContentType);
         try (ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashWithOrigin(ML_ORIGIN)) {
             PostDataAction.Response response = client.execute(PostDataAction.INSTANCE, request).actionGet();
             return response.getDataCounts();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessor.java
index 8cf3ed39651d6..365cfc6c54a20 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessor.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
 import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregation;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
+import org.elasticsearch.search.aggregations.metrics.GeoCentroid;
 import org.elasticsearch.search.aggregations.metrics.Max;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.search.aggregations.metrics.Percentile;
@@ -275,7 +276,7 @@ private void processBucket(MultiBucketsAggregation bucketAgg, boolean addField)
     }
 
     /**
-     * Adds a leaf key-value. It returns the name of the key added or {@code null} when nothing was added.
+     * Adds a leaf key-value. It returns {@code true} if the key added or {@code false} when nothing was added.
      * Non-finite metric values are not added.
      */
     private boolean processLeaf(Aggregation agg) throws IOException {
@@ -283,6 +284,8 @@ private boolean processLeaf(Aggregation agg) throws IOException {
             return processSingleValue((NumericMetricsAggregation.SingleValue) agg);
         } else if (agg instanceof Percentiles) {
             return processPercentiles((Percentiles) agg);
+        } else if (agg instanceof GeoCentroid){
+            return processGeoCentroid((GeoCentroid) agg);
         } else {
             throw new IllegalArgumentException("Unsupported aggregation type [" + agg.getName() + "]");
         }
@@ -300,6 +303,14 @@ private boolean addMetricIfFinite(String key, double value) {
         return false;
     }
 
+    private boolean processGeoCentroid(GeoCentroid agg) {
+        if (agg.count() > 0) {
+            keyValuePairs.put(agg.getName(), agg.centroid().getLat() + "," + agg.centroid().getLon());
+            return true;
+        }
+        return false;
+    }
+
     private boolean processPercentiles(Percentiles percentiles) throws IOException {
         Iterator<Percentile> percentileIterator = percentiles.iterator();
         boolean aggregationAdded = addMetricIfFinite(percentiles.getName(), percentileIterator.next().getValue());
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java
index 451f480b27830..e297695152435 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java
@@ -6,9 +6,18 @@
 package org.elasticsearch.xpack.ml.datafeed.extractor.fields;
 
 import org.elasticsearch.common.document.DocumentField;
+import org.elasticsearch.geo.geometry.Geometry;
+import org.elasticsearch.geo.geometry.Point;
+import org.elasticsearch.geo.geometry.ShapeType;
+import org.elasticsearch.geo.utils.GeographyValidator;
+import org.elasticsearch.geo.utils.WellKnownText;
 import org.elasticsearch.search.SearchHit;
 
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.Arrays;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 
@@ -50,6 +59,8 @@ public ExtractionMethod getExtractionMethod() {
 
     public abstract Object[] value(SearchHit hit);
 
+    public abstract boolean supportsFromSource();
+
     public String getDocValueFormat() {
         return null;
     }
@@ -61,6 +72,14 @@ public static ExtractedField newTimeField(String name, ExtractionMethod extracti
         return new TimeField(name, extractionMethod);
     }
 
+    public static ExtractedField newGeoShapeField(String alias, String name) {
+        return new GeoShapeField(alias, name);
+    }
+
+    public static ExtractedField newGeoPointField(String alias, String name) {
+        return new GeoPointField(alias, name);
+    }
+
     public static ExtractedField newField(String name, ExtractionMethod extractionMethod) {
         return newField(name, name, extractionMethod);
     }
@@ -71,12 +90,20 @@ public static ExtractedField newField(String alias, String name, ExtractionMetho
             case SCRIPT_FIELD:
                 return new FromFields(alias, name, extractionMethod);
             case SOURCE:
-                return new FromSource(alias, name, extractionMethod);
+                return new FromSource(alias, name);
             default:
                 throw new IllegalArgumentException("Invalid extraction method [" + extractionMethod + "]");
         }
     }
 
+    public ExtractedField newFromSource() {
+        if (supportsFromSource()) {
+            return new FromSource(alias, name);
+        }
+        throw new IllegalStateException("Field (alias [" + alias + "], name [" + name + "]) should be extracted via ["
+            + extractionMethod + "] and cannot be extracted from source");
+    }
+
     private static class FromFields extends ExtractedField {
 
         FromFields(String alias, String name, ExtractionMethod extractionMethod) {
@@ -88,10 +115,107 @@ public Object[] value(SearchHit hit) {
             DocumentField keyValue = hit.field(name);
             if (keyValue != null) {
                 List<Object> values = keyValue.getValues();
-                return values.toArray(new Object[values.size()]);
+                return values.toArray(new Object[0]);
             }
             return new Object[0];
         }
+
+        @Override
+        public boolean supportsFromSource() {
+            return getExtractionMethod() == ExtractionMethod.DOC_VALUE;
+        }
+    }
+
+    private static class GeoShapeField extends FromSource {
+        private static final WellKnownText wkt = new WellKnownText(true, new GeographyValidator(true));
+
+        GeoShapeField(String alias, String name) {
+            super(alias, name);
+        }
+
+        @Override
+        public Object[] value(SearchHit hit) {
+            Object[] value = super.value(hit);
+            if (value.length != 1) {
+                throw new IllegalStateException("Unexpected values for a geo_shape field: " + Arrays.toString(value));
+            }
+            if (value[0] instanceof String) {
+                value[0] = handleString((String) value[0]);
+            } else if (value[0] instanceof Map<?, ?>) {
+                @SuppressWarnings("unchecked")
+                Map<String, Object> geoObject = (Map<String, Object>) value[0];
+                value[0] = handleObject(geoObject);
+            } else {
+                throw new IllegalStateException("Unexpected value type for a geo_shape field: " + value[0].getClass());
+            }
+            return value;
+        }
+
+        private String handleString(String geoString) {
+            try {
+                if (geoString.startsWith("POINT")) { // Entry is of the form "POINT (-77.03653 38.897676)"
+                    Geometry geometry = wkt.fromWKT(geoString);
+                    if (geometry.type() != ShapeType.POINT) {
+                        throw new IllegalArgumentException("Unexpected non-point geo_shape type: " + geometry.type().name());
+                    }
+                    Point pt = ((Point)geometry);
+                    return pt.getLat() + "," + pt.getLon();
+                } else {
+                    throw new IllegalArgumentException("Unexpected value for a geo_shape field: " + geoString);
+                }
+            } catch (IOException | ParseException ex) {
+                throw new IllegalArgumentException("Unexpected value for a geo_shape field: " + geoString);
+            }
+        }
+
+        private String handleObject(Map<String, Object> geoObject) {
+            String geoType = (String) geoObject.get("type");
+            if (geoType != null && "point".equals(geoType.toLowerCase(Locale.ROOT))) {
+                @SuppressWarnings("unchecked")
+                List<Double> coordinates = (List<Double>) geoObject.get("coordinates");
+                if (coordinates == null || coordinates.size() != 2) {
+                    throw new IllegalArgumentException("Invalid coordinates for geo_shape point: " + geoObject);
+                }
+                return coordinates.get(1) + "," + coordinates.get(0);
+            } else {
+                throw new IllegalArgumentException("Unexpected value for a geo_shape field: " + geoObject);
+            }
+        }
+
+    }
+
+    private static class GeoPointField extends FromFields {
+
+        GeoPointField(String alias, String name) {
+            super(alias, name, ExtractionMethod.DOC_VALUE);
+        }
+
+        @Override
+        public Object[] value(SearchHit hit) {
+            Object[] value = super.value(hit);
+            if (value.length != 1) {
+                throw new IllegalStateException("Unexpected values for a geo_point field: " + Arrays.toString(value));
+            }
+            if (value[0] instanceof String) {
+                value[0] = handleString((String) value[0]);
+            } else {
+                throw new IllegalStateException("Unexpected value type for a geo_point field: " + value[0].getClass());
+            }
+            return value;
+        }
+
+        private String handleString(String geoString) {
+            if (geoString.contains(",")) { // Entry is of the form "38.897676, -77.03653"
+                return geoString.replace(" ", "");
+            } else {
+                throw new IllegalArgumentException("Unexpected value for a geo_point field: " + geoString);
+            }
+        }
+
+        @Override
+        public boolean supportsFromSource() {
+            return false;
+        }
     }
 
     private static class TimeField extends FromFields {
@@ -120,14 +244,19 @@ public Object[] value(SearchHit hit) {
         public String getDocValueFormat() {
             return EPOCH_MILLIS_FORMAT;
         }
+
+        @Override
+        public boolean supportsFromSource() {
+            return false;
+        }
     }
 
     private static class FromSource extends ExtractedField {
 
         private String[] namePath;
 
-        FromSource(String alias, String name, ExtractionMethod extractionMethod) {
-            super(alias, name, extractionMethod);
+        FromSource(String alias, String name) {
+            super(alias, name, ExtractionMethod.SOURCE);
             namePath = name.split("\\.");
         }
 
@@ -145,7 +274,7 @@ public Object[] value(SearchHit hit) {
                     if (values instanceof List<?>) {
                         @SuppressWarnings("unchecked")
                         List<Object> asList = (List<Object>) values;
-                        return asList.toArray(new Object[asList.size()]);
+                        return asList.toArray(new Object[0]);
                     } else {
                         return new Object[]{values};
                     }
@@ -154,6 +283,11 @@ public Object[] value(SearchHit hit) {
             return new Object[0];
         }
 
+        @Override
+        public boolean supportsFromSource() {
+            return true;
+        }
+
         @SuppressWarnings("unchecked")
         private static Map<String, Object> getNextLevel(Map<String, Object> source, String key) {
             Object nextLevel = source.get(key);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFields.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFields.java
index f9b2467fbcfd3..9495c5a2b405a 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFields.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFields.java
@@ -47,6 +47,15 @@ public List<ExtractedField> getDocValueFields() {
         return docValueFields;
     }
 
+    /**
+     * Returns a new instance which only contains fields matching the given extraction method
+     * @param method the extraction method to filter fields on
+     * @return a new instance which only contains fields matching the given extraction method
+     */
+    public ExtractedFields filterFields(ExtractedField.ExtractionMethod method) {
+        return new ExtractedFields(filterFields(method, allFields));
+    }
+
     private static List<ExtractedField> filterFields(ExtractedField.ExtractionMethod method, List<ExtractedField> fields) {
         return fields.stream().filter(field -> field.getExtractionMethod() == method).collect(Collectors.toList());
     }
@@ -87,6 +96,17 @@ protected ExtractedField detect(String field) {
                             : ExtractedField.ExtractionMethod.SOURCE;
                 }
             }
+
+            if (isFieldOfType(field, "geo_point")) {
+                if (method != ExtractedField.ExtractionMethod.DOC_VALUE) {
+                    throw new IllegalArgumentException("cannot use [geo_point] field with disabled doc values");
+                }
+                return ExtractedField.newGeoPointField(field, internalField);
+            }
+            if (isFieldOfType(field, "geo_shape")) {
+                return ExtractedField.newGeoShapeField(field, internalField);
+            }
+
             return ExtractedField.newField(field, internalField, method);
         }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndex.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndex.java
new file mode 100644
index 0000000000000..661525623575d
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndex.java
@@ -0,0 +1,197 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSortConfig;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+
+import java.time.Clock;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
+
+/**
+ * {@link DataFrameAnalyticsIndex} class encapsulates logic for creating destination index based on source index metadata.
+ */
+public final class DataFrameAnalyticsIndex {
+
+    public static final String ID_COPY = "ml__id_copy";
+
+    // Metadata fields
+    static final String CREATION_DATE_MILLIS = "creation_date_in_millis";
+    static final String VERSION = "version";
+    static final String CREATED = "created";
+    static final String CREATED_BY = "created_by";
+    static final String ANALYTICS = "analytics";
+
+    private static final String PROPERTIES = "properties";
+    private static final String META = "_meta";
+
+    /**
+     * We only preserve the most important settings.
+     * If the user needs other settings on the destination index they
+     * should create the destination index before starting the analytics.
+     */
+    private static final String[] PRESERVED_SETTINGS = new String[] {"index.number_of_shards", "index.number_of_replicas"};
+
+    private DataFrameAnalyticsIndex() {}
+
+    /**
+     * Creates destination index based on source index metadata.
+     */
+    public static void createDestinationIndex(Client client,
+                                              Clock clock,
+                                              DataFrameAnalyticsConfig analyticsConfig,
+                                              ActionListener<CreateIndexResponse> listener) {
+        ActionListener<CreateIndexRequest> createIndexRequestListener = ActionListener.wrap(
+            createIndexRequest -> ClientHelper.executeWithHeadersAsync(analyticsConfig.getHeaders(), ClientHelper.ML_ORIGIN, client,
+                    CreateIndexAction.INSTANCE, createIndexRequest, listener),
+            listener::onFailure
+        );
+
+        prepareCreateIndexRequest(client, clock, analyticsConfig, createIndexRequestListener);
+    }
+
+    private static void prepareCreateIndexRequest(Client client, Clock clock, DataFrameAnalyticsConfig config,
+                                                         ActionListener<CreateIndexRequest> listener) {
+        AtomicReference<Settings> settingsHolder = new AtomicReference<>();
+
+        String[] sourceIndex = config.getSource().getIndex();
+
+        ActionListener<ImmutableOpenMap<String, MappingMetaData>> mappingsListener = ActionListener.wrap(
+            mappings -> listener.onResponse(createIndexRequest(clock, config, settingsHolder.get(), mappings)),
+            listener::onFailure
+        );
+
+        ActionListener<Settings> settingsListener = ActionListener.wrap(
+            settings -> {
+                settingsHolder.set(settings);
+                MappingsMerger.mergeMappings(client, config.getHeaders(), sourceIndex, mappingsListener);
+            },
+            listener::onFailure
+        );
+
+        ActionListener<GetSettingsResponse> getSettingsResponseListener = ActionListener.wrap(
+            settingsResponse -> settingsListener.onResponse(settings(settingsResponse)),
+            listener::onFailure
+        );
+
+        GetSettingsRequest getSettingsRequest = new GetSettingsRequest();
+        getSettingsRequest.indices(sourceIndex);
+        getSettingsRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
+        getSettingsRequest.names(PRESERVED_SETTINGS);
+        ClientHelper.executeWithHeadersAsync(config.getHeaders(), ML_ORIGIN, client, GetSettingsAction.INSTANCE,
+            getSettingsRequest, getSettingsResponseListener);
+    }
+
+    private static CreateIndexRequest createIndexRequest(Clock clock, DataFrameAnalyticsConfig config, Settings settings,
+                                                         ImmutableOpenMap<String, MappingMetaData> mappings) {
+        // There should only be 1 type
+        assert mappings.size() == 1;
+
+        String destinationIndex = config.getDest().getIndex();
+        String type = mappings.keysIt().next();
+        Map<String, Object> mappingsAsMap = mappings.valuesIt().next().sourceAsMap();
+        addProperties(mappingsAsMap);
+        addMetaData(mappingsAsMap, config.getId(), clock);
+        return new CreateIndexRequest(destinationIndex, settings).mapping(type, mappingsAsMap);
+    }
+
+    private static Settings settings(GetSettingsResponse settingsResponse) {
+        Integer maxNumberOfShards = findMaxSettingValue(settingsResponse, IndexMetaData.SETTING_NUMBER_OF_SHARDS);
+        Integer maxNumberOfReplicas = findMaxSettingValue(settingsResponse, IndexMetaData.SETTING_NUMBER_OF_REPLICAS);
+
+        Settings.Builder settingsBuilder = Settings.builder();
+        settingsBuilder.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), ID_COPY);
+        settingsBuilder.put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), SortOrder.ASC);
+        if (maxNumberOfShards != null) {
+            settingsBuilder.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, maxNumberOfShards);
+        }
+        if (maxNumberOfReplicas != null) {
+            settingsBuilder.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, maxNumberOfReplicas);
+        }
+        return settingsBuilder.build();
+    }
+
+    @Nullable
+    private static Integer findMaxSettingValue(GetSettingsResponse settingsResponse, String settingKey) {
+        Integer maxValue = null;
+        Iterator<Settings> settingsIterator = settingsResponse.getIndexToSettings().valuesIt();
+        while (settingsIterator.hasNext()) {
+            Settings settings = settingsIterator.next();
+            Integer indexValue = settings.getAsInt(settingKey, null);
+            if (indexValue != null) {
+                maxValue = maxValue == null ? indexValue : Math.max(indexValue, maxValue);
+            }
+        }
+        return maxValue;
+    }
+
+    private static void addProperties(Map<String, Object> mappingsAsMap) {
+        Map<String, Object> properties = getOrPutDefault(mappingsAsMap, PROPERTIES, HashMap::new);
+        properties.put(ID_COPY, Map.of("type", "keyword"));
+    }
+
+    private static void addMetaData(Map<String, Object> mappingsAsMap, String analyticsId, Clock clock) {
+        Map<String, Object> metadata = getOrPutDefault(mappingsAsMap, META, HashMap::new);
+        metadata.put(CREATION_DATE_MILLIS, clock.millis());
+        metadata.put(CREATED_BY, "data-frame-analytics");
+        metadata.put(VERSION, Map.of(CREATED, Version.CURRENT));
+        metadata.put(ANALYTICS, analyticsId);
+    }
+
+    private static <K, V> V getOrPutDefault(Map<K, Object> map, K key, Supplier<V> valueSupplier) {
+        V value = (V) map.get(key);
+        if (value == null) {
+            value = valueSupplier.get();
+            map.put(key, value);
+        }
+        return value;
+    }
+
+    public static void updateMappingsToDestIndex(Client client, DataFrameAnalyticsConfig analyticsConfig, GetIndexResponse getIndexResponse,
+                                                 ActionListener<AcknowledgedResponse> listener) {
+        // We have validated the destination index should match a single index
+        assert getIndexResponse.indices().length == 1;
+
+        ImmutableOpenMap<String, MappingMetaData> mappings = getIndexResponse.getMappings().get(getIndexResponse.indices()[0]);
+        String type = mappings.keysIt().next();
+
+        Map<String, Object> addedMappings = Map.of(PROPERTIES, Map.of(ID_COPY, Map.of("type", "keyword")));
+
+        PutMappingRequest putMappingRequest = new PutMappingRequest(getIndexResponse.indices());
+        putMappingRequest.type(type);
+        putMappingRequest.source(addedMappings);
+        ClientHelper.executeWithHeadersAsync(analyticsConfig.getHeaders(), ML_ORIGIN, client, PutMappingAction.INSTANCE,
+            putMappingRequest, listener);
+    }
+}
+
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java
new file mode 100644
index 0000000000000..28f277dc84bcf
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java
@@ -0,0 +1,219 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexAction;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
+import org.elasticsearch.action.support.ContextPreservingActionListener;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.elasticsearch.index.reindex.ReindexAction;
+import org.elasticsearch.index.reindex.ReindexRequest;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.ml.action.TransportStartDataFrameAnalyticsAction.DataFrameAnalyticsTask;
+import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractorFactory;
+import org.elasticsearch.xpack.ml.dataframe.persistence.DataFrameAnalyticsConfigProvider;
+import org.elasticsearch.xpack.ml.dataframe.process.AnalyticsProcessManager;
+
+import java.time.Clock;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
+
+public class DataFrameAnalyticsManager {
+
+    private static final Logger LOGGER = LogManager.getLogger(DataFrameAnalyticsManager.class);
+
+    /**
+     * We need a {@link NodeClient} to get the reindexing task and be able to report progress
+     */
+    private final NodeClient client;
+    private final DataFrameAnalyticsConfigProvider configProvider;
+    private final AnalyticsProcessManager processManager;
+
+    public DataFrameAnalyticsManager(NodeClient client, DataFrameAnalyticsConfigProvider configProvider,
+                                     AnalyticsProcessManager processManager) {
+        this.client = Objects.requireNonNull(client);
+        this.configProvider = Objects.requireNonNull(configProvider);
+        this.processManager = Objects.requireNonNull(processManager);
+    }
+
+    public void execute(DataFrameAnalyticsTask task, DataFrameAnalyticsState currentState) {
+        ActionListener<DataFrameAnalyticsConfig> reindexingStateListener = ActionListener.wrap(
+            config -> reindexDataframeAndStartAnalysis(task, config),
+            task::markAsFailed
+        );
+
+        // With config in hand, determine action to take
+        ActionListener<DataFrameAnalyticsConfig> configListener = ActionListener.wrap(
+            config -> {
+                DataFrameAnalyticsTaskState reindexingState = new DataFrameAnalyticsTaskState(DataFrameAnalyticsState.REINDEXING,
+                    task.getAllocationId(), null);
+                switch(currentState) {
+                    // If we are STARTED, we are right at the beginning of our task, we should indicate that we are entering the
+                    // REINDEX state and start reindexing.
+                    case STARTED:
+                        task.updatePersistentTaskState(reindexingState, ActionListener.wrap(
+                            updatedTask -> reindexingStateListener.onResponse(config),
+                            reindexingStateListener::onFailure));
+                        break;
+                    // The task has fully reindexed the documents and we should continue on with our analyses
+                    case ANALYZING:
+                        startAnalytics(task, config, true);
+                        break;
+                    // If we are already at REINDEXING, we are not 100% sure if we reindexed ALL the docs.
+                    // We will delete the destination index, recreate, reindex
+                    case REINDEXING:
+                        ClientHelper.executeAsyncWithOrigin(client,
+                            ML_ORIGIN,
+                            DeleteIndexAction.INSTANCE,
+                            new DeleteIndexRequest(config.getDest().getIndex()),
+                            ActionListener.wrap(
+                                r-> reindexingStateListener.onResponse(config),
+                                e -> {
+                                    if (e instanceof IndexNotFoundException) {
+                                        reindexingStateListener.onResponse(config);
+                                    } else {
+                                        reindexingStateListener.onFailure(e);
+                                    }
+                                }
+                            ));
+                        break;
+                    default:
+                        reindexingStateListener.onFailure(
+                            ExceptionsHelper.conflictStatusException(
+                                "Cannot execute analytics task [{}] as it is currently in state [{}]. " +
+                                "Must be one of [STARTED, REINDEXING, ANALYZING]", config.getId(), currentState));
+                }
+
+            },
+            reindexingStateListener::onFailure
+        );
+
+        // Retrieve configuration
+        configProvider.get(task.getParams().getId(), configListener);
+    }
+
+    private void reindexDataframeAndStartAnalysis(DataFrameAnalyticsTask task, DataFrameAnalyticsConfig config) {
+        if (task.isStopping()) {
+            // The task was requested to stop before we started reindexing
+            task.markAsCompleted();
+            return;
+        }
+
+        // Reindexing is complete; start analytics
+        ActionListener<RefreshResponse> refreshListener = ActionListener.wrap(
+            refreshResponse -> {
+                task.setReindexingTaskId(null);
+                startAnalytics(task, config, false);
+            },
+            task::markAsFailed
+        );
+
+        // Refresh to ensure copied index is fully searchable
+        ActionListener<BulkByScrollResponse> reindexCompletedListener = ActionListener.wrap(
+            bulkResponse ->
+                ClientHelper.executeAsyncWithOrigin(client,
+                    ClientHelper.ML_ORIGIN,
+                    RefreshAction.INSTANCE,
+                    new RefreshRequest(config.getDest().getIndex()),
+                    refreshListener),
+            task::markAsFailed
+        );
+
+        // Reindex
+        ActionListener<CreateIndexResponse> copyIndexCreatedListener = ActionListener.wrap(
+            createIndexResponse -> {
+                ReindexRequest reindexRequest = new ReindexRequest();
+                reindexRequest.setSourceIndices(config.getSource().getIndex());
+                reindexRequest.setSourceQuery(config.getSource().getParsedQuery());
+                reindexRequest.setDestIndex(config.getDest().getIndex());
+                reindexRequest.setScript(new Script("ctx._source." + DataFrameAnalyticsIndex.ID_COPY + " = ctx._id"));
+
+                final ThreadContext threadContext = client.threadPool().getThreadContext();
+                final Supplier<ThreadContext.StoredContext> supplier = threadContext.newRestorableContext(false);
+                try (ThreadContext.StoredContext ignore = threadContext.stashWithOrigin(ML_ORIGIN)) {
+                    Task reindexTask = client.executeLocally(ReindexAction.INSTANCE, reindexRequest,
+                        new ContextPreservingActionListener<>(supplier, reindexCompletedListener));
+                    task.setReindexingTaskId(reindexTask.getId());
+                }
+            },
+            reindexCompletedListener::onFailure
+        );
+
+        // Create destination index if it does not exist
+        ActionListener<GetIndexResponse> destIndexListener = ActionListener.wrap(
+            indexResponse -> {
+                LOGGER.info("[{}] Using existing destination index [{}]", config.getId(), indexResponse.indices()[0]);
+                DataFrameAnalyticsIndex.updateMappingsToDestIndex(client, config, indexResponse, ActionListener.wrap(
+                    acknowledgedResponse -> copyIndexCreatedListener.onResponse(null),
+                    copyIndexCreatedListener::onFailure
+                ));
+            },
+            e -> {
+                if (org.elasticsearch.ExceptionsHelper.unwrapCause(e) instanceof IndexNotFoundException) {
+                    LOGGER.info("[{}] Creating destination index [{}]", config.getId(), config.getDest().getIndex());
+                    DataFrameAnalyticsIndex.createDestinationIndex(client, Clock.systemUTC(), config, copyIndexCreatedListener);
+                } else {
+                    copyIndexCreatedListener.onFailure(e);
+                }
+            }
+        );
+
+        ClientHelper.executeWithHeadersAsync(config.getHeaders(), ML_ORIGIN, client, GetIndexAction.INSTANCE,
+                new GetIndexRequest().indices(config.getDest().getIndex()), destIndexListener);
+    }
+
+    private void startAnalytics(DataFrameAnalyticsTask task, DataFrameAnalyticsConfig config, boolean isTaskRestarting) {
+        // Update state to ANALYZING and start process
+        ActionListener<DataFrameDataExtractorFactory> dataExtractorFactoryListener = ActionListener.wrap(
+            dataExtractorFactory -> {
+                DataFrameAnalyticsTaskState analyzingState = new DataFrameAnalyticsTaskState(DataFrameAnalyticsState.ANALYZING,
+                    task.getAllocationId(), null);
+                task.updatePersistentTaskState(analyzingState, ActionListener.wrap(
+                    updatedTask -> processManager.runJob(task, config, dataExtractorFactory,
+                        error -> {
+                            if (error != null) {
+                                task.markAsFailed(error);
+                            } else {
+                                task.markAsCompleted();
+                            }
+                        }),
+                    task::markAsFailed
+                ));
+            },
+            task::markAsFailed
+        );
+
+        // TODO This could fail with errors. In that case we get stuck with the copied index.
+        // We could delete the index in case of failure or we could try building the factory before reindexing
+        // to catch the error early on.
+        DataFrameDataExtractorFactory.create(client, config, isTaskRestarting, dataExtractorFactoryListener);
+    }
+
+    public void stop(DataFrameAnalyticsTask task) {
+        processManager.stop(task);
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java
new file mode 100644
index 0000000000000..f007831f7cfa4
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe;
+
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
+
+/**
+ * Merges mappings in a best effort and naive manner.
+ * The merge will fail if there is any conflict, i.e. the mappings of a field are not exactly the same.
+ */
+public final class MappingsMerger {
+
+    private MappingsMerger() {}
+
+    public static void mergeMappings(Client client, Map<String, String> headers, String[] index,
+                                     ActionListener<ImmutableOpenMap<String, MappingMetaData>> listener) {
+        ActionListener<GetMappingsResponse> mappingsListener = ActionListener.wrap(
+            getMappingsResponse -> listener.onResponse(MappingsMerger.mergeMappings(getMappingsResponse)),
+            listener::onFailure
+        );
+
+        GetMappingsRequest getMappingsRequest = new GetMappingsRequest();
+        getMappingsRequest.indices(index);
+        ClientHelper.executeWithHeadersAsync(headers, ML_ORIGIN, client, GetMappingsAction.INSTANCE, getMappingsRequest, mappingsListener);
+    }
+
+    static ImmutableOpenMap<String, MappingMetaData> mergeMappings(GetMappingsResponse getMappingsResponse) {
+        ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> indexToMappings = getMappingsResponse.getMappings();
+
+        String type = null;
+        Map<String, Object> mergedMappings = new HashMap<>();
+
+        Iterator<ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>>> iterator = indexToMappings.iterator();
+        while (iterator.hasNext()) {
+            ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> indexMappings = iterator.next();
+            Iterator<ObjectObjectCursor<String, MappingMetaData>> typeIterator = indexMappings.value.iterator();
+            while (typeIterator.hasNext()) {
+                ObjectObjectCursor<String, MappingMetaData> typeMapping = typeIterator.next();
+                if (type == null) {
+                    type = typeMapping.key;
+                } else {
+                    if (type.equals(typeMapping.key) == false) {
+                        throw ExceptionsHelper.badRequestException("source indices contain mappings for different types: [{}, {}]",
+                            type, typeMapping.key);
+                    }
+                }
+                Map<String, Object> currentMappings = typeMapping.value.getSourceAsMap();
+                if (currentMappings.containsKey("properties")) {
+
+                    @SuppressWarnings("unchecked")
+                    Map<String, Object> fieldMappings = (Map<String, Object>) currentMappings.get("properties");
+
+                    for (Map.Entry<String, Object> fieldMapping : fieldMappings.entrySet()) {
+                        if (mergedMappings.containsKey(fieldMapping.getKey())) {
+                            if (mergedMappings.get(fieldMapping.getKey()).equals(fieldMapping.getValue()) == false) {
+                                throw ExceptionsHelper.badRequestException("cannot merge mappings because of differences for field [{}]",
+                                    fieldMapping.getKey());
+                            }
+                        } else {
+                            mergedMappings.put(fieldMapping.getKey(), fieldMapping.getValue());
+                        }
+                    }
+                }
+            }
+        }
+
+        MappingMetaData mappingMetaData = createMappingMetaData(type, mergedMappings);
+        ImmutableOpenMap.Builder<String, MappingMetaData> result = ImmutableOpenMap.builder();
+        result.put(type, mappingMetaData);
+        return result.build();
+    }
+
+    private static MappingMetaData createMappingMetaData(String type, Map<String, Object> mappings) {
+        try {
+            return new MappingMetaData(type, Collections.singletonMap("properties", mappings));
+        } catch (IOException e) {
+            throw ExceptionsHelper.serverError("Failed to parse mappings: " + mappings);
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/SourceDestValidator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/SourceDestValidator.java
new file mode 100644
index 0000000000000..01803dc4359d5
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/SourceDestValidator.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe;
+
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
+public class SourceDestValidator {
+
+    private final ClusterState clusterState;
+    private final IndexNameExpressionResolver indexNameExpressionResolver;
+
+    public SourceDestValidator(ClusterState clusterState, IndexNameExpressionResolver indexNameExpressionResolver) {
+        this.clusterState = Objects.requireNonNull(clusterState);
+        this.indexNameExpressionResolver = Objects.requireNonNull(indexNameExpressionResolver);
+    }
+
+    public void check(DataFrameAnalyticsConfig config) {
+        String[] sourceIndex = config.getSource().getIndex();
+        String destIndex = config.getDest().getIndex();
+
+        String[] sourceExpressions = Arrays.stream(sourceIndex)
+            .map(index -> Strings.tokenizeToStringArray(index, ","))
+            .flatMap(Arrays::stream)
+            .toArray(String[]::new);
+
+        for (String sourceExpression : sourceExpressions) {
+            if (Regex.simpleMatch(sourceExpression, destIndex)) {
+                throw ExceptionsHelper.badRequestException("Destination index [{}] must not be included in source index [{}]",
+                    destIndex, sourceExpression);
+            }
+        }
+
+        Set<String> concreteSourceIndexNames = new HashSet<>(Arrays.asList(indexNameExpressionResolver.concreteIndexNames(clusterState,
+            IndicesOptions.lenientExpandOpen(), sourceExpressions)));
+
+        if (concreteSourceIndexNames.isEmpty()) {
+            throw ExceptionsHelper.badRequestException("No index matches source index {}", Arrays.toString(sourceIndex));
+        }
+
+        final String[] concreteDestIndexNames = indexNameExpressionResolver.concreteIndexNames(clusterState,
+            IndicesOptions.lenientExpandOpen(), destIndex);
+
+        if (concreteDestIndexNames.length > 1) {
+            // In case it is an alias, it may match multiple indices
+            throw ExceptionsHelper.badRequestException("Destination index [{}] should match a single index; matches {}", destIndex,
+                Arrays.toString(concreteDestIndexNames));
+        }
+        if (concreteDestIndexNames.length == 1 && concreteSourceIndexNames.contains(concreteDestIndexNames[0])) {
+            // In case the dest index is an alias, we need to check the concrete index is not matched by source
+            throw ExceptionsHelper.badRequestException("Destination index [{}], which is an alias for [{}], " +
+                    "must not be included in source index {}", destIndex, concreteDestIndexNames[0], Arrays.toString(sourceIndex));
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
new file mode 100644
index 0000000000000..fa18f3bb25b3c
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
@@ -0,0 +1,276 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.extractor;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.search.ClearScrollAction;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchScrollAction;
+import org.elasticsearch.action.search.SearchScrollRequestBuilder;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.fetch.StoredFieldsContext;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedField;
+import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsIndex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+/**
+ * An implementation that extracts data from elasticsearch using search and scroll on a client.
+ * It supports safe and responsive cancellation by continuing the scroll until a new timestamp
+ * is seen.
+ * Note that this class is NOT thread-safe.
+ */
+public class DataFrameDataExtractor {
+
+    private static final Logger LOGGER = LogManager.getLogger(DataFrameDataExtractor.class);
+    private static final TimeValue SCROLL_TIMEOUT = new TimeValue(30, TimeUnit.MINUTES);
+
+    private final Client client;
+    private final DataFrameDataExtractorContext context;
+    private String scrollId;
+    private boolean isCancelled;
+    private boolean hasNext;
+    private boolean searchHasShardFailure;
+
+    DataFrameDataExtractor(Client client, DataFrameDataExtractorContext context) {
+        this.client = Objects.requireNonNull(client);
+        this.context = Objects.requireNonNull(context);
+        hasNext = true;
+        searchHasShardFailure = false;
+    }
+
+    public Map<String, String> getHeaders() {
+        return Collections.unmodifiableMap(context.headers);
+    }
+
+    public boolean hasNext() {
+        return hasNext;
+    }
+
+    public boolean isCancelled() {
+        return isCancelled;
+    }
+
+    public void cancel() {
+        LOGGER.debug("[{}] Data extractor was cancelled", context.jobId);
+        isCancelled = true;
+    }
+
+    public Optional<List<Row>> next() throws IOException {
+        if (!hasNext()) {
+            throw new NoSuchElementException();
+        }
+        Optional<List<Row>> hits = scrollId == null ? Optional.ofNullable(initScroll()) : Optional.ofNullable(continueScroll());
+        if (!hits.isPresent()) {
+            hasNext = false;
+        }
+        return hits;
+    }
+
+    protected List<Row> initScroll() throws IOException {
+        LOGGER.debug("[{}] Initializing scroll", context.jobId);
+        return tryRequestWithSearchResponse(() -> executeSearchRequest(buildSearchRequest()));
+    }
+
+    private List<Row> tryRequestWithSearchResponse(Supplier<SearchResponse> request) throws IOException {
+        try {
+            // We've set allow_partial_search_results to false which means if something
+            // goes wrong the request will throw.
+            SearchResponse searchResponse = request.get();
+            LOGGER.debug("[{}] Search response was obtained", context.jobId);
+
+            // Request was successful so we can restore the flag to retry if a future failure occurs
+            searchHasShardFailure = false;
+
+            return processSearchResponse(searchResponse);
+        } catch (Exception e) {
+            if (searchHasShardFailure) {
+                throw e;
+            }
+            LOGGER.warn(new ParameterizedMessage("[{}] Search resulted to failure; retrying once", context.jobId), e);
+            markScrollAsErrored();
+            return initScroll();
+        }
+    }
+
+    protected SearchResponse executeSearchRequest(SearchRequestBuilder searchRequestBuilder) {
+        return ClientHelper.executeWithHeaders(context.headers, ClientHelper.ML_ORIGIN, client, searchRequestBuilder::get);
+    }
+
+    private SearchRequestBuilder buildSearchRequest() {
+        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE)
+                .setScroll(SCROLL_TIMEOUT)
+                // This ensures the search throws if there are failures and the scroll context gets cleared automatically
+                .setAllowPartialSearchResults(false)
+                .addSort(DataFrameAnalyticsIndex.ID_COPY, SortOrder.ASC)
+                .setIndices(context.indices)
+                .setSize(context.scrollSize)
+                .setQuery(context.query);
+        setFetchSource(searchRequestBuilder);
+
+        for (ExtractedField docValueField : context.extractedFields.getDocValueFields()) {
+            searchRequestBuilder.addDocValueField(docValueField.getName(), docValueField.getDocValueFormat());
+        }
+
+        return searchRequestBuilder;
+    }
+
+    private void setFetchSource(SearchRequestBuilder searchRequestBuilder) {
+        if (context.includeSource) {
+            searchRequestBuilder.setFetchSource(true);
+        } else {
+            String[] sourceFields = context.extractedFields.getSourceFields();
+            if (sourceFields.length == 0) {
+                searchRequestBuilder.setFetchSource(false);
+                searchRequestBuilder.storedFields(StoredFieldsContext._NONE_);
+            } else {
+                searchRequestBuilder.setFetchSource(sourceFields, null);
+            }
+        }
+    }
+
+    private List<Row> processSearchResponse(SearchResponse searchResponse) throws IOException {
+        scrollId = searchResponse.getScrollId();
+        if (searchResponse.getHits().getHits().length == 0) {
+            hasNext = false;
+            clearScroll(scrollId);
+            return null;
+        }
+
+        SearchHit[] hits = searchResponse.getHits().getHits();
+        List<Row> rows = new ArrayList<>(hits.length);
+        for (SearchHit hit : hits) {
+            if (isCancelled) {
+                hasNext = false;
+                clearScroll(scrollId);
+                break;
+            }
+            rows.add(createRow(hit));
+        }
+        return rows;
+    }
+
+    private Row createRow(SearchHit hit) {
+        String[] extractedValues = new String[context.extractedFields.getAllFields().size()];
+        for (int i = 0; i < extractedValues.length; ++i) {
+            ExtractedField field = context.extractedFields.getAllFields().get(i);
+            Object[] values = field.value(hit);
+            if (values.length == 1 && values[0] instanceof Number) {
+                extractedValues[i] = Objects.toString(values[0]);
+            } else {
+                extractedValues = null;
+                break;
+            }
+        }
+        return new Row(extractedValues, hit);
+    }
+
+    private List<Row> continueScroll() throws IOException {
+        LOGGER.debug("[{}] Continuing scroll with id [{}]", context.jobId, scrollId);
+        return tryRequestWithSearchResponse(() -> executeSearchScrollRequest(scrollId));
+    }
+
+    private void markScrollAsErrored() {
+        // This could be a transient error with the scroll Id.
+        // Reinitialise the scroll and try again but only once.
+        scrollId = null;
+        searchHasShardFailure = true;
+    }
+
+    protected SearchResponse executeSearchScrollRequest(String scrollId) {
+        return ClientHelper.executeWithHeaders(context.headers, ClientHelper.ML_ORIGIN, client,
+                () -> new SearchScrollRequestBuilder(client, SearchScrollAction.INSTANCE)
+                .setScroll(SCROLL_TIMEOUT)
+                .setScrollId(scrollId)
+                .get());
+    }
+
+    private void clearScroll(String scrollId) {
+        if (scrollId != null) {
+            ClearScrollRequest request = new ClearScrollRequest();
+            request.addScrollId(scrollId);
+            ClientHelper.executeWithHeaders(context.headers, ClientHelper.ML_ORIGIN, client,
+                    () -> client.execute(ClearScrollAction.INSTANCE, request).actionGet());
+        }
+    }
+
+    public List<String> getFieldNames() {
+        return context.extractedFields.getAllFields().stream().map(ExtractedField::getAlias).collect(Collectors.toList());
+    }
+
+    public DataSummary collectDataSummary() {
+        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE)
+            .setIndices(context.indices)
+            .setSize(0)
+            .setQuery(context.query)
+            .setTrackTotalHits(true);
+
+        SearchResponse searchResponse = executeSearchRequest(searchRequestBuilder);
+        return new DataSummary(searchResponse.getHits().getTotalHits().value, context.extractedFields.getAllFields().size());
+    }
+
+    public static class DataSummary {
+
+        public final long rows;
+        public final int cols;
+
+        public DataSummary(long rows, int cols) {
+            this.rows = rows;
+            this.cols = cols;
+        }
+    }
+
+    public static class Row {
+
+        private SearchHit hit;
+
+        @Nullable
+        private String[] values;
+
+        private Row(String[] values, SearchHit hit) {
+            this.values = values;
+            this.hit = hit;
+        }
+
+        @Nullable
+        public String[] getValues() {
+            return values;
+        }
+
+        public SearchHit getHit() {
+            return hit;
+        }
+
+        public boolean shouldSkip() {
+            return values == null;
+        }
+
+        public int getChecksum() {
+            return Arrays.hashCode(values);
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java
new file mode 100644
index 0000000000000..f602a66221f7c
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.extractor;
+
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedFields;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class DataFrameDataExtractorContext {
+
+    final String jobId;
+    final ExtractedFields extractedFields;
+    final String[] indices;
+    final QueryBuilder query;
+    final int scrollSize;
+    final Map<String, String> headers;
+    final boolean includeSource;
+
+    DataFrameDataExtractorContext(String jobId, ExtractedFields extractedFields, List<String> indices, QueryBuilder query, int scrollSize,
+                                  Map<String, String> headers, boolean includeSource) {
+        this.jobId = Objects.requireNonNull(jobId);
+        this.extractedFields = Objects.requireNonNull(extractedFields);
+        this.indices = indices.toArray(new String[indices.size()]);
+        this.query = Objects.requireNonNull(query);
+        this.scrollSize = scrollSize;
+        this.headers = headers;
+        this.includeSource = includeSource;
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java
new file mode 100644
index 0000000000000..cacf00ad9e9bc
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java
@@ -0,0 +1,170 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.extractor;
+
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesAction;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedFields;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class DataFrameDataExtractorFactory {
+
+    private final Client client;
+    private final String analyticsId;
+    private final String index;
+    private final ExtractedFields extractedFields;
+    private final Map<String, String> headers;
+
+    private DataFrameDataExtractorFactory(Client client, String analyticsId, String index, ExtractedFields extractedFields,
+                                          Map<String, String> headers) {
+        this.client = Objects.requireNonNull(client);
+        this.analyticsId = Objects.requireNonNull(analyticsId);
+        this.index = Objects.requireNonNull(index);
+        this.extractedFields = Objects.requireNonNull(extractedFields);
+        this.headers = headers;
+    }
+
+    public DataFrameDataExtractor newExtractor(boolean includeSource) {
+        DataFrameDataExtractorContext context = new DataFrameDataExtractorContext(
+                analyticsId,
+                extractedFields,
+                Arrays.asList(index),
+                QueryBuilders.matchAllQuery(),
+                1000,
+                headers,
+                includeSource
+            );
+        return new DataFrameDataExtractor(client, context);
+    }
+
+    /**
+     * Validate and create a new extractor factory
+     *
+     * The destination index must exist and contain at least 1 compatible field or validations will fail.
+     *
+     * @param client ES Client used to make calls against the cluster
+     * @param config The config from which to create the extractor factory
+     * @param isTaskRestarting Whether the task is restarting
+     * @param listener The listener to notify on creation or failure
+     */
+    public static void create(Client client,
+                              DataFrameAnalyticsConfig config,
+                              boolean isTaskRestarting,
+                              ActionListener<DataFrameDataExtractorFactory> listener) {
+        validateIndexAndExtractFields(client, new String[] {config.getDest().getIndex()}, config, isTaskRestarting,
+            ActionListener.wrap(extractedFields -> listener.onResponse(new DataFrameDataExtractorFactory(
+                    client, config.getId(), config.getDest().getIndex(), extractedFields, config.getHeaders())),
+                listener::onFailure
+        ));
+    }
+
+    /**
+     * Validates the source index and analytics config
+     *
+     * @param client ES Client to make calls
+     * @param config Analytics config to validate
+     * @param listener The listener to notify on failure or completion
+     */
+    public static void validateConfigAndSourceIndex(Client client,
+                                                    DataFrameAnalyticsConfig config,
+                                                    ActionListener<DataFrameAnalyticsConfig> listener) {
+        validateIndexAndExtractFields(client, config.getSource().getIndex(), config, false, ActionListener.wrap(
+                fields -> {
+                    config.getSource().getParsedQuery(); // validate query is acceptable
+                    listener.onResponse(config);
+                },
+                listener::onFailure
+        ));
+    }
+
+    private static void validateIndexAndExtractFields(Client client,
+                                                      String[] index,
+                                                      DataFrameAnalyticsConfig config,
+                                                      boolean isTaskRestarting,
+                                                      ActionListener<ExtractedFields> listener) {
+        AtomicInteger docValueFieldsLimitHolder = new AtomicInteger();
+
+        // Step 3. Extract fields (if possible) and notify listener
+        ActionListener<FieldCapabilitiesResponse> fieldCapabilitiesHandler = ActionListener.wrap(
+            fieldCapabilitiesResponse -> listener.onResponse(new ExtractedFieldsDetector(index, config, isTaskRestarting,
+                docValueFieldsLimitHolder.get(), fieldCapabilitiesResponse).detect()),
+            listener::onFailure
+        );
+
+        // Step 2. Get field capabilities necessary to build the information of how to extract fields
+        ActionListener<Integer> docValueFieldsLimitListener = ActionListener.wrap(
+            docValueFieldsLimit -> {
+                docValueFieldsLimitHolder.set(docValueFieldsLimit);
+
+                FieldCapabilitiesRequest fieldCapabilitiesRequest = new FieldCapabilitiesRequest();
+                fieldCapabilitiesRequest.indices(index);
+                fieldCapabilitiesRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
+                fieldCapabilitiesRequest.fields("*");
+                ClientHelper.executeWithHeaders(config.getHeaders(), ClientHelper.ML_ORIGIN, client, () -> {
+                    client.execute(FieldCapabilitiesAction.INSTANCE, fieldCapabilitiesRequest, fieldCapabilitiesHandler);
+                    // This response gets discarded - the listener handles the real response
+                    return null;
+                });
+            },
+            listener::onFailure
+        );
+
+        // Step 1. Get doc value fields limit
+        getDocValueFieldsLimit(client, index, docValueFieldsLimitListener);
+    }
+
+    private static void getDocValueFieldsLimit(Client client, String[] index, ActionListener<Integer> docValueFieldsLimitListener) {
+        ActionListener<GetSettingsResponse> settingsListener = ActionListener.wrap(getSettingsResponse -> {
+                Integer minDocValueFieldsLimit = Integer.MAX_VALUE;
+
+                ImmutableOpenMap<String, Settings> indexToSettings = getSettingsResponse.getIndexToSettings();
+                Iterator<ObjectObjectCursor<String, Settings>> iterator = indexToSettings.iterator();
+                while (iterator.hasNext()) {
+                    ObjectObjectCursor<String, Settings> indexSettings = iterator.next();
+                    Integer indexMaxDocValueFields = IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING.get(indexSettings.value);
+                    if (indexMaxDocValueFields < minDocValueFieldsLimit) {
+                        minDocValueFieldsLimit = indexMaxDocValueFields;
+                    }
+                }
+                docValueFieldsLimitListener.onResponse(minDocValueFieldsLimit);
+            },
+            e -> {
+                if (e instanceof IndexNotFoundException) {
+                    docValueFieldsLimitListener.onFailure(new ResourceNotFoundException("cannot retrieve data because index "
+                        + ((IndexNotFoundException) e).getIndex() + " does not exist"));
+                } else {
+                    docValueFieldsLimitListener.onFailure(e);
+                }
+            }
+        );
+
+        GetSettingsRequest getSettingsRequest = new GetSettingsRequest();
+        getSettingsRequest.indices(index);
+        getSettingsRequest.includeDefaults(true);
+        getSettingsRequest.names(IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING.getKey());
+        client.admin().indices().getSettings(getSettingsRequest, settingsListener);
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java
new file mode 100644
index 0000000000000..d58eaebe353e9
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.extractor;
+
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.fieldcaps.FieldCapabilities;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest;
+import org.elasticsearch.xpack.core.ml.job.messages.Messages;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.utils.NameResolver;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedField;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedFields;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class ExtractedFieldsDetector {
+
+    /**
+     * Fields to ignore. These are mostly internal meta fields.
+     */
+    private static final List<String> IGNORE_FIELDS = Arrays.asList("_id", "_field_names", "_index", "_parent", "_routing", "_seq_no",
+        "_source", "_type", "_uid", "_version", "_feature", "_ignored");
+
+    /**
+     * The types supported by data frames
+     */
+    private static final Set<String> COMPATIBLE_FIELD_TYPES;
+
+    static {
+        Set<String> compatibleTypes = Stream.of(NumberFieldMapper.NumberType.values())
+            .map(NumberFieldMapper.NumberType::typeName)
+            .collect(Collectors.toSet());
+        compatibleTypes.add("scaled_float"); // have to add manually since scaled_float is in a module
+
+        COMPATIBLE_FIELD_TYPES = Collections.unmodifiableSet(compatibleTypes);
+    }
+
+    private final String[] index;
+    private final DataFrameAnalyticsConfig config;
+    private final boolean isTaskRestarting;
+    private final int docValueFieldsLimit;
+    private final FieldCapabilitiesResponse fieldCapabilitiesResponse;
+
+    ExtractedFieldsDetector(String[] index, DataFrameAnalyticsConfig config, boolean isTaskRestarting, int docValueFieldsLimit,
+                            FieldCapabilitiesResponse fieldCapabilitiesResponse) {
+        this.index = Objects.requireNonNull(index);
+        this.config = Objects.requireNonNull(config);
+        this.isTaskRestarting = isTaskRestarting;
+        this.docValueFieldsLimit = docValueFieldsLimit;
+        this.fieldCapabilitiesResponse = Objects.requireNonNull(fieldCapabilitiesResponse);
+    }
+
+    public ExtractedFields detect() {
+        Set<String> fields = new HashSet<>(fieldCapabilitiesResponse.get().keySet());
+        fields.removeAll(IGNORE_FIELDS);
+
+        checkResultsFieldIsNotPresent();
+
+        // Ignore fields under the results object
+        fields.removeIf(field -> field.startsWith(config.getDest().getResultsField() + "."));
+
+        removeFieldsWithIncompatibleTypes(fields);
+        includeAndExcludeFields(fields, index);
+        List<String> sortedFields = new ArrayList<>(fields);
+        // We sort the fields to ensure the checksum for each document is deterministic
+        Collections.sort(sortedFields);
+        ExtractedFields extractedFields = ExtractedFields.build(sortedFields, Collections.emptySet(), fieldCapabilitiesResponse)
+            .filterFields(ExtractedField.ExtractionMethod.DOC_VALUE);
+        if (extractedFields.getAllFields().isEmpty()) {
+            throw ExceptionsHelper.badRequestException("No compatible fields could be detected in index {}", Arrays.toString(index));
+        }
+        if (extractedFields.getDocValueFields().size() > docValueFieldsLimit) {
+            extractedFields = fetchFromSourceIfSupported(extractedFields);
+            if (extractedFields.getDocValueFields().size() > docValueFieldsLimit) {
+                throw ExceptionsHelper.badRequestException("[{}] fields must be retrieved from doc_values but the limit is [{}]; " +
+                    "please adjust the index level setting [{}]", extractedFields.getDocValueFields().size(), docValueFieldsLimit,
+                    IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING.getKey());
+            }
+        }
+        return extractedFields;
+    }
+
+    private void checkResultsFieldIsNotPresent() {
+        // If the task is restarting we do not mind the index containing the results field, we will overwrite all docs
+        if (isTaskRestarting) {
+            return;
+        }
+
+        Map<String, FieldCapabilities> indexToFieldCaps = fieldCapabilitiesResponse.getField(config.getDest().getResultsField());
+        if (indexToFieldCaps != null && indexToFieldCaps.isEmpty() == false) {
+            throw ExceptionsHelper.badRequestException("A field that matches the {}.{} [{}] already exists;" +
+                    " please set a different {}", DataFrameAnalyticsConfig.DEST.getPreferredName(),
+                DataFrameAnalyticsDest.RESULTS_FIELD.getPreferredName(), config.getDest().getResultsField(),
+                DataFrameAnalyticsDest.RESULTS_FIELD.getPreferredName());
+        }
+    }
+
+    private void removeFieldsWithIncompatibleTypes(Set<String> fields) {
+        Iterator<String> fieldsIterator = fields.iterator();
+        while (fieldsIterator.hasNext()) {
+            String field = fieldsIterator.next();
+            Map<String, FieldCapabilities> fieldCaps = fieldCapabilitiesResponse.getField(field);
+            if (fieldCaps == null || COMPATIBLE_FIELD_TYPES.containsAll(fieldCaps.keySet()) == false) {
+                fieldsIterator.remove();
+            }
+        }
+    }
+
+    private void includeAndExcludeFields(Set<String> fields, String[] index) {
+        FetchSourceContext analyzedFields = config.getAnalyzedFields();
+        if (analyzedFields == null) {
+            return;
+        }
+        String includes = analyzedFields.includes().length == 0 ? "*" : Strings.arrayToCommaDelimitedString(analyzedFields.includes());
+        String excludes = Strings.arrayToCommaDelimitedString(analyzedFields.excludes());
+
+        if (Regex.isMatchAllPattern(includes) && excludes.isEmpty()) {
+            return;
+        }
+        try {
+            // If the inclusion set does not match anything, that means the user's desired fields cannot be found in
+            // the collection of supported field types. We should let the user know.
+            Set<String> includedSet = NameResolver.newUnaliased(fields,
+                (ex) -> new ResourceNotFoundException(
+                    Messages.getMessage(Messages.DATA_FRAME_ANALYTICS_BAD_FIELD_FILTER, ex)))
+                .expand(includes, false);
+            // If the exclusion set does not match anything, that means the fields are already not present
+            // no need to raise if nothing matched
+            Set<String> excludedSet = NameResolver.newUnaliased(fields,
+                (ex) -> new ResourceNotFoundException(
+                    Messages.getMessage(Messages.DATA_FRAME_ANALYTICS_BAD_FIELD_FILTER, ex)))
+                .expand(excludes, true);
+
+            fields.retainAll(includedSet);
+            fields.removeAll(excludedSet);
+        } catch (ResourceNotFoundException ex) {
+            // Re-wrap our exception so that we throw the same exception type when there are no fields.
+            throw ExceptionsHelper.badRequestException(ex.getMessage());
+        }
+    }
+
+    private ExtractedFields fetchFromSourceIfSupported(ExtractedFields extractedFields) {
+        List<ExtractedField> adjusted = new ArrayList<>(extractedFields.getAllFields().size());
+        for (ExtractedField field : extractedFields.getDocValueFields()) {
+            adjusted.add(field.supportsFromSource() ? field.newFromSource() : field);
+        }
+        return new ExtractedFields(adjusted);
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/persistence/DataFrameAnalyticsConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/persistence/DataFrameAnalyticsConfigProvider.java
new file mode 100644
index 0000000000000..569469452cf64
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/persistence/DataFrameAnalyticsConfigProvider.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.persistence;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.index.IndexAction;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.engine.VersionConflictEngineException;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.action.util.PageParams;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
+import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
+
+public class DataFrameAnalyticsConfigProvider {
+
+    private static final int MAX_CONFIGS_SIZE = 10000;
+
+    private static final Map<String, String> TO_XCONTENT_PARAMS;
+
+    static {
+        Map<String, String> modifiable = new HashMap<>();
+        modifiable.put(ToXContentParams.INCLUDE_TYPE, "true");
+        modifiable.put(ToXContentParams.FOR_INTERNAL_STORAGE, "true");
+        TO_XCONTENT_PARAMS = Collections.unmodifiableMap(modifiable);
+    }
+
+    private final Client client;
+
+    public DataFrameAnalyticsConfigProvider(Client client) {
+        this.client = Objects.requireNonNull(client);
+    }
+
+    public void put(DataFrameAnalyticsConfig config, Map<String, String> headers, ActionListener<IndexResponse> listener) {
+        String id = config.getId();
+
+        if (headers.isEmpty() == false) {
+            // Filter any values in headers that aren't security fields
+            DataFrameAnalyticsConfig.Builder builder = new DataFrameAnalyticsConfig.Builder(config);
+            Map<String, String> securityHeaders = headers.entrySet().stream()
+                .filter(e -> ClientHelper.SECURITY_HEADER_FILTERS.contains(e.getKey()))
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+            builder.setHeaders(securityHeaders);
+            config = builder.build();
+        }
+        try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
+            config.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS));
+            IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.configIndexName())
+                    .id(DataFrameAnalyticsConfig.documentId(config.getId()))
+                    .opType(DocWriteRequest.OpType.CREATE)
+                    .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                    .source(builder);
+
+            executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap(
+                listener::onResponse,
+                e -> {
+                    if (e instanceof VersionConflictEngineException) {
+                        listener.onFailure(ExceptionsHelper.dataFrameAnalyticsAlreadyExists(id));
+                    } else {
+                        listener.onFailure(e);
+                    }
+                }
+            ));
+        } catch (IOException e) {
+            listener.onFailure(new ElasticsearchParseException("Failed to serialise data frame analytics with id [" + config.getId()
+                + "]"));
+        }
+    }
+
+    public void get(String id, ActionListener<DataFrameAnalyticsConfig> listener) {
+        GetDataFrameAnalyticsAction.Request request = new GetDataFrameAnalyticsAction.Request();
+        request.setResourceId(id);
+        executeAsyncWithOrigin(client, ML_ORIGIN, GetDataFrameAnalyticsAction.INSTANCE, request, ActionListener.wrap(
+            response -> {
+                List<DataFrameAnalyticsConfig> analytics = response.getResources().results();
+                if (analytics.size() != 1) {
+                    listener.onFailure(ExceptionsHelper.badRequestException("Expected a single match for data frame analytics [{}] " +
+                        "but got [{}]", id, analytics.size()));
+                } else {
+                    listener.onResponse(analytics.get(0));
+                }
+            },
+            listener::onFailure
+        ));
+    }
+
+    /**
+     * @param ids a comma separated list of single IDs and/or wildcards
+     */
+    public void getMultiple(String ids, boolean allowNoMatch, ActionListener<List<DataFrameAnalyticsConfig>> listener) {
+        GetDataFrameAnalyticsAction.Request request = new GetDataFrameAnalyticsAction.Request();
+        request.setPageParams(new PageParams(0, MAX_CONFIGS_SIZE));
+        request.setResourceId(ids);
+        request.setAllowNoResources(allowNoMatch);
+        executeAsyncWithOrigin(client, ML_ORIGIN, GetDataFrameAnalyticsAction.INSTANCE, request, ActionListener.wrap(
+            response -> listener.onResponse(response.getResources().results()), listener::onFailure));
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsBuilder.java
new file mode 100644
index 0000000000000..4d58a132bab5b
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsBuilder.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.ml.process.NativeController;
+import org.elasticsearch.xpack.ml.process.ProcessPipes;
+
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+public class AnalyticsBuilder {
+
+    public static final String ANALYTICS = "data_frame_analyzer";
+    private static final String ANALYTICS_PATH = "./" + ANALYTICS;
+
+    private static final String LENGTH_ENCODED_INPUT_ARG = "--lengthEncodedInput";
+    private static final String CONFIG_ARG = "--config=";
+
+    private final Environment env;
+    private final NativeController nativeController;
+    private final ProcessPipes processPipes;
+    private final AnalyticsProcessConfig config;
+    private final List<Path> filesToDelete;
+
+    public AnalyticsBuilder(Environment env, NativeController nativeController, ProcessPipes processPipes, AnalyticsProcessConfig config,
+                            List<Path> filesToDelete) {
+        this.env = Objects.requireNonNull(env);
+        this.nativeController = Objects.requireNonNull(nativeController);
+        this.processPipes = Objects.requireNonNull(processPipes);
+        this.config = Objects.requireNonNull(config);
+        this.filesToDelete = Objects.requireNonNull(filesToDelete);
+    }
+
+    public void build() throws IOException {
+        List<String> command = buildAnalyticsCommand();
+        processPipes.addArgs(command);
+        nativeController.startProcess(command);
+    }
+
+    List<String> buildAnalyticsCommand() throws IOException {
+        List<String> command = new ArrayList<>();
+        command.add(ANALYTICS_PATH);
+        command.add(LENGTH_ENCODED_INPUT_ARG);
+        addConfigFile(command);
+        return command;
+    }
+
+    private void addConfigFile(List<String> command) throws IOException {
+        Path configFile = Files.createTempFile(env.tmpFile(), "analysis", ".conf");
+        filesToDelete.add(configFile);
+        try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(configFile),StandardCharsets.UTF_8);
+             XContentBuilder jsonBuilder = JsonXContent.contentBuilder()) {
+
+            config.toXContent(jsonBuilder, ToXContent.EMPTY_PARAMS);
+            osw.write(Strings.toString(jsonBuilder));
+        }
+
+        command.add(CONFIG_ARG + configFile.toString());
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsControlMessageWriter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsControlMessageWriter.java
new file mode 100644
index 0000000000000..c66754171fc29
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsControlMessageWriter.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import org.elasticsearch.xpack.ml.process.writer.AbstractControlMsgWriter;
+import org.elasticsearch.xpack.ml.process.writer.LengthEncodedWriter;
+
+import java.io.IOException;
+
+public class AnalyticsControlMessageWriter extends AbstractControlMsgWriter {
+
+    /**
+     * This must match the code defined in the api::CDataFrameAnalyzer C++ class.
+     * The constant there is referred as RUN_ANALYSIS_CONTROL_MESSAGE_FIELD_VALUE
+     * but in the context of the java side it is more descriptive to call this the
+     * end of data message.
+     */
+    private static final String END_OF_DATA_MESSAGE_CODE = "$";
+
+    /**
+     * Construct the control message writer with a LengthEncodedWriter
+     *
+     * @param lengthEncodedWriter The writer
+     * @param numberOfFields      The number of fields the process expects in each record
+     */
+    public AnalyticsControlMessageWriter(LengthEncodedWriter lengthEncodedWriter, int numberOfFields) {
+        super(lengthEncodedWriter, numberOfFields);
+    }
+
+    public void writeEndOfData() throws IOException {
+        writeMessage(END_OF_DATA_MESSAGE_CODE);
+        fillCommandBuffer();
+        lengthEncodedWriter.flush();
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcess.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcess.java
new file mode 100644
index 0000000000000..c5e361c3e1215
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcess.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import org.elasticsearch.xpack.ml.process.NativeProcess;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+public interface AnalyticsProcess extends NativeProcess {
+
+    /**
+     * Writes a control message that informs the process
+     * all data has been sent
+     * @throws IOException If an error occurs writing to the process
+     */
+    void writeEndOfDataMessage() throws IOException;
+
+    /**
+     * @return stream of data frame analytics results.
+     */
+    Iterator<AnalyticsResult> readAnalyticsResults();
+
+    /**
+     * Read anything left in the stream before
+     * closing the stream otherwise if the process
+     * tries to write more after the close it gets
+     * a SIGPIPE
+     */
+    void consumeAndCloseOutputStream();
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java
new file mode 100644
index 0000000000000..226498376bbe1
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.DataFrameAnalysis;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class AnalyticsProcessConfig implements ToXContentObject {
+
+    private static final String ROWS = "rows";
+    private static final String COLS = "cols";
+    private static final String MEMORY_LIMIT = "memory_limit";
+    private static final String THREADS = "threads";
+    private static final String ANALYSIS = "analysis";
+    private static final String RESULTS_FIELD = "results_field";
+
+    private final long rows;
+    private final int cols;
+    private final ByteSizeValue memoryLimit;
+    private final int threads;
+    private final DataFrameAnalysis analysis;
+    private final String resultsField;
+
+    public AnalyticsProcessConfig(long rows, int cols, ByteSizeValue memoryLimit, int threads, String resultsField,
+                                  DataFrameAnalysis analysis) {
+        this.rows = rows;
+        this.cols = cols;
+        this.memoryLimit = Objects.requireNonNull(memoryLimit);
+        this.threads = threads;
+        this.resultsField = Objects.requireNonNull(resultsField);
+        this.analysis = Objects.requireNonNull(analysis);
+    }
+
+    public int cols() {
+        return cols;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(ROWS, rows);
+        builder.field(COLS, cols);
+        builder.field(MEMORY_LIMIT, memoryLimit.getBytes());
+        builder.field(THREADS, threads);
+        builder.field(RESULTS_FIELD, resultsField);
+        builder.field(ANALYSIS, new DataFrameAnalysisWrapper(analysis));
+        builder.endObject();
+        return builder;
+    }
+
+    private static class DataFrameAnalysisWrapper implements ToXContentObject {
+
+        private final DataFrameAnalysis analysis;
+
+        private DataFrameAnalysisWrapper(DataFrameAnalysis analysis) {
+            this.analysis = analysis;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field("name", analysis.getWriteableName());
+            builder.field("parameters", analysis.getParams());
+            builder.endObject();
+            return builder;
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessFactory.java
new file mode 100644
index 0000000000000..1df29b88ba4e3
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessFactory.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import java.util.concurrent.ExecutorService;
+import java.util.function.Consumer;
+
+public interface AnalyticsProcessFactory {
+
+    /**
+     * Create an implementation of {@link AnalyticsProcess}
+     *
+     * @param jobId             The job id
+     * @param analyticsProcessConfig The process configuration
+     * @param executorService   Executor service used to start the async tasks a job needs to operate the analytical process
+     * @param onProcessCrash    Callback to execute if the process stops unexpectedly
+     * @return The process
+     */
+    AnalyticsProcess createAnalyticsProcess(String jobId, AnalyticsProcessConfig analyticsProcessConfig, ExecutorService executorService,
+                                            Consumer<String> onProcessCrash);
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
new file mode 100644
index 0000000000000..bb54895b2fa97
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java
@@ -0,0 +1,301 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.ml.MachineLearning;
+import org.elasticsearch.xpack.ml.action.TransportStartDataFrameAnalyticsAction.DataFrameAnalyticsTask;
+import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractor;
+import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractorFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
+
+public class AnalyticsProcessManager {
+
+    private static final Logger LOGGER = LogManager.getLogger(AnalyticsProcessManager.class);
+
+    private final Client client;
+    private final ThreadPool threadPool;
+    private final AnalyticsProcessFactory processFactory;
+    private final ConcurrentMap<Long, ProcessContext> processContextByAllocation = new ConcurrentHashMap<>();
+
+    public AnalyticsProcessManager(Client client, ThreadPool threadPool, AnalyticsProcessFactory analyticsProcessFactory) {
+        this.client = Objects.requireNonNull(client);
+        this.threadPool = Objects.requireNonNull(threadPool);
+        this.processFactory = Objects.requireNonNull(analyticsProcessFactory);
+    }
+
+    public void runJob(DataFrameAnalyticsTask task, DataFrameAnalyticsConfig config,
+                       DataFrameDataExtractorFactory dataExtractorFactory, Consumer<Exception> finishHandler) {
+        threadPool.generic().execute(() -> {
+            if (task.isStopping()) {
+                // The task was requested to stop before we created the process context
+                finishHandler.accept(null);
+                return;
+            }
+
+            ProcessContext processContext = new ProcessContext(config.getId());
+            if (processContextByAllocation.putIfAbsent(task.getAllocationId(), processContext) != null) {
+                finishHandler.accept(ExceptionsHelper.serverError("[" + processContext.id
+                    + "] Could not create process as one already exists"));
+                return;
+            }
+            if (processContext.startProcess(dataExtractorFactory, config, task)) {
+                ExecutorService executorService = threadPool.executor(MachineLearning.JOB_COMMS_THREAD_POOL_NAME);
+                executorService.execute(() -> processResults(processContext));
+                executorService.execute(() -> processData(task, config, processContext.dataExtractor,
+                    processContext.process, processContext.resultProcessor, finishHandler));
+            } else {
+                finishHandler.accept(null);
+            }
+        });
+    }
+
+    private void processResults(ProcessContext processContext) {
+        try {
+            processContext.resultProcessor.process(processContext.process);
+        } catch (Exception e) {
+            processContext.setFailureReason(e.getMessage());
+        }
+    }
+
+    private void processData(DataFrameAnalyticsTask task, DataFrameAnalyticsConfig config, DataFrameDataExtractor dataExtractor,
+                             AnalyticsProcess process, AnalyticsResultProcessor resultProcessor, Consumer<Exception> finishHandler) {
+
+        try {
+            writeHeaderRecord(dataExtractor, process);
+            writeDataRows(dataExtractor, process);
+            process.writeEndOfDataMessage();
+            process.flushStream();
+
+            LOGGER.info("[{}] Waiting for result processor to complete", config.getId());
+            resultProcessor.awaitForCompletion();
+            processContextByAllocation.get(task.getAllocationId()).setFailureReason(resultProcessor.getFailure());
+
+            refreshDest(config);
+            LOGGER.info("[{}] Result processor has completed", config.getId());
+        } catch (Exception e) {
+            String errorMsg = new ParameterizedMessage("[{}] Error while processing data", config.getId()).getFormattedMessage();
+            processContextByAllocation.get(task.getAllocationId()).setFailureReason(errorMsg);
+        } finally {
+            closeProcess(task);
+
+            ProcessContext processContext = processContextByAllocation.remove(task.getAllocationId());
+            LOGGER.debug("Removed process context for task [{}]; [{}] processes still running", config.getId(),
+                processContextByAllocation.size());
+
+            if (processContext.getFailureReason() == null) {
+                // This results in marking the persistent task as complete
+                LOGGER.info("[{}] Marking task completed", config.getId());
+                finishHandler.accept(null);
+            } else {
+                LOGGER.error("[{}] Marking task failed; {}", config.getId(), processContext.getFailureReason());
+                updateTaskState(task, DataFrameAnalyticsState.FAILED, processContext.getFailureReason());
+            }
+        }
+    }
+
+    private void writeDataRows(DataFrameDataExtractor dataExtractor, AnalyticsProcess process) throws IOException {
+        // The extra fields are for the doc hash and the control field (should be an empty string)
+        String[] record = new String[dataExtractor.getFieldNames().size() + 2];
+        // The value of the control field should be an empty string for data frame rows
+        record[record.length - 1] = "";
+
+        while (dataExtractor.hasNext()) {
+            Optional<List<DataFrameDataExtractor.Row>> rows = dataExtractor.next();
+            if (rows.isPresent()) {
+                for (DataFrameDataExtractor.Row row : rows.get()) {
+                    if (row.shouldSkip() == false) {
+                        String[] rowValues = row.getValues();
+                        System.arraycopy(rowValues, 0, record, 0, rowValues.length);
+                        record[record.length - 2] = String.valueOf(row.getChecksum());
+                        process.writeRecord(record);
+                    }
+                }
+            }
+        }
+    }
+
+    private void writeHeaderRecord(DataFrameDataExtractor dataExtractor, AnalyticsProcess process) throws IOException {
+        List<String> fieldNames = dataExtractor.getFieldNames();
+
+        // We add 2 extra fields, both named dot:
+        //   - the document hash
+        //   - the control message
+        String[] headerRecord = new String[fieldNames.size() + 2];
+        for (int i = 0; i < fieldNames.size(); i++) {
+            headerRecord[i] = fieldNames.get(i);
+        }
+
+        headerRecord[headerRecord.length - 2] = ".";
+        headerRecord[headerRecord.length - 1] = ".";
+        process.writeRecord(headerRecord);
+    }
+
+    private AnalyticsProcess createProcess(DataFrameAnalyticsTask task, AnalyticsProcessConfig analyticsProcessConfig) {
+        ExecutorService executorService = threadPool.executor(MachineLearning.JOB_COMMS_THREAD_POOL_NAME);
+        AnalyticsProcess process = processFactory.createAnalyticsProcess(task.getParams().getId(), analyticsProcessConfig,
+            executorService, onProcessCrash(task));
+        if (process.isProcessAlive() == false) {
+            throw ExceptionsHelper.serverError("Failed to start data frame analytics process");
+        }
+        return process;
+    }
+
+    private Consumer<String> onProcessCrash(DataFrameAnalyticsTask task) {
+        return reason -> {
+            ProcessContext processContext = processContextByAllocation.get(task.getAllocationId());
+            if (processContext != null) {
+                processContext.setFailureReason(reason);
+                processContext.stop();
+            }
+        };
+    }
+
+    private void updateTaskState(DataFrameAnalyticsTask task, DataFrameAnalyticsState state, @Nullable String reason) {
+        DataFrameAnalyticsTaskState newTaskState = new DataFrameAnalyticsTaskState(state, task.getAllocationId(), reason);
+        task.updatePersistentTaskState(newTaskState, ActionListener.wrap(
+            updatedTask -> LOGGER.info("[{}] Successfully update task state to [{}]", task.getParams().getId(), state),
+            e -> LOGGER.error(new ParameterizedMessage("[{}] Could not update task state to [{}]", task.getParams().getId(), state), e)
+        ));
+    }
+
+    @Nullable
+    public Integer getProgressPercent(long allocationId) {
+        ProcessContext processContext = processContextByAllocation.get(allocationId);
+        return processContext == null ? null : processContext.progressPercent.get();
+    }
+
+    private void refreshDest(DataFrameAnalyticsConfig config) {
+        ClientHelper.executeWithHeaders(config.getHeaders(), ClientHelper.ML_ORIGIN, client,
+            () -> client.execute(RefreshAction.INSTANCE, new RefreshRequest(config.getDest().getIndex())).actionGet());
+    }
+
+    private void closeProcess(DataFrameAnalyticsTask task) {
+        String configId = task.getParams().getId();
+        LOGGER.info("[{}] Closing process", configId);
+
+        ProcessContext processContext = processContextByAllocation.get(task.getAllocationId());
+        try {
+            processContext.process.close();
+            LOGGER.info("[{}] Closed process", configId);
+        } catch (IOException e) {
+            String errorMsg = new ParameterizedMessage("[{}] Error closing data frame analyzer process [{}]"
+                , configId, e.getMessage()).getFormattedMessage();
+            processContext.setFailureReason(errorMsg);
+        }
+    }
+
+    public void stop(DataFrameAnalyticsTask task) {
+        ProcessContext processContext = processContextByAllocation.get(task.getAllocationId());
+        if (processContext != null) {
+            LOGGER.debug("[{}] Stopping process", task.getParams().getId() );
+            processContext.stop();
+        } else {
+            LOGGER.debug("[{}] No process context to stop", task.getParams().getId() );
+            task.markAsCompleted();
+        }
+    }
+
+    class ProcessContext {
+
+        private final String id;
+        private volatile AnalyticsProcess process;
+        private volatile DataFrameDataExtractor dataExtractor;
+        private volatile AnalyticsResultProcessor resultProcessor;
+        private final AtomicInteger progressPercent = new AtomicInteger(0);
+        private volatile boolean processKilled;
+        private volatile String failureReason;
+
+        ProcessContext(String id) {
+            this.id = Objects.requireNonNull(id);
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        public boolean isProcessKilled() {
+            return processKilled;
+        }
+
+        void setProgressPercent(int progressPercent) {
+            this.progressPercent.set(progressPercent);
+        }
+
+        private synchronized void setFailureReason(String failureReason) {
+            // Only set the new reason if there isn't one already as we want to keep the first reason
+            if (failureReason != null) {
+                this.failureReason = failureReason;
+            }
+        }
+
+        private String getFailureReason() {
+            return failureReason;
+        }
+
+        public synchronized void stop() {
+            LOGGER.debug("[{}] Stopping process", id);
+            processKilled = true;
+            if (dataExtractor != null) {
+                dataExtractor.cancel();
+            }
+            if (process != null) {
+                try {
+                    process.kill();
+                } catch (IOException e) {
+                    LOGGER.error(new ParameterizedMessage("[{}] Failed to kill process", id), e);
+                }
+            }
+        }
+
+        /**
+         * @return {@code true} if the process was started or {@code false} if it was not because it was stopped in the meantime
+         */
+        private synchronized boolean startProcess(DataFrameDataExtractorFactory dataExtractorFactory, DataFrameAnalyticsConfig config,
+                                                  DataFrameAnalyticsTask task) {
+            if (processKilled) {
+                // The job was stopped before we started the process so no need to start it
+                return false;
+            }
+
+            dataExtractor = dataExtractorFactory.newExtractor(false);
+            process = createProcess(task, createProcessConfig(config, dataExtractor));
+            DataFrameRowsJoiner dataFrameRowsJoiner = new DataFrameRowsJoiner(config.getId(), client,
+                dataExtractorFactory.newExtractor(true));
+            resultProcessor = new AnalyticsResultProcessor(id, dataFrameRowsJoiner, this::isProcessKilled, this::setProgressPercent);
+            return true;
+        }
+
+        private AnalyticsProcessConfig createProcessConfig(DataFrameAnalyticsConfig config, DataFrameDataExtractor dataExtractor) {
+            DataFrameDataExtractor.DataSummary dataSummary = dataExtractor.collectDataSummary();
+            AnalyticsProcessConfig processConfig = new AnalyticsProcessConfig(dataSummary.rows, dataSummary.cols,
+                config.getModelMemoryLimit(), 1, config.getDest().getResultsField(), config.getAnalysis());
+            return processConfig;
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResult.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResult.java
new file mode 100644
index 0000000000000..ced64ab04a280
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResult.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.ml.dataframe.process.results.RowResults;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class AnalyticsResult implements ToXContentObject {
+
+    public static final ParseField TYPE = new ParseField("analytics_result");
+
+    public static final ParseField PROGRESS_PERCENT = new ParseField("progress_percent");
+
+    static final ConstructingObjectParser<AnalyticsResult, Void> PARSER = new ConstructingObjectParser<>(TYPE.getPreferredName(),
+            a -> new AnalyticsResult((RowResults) a[0], (Integer) a[1]));
+
+    static {
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), RowResults.PARSER, RowResults.TYPE);
+        PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), PROGRESS_PERCENT);
+    }
+
+    private final RowResults rowResults;
+    private final Integer progressPercent;
+
+    public AnalyticsResult(RowResults rowResults, Integer progressPercent) {
+        this.rowResults = rowResults;
+        this.progressPercent = progressPercent;
+    }
+
+    public RowResults getRowResults() {
+        return rowResults;
+    }
+
+    public Integer getProgressPercent() {
+        return progressPercent;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        if (rowResults != null) {
+            builder.field(RowResults.TYPE.getPreferredName(), rowResults);
+        }
+        if (progressPercent != null) {
+            builder.field(PROGRESS_PERCENT.getPreferredName(), progressPercent);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        AnalyticsResult that = (AnalyticsResult) other;
+        return Objects.equals(rowResults, that.rowResults) && Objects.equals(progressPercent, that.progressPercent);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(rowResults, progressPercent);
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
new file mode 100644
index 0000000000000..11c451e9c3932
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.xpack.ml.dataframe.process.results.RowResults;
+
+import java.util.Iterator;
+import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+
+public class AnalyticsResultProcessor {
+
+    private static final Logger LOGGER = LogManager.getLogger(AnalyticsResultProcessor.class);
+
+    private final String dataFrameAnalyticsId;
+    private final DataFrameRowsJoiner dataFrameRowsJoiner;
+    private final Supplier<Boolean> isProcessKilled;
+    private final Consumer<Integer> progressConsumer;
+    private final CountDownLatch completionLatch = new CountDownLatch(1);
+    private volatile String failure;
+
+    public AnalyticsResultProcessor(String dataFrameAnalyticsId, DataFrameRowsJoiner dataFrameRowsJoiner, Supplier<Boolean> isProcessKilled,
+                                    Consumer<Integer> progressConsumer) {
+        this.dataFrameAnalyticsId = Objects.requireNonNull(dataFrameAnalyticsId);
+        this.dataFrameRowsJoiner = Objects.requireNonNull(dataFrameRowsJoiner);
+        this.isProcessKilled = Objects.requireNonNull(isProcessKilled);
+        this.progressConsumer = Objects.requireNonNull(progressConsumer);
+    }
+
+    @Nullable
+    public String getFailure() {
+        return failure == null ? dataFrameRowsJoiner.getFailure() : failure;
+    }
+
+    public void awaitForCompletion() {
+        try {
+            if (completionLatch.await(30, TimeUnit.MINUTES) == false) {
+                LOGGER.warn("[{}] Timeout waiting for results processor to complete", dataFrameAnalyticsId);
+            }
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            LOGGER.info("[{}] Interrupted waiting for results processor to complete", dataFrameAnalyticsId);
+        }
+    }
+
+    public void process(AnalyticsProcess process) {
+        // TODO When java 9 features can be used, we will not need the local variable here
+        try (DataFrameRowsJoiner resultsJoiner = dataFrameRowsJoiner) {
+            Iterator<AnalyticsResult> iterator = process.readAnalyticsResults();
+            while (iterator.hasNext()) {
+                AnalyticsResult result = iterator.next();
+                processResult(result, resultsJoiner);
+            }
+        } catch (Exception e) {
+            if (isProcessKilled.get()) {
+                // No need to log error as it's due to stopping
+            } else {
+                LOGGER.error(new ParameterizedMessage("[{}] Error parsing data frame analytics output", dataFrameAnalyticsId), e);
+                failure = "error parsing data frame analytics output: [" + e.getMessage() + "]";
+            }
+        } finally {
+            completionLatch.countDown();
+            process.consumeAndCloseOutputStream();
+        }
+    }
+
+    private void processResult(AnalyticsResult result, DataFrameRowsJoiner resultsJoiner) {
+        RowResults rowResults = result.getRowResults();
+        if (rowResults != null) {
+            resultsJoiner.processRowResults(rowResults);
+        }
+        Integer progressPercent = result.getProgressPercent();
+        if (progressPercent != null) {
+            progressConsumer.accept(progressPercent);
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoiner.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoiner.java
new file mode 100644
index 0000000000000..1b3dd2932ab9e
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoiner.java
@@ -0,0 +1,186 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.bulk.BulkAction;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractor;
+import org.elasticsearch.xpack.ml.dataframe.process.results.RowResults;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
+class DataFrameRowsJoiner implements AutoCloseable {
+
+    private static final Logger LOGGER = LogManager.getLogger(DataFrameRowsJoiner.class);
+
+    private static final int RESULTS_BATCH_SIZE = 1000;
+
+    private final String analyticsId;
+    private final Client client;
+    private final DataFrameDataExtractor dataExtractor;
+    private final Iterator<DataFrameDataExtractor.Row> dataFrameRowsIterator;
+    private LinkedList<RowResults> currentResults;
+    private volatile String failure;
+
+    DataFrameRowsJoiner(String analyticsId, Client client, DataFrameDataExtractor dataExtractor) {
+        this.analyticsId = Objects.requireNonNull(analyticsId);
+        this.client = Objects.requireNonNull(client);
+        this.dataExtractor = Objects.requireNonNull(dataExtractor);
+        this.dataFrameRowsIterator = new ResultMatchingDataFrameRows();
+        this.currentResults = new LinkedList<>();
+    }
+
+    @Nullable
+    String getFailure() {
+        return failure;
+    }
+
+    void processRowResults(RowResults rowResults) {
+        if (failure != null) {
+            // If we are in failed state we drop the results but we let the processor
+            // parse the output
+            return;
+        }
+
+        try {
+            addResultAndJoinIfEndOfBatch(rowResults);
+        } catch (Exception e) {
+            LOGGER.error(new ParameterizedMessage("[{}] Failed to join results ", analyticsId), e);
+            failure = "[" + analyticsId + "] Failed to join results: " + e.getMessage();
+        }
+    }
+
+    private void addResultAndJoinIfEndOfBatch(RowResults rowResults) {
+        currentResults.add(rowResults);
+        if (currentResults.size() == RESULTS_BATCH_SIZE) {
+            joinCurrentResults();
+        }
+    }
+
+    private void joinCurrentResults() {
+        BulkRequest bulkRequest = new BulkRequest();
+        while (currentResults.isEmpty() == false) {
+            RowResults result = currentResults.pop();
+            DataFrameDataExtractor.Row row = dataFrameRowsIterator.next();
+            checkChecksumsMatch(row, result);
+            bulkRequest.add(createIndexRequest(result, row.getHit()));
+        }
+        if (bulkRequest.numberOfActions() > 0) {
+            executeBulkRequest(bulkRequest);
+        }
+        currentResults = new LinkedList<>();
+    }
+
+    private void checkChecksumsMatch(DataFrameDataExtractor.Row row, RowResults result) {
+        if (row.getChecksum() != result.getChecksum()) {
+            String msg = "Detected checksum mismatch for document with id [" + row.getHit().getId() + "]; ";
+            msg += "expected [" + row.getChecksum() + "] but result had [" + result.getChecksum() + "]; ";
+            msg += "this implies the data frame index [" + row.getHit().getIndex() + "] was modified while the analysis was running. ";
+            msg += "We rely on this index being immutable during a running analysis and so the results will be unreliable.";
+            throw ExceptionsHelper.serverError(msg);
+        }
+    }
+
+    private IndexRequest createIndexRequest(RowResults result, SearchHit hit) {
+        Map<String, Object> source = new LinkedHashMap(hit.getSourceAsMap());
+        source.putAll(result.getResults());
+        IndexRequest indexRequest = new IndexRequest(hit.getIndex());
+        indexRequest.id(hit.getId());
+        indexRequest.source(source);
+        indexRequest.opType(DocWriteRequest.OpType.INDEX);
+        return indexRequest;
+    }
+
+    private void executeBulkRequest(BulkRequest bulkRequest) {
+        BulkResponse bulkResponse = ClientHelper.executeWithHeaders(dataExtractor.getHeaders(), ClientHelper.ML_ORIGIN, client,
+                () -> client.execute(BulkAction.INSTANCE, bulkRequest).actionGet());
+        if (bulkResponse.hasFailures()) {
+            throw ExceptionsHelper.serverError("failures while writing results [" + bulkResponse.buildFailureMessage() + "]");
+        }
+    }
+
+    @Override
+    public void close() {
+        try {
+            joinCurrentResults();
+        } catch (Exception e) {
+            LOGGER.error(new ParameterizedMessage("[{}] Failed to join results", analyticsId), e);
+            failure = "[" + analyticsId + "] Failed to join results: " + e.getMessage();
+        } finally {
+            try {
+                consumeDataExtractor();
+            } catch (Exception e) {
+                LOGGER.error(new ParameterizedMessage("[{}] Failed to consume data extractor", analyticsId), e);
+            }
+        }
+    }
+
+    private void consumeDataExtractor() throws IOException {
+        dataExtractor.cancel();
+        while (dataExtractor.hasNext()) {
+            dataExtractor.next();
+        }
+    }
+
+    private class ResultMatchingDataFrameRows implements Iterator<DataFrameDataExtractor.Row> {
+
+        private List<DataFrameDataExtractor.Row> currentDataFrameRows = Collections.emptyList();
+        private int currentDataFrameRowsIndex;
+
+        @Override
+        public boolean hasNext() {
+            return dataExtractor.hasNext() || currentDataFrameRowsIndex < currentDataFrameRows.size();
+        }
+
+        @Override
+        public DataFrameDataExtractor.Row next() {
+            DataFrameDataExtractor.Row row = null;
+            while ((row == null || row.shouldSkip()) && hasNext()) {
+                advanceToNextBatchIfNecessary();
+                row = currentDataFrameRows.get(currentDataFrameRowsIndex++);
+            }
+
+            if (row == null || row.shouldSkip()) {
+                throw ExceptionsHelper.serverError("no more data frame rows could be found while joining results");
+            }
+            return row;
+        }
+
+        private void advanceToNextBatchIfNecessary() {
+            if (currentDataFrameRowsIndex >= currentDataFrameRows.size()) {
+                currentDataFrameRows = getNextDataRowsBatch().orElse(Collections.emptyList());
+                currentDataFrameRowsIndex = 0;
+            }
+        }
+
+        private Optional<List<DataFrameDataExtractor.Row>> getNextDataRowsBatch() {
+            try {
+                return dataExtractor.next();
+            } catch (IOException e) {
+                throw ExceptionsHelper.serverError("error reading next batch of data frame rows [" + e.getMessage() + "]");
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcess.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcess.java
new file mode 100644
index 0000000000000..22c96d648b4a5
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcess.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import org.elasticsearch.xpack.ml.process.AbstractNativeProcess;
+import org.elasticsearch.xpack.ml.process.NativeController;
+import org.elasticsearch.xpack.ml.process.ProcessResultsParser;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Path;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Consumer;
+
+public class NativeAnalyticsProcess extends AbstractNativeProcess implements AnalyticsProcess {
+
+    private static final String NAME = "analytics";
+
+    private final ProcessResultsParser<AnalyticsResult> resultsParser = new ProcessResultsParser<>(AnalyticsResult.PARSER);
+
+    protected NativeAnalyticsProcess(String jobId, NativeController nativeController, InputStream logStream, OutputStream processInStream,
+                                     InputStream processOutStream, OutputStream processRestoreStream, int numberOfFields,
+                                     List<Path> filesToDelete, Consumer<String> onProcessCrash) {
+        super(jobId, nativeController, logStream, processInStream, processOutStream, processRestoreStream, numberOfFields, filesToDelete,
+            onProcessCrash);
+    }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
+
+    @Override
+    public void persistState() {
+        // Nothing to persist
+    }
+
+    @Override
+    public void writeEndOfDataMessage() throws IOException {
+        new AnalyticsControlMessageWriter(recordWriter(), numberOfFields()).writeEndOfData();
+    }
+
+    @Override
+    public Iterator<AnalyticsResult> readAnalyticsResults() {
+        return resultsParser.parseResults(processOutStream());
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java
new file mode 100644
index 0000000000000..4f765e68b73f4
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.core.internal.io.IOUtils;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.ml.MachineLearning;
+import org.elasticsearch.xpack.ml.process.NativeController;
+import org.elasticsearch.xpack.ml.process.ProcessPipes;
+import org.elasticsearch.xpack.ml.utils.NamedPipeHelper;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Consumer;
+
+public class NativeAnalyticsProcessFactory implements AnalyticsProcessFactory {
+
+    private static final Logger LOGGER = LogManager.getLogger(NativeAnalyticsProcessFactory.class);
+
+    private static final NamedPipeHelper NAMED_PIPE_HELPER = new NamedPipeHelper();
+
+    private final Environment env;
+    private final NativeController nativeController;
+    private volatile Duration processConnectTimeout;
+
+    public NativeAnalyticsProcessFactory(Environment env, NativeController nativeController, ClusterService clusterService) {
+        this.env = Objects.requireNonNull(env);
+        this.nativeController = Objects.requireNonNull(nativeController);
+        setProcessConnectTimeout(MachineLearning.PROCESS_CONNECT_TIMEOUT.get(env.settings()));
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.PROCESS_CONNECT_TIMEOUT,
+            this::setProcessConnectTimeout);
+    }
+
+    void setProcessConnectTimeout(TimeValue processConnectTimeout) {
+        this.processConnectTimeout = Duration.ofMillis(processConnectTimeout.getMillis());
+    }
+
+    @Override
+    public AnalyticsProcess createAnalyticsProcess(String jobId, AnalyticsProcessConfig analyticsProcessConfig,
+                                                   ExecutorService executorService, Consumer<String> onProcessCrash) {
+        List<Path> filesToDelete = new ArrayList<>();
+        ProcessPipes processPipes = new ProcessPipes(env, NAMED_PIPE_HELPER, AnalyticsBuilder.ANALYTICS, jobId,
+                true, false, true, true, false, false);
+
+        // The extra 2 are for the checksum and the control field
+        int numberOfFields = analyticsProcessConfig.cols() + 2;
+
+        createNativeProcess(jobId, analyticsProcessConfig, filesToDelete, processPipes);
+
+        NativeAnalyticsProcess analyticsProcess = new NativeAnalyticsProcess(jobId, nativeController, processPipes.getLogStream().get(),
+                processPipes.getProcessInStream().get(), processPipes.getProcessOutStream().get(), null, numberOfFields,
+                filesToDelete, onProcessCrash);
+
+        try {
+            analyticsProcess.start(executorService);
+            return analyticsProcess;
+        } catch (EsRejectedExecutionException e) {
+            try {
+                IOUtils.close(analyticsProcess);
+            } catch (IOException ioe) {
+                LOGGER.error("Can't close data frame analytics process", ioe);
+            }
+            throw e;
+        }
+    }
+
+    private void createNativeProcess(String jobId, AnalyticsProcessConfig analyticsProcessConfig, List<Path> filesToDelete,
+                                     ProcessPipes processPipes) {
+        AnalyticsBuilder analyticsBuilder = new AnalyticsBuilder(env, nativeController, processPipes, analyticsProcessConfig,
+                filesToDelete);
+        try {
+            analyticsBuilder.build();
+            processPipes.connectStreams(processConnectTimeout);
+        } catch (IOException e) {
+            String msg = "Failed to launch data frame analytics process for job " + jobId;
+            LOGGER.error(msg);
+            throw ExceptionsHelper.serverError(msg, e);
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/results/RowResults.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/results/RowResults.java
new file mode 100644
index 0000000000000..ba4aebededa2e
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/results/RowResults.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process.results;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+public class RowResults implements ToXContentObject {
+
+    public static final ParseField TYPE = new ParseField("row_results");
+    public static final ParseField CHECKSUM = new ParseField("checksum");
+    public static final ParseField RESULTS = new ParseField("results");
+
+    public static final ConstructingObjectParser<RowResults, Void> PARSER = new ConstructingObjectParser<>(TYPE.getPreferredName(),
+            a -> new RowResults((Integer) a[0], (Map<String, Object>) a[1]));
+
+    static {
+        PARSER.declareInt(ConstructingObjectParser.constructorArg(), CHECKSUM);
+        PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, context) -> p.map(), RESULTS);
+    }
+
+    private final int checksum;
+    private final Map<String, Object> results;
+
+    public RowResults(int checksum, Map<String, Object> results) {
+        this.checksum = Objects.requireNonNull(checksum);
+        this.results = Objects.requireNonNull(results);
+    }
+
+    public int getChecksum() {
+        return checksum;
+    }
+
+    public Map<String, Object> getResults() {
+        return results;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(CHECKSUM.getPreferredName(), checksum);
+        builder.field(RESULTS.getPreferredName(), results);
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+
+        RowResults that = (RowResults) other;
+        return checksum == that.checksum && Objects.equals(results, that.results);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(checksum, results);
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinder.java
index aa88905962638..0eb4bfd4b90d1 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinder.java
@@ -62,12 +62,12 @@ static DelimitedFileStructureFinder makeDelimitedFileStructureFinder(List<String
             }
             columnNames = overriddenColumnNames.toArray(new String[0]);
         } else {
-            // The column names are the header names but with blanks named column1, column2, etc.
+            // The column names are the header names but with dots replaced with underscores and blanks named column1, column2, etc.
             columnNames = new String[header.length];
             for (int i = 0; i < header.length; ++i) {
                 assert header[i] != null;
                 String rawHeader = trimFields ? header[i].trim() : header[i];
-                columnNames[i] = rawHeader.isEmpty() ? "column" + (i + 1) : rawHeader;
+                columnNames[i] = rawHeader.isEmpty() ? "column" + (i + 1) : rawHeader.replace('.', '_');
             }
         }
 
@@ -159,8 +159,7 @@ static DelimitedFileStructureFinder makeDelimitedFileStructureFinder(List<String
 
         SortedMap<String, Object> mappings = mappingsAndFieldStats.v1();
         if (timeField != null) {
-            mappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD,
-                Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "date"));
+            mappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD, FileStructureUtils.DATE_MAPPING_WITHOUT_FORMAT);
         }
 
         if (mappingsAndFieldStats.v2() != null) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinderFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinderFactory.java
index 982a6ff703572..cc15491f2e6ca 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinderFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinderFactory.java
@@ -62,7 +62,8 @@ public boolean canCreateFromSample(List<String> explanation, String sample) {
 
     @Override
     public FileStructureFinder createFromSample(List<String> explanation, String sample, String charsetName, Boolean hasByteOrderMarker,
-                                                FileStructureOverrides overrides, TimeoutChecker timeoutChecker) throws IOException {
+                                                int lineMergeSizeLimit, FileStructureOverrides overrides, TimeoutChecker timeoutChecker)
+        throws IOException {
         return DelimitedFileStructureFinder.makeDelimitedFileStructureFinder(explanation, sample, charsetName, hasByteOrderMarker,
             csvPreference, trimFields, overrides, timeoutChecker);
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FieldStatsCalculator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FieldStatsCalculator.java
index 40dff9116d7ab..39bf613165e2e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FieldStatsCalculator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FieldStatsCalculator.java
@@ -5,10 +5,15 @@
  */
 package org.elasticsearch.xpack.ml.filestructurefinder;
 
+import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.time.DateFormatters;
+import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.xpack.core.ml.filestructurefinder.FieldStats;
 
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -26,8 +31,52 @@
 public class FieldStatsCalculator {
 
     private long count;
-    private SortedMap<String, Integer> countsByStringValue = new TreeMap<>();
-    private SortedMap<Double, Integer> countsByNumericValue = new TreeMap<>();
+    private SortedMap<String, Integer> countsByStringValue;
+    private SortedMap<Double, Integer> countsByNumericValue;
+    private DateFormatter dateFormatter;
+    /**
+     * Parsed earliest and latest times.  Some date formats may cause these to be
+     * wrong due to lack of information.  For example, if the date format does not
+     * contain a year then these will be in 1970, and if there's no timezone in
+     * the format then these will be on the assumption the time was in UTC.  However,
+     * since all the timestamps will be inaccurate in the same way the determination
+     * of the earliest and latest will still be correct.  The trick then is to never
+     * print them out...
+     */
+    private Instant earliestTimestamp;
+    private Instant latestTimestamp;
+    /**
+     * Earliest and latest times in the exact form they were present in the input,
+     * making the output immune to issues like not knowing the correct timezone
+     * or year when parsing.
+     */
+    private String earliestTimeString;
+    private String latestTimeString;
+
+    public FieldStatsCalculator(Map<String, String> mapping) {
+
+        switch (mapping.get(FileStructureUtils.MAPPING_TYPE_SETTING)) {
+            case "byte":
+            case "short":
+            case "integer":
+            case "long":
+            case "half_float":
+            case "float":
+            case "double":
+                countsByNumericValue = new TreeMap<>();
+                break;
+            case "date":
+            case "date_nanos":
+                String format = mapping.get(FileStructureUtils.MAPPING_FORMAT_SETTING);
+                dateFormatter = (format == null) ? DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER : DateFormatter.forPattern(format);
+                // Dates are treated like strings for top hits
+                countsByStringValue = new TreeMap<>();
+                break;
+            default:
+                countsByStringValue = new TreeMap<>();
+                break;
+        }
+    }
 
     /**
      * Add a collection of values to the calculator.
@@ -41,14 +90,27 @@ public void accept(Collection<String> fieldValues) {
 
         for (String fieldValue : fieldValues) {
 
-            countsByStringValue.compute(fieldValue, (k, v) -> (v == null) ? 1 : (1 + v));
-
             if (countsByNumericValue != null) {
-
                 try {
                     countsByNumericValue.compute(Double.valueOf(fieldValue), (k, v) -> (v == null) ? 1 : (1 + v));
                 } catch (NumberFormatException e) {
-                    countsByNumericValue = null;
+                    // This should not happen in the usual context this class is used in within the file structure finder,
+                    // as "double" should be big enough to hold any value that the file structure finder considers numeric
+                    throw new IllegalArgumentException("Field with numeric mapping [" + fieldValue + "] could not be parsed as type double",
+                        e);
+                }
+            } else {
+                countsByStringValue.compute(fieldValue, (k, v) -> (v == null) ? 1 : (1 + v));
+                if (dateFormatter != null) {
+                    Instant parsedTimestamp = DateFormatters.from(dateFormatter.parse(fieldValue)).toInstant();
+                    if (earliestTimestamp == null || earliestTimestamp.isAfter(parsedTimestamp)) {
+                        earliestTimestamp = parsedTimestamp;
+                        earliestTimeString = fieldValue;
+                    }
+                    if (latestTimestamp == null || latestTimestamp.isBefore(parsedTimestamp)) {
+                        latestTimestamp = parsedTimestamp;
+                        latestTimeString = fieldValue;
+                    }
                 }
             }
         }
@@ -61,11 +123,17 @@ public void accept(Collection<String> fieldValues) {
      */
     public FieldStats calculate(int numTopHits) {
 
-        if (countsByNumericValue != null && countsByNumericValue.isEmpty() == false) {
-            return new FieldStats(count, countsByNumericValue.size(), countsByNumericValue.firstKey(), countsByNumericValue.lastKey(),
-                calculateMean(), calculateMedian(), findNumericTopHits(numTopHits));
+        if (countsByNumericValue != null) {
+            if (countsByNumericValue.isEmpty()) {
+                assert count == 0;
+                return new FieldStats(count, 0, Collections.emptyList());
+            } else {
+                assert count > 0;
+                return new FieldStats(count, countsByNumericValue.size(), countsByNumericValue.firstKey(), countsByNumericValue.lastKey(),
+                    calculateMean(), calculateMedian(), findNumericTopHits(numTopHits));
+            }
         } else {
-            return new FieldStats(count, countsByStringValue.size(), findStringTopHits(numTopHits));
+            return new FieldStats(count, countsByStringValue.size(), earliestTimeString, latestTimeString, findStringTopHits(numTopHits));
         }
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureFinderFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureFinderFactory.java
index 8790b8f526864..45edf96ce564f 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureFinderFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureFinderFactory.java
@@ -37,6 +37,7 @@ public interface FileStructureFinderFactory {
      * @param sample A sample from the file to be ingested.
      * @param charsetName The name of the character set in which the sample was provided.
      * @param hasByteOrderMarker Did the sample have a byte order marker?  <code>null</code> means "not relevant".
+     * @param lineMergeSizeLimit Maximum number of characters permitted when lines are merged to create messages.
      * @param overrides Stores structure decisions that have been made by the end user, and should
      *                  take precedence over anything the {@link FileStructureFinder} may decide.
      * @param timeoutChecker Will abort the operation if its timeout is exceeded.
@@ -44,5 +45,6 @@ public interface FileStructureFinderFactory {
      * @throws Exception if something goes wrong during creation.
      */
     FileStructureFinder createFromSample(List<String> explanation, String sample, String charsetName, Boolean hasByteOrderMarker,
-                                         FileStructureOverrides overrides, TimeoutChecker timeoutChecker) throws Exception;
+                                         int lineMergeSizeLimit, FileStructureOverrides overrides,
+                                         TimeoutChecker timeoutChecker) throws Exception;
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureFinderManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureFinderManager.java
index 1a0e41c5e135e..32a2d7244d8ae 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureFinderManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureFinderManager.java
@@ -42,6 +42,7 @@ public final class FileStructureFinderManager {
 
     public static final int MIN_SAMPLE_LINE_COUNT = 2;
     public static final int DEFAULT_IDEAL_SAMPLE_LINE_COUNT = 1000;
+    public static final int DEFAULT_LINE_MERGE_SIZE_LIMIT = 10000;
 
     static final Set<String> FILEBEAT_SUPPORTED_ENCODINGS = Set.of(
             "866",
@@ -294,8 +295,9 @@ public FileStructureFinderManager(ScheduledExecutorService scheduler) {
         this.scheduler = Objects.requireNonNull(scheduler);
     }
 
-    public FileStructureFinder findFileStructure(Integer idealSampleLineCount, InputStream fromFile) throws Exception {
-        return findFileStructure(idealSampleLineCount, fromFile, FileStructureOverrides.EMPTY_OVERRIDES, null);
+    public FileStructureFinder findFileStructure(Integer idealSampleLineCount, Integer lineMergeSizeLimit,
+                                                 InputStream fromFile) throws Exception {
+        return findFileStructure(idealSampleLineCount, lineMergeSizeLimit, fromFile, FileStructureOverrides.EMPTY_OVERRIDES, null);
     }
 
     /**
@@ -304,6 +306,8 @@ public FileStructureFinder findFileStructure(Integer idealSampleLineCount, Input
      *                             If the stream has fewer lines then an attempt will still be made, providing at
      *                             least {@link #MIN_SAMPLE_LINE_COUNT} lines can be read.  If <code>null</code>
      *                             the value of {@link #DEFAULT_IDEAL_SAMPLE_LINE_COUNT} will be used.
+     * @param lineMergeSizeLimit Maximum number of characters permitted when lines are merged to create messages.
+     *                           If <code>null</code> the value of {@link #DEFAULT_LINE_MERGE_SIZE_LIMIT} will be used.
      * @param fromFile A stream from which the sample will be read.
      * @param overrides Aspects of the file structure that are known in advance.  These take precedence over
      *                  values determined by structure analysis.  An exception will be thrown if the file structure
@@ -314,20 +318,21 @@ public FileStructureFinder findFileStructure(Integer idealSampleLineCount, Input
      * @return A {@link FileStructureFinder} object from which the structure and messages can be queried.
      * @throws Exception A variety of problems could occur at various stages of the structure finding process.
      */
-    public FileStructureFinder findFileStructure(Integer idealSampleLineCount, InputStream fromFile, FileStructureOverrides overrides,
-                                                 TimeValue timeout)
-        throws Exception {
+    public FileStructureFinder findFileStructure(Integer idealSampleLineCount, Integer lineMergeSizeLimit, InputStream fromFile,
+                                                 FileStructureOverrides overrides, TimeValue timeout) throws Exception {
         return findFileStructure(new ArrayList<>(), (idealSampleLineCount == null) ? DEFAULT_IDEAL_SAMPLE_LINE_COUNT : idealSampleLineCount,
-            fromFile, overrides, timeout);
+            (lineMergeSizeLimit == null) ? DEFAULT_LINE_MERGE_SIZE_LIMIT : lineMergeSizeLimit, fromFile, overrides, timeout);
     }
 
-    public FileStructureFinder findFileStructure(List<String> explanation, int idealSampleLineCount, InputStream fromFile)
-        throws Exception {
-        return findFileStructure(explanation, idealSampleLineCount, fromFile, FileStructureOverrides.EMPTY_OVERRIDES, null);
+    public FileStructureFinder findFileStructure(List<String> explanation, int idealSampleLineCount, int lineMergeSizeLimit,
+                                                 InputStream fromFile) throws Exception {
+        return findFileStructure(explanation, idealSampleLineCount, lineMergeSizeLimit, fromFile, FileStructureOverrides.EMPTY_OVERRIDES,
+            null);
     }
 
-    public FileStructureFinder findFileStructure(List<String> explanation, int idealSampleLineCount, InputStream fromFile,
-                                                 FileStructureOverrides overrides, TimeValue timeout) throws Exception {
+    public FileStructureFinder findFileStructure(List<String> explanation, int idealSampleLineCount, int lineMergeSizeLimit,
+                                                 InputStream fromFile, FileStructureOverrides overrides,
+                                                 TimeValue timeout) throws Exception {
 
         try (TimeoutChecker timeoutChecker = new TimeoutChecker("structure analysis", timeout, scheduler)) {
 
@@ -346,7 +351,8 @@ public FileStructureFinder findFileStructure(List<String> explanation, int ideal
             Tuple<String, Boolean> sampleInfo = sampleFile(sampleReader, charsetName, MIN_SAMPLE_LINE_COUNT,
                 Math.max(MIN_SAMPLE_LINE_COUNT, idealSampleLineCount), timeoutChecker);
 
-            return makeBestStructureFinder(explanation, sampleInfo.v1(), charsetName, sampleInfo.v2(), overrides, timeoutChecker);
+            return makeBestStructureFinder(explanation, sampleInfo.v1(), charsetName, sampleInfo.v2(), lineMergeSizeLimit, overrides,
+                timeoutChecker);
         } catch (Exception e) {
             // Add a dummy exception containing the explanation so far - this can be invaluable for troubleshooting as incorrect
             // decisions made early on in the structure analysis can result in seemingly crazy decisions or timeouts later on
@@ -373,7 +379,8 @@ CharsetMatch findCharset(List<String> explanation, InputStream inputStream, Time
 
         // Determine some extra characteristics of the input to compensate for some deficiencies of ICU4J
         boolean pureAscii = true;
-        boolean containsZeroBytes = false;
+        int evenPosZeroCount = 0;
+        int oddPosZeroCount = 0;
         inputStream.mark(BUFFER_SIZE);
         byte[] workspace = new byte[BUFFER_SIZE];
         int remainingLength = BUFFER_SIZE;
@@ -382,17 +389,22 @@ CharsetMatch findCharset(List<String> explanation, InputStream inputStream, Time
             if (bytesRead <= 0) {
                 break;
             }
-            for (int i = 0; i < bytesRead && containsZeroBytes == false; ++i) {
+            for (int i = 0; i < bytesRead; ++i) {
                 if (workspace[i] == 0) {
-                    containsZeroBytes = true;
                     pureAscii = false;
+                    if (i % 2 == 0) {
+                        ++evenPosZeroCount;
+                    } else {
+                        ++oddPosZeroCount;
+                    }
                 } else {
                     pureAscii = pureAscii && workspace[i] > 0 && workspace[i] < 128;
                 }
             }
             remainingLength -= bytesRead;
-        } while (containsZeroBytes == false && remainingLength > 0);
+        } while (remainingLength > 0);
         inputStream.reset();
+        boolean containsZeroBytes = evenPosZeroCount > 0 || oddPosZeroCount > 0;
         timeoutChecker.check("character set detection");
 
         if (pureAscii) {
@@ -433,6 +445,11 @@ CharsetMatch findCharset(List<String> explanation, InputStream inputStream, Time
                 if (containsZeroBytes && spaceEncodingContainsZeroByte == false) {
                     explanation.add("Character encoding [" + name + "] matched the input with [" + charsetMatch.getConfidence() +
                         "%] confidence but was rejected as the input contains zero bytes and the [" + name + "] encoding does not");
+                } else if (containsZeroBytes && 3 * oddPosZeroCount > 2 * evenPosZeroCount && 3 * evenPosZeroCount > 2 * oddPosZeroCount) {
+                    explanation.add("Character encoding [" + name + "] matched the input with [" + charsetMatch.getConfidence() +
+                        "%] confidence but was rejected as the distribution of zero bytes between odd and even positions in the " +
+                        "file is very close - [" + evenPosZeroCount + "] and [" + oddPosZeroCount + "] in the first [" +
+                        (BUFFER_SIZE / 1024) + "kB] of input");
                 } else {
                     explanation.add("Using character encoding [" + name + "], which matched the input with [" +
                         charsetMatch.getConfidence() + "%] confidence");
@@ -450,7 +467,8 @@ CharsetMatch findCharset(List<String> explanation, InputStream inputStream, Time
     }
 
     FileStructureFinder makeBestStructureFinder(List<String> explanation, String sample, String charsetName, Boolean hasByteOrderMarker,
-                                                FileStructureOverrides overrides, TimeoutChecker timeoutChecker) throws Exception {
+                                                int lineMergeSizeLimit, FileStructureOverrides overrides,
+                                                TimeoutChecker timeoutChecker) throws Exception {
 
         Character delimiter = overrides.getDelimiter();
         Character quote = overrides.getQuote();
@@ -482,7 +500,8 @@ FileStructureFinder makeBestStructureFinder(List<String> explanation, String sam
         for (FileStructureFinderFactory factory : factories) {
             timeoutChecker.check("high level format detection");
             if (factory.canCreateFromSample(explanation, sample)) {
-                return factory.createFromSample(explanation, sample, charsetName, hasByteOrderMarker, overrides, timeoutChecker);
+                return factory.createFromSample(explanation, sample, charsetName, hasByteOrderMarker, lineMergeSizeLimit, overrides,
+                    timeoutChecker);
             }
         }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtils.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtils.java
index 90cc74c8d259c..e4945d3709860 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtils.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtils.java
@@ -29,6 +29,8 @@ public final class FileStructureUtils {
     public static final String MAPPING_TYPE_SETTING = "type";
     public static final String MAPPING_FORMAT_SETTING = "format";
     public static final String MAPPING_PROPERTIES_SETTING = "properties";
+    public static final Map<String, String> DATE_MAPPING_WITHOUT_FORMAT =
+        Collections.singletonMap(MAPPING_TYPE_SETTING, "date");
 
     private static final int NUM_TOP_HITS = 10;
     // NUMBER Grok pattern doesn't support scientific notation, so we extend it
@@ -231,7 +233,7 @@ static Tuple<Map<String, String>, FieldStats> guessMappingAndCalculateFieldStats
         Collection<String> fieldValuesAsStrings = fieldValues.stream().map(Object::toString).collect(Collectors.toList());
         Map<String, String> mapping = guessScalarMapping(explanation, fieldName, fieldValuesAsStrings, timeoutChecker);
         timeoutChecker.check("mapping determination");
-        return new Tuple<>(mapping, calculateFieldStats(fieldValuesAsStrings, timeoutChecker));
+        return new Tuple<>(mapping, calculateFieldStats(mapping, fieldValuesAsStrings, timeoutChecker));
     }
 
     private static Stream<Object> flatten(Object value) {
@@ -323,13 +325,14 @@ else if (fieldValues.stream().allMatch(IP_GROK::match)) {
 
     /**
      * Calculate stats for a set of field values.
+     * @param mapping The  mapping for the field.
      * @param fieldValues Values of the field for which field stats are to be calculated.
      * @param timeoutChecker Will abort the operation if its timeout is exceeded.
      * @return The stats calculated from the field values.
      */
-    static FieldStats calculateFieldStats(Collection<String> fieldValues, TimeoutChecker timeoutChecker) {
+    static FieldStats calculateFieldStats(Map<String, String> mapping, Collection<String> fieldValues, TimeoutChecker timeoutChecker) {
 
-        FieldStatsCalculator calculator = new FieldStatsCalculator();
+        FieldStatsCalculator calculator = new FieldStatsCalculator(mapping);
         calculator.accept(fieldValues);
         timeoutChecker.check("field stats calculation");
         return calculator.calculate(NUM_TOP_HITS);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/GrokPatternCreator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/GrokPatternCreator.java
index 7a5c9a48f8757..cb2bfeb00f268 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/GrokPatternCreator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/GrokPatternCreator.java
@@ -195,14 +195,16 @@ public void validateFullLineGrokPattern(String grokPattern, String timestampFiel
     /**
      * Build a Grok pattern that will match all of the sample messages in their entirety.
      * @param seedPatternName A pattern that has already been determined to match some portion of every sample message.
-     * @param seedFieldName The field name to be used for the portion of every sample message that the seed pattern matches.
+     * @param seedMapping     The mapping for the seed field.
+     * @param seedFieldName   The field name to be used for the portion of every sample message that the seed pattern matches.
      * @return The built Grok pattern.
      */
-    public String createGrokPatternFromExamples(String seedPatternName, String seedFieldName) {
+    public String createGrokPatternFromExamples(String seedPatternName, Map<String, String> seedMapping, String seedFieldName) {
 
         overallGrokPatternBuilder.setLength(0);
 
-        GrokPatternCandidate seedCandidate = new NoMappingGrokPatternCandidate(seedPatternName, seedFieldName, grokPatternDefinitions);
+        GrokPatternCandidate seedCandidate = new PrecalculatedMappingGrokPatternCandidate(seedPatternName, seedMapping, seedFieldName,
+            grokPatternDefinitions);
 
         processCandidateAndSplit(seedCandidate, true, sampleMessages, false, 0, false, 0);
 
@@ -433,7 +435,7 @@ String processCaptures(List<String> explanation, Map<String, Integer> fieldNameC
     static class ValueOnlyGrokPatternCandidate implements GrokPatternCandidate {
 
         private final String grokPatternName;
-        private final String mappingType;
+        private final Map<String, String> mapping;
         private final String fieldName;
         private final Grok grok;
 
@@ -451,7 +453,8 @@ static class ValueOnlyGrokPatternCandidate implements GrokPatternCandidate {
          * @param fieldName       Name of the field to extract from the match.
          */
         ValueOnlyGrokPatternCandidate(String grokPatternName, String mappingType, String fieldName) {
-            this(grokPatternName, mappingType, fieldName, "\\b", "\\b", Grok.getBuiltinPatterns());
+            this(grokPatternName, Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, mappingType), fieldName,
+                "\\b", "\\b", Grok.getBuiltinPatterns());
         }
 
         /**
@@ -462,7 +465,8 @@ static class ValueOnlyGrokPatternCandidate implements GrokPatternCandidate {
          */
         ValueOnlyGrokPatternCandidate(String grokPatternName, String mappingType, String fieldName,
                                       Map<String, String> grokPatternDefinitions) {
-            this(grokPatternName, mappingType, fieldName, "\\b", "\\b", grokPatternDefinitions);
+            this(grokPatternName, Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, mappingType), fieldName,
+                "\\b", "\\b", grokPatternDefinitions);
         }
 
         /**
@@ -473,25 +477,28 @@ static class ValueOnlyGrokPatternCandidate implements GrokPatternCandidate {
          * @param postBreak       Only consider the match if it's broken from the following text by this.
          */
         ValueOnlyGrokPatternCandidate(String grokPatternName, String mappingType, String fieldName, String preBreak, String postBreak) {
-            this(grokPatternName, mappingType, fieldName, preBreak, postBreak, Grok.getBuiltinPatterns());
+            this(grokPatternName, Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, mappingType), fieldName,
+                preBreak, postBreak, Grok.getBuiltinPatterns());
         }
 
         /**
          * @param grokPatternName        Name of the Grok pattern to try to match - must match one defined in Logstash.
-         * @param mappingType            Data type for field in Elasticsearch mappings.
+         * @param mapping                Elasticsearch mapping for the field.
          * @param fieldName              Name of the field to extract from the match.
          * @param preBreak               Only consider the match if it's broken from the previous text by this.
          * @param postBreak              Only consider the match if it's broken from the following text by this.
          * @param grokPatternDefinitions Definitions of Grok patterns to be used.
          */
-        ValueOnlyGrokPatternCandidate(String grokPatternName, String mappingType, String fieldName, String preBreak, String postBreak,
-                                      Map<String, String> grokPatternDefinitions) {
-            this.grokPatternName = grokPatternName;
-            this.mappingType = mappingType;
-            this.fieldName = fieldName;
+        ValueOnlyGrokPatternCandidate(String grokPatternName, Map<String, String> mapping, String fieldName, String preBreak,
+                                      String postBreak, Map<String, String> grokPatternDefinitions) {
+            this.grokPatternName = Objects.requireNonNull(grokPatternName);
+            this.mapping = Collections.unmodifiableMap(mapping);
+            this.fieldName = Objects.requireNonNull(fieldName);
             // The (?m) here has the Ruby meaning, which is equivalent to (?s) in Java
-            grok = new Grok(grokPatternDefinitions, "(?m)%{DATA:" + PREFACE + "}" + preBreak +
-                "%{" + grokPatternName + ":" + VALUE + "}" + postBreak + "%{GREEDYDATA:" + EPILOGUE + "}", TimeoutChecker.watchdog);
+            grok = new Grok(grokPatternDefinitions,
+                "(?m)%{DATA:" + PREFACE + "}" + Objects.requireNonNull(preBreak) +
+                    "%{" + grokPatternName + ":" + VALUE + "}" + Objects.requireNonNull(postBreak) + "%{GREEDYDATA:" + EPILOGUE + "}",
+                TimeoutChecker.watchdog);
         }
 
         @Override
@@ -520,23 +527,24 @@ public String processCaptures(List<String> explanation, Map<String, Integer> fie
                 epilogues.add(captures.getOrDefault(EPILOGUE, "").toString());
             }
             String adjustedFieldName = buildFieldName(fieldNameCountStore, fieldName);
-            if (mappings != null) {
-                Map<String, String> fullMappingType = Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, mappingType);
-                if ("date".equals(mappingType)) {
-                    try {
-                        fullMappingType = FileStructureUtils.findTimestampMapping(explanation, values, timeoutChecker);
-                    } catch (IllegalArgumentException e) {
-                        // This feels like it shouldn't happen, but there may be some obscure edge case
-                        // where it does, and in production it will cause less frustration to just return
-                        // a mapping type of "date" with no format than to fail the whole analysis
-                        assert e == null : e.getMessage();
-                    }
-                    timeoutChecker.check("mapping determination");
+            Map<String, String> adjustedMapping = mapping;
+            // If the mapping is type "date" with no format, try to adjust it to include the format
+            if (FileStructureUtils.DATE_MAPPING_WITHOUT_FORMAT.equals(adjustedMapping)) {
+                try {
+                    adjustedMapping = FileStructureUtils.findTimestampMapping(explanation, values, timeoutChecker);
+                } catch (IllegalArgumentException e) {
+                    // This feels like it shouldn't happen, but there may be some obscure edge case
+                    // where it does, and in production it will cause less frustration to just return
+                    // a mapping type of "date" with no format than to fail the whole analysis
+                    assert e == null : e.getMessage();
                 }
-                mappings.put(adjustedFieldName, fullMappingType);
+                timeoutChecker.check("mapping determination");
+            }
+            if (mappings != null) {
+                mappings.put(adjustedFieldName, adjustedMapping);
             }
             if (fieldStats != null) {
-                fieldStats.put(adjustedFieldName, FileStructureUtils.calculateFieldStats(values, timeoutChecker));
+                fieldStats.put(adjustedFieldName, FileStructureUtils.calculateFieldStats(adjustedMapping, values, timeoutChecker));
             }
             return "%{" + grokPatternName + ":" + adjustedFieldName + "}";
         }
@@ -598,13 +606,13 @@ public String processCaptures(List<String> explanation, Map<String, Integer> fie
                 timeoutChecker.check("full message Grok pattern field extraction");
             }
             String adjustedFieldName = buildFieldName(fieldNameCountStore, fieldName);
+            Map<String, String> mapping = FileStructureUtils.guessScalarMapping(explanation, adjustedFieldName, values, timeoutChecker);
+            timeoutChecker.check("mapping determination");
             if (mappings != null) {
-                mappings.put(adjustedFieldName,
-                    FileStructureUtils.guessScalarMapping(explanation, adjustedFieldName, values, timeoutChecker));
-                timeoutChecker.check("mapping determination");
+                mappings.put(adjustedFieldName, mapping);
             }
             if (fieldStats != null) {
-                fieldStats.put(adjustedFieldName, FileStructureUtils.calculateFieldStats(values, timeoutChecker));
+                fieldStats.put(adjustedFieldName, FileStructureUtils.calculateFieldStats(mapping, values, timeoutChecker));
             }
             return "\\b" + fieldName + "=%{USER:" + adjustedFieldName + "}";
         }
@@ -613,10 +621,11 @@ public String processCaptures(List<String> explanation, Map<String, Integer> fie
     /**
      * A Grok pattern candidate that matches a single named Grok pattern but will not update mappings.
      */
-    static class NoMappingGrokPatternCandidate extends ValueOnlyGrokPatternCandidate {
+    static class PrecalculatedMappingGrokPatternCandidate extends ValueOnlyGrokPatternCandidate {
 
-        NoMappingGrokPatternCandidate(String grokPatternName, String fieldName, Map<String, String> grokPatternDefinitions) {
-            super(grokPatternName, null, fieldName, grokPatternDefinitions);
+        PrecalculatedMappingGrokPatternCandidate(String grokPatternName, Map<String, String> mapping, String fieldName,
+                                                 Map<String, String> grokPatternDefinitions) {
+            super(grokPatternName, mapping, fieldName, "\\b", "\\b", grokPatternDefinitions);
         }
 
         @Override
@@ -710,16 +719,16 @@ public Tuple<String, String> processMatch(List<String> explanation, Collection<S
 
                 for (Map.Entry<String, Collection<String>> valuesForField : valuesPerField.entrySet()) {
                     String fieldName = valuesForField.getKey();
-                    if (mappings != null) {
-                        // Exclude the time field because that will be dropped and replaced with @timestamp
-                        if (fieldName.equals(timeField) == false) {
-                            mappings.put(fieldName,
-                                FileStructureUtils.guessScalarMapping(explanation, fieldName, valuesForField.getValue(), timeoutChecker));
-                            timeoutChecker.check("mapping determination");
-                        }
+                    Map<String, String> mapping =
+                        FileStructureUtils.guessScalarMapping(explanation, fieldName, valuesForField.getValue(), timeoutChecker);
+                    timeoutChecker.check("mapping determination");
+                    // Exclude the time field because that will be dropped and replaced with @timestamp
+                    if (mappings != null && fieldName.equals(timeField) == false) {
+                        mappings.put(fieldName, mapping);
                     }
                     if (fieldStats != null) {
-                        fieldStats.put(fieldName, FileStructureUtils.calculateFieldStats(valuesForField.getValue(), timeoutChecker));
+                        fieldStats.put(fieldName,
+                            FileStructureUtils.calculateFieldStats(mapping, valuesForField.getValue(), timeoutChecker));
                     }
                 }
             }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/NdJsonFileStructureFinder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/NdJsonFileStructureFinder.java
index 116de8f7679d2..1b405eb685fa2 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/NdJsonFileStructureFinder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/NdJsonFileStructureFinder.java
@@ -70,8 +70,7 @@ static NdJsonFileStructureFinder makeNdJsonFileStructureFinder(List<String> expl
 
         SortedMap<String, Object> mappings = mappingsAndFieldStats.v1();
         if (timeField != null) {
-            mappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD,
-                Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "date"));
+            mappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD, FileStructureUtils.DATE_MAPPING_WITHOUT_FORMAT);
         }
 
         if (mappingsAndFieldStats.v2() != null) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/NdJsonFileStructureFinderFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/NdJsonFileStructureFinderFactory.java
index 43612890bc8a8..6970af01bb79d 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/NdJsonFileStructureFinderFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/NdJsonFileStructureFinderFactory.java
@@ -68,7 +68,8 @@ DeprecationHandler.THROW_UNSUPPORTED_OPERATION, new ContextPrintingStringReader(
 
     @Override
     public FileStructureFinder createFromSample(List<String> explanation, String sample, String charsetName, Boolean hasByteOrderMarker,
-                                                FileStructureOverrides overrides, TimeoutChecker timeoutChecker) throws IOException {
+                                                int lineMergeSizeLimit, FileStructureOverrides overrides, TimeoutChecker timeoutChecker)
+        throws IOException {
         return NdJsonFileStructureFinder.makeNdJsonFileStructureFinder(explanation, sample, charsetName, hasByteOrderMarker, overrides,
             timeoutChecker);
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinder.java
index d07eea15f973f..e47d045dd257e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinder.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.ml.filestructurefinder;
 
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.xpack.core.ml.action.FindFileStructureAction;
 import org.elasticsearch.xpack.core.ml.filestructurefinder.FieldStats;
 import org.elasticsearch.xpack.core.ml.filestructurefinder.FileStructure;
 
@@ -24,8 +25,8 @@ public class TextLogFileStructureFinder implements FileStructureFinder {
     private final FileStructure structure;
 
     static TextLogFileStructureFinder makeTextLogFileStructureFinder(List<String> explanation, String sample, String charsetName,
-                                                                     Boolean hasByteOrderMarker, FileStructureOverrides overrides,
-                                                                     TimeoutChecker timeoutChecker) {
+                                                                     Boolean hasByteOrderMarker, int lineMergeSizeLimit,
+                                                                     FileStructureOverrides overrides, TimeoutChecker timeoutChecker) {
         String[] sampleLines = sample.split("\n");
         TimestampFormatFinder timestampFormatFinder = populateTimestampFormatFinder(explanation, sampleLines, overrides, timeoutChecker);
         switch (timestampFormatFinder.getNumMatchedFormats()) {
@@ -69,6 +70,16 @@ static TextLogFileStructureFinder makeTextLogFileStructureFinder(List<String> ex
                     // for the CSV header or lines before the first XML document starts)
                     ++linesConsumed;
                 } else {
+                    // This check avoids subsequent problems when a massive message is unwieldy and slow to process
+                    long lengthAfterAppend = message.length() + 1L + sampleLine.length();
+                    if (lengthAfterAppend > lineMergeSizeLimit) {
+                        assert linesInMessage > 0;
+                        throw new IllegalArgumentException("Merging lines into messages resulted in an unacceptably long message. "
+                            + "Merged message would have [" + (linesInMessage + 1) + "] lines and [" + lengthAfterAppend + "] "
+                            + "characters (limit [" + lineMergeSizeLimit + "]). If you have messages this big please increase "
+                            + "the value of [" + FindFileStructureAction.Request.LINE_MERGE_SIZE_LIMIT + "]. Otherwise it "
+                            + "probably means the timestamp has been incorrectly detected, so try overriding that.");
+                    }
                     message.append('\n').append(sampleLine);
                     ++linesInMessage;
                 }
@@ -97,12 +108,13 @@ static TextLogFileStructureFinder makeTextLogFileStructureFinder(List<String> ex
             .setNumMessagesAnalyzed(sampleMessages.size())
             .setMultilineStartPattern(multiLineRegex);
 
+        Map<String, String> messageMapping = Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "text");
         SortedMap<String, Object> mappings = new TreeMap<>();
-        mappings.put("message", Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "text"));
-        mappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD, Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "date"));
+        mappings.put("message", messageMapping);
+        mappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD, FileStructureUtils.DATE_MAPPING_WITHOUT_FORMAT);
 
         SortedMap<String, FieldStats> fieldStats = new TreeMap<>();
-        fieldStats.put("message", FileStructureUtils.calculateFieldStats(sampleMessages, timeoutChecker));
+        fieldStats.put("message", FileStructureUtils.calculateFieldStats(messageMapping, sampleMessages, timeoutChecker));
 
         Map<String, String> customGrokPatternDefinitions = timestampFormatFinder.getCustomGrokPatternDefinitions();
         GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, sampleMessages, mappings, fieldStats,
@@ -125,8 +137,8 @@ static TextLogFileStructureFinder makeTextLogFileStructureFinder(List<String> ex
                 if (interimTimestampField == null) {
                     interimTimestampField = "timestamp";
                 }
-                grokPattern =
-                    grokPatternCreator.createGrokPatternFromExamples(timestampFormatFinder.getGrokPatternName(), interimTimestampField);
+                grokPattern = grokPatternCreator.createGrokPatternFromExamples(timestampFormatFinder.getGrokPatternName(),
+                    timestampFormatFinder.getEsDateMappingTypeWithFormat(), interimTimestampField);
             }
         }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinderFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinderFactory.java
index 5931fea5f1abf..2980d5d0678ca 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinderFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinderFactory.java
@@ -41,8 +41,8 @@ public boolean canCreateFromSample(List<String> explanation, String sample) {
 
     @Override
     public FileStructureFinder createFromSample(List<String> explanation, String sample, String charsetName, Boolean hasByteOrderMarker,
-                                                FileStructureOverrides overrides, TimeoutChecker timeoutChecker) {
+                                                int lineMergeSizeLimit, FileStructureOverrides overrides, TimeoutChecker timeoutChecker) {
         return TextLogFileStructureFinder.makeTextLogFileStructureFinder(explanation, sample, charsetName, hasByteOrderMarker,
-            overrides, timeoutChecker);
+            lineMergeSizeLimit, overrides, timeoutChecker);
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimeoutChecker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimeoutChecker.java
index f8a9368b8425e..97eb6146a6654 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimeoutChecker.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimeoutChecker.java
@@ -44,6 +44,7 @@ public class TimeoutChecker implements Closeable {
     private final TimeValue timeout;
     private final Thread checkedThread;
     private final ScheduledFuture<?> future;
+    private boolean isClosed; // only accessed within synchronized methods
     private volatile boolean timeoutExceeded;
 
     /**
@@ -66,9 +67,13 @@ public TimeoutChecker(String operation, TimeValue timeout, ScheduledExecutorServ
      * Stops the timer if running.
      */
     @Override
-    public void close() {
+    public synchronized void close() {
+        if (isClosed) {
+            return;
+        }
         FutureUtils.cancel(future);
         timeoutCheckerWatchdog.remove(checkedThread);
+        isClosed = true;
     }
 
     /**
@@ -104,7 +109,12 @@ public Map<String, Object> grokCaptures(Grok grok, String text, String where) {
         }
     }
 
-    private void setTimeoutExceeded() {
+    private synchronized void setTimeoutExceeded() {
+        // Even though close() cancels the timer, it's possible that it can already be running when close()
+        // is called, so this check prevents the effects of this method occurring after close() returns
+        if (isClosed) {
+            return;
+        }
         timeoutExceeded = true;
         timeoutCheckerWatchdog.interruptLongRunningThreadIfRegistered(checkedThread);
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinder.java
index 0283437d64808..9dec0ddbf5ba0 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinder.java
@@ -97,10 +97,10 @@ public final class TimestampFormatFinder {
             "\\b\\d{4}-\\d{2}-\\d{2}[T ]\\d{2}:\\d{2}", "\\b%{TIMESTAMP_ISO8601}\\b", "TIMESTAMP_ISO8601",
             "1111 11 11 11 11", 0, 19);
     static final CandidateTimestampFormat UNIX_MS_CANDIDATE_FORMAT =
-        new CandidateTimestampFormat(example -> Collections.singletonList("UNIX_MS"), "\\b\\d{13}\\b", "\\b\\d{13}\\b", "POSINT",
+        new CandidateTimestampFormat(example -> Collections.singletonList("UNIX_MS"), "\\b\\d{13}\\b", "\\b[12]\\d{12}\\b", "POSINT",
             "1111111111111", 0, 0);
     static final CandidateTimestampFormat UNIX_CANDIDATE_FORMAT =
-        new CandidateTimestampFormat(example -> Collections.singletonList("UNIX"), "\\b\\d{10}\\b", "\\b\\d{10}(?:\\.\\d{3,9})?\\b",
+        new CandidateTimestampFormat(example -> Collections.singletonList("UNIX"), "\\b\\d{10}\\b", "\\b[12]\\d{9}(?:\\.\\d{3,9})?\\b",
             "NUMBER", "1111111111", 0, 10);
     static final CandidateTimestampFormat TAI64N_CANDIDATE_FORMAT =
         new CandidateTimestampFormat(example -> Collections.singletonList("TAI64N"), "\\b[0-9A-Fa-f]{24}\\b", "\\b[0-9A-Fa-f]{24}\\b",
@@ -275,7 +275,7 @@ static Tuple<String, String> overrideFormatToGrokAndRegex(String overrideFormat)
                         "ss".equals(prevLetterGroup) == false || endPos - startPos > 9) {
                         String msg = "Letter group [" + letterGroup + "] in [" + overrideFormat + "] is not supported";
                         if (curChar == 'S') {
-                            msg += " because it is not preceeded by [ss] and a separator from [" + FRACTIONAL_SECOND_SEPARATORS + "]";
+                            msg += " because it is not preceded by [ss] and a separator from [" + FRACTIONAL_SECOND_SEPARATORS + "]";
                         }
                         throw new IllegalArgumentException(msg);
                     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinder.java
index d2572b7fd2085..91fc61bcbd4b4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinder.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinder.java
@@ -120,8 +120,7 @@ static XmlFileStructureFinder makeXmlFileStructureFinder(List<String> explanatio
         SortedMap<String, Object> outerMappings = new TreeMap<>();
         outerMappings.put(topLevelTag, secondLevelProperties);
         if (timeField != null) {
-            outerMappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD,
-                Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "date"));
+            outerMappings.put(FileStructureUtils.DEFAULT_TIMESTAMP_FIELD, FileStructureUtils.DATE_MAPPING_WITHOUT_FORMAT);
         }
 
         FileStructure structure = structureBuilder
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinderFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinderFactory.java
index 97984d1d77560..382f2e7502719 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinderFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinderFactory.java
@@ -125,7 +125,7 @@ public boolean canCreateFromSample(List<String> explanation, String sample) {
 
     @Override
     public FileStructureFinder createFromSample(List<String> explanation, String sample, String charsetName, Boolean hasByteOrderMarker,
-                                                FileStructureOverrides overrides, TimeoutChecker timeoutChecker)
+                                                int lineMergeSizeLimit, FileStructureOverrides overrides, TimeoutChecker timeoutChecker)
         throws IOException, ParserConfigurationException, SAXException {
         return XmlFileStructureFinder.makeXmlFileStructureFinder(explanation, sample, charsetName, hasByteOrderMarker, overrides,
             timeoutChecker);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
index fbd17cb766844..8d43ec9b75df4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
@@ -226,7 +226,7 @@ static void validateCategorizationAnalyzer(Job.Builder jobBuilder, AnalysisRegis
         CategorizationAnalyzerConfig categorizationAnalyzerConfig = jobBuilder.getAnalysisConfig().getCategorizationAnalyzerConfig();
         if (categorizationAnalyzerConfig != null) {
             CategorizationAnalyzer.verifyConfigBuilder(new CategorizationAnalyzerConfig.Builder(categorizationAnalyzerConfig),
-                analysisRegistry, environment);
+                analysisRegistry);
         }
     }
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobNodeSelector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobNodeSelector.java
new file mode 100644
index 0000000000000..aa97c13b21d69
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobNodeSelector.java
@@ -0,0 +1,329 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.job;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.xpack.core.ml.MlTasks;
+import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
+import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
+import org.elasticsearch.xpack.core.ml.job.config.JobState;
+import org.elasticsearch.xpack.ml.MachineLearning;
+import org.elasticsearch.xpack.ml.process.MlMemoryTracker;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
+import static org.elasticsearch.xpack.ml.MachineLearning.MAX_OPEN_JOBS_PER_NODE;
+
+/**
+ * Class that contains the logic to decide which node to assign each job to.
+ *
+ * The assignment rules are as follows:
+ *
+ * 1. Reject nodes that are not ML nodes
+ * 2. Reject nodes for which the node filter returns a rejection reason
+ * 3. Reject nodes where the new job would result in more than the permitted number of concurrent "opening" jobs
+ * 4. Reject nodes where the new job would result in more than the permitted number of assigned jobs
+ * 5. If assigning by memory, reject nodes where the new job would result in the permitted amount of memory being exceeded
+ * 6. If assigning by memory, pick the node that remains after rejections that has the most remaining memory
+ * 7. If assigning by count, pick the node that remains after rejections that has the fewest jobs assigned to it
+ *
+ * The decision on whether to assign by memory or by count is:
+ * - If values are available for every node's memory size and every job's memory requirement then assign by memory
+ * - Otherwise assign by count
+ */
+public class JobNodeSelector {
+
+    public static final PersistentTasksCustomMetaData.Assignment AWAITING_LAZY_ASSIGNMENT =
+        new PersistentTasksCustomMetaData.Assignment(null, "persistent task is awaiting node assignment.");
+
+    private static final Logger logger = LogManager.getLogger(JobNodeSelector.class);
+
+    private final String jobId;
+    private final String taskName;
+    private final ClusterState clusterState;
+    private final MlMemoryTracker memoryTracker;
+    private final Function<DiscoveryNode, String> nodeFilter;
+    private final int maxLazyNodes;
+
+    /**
+     * @param nodeFilter Optionally a function that returns a reason beyond the general
+     *                   reasons why a job cannot be assigned to a particular node.  May
+     *                   be <code>null</code> if no such function is needed.
+     */
+    public JobNodeSelector(ClusterState clusterState, String jobId, String taskName, MlMemoryTracker memoryTracker, int maxLazyNodes,
+                           Function<DiscoveryNode, String> nodeFilter) {
+        this.jobId = Objects.requireNonNull(jobId);
+        this.taskName = Objects.requireNonNull(taskName);
+        this.clusterState = Objects.requireNonNull(clusterState);
+        this.memoryTracker = Objects.requireNonNull(memoryTracker);
+        this.maxLazyNodes = maxLazyNodes;
+        this.nodeFilter = node -> {
+            if (MachineLearning.isMlNode(node)) {
+                return (nodeFilter != null) ? nodeFilter.apply(node) : null;
+            }
+            return "Not opening job [" + jobId + "] on node [" + nodeNameOrId(node) + "], because this node isn't a ml node.";
+        };
+    }
+
+    public PersistentTasksCustomMetaData.Assignment selectNode(int dynamicMaxOpenJobs, int maxConcurrentJobAllocations,
+                                                               int maxMachineMemoryPercent, boolean isMemoryTrackerRecentlyRefreshed) {
+        // TODO: remove in 8.0.0
+        boolean allNodesHaveDynamicMaxWorkers = clusterState.getNodes().getMinNodeVersion().onOrAfter(Version.V_7_2_0);
+
+        // Try to allocate jobs according to memory usage, but if that's not possible (maybe due to a mixed version cluster or maybe
+        // because of some weird OS problem) then fall back to the old mechanism of only considering numbers of assigned jobs
+        boolean allocateByMemory = isMemoryTrackerRecentlyRefreshed;
+        if (isMemoryTrackerRecentlyRefreshed == false) {
+            logger.warn("Falling back to allocating job [{}] by job counts because a memory requirement refresh could not be scheduled",
+                jobId);
+        }
+
+        List<String> reasons = new LinkedList<>();
+        long maxAvailableCount = Long.MIN_VALUE;
+        long maxAvailableMemory = Long.MIN_VALUE;
+        DiscoveryNode minLoadedNodeByCount = null;
+        DiscoveryNode minLoadedNodeByMemory = null;
+        PersistentTasksCustomMetaData persistentTasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
+        for (DiscoveryNode node : clusterState.getNodes()) {
+
+            // First check conditions that would rule out the node regardless of what other tasks are assigned to it
+            String reason = nodeFilter.apply(node);
+            if (reason != null) {
+                logger.trace(reason);
+                reasons.add(reason);
+                continue;
+            }
+
+            // Assuming the node is elligible at all, check loading
+            CurrentLoad currentLoad = calculateCurrentLoadForNode(node, persistentTasks, allocateByMemory);
+            allocateByMemory = currentLoad.allocateByMemory;
+
+            if (currentLoad.numberOfAllocatingJobs >= maxConcurrentJobAllocations) {
+                reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndMlAttributes(node) + "], because node exceeds ["
+                    + currentLoad.numberOfAllocatingJobs + "] the maximum number of jobs [" + maxConcurrentJobAllocations
+                    + "] in opening state";
+                logger.trace(reason);
+                reasons.add(reason);
+                continue;
+            }
+
+            Map<String, String> nodeAttributes = node.getAttributes();
+            int maxNumberOfOpenJobs = dynamicMaxOpenJobs;
+            // TODO: remove this in 8.0.0
+            if (allNodesHaveDynamicMaxWorkers == false) {
+                String maxNumberOfOpenJobsStr = nodeAttributes.get(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR);
+                try {
+                    maxNumberOfOpenJobs = Integer.parseInt(maxNumberOfOpenJobsStr);
+                } catch (NumberFormatException e) {
+                    reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndMlAttributes(node) + "], because " +
+                        MachineLearning.MAX_OPEN_JOBS_NODE_ATTR + " attribute [" + maxNumberOfOpenJobsStr + "] is not an integer";
+                    logger.trace(reason);
+                    reasons.add(reason);
+                    continue;
+                }
+            }
+            long availableCount = maxNumberOfOpenJobs - currentLoad.numberOfAssignedJobs;
+            if (availableCount == 0) {
+                reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndMlAttributes(node)
+                    + "], because this node is full. Number of opened jobs [" + currentLoad.numberOfAssignedJobs
+                    + "], " + MAX_OPEN_JOBS_PER_NODE.getKey() + " [" + maxNumberOfOpenJobs + "]";
+                logger.trace(reason);
+                reasons.add(reason);
+                continue;
+            }
+
+            if (maxAvailableCount < availableCount) {
+                maxAvailableCount = availableCount;
+                minLoadedNodeByCount = node;
+            }
+
+            String machineMemoryStr = nodeAttributes.get(MachineLearning.MACHINE_MEMORY_NODE_ATTR);
+            long machineMemory;
+            try {
+                machineMemory = Long.parseLong(machineMemoryStr);
+            } catch (NumberFormatException e) {
+                reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndMlAttributes(node) + "], because " +
+                    MachineLearning.MACHINE_MEMORY_NODE_ATTR + " attribute [" + machineMemoryStr + "] is not a long";
+                logger.trace(reason);
+                reasons.add(reason);
+                continue;
+            }
+
+            if (allocateByMemory) {
+                if (machineMemory > 0) {
+                    long maxMlMemory = machineMemory * maxMachineMemoryPercent / 100;
+                    Long estimatedMemoryFootprint = memoryTracker.getJobMemoryRequirement(taskName, jobId);
+                    if (estimatedMemoryFootprint != null) {
+                        long availableMemory = maxMlMemory - currentLoad.assignedJobMemory;
+                        if (estimatedMemoryFootprint > availableMemory) {
+                            reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndMlAttributes(node)
+                                + "], because this node has insufficient available memory. Available memory for ML [" + maxMlMemory
+                                + "], memory required by existing jobs [" + currentLoad.assignedJobMemory
+                                + "], estimated memory required for this job [" + estimatedMemoryFootprint + "]";
+                            logger.trace(reason);
+                            reasons.add(reason);
+                            continue;
+                        }
+
+                        if (maxAvailableMemory < availableMemory) {
+                            maxAvailableMemory = availableMemory;
+                            minLoadedNodeByMemory = node;
+                        }
+                    } else {
+                        // If we cannot get the job memory requirement,
+                        // fall back to simply allocating by job count
+                        allocateByMemory = false;
+                        logger.debug("Falling back to allocating job [{}] by job counts because its memory requirement was not available",
+                            jobId);
+                    }
+                } else {
+                    // If we cannot get the available memory on any machine in
+                    // the cluster, fall back to simply allocating by job count
+                    allocateByMemory = false;
+                    logger.debug("Falling back to allocating job [{}] by job counts because machine memory was not available for node [{}]",
+                        jobId, nodeNameAndMlAttributes(node));
+                }
+            }
+        }
+        return createAssignment(allocateByMemory ? minLoadedNodeByMemory : minLoadedNodeByCount, reasons);
+    }
+
+    private PersistentTasksCustomMetaData.Assignment createAssignment(DiscoveryNode minLoadedNode, List<String> reasons) {
+        if (minLoadedNode == null) {
+            String explanation = String.join("|", reasons);
+            logger.debug("no node selected for job [{}], reasons [{}]", jobId, explanation);
+            return considerLazyAssignment(new PersistentTasksCustomMetaData.Assignment(null, explanation));
+        }
+        logger.debug("selected node [{}] for job [{}]", minLoadedNode, jobId);
+        return new PersistentTasksCustomMetaData.Assignment(minLoadedNode.getId(), "");
+    }
+
+    PersistentTasksCustomMetaData.Assignment considerLazyAssignment(PersistentTasksCustomMetaData.Assignment currentAssignment) {
+
+        assert currentAssignment.getExecutorNode() == null;
+
+        int numMlNodes = 0;
+        for (DiscoveryNode node : clusterState.getNodes()) {
+            if (MachineLearning.isMlNode(node)) {
+                numMlNodes++;
+            }
+        }
+
+        if (numMlNodes < maxLazyNodes) { // Means we have lazy nodes left to allocate
+            return AWAITING_LAZY_ASSIGNMENT;
+        }
+
+        return currentAssignment;
+    }
+
+    private CurrentLoad calculateCurrentLoadForNode(DiscoveryNode node, PersistentTasksCustomMetaData persistentTasks,
+                                                    final boolean allocateByMemory) {
+        CurrentLoad result = new CurrentLoad(allocateByMemory);
+
+        if (persistentTasks != null) {
+            // find all the anomaly detector job tasks assigned to this node
+            Collection<PersistentTasksCustomMetaData.PersistentTask<?>> assignedAnomalyDetectorTasks = persistentTasks.findTasks(
+                MlTasks.JOB_TASK_NAME, task -> node.getId().equals(task.getExecutorNode()));
+            for (PersistentTasksCustomMetaData.PersistentTask<?> assignedTask : assignedAnomalyDetectorTasks) {
+                JobState jobState = MlTasks.getJobStateModifiedForReassignments(assignedTask);
+                if (jobState.isAnyOf(JobState.CLOSED, JobState.FAILED) == false) {
+                    // Don't count CLOSED or FAILED jobs, as they don't consume native memory
+                    ++result.numberOfAssignedJobs;
+                    if (jobState == JobState.OPENING) {
+                        ++result.numberOfAllocatingJobs;
+                    }
+                    OpenJobAction.JobParams params = (OpenJobAction.JobParams) assignedTask.getParams();
+                    Long jobMemoryRequirement = memoryTracker.getAnomalyDetectorJobMemoryRequirement(params.getJobId());
+                    if (jobMemoryRequirement == null) {
+                        result.allocateByMemory = false;
+                        logger.debug("Falling back to allocating job [{}] by job counts because " +
+                            "the memory requirement for job [{}] was not available", jobId, params.getJobId());
+                    } else {
+                        logger.debug("adding " + jobMemoryRequirement);
+                        result.assignedJobMemory += jobMemoryRequirement;
+                    }
+                }
+            }
+            // find all the data frame analytics job tasks assigned to this node
+            Collection<PersistentTasksCustomMetaData.PersistentTask<?>> assignedAnalyticsTasks = persistentTasks.findTasks(
+                MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, task -> node.getId().equals(task.getExecutorNode()));
+            for (PersistentTasksCustomMetaData.PersistentTask<?> assignedTask : assignedAnalyticsTasks) {
+                DataFrameAnalyticsState dataFrameAnalyticsState = ((DataFrameAnalyticsTaskState) assignedTask.getState()).getState();
+
+                // Don't count stopped and failed df-analytics tasks as they don't consume native memory
+                if (dataFrameAnalyticsState.isAnyOf(DataFrameAnalyticsState.STOPPED, DataFrameAnalyticsState.FAILED) == false) {
+                    // The native process is only running in the ANALYZING and STOPPING states, but in the STARTED
+                    // and REINDEXING states we're committed to using the memory soon, so account for it here
+                    ++result.numberOfAssignedJobs;
+                    StartDataFrameAnalyticsAction.TaskParams params =
+                        (StartDataFrameAnalyticsAction.TaskParams) assignedTask.getParams();
+                    Long jobMemoryRequirement = memoryTracker.getDataFrameAnalyticsJobMemoryRequirement(params.getId());
+                    if (jobMemoryRequirement == null) {
+                        result.allocateByMemory = false;
+                        logger.debug("Falling back to allocating job [{}] by job counts because " +
+                            "the memory requirement for job [{}] was not available", jobId, params.getId());
+                    } else {
+                        result.assignedJobMemory += jobMemoryRequirement;
+                    }
+                }
+            }
+        }
+
+        return result;
+    }
+
+    static String nodeNameOrId(DiscoveryNode node) {
+        String nodeNameOrID = node.getName();
+        if (Strings.isNullOrEmpty(nodeNameOrID)) {
+            nodeNameOrID = node.getId();
+        }
+        return nodeNameOrID;
+    }
+
+    public static String nodeNameAndVersion(DiscoveryNode node) {
+        String nodeNameOrID = nodeNameOrId(node);
+        StringBuilder builder = new StringBuilder("{").append(nodeNameOrID).append('}');
+        builder.append('{').append("version=").append(node.getVersion()).append('}');
+        return builder.toString();
+    }
+
+    static String nodeNameAndMlAttributes(DiscoveryNode node) {
+        String nodeNameOrID = nodeNameOrId(node);
+
+        StringBuilder builder = new StringBuilder("{").append(nodeNameOrID).append('}');
+        for (Map.Entry<String, String> entry : node.getAttributes().entrySet()) {
+            if (entry.getKey().startsWith("ml.") || entry.getKey().equals("node.ml")) {
+                builder.append('{').append(entry).append('}');
+            }
+        }
+        return builder.toString();
+    }
+
+    private static class CurrentLoad {
+
+        long numberOfAssignedJobs = 0;
+        long numberOfAllocatingJobs = 0;
+        long assignedJobMemory = 0;
+        boolean allocateByMemory;
+
+        CurrentLoad(boolean allocateByMemory) {
+            this.allocateByMemory = allocateByMemory;
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzer.java
index d9af90e00f6f0..52efb5bdad071 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzer.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzer.java
@@ -9,20 +9,8 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
-import org.elasticsearch.index.analysis.CharFilterFactory;
-import org.elasticsearch.index.analysis.CustomAnalyzer;
-import org.elasticsearch.index.analysis.TokenFilterFactory;
-import org.elasticsearch.index.analysis.TokenizerFactory;
-import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig;
 
 import java.io.Closeable;
@@ -35,21 +23,16 @@
  *
  * Converts messages to lists of tokens that will be fed to the ML categorization algorithm.
  *
- * The code in {@link #makeAnalyzer} and the methods it calls is largely copied from {@link TransportAnalyzeAction}.
- * Unfortunately there is no easy way to reuse a subset of the <code>_analyze</code> action implementation, as the
- * logic required here is not quite identical to that of {@link TransportAnalyzeAction}, and the required code is
- * hard to partially reuse.
- * TODO: consider refactoring ES core to allow more reuse.
  */
 public class CategorizationAnalyzer implements Closeable {
 
     private final Analyzer analyzer;
     private final boolean closeAnalyzer;
 
-    public CategorizationAnalyzer(AnalysisRegistry analysisRegistry, Environment environment,
+    public CategorizationAnalyzer(AnalysisRegistry analysisRegistry,
                                   CategorizationAnalyzerConfig categorizationAnalyzerConfig) throws IOException {
 
-        Tuple<Analyzer, Boolean> tuple = makeAnalyzer(categorizationAnalyzerConfig, analysisRegistry, environment);
+        Tuple<Analyzer, Boolean> tuple = makeAnalyzer(categorizationAnalyzerConfig, analysisRegistry);
         analyzer = tuple.v1();
         closeAnalyzer = tuple.v2();
     }
@@ -93,9 +76,9 @@ public List<String> tokenizeField(String fieldName, String fieldValue) {
      * server-side rather than client-side, as the client will not have loaded the appropriate analysis
      * modules/plugins.
      */
-    public static void verifyConfigBuilder(CategorizationAnalyzerConfig.Builder configBuilder, AnalysisRegistry analysisRegistry,
-                                           Environment environment) throws IOException {
-        Tuple<Analyzer, Boolean> tuple = makeAnalyzer(configBuilder.build(), analysisRegistry, environment);
+    public static void verifyConfigBuilder(CategorizationAnalyzerConfig.Builder configBuilder, AnalysisRegistry analysisRegistry)
+        throws IOException {
+        Tuple<Analyzer, Boolean> tuple = makeAnalyzer(configBuilder.build(), analysisRegistry);
         if (tuple.v2()) {
             tuple.v1().close();
         }
@@ -108,8 +91,8 @@ public static void verifyConfigBuilder(CategorizationAnalyzerConfig.Builder conf
      * @return The first tuple member is the {@link Analyzer}; the second indicates whether the caller is responsible
      *         for closing it.
      */
-    private static Tuple<Analyzer, Boolean> makeAnalyzer(CategorizationAnalyzerConfig config, AnalysisRegistry analysisRegistry,
-                                                         Environment environment) throws IOException {
+    private static Tuple<Analyzer, Boolean> makeAnalyzer(CategorizationAnalyzerConfig config, AnalysisRegistry analysisRegistry)
+        throws IOException {
         String analyzer = config.getAnalyzer();
         if (analyzer != null) {
             Analyzer globalAnalyzer = analysisRegistry.getAnalyzer(analyzer);
@@ -118,162 +101,9 @@ private static Tuple<Analyzer, Boolean> makeAnalyzer(CategorizationAnalyzerConfi
             }
             return new Tuple<>(globalAnalyzer, Boolean.FALSE);
         } else {
-            List<CharFilterFactory> charFilterFactoryList = parseCharFilterFactories(config, analysisRegistry, environment);
-
-            Tuple<String, TokenizerFactory> tokenizerFactory = parseTokenizerFactory(config, analysisRegistry, environment);
-
-            List<TokenFilterFactory> tokenFilterFactoryList = parseTokenFilterFactories(config, analysisRegistry, environment,
-                tokenizerFactory, charFilterFactoryList);
-
-            return new Tuple<>(new CustomAnalyzer(tokenizerFactory.v1(), tokenizerFactory.v2(),
-                charFilterFactoryList.toArray(new CharFilterFactory[charFilterFactoryList.size()]),
-                tokenFilterFactoryList.toArray(new TokenFilterFactory[tokenFilterFactoryList.size()])), Boolean.TRUE);
-        }
-    }
-
-
-    /**
-     * Get char filter factories for each configured char filter.  Each configuration
-     * element can be the name of an out-of-the-box char filter, or a custom definition.
-     */
-    private static List<CharFilterFactory> parseCharFilterFactories(CategorizationAnalyzerConfig config, AnalysisRegistry analysisRegistry,
-                                                                    Environment environment) throws IOException {
-        List<CategorizationAnalyzerConfig.NameOrDefinition> charFilters = config.getCharFilters();
-        final List<CharFilterFactory> charFilterFactoryList = new ArrayList<>();
-        for (CategorizationAnalyzerConfig.NameOrDefinition charFilter : charFilters) {
-            final CharFilterFactory charFilterFactory;
-            if (charFilter.name != null) {
-                AnalysisModule.AnalysisProvider<CharFilterFactory> charFilterFactoryFactory =
-                    analysisRegistry.getCharFilterProvider(charFilter.name);
-                if (charFilterFactoryFactory == null) {
-                    throw new IllegalArgumentException("Failed to find global char filter under [" + charFilter.name + "]");
-                }
-                charFilterFactory = charFilterFactoryFactory.get(environment, charFilter.name);
-            } else {
-                String charFilterTypeName = charFilter.definition.get("type");
-                if (charFilterTypeName == null) {
-                    throw new IllegalArgumentException("Missing [type] setting for char filter: " + charFilter.definition);
-                }
-                AnalysisModule.AnalysisProvider<CharFilterFactory> charFilterFactoryFactory =
-                    analysisRegistry.getCharFilterProvider(charFilterTypeName);
-                if (charFilterFactoryFactory == null) {
-                    throw new IllegalArgumentException("Failed to find global char filter under [" + charFilterTypeName + "]");
-                }
-                Settings settings = augmentSettings(charFilter.definition);
-                // Need to set anonymous "name" of char_filter
-                charFilterFactory = charFilterFactoryFactory.get(buildDummyIndexSettings(settings), environment, "_anonymous_charfilter",
-                    settings);
-            }
-            if (charFilterFactory == null) {
-                throw new IllegalArgumentException("Failed to find char filter [" + charFilter + "]");
-            }
-            charFilterFactoryList.add(charFilterFactory);
+            return new Tuple<>(analysisRegistry.buildCustomAnalyzer(null, false,
+                config.getTokenizer(), config.getCharFilters(), config.getTokenFilters()), Boolean.TRUE);
         }
-        return charFilterFactoryList;
     }
 
-    /**
-     * Get the tokenizer factory for the configured tokenizer.  The configuration
-     * can be the name of an out-of-the-box tokenizer, or a custom definition.
-     */
-    private static Tuple<String, TokenizerFactory> parseTokenizerFactory(CategorizationAnalyzerConfig config,
-                                                                         AnalysisRegistry analysisRegistry, Environment environment)
-        throws IOException {
-        CategorizationAnalyzerConfig.NameOrDefinition tokenizer = config.getTokenizer();
-        final String name;
-        final TokenizerFactory tokenizerFactory;
-        if (tokenizer.name != null) {
-            name = tokenizer.name;
-            AnalysisModule.AnalysisProvider<TokenizerFactory> tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider(name);
-            if (tokenizerFactoryFactory == null) {
-                throw new IllegalArgumentException("Failed to find global tokenizer under [" + name + "]");
-            }
-            tokenizerFactory = tokenizerFactoryFactory.get(environment, name);
-        } else {
-            String tokenizerTypeName = tokenizer.definition.get("type");
-            if (tokenizerTypeName == null) {
-                throw new IllegalArgumentException("Missing [type] setting for tokenizer: " + tokenizer.definition);
-            }
-            AnalysisModule.AnalysisProvider<TokenizerFactory> tokenizerFactoryFactory =
-                analysisRegistry.getTokenizerProvider(tokenizerTypeName);
-            if (tokenizerFactoryFactory == null) {
-                throw new IllegalArgumentException("Failed to find global tokenizer under [" + tokenizerTypeName + "]");
-            }
-            Settings settings = augmentSettings(tokenizer.definition);
-            // Need to set anonymous "name" of tokenizer
-            name = "_anonymous_tokenizer";
-            tokenizerFactory = tokenizerFactoryFactory.get(buildDummyIndexSettings(settings), environment, name, settings);
-        }
-        return new Tuple<>(name, tokenizerFactory);
-    }
-
-    /**
-     * Get token filter factories for each configured token filter.  Each configuration
-     * element can be the name of an out-of-the-box token filter, or a custom definition.
-     */
-    private static List<TokenFilterFactory> parseTokenFilterFactories(CategorizationAnalyzerConfig config,
-                                                                      AnalysisRegistry analysisRegistry, Environment environment,
-                                                                      Tuple<String, TokenizerFactory> tokenizerFactory,
-                                                                      List<CharFilterFactory> charFilterFactoryList) throws IOException {
-        List<CategorizationAnalyzerConfig.NameOrDefinition> tokenFilters = config.getTokenFilters();
-        TransportAnalyzeAction.DeferredTokenFilterRegistry deferredRegistry
-            = new TransportAnalyzeAction.DeferredTokenFilterRegistry(analysisRegistry, null);
-        final List<TokenFilterFactory> tokenFilterFactoryList = new ArrayList<>();
-        for (CategorizationAnalyzerConfig.NameOrDefinition tokenFilter : tokenFilters) {
-            TokenFilterFactory tokenFilterFactory;
-            if (tokenFilter.name != null) {
-                AnalysisModule.AnalysisProvider<TokenFilterFactory> tokenFilterFactoryFactory;
-                tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider(tokenFilter.name);
-                if (tokenFilterFactoryFactory == null) {
-                    throw new IllegalArgumentException("Failed to find global token filter under [" + tokenFilter.name + "]");
-                }
-                tokenFilterFactory = tokenFilterFactoryFactory.get(environment, tokenFilter.name);
-            } else {
-                String filterTypeName = tokenFilter.definition.get("type");
-                if (filterTypeName == null) {
-                    throw new IllegalArgumentException("Missing [type] setting for token filter: " + tokenFilter.definition);
-                }
-                AnalysisModule.AnalysisProvider<TokenFilterFactory> tokenFilterFactoryFactory =
-                    analysisRegistry.getTokenFilterProvider(filterTypeName);
-                if (tokenFilterFactoryFactory == null) {
-                    throw new IllegalArgumentException("Failed to find global token filter under [" + filterTypeName + "]");
-                }
-                Settings settings = augmentSettings(tokenFilter.definition);
-                // Need to set anonymous "name" of token_filter
-                tokenFilterFactory = tokenFilterFactoryFactory.get(buildDummyIndexSettings(settings), environment, "_anonymous_tokenfilter",
-                    settings);
-                tokenFilterFactory = tokenFilterFactory.getChainAwareTokenFilterFactory(tokenizerFactory.v2(),
-                    charFilterFactoryList, tokenFilterFactoryList, deferredRegistry);
-            }
-            if (tokenFilterFactory == null) {
-                throw new IllegalArgumentException("Failed to find or create token filter [" + tokenFilter + "]");
-            }
-            tokenFilterFactoryList.add(tokenFilterFactory);
-        }
-        return tokenFilterFactoryList;
-    }
-
-    /**
-     * The Elasticsearch analysis functionality is designed to work with indices.  For
-     * categorization we have to pretend we've got some index settings.
-     */
-    private static IndexSettings buildDummyIndexSettings(Settings settings) {
-        IndexMetaData metaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(settings).build();
-        return new IndexSettings(metaData, Settings.EMPTY);
-    }
-
-    /**
-     * The behaviour of Elasticsearch analyzers can vary between versions.
-     * For categorization we'll always use the latest version of the text analysis.
-     * The other settings are just to stop classes that expect to be associated with
-     * an index from complaining.
-     */
-    private static Settings augmentSettings(Settings settings) {
-        return Settings.builder().put(settings)
-            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
-            .build();
-    }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java
index a0017af4b8c47..9866e3f56af19 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java
@@ -49,7 +49,7 @@ private XContentBuilder serialiseCounts(DataCounts counts) throws IOException {
      *
      * @param jobId Job to update
      * @param counts The counts
-     * @param listener Action response listener
+     * @param listener ActionType response listener
      */
     public void persistDataCounts(String jobId, DataCounts counts, ActionListener<Boolean> listener) {
         try (XContentBuilder content = serialiseCounts(counts)) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java
index 9380be0c15b78..f48dcb977e59c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.index.query.IdsQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.BulkByScrollTask;
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
@@ -129,7 +130,7 @@ private static class DeleteByQueryHolder {
         private DeleteByQueryHolder(String index) {
             dbqRequest = new DeleteByQueryRequest();
             dbqRequest.indices(index);
-            dbqRequest.setSlices(5);
+            dbqRequest.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES);
             dbqRequest.setAbortOnVersionConflict(false);
         }
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java
index fc5b87d4afebe..b7d5214ecf304 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord;
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
 import org.elasticsearch.xpack.core.ml.job.results.BucketInfluencer;
@@ -121,6 +122,17 @@ private void persistBucketInfluencersStandalone(String jobId, List<BucketInfluen
             }
         }
 
+        /**
+         * Persist timing stats
+         *
+         * @param timingStats timing stats to persist
+         * @return this
+         */
+        public Builder persistTimingStats(TimingStats timingStats) {
+            indexResult(TimingStats.documentId(timingStats.getJobId()), timingStats, TimingStats.TYPE.getPreferredName());
+            return this;
+        }
+
         /**
          * Persist a list of anomaly records
          *
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
index e5d343bb304b6..bb0ed2b3ddf0e 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
@@ -91,6 +91,7 @@
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshotField;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord;
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
 import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition;
@@ -417,6 +418,30 @@ private SearchRequestBuilder createLatestDataCountsSearch(String indexName, Stri
                 .addSort(SortBuilders.fieldSort(DataCounts.LATEST_RECORD_TIME.getPreferredName()).order(SortOrder.DESC));
     }
 
+    /**
+     * Get the job's timing stats
+     *
+     * @param jobId The job id
+     */
+    public void timingStats(String jobId, Consumer<TimingStats> handler, Consumer<Exception> errorHandler) {
+        String indexName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId);
+        searchSingleResult(
+            jobId,
+            TimingStats.TYPE.getPreferredName(),
+            createTimingStatsSearch(indexName, jobId),
+            TimingStats.PARSER,
+            result -> handler.accept(result.result),
+            errorHandler,
+            () -> new TimingStats(jobId));
+    }
+
+    private SearchRequestBuilder createTimingStatsSearch(String indexName, String jobId) {
+        return client.prepareSearch(indexName)
+            .setSize(1)
+            .setIndicesOptions(IndicesOptions.lenientExpandOpen())
+            .setQuery(QueryBuilders.idsQuery().addIds(TimingStats.documentId(jobId)));
+    }
+
     public void getAutodetectParams(Job job, Consumer<AutodetectParams> consumer, Consumer<Exception> errorHandler) {
 
         String jobId = job.getId();
@@ -443,6 +468,7 @@ public void getAutodetectParams(Job job, Consumer<AutodetectParams> consumer, Co
         MultiSearchRequestBuilder msearch = client.prepareMultiSearch()
                 .add(createLatestDataCountsSearch(resultsIndex, jobId))
                 .add(createLatestModelSizeStatsSearch(resultsIndex))
+                .add(createTimingStatsSearch(resultsIndex, jobId))
                 // These next two document IDs never need to be the legacy ones due to the rule
                 // that you cannot open a 5.4 job in a subsequent version of the product
                 .add(createDocIdSearch(resultsIndex, ModelSnapshot.documentId(jobId, job.getModelSnapshotId())))
@@ -504,6 +530,8 @@ private void parseAutodetectParamSearchHit(String jobId, AutodetectParams.Builde
         String hitId = hit.getId();
         if (DataCounts.documentId(jobId).equals(hitId)) {
             paramsBuilder.setDataCounts(parseSearchHit(hit, DataCounts.PARSER, errorHandler));
+        } else if (TimingStats.documentId(jobId).equals(hitId)) {
+            paramsBuilder.setTimingStats(parseSearchHit(hit, TimingStats.PARSER, errorHandler));
         } else if (hitId.startsWith(ModelSizeStats.documentIdPrefix(jobId))) {
             ModelSizeStats.Builder modelSizeStats = parseSearchHit(hit, ModelSizeStats.LENIENT_PARSER, errorHandler);
             paramsBuilder.setModelSizeStats(modelSizeStats == null ? null : modelSizeStats.build());
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/TimingStatsReporter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/TimingStatsReporter.java
new file mode 100644
index 0000000000000..d30335a5f06e9
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/TimingStatsReporter.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.job.persistence;
+
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
+
+import java.util.Objects;
+
+/**
+ * {@link TimingStatsReporter} class handles the logic of persisting {@link TimingStats} if they changed significantly since the last time
+ * they were persisted.
+ *
+ * This class is not thread-safe.
+ */
+public class TimingStatsReporter {
+
+    /** Persisted timing stats. May be stale. */
+    private TimingStats persistedTimingStats;
+    /** Current timing stats. */
+    private TimingStats currentTimingStats;
+    /** Object used to persist current timing stats. */
+    private JobResultsPersister.Builder bulkResultsPersister;
+
+    public TimingStatsReporter(TimingStats timingStats, JobResultsPersister.Builder jobResultsPersister) {
+        Objects.requireNonNull(timingStats);
+        this.persistedTimingStats = new TimingStats(timingStats);
+        this.currentTimingStats = new TimingStats(timingStats);
+        this.bulkResultsPersister = Objects.requireNonNull(jobResultsPersister);
+    }
+
+    public TimingStats getCurrentTimingStats() {
+        return new TimingStats(currentTimingStats);
+    }
+
+    public void reportBucketProcessingTime(long bucketProcessingTimeMs) {
+        currentTimingStats.updateStats(bucketProcessingTimeMs);
+        if (differSignificantly(currentTimingStats, persistedTimingStats)) {
+            flush();
+        }
+    }
+
+    public void finishReporting() {
+        // Don't flush if current timing stats are identical to the persisted ones
+        if (currentTimingStats.equals(persistedTimingStats)) {
+            return;
+        }
+        flush();
+    }
+
+    public void flush() {
+        persistedTimingStats = new TimingStats(currentTimingStats);
+        bulkResultsPersister.persistTimingStats(persistedTimingStats);
+    }
+
+    /**
+     * Returns true if given stats objects differ from each other by more than 10% for at least one of the statistics.
+     */
+    public static boolean differSignificantly(TimingStats stats1, TimingStats stats2) {
+        return differSignificantly(stats1.getMinBucketProcessingTimeMs(), stats2.getMinBucketProcessingTimeMs())
+            || differSignificantly(stats1.getMaxBucketProcessingTimeMs(), stats2.getMaxBucketProcessingTimeMs())
+            || differSignificantly(stats1.getAvgBucketProcessingTimeMs(), stats2.getAvgBucketProcessingTimeMs())
+            || differSignificantly(stats1.getExponentialAvgBucketProcessingTimeMs(), stats2.getExponentialAvgBucketProcessingTimeMs());
+    }
+
+    /**
+     * Returns {@code true} if one of the ratios { value1 / value2, value2 / value1 } is smaller than MIN_VALID_RATIO.
+     * This can be interpreted as values { value1, value2 } differing significantly from each other.
+     * This method also returns:
+     *   - {@code true} in case one value is {@code null} while the other is not.
+     *   - {@code false} in case both values are {@code null}.
+     */
+    static boolean differSignificantly(Double value1, Double value2) {
+        if (value1 != null && value2 != null) {
+            return (value2 / value1 < MIN_VALID_RATIO) || (value1 / value2 < MIN_VALID_RATIO);
+        }
+        return (value1 != null) || (value2 != null);
+    }
+
+    /**
+     * Minimum ratio of values that is interpreted as values being similar.
+     * If the values ratio is less than MIN_VALID_RATIO, the values are interpreted as significantly different.
+     */
+    private static final double MIN_VALID_RATIO = 0.9;
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java
index 7e778e48524ba..8d49a0bfdbccf 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java
@@ -26,13 +26,14 @@
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzer;
 import org.elasticsearch.xpack.ml.job.persistence.StateStreamer;
 import org.elasticsearch.xpack.ml.job.process.CountingInputStream;
 import org.elasticsearch.xpack.ml.job.process.DataCountsReporter;
-import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutoDetectResultProcessor;
+import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutodetectResultProcessor;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.FlushJobParams;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.ForecastParams;
@@ -65,7 +66,7 @@ public class AutodetectCommunicator implements Closeable {
     private final AutodetectProcess autodetectProcess;
     private final StateStreamer stateStreamer;
     private final DataCountsReporter dataCountsReporter;
-    private final AutoDetectResultProcessor autoDetectResultProcessor;
+    private final AutodetectResultProcessor autodetectResultProcessor;
     private final BiConsumer<Exception, Boolean> onFinishHandler;
     private final ExecutorService autodetectWorkerExecutor;
     private final NamedXContentRegistry xContentRegistry;
@@ -74,7 +75,7 @@ public class AutodetectCommunicator implements Closeable {
     private volatile boolean processKilled;
 
     AutodetectCommunicator(Job job, Environment environment, AutodetectProcess process, StateStreamer stateStreamer,
-                           DataCountsReporter dataCountsReporter, AutoDetectResultProcessor autoDetectResultProcessor,
+                           DataCountsReporter dataCountsReporter, AutodetectResultProcessor autodetectResultProcessor,
                            BiConsumer<Exception, Boolean> onFinishHandler, NamedXContentRegistry xContentRegistry,
                            ExecutorService autodetectWorkerExecutor) {
         this.job = job;
@@ -82,7 +83,7 @@ public class AutodetectCommunicator implements Closeable {
         this.autodetectProcess = process;
         this.stateStreamer = stateStreamer;
         this.dataCountsReporter = dataCountsReporter;
-        this.autoDetectResultProcessor = autoDetectResultProcessor;
+        this.autodetectResultProcessor = autodetectResultProcessor;
         this.onFinishHandler = onFinishHandler;
         this.xContentRegistry = xContentRegistry;
         this.autodetectWorkerExecutor = autodetectWorkerExecutor;
@@ -119,7 +120,7 @@ public void writeToJob(InputStream inputStream, AnalysisRegistry analysisRegistr
             }
 
             CountingInputStream countingStream = new CountingInputStream(inputStream, dataCountsReporter);
-            DataToProcessWriter autoDetectWriter = createProcessWriter(params.getDataDescription());
+            DataToProcessWriter autodetectWriter = createProcessWriter(params.getDataDescription());
 
             if (includeTokensField && categorizationAnalyzer == null) {
                 createCategorizationAnalyzer(analysisRegistry);
@@ -128,14 +129,14 @@ public void writeToJob(InputStream inputStream, AnalysisRegistry analysisRegistr
             CountDownLatch latch = new CountDownLatch(1);
             AtomicReference<DataCounts> dataCountsAtomicReference = new AtomicReference<>();
             AtomicReference<Exception> exceptionAtomicReference = new AtomicReference<>();
-            autoDetectWriter.write(countingStream, categorizationAnalyzer, xContentType, (dataCounts, e) -> {
+            autodetectWriter.write(countingStream, categorizationAnalyzer, xContentType, (dataCounts, e) -> {
                 dataCountsAtomicReference.set(dataCounts);
                 exceptionAtomicReference.set(e);
                 latch.countDown();
             });
 
             latch.await();
-            autoDetectWriter.flushStream();
+            autodetectWriter.flushStream();
 
             if (exceptionAtomicReference.get() != null) {
                 throw exceptionAtomicReference.get();
@@ -167,7 +168,7 @@ public void close(boolean restart, String reason) {
                     killProcess(false, false);
                     stateStreamer.cancel();
                 }
-                autoDetectResultProcessor.awaitCompletion();
+                autodetectResultProcessor.awaitCompletion();
             } finally {
                 onFinishHandler.accept(restart ? new ElasticsearchException(reason) : null, true);
             }
@@ -198,13 +199,13 @@ public void killProcess(boolean awaitCompletion, boolean finish) throws IOExcept
     public void killProcess(boolean awaitCompletion, boolean finish, boolean finalizeJob) throws IOException {
         try {
             processKilled = true;
-            autoDetectResultProcessor.setProcessKilled();
+            autodetectResultProcessor.setProcessKilled();
             autodetectWorkerExecutor.shutdown();
             autodetectProcess.kill();
 
             if (awaitCompletion) {
                 try {
-                    autoDetectResultProcessor.awaitCompletion();
+                    autodetectResultProcessor.awaitCompletion();
                 } catch (TimeoutException e) {
                     LOGGER.warn(new ParameterizedMessage("[{}] Timed out waiting for killed job", job.getId()), e);
                 }
@@ -288,20 +289,20 @@ FlushAcknowledgement waitFlushToCompletion(String flushId) throws InterruptedExc
 
         FlushAcknowledgement flushAcknowledgement;
         try {
-            flushAcknowledgement = autoDetectResultProcessor.waitForFlushAcknowledgement(flushId, FLUSH_PROCESS_CHECK_FREQUENCY);
+            flushAcknowledgement = autodetectResultProcessor.waitForFlushAcknowledgement(flushId, FLUSH_PROCESS_CHECK_FREQUENCY);
             while (flushAcknowledgement == null) {
                 checkProcessIsAlive();
                 checkResultsProcessorIsAlive();
-                flushAcknowledgement = autoDetectResultProcessor.waitForFlushAcknowledgement(flushId, FLUSH_PROCESS_CHECK_FREQUENCY);
+                flushAcknowledgement = autodetectResultProcessor.waitForFlushAcknowledgement(flushId, FLUSH_PROCESS_CHECK_FREQUENCY);
             }
         } finally {
-            autoDetectResultProcessor.clearAwaitingFlush(flushId);
+            autodetectResultProcessor.clearAwaitingFlush(flushId);
         }
 
         if (processKilled == false) {
             // We also have to wait for the normalizer to become idle so that we block
             // clients from querying results in the middle of normalization.
-            autoDetectResultProcessor.waitUntilRenormalizerIsIdle();
+            autodetectResultProcessor.waitUntilRenormalizerIsIdle();
 
             LOGGER.debug("[{}] Flush completed", job.getId());
         }
@@ -320,7 +321,7 @@ private void checkProcessIsAlive() {
     }
 
     private void checkResultsProcessorIsAlive() {
-        if (autoDetectResultProcessor.isFailed()) {
+        if (autodetectResultProcessor.isFailed()) {
             // Don't log here - it just causes double logging when the exception gets logged
             throw new ElasticsearchException("[{}] Unexpected death of the result processor", job.getId());
         }
@@ -331,7 +332,11 @@ public ZonedDateTime getProcessStartTime() {
     }
 
     public ModelSizeStats getModelSizeStats() {
-        return autoDetectResultProcessor.modelSizeStats();
+        return autodetectResultProcessor.modelSizeStats();
+    }
+
+    public TimingStats getTimingStats() {
+        return autodetectResultProcessor.timingStats();
     }
 
     public DataCounts getDataCounts() {
@@ -384,6 +389,6 @@ private void createCategorizationAnalyzer(AnalysisRegistry analysisRegistry) thr
             categorizationAnalyzerConfig =
                     CategorizationAnalyzerConfig.buildDefaultCategorizationAnalyzer(analysisConfig.getCategorizationFilters());
         }
-        categorizationAnalyzer = new CategorizationAnalyzer(analysisRegistry, environment, categorizationAnalyzerConfig);
+        categorizationAnalyzer = new CategorizationAnalyzer(analysisRegistry, categorizationAnalyzerConfig);
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java
index cbcaf54b46b9e..c6c91bcad09be 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java
@@ -45,6 +45,7 @@
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.action.TransportOpenJobAction.JobTask;
@@ -56,7 +57,7 @@
 import org.elasticsearch.xpack.ml.job.persistence.ScheduledEventsQueryBuilder;
 import org.elasticsearch.xpack.ml.job.persistence.StateStreamer;
 import org.elasticsearch.xpack.ml.job.process.DataCountsReporter;
-import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutoDetectResultProcessor;
+import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutodetectResultProcessor;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.FlushJobParams;
@@ -499,7 +500,7 @@ AutodetectCommunicator create(JobTask jobTask, Job job, AutodetectParams autodet
         }
 
         // A TP with no queue, so that we fail immediately if there are no threads available
-        ExecutorService autoDetectExecutorService = threadPool.executor(MachineLearning.JOB_COMMS_THREAD_POOL_NAME);
+        ExecutorService autodetectExecutorService = threadPool.executor(MachineLearning.JOB_COMMS_THREAD_POOL_NAME);
         DataCountsReporter dataCountsReporter = new DataCountsReporter(job, autodetectParams.dataCounts(), jobDataCountsPersister);
         ScoresUpdater scoresUpdater = new ScoresUpdater(job, jobResultsProvider,
                 new JobRenormalizedResultsPersister(job.getId(), client), normalizerFactory);
@@ -507,14 +508,22 @@ AutodetectCommunicator create(JobTask jobTask, Job job, AutodetectParams autodet
         Renormalizer renormalizer = new ShortCircuitingRenormalizer(jobId, scoresUpdater,
                 renormalizerExecutorService);
 
-        AutodetectProcess process = autodetectProcessFactory.createAutodetectProcess(job, autodetectParams, autoDetectExecutorService,
+        AutodetectProcess process = autodetectProcessFactory.createAutodetectProcess(job, autodetectParams, autodetectExecutorService,
                 onProcessCrash(jobTask));
-        AutoDetectResultProcessor processor = new AutoDetectResultProcessor(
-                client, auditor, jobId, renormalizer, jobResultsPersister, autodetectParams.modelSizeStats());
+        AutodetectResultProcessor processor =
+            new AutodetectResultProcessor(
+                client,
+                auditor,
+                jobId,
+                renormalizer,
+                jobResultsPersister,
+                process,
+                autodetectParams.modelSizeStats(),
+                autodetectParams.timingStats());
         ExecutorService autodetectWorkerExecutor;
         try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) {
-            autodetectWorkerExecutor = createAutodetectExecutorService(autoDetectExecutorService);
-            autoDetectExecutorService.submit(() -> processor.process(process));
+            autodetectWorkerExecutor = createAutodetectExecutorService(autodetectExecutorService);
+            autodetectExecutorService.submit(processor::process);
         } catch (EsRejectedExecutionException e) {
             // If submitting the operation to read the results from the process fails we need to close
             // the process too, so that other submitted operations to threadpool are stopped.
@@ -716,18 +725,19 @@ public void onFailure(Exception e) {
         });
     }
 
-    public Optional<Tuple<DataCounts, ModelSizeStats>> getStatistics(JobTask jobTask) {
+    public Optional<Tuple<DataCounts, Tuple<ModelSizeStats, TimingStats>>> getStatistics(JobTask jobTask) {
         AutodetectCommunicator communicator = getAutodetectCommunicator(jobTask);
         if (communicator == null) {
             return Optional.empty();
         }
-        return Optional.of(new Tuple<>(communicator.getDataCounts(), communicator.getModelSizeStats()));
+        return Optional.of(
+            new Tuple<>(communicator.getDataCounts(), new Tuple<>(communicator.getModelSizeStats(), communicator.getTimingStats())));
     }
 
     ExecutorService createAutodetectExecutorService(ExecutorService executorService) {
-        AutodetectWorkerExecutorService autoDetectWorkerExecutor = new AutodetectWorkerExecutorService(threadPool.getThreadContext());
-        executorService.submit(autoDetectWorkerExecutor::start);
-        return autoDetectWorkerExecutor;
+        AutodetectWorkerExecutorService autodetectWorkerExecutor = new AutodetectWorkerExecutorService(threadPool.getThreadContext());
+        executorService.submit(autodetectWorkerExecutor::start);
+        return autodetectWorkerExecutor;
     }
 
     public ByteSizeValue getMinLocalStorageAvailable() {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java
index 96d5a74097532..8f4d3d428e3d4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java
@@ -14,13 +14,14 @@
 import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.ml.job.persistence.StateStreamer;
-import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutodetectResultsParser;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.FlushJobParams;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.ForecastParams;
 import org.elasticsearch.xpack.ml.job.process.autodetect.writer.AutodetectControlMsgWriter;
 import org.elasticsearch.xpack.ml.job.results.AutodetectResult;
 import org.elasticsearch.xpack.ml.process.AbstractNativeProcess;
+import org.elasticsearch.xpack.ml.process.ProcessResultsParser;
+import org.elasticsearch.xpack.ml.process.NativeController;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -39,12 +40,13 @@ class NativeAutodetectProcess extends AbstractNativeProcess implements Autodetec
 
     private static final String NAME = "autodetect";
 
-    private final AutodetectResultsParser resultsParser;
+    private final ProcessResultsParser<AutodetectResult> resultsParser;
 
-    NativeAutodetectProcess(String jobId, InputStream logStream, OutputStream processInStream, InputStream processOutStream,
-                            OutputStream processRestoreStream, int numberOfFields, List<Path> filesToDelete,
-                            AutodetectResultsParser resultsParser, Consumer<String> onProcessCrash) {
-        super(jobId, logStream, processInStream, processOutStream, processRestoreStream, numberOfFields, filesToDelete, onProcessCrash);
+    NativeAutodetectProcess(String jobId, NativeController nativeController, InputStream logStream, OutputStream processInStream,
+                            InputStream processOutStream, OutputStream processRestoreStream, int numberOfFields, List<Path> filesToDelete,
+                            ProcessResultsParser<AutodetectResult> resultsParser, Consumer<String> onProcessCrash) {
+        super(jobId, nativeController, logStream, processInStream, processOutStream, processRestoreStream, numberOfFields, filesToDelete,
+            onProcessCrash);
         this.resultsParser = resultsParser;
     }
 
@@ -118,17 +120,4 @@ public Iterator<AutodetectResult> readAutodetectResults() {
     private AutodetectControlMsgWriter newMessageWriter() {
         return new AutodetectControlMsgWriter(recordWriter(), numberOfFields());
     }
-
-    @Override
-    public void consumeAndCloseOutputStream() {
-        try {
-            byte[] buff = new byte[512];
-            while (processOutStream().read(buff) >= 0) {
-                // Do nothing
-            }
-            processOutStream().close();
-        } catch (IOException e) {
-            throw new RuntimeException("Error closing result parser input stream", e);
-        }
-    }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java
index 27bf1dd675325..9987083159905 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java
@@ -10,17 +10,19 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.ml.MachineLearning;
-import org.elasticsearch.xpack.ml.process.NativeController;
-import org.elasticsearch.xpack.ml.process.ProcessPipes;
-import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutodetectResultsParser;
 import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutodetectStateProcessor;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams;
+import org.elasticsearch.xpack.ml.job.results.AutodetectResult;
+import org.elasticsearch.xpack.ml.process.NativeController;
+import org.elasticsearch.xpack.ml.process.ProcessPipes;
+import org.elasticsearch.xpack.ml.process.ProcessResultsParser;
 import org.elasticsearch.xpack.ml.utils.NamedPipeHelper;
 
 import java.io.IOException;
@@ -36,13 +38,13 @@ public class NativeAutodetectProcessFactory implements AutodetectProcessFactory
 
     private static final Logger LOGGER = LogManager.getLogger(NativeAutodetectProcessFactory.class);
     private static final NamedPipeHelper NAMED_PIPE_HELPER = new NamedPipeHelper();
-    public static final Duration PROCESS_STARTUP_TIMEOUT = Duration.ofSeconds(10);
 
     private final Client client;
     private final Environment env;
     private final Settings settings;
     private final NativeController nativeController;
     private final ClusterService clusterService;
+    private volatile Duration processConnectTimeout;
 
     public NativeAutodetectProcessFactory(Environment env, Settings settings, NativeController nativeController, Client client,
                                           ClusterService clusterService) {
@@ -51,6 +53,13 @@ public NativeAutodetectProcessFactory(Environment env, Settings settings, Native
         this.nativeController = Objects.requireNonNull(nativeController);
         this.client = client;
         this.clusterService = clusterService;
+        setProcessConnectTimeout(MachineLearning.PROCESS_CONNECT_TIMEOUT.get(settings));
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.PROCESS_CONNECT_TIMEOUT,
+            this::setProcessConnectTimeout);
+    }
+
+    void setProcessConnectTimeout(TimeValue processConnectTimeout) {
+        this.processConnectTimeout = Duration.ofMillis(processConnectTimeout.getMillis());
     }
 
     @Override
@@ -69,9 +78,9 @@ public AutodetectProcess createAutodetectProcess(Job job,
         int numberOfFields = job.allInputFields().size() + (includeTokensField ? 1 : 0) + 1;
 
         AutodetectStateProcessor stateProcessor = new AutodetectStateProcessor(client, job.getId());
-        AutodetectResultsParser resultsParser = new AutodetectResultsParser();
+        ProcessResultsParser<AutodetectResult> resultsParser = new ProcessResultsParser<>(AutodetectResult.PARSER);
         NativeAutodetectProcess autodetect = new NativeAutodetectProcess(
-                job.getId(), processPipes.getLogStream().get(), processPipes.getProcessInStream().get(),
+                job.getId(), nativeController, processPipes.getLogStream().get(), processPipes.getProcessInStream().get(),
                 processPipes.getProcessOutStream().get(), processPipes.getRestoreStream().orElse(null), numberOfFields,
                 filesToDelete, resultsParser, onProcessCrash);
         try {
@@ -87,8 +96,8 @@ public AutodetectProcess createAutodetectProcess(Job job,
         }
     }
 
-    private void createNativeProcess(Job job, AutodetectParams autodetectParams, ProcessPipes processPipes,
-                                     List<Path> filesToDelete) {
+    void createNativeProcess(Job job, AutodetectParams autodetectParams, ProcessPipes processPipes,
+                             List<Path> filesToDelete) {
         try {
 
             Settings updatedSettings = Settings.builder()
@@ -108,7 +117,7 @@ private void createNativeProcess(Job job, AutodetectParams autodetectParams, Pro
                 autodetectBuilder.quantiles(autodetectParams.quantiles());
             }
             autodetectBuilder.build();
-            processPipes.connectStreams(PROCESS_STARTUP_TIMEOUT);
+            processPipes.connectStreams(processConnectTimeout);
         } catch (IOException e) {
             String msg = "Failed to launch autodetect for job " + job.getId();
             LOGGER.error(msg);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessor.java
similarity index 79%
rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java
rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessor.java
index 4516686b65202..d2d052b1a3e6c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessor.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord;
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
 import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition;
@@ -33,6 +34,7 @@
 import org.elasticsearch.xpack.core.ml.job.results.Influencer;
 import org.elasticsearch.xpack.core.ml.job.results.ModelPlot;
 import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister;
+import org.elasticsearch.xpack.ml.job.persistence.TimingStatsReporter;
 import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess;
 import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer;
 import org.elasticsearch.xpack.ml.job.results.AutodetectResult;
@@ -52,7 +54,7 @@
 
 /**
  * A runnable class that reads the autodetect process output in the
- * {@link #process(AutodetectProcess)} method and persists parsed
+ * {@link #process()} method and persists parsed
  * results via the {@linkplain JobResultsPersister} passed in the constructor.
  * <p>
  * Has methods to register and remove alert observers.
@@ -67,15 +69,17 @@
  * interim results and the old interim results have to be cleared out
  * before the new ones are written.
  */
-public class AutoDetectResultProcessor {
+public class AutodetectResultProcessor {
 
-    private static final Logger LOGGER = LogManager.getLogger(AutoDetectResultProcessor.class);
+    private static final Logger LOGGER = LogManager.getLogger(AutodetectResultProcessor.class);
 
     private final Client client;
     private final Auditor auditor;
     private final String jobId;
     private final Renormalizer renormalizer;
     private final JobResultsPersister persister;
+    private final AutodetectProcess process;
+    private final TimingStatsReporter timingStatsReporter;
 
     final CountDownLatch completionLatch = new CountDownLatch(1);
     final Semaphore updateModelSnapshotSemaphore = new Semaphore(1);
@@ -83,40 +87,55 @@ public class AutoDetectResultProcessor {
     private volatile boolean processKilled;
     private volatile boolean failed;
     private int bucketCount; // only used from the process() thread, so doesn't need to be volatile
+    private final JobResultsPersister.Builder bulkResultsPersister;
+    private boolean deleteInterimRequired;
 
     /**
      * New model size stats are read as the process is running
      */
     private volatile ModelSizeStats latestModelSizeStats;
 
-    public AutoDetectResultProcessor(Client client, Auditor auditor, String jobId, Renormalizer renormalizer,
-                                     JobResultsPersister persister, ModelSizeStats latestModelSizeStats) {
-        this(client, auditor, jobId, renormalizer, persister, latestModelSizeStats, new FlushListener());
+    public AutodetectResultProcessor(Client client,
+                                     Auditor auditor,
+                                     String jobId,
+                                     Renormalizer renormalizer,
+                                     JobResultsPersister persister,
+                                     AutodetectProcess process,
+                                     ModelSizeStats latestModelSizeStats,
+                                     TimingStats timingStats) {
+        this(client, auditor, jobId, renormalizer, persister, process, latestModelSizeStats, timingStats, new FlushListener());
     }
 
-    AutoDetectResultProcessor(Client client, Auditor auditor, String jobId, Renormalizer renormalizer,
-                              JobResultsPersister persister, ModelSizeStats latestModelSizeStats, FlushListener flushListener) {
+    // Visible for testing
+    AutodetectResultProcessor(Client client, Auditor auditor, String jobId, Renormalizer renormalizer,
+                              JobResultsPersister persister, AutodetectProcess autodetectProcess, ModelSizeStats latestModelSizeStats,
+                              TimingStats timingStats,
+                              FlushListener flushListener) {
         this.client = Objects.requireNonNull(client);
         this.auditor = Objects.requireNonNull(auditor);
         this.jobId = Objects.requireNonNull(jobId);
         this.renormalizer = Objects.requireNonNull(renormalizer);
         this.persister = Objects.requireNonNull(persister);
+        this.process = Objects.requireNonNull(autodetectProcess);
         this.flushListener = Objects.requireNonNull(flushListener);
         this.latestModelSizeStats = Objects.requireNonNull(latestModelSizeStats);
+        this.bulkResultsPersister = persister.bulkPersisterBuilder(jobId);
+        this.timingStatsReporter = new TimingStatsReporter(timingStats, bulkResultsPersister);
+        this.deleteInterimRequired = true;
     }
 
-    public void process(AutodetectProcess process) {
-        Context context = new Context(jobId, persister.bulkPersisterBuilder(jobId));
+    public void process() {
 
         // If a function call in this throws for some reason we don't want it
         // to kill the results reader thread as autodetect will be blocked
         // trying to write its output.
         try {
-            readResults(process, context);
+            readResults();
 
             try {
                 if (processKilled == false) {
-                    context.bulkResultsPersister.executeRequest();
+                    timingStatsReporter.finishReporting();
+                    bulkResultsPersister.executeRequest();
                 }
             } catch (Exception e) {
                 LOGGER.warn(new ParameterizedMessage("[{}] Error persisting autodetect results", jobId), e);
@@ -146,14 +165,14 @@ public void process(AutodetectProcess process) {
         }
     }
 
-    private void readResults(AutodetectProcess process, Context context) {
+    private void readResults() {
         bucketCount = 0;
         try {
             Iterator<AutodetectResult> iterator = process.readAutodetectResults();
             while (iterator.hasNext()) {
                 try {
                     AutodetectResult result = iterator.next();
-                    processResult(context, result);
+                    processResult(result);
                     if (result.getBucket() != null) {
                         LOGGER.trace("[{}] Bucket number {} parsed from output", jobId, bucketCount);
                     }
@@ -177,33 +196,35 @@ public void setProcessKilled() {
         renormalizer.shutdown();
     }
 
-    void processResult(Context context, AutodetectResult result) {
+    void processResult(AutodetectResult result) {
         if (processKilled) {
             return;
         }
 
         Bucket bucket = result.getBucket();
         if (bucket != null) {
-            if (context.deleteInterimRequired) {
+            if (deleteInterimRequired) {
                 // Delete any existing interim results generated by a Flush command
                 // which have not been replaced or superseded by new results.
-                LOGGER.trace("[{}] Deleting interim results", context.jobId);
-                persister.deleteInterimResults(context.jobId);
-                context.deleteInterimRequired = false;
+                LOGGER.trace("[{}] Deleting interim results", jobId);
+                persister.deleteInterimResults(jobId);
+                deleteInterimRequired = false;
             }
 
             // persist after deleting interim results in case the new
             // results are also interim
-            context.bulkResultsPersister.persistBucket(bucket).executeRequest();
+            timingStatsReporter.reportBucketProcessingTime(bucket.getProcessingTimeMs());
+            bulkResultsPersister.persistBucket(bucket).executeRequest();
+
             ++bucketCount;
         }
         List<AnomalyRecord> records = result.getRecords();
         if (records != null && !records.isEmpty()) {
-            context.bulkResultsPersister.persistRecords(records);
+            bulkResultsPersister.persistRecords(records);
         }
         List<Influencer> influencers = result.getInfluencers();
         if (influencers != null && !influencers.isEmpty()) {
-            context.bulkResultsPersister.persistInfluencers(influencers);
+            bulkResultsPersister.persistInfluencers(influencers);
         }
         CategoryDefinition categoryDefinition = result.getCategoryDefinition();
         if (categoryDefinition != null) {
@@ -211,16 +232,16 @@ void processResult(Context context, AutodetectResult result) {
         }
         ModelPlot modelPlot = result.getModelPlot();
         if (modelPlot != null) {
-            context.bulkResultsPersister.persistModelPlot(modelPlot);
+            bulkResultsPersister.persistModelPlot(modelPlot);
         }
         Forecast forecast = result.getForecast();
         if (forecast != null) {
-            context.bulkResultsPersister.persistForecast(forecast);
+            bulkResultsPersister.persistForecast(forecast);
         }
         ForecastRequestStats forecastRequestStats = result.getForecastRequestStats();
         if (forecastRequestStats != null) {
             LOGGER.trace("Received Forecast Stats [{}]", forecastRequestStats.getId());
-            context.bulkResultsPersister.persistForecastRequestStats(forecastRequestStats);
+            bulkResultsPersister.persistForecastRequestStats(forecastRequestStats);
 
             // execute the bulk request only in some cases or in doubt
             // otherwise rely on the count-based trigger
@@ -232,13 +253,13 @@ void processResult(Context context, AutodetectResult result) {
                 case SCHEDULED:
                 case FINISHED:
                 default:
-                    context.bulkResultsPersister.executeRequest();
+                    bulkResultsPersister.executeRequest();
 
             }
         }
         ModelSizeStats modelSizeStats = result.getModelSizeStats();
         if (modelSizeStats != null) {
-            processModelSizeStats(context, modelSizeStats);
+            processModelSizeStats(modelSizeStats);
         }
         ModelSnapshot modelSnapshot = result.getModelSnapshot();
         if (modelSnapshot != null) {
@@ -250,55 +271,55 @@ void processResult(Context context, AutodetectResult result) {
         }
         Quantiles quantiles = result.getQuantiles();
         if (quantiles != null) {
-            LOGGER.debug("[{}] Parsed Quantiles with timestamp {}", context.jobId, quantiles.getTimestamp());
+            LOGGER.debug("[{}] Parsed Quantiles with timestamp {}", jobId, quantiles.getTimestamp());
             persister.persistQuantiles(quantiles);
-            context.bulkResultsPersister.executeRequest();
+            bulkResultsPersister.executeRequest();
 
             if (processKilled == false && renormalizer.isEnabled()) {
                 // We need to make all results written up to these quantiles available for renormalization
-                persister.commitResultWrites(context.jobId);
-                LOGGER.debug("[{}] Quantiles queued for renormalization", context.jobId);
+                persister.commitResultWrites(jobId);
+                LOGGER.debug("[{}] Quantiles queued for renormalization", jobId);
                 renormalizer.renormalize(quantiles);
             }
         }
         FlushAcknowledgement flushAcknowledgement = result.getFlushAcknowledgement();
         if (flushAcknowledgement != null) {
-            LOGGER.debug("[{}] Flush acknowledgement parsed from output for ID {}", context.jobId, flushAcknowledgement.getId());
+            LOGGER.debug("[{}] Flush acknowledgement parsed from output for ID {}", jobId, flushAcknowledgement.getId());
             // Commit previous writes here, effectively continuing
             // the flush from the C++ autodetect process right
             // through to the data store
-            context.bulkResultsPersister.executeRequest();
-            persister.commitResultWrites(context.jobId);
+            bulkResultsPersister.executeRequest();
+            persister.commitResultWrites(jobId);
             flushListener.acknowledgeFlush(flushAcknowledgement);
             // Interim results may have been produced by the flush,
             // which need to be
             // deleted when the next finalized results come through
-            context.deleteInterimRequired = true;
+            deleteInterimRequired = true;
         }
     }
 
-    private void processModelSizeStats(Context context, ModelSizeStats modelSizeStats) {
+    private void processModelSizeStats(ModelSizeStats modelSizeStats) {
         LOGGER.trace("[{}] Parsed ModelSizeStats: {} / {} / {} / {} / {} / {}",
-                context.jobId, modelSizeStats.getModelBytes(), modelSizeStats.getTotalByFieldCount(),
+                jobId, modelSizeStats.getModelBytes(), modelSizeStats.getTotalByFieldCount(),
                 modelSizeStats.getTotalOverFieldCount(), modelSizeStats.getTotalPartitionFieldCount(),
                 modelSizeStats.getBucketAllocationFailuresCount(), modelSizeStats.getMemoryStatus());
 
         persister.persistModelSizeStats(modelSizeStats);
-        notifyModelMemoryStatusChange(context, modelSizeStats);
+        notifyModelMemoryStatusChange(modelSizeStats);
         latestModelSizeStats = modelSizeStats;
     }
 
-    private void notifyModelMemoryStatusChange(Context context, ModelSizeStats modelSizeStats) {
+    private void notifyModelMemoryStatusChange(ModelSizeStats modelSizeStats) {
         ModelSizeStats.MemoryStatus memoryStatus = modelSizeStats.getMemoryStatus();
         if (memoryStatus != latestModelSizeStats.getMemoryStatus()) {
             if (memoryStatus == ModelSizeStats.MemoryStatus.SOFT_LIMIT) {
-                auditor.warning(context.jobId, Messages.getMessage(Messages.JOB_AUDIT_MEMORY_STATUS_SOFT_LIMIT));
+                auditor.warning(jobId, Messages.getMessage(Messages.JOB_AUDIT_MEMORY_STATUS_SOFT_LIMIT));
             } else if (memoryStatus == ModelSizeStats.MemoryStatus.HARD_LIMIT) {
                 if (modelSizeStats.getModelBytesMemoryLimit() == null || modelSizeStats.getModelBytesExceeded() == null) {
-                    auditor.error(context.jobId, Messages.getMessage(Messages.JOB_AUDIT_MEMORY_STATUS_HARD_LIMIT_PRE_7_2,
+                    auditor.error(jobId, Messages.getMessage(Messages.JOB_AUDIT_MEMORY_STATUS_HARD_LIMIT_PRE_7_2,
                         new ByteSizeValue(modelSizeStats.getModelBytes(), ByteSizeUnit.BYTES).toString()));
                 } else {
-                    auditor.error(context.jobId, Messages.getMessage(Messages.JOB_AUDIT_MEMORY_STATUS_HARD_LIMIT,
+                    auditor.error(jobId, Messages.getMessage(Messages.JOB_AUDIT_MEMORY_STATUS_HARD_LIMIT,
                         new ByteSizeValue(modelSizeStats.getModelBytesMemoryLimit(), ByteSizeUnit.BYTES).toString(),
                         new ByteSizeValue(modelSizeStats.getModelBytesExceeded(), ByteSizeUnit.BYTES).toString()));
                 }
@@ -390,22 +411,19 @@ public boolean isFailed() {
         return failed;
     }
 
-    static class Context {
-
-        private final String jobId;
-        private JobResultsPersister.Builder bulkResultsPersister;
+    public ModelSizeStats modelSizeStats() {
+        return latestModelSizeStats;
+    }
 
-        boolean deleteInterimRequired;
+    public TimingStats timingStats() {
+        return timingStatsReporter.getCurrentTimingStats();
+    }
 
-        Context(String jobId, JobResultsPersister.Builder bulkResultsPersister) {
-            this.jobId = jobId;
-            this.deleteInterimRequired = true;
-            this.bulkResultsPersister = bulkResultsPersister;
-        }
+    boolean isDeleteInterimRequired() {
+        return deleteInterimRequired;
     }
 
-    public ModelSizeStats modelSizeStats() {
-        return latestModelSizeStats;
+    void setDeleteInterimRequired(boolean deleteInterimRequired) {
+        this.deleteInterimRequired = deleteInterimRequired;
     }
 }
-
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/AutodetectParams.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/AutodetectParams.java
index 9421a3662e1ad..7c56d9db1d08c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/AutodetectParams.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/AutodetectParams.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -24,6 +25,8 @@ public class AutodetectParams {
     private final DataCounts dataCounts;
     private final ModelSizeStats modelSizeStats;
     @Nullable
+    private final TimingStats timingStats;
+    @Nullable
     private final ModelSnapshot modelSnapshot;
     @Nullable
     private final Quantiles quantiles;
@@ -31,12 +34,13 @@ public class AutodetectParams {
     private final List<ScheduledEvent> scheduledEvents;
 
 
-    private AutodetectParams(DataCounts dataCounts, ModelSizeStats modelSizeStats,
+    private AutodetectParams(DataCounts dataCounts, ModelSizeStats modelSizeStats, TimingStats timingStats,
                              @Nullable ModelSnapshot modelSnapshot,
                              @Nullable Quantiles quantiles, Set<MlFilter> filters,
                              List<ScheduledEvent> scheduledEvents) {
         this.dataCounts = Objects.requireNonNull(dataCounts);
         this.modelSizeStats = Objects.requireNonNull(modelSizeStats);
+        this.timingStats = timingStats;
         this.modelSnapshot = modelSnapshot;
         this.quantiles = quantiles;
         this.filters = filters;
@@ -51,6 +55,10 @@ public ModelSizeStats modelSizeStats() {
         return modelSizeStats;
     }
 
+    public TimingStats timingStats() {
+        return timingStats;
+    }
+
     @Nullable
     public ModelSnapshot modelSnapshot() {
         return modelSnapshot;
@@ -83,6 +91,7 @@ public boolean equals(Object other) {
 
         return Objects.equals(this.dataCounts, that.dataCounts)
                 && Objects.equals(this.modelSizeStats, that.modelSizeStats)
+                && Objects.equals(this.timingStats, that.timingStats)
                 && Objects.equals(this.modelSnapshot, that.modelSnapshot)
                 && Objects.equals(this.quantiles, that.quantiles)
                 && Objects.equals(this.filters, that.filters)
@@ -91,13 +100,14 @@ public boolean equals(Object other) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(dataCounts, modelSizeStats, modelSnapshot, quantiles, filters, scheduledEvents);
+        return Objects.hash(dataCounts, modelSizeStats, timingStats, modelSnapshot, quantiles, filters, scheduledEvents);
     }
 
     public static class Builder {
 
         private DataCounts dataCounts;
         private ModelSizeStats modelSizeStats;
+        private TimingStats timingStats;
         private ModelSnapshot modelSnapshot;
         private Quantiles quantiles;
         private Set<MlFilter> filters;
@@ -106,6 +116,7 @@ public static class Builder {
         public Builder(String jobId) {
             dataCounts = new DataCounts(jobId);
             modelSizeStats = new ModelSizeStats.Builder(jobId).build();
+            timingStats = new TimingStats(jobId);
             filters = new HashSet<>();
             scheduledEvents = new ArrayList<>();
         }
@@ -124,6 +135,11 @@ public Builder setModelSizeStats(ModelSizeStats modelSizeStats) {
             return this;
         }
 
+        public Builder setTimingStats(TimingStats timingStats) {
+            this.timingStats = new TimingStats(timingStats);
+            return this;
+        }
+
         public Builder setModelSnapshot(ModelSnapshot modelSnapshot) {
             this.modelSnapshot = modelSnapshot;
             return this;
@@ -150,7 +166,7 @@ public Builder setFilters(Set<MlFilter> filters) {
         }
 
         public AutodetectParams build() {
-            return new AutodetectParams(dataCounts, modelSizeStats, modelSnapshot, quantiles,
+            return new AutodetectParams(dataCounts, modelSizeStats, timingStats, modelSnapshot, quantiles,
                     filters, scheduledEvents);
         }
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/FlushJobParams.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/FlushJobParams.java
index fd813e27fda93..354b2d4c1b669 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/FlushJobParams.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/FlushJobParams.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.util.Objects;
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/TimeRange.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/TimeRange.java
index a14d810d0d2f9..d99a8351c7bef 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/TimeRange.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/TimeRange.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.common.time.TimeUtils;
 
 import java.util.Objects;
 
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriter.java
index dc9d77cd68784..799954619b315 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriter.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriter.java
@@ -122,10 +122,10 @@ public void writeHeader() throws IOException {
 
     /**
      * Tokenize the field that has been configured for categorization, and store the resulting list of tokens in CSV
-     * format in the appropriate field of the record to be sent to the analytics.
+     * format in the appropriate field of the record to be sent to the process.
      * @param categorizationAnalyzer   The analyzer to use to convert the categorization field to a list of tokens
      * @param categorizationFieldValue The value of the categorization field to be tokenized
-     * @param record                   The record to be sent to the analytics
+     * @param record                   The record to be sent to the process
      */
     protected void tokenizeForCategorization(CategorizationAnalyzer categorizationAnalyzer, String categorizationFieldValue,
                                              String[] record) {
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcess.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcess.java
index ec22d35f16872..69c500f9d1637 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcess.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcess.java
@@ -7,6 +7,7 @@
 
 import org.elasticsearch.xpack.ml.job.process.normalizer.output.NormalizerResultHandler;
 import org.elasticsearch.xpack.ml.process.AbstractNativeProcess;
+import org.elasticsearch.xpack.ml.process.NativeController;
 
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -19,8 +20,9 @@ class NativeNormalizerProcess extends AbstractNativeProcess implements Normalize
 
     private static final String NAME = "normalizer";
 
-    NativeNormalizerProcess(String jobId, InputStream logStream, OutputStream processInStream, InputStream processOutStream) {
-        super(jobId, logStream, processInStream, processOutStream, null, 0, Collections.emptyList(), (ignore) -> {});
+    NativeNormalizerProcess(String jobId, NativeController nativeController, InputStream logStream, OutputStream processInStream,
+                            InputStream processOutStream) {
+        super(jobId, nativeController, logStream, processInStream, processOutStream, null, 0, Collections.emptyList(), (ignore) -> {});
     }
 
     @Override
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java
index 21f7229aef123..0022a44f42fb9 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java
@@ -7,10 +7,13 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.process.NativeController;
 import org.elasticsearch.xpack.ml.process.ProcessPipes;
 import org.elasticsearch.xpack.ml.utils.NamedPipeHelper;
@@ -25,14 +28,21 @@ public class NativeNormalizerProcessFactory implements NormalizerProcessFactory
 
     private static final Logger LOGGER = LogManager.getLogger(NativeNormalizerProcessFactory.class);
     private static final NamedPipeHelper NAMED_PIPE_HELPER = new NamedPipeHelper();
-    private static final Duration PROCESS_STARTUP_TIMEOUT = Duration.ofSeconds(10);
 
     private final Environment env;
     private final NativeController nativeController;
+    private volatile Duration processConnectTimeout;
 
-    public NativeNormalizerProcessFactory(Environment env, NativeController nativeController) {
+    public NativeNormalizerProcessFactory(Environment env, NativeController nativeController, ClusterService clusterService) {
         this.env = Objects.requireNonNull(env);
         this.nativeController = Objects.requireNonNull(nativeController);
+        setProcessConnectTimeout(MachineLearning.PROCESS_CONNECT_TIMEOUT.get(env.settings()));
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(MachineLearning.PROCESS_CONNECT_TIMEOUT,
+            this::setProcessConnectTimeout);
+    }
+
+    void setProcessConnectTimeout(TimeValue processConnectTimeout) {
+        this.processConnectTimeout = Duration.ofMillis(processConnectTimeout.getMillis());
     }
 
     @Override
@@ -42,7 +52,7 @@ public NormalizerProcess createNormalizerProcess(String jobId, String quantilesS
                 true, false, true, true, false, false);
         createNativeProcess(jobId, quantilesState, processPipes, bucketSpan);
 
-        NativeNormalizerProcess normalizerProcess = new NativeNormalizerProcess(jobId, processPipes.getLogStream().get(),
+        NativeNormalizerProcess normalizerProcess = new NativeNormalizerProcess(jobId, nativeController, processPipes.getLogStream().get(),
                 processPipes.getProcessInStream().get(), processPipes.getProcessOutStream().get());
 
         try {
@@ -64,7 +74,7 @@ private void createNativeProcess(String jobId, String quantilesState, ProcessPip
             List<String> command = new NormalizerBuilder(env, jobId, quantilesState, bucketSpan).build();
             processPipes.addArgs(command);
             nativeController.startProcess(command);
-            processPipes.connectStreams(PROCESS_STARTUP_TIMEOUT);
+            processPipes.connectStreams(processConnectTimeout);
         } catch (IOException e) {
             String msg = "Failed to launch normalizer for job " + jobId;
             LOGGER.error(msg);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResult.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResult.java
index 269792dbe7797..43f87f33ed7a4 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResult.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResult.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.ml.job.process.normalizer;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -66,9 +65,7 @@ public NormalizerResult(StreamInput in) throws IOException {
         partitionFieldName = in.readOptionalString();
         partitionFieldValue = in.readOptionalString();
         personFieldName = in.readOptionalString();
-        if (in.getVersion().onOrAfter(Version.V_6_5_0)) {
-            personFieldValue = in.readOptionalString();
-        }
+        personFieldValue = in.readOptionalString();
         functionName = in.readOptionalString();
         valueFieldName = in.readOptionalString();
         probability = in.readDouble();
@@ -81,9 +78,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(partitionFieldName);
         out.writeOptionalString(partitionFieldValue);
         out.writeOptionalString(personFieldName);
-        if (out.getVersion().onOrAfter(Version.V_6_5_0)) {
-            out.writeOptionalString(personFieldValue);
-        }
+        out.writeOptionalString(personFieldValue);
         out.writeOptionalString(functionName);
         out.writeOptionalString(valueFieldName);
         out.writeDouble(probability);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java
index 3225a7eb9212e..3556a3bc9a961 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
@@ -143,7 +144,7 @@ private List<ForecastRequestStats> findForecastsToDelete(SearchResponse searchRe
 
     private DeleteByQueryRequest buildDeleteByQuery(List<ForecastRequestStats> forecastsToDelete) {
         DeleteByQueryRequest request = new DeleteByQueryRequest();
-        request.setSlices(5);
+        request.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES);
 
         request.indices(RESULTS_INDEX_PATTERN);
         BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().minimumShouldMatch(1);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java
index d7b16ef8c468e..e24ecfbe8b0f0 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.DeleteByQueryAction;
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
@@ -85,7 +86,7 @@ public void onFailure(Exception e) {
 
     private DeleteByQueryRequest createDBQRequest(Job job, long cutoffEpochMs) {
         DeleteByQueryRequest request = new DeleteByQueryRequest();
-        request.setSlices(5);
+        request.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES);
 
         request.indices(AnomalyDetectorsIndex.jobResultsAliasedName(job.getId()));
         QueryBuilder excludeFilter = QueryBuilders.termsQuery(Result.RESULT_TYPE.getPreferredName(),
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/AbstractNativeProcess.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/AbstractNativeProcess.java
index 25e671a6de1e9..6da48cf1e5383 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/AbstractNativeProcess.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/AbstractNativeProcess.java
@@ -42,6 +42,7 @@ public abstract class AbstractNativeProcess implements NativeProcess {
     private static final Duration WAIT_FOR_KILL_TIMEOUT = Duration.ofMillis(1000);
 
     private final String jobId;
+    private final NativeController nativeController;
     private final CppLogMessageHandler cppLogHandler;
     private final OutputStream processInStream;
     private final InputStream processOutStream;
@@ -57,10 +58,11 @@ public abstract class AbstractNativeProcess implements NativeProcess {
     private volatile boolean processKilled;
     private volatile boolean isReady;
 
-    protected AbstractNativeProcess(String jobId, InputStream logStream, OutputStream processInStream, InputStream processOutStream,
-                                    OutputStream processRestoreStream, int numberOfFields, List<Path> filesToDelete,
-                                    Consumer<String> onProcessCrash) {
+    protected AbstractNativeProcess(String jobId, NativeController nativeController, InputStream logStream, OutputStream processInStream,
+                                    InputStream processOutStream, OutputStream processRestoreStream, int numberOfFields,
+                                    List<Path> filesToDelete, Consumer<String> onProcessCrash) {
         this.jobId = jobId;
+        this.nativeController = nativeController;
         cppLogHandler = new CppLogMessageHandler(jobId, logStream);
         this.processInStream = new BufferedOutputStream(processInStream);
         this.processOutStream = processOutStream;
@@ -177,12 +179,13 @@ public void close() throws IOException {
 
     @Override
     public void kill() throws IOException {
+        LOGGER.debug("[{}] Killing {} process", jobId, getName());
         processKilled = true;
         try {
             // The PID comes via the processes log stream.  We don't wait for it to arrive here,
             // but if the wait times out it implies the process has only just started, in which
             // case it should die very quickly when we close its input stream.
-            NativeControllerHolder.getNativeController().killProcess(cppLogHandler.getPid(Duration.ZERO));
+            nativeController.killProcess(cppLogHandler.getPid(Duration.ZERO));
 
             // Wait for the process to die before closing processInStream as if the process
             // is still alive when processInStream is closed it may start persisting state
@@ -265,4 +268,16 @@ protected LengthEncodedWriter recordWriter() {
     protected boolean isProcessKilled() {
         return processKilled;
     }
+
+    public void consumeAndCloseOutputStream() {
+        try {
+            byte[] buff = new byte[512];
+            while (processOutStream().read(buff) >= 0) {
+                // Do nothing
+            }
+            processOutStream().close();
+        } catch (IOException e) {
+            // Given we are closing down the process there is no point propagating IO exceptions here
+        }
+    }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/DummyController.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/DummyController.java
new file mode 100644
index 0000000000000..b79be3958ca67
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/DummyController.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.process;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * A dummy ML controller for use in internal cluster tests where it
+ * is not possible/appropriate to start a native controller.
+ */
+public class DummyController implements MlController {
+
+    @Override
+    public Map<String, Object> getNativeCodeInfo() {
+        return Collections.emptyMap();
+    }
+
+    @Override
+    public void stop() {
+        // no-op
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlController.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlController.java
new file mode 100644
index 0000000000000..d31b164814a49
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlController.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.process;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Interface for the controller that ML uses to start native processes.
+ */
+public interface MlController {
+
+    /**
+     * Get build/version information about the native code being used by the ML plugin.
+     * @return A map containing native code build/version information.
+     *         This map will be empty in the case where no native code exists.
+     * @throws TimeoutException If native code information cannot be obtained
+     *                          within a reasonable amount of time.
+     */
+    Map<String, Object> getNativeCodeInfo() throws TimeoutException;
+
+    /**
+     * Stop the controller.  For implementations where the controller is an external
+     * process this will instruct that external process to exit, thus preventing any
+     * subsequent controller operations from working.  Stopping the controller is
+     * irreversible; the only way to restart a controller is to restart the
+     * Elasticsearch node.
+     * @throws IOException If it is not possible to communicate the stop request to
+     *                     the controller.
+     */
+    void stop() throws IOException;
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlControllerHolder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlControllerHolder.java
new file mode 100644
index 0000000000000..22ee89bc2c369
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlControllerHolder.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.process;
+
+import org.elasticsearch.node.Node;
+
+import java.util.Objects;
+
+/**
+ * Wrapper for the {@link MlController} interface that allows it to be used
+ * given the way {@link Node} does Guice bindings for plugin components.
+ * TODO: remove this class entirely once Guice is removed entirely.
+ */
+public class MlControllerHolder {
+
+    private MlController mlController;
+
+    public MlControllerHolder(MlController mlController) {
+        this.mlController = Objects.requireNonNull(mlController);
+    }
+
+    public MlController getMlController() {
+        return mlController;
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java
index 50d2515046a22..afd670a180384 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/MlMemoryTracker.java
@@ -20,17 +20,23 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
+import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.ml.MachineLearning;
+import org.elasticsearch.xpack.ml.dataframe.persistence.DataFrameAnalyticsConfigProvider;
 import org.elasticsearch.xpack.ml.job.JobManager;
 import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 
 import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Phaser;
 import java.util.stream.Collectors;
@@ -38,36 +44,49 @@
 /**
  * This class keeps track of the memory requirement of ML jobs.
  * It only functions on the master node - for this reason it should only be used by master node actions.
- * The memory requirement for ML jobs can be updated in 3 ways:
- * 1. For all open ML jobs (via {@link #asyncRefresh})
- * 2. For all open ML jobs, plus one named ML job that is not open (via {@link #refreshJobMemoryAndAllOthers})
- * 3. For one named ML job (via {@link #refreshJobMemory})
- * In cases 2 and 3 a listener informs the caller when the requested updates are complete.
+ * The memory requirement for ML jobs can be updated in 4 ways:
+ * 1. For all open ML data frame analytics jobs and anomaly detector jobs (via {@link #asyncRefresh})
+ * 2. For all open/started ML jobs, plus one named ML anomaly detector job that is not open
+ *    (via {@link #refreshAnomalyDetectorJobMemoryAndAllOthers})
+ * 3. For all open/started ML jobs, plus one named ML data frame analytics job that is not started
+ *    (via {@link #addDataFrameAnalyticsJobMemoryAndRefreshAllOthers})
+ * 4. For one named ML anomaly detector job (via {@link #refreshAnomalyDetectorJobMemory})
+ * In cases 2, 3 and 4 a listener informs the caller when the requested updates are complete.
  */
 public class MlMemoryTracker implements LocalNodeMasterListener {
 
     private static final Duration RECENT_UPDATE_THRESHOLD = Duration.ofMinutes(1);
 
     private final Logger logger = LogManager.getLogger(MlMemoryTracker.class);
-    private final ConcurrentHashMap<String, Long> memoryRequirementByJob = new ConcurrentHashMap<>();
+    private final Map<String, Long> memoryRequirementByAnomalyDetectorJob = new ConcurrentHashMap<>();
+    private final Map<String, Long> memoryRequirementByDataFrameAnalyticsJob = new ConcurrentHashMap<>();
+    private final Map<String, Map<String, Long>> memoryRequirementByTaskName;
     private final List<ActionListener<Void>> fullRefreshCompletionListeners = new ArrayList<>();
 
     private final ThreadPool threadPool;
     private final ClusterService clusterService;
     private final JobManager jobManager;
     private final JobResultsProvider jobResultsProvider;
+    private final DataFrameAnalyticsConfigProvider configProvider;
     private final Phaser stopPhaser;
     private volatile boolean isMaster;
     private volatile Instant lastUpdateTime;
     private volatile Duration reassignmentRecheckInterval;
 
     public MlMemoryTracker(Settings settings, ClusterService clusterService, ThreadPool threadPool, JobManager jobManager,
-                           JobResultsProvider jobResultsProvider) {
+                           JobResultsProvider jobResultsProvider, DataFrameAnalyticsConfigProvider configProvider) {
         this.threadPool = threadPool;
         this.clusterService = clusterService;
         this.jobManager = jobManager;
         this.jobResultsProvider = jobResultsProvider;
+        this.configProvider = configProvider;
         this.stopPhaser = new Phaser(1);
+
+        Map<String, Map<String, Long>> memoryRequirementByTaskName = new TreeMap<>();
+        memoryRequirementByTaskName.put(MlTasks.JOB_TASK_NAME, memoryRequirementByAnomalyDetectorJob);
+        memoryRequirementByTaskName.put(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, memoryRequirementByDataFrameAnalyticsJob);
+        this.memoryRequirementByTaskName = Collections.unmodifiableMap(memoryRequirementByTaskName);
+
         setReassignmentRecheckInterval(PersistentTasksClusterService.CLUSTER_TASKS_ALLOCATION_RECHECK_INTERVAL_SETTING.get(settings));
         clusterService.addLocalNodeMasterListener(this);
         clusterService.getClusterSettings().addSettingsUpdateConsumer(
@@ -88,7 +107,9 @@ public void onMaster() {
     public void offMaster() {
         isMaster = false;
         logger.trace("ML memory tracker off master");
-        memoryRequirementByJob.clear();
+        for (Map<String, Long> memoryRequirementByJob : memoryRequirementByTaskName.values()) {
+            memoryRequirementByJob.clear();
+        }
         lastUpdateTime = null;
     }
 
@@ -125,37 +146,69 @@ public boolean isRecentlyRefreshed() {
     }
 
     /**
-     * Get the memory requirement for a job.
+     * Get the memory requirement for an anomaly detector job.
      * This method only works on the master node.
      * @param jobId The job ID.
      * @return The memory requirement of the job specified by {@code jobId},
      *         or <code>null</code> if it cannot be calculated.
      */
-    public Long getJobMemoryRequirement(String jobId) {
+    public Long getAnomalyDetectorJobMemoryRequirement(String jobId) {
+        return getJobMemoryRequirement(MlTasks.JOB_TASK_NAME, jobId);
+    }
+
+    /**
+     * Get the memory requirement for a data frame analytics job.
+     * This method only works on the master node.
+     * @param id The job ID.
+     * @return The memory requirement of the job specified by {@code id},
+     *         or <code>null</code> if it cannot be found.
+     */
+    public Long getDataFrameAnalyticsJobMemoryRequirement(String id) {
+        return getJobMemoryRequirement(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, id);
+    }
+
+    /**
+     * Get the memory requirement for the type of job corresponding to a specified persistent task name.
+     * This method only works on the master node.
+     * @param taskName The persistent task name.
+     * @param id The job ID.
+     * @return The memory requirement of the job specified by {@code id},
+     *         or <code>null</code> if it cannot be found.
+     */
+    public Long getJobMemoryRequirement(String taskName, String id) {
 
         if (isMaster == false) {
             return null;
         }
 
-        Long memoryRequirement = memoryRequirementByJob.get(jobId);
-        if (memoryRequirement != null) {
-            return memoryRequirement;
+        Map<String, Long> memoryRequirementByJob = memoryRequirementByTaskName.get(taskName);
+        if (memoryRequirementByJob == null) {
+            return null;
         }
 
-        return null;
+        return memoryRequirementByJob.get(id);
+    }
+
+    /**
+     * Remove any memory requirement that is stored for the specified anomaly detector job.
+     * It doesn't matter if this method is called for a job that doesn't have a
+     * stored memory requirement.
+     */
+    public void removeAnomalyDetectorJob(String jobId) {
+        memoryRequirementByAnomalyDetectorJob.remove(jobId);
     }
 
     /**
-     * Remove any memory requirement that is stored for the specified job.
-     * It doesn't matter if this method is called for a job that doesn't have
-     * a stored memory requirement.
+     * Remove any memory requirement that is stored for the specified data frame analytics
+     * job.  It doesn't matter if this method is called for a job that doesn't have a
+     * stored memory requirement.
      */
-    public void removeJob(String jobId) {
-        memoryRequirementByJob.remove(jobId);
+    public void removeDataFrameAnalyticsJob(String id) {
+        memoryRequirementByDataFrameAnalyticsJob.remove(id);
     }
 
     /**
-     * Uses a separate thread to refresh the memory requirement for every ML job that has
+     * Uses a separate thread to refresh the memory requirement for every ML anomaly detector job that has
      * a corresponding persistent task.  This method only works on the master node.
      * @return <code>true</code> if the async refresh is scheduled, and <code>false</code>
      *         if this is not possible for some reason.
@@ -188,7 +241,7 @@ public boolean asyncRefresh() {
      * @param listener Receives the memory requirement of the job specified by {@code jobId},
      *                 or <code>null</code> if it cannot be calculated.
      */
-    public void refreshJobMemoryAndAllOthers(String jobId, ActionListener<Long> listener) {
+    public void refreshAnomalyDetectorJobMemoryAndAllOthers(String jobId, ActionListener<Long> listener) {
 
         if (isMaster == false) {
             listener.onResponse(null);
@@ -196,14 +249,36 @@ public void refreshJobMemoryAndAllOthers(String jobId, ActionListener<Long> list
         }
 
         PersistentTasksCustomMetaData persistentTasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
-        refresh(persistentTasks, ActionListener.wrap(aVoid -> refreshJobMemory(jobId, listener), listener::onFailure));
+        refresh(persistentTasks,
+            ActionListener.wrap(aVoid -> refreshAnomalyDetectorJobMemory(jobId, listener), listener::onFailure));
+    }
+
+    /**
+     * This refreshes the memory requirement for every ML job that has a corresponding
+     * persistent task and, in addition, adds the memory requirement of one data frame analytics
+     * job that doesn't have a persistent task.  This method only works on the master node.
+     * @param id The job ID of the job whose memory requirement is to be added.
+     * @param mem The memory requirement (in bytes) of the job specified by {@code id}.
+     * @param listener Called when the refresh is complete or fails.
+     */
+    public void addDataFrameAnalyticsJobMemoryAndRefreshAllOthers(String id, long mem, ActionListener<Void> listener) {
+
+        if (isMaster == false) {
+            listener.onResponse(null);
+            return;
+        }
+
+        memoryRequirementByDataFrameAnalyticsJob.put(id, mem + DataFrameAnalyticsConfig.PROCESS_MEMORY_OVERHEAD.getBytes());
+
+        PersistentTasksCustomMetaData persistentTasks = clusterService.state().getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
+        refresh(persistentTasks, listener);
     }
 
     /**
      * This refreshes the memory requirement for every ML job that has a corresponding persistent task.
-     * It does NOT remove entries for jobs that no longer have a persistent task, because that would
-     * lead to a race where a job was opened part way through the refresh.  (Instead, entries are removed
-     * when jobs are deleted.)
+     * It does NOT remove entries for jobs that no longer have a persistent task, because that would lead
+     * to a race where a job was opened part way through the refresh.  (Instead, entries are removed when
+     * jobs are deleted.)
      */
     void refresh(PersistentTasksCustomMetaData persistentTasks, ActionListener<Void> onCompletion) {
 
@@ -230,37 +305,65 @@ void refresh(PersistentTasksCustomMetaData persistentTasks, ActionListener<Void>
         if (persistentTasks == null) {
             refreshComplete.onResponse(null);
         } else {
-            List<PersistentTasksCustomMetaData.PersistentTask<?>> mlJobTasks = persistentTasks.tasks().stream()
+            List<PersistentTasksCustomMetaData.PersistentTask<?>> mlDataFrameAnalyticsJobTasks = persistentTasks.tasks().stream()
+                .filter(task -> MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME.equals(task.getTaskName())).collect(Collectors.toList());
+            ActionListener<Void> refreshDataFrameAnalyticsJobs =
+                ActionListener.wrap(aVoid -> refreshAllDataFrameAnalyticsJobTasks(mlDataFrameAnalyticsJobTasks, refreshComplete),
+                    refreshComplete::onFailure);
+
+            List<PersistentTasksCustomMetaData.PersistentTask<?>> mlAnomalyDetectorJobTasks = persistentTasks.tasks().stream()
                 .filter(task -> MlTasks.JOB_TASK_NAME.equals(task.getTaskName())).collect(Collectors.toList());
-            iterateMlJobTasks(mlJobTasks.iterator(), refreshComplete);
+            iterateAnomalyDetectorJobTasks(mlAnomalyDetectorJobTasks.iterator(), refreshDataFrameAnalyticsJobs);
         }
     }
 
-    private void iterateMlJobTasks(Iterator<PersistentTasksCustomMetaData.PersistentTask<?>> iterator,
-                                   ActionListener<Void> refreshComplete) {
+    private void iterateAnomalyDetectorJobTasks(Iterator<PersistentTasksCustomMetaData.PersistentTask<?>> iterator,
+                                                ActionListener<Void> refreshComplete) {
         if (iterator.hasNext()) {
             OpenJobAction.JobParams jobParams = (OpenJobAction.JobParams) iterator.next().getParams();
-            refreshJobMemory(jobParams.getJobId(),
+            refreshAnomalyDetectorJobMemory(jobParams.getJobId(),
                 ActionListener.wrap(
                     // Do the next iteration in a different thread, otherwise stack overflow
                     // can occur if the searches happen to be on the local node, as the huge
                     // chain of listeners are all called in the same thread if only one node
                     // is involved
-                    mem -> threadPool.executor(executorName()).execute(() -> iterateMlJobTasks(iterator, refreshComplete)),
+                    mem -> threadPool.executor(executorName()).execute(() -> iterateAnomalyDetectorJobTasks(iterator, refreshComplete)),
                     refreshComplete::onFailure));
         } else {
             refreshComplete.onResponse(null);
         }
     }
 
+    private void refreshAllDataFrameAnalyticsJobTasks(List<PersistentTasksCustomMetaData.PersistentTask<?>> mlDataFrameAnalyticsJobTasks,
+                                                      ActionListener<Void> listener) {
+        if (mlDataFrameAnalyticsJobTasks.isEmpty()) {
+            listener.onResponse(null);
+            return;
+        }
+
+        String startedJobIds = mlDataFrameAnalyticsJobTasks.stream()
+            .map(task -> ((StartDataFrameAnalyticsAction.TaskParams) task.getParams()).getId()).sorted().collect(Collectors.joining(","));
+
+        configProvider.getMultiple(startedJobIds, false, ActionListener.wrap(
+            analyticsConfigs -> {
+                for (DataFrameAnalyticsConfig analyticsConfig : analyticsConfigs) {
+                    memoryRequirementByDataFrameAnalyticsJob.put(analyticsConfig.getId(),
+                        analyticsConfig.getModelMemoryLimit().getBytes() + DataFrameAnalyticsConfig.PROCESS_MEMORY_OVERHEAD.getBytes());
+                }
+                listener.onResponse(null);
+            },
+            listener::onFailure
+        ));
+    }
+
     /**
-     * Refresh the memory requirement for a single job.
+     * Refresh the memory requirement for a single anomaly detector job.
      * This method only works on the master node.
      * @param jobId    The ID of the job to refresh the memory requirement for.
      * @param listener Receives the job's memory requirement, or <code>null</code>
      *                 if it cannot be calculated.
      */
-    public void refreshJobMemory(String jobId, ActionListener<Long> listener) {
+    public void refreshAnomalyDetectorJobMemory(String jobId, ActionListener<Long> listener) {
         if (isMaster == false) {
             listener.onResponse(null);
             return;
@@ -288,25 +391,25 @@ public void refreshJobMemory(String jobId, ActionListener<Long> listener) {
             jobResultsProvider.getEstablishedMemoryUsage(jobId, null, null,
                 establishedModelMemoryBytes -> {
                     if (establishedModelMemoryBytes <= 0L) {
-                        setJobMemoryToLimit(jobId, phaserListener);
+                        setAnomalyDetectorJobMemoryToLimit(jobId, phaserListener);
                     } else {
                         Long memoryRequirementBytes = establishedModelMemoryBytes + Job.PROCESS_MEMORY_OVERHEAD.getBytes();
-                        memoryRequirementByJob.put(jobId, memoryRequirementBytes);
+                        memoryRequirementByAnomalyDetectorJob.put(jobId, memoryRequirementBytes);
                         phaserListener.onResponse(memoryRequirementBytes);
                     }
                 },
                 e -> {
-                    logger.error("[" + jobId + "] failed to calculate job established model memory requirement", e);
-                    setJobMemoryToLimit(jobId, phaserListener);
+                    logger.error("[" + jobId + "] failed to calculate anomaly detector job established model memory requirement", e);
+                    setAnomalyDetectorJobMemoryToLimit(jobId, phaserListener);
                 }
             );
         } catch (Exception e) {
-            logger.error("[" + jobId + "] failed to calculate job established model memory requirement", e);
-            setJobMemoryToLimit(jobId, phaserListener);
+            logger.error("[" + jobId + "] failed to calculate anomaly detector job established model memory requirement", e);
+            setAnomalyDetectorJobMemoryToLimit(jobId, phaserListener);
         }
     }
 
-    private void setJobMemoryToLimit(String jobId, ActionListener<Long> listener) {
+    private void setAnomalyDetectorJobMemoryToLimit(String jobId, ActionListener<Long> listener) {
         jobManager.getJob(jobId, ActionListener.wrap(job -> {
             Long memoryLimitMb = (job.getAnalysisLimits() != null) ? job.getAnalysisLimits().getModelMemoryLimit() : null;
             // Although recent versions of the code enforce a non-null model_memory_limit
@@ -316,16 +419,16 @@ private void setJobMemoryToLimit(String jobId, ActionListener<Long> listener) {
                 memoryLimitMb = AnalysisLimits.PRE_6_1_DEFAULT_MODEL_MEMORY_LIMIT_MB;
             }
             Long memoryRequirementBytes = ByteSizeUnit.MB.toBytes(memoryLimitMb) + Job.PROCESS_MEMORY_OVERHEAD.getBytes();
-            memoryRequirementByJob.put(jobId, memoryRequirementBytes);
+            memoryRequirementByAnomalyDetectorJob.put(jobId, memoryRequirementBytes);
             listener.onResponse(memoryRequirementBytes);
         }, e -> {
             if (e instanceof ResourceNotFoundException) {
                 // TODO: does this also happen if the .ml-config index exists but is unavailable?
-                logger.trace("[{}] job deleted during ML memory update", jobId);
+                logger.trace("[{}] anomaly detector job deleted during ML memory update", jobId);
             } else {
-                logger.error("[" + jobId + "] failed to get job during ML memory update", e);
+                logger.error("[" + jobId + "] failed to get anomaly detector job during ML memory update", e);
             }
-            memoryRequirementByJob.remove(jobId);
+            memoryRequirementByAnomalyDetectorJob.remove(jobId);
             listener.onResponse(null);
         }));
     }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeController.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeController.java
index 5dfa86ad22583..602782cc4c930 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeController.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeController.java
@@ -26,7 +26,7 @@
  * Maintains the connection to the native controller daemon that can start other processes.
  */
 @SuppressWarnings("ALL")
-public class NativeController {
+public class NativeController implements MlController {
     private static final Logger LOGGER = LogManager.getLogger(NativeController.class);
 
     /**
@@ -41,12 +41,17 @@ public class NativeController {
     private static final String START_COMMAND = "start";
     private static final String KILL_COMMAND = "kill";
 
-    public static final Map<String, Object> UNKNOWN_NATIVE_CODE_INFO = Map.of("version", "N/A", "build_hash", "N/A");
-
     private final String localNodeName;
     private final CppLogMessageHandler cppLogHandler;
     private final OutputStream commandStream;
 
+    public static NativeController makeNativeController(String localNodeName, Environment env)
+        throws IOException {
+        NativeController nativeController = new NativeController(localNodeName, env, new NamedPipeHelper());
+        nativeController.tailLogsInThread();
+        return nativeController;
+    }
+
     NativeController(String localNodeName, Environment env, NamedPipeHelper namedPipeHelper) throws IOException {
         ProcessPipes processPipes = new ProcessPipes(env, namedPipeHelper, CONTROLLER, null,
                 true, true, false, false, false, false);
@@ -80,6 +85,7 @@ public long getPid() throws TimeoutException {
         return cppLogHandler.getPid(CONTROLLER_CONNECT_TIMEOUT);
     }
 
+    @Override
     public Map<String, Object> getNativeCodeInfo() throws TimeoutException {
         return cppLogHandler.getNativeCodeInfo(CONTROLLER_CONNECT_TIMEOUT);
     }
@@ -143,6 +149,7 @@ public void killProcess(long pid) throws TimeoutException, IOException {
         }
     }
 
+    @Override
     public void stop() throws IOException {
         // The C++ process will exit when it gets EOF on the command stream
         commandStream.close();
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeControllerHolder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeControllerHolder.java
deleted file mode 100644
index 5365a11f6b560..0000000000000
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeControllerHolder.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.ml.process;
-
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.xpack.core.ml.MachineLearningField;
-import org.elasticsearch.xpack.ml.utils.NamedPipeHelper;
-
-import java.io.IOException;
-
-/**
- * Manages a singleton NativeController so that both the MachineLearningFeatureSet and MachineLearning classes can
- * get access to the same one.
- */
-public class NativeControllerHolder {
-
-    private static final Object lock = new Object();
-    private static NativeController nativeController;
-
-    private NativeControllerHolder() {
-    }
-
-    /**
-     * Get a reference to the singleton native process controller.
-     *
-     * The NativeController is created lazily to allow time for the C++ process to be started before connection is attempted.
-     *
-     * <code>null</code> is returned to tests where xpack.ml.autodetect_process=false.
-     *
-     * Calls may throw an exception if initial connection to the C++ process fails.
-     */
-    public static NativeController getNativeController(String localNodeName, Environment environment) throws IOException {
-
-        if (MachineLearningField.AUTODETECT_PROCESS.get(environment.settings())) {
-            synchronized (lock) {
-                if (nativeController == null) {
-                    nativeController = new NativeController(localNodeName, environment, new NamedPipeHelper());
-                    nativeController.tailLogsInThread();
-                }
-            }
-            return nativeController;
-        }
-        return null;
-    }
-
-    /**
-     * Get a reference to the singleton native process controller.
-     *
-     * Assumes that if it is possible for a native controller to exist that it will already have been created.
-     * Designed for use by objects that don't have access to the environment but know a native controller must exist
-     * for the object calling this method to exist.
-     */
-    public static NativeController getNativeController() {
-        return nativeController;
-    }
-
-}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParser.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/ProcessResultsParser.java
similarity index 72%
rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParser.java
rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/ProcessResultsParser.java
index 2ec37a0f86e5d..609c45659dd6c 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParser.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/ProcessResultsParser.java
@@ -3,31 +3,41 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.ml.job.process.autodetect.output;
+package org.elasticsearch.xpack.ml.process;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.xpack.ml.job.results.AutodetectResult;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Iterator;
+import java.util.Objects;
 
 
 /**
- * Parses the JSON output of the autodetect program.
+ * Parses the JSON output of a process.
  * <p>
- * Expects an array of buckets so the first element will always be the
+ * Expects an array of objects so the first element will always be the
  * start array symbol and the data must be terminated with the end array symbol.
  */
-public class AutodetectResultsParser {
-    public Iterator<AutodetectResult> parseResults(InputStream in) throws ElasticsearchParseException {
+public class ProcessResultsParser<T> {
+
+    private static final Logger logger = LogManager.getLogger(ProcessResultsParser.class);
+
+    private final ConstructingObjectParser<T, Void> resultParser;
+
+    public ProcessResultsParser(ConstructingObjectParser<T, Void> resultParser) {
+        this.resultParser = Objects.requireNonNull(resultParser);
+    }
+
+    public Iterator<T> parseResults(InputStream in) throws ElasticsearchParseException {
         try {
             XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                     .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, in);
@@ -36,21 +46,19 @@ public Iterator<AutodetectResult> parseResults(InputStream in) throws Elasticsea
             if (token != XContentParser.Token.START_ARRAY) {
                 throw new ElasticsearchParseException("unexpected token [" + token + "]");
             }
-            return new AutodetectResultIterator(in, parser);
+            return new ResultIterator(in, parser);
         } catch (IOException e) {
             throw new ElasticsearchParseException(e.getMessage(), e);
         }
     }
 
-    private static class AutodetectResultIterator implements Iterator<AutodetectResult> {
-
-        private static final Logger logger = LogManager.getLogger(AutodetectResultIterator.class);
+    private class ResultIterator implements Iterator<T> {
 
         private final InputStream in;
         private final XContentParser parser;
         private XContentParser.Token token;
 
-        private AutodetectResultIterator(InputStream in, XContentParser parser) {
+        private ResultIterator(InputStream in, XContentParser parser) {
             this.in = in;
             this.parser = parser;
             token = parser.currentToken();
@@ -74,8 +82,8 @@ public boolean hasNext() {
         }
 
         @Override
-        public AutodetectResult next() {
-            return AutodetectResult.PARSER.apply(parser, null);
+        public T next() {
+            return resultParser.apply(parser, null);
         }
     }
 }
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestFindFileStructureAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestFindFileStructureAction.java
index 5810a2e929daa..03c3fb2a39f54 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestFindFileStructureAction.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/RestFindFileStructureAction.java
@@ -53,6 +53,8 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient
         FindFileStructureAction.Request request = new FindFileStructureAction.Request();
         request.setLinesToSample(restRequest.paramAsInt(FindFileStructureAction.Request.LINES_TO_SAMPLE.getPreferredName(),
             FileStructureFinderManager.DEFAULT_IDEAL_SAMPLE_LINE_COUNT));
+        request.setLineMergeSizeLimit(restRequest.paramAsInt(FindFileStructureAction.Request.LINE_MERGE_SIZE_LIMIT.getPreferredName(),
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT));
         request.setTimeout(TimeValue.parseTimeValue(restRequest.param(FindFileStructureAction.Request.TIMEOUT.getPreferredName()),
             DEFAULT_TIMEOUT, FindFileStructureAction.Request.TIMEOUT.getPreferredName()));
         request.setCharset(restRequest.param(FindFileStructureAction.Request.CHARSET.getPreferredName()));
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestDeleteDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestDeleteDataFrameAnalyticsAction.java
new file mode 100644
index 0000000000000..31a9ba690a9b2
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestDeleteDataFrameAnalyticsAction.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.rest.dataframe;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.ml.action.DeleteDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.ml.MachineLearning;
+
+import java.io.IOException;
+
+public class RestDeleteDataFrameAnalyticsAction extends BaseRestHandler {
+
+    public RestDeleteDataFrameAnalyticsAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.DELETE, MachineLearning.BASE_PATH + "data_frame/analytics/{"
+            + DataFrameAnalyticsConfig.ID.getPreferredName() + "}", this);
+    }
+
+    @Override
+    public String getName() {
+        return "xpack_ml_delete_data_frame_analytics_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        String id = restRequest.param(DataFrameAnalyticsConfig.ID.getPreferredName());
+        DeleteDataFrameAnalyticsAction.Request request = new DeleteDataFrameAnalyticsAction.Request(id);
+        return channel -> client.execute(DeleteDataFrameAnalyticsAction.INSTANCE, request, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestEvaluateDataFrameAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestEvaluateDataFrameAction.java
new file mode 100644
index 0000000000000..3b514e1283859
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestEvaluateDataFrameAction.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.rest.dataframe;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.ml.action.EvaluateDataFrameAction;
+import org.elasticsearch.xpack.ml.MachineLearning;
+
+import java.io.IOException;
+
+public class RestEvaluateDataFrameAction extends BaseRestHandler {
+
+    public RestEvaluateDataFrameAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.POST, MachineLearning.BASE_PATH + "data_frame/_evaluate", this);
+    }
+
+    @Override
+    public String getName() {
+        return "ml_evaluate_data_frame_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        EvaluateDataFrameAction.Request request = EvaluateDataFrameAction.Request.parseRequest(restRequest.contentOrSourceParamParser());
+        return channel -> client.execute(EvaluateDataFrameAction.INSTANCE, request, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java
new file mode 100644
index 0000000000000..b37ff2b7e5982
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.rest.dataframe;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.action.util.PageParams;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.ml.MachineLearning;
+
+import java.io.IOException;
+
+public class RestGetDataFrameAnalyticsAction extends BaseRestHandler {
+
+    public RestGetDataFrameAnalyticsAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.GET, MachineLearning.BASE_PATH + "data_frame/analytics", this);
+        controller.registerHandler(RestRequest.Method.GET, MachineLearning.BASE_PATH + "data_frame/analytics/{"
+            + DataFrameAnalyticsConfig.ID.getPreferredName() + "}", this);
+    }
+
+    @Override
+    public String getName() {
+        return "xpack_ml_get_data_frame_analytics_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        GetDataFrameAnalyticsAction.Request request = new GetDataFrameAnalyticsAction.Request();
+        String id = restRequest.param(DataFrameAnalyticsConfig.ID.getPreferredName());
+        if (Strings.isNullOrEmpty(id) == false) {
+            request.setResourceId(id);
+        }
+        if (restRequest.hasParam(PageParams.FROM.getPreferredName()) || restRequest.hasParam(PageParams.SIZE.getPreferredName())) {
+            request.setPageParams(new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), PageParams.DEFAULT_FROM),
+                    restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), PageParams.DEFAULT_SIZE)));
+        }
+        request.setAllowNoResources(restRequest.paramAsBoolean(GetDataFrameAnalyticsAction.Request.ALLOW_NO_MATCH.getPreferredName(),
+                request.isAllowNoResources()));
+        return channel -> client.execute(GetDataFrameAnalyticsAction.INSTANCE, request, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsStatsAction.java
new file mode 100644
index 0000000000000..3c363762817ba
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsStatsAction.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.rest.dataframe;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.action.util.PageParams;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.ml.MachineLearning;
+
+import java.io.IOException;
+
+public class RestGetDataFrameAnalyticsStatsAction extends BaseRestHandler {
+
+    public RestGetDataFrameAnalyticsStatsAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.GET, MachineLearning.BASE_PATH + "data_frame/analytics/_stats", this);
+        controller.registerHandler(RestRequest.Method.GET, MachineLearning.BASE_PATH + "data_frame/analytics/{"
+            + DataFrameAnalyticsConfig.ID.getPreferredName() + "}/_stats", this);
+    }
+
+    @Override
+    public String getName() {
+        return "xpack_ml_get_data_frame_analytics_stats_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        String id = restRequest.param(DataFrameAnalyticsConfig.ID.getPreferredName());
+        GetDataFrameAnalyticsStatsAction.Request request = new GetDataFrameAnalyticsStatsAction.Request();
+        if (Strings.isNullOrEmpty(id) == false) {
+            request.setId(id);
+        }
+        if (restRequest.hasParam(PageParams.FROM.getPreferredName()) || restRequest.hasParam(PageParams.SIZE.getPreferredName())) {
+            request.setPageParams(new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), PageParams.DEFAULT_FROM),
+                    restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), PageParams.DEFAULT_SIZE)));
+        }
+        request.setAllowNoMatch(restRequest.paramAsBoolean(GetDataFrameAnalyticsStatsAction.Request.ALLOW_NO_MATCH.getPreferredName(),
+            request.isAllowNoMatch()));
+
+        return channel -> client.execute(GetDataFrameAnalyticsStatsAction.INSTANCE, request, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestPutDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestPutDataFrameAnalyticsAction.java
new file mode 100644
index 0000000000000..e2422c6cdeba9
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestPutDataFrameAnalyticsAction.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.rest.dataframe;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.ml.MachineLearning;
+
+import java.io.IOException;
+
+public class RestPutDataFrameAnalyticsAction extends BaseRestHandler {
+
+    public RestPutDataFrameAnalyticsAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.PUT, MachineLearning.BASE_PATH + "data_frame/analytics/{"
+            + DataFrameAnalyticsConfig.ID.getPreferredName() + "}", this);
+    }
+
+    @Override
+    public String getName() {
+        return "xpack_ml_put_data_frame_analytics_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        String id = restRequest.param(DataFrameAnalyticsConfig.ID.getPreferredName());
+        XContentParser parser = restRequest.contentParser();
+        PutDataFrameAnalyticsAction.Request putRequest = PutDataFrameAnalyticsAction.Request.parseRequest(id, parser);
+        putRequest.timeout(restRequest.paramAsTime("timeout", putRequest.timeout()));
+
+        return channel -> client.execute(PutDataFrameAnalyticsAction.INSTANCE, putRequest, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestStartDataFrameAnalyticsAction.java
new file mode 100644
index 0000000000000..7502f31375f1a
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestStartDataFrameAnalyticsAction.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.rest.dataframe;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.ml.MachineLearning;
+
+import java.io.IOException;
+
+public class RestStartDataFrameAnalyticsAction extends BaseRestHandler {
+
+    public RestStartDataFrameAnalyticsAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.POST, MachineLearning.BASE_PATH + "data_frame/analytics/{"
+            + DataFrameAnalyticsConfig.ID.getPreferredName() + "}/_start", this);
+    }
+
+    @Override
+    public String getName() {
+        return "xpack_ml_start_data_frame_analytics_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        String id = restRequest.param(DataFrameAnalyticsConfig.ID.getPreferredName());
+        StartDataFrameAnalyticsAction.Request request;
+        if (restRequest.hasContentOrSourceParam()) {
+            request = StartDataFrameAnalyticsAction.Request.parseRequest(id, restRequest.contentOrSourceParamParser());
+        } else {
+            request = new StartDataFrameAnalyticsAction.Request(id);
+            if (restRequest.hasParam(StartDataFrameAnalyticsAction.Request.TIMEOUT.getPreferredName())) {
+                TimeValue timeout = restRequest.paramAsTime(StartDataFrameAnalyticsAction.Request.TIMEOUT.getPreferredName(),
+                    request.getTimeout());
+                request.setTimeout(timeout);
+            }
+        }
+        return channel -> client.execute(StartDataFrameAnalyticsAction.INSTANCE, request, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestStopDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestStopDataFrameAnalyticsAction.java
new file mode 100644
index 0000000000000..65f1d40273540
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestStopDataFrameAnalyticsAction.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.rest.dataframe;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+import org.elasticsearch.xpack.core.ml.action.StopDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.ml.MachineLearning;
+
+import java.io.IOException;
+
+public class RestStopDataFrameAnalyticsAction extends BaseRestHandler {
+
+    public RestStopDataFrameAnalyticsAction(Settings settings, RestController controller) {
+        super(settings);
+        controller.registerHandler(RestRequest.Method.POST, MachineLearning.BASE_PATH + "data_frame/analytics/{"
+            + DataFrameAnalyticsConfig.ID.getPreferredName() + "}/_stop", this);
+    }
+
+    @Override
+    public String getName() {
+        return "xpack_ml_stop_data_frame_analytics_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
+        String id = restRequest.param(DataFrameAnalyticsConfig.ID.getPreferredName());
+        StopDataFrameAnalyticsAction.Request request;
+        if (restRequest.hasContentOrSourceParam()) {
+            request = StopDataFrameAnalyticsAction.Request.parseRequest(id, restRequest.contentOrSourceParamParser());
+        } else {
+            request = new StopDataFrameAnalyticsAction.Request(id);
+            request.setTimeout(restRequest.paramAsTime(StopDataFrameAnalyticsAction.Request.TIMEOUT.getPreferredName(),
+                request.getTimeout()));
+            request.setAllowNoMatch(restRequest.paramAsBoolean(StopDataFrameAnalyticsAction.Request.ALLOW_NO_MATCH.getPreferredName(),
+                request.allowNoMatch()));
+            request.setForce(restRequest.paramAsBoolean(StopDataFrameAnalyticsAction.Request.FORCE.getPreferredName(), request.isForce()));
+        }
+        return channel -> client.execute(StopDataFrameAnalyticsAction.INSTANCE, request, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security-test.policy b/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security-test.policy
new file mode 100644
index 0000000000000..d090016eac620
--- /dev/null
+++ b/x-pack/plugin/ml/src/main/plugin-metadata/plugin-security-test.policy
@@ -0,0 +1,5 @@
+// Needed for painless script to run
+grant {
+  // needed to create the classloader which allows plugins to extend other plugins
+  permission java.lang.RuntimePermission "createClassLoader";
+};
\ No newline at end of file
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
index 9e0692672349d..19da0d1a682e5 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.license.License.OperationMode;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteDatafeedAction;
@@ -26,7 +25,6 @@
 import org.elasticsearch.xpack.core.ml.action.PutJobAction;
 import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction;
-import org.elasticsearch.xpack.core.ml.client.MachineLearningClient;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
@@ -39,7 +37,6 @@
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
 
-@TestLogging("org.elasticsearch.xpack.ml.action:DEBUG")
 public class MachineLearningLicensingTests extends BaseMlIntegTestCase {
 
     @Before
@@ -60,7 +57,7 @@ public void testMachineLearningPutJobActionRestricted() {
         // test that license restricted apis do not work
         ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> {
             PlainActionFuture<PutJobAction.Response> listener = PlainActionFuture.newFuture();
-            new MachineLearningClient(client()).putJob(new PutJobAction.Request(createJob(jobId)), listener);
+            client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(createJob(jobId)), listener);
             listener.actionGet();
         });
         assertThat(e.status(), is(RestStatus.FORBIDDEN));
@@ -73,7 +70,7 @@ public void testMachineLearningPutJobActionRestricted() {
         assertMLAllowed(true);
         // test that license restricted apis do now work
         PlainActionFuture<PutJobAction.Response> listener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).putJob(new PutJobAction.Request(createJob(jobId)), listener);
+        client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(createJob(jobId)), listener);
         PutJobAction.Response response = listener.actionGet();
         assertNotNull(response);
     }
@@ -83,7 +80,7 @@ public void testMachineLearningOpenJobActionRestricted() throws Exception {
         assertMLAllowed(true);
         // test that license restricted apis do now work
         PlainActionFuture<PutJobAction.Response> putJobListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).putJob(new PutJobAction.Request(createJob(jobId)), putJobListener);
+        client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(createJob(jobId)), putJobListener);
         PutJobAction.Response response = putJobListener.actionGet();
         assertNotNull(response);
 
@@ -94,7 +91,7 @@ public void testMachineLearningOpenJobActionRestricted() throws Exception {
         // test that license restricted apis do not work
         ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> {
             PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-            new MachineLearningClient(client()).openJob(new OpenJobAction.Request(jobId), listener);
+            client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request(jobId), listener);
             listener.actionGet();
         });
         assertThat(e.status(), is(RestStatus.FORBIDDEN));
@@ -114,7 +111,7 @@ public void testMachineLearningOpenJobActionRestricted() throws Exception {
 
         // test that license restricted apis do now work
         PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).openJob(new OpenJobAction.Request(jobId), listener);
+        client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request(jobId), listener);
         AcknowledgedResponse response2 = listener.actionGet();
         assertNotNull(response2);
     }
@@ -125,7 +122,7 @@ public void testMachineLearningPutDatafeedActionRestricted() throws Exception {
         assertMLAllowed(true);
         // test that license restricted apis do now work
         PlainActionFuture<PutJobAction.Response> putJobListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).putJob(new PutJobAction.Request(createJob(jobId)), putJobListener);
+        client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(createJob(jobId)), putJobListener);
         PutJobAction.Response putJobResponse = putJobListener.actionGet();
         assertNotNull(putJobResponse);
 
@@ -136,7 +133,7 @@ public void testMachineLearningPutDatafeedActionRestricted() throws Exception {
         // test that license restricted apis do not work
         ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> {
             PlainActionFuture<PutDatafeedAction.Response> listener = PlainActionFuture.newFuture();
-            new MachineLearningClient(client()).putDatafeed(
+            client().execute(PutDatafeedAction.INSTANCE, 
                 new PutDatafeedAction.Request(createDatafeed(datafeedId, jobId, Collections.singletonList(jobId))), listener);
             listener.actionGet();
         });
@@ -150,7 +147,7 @@ public void testMachineLearningPutDatafeedActionRestricted() throws Exception {
         assertMLAllowed(true);
         // test that license restricted apis do now work
         PlainActionFuture<PutDatafeedAction.Response> listener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).putDatafeed(
+        client().execute(PutDatafeedAction.INSTANCE, 
                 new PutDatafeedAction.Request(createDatafeed(datafeedId, jobId, Collections.singletonList(jobId))), listener);
         PutDatafeedAction.Response response = listener.actionGet();
         assertNotNull(response);
@@ -166,24 +163,24 @@ public void testAutoCloseJobWithDatafeed() throws Exception {
 
         // put job
         PlainActionFuture<PutJobAction.Response> putJobListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).putJob(new PutJobAction.Request(createJob(jobId)), putJobListener);
+        client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(createJob(jobId)), putJobListener);
         PutJobAction.Response putJobResponse = putJobListener.actionGet();
         assertNotNull(putJobResponse);
         // put datafeed
         PlainActionFuture<PutDatafeedAction.Response> putDatafeedListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).putDatafeed(
+        client().execute(PutDatafeedAction.INSTANCE, 
                 new PutDatafeedAction.Request(createDatafeed(datafeedId, jobId,
                         Collections.singletonList(datafeedIndex))), putDatafeedListener);
         PutDatafeedAction.Response putDatafeedResponse = putDatafeedListener.actionGet();
         assertNotNull(putDatafeedResponse);
         // open job
         PlainActionFuture<AcknowledgedResponse> openJobListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).openJob(new OpenJobAction.Request(jobId), openJobListener);
+        client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request(jobId), openJobListener);
         AcknowledgedResponse openJobResponse = openJobListener.actionGet();
         assertNotNull(openJobResponse);
         // start datafeed
         PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).startDatafeed(new StartDatafeedAction.Request(datafeedId, 0L), listener);
+        client().execute(StartDatafeedAction.INSTANCE, new StartDatafeedAction.Request(datafeedId, 0L), listener);
         listener.actionGet();
 
 
@@ -214,12 +211,12 @@ public void testAutoCloseJobWithDatafeed() throws Exception {
 
         // open job
         PlainActionFuture<AcknowledgedResponse> openJobListener2 = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).openJob(new OpenJobAction.Request(jobId), openJobListener2);
+        client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request(jobId), openJobListener2);
         AcknowledgedResponse openJobResponse3 = openJobListener2.actionGet();
         assertNotNull(openJobResponse3);
         // start datafeed
         PlainActionFuture<AcknowledgedResponse> listener2 = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).startDatafeed(new StartDatafeedAction.Request(datafeedId, 0L), listener2);
+        client().execute(StartDatafeedAction.INSTANCE, new StartDatafeedAction.Request(datafeedId, 0L), listener2);
         listener2.actionGet();
 
         assertBusy(() -> {
@@ -264,17 +261,17 @@ public void testMachineLearningStartDatafeedActionRestricted() throws Exception
             XContentType.JSON).get();
         // test that license restricted apis do now work
         PlainActionFuture<PutJobAction.Response> putJobListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).putJob(new PutJobAction.Request(createJob(jobId)), putJobListener);
+        client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(createJob(jobId)), putJobListener);
         PutJobAction.Response putJobResponse = putJobListener.actionGet();
         assertNotNull(putJobResponse);
         PlainActionFuture<PutDatafeedAction.Response> putDatafeedListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).putDatafeed(
+        client().execute(PutDatafeedAction.INSTANCE, 
                 new PutDatafeedAction.Request(createDatafeed(datafeedId, jobId,
                         Collections.singletonList(datafeedIndex))), putDatafeedListener);
         PutDatafeedAction.Response putDatafeedResponse = putDatafeedListener.actionGet();
         assertNotNull(putDatafeedResponse);
         PlainActionFuture<AcknowledgedResponse> openJobListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).openJob(new OpenJobAction.Request(jobId), openJobListener);
+        client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request(jobId), openJobListener);
         AcknowledgedResponse openJobResponse = openJobListener.actionGet();
         assertNotNull(openJobResponse);
 
@@ -295,7 +292,7 @@ public void testMachineLearningStartDatafeedActionRestricted() throws Exception
         // test that license restricted apis do not work
         ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> {
             PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-            new MachineLearningClient(client()).startDatafeed(new StartDatafeedAction.Request(datafeedId, 0L), listener);
+            client().execute(StartDatafeedAction.INSTANCE, new StartDatafeedAction.Request(datafeedId, 0L), listener);
             listener.actionGet();
         });
         assertThat(e.status(), is(RestStatus.FORBIDDEN));
@@ -309,12 +306,12 @@ public void testMachineLearningStartDatafeedActionRestricted() throws Exception
         // test that license restricted apis do now work
         // re-open job now that the license is valid again
         PlainActionFuture<AcknowledgedResponse> openJobListener2 = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).openJob(new OpenJobAction.Request(jobId), openJobListener2);
+        client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request(jobId), openJobListener2);
         AcknowledgedResponse openJobResponse3 = openJobListener2.actionGet();
         assertNotNull(openJobResponse3);
 
         PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).startDatafeed(new StartDatafeedAction.Request(datafeedId, 0L), listener);
+        client().execute(StartDatafeedAction.INSTANCE, new StartDatafeedAction.Request(datafeedId, 0L), listener);
         AcknowledgedResponse response = listener.actionGet();
         assertNotNull(response);
     }
@@ -328,21 +325,21 @@ public void testMachineLearningStopDatafeedActionNotRestricted() throws Exceptio
             XContentType.JSON).get();
         // test that license restricted apis do now work
         PlainActionFuture<PutJobAction.Response> putJobListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).putJob(new PutJobAction.Request(createJob(jobId)), putJobListener);
+        client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(createJob(jobId)), putJobListener);
         PutJobAction.Response putJobResponse = putJobListener.actionGet();
         assertNotNull(putJobResponse);
         PlainActionFuture<PutDatafeedAction.Response> putDatafeedListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).putDatafeed(
+        client().execute(PutDatafeedAction.INSTANCE, 
                 new PutDatafeedAction.Request(createDatafeed(datafeedId, jobId,
                         Collections.singletonList(datafeedIndex))), putDatafeedListener);
         PutDatafeedAction.Response putDatafeedResponse = putDatafeedListener.actionGet();
         assertNotNull(putDatafeedResponse);
         PlainActionFuture<AcknowledgedResponse> openJobListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).openJob(new OpenJobAction.Request(jobId), openJobListener);
+        client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request(jobId), openJobListener);
         AcknowledgedResponse openJobResponse = openJobListener.actionGet();
         assertNotNull(openJobResponse);
         PlainActionFuture<AcknowledgedResponse> startDatafeedListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).startDatafeed(
+        client().execute(StartDatafeedAction.INSTANCE, 
             new StartDatafeedAction.Request(datafeedId, 0L), startDatafeedListener);
         AcknowledgedResponse startDatafeedResponse = startDatafeedListener.actionGet();
         assertNotNull(startDatafeedResponse);
@@ -355,13 +352,12 @@ public void testMachineLearningStopDatafeedActionNotRestricted() throws Exceptio
         }
 
         PlainActionFuture<StopDatafeedAction.Response> listener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).stopDatafeed(new StopDatafeedAction.Request(datafeedId), listener);
+        client().execute(StopDatafeedAction.INSTANCE, new StopDatafeedAction.Request(datafeedId), listener);
         if (invalidLicense) {
             // the stop datafeed due to invalid license happens async, so check if the datafeed turns into stopped state:
             assertBusy(() -> {
                 GetDatafeedsStatsAction.Response response =
-                        new MachineLearningClient(client())
-                                .getDatafeedsStats(new GetDatafeedsStatsAction.Request(datafeedId)).actionGet();
+                        client().execute(GetDatafeedsStatsAction.INSTANCE, new GetDatafeedsStatsAction.Request(datafeedId)).actionGet();
                 assertEquals(DatafeedState.STOPPED, response.getResponse().results().get(0).getDatafeedState());
             });
         } else {
@@ -372,7 +368,7 @@ public void testMachineLearningStopDatafeedActionNotRestricted() throws Exceptio
             // the close due to invalid license happens async, so check if the job turns into closed state:
             assertBusy(() -> {
                 GetJobsStatsAction.Response response =
-                        new MachineLearningClient(client()).getJobsStats(new GetJobsStatsAction.Request(jobId)).actionGet();
+                        client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request(jobId)).actionGet();
                 assertEquals(JobState.CLOSED, response.getResponse().results().get(0).getState());
             });
         }
@@ -383,11 +379,11 @@ public void testMachineLearningCloseJobActionNotRestricted() throws Exception {
         assertMLAllowed(true);
         // test that license restricted apis do now work
         PlainActionFuture<PutJobAction.Response> putJobListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).putJob(new PutJobAction.Request(createJob(jobId)), putJobListener);
+        client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(createJob(jobId)), putJobListener);
         PutJobAction.Response putJobResponse = putJobListener.actionGet();
         assertNotNull(putJobResponse);
         PlainActionFuture<AcknowledgedResponse> openJobListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).openJob(new OpenJobAction.Request(jobId), openJobListener);
+        client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request(jobId), openJobListener);
         AcknowledgedResponse openJobResponse = openJobListener.actionGet();
         assertNotNull(openJobResponse);
 
@@ -405,11 +401,11 @@ public void testMachineLearningCloseJobActionNotRestricted() throws Exception {
             // the close due to invalid license happens async, so check if the job turns into closed state:
             assertBusy(() -> {
                 GetJobsStatsAction.Response response =
-                        new MachineLearningClient(client()).getJobsStats(new GetJobsStatsAction.Request(jobId)).actionGet();
+                        client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request(jobId)).actionGet();
                 assertEquals(JobState.CLOSED, response.getResponse().results().get(0).getState());
             });
         } else {
-            new MachineLearningClient(client()).closeJob(request, listener);
+            client().execute(CloseJobAction.INSTANCE, request, listener);
             listener.actionGet();
         }
     }
@@ -419,7 +415,7 @@ public void testMachineLearningDeleteJobActionNotRestricted() throws Exception {
         assertMLAllowed(true);
         // test that license restricted apis do now work
         PlainActionFuture<PutJobAction.Response> putJobListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).putJob(new PutJobAction.Request(createJob(jobId)), putJobListener);
+        client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(createJob(jobId)), putJobListener);
         PutJobAction.Response putJobResponse = putJobListener.actionGet();
         assertNotNull(putJobResponse);
 
@@ -428,7 +424,7 @@ public void testMachineLearningDeleteJobActionNotRestricted() throws Exception {
         enableLicensing(mode);
 
         PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).deleteJob(new DeleteJobAction.Request(jobId), listener);
+        client().execute(DeleteJobAction.INSTANCE, new DeleteJobAction.Request(jobId), listener);
         listener.actionGet();
     }
 
@@ -438,11 +434,11 @@ public void testMachineLearningDeleteDatafeedActionNotRestricted() throws Except
         assertMLAllowed(true);
         // test that license restricted apis do now work
         PlainActionFuture<PutJobAction.Response> putJobListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).putJob(new PutJobAction.Request(createJob(jobId)), putJobListener);
+        client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(createJob(jobId)), putJobListener);
         PutJobAction.Response putJobResponse = putJobListener.actionGet();
         assertNotNull(putJobResponse);
         PlainActionFuture<PutDatafeedAction.Response> putDatafeedListener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).putDatafeed(
+        client().execute(PutDatafeedAction.INSTANCE, 
                 new PutDatafeedAction.Request(createDatafeed(datafeedId, jobId,
                         Collections.singletonList(jobId))), putDatafeedListener);
         PutDatafeedAction.Response putDatafeedResponse = putDatafeedListener.actionGet();
@@ -453,7 +449,7 @@ public void testMachineLearningDeleteDatafeedActionNotRestricted() throws Except
         enableLicensing(mode);
 
         PlainActionFuture<AcknowledgedResponse> listener = PlainActionFuture.newFuture();
-        new MachineLearningClient(client()).deleteDatafeed(new DeleteDatafeedAction.Request(datafeedId), listener);
+        client().execute(DeleteDatafeedAction.INSTANCE, new DeleteDatafeedAction.Request(datafeedId), listener);
         listener.actionGet();
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java
similarity index 82%
rename from x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java
rename to x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java
index 2de1f79c64aa6..29f348c3857b9 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningInfoTransportActionTests.java
@@ -5,14 +5,16 @@
  */
 package org.elasticsearch.xpack.ml;
 
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -25,9 +27,10 @@
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.XPackFeatureSet.Usage;
 import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.action.util.QueryPage;
 import org.elasticsearch.xpack.core.ml.MachineLearningFeatureSetUsage;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
@@ -68,7 +71,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class MachineLearningFeatureSetTests extends ESTestCase {
+public class MachineLearningInfoTransportActionTests extends ESTestCase {
 
     private Settings commonSettings;
     private ClusterService clusterService;
@@ -94,30 +97,22 @@ public void init() {
         givenDatafeeds(Collections.emptyList());
     }
 
-    public void testIsRunningOnMlPlatform() {
-        assertTrue(MachineLearningFeatureSet.isRunningOnMlPlatform("Linux", "amd64", true));
-        assertTrue(MachineLearningFeatureSet.isRunningOnMlPlatform("Windows 10", "amd64", true));
-        assertTrue(MachineLearningFeatureSet.isRunningOnMlPlatform("Mac OS X", "x86_64", true));
-        assertFalse(MachineLearningFeatureSet.isRunningOnMlPlatform("Linux", "i386", false));
-        assertFalse(MachineLearningFeatureSet.isRunningOnMlPlatform("Windows 10", "i386", false));
-        assertFalse(MachineLearningFeatureSet.isRunningOnMlPlatform("SunOS", "amd64", false));
-        expectThrows(ElasticsearchException.class,
-                () -> MachineLearningFeatureSet.isRunningOnMlPlatform("Linux", "i386", true));
-        expectThrows(ElasticsearchException.class,
-                () -> MachineLearningFeatureSet.isRunningOnMlPlatform("Windows 10", "i386", true));
-        expectThrows(ElasticsearchException.class,
-                () -> MachineLearningFeatureSet.isRunningOnMlPlatform("SunOS", "amd64", true));
+    private MachineLearningUsageTransportAction newUsageAction(Settings settings) {
+        return new MachineLearningUsageTransportAction(mock(TransportService.class), clusterService,
+            null, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class),
+            TestEnvironment.newEnvironment(settings), client, licenseState, jobManagerHolder);
     }
 
     public void testAvailable() throws Exception {
-        MachineLearningFeatureSet featureSet = new MachineLearningFeatureSet(TestEnvironment.newEnvironment(commonSettings), clusterService,
-                client, licenseState, jobManagerHolder);
+        MachineLearningInfoTransportAction featureSet = new MachineLearningInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), commonSettings, licenseState);
         boolean available = randomBoolean();
         when(licenseState.isMachineLearningAllowed()).thenReturn(available);
         assertThat(featureSet.available(), is(available));
-        PlainActionFuture<Usage> future = new PlainActionFuture<>();
-        featureSet.usage(future);
-        XPackFeatureSet.Usage usage = future.get();
+        var usageAction = newUsageAction(commonSettings);
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, ClusterState.EMPTY_STATE, future);
+        XPackFeatureSet.Usage usage = future.get().getUsage();
         assertThat(usage.available(), is(available));
 
         BytesStreamOutput out = new BytesStreamOutput();
@@ -135,12 +130,13 @@ public void testEnabled() throws Exception {
             settings.put("xpack.ml.enabled", enabled);
         }
         boolean expected = enabled || useDefault;
-        MachineLearningFeatureSet featureSet = new MachineLearningFeatureSet(TestEnvironment.newEnvironment(settings.build()),
-                clusterService, client, licenseState, jobManagerHolder);
+        MachineLearningInfoTransportAction featureSet = new MachineLearningInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState);
         assertThat(featureSet.enabled(), is(expected));
-        PlainActionFuture<Usage> future = new PlainActionFuture<>();
-        featureSet.usage(future);
-        XPackFeatureSet.Usage usage = future.get();
+        var usageAction = newUsageAction(settings.build());
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, ClusterState.EMPTY_STATE, future);
+        XPackFeatureSet.Usage usage = future.get().getUsage();
         assertThat(usage.enabled(), is(expected));
 
         BytesStreamOutput out = new BytesStreamOutput();
@@ -170,11 +166,10 @@ public void testUsage() throws Exception {
                 buildDatafeedStats(DatafeedState.STOPPED)
         ));
 
-        MachineLearningFeatureSet featureSet = new MachineLearningFeatureSet(TestEnvironment.newEnvironment(settings.build()),
-                clusterService, client, licenseState, jobManagerHolder);
-        PlainActionFuture<Usage> future = new PlainActionFuture<>();
-        featureSet.usage(future);
-        XPackFeatureSet.Usage mlUsage = future.get();
+        var usageAction = newUsageAction(settings.build());
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, ClusterState.EMPTY_STATE, future);
+        XPackFeatureSet.Usage mlUsage = future.get().getUsage();
 
         BytesStreamOutput out = new BytesStreamOutput();
         mlUsage.writeTo(out);
@@ -251,12 +246,10 @@ public void testUsageDisabledML() throws Exception {
         Settings.Builder settings = Settings.builder().put(commonSettings);
         settings.put("xpack.ml.enabled", false);
 
-        JobManagerHolder emptyJobManagerHolder = new JobManagerHolder();
-        MachineLearningFeatureSet featureSet = new MachineLearningFeatureSet(TestEnvironment.newEnvironment(settings.build()),
-                clusterService, client, licenseState, emptyJobManagerHolder);
-        PlainActionFuture<Usage> future = new PlainActionFuture<>();
-        featureSet.usage(future);
-        XPackFeatureSet.Usage mlUsage = future.get();
+        var usageAction = newUsageAction(settings.build());
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, ClusterState.EMPTY_STATE, future);
+        XPackFeatureSet.Usage mlUsage = future.get().getUsage();
         BytesStreamOutput out = new BytesStreamOutput();
         mlUsage.writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new MachineLearningFeatureSetUsage(out.bytes().streamInput());
@@ -271,15 +264,14 @@ public void testUsageDisabledML() throws Exception {
     public void testNodeCount() throws Exception {
         when(licenseState.isMachineLearningAllowed()).thenReturn(true);
         int nodeCount = randomIntBetween(1, 3);
-        givenNodeCount(nodeCount);
+        ClusterState clusterState = givenNodeCount(nodeCount);
         Settings.Builder settings = Settings.builder().put(commonSettings);
         settings.put("xpack.ml.enabled", true);
-        MachineLearningFeatureSet featureSet = new MachineLearningFeatureSet(TestEnvironment.newEnvironment(settings.build()),
-            clusterService, client, licenseState, jobManagerHolder);
 
-        PlainActionFuture<Usage> future = new PlainActionFuture<>();
-        featureSet.usage(future);
-        XPackFeatureSet.Usage usage = future.get();
+        var usageAction = newUsageAction(settings.build());
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, clusterState, future);
+        XPackFeatureSet.Usage usage = future.get().getUsage();
 
         assertThat(usage.available(), is(true));
         assertThat(usage.enabled(), is(true));
@@ -302,11 +294,10 @@ public void testUsageGivenMlMetadataNotInstalled() throws Exception {
         settings.put("xpack.ml.enabled", true);
         when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE);
 
-        MachineLearningFeatureSet featureSet = new MachineLearningFeatureSet(TestEnvironment.newEnvironment(settings.build()),
-                clusterService, client, licenseState, jobManagerHolder);
-        PlainActionFuture<Usage> future = new PlainActionFuture<>();
-        featureSet.usage(future);
-        XPackFeatureSet.Usage usage = future.get();
+        var usageAction = newUsageAction(settings.build());
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, ClusterState.EMPTY_STATE, future);
+        XPackFeatureSet.Usage usage = future.get().getUsage();
 
         assertThat(usage.available(), is(true));
         assertThat(usage.enabled(), is(true));
@@ -359,15 +350,15 @@ private void givenJobs(List<Job> jobs, List<GetJobsStatsAction.Response.JobStats
         }).when(client).execute(same(GetJobsStatsAction.INSTANCE), any(), any());
     }
 
-    private void givenNodeCount(int nodeCount) {
+    private ClusterState givenNodeCount(int nodeCount) {
         DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder();
         for (int i = 0; i < nodeCount; i++) {
             Map<String, String> attrs = new HashMap<>();
             attrs.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, Integer.toString(20));
-            Set<DiscoveryNode.Role> roles = new HashSet<>();
-            roles.add(DiscoveryNode.Role.DATA);
-            roles.add(DiscoveryNode.Role.MASTER);
-            roles.add(DiscoveryNode.Role.INGEST);
+            Set<DiscoveryNodeRole> roles = new HashSet<>();
+            roles.add(DiscoveryNodeRole.DATA_ROLE);
+            roles.add(DiscoveryNodeRole.MASTER_ROLE);
+            roles.add(DiscoveryNodeRole.INGEST_ROLE);
             nodesBuilder.add(new DiscoveryNode("ml-feature-set-given-ml-node-" + i,
                 new TransportAddress(TransportAddress.META_ADDRESS, 9100 + i),
                 attrs,
@@ -376,18 +367,17 @@ private void givenNodeCount(int nodeCount) {
         }
         for (int i = 0; i < randomIntBetween(1, 3); i++) {
             Map<String, String> attrs = new HashMap<>();
-            Set<DiscoveryNode.Role> roles = new HashSet<>();
-            roles.add(DiscoveryNode.Role.DATA);
-            roles.add(DiscoveryNode.Role.MASTER);
-            roles.add(DiscoveryNode.Role.INGEST);
+            Set<DiscoveryNodeRole> roles = new HashSet<>();
+            roles.add(DiscoveryNodeRole.DATA_ROLE);
+            roles.add(DiscoveryNodeRole.MASTER_ROLE);
+            roles.add(DiscoveryNodeRole.INGEST_ROLE);
             nodesBuilder.add(new DiscoveryNode("ml-feature-set-given-non-ml-node-" + i,
                 new TransportAddress(TransportAddress.META_ADDRESS, 9300 + i),
                 attrs,
                 roles,
                 Version.CURRENT));
         }
-        ClusterState clusterState = new ClusterState.Builder(ClusterState.EMPTY_STATE).nodes(nodesBuilder.build()).build();
-        when(clusterService.state()).thenReturn(clusterState);
+        return new ClusterState.Builder(ClusterState.EMPTY_STATE).nodes(nodesBuilder.build()).build();
     }
 
     private void givenDatafeeds(List<GetDatafeedsStatsAction.Response.DatafeedStats> datafeedStats) {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java
index 9504cbe7a7011..e83fcdb5af6fb 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTests.java
@@ -97,8 +97,8 @@ public void testNoAttributes_givenClash() {
 
     public void testMachineMemory_givenStatsFailure() throws IOException {
         OsStats stats = mock(OsStats.class);
-        when(stats.getMem()).thenReturn(new OsStats.Mem(-1, -1));
-        assertEquals(-1L, MachineLearning.machineMemoryFromStats(stats));
+        when(stats.getMem()).thenReturn(new OsStats.Mem(0, 0));
+        assertEquals(0L, MachineLearning.machineMemoryFromStats(stats));
     }
 
     public void testMachineMemory_givenNoCgroup() throws IOException {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheckTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheckTests.java
index 4d07a93e1019c..d44a7792b9d7c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheckTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigrationEligibilityCheckTests.java
@@ -10,8 +10,6 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RecoverySource;
@@ -21,7 +19,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
@@ -38,7 +35,6 @@
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.junit.Before;
 
-import java.net.InetAddress;
 import java.util.Collections;
 import java.util.HashSet;
 
@@ -56,7 +52,7 @@ public void setUpTests() {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
@@ -70,50 +66,6 @@ public void testCanStartMigration_givenMigrationIsDisabled() {
         assertFalse(check.canStartMigration(clusterState));
     }
 
-    public void testCanStartMigration_givenNodesNotUpToVersion() {
-        MetaData.Builder metaData = MetaData.builder();
-        RoutingTable.Builder routingTable = RoutingTable.builder();
-        addMlConfigIndex(metaData, routingTable);
-
-        // mixed 6.5 and 6.6 nodes
-        ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
-                .nodes(DiscoveryNodes.builder()
-                        .add(new DiscoveryNode("node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), Version.V_6_5_0))
-                        .add(new DiscoveryNode("node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301), Version.V_6_6_0)))
-                .routingTable(routingTable.build())
-                .metaData(metaData)
-                .build();
-
-        Settings settings = newSettings(true);
-        givenClusterSettings(settings);
-
-        MlConfigMigrationEligibilityCheck check = new MlConfigMigrationEligibilityCheck(settings, clusterService);
-
-        assertFalse(check.canStartMigration(clusterState));
-    }
-
-    public void testCanStartMigration_givenNodesNotUpToVersionAndMigrationIsEnabled() {
-        MetaData.Builder metaData = MetaData.builder();
-        RoutingTable.Builder routingTable = RoutingTable.builder();
-        addMlConfigIndex(metaData, routingTable);
-
-        // mixed 6.5 and 6.6 nodes
-        ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
-                .nodes(DiscoveryNodes.builder()
-                        .add(new DiscoveryNode("node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), Version.V_6_6_0))
-                        .add(new DiscoveryNode("node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301), Version.V_6_6_0)))
-                .routingTable(routingTable.build())
-                .metaData(metaData)
-                .build();
-
-        Settings settings = newSettings(true);
-        givenClusterSettings(settings);
-
-        MlConfigMigrationEligibilityCheck check = new MlConfigMigrationEligibilityCheck(settings, clusterService);
-
-        assertTrue(check.canStartMigration(clusterState));
-    }
-
     public void testCanStartMigration_givenMissingIndex() {
         Settings settings = newSettings(true);
         givenClusterSettings(settings);
@@ -159,23 +111,6 @@ private void addMlConfigIndex(MetaData.Builder metaData, RoutingTable.Builder ro
                 .addIndexShard(new IndexShardRoutingTable.Builder(shardId).addShard(shardRouting).build()));
     }
 
-
-    public void testJobIsEligibleForMigration_givenNodesNotUpToVersion() {
-        // mixed 6.5 and 6.6 nodes
-        ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
-            .nodes(DiscoveryNodes.builder()
-                .add(new DiscoveryNode("node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), Version.V_6_5_0))
-                .add(new DiscoveryNode("node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301), Version.V_6_6_0)))
-            .build();
-
-        Settings settings = newSettings(true);
-        givenClusterSettings(settings);
-
-        MlConfigMigrationEligibilityCheck check = new MlConfigMigrationEligibilityCheck(settings, clusterService);
-
-        assertFalse(check.jobIsEligibleForMigration("pre-min-version", clusterState));
-    }
-
     public void testJobIsEligibleForMigration_givenJobNotInClusterState() {
         ClusterState clusterState = ClusterState.builder(new ClusterName("migratortests")).build();
 
@@ -306,22 +241,6 @@ public void testJobIsEligibleForMigration_givenOpenAndUnallocatedJob() {
         assertTrue(check.jobIsEligibleForMigration(openJob.getId(), clusterState));
     }
 
-    public void testDatafeedIsEligibleForMigration_givenNodesNotUpToVersion() {
-        // mixed 6.5 and 6.6 nodes
-        ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
-            .nodes(DiscoveryNodes.builder()
-                .add(new DiscoveryNode("node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), Version.V_6_5_0))
-                .add(new DiscoveryNode("node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301), Version.V_6_6_0)))
-            .build();
-
-        Settings settings = newSettings(true);
-        givenClusterSettings(settings);
-
-        MlConfigMigrationEligibilityCheck check = new MlConfigMigrationEligibilityCheck(settings, clusterService);
-
-        assertFalse(check.datafeedIsEligibleForMigration("pre-min-version", clusterState));
-    }
-
     public void testDatafeedIsEligibleForMigration_givenDatafeedNotInClusterState() {
         ClusterState clusterState = ClusterState.builder(new ClusterName("migratortests")).build();
         Settings settings = newSettings(true);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigratorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigratorTests.java
index 802f969d88609..f368111829c87 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigratorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlConfigMigratorTests.java
@@ -53,7 +53,7 @@ public class MlConfigMigratorTests extends ESTestCase {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java
index 0e6df7db57ab8..551f9eb87ff94 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlMetadataTests.java
@@ -70,13 +70,13 @@ protected MlMetadata doParseInstance(XContentParser parser) {
 
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedWriteableRegistry(searchModule.getNamedWriteables());
     }
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java
index bdb1be97bd2e8..734533cac54b0 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java
@@ -49,7 +49,7 @@ protected Settings nodeSettings() {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java
index fc44c520ebfc7..11fdd9a2c09a4 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportFinalizeJobExecutionActionTests.java
@@ -71,7 +71,7 @@ public void testOperation() {
 
         FinalizeJobExecutionAction.Request request = new FinalizeJobExecutionAction.Request(new String[]{"job1", "job2"});
         AtomicReference<AcknowledgedResponse> ack = new AtomicReference<>();
-        action.masterOperation(request, clusterState, ActionListener.wrap(
+        action.masterOperation(null, request, clusterState, ActionListener.wrap(
                 ack::set,
                 e -> assertNull(e.getMessage())
         ));
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsActionTests.java
index eb5a5a3dda526..73eb9a1c7adca 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetJobsStatsActionTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 
 import java.time.Duration;
 import java.util.Arrays;
@@ -27,8 +28,11 @@ public void testDetermineJobIds() {
         assertEquals(1, result.size());
         assertEquals("id1", result.get(0));
 
-        result = determineJobIdsWithoutLiveStats(Collections.singletonList("id1"), Collections.singletonList(
-                new GetJobsStatsAction.Response.JobStats("id1", new DataCounts("id1"), null, null, JobState.OPENED, null, null, null)));
+        result = determineJobIdsWithoutLiveStats(
+            Collections.singletonList("id1"),
+            Collections.singletonList(
+                new GetJobsStatsAction.Response.JobStats(
+                    "id1", new DataCounts("id1"), null, null, JobState.OPENED, null, null, null, new TimingStats("id1"))));
         assertEquals(0, result.size());
 
         result = determineJobIdsWithoutLiveStats(Arrays.asList("id1", "id2", "id3"), Collections.emptyList());
@@ -39,23 +43,28 @@ public void testDetermineJobIds() {
 
         result = determineJobIdsWithoutLiveStats(Arrays.asList("id1", "id2", "id3"),
                 Collections.singletonList(new GetJobsStatsAction.Response.JobStats("id1", new DataCounts("id1"), null, null,
-                        JobState.OPENED, null, null, null))
+                        JobState.OPENED, null, null, null, new TimingStats("id1")))
         );
         assertEquals(2, result.size());
         assertEquals("id2", result.get(0));
         assertEquals("id3", result.get(1));
 
         result = determineJobIdsWithoutLiveStats(Arrays.asList("id1", "id2", "id3"), Arrays.asList(
-                new GetJobsStatsAction.Response.JobStats("id1", new DataCounts("id1"), null, null, JobState.OPENED, null, null, null),
-                new GetJobsStatsAction.Response.JobStats("id3", new DataCounts("id3"), null, null, JobState.OPENED, null, null, null)
+                new GetJobsStatsAction.Response.JobStats(
+                    "id1", new DataCounts("id1"), null, null, JobState.OPENED, null, null, null, new TimingStats("id1")),
+                new GetJobsStatsAction.Response.JobStats(
+                    "id3", new DataCounts("id3"), null, null, JobState.OPENED, null, null, null, new TimingStats("id3"))
         ));
         assertEquals(1, result.size());
         assertEquals("id2", result.get(0));
 
         result = determineJobIdsWithoutLiveStats(Arrays.asList("id1", "id2", "id3"), Arrays.asList(
-                new GetJobsStatsAction.Response.JobStats("id1", new DataCounts("id1"), null, null, JobState.OPENED, null, null, null),
-                new GetJobsStatsAction.Response.JobStats("id2", new DataCounts("id2"), null, null, JobState.OPENED, null, null, null),
-                new GetJobsStatsAction.Response.JobStats("id3", new DataCounts("id3"), null, null, JobState.OPENED, null, null, null)));
+                new GetJobsStatsAction.Response.JobStats(
+                    "id1", new DataCounts("id1"), null, null, JobState.OPENED, null, null, null, new TimingStats("id1")),
+                new GetJobsStatsAction.Response.JobStats(
+                    "id2", new DataCounts("id2"), null, null, JobState.OPENED, null, null, null, new TimingStats("id2")),
+                new GetJobsStatsAction.Response.JobStats(
+                    "id3", new DataCounts("id3"), null, null, JobState.OPENED, null, null, null, new TimingStats("id3"))));
         assertEquals(0, result.size());
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java
index 5f1a4050d1f3e..cc9a0ba0181ad 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java
@@ -16,8 +16,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RecoverySource;
@@ -27,9 +25,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
@@ -38,7 +33,6 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.xpack.core.ml.MlMetaIndex;
 import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
@@ -57,38 +51,19 @@
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager;
 import org.elasticsearch.xpack.ml.process.MlMemoryTracker;
-import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
-import org.junit.Before;
 
-import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
 
 import static org.elasticsearch.xpack.core.ml.job.config.JobTests.buildJobBuilder;
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-// TODO: in 8.0.0 remove all instances of MAX_OPEN_JOBS_NODE_ATTR from this file
 public class TransportOpenJobActionTests extends ESTestCase {
 
-    private MlMemoryTracker memoryTracker;
-    private boolean isMemoryTrackerRecentlyRefreshed;
-
-    @Before
-    public void setup() {
-        memoryTracker = mock(MlMemoryTracker.class);
-        isMemoryTrackerRecentlyRefreshed = true;
-        when(memoryTracker.isRecentlyRefreshed()).thenReturn(isMemoryTrackerRecentlyRefreshed);
-    }
-
     public void testValidate_jobMissing() {
         expectThrows(ResourceNotFoundException.class, () -> TransportOpenJobAction.validate("job_id2", null));
     }
@@ -114,351 +89,6 @@ public void testValidate_givenValidJob() {
         TransportOpenJobAction.validate("job_id", jobBuilder.build(new Date()));
     }
 
-    public void testSelectLeastLoadedMlNode_byCount() {
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "-1");
-        // MachineLearning.MACHINE_MEMORY_NODE_ATTR negative, so this will fall back to allocating by count
-        DiscoveryNodes nodes = DiscoveryNodes.builder()
-                .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
-                        nodeAttr, Collections.emptySet(), Version.CURRENT))
-                .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
-                        nodeAttr, Collections.emptySet(), Version.CURRENT))
-                .add(new DiscoveryNode("_node_name3", "_node_id3", new TransportAddress(InetAddress.getLoopbackAddress(), 9302),
-                        nodeAttr, Collections.emptySet(), Version.CURRENT))
-                .build();
-
-        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
-        addJobTask("job_id1", "_node_id1", null, tasksBuilder);
-        addJobTask("job_id2", "_node_id1", null, tasksBuilder);
-        addJobTask("job_id3", "_node_id2", null, tasksBuilder);
-        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
-
-        ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
-        cs.nodes(nodes);
-        MetaData.Builder metaData = MetaData.builder();
-        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
-        cs.metaData(metaData);
-
-        Job.Builder jobBuilder = buildJobBuilder("job_id4");
-        jobBuilder.setJobVersion(Version.CURRENT);
-
-        Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id4", jobBuilder.build(),
-                cs.build(), 10, 2, 30, memoryTracker, isMemoryTrackerRecentlyRefreshed, logger);
-        assertEquals("", result.getExplanation());
-        assertEquals("_node_id3", result.getExecutorNode());
-    }
-
-    public void testSelectLeastLoadedMlNode_maxCapacity() {
-        int numNodes = randomIntBetween(1, 10);
-        int maxRunningJobsPerNode = randomIntBetween(1, 100);
-
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, Integer.toString(maxRunningJobsPerNode));
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
-        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
-        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
-        String[] jobIds = new String[numNodes * maxRunningJobsPerNode];
-        for (int i = 0; i < numNodes; i++) {
-            String nodeId = "_node_id" + i;
-            TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), 9300 + i);
-            nodes.add(new DiscoveryNode("_node_name" + i, nodeId, address, nodeAttr, Collections.emptySet(), Version.CURRENT));
-            for (int j = 0; j < maxRunningJobsPerNode; j++) {
-                int id = j + (maxRunningJobsPerNode * i);
-                jobIds[id] = "job_id" + id;
-                addJobTask(jobIds[id], nodeId, JobState.OPENED, tasksBuilder);
-            }
-        }
-        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
-
-        ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
-        MetaData.Builder metaData = MetaData.builder();
-        cs.nodes(nodes);
-        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
-        cs.metaData(metaData);
-
-        Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id0", new ByteSizeValue(150, ByteSizeUnit.MB)).build(new Date());
-
-        Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id0", job, cs.build(), maxRunningJobsPerNode, 2,
-                30, memoryTracker, isMemoryTrackerRecentlyRefreshed, logger);
-        assertNull(result.getExecutorNode());
-        assertTrue(result.getExplanation(), result.getExplanation().contains("because this node is full. Number of opened jobs ["
-            + maxRunningJobsPerNode + "], xpack.ml.max_open_jobs [" + maxRunningJobsPerNode + "]"));
-    }
-
-    public void testSelectLeastLoadedMlNode_noMlNodes() {
-        DiscoveryNodes nodes = DiscoveryNodes.builder()
-                .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
-                        Collections.emptyMap(), Collections.emptySet(), Version.CURRENT))
-                .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
-                        Collections.emptyMap(), Collections.emptySet(), Version.CURRENT))
-                .build();
-
-        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
-        addJobTask("job_id1", "_node_id1", null, tasksBuilder);
-        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
-
-        ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
-        MetaData.Builder metaData = MetaData.builder();
-        cs.nodes(nodes);
-        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
-        cs.metaData(metaData);
-
-        Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id2", new ByteSizeValue(2, ByteSizeUnit.MB)).build(new Date());
-
-        Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id2", job, cs.build(), 20, 2, 30, memoryTracker,
-            isMemoryTrackerRecentlyRefreshed, logger);
-        assertTrue(result.getExplanation().contains("because this node isn't a ml node"));
-        assertNull(result.getExecutorNode());
-    }
-
-    public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() {
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
-        DiscoveryNodes nodes = DiscoveryNodes.builder()
-                .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
-                        nodeAttr, Collections.emptySet(), Version.CURRENT))
-                .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
-                        nodeAttr, Collections.emptySet(), Version.CURRENT))
-                .add(new DiscoveryNode("_node_name3", "_node_id3", new TransportAddress(InetAddress.getLoopbackAddress(), 9302),
-                        nodeAttr, Collections.emptySet(), Version.CURRENT))
-                .build();
-
-        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
-        addJobTask("job_id1", "_node_id1", null, tasksBuilder);
-        addJobTask("job_id2", "_node_id1", null, tasksBuilder);
-        addJobTask("job_id3", "_node_id2", null, tasksBuilder);
-        addJobTask("job_id4", "_node_id2", null, tasksBuilder);
-        addJobTask("job_id5", "_node_id3", null, tasksBuilder);
-        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
-
-        ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
-        csBuilder.nodes(nodes);
-        MetaData.Builder metaData = MetaData.builder();
-        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
-        csBuilder.metaData(metaData);
-
-        Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id6", new ByteSizeValue(2, ByteSizeUnit.MB)).build(new Date());
-
-        ClusterState cs = csBuilder.build();
-        Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id6", job, cs, 10, 2, 30, memoryTracker,
-            isMemoryTrackerRecentlyRefreshed, logger);
-        assertEquals("_node_id3", result.getExecutorNode());
-
-        tasksBuilder = PersistentTasksCustomMetaData.builder(tasks);
-        addJobTask("job_id6", "_node_id3", null, tasksBuilder);
-        tasks = tasksBuilder.build();
-
-        csBuilder = ClusterState.builder(cs);
-        csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks));
-        cs = csBuilder.build();
-        result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 10, 2, 30, memoryTracker,
-            isMemoryTrackerRecentlyRefreshed, logger);
-        assertNull("no node selected, because OPENING state", result.getExecutorNode());
-        assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state"));
-
-        tasksBuilder = PersistentTasksCustomMetaData.builder(tasks);
-        tasksBuilder.reassignTask(MlTasks.jobTaskId("job_id6"), new Assignment("_node_id3", "test assignment"));
-        tasks = tasksBuilder.build();
-
-        csBuilder = ClusterState.builder(cs);
-        csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks));
-        cs = csBuilder.build();
-        result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 10, 2, 30, memoryTracker,
-            isMemoryTrackerRecentlyRefreshed, logger);
-        assertNull("no node selected, because stale task", result.getExecutorNode());
-        assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state"));
-
-        tasksBuilder = PersistentTasksCustomMetaData.builder(tasks);
-        tasksBuilder.updateTaskState(MlTasks.jobTaskId("job_id6"), null);
-        tasks = tasksBuilder.build();
-
-        csBuilder = ClusterState.builder(cs);
-        csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks));
-        cs = csBuilder.build();
-        result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 10, 2, 30, memoryTracker,
-            isMemoryTrackerRecentlyRefreshed, logger);
-        assertNull("no node selected, because null state", result.getExecutorNode());
-        assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state"));
-    }
-
-    public void testSelectLeastLoadedMlNode_concurrentOpeningJobsAndStaleFailedJob() {
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
-        DiscoveryNodes nodes = DiscoveryNodes.builder()
-            .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
-                nodeAttr, Collections.emptySet(), Version.CURRENT))
-            .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
-                nodeAttr, Collections.emptySet(), Version.CURRENT))
-            .add(new DiscoveryNode("_node_name3", "_node_id3", new TransportAddress(InetAddress.getLoopbackAddress(), 9302),
-                nodeAttr, Collections.emptySet(), Version.CURRENT))
-            .build();
-
-        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
-        addJobTask("job_id1", "_node_id1", JobState.fromString("failed"), tasksBuilder);
-        // This will make the allocation stale for job_id1
-        tasksBuilder.reassignTask(MlTasks.jobTaskId("job_id1"), new Assignment("_node_id1", "test assignment"));
-        addJobTask("job_id2", "_node_id1", null, tasksBuilder);
-        addJobTask("job_id3", "_node_id2", null, tasksBuilder);
-        addJobTask("job_id4", "_node_id2", null, tasksBuilder);
-        addJobTask("job_id5", "_node_id3", null, tasksBuilder);
-        addJobTask("job_id6", "_node_id3", null, tasksBuilder);
-        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
-
-        ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
-        csBuilder.nodes(nodes);
-        MetaData.Builder metaData = MetaData.builder();
-        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
-        csBuilder.metaData(metaData);
-
-        ClusterState cs = csBuilder.build();
-        Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id7", new ByteSizeValue(2, ByteSizeUnit.MB)).build(new Date());
-
-        // Allocation won't be possible if the stale failed job is treated as opening
-        Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 10, 2, 30, memoryTracker,
-            isMemoryTrackerRecentlyRefreshed, logger);
-        assertEquals("_node_id1", result.getExecutorNode());
-
-        tasksBuilder = PersistentTasksCustomMetaData.builder(tasks);
-        addJobTask("job_id7", "_node_id1", null, tasksBuilder);
-        tasks = tasksBuilder.build();
-
-        csBuilder = ClusterState.builder(cs);
-        csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks));
-        cs = csBuilder.build();
-        result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id8", job, cs, 10, 2, 30, memoryTracker,
-            isMemoryTrackerRecentlyRefreshed, logger);
-        assertNull("no node selected, because OPENING state", result.getExecutorNode());
-        assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state"));
-    }
-
-    public void testSelectLeastLoadedMlNode_noCompatibleJobTypeNodes() {
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
-        DiscoveryNodes nodes = DiscoveryNodes.builder()
-                .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
-                        nodeAttr, Collections.emptySet(), Version.CURRENT))
-                .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
-                        nodeAttr, Collections.emptySet(), Version.CURRENT))
-                .build();
-
-        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
-        addJobTask("incompatible_type_job", "_node_id1", null, tasksBuilder);
-        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
-
-        ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
-        MetaData.Builder metaData = MetaData.builder();
-
-        Job job = mock(Job.class);
-        when(job.getId()).thenReturn("incompatible_type_job");
-        when(job.getJobVersion()).thenReturn(Version.CURRENT);
-        when(job.getJobType()).thenReturn("incompatible_type");
-        when(job.getInitialResultsIndexName()).thenReturn("shared");
-
-        cs.nodes(nodes);
-        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
-        cs.metaData(metaData);
-        Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("incompatible_type_job", job, cs.build(), 10, 2, 30,
-            memoryTracker, isMemoryTrackerRecentlyRefreshed, logger);
-        assertThat(result.getExplanation(), containsString("because this node does not support jobs of type [incompatible_type]"));
-        assertNull(result.getExecutorNode());
-    }
-
-    public void testSelectLeastLoadedMlNode_noNodesMatchingModelSnapshotMinVersion() {
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
-        Version node1Version = VersionUtils.randomCompatibleVersion(random(), VersionUtils.getPreviousVersion(Version.CURRENT));
-        Version node2Version = randomValueOtherThan(node1Version,
-            () -> VersionUtils.randomCompatibleVersion(random(), VersionUtils.getPreviousVersion(Version.CURRENT)));
-        DiscoveryNodes nodes = DiscoveryNodes.builder()
-                .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
-                        nodeAttr, Collections.emptySet(), node1Version))
-                .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
-                        nodeAttr, Collections.emptySet(), node2Version))
-                .build();
-
-        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
-        addJobTask("job_with_incompatible_model_snapshot", "_node_id1", null, tasksBuilder);
-        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
-
-        ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
-        MetaData.Builder metaData = MetaData.builder();
-
-        Job job = BaseMlIntegTestCase.createFareQuoteJob("job_with_incompatible_model_snapshot")
-                .setModelSnapshotId("incompatible_snapshot")
-                .setModelSnapshotMinVersion(Version.CURRENT)
-                .build(new Date());
-        cs.nodes(nodes);
-        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
-        cs.metaData(metaData);
-        Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_incompatible_model_snapshot", job, cs.build(), 10,
-                2, 30, memoryTracker, isMemoryTrackerRecentlyRefreshed, logger);
-        assertThat(result.getExplanation(), containsString(
-                "because the job's model snapshot requires a node of version [" + Version.CURRENT + "] or higher"));
-        assertNull(result.getExecutorNode());
-    }
-
-    public void testSelectLeastLoadedMlNode_jobWithRulesButNoNodeMeetsRequiredVersion() {
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
-        Version version = Version.fromString("6.3.0");
-        DiscoveryNodes nodes = DiscoveryNodes.builder()
-                .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
-                        nodeAttr, Collections.emptySet(), version))
-                .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
-                        nodeAttr, Collections.emptySet(), version))
-                .build();
-
-        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
-        addJobTask("job_with_rules", "_node_id1", null, tasksBuilder);
-        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
-
-        ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
-        MetaData.Builder metaData = MetaData.builder();
-        cs.nodes(nodes);
-        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
-        cs.metaData(metaData);
-
-        Job job = jobWithRules("job_with_rules");
-        Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_rules", job, cs.build(), 10, 2, 30, memoryTracker,
-            isMemoryTrackerRecentlyRefreshed, logger);
-        assertThat(result.getExplanation(), containsString(
-                "because jobs using custom_rules require a node of version [6.4.0] or higher"));
-        assertNull(result.getExecutorNode());
-    }
-
-    public void testSelectLeastLoadedMlNode_jobWithRulesAndNodeMeetsRequiredVersion() {
-        Map<String, String> nodeAttr = new HashMap<>();
-        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
-        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
-        DiscoveryNodes nodes = DiscoveryNodes.builder()
-                .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
-                        nodeAttr, Collections.emptySet(), Version.fromString("6.2.0")))
-                .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
-                        nodeAttr, Collections.emptySet(), Version.fromString("6.4.0")))
-                .build();
-
-        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
-        addJobTask("job_with_rules", "_node_id1", null, tasksBuilder);
-        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
-
-        ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
-        MetaData.Builder metaData = MetaData.builder();
-        cs.nodes(nodes);
-        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
-        cs.metaData(metaData);
-
-        Job job = jobWithRules("job_with_rules");
-        Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_rules", job, cs.build(), 10, 2, 30, memoryTracker,
-            isMemoryTrackerRecentlyRefreshed, logger);
-        assertNotNull(result.getExecutorNode());
-    }
-
     public void testVerifyIndicesPrimaryShardsAreActive() {
         MetaData.Builder metaData = MetaData.builder();
         RoutingTable.Builder routingTable = RoutingTable.builder();
@@ -495,33 +125,6 @@ public void testVerifyIndicesPrimaryShardsAreActive() {
         assertEquals(indexToRemove, result.get(0));
     }
 
-    public void testNodeNameAndVersion() {
-        TransportAddress ta = new TransportAddress(InetAddress.getLoopbackAddress(), 9300);
-        Map<String, String> attributes = new HashMap<>();
-        attributes.put("unrelated", "attribute");
-        DiscoveryNode node = new DiscoveryNode("_node_name1", "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT);
-        assertEquals("{_node_name1}{version=" + node.getVersion() + "}", TransportOpenJobAction.nodeNameAndVersion(node));
-    }
-
-    public void testNodeNameAndMlAttributes() {
-        TransportAddress ta = new TransportAddress(InetAddress.getLoopbackAddress(), 9300);
-        SortedMap<String, String> attributes = new TreeMap<>();
-        attributes.put("unrelated", "attribute");
-        DiscoveryNode node = new DiscoveryNode("_node_name1", "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT);
-        assertEquals("{_node_name1}", TransportOpenJobAction.nodeNameAndMlAttributes(node));
-
-        attributes.put("ml.machine_memory", "5");
-        node = new DiscoveryNode("_node_name1", "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT);
-        assertEquals("{_node_name1}{ml.machine_memory=5}", TransportOpenJobAction.nodeNameAndMlAttributes(node));
-
-        node = new DiscoveryNode(null, "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT);
-        assertEquals("{_node_id1}{ml.machine_memory=5}", TransportOpenJobAction.nodeNameAndMlAttributes(node));
-
-        attributes.put("node.ml", "true");
-        node = new DiscoveryNode("_node_name1", "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT);
-        assertEquals("{_node_name1}{ml.machine_memory=5}{node.ml=true}", TransportOpenJobAction.nodeNameAndMlAttributes(node));
-    }
-
     public void testJobTaskMatcherMatch() {
         Task nonJobTask1 = mock(Task.class);
         Task nonJobTask2 = mock(Task.class);
@@ -625,7 +228,7 @@ private void addIndices(MetaData.Builder metaData, RoutingTable.Builder routingT
         }
     }
 
-    private static Job jobWithRules(String jobId) {
+    public static Job jobWithRules(String jobId) {
         DetectionRule rule = new DetectionRule.Builder(Collections.singletonList(
                 new RuleCondition(RuleCondition.AppliesTo.TYPICAL, Operator.LT, 100.0)
         )).build();
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedActionTests.java
index 2216aa0c39167..dc6b45d126a4e 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedActionTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedActionTests.java
@@ -38,7 +38,7 @@ public class TransportStartDatafeedActionTests extends ESTestCase {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsActionTests.java
new file mode 100644
index 0000000000000..2f28463c5206a
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportStopDataFrameAnalyticsActionTests.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.action;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.Version;
+import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ml.MlTasks;
+import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+
+public class TransportStopDataFrameAnalyticsActionTests extends ESTestCase {
+
+    public void testFindAnalyticsToStop_GivenOneFailedTaskAndNotForce() {
+        PersistentTasksCustomMetaData.Builder tasksBuilder =  PersistentTasksCustomMetaData.builder();
+        addAnalyticsTask(tasksBuilder, "started", "foo-node", DataFrameAnalyticsState.STARTED);
+        addAnalyticsTask(tasksBuilder, "reindexing", "foo-node", DataFrameAnalyticsState.REINDEXING);
+        addAnalyticsTask(tasksBuilder, "analyzing", "foo-node", DataFrameAnalyticsState.ANALYZING);
+        addAnalyticsTask(tasksBuilder, "stopping", "foo-node", DataFrameAnalyticsState.STOPPING);
+        addAnalyticsTask(tasksBuilder, "stopped", "foo-node", DataFrameAnalyticsState.STOPPED);
+        addAnalyticsTask(tasksBuilder, "failed", "foo-node", DataFrameAnalyticsState.FAILED);
+
+        Set<String> ids = new HashSet<>(Arrays.asList("started", "reindexing", "analyzing", "stopping", "stopped", "failed"));
+
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
+            () -> TransportStopDataFrameAnalyticsAction.findAnalyticsToStop(tasksBuilder.build(), ids, false));
+
+        assertThat(e.status(), equalTo(RestStatus.CONFLICT));
+        assertThat(e.getMessage(), equalTo("cannot close data frame analytics [failed] because it failed, use force stop instead"));
+    }
+
+    public void testFindAnalyticsToStop_GivenTwoFailedTasksAndNotForce() {
+        PersistentTasksCustomMetaData.Builder tasksBuilder =  PersistentTasksCustomMetaData.builder();
+        addAnalyticsTask(tasksBuilder, "failed", "foo-node", DataFrameAnalyticsState.FAILED);
+        addAnalyticsTask(tasksBuilder, "another_failed", "foo-node", DataFrameAnalyticsState.FAILED);
+
+        Set<String> ids = new HashSet<>(Arrays.asList("failed", "another_failed"));
+
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
+            () -> TransportStopDataFrameAnalyticsAction.findAnalyticsToStop(tasksBuilder.build(), ids, false));
+
+        assertThat(e.status(), equalTo(RestStatus.CONFLICT));
+        assertThat(e.getMessage(), equalTo("one or more data frame analytics are in failed state, use force stop instead"));
+    }
+
+    public void testFindAnalyticsToStop_GivenFailedTaskAndForce() {
+        PersistentTasksCustomMetaData.Builder tasksBuilder =  PersistentTasksCustomMetaData.builder();
+        addAnalyticsTask(tasksBuilder, "started", "foo-node", DataFrameAnalyticsState.STARTED);
+        addAnalyticsTask(tasksBuilder, "reindexing", "foo-node", DataFrameAnalyticsState.REINDEXING);
+        addAnalyticsTask(tasksBuilder, "analyzing", "foo-node", DataFrameAnalyticsState.ANALYZING);
+        addAnalyticsTask(tasksBuilder, "stopping", "foo-node", DataFrameAnalyticsState.STOPPING);
+        addAnalyticsTask(tasksBuilder, "stopped", "foo-node", DataFrameAnalyticsState.STOPPED);
+        addAnalyticsTask(tasksBuilder, "failed", "foo-node", DataFrameAnalyticsState.FAILED);
+
+        Set<String> ids = new HashSet<>(Arrays.asList("started", "reindexing", "analyzing", "stopping", "stopped", "failed"));
+
+        Set<String> analyticsToStop = TransportStopDataFrameAnalyticsAction.findAnalyticsToStop(tasksBuilder.build(), ids, true);
+
+        assertThat(analyticsToStop, containsInAnyOrder("started", "reindexing", "analyzing", "failed"));
+    }
+
+    private static void addAnalyticsTask(PersistentTasksCustomMetaData.Builder builder, String analyticsId, String nodeId,
+                                         DataFrameAnalyticsState state) {
+        builder.addTask(MlTasks.dataFrameAnalyticsTaskId(analyticsId), MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME,
+            new StartDataFrameAnalyticsAction.TaskParams(analyticsId, Version.CURRENT),
+            new PersistentTasksCustomMetaData.Assignment(nodeId, "test assignment"));
+
+        builder.updateTaskState(MlTasks.dataFrameAnalyticsTaskId(analyticsId),
+            new DataFrameAnalyticsTaskState(state, builder.getLastAllocationId(), null));
+
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidatorTests.java
index 00db9462c6087..c50edd215d037 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidatorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidatorTests.java
@@ -33,7 +33,7 @@ public class DatafeedJobValidatorTests extends ESTestCase {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/delayeddatacheck/DelayedDataDetectorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/delayeddatacheck/DelayedDataDetectorFactoryTests.java
index 8857472062f81..5eef1cde7ac55 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/delayeddatacheck/DelayedDataDetectorFactoryTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/delayeddatacheck/DelayedDataDetectorFactoryTests.java
@@ -31,7 +31,7 @@ public class DelayedDataDetectorFactoryTests extends ESTestCase {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactoryTests.java
index ed24af15962a2..e82a99378c06c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactoryTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactoryTests.java
@@ -65,7 +65,7 @@ public class DataExtractorFactoryTests extends ESTestCase {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactoryTests.java
index fdfd75759db13..a082398772337 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactoryTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactoryTests.java
@@ -37,7 +37,7 @@ public void setUpMocks() {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationTestUtils.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationTestUtils.java
index 38202eee0ff06..afdf9c2eaf852 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationTestUtils.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationTestUtils.java
@@ -5,12 +5,14 @@
  */
 package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation;
 
+import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregation;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.aggregations.metrics.GeoCentroid;
 import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.search.aggregations.metrics.Max;
 import org.elasticsearch.search.aggregations.metrics.Percentile;
@@ -70,6 +72,15 @@ static Max createMax(String name, double value) {
         return max;
     }
 
+    static GeoCentroid createGeoCentroid(String name, long count, double lat, double lon) {
+        GeoCentroid centroid = mock(GeoCentroid.class);
+        when(centroid.count()).thenReturn(count);
+        when(centroid.getName()).thenReturn(name);
+        GeoPoint point = count > 0 ? new GeoPoint(lat, lon) : null;
+        when(centroid.centroid()).thenReturn(point);
+        return centroid;
+    }
+
     static NumericMetricsAggregation.SingleValue createSingleValue(String name, double value) {
         NumericMetricsAggregation.SingleValue singleValue = mock(NumericMetricsAggregation.SingleValue.class);
         when(singleValue.getName()).thenReturn(name);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessorTests.java
index be79b461eeb18..d45f294d1887a 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessorTests.java
@@ -27,6 +27,7 @@
 
 import static org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationTestUtils.Term;
 import static org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationTestUtils.createAggs;
+import static org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationTestUtils.createGeoCentroid;
 import static org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationTestUtils.createHistogramAggregation;
 import static org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationTestUtils.createHistogramBucket;
 import static org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationTestUtils.createMax;
@@ -472,6 +473,20 @@ public void testSingleBucketAgg_failureWithSubMultiBucket() throws IOException {
             () -> aggToString(Sets.newHashSet("my_field"), histogramBuckets));
     }
 
+    public void testGeoCentroidAgg() throws IOException {
+        List<Histogram.Bucket> histogramBuckets = Arrays.asList(
+            createHistogramBucket(1000L, 4, Arrays.asList(
+                createMax("time", 1000),
+                createGeoCentroid("geo_field", 4, 92.1, 93.1))),
+            createHistogramBucket(2000L, 7, Arrays.asList(
+                createMax("time", 2000),
+                createGeoCentroid("geo_field", 0, -1, -1))));
+        String json = aggToString(Sets.newHashSet("geo_field"), histogramBuckets);
+
+        assertThat(json, equalTo("{\"time\":1000,\"geo_field\":\"92.1,93.1\",\"doc_count\":4}" +
+            " {\"time\":2000,\"doc_count\":7}"));
+    }
+
     private String aggToString(Set<String> fields, Histogram.Bucket bucket) throws IOException {
         return aggToString(fields, Collections.singletonList(bucket));
     }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactoryTests.java
index 058e3c9e78645..9f345dfe77b40 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactoryTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactoryTests.java
@@ -34,7 +34,7 @@ public class ChunkedDataExtractorFactoryTests extends ESTestCase {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldTests.java
index 098aa48a4f75c..6969c97be0a9b 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldTests.java
@@ -62,6 +62,34 @@ public void testValueGivenNestedSource() {
         assertThat(nested.value(hit), equalTo(new String[] { "bar" }));
     }
 
+    public void testGeoPoint() {
+        double lat = 38.897676;
+        double lon = -77.03653;
+        String[] expected = new String[] {lat + "," + lon};
+
+        // doc_value field
+        ExtractedField geo = ExtractedField.newGeoPointField("geo", "geo");
+        SearchHit hit = new SearchHitBuilder(42).addField("geo", lat + ", " + lon).build();
+        assertThat(geo.value(hit), equalTo(expected));
+    }
+
+    public void testGeoShape() {
+        double lat = 38.897676;
+        double lon = -77.03653;
+        String[] expected = new String[] {lat + "," + lon};
+        // object format
+        SearchHit hit = new SearchHitBuilder(42)
+            .setSource("{\"geo\":{\"type\":\"point\", \"coordinates\": [" + lon + ", " + lat + "]}}")
+            .build();
+        ExtractedField geo = ExtractedField.newGeoShapeField("geo", "geo");
+        assertThat(geo.value(hit), equalTo(expected));
+
+        // WKT format
+        hit = new SearchHitBuilder(42).setSource("{\"geo\":\"POINT ("+ lon + " " + lat + ")\"}").build();
+        geo = ExtractedField.newGeoShapeField("geo", "geo");
+        assertThat(geo.value(hit), equalTo(expected));
+    }
+
     public void testValueGivenSourceAndHitWithNoSource() {
         ExtractedField missing = ExtractedField.newField("missing", ExtractedField.ExtractionMethod.SOURCE);
         assertThat(missing.value(new SearchHitBuilder(3).build()), equalTo(new Object[0]));
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndexTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndexTests.java
new file mode 100644
index 0000000000000..b81df21e4c63a
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsIndexTests.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsSource;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetection;
+import org.mockito.ArgumentCaptor;
+
+import java.io.IOException;
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.Arrays;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DataFrameAnalyticsIndexTests extends ESTestCase {
+
+    private static final String ANALYTICS_ID = "some-analytics-id";
+    private static final String[] SOURCE_INDEX = new String[] {"source-index"};
+    private static final String DEST_INDEX = "dest-index";
+    private static final DataFrameAnalyticsConfig ANALYTICS_CONFIG =
+        new DataFrameAnalyticsConfig.Builder(ANALYTICS_ID)
+            .setSource(new DataFrameAnalyticsSource(SOURCE_INDEX, null))
+            .setDest(new DataFrameAnalyticsDest(DEST_INDEX, null))
+            .setAnalysis(new OutlierDetection())
+            .build();
+    private static final int CURRENT_TIME_MILLIS = 123456789;
+    private static final String CREATED_BY = "data-frame-analytics";
+
+    private ThreadPool threadPool = mock(ThreadPool.class);
+    private Client client = mock(Client.class);
+    private Clock clock = Clock.fixed(Instant.ofEpochMilli(123456789L), ZoneId.systemDefault());
+
+    public void testCreateDestinationIndex() throws IOException {
+        when(client.threadPool()).thenReturn(threadPool);
+        when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
+
+        ArgumentCaptor<CreateIndexRequest> createIndexRequestCaptor = ArgumentCaptor.forClass(CreateIndexRequest.class);
+        doAnswer(
+            invocationOnMock -> {
+                @SuppressWarnings("unchecked")
+                ActionListener<CreateIndexResponse> listener = (ActionListener<CreateIndexResponse>) invocationOnMock.getArguments()[2];
+                listener.onResponse(null);
+                return null;
+            })
+            .when(client).execute(eq(CreateIndexAction.INSTANCE), createIndexRequestCaptor.capture(), any());
+
+        Settings index1Settings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .build();
+
+        Settings index2Settings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+            .build();
+
+        ArgumentCaptor<GetSettingsRequest> getSettingsRequestCaptor = ArgumentCaptor.forClass(GetSettingsRequest.class);
+        ArgumentCaptor<GetMappingsRequest> getMappingsRequestCaptor = ArgumentCaptor.forClass(GetMappingsRequest.class);
+
+        ImmutableOpenMap.Builder<String, Settings> indexToSettings = ImmutableOpenMap.builder();
+        indexToSettings.put("index_1", index1Settings);
+        indexToSettings.put("index_2", index2Settings);
+
+        GetSettingsResponse getSettingsResponse = new GetSettingsResponse(indexToSettings.build(), ImmutableOpenMap.of());
+
+        doAnswer(
+            invocationOnMock -> {
+                @SuppressWarnings("unchecked")
+                ActionListener<GetSettingsResponse> listener = (ActionListener<GetSettingsResponse>) invocationOnMock.getArguments()[2];
+                listener.onResponse(getSettingsResponse);
+                return null;
+            }
+        ).when(client).execute(eq(GetSettingsAction.INSTANCE), getSettingsRequestCaptor.capture(), any());
+
+        Map<String, Object> index1Mappings = Map.of("properties", Map.of("field_1", "field_1_mappings", "field_2", "field_2_mappings"));
+        MappingMetaData index1MappingMetaData = new MappingMetaData("_doc", index1Mappings);
+
+        Map<String, Object> index2Mappings = Map.of("properties", Map.of("field_1", "field_1_mappings", "field_2", "field_2_mappings"));
+        MappingMetaData index2MappingMetaData = new MappingMetaData("_doc", index2Mappings);
+
+        ImmutableOpenMap.Builder<String, MappingMetaData> index1MappingsMap = ImmutableOpenMap.builder();
+        index1MappingsMap.put("_doc", index1MappingMetaData);
+        ImmutableOpenMap.Builder<String, MappingMetaData> index2MappingsMap = ImmutableOpenMap.builder();
+        index2MappingsMap.put("_doc", index2MappingMetaData);
+
+        ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> mappings = ImmutableOpenMap.builder();
+        mappings.put("index_1", index1MappingsMap.build());
+        mappings.put("index_2", index2MappingsMap.build());
+
+        GetMappingsResponse getMappingsResponse = new GetMappingsResponse(mappings.build());
+
+        doAnswer(
+            invocationOnMock -> {
+                @SuppressWarnings("unchecked")
+                ActionListener<GetMappingsResponse> listener = (ActionListener<GetMappingsResponse>) invocationOnMock.getArguments()[2];
+                listener.onResponse(getMappingsResponse);
+                return null;
+            }
+        ).when(client).execute(eq(GetMappingsAction.INSTANCE), getMappingsRequestCaptor.capture(), any());
+
+        DataFrameAnalyticsIndex.createDestinationIndex(
+            client,
+            clock,
+            ANALYTICS_CONFIG,
+            ActionListener.wrap(
+                response -> {},
+                e -> fail(e.getMessage())));
+
+        GetSettingsRequest capturedGetSettingsRequest = getSettingsRequestCaptor.getValue();
+        assertThat(capturedGetSettingsRequest.indices(), equalTo(SOURCE_INDEX));
+        assertThat(capturedGetSettingsRequest.indicesOptions(), equalTo(IndicesOptions.lenientExpandOpen()));
+        assertThat(Arrays.asList(capturedGetSettingsRequest.names()), contains("index.number_of_shards", "index.number_of_replicas"));
+
+        assertThat(getMappingsRequestCaptor.getValue().indices(), equalTo(SOURCE_INDEX));
+
+        CreateIndexRequest createIndexRequest = createIndexRequestCaptor.getValue();
+
+        assertThat(createIndexRequest.settings().keySet(),
+            containsInAnyOrder("index.number_of_shards", "index.number_of_replicas", "index.sort.field", "index.sort.order"));
+        assertThat(createIndexRequest.settings().getAsInt("index.number_of_shards", -1), equalTo(5));
+        assertThat(createIndexRequest.settings().getAsInt("index.number_of_replicas", -1), equalTo(1));
+        assertThat(createIndexRequest.settings().get("index.sort.field"), equalTo("ml__id_copy"));
+        assertThat(createIndexRequest.settings().get("index.sort.order"), equalTo("asc"));
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, createIndexRequest.mappings().get("_doc"))) {
+            Map<String, Object> map = parser.map();
+            assertThat(extractValue("_doc.properties.ml__id_copy.type", map), equalTo("keyword"));
+            assertThat(extractValue("_doc.properties.field_1", map), equalTo("field_1_mappings"));
+            assertThat(extractValue("_doc.properties.field_2", map), equalTo("field_2_mappings"));
+            assertThat(extractValue("_doc._meta.analytics", map), equalTo(ANALYTICS_ID));
+            assertThat(extractValue("_doc._meta.creation_date_in_millis", map), equalTo(CURRENT_TIME_MILLIS));
+            assertThat(extractValue("_doc._meta.created_by", map), equalTo(CREATED_BY));
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/MappingsMergerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/MappingsMergerTests.java
new file mode 100644
index 0000000000000..670f6e559bd4f
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/MappingsMergerTests.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class MappingsMergerTests extends ESTestCase {
+
+    public void testMergeMappings_GivenIndicesWithIdenticalMappings() throws IOException {
+        Map<String, Object> index1Mappings = Map.of("properties", Map.of("field_1", "field_1_mappings", "field_2", "field_2_mappings"));
+        MappingMetaData index1MappingMetaData = new MappingMetaData("_doc", index1Mappings);
+
+        Map<String, Object> index2Mappings = Map.of("properties", Map.of("field_1", "field_1_mappings", "field_2", "field_2_mappings"));
+        MappingMetaData index2MappingMetaData = new MappingMetaData("_doc", index2Mappings);
+
+        ImmutableOpenMap.Builder<String, MappingMetaData> index1MappingsMap = ImmutableOpenMap.builder();
+        index1MappingsMap.put("_doc", index1MappingMetaData);
+        ImmutableOpenMap.Builder<String, MappingMetaData> index2MappingsMap = ImmutableOpenMap.builder();
+        index2MappingsMap.put("_doc", index2MappingMetaData);
+
+        ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> mappings = ImmutableOpenMap.builder();
+        mappings.put("index_1", index1MappingsMap.build());
+        mappings.put("index_2", index2MappingsMap.build());
+
+        GetMappingsResponse getMappingsResponse = new GetMappingsResponse(mappings.build());
+
+        ImmutableOpenMap<String, MappingMetaData> mergedMappings = MappingsMerger.mergeMappings(getMappingsResponse);
+
+        assertThat(mergedMappings.size(), equalTo(1));
+        assertThat(mergedMappings.containsKey("_doc"), is(true));
+        assertThat(mergedMappings.valuesIt().next().getSourceAsMap(), equalTo(index1Mappings));
+    }
+
+    public void testMergeMappings_GivenIndicesWithDifferentTypes() throws IOException {
+        Map<String, Object> index1Mappings = Map.of("properties", Map.of("field_1", "field_1_mappings"));
+        MappingMetaData index1MappingMetaData = new MappingMetaData("_doc", index1Mappings);
+
+        Map<String, Object> index2Mappings = Map.of("properties", Map.of("field_1", "field_1_mappings"));
+        MappingMetaData index2MappingMetaData = new MappingMetaData("_doc", index2Mappings);
+
+        ImmutableOpenMap.Builder<String, MappingMetaData> index1MappingsMap = ImmutableOpenMap.builder();
+        index1MappingsMap.put("type_1", index1MappingMetaData);
+        ImmutableOpenMap.Builder<String, MappingMetaData> index2MappingsMap = ImmutableOpenMap.builder();
+        index2MappingsMap.put("type_2", index2MappingMetaData);
+
+        ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> mappings = ImmutableOpenMap.builder();
+        mappings.put("index_1", index1MappingsMap.build());
+        mappings.put("index_2", index2MappingsMap.build());
+
+        GetMappingsResponse getMappingsResponse = new GetMappingsResponse(mappings.build());
+
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
+            () -> MappingsMerger.mergeMappings(getMappingsResponse));
+        assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        assertThat(e.getMessage(), containsString("source indices contain mappings for different types:"));
+        assertThat(e.getMessage(), containsString("type_1"));
+        assertThat(e.getMessage(), containsString("type_2"));
+    }
+
+    public void testMergeMappings_GivenFieldWithDifferentMapping() throws IOException {
+        Map<String, Object> index1Mappings = Map.of("properties", Map.of("field_1", "field_1_mappings"));
+        MappingMetaData index1MappingMetaData = new MappingMetaData("_doc", index1Mappings);
+
+        Map<String, Object> index2Mappings = Map.of("properties", Map.of("field_1", "different_field_1_mappings"));
+        MappingMetaData index2MappingMetaData = new MappingMetaData("_doc", index2Mappings);
+
+        ImmutableOpenMap.Builder<String, MappingMetaData> index1MappingsMap = ImmutableOpenMap.builder();
+        index1MappingsMap.put("_doc", index1MappingMetaData);
+        ImmutableOpenMap.Builder<String, MappingMetaData> index2MappingsMap = ImmutableOpenMap.builder();
+        index2MappingsMap.put("_doc", index2MappingMetaData);
+
+        ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> mappings = ImmutableOpenMap.builder();
+        mappings.put("index_1", index1MappingsMap.build());
+        mappings.put("index_2", index2MappingsMap.build());
+
+        GetMappingsResponse getMappingsResponse = new GetMappingsResponse(mappings.build());
+
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
+            () -> MappingsMerger.mergeMappings(getMappingsResponse));
+        assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        assertThat(e.getMessage(), equalTo("cannot merge mappings because of differences for field [field_1]"));
+    }
+
+    public void testMergeMappings_GivenIndicesWithDifferentMappingsButNoConflicts() throws IOException {
+        Map<String, Object> index1Mappings = Map.of("properties",
+            Map.of("field_1", "field_1_mappings", "field_2", "field_2_mappings"));
+        MappingMetaData index1MappingMetaData = new MappingMetaData("_doc", index1Mappings);
+
+        Map<String, Object> index2Mappings = Map.of("properties",
+            Map.of("field_1", "field_1_mappings", "field_3", "field_3_mappings"));
+        MappingMetaData index2MappingMetaData = new MappingMetaData("_doc", index2Mappings);
+
+        ImmutableOpenMap.Builder<String, MappingMetaData> index1MappingsMap = ImmutableOpenMap.builder();
+        index1MappingsMap.put("_doc", index1MappingMetaData);
+        ImmutableOpenMap.Builder<String, MappingMetaData> index2MappingsMap = ImmutableOpenMap.builder();
+        index2MappingsMap.put("_doc", index2MappingMetaData);
+
+        ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> mappings = ImmutableOpenMap.builder();
+        mappings.put("index_1", index1MappingsMap.build());
+        mappings.put("index_2", index2MappingsMap.build());
+
+        GetMappingsResponse getMappingsResponse = new GetMappingsResponse(mappings.build());
+
+        ImmutableOpenMap<String, MappingMetaData> mergedMappings = MappingsMerger.mergeMappings(getMappingsResponse);
+
+        assertThat(mergedMappings.size(), equalTo(1));
+        assertThat(mergedMappings.containsKey("_doc"), is(true));
+        Map<String, Object> mappingsAsMap = mergedMappings.valuesIt().next().getSourceAsMap();
+        assertThat(mappingsAsMap.size(), equalTo(1));
+        assertThat(mappingsAsMap.containsKey("properties"), is(true));
+
+        @SuppressWarnings("unchecked")
+        Map<String, Object> fieldMappings = (Map<String, Object>) mappingsAsMap.get("properties");
+
+        assertThat(fieldMappings.size(), equalTo(3));
+        assertThat(fieldMappings.keySet(), containsInAnyOrder("field_1", "field_2", "field_3"));
+        assertThat(fieldMappings.get("field_1"), equalTo("field_1_mappings"));
+        assertThat(fieldMappings.get("field_2"), equalTo("field_2_mappings"));
+        assertThat(fieldMappings.get("field_3"), equalTo("field_3_mappings"));
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/SourceDestValidatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/SourceDestValidatorTests.java
new file mode 100644
index 0000000000000..d48d079314aa6
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/SourceDestValidatorTests.java
@@ -0,0 +1,180 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsSource;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetection;
+
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_CREATION_DATE;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED;
+import static org.hamcrest.Matchers.equalTo;
+
+public class SourceDestValidatorTests extends ESTestCase {
+
+    private static final String SOURCE_1 = "source-1";
+    private static final String SOURCE_2 = "source-2";
+    private static final String ALIASED_DEST = "aliased-dest";
+
+    private static final ClusterState CLUSTER_STATE;
+
+    static {
+        IndexMetaData source1 = IndexMetaData.builder(SOURCE_1).settings(Settings.builder()
+            .put(SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(SETTING_CREATION_DATE, System.currentTimeMillis()))
+            .putAlias(AliasMetaData.builder("source-1-alias").build())
+            .build();
+        IndexMetaData source2 = IndexMetaData.builder(SOURCE_2).settings(Settings.builder()
+            .put(SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(SETTING_CREATION_DATE, System.currentTimeMillis()))
+            .putAlias(AliasMetaData.builder("dest-alias").build())
+            .build();
+        IndexMetaData aliasedDest = IndexMetaData.builder(ALIASED_DEST).settings(Settings.builder()
+            .put(SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(SETTING_CREATION_DATE, System.currentTimeMillis()))
+            .putAlias(AliasMetaData.builder("dest-alias").build())
+            .build();
+        ClusterState.Builder state = ClusterState.builder(new ClusterName("test"));
+        state.metaData(MetaData.builder()
+            .put(IndexMetaData.builder(source1).build(), false)
+            .put(IndexMetaData.builder(source2).build(), false)
+            .put(IndexMetaData.builder(aliasedDest).build(), false));
+        CLUSTER_STATE = state.build();
+    }
+
+    public void testCheck_GivenSimpleSourceIndexAndValidDestIndex() {
+        DataFrameAnalyticsConfig config = new DataFrameAnalyticsConfig.Builder("test")
+            .setSource(createSource("source-1"))
+            .setDest(new DataFrameAnalyticsDest("dest", null))
+            .setAnalysis(new OutlierDetection())
+            .build();
+
+        SourceDestValidator validator = new SourceDestValidator(CLUSTER_STATE, new IndexNameExpressionResolver());
+        validator.check(config);
+    }
+
+    public void testCheck_GivenMissingConcreteSourceIndex() {
+        DataFrameAnalyticsConfig config = new DataFrameAnalyticsConfig.Builder("test")
+            .setSource(createSource("missing"))
+            .setDest(new DataFrameAnalyticsDest("dest", null))
+            .setAnalysis(new OutlierDetection())
+            .build();
+
+        SourceDestValidator validator = new SourceDestValidator(CLUSTER_STATE, new IndexNameExpressionResolver());
+
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> validator.check(config));
+        assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        assertThat(e.getMessage(), equalTo("No index matches source index [missing]"));
+    }
+
+    public void testCheck_GivenMissingWildcardSourceIndex() {
+        DataFrameAnalyticsConfig config = new DataFrameAnalyticsConfig.Builder("test")
+            .setSource(createSource("missing*"))
+            .setDest(new DataFrameAnalyticsDest("dest", null))
+            .setAnalysis(new OutlierDetection())
+            .build();
+
+        SourceDestValidator validator = new SourceDestValidator(CLUSTER_STATE, new IndexNameExpressionResolver());
+
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> validator.check(config));
+        assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        assertThat(e.getMessage(), equalTo("No index matches source index [missing*]"));
+    }
+
+    public void testCheck_GivenDestIndexSameAsSourceIndex() {
+        DataFrameAnalyticsConfig config = new DataFrameAnalyticsConfig.Builder("test")
+            .setSource(createSource("source-1"))
+            .setDest(new DataFrameAnalyticsDest("source-1", null))
+            .setAnalysis(new OutlierDetection())
+            .build();
+
+        SourceDestValidator validator = new SourceDestValidator(CLUSTER_STATE, new IndexNameExpressionResolver());
+
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> validator.check(config));
+        assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        assertThat(e.getMessage(), equalTo("Destination index [source-1] must not be included in source index [source-1]"));
+    }
+
+    public void testCheck_GivenDestIndexMatchesSourceIndex() {
+        DataFrameAnalyticsConfig config = new DataFrameAnalyticsConfig.Builder("test")
+            .setSource(createSource("source-*"))
+            .setDest(new DataFrameAnalyticsDest(SOURCE_2, null))
+            .setAnalysis(new OutlierDetection())
+            .build();
+
+        SourceDestValidator validator = new SourceDestValidator(CLUSTER_STATE, new IndexNameExpressionResolver());
+
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> validator.check(config));
+        assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        assertThat(e.getMessage(), equalTo("Destination index [source-2] must not be included in source index [source-*]"));
+    }
+
+    public void testCheck_GivenDestIndexMatchesOneOfSourceIndices() {
+        DataFrameAnalyticsConfig config = new DataFrameAnalyticsConfig.Builder("test")
+            .setSource(createSource("source-1,source-*"))
+            .setDest(new DataFrameAnalyticsDest(SOURCE_2, null))
+            .setAnalysis(new OutlierDetection())
+            .build();
+
+        SourceDestValidator validator = new SourceDestValidator(CLUSTER_STATE, new IndexNameExpressionResolver());
+
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> validator.check(config));
+        assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        assertThat(e.getMessage(), equalTo("Destination index [source-2] must not be included in source index [source-*]"));
+    }
+
+    public void testCheck_GivenDestIndexIsAliasThatMatchesMultipleIndices() {
+        DataFrameAnalyticsConfig config = new DataFrameAnalyticsConfig.Builder("test")
+            .setSource(createSource(SOURCE_1))
+            .setDest(new DataFrameAnalyticsDest("dest-alias", null))
+            .setAnalysis(new OutlierDetection())
+            .build();
+
+        SourceDestValidator validator = new SourceDestValidator(CLUSTER_STATE, new IndexNameExpressionResolver());
+
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> validator.check(config));
+        assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        assertThat(e.getMessage(),
+            equalTo("Destination index [dest-alias] should match a single index; matches [source-2, aliased-dest]"));
+    }
+
+    public void testCheck_GivenDestIndexIsAliasThatIsIncludedInSource() {
+        DataFrameAnalyticsConfig config = new DataFrameAnalyticsConfig.Builder("test")
+            .setSource(createSource("source-1"))
+            .setDest(new DataFrameAnalyticsDest("source-1-alias", null))
+            .setAnalysis(new OutlierDetection())
+            .build();
+
+        SourceDestValidator validator = new SourceDestValidator(CLUSTER_STATE, new IndexNameExpressionResolver());
+
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> validator.check(config));
+        assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
+        assertThat(e.getMessage(),
+            equalTo("Destination index [source-1-alias], which is an alias for [source-1], " +
+                "must not be included in source index [source-1]"));
+    }
+
+    private static DataFrameAnalyticsSource createSource(String... index) {
+        return new DataFrameAnalyticsSource(index, null);
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
new file mode 100644
index 0000000000000..47c5aa26390a5
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
@@ -0,0 +1,392 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.extractor;
+
+import org.apache.lucene.search.TotalHits;
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.search.ClearScrollAction;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedField;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedFields;
+import org.elasticsearch.xpack.ml.test.SearchHitBuilder;
+import org.junit.Before;
+import org.mockito.ArgumentCaptor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Queue;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Matchers.same;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DataFrameDataExtractorTests extends ESTestCase {
+
+    private static final String JOB_ID = "foo";
+
+    private Client client;
+    private List<String> indices;
+    private ExtractedFields extractedFields;
+    private QueryBuilder query;
+    private int scrollSize;
+    private Map<String, String> headers;
+    private ArgumentCaptor<ClearScrollRequest> capturedClearScrollRequests;
+    private ActionFuture<ClearScrollResponse> clearScrollFuture;
+
+    @Before
+    public void setUpTests() {
+        ThreadPool threadPool = mock(ThreadPool.class);
+        when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY));
+        client = mock(Client.class);
+        when(client.threadPool()).thenReturn(threadPool);
+
+        indices = Arrays.asList("index-1", "index-2");
+        query = QueryBuilders.matchAllQuery();
+        extractedFields = new ExtractedFields(Arrays.asList(
+            ExtractedField.newField("field_1", ExtractedField.ExtractionMethod.DOC_VALUE),
+            ExtractedField.newField("field_2", ExtractedField.ExtractionMethod.DOC_VALUE)));
+        scrollSize = 1000;
+        headers = Collections.emptyMap();
+
+        clearScrollFuture = mock(ActionFuture.class);
+        capturedClearScrollRequests = ArgumentCaptor.forClass(ClearScrollRequest.class);
+        when(client.execute(same(ClearScrollAction.INSTANCE), capturedClearScrollRequests.capture())).thenReturn(clearScrollFuture);
+    }
+
+    public void testTwoPageExtraction() throws IOException {
+        TestExtractor dataExtractor = createExtractor(true);
+
+        // First batch
+        SearchResponse response1 = createSearchResponse(Arrays.asList(1_1, 1_2, 1_3), Arrays.asList(2_1, 2_2, 2_3));
+        dataExtractor.setNextResponse(response1);
+
+        // Second batch
+        SearchResponse response2 = createSearchResponse(Arrays.asList(3_1), Arrays.asList(4_1));
+        dataExtractor.setNextResponse(response2);
+
+        // Third batch is empty
+        SearchResponse lastAndEmptyResponse = createEmptySearchResponse();
+        dataExtractor.setNextResponse(lastAndEmptyResponse);
+
+        assertThat(dataExtractor.hasNext(), is(true));
+
+        // First batch
+        Optional<List<DataFrameDataExtractor.Row>> rows = dataExtractor.next();
+        assertThat(rows.isPresent(), is(true));
+        assertThat(rows.get().size(), equalTo(3));
+        assertThat(rows.get().get(0).getValues(), equalTo(new String[] {"11", "21"}));
+        assertThat(rows.get().get(1).getValues(), equalTo(new String[] {"12", "22"}));
+        assertThat(rows.get().get(2).getValues(), equalTo(new String[] {"13", "23"}));
+        assertThat(dataExtractor.hasNext(), is(true));
+
+        // Second batch
+        rows = dataExtractor.next();
+        assertThat(rows.isPresent(), is(true));
+        assertThat(rows.get().size(), equalTo(1));
+        assertThat(rows.get().get(0).getValues(), equalTo(new String[] {"31", "41"}));
+        assertThat(dataExtractor.hasNext(), is(true));
+
+        // Third batch should return empty
+        rows = dataExtractor.next();
+        assertThat(rows.isEmpty(), is(true));
+        assertThat(dataExtractor.hasNext(), is(false));
+
+        // Now let's assert we're sending the expected search request
+        assertThat(dataExtractor.capturedSearchRequests.size(), equalTo(1));
+        String searchRequest = dataExtractor.capturedSearchRequests.get(0).request().toString().replaceAll("\\s", "");
+        assertThat(searchRequest, containsString("allowPartialSearchResults=false"));
+        assertThat(searchRequest, containsString("indices=[index-1,index-2]"));
+        assertThat(searchRequest, containsString("\"size\":1000"));
+        assertThat(searchRequest, containsString("\"query\":{\"match_all\":{\"boost\":1.0}}"));
+        assertThat(searchRequest, containsString("\"docvalue_fields\":[{\"field\":\"field_1\"},{\"field\":\"field_2\"}]"));
+        assertThat(searchRequest, containsString("\"_source\":{\"includes\":[],\"excludes\":[]}"));
+        assertThat(searchRequest, containsString("\"sort\":[{\"ml__id_copy\":{\"order\":\"asc\"}}]"));
+
+        // Check continue scroll requests had correct ids
+        assertThat(dataExtractor.capturedContinueScrollIds.size(), equalTo(2));
+        assertThat(dataExtractor.capturedContinueScrollIds.get(0), equalTo(response1.getScrollId()));
+        assertThat(dataExtractor.capturedContinueScrollIds.get(1), equalTo(response2.getScrollId()));
+
+        // Check we cleared the scroll with the latest scroll id
+        List<String> capturedClearScrollRequests = getCapturedClearScrollIds();
+        assertThat(capturedClearScrollRequests.size(), equalTo(1));
+        assertThat(capturedClearScrollRequests.get(0), equalTo(lastAndEmptyResponse.getScrollId()));
+    }
+
+    public void testRecoveryFromErrorOnSearchAfterRetry() throws IOException {
+        TestExtractor dataExtractor = createExtractor(true);
+
+        // First search will fail
+        dataExtractor.setNextResponse(createResponseWithShardFailures());
+
+        // Next one will succeed
+        SearchResponse response = createSearchResponse(Arrays.asList(1_1), Arrays.asList(2_1));
+        dataExtractor.setNextResponse(response);
+
+        // Last one
+        SearchResponse lastAndEmptyResponse = createEmptySearchResponse();
+        dataExtractor.setNextResponse(lastAndEmptyResponse);
+
+        assertThat(dataExtractor.hasNext(), is(true));
+
+        // First batch expected as normally since we'll retry after the error
+        Optional<List<DataFrameDataExtractor.Row>> rows = dataExtractor.next();
+        assertThat(rows.isPresent(), is(true));
+        assertThat(rows.get().size(), equalTo(1));
+        assertThat(rows.get().get(0).getValues(), equalTo(new String[] {"11", "21"}));
+        assertThat(dataExtractor.hasNext(), is(true));
+
+        // Next batch should return empty
+        rows = dataExtractor.next();
+        assertThat(rows.isEmpty(), is(true));
+        assertThat(dataExtractor.hasNext(), is(false));
+
+        // Check we cleared the scroll with the latest scroll id
+        List<String> capturedClearScrollRequests = getCapturedClearScrollIds();
+        assertThat(capturedClearScrollRequests.size(), equalTo(1));
+        assertThat(capturedClearScrollRequests.get(0), equalTo(lastAndEmptyResponse.getScrollId()));
+    }
+
+    public void testErrorOnSearchTwiceLeadsToFailure() {
+        TestExtractor dataExtractor = createExtractor(true);
+
+        // First search will fail
+        dataExtractor.setNextResponse(createResponseWithShardFailures());
+        // Next one fails again
+        dataExtractor.setNextResponse(createResponseWithShardFailures());
+
+        assertThat(dataExtractor.hasNext(), is(true));
+
+        expectThrows(RuntimeException.class, () -> dataExtractor.next());
+    }
+
+    public void testRecoveryFromErrorOnContinueScrollAfterRetry() throws IOException {
+        TestExtractor dataExtractor = createExtractor(true);
+
+        // Search will succeed
+        SearchResponse response1 = createSearchResponse(Arrays.asList(1_1), Arrays.asList(2_1));
+        dataExtractor.setNextResponse(response1);
+
+        // But the first continue scroll fails
+        dataExtractor.setNextResponse(createResponseWithShardFailures());
+
+        // The next one succeeds and we shall recover
+        SearchResponse response2 = createSearchResponse(Arrays.asList(1_2), Arrays.asList(2_2));
+        dataExtractor.setNextResponse(response2);
+
+        // Last one
+        SearchResponse lastAndEmptyResponse = createEmptySearchResponse();
+        dataExtractor.setNextResponse(lastAndEmptyResponse);
+
+        assertThat(dataExtractor.hasNext(), is(true));
+
+        // First batch expected as normally since we'll retry after the error
+        Optional<List<DataFrameDataExtractor.Row>> rows = dataExtractor.next();
+        assertThat(rows.isPresent(), is(true));
+        assertThat(rows.get().size(), equalTo(1));
+        assertThat(rows.get().get(0).getValues(), equalTo(new String[] {"11", "21"}));
+        assertThat(dataExtractor.hasNext(), is(true));
+
+        // We get second batch as we retried after the error
+        rows = dataExtractor.next();
+        assertThat(rows.isPresent(), is(true));
+        assertThat(rows.get().size(), equalTo(1));
+        assertThat(rows.get().get(0).getValues(), equalTo(new String[] {"12", "22"}));
+        assertThat(dataExtractor.hasNext(), is(true));
+
+        // Next batch should return empty
+        rows = dataExtractor.next();
+        assertThat(rows.isEmpty(), is(true));
+        assertThat(dataExtractor.hasNext(), is(false));
+
+        // Notice we've done two searches and two continues here
+        assertThat(dataExtractor.capturedSearchRequests.size(), equalTo(2));
+        assertThat(dataExtractor.capturedContinueScrollIds.size(), equalTo(2));
+
+        // Check we cleared the scroll with the latest scroll id
+        List<String> capturedClearScrollRequests = getCapturedClearScrollIds();
+        assertThat(capturedClearScrollRequests.size(), equalTo(1));
+        assertThat(capturedClearScrollRequests.get(0), equalTo(lastAndEmptyResponse.getScrollId()));
+    }
+
+    public void testErrorOnContinueScrollTwiceLeadsToFailure() throws IOException {
+        TestExtractor dataExtractor = createExtractor(true);
+
+        // Search will succeed
+        SearchResponse response1 = createSearchResponse(Arrays.asList(1_1), Arrays.asList(2_1));
+        dataExtractor.setNextResponse(response1);
+
+        // But the first continue scroll fails
+        dataExtractor.setNextResponse(createResponseWithShardFailures());
+        // As well as the second
+        dataExtractor.setNextResponse(createResponseWithShardFailures());
+
+        assertThat(dataExtractor.hasNext(), is(true));
+
+        // First batch expected as normally since we'll retry after the error
+        Optional<List<DataFrameDataExtractor.Row>> rows = dataExtractor.next();
+        assertThat(rows.isPresent(), is(true));
+        assertThat(rows.get().size(), equalTo(1));
+        assertThat(rows.get().get(0).getValues(), equalTo(new String[] {"11", "21"}));
+        assertThat(dataExtractor.hasNext(), is(true));
+
+        // We get second batch as we retried after the error
+        expectThrows(RuntimeException.class, () -> dataExtractor.next());
+    }
+
+    public void testIncludeSourceIsFalseAndNoSourceFields() throws IOException {
+        TestExtractor dataExtractor = createExtractor(false);
+
+        SearchResponse response = createSearchResponse(Arrays.asList(1_1), Arrays.asList(2_1));
+        dataExtractor.setNextResponse(response);
+        dataExtractor.setNextResponse(createEmptySearchResponse());
+
+        assertThat(dataExtractor.hasNext(), is(true));
+
+        Optional<List<DataFrameDataExtractor.Row>> rows = dataExtractor.next();
+        assertThat(rows.isPresent(), is(true));
+        assertThat(rows.get().size(), equalTo(1));
+        assertThat(rows.get().get(0).getValues(), equalTo(new String[] {"11", "21"}));
+        assertThat(dataExtractor.hasNext(), is(true));
+
+        assertThat(dataExtractor.next().isEmpty(), is(true));
+        assertThat(dataExtractor.hasNext(), is(false));
+
+        assertThat(dataExtractor.capturedSearchRequests.size(), equalTo(1));
+        String searchRequest = dataExtractor.capturedSearchRequests.get(0).request().toString().replaceAll("\\s", "");
+        assertThat(searchRequest, containsString("\"docvalue_fields\":[{\"field\":\"field_1\"},{\"field\":\"field_2\"}]"));
+        assertThat(searchRequest, containsString("\"_source\":false"));
+    }
+
+    public void testIncludeSourceIsFalseAndAtLeastOneSourceField() throws IOException {
+        extractedFields = new ExtractedFields(Arrays.asList(
+            ExtractedField.newField("field_1", ExtractedField.ExtractionMethod.DOC_VALUE),
+            ExtractedField.newField("field_2", ExtractedField.ExtractionMethod.SOURCE)));
+
+        TestExtractor dataExtractor = createExtractor(false);
+
+        SearchResponse response = createSearchResponse(Arrays.asList(1_1), Arrays.asList(2_1));
+        dataExtractor.setNextResponse(response);
+        dataExtractor.setNextResponse(createEmptySearchResponse());
+
+        assertThat(dataExtractor.hasNext(), is(true));
+
+        Optional<List<DataFrameDataExtractor.Row>> rows = dataExtractor.next();
+        assertThat(rows.isPresent(), is(true));
+        assertThat(rows.get().size(), equalTo(1));
+        assertThat(rows.get().get(0).getValues(), equalTo(new String[] {"11", "21"}));
+        assertThat(dataExtractor.hasNext(), is(true));
+
+        assertThat(dataExtractor.next().isEmpty(), is(true));
+        assertThat(dataExtractor.hasNext(), is(false));
+
+        assertThat(dataExtractor.capturedSearchRequests.size(), equalTo(1));
+        String searchRequest = dataExtractor.capturedSearchRequests.get(0).request().toString().replaceAll("\\s", "");
+        assertThat(searchRequest, containsString("\"docvalue_fields\":[{\"field\":\"field_1\"}]"));
+        assertThat(searchRequest, containsString("\"_source\":{\"includes\":[\"field_2\"],\"excludes\":[]}"));
+    }
+
+    private TestExtractor createExtractor(boolean includeSource) {
+        DataFrameDataExtractorContext context = new DataFrameDataExtractorContext(
+            JOB_ID, extractedFields, indices, query, scrollSize, headers, includeSource);
+        return new TestExtractor(client, context);
+    }
+
+    private SearchResponse createSearchResponse(List<Number> field1Values, List<Number> field2Values) {
+        assertThat(field1Values.size(), equalTo(field2Values.size()));
+        SearchResponse searchResponse = mock(SearchResponse.class);
+        when(searchResponse.getScrollId()).thenReturn(randomAlphaOfLength(1000));
+        List<SearchHit> hits = new ArrayList<>();
+        for (int i = 0; i < field1Values.size(); i++) {
+            SearchHit hit = new SearchHit(randomInt());
+            SearchHitBuilder searchHitBuilder = new SearchHitBuilder(randomInt())
+                .addField("field_1", Collections.singletonList(field1Values.get(i)))
+                .addField("field_2", Collections.singletonList(field2Values.get(i)))
+                .setSource("{\"field_1\":" + field1Values.get(i) + ",\"field_2\":" + field2Values.get(i) + "}");
+            hits.add(searchHitBuilder.build());
+        }
+        SearchHits searchHits = new SearchHits(hits.toArray(new SearchHit[0]), new TotalHits(hits.size(), TotalHits.Relation.EQUAL_TO), 1);
+        when(searchResponse.getHits()).thenReturn(searchHits);
+        return searchResponse;
+    }
+
+    private SearchResponse createEmptySearchResponse() {
+        return createSearchResponse(Collections.emptyList(), Collections.emptyList());
+    }
+
+    private SearchResponse createResponseWithShardFailures() {
+        SearchResponse searchResponse = mock(SearchResponse.class);
+        when(searchResponse.status()).thenReturn(RestStatus.OK);
+        when(searchResponse.getShardFailures()).thenReturn(
+            new ShardSearchFailure[] { new ShardSearchFailure(new RuntimeException("shard failed"))});
+        when(searchResponse.getFailedShards()).thenReturn(1);
+        when(searchResponse.getScrollId()).thenReturn(randomAlphaOfLength(1000));
+        return searchResponse;
+    }
+
+    private List<String> getCapturedClearScrollIds() {
+        return capturedClearScrollRequests.getAllValues().stream().map(r -> r.getScrollIds().get(0)).collect(Collectors.toList());
+    }
+
+    private static class TestExtractor extends DataFrameDataExtractor {
+
+        private Queue<SearchResponse> responses = new LinkedList<>();
+        private List<SearchRequestBuilder> capturedSearchRequests = new ArrayList<>();
+        private List<String> capturedContinueScrollIds = new ArrayList<>();
+
+        TestExtractor(Client client, DataFrameDataExtractorContext context) {
+            super(client, context);
+        }
+
+        void setNextResponse(SearchResponse searchResponse) {
+            responses.add(searchResponse);
+        }
+
+        @Override
+        protected SearchResponse executeSearchRequest(SearchRequestBuilder searchRequestBuilder) {
+            capturedSearchRequests.add(searchRequestBuilder);
+            SearchResponse searchResponse = responses.remove();
+            if (searchResponse.getShardFailures() != null) {
+                throw new RuntimeException(searchResponse.getShardFailures()[0].getCause());
+            }
+            return searchResponse;
+        }
+
+        @Override
+        protected SearchResponse executeSearchScrollRequest(String scrollId) {
+            capturedContinueScrollIds.add(scrollId);
+            SearchResponse searchResponse = responses.remove();
+            if (searchResponse.getShardFailures() != null) {
+                throw new RuntimeException(searchResponse.getShardFailures()[0].getCause());
+            }
+            return searchResponse;
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java
new file mode 100644
index 0000000000000..1345a1fe1287b
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java
@@ -0,0 +1,319 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.extractor;
+
+import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.fieldcaps.FieldCapabilities;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsSource;
+import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetection;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedField;
+import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedFields;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ExtractedFieldsDetectorTests extends ESTestCase {
+
+    private static final String[] SOURCE_INDEX = new String[] { "source_index" };
+    private static final String DEST_INDEX = "dest_index";
+    private static final String RESULTS_FIELD = "ml";
+
+    public void testDetect_GivenFloatField() {
+        FieldCapabilitiesResponse fieldCapabilities= new MockFieldCapsResponseBuilder()
+            .addAggregatableField("some_float", "float").build();
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildAnalyticsConfig(), false, 100, fieldCapabilities);
+        ExtractedFields extractedFields = extractedFieldsDetector.detect();
+
+        List<ExtractedField> allFields = extractedFields.getAllFields();
+        assertThat(allFields.size(), equalTo(1));
+        assertThat(allFields.get(0).getName(), equalTo("some_float"));
+        assertThat(allFields.get(0).getExtractionMethod(), equalTo(ExtractedField.ExtractionMethod.DOC_VALUE));
+    }
+
+    public void testDetect_GivenNumericFieldWithMultipleTypes() {
+        FieldCapabilitiesResponse fieldCapabilities= new MockFieldCapsResponseBuilder()
+            .addAggregatableField("some_number", "long", "integer", "short", "byte", "double", "float", "half_float", "scaled_float")
+            .build();
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildAnalyticsConfig(), false, 100, fieldCapabilities);
+        ExtractedFields extractedFields = extractedFieldsDetector.detect();
+
+        List<ExtractedField> allFields = extractedFields.getAllFields();
+        assertThat(allFields.size(), equalTo(1));
+        assertThat(allFields.get(0).getName(), equalTo("some_number"));
+        assertThat(allFields.get(0).getExtractionMethod(), equalTo(ExtractedField.ExtractionMethod.DOC_VALUE));
+    }
+
+    public void testDetect_GivenNonNumericField() {
+        FieldCapabilitiesResponse fieldCapabilities= new MockFieldCapsResponseBuilder()
+            .addAggregatableField("some_keyword", "keyword").build();
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildAnalyticsConfig(), false, 100, fieldCapabilities);
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> extractedFieldsDetector.detect());
+
+        assertThat(e.getMessage(), equalTo("No compatible fields could be detected in index [source_index]"));
+    }
+
+    public void testDetect_GivenFieldWithNumericAndNonNumericTypes() {
+        FieldCapabilitiesResponse fieldCapabilities= new MockFieldCapsResponseBuilder()
+            .addAggregatableField("indecisive_field", "float", "keyword").build();
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildAnalyticsConfig(), false, 100, fieldCapabilities);
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> extractedFieldsDetector.detect());
+
+        assertThat(e.getMessage(), equalTo("No compatible fields could be detected in index [source_index]"));
+    }
+
+    public void testDetect_GivenMultipleFields() {
+        FieldCapabilitiesResponse fieldCapabilities= new MockFieldCapsResponseBuilder()
+            .addAggregatableField("some_float", "float")
+            .addAggregatableField("some_long", "long")
+            .addAggregatableField("some_keyword", "keyword")
+            .build();
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildAnalyticsConfig(), false, 100, fieldCapabilities);
+        ExtractedFields extractedFields = extractedFieldsDetector.detect();
+
+        List<ExtractedField> allFields = extractedFields.getAllFields();
+        assertThat(allFields.size(), equalTo(2));
+        assertThat(allFields.stream().map(ExtractedField::getName).collect(Collectors.toSet()),
+            containsInAnyOrder("some_float", "some_long"));
+        assertThat(allFields.stream().map(ExtractedField::getExtractionMethod).collect(Collectors.toSet()),
+            contains(equalTo(ExtractedField.ExtractionMethod.DOC_VALUE)));
+    }
+
+    public void testDetect_GivenIgnoredField() {
+        FieldCapabilitiesResponse fieldCapabilities= new MockFieldCapsResponseBuilder()
+            .addAggregatableField("_id", "float").build();
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildAnalyticsConfig(), false, 100, fieldCapabilities);
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> extractedFieldsDetector.detect());
+
+        assertThat(e.getMessage(), equalTo("No compatible fields could be detected in index [source_index]"));
+    }
+
+    public void testDetect_ShouldSortFieldsAlphabetically() {
+        int fieldCount = randomIntBetween(10, 20);
+        List<String> fields = new ArrayList<>();
+        for (int i = 0; i < fieldCount; i++) {
+            fields.add(randomAlphaOfLength(20));
+        }
+        List<String> sortedFields = new ArrayList<>(fields);
+        Collections.sort(sortedFields);
+
+        MockFieldCapsResponseBuilder mockFieldCapsResponseBuilder = new MockFieldCapsResponseBuilder();
+        for (String field : fields) {
+            mockFieldCapsResponseBuilder.addAggregatableField(field, "float");
+        }
+        FieldCapabilitiesResponse fieldCapabilities = mockFieldCapsResponseBuilder.build();
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildAnalyticsConfig(), false, 100, fieldCapabilities);
+        ExtractedFields extractedFields = extractedFieldsDetector.detect();
+
+        List<String> extractedFieldNames = extractedFields.getAllFields().stream().map(ExtractedField::getName)
+            .collect(Collectors.toList());
+        assertThat(extractedFieldNames, equalTo(sortedFields));
+    }
+
+    public void testDetectedExtractedFields_GivenIncludeWithMissingField() {
+        FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
+            .addAggregatableField("my_field1", "float")
+            .addAggregatableField("my_field2", "float")
+            .build();
+
+        FetchSourceContext desiredFields = new FetchSourceContext(true, new String[]{"your_field1", "my*"}, new String[0]);
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildAnalyticsConfig(desiredFields), false, 100, fieldCapabilities);
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> extractedFieldsDetector.detect());
+
+        assertThat(e.getMessage(), equalTo("No field [your_field1] could be detected"));
+    }
+
+    public void testDetectedExtractedFields_GivenExcludeAllValidFields() {
+        FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
+            .addAggregatableField("my_field1", "float")
+            .addAggregatableField("my_field2", "float")
+            .build();
+
+        FetchSourceContext desiredFields = new FetchSourceContext(true, new String[0], new String[]{"my_*"});
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildAnalyticsConfig(desiredFields), false, 100, fieldCapabilities);
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> extractedFieldsDetector.detect());
+        assertThat(e.getMessage(), equalTo("No compatible fields could be detected in index [source_index]"));
+    }
+
+    public void testDetectedExtractedFields_GivenInclusionsAndExclusions() {
+        FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
+            .addAggregatableField("my_field1_nope", "float")
+            .addAggregatableField("my_field1", "float")
+            .addAggregatableField("your_field2", "float")
+            .addAggregatableField("your_keyword", "keyword")
+            .build();
+
+        FetchSourceContext desiredFields = new FetchSourceContext(true, new String[]{"your*", "my_*"}, new String[]{"*nope"});
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildAnalyticsConfig(desiredFields), false, 100, fieldCapabilities);
+        ExtractedFields extractedFields = extractedFieldsDetector.detect();
+
+        List<String> extractedFieldNames = extractedFields.getAllFields().stream().map(ExtractedField::getName)
+            .collect(Collectors.toList());
+        assertThat(extractedFieldNames, equalTo(Arrays.asList("my_field1", "your_field2")));
+    }
+
+    public void testDetectedExtractedFields_GivenIndexContainsResultsField() {
+        FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
+            .addAggregatableField(RESULTS_FIELD, "float")
+            .addAggregatableField("my_field1", "float")
+            .addAggregatableField("your_field2", "float")
+            .addAggregatableField("your_keyword", "keyword")
+            .build();
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildAnalyticsConfig(), false, 100, fieldCapabilities);
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> extractedFieldsDetector.detect());
+
+        assertThat(e.getMessage(), equalTo("A field that matches the dest.results_field [ml] already exists; " +
+            "please set a different results_field"));
+    }
+
+    public void testDetectedExtractedFields_GivenIndexContainsResultsFieldAndTaskIsRestarting() {
+        FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
+            .addAggregatableField(RESULTS_FIELD + ".outlier_score", "float")
+            .addAggregatableField("my_field1", "float")
+            .addAggregatableField("your_field2", "float")
+            .addAggregatableField("your_keyword", "keyword")
+            .build();
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildAnalyticsConfig(), true, 100, fieldCapabilities);
+        ExtractedFields extractedFields = extractedFieldsDetector.detect();
+
+        List<String> extractedFieldNames = extractedFields.getAllFields().stream().map(ExtractedField::getName)
+            .collect(Collectors.toList());
+        assertThat(extractedFieldNames, equalTo(Arrays.asList("my_field1", "your_field2")));
+    }
+
+    public void testDetectedExtractedFields_GivenLessFieldsThanDocValuesLimit() {
+        FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
+            .addAggregatableField("field_1", "float")
+            .addAggregatableField("field_2", "float")
+            .addAggregatableField("field_3", "float")
+            .addAggregatableField("a_keyword", "keyword")
+            .build();
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildAnalyticsConfig(), true, 4, fieldCapabilities);
+        ExtractedFields extractedFields = extractedFieldsDetector.detect();
+
+        List<String> extractedFieldNames = extractedFields.getAllFields().stream().map(ExtractedField::getName)
+            .collect(Collectors.toList());
+        assertThat(extractedFieldNames, equalTo(Arrays.asList("field_1", "field_2", "field_3")));
+        assertThat(extractedFields.getAllFields().stream().map(ExtractedField::getExtractionMethod).collect(Collectors.toSet()),
+            contains(equalTo(ExtractedField.ExtractionMethod.DOC_VALUE)));
+    }
+
+    public void testDetectedExtractedFields_GivenEqualFieldsToDocValuesLimit() {
+        FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
+            .addAggregatableField("field_1", "float")
+            .addAggregatableField("field_2", "float")
+            .addAggregatableField("field_3", "float")
+            .addAggregatableField("a_keyword", "keyword")
+            .build();
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildAnalyticsConfig(), true, 3, fieldCapabilities);
+        ExtractedFields extractedFields = extractedFieldsDetector.detect();
+
+        List<String> extractedFieldNames = extractedFields.getAllFields().stream().map(ExtractedField::getName)
+            .collect(Collectors.toList());
+        assertThat(extractedFieldNames, equalTo(Arrays.asList("field_1", "field_2", "field_3")));
+        assertThat(extractedFields.getAllFields().stream().map(ExtractedField::getExtractionMethod).collect(Collectors.toSet()),
+            contains(equalTo(ExtractedField.ExtractionMethod.DOC_VALUE)));
+    }
+
+    public void testDetectedExtractedFields_GivenMoreFieldsThanDocValuesLimit() {
+        FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
+            .addAggregatableField("field_1", "float")
+            .addAggregatableField("field_2", "float")
+            .addAggregatableField("field_3", "float")
+            .addAggregatableField("a_keyword", "keyword")
+            .build();
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            SOURCE_INDEX, buildAnalyticsConfig(), true, 2, fieldCapabilities);
+        ExtractedFields extractedFields = extractedFieldsDetector.detect();
+
+        List<String> extractedFieldNames = extractedFields.getAllFields().stream().map(ExtractedField::getName)
+            .collect(Collectors.toList());
+        assertThat(extractedFieldNames, equalTo(Arrays.asList("field_1", "field_2", "field_3")));
+        assertThat(extractedFields.getAllFields().stream().map(ExtractedField::getExtractionMethod).collect(Collectors.toSet()),
+            contains(equalTo(ExtractedField.ExtractionMethod.SOURCE)));
+    }
+
+    private static DataFrameAnalyticsConfig buildAnalyticsConfig() {
+        return buildAnalyticsConfig(null);
+    }
+
+    private static DataFrameAnalyticsConfig buildAnalyticsConfig(FetchSourceContext analyzedFields) {
+        return new DataFrameAnalyticsConfig.Builder("foo")
+            .setSource(new DataFrameAnalyticsSource(SOURCE_INDEX, null))
+            .setDest(new DataFrameAnalyticsDest(DEST_INDEX, null))
+            .setAnalyzedFields(analyzedFields)
+            .setAnalysis(new OutlierDetection())
+            .build();
+    }
+
+    private static class MockFieldCapsResponseBuilder {
+
+        private final Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
+
+        private MockFieldCapsResponseBuilder addAggregatableField(String field, String... types) {
+            Map<String, FieldCapabilities> caps = new HashMap<>();
+            for (String type : types) {
+                caps.put(type, new FieldCapabilities(field, type, true, true));
+            }
+            fieldCaps.put(field, caps);
+            return this;
+        }
+
+        private FieldCapabilitiesResponse build() {
+            FieldCapabilitiesResponse response = mock(FieldCapabilitiesResponse.class);
+            when(response.get()).thenReturn(fieldCaps);
+
+            for (String field : fieldCaps.keySet()) {
+                when(response.getField(field)).thenReturn(fieldCaps.get(field));
+            }
+            return response;
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsControlMessageWriterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsControlMessageWriterTests.java
new file mode 100644
index 0000000000000..5f0cce1770227
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsControlMessageWriterTests.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.ml.process.writer.LengthEncodedWriter;
+import org.junit.Before;
+import org.mockito.InOrder;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.stream.IntStream;
+
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+public class AnalyticsControlMessageWriterTests extends ESTestCase {
+
+    private LengthEncodedWriter lengthEncodedWriter;
+
+    @Before
+    public void setUpMocks() {
+        lengthEncodedWriter = Mockito.mock(LengthEncodedWriter.class);
+    }
+
+    public void testWriteEndOfData() throws IOException {
+        AnalyticsControlMessageWriter writer = new AnalyticsControlMessageWriter(lengthEncodedWriter, 4);
+
+        writer.writeEndOfData();
+
+        InOrder inOrder = inOrder(lengthEncodedWriter);
+        inOrder.verify(lengthEncodedWriter).writeNumFields(4);
+        inOrder.verify(lengthEncodedWriter, times(3)).writeField("");
+        inOrder.verify(lengthEncodedWriter).writeField("$");
+
+        StringBuilder spaces = new StringBuilder();
+        IntStream.rangeClosed(1, 8192).forEach(i -> spaces.append(' '));
+        inOrder.verify(lengthEncodedWriter).writeNumFields(4);
+        inOrder.verify(lengthEncodedWriter, times(3)).writeField("");
+        inOrder.verify(lengthEncodedWriter).writeField(spaces.toString());
+
+        inOrder.verify(lengthEncodedWriter).flush();
+
+        verifyNoMoreInteractions(lengthEncodedWriter);
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
new file mode 100644
index 0000000000000..4032f2d65bf34
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.ml.dataframe.process.results.RowResults;
+import org.junit.Before;
+import org.mockito.InOrder;
+import org.mockito.Mockito;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class AnalyticsResultProcessorTests extends ESTestCase {
+
+    private static final String JOB_ID = "analytics-result-processor-tests";
+
+    private AnalyticsProcess process;
+    private DataFrameRowsJoiner dataFrameRowsJoiner;
+    private int progressPercent;
+
+
+    @Before
+    public void setUpMocks() {
+        process = mock(AnalyticsProcess.class);
+        dataFrameRowsJoiner = mock(DataFrameRowsJoiner.class);
+    }
+
+    public void testProcess_GivenNoResults() {
+        givenProcessResults(Collections.emptyList());
+        AnalyticsResultProcessor resultProcessor = createResultProcessor();
+
+        resultProcessor.process(process);
+        resultProcessor.awaitForCompletion();
+
+        verify(dataFrameRowsJoiner).close();
+        verifyNoMoreInteractions(dataFrameRowsJoiner);
+    }
+
+    public void testProcess_GivenEmptyResults() {
+        givenProcessResults(Arrays.asList(new AnalyticsResult(null, 50), new AnalyticsResult(null, 100)));
+        AnalyticsResultProcessor resultProcessor = createResultProcessor();
+
+        resultProcessor.process(process);
+        resultProcessor.awaitForCompletion();
+
+        verify(dataFrameRowsJoiner).close();
+        Mockito.verifyNoMoreInteractions(dataFrameRowsJoiner);
+        assertThat(progressPercent, equalTo(100));
+    }
+
+    public void testProcess_GivenRowResults() {
+        RowResults rowResults1 = mock(RowResults.class);
+        RowResults rowResults2 = mock(RowResults.class);
+        givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, 50), new AnalyticsResult(rowResults2, 100)));
+        AnalyticsResultProcessor resultProcessor = createResultProcessor();
+
+        resultProcessor.process(process);
+        resultProcessor.awaitForCompletion();
+
+        InOrder inOrder = Mockito.inOrder(dataFrameRowsJoiner);
+        inOrder.verify(dataFrameRowsJoiner).processRowResults(rowResults1);
+        inOrder.verify(dataFrameRowsJoiner).processRowResults(rowResults2);
+
+        assertThat(progressPercent, equalTo(100));
+    }
+
+    private void givenProcessResults(List<AnalyticsResult> results) {
+        when(process.readAnalyticsResults()).thenReturn(results.iterator());
+    }
+
+    private AnalyticsResultProcessor createResultProcessor() {
+        return new AnalyticsResultProcessor(JOB_ID, dataFrameRowsJoiner, () -> false,
+            progressPercent -> this.progressPercent = progressPercent);
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultTests.java
new file mode 100644
index 0000000000000..22c03d47682e8
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultTests.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+import org.elasticsearch.xpack.ml.dataframe.process.results.RowResults;
+import org.elasticsearch.xpack.ml.dataframe.process.results.RowResultsTests;
+
+import java.io.IOException;
+
+public class AnalyticsResultTests extends AbstractXContentTestCase<AnalyticsResult> {
+
+    @Override
+    protected AnalyticsResult createTestInstance() {
+        RowResults rowResults = null;
+        Integer progressPercent = null;
+        if (randomBoolean()) {
+            rowResults = RowResultsTests.createRandom();
+        }
+        if (randomBoolean()) {
+            progressPercent = randomIntBetween(0, 100);
+        }
+        return new AnalyticsResult(rowResults, progressPercent);
+    }
+
+    @Override
+    protected AnalyticsResult doParseInstance(XContentParser parser) throws IOException {
+        return AnalyticsResult.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoinerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoinerTests.java
new file mode 100644
index 0000000000000..fd2b396d62541
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoinerTests.java
@@ -0,0 +1,280 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process;
+
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.bulk.BulkAction;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractor;
+import org.elasticsearch.xpack.ml.dataframe.process.results.RowResults;
+import org.junit.Before;
+import org.mockito.ArgumentCaptor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.IntStream;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.same;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class DataFrameRowsJoinerTests extends ESTestCase {
+
+    private static final String ANALYTICS_ID = "my_analytics";
+
+    private Client client;
+    private DataFrameDataExtractor dataExtractor;
+    private ArgumentCaptor<BulkRequest> bulkRequestCaptor = ArgumentCaptor.forClass(BulkRequest.class);
+
+    @Before
+    public void setUpMocks() {
+        client = mock(Client.class);
+        dataExtractor = mock(DataFrameDataExtractor.class);
+    }
+
+    public void testProcess_GivenNoResults() {
+        givenProcessResults(Collections.emptyList());
+        verifyNoMoreInteractions(client);
+    }
+
+    public void testProcess_GivenSingleRowAndResult() throws IOException {
+        givenClientHasNoFailures();
+
+        String dataDoc = "{\"f_1\": \"foo\", \"f_2\": 42.0}";
+        String[] dataValues = {"42.0"};
+        DataFrameDataExtractor.Row row = newRow(newHit(dataDoc), dataValues, 1);
+        givenDataFrameBatches(Arrays.asList(row));
+
+        Map<String, Object> resultFields = new HashMap<>();
+        resultFields.put("a", "1");
+        resultFields.put("b", "2");
+        RowResults result = new RowResults(1, resultFields);
+        givenProcessResults(Arrays.asList(result));
+
+        List<BulkRequest> capturedBulkRequests = bulkRequestCaptor.getAllValues();
+        assertThat(capturedBulkRequests.size(), equalTo(1));
+        BulkRequest capturedBulkRequest = capturedBulkRequests.get(0);
+        assertThat(capturedBulkRequest.numberOfActions(), equalTo(1));
+        IndexRequest indexRequest = (IndexRequest) capturedBulkRequest.requests().get(0);
+        Map<String, Object> indexedDocSource = indexRequest.sourceAsMap();
+        assertThat(indexedDocSource.size(), equalTo(4));
+        assertThat(indexedDocSource.get("f_1"), equalTo("foo"));
+        assertThat(indexedDocSource.get("f_2"), equalTo(42.0));
+        assertThat(indexedDocSource.get("a"), equalTo("1"));
+        assertThat(indexedDocSource.get("b"), equalTo("2"));
+    }
+
+    public void testProcess_GivenFullResultsBatch() throws IOException {
+        givenClientHasNoFailures();
+
+        String dataDoc = "{\"f_1\": \"foo\", \"f_2\": 42.0}";
+        String[] dataValues = {"42.0"};
+        List<DataFrameDataExtractor.Row> firstBatch = new ArrayList<>(1000);
+        IntStream.range(0, 1000).forEach(i -> firstBatch.add(newRow(newHit(dataDoc), dataValues, i)));
+        List<DataFrameDataExtractor.Row> secondBatch = new ArrayList<>(1);
+        secondBatch.add(newRow(newHit(dataDoc), dataValues, 1000));
+        givenDataFrameBatches(firstBatch, secondBatch);
+
+        Map<String, Object> resultFields = new HashMap<>();
+        resultFields.put("a", "1");
+        resultFields.put("b", "2");
+        List<RowResults> results = new ArrayList<>(1001);
+        IntStream.range(0, 1001).forEach(i -> results.add(new RowResults(i, resultFields)));
+        givenProcessResults(results);
+
+        List<BulkRequest> capturedBulkRequests = bulkRequestCaptor.getAllValues();
+        assertThat(capturedBulkRequests.size(), equalTo(2));
+        assertThat(capturedBulkRequests.get(0).numberOfActions(), equalTo(1000));
+        assertThat(capturedBulkRequests.get(1).numberOfActions(), equalTo(1));
+    }
+
+    public void testProcess_GivenSingleRowAndResultWithMismatchingIdHash() throws IOException {
+        givenClientHasNoFailures();
+
+        String dataDoc = "{\"f_1\": \"foo\", \"f_2\": 42.0}";
+        String[] dataValues = {"42.0"};
+        DataFrameDataExtractor.Row row = newRow(newHit(dataDoc), dataValues, 1);
+        givenDataFrameBatches(Arrays.asList(row));
+
+        Map<String, Object> resultFields = new HashMap<>();
+        resultFields.put("a", "1");
+        resultFields.put("b", "2");
+        RowResults result = new RowResults(2, resultFields);
+        givenProcessResults(Arrays.asList(result));
+
+        verifyNoMoreInteractions(client);
+    }
+
+    public void testProcess_GivenSingleBatchWithSkippedRows() throws IOException {
+        givenClientHasNoFailures();
+
+        DataFrameDataExtractor.Row skippedRow = newRow(newHit("{}"), null, 1);
+        String dataDoc = "{\"f_1\": \"foo\", \"f_2\": 42.0}";
+        String[] dataValues = {"42.0"};
+        DataFrameDataExtractor.Row normalRow = newRow(newHit(dataDoc), dataValues, 2);
+        givenDataFrameBatches(Arrays.asList(skippedRow, normalRow));
+
+        Map<String, Object> resultFields = new HashMap<>();
+        resultFields.put("a", "1");
+        resultFields.put("b", "2");
+        RowResults result = new RowResults(2, resultFields);
+        givenProcessResults(Arrays.asList(result));
+
+        List<BulkRequest> capturedBulkRequests = bulkRequestCaptor.getAllValues();
+        assertThat(capturedBulkRequests.size(), equalTo(1));
+        BulkRequest capturedBulkRequest = capturedBulkRequests.get(0);
+        assertThat(capturedBulkRequest.numberOfActions(), equalTo(1));
+        IndexRequest indexRequest = (IndexRequest) capturedBulkRequest.requests().get(0);
+        Map<String, Object> indexedDocSource = indexRequest.sourceAsMap();
+        assertThat(indexedDocSource.size(), equalTo(4));
+        assertThat(indexedDocSource.get("f_1"), equalTo("foo"));
+        assertThat(indexedDocSource.get("f_2"), equalTo(42.0));
+        assertThat(indexedDocSource.get("a"), equalTo("1"));
+        assertThat(indexedDocSource.get("b"), equalTo("2"));
+    }
+
+    public void testProcess_GivenTwoBatchesWhereFirstEndsWithSkippedRow() throws IOException {
+        givenClientHasNoFailures();
+
+        String dataDoc = "{\"f_1\": \"foo\", \"f_2\": 42.0}";
+        String[] dataValues = {"42.0"};
+        DataFrameDataExtractor.Row normalRow1 = newRow(newHit(dataDoc), dataValues, 1);
+        DataFrameDataExtractor.Row normalRow2 = newRow(newHit(dataDoc), dataValues, 2);
+        DataFrameDataExtractor.Row skippedRow = newRow(newHit("{}"), null, 3);
+        DataFrameDataExtractor.Row normalRow3 = newRow(newHit(dataDoc), dataValues, 4);
+        givenDataFrameBatches(Arrays.asList(normalRow1, normalRow2, skippedRow), Arrays.asList(normalRow3));
+
+        Map<String, Object> resultFields = new HashMap<>();
+        resultFields.put("a", "1");
+        resultFields.put("b", "2");
+        RowResults result1 = new RowResults(1, resultFields);
+        RowResults result2 = new RowResults(2, resultFields);
+        RowResults result3 = new RowResults(4, resultFields);
+        givenProcessResults(Arrays.asList(result1, result2, result3));
+
+        List<BulkRequest> capturedBulkRequests = bulkRequestCaptor.getAllValues();
+        assertThat(capturedBulkRequests.size(), equalTo(1));
+        BulkRequest capturedBulkRequest = capturedBulkRequests.get(0);
+        assertThat(capturedBulkRequest.numberOfActions(), equalTo(3));
+        IndexRequest indexRequest = (IndexRequest) capturedBulkRequest.requests().get(0);
+        Map<String, Object> indexedDocSource = indexRequest.sourceAsMap();
+        assertThat(indexedDocSource.size(), equalTo(4));
+        assertThat(indexedDocSource.get("f_1"), equalTo("foo"));
+        assertThat(indexedDocSource.get("f_2"), equalTo(42.0));
+        assertThat(indexedDocSource.get("a"), equalTo("1"));
+        assertThat(indexedDocSource.get("b"), equalTo("2"));
+    }
+
+    public void testProcess_GivenMoreResultsThanRows() throws IOException {
+        givenClientHasNoFailures();
+
+        String dataDoc = "{\"f_1\": \"foo\", \"f_2\": 42.0}";
+        String[] dataValues = {"42.0"};
+        DataFrameDataExtractor.Row row = newRow(newHit(dataDoc), dataValues, 1);
+        givenDataFrameBatches(Arrays.asList(row));
+
+        Map<String, Object> resultFields = new HashMap<>();
+        resultFields.put("a", "1");
+        resultFields.put("b", "2");
+        RowResults result1 = new RowResults(1, resultFields);
+        RowResults result2 = new RowResults(2, resultFields);
+        givenProcessResults(Arrays.asList(result1, result2));
+
+        verifyNoMoreInteractions(client);
+    }
+
+    public void testProcess_GivenNoResults_ShouldCancelAndConsumeExtractor() throws IOException {
+        givenClientHasNoFailures();
+
+        String dataDoc = "{\"f_1\": \"foo\", \"f_2\": 42.0}";
+        String[] dataValues = {"42.0"};
+        DataFrameDataExtractor.Row row1 = newRow(newHit(dataDoc), dataValues, 1);
+        DataFrameDataExtractor.Row row2 = newRow(newHit(dataDoc), dataValues, 1);
+        givenDataFrameBatches(Arrays.asList(row1), Arrays.asList(row2));
+
+        givenProcessResults(Collections.emptyList());
+
+        verifyNoMoreInteractions(client);
+        verify(dataExtractor).cancel();
+        verify(dataExtractor, times(2)).next();
+    }
+
+    private void givenProcessResults(List<RowResults> results) {
+        try (DataFrameRowsJoiner joiner = new DataFrameRowsJoiner(ANALYTICS_ID, client, dataExtractor)) {
+            results.forEach(joiner::processRowResults);
+        }
+    }
+
+    private void givenDataFrameBatches(List<DataFrameDataExtractor.Row>... batches) throws IOException {
+        DelegateStubDataExtractor delegateStubDataExtractor = new DelegateStubDataExtractor(Arrays.asList(batches));
+        when(dataExtractor.hasNext()).thenAnswer(a -> delegateStubDataExtractor.hasNext());
+        when(dataExtractor.next()).thenAnswer(a -> delegateStubDataExtractor.next());
+    }
+
+    private static SearchHit newHit(String json) {
+        SearchHit hit = new SearchHit(randomInt(), randomAlphaOfLength(10), new Text("doc"), Collections.emptyMap());
+        hit.sourceRef(new BytesArray(json));
+        return hit;
+    }
+
+    private static DataFrameDataExtractor.Row newRow(SearchHit hit, String[] values, int checksum) {
+        DataFrameDataExtractor.Row row = mock(DataFrameDataExtractor.Row.class);
+        when(row.getHit()).thenReturn(hit);
+        when(row.getValues()).thenReturn(values);
+        when(row.getChecksum()).thenReturn(checksum);
+        when(row.shouldSkip()).thenReturn(values == null);
+        return row;
+    }
+
+    private void givenClientHasNoFailures() {
+        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        ThreadPool threadPool = mock(ThreadPool.class);
+        when(threadPool.getThreadContext()).thenReturn(threadContext);
+        ActionFuture<BulkResponse> responseFuture = mock(ActionFuture.class);
+        when(responseFuture.actionGet()).thenReturn(new BulkResponse(new BulkItemResponse[0], 0));
+        when(client.execute(same(BulkAction.INSTANCE), bulkRequestCaptor.capture())).thenReturn(responseFuture);
+        when(client.threadPool()).thenReturn(threadPool);
+    }
+
+    private static class DelegateStubDataExtractor {
+
+        private final List<List<DataFrameDataExtractor.Row>> batches;
+        private int batchIndex;
+
+        private DelegateStubDataExtractor(List<List<DataFrameDataExtractor.Row>> batches) {
+            this.batches = batches;
+        }
+
+        public boolean hasNext() {
+            return batchIndex < batches.size();
+        }
+
+        public Optional<List<DataFrameDataExtractor.Row>> next() {
+            return Optional.of(batches.get(batchIndex++));
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/results/RowResultsTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/results/RowResultsTests.java
new file mode 100644
index 0000000000000..5fdeee90329ae
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/results/RowResultsTests.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.dataframe.process.results;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractXContentTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class RowResultsTests extends AbstractXContentTestCase<RowResults> {
+
+    @Override
+    protected RowResults createTestInstance() {
+        return createRandom();
+    }
+
+    public static RowResults createRandom() {
+        int checksum = randomInt();
+        Map<String, Object> results = new HashMap<>();
+        int resultsSize = randomIntBetween(1, 10);
+        for (int i = 0; i < resultsSize; i++) {
+            String resultField = randomAlphaOfLength(20);
+            Object resultValue = randomBoolean() ? randomAlphaOfLength(20) : randomDouble();
+            results.put(resultField, resultValue);
+        }
+        return new RowResults(checksum, results);
+    }
+
+    @Override
+    protected RowResults doParseInstance(XContentParser parser) {
+        return RowResults.PARSER.apply(parser, null);
+    }
+
+    @Override
+    protected boolean supportsUnknownFields() {
+        return false;
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinderTests.java
index 280a50324e447..4a7c5b87d2186 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/DelimitedFileStructureFinderTests.java
@@ -30,7 +30,7 @@ public void testCreateConfigsGivenCompleteCsv() throws Exception {
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
         FileStructureFinder structureFinder = csvFactory.createFromSample(explanation, sample, charset, hasByteOrderMarker,
-            FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
 
         FileStructure structure = structureFinder.getStructure();
 
@@ -64,8 +64,8 @@ public void testCreateConfigsGivenCompleteCsvAndColumnNamesOverride() throws Exc
 
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
-        FileStructureFinder structureFinder = csvFactory.createFromSample(explanation, sample, charset, hasByteOrderMarker, overrides,
-            NOOP_TIMEOUT_CHECKER);
+        FileStructureFinder structureFinder = csvFactory.createFromSample(explanation, sample, charset, hasByteOrderMarker,
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, overrides, NOOP_TIMEOUT_CHECKER);
 
         FileStructure structure = structureFinder.getStructure();
 
@@ -101,8 +101,8 @@ public void testCreateConfigsGivenCompleteCsvAndHasHeaderRowOverride() throws Ex
 
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
-        FileStructureFinder structureFinder = csvFactory.createFromSample(explanation, sample, charset, hasByteOrderMarker, overrides,
-            NOOP_TIMEOUT_CHECKER);
+        FileStructureFinder structureFinder = csvFactory.createFromSample(explanation, sample, charset, hasByteOrderMarker,
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, overrides, NOOP_TIMEOUT_CHECKER);
 
         FileStructure structure = structureFinder.getStructure();
 
@@ -135,7 +135,7 @@ public void testCreateConfigsGivenCsvWithIncompleteLastRecord() throws Exception
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
         FileStructureFinder structureFinder = csvFactory.createFromSample(explanation, sample, charset, hasByteOrderMarker,
-            FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
 
         FileStructure structure = structureFinder.getStructure();
 
@@ -170,7 +170,7 @@ public void testCreateConfigsGivenCsvWithTrailingNulls() throws Exception {
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
         FileStructureFinder structureFinder = csvFactory.createFromSample(explanation, sample, charset, hasByteOrderMarker,
-            FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
 
         FileStructure structure = structureFinder.getStructure();
 
@@ -214,8 +214,8 @@ public void testCreateConfigsGivenCsvWithTrailingNullsAndOverriddenTimeField() t
 
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
-        FileStructureFinder structureFinder = csvFactory.createFromSample(explanation, sample, charset, hasByteOrderMarker, overrides,
-            NOOP_TIMEOUT_CHECKER);
+        FileStructureFinder structureFinder = csvFactory.createFromSample(explanation, sample, charset, hasByteOrderMarker,
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, overrides, NOOP_TIMEOUT_CHECKER);
 
         FileStructure structure = structureFinder.getStructure();
 
@@ -255,7 +255,7 @@ public void testCreateConfigsGivenCsvWithTrailingNullsExceptHeader() throws Exce
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
         FileStructureFinder structureFinder = csvFactory.createFromSample(explanation, sample, charset, hasByteOrderMarker,
-            FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
 
         FileStructure structure = structureFinder.getStructure();
 
@@ -301,8 +301,8 @@ public void testCreateConfigsGivenCsvWithTrailingNullsExceptHeaderAndColumnNames
 
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
-        FileStructureFinder structureFinder = csvFactory.createFromSample(explanation, sample, charset, hasByteOrderMarker, overrides,
-            NOOP_TIMEOUT_CHECKER);
+        FileStructureFinder structureFinder = csvFactory.createFromSample(explanation, sample, charset, hasByteOrderMarker,
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, overrides, NOOP_TIMEOUT_CHECKER);
 
         FileStructure structure = structureFinder.getStructure();
 
@@ -340,7 +340,7 @@ public void testCreateConfigsGivenCsvWithTimeLastColumn() throws Exception {
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
         FileStructureFinder structureFinder = csvFactory.createFromSample(explanation, sample, charset, hasByteOrderMarker,
-            FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
 
         FileStructure structure = structureFinder.getStructure();
 
@@ -364,6 +364,39 @@ public void testCreateConfigsGivenCsvWithTimeLastColumn() throws Exception {
         assertEquals(Collections.singletonList("YYYY-MM-dd HH:mm:ss.SSSSSS"), structure.getJodaTimestampFormats());
     }
 
+    public void testCreateConfigsGivenDotInFieldName() throws Exception {
+        String sample = "time.iso8601,message\n" +
+            "2018-05-17T13:41:23,hello\n" +
+            "2018-05-17T13:41:32,hello again\n";
+        assertTrue(csvFactory.canCreateFromSample(explanation, sample));
+
+        String charset = randomFrom(POSSIBLE_CHARSETS);
+        Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
+        FileStructureFinder structureFinder = csvFactory.createFromSample(explanation, sample, charset, hasByteOrderMarker,
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
+
+        FileStructure structure = structureFinder.getStructure();
+
+        assertEquals(FileStructure.Format.DELIMITED, structure.getFormat());
+        assertEquals(charset, structure.getCharset());
+        if (hasByteOrderMarker == null) {
+            assertNull(structure.getHasByteOrderMarker());
+        } else {
+            assertEquals(hasByteOrderMarker, structure.getHasByteOrderMarker());
+        }
+        // The exclude pattern needs to work on the raw text, so reflects the unmodified field names
+        assertEquals("^\"?time\\.iso8601\"?,\"?message\"?", structure.getExcludeLinesPattern());
+        assertEquals("^\"?\\d{4}-\\d{2}-\\d{2}[T ]\\d{2}:\\d{2}", structure.getMultilineStartPattern());
+        assertEquals(Character.valueOf(','), structure.getDelimiter());
+        assertEquals(Character.valueOf('"'), structure.getQuote());
+        assertTrue(structure.getHasHeaderRow());
+        assertNull(structure.getShouldTrimFields());
+        assertEquals(Arrays.asList("time_iso8601", "message"), structure.getColumnNames());
+        assertNull(structure.getGrokPattern());
+        assertEquals("time_iso8601", structure.getTimestampField());
+        assertEquals(Collections.singletonList("ISO8601"), structure.getJodaTimestampFormats());
+    }
+
     public void testFindHeaderFromSampleGivenHeaderInSample() throws IOException {
         String withHeader = "time,airline,responsetime,sourcetype\n" +
             "2014-06-23 00:00:00Z,AAL,132.2046,farequote\n" +
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FieldStatsCalculatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FieldStatsCalculatorTests.java
index 30445a4a77c10..4efaf64bd092c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FieldStatsCalculatorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FieldStatsCalculatorTests.java
@@ -10,14 +10,19 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.DoubleSummaryStatistics;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 public class FieldStatsCalculatorTests extends FileStructureTestCase {
 
+    private static final Map<String, String> LONG = Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "long");
+    private static final Map<String, String> DOUBLE = Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "double");
+    private static final Map<String, String> KEYWORD = Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "keyword");
+
     public void testMean() {
 
-        FieldStatsCalculator calculator = new FieldStatsCalculator();
+        FieldStatsCalculator calculator = new FieldStatsCalculator(DOUBLE);
 
         calculator.accept(Arrays.asList("1", "3.5", "2.5", "9"));
 
@@ -26,7 +31,7 @@ public void testMean() {
 
     public void testMedianGivenOddCount() {
 
-        FieldStatsCalculator calculator = new FieldStatsCalculator();
+        FieldStatsCalculator calculator = new FieldStatsCalculator(LONG);
 
         calculator.accept(Arrays.asList("3", "23", "-1", "5", "1000"));
 
@@ -35,7 +40,7 @@ public void testMedianGivenOddCount() {
 
     public void testMedianGivenOddCountMinimal() {
 
-        FieldStatsCalculator calculator = new FieldStatsCalculator();
+        FieldStatsCalculator calculator = new FieldStatsCalculator(LONG);
 
         calculator.accept(Collections.singletonList("3"));
 
@@ -44,7 +49,7 @@ public void testMedianGivenOddCountMinimal() {
 
     public void testMedianGivenEvenCountMiddleValuesDifferent() {
 
-        FieldStatsCalculator calculator = new FieldStatsCalculator();
+        FieldStatsCalculator calculator = new FieldStatsCalculator(LONG);
 
         calculator.accept(Arrays.asList("3", "23", "-1", "5", "1000", "6"));
 
@@ -53,7 +58,7 @@ public void testMedianGivenEvenCountMiddleValuesDifferent() {
 
     public void testMedianGivenEvenCountMiddleValuesSame() {
 
-        FieldStatsCalculator calculator = new FieldStatsCalculator();
+        FieldStatsCalculator calculator = new FieldStatsCalculator(LONG);
 
         calculator.accept(Arrays.asList("3", "23", "-1", "5", "1000", "5"));
 
@@ -62,7 +67,7 @@ public void testMedianGivenEvenCountMiddleValuesSame() {
 
     public void testMedianGivenEvenCountMinimal() {
 
-        FieldStatsCalculator calculator = new FieldStatsCalculator();
+        FieldStatsCalculator calculator = new FieldStatsCalculator(LONG);
 
         calculator.accept(Arrays.asList("4", "4"));
 
@@ -71,7 +76,7 @@ public void testMedianGivenEvenCountMinimal() {
 
     public void testTopHitsNumeric() {
 
-        FieldStatsCalculator calculator = new FieldStatsCalculator();
+        FieldStatsCalculator calculator = new FieldStatsCalculator(DOUBLE);
 
         calculator.accept(Arrays.asList("4", "4", "7", "4", "6", "5.2", "6", "5.2", "16", "4", "5.2"));
 
@@ -88,7 +93,7 @@ public void testTopHitsNumeric() {
 
     public void testTopHitsString() {
 
-        FieldStatsCalculator calculator = new FieldStatsCalculator();
+        FieldStatsCalculator calculator = new FieldStatsCalculator(KEYWORD);
 
         calculator.accept(Arrays.asList("s", "s", "d", "s", "f", "x", "f", "x", "n", "s", "x"));
 
@@ -105,7 +110,8 @@ public void testTopHitsString() {
 
     public void testCalculateGivenEmpty() {
 
-        FieldStatsCalculator calculator = new FieldStatsCalculator();
+        FieldStatsCalculator calculator =
+            new FieldStatsCalculator(randomFrom(Arrays.asList(LONG, DOUBLE, KEYWORD, FileStructureUtils.DATE_MAPPING_WITHOUT_FORMAT)));
 
         calculator.accept(Collections.emptyList());
 
@@ -117,12 +123,15 @@ public void testCalculateGivenEmpty() {
         assertNull(stats.getMaxValue());
         assertNull(stats.getMeanValue());
         assertNull(stats.getMedianValue());
+        assertNull(stats.getEarliestTimestamp());
+        assertNull(stats.getLatestTimestamp());
+
         assertEquals(0, stats.getTopHits().size());
     }
 
     public void testCalculateGivenNumericField() {
 
-        FieldStatsCalculator calculator = new FieldStatsCalculator();
+        FieldStatsCalculator calculator = new FieldStatsCalculator(DOUBLE);
 
         calculator.accept(Arrays.asList("4.5", "4.5", "7", "4.5", "6", "5", "6", "5", "25", "4.5", "5"));
 
@@ -134,6 +143,8 @@ public void testCalculateGivenNumericField() {
         assertEquals(25.0, stats.getMaxValue(), 1e-10);
         assertEquals(7.0, stats.getMeanValue(), 1e-10);
         assertEquals(5.0, stats.getMedianValue(), 1e-10);
+        assertNull(stats.getEarliestTimestamp());
+        assertNull(stats.getLatestTimestamp());
 
         List<Map<String, Object>> topHits = stats.getTopHits();
 
@@ -148,7 +159,7 @@ public void testCalculateGivenNumericField() {
 
     public void testCalculateGivenStringField() {
 
-        FieldStatsCalculator calculator = new FieldStatsCalculator();
+        FieldStatsCalculator calculator = new FieldStatsCalculator(KEYWORD);
 
         calculator.accept(Arrays.asList("s", "s", "d", "s", "f", "x", "f", "x", "n", "s", "x"));
 
@@ -160,6 +171,8 @@ public void testCalculateGivenStringField() {
         assertNull(stats.getMaxValue());
         assertNull(stats.getMeanValue());
         assertNull(stats.getMedianValue());
+        assertNull(stats.getEarliestTimestamp());
+        assertNull(stats.getLatestTimestamp());
 
         List<Map<String, Object>> topHits = stats.getTopHits();
 
@@ -174,7 +187,7 @@ public void testCalculateGivenStringField() {
 
     public void testCalculateGivenMixedField() {
 
-        FieldStatsCalculator calculator = new FieldStatsCalculator();
+        FieldStatsCalculator calculator = new FieldStatsCalculator(KEYWORD);
 
         calculator.accept(Arrays.asList("4", "4", "d", "4", "f", "x", "f", "x", "16", "4", "x"));
 
@@ -186,6 +199,8 @@ public void testCalculateGivenMixedField() {
         assertNull(stats.getMaxValue());
         assertNull(stats.getMeanValue());
         assertNull(stats.getMedianValue());
+        assertNull(stats.getEarliestTimestamp());
+        assertNull(stats.getLatestTimestamp());
 
         List<Map<String, Object>> topHits = stats.getTopHits();
 
@@ -198,10 +213,71 @@ public void testCalculateGivenMixedField() {
         assertEquals(2, topHits.get(2).get("count"));
     }
 
+    public void testGivenDateFieldWithoutFormat() {
+
+        FieldStatsCalculator calculator = new FieldStatsCalculator(FileStructureUtils.DATE_MAPPING_WITHOUT_FORMAT);
+
+        calculator.accept(Arrays.asList("2018-10-08T10:49:16.642", "2018-10-08T10:49:16.642", "2018-10-08T10:49:16.642",
+            "2018-09-08T11:12:13.789", "2019-01-28T01:02:03.456", "2018-09-08T11:12:13.789"));
+
+        FieldStats stats = calculator.calculate(3);
+
+        assertEquals(6L, stats.getCount());
+        assertEquals(3, stats.getCardinality());
+        assertNull(stats.getMinValue());
+        assertNull(stats.getMaxValue());
+        assertNull(stats.getMeanValue());
+        assertNull(stats.getMedianValue());
+        assertEquals("2018-09-08T11:12:13.789", stats.getEarliestTimestamp());
+        assertEquals("2019-01-28T01:02:03.456", stats.getLatestTimestamp());
+
+        List<Map<String, Object>> topHits = stats.getTopHits();
+
+        assertEquals(3, topHits.size());
+        assertEquals("2018-10-08T10:49:16.642", topHits.get(0).get("value"));
+        assertEquals(3, topHits.get(0).get("count"));
+        assertEquals("2018-09-08T11:12:13.789", topHits.get(1).get("value"));
+        assertEquals(2, topHits.get(1).get("count"));
+        assertEquals("2019-01-28T01:02:03.456", topHits.get(2).get("value"));
+        assertEquals(1, topHits.get(2).get("count"));
+    }
+
+    public void testGivenDateFieldWithFormat() {
+
+        Map<String, String> dateMapping = new HashMap<>();
+        dateMapping.put(FileStructureUtils.MAPPING_TYPE_SETTING, "date");
+        dateMapping.put(FileStructureUtils.MAPPING_FORMAT_SETTING, "M/dd/yyyy h:mma");
+        FieldStatsCalculator calculator = new FieldStatsCalculator(dateMapping);
+
+        calculator.accept(Arrays.asList("10/08/2018 10:49AM", "10/08/2018 10:49AM", "10/08/2018 10:49AM",
+            "9/08/2018 11:12AM", "1/28/2019 1:02AM", "9/08/2018 11:12AM"));
+
+        FieldStats stats = calculator.calculate(3);
+
+        assertEquals(6L, stats.getCount());
+        assertEquals(3, stats.getCardinality());
+        assertNull(stats.getMinValue());
+        assertNull(stats.getMaxValue());
+        assertNull(stats.getMeanValue());
+        assertNull(stats.getMedianValue());
+        assertEquals("9/08/2018 11:12AM", stats.getEarliestTimestamp());
+        assertEquals("1/28/2019 1:02AM", stats.getLatestTimestamp());
+
+        List<Map<String, Object>> topHits = stats.getTopHits();
+
+        assertEquals(3, topHits.size());
+        assertEquals("10/08/2018 10:49AM", topHits.get(0).get("value"));
+        assertEquals(3, topHits.get(0).get("count"));
+        assertEquals("9/08/2018 11:12AM", topHits.get(1).get("value"));
+        assertEquals(2, topHits.get(1).get("count"));
+        assertEquals("1/28/2019 1:02AM", topHits.get(2).get("value"));
+        assertEquals(1, topHits.get(2).get("count"));
+    }
+
     public void testJavaStatsEquivalence() {
 
         DoubleSummaryStatistics summaryStatistics = new DoubleSummaryStatistics();
-        FieldStatsCalculator calculator = new FieldStatsCalculator();
+        FieldStatsCalculator calculator = new FieldStatsCalculator(DOUBLE);
 
         for (int numValues = randomIntBetween(1000, 10000); numValues > 0; --numValues) {
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureFinderManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureFinderManagerTests.java
index 978f1c5286de8..188bc9a628bd8 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureFinderManagerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureFinderManagerTests.java
@@ -14,6 +14,7 @@
 import org.junit.Before;
 
 import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PipedInputStream;
@@ -25,6 +26,7 @@
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.xpack.ml.filestructurefinder.FileStructureOverrides.EMPTY_OVERRIDES;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.startsWith;
 import static org.hamcrest.core.IsInstanceOf.instanceOf;
@@ -54,7 +56,7 @@ public void testFindCharsetGivenCharacterWidths() throws Exception {
         }
     }
 
-    public void testFindCharsetGivenBinary() throws Exception {
+    public void testFindCharsetGivenRandomBinary() throws Exception {
 
         // This input should never match a single byte character set.  ICU4J will sometimes decide
         // that it matches a double byte character set, hence the two assertion branches.
@@ -73,9 +75,35 @@ public void testFindCharsetGivenBinary() throws Exception {
         }
     }
 
+    public void testFindCharsetGivenBinaryNearUtf16() throws Exception {
+
+        // This input should never match a single byte character set.  ICU4J will probably decide
+        // that it matches both UTF-16BE and UTF-16LE, but we should reject these as there's no
+        // clear winner.
+        ByteArrayOutputStream stream = new ByteArrayOutputStream();
+        if (randomBoolean()) {
+            stream.write(randomAlphaOfLengthBetween(3, 4).getBytes(StandardCharsets.UTF_16LE));
+        }
+        for (int i = 0; i < 50; ++i) {
+            stream.write(randomAlphaOfLengthBetween(5, 6).getBytes(StandardCharsets.UTF_16BE));
+            stream.write(randomAlphaOfLengthBetween(5, 6).getBytes(StandardCharsets.UTF_16LE));
+        }
+        if (randomBoolean()) {
+            stream.write(randomAlphaOfLengthBetween(3, 4).getBytes(StandardCharsets.UTF_16BE));
+        }
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> structureFinderManager.findCharset(explanation, new ByteArrayInputStream(stream.toByteArray()), NOOP_TIMEOUT_CHECKER));
+
+        assertEquals("Could not determine a usable character encoding for the input - could it be binary data?", e.getMessage());
+        assertThat(explanation.toString(),
+            containsString("but was rejected as the distribution of zero bytes between odd and even positions in the file is very close"));
+    }
+
     public void testMakeBestStructureGivenNdJson() throws Exception {
         assertThat(structureFinderManager.makeBestStructureFinder(explanation, NDJSON_SAMPLE, StandardCharsets.UTF_8.name(),
-            randomBoolean(), EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER), instanceOf(NdJsonFileStructureFinder.class));
+            randomBoolean(), FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER),
+            instanceOf(NdJsonFileStructureFinder.class));
     }
 
     public void testMakeBestStructureGivenNdJsonAndDelimitedOverride() throws Exception {
@@ -86,12 +114,14 @@ public void testMakeBestStructureGivenNdJsonAndDelimitedOverride() throws Except
             .setFormat(FileStructure.Format.DELIMITED).setQuote('\'').build();
 
         assertThat(structureFinderManager.makeBestStructureFinder(explanation, NDJSON_SAMPLE, StandardCharsets.UTF_8.name(),
-            randomBoolean(), overrides, NOOP_TIMEOUT_CHECKER), instanceOf(DelimitedFileStructureFinder.class));
+            randomBoolean(), FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, overrides, NOOP_TIMEOUT_CHECKER),
+            instanceOf(DelimitedFileStructureFinder.class));
     }
 
     public void testMakeBestStructureGivenXml() throws Exception {
         assertThat(structureFinderManager.makeBestStructureFinder(explanation, XML_SAMPLE, StandardCharsets.UTF_8.name(), randomBoolean(),
-            EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER), instanceOf(XmlFileStructureFinder.class));
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER),
+            instanceOf(XmlFileStructureFinder.class));
     }
 
     public void testMakeBestStructureGivenXmlAndTextOverride() throws Exception {
@@ -99,12 +129,14 @@ public void testMakeBestStructureGivenXmlAndTextOverride() throws Exception {
         FileStructureOverrides overrides = FileStructureOverrides.builder().setFormat(FileStructure.Format.SEMI_STRUCTURED_TEXT).build();
 
         assertThat(structureFinderManager.makeBestStructureFinder(explanation, XML_SAMPLE, StandardCharsets.UTF_8.name(), randomBoolean(),
-            overrides, NOOP_TIMEOUT_CHECKER), instanceOf(TextLogFileStructureFinder.class));
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, overrides, NOOP_TIMEOUT_CHECKER),
+            instanceOf(TextLogFileStructureFinder.class));
     }
 
     public void testMakeBestStructureGivenCsv() throws Exception {
         assertThat(structureFinderManager.makeBestStructureFinder(explanation, CSV_SAMPLE, StandardCharsets.UTF_8.name(), randomBoolean(),
-            EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER), instanceOf(DelimitedFileStructureFinder.class));
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER),
+            instanceOf(DelimitedFileStructureFinder.class));
     }
 
     public void testMakeBestStructureGivenCsvAndJsonOverride() {
@@ -113,14 +145,15 @@ public void testMakeBestStructureGivenCsvAndJsonOverride() {
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
             () -> structureFinderManager.makeBestStructureFinder(explanation, CSV_SAMPLE, StandardCharsets.UTF_8.name(), randomBoolean(),
-                overrides, NOOP_TIMEOUT_CHECKER));
+                FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, overrides, NOOP_TIMEOUT_CHECKER));
 
         assertEquals("Input did not match the specified format [ndjson]", e.getMessage());
     }
 
     public void testMakeBestStructureGivenText() throws Exception {
         assertThat(structureFinderManager.makeBestStructureFinder(explanation, TEXT_SAMPLE, StandardCharsets.UTF_8.name(), randomBoolean(),
-            EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER), instanceOf(TextLogFileStructureFinder.class));
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER),
+            instanceOf(TextLogFileStructureFinder.class));
     }
 
     public void testMakeBestStructureGivenTextAndDelimitedOverride() throws Exception {
@@ -130,7 +163,8 @@ public void testMakeBestStructureGivenTextAndDelimitedOverride() throws Exceptio
             .setFormat(FileStructure.Format.DELIMITED).setDelimiter(':').build();
 
         assertThat(structureFinderManager.makeBestStructureFinder(explanation, TEXT_SAMPLE, StandardCharsets.UTF_8.name(), randomBoolean(),
-            overrides, NOOP_TIMEOUT_CHECKER), instanceOf(DelimitedFileStructureFinder.class));
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, overrides, NOOP_TIMEOUT_CHECKER),
+            instanceOf(DelimitedFileStructureFinder.class));
     }
 
     public void testFindFileStructureTimeout() throws IOException, InterruptedException {
@@ -163,7 +197,8 @@ public void testFindFileStructureTimeout() throws IOException, InterruptedExcept
                 junkProducer.start();
 
                 ElasticsearchTimeoutException e = expectThrows(ElasticsearchTimeoutException.class,
-                    () -> structureFinderManager.findFileStructure(explanation, linesOfJunk - 1, bigInput, EMPTY_OVERRIDES, timeout));
+                    () -> structureFinderManager.findFileStructure(explanation, FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT,
+                        linesOfJunk - 1, bigInput, EMPTY_OVERRIDES, timeout));
 
                 assertThat(e.getMessage(), startsWith("Aborting structure analysis during ["));
                 assertThat(e.getMessage(), endsWith("] as it has taken longer than the timeout of [" + timeout + "]"));
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtilsTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtilsTests.java
index c0adccd0eb477..91568573c9d71 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtilsTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtilsTests.java
@@ -323,8 +323,7 @@ public void testGuessMappingsAndCalculateFieldStats() {
         sample2.put("nothing", null);
 
         Tuple<SortedMap<String, Object>, SortedMap<String, FieldStats>> mappingsAndFieldStats =
-            FileStructureUtils.guessMappingsAndCalculateFieldStats(explanation, Arrays.asList(sample1, sample2),
-                NOOP_TIMEOUT_CHECKER);
+            FileStructureUtils.guessMappingsAndCalculateFieldStats(explanation, Arrays.asList(sample1, sample2), NOOP_TIMEOUT_CHECKER);
         assertNotNull(mappingsAndFieldStats);
 
         Map<String, Object> mappings = mappingsAndFieldStats.v1();
@@ -341,7 +340,8 @@ public void testGuessMappingsAndCalculateFieldStats() {
         assertNotNull(fieldStats);
         assertEquals(3, fieldStats.size());
         assertEquals(new FieldStats(2, 2, makeTopHits("not a time", 1, "whatever", 1)), fieldStats.get("foo"));
-        assertEquals(new FieldStats(2, 2, makeTopHits("2018-05-24 17:28:31,735", 1, "2018-05-29 11:53:02,837", 1)), fieldStats.get("time"));
+        assertEquals(new FieldStats(2, 2, "2018-05-24 17:28:31,735", "2018-05-29 11:53:02,837",
+            makeTopHits("2018-05-24 17:28:31,735", 1, "2018-05-29 11:53:02,837", 1)), fieldStats.get("time"));
         assertEquals(new FieldStats(2, 2, 17.0, 42.0, 29.5, 29.5, makeTopHits(17, 1, 42, 1)), fieldStats.get("bar"));
         assertNull(fieldStats.get("nothing"));
     }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/GrokPatternCreatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/GrokPatternCreatorTests.java
index 7e6363602dcdd..967c6d42921e4 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/GrokPatternCreatorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/GrokPatternCreatorTests.java
@@ -203,7 +203,8 @@ public void testCreateGrokPatternFromExamplesGivenNamedLogs() {
 
         assertEquals("%{SYSLOGTIMESTAMP:timestamp} .*? .*?\\[%{INT:field}\\]: %{LOGLEVEL:loglevel} \\(.*? .*? .*?\\) .*? " +
                 "%{QUOTEDSTRING:field2}: %{IP:ipaddress}#%{INT:field3}",
-            grokPatternCreator.createGrokPatternFromExamples("SYSLOGTIMESTAMP", "timestamp"));
+            grokPatternCreator.createGrokPatternFromExamples("SYSLOGTIMESTAMP", FileStructureUtils.DATE_MAPPING_WITHOUT_FORMAT,
+                "timestamp"));
         assertEquals(5, mappings.size());
         assertEquals(Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "long"), mappings.get("field"));
         assertEquals(Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "keyword"), mappings.get("loglevel"));
@@ -229,7 +230,8 @@ public void testCreateGrokPatternFromExamplesGivenCatalinaLogs() {
             NOOP_TIMEOUT_CHECKER);
 
         assertEquals("%{CATALINA_DATESTAMP:timestamp} .*? .*?\\n%{LOGLEVEL:loglevel}: .*",
-            grokPatternCreator.createGrokPatternFromExamples("CATALINA_DATESTAMP", "timestamp"));
+            grokPatternCreator.createGrokPatternFromExamples("CATALINA_DATESTAMP", FileStructureUtils.DATE_MAPPING_WITHOUT_FORMAT,
+                "timestamp"));
         assertEquals(1, mappings.size());
         assertEquals(Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "keyword"), mappings.get("loglevel"));
     }
@@ -253,7 +255,8 @@ public void testCreateGrokPatternFromExamplesGivenMultiTimestampLogs() {
 
         assertEquals("%{INT:field}\\t%{TIMESTAMP_ISO8601:timestamp}\\t%{TIMESTAMP_ISO8601:extra_timestamp}\\t%{INT:field2}\\t.*?\\t" +
                 "%{IP:ipaddress}\\t.*?\\t%{LOGLEVEL:loglevel}\\t.*",
-            grokPatternCreator.createGrokPatternFromExamples("TIMESTAMP_ISO8601", "timestamp"));
+            grokPatternCreator.createGrokPatternFromExamples("TIMESTAMP_ISO8601", FileStructureUtils.DATE_MAPPING_WITHOUT_FORMAT,
+                "timestamp"));
         assertEquals(5, mappings.size());
         assertEquals(Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "long"), mappings.get("field"));
         Map<String, String> expectedDateMapping = new HashMap<>();
@@ -284,7 +287,8 @@ public void testCreateGrokPatternFromExamplesGivenMultiTimestampLogsAndIndetermi
 
         assertEquals("%{INT:field}\\t%{TIMESTAMP_ISO8601:timestamp}\\t%{DATESTAMP:extra_timestamp}\\t%{INT:field2}\\t.*?\\t" +
                 "%{IP:ipaddress}\\t.*?\\t%{LOGLEVEL:loglevel}\\t.*",
-            grokPatternCreator.createGrokPatternFromExamples("TIMESTAMP_ISO8601", "timestamp"));
+            grokPatternCreator.createGrokPatternFromExamples("TIMESTAMP_ISO8601", FileStructureUtils.DATE_MAPPING_WITHOUT_FORMAT,
+                "timestamp"));
         assertEquals(5, mappings.size());
         assertEquals(Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "long"), mappings.get("field"));
         Map<String, String> expectedDateMapping = new HashMap<>();
@@ -314,9 +318,12 @@ public void testCreateGrokPatternFromExamplesGivenMultiTimestampLogsAndCustomDef
             Collections.singletonMap("CUSTOM_TIMESTAMP", "%{MONTHNUM}/%{MONTHDAY}/%{YEAR} %{HOUR}:%{MINUTE}(?:AM|PM)"),
             NOOP_TIMEOUT_CHECKER);
 
+        Map<String, String> customMapping = new HashMap<>();
+        customMapping.put(FileStructureUtils.MAPPING_TYPE_SETTING, "date");
+        customMapping.put(FileStructureUtils.MAPPING_FORMAT_SETTING, "M/dd/yyyy h:mma");
         assertEquals("%{INT:field}\\t%{CUSTOM_TIMESTAMP:timestamp}\\t%{TIMESTAMP_ISO8601:extra_timestamp}\\t%{INT:field2}\\t.*?\\t" +
                 "%{IP:ipaddress}\\t.*?\\t%{LOGLEVEL:loglevel}\\t.*",
-            grokPatternCreator.createGrokPatternFromExamples("CUSTOM_TIMESTAMP", "timestamp"));
+            grokPatternCreator.createGrokPatternFromExamples("CUSTOM_TIMESTAMP", customMapping, "timestamp"));
         assertEquals(5, mappings.size());
         assertEquals(Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "long"), mappings.get("field"));
         Map<String, String> expectedDateMapping = new HashMap<>();
@@ -347,7 +354,8 @@ public void testCreateGrokPatternFromExamplesGivenTimestampAndTimeWithoutDate()
 
         assertEquals("%{INT:field}\\t%{TIMESTAMP_ISO8601:timestamp}\\t%{TIME:time}\\t%{INT:field2}\\t.*?\\t" +
                 "%{IP:ipaddress}\\t.*?\\t%{LOGLEVEL:loglevel}\\t.*",
-            grokPatternCreator.createGrokPatternFromExamples("TIMESTAMP_ISO8601", "timestamp"));
+            grokPatternCreator.createGrokPatternFromExamples("TIMESTAMP_ISO8601", FileStructureUtils.DATE_MAPPING_WITHOUT_FORMAT,
+                "timestamp"));
         assertEquals(5, mappings.size());
         assertEquals(Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "long"), mappings.get("field"));
         assertEquals(Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "keyword"), mappings.get("time"));
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/NdJsonFileStructureFinderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/NdJsonFileStructureFinderTests.java
index a220bdf3b0690..048d2708e7740 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/NdJsonFileStructureFinderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/NdJsonFileStructureFinderTests.java
@@ -19,7 +19,7 @@ public void testCreateConfigsGivenGoodJson() throws Exception {
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
         FileStructureFinder structureFinder = factory.createFromSample(explanation, NDJSON_SAMPLE, charset, hasByteOrderMarker,
-            FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
 
         FileStructure structure = structureFinder.getStructure();
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinderTests.java
index 6ac672f61780e..4c921c8a9f9ba 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/TextLogFileStructureFinderTests.java
@@ -20,13 +20,36 @@ public class TextLogFileStructureFinderTests extends FileStructureTestCase {
 
     private FileStructureFinderFactory factory = new TextLogFileStructureFinderFactory();
 
+    public void testCreateConfigsGivenLowLineMergeSizeLimit() {
+
+        String sample = "2019-05-16 16:56:14 line 1 abcdefghijklmnopqrstuvwxyz\n" +
+            "2019-05-16 16:56:14 line 2 abcdefghijklmnopqrstuvwxyz\n" +
+            "continuation line 2.1\n" +
+            "continuation line 2.2\n" +
+            "continuation line 2.3\n" +
+            "continuation line 2.4\n" +
+            "2019-05-16 16:56:14 line 3 abcdefghijklmnopqrstuvwxyz\n";
+
+        assertTrue(factory.canCreateFromSample(explanation, sample));
+
+        String charset = randomFrom(POSSIBLE_CHARSETS);
+        Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> factory.createFromSample(explanation, sample, charset, hasByteOrderMarker, 100,
+                FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER));
+
+        assertEquals("Merging lines into messages resulted in an unacceptably long message. Merged message would have [4] lines and "
+            + "[119] characters (limit [100]). If you have messages this big please increase the value of [line_merge_size_limit]. "
+            + "Otherwise it probably means the timestamp has been incorrectly detected, so try overriding that.", e.getMessage());
+    }
+
     public void testCreateConfigsGivenElasticsearchLog() throws Exception {
         assertTrue(factory.canCreateFromSample(explanation, TEXT_SAMPLE));
 
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
         FileStructureFinder structureFinder = factory.createFromSample(explanation, TEXT_SAMPLE, charset, hasByteOrderMarker,
-            FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
 
         FileStructure structure = structureFinder.getStructure();
 
@@ -66,8 +89,8 @@ public void testCreateConfigsGivenElasticsearchLogAndTimestampFormatOverride() t
 
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
-        FileStructureFinder structureFinder = factory.createFromSample(explanation, sample, charset, hasByteOrderMarker, overrides,
-            NOOP_TIMEOUT_CHECKER);
+        FileStructureFinder structureFinder = factory.createFromSample(explanation, sample, charset, hasByteOrderMarker,
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, overrides, NOOP_TIMEOUT_CHECKER);
 
         FileStructure structure = structureFinder.getStructure();
 
@@ -102,8 +125,8 @@ public void testCreateConfigsGivenElasticsearchLogAndTimestampFieldOverride() th
 
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
-        FileStructureFinder structureFinder = factory.createFromSample(explanation, TEXT_SAMPLE, charset, hasByteOrderMarker, overrides,
-            NOOP_TIMEOUT_CHECKER);
+        FileStructureFinder structureFinder = factory.createFromSample(explanation, TEXT_SAMPLE, charset, hasByteOrderMarker,
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, overrides, NOOP_TIMEOUT_CHECKER);
 
         FileStructure structure = structureFinder.getStructure();
 
@@ -139,8 +162,8 @@ public void testCreateConfigsGivenElasticsearchLogAndGrokPatternOverride() throw
 
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
-        FileStructureFinder structureFinder = factory.createFromSample(explanation, TEXT_SAMPLE, charset, hasByteOrderMarker, overrides,
-            NOOP_TIMEOUT_CHECKER);
+        FileStructureFinder structureFinder = factory.createFromSample(explanation, TEXT_SAMPLE, charset, hasByteOrderMarker,
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, overrides, NOOP_TIMEOUT_CHECKER);
 
         FileStructure structure = structureFinder.getStructure();
 
@@ -181,7 +204,8 @@ public void testCreateConfigsGivenElasticsearchLogAndImpossibleGrokPatternOverri
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> factory.createFromSample(explanation, TEXT_SAMPLE, charset, hasByteOrderMarker, overrides, NOOP_TIMEOUT_CHECKER));
+            () -> factory.createFromSample(explanation, TEXT_SAMPLE, charset, hasByteOrderMarker,
+                FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, overrides, NOOP_TIMEOUT_CHECKER));
 
         assertEquals("Supplied Grok pattern [\\[%{LOGLEVEL:loglevel} *\\]\\[%{HOSTNAME:node}\\]\\[%{TIMESTAMP_ISO8601:timestamp}\\] " +
             "\\[%{JAVACLASS:class} *\\] %{JAVALOGMESSAGE:message}] does not match sample messages", e.getMessage());
@@ -200,8 +224,8 @@ public void testErrorOnIncorrectMessageFormation() {
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> factory.createFromSample(explanation, sample, charset, hasByteOrderMarker, FileStructureOverrides.EMPTY_OVERRIDES,
-                NOOP_TIMEOUT_CHECKER));
+            () -> factory.createFromSample(explanation, sample, charset, hasByteOrderMarker,
+                FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER));
 
         assertEquals("Failed to create more than one message from the sample lines provided. (The last is discarded in "
             + "case the sample is incomplete.) If your sample does contain multiple messages the problem is probably that "
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinderTests.java
index b80e8a5712aaa..0b872e630b6ab 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinderTests.java
@@ -136,11 +136,11 @@ public void testInvalidOverrideFormatToGrokAndRegex() {
         e = expectThrows(IllegalArgumentException.class,
             () -> TimestampFormatFinder.overrideFormatToGrokAndRegex("MM/dd/yyyy H:mm:ss+SSSSSS"));
         assertEquals("Letter group [SSSSSS] in [MM/dd/yyyy H:mm:ss+SSSSSS] is not supported"
-            + " because it is not preceeded by [ss] and a separator from [:.,]", e.getMessage());
+            + " because it is not preceded by [ss] and a separator from [:.,]", e.getMessage());
         e = expectThrows(IllegalArgumentException.class,
             () -> TimestampFormatFinder.overrideFormatToGrokAndRegex("MM/dd/yyyy H:mm,SSSSSS"));
         assertEquals("Letter group [SSSSSS] in [MM/dd/yyyy H:mm,SSSSSS] is not supported"
-            + " because it is not preceeded by [ss] and a separator from [:.,]", e.getMessage());
+            + " because it is not preceded by [ss] and a separator from [:.,]", e.getMessage());
         e = expectThrows(IllegalArgumentException.class,
             () -> TimestampFormatFinder.overrideFormatToGrokAndRegex(" 'T' "));
         assertEquals("No time format letter groups in override format [ 'T' ]", e.getMessage());
@@ -562,6 +562,11 @@ public void testFindFormatGivenNoMatch() {
         validateNoTimestampMatch("no timestamps in here");
         validateNoTimestampMatch(":::");
         validateNoTimestampMatch("/+");
+        // These two don't match because they're too far in the future
+        // when interpreted as seconds/milliseconds from the epoch
+        // (they need to be 10 or 13 digits beginning with 1 or 2)
+        validateNoTimestampMatch("3213213213");
+        validateNoTimestampMatch("9789522792167");
     }
 
     public void testFindFormatGivenOnlyIso8601() {
@@ -693,10 +698,14 @@ public void testFindFormatGivenOnlyKnownTimestampFormat() {
 
     public void testFindFormatGivenOnlySystemDate() {
 
+        validateTimestampMatch("1000000000000", "POSINT", "\\b\\d{13}\\b", "UNIX_MS", 1000000000000L);
         validateTimestampMatch("1526400896374", "POSINT", "\\b\\d{13}\\b", "UNIX_MS", 1526400896374L);
+        validateTimestampMatch("2999999999999", "POSINT", "\\b\\d{13}\\b", "UNIX_MS", 2999999999999L);
 
+        validateTimestampMatch("1000000000", "NUMBER", "\\b\\d{10}\\b", "UNIX", 1000000000000L);
         validateTimestampMatch("1526400896.736", "NUMBER", "\\b\\d{10}\\b", "UNIX", 1526400896736L);
         validateTimestampMatch("1526400896", "NUMBER", "\\b\\d{10}\\b", "UNIX", 1526400896000L);
+        validateTimestampMatch("2999999999.999", "NUMBER", "\\b\\d{10}\\b", "UNIX", 2999999999999L);
 
         validateTimestampMatch("400000005afb078a164ac980", "BASE16NUM", "\\b[0-9A-Fa-f]{24}\\b", "TAI64N", 1526400896374L);
     }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinderTests.java
index b6f93a6e39b1d..9ad07f6142782 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/XmlFileStructureFinderTests.java
@@ -19,7 +19,7 @@ public void testCreateConfigsGivenGoodXml() throws Exception {
         String charset = randomFrom(POSSIBLE_CHARSETS);
         Boolean hasByteOrderMarker = randomHasByteOrderMarker(charset);
         FileStructureFinder structureFinder = factory.createFromSample(explanation, XML_SAMPLE, charset, hasByteOrderMarker,
-            FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
+            FileStructureFinderManager.DEFAULT_LINE_MERGE_SIZE_LIMIT, FileStructureOverrides.EMPTY_OVERRIDES, NOOP_TIMEOUT_CHECKER);
 
         FileStructure structure = structureFinder.getStructure();
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
index b19266956cb8d..d99268e95a3e2 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex;
 import org.elasticsearch.xpack.ml.LocalStateMachineLearning;
@@ -66,7 +67,7 @@ public void testCreatedWhenAfterOtherMlIndex() throws Exception {
     }
 
     private boolean annotationsIndexExists() {
-        return client().admin().indices().prepareExists(AnnotationIndex.INDEX_NAME).get().isExists();
+        return ESIntegTestCase.indexExists(AnnotationIndex.INDEX_NAME, client());
     }
 
     private int numberOfAnnotationsAliases() {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
index b778e4d06fe8e..976f414f11fb8 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord;
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
 import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition;
@@ -38,7 +39,7 @@
 import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.elasticsearch.xpack.ml.job.persistence.RecordsQueryBuilder;
 import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess;
-import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutoDetectResultProcessor;
+import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutodetectResultProcessor;
 import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer;
 import org.elasticsearch.xpack.ml.job.results.AutodetectResult;
 import org.elasticsearch.xpack.ml.job.results.BucketTests;
@@ -49,7 +50,6 @@
 import org.junit.Before;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
@@ -62,6 +62,8 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReference;
 
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
@@ -72,8 +74,9 @@ public class AutodetectResultProcessorIT extends MlSingleNodeTestCase {
 
     private JobResultsProvider jobResultsProvider;
     private List<ModelSnapshot> capturedUpdateModelSnapshotOnJobRequests;
-    private AutoDetectResultProcessor resultProcessor;
+    private AutodetectResultProcessor resultProcessor;
     private Renormalizer renormalizer;
+    private AutodetectProcess process;
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
@@ -87,9 +90,17 @@ public void createComponents() throws Exception {
         Auditor auditor = new Auditor(client(), "test_node");
         jobResultsProvider = new JobResultsProvider(client(), builder.build());
         renormalizer = mock(Renormalizer.class);
+        process = mock(AutodetectProcess.class);
         capturedUpdateModelSnapshotOnJobRequests = new ArrayList<>();
-        resultProcessor = new AutoDetectResultProcessor(client(), auditor, JOB_ID, renormalizer,
-                new JobResultsPersister(client()), new ModelSizeStats.Builder(JOB_ID).build()) {
+        resultProcessor = new AutodetectResultProcessor(
+                client(),
+                auditor,
+                JOB_ID,
+                renormalizer,
+                new JobResultsPersister(client()),
+                process,
+                new ModelSizeStats.Builder(JOB_ID).build(),
+                new TimingStats(JOB_ID)) {
             @Override
             protected void updateModelSnapshotOnJob(ModelSnapshot modelSnapshot) {
                 capturedUpdateModelSnapshotOnJobRequests.add(modelSnapshot);
@@ -107,25 +118,26 @@ public void deleteJob() {
     }
 
     public void testProcessResults() throws Exception {
-        ResultsBuilder builder = new ResultsBuilder();
+        ResultsBuilder resultsBuilder = new ResultsBuilder();
         Bucket bucket = createBucket(false);
-        builder.addBucket(bucket);
+        resultsBuilder.addBucket(bucket);
         List<AnomalyRecord> records = createRecords(false);
-        builder.addRecords(records);
+        resultsBuilder.addRecords(records);
         List<Influencer> influencers = createInfluencers(false);
-        builder.addInfluencers(influencers);
+        resultsBuilder.addInfluencers(influencers);
         CategoryDefinition categoryDefinition = createCategoryDefinition();
-        builder.addCategoryDefinition(categoryDefinition);
-        ModelPlot modelPlot = createmodelPlot();
-        builder.addmodelPlot(modelPlot);
+        resultsBuilder.addCategoryDefinition(categoryDefinition);
+        ModelPlot modelPlot = createModelPlot();
+        resultsBuilder.addModelPlot(modelPlot);
         ModelSizeStats modelSizeStats = createModelSizeStats();
-        builder.addModelSizeStats(modelSizeStats);
+        resultsBuilder.addModelSizeStats(modelSizeStats);
         ModelSnapshot modelSnapshot = createModelSnapshot();
-        builder.addModelSnapshot(modelSnapshot);
+        resultsBuilder.addModelSnapshot(modelSnapshot);
         Quantiles quantiles = createQuantiles();
-        builder.addQuantiles(quantiles);
+        resultsBuilder.addQuantiles(quantiles);
+        when(process.readAutodetectResults()).thenReturn(resultsBuilder.build().iterator());
 
-        resultProcessor.process(builder.buildTestProcess());
+        resultProcessor.process();
         resultProcessor.awaitCompletion();
 
         BucketsQueryBuilder bucketsQuery = new BucketsQueryBuilder().includeInterim(true);
@@ -163,14 +175,41 @@ public void testProcessResults() throws Exception {
         assertEquals(quantiles, persistedQuantiles.get());
     }
 
+    public void testProcessResults_TimingStats() throws Exception {
+        ResultsBuilder resultsBuilder = new ResultsBuilder()
+                .addBucket(createBucket(true, 100))
+                .addBucket(createBucket(true, 1000))
+                .addBucket(createBucket(true, 100))
+                .addBucket(createBucket(true, 1000))
+                .addBucket(createBucket(true, 100))
+                .addBucket(createBucket(true, 1000))
+                .addBucket(createBucket(true, 100))
+                .addBucket(createBucket(true, 1000))
+                .addBucket(createBucket(true, 100))
+                .addBucket(createBucket(true, 1000));
+        when(process.readAutodetectResults()).thenReturn(resultsBuilder.build().iterator());
+
+        resultProcessor.process();
+        resultProcessor.awaitCompletion();
+
+        TimingStats timingStats = resultProcessor.timingStats();
+        assertThat(timingStats.getJobId(), equalTo(JOB_ID));
+        assertThat(timingStats.getBucketCount(), equalTo(10L));
+        assertThat(timingStats.getMinBucketProcessingTimeMs(), equalTo(100.0));
+        assertThat(timingStats.getMaxBucketProcessingTimeMs(), equalTo(1000.0));
+        assertThat(timingStats.getAvgBucketProcessingTimeMs(), equalTo(550.0));
+        assertThat(timingStats.getExponentialAvgBucketProcessingTimeMs(), closeTo(143.244, 1e-3));
+    }
+
     public void testParseQuantiles_GivenRenormalizationIsEnabled() throws Exception {
         when(renormalizer.isEnabled()).thenReturn(true);
 
-        ResultsBuilder builder = new ResultsBuilder();
+        ResultsBuilder resultsBuilder = new ResultsBuilder();
         Quantiles quantiles = createQuantiles();
-        builder.addQuantiles(quantiles);
+        resultsBuilder.addQuantiles(quantiles);
+        when(process.readAutodetectResults()).thenReturn(resultsBuilder.build().iterator());
 
-        resultProcessor.process(builder.buildTestProcess());
+        resultProcessor.process();
         resultProcessor.awaitCompletion();
 
         Optional<Quantiles> persistedQuantiles = getQuantiles();
@@ -182,11 +221,12 @@ public void testParseQuantiles_GivenRenormalizationIsEnabled() throws Exception
     public void testParseQuantiles_GivenRenormalizationIsDisabled() throws Exception {
         when(renormalizer.isEnabled()).thenReturn(false);
 
-        ResultsBuilder builder = new ResultsBuilder();
+        ResultsBuilder resultsBuilder = new ResultsBuilder();
         Quantiles quantiles = createQuantiles();
-        builder.addQuantiles(quantiles);
+        resultsBuilder.addQuantiles(quantiles);
+        when(process.readAutodetectResults()).thenReturn(resultsBuilder.build().iterator());
 
-        resultProcessor.process(builder.buildTestProcess());
+        resultProcessor.process();
         resultProcessor.awaitCompletion();
 
         Optional<Quantiles> persistedQuantiles = getQuantiles();
@@ -199,14 +239,15 @@ public void testDeleteInterimResults() throws Exception {
         Bucket nonInterimBucket = createBucket(false);
         Bucket interimBucket = createBucket(true);
 
-        ResultsBuilder resultBuilder = new ResultsBuilder()
+        ResultsBuilder resultsBuilder = new ResultsBuilder()
                 .addRecords(createRecords(true))
                 .addInfluencers(createInfluencers(true))
                 .addBucket(interimBucket)  // this will persist the interim results
                 .addFlushAcknowledgement(createFlushAcknowledgement())
                 .addBucket(nonInterimBucket); // and this will delete the interim results
+        when(process.readAutodetectResults()).thenReturn(resultsBuilder.build().iterator());
 
-        resultProcessor.process(resultBuilder.buildTestProcess());
+        resultProcessor.process();
         resultProcessor.awaitCompletion();
 
         QueryPage<Bucket> persistedBucket = getBucketQueryPage(new BucketsQueryBuilder().includeInterim(true));
@@ -227,7 +268,7 @@ public void testMultipleFlushesBetweenPersisting() throws Exception {
         Bucket finalBucket = createBucket(true);
         List<AnomalyRecord> finalAnomalyRecords = createRecords(true);
 
-        ResultsBuilder resultBuilder = new ResultsBuilder()
+        ResultsBuilder resultsBuilder = new ResultsBuilder()
                 .addRecords(createRecords(true))
                 .addInfluencers(createInfluencers(true))
                 .addBucket(createBucket(true))  // this will persist the interim results
@@ -237,8 +278,9 @@ public void testMultipleFlushesBetweenPersisting() throws Exception {
                 .addFlushAcknowledgement(createFlushAcknowledgement())
                 .addRecords(finalAnomalyRecords)
                 .addBucket(finalBucket); // this deletes the previous interim and persists final bucket & records
+        when(process.readAutodetectResults()).thenReturn(resultsBuilder.build().iterator());
 
-        resultProcessor.process(resultBuilder.buildTestProcess());
+        resultProcessor.process();
         resultProcessor.awaitCompletion();
 
         QueryPage<Bucket> persistedBucket = getBucketQueryPage(new BucketsQueryBuilder().includeInterim(true));
@@ -257,12 +299,13 @@ public void testEndOfStreamTriggersPersisting() throws Exception {
         List<AnomalyRecord> firstSetOfRecords = createRecords(false);
         List<AnomalyRecord> secondSetOfRecords = createRecords(false);
 
-        ResultsBuilder resultBuilder = new ResultsBuilder()
+        ResultsBuilder resultsBuilder = new ResultsBuilder()
                 .addRecords(firstSetOfRecords)
                 .addBucket(bucket)  // bucket triggers persistence
                 .addRecords(secondSetOfRecords);
+        when(process.readAutodetectResults()).thenReturn(resultsBuilder.build().iterator());
 
-        resultProcessor.process(resultBuilder.buildTestProcess());
+        resultProcessor.process();
         resultProcessor.awaitCompletion();
 
         QueryPage<Bucket> persistedBucket = getBucketQueryPage(new BucketsQueryBuilder().includeInterim(true));
@@ -283,18 +326,24 @@ private void putJob() {
         client().execute(PutJobAction.INSTANCE, request).actionGet();
     }
 
-    private Bucket createBucket(boolean isInterim) {
+    private static Bucket createBucket(boolean isInterim) {
         Bucket bucket = new BucketTests().createTestInstance(JOB_ID);
         bucket.setInterim(isInterim);
         return bucket;
     }
 
-    private Date randomDate() {
+    private static Bucket createBucket(boolean isInterim, long processingTimeMs) {
+        Bucket bucket = createBucket(isInterim);
+        bucket.setProcessingTimeMs(processingTimeMs);
+        return bucket;
+    }
+
+    private static Date randomDate() {
         // between 1970 and 2065
         return new Date(randomLongBetween(0, 3000000000000L));
     }
 
-    private List<AnomalyRecord> createRecords(boolean isInterim) {
+    private static List<AnomalyRecord> createRecords(boolean isInterim) {
         List<AnomalyRecord> records = new ArrayList<>();
 
         int count = randomIntBetween(0, 100);
@@ -309,7 +358,7 @@ private List<AnomalyRecord> createRecords(boolean isInterim) {
         return records;
     }
 
-    private List<Influencer> createInfluencers(boolean isInterim) {
+    private static List<Influencer> createInfluencers(boolean isInterim) {
         List<Influencer> influencers = new ArrayList<>();
 
         int count = randomIntBetween(0, 100);
@@ -322,15 +371,15 @@ private List<Influencer> createInfluencers(boolean isInterim) {
         return influencers;
     }
 
-    private CategoryDefinition createCategoryDefinition() {
+    private static CategoryDefinition createCategoryDefinition() {
         return new CategoryDefinitionTests().createTestInstance(JOB_ID);
     }
 
-    private ModelPlot createmodelPlot() {
+    private static ModelPlot createModelPlot() {
         return new ModelPlotTests().createTestInstance(JOB_ID);
     }
 
-    private ModelSizeStats createModelSizeStats() {
+    private static ModelSizeStats createModelSizeStats() {
         ModelSizeStats.Builder builder = new ModelSizeStats.Builder(JOB_ID);
         builder.setTimestamp(randomDate());
         builder.setLogTime(randomDate());
@@ -343,21 +392,21 @@ private ModelSizeStats createModelSizeStats() {
         return builder.build();
     }
 
-    private ModelSnapshot createModelSnapshot() {
+    private static ModelSnapshot createModelSnapshot() {
         return new ModelSnapshot.Builder(JOB_ID).setSnapshotId(randomAlphaOfLength(12)).build();
     }
 
-    private Quantiles createQuantiles() {
+    private static Quantiles createQuantiles() {
         return new Quantiles(JOB_ID, randomDate(), randomAlphaOfLength(100));
     }
 
-    private FlushAcknowledgement createFlushAcknowledgement() {
+    private static FlushAcknowledgement createFlushAcknowledgement() {
         return new FlushAcknowledgement(randomAlphaOfLength(5), randomDate());
     }
 
-    private class ResultsBuilder {
+    private static class ResultsBuilder {
 
-        private List<AutodetectResult> results = new ArrayList<>();
+        private final List<AutodetectResult> results = new ArrayList<>();
 
         ResultsBuilder addBucket(Bucket bucket) {
             results.add(new AutodetectResult(Objects.requireNonNull(bucket), null, null, null, null, null, null, null, null, null, null));
@@ -379,7 +428,7 @@ ResultsBuilder addCategoryDefinition(CategoryDefinition categoryDefinition) {
             return this;
         }
 
-        ResultsBuilder addmodelPlot(ModelPlot modelPlot) {
+        ResultsBuilder addModelPlot(ModelPlot modelPlot) {
             results.add(new AutodetectResult(null, null, null, null, null, null, modelPlot, null, null, null, null));
             return this;
         }
@@ -404,12 +453,8 @@ ResultsBuilder addFlushAcknowledgement(FlushAcknowledgement flushAcknowledgement
             return this;
         }
 
-
-        AutodetectProcess buildTestProcess() {
-            AutodetectResult[] results = this.results.toArray(new AutodetectResult[0]);
-            AutodetectProcess process = mock(AutodetectProcess.class);
-            when(process.readAutodetectResults()).thenReturn(Arrays.asList(results).iterator());
-            return process;
+        Iterable<AutodetectResult> build() {
+            return results;
         }
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
index 97d0824d2ac0f..e59f724a36371 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java
@@ -23,7 +23,6 @@
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
 import org.elasticsearch.test.InternalTestCluster;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
 import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction;
@@ -64,19 +63,19 @@ public void testFailOverBasics() throws Exception {
         Job.Builder job = createJob("fail-over-basics-job", new ByteSizeValue(2, ByteSizeUnit.MB));
         PutJobAction.Request putJobRequest = new PutJobAction.Request(job);
         client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet();
-        ensureGreen();
+        ensureYellow(); // at least the primary shards of the indices a job uses should be started
         OpenJobAction.Request openJobRequest = new OpenJobAction.Request(job.getId());
         client().execute(OpenJobAction.INSTANCE, openJobRequest).actionGet();
         awaitJobOpenedAndAssigned(job.getId(), null);
 
+        ensureGreen(); // replicas must be assigned, otherwise we could lose a whole index
         internalCluster().stopRandomDataNode();
         ensureStableCluster(3);
-        ensureGreen();
         awaitJobOpenedAndAssigned(job.getId(), null);
 
+        ensureGreen(); // replicas must be assigned, otherwise we could lose a whole index
         internalCluster().stopRandomDataNode();
         ensureStableCluster(2);
-        ensureGreen();
         awaitJobOpenedAndAssigned(job.getId(), null);
     }
 
@@ -106,7 +105,7 @@ public void testFailOverBasics_withDataFeeder() throws Exception {
         PutDatafeedAction.Request putDatafeedRequest = new PutDatafeedAction.Request(config);
         client().execute(PutDatafeedAction.INSTANCE, putDatafeedRequest).actionGet();
 
-        ensureGreen();
+        ensureYellow(); // at least the primary shards of the indices a job uses should be started
         OpenJobAction.Request openJobRequest = new OpenJobAction.Request(job.getId());
         client().execute(OpenJobAction.INSTANCE, openJobRequest).actionGet();
         awaitJobOpenedAndAssigned(job.getId(), null);
@@ -120,9 +119,9 @@ public void testFailOverBasics_withDataFeeder() throws Exception {
             assertEquals(DatafeedState.STARTED, statsResponse.getResponse().results().get(0).getDatafeedState());
         });
 
+        ensureGreen(); // replicas must be assigned, otherwise we could lose a whole index
         internalCluster().stopRandomDataNode();
         ensureStableCluster(3);
-        ensureGreen();
         awaitJobOpenedAndAssigned(job.getId(), null);
         assertBusy(() -> {
             GetDatafeedsStatsAction.Response statsResponse =
@@ -131,9 +130,9 @@ public void testFailOverBasics_withDataFeeder() throws Exception {
             assertEquals(DatafeedState.STARTED, statsResponse.getResponse().results().get(0).getDatafeedState());
         });
 
+        ensureGreen(); // replicas must be assigned, otherwise we could lose a whole index
         internalCluster().stopRandomDataNode();
         ensureStableCluster(2);
-        ensureGreen();
         awaitJobOpenedAndAssigned(job.getId(), null);
         assertBusy(() -> {
             GetDatafeedsStatsAction.Response statsResponse =
@@ -171,6 +170,7 @@ public void testJobAutoClose() throws Exception {
         PutDatafeedAction.Request putDatafeedRequest = new PutDatafeedAction.Request(config);
         client().execute(PutDatafeedAction.INSTANCE, putDatafeedRequest).actionGet();
 
+        ensureYellow(); // at least the primary shards of the indices a job uses should be started
         client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request(job.getId())).get();
 
         StartDatafeedAction.Request startDatafeedRequest = new StartDatafeedAction.Request(config.getId(), 0L);
@@ -183,7 +183,6 @@ public void testJobAutoClose() throws Exception {
         });
     }
 
-    @TestLogging("org.elasticsearch.xpack.persistent:TRACE,org.elasticsearch.cluster.service:DEBUG,org.elasticsearch.xpack.ml.action:DEBUG")
     public void testDedicatedMlNode() throws Exception {
         internalCluster().ensureAtMostNumDataNodes(0);
         // start 2 non ml node that will never get a job allocated. (but ml apis are accessible from this node)
@@ -203,6 +202,7 @@ public void testDedicatedMlNode() throws Exception {
         PutJobAction.Request putJobRequest = new PutJobAction.Request(job);
         client().execute(PutJobAction.INSTANCE, putJobRequest).actionGet();
 
+        ensureYellow(); // at least the primary shards of the indices a job uses should be started
         OpenJobAction.Request openJobRequest = new OpenJobAction.Request(job.getId());
         client().execute(OpenJobAction.INSTANCE, openJobRequest).actionGet();
         assertBusy(() -> {
@@ -277,6 +277,7 @@ public void testMaxConcurrentJobAllocations() throws Exception {
             }
         });
 
+        ensureYellow(); // at least the primary shards of the indices a job uses should be started
         int numJobs = numMlNodes * 10;
         for (int i = 0; i < numJobs; i++) {
             Job.Builder job = createJob(Integer.toString(i), new ByteSizeValue(2, ByteSizeUnit.MB));
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java
index 69b82ef984671..140bc0564bd93 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlConfigMigratorIT.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xpack.core.ml.MlMetadata;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
@@ -447,7 +448,7 @@ public void testConfigIndexIsCreated() throws Exception {
     }
 
     private boolean configIndexExists() {
-        return client().admin().indices().prepareExists(AnomalyDetectorsIndex.configIndexName()).get().isExists();
+        return ESIntegTestCase.indexExists(AnomalyDetectorsIndex.configIndexName(), client());
     }
 }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java
index 40249c0bc771e..711c9687144ca 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java
@@ -25,7 +25,7 @@
 import org.elasticsearch.persistent.PersistentTasksClusterService;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData.PersistentTask;
-import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.xpack.core.action.util.QueryPage;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
 import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction.Response.DatafeedStats;
@@ -36,7 +36,6 @@
 import org.elasticsearch.xpack.core.ml.action.PutJobAction;
 import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction;
-import org.elasticsearch.xpack.core.action.util.QueryPage;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
@@ -117,6 +116,7 @@ public void testFullClusterRestart() throws Exception {
         });
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/43670")
     public void testCloseUnassignedJobAndDatafeed() throws Exception {
         internalCluster().ensureAtMostNumDataNodes(0);
         logger.info("Starting dedicated master node...");
@@ -147,7 +147,9 @@ public void testCloseUnassignedJobAndDatafeed() throws Exception {
         waitForDatafeed(jobId, numDocs1);
 
         // stop the only ML node
+        ensureGreen(); // replicas must be assigned, otherwise we could lose a whole index
         internalCluster().stopRandomNonMasterNode();
+        ensureStableCluster(1);
 
         // Job state is opened but the job is not assigned to a node (because we just killed the only ML node)
         GetJobsStatsAction.Request jobStatsRequest = new GetJobsStatsAction.Request(jobId);
@@ -179,7 +181,6 @@ public void testCloseUnassignedJobAndDatafeed() throws Exception {
         assertTrue(closeJobResponse.isClosed());
     }
 
-    @TestLogging("org.elasticsearch.xpack.ml.action:TRACE,org.elasticsearch.xpack.ml.process:TRACE")
     public void testJobRelocationIsMemoryAware() throws Exception {
 
         internalCluster().ensureAtLeastNumDataNodes(1);
@@ -200,7 +201,7 @@ public void testJobRelocationIsMemoryAware() throws Exception {
 
         // Wait for the cluster to be green - this means the indices have been replicated.
 
-        ensureGreen(".ml-config", ".ml-anomalies-shared", ".ml-notifications");
+        ensureGreen();
 
         // Open a big job.  This should go on a different node to the 4 small ones.
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/NetworkDisruptionIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/NetworkDisruptionIT.java
index e05263014d33a..8a257baa3d628 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/NetworkDisruptionIT.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/NetworkDisruptionIT.java
@@ -38,12 +38,6 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return plugins;
     }
 
-    // Remove this once the AwaitsFix below has been resolved
-    public void testDummy() {
-        assertTrue(true);
-    }
-
-    @AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/39858")
     public void testJobRelocation() throws Exception {
         internalCluster().ensureAtLeastNumDataNodes(5);
         ensureStableCluster(5);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java
index ed9709b2d48f5..65a6c6c88f6ea 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java
@@ -108,7 +108,7 @@ public class JobManagerTests extends ESTestCase {
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobNodeSelectorTests.java
new file mode 100644
index 0000000000000..f0388abd5d5fd
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobNodeSelectorTests.java
@@ -0,0 +1,576 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.job;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ml.MlTasks;
+import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState;
+import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.core.ml.job.config.JobState;
+import org.elasticsearch.xpack.ml.MachineLearning;
+import org.elasticsearch.xpack.ml.action.TransportOpenJobAction;
+import org.elasticsearch.xpack.ml.action.TransportOpenJobActionTests;
+import org.elasticsearch.xpack.ml.action.TransportStartDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.ml.process.MlMemoryTracker;
+import org.elasticsearch.xpack.ml.support.BaseMlIntegTestCase;
+import org.junit.Before;
+
+import java.net.InetAddress;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import static org.elasticsearch.xpack.core.ml.job.config.JobTests.buildJobBuilder;
+import static org.hamcrest.Matchers.containsString;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+// TODO: in 8.0.0 remove all instances of MAX_OPEN_JOBS_NODE_ATTR from this file
+public class JobNodeSelectorTests extends ESTestCase {
+
+    // To simplify the logic in this class all jobs have the same memory requirement
+    private static final ByteSizeValue JOB_MEMORY_REQUIREMENT = new ByteSizeValue(10, ByteSizeUnit.MB);
+
+    private MlMemoryTracker memoryTracker;
+    private boolean isMemoryTrackerRecentlyRefreshed;
+
+    @Before
+    public void setup() {
+        memoryTracker = mock(MlMemoryTracker.class);
+        isMemoryTrackerRecentlyRefreshed = true;
+        when(memoryTracker.isRecentlyRefreshed()).thenReturn(isMemoryTrackerRecentlyRefreshed);
+        when(memoryTracker.getAnomalyDetectorJobMemoryRequirement(anyString())).thenReturn(JOB_MEMORY_REQUIREMENT.getBytes());
+        when(memoryTracker.getDataFrameAnalyticsJobMemoryRequirement(anyString())).thenReturn(JOB_MEMORY_REQUIREMENT.getBytes());
+        when(memoryTracker.getJobMemoryRequirement(anyString(), anyString())).thenReturn(JOB_MEMORY_REQUIREMENT.getBytes());
+    }
+
+    public void testNodeNameAndVersion() {
+        TransportAddress ta = new TransportAddress(InetAddress.getLoopbackAddress(), 9300);
+        Map<String, String> attributes = new HashMap<>();
+        attributes.put("unrelated", "attribute");
+        DiscoveryNode node = new DiscoveryNode("_node_name1", "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT);
+        assertEquals("{_node_name1}{version=" + node.getVersion() + "}", JobNodeSelector.nodeNameAndVersion(node));
+    }
+
+    public void testNodeNameAndMlAttributes() {
+        TransportAddress ta = new TransportAddress(InetAddress.getLoopbackAddress(), 9300);
+        SortedMap<String, String> attributes = new TreeMap<>();
+        attributes.put("unrelated", "attribute");
+        DiscoveryNode node = new DiscoveryNode("_node_name1", "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT);
+        assertEquals("{_node_name1}", JobNodeSelector.nodeNameAndMlAttributes(node));
+
+        attributes.put("ml.machine_memory", "5");
+        node = new DiscoveryNode("_node_name1", "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT);
+        assertEquals("{_node_name1}{ml.machine_memory=5}", JobNodeSelector.nodeNameAndMlAttributes(node));
+
+        node = new DiscoveryNode(null, "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT);
+        assertEquals("{_node_id1}{ml.machine_memory=5}", JobNodeSelector.nodeNameAndMlAttributes(node));
+
+        attributes.put("node.ml", "true");
+        node = new DiscoveryNode("_node_name1", "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT);
+        assertEquals("{_node_name1}{ml.machine_memory=5}{node.ml=true}", JobNodeSelector.nodeNameAndMlAttributes(node));
+    }
+
+    public void testSelectLeastLoadedMlNode_byCount() {
+        Map<String, String> nodeAttr = new HashMap<>();
+        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
+        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "-1");
+        // MachineLearning.MACHINE_MEMORY_NODE_ATTR negative, so this will fall back to allocating by count
+        DiscoveryNodes nodes = DiscoveryNodes.builder()
+            .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
+                nodeAttr, Collections.emptySet(), Version.CURRENT))
+            .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
+                nodeAttr, Collections.emptySet(), Version.CURRENT))
+            .add(new DiscoveryNode("_node_name3", "_node_id3", new TransportAddress(InetAddress.getLoopbackAddress(), 9302),
+                nodeAttr, Collections.emptySet(), Version.CURRENT))
+            .build();
+
+        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
+        TransportOpenJobActionTests.addJobTask("job_id1", "_node_id1", null, tasksBuilder);
+        TransportOpenJobActionTests.addJobTask("job_id2", "_node_id1", null, tasksBuilder);
+        TransportOpenJobActionTests.addJobTask("job_id3", "_node_id2", null, tasksBuilder);
+        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
+
+        ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
+        cs.nodes(nodes);
+        MetaData.Builder metaData = MetaData.builder();
+        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
+        cs.metaData(metaData);
+
+        Job.Builder jobBuilder = buildJobBuilder("job_id4");
+        jobBuilder.setJobVersion(Version.CURRENT);
+
+        Job job = jobBuilder.build();
+        JobNodeSelector jobNodeSelector = new JobNodeSelector(cs.build(), job.getId(), MlTasks.JOB_TASK_NAME, memoryTracker, 0,
+            node -> TransportOpenJobAction.nodeFilter(node, job));
+        PersistentTasksCustomMetaData.Assignment result = jobNodeSelector.selectNode(10, 2, 30, isMemoryTrackerRecentlyRefreshed);
+        assertEquals("", result.getExplanation());
+        assertEquals("_node_id3", result.getExecutorNode());
+    }
+
+    public void testSelectLeastLoadedMlNodeForAnomalyDetectorJob_maxCapacityCountLimiting() {
+        int numNodes = randomIntBetween(1, 10);
+        int maxRunningJobsPerNode = randomIntBetween(1, 100);
+        int maxMachineMemoryPercent = 30;
+        long machineMemory = (maxRunningJobsPerNode + 1) * JOB_MEMORY_REQUIREMENT.getBytes() * 100 / maxMachineMemoryPercent;
+
+        Map<String, String> nodeAttr = new HashMap<>();
+        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, Integer.toString(maxRunningJobsPerNode));
+        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
+
+        ClusterState.Builder cs = fillNodesWithRunningJobs(nodeAttr, numNodes, maxRunningJobsPerNode);
+
+        Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id1000", JOB_MEMORY_REQUIREMENT).build(new Date());
+
+        JobNodeSelector jobNodeSelector = new JobNodeSelector(cs.build(), job.getId(), MlTasks.JOB_TASK_NAME, memoryTracker, 0,
+            node -> TransportOpenJobAction.nodeFilter(node, job));
+        PersistentTasksCustomMetaData.Assignment result =
+            jobNodeSelector.selectNode(maxRunningJobsPerNode, 2, maxMachineMemoryPercent, isMemoryTrackerRecentlyRefreshed);
+        assertNull(result.getExecutorNode());
+        assertThat(result.getExplanation(), containsString("because this node is full. Number of opened jobs ["
+            + maxRunningJobsPerNode + "], xpack.ml.max_open_jobs [" + maxRunningJobsPerNode + "]"));
+    }
+
+    public void testSelectLeastLoadedMlNodeForDataFrameAnalyticsJob_maxCapacityCountLimiting() {
+        int numNodes = randomIntBetween(1, 10);
+        int maxRunningJobsPerNode = randomIntBetween(1, 100);
+        int maxMachineMemoryPercent = 30;
+        long machineMemory = (maxRunningJobsPerNode + 1) * JOB_MEMORY_REQUIREMENT.getBytes() * 100 / maxMachineMemoryPercent;
+
+        Map<String, String> nodeAttr = new HashMap<>();
+        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, Integer.toString(maxRunningJobsPerNode));
+        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
+
+        ClusterState.Builder cs = fillNodesWithRunningJobs(nodeAttr, numNodes, maxRunningJobsPerNode);
+
+        String dataFrameAnalyticsId = "data_frame_analytics_id1000";
+
+        JobNodeSelector jobNodeSelector = new JobNodeSelector(cs.build(), dataFrameAnalyticsId,
+            MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, memoryTracker, 0,
+            node -> TransportStartDataFrameAnalyticsAction.TaskExecutor.nodeFilter(node, dataFrameAnalyticsId));
+        PersistentTasksCustomMetaData.Assignment result =
+            jobNodeSelector.selectNode(maxRunningJobsPerNode, 2, maxMachineMemoryPercent, isMemoryTrackerRecentlyRefreshed);
+        assertNull(result.getExecutorNode());
+        assertThat(result.getExplanation(), containsString("because this node is full. Number of opened jobs ["
+            + maxRunningJobsPerNode + "], xpack.ml.max_open_jobs [" + maxRunningJobsPerNode + "]"));
+    }
+
+    public void testSelectLeastLoadedMlNodeForAnomalyDetectorJob_maxCapacityMemoryLimiting() {
+        int numNodes = randomIntBetween(1, 10);
+        int currentlyRunningJobsPerNode = randomIntBetween(1, 100);
+        int maxRunningJobsPerNode = currentlyRunningJobsPerNode + 1;
+        // Be careful if changing this - in order for the error message to be exactly as expected
+        // the value here must divide exactly into (JOB_MEMORY_REQUIREMENT.getBytes() * 100)
+        int maxMachineMemoryPercent = 40;
+        long machineMemory = currentlyRunningJobsPerNode * JOB_MEMORY_REQUIREMENT.getBytes() * 100 / maxMachineMemoryPercent;
+
+        Map<String, String> nodeAttr = new HashMap<>();
+        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, Integer.toString(maxRunningJobsPerNode));
+        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
+
+        ClusterState.Builder cs = fillNodesWithRunningJobs(nodeAttr, numNodes, currentlyRunningJobsPerNode);
+
+        Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id1000", JOB_MEMORY_REQUIREMENT).build(new Date());
+
+        JobNodeSelector jobNodeSelector = new JobNodeSelector(cs.build(), job.getId(), MlTasks.JOB_TASK_NAME, memoryTracker, 0,
+            node -> TransportOpenJobAction.nodeFilter(node, job));
+        PersistentTasksCustomMetaData.Assignment result =
+            jobNodeSelector.selectNode(maxRunningJobsPerNode, 2, maxMachineMemoryPercent, isMemoryTrackerRecentlyRefreshed);
+        assertNull(result.getExecutorNode());
+        assertThat(result.getExplanation(), containsString("because this node has insufficient available memory. "
+            + "Available memory for ML [" + (machineMemory * maxMachineMemoryPercent / 100) + "], memory required by existing jobs ["
+            + (JOB_MEMORY_REQUIREMENT.getBytes() * currentlyRunningJobsPerNode) + "], estimated memory required for this job ["
+            + JOB_MEMORY_REQUIREMENT.getBytes() + "]"));
+    }
+
+    public void testSelectLeastLoadedMlNodeForDataFrameAnalyticsJob_maxCapacityMemoryLimiting() {
+        int numNodes = randomIntBetween(1, 10);
+        int currentlyRunningJobsPerNode = randomIntBetween(1, 100);
+        int maxRunningJobsPerNode = currentlyRunningJobsPerNode + 1;
+        // Be careful if changing this - in order for the error message to be exactly as expected
+        // the value here must divide exactly into (JOB_MEMORY_REQUIREMENT.getBytes() * 100)
+        int maxMachineMemoryPercent = 40;
+        long machineMemory = currentlyRunningJobsPerNode * JOB_MEMORY_REQUIREMENT.getBytes() * 100 / maxMachineMemoryPercent;
+
+        Map<String, String> nodeAttr = new HashMap<>();
+        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, Integer.toString(maxRunningJobsPerNode));
+        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, Long.toString(machineMemory));
+
+        ClusterState.Builder cs = fillNodesWithRunningJobs(nodeAttr, numNodes, currentlyRunningJobsPerNode);
+
+        String dataFrameAnalyticsId = "data_frame_analytics_id1000";
+
+        JobNodeSelector jobNodeSelector = new JobNodeSelector(cs.build(), dataFrameAnalyticsId,
+            MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, memoryTracker, 0,
+            node -> TransportStartDataFrameAnalyticsAction.TaskExecutor.nodeFilter(node, dataFrameAnalyticsId));
+        PersistentTasksCustomMetaData.Assignment result =
+            jobNodeSelector.selectNode(maxRunningJobsPerNode, 2, maxMachineMemoryPercent, isMemoryTrackerRecentlyRefreshed);
+        assertNull(result.getExecutorNode());
+        assertThat(result.getExplanation(), containsString("because this node has insufficient available memory. "
+            + "Available memory for ML [" + (machineMemory * maxMachineMemoryPercent / 100) + "], memory required by existing jobs ["
+            + (JOB_MEMORY_REQUIREMENT.getBytes() * currentlyRunningJobsPerNode) + "], estimated memory required for this job ["
+            + JOB_MEMORY_REQUIREMENT.getBytes() + "]"));
+    }
+
+    public void testSelectLeastLoadedMlNode_noMlNodes() {
+        DiscoveryNodes nodes = DiscoveryNodes.builder()
+            .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
+                Collections.emptyMap(), Collections.emptySet(), Version.CURRENT))
+            .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
+                Collections.emptyMap(), Collections.emptySet(), Version.CURRENT))
+            .build();
+
+        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
+        TransportOpenJobActionTests.addJobTask("job_id1", "_node_id1", null, tasksBuilder);
+        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
+
+        ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
+        MetaData.Builder metaData = MetaData.builder();
+        cs.nodes(nodes);
+        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
+        cs.metaData(metaData);
+
+        Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id2", JOB_MEMORY_REQUIREMENT).build(new Date());
+
+        JobNodeSelector jobNodeSelector = new JobNodeSelector(cs.build(), job.getId(), MlTasks.JOB_TASK_NAME, memoryTracker, 0,
+            node -> TransportOpenJobAction.nodeFilter(node, job));
+        PersistentTasksCustomMetaData.Assignment result = jobNodeSelector.selectNode(20, 2, 30, isMemoryTrackerRecentlyRefreshed);
+        assertTrue(result.getExplanation().contains("because this node isn't a ml node"));
+        assertNull(result.getExecutorNode());
+    }
+
+    public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() {
+        Map<String, String> nodeAttr = new HashMap<>();
+        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
+        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
+        DiscoveryNodes nodes = DiscoveryNodes.builder()
+            .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
+                nodeAttr, Collections.emptySet(), Version.CURRENT))
+            .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
+                nodeAttr, Collections.emptySet(), Version.CURRENT))
+            .add(new DiscoveryNode("_node_name3", "_node_id3", new TransportAddress(InetAddress.getLoopbackAddress(), 9302),
+                nodeAttr, Collections.emptySet(), Version.CURRENT))
+            .build();
+
+        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
+        TransportOpenJobActionTests.addJobTask("job_id1", "_node_id1", null, tasksBuilder);
+        TransportOpenJobActionTests.addJobTask("job_id2", "_node_id1", null, tasksBuilder);
+        TransportOpenJobActionTests.addJobTask("job_id3", "_node_id2", null, tasksBuilder);
+        TransportOpenJobActionTests.addJobTask("job_id4", "_node_id2", null, tasksBuilder);
+        TransportOpenJobActionTests.addJobTask("job_id5", "_node_id3", null, tasksBuilder);
+        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
+
+        ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
+        csBuilder.nodes(nodes);
+        MetaData.Builder metaData = MetaData.builder();
+        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
+        csBuilder.metaData(metaData);
+
+        Job job6 = BaseMlIntegTestCase.createFareQuoteJob("job_id6", JOB_MEMORY_REQUIREMENT).build(new Date());
+
+        ClusterState cs = csBuilder.build();
+        JobNodeSelector jobNodeSelector = new JobNodeSelector(cs, job6.getId(), MlTasks.JOB_TASK_NAME, memoryTracker, 0,
+            node -> TransportOpenJobAction.nodeFilter(node, job6));
+        PersistentTasksCustomMetaData.Assignment result = jobNodeSelector.selectNode(10, 2, 30, isMemoryTrackerRecentlyRefreshed);
+        assertEquals("_node_id3", result.getExecutorNode());
+
+        tasksBuilder = PersistentTasksCustomMetaData.builder(tasks);
+        TransportOpenJobActionTests.addJobTask(job6.getId(), "_node_id3", null, tasksBuilder);
+        tasks = tasksBuilder.build();
+
+        csBuilder = ClusterState.builder(cs);
+        csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks));
+        cs = csBuilder.build();
+
+        Job job7 = BaseMlIntegTestCase.createFareQuoteJob("job_id7", JOB_MEMORY_REQUIREMENT).build(new Date());
+        jobNodeSelector = new JobNodeSelector(cs, job7.getId(), MlTasks.JOB_TASK_NAME, memoryTracker, 0,
+            node -> TransportOpenJobAction.nodeFilter(node, job7));
+        result = jobNodeSelector.selectNode(10, 2, 30, isMemoryTrackerRecentlyRefreshed);
+        assertNull("no node selected, because OPENING state", result.getExecutorNode());
+        assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state"));
+
+        tasksBuilder = PersistentTasksCustomMetaData.builder(tasks);
+        tasksBuilder.reassignTask(MlTasks.jobTaskId(job6.getId()),
+            new PersistentTasksCustomMetaData.Assignment("_node_id3", "test assignment"));
+        tasks = tasksBuilder.build();
+
+        csBuilder = ClusterState.builder(cs);
+        csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks));
+        cs = csBuilder.build();
+        jobNodeSelector = new JobNodeSelector(cs, job7.getId(), MlTasks.JOB_TASK_NAME, memoryTracker, 0,
+            node -> TransportOpenJobAction.nodeFilter(node, job7));
+        result = jobNodeSelector.selectNode(10, 2, 30, isMemoryTrackerRecentlyRefreshed);
+        assertNull("no node selected, because stale task", result.getExecutorNode());
+        assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state"));
+
+        tasksBuilder = PersistentTasksCustomMetaData.builder(tasks);
+        tasksBuilder.updateTaskState(MlTasks.jobTaskId(job6.getId()), null);
+        tasks = tasksBuilder.build();
+
+        csBuilder = ClusterState.builder(cs);
+        csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks));
+        cs = csBuilder.build();
+        jobNodeSelector = new JobNodeSelector(cs, job7.getId(), MlTasks.JOB_TASK_NAME, memoryTracker, 0,
+            node -> TransportOpenJobAction.nodeFilter(node, job7));
+        result = jobNodeSelector.selectNode(10, 2, 30, isMemoryTrackerRecentlyRefreshed);
+        assertNull("no node selected, because null state", result.getExecutorNode());
+        assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state"));
+    }
+
+    public void testSelectLeastLoadedMlNode_concurrentOpeningJobsAndStaleFailedJob() {
+        Map<String, String> nodeAttr = new HashMap<>();
+        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
+        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
+        DiscoveryNodes nodes = DiscoveryNodes.builder()
+            .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
+                nodeAttr, Collections.emptySet(), Version.CURRENT))
+            .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
+                nodeAttr, Collections.emptySet(), Version.CURRENT))
+            .add(new DiscoveryNode("_node_name3", "_node_id3", new TransportAddress(InetAddress.getLoopbackAddress(), 9302),
+                nodeAttr, Collections.emptySet(), Version.CURRENT))
+            .build();
+
+        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
+        TransportOpenJobActionTests.addJobTask("job_id1", "_node_id1", JobState.fromString("failed"), tasksBuilder);
+        // This will make the allocation stale for job_id1
+        tasksBuilder.reassignTask(MlTasks.jobTaskId("job_id1"),
+            new PersistentTasksCustomMetaData.Assignment("_node_id1", "test assignment"));
+        TransportOpenJobActionTests.addJobTask("job_id2", "_node_id1", null, tasksBuilder);
+        TransportOpenJobActionTests.addJobTask("job_id3", "_node_id2", null, tasksBuilder);
+        TransportOpenJobActionTests.addJobTask("job_id4", "_node_id2", null, tasksBuilder);
+        TransportOpenJobActionTests.addJobTask("job_id5", "_node_id3", null, tasksBuilder);
+        TransportOpenJobActionTests.addJobTask("job_id6", "_node_id3", null, tasksBuilder);
+        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
+
+        ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
+        csBuilder.nodes(nodes);
+        MetaData.Builder metaData = MetaData.builder();
+        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
+        csBuilder.metaData(metaData);
+
+        ClusterState cs = csBuilder.build();
+        Job job7 = BaseMlIntegTestCase.createFareQuoteJob("job_id7", JOB_MEMORY_REQUIREMENT).build(new Date());
+
+        // Allocation won't be possible if the stale failed job is treated as opening
+        JobNodeSelector jobNodeSelector = new JobNodeSelector(cs, job7.getId(), MlTasks.JOB_TASK_NAME, memoryTracker, 0,
+            node -> TransportOpenJobAction.nodeFilter(node, job7));
+        PersistentTasksCustomMetaData.Assignment result = jobNodeSelector.selectNode(10, 2, 30, isMemoryTrackerRecentlyRefreshed);
+        assertEquals("_node_id1", result.getExecutorNode());
+
+        tasksBuilder = PersistentTasksCustomMetaData.builder(tasks);
+        TransportOpenJobActionTests.addJobTask("job_id7", "_node_id1", null, tasksBuilder);
+        tasks = tasksBuilder.build();
+
+        csBuilder = ClusterState.builder(cs);
+        csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks));
+        cs = csBuilder.build();
+        Job job8 = BaseMlIntegTestCase.createFareQuoteJob("job_id8", JOB_MEMORY_REQUIREMENT).build(new Date());
+        jobNodeSelector = new JobNodeSelector(cs, job8.getId(), MlTasks.JOB_TASK_NAME, memoryTracker, 0,
+            node -> TransportOpenJobAction.nodeFilter(node, job8));
+        result = jobNodeSelector.selectNode(10, 2, 30, isMemoryTrackerRecentlyRefreshed);
+        assertNull("no node selected, because OPENING state", result.getExecutorNode());
+        assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state"));
+    }
+
+    public void testSelectLeastLoadedMlNode_noCompatibleJobTypeNodes() {
+        Map<String, String> nodeAttr = new HashMap<>();
+        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
+        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
+        DiscoveryNodes nodes = DiscoveryNodes.builder()
+            .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
+                nodeAttr, Collections.emptySet(), Version.CURRENT))
+            .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
+                nodeAttr, Collections.emptySet(), Version.CURRENT))
+            .build();
+
+        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
+        TransportOpenJobActionTests.addJobTask("incompatible_type_job", "_node_id1", null, tasksBuilder);
+        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
+
+        ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
+        MetaData.Builder metaData = MetaData.builder();
+
+        Job job = mock(Job.class);
+        when(job.getId()).thenReturn("incompatible_type_job");
+        when(job.getJobVersion()).thenReturn(Version.CURRENT);
+        when(job.getJobType()).thenReturn("incompatible_type");
+        when(job.getInitialResultsIndexName()).thenReturn("shared");
+
+        cs.nodes(nodes);
+        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
+        cs.metaData(metaData);
+        JobNodeSelector jobNodeSelector = new JobNodeSelector(cs.build(), job.getId(), MlTasks.JOB_TASK_NAME, memoryTracker, 0,
+            node -> TransportOpenJobAction.nodeFilter(node, job));
+        PersistentTasksCustomMetaData.Assignment result = jobNodeSelector.selectNode(10, 2, 30, isMemoryTrackerRecentlyRefreshed);
+        assertThat(result.getExplanation(), containsString("because this node does not support jobs of type [incompatible_type]"));
+        assertNull(result.getExecutorNode());
+    }
+
+    public void testSelectLeastLoadedMlNode_noNodesMatchingModelSnapshotMinVersion() {
+        Map<String, String> nodeAttr = new HashMap<>();
+        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
+        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
+        DiscoveryNodes nodes = DiscoveryNodes.builder()
+            .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
+                nodeAttr, Collections.emptySet(), Version.fromString("6.2.0")))
+            .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
+                nodeAttr, Collections.emptySet(), Version.fromString("6.1.0")))
+            .build();
+
+        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
+        TransportOpenJobActionTests.addJobTask("job_with_incompatible_model_snapshot", "_node_id1", null, tasksBuilder);
+        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
+
+        ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
+        MetaData.Builder metaData = MetaData.builder();
+
+        Job job = BaseMlIntegTestCase.createFareQuoteJob("job_with_incompatible_model_snapshot")
+            .setModelSnapshotId("incompatible_snapshot")
+            .setModelSnapshotMinVersion(Version.fromString("6.3.0"))
+            .build(new Date());
+        cs.nodes(nodes);
+        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
+        cs.metaData(metaData);
+        JobNodeSelector jobNodeSelector = new JobNodeSelector(cs.build(), job.getId(),
+            MlTasks.JOB_TASK_NAME, memoryTracker, 0, node -> TransportOpenJobAction.nodeFilter(node, job));
+        PersistentTasksCustomMetaData.Assignment result = jobNodeSelector.selectNode(10, 2, 30, isMemoryTrackerRecentlyRefreshed);
+        assertThat(result.getExplanation(), containsString(
+            "because the job's model snapshot requires a node of version [6.3.0] or higher"));
+        assertNull(result.getExecutorNode());
+    }
+
+    public void testSelectLeastLoadedMlNode_jobWithRules() {
+        Map<String, String> nodeAttr = new HashMap<>();
+        nodeAttr.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, "10");
+        nodeAttr.put(MachineLearning.MACHINE_MEMORY_NODE_ATTR, "1000000000");
+        DiscoveryNodes nodes = DiscoveryNodes.builder()
+            .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
+                nodeAttr, Collections.emptySet(), Version.fromString("6.2.0")))
+            .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
+                nodeAttr, Collections.emptySet(), Version.fromString("6.4.0")))
+            .build();
+
+        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
+        TransportOpenJobActionTests.addJobTask("job_with_rules", "_node_id1", null, tasksBuilder);
+        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
+
+        ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
+        MetaData.Builder metaData = MetaData.builder();
+        cs.nodes(nodes);
+        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
+        cs.metaData(metaData);
+
+        Job job = TransportOpenJobActionTests.jobWithRules("job_with_rules");
+        JobNodeSelector jobNodeSelector = new JobNodeSelector(cs.build(), job.getId(), MlTasks.JOB_TASK_NAME, memoryTracker, 0,
+            node -> TransportOpenJobAction.nodeFilter(node, job));
+        PersistentTasksCustomMetaData.Assignment result = jobNodeSelector.selectNode(10, 2, 30, isMemoryTrackerRecentlyRefreshed);
+        assertNotNull(result.getExecutorNode());
+    }
+
+    public void testConsiderLazyAssignmentWithNoLazyNodes() {
+        DiscoveryNodes nodes = DiscoveryNodes.builder()
+            .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
+                Collections.emptyMap(), Collections.emptySet(), Version.CURRENT))
+            .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
+                Collections.emptyMap(), Collections.emptySet(), Version.CURRENT))
+            .build();
+
+        ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
+        cs.nodes(nodes);
+
+        Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id1000", JOB_MEMORY_REQUIREMENT).build(new Date());
+        JobNodeSelector jobNodeSelector = new JobNodeSelector(cs.build(), job.getId(), MlTasks.JOB_TASK_NAME, memoryTracker, 0,
+            node -> TransportOpenJobAction.nodeFilter(node, job));
+        PersistentTasksCustomMetaData.Assignment result =
+            jobNodeSelector.considerLazyAssignment(new PersistentTasksCustomMetaData.Assignment(null, "foo"));
+        assertEquals("foo", result.getExplanation());
+        assertNull(result.getExecutorNode());
+    }
+
+    public void testConsiderLazyAssignmentWithLazyNodes() {
+        DiscoveryNodes nodes = DiscoveryNodes.builder()
+            .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300),
+                Collections.emptyMap(), Collections.emptySet(), Version.CURRENT))
+            .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301),
+                Collections.emptyMap(), Collections.emptySet(), Version.CURRENT))
+            .build();
+
+        ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
+        cs.nodes(nodes);
+
+        Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id1000", JOB_MEMORY_REQUIREMENT).build(new Date());
+        JobNodeSelector jobNodeSelector = new JobNodeSelector(cs.build(), job.getId(), MlTasks.JOB_TASK_NAME, memoryTracker,
+            randomIntBetween(1, 3), node -> TransportOpenJobAction.nodeFilter(node, job));
+        PersistentTasksCustomMetaData.Assignment result =
+            jobNodeSelector.considerLazyAssignment(new PersistentTasksCustomMetaData.Assignment(null, "foo"));
+        assertEquals(JobNodeSelector.AWAITING_LAZY_ASSIGNMENT.getExplanation(), result.getExplanation());
+        assertNull(result.getExecutorNode());
+    }
+
+    private ClusterState.Builder fillNodesWithRunningJobs(Map<String, String> nodeAttr, int numNodes, int numRunningJobsPerNode) {
+
+        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
+        PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder();
+        String[] jobIds = new String[numNodes * numRunningJobsPerNode];
+        for (int i = 0; i < numNodes; i++) {
+            String nodeId = "_node_id" + i;
+            TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), 9300 + i);
+            nodes.add(new DiscoveryNode("_node_name" + i, nodeId, address, nodeAttr, Collections.emptySet(), Version.CURRENT));
+            for (int j = 0; j < numRunningJobsPerNode; j++) {
+                int id = j + (numRunningJobsPerNode * i);
+                // Both anomaly detector jobs and data frame analytics jobs should count towards the limit
+                if (randomBoolean()) {
+                    jobIds[id] = "job_id" + id;
+                    TransportOpenJobActionTests.addJobTask(jobIds[id], nodeId, JobState.OPENED, tasksBuilder);
+                } else {
+                    jobIds[id] = "data_frame_analytics_id" + id;
+                    addDataFrameAnalyticsJobTask(jobIds[id], nodeId, DataFrameAnalyticsState.STARTED, tasksBuilder);
+                }
+            }
+        }
+        PersistentTasksCustomMetaData tasks = tasksBuilder.build();
+
+        ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name"));
+        MetaData.Builder metaData = MetaData.builder();
+        cs.nodes(nodes);
+        metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks);
+        cs.metaData(metaData);
+
+        return cs;
+    }
+
+    static void addDataFrameAnalyticsJobTask(String id, String nodeId, DataFrameAnalyticsState state,
+                                             PersistentTasksCustomMetaData.Builder builder) {
+        addDataFrameAnalyticsJobTask(id, nodeId, state, builder, false);
+    }
+
+    static void addDataFrameAnalyticsJobTask(String id, String nodeId, DataFrameAnalyticsState state,
+                                             PersistentTasksCustomMetaData.Builder builder, boolean isStale) {
+        builder.addTask(MlTasks.dataFrameAnalyticsTaskId(id), MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME,
+            new StartDataFrameAnalyticsAction.TaskParams(id, Version.CURRENT),
+            new PersistentTasksCustomMetaData.Assignment(nodeId, "test assignment"));
+        if (state != null) {
+            builder.updateTaskState(MlTasks.dataFrameAnalyticsTaskId(id),
+                new DataFrameAnalyticsTaskState(state, builder.getLastAllocationId() - (isStale ? 1 : 0), null));
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzerTests.java
index 59413f6a61879..988fa451c6540 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/categorization/CategorizationAnalyzerTests.java
@@ -25,7 +25,6 @@
 public class CategorizationAnalyzerTests extends ESTestCase {
 
     private AnalysisRegistry analysisRegistry;
-    private Environment environment;
 
     public static AnalysisRegistry buildTestAnalysisRegistry(Environment environment) throws Exception {
         CommonAnalysisPlugin commonAnalysisPlugin = new CommonAnalysisPlugin();
@@ -36,32 +35,32 @@ public static AnalysisRegistry buildTestAnalysisRegistry(Environment environment
     @Before
     public void setup() throws Exception {
         Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
-        environment = TestEnvironment.newEnvironment(settings);
+        Environment environment = TestEnvironment.newEnvironment(settings);
         analysisRegistry = buildTestAnalysisRegistry(environment);
     }
 
     public void testVerifyConfigBuilder_GivenNoConfig() {
         CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder();
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry));
         assertEquals("categorization_analyzer that is not a global analyzer must specify a [tokenizer] field", e.getMessage());
     }
 
     public void testVerifyConfigBuilder_GivenDefault() throws IOException {
         CategorizationAnalyzerConfig defaultConfig = CategorizationAnalyzerConfig.buildDefaultCategorizationAnalyzer(null);
         CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder(defaultConfig);
-        CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment);
+        CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry);
     }
 
     public void testVerifyConfigBuilder_GivenValidAnalyzer() throws IOException {
         CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder().setAnalyzer("standard");
-        CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment);
+        CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry);
     }
 
     public void testVerifyConfigBuilder_GivenInvalidAnalyzer() {
         CategorizationAnalyzerConfig.Builder builder = new CategorizationAnalyzerConfig.Builder().setAnalyzer("does not exist");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry));
         assertEquals("Failed to find global analyzer [does not exist]", e.getMessage());
     }
 
@@ -78,7 +77,7 @@ public void testVerifyConfigBuilder_GivenValidCustomConfig() throws IOException
             .addTokenFilter("lowercase")
             .addTokenFilter(ignoreStuffThatBeginsWithADigit)
             .addTokenFilter("snowball");
-        CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment);
+        CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry);
     }
 
     public void testVerifyConfigBuilder_GivenCustomConfigWithInvalidCharFilter() {
@@ -88,8 +87,8 @@ public void testVerifyConfigBuilder_GivenCustomConfigWithInvalidCharFilter() {
             .addTokenFilter("lowercase")
             .addTokenFilter("snowball");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
-        assertEquals("Failed to find global char filter under [wrong!]", e.getMessage());
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry));
+        assertEquals("failed to find global char_filter under [wrong!]", e.getMessage());
     }
 
     public void testVerifyConfigBuilder_GivenCustomConfigWithMisconfiguredCharFilter() {
@@ -102,8 +101,8 @@ public void testVerifyConfigBuilder_GivenCustomConfigWithMisconfiguredCharFilter
             .addTokenFilter("lowercase")
             .addTokenFilter("snowball");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
-        assertEquals("pattern is missing for [_anonymous_charfilter] char filter of type 'pattern_replace'", e.getMessage());
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry));
+        assertEquals("pattern is missing for [__anonymous__pattern_replace] char filter of type 'pattern_replace'", e.getMessage());
     }
 
     public void testVerifyConfigBuilder_GivenCustomConfigWithInvalidTokenizer() {
@@ -116,8 +115,8 @@ public void testVerifyConfigBuilder_GivenCustomConfigWithInvalidTokenizer() {
             .addTokenFilter("lowercase")
             .addTokenFilter("snowball");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
-        assertEquals("Failed to find global tokenizer under [oops!]", e.getMessage());
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry));
+        assertEquals("failed to find global tokenizer under [oops!]", e.getMessage());
     }
 
     public void testVerifyConfigBuilder_GivenNoTokenizer() {
@@ -133,7 +132,7 @@ public void testVerifyConfigBuilder_GivenNoTokenizer() {
             .addTokenFilter(ignoreStuffThatBeginsWithADigit)
             .addTokenFilter("snowball");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry));
         assertEquals("categorization_analyzer that is not a global analyzer must specify a [tokenizer] field", e.getMessage());
     }
 
@@ -147,8 +146,8 @@ public void testVerifyConfigBuilder_GivenCustomConfigWithInvalidTokenFilter() {
             .addTokenFilter("lowercase")
             .addTokenFilter("oh dear!");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
-        assertEquals("Failed to find global token filter under [oh dear!]", e.getMessage());
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry));
+        assertEquals("failed to find global filter under [oh dear!]", e.getMessage());
     }
 
     public void testVerifyConfigBuilder_GivenCustomConfigWithMisconfiguredTokenFilter() {
@@ -161,8 +160,8 @@ public void testVerifyConfigBuilder_GivenCustomConfigWithMisconfiguredTokenFilte
             .addTokenFilter("lowercase")
             .addTokenFilter(noPattern);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
-        assertEquals("pattern is missing for [_anonymous_tokenfilter] token filter of type 'pattern_replace'", e.getMessage());
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry));
+        assertEquals("pattern is missing for [__anonymous__pattern_replace] token filter of type 'pattern_replace'", e.getMessage());
     }
 
     public void testVerifyConfigBuilder_GivenAnalyzerAndCharFilter() {
@@ -170,7 +169,7 @@ public void testVerifyConfigBuilder_GivenAnalyzerAndCharFilter() {
             .setAnalyzer("standard")
             .addCharFilter("html_strip");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry));
         assertEquals("categorization_analyzer that is a global analyzer cannot also specify a [char_filter] field", e.getMessage());
     }
 
@@ -179,7 +178,7 @@ public void testVerifyConfigBuilder_GivenAnalyzerAndTokenizer() {
             .setAnalyzer("standard")
             .setTokenizer("classic");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry));
         assertEquals("categorization_analyzer that is a global analyzer cannot also specify a [tokenizer] field", e.getMessage());
     }
 
@@ -188,14 +187,14 @@ public void testVerifyConfigBuilder_GivenAnalyzerAndTokenFilter() {
             .setAnalyzer("standard")
             .addTokenFilter("lowercase");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry, environment));
+            () -> CategorizationAnalyzer.verifyConfigBuilder(builder, analysisRegistry));
         assertEquals("categorization_analyzer that is a global analyzer cannot also specify a [filter] field", e.getMessage());
     }
 
     // The default categorization analyzer matches what the analyzer in the ML C++ does
     public void testDefaultCategorizationAnalyzer() throws IOException {
         CategorizationAnalyzerConfig defaultConfig = CategorizationAnalyzerConfig.buildDefaultCategorizationAnalyzer(null);
-        try (CategorizationAnalyzer categorizationAnalyzer = new CategorizationAnalyzer(analysisRegistry, environment, defaultConfig)) {
+        try (CategorizationAnalyzer categorizationAnalyzer = new CategorizationAnalyzer(analysisRegistry, defaultConfig)) {
 
             assertEquals(Arrays.asList("ml13-4608.1.p2ps", "Info", "Source", "ML_SERVICE2", "on", "has", "shut", "down"),
                     categorizationAnalyzer.tokenizeField("p2ps",
@@ -225,7 +224,7 @@ public void testDefaultCategorizationAnalyzerWithCategorizationFilter() throws I
         // A categorization filter that removes stuff in square brackets
         CategorizationAnalyzerConfig defaultConfigWithCategorizationFilter =
                 CategorizationAnalyzerConfig.buildDefaultCategorizationAnalyzer(Collections.singletonList("\\[[^\\]]*\\]"));
-        try (CategorizationAnalyzer categorizationAnalyzer = new CategorizationAnalyzer(analysisRegistry, environment,
+        try (CategorizationAnalyzer categorizationAnalyzer = new CategorizationAnalyzer(analysisRegistry,
                 defaultConfigWithCategorizationFilter)) {
 
             assertEquals(Arrays.asList("ml13-4608.1.p2ps", "Info", "Source", "ML_SERVICE2", "on", "has", "shut", "down"),
@@ -255,7 +254,7 @@ public void testDefaultCategorizationAnalyzerWithCategorizationFilter() throws I
     // NOT for ML categorization (and you'll see why if you look at the expected results of this test!)
     public void testStandardAnalyzer() throws IOException {
         CategorizationAnalyzerConfig config = new CategorizationAnalyzerConfig.Builder().setAnalyzer("standard").build();
-        try (CategorizationAnalyzer categorizationAnalyzer = new CategorizationAnalyzer(analysisRegistry, environment, config)) {
+        try (CategorizationAnalyzer categorizationAnalyzer = new CategorizationAnalyzer(analysisRegistry, config)) {
 
             assertEquals(Arrays.asList("ml13", "4608.1", "p2ps", "info", "source", "ml_service2", "on", "13122", "867", "has", "shut",
                     "down"),
@@ -298,7 +297,7 @@ public void testCustomAnalyzer() throws IOException {
                 .addTokenFilter(ignoreStuffThatBeginsWithADigit)
                 .addTokenFilter("snowball")
                 .build();
-        try (CategorizationAnalyzer categorizationAnalyzer = new CategorizationAnalyzer(analysisRegistry, environment, config)) {
+        try (CategorizationAnalyzer categorizationAnalyzer = new CategorizationAnalyzer(analysisRegistry, config)) {
 
             assertEquals(Arrays.asList("ml13-4608.1.p2ps", "info", "sourc", "ml_service2", "on", "has", "shut", "down"),
                     categorizationAnalyzer.tokenizeField("p2ps",
@@ -325,14 +324,14 @@ public void testCustomAnalyzer() throws IOException {
 
     public void testEmptyString() throws IOException {
         CategorizationAnalyzerConfig defaultConfig = CategorizationAnalyzerConfig.buildDefaultCategorizationAnalyzer(null);
-        try (CategorizationAnalyzer categorizationAnalyzer = new CategorizationAnalyzer(analysisRegistry, environment, defaultConfig)) {
+        try (CategorizationAnalyzer categorizationAnalyzer = new CategorizationAnalyzer(analysisRegistry, defaultConfig)) {
             assertEquals(Collections.emptyList(), categorizationAnalyzer.tokenizeField("foo", ""));
         }
     }
 
     public void testThaiAnalyzer() throws IOException {
         CategorizationAnalyzerConfig config = new CategorizationAnalyzerConfig.Builder().setAnalyzer("thai").build();
-        try (CategorizationAnalyzer categorizationAnalyzer = new CategorizationAnalyzer(analysisRegistry, environment, config)) {
+        try (CategorizationAnalyzer categorizationAnalyzer = new CategorizationAnalyzer(analysisRegistry, config)) {
 
             // An example from the ES docs - no idea what it means or whether it's remotely sensible from a categorization point-of-view
             assertEquals(Arrays.asList("แสดง", "งาน", "ดี"),
@@ -343,6 +342,6 @@ public void testThaiAnalyzer() throws IOException {
 
     public void testInvalidAnalyzer() {
         CategorizationAnalyzerConfig config = new CategorizationAnalyzerConfig.Builder().setAnalyzer("does not exist").build();
-        expectThrows(IllegalArgumentException.class, () -> new CategorizationAnalyzer(analysisRegistry, environment, config));
+        expectThrows(IllegalArgumentException.class, () -> new CategorizationAnalyzer(analysisRegistry, config));
     }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/BatchedDocumentsIteratorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/BatchedDocumentsIteratorTests.java
index 0024eb5f8c648..2ff21243979be 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/BatchedDocumentsIteratorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/BatchedDocumentsIteratorTests.java
@@ -138,7 +138,6 @@ private void assertSearchRequest() {
         SearchRequest searchRequest = searchRequests.get(0);
         assertThat(searchRequest.indices(), equalTo(new String[] {INDEX_NAME}));
         assertThat(searchRequest.scroll().keepAlive(), equalTo(TimeValue.timeValueMinutes(5)));
-        assertThat(searchRequest.types().length, equalTo(0));
         assertThat(searchRequest.source().query(), equalTo(QueryBuilders.matchAllQuery()));
         assertThat(searchRequest.source().trackTotalHitsUpTo(), is(SearchContext.TRACK_TOTAL_HITS_ACCURATE));
     }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java
index 0035531d55a57..721e0d9cdca94 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord;
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
 import org.elasticsearch.xpack.core.ml.job.results.BucketInfluencer;
@@ -27,7 +28,9 @@
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
@@ -195,6 +198,35 @@ public void testBulkRequestExecutesWhenReachMaxDocs() {
         verifyNoMoreInteractions(client);
     }
 
+    public void testPersistTimingStats() {
+        ArgumentCaptor<BulkRequest> bulkRequestCaptor = ArgumentCaptor.forClass(BulkRequest.class);
+        Client client = mockClient(bulkRequestCaptor);
+
+        JobResultsPersister persister = new JobResultsPersister(client);
+        TimingStats timingStats = new TimingStats("foo", 7, 1.0, 2.0, 1.23, 7.89);
+        persister.bulkPersisterBuilder(JOB_ID).persistTimingStats(timingStats).executeRequest();
+
+        verify(client, times(1)).bulk(bulkRequestCaptor.capture());
+        BulkRequest bulkRequest = bulkRequestCaptor.getValue();
+        assertThat(bulkRequest.requests().size(), equalTo(1));
+        IndexRequest indexRequest = (IndexRequest) bulkRequest.requests().get(0);
+        assertThat(indexRequest.index(), equalTo(".ml-anomalies-.write-foo"));
+        assertThat(indexRequest.id(), equalTo("foo_timing_stats"));
+        assertThat(
+            indexRequest.sourceAsMap(),
+            equalTo(
+                Map.of(
+                    "job_id", "foo",
+                    "bucket_count", 7,
+                    "minimum_bucket_processing_time_ms", 1.0,
+                    "maximum_bucket_processing_time_ms", 2.0,
+                    "average_bucket_processing_time_ms", 1.23,
+                    "exponential_average_bucket_processing_time_ms", 7.89)));
+
+        verify(client, times(1)).threadPool();
+        verifyNoMoreInteractions(client);
+    }
+
     @SuppressWarnings({"unchecked"})
     private Client mockClient(ArgumentCaptor<BulkRequest> captor) {
         Client client = mock(Client.class);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java
index 2513aeac596c7..af821cf48fdc2 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java
@@ -13,7 +13,9 @@
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.MultiSearchResponse;
+import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.Client;
@@ -44,6 +46,7 @@
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
 import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord;
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
 import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition;
@@ -54,6 +57,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
@@ -70,6 +74,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
 public class JobResultsProviderTests extends ESTestCase {
@@ -824,6 +829,56 @@ public void testCountFields() {
         assertEquals(7, JobResultsProvider.countFields(Collections.singletonMap("properties", mapping)));
     }
 
+    public void testTimingStats_Ok() throws IOException {
+        String indexName = AnomalyDetectorsIndex.jobResultsAliasedName("foo");
+        List<Map<String, Object>> source =
+            Arrays.asList(
+                Map.of(
+                    Job.ID.getPreferredName(), "foo",
+                    TimingStats.BUCKET_COUNT.getPreferredName(), 7,
+                    TimingStats.MIN_BUCKET_PROCESSING_TIME_MS.getPreferredName(), 1.0,
+                    TimingStats.MAX_BUCKET_PROCESSING_TIME_MS.getPreferredName(), 1000.0,
+                    TimingStats.AVG_BUCKET_PROCESSING_TIME_MS.getPreferredName(), 666.0,
+                    TimingStats.EXPONENTIAL_AVG_BUCKET_PROCESSING_TIME_MS.getPreferredName(), 777.0));
+        SearchResponse response = createSearchResponse(source);
+        Client client = getMockedClient(
+            queryBuilder -> assertThat(queryBuilder.getName(), equalTo("ids")),
+            response);
+
+        when(client.prepareSearch(indexName)).thenReturn(new SearchRequestBuilder(client, SearchAction.INSTANCE).setIndices(indexName));
+        JobResultsProvider provider = createProvider(client);
+        provider.timingStats(
+            "foo",
+            stats -> assertThat(stats, equalTo(new TimingStats("foo", 7, 1.0, 1000.0, 666.0, 777.0))),
+            e -> { throw new AssertionError(); });
+
+        verify(client).prepareSearch(indexName);
+        verify(client).threadPool();
+        verify(client).search(any(SearchRequest.class), any(ActionListener.class));
+        verifyNoMoreInteractions(client);
+    }
+
+    public void testTimingStats_NotFound() throws IOException {
+        String indexName = AnomalyDetectorsIndex.jobResultsAliasedName("foo");
+        List<Map<String, Object>> source = new ArrayList<>();
+        SearchResponse response = createSearchResponse(source);
+        Client client = getMockedClient(
+            queryBuilder -> assertThat(queryBuilder.getName(), equalTo("ids")),
+            response);
+
+        when(client.prepareSearch(indexName)).thenReturn(new SearchRequestBuilder(client, SearchAction.INSTANCE).setIndices(indexName));
+        JobResultsProvider provider = createProvider(client);
+        provider.timingStats(
+            "foo",
+            stats -> assertThat(stats, equalTo(new TimingStats("foo"))),
+            e -> { throw new AssertionError(); });
+
+        verify(client).prepareSearch(indexName);
+        verify(client).threadPool();
+        verify(client).search(any(SearchRequest.class), any(ActionListener.class));
+        verifyNoMoreInteractions(client);
+    }
+
     private Bucket createBucketAtEpochTime(long epoch) {
         return new Bucket("foo", new Date(epoch), 123);
     }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java
index 999c36c7b4f86..b64bc42cb2c63 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java
@@ -17,8 +17,6 @@
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder;
@@ -122,31 +120,15 @@ public MockClientBuilder addClusterStatusYellowResponse(String index) throws Int
         return this;
     }
 
-    @SuppressWarnings({ "rawtypes", "unchecked" })
-    public MockClientBuilder addIndicesExistsResponse(String index, boolean exists) throws InterruptedException, ExecutionException {
-        ActionFuture actionFuture = mock(ActionFuture.class);
-        ArgumentCaptor<IndicesExistsRequest> requestCaptor = ArgumentCaptor.forClass(IndicesExistsRequest.class);
-
-        when(indicesAdminClient.exists(requestCaptor.capture())).thenReturn(actionFuture);
-        doAnswer(invocation -> {
-            IndicesExistsRequest request = (IndicesExistsRequest) invocation.getArguments()[0];
-            return request.indices()[0].equals(index) ? actionFuture : null;
-        }).when(indicesAdminClient).exists(any(IndicesExistsRequest.class));
-        when(actionFuture.get()).thenReturn(new IndicesExistsResponse(exists));
-        when(actionFuture.actionGet()).thenReturn(new IndicesExistsResponse(exists));
-        return this;
-    }
-
     @SuppressWarnings({ "unchecked" })
     public MockClientBuilder addIndicesDeleteResponse(String index, boolean exists, boolean exception,
             ActionListener<AcknowledgedResponse> actionListener) throws InterruptedException, ExecutionException, IOException {
-        AcknowledgedResponse response = DeleteIndexAction.INSTANCE.newResponse();
         StreamInput si = mock(StreamInput.class);
         // this looks complicated but Mockito can't mock the final method
         // DeleteIndexResponse.isAcknowledged() and the only way to create
         // one with a true response is reading from a stream.
         when(si.readByte()).thenReturn((byte) 0x01);
-        response.readFrom(si);
+        AcknowledgedResponse response = DeleteIndexAction.INSTANCE.getResponseReader().read(si);
 
         doAnswer(invocation -> {
             DeleteIndexRequest deleteIndexRequest = (DeleteIndexRequest) invocation.getArguments()[0];
@@ -206,7 +188,6 @@ public MockClientBuilder createIndexRequest(ArgumentCaptor<CreateIndexRequest> r
     @SuppressWarnings("unchecked")
     public MockClientBuilder prepareSearchExecuteListener(String index, SearchResponse response) {
         SearchRequestBuilder builder = mock(SearchRequestBuilder.class);
-        when(builder.setTypes(anyString())).thenReturn(builder);
         when(builder.addSort(any(SortBuilder.class))).thenReturn(builder);
         when(builder.setFetchSource(anyBoolean())).thenReturn(builder);
         when(builder.setScroll(anyString())).thenReturn(builder);
@@ -250,10 +231,9 @@ public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
         return this;
     }
 
-    public MockClientBuilder prepareSearch(String index, String type, int from, int size, SearchResponse response,
+    public MockClientBuilder prepareSearch(String index, int from, int size, SearchResponse response,
             ArgumentCaptor<QueryBuilder> filter) {
         SearchRequestBuilder builder = mock(SearchRequestBuilder.class);
-        when(builder.setTypes(eq(type))).thenReturn(builder);
         when(builder.addSort(any(SortBuilder.class))).thenReturn(builder);
         when(builder.setQuery(filter.capture())).thenReturn(builder);
         when(builder.setPostFilter(filter.capture())).thenReturn(builder);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/StateStreamerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/StateStreamerTests.java
index 1629a8bcdbad5..9337095974eba 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/StateStreamerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/StateStreamerTests.java
@@ -106,7 +106,6 @@ private static SearchResponse createSearchResponse(List<Map<String, Object>> sou
 
     private static SearchRequestBuilder prepareSearchBuilder(SearchResponse response, QueryBuilder queryBuilder) {
         SearchRequestBuilder builder = mock(SearchRequestBuilder.class);
-        when(builder.setTypes(any())).thenReturn(builder);
         when(builder.addSort(any(SortBuilder.class))).thenReturn(builder);
         when(builder.setQuery(queryBuilder)).thenReturn(builder);
         when(builder.setPostFilter(any())).thenReturn(builder);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/TimingStatsReporterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/TimingStatsReporterTests.java
new file mode 100644
index 0000000000000..f2314e6de3ed8
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/TimingStatsReporterTests.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.job.persistence;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
+import org.junit.Before;
+import org.mockito.InOrder;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+
+public class TimingStatsReporterTests extends ESTestCase {
+
+    private static final String JOB_ID = "my-job-id";
+
+    private JobResultsPersister.Builder bulkResultsPersister;
+
+    @Before
+    public void setUpTests() {
+        bulkResultsPersister = mock(JobResultsPersister.Builder.class);
+    }
+
+    public void testGetCurrentTimingStats() {
+        TimingStats stats = new TimingStats(JOB_ID, 7, 1.0, 2.0, 1.23, 7.89);
+        TimingStatsReporter reporter = new TimingStatsReporter(stats, bulkResultsPersister);
+        assertThat(reporter.getCurrentTimingStats(), equalTo(stats));
+
+        verifyZeroInteractions(bulkResultsPersister);
+    }
+
+    public void testReporting() {
+        TimingStatsReporter reporter = new TimingStatsReporter(new TimingStats(JOB_ID), bulkResultsPersister);
+        assertThat(reporter.getCurrentTimingStats(), equalTo(new TimingStats(JOB_ID)));
+
+        reporter.reportBucketProcessingTime(10);
+        assertThat(reporter.getCurrentTimingStats(), equalTo(new TimingStats(JOB_ID, 1, 10.0, 10.0, 10.0, 10.0)));
+
+        reporter.reportBucketProcessingTime(20);
+        assertThat(reporter.getCurrentTimingStats(), equalTo(new TimingStats(JOB_ID, 2, 10.0, 20.0, 15.0, 10.1)));
+
+        reporter.reportBucketProcessingTime(15);
+        assertThat(reporter.getCurrentTimingStats(), equalTo(new TimingStats(JOB_ID, 3, 10.0, 20.0, 15.0, 10.149)));
+
+        InOrder inOrder = inOrder(bulkResultsPersister);
+        inOrder.verify(bulkResultsPersister).persistTimingStats(new TimingStats(JOB_ID, 1, 10.0, 10.0, 10.0, 10.0));
+        inOrder.verify(bulkResultsPersister).persistTimingStats(new TimingStats(JOB_ID, 2, 10.0, 20.0, 15.0, 10.1));
+        inOrder.verifyNoMoreInteractions();
+    }
+
+    public void testFlush() {
+        TimingStatsReporter reporter = new TimingStatsReporter(new TimingStats(JOB_ID), bulkResultsPersister);
+        assertThat(reporter.getCurrentTimingStats(), equalTo(new TimingStats(JOB_ID)));
+
+        reporter.reportBucketProcessingTime(10);
+        assertThat(reporter.getCurrentTimingStats(), equalTo(new TimingStats(JOB_ID, 1, 10.0, 10.0, 10.0, 10.0)));
+
+        reporter.reportBucketProcessingTime(10);
+        assertThat(reporter.getCurrentTimingStats(), equalTo(new TimingStats(JOB_ID, 2, 10.0, 10.0, 10.0, 10.0)));
+
+        reporter.reportBucketProcessingTime(10);
+        assertThat(reporter.getCurrentTimingStats(), equalTo(new TimingStats(JOB_ID, 3, 10.0, 10.0, 10.0, 10.0)));
+
+        reporter.flush();
+        assertThat(reporter.getCurrentTimingStats(), equalTo(new TimingStats(JOB_ID, 3, 10.0, 10.0, 10.0, 10.0)));
+
+        InOrder inOrder = inOrder(bulkResultsPersister);
+        inOrder.verify(bulkResultsPersister).persistTimingStats(new TimingStats(JOB_ID, 1, 10.0, 10.0, 10.0, 10.0));
+        inOrder.verify(bulkResultsPersister).persistTimingStats(new TimingStats(JOB_ID, 3, 10.0, 10.0, 10.0, 10.0));
+        inOrder.verifyNoMoreInteractions();
+    }
+
+    public void testFinishReportingNoChange() {
+        TimingStatsReporter reporter = new TimingStatsReporter(new TimingStats(JOB_ID), bulkResultsPersister);
+
+        reporter.finishReporting();
+
+        verifyZeroInteractions(bulkResultsPersister);
+    }
+
+    public void testFinishReportingWithChange() {
+        TimingStatsReporter reporter = new TimingStatsReporter(new TimingStats(JOB_ID), bulkResultsPersister);
+
+        reporter.reportBucketProcessingTime(10);
+
+        reporter.finishReporting();
+
+        verify(bulkResultsPersister).persistTimingStats(new TimingStats(JOB_ID, 1, 10.0, 10.0, 10.0, 10.0));
+    }
+
+    public void testTimingStatsDifferSignificantly() {
+        assertThat(
+            TimingStatsReporter.differSignificantly(
+                new TimingStats(JOB_ID, 10, 10.0, 10.0, 1.0, 10.0), new TimingStats(JOB_ID, 10, 10.0, 10.0, 1.0, 10.0)),
+            is(false));
+        assertThat(
+            TimingStatsReporter.differSignificantly(
+                new TimingStats(JOB_ID, 10, 10.0, 10.0, 1.0, 10.0), new TimingStats(JOB_ID, 10, 10.0, 11.0, 1.0, 10.0)),
+            is(false));
+        assertThat(
+            TimingStatsReporter.differSignificantly(
+                new TimingStats(JOB_ID, 10, 10.0, 10.0, 1.0, 10.0), new TimingStats(JOB_ID, 10, 10.0, 12.0, 1.0, 10.0)),
+            is(true));
+    }
+
+    public void testValuesDifferSignificantly() {
+        assertThat(TimingStatsReporter.differSignificantly((Double) null, (Double) null), is(false));
+        assertThat(TimingStatsReporter.differSignificantly(1.0, null), is(true));
+        assertThat(TimingStatsReporter.differSignificantly(null, 1.0), is(true));
+        assertThat(TimingStatsReporter.differSignificantly(0.9, 1.0), is(false));
+        assertThat(TimingStatsReporter.differSignificantly(1.0, 0.9), is(false));
+        assertThat(TimingStatsReporter.differSignificantly(0.9, 1.000001), is(true));
+        assertThat(TimingStatsReporter.differSignificantly(1.0, 0.899999), is(true));
+        assertThat(TimingStatsReporter.differSignificantly(0.0, 1.0), is(true));
+        assertThat(TimingStatsReporter.differSignificantly(1.0, 0.0), is(true));
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicatorTests.java
index 6705773dc7329..9f6d52950732b 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicatorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicatorTests.java
@@ -27,7 +27,7 @@
 import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzerTests;
 import org.elasticsearch.xpack.ml.job.persistence.StateStreamer;
 import org.elasticsearch.xpack.ml.job.process.DataCountsReporter;
-import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutoDetectResultProcessor;
+import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutodetectResultProcessor;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.FlushJobParams;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.TimeRange;
@@ -79,7 +79,7 @@ public void setup() throws Exception {
     public void testWriteResetBucketsControlMessage() throws IOException {
         DataLoadParams params = new DataLoadParams(TimeRange.builder().startTime("1").endTime("2").build(), Optional.empty());
         AutodetectProcess process = mockAutodetectProcessWithOutputStream();
-        try (AutodetectCommunicator communicator = createAutodetectCommunicator(process, mock(AutoDetectResultProcessor.class))) {
+        try (AutodetectCommunicator communicator = createAutodetectCommunicator(process, mock(AutodetectResultProcessor.class))) {
             communicator.writeToJob(new ByteArrayInputStream(new byte[0]), analysisRegistry,
                     randomFrom(XContentType.values()), params, (dataCounts, e) -> {});
             verify(process).writeResetBucketsControlMessage(params);
@@ -89,7 +89,7 @@ public void testWriteResetBucketsControlMessage() throws IOException {
     public void testWriteUpdateProcessMessage() throws IOException {
         AutodetectProcess process = mockAutodetectProcessWithOutputStream();
         when(process.isReady()).thenReturn(true);
-        AutodetectCommunicator communicator = createAutodetectCommunicator(process, mock(AutoDetectResultProcessor.class));
+        AutodetectCommunicator communicator = createAutodetectCommunicator(process, mock(AutodetectResultProcessor.class));
 
         DetectionRule updatedRule = new DetectionRule.Builder(RuleScope.builder().exclude("foo", "bar")).build();
         List<JobUpdate.DetectorUpdate> detectorUpdates = Collections.singletonList(
@@ -111,7 +111,7 @@ public void testWriteUpdateProcessMessage() throws IOException {
     public void testFlushJob() throws IOException, InterruptedException {
         AutodetectProcess process = mockAutodetectProcessWithOutputStream();
         when(process.isProcessAlive()).thenReturn(true);
-        AutoDetectResultProcessor processor = mock(AutoDetectResultProcessor.class);
+        AutodetectResultProcessor processor = mock(AutodetectResultProcessor.class);
         FlushAcknowledgement flushAcknowledgement = mock(FlushAcknowledgement.class);
         when(processor.waitForFlushAcknowledgement(anyString(), any())).thenReturn(flushAcknowledgement);
         try (AutodetectCommunicator communicator = createAutodetectCommunicator(process, processor)) {
@@ -126,7 +126,7 @@ public void testFlushJob() throws IOException, InterruptedException {
     public void testWaitForFlushReturnsIfParserFails() throws IOException, InterruptedException {
         AutodetectProcess process = mockAutodetectProcessWithOutputStream();
         when(process.isProcessAlive()).thenReturn(true);
-        AutoDetectResultProcessor processor = mock(AutoDetectResultProcessor.class);
+        AutodetectResultProcessor processor = mock(AutodetectResultProcessor.class);
         when(processor.isFailed()).thenReturn(true);
         when(processor.waitForFlushAcknowledgement(anyString(), any())).thenReturn(null);
         AutodetectCommunicator communicator = createAutodetectCommunicator(process, processor);
@@ -137,7 +137,7 @@ public void testFlushJob_throwsIfProcessIsDead() throws IOException {
         AutodetectProcess process = mockAutodetectProcessWithOutputStream();
         when(process.isProcessAlive()).thenReturn(false);
         when(process.readError()).thenReturn("Mock process is dead");
-        AutodetectCommunicator communicator = createAutodetectCommunicator(process, mock(AutoDetectResultProcessor.class));
+        AutodetectCommunicator communicator = createAutodetectCommunicator(process, mock(AutodetectResultProcessor.class));
         FlushJobParams params = FlushJobParams.builder().build();
         Exception[] holder = new ElasticsearchException[1];
         communicator.flushJob(params, (aVoid, e1) -> holder[0] = e1);
@@ -147,17 +147,17 @@ public void testFlushJob_throwsIfProcessIsDead() throws IOException {
     public void testFlushJob_givenFlushWaitReturnsTrueOnSecondCall() throws IOException, InterruptedException {
         AutodetectProcess process = mockAutodetectProcessWithOutputStream();
         when(process.isProcessAlive()).thenReturn(true);
-        AutoDetectResultProcessor autoDetectResultProcessor = Mockito.mock(AutoDetectResultProcessor.class);
+        AutodetectResultProcessor autodetectResultProcessor = Mockito.mock(AutodetectResultProcessor.class);
         FlushAcknowledgement flushAcknowledgement = mock(FlushAcknowledgement.class);
-        when(autoDetectResultProcessor.waitForFlushAcknowledgement(anyString(), eq(Duration.ofSeconds(1))))
+        when(autodetectResultProcessor.waitForFlushAcknowledgement(anyString(), eq(Duration.ofSeconds(1))))
                 .thenReturn(null).thenReturn(flushAcknowledgement);
         FlushJobParams params = FlushJobParams.builder().build();
 
-        try (AutodetectCommunicator communicator = createAutodetectCommunicator(process, autoDetectResultProcessor)) {
+        try (AutodetectCommunicator communicator = createAutodetectCommunicator(process, autodetectResultProcessor)) {
             communicator.flushJob(params, (aVoid, e) -> {});
         }
 
-        verify(autoDetectResultProcessor, times(2)).waitForFlushAcknowledgement(anyString(), eq(Duration.ofSeconds(1)));
+        verify(autodetectResultProcessor, times(2)).waitForFlushAcknowledgement(anyString(), eq(Duration.ofSeconds(1)));
         // First in checkAndRun, second due to check between calls to waitForFlushAcknowledgement and third due to close()
         verify(process, times(3)).isProcessAlive();
     }
@@ -165,7 +165,7 @@ public void testFlushJob_givenFlushWaitReturnsTrueOnSecondCall() throws IOExcept
     public void testCloseGivenProcessIsReady() throws IOException {
         AutodetectProcess process = mockAutodetectProcessWithOutputStream();
         when(process.isReady()).thenReturn(true);
-        AutodetectCommunicator communicator = createAutodetectCommunicator(process, mock(AutoDetectResultProcessor.class));
+        AutodetectCommunicator communicator = createAutodetectCommunicator(process, mock(AutodetectResultProcessor.class));
 
         communicator.close();
 
@@ -177,7 +177,7 @@ public void testCloseGivenProcessIsReady() throws IOException {
     public void testCloseGivenProcessIsNotReady() throws IOException {
         AutodetectProcess process = mockAutodetectProcessWithOutputStream();
         when(process.isReady()).thenReturn(false);
-        AutodetectCommunicator communicator = createAutodetectCommunicator(process, mock(AutoDetectResultProcessor.class));
+        AutodetectCommunicator communicator = createAutodetectCommunicator(process, mock(AutodetectResultProcessor.class));
 
         communicator.close();
 
@@ -188,7 +188,7 @@ public void testCloseGivenProcessIsNotReady() throws IOException {
 
     public void testKill() throws IOException, TimeoutException {
         AutodetectProcess process = mockAutodetectProcessWithOutputStream();
-        AutoDetectResultProcessor resultProcessor = mock(AutoDetectResultProcessor.class);
+        AutodetectResultProcessor resultProcessor = mock(AutodetectResultProcessor.class);
         ExecutorService executorService = mock(ExecutorService.class);
 
         AtomicBoolean finishCalled = new AtomicBoolean(false);
@@ -232,7 +232,7 @@ private AutodetectProcess mockAutodetectProcessWithOutputStream() throws IOExcep
 
     @SuppressWarnings("unchecked")
     private AutodetectCommunicator createAutodetectCommunicator(ExecutorService executorService, AutodetectProcess autodetectProcess,
-                                                                AutoDetectResultProcessor autoDetectResultProcessor,
+                                                                AutodetectResultProcessor autodetectResultProcessor,
                                                                 BiConsumer<Exception, Boolean> finishHandler) throws IOException {
         DataCountsReporter dataCountsReporter = mock(DataCountsReporter.class);
         doAnswer(invocation -> {
@@ -240,13 +240,13 @@ private AutodetectCommunicator createAutodetectCommunicator(ExecutorService exec
             return null;
         }).when(dataCountsReporter).finishReporting(any());
         return new AutodetectCommunicator(createJobDetails(), environment, autodetectProcess,
-                stateStreamer, dataCountsReporter, autoDetectResultProcessor, finishHandler,
+                stateStreamer, dataCountsReporter, autodetectResultProcessor, finishHandler,
                 new NamedXContentRegistry(Collections.emptyList()), executorService);
     }
 
     @SuppressWarnings("unchecked")
     private AutodetectCommunicator createAutodetectCommunicator(AutodetectProcess autodetectProcess,
-                                                                AutoDetectResultProcessor autoDetectResultProcessor) throws IOException {
+                                                                AutodetectResultProcessor autodetectResultProcessor) throws IOException {
         ExecutorService executorService = mock(ExecutorService.class);
         when(executorService.submit(any(Callable.class))).thenReturn(mock(Future.class));
         doAnswer(invocationOnMock -> {
@@ -259,7 +259,7 @@ private AutodetectCommunicator createAutodetectCommunicator(AutodetectProcess au
             return null;
         }).when(executorService).execute(any(Runnable.class));
 
-        return createAutodetectCommunicator(executorService, autodetectProcess, autoDetectResultProcessor, (e, b) -> {});
+        return createAutodetectCommunicator(executorService, autodetectProcess, autodetectResultProcessor, (e, b) -> {});
     }
 
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
index 5f1ce1ddf7d0e..519a81fe7d0a3 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
 import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
@@ -335,9 +334,6 @@ public void testCloseJob() {
         assertEquals(0, manager.numberOfOpenJobs());
     }
 
-    // DEBUG logging makes it possible to see exactly how the two threads
-    // interleaved in the AutodetectProcessManager.close() call
-    @TestLogging("org.elasticsearch.xpack.ml.job.process.autodetect:DEBUG")
     public void testCanCloseClosingJob() throws Exception {
         AtomicInteger numberOfCommunicatorCloses = new AtomicInteger(0);
         doAnswer(invocationOnMock -> {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactoryTests.java
new file mode 100644
index 0000000000000..f486823be7b40
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactoryTests.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.job.process.autodetect;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.ml.MachineLearning;
+import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams;
+import org.elasticsearch.xpack.ml.process.NativeController;
+import org.elasticsearch.xpack.ml.process.ProcessPipes;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.Set;
+
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class NativeAutodetectProcessFactoryTests extends ESTestCase {
+
+    public void testSetProcessConnectTimeout() throws IOException {
+
+        int timeoutSeconds = randomIntBetween(5, 100);
+
+        Settings settings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+        Environment env = TestEnvironment.newEnvironment(settings);
+        NativeController nativeController = mock(NativeController.class);
+        Client client = mock(Client.class);
+        ClusterSettings clusterSettings = new ClusterSettings(settings,
+            Set.of(MachineLearning.PROCESS_CONNECT_TIMEOUT, AutodetectBuilder.MAX_ANOMALY_RECORDS_SETTING_DYNAMIC));
+        ClusterService clusterService = mock(ClusterService.class);
+        when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
+        Job job = mock(Job.class);
+        when(job.getId()).thenReturn("set_process_connect_test_job");
+        AutodetectParams autodetectParams = mock(AutodetectParams.class);
+        ProcessPipes processPipes = mock(ProcessPipes.class);
+
+        NativeAutodetectProcessFactory nativeAutodetectProcessFactory =
+            new NativeAutodetectProcessFactory(env, settings, nativeController, client, clusterService);
+        nativeAutodetectProcessFactory.setProcessConnectTimeout(TimeValue.timeValueSeconds(timeoutSeconds));
+        nativeAutodetectProcessFactory.createNativeProcess(job, autodetectParams, processPipes, Collections.emptyList());
+
+        verify(processPipes, times(1)).connectStreams(eq(Duration.ofSeconds(timeoutSeconds)));
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessTests.java
index 3f1275142b968..0935a4c628d6b 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessTests.java
@@ -7,12 +7,14 @@
 
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig;
-import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutodetectResultsParser;
 import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutodetectStateProcessor;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.FlushJobParams;
 import org.elasticsearch.xpack.ml.job.process.autodetect.params.TimeRange;
 import org.elasticsearch.xpack.ml.job.process.autodetect.writer.AutodetectControlMsgWriter;
+import org.elasticsearch.xpack.ml.job.results.AutodetectResult;
+import org.elasticsearch.xpack.ml.process.ProcessResultsParser;
+import org.elasticsearch.xpack.ml.process.NativeController;
 import org.junit.Assert;
 import org.junit.Before;
 
@@ -56,10 +58,10 @@ public void testProcessStartTime() throws Exception {
         when(logStream.read(new byte[1024])).thenReturn(-1);
         InputStream outputStream = mock(InputStream.class);
         when(outputStream.read(new byte[512])).thenReturn(-1);
-        try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream,
+        try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", mock(NativeController.class), logStream,
                 mock(OutputStream.class), outputStream, mock(OutputStream.class),
                 NUMBER_FIELDS, null,
-                new AutodetectResultsParser(), mock(Consumer.class))) {
+                new ProcessResultsParser<>(AutodetectResult.PARSER), mock(Consumer.class))) {
             process.start(executorService, mock(AutodetectStateProcessor.class), mock(InputStream.class));
 
             ZonedDateTime startTime = process.getProcessStartTime();
@@ -79,9 +81,9 @@ public void testWriteRecord() throws IOException {
         when(outputStream.read(new byte[512])).thenReturn(-1);
         String[] record = {"r1", "r2", "r3", "r4", "r5"};
         ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
-        try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream,
+        try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", mock(NativeController.class), logStream,
                 bos, outputStream, mock(OutputStream.class), NUMBER_FIELDS, Collections.emptyList(),
-                new AutodetectResultsParser(), mock(Consumer.class))) {
+                new ProcessResultsParser<>(AutodetectResult.PARSER), mock(Consumer.class))) {
             process.start(executorService, mock(AutodetectStateProcessor.class), mock(InputStream.class));
 
             process.writeRecord(record);
@@ -113,9 +115,9 @@ public void testFlush() throws IOException {
         InputStream outputStream = mock(InputStream.class);
         when(outputStream.read(new byte[512])).thenReturn(-1);
         ByteArrayOutputStream bos = new ByteArrayOutputStream(AutodetectControlMsgWriter.FLUSH_SPACES_LENGTH + 1024);
-        try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream,
+        try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", mock(NativeController.class), logStream,
                 bos, outputStream, mock(OutputStream.class), NUMBER_FIELDS, Collections.emptyList(),
-                new AutodetectResultsParser(), mock(Consumer.class))) {
+                new ProcessResultsParser<>(AutodetectResult.PARSER), mock(Consumer.class))) {
             process.start(executorService, mock(AutodetectStateProcessor.class), mock(InputStream.class));
 
             FlushJobParams params = FlushJobParams.builder().build();
@@ -146,9 +148,9 @@ public void testConsumeAndCloseOutputStream() throws IOException {
         String json = "some string of data";
         ByteArrayInputStream processOutStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8));
 
-        try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream,
+        try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", mock(NativeController.class), logStream,
             processInStream, processOutStream, mock(OutputStream.class), NUMBER_FIELDS, Collections.emptyList(),
-            new AutodetectResultsParser(), mock(Consumer.class))) {
+            new ProcessResultsParser<AutodetectResult>(AutodetectResult.PARSER), mock(Consumer.class))) {
 
             process.consumeAndCloseOutputStream();
             assertThat(processOutStream.available(), equalTo(0));
@@ -161,9 +163,9 @@ private void testWriteMessage(CheckedConsumer<NativeAutodetectProcess> writeFunc
         InputStream outputStream = mock(InputStream.class);
         when(outputStream.read(new byte[512])).thenReturn(-1);
         ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
-        try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", logStream,
+        try (NativeAutodetectProcess process = new NativeAutodetectProcess("foo", mock(NativeController.class), logStream,
                 bos, outputStream, mock(OutputStream.class), NUMBER_FIELDS, Collections.emptyList(),
-                new AutodetectResultsParser(), mock(Consumer.class))) {
+                new ProcessResultsParser<>(AutodetectResult.PARSER), mock(Consumer.class))) {
             process.start(executorService, mock(AutodetectStateProcessor.class), mock(InputStream.class));
 
             writeFunction.accept(process);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessorTests.java
similarity index 52%
rename from x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java
rename to x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessorTests.java
index 40a1d3f969157..1e0263185798f 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessorTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessorTests.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord;
 import org.elasticsearch.xpack.core.ml.job.results.Bucket;
 import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition;
@@ -51,6 +52,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Matchers.any;
@@ -66,7 +68,7 @@
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
-public class AutoDetectResultProcessorTests extends ESTestCase {
+public class AutodetectResultProcessorTests extends ESTestCase {
 
     private static final String JOB_ID = "valid_id";
     private static final long BUCKET_SPAN_MS = 1000;
@@ -76,8 +78,10 @@ public class AutoDetectResultProcessorTests extends ESTestCase {
     private Auditor auditor;
     private Renormalizer renormalizer;
     private JobResultsPersister persister;
+    private JobResultsPersister.Builder bulkBuilder;
+    private AutodetectProcess process;
     private FlushListener flushListener;
-    private AutoDetectResultProcessor processorUnderTest;
+    private AutodetectResultProcessor processorUnderTest;
     private ScheduledThreadPoolExecutor executor;
 
     @Before
@@ -90,148 +94,138 @@ public void setUpMocks() {
         auditor = mock(Auditor.class);
         renormalizer = mock(Renormalizer.class);
         persister = mock(JobResultsPersister.class);
-        when(persister.persistModelSnapshot(any(), any()))
-                .thenReturn(new IndexResponse(new ShardId("ml", "uid", 0), "doc", "1", 0L, 0L, 0L, true));
+        bulkBuilder = mock(JobResultsPersister.Builder.class);
+        when(persister.bulkPersisterBuilder(JOB_ID)).thenReturn(bulkBuilder);
+        process = mock(AutodetectProcess.class);
         flushListener = mock(FlushListener.class);
-        processorUnderTest = new AutoDetectResultProcessor(client, auditor, JOB_ID, renormalizer, persister,
-                new ModelSizeStats.Builder(JOB_ID).setTimestamp(new Date(BUCKET_SPAN_MS)).build(), flushListener);
+        processorUnderTest = new AutodetectResultProcessor(
+            client,
+            auditor,
+            JOB_ID,
+            renormalizer,
+            persister,
+            process,
+            new ModelSizeStats.Builder(JOB_ID).setTimestamp(new Date(BUCKET_SPAN_MS)).build(),
+            new TimingStats(JOB_ID),
+            flushListener);
     }
 
     @After
     public void cleanup() {
+        verifyNoMoreInteractions(auditor, renormalizer, persister);
         executor.shutdown();
     }
 
     public void testProcess() throws TimeoutException {
-        JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class);
-        when(persister.bulkPersisterBuilder(JOB_ID)).thenReturn(bulkBuilder);
         AutodetectResult autodetectResult = mock(AutodetectResult.class);
-        @SuppressWarnings("unchecked")
-        Iterator<AutodetectResult> iterator = mock(Iterator.class);
-        when(iterator.hasNext()).thenReturn(true).thenReturn(false);
-        when(iterator.next()).thenReturn(autodetectResult);
-        AutodetectProcess process = mock(AutodetectProcess.class);
-        when(process.readAutodetectResults()).thenReturn(iterator);
-        processorUnderTest.process(process);
+        when(process.readAutodetectResults()).thenReturn(Arrays.asList(autodetectResult).iterator());
+
+        processorUnderTest.process();
         processorUnderTest.awaitCompletion();
-        verify(renormalizer, times(1)).waitUntilIdle();
-        assertEquals(0, processorUnderTest.completionLatch.getCount());
+        assertThat(processorUnderTest.completionLatch.getCount(), is(equalTo(0L)));
+
+        verify(renormalizer).waitUntilIdle();
+        verify(persister).bulkPersisterBuilder(JOB_ID);
+        verify(persister).commitResultWrites(JOB_ID);
+        verify(persister).commitStateWrites(JOB_ID);
     }
 
     public void testProcessResult_bucket() {
-        JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class);
-        when(persister.bulkPersisterBuilder(JOB_ID)).thenReturn(bulkBuilder);
+        when(bulkBuilder.persistTimingStats(any(TimingStats.class))).thenReturn(bulkBuilder);
         when(bulkBuilder.persistBucket(any(Bucket.class))).thenReturn(bulkBuilder);
-
-        AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context(JOB_ID, bulkBuilder);
-        context.deleteInterimRequired = false;
         AutodetectResult result = mock(AutodetectResult.class);
         Bucket bucket = mock(Bucket.class);
         when(result.getBucket()).thenReturn(bucket);
-        processorUnderTest.processResult(context, result);
 
-        verify(bulkBuilder, times(1)).persistBucket(bucket);
-        verify(bulkBuilder, times(1)).executeRequest();
+        processorUnderTest.setDeleteInterimRequired(false);
+        processorUnderTest.processResult(result);
+
+        verify(bulkBuilder).persistTimingStats(any(TimingStats.class));
+        verify(bulkBuilder).persistBucket(bucket);
+        verify(bulkBuilder).executeRequest();
+        verify(persister).bulkPersisterBuilder(JOB_ID);
         verify(persister, never()).deleteInterimResults(JOB_ID);
-        verifyNoMoreInteractions(persister);
     }
 
     public void testProcessResult_bucket_deleteInterimRequired() {
-        JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class);
-        when(persister.bulkPersisterBuilder(JOB_ID)).thenReturn(bulkBuilder);
+        when(bulkBuilder.persistTimingStats(any(TimingStats.class))).thenReturn(bulkBuilder);
         when(bulkBuilder.persistBucket(any(Bucket.class))).thenReturn(bulkBuilder);
-
-        AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context(JOB_ID, bulkBuilder);
-        context.deleteInterimRequired = true;
         AutodetectResult result = mock(AutodetectResult.class);
         Bucket bucket = mock(Bucket.class);
         when(result.getBucket()).thenReturn(bucket);
-        processorUnderTest.processResult(context, result);
 
-        verify(bulkBuilder, times(1)).persistBucket(bucket);
-        verify(bulkBuilder, times(1)).executeRequest();
-        verify(persister, times(1)).deleteInterimResults(JOB_ID);
-        verifyNoMoreInteractions(persister);
-        assertFalse(context.deleteInterimRequired);
+        processorUnderTest.processResult(result);
+        assertFalse(processorUnderTest.isDeleteInterimRequired());
+
+        verify(bulkBuilder).persistTimingStats(any(TimingStats.class));
+        verify(bulkBuilder).persistBucket(bucket);
+        verify(bulkBuilder).executeRequest();
+        verify(persister).bulkPersisterBuilder(JOB_ID);
+        verify(persister).deleteInterimResults(JOB_ID);
     }
 
     public void testProcessResult_records() {
-        JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class);
-        when(persister.bulkPersisterBuilder(JOB_ID)).thenReturn(bulkBuilder);
-
-        AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context("foo", bulkBuilder);
-        context.deleteInterimRequired = false;
         AutodetectResult result = mock(AutodetectResult.class);
-        AnomalyRecord record1 = new AnomalyRecord("foo", new Date(123), 123);
-        AnomalyRecord record2 = new AnomalyRecord("foo", new Date(123), 123);
-        List<AnomalyRecord> records = Arrays.asList(record1, record2);
+        List<AnomalyRecord> records =
+            Arrays.asList(
+                new AnomalyRecord(JOB_ID, new Date(123), 123),
+                new AnomalyRecord(JOB_ID, new Date(123), 123));
         when(result.getRecords()).thenReturn(records);
-        processorUnderTest.processResult(context, result);
 
-        verify(bulkBuilder, times(1)).persistRecords(records);
+        processorUnderTest.setDeleteInterimRequired(false);
+        processorUnderTest.processResult(result);
+
+        verify(bulkBuilder).persistRecords(records);
         verify(bulkBuilder, never()).executeRequest();
-        verifyNoMoreInteractions(persister);
+        verify(persister).bulkPersisterBuilder(JOB_ID);
     }
 
-
     public void testProcessResult_influencers() {
-        JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class);
-        when(persister.bulkPersisterBuilder(JOB_ID)).thenReturn(bulkBuilder);
-
-        AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context(JOB_ID, bulkBuilder);
-        context.deleteInterimRequired = false;
         AutodetectResult result = mock(AutodetectResult.class);
-        Influencer influencer1 = new Influencer(JOB_ID, "infField", "infValue", new Date(123), 123);
-        Influencer influencer2 = new Influencer(JOB_ID, "infField2", "infValue2", new Date(123), 123);
-        List<Influencer> influencers = Arrays.asList(influencer1, influencer2);
+        List<Influencer> influencers =
+            Arrays.asList(
+                new Influencer(JOB_ID, "infField", "infValue", new Date(123), 123),
+                new Influencer(JOB_ID, "infField2", "infValue2", new Date(123), 123));
         when(result.getInfluencers()).thenReturn(influencers);
-        processorUnderTest.processResult(context, result);
 
-        verify(bulkBuilder, times(1)).persistInfluencers(influencers);
+        processorUnderTest.setDeleteInterimRequired(false);
+        processorUnderTest.processResult(result);
+
+        verify(bulkBuilder).persistInfluencers(influencers);
         verify(bulkBuilder, never()).executeRequest();
-        verifyNoMoreInteractions(persister);
+        verify(persister).bulkPersisterBuilder(JOB_ID);
     }
 
     public void testProcessResult_categoryDefinition() {
-        JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class);
-        when(persister.bulkPersisterBuilder(JOB_ID)).thenReturn(bulkBuilder);
-
-        AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context(JOB_ID, bulkBuilder);
-        context.deleteInterimRequired = false;
         AutodetectResult result = mock(AutodetectResult.class);
         CategoryDefinition categoryDefinition = mock(CategoryDefinition.class);
         when(result.getCategoryDefinition()).thenReturn(categoryDefinition);
-        processorUnderTest.processResult(context, result);
+
+        processorUnderTest.setDeleteInterimRequired(false);
+        processorUnderTest.processResult(result);
 
         verify(bulkBuilder, never()).executeRequest();
-        verify(persister, times(1)).persistCategoryDefinition(categoryDefinition);
-        verifyNoMoreInteractions(persister);
+        verify(persister).persistCategoryDefinition(categoryDefinition);
+        verify(persister).bulkPersisterBuilder(JOB_ID);
     }
 
     public void testProcessResult_flushAcknowledgement() {
-        JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class);
-        when(persister.bulkPersisterBuilder(JOB_ID)).thenReturn(bulkBuilder);
-
-        AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context(JOB_ID, bulkBuilder);
-        context.deleteInterimRequired = false;
         AutodetectResult result = mock(AutodetectResult.class);
         FlushAcknowledgement flushAcknowledgement = mock(FlushAcknowledgement.class);
         when(flushAcknowledgement.getId()).thenReturn(JOB_ID);
         when(result.getFlushAcknowledgement()).thenReturn(flushAcknowledgement);
-        processorUnderTest.processResult(context, result);
 
-        verify(flushListener, times(1)).acknowledgeFlush(flushAcknowledgement);
-        verify(persister, times(1)).commitResultWrites(JOB_ID);
-        verify(bulkBuilder, times(1)).executeRequest();
-        verifyNoMoreInteractions(persister);
-        assertTrue(context.deleteInterimRequired);
+        processorUnderTest.setDeleteInterimRequired(false);
+        processorUnderTest.processResult(result);
+        assertTrue(processorUnderTest.isDeleteInterimRequired());
+
+        verify(persister).bulkPersisterBuilder(JOB_ID);
+        verify(flushListener).acknowledgeFlush(flushAcknowledgement);
+        verify(persister).commitResultWrites(JOB_ID);
+        verify(bulkBuilder).executeRequest();
     }
 
     public void testProcessResult_flushAcknowledgementMustBeProcessedLast() {
-        JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class);
-
-        AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context(JOB_ID, bulkBuilder);
-        context.deleteInterimRequired = false;
         AutodetectResult result = mock(AutodetectResult.class);
         FlushAcknowledgement flushAcknowledgement = mock(FlushAcknowledgement.class);
         when(flushAcknowledgement.getId()).thenReturn(JOB_ID);
@@ -239,64 +233,61 @@ public void testProcessResult_flushAcknowledgementMustBeProcessedLast() {
         CategoryDefinition categoryDefinition = mock(CategoryDefinition.class);
         when(result.getCategoryDefinition()).thenReturn(categoryDefinition);
 
+        processorUnderTest.setDeleteInterimRequired(false);
+        processorUnderTest.processResult(result);
+        assertTrue(processorUnderTest.isDeleteInterimRequired());
+
         InOrder inOrder = inOrder(persister, bulkBuilder, flushListener);
-        processorUnderTest.processResult(context, result);
-
-        inOrder.verify(persister, times(1)).persistCategoryDefinition(categoryDefinition);
-        inOrder.verify(bulkBuilder, times(1)).executeRequest();
-        inOrder.verify(persister, times(1)).commitResultWrites(JOB_ID);
-        inOrder.verify(flushListener, times(1)).acknowledgeFlush(flushAcknowledgement);
-        verifyNoMoreInteractions(persister);
-        assertTrue(context.deleteInterimRequired);
+        inOrder.verify(persister).bulkPersisterBuilder(JOB_ID);
+        inOrder.verify(persister).persistCategoryDefinition(categoryDefinition);
+        inOrder.verify(bulkBuilder).executeRequest();
+        inOrder.verify(persister).commitResultWrites(JOB_ID);
+        inOrder.verify(flushListener).acknowledgeFlush(flushAcknowledgement);
     }
 
     public void testProcessResult_modelPlot() {
-        JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class);
-
-        AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context(JOB_ID, bulkBuilder);
-        context.deleteInterimRequired = false;
         AutodetectResult result = mock(AutodetectResult.class);
         ModelPlot modelPlot = mock(ModelPlot.class);
         when(result.getModelPlot()).thenReturn(modelPlot);
-        processorUnderTest.processResult(context, result);
 
-        verify(bulkBuilder, times(1)).persistModelPlot(modelPlot);
-        verifyNoMoreInteractions(persister);
+        processorUnderTest.setDeleteInterimRequired(false);
+        processorUnderTest.processResult(result);
+
+        verify(persister).bulkPersisterBuilder(JOB_ID);
+        verify(bulkBuilder).persistModelPlot(modelPlot);
     }
 
     public void testProcessResult_modelSizeStats() {
-        JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class);
-
-        AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context(JOB_ID, bulkBuilder);
-        context.deleteInterimRequired = false;
         AutodetectResult result = mock(AutodetectResult.class);
         ModelSizeStats modelSizeStats = mock(ModelSizeStats.class);
         when(result.getModelSizeStats()).thenReturn(modelSizeStats);
-        processorUnderTest.processResult(context, result);
 
-        verify(persister, times(1)).persistModelSizeStats(modelSizeStats);
-        verifyNoMoreInteractions(persister);
-        assertEquals(modelSizeStats, processorUnderTest.modelSizeStats());
+        processorUnderTest.setDeleteInterimRequired(false);
+        processorUnderTest.processResult(result);
+        assertThat(processorUnderTest.modelSizeStats(), is(equalTo(modelSizeStats)));
+
+        verify(persister).bulkPersisterBuilder(JOB_ID);
+        verify(persister).persistModelSizeStats(modelSizeStats);
     }
 
     public void testProcessResult_modelSizeStatsWithMemoryStatusChanges() {
-        JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class);
-
-        setupScheduleDelayTime(TimeValue.timeValueSeconds(5));
+        TimeValue delay = TimeValue.timeValueSeconds(5);
+        // Set up schedule delay time
+        when(threadPool.schedule(any(Runnable.class), any(TimeValue.class), anyString()))
+            .thenAnswer(i -> executor.schedule((Runnable) i.getArguments()[0], delay.nanos(), TimeUnit.NANOSECONDS));
 
-        AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context(JOB_ID, bulkBuilder);
-        context.deleteInterimRequired = false;
         AutodetectResult result = mock(AutodetectResult.class);
+        processorUnderTest.setDeleteInterimRequired(false);
 
         // First one with soft_limit
         ModelSizeStats modelSizeStats = new ModelSizeStats.Builder(JOB_ID).setMemoryStatus(ModelSizeStats.MemoryStatus.SOFT_LIMIT).build();
         when(result.getModelSizeStats()).thenReturn(modelSizeStats);
-        processorUnderTest.processResult(context, result);
+        processorUnderTest.processResult(result);
 
         // Another with soft_limit
         modelSizeStats = new ModelSizeStats.Builder(JOB_ID).setMemoryStatus(ModelSizeStats.MemoryStatus.SOFT_LIMIT).build();
         when(result.getModelSizeStats()).thenReturn(modelSizeStats);
-        processorUnderTest.processResult(context, result);
+        processorUnderTest.processResult(result);
 
         // Now with hard_limit
         modelSizeStats = new ModelSizeStats.Builder(JOB_ID)
@@ -305,115 +296,104 @@ public void testProcessResult_modelSizeStatsWithMemoryStatusChanges() {
                 .setModelBytesExceeded(new ByteSizeValue(1, ByteSizeUnit.KB).getBytes())
                 .build();
         when(result.getModelSizeStats()).thenReturn(modelSizeStats);
-        processorUnderTest.processResult(context, result);
+        processorUnderTest.processResult(result);
 
         // And another with hard_limit
         modelSizeStats = new ModelSizeStats.Builder(JOB_ID).setMemoryStatus(ModelSizeStats.MemoryStatus.HARD_LIMIT).build();
         when(result.getModelSizeStats()).thenReturn(modelSizeStats);
-        processorUnderTest.processResult(context, result);
+        processorUnderTest.processResult(result);
 
+        verify(persister).bulkPersisterBuilder(JOB_ID);
+        verify(persister, times(4)).persistModelSizeStats(any(ModelSizeStats.class));
         // We should have only fired two notifications: one for soft_limit and one for hard_limit
         verify(auditor).warning(JOB_ID, Messages.getMessage(Messages.JOB_AUDIT_MEMORY_STATUS_SOFT_LIMIT));
         verify(auditor).error(JOB_ID, Messages.getMessage(Messages.JOB_AUDIT_MEMORY_STATUS_HARD_LIMIT, "512mb", "1kb"));
-        verifyNoMoreInteractions(auditor);
     }
 
     public void testProcessResult_modelSnapshot() {
-        JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class);
-
-        AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context(JOB_ID, bulkBuilder);
-        context.deleteInterimRequired = false;
         AutodetectResult result = mock(AutodetectResult.class);
         ModelSnapshot modelSnapshot = new ModelSnapshot.Builder(JOB_ID)
-                .setSnapshotId("a_snapshot_id")
-                .setMinVersion(Version.CURRENT)
-                .build();
+            .setSnapshotId("a_snapshot_id")
+            .setMinVersion(Version.CURRENT)
+            .build();
         when(result.getModelSnapshot()).thenReturn(modelSnapshot);
-        processorUnderTest.processResult(context, result);
 
-        verify(persister, times(1)).persistModelSnapshot(modelSnapshot, WriteRequest.RefreshPolicy.IMMEDIATE);
+        when(persister.persistModelSnapshot(any(), any()))
+            .thenReturn(new IndexResponse(new ShardId("ml", "uid", 0), "doc", "1", 0L, 0L, 0L, true));
+
+        processorUnderTest.setDeleteInterimRequired(false);
+        processorUnderTest.processResult(result);
+
+        verify(persister).bulkPersisterBuilder(JOB_ID);
+        verify(persister).persistModelSnapshot(modelSnapshot, WriteRequest.RefreshPolicy.IMMEDIATE);
 
         UpdateJobAction.Request expectedJobUpdateRequest = UpdateJobAction.Request.internal(JOB_ID,
                 new JobUpdate.Builder(JOB_ID).setModelSnapshotId("a_snapshot_id").build());
 
         verify(client).execute(same(UpdateJobAction.INSTANCE), eq(expectedJobUpdateRequest), any());
-        verifyNoMoreInteractions(persister);
     }
 
     public void testProcessResult_quantiles_givenRenormalizationIsEnabled() {
-        JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class);
-
-        AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context(JOB_ID, bulkBuilder);
-        context.deleteInterimRequired = false;
         AutodetectResult result = mock(AutodetectResult.class);
         Quantiles quantiles = mock(Quantiles.class);
         when(result.getQuantiles()).thenReturn(quantiles);
         when(renormalizer.isEnabled()).thenReturn(true);
-        processorUnderTest.processResult(context, result);
 
-        verify(persister, times(1)).persistQuantiles(quantiles);
+        processorUnderTest.setDeleteInterimRequired(false);
+        processorUnderTest.processResult(result);
+
+        verify(persister).bulkPersisterBuilder(JOB_ID);
+        verify(persister).persistQuantiles(quantiles);
         verify(bulkBuilder).executeRequest();
         verify(persister).commitResultWrites(JOB_ID);
-        verify(renormalizer, times(1)).isEnabled();
-        verify(renormalizer, times(1)).renormalize(quantiles);
-        verifyNoMoreInteractions(persister);
-        verifyNoMoreInteractions(renormalizer);
+        verify(renormalizer).isEnabled();
+        verify(renormalizer).renormalize(quantiles);
     }
 
     public void testProcessResult_quantiles_givenRenormalizationIsDisabled() {
-        JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class);
-
-        AutoDetectResultProcessor.Context context = new AutoDetectResultProcessor.Context(JOB_ID, bulkBuilder);
-        context.deleteInterimRequired = false;
         AutodetectResult result = mock(AutodetectResult.class);
         Quantiles quantiles = mock(Quantiles.class);
         when(result.getQuantiles()).thenReturn(quantiles);
         when(renormalizer.isEnabled()).thenReturn(false);
-        processorUnderTest.processResult(context, result);
 
-        verify(persister, times(1)).persistQuantiles(quantiles);
+        processorUnderTest.setDeleteInterimRequired(false);
+        processorUnderTest.processResult(result);
+
+        verify(persister).bulkPersisterBuilder(JOB_ID);
+        verify(persister).persistQuantiles(quantiles);
         verify(bulkBuilder).executeRequest();
-        verify(renormalizer, times(1)).isEnabled();
-        verifyNoMoreInteractions(persister);
-        verifyNoMoreInteractions(renormalizer);
+        verify(renormalizer).isEnabled();
     }
 
     public void testAwaitCompletion() throws TimeoutException {
-        JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class);
-        when(persister.bulkPersisterBuilder(JOB_ID)).thenReturn(bulkBuilder);
         AutodetectResult autodetectResult = mock(AutodetectResult.class);
-        @SuppressWarnings("unchecked")
-        Iterator<AutodetectResult> iterator = mock(Iterator.class);
-        when(iterator.hasNext()).thenReturn(true).thenReturn(false);
-        when(iterator.next()).thenReturn(autodetectResult);
-        AutodetectProcess process = mock(AutodetectProcess.class);
-        when(process.readAutodetectResults()).thenReturn(iterator);
-        processorUnderTest.process(process);
+        when(process.readAutodetectResults()).thenReturn(Arrays.asList(autodetectResult).iterator());
 
+        processorUnderTest.process();
         processorUnderTest.awaitCompletion();
-        assertEquals(0, processorUnderTest.completionLatch.getCount());
-        assertEquals(1, processorUnderTest.updateModelSnapshotSemaphore.availablePermits());
+        assertThat(processorUnderTest.completionLatch.getCount(), is(equalTo(0L)));
+        assertThat(processorUnderTest.updateModelSnapshotSemaphore.availablePermits(), is(equalTo(1)));
+
+        verify(persister).bulkPersisterBuilder(JOB_ID);
+        verify(persister).commitResultWrites(JOB_ID);
+        verify(persister).commitStateWrites(JOB_ID);
+        verify(renormalizer).waitUntilIdle();
     }
 
     public void testPersisterThrowingDoesntBlockProcessing() {
-        JobResultsPersister.Builder bulkBuilder = mock(JobResultsPersister.Builder.class);
-        when(persister.bulkPersisterBuilder(JOB_ID)).thenReturn(bulkBuilder);
         AutodetectResult autodetectResult = mock(AutodetectResult.class);
         ModelSnapshot modelSnapshot = mock(ModelSnapshot.class);
         when(autodetectResult.getModelSnapshot()).thenReturn(modelSnapshot);
 
-        @SuppressWarnings("unchecked")
-        Iterator<AutodetectResult> iterator = mock(Iterator.class);
-        when(iterator.hasNext()).thenReturn(true).thenReturn(true).thenReturn(false);
-        when(iterator.next()).thenReturn(autodetectResult);
-        AutodetectProcess process = mock(AutodetectProcess.class);
         when(process.isProcessAlive()).thenReturn(true);
         when(process.isProcessAliveAfterWaiting()).thenReturn(true);
-        when(process.readAutodetectResults()).thenReturn(iterator);
+        when(process.readAutodetectResults()).thenReturn(Arrays.asList(autodetectResult, autodetectResult).iterator());
 
         doThrow(new ElasticsearchException("this test throws")).when(persister).persistModelSnapshot(any(), any());
 
-        processorUnderTest.process(process);
+        processorUnderTest.process();
+
+        verify(persister).bulkPersisterBuilder(JOB_ID);
         verify(persister, times(2)).persistModelSnapshot(any(), eq(WriteRequest.RefreshPolicy.IMMEDIATE));
     }
 
@@ -421,44 +401,36 @@ public void testParsingErrorSetsFailed() throws InterruptedException {
         @SuppressWarnings("unchecked")
         Iterator<AutodetectResult> iterator = mock(Iterator.class);
         when(iterator.hasNext()).thenThrow(new ElasticsearchParseException("this test throws"));
-        AutodetectProcess process = mock(AutodetectProcess.class);
         when(process.readAutodetectResults()).thenReturn(iterator);
 
         assertFalse(processorUnderTest.isFailed());
-        processorUnderTest.process(process);
+        processorUnderTest.process();
         assertTrue(processorUnderTest.isFailed());
 
         // Wait for flush should return immediately
-        FlushAcknowledgement flushAcknowledgement = processorUnderTest.waitForFlushAcknowledgement(
-                "foo", Duration.of(300, ChronoUnit.SECONDS));
+        FlushAcknowledgement flushAcknowledgement =
+            processorUnderTest.waitForFlushAcknowledgement(JOB_ID, Duration.of(300, ChronoUnit.SECONDS));
         assertThat(flushAcknowledgement, is(nullValue()));
+
+        verify(persister).bulkPersisterBuilder(JOB_ID);
     }
 
     public void testKill() throws TimeoutException {
         AutodetectResult autodetectResult = mock(AutodetectResult.class);
-        @SuppressWarnings("unchecked")
-        Iterator<AutodetectResult> iterator = mock(Iterator.class);
-        when(iterator.hasNext()).thenReturn(true).thenReturn(false);
-        when(iterator.next()).thenReturn(autodetectResult);
-        AutodetectProcess process = mock(AutodetectProcess.class);
-        when(process.readAutodetectResults()).thenReturn(iterator);
-        processorUnderTest.setProcessKilled();
-        processorUnderTest.process(process);
+        when(process.readAutodetectResults()).thenReturn(Arrays.asList(autodetectResult).iterator());
 
+        processorUnderTest.setProcessKilled();
+        processorUnderTest.process();
         processorUnderTest.awaitCompletion();
-        assertEquals(0, processorUnderTest.completionLatch.getCount());
-        assertEquals(1, processorUnderTest.updateModelSnapshotSemaphore.availablePermits());
+        assertThat(processorUnderTest.completionLatch.getCount(), is(equalTo(0L)));
+        assertThat(processorUnderTest.updateModelSnapshotSemaphore.availablePermits(), is(equalTo(1)));
 
-        verify(persister, times(1)).commitResultWrites(JOB_ID);
-        verify(persister, times(1)).commitStateWrites(JOB_ID);
+        verify(persister).bulkPersisterBuilder(JOB_ID);
+        verify(persister).commitResultWrites(JOB_ID);
+        verify(persister).commitStateWrites(JOB_ID);
         verify(renormalizer, never()).renormalize(any());
         verify(renormalizer).shutdown();
-        verify(renormalizer, times(1)).waitUntilIdle();
-        verify(flushListener, times(1)).clear();
-    }
-
-    private void setupScheduleDelayTime(TimeValue delay) {
-        when(threadPool.schedule(any(Runnable.class), any(TimeValue.class), anyString()))
-            .thenAnswer(i -> executor.schedule((Runnable) i.getArguments()[0], delay.nanos(), TimeUnit.NANOSECONDS));
+        verify(renormalizer).waitUntilIdle();
+        verify(flushListener).clear();
     }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParserTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParserTests.java
deleted file mode 100644
index 1118453154ed8..0000000000000
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParserTests.java
+++ /dev/null
@@ -1,422 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.ml.job.process.autodetect.output;
-
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.xcontent.XContentParseException;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles;
-import org.elasticsearch.xpack.core.ml.job.results.Bucket;
-import org.elasticsearch.xpack.core.ml.job.results.BucketInfluencer;
-import org.elasticsearch.xpack.ml.job.results.AutodetectResult;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Tests for parsing the JSON output of autodetect
- */
-public class AutodetectResultsParserTests extends ESTestCase {
-    private static final double EPSILON = 0.000001;
-
-    private static final String METRIC_OUTPUT_SAMPLE = "[{\"bucket\": {\"job_id\":\"foo\",\"timestamp\":1359450000000,"
-            + "\"bucket_span\":22, \"records\":[],"
-            + "\"anomaly_score\":0,\"event_count\":806,\"bucket_influencers\":["
-            + "{\"timestamp\":1359450000000,\"bucket_span\":22,\"job_id\":\"foo\",\"anomaly_score\":0,"
-            + "\"probability\":0.0, \"influencer_field_name\":\"bucket_time\","
-            + "\"initial_anomaly_score\":0.0}]}},{\"quantiles\": {\"job_id\":\"foo\", \"quantile_state\":\"[normalizer 1.1, normalizer 2" +
-            ".1]\",\"timestamp\":1359450000000}}"
-            + ",{\"bucket\": {\"job_id\":\"foo\",\"timestamp\":1359453600000,\"bucket_span\":22,\"records\":"
-            + "[{\"timestamp\":1359453600000,\"bucket_span\":22,\"job_id\":\"foo\",\"probability\":0.0637541,"
-            + "\"by_field_name\":\"airline\",\"by_field_value\":\"JZA\", \"typical\":[1020.08],\"actual\":[1042.14],"
-            + "\"field_name\":\"responsetime\",\"function\":\"max\",\"partition_field_name\":\"\",\"partition_field_value\":\"\"},"
-            + "{\"timestamp\":1359453600000,\"bucket_span\":22,\"job_id\":\"foo\",\"probability\":0.00748292,"
-            + "\"by_field_name\":\"airline\",\"by_field_value\":\"AMX\", "
-            + "\"typical\":[20.2137],\"actual\":[22.8855],\"field_name\":\"responsetime\",\"function\":\"max\","
-            + "\"partition_field_name\":\"\",\"partition_field_value\":\"\"},{\"timestamp\":1359453600000,\"bucket_span\":22,"
-            + "\"job_id\":\"foo\",\"probability\":0.023494,\"by_field_name\":\"airline\","
-            + "\"by_field_value\":\"DAL\", \"typical\":[382.177],\"actual\":[358.934],\"field_name\":\"responsetime\",\"function\":\"min\","
-            + "\"partition_field_name\":\"\", \"partition_field_value\":\"\"},{\"timestamp\":1359453600000,\"bucket_span\":22,"
-            + "\"job_id\":\"foo\","
-            + "\"probability\":0.0473552,\"by_field_name\":\"airline\",\"by_field_value\":\"SWA\", \"typical\":[152.148],"
-            + "\"actual\":[96.6425],\"field_name\":\"responsetime\",\"function\":\"min\",\"partition_field_name\":\"\","
-            + "\"partition_field_value\":\"\"}],"
-            + "\"initial_anomaly_score\":0.0140005, \"anomaly_score\":20.22688,"
-            + "\"event_count\":820,\"bucket_influencers\":[{\"timestamp\":1359453600000,\"bucket_span\":22,"
-            + "\"job_id\":\"foo\", \"raw_anomaly_score\":0.0140005, \"probability\":0.01,\"influencer_field_name\":\"bucket_time\","
-            + "\"initial_anomaly_score\":20.22688,\"anomaly_score\":20.22688} ,{\"timestamp\":1359453600000,\"bucket_span\":22,"
-            + "\"job_id\":\"foo\",\"raw_anomaly_score\":0.005, \"probability\":0.03,"
-            + "\"influencer_field_name\":\"foo\",\"initial_anomaly_score\":10.5,\"anomaly_score\":10.5}]}},{\"quantiles\": "
-            + "{\"job_id\":\"foo\",\"timestamp\":1359453600000,"
-            + "\"quantile_state\":\"[normalizer 1.2, normalizer 2.2]\"}} ,{\"flush\": {\"id\":\"testing1\"}} ,"
-            + "{\"quantiles\": {\"job_id\":\"foo\",\"timestamp\":1359453600000,\"quantile_state\":\"[normalizer 1.3, normalizer 2.3]\"}} ]";
-
-    private static final String POPULATION_OUTPUT_SAMPLE = "[{\"timestamp\":1379590200,\"records\":[{\"probability\":1.38951e-08,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"mail.google.com\","
-            + "\"function\":\"max\","
-            + "\"causes\":[{\"probability\":1.38951e-08,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"mail.google.com\",\"function\":\"max\",\"typical\":[101534],\"actual\":[9.19027e+07]}],"
-            + "\"record_score\":100,\"anomaly_score\":44.7324},{\"probability\":3.86587e-07,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"armmf.adobe.com\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":3.86587e-07,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"armmf.adobe.com\",\"function\":\"max\",\"typical\":[101534],\"actual\":[3.20093e+07]}],"
-            + "\"record_score\":89.5834,\"anomaly_score\":44.7324},{\"probability\":0.00500083,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"0.docs.google.com\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":0.00500083,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"0.docs.google.com\",\"function\":\"max\",\"typical\":[101534],\"actual\":[6.61812e+06]}],"
-            + "\"record_score\":1.19856,\"anomaly_score\":44.7324},{\"probability\":0.0152333,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"emea.salesforce.com\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":0.0152333,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"emea.salesforce.com\",\"function\":\"max\",\"typical\":[101534],\"actual\":[5.36373e+06]}],"
-            + "\"record_score\":0.303996,\"anomaly_score\":44.7324}],\"raw_anomaly_score\":1.30397,\"anomaly_score\":44.7324,"
-            + "\"event_count\":1235}" + ",{\"flush\":\"testing2\"}"
-            + ",{\"timestamp\":1379590800,\"records\":[{\"probability\":1.9008e-08,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"mail.google.com\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":1.9008e-08,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"mail.google.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[1.1498e+08]}],"
-            + "\"record_score\":93.6213,\"anomaly_score\":1.19192},{\"probability\":1.01013e-06,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"armmf.adobe.com\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":1.01013e-06,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"armmf.adobe.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[3.25808e+07]}],"
-            + "\"record_score\":86.5825,\"anomaly_score\":1.19192},{\"probability\":0.000386185,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"0.docs.google.com\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":0.000386185,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"0.docs.google.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[3.22855e+06]}],"
-            + "\"record_score\":17.1179,\"anomaly_score\":1.19192},{\"probability\":0.00208033,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"docs.google.com\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":0.00208033,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"docs.google.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[1.43328e+06]}],"
-            + "\"record_score\":3.0692,\"anomaly_score\":1.19192},{\"probability\":0.00312988,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"booking2.airasia.com\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":0.00312988,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"booking2.airasia.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[1.15764e+06]}],"
-            + "\"record_score\":1.99532,\"anomaly_score\":1.19192},{\"probability\":0.00379229,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.facebook.com\",\"function\":\"max\",\"causes\":["
-            + "{\"probability\":0.00379229,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"www.facebook.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[1.0443e+06]}],"
-            + "\"record_score\":1.62352,\"anomaly_score\":1.19192},{\"probability\":0.00623576,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.airasia.com\",\"function\":\"max\",\"causes\":["
-            + "{\"probability\":0.00623576,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"www.airasia.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[792699]}],"
-            + "\"record_score\":0.935134,\"anomaly_score\":1.19192},{\"probability\":0.00665308,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.google.com\",\"function\":\"max\",\"causes\":["
-            + "{\"probability\":0.00665308,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"www.google.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[763985]}],"
-            + "\"record_score\":0.868119,\"anomaly_score\":1.19192},{\"probability\":0.00709315,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"0.drive.google.com\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":0.00709315,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"0.drive.google.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[736442]}],"
-            + "\"record_score\":0.805994,\"anomaly_score\":1.19192},{\"probability\":0.00755789,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"resources2.news.com.au\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":0.00755789,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"resources2.news.com.au\",\"function\":\"max\",\"typical\":[31356],\"actual\":[709962]}],"
-            + "\"record_score\":0.748239,\"anomaly_score\":1.19192},{\"probability\":0.00834974,\"field_name\":"
-            + "\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.calypso.net.au\",\"function\":\"max\","
-            + "\"causes\":[{\"probability\":0.00834974,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"www.calypso.net.au\",\"function\":\"max\",\"typical\":[31356],\"actual\":[669968]}],"
-            + "\"record_score\":0.664644,\"anomaly_score\":1.19192},{\"probability\":0.0107711,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"ad.yieldmanager.com\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":0.0107711,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"ad.yieldmanager.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[576067]}],"
-            + "\"record_score\":0.485277,\"anomaly_score\":1.19192},{\"probability\":0.0123367,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.google-analytics.com\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":0.0123367,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"www.google-analytics.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[530594]}],"
-            + "\"record_score\":0.406783,\"anomaly_score\":1.19192},{\"probability\":0.0125647,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"bs.serving-sys.com\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":0.0125647,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"bs.serving-sys.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[524690]}],"
-            + "\"record_score\":0.396986,\"anomaly_score\":1.19192},{\"probability\":0.0141652,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.google.com.au\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":0.0141652,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"www.google.com.au\",\"function\":\"max\",\"typical\":[31356],\"actual\":[487328]}],"
-            + "\"record_score\":0.337075,\"anomaly_score\":1.19192},{\"probability\":0.0141742,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"resources1.news.com.au\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":0.0141742,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"resources1.news.com.au\",\"function\":\"max\",\"typical\":[31356],\"actual\":[487136]}],"
-            + "\"record_score\":0.336776,\"anomaly_score\":1.19192},{\"probability\":0.0145263,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"b.mail.google.com\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":0.0145263,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"b.mail.google.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[479766]}],"
-            + "\"record_score\":0.325385,\"anomaly_score\":1.19192},{\"probability\":0.0151447,\"field_name\":\"sum_cs_bytes_\","
-            + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.rei.com\",\"function\":\"max\",\"causes\":[{"
-            + "\"probability\":0.0151447,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.rei" +
-            ".com\","
-            + "\"function\":\"max\",\"typical\":[31356],\"actual\":[467450]}],\"record_score\":0.306657,\"anomaly_score\":1" +
-            ".19192},"
-            + "{\"probability\":0.0164073,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"s3.amazonaws.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0164073,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"s3.amazonaws.com\","
-            + "\"function\":\"max\",\"typical\":[31356],\"actual\":[444511]}],\"record_score\":0.272805,\"anomaly_score\":1" +
-            ".19192},"
-            + "{\"probability\":0.0201927,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"0-p-06-ash2.channel.facebook.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0201927,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"0-p-06-ash2.channel.facebook.com\","
-            + "\"function\":\"max\",\"typical\":[31356],\"actual\":[389243]}],\"record_score\":0.196685,\"anomaly_score\":1" +
-            ".19192},"
-            + "{\"probability\":0.0218721,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"booking.airasia.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0218721,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"booking.airasia.com\","
-            + "\"function\":\"max\",\"typical\":[31356],\"actual\":[369509]}],\"record_score\":0.171353,"
-            + "\"anomaly_score\":1.19192},{\"probability\":0.0242411,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"www.yammer.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0242411,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.yammer.com\"," +
-            "\"function\":\"max\","
-            + "\"typical\":[31356],\"actual\":[345295]}],\"record_score\":0.141585,\"anomaly_score\":1.19192},"
-            + "{\"probability\":0.0258232,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"safebrowsing-cache.google.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0258232,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"safebrowsing-cache.google.com\","
-            + "\"function\":\"max\",\"typical\":[31356],\"actual\":[331051]}],\"record_score\":0.124748,\"anomaly_score\":1" +
-            ".19192},"
-            + "{\"probability\":0.0259695,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"fbcdn-profile-a.akamaihd.net\",\"function\":\"max\",\"causes\":[{\"probability\":0.0259695,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"fbcdn-profile-a.akamaihd.net\","
-            + "\"function\":\"max\",\"typical\":[31356],\"actual\":[329801]}],\"record_score\":0.123294,\"anomaly_score\":1" +
-            ".19192},"
-            + "{\"probability\":0.0268874,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"www.oag.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0268874,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.oag.com\","
-            + "\"function\":\"max\",\"typical\":[31356],\"actual\":[322200]}],\"record_score\":0.114537,"
-            + "\"anomaly_score\":1.19192},{\"probability\":0.0279146,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"booking.qatarairways.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0279146,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"booking.qatarairways.com\","
-            + "\"function\":\"max\",\"typical\":[31356],\"actual\":[314153]}],\"record_score\":0.105419,\"anomaly_score\":1" +
-            ".19192},"
-            + "{\"probability\":0.0309351,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"resources3.news.com.au\",\"function\":\"max\",\"causes\":[{\"probability\":0.0309351,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"resources3.news.com.au\","
-            + "\"function\":\"max\",\"typical\":[31356],\"actual\":[292918]}],\"record_score\":0.0821156,\"anomaly_score\":1" +
-            ".19192}"
-            + ",{\"probability\":0.0335204,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"resources0.news.com.au\",\"function\":\"max\",\"causes\":[{\"probability\":0.0335204,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"resources0.news.com.au\","
-            + "\"function\":\"max\",\"typical\":[31356],\"actual\":[277136]}],\"record_score\":0.0655063,\"anomaly_score\":1" +
-            ".19192}"
-            + ",{\"probability\":0.0354927,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"www.southwest.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0354927,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.southwest.com\","
-            + "\"function\":\"max\",\"typical\":[31356],\"actual\":[266310]}],\"record_score\":0.0544615,"
-            + "\"anomaly_score\":1.19192},{\"probability\":0.0392043,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"syndication.twimg.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0392043,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"syndication.twimg.com\","
-            + "\"function\":\"max\",\"typical\":[31356],\"actual\":[248276]}],\"record_score\":0.0366913,\"anomaly_score\":1" +
-            ".19192}"
-            + ",{\"probability\":0.0400853,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\""
-            + ",\"over_field_value\":\"mts0.google.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0400853,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"mts0.google.com\","
-            + "\"function\":\"max\",\"typical\":[31356],\"actual\":[244381]}],\"record_score\":0.0329562,"
-            + "\"anomaly_score\":1.19192},{\"probability\":0.0407335,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"www.onthegotours.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0407335,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.onthegotours.com\","
-            + "\"function\":\"max\",\"typical\":[31356],\"actual\":[241600]}],\"record_score\":0.0303116,"
-            + "\"anomaly_score\":1.19192},{\"probability\":0.0470889,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"chatenabled.mail.google.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0470889,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"chatenabled.mail.google.com\","
-            + "\"function\":\"max\",\"typical\":[31356],\"actual\":[217573]}],\"record_score\":0.00823738,"
-            + "\"anomaly_score\":1.19192},{\"probability\":0.0491243,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\","
-            + "\"over_field_value\":\"googleads.g.doubleclick.net\",\"function\":\"max\",\"causes\":[{\"probability\":0.0491243,"
-            + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"googleads.g.doubleclick.net\","
-            + "\"function\":\"max\",\"typical\":[31356],\"actual\":[210926]}],\"record_score\":0.00237509,"
-            + "\"anomaly_score\":1.19192}],\"raw_anomaly_score\":1.26918,\"anomaly_score\":1.19192,"
-            + "\"event_count\":1159}" + "]";
-
-    public void testParser() throws IOException {
-        try (InputStream inputStream = new ByteArrayInputStream(METRIC_OUTPUT_SAMPLE.getBytes(StandardCharsets.UTF_8))) {
-            AutodetectResultsParser parser = new AutodetectResultsParser();
-            List<AutodetectResult> results = new ArrayList<>();
-            parser.parseResults(inputStream).forEachRemaining(results::add);
-            List<Bucket> buckets = results.stream().map(AutodetectResult::getBucket)
-                .filter(b -> b != null)
-                .collect(Collectors.toList());
-
-            assertEquals(2, buckets.size());
-            assertEquals(new Date(1359450000000L), buckets.get(0).getTimestamp());
-
-            assertEquals(buckets.get(0).getEventCount(), 806);
-
-            List<BucketInfluencer> bucketInfluencers = buckets.get(0).getBucketInfluencers();
-            assertEquals(1, bucketInfluencers.size());
-            assertEquals(0.0, bucketInfluencers.get(0).getRawAnomalyScore(), EPSILON);
-            assertEquals(0.0, bucketInfluencers.get(0).getAnomalyScore(), EPSILON);
-            assertEquals(0.0, bucketInfluencers.get(0).getProbability(), EPSILON);
-            assertEquals("bucket_time", bucketInfluencers.get(0).getInfluencerFieldName());
-
-            assertEquals(new Date(1359453600000L), buckets.get(1).getTimestamp());
-
-            assertEquals(buckets.get(1).getEventCount(), 820);
-            bucketInfluencers = buckets.get(1).getBucketInfluencers();
-            assertEquals(2, bucketInfluencers.size());
-            assertEquals(0.0140005, bucketInfluencers.get(0).getRawAnomalyScore(), EPSILON);
-            assertEquals(20.22688, bucketInfluencers.get(0).getAnomalyScore(), EPSILON);
-            assertEquals(0.01, bucketInfluencers.get(0).getProbability(), EPSILON);
-            assertEquals("bucket_time", bucketInfluencers.get(0).getInfluencerFieldName());
-            assertEquals(0.005, bucketInfluencers.get(1).getRawAnomalyScore(), EPSILON);
-            assertEquals(10.5, bucketInfluencers.get(1).getAnomalyScore(), EPSILON);
-            assertEquals(0.03, bucketInfluencers.get(1).getProbability(), EPSILON);
-            assertEquals("foo", bucketInfluencers.get(1).getInfluencerFieldName());
-
-            Bucket secondBucket = buckets.get(1);
-
-            assertEquals(0.0637541, secondBucket.getRecords().get(0).getProbability(), EPSILON);
-            assertEquals("airline", secondBucket.getRecords().get(0).getByFieldName());
-            assertEquals("JZA", secondBucket.getRecords().get(0).getByFieldValue());
-            assertEquals(1020.08, secondBucket.getRecords().get(0).getTypical().get(0), EPSILON);
-            assertEquals(1042.14, secondBucket.getRecords().get(0).getActual().get(0), EPSILON);
-            assertEquals("responsetime", secondBucket.getRecords().get(0).getFieldName());
-            assertEquals("max", secondBucket.getRecords().get(0).getFunction());
-            assertEquals("", secondBucket.getRecords().get(0).getPartitionFieldName());
-            assertEquals("", secondBucket.getRecords().get(0).getPartitionFieldValue());
-
-            assertEquals(0.00748292, secondBucket.getRecords().get(1).getProbability(), EPSILON);
-            assertEquals("airline", secondBucket.getRecords().get(1).getByFieldName());
-            assertEquals("AMX", secondBucket.getRecords().get(1).getByFieldValue());
-            assertEquals(20.2137, secondBucket.getRecords().get(1).getTypical().get(0), EPSILON);
-            assertEquals(22.8855, secondBucket.getRecords().get(1).getActual().get(0), EPSILON);
-            assertEquals("responsetime", secondBucket.getRecords().get(1).getFieldName());
-            assertEquals("max", secondBucket.getRecords().get(1).getFunction());
-            assertEquals("", secondBucket.getRecords().get(1).getPartitionFieldName());
-            assertEquals("", secondBucket.getRecords().get(1).getPartitionFieldValue());
-
-            assertEquals(0.023494, secondBucket.getRecords().get(2).getProbability(), EPSILON);
-            assertEquals("airline", secondBucket.getRecords().get(2).getByFieldName());
-            assertEquals("DAL", secondBucket.getRecords().get(2).getByFieldValue());
-            assertEquals(382.177, secondBucket.getRecords().get(2).getTypical().get(0), EPSILON);
-            assertEquals(358.934, secondBucket.getRecords().get(2).getActual().get(0), EPSILON);
-            assertEquals("responsetime", secondBucket.getRecords().get(2).getFieldName());
-            assertEquals("min", secondBucket.getRecords().get(2).getFunction());
-            assertEquals("", secondBucket.getRecords().get(2).getPartitionFieldName());
-            assertEquals("", secondBucket.getRecords().get(2).getPartitionFieldValue());
-
-            assertEquals(0.0473552, secondBucket.getRecords().get(3).getProbability(), EPSILON);
-            assertEquals("airline", secondBucket.getRecords().get(3).getByFieldName());
-            assertEquals("SWA", secondBucket.getRecords().get(3).getByFieldValue());
-            assertEquals(152.148, secondBucket.getRecords().get(3).getTypical().get(0), EPSILON);
-            assertEquals(96.6425, secondBucket.getRecords().get(3).getActual().get(0), EPSILON);
-            assertEquals("responsetime", secondBucket.getRecords().get(3).getFieldName());
-            assertEquals("min", secondBucket.getRecords().get(3).getFunction());
-            assertEquals("", secondBucket.getRecords().get(3).getPartitionFieldName());
-            assertEquals("", secondBucket.getRecords().get(3).getPartitionFieldValue());
-
-            List<Quantiles> quantiles = results.stream().map(AutodetectResult::getQuantiles)
-                .filter(q -> q != null)
-                .collect(Collectors.toList());
-            assertEquals(3, quantiles.size());
-            assertEquals("foo", quantiles.get(0).getJobId());
-            assertEquals(new Date(1359450000000L), quantiles.get(0).getTimestamp());
-            assertEquals("[normalizer 1.1, normalizer 2.1]", quantiles.get(0).getQuantileState());
-            assertEquals("foo", quantiles.get(1).getJobId());
-            assertEquals(new Date(1359453600000L), quantiles.get(1).getTimestamp());
-            assertEquals("[normalizer 1.2, normalizer 2.2]", quantiles.get(1).getQuantileState());
-            assertEquals("foo", quantiles.get(2).getJobId());
-            assertEquals(new Date(1359453600000L), quantiles.get(2).getTimestamp());
-            assertEquals("[normalizer 1.3, normalizer 2.3]", quantiles.get(2).getQuantileState());
-        }
-    }
-
-    @AwaitsFix(bugUrl = "rewrite this test so it doesn't use ~200 lines of json")
-    public void testPopulationParser() throws IOException {
-        try (InputStream inputStream = new ByteArrayInputStream(POPULATION_OUTPUT_SAMPLE.getBytes(StandardCharsets.UTF_8))) {
-            AutodetectResultsParser parser = new AutodetectResultsParser();
-            List<AutodetectResult> results = new ArrayList<>();
-            parser.parseResults(inputStream).forEachRemaining(results::add);
-            List<Bucket> buckets = results.stream().map(AutodetectResult::getBucket)
-                .filter(b -> b != null)
-                .collect(Collectors.toList());
-
-            assertEquals(2, buckets.size());
-            assertEquals(new Date(1379590200000L), buckets.get(0).getTimestamp());
-            assertEquals(buckets.get(0).getEventCount(), 1235);
-
-            Bucket firstBucket = buckets.get(0);
-            assertEquals(1.38951e-08, firstBucket.getRecords().get(0).getProbability(), EPSILON);
-            assertEquals("sum_cs_bytes_", firstBucket.getRecords().get(0).getFieldName());
-            assertEquals("max", firstBucket.getRecords().get(0).getFunction());
-            assertEquals("cs_host", firstBucket.getRecords().get(0).getOverFieldName());
-            assertEquals("mail.google.com", firstBucket.getRecords().get(0).getOverFieldValue());
-            assertNotNull(firstBucket.getRecords().get(0).getCauses());
-
-            assertEquals(new Date(1379590800000L), buckets.get(1).getTimestamp());
-            assertEquals(buckets.get(1).getEventCount(), 1159);
-        }
-    }
-
-    public void testParse_GivenEmptyArray() throws ElasticsearchParseException, IOException {
-        String json = "[]";
-        try (InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8))) {
-            AutodetectResultsParser parser = new AutodetectResultsParser();
-            assertFalse(parser.parseResults(inputStream).hasNext());
-        }
-    }
-
-    public void testParse_GivenModelSizeStats() throws ElasticsearchParseException, IOException {
-        String json = "[{\"model_size_stats\": {\"job_id\": \"foo\", \"model_bytes\":300}}]";
-        try (InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8))) {
-
-            AutodetectResultsParser parser = new AutodetectResultsParser();
-            List<AutodetectResult> results = new ArrayList<>();
-            parser.parseResults(inputStream).forEachRemaining(results::add);
-
-            assertEquals(1, results.size());
-            assertEquals(300, results.get(0).getModelSizeStats().getModelBytes());
-        }
-    }
-
-    public void testParse_GivenCategoryDefinition() throws IOException {
-        String json = "[{\"category_definition\": {\"job_id\":\"foo\", \"category_id\":18}}]";
-        try (InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8))) {
-            AutodetectResultsParser parser = new AutodetectResultsParser();
-            List<AutodetectResult> results = new ArrayList<>();
-            parser.parseResults(inputStream).forEachRemaining(results::add);
-
-
-            assertEquals(1, results.size());
-            assertEquals(18, results.get(0).getCategoryDefinition().getCategoryId());
-        }
-    }
-
-    public void testParse_GivenUnknownObject() throws ElasticsearchParseException, IOException {
-        String json = "[{\"unknown\":{\"id\": 18}}]";
-        try (InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8))) {
-            AutodetectResultsParser parser = new AutodetectResultsParser();
-            XContentParseException e = expectThrows(XContentParseException.class,
-                () -> parser.parseResults(inputStream).forEachRemaining(a -> {
-                }));
-            assertEquals("[1:3] [autodetect_result] unknown field [unknown], parser not found", e.getMessage());
-        }
-    }
-
-    public void testParse_GivenArrayContainsAnotherArray() throws ElasticsearchParseException, IOException {
-        String json = "[[]]";
-        try (InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8))) {
-            AutodetectResultsParser parser = new AutodetectResultsParser();
-            ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class,
-                () -> parser.parseResults(inputStream).forEachRemaining(a -> {
-                }));
-            assertEquals("unexpected token [START_ARRAY]", e.getMessage());
-        }
-    }
-
-    /**
-     * Ensure that we do not accept NaN values
-     */
-    public void testParsingExceptionNaN() {
-        String json = "[{\"bucket\": {\"job_id\":\"foo\",\"timestamp\":1359453600000,\"bucket_span\":10,\"records\":"
-                + "[{\"timestamp\":1359453600000,\"bucket_span\":10,\"job_id\":\"foo\",\"probability\":NaN,"
-                + "\"by_field_name\":\"airline\",\"by_field_value\":\"JZA\", \"typical\":[1020.08],\"actual\":[0],"
-                + "\"field_name\":\"responsetime\",\"function\":\"max\",\"partition_field_name\":\"\",\"partition_field_value\":\"\"}]}}]";
-        InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8));
-        AutodetectResultsParser parser = new AutodetectResultsParser();
-
-        expectThrows(XContentParseException.class,
-                () -> parser.parseResults(inputStream).forEachRemaining(a -> {}));
-    }
-}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/AutodetectParamsTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/AutodetectParamsTests.java
new file mode 100644
index 0000000000000..616dd6cba5a14
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/AutodetectParamsTests.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.job.process.autodetect.params;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class AutodetectParamsTests extends ESTestCase {
+
+    private static final String JOB_ID = "my-job";
+
+    public void testBuilder_WithTimingStats() {
+        TimingStats timingStats = new TimingStats(JOB_ID, 7, 1.0, 1000.0, 666.0, 1000.0);
+        AutodetectParams params = new AutodetectParams.Builder(JOB_ID).setTimingStats(timingStats).build();
+        assertThat(params.timingStats(), equalTo(timingStats));
+
+        timingStats.updateStats(2000.0);
+        assertThat(timingStats, equalTo(new TimingStats(JOB_ID, 8, 1.0, 2000.0, 832.75, 1010.0)));
+        assertThat(params.timingStats(), equalTo(new TimingStats(JOB_ID, 7, 1.0, 1000.0, 666.0, 1000.0)));
+    }
+
+    public void testBuilder_WithoutTimingStats() {
+        AutodetectParams params = new AutodetectParams.Builder(JOB_ID).build();
+        assertThat(params.timingStats(), equalTo(new TimingStats(JOB_ID)));
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriterTests.java
index 01bdd6a999f26..e20950bede2d5 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriterTests.java
@@ -109,7 +109,7 @@ public void testInputFields() throws IOException {
 
     public void testTokenizeForCategorization() throws IOException {
         CategorizationAnalyzerConfig defaultConfig = CategorizationAnalyzerConfig.buildDefaultCategorizationAnalyzer(null);
-        try (CategorizationAnalyzer categorizationAnalyzer = new CategorizationAnalyzer(analysisRegistry, environment, defaultConfig)) {
+        try (CategorizationAnalyzer categorizationAnalyzer = new CategorizationAnalyzer(analysisRegistry, defaultConfig)) {
 
             assertEquals("sol13m-8608.1.p2ps,Info,Source,AES_SERVICE2,on,has,shut,down",
                     AbstractDataToProcessWriter.tokenizeForCategorization(categorizationAnalyzer, "p2ps",
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CsvDataToProcessWriterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CsvDataToProcessWriterTests.java
index abe4a44f69fe7..6e846b000237d 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CsvDataToProcessWriterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CsvDataToProcessWriterTests.java
@@ -125,7 +125,7 @@ public void testWrite_GivenTimeFormatIsEpochAndCategorization() throws IOExcepti
         CsvDataToProcessWriter writer = createWriter();
         writer.writeHeader();
         try (CategorizationAnalyzer categorizationAnalyzer =
-                     new CategorizationAnalyzer(analysisRegistry, environment, analysisConfig.getCategorizationAnalyzerConfig())) {
+                     new CategorizationAnalyzer(analysisRegistry, analysisConfig.getCategorizationAnalyzerConfig())) {
             writer.write(inputStream, categorizationAnalyzer, null, (r, e) -> {});
         }
         verify(dataCountsReporter, times(1)).startNewIncrementalCount();
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriterTests.java
index 39860033e6712..f16b388edee6f 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriterTests.java
@@ -124,7 +124,7 @@ public void testWrite_GivenTimeFormatIsEpochAndCategorization() throws Exception
         JsonDataToProcessWriter writer = createWriter();
         writer.writeHeader();
         try (CategorizationAnalyzer categorizationAnalyzer =
-                     new CategorizationAnalyzer(analysisRegistry, environment, analysisConfig.getCategorizationAnalyzerConfig())) {
+                     new CategorizationAnalyzer(analysisRegistry, analysisConfig.getCategorizationAnalyzerConfig())) {
             writer.write(inputStream, categorizationAnalyzer, XContentType.JSON, (r, e) -> {});
         }
         verify(dataCountsReporter, times(1)).startNewIncrementalCount();
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/BucketTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/BucketTests.java
index a49ef0a5e26fa..aa56e392de2bb 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/BucketTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/BucketTests.java
@@ -61,7 +61,7 @@ public Bucket createTestInstance(String jobId) {
             bucket.setInterim(randomBoolean());
         }
         if (randomBoolean()) {
-            bucket.setProcessingTimeMs(randomLong());
+            bucket.setProcessingTimeMs(randomNonNegativeLong());
         }
         if (randomBoolean()) {
             int size = randomInt(10);
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/MlMemoryTrackerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/MlMemoryTrackerTests.java
index 1dd2ba923ef00..429575902b0b8 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/MlMemoryTrackerTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/MlMemoryTrackerTests.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.ml.process;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -17,14 +18,18 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.ml.MlTasks;
 import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
+import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.ml.dataframe.persistence.DataFrameAnalyticsConfigProvider;
 import org.elasticsearch.xpack.ml.job.JobManager;
 import org.elasticsearch.xpack.ml.job.persistence.JobResultsProvider;
 import org.junit.Before;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicReference;
@@ -45,6 +50,7 @@ public class MlMemoryTrackerTests extends ESTestCase {
 
     private JobManager jobManager;
     private JobResultsProvider jobResultsProvider;
+    private DataFrameAnalyticsConfigProvider configProvider;
     private MlMemoryTracker memoryTracker;
 
     @Before
@@ -65,7 +71,8 @@ public void setup() {
         when(threadPool.executor(anyString())).thenReturn(executorService);
         jobManager = mock(JobManager.class);
         jobResultsProvider = mock(JobResultsProvider.class);
-        memoryTracker = new MlMemoryTracker(Settings.EMPTY, clusterService, threadPool, jobManager, jobResultsProvider);
+        configProvider = mock(DataFrameAnalyticsConfigProvider.class);
+        memoryTracker = new MlMemoryTracker(Settings.EMPTY, clusterService, threadPool, jobManager, jobResultsProvider, configProvider);
     }
 
     public void testRefreshAll() {
@@ -77,14 +84,26 @@ public void testRefreshAll() {
             memoryTracker.offMaster();
         }
 
-        int numMlJobTasks = randomIntBetween(2, 5);
         Map<String, PersistentTasksCustomMetaData.PersistentTask<?>> tasks = new HashMap<>();
-        for (int i = 1; i <= numMlJobTasks; ++i) {
+
+        int numAnomalyDetectorJobTasks = randomIntBetween(2, 5);
+        for (int i = 1; i <= numAnomalyDetectorJobTasks; ++i) {
             String jobId = "job" + i;
-            PersistentTasksCustomMetaData.PersistentTask<?> task = makeTestTask(jobId);
+            PersistentTasksCustomMetaData.PersistentTask<?> task = makeTestAnomalyDetectorTask(jobId);
+            tasks.put(task.getId(), task);
+        }
+
+        List<String> allIds = new ArrayList<>();
+        int numDataFrameAnalyticsTasks = randomIntBetween(2, 5);
+        for (int i = 1; i <= numDataFrameAnalyticsTasks; ++i) {
+            String id = "analytics" + i;
+            allIds.add(id);
+            PersistentTasksCustomMetaData.PersistentTask<?> task = makeTestDataFrameAnalyticsTask(id);
             tasks.put(task.getId(), task);
         }
-        PersistentTasksCustomMetaData persistentTasks = new PersistentTasksCustomMetaData(numMlJobTasks, tasks);
+
+        PersistentTasksCustomMetaData persistentTasks =
+            new PersistentTasksCustomMetaData(numAnomalyDetectorJobTasks + numDataFrameAnalyticsTasks, tasks);
 
         doAnswer(invocation -> {
             @SuppressWarnings("unchecked")
@@ -96,16 +115,17 @@ public void testRefreshAll() {
         memoryTracker.refresh(persistentTasks, ActionListener.wrap(aVoid -> {}, ESTestCase::assertNull));
 
         if (isMaster) {
-            for (int i = 1; i <= numMlJobTasks; ++i) {
+            for (int i = 1; i <= numAnomalyDetectorJobTasks; ++i) {
                 String jobId = "job" + i;
                 verify(jobResultsProvider, times(1)).getEstablishedMemoryUsage(eq(jobId), any(), any(), any(), any());
             }
+            verify(configProvider, times(1)).getMultiple(eq(String.join(",", allIds)), eq(false), any(ActionListener.class));
         } else {
             verify(jobResultsProvider, never()).getEstablishedMemoryUsage(anyString(), any(), any(), any(), any());
         }
     }
 
-    public void testRefreshOne() {
+    public void testRefreshOneAnomalyDetectorJob() {
 
         boolean isMaster = randomBoolean();
         if (isMaster) {
@@ -137,26 +157,26 @@ public void testRefreshOne() {
         }).when(jobManager).getJob(eq(jobId), any(ActionListener.class));
 
         AtomicReference<Long> refreshedMemoryRequirement = new AtomicReference<>();
-        memoryTracker.refreshJobMemory(jobId, ActionListener.wrap(refreshedMemoryRequirement::set, ESTestCase::assertNull));
+        memoryTracker.refreshAnomalyDetectorJobMemory(jobId, ActionListener.wrap(refreshedMemoryRequirement::set, ESTestCase::assertNull));
 
         if (isMaster) {
             if (haveEstablishedModelMemory) {
                 assertEquals(Long.valueOf(modelBytes + Job.PROCESS_MEMORY_OVERHEAD.getBytes()),
-                    memoryTracker.getJobMemoryRequirement(jobId));
+                    memoryTracker.getAnomalyDetectorJobMemoryRequirement(jobId));
             } else {
                 long expectedModelMemoryLimit =
                     simulateVeryOldJob ? AnalysisLimits.PRE_6_1_DEFAULT_MODEL_MEMORY_LIMIT_MB : recentJobModelMemoryLimitMb;
                 assertEquals(Long.valueOf(ByteSizeUnit.MB.toBytes(expectedModelMemoryLimit) + Job.PROCESS_MEMORY_OVERHEAD.getBytes()),
-                    memoryTracker.getJobMemoryRequirement(jobId));
+                    memoryTracker.getAnomalyDetectorJobMemoryRequirement(jobId));
             }
         } else {
-            assertNull(memoryTracker.getJobMemoryRequirement(jobId));
+            assertNull(memoryTracker.getAnomalyDetectorJobMemoryRequirement(jobId));
         }
 
-        assertEquals(memoryTracker.getJobMemoryRequirement(jobId), refreshedMemoryRequirement.get());
+        assertEquals(memoryTracker.getAnomalyDetectorJobMemoryRequirement(jobId), refreshedMemoryRequirement.get());
 
-        memoryTracker.removeJob(jobId);
-        assertNull(memoryTracker.getJobMemoryRequirement(jobId));
+        memoryTracker.removeAnomalyDetectorJob(jobId);
+        assertNull(memoryTracker.getAnomalyDetectorJobMemoryRequirement(jobId));
     }
 
     public void testStop() {
@@ -165,15 +185,22 @@ public void testStop() {
         memoryTracker.stop();
 
         AtomicReference<Exception> exception = new AtomicReference<>();
-        memoryTracker.refreshJobMemory("job", ActionListener.wrap(ESTestCase::assertNull, exception::set));
+        memoryTracker.refreshAnomalyDetectorJobMemory("job", ActionListener.wrap(ESTestCase::assertNull, exception::set));
 
         assertNotNull(exception.get());
         assertThat(exception.get(), instanceOf(EsRejectedExecutionException.class));
         assertEquals("Couldn't run ML memory update - node is shutting down", exception.get().getMessage());
     }
 
-    private PersistentTasksCustomMetaData.PersistentTask<OpenJobAction.JobParams> makeTestTask(String jobId) {
-        return new PersistentTasksCustomMetaData.PersistentTask<>("job-" + jobId, MlTasks.JOB_TASK_NAME, new OpenJobAction.JobParams(jobId),
-            0, PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT);
+    private PersistentTasksCustomMetaData.PersistentTask<OpenJobAction.JobParams> makeTestAnomalyDetectorTask(String jobId) {
+        return new PersistentTasksCustomMetaData.PersistentTask<>(MlTasks.jobTaskId(jobId), MlTasks.JOB_TASK_NAME,
+            new OpenJobAction.JobParams(jobId), 0, PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT);
+    }
+
+    private
+    PersistentTasksCustomMetaData.PersistentTask<StartDataFrameAnalyticsAction.TaskParams> makeTestDataFrameAnalyticsTask(String id) {
+        return new PersistentTasksCustomMetaData.PersistentTask<>(MlTasks.dataFrameAnalyticsTaskId(id),
+            MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, new StartDataFrameAnalyticsAction.TaskParams(id, Version.CURRENT), 0,
+            PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT);
     }
 }
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/ProcessResultsParserTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/ProcessResultsParserTests.java
new file mode 100644
index 0000000000000..32ab15a27019f
--- /dev/null
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/process/ProcessResultsParserTests.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.process;
+
+import com.google.common.base.Charsets;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParseException;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+
+import static org.hamcrest.Matchers.contains;
+
+public class ProcessResultsParserTests extends ESTestCase {
+
+    public void testParse_GivenEmptyArray() throws IOException {
+        String json = "[]";
+        try (InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8))) {
+            ProcessResultsParser<TestResult> parser = new ProcessResultsParser<>(TestResult.PARSER);
+            assertFalse(parser.parseResults(inputStream).hasNext());
+        }
+    }
+
+    public void testParse_GivenUnknownObject() throws IOException {
+        String json = "[{\"unknown\":{\"id\": 18}}]";
+        try (InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8))) {
+            ProcessResultsParser<TestResult> parser = new ProcessResultsParser<>(TestResult.PARSER);
+            XContentParseException e = expectThrows(XContentParseException.class,
+                () -> parser.parseResults(inputStream).forEachRemaining(a -> {
+                }));
+            assertEquals("[1:3] [test_result] unknown field [unknown], parser not found", e.getMessage());
+        }
+    }
+
+    public void testParse_GivenArrayContainsAnotherArray() throws IOException {
+        String json = "[[]]";
+        try (InputStream inputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8))) {
+            ProcessResultsParser<TestResult> parser = new ProcessResultsParser<>(TestResult.PARSER);
+            ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class,
+                () -> parser.parseResults(inputStream).forEachRemaining(a -> {
+                }));
+            assertEquals("unexpected token [START_ARRAY]", e.getMessage());
+        }
+    }
+
+    public void testParseResults() throws IOException {
+        String input = "[{\"field_1\": \"a\", \"field_2\": 1.0}, {\"field_1\": \"b\", \"field_2\": 2.0},"
+                + " {\"field_1\": \"c\", \"field_2\": 3.0}]";
+        try (InputStream inputStream = new ByteArrayInputStream(input.getBytes(Charsets.UTF_8))) {
+
+            ProcessResultsParser<TestResult> parser = new ProcessResultsParser<>(TestResult.PARSER);
+            Iterator<TestResult> testResultIterator = parser.parseResults(inputStream);
+
+            List<TestResult> parsedResults = new ArrayList<>();
+            while (testResultIterator.hasNext()) {
+                parsedResults.add(testResultIterator.next());
+            }
+
+            assertThat(parsedResults, contains(new TestResult("a", 1.0), new TestResult("b", 2.0), new TestResult("c", 3.0)));
+        }
+    }
+
+    private static class TestResult {
+
+        private static final ParseField FIELD_1 = new ParseField("field_1");
+        private static final ParseField FIELD_2 = new ParseField("field_2");
+
+        private static final ConstructingObjectParser<TestResult, Void> PARSER = new ConstructingObjectParser<>("test_result",
+                a -> new TestResult((String) a[0], (Double) a[1]));
+
+        static {
+            PARSER.declareString(ConstructingObjectParser.constructorArg(), FIELD_1);
+            PARSER.declareDouble(ConstructingObjectParser.constructorArg(), FIELD_2);
+        }
+
+        private final String field1;
+        private final double field2;
+
+        private TestResult(String field1, double field2) {
+            this.field1 = field1;
+            this.field2 = field2;
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            if (this == other) {
+                return true;
+            }
+            if (other == null || getClass() != other.getClass()) {
+                return false;
+            }
+            TestResult that = (TestResult) other;
+            return Objects.equals(field1, that.field1) && Objects.equals(field2, that.field2);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(field1, field2);
+        }
+    }
+}
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
index aae9b0d89c2a0..b2c62b5ffb230 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
@@ -23,23 +23,28 @@
 import org.elasticsearch.index.reindex.ReindexPlugin;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.license.LicenseService;
+import org.elasticsearch.persistent.PersistentTasksClusterService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.MockHttpTransport;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.util.QueryPage;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 import org.elasticsearch.xpack.core.ml.action.CloseJobAction;
+import org.elasticsearch.xpack.core.ml.action.DeleteDataFrameAnalyticsAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteDatafeedAction;
 import org.elasticsearch.xpack.core.ml.action.DeleteJobAction;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction;
+import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction;
 import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.GetJobsAction;
 import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction;
-import org.elasticsearch.xpack.core.action.util.QueryPage;
-import org.elasticsearch.xpack.core.ml.client.MachineLearningClient;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
 import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
+import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
 import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
 import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
@@ -189,6 +194,7 @@ public void cleanupWorkaround() throws Exception {
         logger.info("[{}#{}]: Cleaning up datafeeds and jobs after test", getTestClass().getSimpleName(), getTestName());
         deleteAllDatafeeds(logger, client());
         deleteAllJobs(logger, client());
+        deleteAllDataFrameAnalytics(client());
         assertBusy(() -> {
             RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries()
                     .setActiveOnly(true)
@@ -255,9 +261,8 @@ public static GetDatafeedsStatsAction.Response.DatafeedStats getDatafeedStats(St
     }
 
     public static void deleteAllDatafeeds(Logger logger, Client client) throws Exception {
-        final MachineLearningClient mlClient = new MachineLearningClient(client);
         final QueryPage<DatafeedConfig> datafeeds =
-                mlClient.getDatafeeds(new GetDatafeedsAction.Request(GetDatafeedsAction.ALL)).actionGet().getResponse();
+            client.execute(GetDatafeedsAction.INSTANCE, new GetDatafeedsAction.Request(GetDatafeedsAction.ALL)).actionGet().getResponse();
         try {
             logger.info("Closing all datafeeds (using _all)");
             StopDatafeedAction.Response stopResponse = client
@@ -295,8 +300,8 @@ public static void deleteAllDatafeeds(Logger logger, Client client) throws Excep
     }
 
     public static void deleteAllJobs(Logger logger, Client client) throws Exception {
-        final MachineLearningClient mlClient = new MachineLearningClient(client);
-        final QueryPage<Job> jobs = mlClient.getJobs(new GetJobsAction.Request(MetaData.ALL)).actionGet().getResponse();
+        final QueryPage<Job> jobs =
+            client.execute(GetJobsAction.INSTANCE, new GetJobsAction.Request(MetaData.ALL)).actionGet().getResponse();
 
         try {
             CloseJobAction.Request closeRequest = new CloseJobAction.Request(MetaData.ALL);
@@ -332,7 +337,33 @@ public static void deleteAllJobs(Logger logger, Client client) throws Exception
         }
     }
 
+    public static void deleteAllDataFrameAnalytics(Client client) throws Exception {
+        final QueryPage<DataFrameAnalyticsConfig> analytics =
+            client.execute(GetDataFrameAnalyticsAction.INSTANCE,
+                new GetDataFrameAnalyticsAction.Request("_all")).get().getResources();
+
+        assertBusy(() -> {
+            GetDataFrameAnalyticsStatsAction.Response statsResponse =
+                client().execute(GetDataFrameAnalyticsStatsAction.INSTANCE, new GetDataFrameAnalyticsStatsAction.Request("_all")).get();
+            assertTrue(statsResponse.getResponse().results().stream().allMatch(s -> s.getState().equals(DataFrameAnalyticsState.STOPPED)));
+        });
+        for (final DataFrameAnalyticsConfig config : analytics.results()) {
+            client.execute(DeleteDataFrameAnalyticsAction.INSTANCE, new DeleteDataFrameAnalyticsAction.Request(config.getId())).actionGet();
+        }
+    }
+
     protected String awaitJobOpenedAndAssigned(String jobId, String queryNode) throws Exception {
+
+        PersistentTasksClusterService persistentTasksClusterService =
+            internalCluster().getInstance(PersistentTasksClusterService.class, internalCluster().getMasterName(queryNode));
+        // Speed up rechecks to a rate that is quicker than what settings would allow.
+        // The check would work eventually without doing this, but the assertBusy() below
+        // would need to wait 30 seconds, which would make the test run very slowly.
+        // The 1 second refresh puts a greater burden on the master node to recheck
+        // persistent tasks, but it will cope in these tests as it's not doing much
+        // else.
+        persistentTasksClusterService.setRecheckInterval(TimeValue.timeValueSeconds(1));
+
         AtomicReference<String> jobNode = new AtomicReference<>();
         assertBusy(() -> {
             GetJobsStatsAction.Response statsResponse =
diff --git a/x-pack/plugin/monitoring/build.gradle b/x-pack/plugin/monitoring/build.gradle
index cb6395b18a4c2..0eb0089e237a1 100644
--- a/x-pack/plugin/monitoring/build.gradle
+++ b/x-pack/plugin/monitoring/build.gradle
@@ -10,7 +10,6 @@ esplugin {
 archivesBaseName = 'x-pack-monitoring'
 
 dependencies {
-    // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
     compileOnly project(path: xpackModule('core'), configuration: 'default')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     if (isEclipse) {
@@ -18,13 +17,13 @@ dependencies {
     }
 
     // monitoring deps
-    compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
-    compile "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}"
+    compile project(':client:rest')
+    compile project(':client:sniffer')
 
     // baz - this goes away after we separate out the actions #27759
-    testCompile "org.elasticsearch.plugin:x-pack-watcher:${version}"
+    testCompile project(xpackModule('watcher'))
 
-    testCompile "org.elasticsearch.plugin:x-pack-ilm:${version}"
+    testCompile project(xpackModule('ilm'))
 }
 
 compileJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked"
@@ -38,7 +37,7 @@ task testJar(type: Jar) {
     from sourceSets.test.output
 }
 artifacts {
-    // normal es plugins do not publish the jar but we need to since users need it for Transport Clients and extensions
+    // normal es plugins do not publish the jar but we need to since users need it for extensions
     archives jar
     testArtifacts testJar
 }
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
index 0472d909d0fa9..29a3caec10bcb 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
@@ -11,8 +11,6 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -33,6 +31,8 @@
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.monitoring.MonitoringField;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkAction;
 import org.elasticsearch.xpack.core.ssl.SSLService;
@@ -67,11 +67,6 @@
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.common.settings.Setting.boolSetting;
 
-/**
- * This class activates/deactivates the monitoring modules depending if we're running a node client, transport client:
- * - node clients: all modules are bound
- * - transport clients: only action/transport actions are bound
- */
 public class Monitoring extends Plugin implements ActionPlugin {
 
     /**
@@ -98,26 +93,13 @@ boolean isEnabled() {
         return enabled;
     }
 
-    @Override
-    public Collection<Module> createGuiceModules() {
-        List<Module> modules = new ArrayList<>();
-        modules.add(b -> {
-            XPackPlugin.bindFeatureSet(b, MonitoringFeatureSet.class);
-            if (enabled == false) {
-                b.bind(MonitoringService.class).toProvider(Providers.of(null));
-                b.bind(Exporters.class).toProvider(Providers.of(null));
-            }
-        });
-        return modules;
-    }
-
     @Override
     public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
                                                ResourceWatcherService resourceWatcherService, ScriptService scriptService,
                                                NamedXContentRegistry xContentRegistry, Environment environment,
                                                NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) {
         if (enabled == false) {
-            return Collections.emptyList();
+            return Collections.singletonList(new MonitoringUsageServices(null, null));
         }
 
         final ClusterSettings clusterSettings = clusterService.getClusterSettings();
@@ -141,15 +123,21 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
 
         final MonitoringService monitoringService = new MonitoringService(settings, clusterService, threadPool, collectors, exporters);
 
-        return Arrays.asList(monitoringService, exporters, cleanerService);
+        var usageServices = new MonitoringUsageServices(monitoringService, exporters);
+        return Arrays.asList(monitoringService, exporters, cleanerService, usageServices);
     }
 
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        var usageAction = new ActionHandler<>(XPackUsageFeatureAction.MONITORING, MonitoringUsageTransportAction.class);
+        var infoAction = new ActionHandler<>(XPackInfoFeatureAction.MONITORING, MonitoringInfoTransportAction.class);
         if (false == enabled) {
-            return emptyList();
+            return Arrays.asList(usageAction, infoAction);
         }
-        return singletonList(new ActionHandler<>(MonitoringBulkAction.INSTANCE, TransportMonitoringBulkAction.class));
+        return Arrays.asList(
+            new ActionHandler<>(MonitoringBulkAction.INSTANCE, TransportMonitoringBulkAction.class),
+            usageAction,
+            infoAction);
     }
 
     @Override
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringFeatureSet.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringFeatureSet.java
deleted file mode 100644
index 8482bb52de609..0000000000000
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringFeatureSet.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.XPackField;
-import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.monitoring.MonitoringFeatureSetUsage;
-import org.elasticsearch.xpack.monitoring.exporter.Exporter;
-import org.elasticsearch.xpack.monitoring.exporter.Exporters;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class MonitoringFeatureSet implements XPackFeatureSet {
-
-    private final boolean enabled;
-    private final MonitoringService monitoring;
-    private final XPackLicenseState licenseState;
-    private final Exporters exporters;
-
-    @Inject
-    public MonitoringFeatureSet(Settings settings,
-                                @Nullable MonitoringService monitoring,
-                                @Nullable XPackLicenseState licenseState,
-                                @Nullable Exporters exporters) {
-        this.enabled = XPackSettings.MONITORING_ENABLED.get(settings);
-        this.monitoring = monitoring;
-        this.licenseState = licenseState;
-        this.exporters = exporters;
-    }
-
-    @Override
-    public String name() {
-        return XPackField.MONITORING;
-    }
-
-    @Override
-    public String description() {
-        return "Monitoring for the Elastic Stack";
-    }
-
-    @Override
-    public boolean available() {
-        return licenseState != null && licenseState.isMonitoringAllowed();
-    }
-
-    @Override
-    public boolean enabled() {
-        return enabled;
-    }
-
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return null;
-    }
-
-    @Override
-    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
-        final boolean collectionEnabled = monitoring != null && monitoring.isMonitoringActive();
-
-        listener.onResponse(new MonitoringFeatureSetUsage(available(), enabled(), collectionEnabled, exportersUsage(exporters)));
-    }
-
-    static Map<String, Object> exportersUsage(Exporters exporters) {
-        if (exporters == null) {
-            return null;
-        }
-        Map<String, Object> usage = new HashMap<>();
-        for (Exporter exporter : exporters.getEnabledExporters()) {
-            if (exporter.config().enabled()) {
-                String type = exporter.config().type();
-                int count = (Integer) usage.getOrDefault(type, 0);
-                usage.put(type, count + 1);
-            }
-        }
-        return usage;
-    }
-
-}
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportAction.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportAction.java
new file mode 100644
index 0000000000000..6769138e4caa0
--- /dev/null
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportAction.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
+
+public class MonitoringInfoTransportAction extends XPackInfoFeatureTransportAction {
+
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public MonitoringInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                         Settings settings, XPackLicenseState licenseState) {
+        super(XPackInfoFeatureAction.MONITORING.name(), transportService, actionFilters);
+        this.enabled = XPackSettings.MONITORING_ENABLED.get(settings);
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    public String name() {
+        return XPackField.MONITORING;
+    }
+
+    @Override
+    public boolean available() {
+        return licenseState.isMonitoringAllowed();
+    }
+
+    @Override
+    public boolean enabled() {
+        return enabled;
+    }
+}
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringUsageServices.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringUsageServices.java
new file mode 100644
index 0000000000000..59555e073730d
--- /dev/null
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringUsageServices.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring;
+
+import org.elasticsearch.xpack.monitoring.exporter.Exporters;
+
+/**
+ * A wrapper around the services needed to produce usage information for the monitoring feature.
+ *
+ * This class is temporary until actions can be constructed directly by plugins.
+ */
+class MonitoringUsageServices {
+    final MonitoringService monitoringService;
+    final Exporters exporters;
+
+    MonitoringUsageServices(MonitoringService monitoringService, Exporters exporters) {
+        this.monitoringService = monitoringService;
+        this.exporters = exporters;
+    }
+}
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringUsageTransportAction.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringUsageTransportAction.java
new file mode 100644
index 0000000000000..b7e3bf98b1c06
--- /dev/null
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringUsageTransportAction.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.monitoring.MonitoringFeatureSetUsage;
+import org.elasticsearch.xpack.monitoring.exporter.Exporter;
+import org.elasticsearch.xpack.monitoring.exporter.Exporters;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class MonitoringUsageTransportAction extends XPackUsageFeatureTransportAction {
+    private final boolean enabled;
+    private final MonitoringService monitoringService;
+    private final XPackLicenseState licenseState;
+    private final Exporters exporters;
+
+    @Inject
+    public MonitoringUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                          ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                          Settings settings, XPackLicenseState licenseState, MonitoringUsageServices monitoringServices) {
+        super(XPackUsageFeatureAction.MONITORING.name(), transportService, clusterService, threadPool,
+            actionFilters, indexNameExpressionResolver);
+        this.enabled = XPackSettings.MONITORING_ENABLED.get(settings);
+        this.licenseState = licenseState;
+        this.monitoringService = monitoringServices.monitoringService;
+        this.exporters = monitoringServices.exporters;
+    }
+
+    @Override
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
+        final boolean collectionEnabled = monitoringService != null && monitoringService.isMonitoringActive();
+        var usage =
+            new MonitoringFeatureSetUsage(licenseState.isMonitoringAllowed(), enabled, collectionEnabled, exportersUsage(exporters));
+        listener.onResponse(new XPackUsageFeatureResponse(usage));
+    }
+
+    static Map<String, Object> exportersUsage(Exporters exporters) {
+        if (exporters == null) {
+            return null;
+        }
+        Map<String, Object> usage = new HashMap<>();
+        for (Exporter exporter : exporters.getEnabledExporters()) {
+            if (exporter.config().enabled()) {
+                String type = exporter.config().type();
+                int count = (Integer) usage.getOrDefault(type, 0);
+                usage.put(type, count + 1);
+            }
+        }
+        return usage;
+    }
+}
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollector.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollector.java
index 6aab3114b7807..8f8072ef5b864 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollector.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollector.java
@@ -14,10 +14,8 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.xpack.core.XPackClient;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
-import org.elasticsearch.xpack.core.ccr.client.CcrClient;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc;
 import org.elasticsearch.xpack.monitoring.collector.Collector;
 
@@ -37,25 +35,25 @@ public final class StatsCollector extends Collector {
 
     private final Settings settings;
     private final ThreadContext threadContext;
-    private final CcrClient ccrClient;
+    private final Client client;
 
     public StatsCollector(
             final Settings settings,
             final ClusterService clusterService,
             final XPackLicenseState licenseState,
             final Client client) {
-        this(settings, clusterService, licenseState, new XPackClient(client).ccr(), client.threadPool().getThreadContext());
+        this(settings, clusterService, licenseState, client, client.threadPool().getThreadContext());
     }
 
     StatsCollector(
             final Settings settings,
             final ClusterService clusterService,
             final XPackLicenseState licenseState,
-            final CcrClient ccrClient,
+            final Client client,
             final ThreadContext threadContext) {
         super(TYPE, clusterService, CCR_STATS_TIMEOUT, licenseState);
         this.settings = settings;
-        this.ccrClient = ccrClient;
+        this.client = client;
         this.threadContext = threadContext;
     }
 
@@ -79,7 +77,7 @@ protected Collection<MonitoringDoc> doCollect(
             final String clusterUuid = clusterUuid(clusterState);
 
             final CcrStatsAction.Request request = new CcrStatsAction.Request();
-            final CcrStatsAction.Response response = ccrClient.stats(request).actionGet(getCollectionTimeout());
+            final CcrStatsAction.Response response = client.execute(CcrStatsAction.INSTANCE, request).actionGet(getCollectionTimeout());
 
             final AutoFollowStatsMonitoringDoc autoFollowStatsDoc =
                 new AutoFollowStatsMonitoringDoc(clusterUuid, timestamp, interval, node, response.getAutoFollowStats());
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsCollector.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsCollector.java
index 855780d4836ae..be34af2850ec6 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsCollector.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsCollector.java
@@ -14,10 +14,8 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.xpack.core.XPackClient;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
-import org.elasticsearch.xpack.core.ml.client.MachineLearningClient;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc;
 import org.elasticsearch.xpack.monitoring.collector.Collector;
 
@@ -43,15 +41,15 @@ public class JobStatsCollector extends Collector {
 
     private final Settings settings;
     private final ThreadContext threadContext;
-    private final MachineLearningClient client;
+    private final Client client;
 
     public JobStatsCollector(final Settings settings, final ClusterService clusterService,
                              final XPackLicenseState licenseState, final Client client) {
-        this(settings, clusterService, licenseState, new XPackClient(client).machineLearning(), client.threadPool().getThreadContext());
+        this(settings, clusterService, licenseState, client, client.threadPool().getThreadContext());
     }
 
     JobStatsCollector(final Settings settings, final ClusterService clusterService,
-                      final XPackLicenseState licenseState, final MachineLearningClient client, final ThreadContext threadContext) {
+                      final XPackLicenseState licenseState, final Client client, final ThreadContext threadContext) {
         super(JobStatsMonitoringDoc.TYPE, clusterService, JOB_STATS_TIMEOUT, licenseState);
         this.settings = settings;
         this.client = client;
@@ -74,7 +72,7 @@ protected List<MonitoringDoc> doCollect(final MonitoringDoc.Node node,
         // fetch details about all jobs
         try (ThreadContext.StoredContext ignore = threadContext.stashWithOrigin(MONITORING_ORIGIN)) {
             final GetJobsStatsAction.Response jobs =
-                    client.getJobsStats(new GetJobsStatsAction.Request(MetaData.ALL))
+                    client.execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request(MetaData.ALL))
                             .actionGet(getCollectionTimeout());
 
             final long timestamp = timestamp();
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/ClusterAlertsUtil.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/ClusterAlertsUtil.java
index 782ecba1c30f9..2fe7e983a7a5b 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/ClusterAlertsUtil.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/ClusterAlertsUtil.java
@@ -7,14 +7,11 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.core.internal.io.Streams;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
@@ -124,13 +121,7 @@ public static String loadWatch(final ClusterService clusterService, final String
     }
 
     private static BytesReference loadResource(final String resource) throws IOException {
-        try (InputStream is = ClusterAlertsUtil.class.getResourceAsStream(resource)) {
-            try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
-                Streams.copy(is, out);
-
-                return new BytesArray(out.toByteArray());
-            }
-        }
+        return Streams.readFully(ClusterAlertsUtil.class.getResourceAsStream(resource));
     }
 
     /**
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
index 12f2f77a9d40c..ee2422ef4c2c2 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
@@ -39,14 +39,14 @@
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
-import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
-import org.elasticsearch.xpack.core.XPackClient;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
+import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchAction;
+import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.monitoring.cleaner.CleanerService;
 import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
@@ -431,8 +431,6 @@ private boolean hasValidVersion(final Object version, final long minimumVersion)
      */
     private void getClusterAlertsInstallationAsyncActions(final boolean indexExists, final List<Runnable> asyncActions,
                                                           final AtomicInteger pendingResponses) {
-        final XPackClient xpackClient = new XPackClient(client);
-        final WatcherClient watcher = xpackClient.watcher();
         final boolean canAddWatches = licenseState.isMonitoringClusterAlertsAllowed();
 
         for (final String watchId : ClusterAlertsUtil.WATCH_IDS) {
@@ -444,31 +442,30 @@ private void getClusterAlertsInstallationAsyncActions(final boolean indexExists,
                 if (addWatch) {
                     logger.trace("checking monitoring watch [{}]", uniqueWatchId);
 
-                    asyncActions.add(() -> watcher.getWatch(new GetWatchRequest(uniqueWatchId),
-                                                            new GetAndPutWatchResponseActionListener(watcher, watchId, uniqueWatchId,
+                    asyncActions.add(() -> client.execute(GetWatchAction.INSTANCE, new GetWatchRequest(uniqueWatchId),
+                                                            new GetAndPutWatchResponseActionListener(client, watchId, uniqueWatchId,
                                                                                                      pendingResponses)));
                 } else {
                     logger.trace("pruning monitoring watch [{}]", uniqueWatchId);
 
-                    asyncActions.add(() -> watcher.deleteWatch(new DeleteWatchRequest(uniqueWatchId),
+                    asyncActions.add(() -> client.execute(DeleteWatchAction.INSTANCE, new DeleteWatchRequest(uniqueWatchId),
                                                                new ResponseActionListener<>("watch", uniqueWatchId, pendingResponses)));
                 }
             } else if (addWatch) {
-                asyncActions.add(() -> putWatch(watcher, watchId, uniqueWatchId, pendingResponses));
+                asyncActions.add(() -> putWatch(client, watchId, uniqueWatchId, pendingResponses));
             }
         }
     }
 
-    private void putWatch(final WatcherClient watcher, final String watchId, final String uniqueWatchId,
+    private void putWatch(final Client client, final String watchId, final String uniqueWatchId,
                           final AtomicInteger pendingResponses) {
         final String watch = ClusterAlertsUtil.loadWatch(clusterService, watchId);
 
         logger.trace("adding monitoring watch [{}]", uniqueWatchId);
 
-        executeAsyncWithOrigin(client.threadPool().getThreadContext(), MONITORING_ORIGIN,
+        executeAsyncWithOrigin(client, MONITORING_ORIGIN, PutWatchAction.INSTANCE,
                 new PutWatchRequest(uniqueWatchId, new BytesArray(watch), XContentType.JSON),
-                new ResponseActionListener<PutWatchResponse>("watch", uniqueWatchId, pendingResponses, watcherSetup),
-                watcher::putWatch);
+                new ResponseActionListener<>("watch", uniqueWatchId, pendingResponses, watcherSetup));
     }
 
     /**
@@ -616,15 +613,15 @@ public void onFailure(Exception e) {
 
     private class GetAndPutWatchResponseActionListener implements ActionListener<GetWatchResponse> {
 
-        private final WatcherClient watcher;
+        private final Client client;
         private final String watchId;
         private final String uniqueWatchId;
         private final AtomicInteger countDown;
 
-        private GetAndPutWatchResponseActionListener(final WatcherClient watcher,
+        private GetAndPutWatchResponseActionListener(final Client client,
                                                      final String watchId, final String uniqueWatchId,
                                                      final AtomicInteger countDown) {
-            this.watcher = Objects.requireNonNull(watcher);
+            this.client = Objects.requireNonNull(client);
             this.watchId = Objects.requireNonNull(watchId);
             this.uniqueWatchId = Objects.requireNonNull(uniqueWatchId);
             this.countDown = Objects.requireNonNull(countDown);
@@ -638,7 +635,7 @@ public void onResponse(GetWatchResponse response) {
 
                 responseReceived(countDown, true, watcherSetup);
             } else {
-                putWatch(watcher, watchId, uniqueWatchId, countDown);
+                putWatch(client, watchId, uniqueWatchId, countDown);
             }
         }
 
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java
index 0a1576393ba26..04faf82d8c1d1 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java
@@ -8,21 +8,22 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
-import org.elasticsearch.xpack.core.XPackClient;
 import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkRequestBuilder;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkResponse;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.core.rest.XPackRestHandler;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -34,7 +35,7 @@
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 
-public class RestMonitoringBulkAction extends XPackRestHandler {
+public class RestMonitoringBulkAction extends BaseRestHandler {
 
     public static final String MONITORING_ID = "system_id";
     public static final String MONITORING_VERSION = "system_api_version";
@@ -68,7 +69,7 @@ public String getName() {
     }
 
     @Override
-    public RestChannelConsumer doPrepareRequest(RestRequest request, XPackClient client) throws IOException {
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
 
         final String id = request.param(MONITORING_ID);
         if (Strings.isEmpty(id)) {
@@ -98,27 +99,9 @@ public RestChannelConsumer doPrepareRequest(RestRequest request, XPackClient cli
         final long timestamp = System.currentTimeMillis();
         final long intervalMillis = parseTimeValue(intervalAsString, INTERVAL).getMillis();
 
-        final MonitoringBulkRequestBuilder requestBuilder = client.monitoring().prepareMonitoringBulk();
+        final MonitoringBulkRequestBuilder requestBuilder = new MonitoringBulkRequestBuilder(client);
         requestBuilder.add(system, request.content(), request.getXContentType(), timestamp, intervalMillis);
-        return channel -> requestBuilder.execute(new RestBuilderListener<MonitoringBulkResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(MonitoringBulkResponse response, XContentBuilder builder) throws Exception {
-                builder.startObject();
-                {
-                    builder.field("took", response.getTookInMillis());
-                    builder.field("ignored", response.isIgnored());
-
-                    final MonitoringBulkResponse.Error error = response.getError();
-                    builder.field("errors", error != null);
-
-                    if (error != null) {
-                        builder.field("error", response.getError());
-                    }
-                }
-                builder.endObject();
-                return new BytesRestResponse(response.status(), builder);
-            }
-        });
+        return channel -> requestBuilder.execute(getRestBuilderListener(channel));
     }
 
     @Override
@@ -138,4 +121,26 @@ private boolean isSupportedSystemVersion(final MonitoredSystem system, final Str
         final List<String> monitoredSystem = supportedApiVersions.getOrDefault(system, emptyList());
         return monitoredSystem.contains(version);
     }
+
+    static RestBuilderListener<MonitoringBulkResponse> getRestBuilderListener(RestChannel channel) {
+        return new RestBuilderListener<>(channel) {
+            @Override
+            public RestResponse buildResponse(MonitoringBulkResponse response, XContentBuilder builder) throws Exception {
+                builder.startObject();
+                {
+                    builder.field("took", response.getTookInMillis());
+                    builder.field("ignored", response.isIgnored());
+
+                    final MonitoringBulkResponse.Error error = response.getError();
+                    builder.field("errors", error != null);
+
+                    if (error != null) {
+                        builder.field("error", response.getError());
+                    }
+                }
+                builder.endObject();
+                return new BytesRestResponse(response.status(), builder);
+            }
+        };
+    }
 }
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/LocalStateMonitoring.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/LocalStateMonitoring.java
index 8e0346270ed47..3ab34652e229e 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/LocalStateMonitoring.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/LocalStateMonitoring.java
@@ -5,18 +5,45 @@
  */
 package org.elasticsearch.xpack.monitoring;
 
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.action.TransportXPackUsageAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageResponse;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.indexlifecycle.IndexLifecycle;
 import org.elasticsearch.xpack.watcher.Watcher;
 
 import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
 
 public class LocalStateMonitoring extends LocalStateCompositeXPackPlugin {
 
+    public static class MonitoringTransportXPackUsageAction extends TransportXPackUsageAction {
+        @Inject
+        public MonitoringTransportXPackUsageAction(ThreadPool threadPool, TransportService transportService,
+                                         ClusterService clusterService, ActionFilters actionFilters,
+                                         IndexNameExpressionResolver indexNameExpressionResolver, NodeClient client) {
+            super(threadPool, transportService, clusterService, actionFilters, indexNameExpressionResolver, client);
+        }
+        @Override
+        protected List<XPackUsageFeatureAction> usageActions() {
+            return Collections.singletonList(XPackUsageFeatureAction.MONITORING);
+        }
+    }
+
     public LocalStateMonitoring(final Settings settings, final Path configPath) throws Exception {
         super(settings, configPath);
         LocalStateMonitoring thisVar = this;
@@ -50,4 +77,9 @@ protected XPackLicenseState getLicenseState() {
         });
         plugins.add(new IndexLifecycle(settings));
     }
+
+    @Override
+    protected Class<? extends TransportAction<XPackUsageRequest, XPackUsageResponse>> getUsageAction() {
+        return MonitoringTransportXPackUsageAction.class;
+    }
 }
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringFeatureSetTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java
similarity index 82%
rename from x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringFeatureSetTests.java
rename to x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java
index ff545549e80ef..920b201bfc97b 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringFeatureSetTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringInfoTransportActionTests.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.monitoring;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -17,8 +18,9 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.rest.yaml.ObjectPath;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.XPackFeatureSet.Usage;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.monitoring.MonitoringFeatureSetUsage;
 import org.elasticsearch.xpack.monitoring.exporter.Exporter;
 import org.elasticsearch.xpack.monitoring.exporter.Exporters;
@@ -36,14 +38,15 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class MonitoringFeatureSetTests extends ESTestCase {
+public class MonitoringInfoTransportActionTests extends ESTestCase {
 
     private final MonitoringService monitoring = mock(MonitoringService.class);
     private final XPackLicenseState licenseState = mock(XPackLicenseState.class);
     private final Exporters exporters = mock(Exporters.class);
 
     public void testAvailable() {
-        MonitoringFeatureSet featureSet = new MonitoringFeatureSet(Settings.EMPTY, monitoring, licenseState, exporters);
+        MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
         boolean available = randomBoolean();
         when(licenseState.isMonitoringAllowed()).thenReturn(available);
         assertThat(featureSet.available(), is(available));
@@ -53,12 +56,14 @@ public void testEnabledSetting() {
         boolean enabled = randomBoolean();
         Settings.Builder settings = Settings.builder();
         settings.put("xpack.monitoring.enabled", enabled);
-        MonitoringFeatureSet featureSet = new MonitoringFeatureSet(settings.build(), monitoring, licenseState, exporters);
+        MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState);
         assertThat(featureSet.enabled(), is(enabled));
     }
 
     public void testEnabledDefault() {
-        MonitoringFeatureSet featureSet = new MonitoringFeatureSet(Settings.EMPTY, monitoring, licenseState, exporters);
+        MonitoringInfoTransportAction featureSet = new MonitoringInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
         assertThat(featureSet.enabled(), is(true));
     }
 
@@ -97,10 +102,12 @@ public void testUsage() throws Exception {
         when(exporters.getEnabledExporters()).thenReturn(exporterList);
         when(monitoring.isMonitoringActive()).thenReturn(collectionEnabled);
 
-        MonitoringFeatureSet featureSet = new MonitoringFeatureSet(Settings.EMPTY, monitoring, licenseState, exporters);
-        PlainActionFuture<Usage> future = new PlainActionFuture<>();
-        featureSet.usage(future);
-        XPackFeatureSet.Usage monitoringUsage = future.get();
+        var usageAction = new MonitoringUsageTransportAction(mock(TransportService.class), null, null,
+            mock(ActionFilters.class), null, Settings.EMPTY,licenseState,
+            new MonitoringUsageServices(monitoring, exporters));
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, null, future);
+        MonitoringFeatureSetUsage monitoringUsage = (MonitoringFeatureSetUsage) future.get().getUsage();
         BytesStreamOutput out = new BytesStreamOutput();
         out.setVersion(serializedVersion);
         monitoringUsage.writeTo(out);
@@ -108,8 +115,6 @@ public void testUsage() throws Exception {
         in.setVersion(serializedVersion);
         XPackFeatureSet.Usage serializedUsage = new MonitoringFeatureSetUsage(in);
         for (XPackFeatureSet.Usage usage : Arrays.asList(monitoringUsage, serializedUsage)) {
-            assertThat(usage.name(), is(featureSet.name()));
-            assertThat(usage.enabled(), is(featureSet.enabled()));
             ObjectPath  source;
             try (XContentBuilder builder = jsonBuilder()) {
                 usage.toXContent(builder, ToXContent.EMPTY_PARAMS);
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
index 94b205b3d85d1..a4e76d60f041c 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -175,7 +176,7 @@ public void testNodesHash() {
                                   randomAlphaOfLength(5),
                                   new TransportAddress(TransportAddress.META_ADDRESS, 9301 + i),
                                   randomBoolean() ? singletonMap("attr", randomAlphaOfLength(3)) : emptyMap,
-                                  singleton(randomFrom(DiscoveryNode.Role.values())),
+                                  singleton(randomFrom(DiscoveryNodeRole.BUILT_IN_ROLES)),
                                   Version.CURRENT));
         }
 
@@ -201,7 +202,7 @@ public void testToXContent() throws IOException {
                                                                 "_host_address",
                                                                 transportAddress,
                                                                 singletonMap("attr", "value"),
-                                                                singleton(DiscoveryNode.Role.MASTER),
+                                                                singleton(DiscoveryNodeRole.MASTER_ROLE),
                                                                 Version.CURRENT);
 
         final ClusterState clusterState = ClusterState.builder(clusterName)
@@ -441,10 +442,10 @@ public void testToXContent() throws IOException {
                     + "\"nodes\":{"
                       + "\"count\":{"
                         + "\"total\":1,"
-                        + "\"data\":0,"
                         + "\"coordinating_only\":0,"
-                        + "\"master\":1,"
-                        + "\"ingest\":0"
+                        + "\"data\":0,"
+                        + "\"ingest\":0,"
+                        + "\"master\":1"
                       + "},"
                       + "\"versions\":["
                         + "\"" + mockNodeVersion + "\""
@@ -586,7 +587,7 @@ private DiscoveryNode masterNode() {
                                  "_host_address",
                                  new TransportAddress(TransportAddress.META_ADDRESS, 9300),
                                  singletonMap("attr", "value"),
-                                 singleton(DiscoveryNode.Role.MASTER),
+                                 singleton(DiscoveryNodeRole.MASTER_ROLE),
                                  Version.CURRENT);
     }
 
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryMonitoringDocTests.java
index 08e04725c2e0b..f018828c3a46d 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryMonitoringDocTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
@@ -78,7 +79,7 @@ public void testToXContent() throws IOException {
                                                                     "_host_address_0",
                                                                     new TransportAddress(TransportAddress.META_ADDRESS, 9300),
                                                                     singletonMap("attr", "value_0"),
-                                                                    singleton(DiscoveryNode.Role.MASTER),
+                                                                    singleton(DiscoveryNodeRole.MASTER_ROLE),
                                                                     Version.CURRENT);
 
         final DiscoveryNode discoveryNodeOne = new DiscoveryNode("_node_1",
@@ -88,7 +89,7 @@ public void testToXContent() throws IOException {
                                                                     "_host_address_1",
                                                                     new TransportAddress(TransportAddress.META_ADDRESS, 9301),
                                                                     singletonMap("attr", "value_1"),
-                                                                    singleton(DiscoveryNode.Role.DATA),
+                                                                    singleton(DiscoveryNodeRole.DATA_ROLE),
                                                                     Version.CURRENT.minimumIndexCompatibilityVersion());
 
         final ShardId shardId = new ShardId("_index_a", "_uuid_a", 0);
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsCollectorTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsCollectorTests.java
index 0713e26f80667..61b485e77ae27 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsCollectorTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsCollectorTests.java
@@ -6,16 +6,17 @@
 package org.elasticsearch.xpack.monitoring.collector.ml;
 
 import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.util.QueryPage;
+import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction;
 import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction.Request;
 import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction.Response;
 import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction.Response.JobStats;
-import org.elasticsearch.xpack.core.action.util.QueryPage;
-import org.elasticsearch.xpack.core.ml.client.MachineLearningClient;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc;
@@ -128,7 +129,7 @@ public void testDoCollect() throws Exception {
         whenClusterStateWithUUID(clusterUuid);
 
         final MonitoringDoc.Node node = randomMonitoringNode(random());
-        final MachineLearningClient client = mock(MachineLearningClient.class);
+        final Client client = mock(Client.class);
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
 
         final TimeValue timeout = TimeValue.timeValueSeconds(randomIntBetween(1, 120));
@@ -143,7 +144,7 @@ public void testDoCollect() throws Exception {
         final ActionFuture<Response> future = (ActionFuture<Response>)mock(ActionFuture.class);
         final Response response = new Response(new QueryPage<>(jobStats, jobStats.size(), Job.RESULTS_FIELD));
 
-        when(client.getJobsStats(eq(new Request(MetaData.ALL)))).thenReturn(future);
+        when(client.execute(eq(GetJobsStatsAction.INSTANCE), eq(new Request(MetaData.ALL)))).thenReturn(future);
         when(future.actionGet(timeout)).thenReturn(response);
 
         final long interval = randomNonNegativeLong();
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsMonitoringDocTests.java
index 2bc3ac48b99ec..dd64e0d2ded95 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsMonitoringDocTests.java
@@ -7,6 +7,7 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
@@ -16,6 +17,7 @@
 import org.elasticsearch.xpack.core.ml.job.config.JobState;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats;
+import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
 import org.elasticsearch.xpack.core.ml.stats.ForecastStats;
 import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc;
@@ -85,7 +87,7 @@ public void testToXContent() throws IOException {
                                                              "_host_address",
                                                              new TransportAddress(TransportAddress.META_ADDRESS, 9300),
                                                              singletonMap("attr", "value"),
-                                                             singleton(DiscoveryNode.Role.MASTER),
+                                                             singleton(DiscoveryNodeRole.MASTER_ROLE),
                                                              Version.CURRENT);
 
         final ModelSizeStats modelStats = new ModelSizeStats.Builder("_model")
@@ -101,8 +103,9 @@ public void testToXContent() throws IOException {
 
         final DataCounts dataCounts = new DataCounts("_job_id", 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, date3, date4, date5, date6, date7);
         final ForecastStats forecastStats = new ForecastStats();
-        final JobStats jobStats = new JobStats("_job", dataCounts, modelStats, forecastStats, JobState.OPENED, discoveryNode,
-                "_explanation", time);
+        final TimingStats timingStats = new TimingStats("_job_id", 100, 10.0, 30.0, 20.0, 25.0);
+        final JobStats jobStats = new JobStats(
+            "_job", dataCounts, modelStats, forecastStats, JobState.OPENED, discoveryNode, "_explanation", time, timingStats);
         final MonitoringDoc.Node node = new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L);
 
         final JobStatsMonitoringDoc document = new JobStatsMonitoringDoc("_cluster", 1502266739402L, 1506593717631L, node, jobStats);
@@ -168,8 +171,16 @@ public void testToXContent() throws IOException {
                           + "}"
                         + "},"
                        + "\"assignment_explanation\":\"_explanation\","
-                       + "\"open_time\":\"13h\""
-                      + "}"
+                       + "\"open_time\":\"13h\","
+                       + "\"timing_stats\":{"
+                        + "\"job_id\":\"_job_id\","
+                        + "\"bucket_count\":100,"
+                        + "\"minimum_bucket_processing_time_ms\":10.0,"
+                        + "\"maximum_bucket_processing_time_ms\":30.0,"
+                        + "\"average_bucket_processing_time_ms\":20.0,"
+                        + "\"exponential_average_bucket_processing_time_ms\":25.0"
+                       + "}"
+                     + "}"
                     + "}", xContent.utf8ToString());
     }
 }
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java
index 4296bedfe0da3..2906adac892b0 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java
@@ -329,7 +329,7 @@ private static NodeStats mockNodeStats() {
         final OsStats.Cgroup osCgroup = new OsStats.Cgroup("_cpu_acct_ctrl_group", ++iota, "_cpu_ctrl_group", ++iota, ++iota, osCpuStat,
                 "_memory_ctrl_group", "2000000000", "1000000000");
 
-        final OsStats.Mem osMem = new OsStats.Mem(no, no);
+        final OsStats.Mem osMem = new OsStats.Mem(0, 0);
         final OsStats.Swap osSwap = new OsStats.Swap(no, no);
         final OsStats os = new OsStats(no, osCpu, osMem, osSwap, osCgroup);
 
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterSslIT.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterSslIT.java
index 34885563c87ac..19a981939c480 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterSslIT.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterSslIT.java
@@ -203,7 +203,8 @@ private static List<String> getProtocols() {
             return List.of("TLSv1.2");
         } else {
             JavaVersion full =
-                AccessController.doPrivileged((PrivilegedAction<JavaVersion>) () -> JavaVersion.parse(System.getProperty("java.version")));
+                AccessController.doPrivileged(
+                        (PrivilegedAction<JavaVersion>) () -> JavaVersion.parse(System.getProperty("java.specification.version")));
             if (full.compareTo(JavaVersion.parse("12.0.1")) < 0) {
                 return List.of("TLSv1.2");
             }
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java
index 5db71f72cf6ef..898f8cb491e6c 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java
@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.monitoring.exporter.local;
 
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.ingest.GetPipelineResponse;
@@ -105,12 +104,12 @@ public void testExport() throws Exception {
                 }
 
                 assertBusy(() -> {
-                    MonitoringBulkRequestBuilder bulk = monitoringClient().prepareMonitoringBulk();
+                    MonitoringBulkRequestBuilder bulk = new MonitoringBulkRequestBuilder(client());
                     monitoringDocs.forEach(bulk::add);
                     assertEquals(RestStatus.OK, bulk.get().status());
                     refresh();
 
-                    assertThat(client().admin().indices().prepareExists(".monitoring-*").get().isExists(), is(true));
+                    assertThat(indexExists(".monitoring-*"), is(true));
                     ensureYellowAndNoInitializingShards(".monitoring-*");
 
                     SearchResponse response = client().prepareSearch(".monitoring-*").get();
@@ -124,7 +123,7 @@ public void testExport() throws Exception {
 
             final int numNodes = internalCluster().getNodeNames().length;
             assertBusy(() -> {
-                assertThat(client().admin().indices().prepareExists(".monitoring-*").get().isExists(), is(true));
+                assertThat(indexExists(".monitoring-*"), is(true));
                 ensureYellowAndNoInitializingShards(".monitoring-*");
 
                 assertThat(client().prepareSearch(".monitoring-es-*")
@@ -184,8 +183,7 @@ public void testExport() throws Exception {
         final int elapsedInSeconds = 10;
         final ZonedDateTime startTime = ZonedDateTime.now(ZoneOffset.UTC);
         assertBusy(() -> {
-            IndicesExistsResponse indicesExistsResponse = client().admin().indices().prepareExists(".monitoring-*").get();
-            if (indicesExistsResponse.isExists()) {
+            if (indexExists(".monitoring-*")) {
                 ensureYellowAndNoInitializingShards(".monitoring-*");
                 refresh(".monitoring-es-*");
 
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkActionTests.java
index 7a4427c9f0fdc..fb79751f797e6 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkActionTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkActionTests.java
@@ -6,8 +6,7 @@
 package org.elasticsearch.xpack.monitoring.rest.action;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -25,22 +24,16 @@
 import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
-import org.elasticsearch.xpack.core.XPackClient;
 import org.elasticsearch.xpack.core.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkRequestBuilder;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkResponse;
-import org.elasticsearch.xpack.core.monitoring.client.MonitoringClient;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
 
 import java.util.HashMap;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_VERSION;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -115,8 +108,7 @@ public void testUnknownSystemVersion() {
 
     public void testNoErrors() throws Exception {
         final MonitoringBulkResponse response = new MonitoringBulkResponse(randomLong(), false);
-        final FakeRestRequest request = createRestRequest(randomSystemId(), TEMPLATE_VERSION, "10s");
-        final RestResponse restResponse = getRestBuilderListener(request).buildResponse(response);
+        final RestResponse restResponse = getRestBuilderListener().buildResponse(response);
 
         assertThat(restResponse.status(), is(RestStatus.OK));
         assertThat(restResponse.content().utf8ToString(),
@@ -125,8 +117,7 @@ public void testNoErrors() throws Exception {
 
     public void testNoErrorsButIgnored() throws Exception {
         final MonitoringBulkResponse response = new MonitoringBulkResponse(randomLong(), true);
-        final FakeRestRequest request = createRestRequest(randomSystemId(), TEMPLATE_VERSION, "10s");
-        final RestResponse restResponse = getRestBuilderListener(request).buildResponse(response);
+        final RestResponse restResponse = getRestBuilderListener().buildResponse(response);
 
         assertThat(restResponse.status(), is(RestStatus.OK));
         assertThat(restResponse.content().utf8ToString(),
@@ -139,8 +130,7 @@ public void testWithErrors() throws Exception {
         final MonitoringBulkResponse response = new MonitoringBulkResponse(randomLong(), error);
         final String errorJson;
 
-        final FakeRestRequest request = createRestRequest(randomSystemId(), TEMPLATE_VERSION, "10s");
-        final RestResponse restResponse = getRestBuilderListener(request).buildResponse(response);
+        final RestResponse restResponse = getRestBuilderListener().buildResponse(response);
 
         try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
             error.toXContent(builder, ToXContent.EMPTY_PARAMS);
@@ -167,35 +157,18 @@ private static String randomSystemId() {
     }
 
     private void prepareRequest(final RestRequest restRequest) throws Exception {
-        getRestBuilderListener(restRequest);
-    }
-
-    private RestBuilderListener<MonitoringBulkResponse> getRestBuilderListener(final RestRequest restRequest) throws Exception {
-        final Client client = mock(Client.class);
-        final XPackClient xpackClient = mock(XPackClient.class);
-        final MonitoringClient monitoringClient = mock(MonitoringClient.class);
-        final AtomicReference<RestBuilderListener<MonitoringBulkResponse>> listenerReference = new AtomicReference<>();
-        final MonitoringBulkRequestBuilder builder = new MonitoringBulkRequestBuilder(client){
-            @SuppressWarnings("unchecked")
-            @Override
-            public void execute(ActionListener<MonitoringBulkResponse> listener) {
-                listenerReference.set((RestBuilderListener)listener);
-            }
-        };
-        when(monitoringClient.prepareMonitoringBulk()).thenReturn(builder);
-        when(xpackClient.monitoring()).thenReturn(monitoringClient);
-
-        final CheckedConsumer<RestChannel, Exception> consumer = action.doPrepareRequest(restRequest, xpackClient);
-
+        final NodeClient client = mock(NodeClient.class);
+        final CheckedConsumer<RestChannel, Exception> consumer = action.prepareRequest(restRequest, client);
         final RestChannel channel = mock(RestChannel.class);
         when(channel.newBuilder()).thenReturn(JsonXContent.contentBuilder());
-
-        // trigger/capture execution
+        // trigger execution
         consumer.accept(channel);
+    }
 
-        assertThat(listenerReference.get(), not(nullValue()));
-
-        return listenerReference.get();
+    private RestBuilderListener<MonitoringBulkResponse> getRestBuilderListener() throws Exception {
+        final RestChannel channel = mock(RestChannel.class);
+        when(channel.newBuilder()).thenReturn(JsonXContent.contentBuilder());
+        return RestMonitoringBulkAction.getRestBuilderListener(channel);
     }
 
     private static FakeRestRequest createRestRequest(final String systemId, final String systemApiVersion, final String interval) {
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java
index c350b9a374ab2..22f3b59220998 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java
@@ -19,8 +19,6 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.store.MockFSIndexStore;
 import org.elasticsearch.test.transport.MockTransportService;
-import org.elasticsearch.xpack.core.XPackClient;
-import org.elasticsearch.xpack.core.monitoring.client.MonitoringClient;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
 import org.elasticsearch.xpack.core.monitoring.test.MockPainlessScriptEngine;
 import org.elasticsearch.xpack.monitoring.LocalStateMonitoring;
@@ -75,10 +73,6 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
                 MockIngestPlugin.class, CommonAnalysisPlugin.class);
     }
 
-    protected MonitoringClient monitoringClient() {
-        return randomBoolean() ? new XPackClient(client()).monitoring() : new MonitoringClient(client());
-    }
-
     @Override
     protected Set<String> excludeTemplates() {
         return new HashSet<>(monitoringTemplateNames());
@@ -108,9 +102,7 @@ protected void wipeMonitoringIndices() throws Exception {
         CountDown retries = new CountDown(3);
         assertBusy(() -> {
             try {
-                boolean exist = client().admin().indices().prepareExists(ALL_MONITORING_INDICES)
-                        .get().isExists();
-                if (exist) {
+                if (indexExists(ALL_MONITORING_INDICES)) {
                     deleteMonitoringIndices();
                 } else {
                     retries.countDown();
@@ -192,7 +184,9 @@ private void awaitIndexExists(final String index) throws Exception {
 
     private void assertIndicesExists(String... indices) {
         logger.trace("checking if index exists [{}]", Strings.arrayToCommaDelimitedString(indices));
-        assertThat(client().admin().indices().prepareExists(indices).get().isExists(), is(true));
+        for (String index : indices) {
+            assertThat(indexExists(index), is(true));
+        }
     }
 
     protected void enableMonitoringCollection() {
@@ -204,5 +198,4 @@ protected void disableMonitoringCollection() {
         assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(
                     Settings.builder().putNull(MonitoringService.ENABLED.getKey())));
     }
-
 }
diff --git a/x-pack/plugin/rollup/build.gradle b/x-pack/plugin/rollup/build.gradle
index ae53f78dad2ff..39afbd4349606 100644
--- a/x-pack/plugin/rollup/build.gradle
+++ b/x-pack/plugin/rollup/build.gradle
@@ -14,9 +14,8 @@ compileTestJava.options.compilerArgs << "-Xlint:-rawtypes"
 
 
 dependencies {
-    compileOnly "org.elasticsearch:elasticsearch:${version}"
-
-    // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
+    compileOnly project(":server")
+    
     compileOnly project(path: xpackModule('core'), configuration: 'default')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     if (isEclipse) {
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java
index faa713efb7d11..20c8fb4502891 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/Rollup.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -23,7 +22,6 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
-import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.persistent.PersistentTasksExecutor;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.PersistentTaskPlugin;
@@ -35,8 +33,9 @@
 import org.elasticsearch.threadpool.FixedExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
-import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.rollup.RollupField;
 import org.elasticsearch.xpack.core.rollup.action.DeleteRollupJobAction;
 import org.elasticsearch.xpack.core.rollup.action.GetRollupCapsAction;
@@ -67,7 +66,6 @@
 import org.elasticsearch.xpack.rollup.rest.RestStopRollupJobAction;
 
 import java.time.Clock;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -122,15 +120,6 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
         return emptyList();
     }
 
-    @Override
-    public Collection<Module> createGuiceModules() {
-        List<Module> modules = new ArrayList<>();
-        modules.add(b -> XPackPlugin.bindFeatureSet(b, RollupFeatureSet.class));
-        return modules;
-    }
-
-    protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); }
-
     @Override
     public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
                                              IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
@@ -155,8 +144,10 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
 
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
-        if (!enabled) {
-            return emptyList();
+        var usageAction = new ActionHandler<>(XPackUsageFeatureAction.ROLLUP, RollupUsageTransportAction.class);
+        var infoAction = new ActionHandler<>(XPackInfoFeatureAction.ROLLUP, RollupInfoTransportAction.class);
+        if (enabled == false) {
+            return Arrays.asList(usageAction, infoAction);
         }
         return Arrays.asList(
             new ActionHandler<>(RollupSearchAction.INSTANCE, TransportRollupSearchAction.class),
@@ -166,8 +157,9 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
             new ActionHandler<>(DeleteRollupJobAction.INSTANCE, TransportDeleteRollupJobAction.class),
             new ActionHandler<>(GetRollupJobsAction.INSTANCE, TransportGetRollupJobAction.class),
             new ActionHandler<>(GetRollupCapsAction.INSTANCE, TransportGetRollupCapsAction.class),
-            new ActionHandler<>(GetRollupIndexCapsAction.INSTANCE, TransportGetRollupIndexCapsAction.class)
-        );
+            new ActionHandler<>(GetRollupIndexCapsAction.INSTANCE, TransportGetRollupIndexCapsAction.class),
+            usageAction,
+            infoAction);
     }
 
     @Override
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupFeatureSet.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupInfoTransportAction.java
similarity index 50%
rename from x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupFeatureSet.java
rename to x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupInfoTransportAction.java
index 780b5fa6ffa8b..93330872554cc 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupFeatureSet.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupInfoTransportAction.java
@@ -5,25 +5,25 @@
  */
 package org.elasticsearch.xpack.rollup;
 
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.common.Nullable;
+import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.rollup.RollupFeatureSetUsage;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
 
-import java.util.Map;
-
-public class RollupFeatureSet implements XPackFeatureSet {
+public class RollupInfoTransportAction extends XPackInfoFeatureTransportAction {
 
     private final boolean enabled;
     private final XPackLicenseState licenseState;
 
     @Inject
-    public RollupFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState) {
+    public RollupInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                     Settings settings, XPackLicenseState licenseState) {
+        super(XPackInfoFeatureAction.ROLLUP.name(), transportService, actionFilters);
         this.enabled = XPackSettings.ROLLUP_ENABLED.get(settings);
         this.licenseState = licenseState;
     }
@@ -33,14 +33,9 @@ public String name() {
         return XPackField.ROLLUP;
     }
 
-    @Override
-    public String description() {
-        return "Time series pre-aggregation and rollup";
-    }
-
     @Override
     public boolean available() {
-        return licenseState != null && licenseState.isRollupAllowed();
+        return licenseState.isRollupAllowed();
     }
 
     @Override
@@ -48,14 +43,4 @@ public boolean enabled() {
         return enabled;
     }
 
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return null;
-    }
-
-    @Override
-    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
-        // TODO expose the currently running rollup tasks on this node?  Unclear the best way to do that
-        listener.onResponse(new RollupFeatureSetUsage(available(), enabled()));
-    }
 }
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupUsageTransportAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupUsageTransportAction.java
new file mode 100644
index 0000000000000..5253c05517e84
--- /dev/null
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/RollupUsageTransportAction.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.rollup;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.rollup.RollupFeatureSetUsage;
+
+public class RollupUsageTransportAction extends XPackUsageFeatureTransportAction {
+
+    private final Settings settings;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public RollupUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                      ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                      Settings settings, XPackLicenseState licenseState) {
+        super(XPackUsageFeatureAction.ROLLUP.name(), transportService, clusterService,
+            threadPool, actionFilters, indexNameExpressionResolver);
+        this.settings = settings;
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
+        boolean available = licenseState.isRollupAllowed();
+        // TODO expose the currently running rollup tasks on this node?  Unclear the best way to do that
+        RollupFeatureSetUsage usage =
+            new RollupFeatureSetUsage(available, XPackSettings.ROLLUP_ENABLED.get(settings));
+        listener.onResponse(new XPackUsageFeatureResponse(usage));
+    }
+}
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java
index db6c1c5ddeaa7..571d37e9652ba 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.common.unit.TimeValue;
@@ -42,6 +43,7 @@
 import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackField;
@@ -52,6 +54,7 @@
 import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig;
 import org.elasticsearch.xpack.rollup.Rollup;
 
+import java.io.IOException;
 import java.util.Map;
 import java.util.Objects;
 import java.util.stream.Collectors;
@@ -80,13 +83,18 @@ protected String executor() {
         return ThreadPool.Names.SAME;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(PutRollupJobAction.Request request, ClusterState clusterState,
+    protected void masterOperation(Task task, PutRollupJobAction.Request request, ClusterState clusterState,
                                    ActionListener<AcknowledgedResponse> listener) {
 
         if (!licenseState.isRollupAllowed()) {
@@ -100,6 +108,7 @@ protected void masterOperation(PutRollupJobAction.Request request, ClusterState
         FieldCapabilitiesRequest fieldCapsRequest = new FieldCapabilitiesRequest()
             .indices(request.getConfig().getIndexPattern())
             .fields(request.getConfig().getAllFields().toArray(new String[0]));
+        fieldCapsRequest.setParentTask(clusterService.localNode().getId(), task.getId());
 
         client.fieldCaps(fieldCapsRequest, new ActionListener<FieldCapabilitiesResponse>() {
             @Override
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
index 2a1308353d6ad..9542f2129bcbf 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java
@@ -161,7 +161,7 @@ static MultiSearchRequest createMSearchRequest(SearchRequest request, NamedWrite
             // Note: we can't apply any query rewriting or filtering on the query because there
             // are no validated caps, so we have no idea what job is intended here.  The only thing
             // this affects is doc count, since hits and aggs will both be empty it doesn't really matter.
-            msearch.add(new SearchRequest(context.getRollupIndices(), request.source()).types(request.types()));
+            msearch.add(new SearchRequest(context.getRollupIndices(), request.source()));
             return msearch;
         }
 
@@ -208,7 +208,7 @@ static MultiSearchRequest createMSearchRequest(SearchRequest request, NamedWrite
                         new long[]{Rollup.ROLLUP_VERSION_V1, Rollup.ROLLUP_VERSION_V2})));
 
             // And add a new msearch per JobID
-            msearch.add(new SearchRequest(context.getRollupIndices(), copiedSource).types(request.types()));
+            msearch.add(new SearchRequest(context.getRollupIndices(), copiedSource));
         }
 
         return msearch;
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/IndexerUtils.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/IndexerUtils.java
index d8dbc0ceade7a..4e3789afa0133 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/IndexerUtils.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/IndexerUtils.java
@@ -5,7 +5,8 @@
  */
 package org.elasticsearch.xpack.rollup.job;
 
-import org.apache.log4j.Logger;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.search.aggregations.Aggregation;
@@ -32,7 +33,7 @@
  * They are extracted out as static classes mainly to make testing easier.
  */
 class IndexerUtils {
-    private static final Logger logger = Logger.getLogger(IndexerUtils.class.getName());
+    private static final Logger logger = LogManager.getLogger(IndexerUtils.class);
 
     /**
      * The only entry point in this class.  You hand this method an aggregation and an index
diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java
index 038c69365c839..0421f72aaf5b8 100644
--- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java
+++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/job/RollupJobTask.java
@@ -5,7 +5,8 @@
  */
 package org.elasticsearch.xpack.rollup.job;
 
-import org.apache.log4j.Logger;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.bulk.BulkAction;
@@ -45,7 +46,7 @@
  * Each RollupJobTask also registers itself into the Scheduler so that it can be triggered on the cron's interval.
  */
 public class RollupJobTask extends AllocatedPersistentTask implements SchedulerEngine.Listener {
-    private static final Logger logger = Logger.getLogger(RollupJobTask.class.getName());
+    private static final Logger logger = LogManager.getLogger(RollupJobTask.class.getName());
 
     static final String SCHEDULE_NAME = RollupField.TASK_NAME + "/schedule";
 
@@ -210,8 +211,10 @@ public RollupJobConfig getConfig() {
     }
 
     /**
-     * Attempt to start the indexer.  If the state is anything other than STOPPED, this will fail.
-     * Otherwise, the persistent task's status will be updated to reflect the change.
+     * Attempt to start the indexer.
+     * - If the indexer is started/indexing, returns OK
+     * - If the indexer is stopped, starts task, updates persistent task's status, returns ok
+     * - Anything else returns error
      *
      * Note that while the job is started, the indexer will not necessarily run immediately.  That
      * will only occur when the scheduler triggers it based on the cron
@@ -220,8 +223,14 @@ public RollupJobConfig getConfig() {
      */
     public synchronized void start(ActionListener<StartRollupJobAction.Response> listener) {
         final IndexerState prevState = indexer.getState();
-        if (prevState != IndexerState.STOPPED) {
-            // fails if the task is not STOPPED
+
+        if (prevState == IndexerState.STARTED || prevState == IndexerState.INDEXING) {
+            // We're already running so just return acknowledgement
+            logger.debug("Indexer already running (State: [" + prevState + "]), acknowledging start without change.");
+            listener.onResponse(new StartRollupJobAction.Response(true));
+            return;
+        } else if (prevState != IndexerState.STOPPED) {
+            // if we're not already started/indexing, we must be STOPPED to get started
             listener.onFailure(new ElasticsearchException("Cannot start task for Rollup Job [" + job.getConfig().getId() + "] because"
                     + " state was [" + prevState + "]"));
             return;
@@ -230,11 +239,10 @@ public synchronized void start(ActionListener<StartRollupJobAction.Response> lis
         final IndexerState newState = indexer.start();
         if (newState != IndexerState.STARTED) {
             listener.onFailure(new ElasticsearchException("Cannot start task for Rollup Job [" + job.getConfig().getId() + "] because"
-                    + " state was [" + newState + "]"));
+                    + " new state was [" + newState + "]"));
             return;
         }
 
-
         final RollupJobStatus state = new RollupJobStatus(IndexerState.STARTED, indexer.getPosition());
         logger.debug("Updating state for rollup job [" + job.getConfig().getId() + "] to [" + state.getIndexerState() + "][" +
                 state.getPosition() + "]");
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupFeatureSetTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java
similarity index 58%
rename from x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupFeatureSetTests.java
rename to x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java
index 092fea16f289b..135d67b7eaef3 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupFeatureSetTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupInfoTransportActionTests.java
@@ -5,24 +5,26 @@
  */
 package org.elasticsearch.xpack.rollup;
 
+import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.rollup.RollupFeatureSetUsage;
 import org.junit.Before;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.concurrent.ExecutionException;
 
 import static org.hamcrest.core.Is.is;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class RollupFeatureSetTests extends ESTestCase {
+public class RollupInfoTransportActionTests extends ESTestCase {
     private XPackLicenseState licenseState;
 
     @Before
@@ -31,7 +33,8 @@ public void init() {
     }
 
     public void testAvailable() {
-        RollupFeatureSet featureSet = new RollupFeatureSet(Settings.EMPTY, licenseState);
+        RollupInfoTransportAction featureSet = new RollupInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
         boolean available = randomBoolean();
         when(licenseState.isRollupAllowed()).thenReturn(available);
         assertThat(featureSet.available(), is(available));
@@ -41,27 +44,28 @@ public void testEnabledSetting() {
         boolean enabled = randomBoolean();
         Settings.Builder settings = Settings.builder();
         settings.put("xpack.rollup.enabled", enabled);
-        RollupFeatureSet featureSet = new RollupFeatureSet(settings.build(), licenseState);
+        RollupInfoTransportAction featureSet = new RollupInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState);
         assertThat(featureSet.enabled(), is(enabled));
     }
 
     public void testEnabledDefault() {
-        RollupFeatureSet featureSet = new RollupFeatureSet(Settings.EMPTY, licenseState);
+        RollupInfoTransportAction featureSet = new RollupInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
         assertThat(featureSet.enabled(), is(true));
     }
 
     public void testUsage() throws ExecutionException, InterruptedException, IOException {
-        RollupFeatureSet featureSet = new RollupFeatureSet(Settings.EMPTY, licenseState);
-        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
-        featureSet.usage(future);
-        XPackFeatureSet.Usage rollupUsage = future.get();
+        var usageAction = new RollupUsageTransportAction(mock(TransportService.class), null, null,
+            mock(ActionFilters.class), null, Settings.EMPTY, licenseState);
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, null, future);
+        XPackFeatureSet.Usage rollupUsage = future.get().getUsage();
         BytesStreamOutput out = new BytesStreamOutput();
         rollupUsage.writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new RollupFeatureSetUsage(out.bytes().streamInput());
-        for (XPackFeatureSet.Usage usage : Arrays.asList(rollupUsage, serializedUsage)) {
-            assertThat(usage.name(), is(featureSet.name()));
-            assertThat(usage.enabled(), is(featureSet.enabled()));
-        }
+        assertThat(rollupUsage.name(), is(serializedUsage.name()));
+        assertThat(rollupUsage.enabled(), is(serializedUsage.enabled()));
     }
 
 }
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java
index 27dcc751860bc..cd169c1ae7ce1 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java
@@ -49,7 +49,7 @@ public class RollupRequestTranslationTests extends ESTestCase {
     @Before
     public void setUp() throws Exception {
         super.setUp();
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
         entries.addAll(searchModule.getNamedWriteables());
         namedWriteableRegistry = new NamedWriteableRegistry(entries);
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java
index 3f49609953ea9..19f241440c438 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers;
 import org.elasticsearch.xpack.core.rollup.RollupField;
 import org.elasticsearch.xpack.core.rollup.action.PutRollupJobAction;
@@ -303,7 +304,7 @@ public void testJobAlreadyInMapping() {
         doAnswer(invocation -> {
             GetMappingsResponse response = mock(GetMappingsResponse.class);
             Map<String, Object> m = new HashMap<>(2);
-            m.put(Rollup.ROLLUP_TEMPLATE_VERSION_FIELD, Version.V_6_4_0);
+            m.put(Rollup.ROLLUP_TEMPLATE_VERSION_FIELD, VersionUtils.randomIndexCompatibleVersion(random()));
             m.put(RollupField.ROLLUP_META,
                 Collections.singletonMap(job.getConfig().getId(), job.getConfig()));
             MappingMetaData meta = new MappingMetaData(RollupField.TYPE_NAME,
@@ -344,7 +345,7 @@ public void testAddJobToMapping() {
         doAnswer(invocation -> {
             GetMappingsResponse response = mock(GetMappingsResponse.class);
             Map<String, Object> m = new HashMap<>(2);
-            m.put(Rollup.ROLLUP_TEMPLATE_VERSION_FIELD, Version.V_6_4_0);
+            m.put(Rollup.ROLLUP_TEMPLATE_VERSION_FIELD, VersionUtils.randomIndexCompatibleVersion(random()));
             m.put(RollupField.ROLLUP_META,
                 Collections.singletonMap(unrelatedJob.getId(), unrelatedJob));
             MappingMetaData meta = new MappingMetaData(RollupField.TYPE_NAME,
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java
index 00f2fbd3171d4..a2f843ea7b205 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java
@@ -85,7 +85,7 @@ public class SearchActionTests extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         IndicesModule indicesModule = new IndicesModule(emptyList());
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
         List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
         entries.addAll(indicesModule.getNamedWriteables());
         entries.addAll(searchModule.getNamedWriteables());
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java
index 1fe01a8246267..1bd710abb6d77 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java
@@ -89,7 +89,7 @@ public class RollupIndexerIndexingTests extends AggregatorTestCase {
     private void setup() {
         settings = createIndexSettings();
         queryShardContext = new QueryShardContext(0, settings,
-                null, null, null, null, null,
+                null, null, null, null, null, null,
                 null, null, null, null, () -> 0L, null);
     }
 
diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java
index 85962a5cfdbcf..a32cd975b6411 100644
--- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java
+++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java
@@ -163,17 +163,16 @@ public void testStartWhenStarted() throws InterruptedException {
         assertTrue(((RollupJobStatus)task.getStatus()).getPosition().containsKey("foo"));
 
         CountDownLatch latch = new CountDownLatch(1);
-        task.start(new ActionListener<StartRollupJobAction.Response>() {
+        task.start(new ActionListener<>() {
             @Override
             public void onResponse(StartRollupJobAction.Response response) {
-                fail("Should not have entered onResponse.");
+                assertTrue(response.isStarted());
+                latch.countDown();
             }
 
             @Override
             public void onFailure(Exception e) {
-                assertThat(e.getMessage(), equalTo("Cannot start task for Rollup Job ["
-                    + job.getConfig().getId() + "] because state was [STARTED]"));
-                latch.countDown();
+                fail("Should not have throw exception: " + e.getMessage());
             }
         });
         latch.await(3, TimeUnit.SECONDS);
diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle
index cc5833d20e440..c768b51a9c09f 100644
--- a/x-pack/plugin/security/build.gradle
+++ b/x-pack/plugin/security/build.gradle
@@ -13,7 +13,6 @@ esplugin {
 archivesBaseName = 'x-pack-security'
 
 dependencies {
-    // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
     compileOnly project(path: xpackModule('core'), configuration: 'default')
     compileOnly project(path: ':modules:transport-netty4', configuration: 'runtime')
     compileOnly project(path: ':plugins:transport-nio', configuration: 'runtime')
@@ -152,7 +151,7 @@ task testJar(type: Jar) {
     from sourceSets.test.output
 }
 artifacts {
-    // normal es plugins do not publish the jar but we need to since users need it for Transport Clients and extensions
+    // normal es plugins do not publish the jar but we need to since users need it for extensions
     archives jar
     testArtifacts testJar
 }
diff --git a/x-pack/plugin/security/cli/build.gradle b/x-pack/plugin/security/cli/build.gradle
index 5a95594b292ed..0097adf30f511 100644
--- a/x-pack/plugin/security/cli/build.gradle
+++ b/x-pack/plugin/security/cli/build.gradle
@@ -5,13 +5,12 @@ apply plugin: 'elasticsearch.build'
 archivesBaseName = 'elasticsearch-security-cli'
 
 dependencies {
-    compileOnly "org.elasticsearch:elasticsearch:${version}"
-    // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
+    compileOnly project(":server")
     compileOnly project(path: xpackModule('core'), configuration: 'default')
     compile "org.bouncycastle:bcpkix-jdk15on:${versions.bouncycastle}"
     compile "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}"
     testImplementation 'com.google.jimfs:jimfs:1.1'
-    testCompile "org.elasticsearch.test:framework:${version}"
+    testCompile project(":test:framework")
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     if (isEclipse) {
       testCompile project(path: xpackModule('core-tests'), configuration: 'testArtifacts')
diff --git a/x-pack/plugin/security/qa/basic-enable-security/build.gradle b/x-pack/plugin/security/qa/basic-enable-security/build.gradle
index a21e3c68d3fc4..94bfb73c07679 100644
--- a/x-pack/plugin/security/qa/basic-enable-security/build.gradle
+++ b/x-pack/plugin/security/qa/basic-enable-security/build.gradle
@@ -1,68 +1,60 @@
-import org.elasticsearch.gradle.test.RestIntegTestTask
-
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
   testCompile project(path: xpackModule('core'), configuration: 'default')
   testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
-task integTestNoSecurity(type: RestIntegTestTask) {
-  description = "Run tests against a cluster that doesn't have security"
-}
-tasks.getByName("integTestNoSecurityRunner").configure {
-    systemProperty 'tests.has_security', 'false'
-}
-check.dependsOn(integTestNoSecurity)
-
-task integTestSecurity(type: RestIntegTestTask) {
-  dependsOn integTestNoSecurity
-  description = "Run tests against a cluster that has security"
+integTest {
+    description = "Run tests against a cluster that doesn't have security"
+    runner {
+        systemProperty 'tests.has_security', 'false'
+    }
 }
-tasks.getByName("integTestSecurityRunner").configure {
-    systemProperty 'tests.has_security', 'true'
-}
-check.dependsOn(integTestSecurity)
-
-configure(extensions.findByName("integTestNoSecurityCluster")) {
-  clusterName = "enable-security-on-basic"
-  numNodes = 2
-
-  setting 'xpack.ilm.enabled', 'false'
-  setting 'xpack.ml.enabled', 'false'
-  setting 'xpack.license.self_generated.type', 'basic'
-  setting 'xpack.security.enabled', 'false'
-}
-
-Task noSecurityTest = tasks.findByName("integTestNoSecurity")
-configure(extensions.findByName("integTestSecurityCluster")) {
-    clusterName = "basic-license"
-    numNodes = 2
-    dataDir = { nodeNum -> noSecurityTest.nodes[nodeNum].dataDir }
 
+testClusters.integTest {
+    distribution = 'DEFAULT'
+    numberOfNodes = 2
     setting 'xpack.ilm.enabled', 'false'
     setting 'xpack.ml.enabled', 'false'
     setting 'xpack.license.self_generated.type', 'basic'
-    setting 'xpack.security.enabled', 'true'
-    setting 'xpack.security.authc.anonymous.roles', 'anonymous'
-    setting 'xpack.security.transport.ssl.enabled', 'true'
-    setting 'xpack.security.transport.ssl.certificate', 'transport.crt'
-    setting 'xpack.security.transport.ssl.key', 'transport.key'
-    setting 'xpack.security.transport.ssl.key_passphrase', 'transport-password'
-    setting 'xpack.security.transport.ssl.certificate_authorities', 'ca.crt'
-
-    extraConfigFile 'transport.key', project.projectDir.toPath().resolve('src/test/resources/ssl/transport.key').toFile()
-    extraConfigFile 'transport.crt', project.projectDir.toPath().resolve('src/test/resources/ssl/transport.crt').toFile()
-    extraConfigFile 'ca.crt', project.projectDir.toPath().resolve('src/test/resources/ssl/ca.crt').toFile()
+    setting 'xpack.security.enabled', 'false'
+}
 
-    setupCommand 'setupAdminUser',
-                 'bin/elasticsearch-users', 'useradd', 'admin_user', '-p', 'admin-password', '-r', 'superuser'
-    setupCommand 'setupTestUser' ,
-                 'bin/elasticsearch-users', 'useradd', 'security_test_user', '-p', 'security-test-password', '-r', 'security_test_role'
-    extraConfigFile 'roles.yml', project.projectDir.toPath().resolve('src/test/resources/roles.yml').toFile()
+task integTestSecurity(type: Test) {
+    description = "Run tests against a cluster that has security"
+    useCluster testClusters.integTest
+    dependsOn integTest
+    systemProperty 'tests.has_security', 'true'
+    maxParallelForks = 1
+    outputs.cacheIf "Caching of REST tests not implemented yet", { false }
+
+    doFirst {
+        testClusters.integTest {
+            // Reconfigure cluster to enable security
+            setting 'xpack.security.enabled', 'true'
+            setting 'xpack.security.authc.anonymous.roles', 'anonymous'
+            setting 'xpack.security.transport.ssl.enabled', 'true'
+            setting 'xpack.security.transport.ssl.certificate', 'transport.crt'
+            setting 'xpack.security.transport.ssl.key', 'transport.key'
+            setting 'xpack.security.transport.ssl.key_passphrase', 'transport-password'
+            setting 'xpack.security.transport.ssl.certificate_authorities', 'ca.crt'
+
+            extraConfigFile 'transport.key', file('src/test/resources/ssl/transport.key')
+            extraConfigFile 'transport.crt', file('src/test/resources/ssl/transport.crt')
+            extraConfigFile 'ca.crt', file('src/test/resources/ssl/ca.crt')
+            extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
+
+            user username: "admin_user", password: "admin-password"
+            user username: "security_test_user", password: "security-test-password", role: "security_test_role"
+
+            restart()
+        }
+        nonInputProperties.systemProperty 'tests.rest.cluster', "${-> testClusters.integTest.getAllHttpSocketURI().join(",")}"
+    }
 }
+check.dependsOn(integTestSecurity)
 
-integTest.enabled = false
diff --git a/x-pack/plugin/security/qa/security-basic/build.gradle b/x-pack/plugin/security/qa/security-basic/build.gradle
index 864a1e5180934..4dc948e477ed2 100644
--- a/x-pack/plugin/security/qa/security-basic/build.gradle
+++ b/x-pack/plugin/security/qa/security-basic/build.gradle
@@ -1,17 +1,16 @@
-import org.elasticsearch.gradle.http.WaitForHttpResource
-
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
   testCompile project(path: xpackModule('core'), configuration: 'default')
   testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 }
 
-integTestCluster {
-  numNodes=2
+testClusters.integTest {
+  distribution = "DEFAULT"
+  numberOfNodes = 2
 
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
@@ -22,14 +21,7 @@ integTestCluster {
   setting 'xpack.security.authc.token.enabled', 'true'
   setting 'xpack.security.authc.api_key.enabled', 'true'
 
-  extraConfigFile 'roles.yml', project.projectDir.toPath().resolve('src/test/resources/roles.yml')
-  setupCommand 'setupUser#admin_user', 'bin/elasticsearch-users', 'useradd', 'admin_user', '-p', 'admin-password', '-r', 'superuser'
-  setupCommand 'setupUser#security_test_user', 'bin/elasticsearch-users', 'useradd', 'security_test_user', '-p', 'security-test-password', '-r', 'security_test_role'
-
-  waitCondition = { node, ant ->
-    WaitForHttpResource http = new WaitForHttpResource("http", node.httpUri(), numNodes)
-    http.setUsername("admin_user")
-    http.setPassword("admin-password")
-    return http.wait(5000)
-  }
+  extraConfigFile 'roles.yml', file('src/test/resources/roles.yml')
+  user username: "admin_user", password: "admin-password"
+  user username: "security_test_user", password: "security-test-password", role: "security_test_role"
 }
diff --git a/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java b/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java
index 837c9ac4d8ded..837421f6000d5 100644
--- a/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java
+++ b/x-pack/plugin/security/qa/security-basic/src/test/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java
@@ -54,8 +54,11 @@ public void testWithBasicLicense() throws Exception {
         checkAuthentication();
         checkHasPrivileges();
         checkIndexWrite();
+
+        final Tuple<String, String> keyAndId = getApiKeyAndId();
+        assertAuthenticateWithApiKey(keyAndId, true);
+
         assertFailToGetToken();
-        assertFailToGetApiKey();
         assertAddRoleWithDLS(false);
         assertAddRoleWithFLS(false);
     }
@@ -79,9 +82,8 @@ public void testWithTrialLicense() throws Exception {
         } finally {
             revertTrial();
             assertAuthenticateWithToken(accessToken, false);
-            assertAuthenticateWithApiKey(keyAndId, false);
+            assertAuthenticateWithApiKey(keyAndId, true);
             assertFailToGetToken();
-            assertFailToGetApiKey();
             assertAddRoleWithDLS(false);
             assertAddRoleWithFLS(false);
         }
@@ -199,12 +201,6 @@ private void assertFailToGetToken() {
         assertThat(e.getMessage(), containsString("current license is non-compliant for [security tokens]"));
     }
 
-    private void assertFailToGetApiKey() {
-        ResponseException e = expectThrows(ResponseException.class, () -> adminClient().performRequest(buildGetApiKeyRequest()));
-        assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(403));
-        assertThat(e.getMessage(), containsString("current license is non-compliant for [api keys]"));
-    }
-
     private void assertAuthenticateWithToken(String accessToken, boolean shouldSucceed) throws IOException {
         assertNotNull("access token cannot be null", accessToken);
         Request request = new Request("GET", "/_security/_authenticate");
diff --git a/x-pack/plugin/security/qa/tls-basic/build.gradle b/x-pack/plugin/security/qa/tls-basic/build.gradle
index 6487475f7c8c3..a8e9e16aeda1b 100644
--- a/x-pack/plugin/security/qa/tls-basic/build.gradle
+++ b/x-pack/plugin/security/qa/tls-basic/build.gradle
@@ -1,10 +1,8 @@
-import org.elasticsearch.gradle.http.WaitForHttpResource
-
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
   testCompile project(path: xpackModule('core'), configuration: 'default')
   testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
@@ -18,16 +16,16 @@ forbiddenPatterns {
     exclude '**/*.p12'
 }
 
-File caFile = project.file('src/test/resources/ssl/ca.crt')
 
-integTestCluster {
-  numNodes=2
+testClusters.integTest {
+  distribution = "DEFAULT"
+  numberOfNodes = 2
 
-  extraConfigFile 'http.key', project.projectDir.toPath().resolve('src/test/resources/ssl/http.key')
-  extraConfigFile 'http.crt', project.projectDir.toPath().resolve('src/test/resources/ssl/http.crt')
-  extraConfigFile 'transport.key', project.projectDir.toPath().resolve('src/test/resources/ssl/transport.key')
-  extraConfigFile 'transport.crt', project.projectDir.toPath().resolve('src/test/resources/ssl/transport.crt')
-  extraConfigFile 'ca.crt', caFile
+  extraConfigFile 'http.key', file('src/test/resources/ssl/http.key')
+  extraConfigFile 'http.crt', file('src/test/resources/ssl/http.crt')
+  extraConfigFile 'transport.key', file('src/test/resources/ssl/transport.key')
+  extraConfigFile 'transport.crt', file('src/test/resources/ssl/transport.crt')
+  extraConfigFile 'ca.crt', file('src/test/resources/ssl/ca.crt')
 
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
@@ -36,15 +34,10 @@ integTestCluster {
   setting 'xpack.security.http.ssl.certificate', 'http.crt'
   setting 'xpack.security.http.ssl.key', 'http.key'
   setting 'xpack.security.http.ssl.key_passphrase', 'http-password'
+  setting 'xpack.security.http.ssl.certificate_authorities', 'ca.crt'
   setting 'xpack.security.transport.ssl.enabled', 'true'
   setting 'xpack.security.transport.ssl.certificate', 'transport.crt'
   setting 'xpack.security.transport.ssl.key', 'transport.key'
   setting 'xpack.security.transport.ssl.key_passphrase', 'transport-password'
   setting 'xpack.security.transport.ssl.certificate_authorities', 'ca.crt'
-
-  waitCondition = { node, ant ->
-    WaitForHttpResource http = new WaitForHttpResource("https", node.httpUri(), numNodes)
-    http.setCertificateAuthorities(caFile)
-    return http.wait(5000)
-  }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
index c7ada6e79a9ac..fc443d14e01d7 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -24,8 +24,6 @@
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.network.NetworkService;
@@ -40,7 +38,6 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.http.HttpServerTransport;
@@ -73,6 +70,8 @@
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.security.SecurityContext;
 import org.elasticsearch.xpack.core.security.SecurityExtension;
 import org.elasticsearch.xpack.core.security.SecurityField;
@@ -84,6 +83,7 @@
 import org.elasticsearch.xpack.core.security.action.oidc.OpenIdConnectLogoutAction;
 import org.elasticsearch.xpack.core.security.action.oidc.OpenIdConnectPrepareAuthenticationAction;
 import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.GetBuiltinPrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesAction;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheAction;
@@ -119,8 +119,9 @@
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetBitsetCache;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
-import org.elasticsearch.xpack.core.security.authz.accesscontrol.SecurityIndexSearcherWrapper;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.SecurityIndexReaderWrapper;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
@@ -142,6 +143,7 @@
 import org.elasticsearch.xpack.security.action.oidc.TransportOpenIdConnectLogoutAction;
 import org.elasticsearch.xpack.security.action.oidc.TransportOpenIdConnectPrepareAuthenticationAction;
 import org.elasticsearch.xpack.security.action.privilege.TransportDeletePrivilegesAction;
+import org.elasticsearch.xpack.security.action.privilege.TransportGetBuiltinPrivilegesAction;
 import org.elasticsearch.xpack.security.action.privilege.TransportGetPrivilegesAction;
 import org.elasticsearch.xpack.security.action.privilege.TransportPutPrivilegesAction;
 import org.elasticsearch.xpack.security.action.realm.TransportClearRealmCacheAction;
@@ -204,6 +206,7 @@
 import org.elasticsearch.xpack.security.rest.action.oidc.RestOpenIdConnectLogoutAction;
 import org.elasticsearch.xpack.security.rest.action.oidc.RestOpenIdConnectPrepareAuthenticationAction;
 import org.elasticsearch.xpack.security.rest.action.privilege.RestDeletePrivilegesAction;
+import org.elasticsearch.xpack.security.rest.action.privilege.RestGetBuiltinPrivilegesAction;
 import org.elasticsearch.xpack.security.rest.action.privilege.RestGetPrivilegesAction;
 import org.elasticsearch.xpack.security.rest.action.privilege.RestPutPrivilegesAction;
 import org.elasticsearch.xpack.security.rest.action.realm.RestClearRealmCacheAction;
@@ -226,6 +229,7 @@
 import org.elasticsearch.xpack.security.rest.action.user.RestPutUserAction;
 import org.elasticsearch.xpack.security.rest.action.user.RestSetEnabledAction;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
+import org.elasticsearch.xpack.security.support.SecurityStatusChangeListener;
 import org.elasticsearch.xpack.security.transport.SecurityHttpSettings;
 import org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor;
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
@@ -283,6 +287,7 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
     private final SetOnce<SecurityActionFilter> securityActionFilter = new SetOnce<>();
     private final SetOnce<SecurityIndexManager> securityIndex = new SetOnce<>();
     private final SetOnce<NioGroupFactory> groupFactory = new SetOnce<>();
+    private final SetOnce<DocumentSubsetBitsetCache> dlsBitsetCache = new SetOnce<>();
     private final List<BootstrapCheck> bootstrapChecks;
     private final List<SecurityExtension> securityExtensions = new ArrayList<>();
 
@@ -322,36 +327,6 @@ private static void runStartupChecks(Settings settings) {
         validateRealmSettings(settings);
     }
 
-    @Override
-    public Collection<Module> createGuiceModules() {
-        List<Module> modules = new ArrayList<>();
-        if (enabled == false) {
-            modules.add(b -> b.bind(IPFilter.class).toProvider(Providers.of(null)));
-        }
-        modules.add(b -> XPackPlugin.bindFeatureSet(b, SecurityFeatureSet.class));
-
-        if (enabled == false) {
-            modules.add(b -> {
-                b.bind(Realms.class).toProvider(Providers.of(null)); // for SecurityFeatureSet
-                b.bind(CompositeRolesStore.class).toProvider(Providers.of(null)); // for SecurityFeatureSet
-                b.bind(NativeRoleMappingStore.class).toProvider(Providers.of(null)); // for SecurityFeatureSet
-                b.bind(AuditTrailService.class)
-                    .toInstance(new AuditTrailService(Collections.emptyList(), getLicenseState()));
-            });
-            return modules;
-        }
-
-        // we can't load that at construction time since the license plugin might not have been loaded at that point
-        // which might not be the case during Plugin class instantiation. Once nodeModules are pulled
-        // everything should have been loaded
-        modules.add(b -> {
-            if (XPackSettings.AUDIT_ENABLED.get(settings)) {
-                b.bind(AuditTrail.class).to(AuditTrailService.class); // interface used by some actions...
-            }
-        });
-        return modules;
-    }
-
     // overridable by tests
     protected Clock getClock() {
         return Clock.systemUTC();
@@ -375,7 +350,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
     Collection<Object> createComponents(Client client, ThreadPool threadPool, ClusterService clusterService,
                                         ResourceWatcherService resourceWatcherService, ScriptService scriptService) throws Exception {
         if (enabled == false) {
-            return Collections.emptyList();
+            return Collections.singletonList(new SecurityUsageServices(null, null, null, null));
         }
 
         threadContext.set(threadPool.getThreadContext());
@@ -383,6 +358,10 @@ Collection<Object> createComponents(Client client, ThreadPool threadPool, Cluste
         securityContext.set(new SecurityContext(settings, threadPool.getThreadContext()));
         components.add(securityContext.get());
 
+        if (XPackSettings.DLS_FLS_ENABLED.get(settings)) {
+            dlsBitsetCache.set(new DocumentSubsetBitsetCache(settings));
+        }
+
         // audit trail service construction
         final List<AuditTrail> auditTrails = XPackSettings.AUDIT_ENABLED.get(settings)
                 ? Collections.singletonList(new LoggingAuditTrail(settings, clusterService, threadPool))
@@ -440,12 +419,13 @@ Collection<Object> createComponents(Client client, ThreadPool threadPool, Cluste
         components.add(apiKeyService);
         final CompositeRolesStore allRolesStore = new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore, reservedRolesStore,
             privilegeStore, rolesProviders, threadPool.getThreadContext(), getLicenseState(), fieldPermissionsCache, apiKeyService,
-            new DeprecationRoleDescriptorConsumer(clusterService, threadPool));
+            dlsBitsetCache.get(), new DeprecationRoleDescriptorConsumer(clusterService, threadPool));
         securityIndex.get().addIndexStateListener(allRolesStore::onSecurityIndexStateChange);
 
         // to keep things simple, just invalidate all cached entries on license change. this happens so rarely that the impact should be
         // minimal
         getLicenseState().addListener(allRolesStore::invalidateAll);
+        getLicenseState().addListener(new SecurityStatusChangeListener(getLicenseState()));
 
         final AuthenticationFailureHandler failureHandler = createAuthenticationFailureHandler(realms);
         authcService.set(new AuthenticationService(settings, realms, auditTrailService, failureHandler, threadPool,
@@ -471,7 +451,7 @@ auditTrailService, failureHandler, threadPool, anonymousUser, getAuthorizationEn
 
         components.add(nativeRolesStore); // used by roles actions
         components.add(reservedRolesStore); // used by roles actions
-        components.add(allRolesStore); // for SecurityFeatureSet and clear roles cache
+        components.add(allRolesStore); // for SecurityInfoTransportAction and clear roles cache
         components.add(authzService);
 
         ipFilter.set(new IPFilter(settings, auditTrailService, clusterService.getClusterSettings(), getLicenseState()));
@@ -483,6 +463,8 @@ auditTrailService, failureHandler, threadPool, anonymousUser, getAuthorizationEn
         securityActionFilter.set(new SecurityActionFilter(authcService.get(), authzService, getLicenseState(),
             threadPool, securityContext.get(), destructiveOperations));
 
+        components.add(new SecurityUsageServices(realms, allRolesStore, nativeRoleMappingStore, ipFilter.get()));
+
         return components;
     }
 
@@ -617,11 +599,11 @@ public static List<Setting<?>> getSettings(List<SecurityExtension> securityExten
         AuthorizationService.addSettings(settingsList);
         Automatons.addSettings(settingsList);
         settingsList.addAll(CompositeRolesStore.getSettings());
+        settingsList.addAll(DocumentSubsetBitsetCache.getSettings());
         settingsList.add(FieldPermissionsCache.CACHE_SIZE_SETTING);
         settingsList.add(TokenService.TOKEN_EXPIRATION);
         settingsList.add(TokenService.DELETE_INTERVAL);
         settingsList.add(TokenService.DELETE_TIMEOUT);
-        settingsList.add(SecurityServerTransportInterceptor.TRANSPORT_TYPE_PROFILE_SETTING);
         settingsList.addAll(SSLConfigurationSettings.getProfileSettings());
         settingsList.add(ApiKeyService.PASSWORD_HASHING_ALGORITHM);
         settingsList.add(ApiKeyService.DELETE_TIMEOUT);
@@ -668,8 +650,9 @@ public void onIndexModule(IndexModule module) {
         if (enabled) {
             assert getLicenseState() != null;
             if (XPackSettings.DLS_FLS_ENABLED.get(settings)) {
-                module.setSearcherWrapper(indexService ->
-                        new SecurityIndexSearcherWrapper(
+                assert dlsBitsetCache.get() != null;
+                module.setReaderWrapper(indexService ->
+                        new SecurityIndexReaderWrapper(
                                 shardId -> indexService.newQueryShardContext(shardId.id(),
                                 // we pass a null index reader, which is legal and will disable rewrite optimizations
                                 // based on index statistics, which is probably safer...
@@ -678,7 +661,7 @@ public void onIndexModule(IndexModule module) {
                                     throw new IllegalArgumentException("permission filters are not allowed to use the current timestamp");
 
                                 }, null),
-                                indexService.cache() != null ? indexService.cache().bitsetFilterCache() : null,
+                                dlsBitsetCache.get(),
                                 indexService.getThreadPool().getThreadContext(), getLicenseState(),
                                 indexService.getScriptService()));
                 /*
@@ -705,8 +688,10 @@ public void onIndexModule(IndexModule module) {
 
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        var usageAction = new ActionHandler<>(XPackUsageFeatureAction.SECURITY, SecurityUsageTransportAction.class);
+        var infoAction = new ActionHandler<>(XPackInfoFeatureAction.SECURITY, SecurityInfoTransportAction.class);
         if (enabled == false) {
-            return emptyList();
+            return Arrays.asList(usageAction, infoAction);
         }
         return Arrays.asList(
                 new ActionHandler<>(ClearRealmCacheAction.INSTANCE, TransportClearRealmCacheAction.class),
@@ -737,13 +722,15 @@ public void onIndexModule(IndexModule module) {
                     TransportOpenIdConnectPrepareAuthenticationAction.class),
                 new ActionHandler<>(OpenIdConnectAuthenticateAction.INSTANCE, TransportOpenIdConnectAuthenticateAction.class),
                 new ActionHandler<>(OpenIdConnectLogoutAction.INSTANCE, TransportOpenIdConnectLogoutAction.class),
+                new ActionHandler<>(GetBuiltinPrivilegesAction.INSTANCE, TransportGetBuiltinPrivilegesAction.class),
                 new ActionHandler<>(GetPrivilegesAction.INSTANCE, TransportGetPrivilegesAction.class),
                 new ActionHandler<>(PutPrivilegesAction.INSTANCE, TransportPutPrivilegesAction.class),
                 new ActionHandler<>(DeletePrivilegesAction.INSTANCE, TransportDeletePrivilegesAction.class),
                 new ActionHandler<>(CreateApiKeyAction.INSTANCE, TransportCreateApiKeyAction.class),
                 new ActionHandler<>(InvalidateApiKeyAction.INSTANCE, TransportInvalidateApiKeyAction.class),
-                new ActionHandler<>(GetApiKeyAction.INSTANCE, TransportGetApiKeyAction.class)
-        );
+                new ActionHandler<>(GetApiKeyAction.INSTANCE, TransportGetApiKeyAction.class),
+                usageAction,
+                infoAction);
     }
 
     @Override
@@ -756,8 +743,9 @@ public List<ActionFilter> getActionFilters() {
 
     @Override
     public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
-            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
-            Supplier<DiscoveryNodes> nodesInCluster) {
+                                             IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
+                                             IndexNameExpressionResolver indexNameExpressionResolver,
+                                             Supplier<DiscoveryNodes> nodesInCluster) {
         if (enabled == false) {
             return emptyList();
         }
@@ -788,6 +776,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
                 new RestOpenIdConnectPrepareAuthenticationAction(settings, restController, getLicenseState()),
                 new RestOpenIdConnectAuthenticateAction(settings, restController, getLicenseState()),
                 new RestOpenIdConnectLogoutAction(settings, restController, getLicenseState()),
+                new RestGetBuiltinPrivilegesAction(settings, restController, getLicenseState()),
                 new RestGetPrivilegesAction(settings, restController, getLicenseState()),
                 new RestPutPrivilegesAction(settings, restController, getLicenseState()),
                 new RestDeletePrivilegesAction(settings, restController, getLicenseState()),
@@ -840,7 +829,6 @@ static void validateRealmSettings(Settings settings) {
     @Override
     public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry, ThreadContext threadContext) {
         if (enabled == false) { // don't register anything if we are not enabled
-            // interceptors are not installed if we are running on the transport client
             return Collections.emptyList();
         }
        return Collections.singletonList(new TransportInterceptor() {
@@ -864,7 +852,7 @@ public AsyncSender interceptSender(AsyncSender sender) {
     public Map<String, Supplier<Transport>> getTransports(Settings settings, ThreadPool threadPool, PageCacheRecycler pageCacheRecycler,
                                                           CircuitBreakerService circuitBreakerService,
                                                           NamedWriteableRegistry namedWriteableRegistry, NetworkService networkService) {
-        if (enabled == false) { // don't register anything if we are not enabled, or in transport client mode
+        if (enabled == false) { // don't register anything if we are not enabled
             return Collections.emptyMap();
         }
 
@@ -975,36 +963,12 @@ public Function<String, Predicate<String>> getFieldFilter() {
     @Override
     public BiConsumer<DiscoveryNode, ClusterState> getJoinValidator() {
         if (enabled) {
-            return new ValidateTLSOnJoin(XPackSettings.TRANSPORT_SSL_ENABLED.get(settings),
-                    DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings), settings)
-                .andThen(new ValidateUpgradedSecurityIndex())
+            return new ValidateUpgradedSecurityIndex()
                 .andThen(new ValidateLicenseForFIPS(XPackSettings.FIPS_MODE_ENABLED.get(settings)));
         }
         return null;
     }
 
-    static final class ValidateTLSOnJoin implements BiConsumer<DiscoveryNode, ClusterState> {
-        private final boolean isTLSEnabled;
-        private final String discoveryType;
-        private final Settings settings;
-
-        ValidateTLSOnJoin(boolean isTLSEnabled, String discoveryType, Settings settings) {
-            this.isTLSEnabled = isTLSEnabled;
-            this.discoveryType = discoveryType;
-            this.settings = settings;
-        }
-
-        @Override
-        public void accept(DiscoveryNode node, ClusterState state) {
-            License license = LicenseService.getLicense(state.metaData());
-            if (isTLSEnabled == false && "single-node".equals(discoveryType) == false
-                && XPackLicenseState.isTransportTlsRequired(license, settings)) {
-                throw new IllegalStateException("Transport TLS ([" + XPackSettings.TRANSPORT_SSL_ENABLED.getKey() +
-                    "]) is required for license type [" + license.operationMode().description() + "] when security is enabled");
-            }
-        }
-    }
-
     static final class ValidateUpgradedSecurityIndex implements BiConsumer<DiscoveryNode, ClusterState> {
         @Override
         public void accept(DiscoveryNode node, ClusterState state) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityInfoTransportAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityInfoTransportAction.java
new file mode 100644
index 0000000000000..b554309def497
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityInfoTransportAction.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
+
+/**
+ * Indicates whether the features of Security are currently in use
+ */
+public class SecurityInfoTransportAction extends XPackInfoFeatureTransportAction {
+
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public SecurityInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                       Settings settings, XPackLicenseState licenseState) {
+        super(XPackInfoFeatureAction.SECURITY.name(), transportService, actionFilters);
+        this.enabled = XPackSettings.SECURITY_ENABLED.get(settings);
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    public String name() {
+        return XPackField.SECURITY;
+    }
+
+    @Override
+    public boolean available() {
+        return licenseState.isSecurityAvailable();
+    }
+
+    @Override
+    public boolean enabled() {
+        return enabled && licenseState.isSecurityDisabledByLicenseDefaults() == false;
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageServices.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageServices.java
new file mode 100644
index 0000000000000..e85459c484b34
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageServices.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.xpack.security.authc.Realms;
+import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
+import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
+import org.elasticsearch.xpack.security.transport.filter.IPFilter;
+
+/**
+ * A wrapper around the services needed to produce usage information for the security feature.
+ *
+ * This class is temporary until actions can be constructed directly by plugins.
+ */
+class SecurityUsageServices {
+    final Realms realms;
+    final CompositeRolesStore rolesStore;
+    final NativeRoleMappingStore roleMappingStore;
+    final IPFilter ipFilter;
+
+    SecurityUsageServices(Realms realms, CompositeRolesStore rolesStore, NativeRoleMappingStore roleMappingStore, IPFilter ipFilter) {
+        this.realms = realms;
+        this.rolesStore = rolesStore;
+        this.roleMappingStore = roleMappingStore;
+        this.ipFilter = ipFilter;
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatureSet.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java
similarity index 73%
rename from x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatureSet.java
rename to x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java
index b24bf4375440f..2b2cbe5364b4c 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityFeatureSet.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java
@@ -6,14 +6,23 @@
 package org.elasticsearch.xpack.security;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
 import org.elasticsearch.xpack.core.security.SecurityFeatureSetUsage;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail;
@@ -34,66 +43,34 @@
 import static org.elasticsearch.xpack.core.XPackSettings.TOKEN_SERVICE_ENABLED_SETTING;
 import static org.elasticsearch.xpack.core.XPackSettings.TRANSPORT_SSL_ENABLED;
 
-/**
- * Indicates whether the features of Security are currently in use
- */
-public class SecurityFeatureSet implements XPackFeatureSet {
+public class SecurityUsageTransportAction extends XPackUsageFeatureTransportAction {
 
+    private final boolean enabledInSettings;
     private final Settings settings;
     private final XPackLicenseState licenseState;
-    @Nullable
     private final Realms realms;
-    @Nullable
     private final CompositeRolesStore rolesStore;
-    @Nullable
     private final NativeRoleMappingStore roleMappingStore;
-    @Nullable
     private final IPFilter ipFilter;
 
     @Inject
-    public SecurityFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState,
-                              @Nullable Realms realms, @Nullable CompositeRolesStore rolesStore,
-                              @Nullable NativeRoleMappingStore roleMappingStore,
-                              @Nullable IPFilter ipFilter) {
-        this.licenseState = licenseState;
-        this.realms = realms;
-        this.rolesStore = rolesStore;
-        this.roleMappingStore = roleMappingStore;
+    public SecurityUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                        ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                        Settings settings, XPackLicenseState licenseState, SecurityUsageServices securityServices) {
+        super(XPackUsageFeatureAction.SECURITY.name(), transportService, clusterService, threadPool,
+              actionFilters, indexNameExpressionResolver);
+        this.enabledInSettings = XPackSettings.SECURITY_ENABLED.get(settings);
         this.settings = settings;
-        this.ipFilter = ipFilter;
-    }
-
-    @Override
-    public String name() {
-        return XPackField.SECURITY;
-    }
-
-    @Override
-    public String description() {
-        return "Security for the Elastic Stack";
-    }
-
-    @Override
-    public boolean available() {
-        return licenseState != null && licenseState.isSecurityAvailable();
-    }
-
-    @Override
-    public boolean enabled() {
-        if (licenseState != null) {
-            return XPackSettings.SECURITY_ENABLED.get(settings) &&
-                licenseState.isSecurityDisabledByLicenseDefaults() == false;
-        }
-        return false;
-    }
-
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return null;
+        this.licenseState = licenseState;
+        this.realms = securityServices.realms;
+        this.rolesStore = securityServices.rolesStore;
+        this.roleMappingStore = securityServices.roleMappingStore;
+        this.ipFilter = securityServices.ipFilter;
     }
 
     @Override
-    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
         Map<String, Object> sslUsage = sslUsage(settings);
         Map<String, Object> tokenServiceUsage = tokenServiceUsage(settings);
         Map<String, Object> apiKeyServiceUsage = apiKeyServiceUsage(settings);
@@ -107,9 +84,11 @@ public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
         final CountDown countDown = new CountDown(3);
         final Runnable doCountDown = () -> {
             if (countDown.countDown()) {
-                listener.onResponse(new SecurityFeatureSetUsage(available(), enabled(), realmsUsageRef.get(), rolesUsageRef.get(),
-                        roleMappingUsageRef.get(), sslUsage, auditUsage, ipFilterUsage, anonymousUsage, tokenServiceUsage,
-                        apiKeyServiceUsage));
+                boolean enabled = enabledInSettings && licenseState.isSecurityDisabledByLicenseDefaults() == false;
+                var usage = new SecurityFeatureSetUsage(licenseState.isSecurityAvailable(), enabled,
+                        realmsUsageRef.get(), rolesUsageRef.get(), roleMappingUsageRef.get(), sslUsage, auditUsage,
+                        ipFilterUsage, anonymousUsage, tokenServiceUsage, apiKeyServiceUsage);
+                listener.onResponse(new XPackUsageFeatureResponse(usage));
             }
         };
 
@@ -147,6 +126,7 @@ public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
         } else {
             realms.usageStats(realmsUsageListener);
         }
+
     }
 
     static Map<String, Object> sslUsage(Settings settings) {
@@ -188,5 +168,4 @@ static Map<String, Object> ipFilterUsage(@Nullable IPFilter ipFilter) {
         }
         return ipFilter.usageStats();
     }
-
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/SecurityActionMapper.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/SecurityActionMapper.java
index 409317bbf89cc..f4eb97fcb3472 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/SecurityActionMapper.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/SecurityActionMapper.java
@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.security.action;
 
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.action.search.ClearScrollAction;
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.transport.TransportRequest;
@@ -36,8 +35,8 @@ public String action(String action, TransportRequest request) {
                 break;
             case AnalyzeAction.NAME:
             case AnalyzeAction.NAME + "[s]":
-                assert request instanceof AnalyzeRequest;
-                String[] indices = ((AnalyzeRequest) request).indices();
+                assert request instanceof AnalyzeAction.Request;
+                String[] indices = ((AnalyzeAction.Request) request).indices();
                 if (indices == null || (indices.length == 1 && indices[0] == null)) {
                     return CLUSTER_PERMISSION_ANALYZE;
                 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetBuiltinPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetBuiltinPrivilegesAction.java
new file mode 100644
index 0000000000000..e179c4bf25093
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetBuiltinPrivilegesAction.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.action.privilege;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.security.action.privilege.GetBuiltinPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.GetBuiltinPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.privilege.GetBuiltinPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
+import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
+
+import java.util.TreeSet;
+
+/**
+ * Transport action to retrieve one or more application privileges from the security index
+ */
+public class TransportGetBuiltinPrivilegesAction extends HandledTransportAction<GetBuiltinPrivilegesRequest, GetBuiltinPrivilegesResponse> {
+
+    @Inject
+    public TransportGetBuiltinPrivilegesAction(ActionFilters actionFilters, TransportService transportService) {
+        super(GetBuiltinPrivilegesAction.NAME, transportService, actionFilters, GetBuiltinPrivilegesRequest::new);
+    }
+
+    @Override
+    protected void doExecute(Task task, GetBuiltinPrivilegesRequest request, ActionListener<GetBuiltinPrivilegesResponse> listener) {
+        final TreeSet<String> cluster = new TreeSet<>(ClusterPrivilege.names());
+        final TreeSet<String> index = new TreeSet<>(IndexPrivilege.names());
+        listener.onResponse(new GetBuiltinPrivilegesResponse(cluster, index));
+    }
+
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetPrivilegesAction.java
index dcf5cb426f20c..9645c8193dc95 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/privilege/TransportGetPrivilegesAction.java
@@ -46,10 +46,12 @@ protected void doExecute(Task task, final GetPrivilegesRequest request, final Ac
         } else {
             names = new HashSet<>(Arrays.asList(request.privileges()));
         }
+
         final Collection<String> applications = isNullOrEmpty(request.application()) ? null : Collections.singleton(request.application());
         this.privilegeStore.getPrivileges(applications, names, ActionListener.wrap(
-                privileges -> listener.onResponse(new GetPrivilegesResponse(privileges)),
-                listener::onFailure
+            privileges -> listener.onResponse(new GetPrivilegesResponse(privileges)),
+            listener::onFailure
         ));
     }
+
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheAction.java
index b4ee8b677c13b..63eb54a0b4ddd 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheAction.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheAction;
@@ -46,8 +47,8 @@ protected ClearRealmCacheResponse newResponse(ClearRealmCacheRequest request,
     }
 
     @Override
-    protected ClearRealmCacheRequest.Node newNodeRequest(String nodeId, ClearRealmCacheRequest request) {
-        return new ClearRealmCacheRequest.Node(request, nodeId);
+    protected ClearRealmCacheRequest.Node newNodeRequest(ClearRealmCacheRequest request) {
+        return new ClearRealmCacheRequest.Node(request);
     }
 
     @Override
@@ -56,7 +57,7 @@ protected ClearRealmCacheResponse.Node newNodeResponse() {
     }
 
     @Override
-    protected ClearRealmCacheResponse.Node nodeOperation(ClearRealmCacheRequest.Node nodeRequest) throws ElasticsearchException {
+    protected ClearRealmCacheResponse.Node nodeOperation(ClearRealmCacheRequest.Node nodeRequest, Task task) throws ElasticsearchException {
         if (nodeRequest.getRealms() == null || nodeRequest.getRealms().length == 0) {
             for (Realm realm : realms) {
                 clearCache(realm, nodeRequest.getUsernames());
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportClearRolesCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportClearRolesCacheAction.java
index 3c19e3deaf6a9..9dd5c09f5b1e4 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportClearRolesCacheAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportClearRolesCacheAction.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheAction;
@@ -39,8 +40,8 @@ protected ClearRolesCacheResponse newResponse(ClearRolesCacheRequest request,
     }
 
     @Override
-    protected ClearRolesCacheRequest.Node newNodeRequest(String nodeId, ClearRolesCacheRequest request) {
-        return new ClearRolesCacheRequest.Node(request, nodeId);
+    protected ClearRolesCacheRequest.Node newNodeRequest(ClearRolesCacheRequest request) {
+        return new ClearRolesCacheRequest.Node(request);
     }
 
     @Override
@@ -49,7 +50,7 @@ protected ClearRolesCacheResponse.Node newNodeResponse() {
     }
 
     @Override
-    protected ClearRolesCacheResponse.Node nodeOperation(ClearRolesCacheRequest.Node request) {
+    protected ClearRolesCacheResponse.Node nodeOperation(ClearRolesCacheRequest.Node request, Task task) {
         if (request.getNames() == null || request.getNames().length == 0) {
             rolesStore.invalidateAll();
         } else {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenAction.java
index 65456ccd2af51..252807779a1d6 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenAction.java
@@ -9,19 +9,25 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.security.action.token.CreateTokenAction;
 import org.elasticsearch.xpack.core.security.action.token.CreateTokenRequest;
+import org.elasticsearch.xpack.core.security.action.token.CreateTokenRequest.GrantType;
 import org.elasticsearch.xpack.core.security.action.token.CreateTokenResponse;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
 import org.elasticsearch.xpack.security.authc.TokenService;
+import org.elasticsearch.xpack.security.authc.kerberos.KerberosAuthenticationToken;
 
+import java.util.Base64;
 import java.util.Collections;
+import java.util.List;
 
 /**
  * Transport action responsible for creating a token based on a request. Requests provide user
@@ -51,11 +57,12 @@ protected void doExecute(Task task, CreateTokenRequest request, ActionListener<C
         assert type != null : "type should have been validated in the action";
         switch (type) {
             case PASSWORD:
-                authenticateAndCreateToken(request, listener);
+            case KERBEROS:
+                authenticateAndCreateToken(type, request, listener);
                 break;
             case CLIENT_CREDENTIALS:
                 Authentication authentication = Authentication.getAuthentication(threadPool.getThreadContext());
-                createToken(request, authentication, authentication, false, listener);
+                createToken(type, request, authentication, authentication, false, listener);
                 break;
             default:
                 listener.onFailure(new IllegalStateException("grant_type [" + request.getGrantType() +
@@ -64,37 +71,86 @@ protected void doExecute(Task task, CreateTokenRequest request, ActionListener<C
         }
     }
 
-    private void authenticateAndCreateToken(CreateTokenRequest request, ActionListener<CreateTokenResponse> listener) {
+    private void authenticateAndCreateToken(GrantType grantType, CreateTokenRequest request, ActionListener<CreateTokenResponse> listener) {
         Authentication originatingAuthentication = Authentication.getAuthentication(threadPool.getThreadContext());
         try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) {
-            final UsernamePasswordToken authToken = new UsernamePasswordToken(request.getUsername(), request.getPassword());
+            final AuthenticationToken authToken = extractAuthenticationToken(grantType, request, listener);
+            if (authToken == null) {
+                listener.onFailure(new IllegalStateException(
+                        "grant_type [" + request.getGrantType() + "] is not supported by the create token action"));
+                return;
+            }
+
             authenticationService.authenticate(CreateTokenAction.NAME, request, authToken,
                 ActionListener.wrap(authentication -> {
-                    request.getPassword().close();
+                    clearCredentialsFromRequest(grantType, request);
+
                     if (authentication != null) {
-                        createToken(request, authentication, originatingAuthentication, true, listener);
+                        createToken(grantType, request, authentication, originatingAuthentication, true, listener);
                     } else {
                         listener.onFailure(new UnsupportedOperationException("cannot create token if authentication is not allowed"));
                     }
                 }, e -> {
-                    // clear the request password
-                    request.getPassword().close();
+                    clearCredentialsFromRequest(grantType, request);
                     listener.onFailure(e);
                 }));
         }
     }
 
-    private void createToken(CreateTokenRequest request, Authentication authentication, Authentication originatingAuth,
+    private AuthenticationToken extractAuthenticationToken(GrantType grantType, CreateTokenRequest request,
+            ActionListener<CreateTokenResponse> listener) {
+        AuthenticationToken authToken = null;
+        if (grantType == GrantType.PASSWORD) {
+            authToken = new UsernamePasswordToken(request.getUsername(), request.getPassword());
+        } else if (grantType == GrantType.KERBEROS) {
+            SecureString kerberosTicket = request.getKerberosTicket();
+            String base64EncodedToken = kerberosTicket.toString();
+            byte[] decodedKerberosTicket = null;
+            try {
+                decodedKerberosTicket = Base64.getDecoder().decode(base64EncodedToken);
+            } catch (IllegalArgumentException iae) {
+                listener.onFailure(new UnsupportedOperationException("could not decode base64 kerberos ticket " + base64EncodedToken));
+            }
+            authToken = new KerberosAuthenticationToken(decodedKerberosTicket);
+        }
+        return authToken;
+    }
+
+    private void clearCredentialsFromRequest(GrantType grantType, CreateTokenRequest request) {
+        if (grantType == GrantType.PASSWORD) {
+            request.getPassword().close();
+        } else if (grantType == GrantType.KERBEROS) {
+            request.getKerberosTicket().close();
+        }
+    }
+
+    private void createToken(GrantType grantType, CreateTokenRequest request, Authentication authentication, Authentication originatingAuth,
             boolean includeRefreshToken, ActionListener<CreateTokenResponse> listener) {
         tokenService.createOAuth2Tokens(authentication, originatingAuth, Collections.emptyMap(), includeRefreshToken,
                 ActionListener.wrap(tuple -> {
                     final String scope = getResponseScopeValue(request.getScope());
+                    final String base64AuthenticateResponse = (grantType == GrantType.KERBEROS) ? extractOutToken() : null;
                     final CreateTokenResponse response = new CreateTokenResponse(tuple.v1(), tokenService.getExpirationDelay(), scope,
-                            tuple.v2());
+                            tuple.v2(), base64AuthenticateResponse);
                     listener.onResponse(response);
                 }, listener::onFailure));
     }
 
+    private String extractOutToken() {
+        List<String> values = threadPool.getThreadContext().getResponseHeaders().get(KerberosAuthenticationToken.WWW_AUTHENTICATE);
+        if (values != null && values.size() == 1) {
+            final String wwwAuthenticateHeaderValue = values.get(0);
+            // it may contain base64 encoded token that needs to be sent to client if mutual auth was requested
+            if (wwwAuthenticateHeaderValue.startsWith(KerberosAuthenticationToken.NEGOTIATE_AUTH_HEADER_PREFIX)) {
+                final String base64EncodedToken = wwwAuthenticateHeaderValue
+                        .substring(KerberosAuthenticationToken.NEGOTIATE_AUTH_HEADER_PREFIX.length()).trim();
+                return base64EncodedToken;
+            }
+        }
+        threadPool.getThreadContext().getResponseHeaders().remove(KerberosAuthenticationToken.WWW_AUTHENTICATE);
+        return null;
+    }
+
     static String getResponseScopeValue(String requestScope) {
         final String scope;
         // the OAuth2.0 RFC requires the scope to be provided in the
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportRefreshTokenAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportRefreshTokenAction.java
index 5c161d889cfb1..9a99e6020ea36 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportRefreshTokenAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportRefreshTokenAction.java
@@ -33,7 +33,7 @@ protected void doExecute(Task task, CreateTokenRequest request, ActionListener<C
         tokenService.refreshToken(request.getRefreshToken(), ActionListener.wrap(tuple -> {
             final String scope = getResponseScopeValue(request.getScope());
             final CreateTokenResponse response =
-                    new CreateTokenResponse(tuple.v1(), tokenService.getExpirationDelay(), scope, tuple.v2());
+                    new CreateTokenResponse(tuple.v1(), tokenService.getExpirationDelay(), scope, tuple.v2(), null);
             listener.onResponse(response);
         }, listener::onFailure));
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java
index 797f7034f496b..11f128e572077 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java
@@ -85,6 +85,8 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
     public static final String IP_FILTER_ORIGIN_FIELD_VALUE = "ip_filter";
 
     // changing any of this names requires changing the log4j2.properties file too
+    public static final String LOG_TYPE = "type";
+    public static final String TIMESTAMP = "timestamp";
     public static final String ORIGIN_TYPE_FIELD_NAME = "origin.type";
     public static final String ORIGIN_ADDRESS_FIELD_NAME = "origin.address";
     public static final String NODE_NAME_FIELD_NAME = "node.name";
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
index 74d4cce8d02de..ee1a15bb8535e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java
@@ -511,7 +511,7 @@ private void validateApiKeyExpiration(Map<String, Object> source, ApiKeyCredenti
         if (expirationEpochMilli == null || Instant.ofEpochMilli(expirationEpochMilli).isAfter(clock.instant())) {
             final Map<String, Object> creator = Objects.requireNonNull((Map<String, Object>) source.get("creator"));
             final String principal = Objects.requireNonNull((String) creator.get("principal"));
-            final Map<String, Object> metadata = (Map<String, Object>) creator.get("metadata");
+            Map<String, Object> metadata = (Map<String, Object>) creator.get("metadata");
             final Map<String, Object> roleDescriptors = (Map<String, Object>) source.get("role_descriptors");
             final Map<String, Object> limitedByRoleDescriptors = (Map<String, Object>) source.get("limited_by_role_descriptors");
             final String[] roleNames = (roleDescriptors != null) ? roleDescriptors.keySet().toArray(Strings.EMPTY_ARRAY)
@@ -743,27 +743,29 @@ private void findApiKeys(final BoolQueryBuilder boolQuery, boolean filterOutInva
             expiredQuery.should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("expiration_time")));
             boolQuery.filter(expiredQuery);
         }
-        final SearchRequest request = client.prepareSearch(SECURITY_MAIN_ALIAS)
-            .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings))
-            .setQuery(boolQuery)
-            .setVersion(false)
-            .setSize(1000)
-            .setFetchSource(true)
-            .request();
-        securityIndex.checkIndexVersionThenExecute(listener::onFailure,
-            () -> ScrollHelper.fetchAllByEntity(client, request, listener,
-                        (SearchHit hit) -> {
-                            Map<String, Object> source = hit.getSourceAsMap();
-                            String name = (String) source.get("name");
-                            String id = hit.getId();
-                            Long creation = (Long) source.get("creation_time");
-                            Long expiration = (Long) source.get("expiration_time");
-                            Boolean invalidated = (Boolean) source.get("api_key_invalidated");
-                            String username = (String) ((Map<String, Object>) source.get("creator")).get("principal");
-                            String realm = (String) ((Map<String, Object>) source.get("creator")).get("realm");
-                            return new ApiKey(name, id, Instant.ofEpochMilli(creation),
-                                    (expiration != null) ? Instant.ofEpochMilli(expiration) : null, invalidated, username, realm);
-                        }));
+        try (ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashWithOrigin(SECURITY_ORIGIN)) {
+            final SearchRequest request = client.prepareSearch(SECURITY_MAIN_ALIAS)
+                .setScroll(DEFAULT_KEEPALIVE_SETTING.get(settings))
+                .setQuery(boolQuery)
+                .setVersion(false)
+                .setSize(1000)
+                .setFetchSource(true)
+                .request();
+            securityIndex.checkIndexVersionThenExecute(listener::onFailure,
+                () -> ScrollHelper.fetchAllByEntity(client, request, listener,
+                    (SearchHit hit) -> {
+                        Map<String, Object> source = hit.getSourceAsMap();
+                        String name = (String) source.get("name");
+                        String id = hit.getId();
+                        Long creation = (Long) source.get("creation_time");
+                        Long expiration = (Long) source.get("expiration_time");
+                        Boolean invalidated = (Boolean) source.get("api_key_invalidated");
+                        String username = (String) ((Map<String, Object>) source.get("creator")).get("principal");
+                        String realm = (String) ((Map<String, Object>) source.get("creator")).get("realm");
+                        return new ApiKey(name, id, Instant.ofEpochMilli(creation),
+                            (expiration != null) ? Instant.ofEpochMilli(expiration) : null, invalidated, username, realm);
+                    }));
+        }
     }
 
     private void findApiKeyForApiKeyName(String apiKeyName, boolean filterOutInvalidatedKeys, boolean filterOutExpiredKeys,
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
index a8f68870556e6..6472220482966 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java
@@ -181,7 +181,7 @@ public final class TokenService {
             TimeValue.MINUS_ONE, Property.NodeScope);
 
     static final String TOKEN_DOC_TYPE = "token";
-    private static final int HASHED_TOKEN_LENGTH = 44;
+    private static final int HASHED_TOKEN_LENGTH = 43;
     // UUIDs are 16 bytes encoded base64 without padding, therefore the length is (16 / 3) * 4 + ((16 % 3) * 8 + 5) / 6 chars
     private static final int TOKEN_LENGTH = 22;
     private static final String TOKEN_DOC_ID_PREFIX = TOKEN_DOC_TYPE + "_";
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java
index 0270e31216c42..b8495e20b10bd 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java
@@ -38,6 +38,7 @@
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.ScrollHelper;
+import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheAction;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequest;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheResponse;
 import org.elasticsearch.xpack.core.security.action.user.ChangePasswordRequest;
@@ -46,7 +47,6 @@
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.esnative.ClientReservedRealm;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.core.security.user.User.Fields;
@@ -617,11 +617,9 @@ public void onFailure(Exception e) {
     }
 
     private <Response> void clearRealmCache(String username, ActionListener<Response> listener, Response response) {
-        SecurityClient securityClient = new SecurityClient(client);
-        ClearRealmCacheRequest request = securityClient.prepareClearRealmCache()
-                .usernames(username).request();
-        executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request,
-                new ActionListener<ClearRealmCacheResponse>() {
+        ClearRealmCacheRequest request = new ClearRealmCacheRequest().usernames(username);
+        executeAsyncWithOrigin(client, SECURITY_ORIGIN, ClearRealmCacheAction.INSTANCE, request,
+                new ActionListener<>() {
                     @Override
                     public void onResponse(ClearRealmCacheResponse nodes) {
                         listener.onResponse(response);
@@ -634,7 +632,7 @@ public void onFailure(Exception e) {
                                 + "] failed. please clear the realm cache manually", e);
                         listener.onFailure(exception);
                     }
-                }, securityClient::clearRealmCache);
+                });
     }
 
     @Nullable
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordTool.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordTool.java
index 5926cdbc01cbb..866f3722e6e76 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordTool.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordTool.java
@@ -193,7 +193,7 @@ private SecureString promptForPassword(Terminal terminal, String user) throws Us
             // loop for two consecutive good passwords
             while (true) {
                 SecureString password1 = new SecureString(terminal.readSecret("Enter password for [" + user + "]: "));
-                Validation.Error err = Validation.Users.validatePassword(password1.getChars());
+                Validation.Error err = Validation.Users.validatePassword(password1);
                 if (err != null) {
                     terminal.println(err.toString());
                     terminal.println("Try again.");
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java
index 6007ef5fd6d03..6d51fc5df938c 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java
@@ -7,7 +7,6 @@
 
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
-
 import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.LoggingAwareMultiCommand;
@@ -111,7 +110,7 @@ protected void execute(Terminal terminal, OptionSet options, Environment env) th
                 throw new UserException(ExitCodes.DATA_ERROR, "Invalid username [" + username + "]... " + validationError);
             }
 
-            char[] password = parsePassword(terminal, passwordOption.value(options));
+            final char[] passwordHash = getPasswordHash(terminal, env, passwordOption.value(options));
             String[] roles = parseRoles(terminal, env, rolesOption.value(options));
 
             Path passwordFile = FileUserPasswdStore.resolveFile(env);
@@ -125,9 +124,8 @@ protected void execute(Terminal terminal, OptionSet options, Environment env) th
             if (users.containsKey(username)) {
                 throw new UserException(ExitCodes.CODE_ERROR, "User [" + username + "] already exists");
             }
-            final Hasher hasher = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(env.settings()));
             users = new HashMap<>(users); // make modifiable
-            users.put(username, hasher.hash(new SecureString(password)));
+            users.put(username, passwordHash);
             FileUserPasswdStore.writeFile(users, passwordFile);
 
             if (roles.length > 0) {
@@ -218,7 +216,7 @@ protected void printAdditionalHelp(Terminal terminal) {
         protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
 
             String username = parseUsername(arguments.values(options), env.settings());
-            char[] password = parsePassword(terminal, passwordOption.value(options));
+            char[] passwordHash = getPasswordHash(terminal, env, passwordOption.value(options));
 
             Path file = FileUserPasswdStore.resolveFile(env);
             FileAttributesChecker attributesChecker = new FileAttributesChecker(file);
@@ -229,9 +227,8 @@ protected void execute(Terminal terminal, OptionSet options, Environment env) th
             if (users.containsKey(username) == false) {
                 throw new UserException(ExitCodes.NO_USER, "User [" + username + "] doesn't exist");
             }
-            final Hasher hasher = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(env.settings()));
             users = new HashMap<>(users); // make modifiable
-            users.put(username, hasher.hash(new SecureString(password)));
+            users.put(username, passwordHash);
             FileUserPasswdStore.writeFile(users, file);
 
             attributesChecker.check(terminal);
@@ -440,23 +437,32 @@ static String parseUsername(List<String> args, Settings settings) throws UserExc
         return username;
     }
 
+    private static char[] getPasswordHash(Terminal terminal, Environment env, String cliPasswordValue) throws UserException {
+        final Hasher hasher = Hasher.resolve(XPackSettings.PASSWORD_HASHING_ALGORITHM.get(env.settings()));
+        final char[] passwordHash;
+        try (SecureString password = parsePassword(terminal, cliPasswordValue)) {
+            passwordHash = hasher.hash(password);
+        }
+        return passwordHash;
+    }
+
     // pkg private for testing
-    static char[] parsePassword(Terminal terminal, String passwordStr) throws UserException {
-        char[] password;
+    static SecureString parsePassword(Terminal terminal, String passwordStr) throws UserException {
+        SecureString password;
         if (passwordStr != null) {
-            password = passwordStr.toCharArray();
+            password = new SecureString(passwordStr.toCharArray());
             Validation.Error validationError = Users.validatePassword(password);
             if (validationError != null) {
                 throw new UserException(ExitCodes.DATA_ERROR, "Invalid password..." + validationError);
             }
         } else {
-            password = terminal.readSecret("Enter new password: ");
+            password = new SecureString(terminal.readSecret("Enter new password: "));
             Validation.Error validationError = Users.validatePassword(password);
             if (validationError != null) {
                 throw new UserException(ExitCodes.DATA_ERROR, "Invalid password..." + validationError);
             }
             char[] retyped = terminal.readSecret("Retype new password: ");
-            if (Arrays.equals(password, retyped) == false) {
+            if (Arrays.equals(password.getChars(), retyped) == false) {
                 throw new UserException(ExitCodes.DATA_ERROR, "Password mismatch");
             }
         }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java
index 56c65574980db..2ca8efd4cd5e5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java
@@ -30,9 +30,9 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import javax.security.auth.login.LoginException;
 
@@ -215,18 +215,17 @@ private void resolveUser(final String userPrincipalName, final String outToken,
             final User user = (userPrincipalNameToUserCache != null) ? userPrincipalNameToUserCache.get(username) : null;
             if (user != null) {
                 listener.onResponse(AuthenticationResult.success(user));
+            } else if (userAndRealmName.length > 1) {
+                final String realmName = userAndRealmName[1];
+                buildUser(username, Map.of(KRB_METADATA_REALM_NAME_KEY, realmName, KRB_METADATA_UPN_KEY, userPrincipalName), listener);
             } else {
-                final String realmName = (userAndRealmName.length > 1) ? userAndRealmName[1] : null;
-                final Map<String, Object> metadata = new HashMap<>();
-                metadata.put(KRB_METADATA_REALM_NAME_KEY, realmName);
-                metadata.put(KRB_METADATA_UPN_KEY, userPrincipalName);
-                buildUser(username, metadata, listener);
+                buildUser(username, Map.of(KRB_METADATA_UPN_KEY, userPrincipalName), listener);
             }
         }
     }
 
     private void buildUser(final String username, final Map<String, Object> metadata, final ActionListener<AuthenticationResult> listener) {
-        final UserRoleMapper.UserData userData = new UserRoleMapper.UserData(username, null, Collections.emptySet(), metadata, this.config);
+        final UserRoleMapper.UserData userData = new UserRoleMapper.UserData(username, null, Set.of(), metadata, this.config);
         userRoleMapper.resolveRoles(userData, ActionListener.wrap(roles -> {
             final User computedUser = new User(username, roles.toArray(new String[roles.size()]), null, null, userData.getMetadata(), true);
             if (userPrincipalNameToUserCache != null) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryGroupsResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryGroupsResolver.java
index a8bf30fa71357..600ed306f9d9b 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryGroupsResolver.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryGroupsResolver.java
@@ -22,7 +22,6 @@
 
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -53,7 +52,7 @@ public void resolve(LDAPInterface connection, String userDn, TimeValue timeout,
         buildGroupQuery(connection, userDn, timeout,
                 ignoreReferralErrors, ActionListener.wrap((filter) -> {
                     if (filter == null) {
-                        listener.onResponse(Collections.emptyList());
+                        listener.onResponse(List.of());
                     } else {
                         logger.debug("group SID to DN [{}] search filter: [{}]", userDn, filter);
                         search(connection, baseDn, scope.scope(), filter,
@@ -61,8 +60,8 @@ public void resolve(LDAPInterface connection, String userDn, TimeValue timeout,
                                 ActionListener.wrap((results) -> {
                                             List<String> groups = results.stream()
                                                     .map(SearchResultEntry::getDN)
-                                                    .collect(Collectors.toList());
-                                            listener.onResponse(Collections.unmodifiableList(groups));
+                                                    .collect(Collectors.toUnmodifiableList());
+                                            listener.onResponse(groups);
                                         },
                                         listener::onFailure),
                                 SearchRequest.NO_ATTRIBUTES);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java
index 33ecf0c233a83..2f104c98cbdc6 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java
@@ -219,8 +219,7 @@ private static void lookupUserFromSession(String username, LdapSession session,
                     .put("ldap_groups", ldapData.groups)
                     .putAll(ldapData.metaData)
                     .map();
-                final UserData user = new UserData(username, session.userDn(), ldapData.groups,
-                    metadata, session.realm());
+                final UserData user = new UserData(username, session.userDn(), ldapData.groups, metadata, session.realm());
                 roleMapper.resolveRoles(user, ActionListener.wrap(
                     roles -> {
                         IOUtils.close(session);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolver.java
index 7e7fcf319a083..2909c956a276c 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolver.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolver.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession.GroupsResolver;
 
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -59,18 +58,17 @@ public void resolve(LDAPInterface connection, String userDn, TimeValue timeout,
                         Collection<Attribute> attributes, ActionListener<List<String>> listener) {
         getUserId(userDn, attributes, connection, timeout, ActionListener.wrap((userId) -> {
             if (userId == null) {
-                listener.onResponse(Collections.emptyList());
+                listener.onResponse(List.of());
             } else {
                 try {
                     Filter userFilter = createFilter(filter, userId);
-
                     search(connection, baseDn, scope.scope(), userFilter,
                             Math.toIntExact(timeout.seconds()), ignoreReferralErrors,
                             ActionListener.wrap(
                                     (results) -> listener.onResponse(results
                                             .stream()
                                             .map((r) -> r.getDN())
-                                            .collect(Collectors.toList())
+                                            .collect(Collectors.toUnmodifiableList())
                                     ),
                                     listener::onFailure),
                             SearchRequest.NO_ATTRIBUTES);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/UserAttributeGroupsResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/UserAttributeGroupsResolver.java
index 32e7cf890a322..ea532d19f594f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/UserAttributeGroupsResolver.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/UserAttributeGroupsResolver.java
@@ -17,7 +17,6 @@
 
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
@@ -47,14 +46,16 @@ private UserAttributeGroupsResolver(String attribute, boolean ignoreReferralErro
     public void resolve(LDAPInterface connection, String userDn, TimeValue timeout, Logger logger, Collection<Attribute> attributes,
                         ActionListener<List<String>> listener) {
         if (attributes != null) {
-            List<String> list = attributes.stream().filter((attr) -> attr.getName().equals(attribute))
-                    .flatMap(attr -> Arrays.stream(attr.getValues())).collect(Collectors.toList());
-            listener.onResponse(Collections.unmodifiableList(list));
+            final List<String> groups = attributes.stream()
+                    .filter((attr) -> attr.getName().equals(attribute))
+                    .flatMap(attr -> Arrays.stream(attr.getValues()))
+                    .collect(Collectors.toUnmodifiableList());
+            listener.onResponse(groups);
         } else {
             searchForEntry(connection, userDn, SearchScope.BASE, OBJECT_CLASS_PRESENCE_FILTER, Math.toIntExact(timeout.seconds()),
                     ignoreReferralErrors, ActionListener.wrap((entry) -> {
                         if (entry == null || entry.hasAttribute(attribute) == false) {
-                            listener.onResponse(Collections.emptyList());
+                            listener.onResponse(List.of());
                         } else {
                             listener.onResponse(List.of(entry.getAttributeValues(attribute)));
                         }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/LdapMetaDataResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/LdapMetaDataResolver.java
index ad5aa7927b68c..82d8cb6e207f4 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/LdapMetaDataResolver.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/support/LdapMetaDataResolver.java
@@ -17,7 +17,7 @@
 
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.Function;
@@ -48,7 +48,7 @@ public void resolve(LDAPInterface connection, String userDn, TimeValue timeout,
                         Collection<Attribute> attributes,
                         ActionListener<Map<String, Object>> listener) {
         if (this.attributeNames.length == 0) {
-            listener.onResponse(Collections.emptyMap());
+            listener.onResponse(Map.of());
         } else if (attributes != null) {
             listener.onResponse(toMap(name -> findAttribute(attributes, name)));
         } else {
@@ -56,7 +56,7 @@ public void resolve(LDAPInterface connection, String userDn, TimeValue timeout,
                     Math.toIntExact(timeout.seconds()), ignoreReferralErrors,
                     ActionListener.wrap((SearchResultEntry entry) -> {
                         if (entry == null) {
-                            listener.onResponse(Collections.emptyMap());
+                            listener.onResponse(Map.of());
                         } else {
                             listener.onResponse(toMap(entry::getAttribute));
                         }
@@ -71,16 +71,14 @@ private Attribute findAttribute(Collection<Attribute> attributes, String name) {
     }
 
     private Map<String, Object> toMap(Function<String, Attribute> attributes) {
-        return Collections.unmodifiableMap(
-                Arrays.stream(this.attributeNames).map(attributes).filter(Objects::nonNull)
-                        .collect(Collectors.toMap(
+        return Arrays.stream(this.attributeNames).map(attributes).filter(Objects::nonNull)
+                        .collect(Collectors.toUnmodifiableMap(
                                 attr -> attr.getName(),
                                 attr -> {
                                     final String[] values = attr.getValues();
-                                    return values.length == 1 ? values[0] : Arrays.asList(values);
+                                    return values.length == 1 ? values[0] : List.of(values);
                                 })
-                        )
-        );
+                        );
     }
 
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java
index c652a39b90912..a3218fc90552f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java
@@ -95,6 +95,8 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+
+import java.net.URLEncoder;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
@@ -465,8 +467,9 @@ private void exchangeCodeForToken(AuthorizationCode code, ActionListener<Tuple<A
             }
             httpPost.setEntity(new UrlEncodedFormEntity(params));
             httpPost.setHeader("Content-type", "application/x-www-form-urlencoded");
-            UsernamePasswordCredentials creds = new UsernamePasswordCredentials(rpConfig.getClientId().getValue(),
-                rpConfig.getClientSecret().toString());
+            UsernamePasswordCredentials creds =
+                new UsernamePasswordCredentials(URLEncoder.encode(rpConfig.getClientId().getValue(), StandardCharsets.UTF_8),
+                    URLEncoder.encode(rpConfig.getClientSecret().toString(), StandardCharsets.UTF_8));
             httpPost.addHeader(new BasicScheme().authenticate(creds, httpPost, null));
             SpecialPermission.check();
             AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
@@ -626,7 +629,7 @@ private void setMetadataFileWatcher(String jwkSetPath) throws IOException {
      * necessary as some OPs return slightly different values for some claims (i.e. Google for the profile picture) and
      * {@link JSONObject#merge(Object)} would throw a runtime exception. The merging is performed based on the following rules:
      * <ul>
-     * <li>If the values for a given claim are primitives (of the the same type), the value from the ID Token is retained</li>
+     * <li>If the values for a given claim are primitives (of the same type), the value from the ID Token is retained</li>
      * <li>If the values for a given claim are Objects, the values are merged</li>
      * <li>If the values for a given claim are Arrays, the values are merged without removing duplicates</li>
      * <li>If the values for a given claim are of different types, an exception is thrown</li>
@@ -777,6 +780,7 @@ public void completed(HttpResponse result) {
                                     StandardCharsets.UTF_8));
                                 reloadFutureRef.set(null);
                                 LOGGER.trace("Successfully refreshed and cached remote JWKSet");
+                                future.onResponse(null);
                             } catch (IOException | ParseException e) {
                                 failed(e);
                             }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectRealm.java
index ac933dcfef878..4267d39cb1f7b 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectRealm.java
@@ -48,13 +48,11 @@
 
 import java.net.URI;
 import java.net.URISyntaxException;
-
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 import java.util.function.Function;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -202,19 +200,17 @@ private void buildUserFromClaims(JWTClaimsSet claims, ActionListener<Authenticat
             return;
         }
 
-        final Map<String, Object> userMetadata = new HashMap<>();
+        final Map<String, Object> userMetadata;
         if (populateUserMetadata) {
-            Map<String, Object> claimsMap = claims.getClaims();
-            /*
-             * We whitelist the Types that we want to parse as metadata from the Claims, explicitly filtering out {@link Date}s
-             */
-            Set<Map.Entry> allowedEntries = claimsMap.entrySet().stream().filter(entry -> {
+            userMetadata = claims.getClaims().entrySet().stream().filter(entry -> {
+                /*
+                 * We whitelist the Types that we want to parse as metadata from the Claims, explicitly filtering out {@link Date}s
+                 */
                 Object v = entry.getValue();
                 return (v instanceof String || v instanceof Boolean || v instanceof Number || v instanceof Collections);
-            }).collect(Collectors.toSet());
-            for (Map.Entry entry : allowedEntries) {
-                userMetadata.put("oidc(" + entry.getKey() + ")", entry.getValue());
-            }
+            }).collect(Collectors.toUnmodifiableMap(entry -> "oidc(" + entry.getKey() + ")", Map.Entry::getValue));
+        } else {
+            userMetadata = Map.of();
         }
         final List<String> groups = groupsAttribute.getClaimValues(claims);
         final String dn = dnAttribute.getClaimValue(claims);
@@ -416,9 +412,9 @@ static ClaimParser forSetting(Logger logger, OpenIdConnectRealmSettings.ClaimSet
                             Object claimValueObject = claims.getClaim(claimName);
                             List<String> values;
                             if (claimValueObject == null) {
-                                values = Collections.emptyList();
+                                values = List.of();
                             } else if (claimValueObject instanceof String) {
-                                values = Collections.singletonList((String) claimValueObject);
+                                values = List.of((String) claimValueObject);
                             } else if (claimValueObject instanceof List) {
                                 values = (List<String>) claimValueObject;
                             } else {
@@ -427,6 +423,10 @@ static ClaimParser forSetting(Logger logger, OpenIdConnectRealmSettings.ClaimSet
                                     + claimValueObject.getClass().getName());
                             }
                             return values.stream().map(s -> {
+                                if (s == null) {
+                                    logger.debug("OpenID Connect Claim [{}] is null", claimName);
+                                    return null;
+                                }
                                 final Matcher matcher = regex.matcher(s);
                                 if (matcher.find() == false) {
                                     logger.debug("OpenID Connect Claim [{}] is [{}], which does not match [{}]",
@@ -440,7 +440,7 @@ static ClaimParser forSetting(Logger logger, OpenIdConnectRealmSettings.ClaimSet
                                     return null;
                                 }
                                 return value;
-                            }).filter(Objects::nonNull).collect(Collectors.toList());
+                            }).filter(Objects::nonNull).collect(Collectors.toUnmodifiableList());
                         });
                 } else {
                     return new ClaimParser(
@@ -448,15 +448,17 @@ static ClaimParser forSetting(Logger logger, OpenIdConnectRealmSettings.ClaimSet
                         claims -> {
                             Object claimValueObject = claims.getClaim(claimName);
                             if (claimValueObject == null) {
-                                return Collections.emptyList();
+                                return List.of();
                             } else if (claimValueObject instanceof String) {
-                                return Collections.singletonList((String) claimValueObject);
+                                return List.of((String) claimValueObject);
                             } else if (claimValueObject instanceof List == false) {
                                 throw new SettingsException("Setting [" + RealmSettings.getFullSettingKey(realmConfig, setting.getClaim())
                                     + " expects a claim with String or a String Array value but found a "
                                     + claimValueObject.getClass().getName());
                             }
-                            return (List<String>) claimValueObject;
+                            return ((List<String>) claimValueObject).stream()
+                                        .filter(Objects::nonNull)
+                                        .collect(Collectors.toUnmodifiableList());
                         });
                 }
             } else if (required) {
@@ -467,8 +469,7 @@ static ClaimParser forSetting(Logger logger, OpenIdConnectRealmSettings.ClaimSet
                     + "] cannot be set unless [" + RealmSettings.getFullSettingKey(realmConfig, setting.getClaim())
                     + "] is also set");
             } else {
-                return new ClaimParser("No OpenID Connect Claim for [" + setting.name(realmConfig) + "]",
-                    attributes -> Collections.emptyList());
+                return new ClaimParser("No OpenID Connect Claim for [" + setting.name(realmConfig) + "]", attributes -> List.of());
             }
         }
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java
index a6488916d7e1e..92a521789ec86 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java
@@ -42,10 +42,10 @@
 import java.security.cert.CertificateEncodingException;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.regex.Matcher;
@@ -149,9 +149,8 @@ public void authenticate(AuthenticationToken authToken, ActionListener<Authentic
     }
 
     private void buildUser(X509AuthenticationToken token, ActionListener<AuthenticationResult> listener) {
-        final Map<String, Object> metadata = Collections.singletonMap("pki_dn", token.dn());
-        final UserRoleMapper.UserData userData = new UserRoleMapper.UserData(token.principal(),
-                token.dn(), Collections.emptySet(), metadata, this.config);
+        final Map<String, Object> metadata = Map.of("pki_dn", token.dn());
+        final UserRoleMapper.UserData userData = new UserRoleMapper.UserData(token.principal(), token.dn(), Set.of(), metadata, config);
         roleMapper.resolveRoles(userData, ActionListener.wrap(roles -> {
             final User computedUser =
                     new User(token.principal(), roles.toArray(new String[roles.size()]), null, null, metadata, true);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlAttributes.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlAttributes.java
index 6915d1aabf635..0200c594f2ff0 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlAttributes.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlAttributes.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.security.authc.saml;
 
-import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
@@ -40,23 +39,23 @@ public class SamlAttributes {
      * @return A list of all matching attribute values (may be empty).
      */
     List<String> getAttributeValues(String attributeId) {
+        if (Strings.isNullOrEmpty(attributeId)) {
+            return List.of();
+        }
         if (attributeId.equals(NAMEID_SYNTHENTIC_ATTRIBUTE)) {
-            return name == null ? Collections.emptyList() : Collections.singletonList(name.value);
+            return name == null ? List.of() : List.of(name.value);
         }
-        if (name != null && NameIDType.PERSISTENT.equals(name.format) && attributeId.equals(PERSISTENT_NAMEID_SYNTHENTIC_ATTRIBUTE)) {
-            return Collections.singletonList(name.value);
-        }
-        if (Strings.isNullOrEmpty(attributeId)) {
-            return Collections.emptyList();
+        if (attributeId.equals(PERSISTENT_NAMEID_SYNTHENTIC_ATTRIBUTE) && name != null && NameIDType.PERSISTENT.equals(name.format)) {
+            return List.of(name.value);
         }
         return attributes.stream()
                 .filter(attr -> attributeId.equals(attr.name) || attributeId.equals(attr.friendlyName))
                 .flatMap(attr -> attr.values.stream())
-                .collect(Collectors.toList());
+                .collect(Collectors.toUnmodifiableList());
     }
 
     List<SamlAttribute> attributes() {
-        return Collections.unmodifiableList(attributes);
+        return attributes;
     }
 
     SamlNameId name() {
@@ -79,13 +78,16 @@ static class SamlAttribute {
 
         SamlAttribute(Attribute attribute) {
             this(attribute.getName(), attribute.getFriendlyName(),
-                    attribute.getAttributeValues().stream().map(x -> x.getDOM().getTextContent()).collect(Collectors.toList()));
+                    attribute.getAttributeValues().stream()
+                        .map(x -> x.getDOM().getTextContent())
+                        .filter(Objects::nonNull)
+                        .collect(Collectors.toUnmodifiableList()));
         }
 
         SamlAttribute(String name, @Nullable String friendlyName, List<String> values) {
             this.name = Objects.requireNonNull(name, "Attribute name cannot be null");
             this.friendlyName = friendlyName;
-            this.values = Collections.unmodifiableList(values);
+            this.values = values;
         }
 
         @Override
@@ -98,5 +100,4 @@ public String toString() {
         }
     }
 
-
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java
index be45ff76ec48f..867bc5d523fff 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java
@@ -439,20 +439,22 @@ private void buildUser(SamlAttributes attributes, ActionListener<AuthenticationR
             return;
         }
 
-        final Map<String, Object> userMeta = new HashMap<>();
+        final Map<String, Object> userMetaBuilder = new HashMap<>();
         if (populateUserMetadata) {
             for (SamlAttributes.SamlAttribute a : attributes.attributes()) {
-                userMeta.put("saml(" + a.name + ")", a.values);
+                userMetaBuilder.put("saml(" + a.name + ")", a.values);
                 if (Strings.hasText(a.friendlyName)) {
-                    userMeta.put("saml_" + a.friendlyName, a.values);
+                    userMetaBuilder.put("saml_" + a.friendlyName, a.values);
                 }
             }
         }
         if (attributes.name() != null) {
-            userMeta.put(USER_METADATA_NAMEID_VALUE, attributes.name().value);
-            userMeta.put(USER_METADATA_NAMEID_FORMAT, attributes.name().format);
+            userMetaBuilder.put(USER_METADATA_NAMEID_VALUE, attributes.name().value);
+            if (attributes.name().format != null) {
+                userMetaBuilder.put(USER_METADATA_NAMEID_FORMAT, attributes.name().format);
+            }
         }
-
+        final Map<String, Object> userMeta = Map.copyOf(userMetaBuilder);
 
         final List<String> groups = groupsAttribute.getAttribute(attributes);
         final String dn = resolveSingleValueAttribute(attributes, dnAttribute, DN_ATTRIBUTE.name(config));
@@ -764,7 +766,7 @@ static AttributeParser forSetting(Logger logger, SamlRealmSettings.AttributeSett
                                     return null;
                                 }
                                 return value;
-                            }).filter(Objects::nonNull).collect(Collectors.toList())
+                            }).filter(Objects::nonNull).collect(Collectors.toUnmodifiableList())
                     );
                 } else {
                     return new AttributeParser(
@@ -780,7 +782,7 @@ static AttributeParser forSetting(Logger logger, SamlRealmSettings.AttributeSett
                         + "] is also set");
             } else {
                 return new AttributeParser("No SAML attribute for [" + setting.name(realmConfig) + "]",
-                        attributes -> Collections.emptyList());
+                        attributes -> List.of());
             }
         }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/UserRoleMapper.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/UserRoleMapper.java
index c23cae1aa878d..6ceb9629d6b4f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/UserRoleMapper.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/UserRoleMapper.java
@@ -20,9 +20,6 @@
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 
 import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
@@ -63,12 +60,8 @@ public UserData(String username, @Nullable String dn, Collection<String> groups,
                         Map<String, Object> metadata, RealmConfig realm) {
             this.username = username;
             this.dn = dn;
-            // noinspection Java9CollectionFactory (because null values happen in some tests, is this realistic?)
-            this.groups = groups == null || groups.isEmpty()
-                    ? Collections.emptySet() : Collections.unmodifiableSet(new HashSet<>(groups));
-            // noinspection Java9CollectionFactory (because null values happen in production code, can such keys be dropped?)
-            this.metadata = metadata == null || metadata.isEmpty()
-                    ? Collections.emptyMap() : Collections.unmodifiableMap(new HashMap<>(metadata));
+            this.groups = Set.copyOf(groups);
+            this.metadata = Map.copyOf(metadata);
             this.realm = realm;
         }
 
@@ -87,7 +80,6 @@ public ExpressionModel asModel() {
                 model.defineField("dn", dn, new DistinguishedNamePredicate(dn));
             }
             model.defineField("groups", groups, groups.stream()
-                    .filter(group -> group != null)
                     .<Predicate<FieldExpression.FieldValue>>map(DistinguishedNamePredicate::new)
                     .reduce(Predicate::or)
                     .orElse(fieldValue -> false)
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java
index bb98dddbe1ddf..0c425d771a4c3 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java
@@ -30,12 +30,11 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.xpack.core.security.ScrollHelper;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheAction;
-import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheResponse;
+import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequest;
 import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequest;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.ExpressionModel;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
 import org.elasticsearch.xpack.security.authc.support.CachingRealm;
 import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
@@ -336,10 +335,8 @@ private <Result> void refreshRealms(ActionListener<Result> listener, Result resu
         }
 
         final String[] realmNames = this.realmsToRefresh.toArray(Strings.EMPTY_ARRAY);
-        final SecurityClient securityClient = new SecurityClient(client);
-        executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN,
-                securityClient.prepareClearRealmCache().realms(realmNames).request(),
-                ActionListener.<ClearRealmCacheResponse>wrap(
+        executeAsyncWithOrigin(client, SECURITY_ORIGIN, ClearRealmCacheAction.INSTANCE, new ClearRealmCacheRequest().realms(realmNames),
+                ActionListener.wrap(
                         response -> {
                             logger.debug((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage(
                                     "Cleared cached in realms [{}] due to role mapping change", Arrays.toString(realmNames)));
@@ -348,8 +345,7 @@ private <Result> void refreshRealms(ActionListener<Result> listener, Result resu
                         ex -> {
                             logger.warn(new ParameterizedMessage("Failed to clear cache for realms [{}]", Arrays.toString(realmNames)), ex);
                             listener.onFailure(ex);
-                        }),
-                securityClient::clearRealmCache);
+                        }));
     }
 
     @Override
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
index b0965a1c5330e..77b539e593b89 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
 import org.elasticsearch.action.search.SearchRequest;
@@ -125,12 +124,6 @@ ResolvedIndices resolveIndicesAndAliases(IndicesRequest indicesRequest, MetaData
             final boolean replaceWildcards = indicesRequest.indicesOptions().expandWildcardsOpen()
                     || indicesRequest.indicesOptions().expandWildcardsClosed();
             IndicesOptions indicesOptions = indicesRequest.indicesOptions();
-            if (indicesRequest instanceof IndicesExistsRequest) {
-                //indices exists api should never throw exception, make sure that ignore_unavailable and allow_no_indices are true
-                //we have to mimic what TransportIndicesExistsAction#checkBlock does in es core
-                indicesOptions = IndicesOptions.fromOptions(true, true,
-                        indicesOptions.expandWildcardsOpen(), indicesOptions.expandWildcardsClosed());
-            }
 
             // check for all and return list of authorized indices
             if (IndexNameExpressionResolver.isAllIndices(indicesList(indicesRequest.indices()))) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java
index 7454ec59da55f..a33a1dc686b41 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetBitsetCache;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition.FieldGrantExcludeGroup;
@@ -39,6 +40,7 @@
 import org.elasticsearch.xpack.core.security.authz.privilege.Privilege;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult;
+import org.elasticsearch.xpack.core.security.support.CacheIteratorHelper;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
 import org.elasticsearch.xpack.core.security.user.User;
@@ -53,14 +55,11 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -84,18 +83,6 @@ public class CompositeRolesStore {
         Setting.intSetting("xpack.security.authz.store.roles.negative_lookup_cache.max_size", 10000, Property.NodeScope);
     private static final Logger logger = LogManager.getLogger(CompositeRolesStore.class);
 
-    // the lock is used in an odd manner; when iterating over the cache we cannot have modifiers other than deletes using
-    // the iterator but when not iterating we can modify the cache without external locking. When making normal modifications to the cache
-    // the read lock is obtained so that we can allow concurrent modifications; however when we need to iterate over the keys or values of
-    // the cache the write lock must obtained to prevent any modifications
-    private final ReleasableLock readLock;
-    private final ReleasableLock writeLock;
-
-    {
-        final ReadWriteLock iterationLock = new ReentrantReadWriteLock();
-        readLock = new ReleasableLock(iterationLock.readLock());
-        writeLock = new ReleasableLock(iterationLock.writeLock());
-    }
 
     private final FileRolesStore fileRolesStore;
     private final NativeRolesStore nativeRolesStore;
@@ -104,7 +91,9 @@ public class CompositeRolesStore {
     private final Consumer<Collection<RoleDescriptor>> effectiveRoleDescriptorsConsumer;
     private final FieldPermissionsCache fieldPermissionsCache;
     private final Cache<RoleKey, Role> roleCache;
+    private final CacheIteratorHelper<RoleKey, Role> roleCacheHelper;
     private final Cache<String, Boolean> negativeLookupCache;
+    private final DocumentSubsetBitsetCache dlsBitsetCache;
     private final ThreadContext threadContext;
     private final AtomicLong numInvalidation = new AtomicLong();
     private final AnonymousUser anonymousUser;
@@ -117,8 +106,10 @@ public CompositeRolesStore(Settings settings, FileRolesStore fileRolesStore, Nat
                                ReservedRolesStore reservedRolesStore, NativePrivilegeStore privilegeStore,
                                List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>> rolesProviders,
                                ThreadContext threadContext, XPackLicenseState licenseState, FieldPermissionsCache fieldPermissionsCache,
-                               ApiKeyService apiKeyService, Consumer<Collection<RoleDescriptor>> effectiveRoleDescriptorsConsumer) {
+                               ApiKeyService apiKeyService, @Nullable DocumentSubsetBitsetCache dlsBitsetCache,
+                               Consumer<Collection<RoleDescriptor>> effectiveRoleDescriptorsConsumer) {
         this.fileRolesStore = fileRolesStore;
+        this.dlsBitsetCache = dlsBitsetCache;
         fileRolesStore.addListener(this::invalidate);
         this.nativeRolesStore = nativeRolesStore;
         this.privilegeStore = privilegeStore;
@@ -132,6 +123,7 @@ public CompositeRolesStore(Settings settings, FileRolesStore fileRolesStore, Nat
             builder.setMaximumWeight(cacheSize);
         }
         this.roleCache = builder.build();
+        this.roleCacheHelper = new CacheIteratorHelper(roleCache);
         this.threadContext = threadContext;
         CacheBuilder<String, Boolean> nlcBuilder = CacheBuilder.builder();
         final int nlcCacheSize = NEGATIVE_LOOKUP_CACHE_SIZE_SETTING.get(settings);
@@ -261,7 +253,7 @@ private void buildThenMaybeCacheRole(RoleKey roleKey, Collection<RoleDescriptor>
         logger.trace("Building role from descriptors [{}] for names [{}] from source [{}]", roleDescriptors, roleKey.names, roleKey.source);
         buildRoleFromDescriptors(roleDescriptors, fieldPermissionsCache, privilegeStore, ActionListener.wrap(role -> {
             if (role != null && tryCache) {
-                try (ReleasableLock ignored = readLock.acquire()) {
+                try (ReleasableLock ignored = roleCacheHelper.acquireUpdateLock()) {
                     /* this is kinda spooky. We use a read/write lock to ensure we don't modify the cache if we hold
                      * the write lock (fetching stats for instance - which is kinda overkill?) but since we fetching
                      * stuff in an async fashion we need to make sure that if the cache got invalidated since we
@@ -420,47 +412,31 @@ public static void buildRoleFromDescriptors(Collection<RoleDescriptor> roleDescr
     public void invalidateAll() {
         numInvalidation.incrementAndGet();
         negativeLookupCache.invalidateAll();
-        try (ReleasableLock ignored = readLock.acquire()) {
+        try (ReleasableLock ignored = roleCacheHelper.acquireUpdateLock()) {
             roleCache.invalidateAll();
         }
+        if (dlsBitsetCache != null) {
+            dlsBitsetCache.clear("role store invalidation");
+        }
     }
 
     public void invalidate(String role) {
         numInvalidation.incrementAndGet();
 
-        // the cache cannot be modified while doing this operation per the terms of the cache iterator
-        try (ReleasableLock ignored = writeLock.acquire()) {
-            Iterator<RoleKey> keyIter = roleCache.keys().iterator();
-            while (keyIter.hasNext()) {
-                RoleKey key = keyIter.next();
-                if (key.names.contains(role)) {
-                    keyIter.remove();
-                }
-            }
-        }
+        roleCacheHelper.removeKeysIf(key -> key.names.contains(role));
         negativeLookupCache.invalidate(role);
     }
 
     public void invalidate(Set<String> roles) {
         numInvalidation.incrementAndGet();
-
-        // the cache cannot be modified while doing this operation per the terms of the cache iterator
-        try (ReleasableLock ignored = writeLock.acquire()) {
-            Iterator<RoleKey> keyIter = roleCache.keys().iterator();
-            while (keyIter.hasNext()) {
-                RoleKey key = keyIter.next();
-                if (Sets.haveEmptyIntersection(key.names, roles) == false) {
-                    keyIter.remove();
-                }
-            }
-        }
-
+        roleCacheHelper.removeKeysIf(key -> Sets.haveEmptyIntersection(key.names, roles) == false);
         roles.forEach(negativeLookupCache::invalidate);
     }
 
     public void usageStats(ActionListener<Map<String, Object>> listener) {
         final Map<String, Object> usage = new HashMap<>(2);
         usage.put("file", fileRolesStore.usageStats());
+        usage.put("dls", Map.of("bit_set_cache", dlsBitsetCache.usageStats()));
         nativeRolesStore.usageStats(ActionListener.wrap(map -> {
             usage.put("native", map);
             listener.onResponse(usage);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java
index 7e70f6116cee7..64c9df8e7005a 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStore.java
@@ -41,10 +41,10 @@
 import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.security.ScrollHelper;
+import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheAction;
 import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheRequest;
 import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheResponse;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import java.io.IOException;
@@ -83,13 +83,11 @@ public class NativePrivilegeStore {
 
     private final Settings settings;
     private final Client client;
-    private final SecurityClient securityClient;
     private final SecurityIndexManager securityIndexManager;
 
     public NativePrivilegeStore(Settings settings, Client client, SecurityIndexManager securityIndexManager) {
         this.settings = settings;
         this.client = client;
-        this.securityClient = new SecurityClient(client);
         this.securityIndexManager = securityIndexManager;
     }
 
@@ -295,8 +293,8 @@ public void deletePrivileges(String application, Collection<String> names, Write
     private <T> void clearRolesCache(ActionListener<T> listener, T value) {
         // This currently clears _all_ roles, but could be improved to clear only those roles that reference the affected application
         ClearRolesCacheRequest request = new ClearRolesCacheRequest();
-        executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request,
-            new ActionListener<ClearRolesCacheResponse>() {
+        executeAsyncWithOrigin(client, SECURITY_ORIGIN, ClearRolesCacheAction.INSTANCE, request,
+            new ActionListener<>() {
                 @Override
                 public void onResponse(ClearRolesCacheResponse nodes) {
                     listener.onResponse(value);
@@ -308,7 +306,7 @@ public void onFailure(Exception e) {
                     listener.onFailure(
                         new ElasticsearchException("clearing the role cache failed. please clear the role cache manually", e));
                 }
-            }, securityClient::clearRolesCache);
+            });
     }
 
     private ApplicationPrivilegeDescriptor buildPrivilege(String docId, BytesReference source) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
index 2169f84082b7f..b962e853cc629 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
@@ -36,6 +36,7 @@
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.core.security.ScrollHelper;
+import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheAction;
 import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheRequest;
 import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheResponse;
 import org.elasticsearch.xpack.core.security.action.role.DeleteRoleRequest;
@@ -43,7 +44,6 @@
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
 import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import java.io.IOException;
@@ -84,13 +84,11 @@ public class NativeRolesStore implements BiConsumer<Set<String>, ActionListener<
     private final Client client;
     private final XPackLicenseState licenseState;
 
-    private SecurityClient securityClient;
     private final SecurityIndexManager securityIndex;
 
     public NativeRolesStore(Settings settings, Client client, XPackLicenseState licenseState, SecurityIndexManager securityIndex) {
         this.settings = settings;
         this.client = client;
-        this.securityClient = new SecurityClient(client);
         this.licenseState = licenseState;
         this.securityIndex = securityIndex;
     }
@@ -338,8 +336,8 @@ private void executeGetRoleRequest(String role, ActionListener<GetResponse> list
 
     private <Response> void clearRoleCache(final String role, ActionListener<Response> listener, Response response) {
         ClearRolesCacheRequest request = new ClearRolesCacheRequest().names(role);
-        executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request,
-                new ActionListener<ClearRolesCacheResponse>() {
+        executeAsyncWithOrigin(client, SECURITY_ORIGIN, ClearRolesCacheAction.INSTANCE, request,
+                new ActionListener<>() {
                     @Override
                     public void onResponse(ClearRolesCacheResponse nodes) {
                         listener.onResponse(response);
@@ -352,7 +350,7 @@ public void onFailure(Exception e) {
                                 + "] failed. please clear the role cache manually", e);
                         listener.onFailure(exception);
                     }
-                }, securityClient::clearRolesCache);
+                });
     }
 
     @Nullable
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java
index 14d4726553dff..b75de301e7e4a 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java
@@ -9,15 +9,12 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequestBuilder;
-import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 
 import java.io.IOException;
 
@@ -44,13 +41,11 @@ public String getName() {
 
     @Override
     protected RestChannelConsumer innerPrepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        try (XContentParser parser = request.contentParser()) {
-            String refresh = request.param("refresh");
-            CreateApiKeyRequestBuilder builder = new SecurityClient(client)
-                    .prepareCreateApiKey(request.requiredContent(), request.getXContentType())
-                    .setRefreshPolicy((refresh != null) ? WriteRequest.RefreshPolicy.parse(request.param("refresh"))
-                            : CreateApiKeyRequest.DEFAULT_REFRESH_POLICY);
-            return channel -> builder.execute(new RestToXContentListener<CreateApiKeyResponse>(channel));
-        }
+        String refresh = request.param("refresh");
+        CreateApiKeyRequestBuilder builder = new CreateApiKeyRequestBuilder(client)
+            .source(request.requiredContent(), request.getXContentType())
+            .setRefreshPolicy((refresh != null) ?
+                WriteRequest.RefreshPolicy.parse(request.param("refresh")) : CreateApiKeyRequest.DEFAULT_REFRESH_POLICY);
+        return channel -> builder.execute(new RestToXContentListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestGetTokenAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestGetTokenAction.java
index 4734c39bc5af5..c3734f37d1a54 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestGetTokenAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestGetTokenAction.java
@@ -8,7 +8,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.client.node.NodeClient;
@@ -30,9 +30,11 @@
 import org.elasticsearch.xpack.core.security.action.token.CreateTokenRequest;
 import org.elasticsearch.xpack.core.security.action.token.CreateTokenResponse;
 import org.elasticsearch.xpack.core.security.action.token.RefreshTokenAction;
+import org.elasticsearch.xpack.security.authc.kerberos.KerberosAuthenticationToken;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.List;
 import java.util.Locale;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
@@ -48,13 +50,17 @@ public final class RestGetTokenAction extends TokenBaseRestHandler {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetTokenAction.class));
     static final ConstructingObjectParser<CreateTokenRequest, Void> PARSER = new ConstructingObjectParser<>("token_request",
-            a -> new CreateTokenRequest((String) a[0], (String) a[1], (SecureString) a[2], (String) a[3], (String) a[4]));
+            a -> new CreateTokenRequest((String) a[0], (String) a[1], (SecureString) a[2], (SecureString) a[3], (String) a[4],
+                    (String) a[5]));
     static {
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("grant_type"));
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("username"));
         PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), parser -> new SecureString(
                 Arrays.copyOfRange(parser.textCharacters(), parser.textOffset(), parser.textOffset() + parser.textLength())),
                 new ParseField("password"), ValueType.STRING);
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), parser -> new SecureString(
+                Arrays.copyOfRange(parser.textCharacters(), parser.textOffset(), parser.textOffset() + parser.textLength())),
+                new ParseField("kerberos_ticket"), ValueType.STRING);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("scope"));
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("refresh_token"));
     }
@@ -76,7 +82,7 @@ public String getName() {
     protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client)throws IOException {
         try (XContentParser parser = request.contentParser()) {
             final CreateTokenRequest tokenRequest = PARSER.parse(parser, null);
-            final Action<CreateTokenResponse> action =
+            final ActionType<CreateTokenResponse> action =
                     "refresh_token".equals(tokenRequest.getGrantType()) ? RefreshTokenAction.INSTANCE : CreateTokenAction.INSTANCE;
             return channel -> client.execute(action, tokenRequest,
                     // this doesn't use the RestBuilderListener since we need to override the
@@ -124,11 +130,28 @@ public void onFailure(Exception e) {
                     ((ElasticsearchSecurityException) e).getHeader("error_description").size() == 1) {
                 sendTokenErrorResponse(TokenRequestError.INVALID_GRANT,
                         ((ElasticsearchSecurityException) e).getHeader("error_description").get(0), e);
+            } else if (e instanceof ElasticsearchSecurityException
+                    && "failed to authenticate user, gss context negotiation not complete".equals(e.getMessage())) {
+                sendTokenErrorResponse(TokenRequestError._UNAUTHORIZED, extractBase64EncodedToken((ElasticsearchSecurityException) e), e);
             } else {
                 sendFailure(e);
             }
         }
 
+        private String extractBase64EncodedToken(ElasticsearchSecurityException e) {
+            String base64EncodedToken = null;
+            List<String> values = e.getHeader(KerberosAuthenticationToken.WWW_AUTHENTICATE);
+            if (values != null && values.size() == 1) {
+                final String wwwAuthenticateHeaderValue = values.get(0);
+                // it may contain base64 encoded token that needs to be sent to client if Spnego GSS context negotiation failed
+                if (wwwAuthenticateHeaderValue.startsWith(KerberosAuthenticationToken.NEGOTIATE_AUTH_HEADER_PREFIX)) {
+                    base64EncodedToken = wwwAuthenticateHeaderValue
+                            .substring(KerberosAuthenticationToken.NEGOTIATE_AUTH_HEADER_PREFIX.length()).trim();
+                }
+            }
+            return base64EncodedToken;
+        }
+
         void sendTokenErrorResponse(TokenRequestError error, String description, Exception e) {
             try (XContentBuilder builder = channel.newErrorBuilder()) {
                 // defined by https://tools.ietf.org/html/rfc6749#section-5.2
@@ -200,6 +223,17 @@ enum TokenRequestError {
          * The requested scope is invalid, unknown, malformed, or exceeds the
          * scope granted by the resource owner.
          */
-        INVALID_SCOPE
+        INVALID_SCOPE,
+
+        // Custom error code
+        /**
+         * When the request for authentication fails using custom grant type for given
+         * credentials.
+         * If the client attempted to authenticate via the "Authorization" request
+         * the authorization server MAY respond with an HTTP 401
+         * (Unauthorized) status code and include the "WWW-Authenticate"
+         * response header field
+         */
+        _UNAUTHORIZED,
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestDeletePrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestDeletePrivilegesAction.java
index 1f96df23e0e77..41758aa26016f 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestDeletePrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestDeletePrivilegesAction.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesResponse;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
 import java.io.IOException;
@@ -54,21 +54,23 @@ public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient c
         final String application = request.param("application");
         final String[] privileges = request.paramAsStringArray("privilege", null);
         final String refresh = request.param("refresh");
-        return channel -> new SecurityClient(client).prepareDeletePrivileges(application, privileges)
-                .setRefreshPolicy(refresh)
-                .execute(new RestBuilderListener<DeletePrivilegesResponse>(channel) {
-                    @Override
-                    public RestResponse buildResponse(DeletePrivilegesResponse response, XContentBuilder builder) throws Exception {
-                        builder.startObject();
-                        builder.startObject(application);
-                        for (String privilege : new HashSet<>(Arrays.asList(privileges))) {
-                            builder.field(privilege, Collections.singletonMap("found", response.found().contains(privilege)));
-                        }
-                        builder.endObject();
-                        builder.endObject();
-                        return new BytesRestResponse(response.found().isEmpty() ? RestStatus.NOT_FOUND : RestStatus.OK, builder);
+        return channel -> new DeletePrivilegesRequestBuilder(client)
+            .application(application)
+            .privileges(privileges)
+            .setRefreshPolicy(refresh)
+            .execute(new RestBuilderListener<>(channel) {
+                @Override
+                public RestResponse buildResponse(DeletePrivilegesResponse response, XContentBuilder builder) throws Exception {
+                    builder.startObject();
+                    builder.startObject(application);
+                    for (String privilege : new HashSet<>(Arrays.asList(privileges))) {
+                        builder.field(privilege, Collections.singletonMap("found", response.found().contains(privilege)));
                     }
-                });
+                    builder.endObject();
+                    builder.endObject();
+                    return new BytesRestResponse(response.found().isEmpty() ? RestStatus.NOT_FOUND : RestStatus.OK, builder);
+                }
+            });
     }
 
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java
new file mode 100644
index 0000000000000..15f4612d90429
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetBuiltinPrivilegesAction.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.rest.action.privilege;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.core.security.action.privilege.GetBuiltinPrivilegesAction;
+import org.elasticsearch.xpack.core.security.action.privilege.GetBuiltinPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.privilege.GetBuiltinPrivilegesResponse;
+import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
+
+import java.io.IOException;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+
+/**
+ * Rest action to retrieve an application privilege from the security index
+ */
+public class RestGetBuiltinPrivilegesAction extends SecurityBaseRestHandler {
+
+    public RestGetBuiltinPrivilegesAction(Settings settings, RestController controller, XPackLicenseState licenseState) {
+        super(settings, licenseState);
+        controller.registerHandler(
+            GET, "/_security/privilege/_builtin", this);
+    }
+
+    @Override
+    public String getName() {
+        return "security_get_builtin_privileges_action";
+    }
+
+    @Override
+    public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
+        return channel -> client.execute(GetBuiltinPrivilegesAction.INSTANCE, new GetBuiltinPrivilegesRequest(),
+            new RestBuilderListener<>(channel) {
+                @Override
+                public RestResponse buildResponse(GetBuiltinPrivilegesResponse response, XContentBuilder builder) throws Exception {
+                    builder.startObject();
+                    builder.array("cluster", response.getClusterPrivileges());
+                    builder.array("index", response.getIndexPrivileges());
+                    builder.endObject();
+                    return new BytesRestResponse(RestStatus.OK, builder);
+                }
+            });
+    }
+
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetPrivilegesAction.java
index e0d0898704e3b..7a5261c42bc93 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestGetPrivilegesAction.java
@@ -19,9 +19,9 @@
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesResponse;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
 import java.io.IOException;
@@ -64,32 +64,38 @@ public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient c
         final String application = request.param("application");
         final String[] privileges = request.paramAsStringArray("privilege", Strings.EMPTY_ARRAY);
 
-        return channel -> new SecurityClient(client).prepareGetPrivileges(application, privileges)
-                .execute(new RestBuilderListener<GetPrivilegesResponse>(channel) {
-                    @Override
-                    public RestResponse buildResponse(GetPrivilegesResponse response, XContentBuilder builder) throws Exception {
-                        final Map<String, Set<ApplicationPrivilegeDescriptor>> privsByApp = groupByApplicationName(response.privileges());
-                        builder.startObject();
-                        for (String app : privsByApp.keySet()) {
-                            builder.startObject(app);
-                            for (ApplicationPrivilegeDescriptor privilege : privsByApp.get(app)) {
-                                builder.field(privilege.getName(), privilege);
-                            }
-                            builder.endObject();
-                        }
-                        builder.endObject();
+        final GetPrivilegesRequestBuilder requestBuilder = new GetPrivilegesRequestBuilder(client);
+        if (Strings.hasText(application)) {
+            requestBuilder.application(application).privileges(privileges);
+        }
 
-                        // if the user asked for specific privileges, but none of them were found
-                        // we'll return an empty result and 404 status code
-                        if (privileges.length != 0 && response.privileges().length == 0) {
-                            return new BytesRestResponse(RestStatus.NOT_FOUND, builder);
-                        }
+        return channel -> requestBuilder.execute(new RestBuilderListener<>(channel) {
+            @Override
+            public RestResponse buildResponse(GetPrivilegesResponse response, XContentBuilder builder) throws Exception {
+                builder.startObject();
 
-                        // either the user asked for all privileges, or at least one of the privileges
-                        // was found
-                        return new BytesRestResponse(RestStatus.OK, builder);
+                final Map<String, Set<ApplicationPrivilegeDescriptor>> appPrivs = groupByApplicationName(response.privileges());
+                for (String app : appPrivs.keySet()) {
+                    builder.startObject(app);
+                    for (ApplicationPrivilegeDescriptor privilege : appPrivs.get(app)) {
+                        builder.field(privilege.getName(), privilege);
                     }
-                });
+                    builder.endObject();
+                }
+
+                builder.endObject();
+
+                // if the user asked for specific privileges, but none of them were found
+                // we'll return an empty result and 404 status code
+                if (privileges.length != 0 && response.isEmpty()) {
+                    return new BytesRestResponse(RestStatus.NOT_FOUND, builder);
+                }
+
+                // either the user asked for all privileges, or at least one of the privileges
+                // was found
+                return new BytesRestResponse(RestStatus.OK, builder);
+            }
+        });
     }
 
     static Map<String, Set<ApplicationPrivilegeDescriptor>> groupByApplicationName(ApplicationPrivilegeDescriptor[] privileges) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegesAction.java
index abc2f00afcd81..082751affb8b5 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/privilege/RestPutPrivilegesAction.java
@@ -21,7 +21,6 @@
 import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesResponse;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
 import java.io.IOException;
@@ -57,9 +56,9 @@ public String getName() {
 
     @Override
     public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
-        PutPrivilegesRequestBuilder requestBuilder = new SecurityClient(client)
-                .preparePutPrivileges(request.requiredContent(), request.getXContentType())
-                .setRefreshPolicy(request.param("refresh"));
+        PutPrivilegesRequestBuilder requestBuilder = new PutPrivilegesRequestBuilder(client)
+            .source(request.requiredContent(), request.getXContentType())
+            .setRefreshPolicy(request.param("refresh"));
 
         return execute(requestBuilder);
     }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/realm/RestClearRealmCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/realm/RestClearRealmCacheAction.java
index da9b1a1828b66..e1590c39fe90e 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/realm/RestClearRealmCacheAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/realm/RestClearRealmCacheAction.java
@@ -13,12 +13,10 @@
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions.NodesResponseRestListener;
+import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheAction;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequest;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
-import java.io.IOException;
-
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public final class RestClearRealmCacheAction extends SecurityBaseRestHandler {
@@ -39,13 +37,13 @@ public String getName() {
     }
 
     @Override
-    public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
+    public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) {
         String[] realms = request.paramAsStringArrayOrEmptyIfAll("realms");
         String[] usernames = request.paramAsStringArrayOrEmptyIfAll("usernames");
 
         ClearRealmCacheRequest req = new ClearRealmCacheRequest().realms(realms).usernames(usernames);
 
-        return channel -> new SecurityClient(client).clearRealmCache(req, new NodesResponseRestListener<>(channel));
+        return channel -> client.execute(ClearRealmCacheAction.INSTANCE, req, new NodesResponseRestListener<>(channel));
     }
 
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestClearRolesCacheAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestClearRolesCacheAction.java
index a51db748ccc29..0fd15051e8231 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestClearRolesCacheAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestClearRolesCacheAction.java
@@ -13,12 +13,10 @@
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions.NodesResponseRestListener;
+import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheAction;
 import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheRequest;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
-import java.io.IOException;
-
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public final class RestClearRolesCacheAction extends SecurityBaseRestHandler {
@@ -39,11 +37,11 @@ public String getName() {
     }
 
     @Override
-    public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
+    public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) {
         String[] roles = request.paramAsStringArrayOrEmptyIfAll("name");
 
         ClearRolesCacheRequest req = new ClearRolesCacheRequest().names(roles);
 
-        return channel -> new SecurityClient(client).clearRolesCache(req, new NodesResponseRestListener<>(channel));
+        return channel -> client.execute(ClearRolesCacheAction.INSTANCE, req, new NodesResponseRestListener<>(channel));
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestDeleteRoleAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestDeleteRoleAction.java
index 26399f4da9118..957cac123486c 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestDeleteRoleAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestDeleteRoleAction.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.core.security.action.role.DeleteRoleRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.role.DeleteRoleResponse;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
 import java.io.IOException;
@@ -50,15 +50,16 @@ public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient c
         final String name = request.param("name");
         final String refresh = request.param("refresh");
 
-        return channel -> new SecurityClient(client).prepareDeleteRole(name)
-                .setRefreshPolicy(refresh)
-                .execute(new RestBuilderListener<DeleteRoleResponse>(channel) {
-                    @Override
-                    public RestResponse buildResponse(DeleteRoleResponse response, XContentBuilder builder) throws Exception {
-                        return new BytesRestResponse(
-                                response.found() ? RestStatus.OK : RestStatus.NOT_FOUND,
-                                builder.startObject().field("found", response.found()).endObject());
-                    }
-                });
+        return channel -> new DeleteRoleRequestBuilder(client)
+            .name(name)
+            .setRefreshPolicy(refresh)
+            .execute(new RestBuilderListener<>(channel) {
+                @Override
+                public RestResponse buildResponse(DeleteRoleResponse response, XContentBuilder builder) throws Exception {
+                    return new BytesRestResponse(
+                            response.found() ? RestStatus.OK : RestStatus.NOT_FOUND,
+                            builder.startObject().field("found", response.found()).endObject());
+                }
+            });
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java
index 850709eea8634..a5141f25f37a1 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestGetRolesAction.java
@@ -18,9 +18,9 @@
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.core.security.action.role.GetRolesRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.role.GetRolesResponse;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
 import java.io.IOException;
@@ -53,7 +53,9 @@ public String getName() {
     @Override
     public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
         final String[] roles = request.paramAsStringArray("name", Strings.EMPTY_ARRAY);
-        return channel -> new SecurityClient(client).prepareGetRoles(roles).execute(new RestBuilderListener<GetRolesResponse>(channel) {
+        return channel -> new GetRolesRequestBuilder(client)
+            .names(roles)
+            .execute(new RestBuilderListener<>(channel) {
             @Override
             public RestResponse buildResponse(GetRolesResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject();
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java
index 10673cd79fac7..e8b2e6a778386 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/RestPutRoleAction.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.xpack.core.security.action.role.PutRoleRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.role.PutRoleResponse;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
 import java.io.IOException;
@@ -52,10 +51,10 @@ public String getName() {
 
     @Override
     public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
-        PutRoleRequestBuilder requestBuilder = new SecurityClient(client)
-                .preparePutRole(request.param("name"), request.requiredContent(), request.getXContentType())
-                .setRefreshPolicy(request.param("refresh"));
-        return channel -> requestBuilder.execute(new RestBuilderListener<PutRoleResponse>(channel) {
+        PutRoleRequestBuilder requestBuilder = new PutRoleRequestBuilder(client)
+            .source(request.param("name"), request.requiredContent(), request.getXContentType())
+            .setRefreshPolicy(request.param("refresh"));
+        return channel -> requestBuilder.execute(new RestBuilderListener<>(channel) {
             @Override
             public RestResponse buildResponse(PutRoleResponse putRoleResponse, XContentBuilder builder) throws Exception {
                 return new BytesRestResponse(RestStatus.OK, builder.startObject().field("role", putRoleResponse).endObject());
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestDeleteRoleMappingAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestDeleteRoleMappingAction.java
index a2be9ba9e041e..a2383534115cc 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestDeleteRoleMappingAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestDeleteRoleMappingAction.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingResponse;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
 import java.io.IOException;
@@ -51,14 +51,15 @@ public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient c
         final String name = request.param("name");
         final String refresh = request.param("refresh");
 
-        return channel -> new SecurityClient(client).prepareDeleteRoleMapping(name)
-                .setRefreshPolicy(refresh)
-                .execute(new RestBuilderListener<DeleteRoleMappingResponse>(channel) {
-                    @Override
-                    public RestResponse buildResponse(DeleteRoleMappingResponse response, XContentBuilder builder) throws Exception {
-                        return new BytesRestResponse(response.isFound() ? RestStatus.OK : RestStatus.NOT_FOUND,
-                                builder.startObject().field("found", response.isFound()).endObject());
-                    }
-                });
+        return channel -> new DeleteRoleMappingRequestBuilder(client)
+            .name(name)
+            .setRefreshPolicy(refresh)
+            .execute(new RestBuilderListener<>(channel) {
+                @Override
+                public RestResponse buildResponse(DeleteRoleMappingResponse response, XContentBuilder builder) throws Exception {
+                    return new BytesRestResponse(response.isFound() ? RestStatus.OK : RestStatus.NOT_FOUND,
+                            builder.startObject().field("found", response.isFound()).endObject());
+                }
+            });
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestGetRoleMappingsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestGetRoleMappingsAction.java
index ceabeb3a62eb2..aeac89e8ffb9a 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestGetRoleMappingsAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestGetRoleMappingsAction.java
@@ -17,9 +17,9 @@
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsResponse;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
 import java.io.IOException;
@@ -52,23 +52,24 @@ public String getName() {
     @Override
     public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
         final String[] names = request.paramAsStringArrayOrEmptyIfAll("name");
-        return channel -> new SecurityClient(client).prepareGetRoleMappings(names)
-                .execute(new RestBuilderListener<GetRoleMappingsResponse>(channel) {
-                    @Override
-                    public RestResponse buildResponse(GetRoleMappingsResponse response, XContentBuilder builder) throws Exception {
-                        builder.startObject();
-                        for (ExpressionRoleMapping mapping : response.mappings()) {
-                            builder.field(mapping.getName(), mapping);
-                        }
-                        builder.endObject();
+        return channel -> new GetRoleMappingsRequestBuilder(client)
+            .names(names)
+            .execute(new RestBuilderListener<>(channel) {
+                @Override
+                public RestResponse buildResponse(GetRoleMappingsResponse response, XContentBuilder builder) throws Exception {
+                    builder.startObject();
+                    for (ExpressionRoleMapping mapping : response.mappings()) {
+                        builder.field(mapping.getName(), mapping);
+                    }
+                    builder.endObject();
 
-                        // if the request specified mapping names, but nothing was found then return a 404 result
-                        if (names.length != 0 && response.mappings().length == 0) {
-                            return new BytesRestResponse(RestStatus.NOT_FOUND, builder);
-                        } else {
-                            return new BytesRestResponse(RestStatus.OK, builder);
-                        }
+                    // if the request specified mapping names, but nothing was found then return a 404 result
+                    if (names.length != 0 && response.mappings().length == 0) {
+                        return new BytesRestResponse(RestStatus.NOT_FOUND, builder);
+                    } else {
+                        return new BytesRestResponse(RestStatus.OK, builder);
                     }
-                });
+                }
+            });
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestPutRoleMappingAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestPutRoleMappingAction.java
index 64c4bc9c52bc1..532a12644a7bd 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestPutRoleMappingAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/rolemapping/RestPutRoleMappingAction.java
@@ -19,7 +19,6 @@
 import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
 import java.io.IOException;
@@ -55,11 +54,11 @@ public String getName() {
     @Override
     public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
         final String name = request.param("name");
-        PutRoleMappingRequestBuilder requestBuilder = new SecurityClient(client)
-                .preparePutRoleMapping(name, request.requiredContent(), request.getXContentType())
-                .setRefreshPolicy(request.param("refresh"));
+        PutRoleMappingRequestBuilder requestBuilder = new PutRoleMappingRequestBuilder(client)
+            .source(name, request.requiredContent(), request.getXContentType())
+            .setRefreshPolicy(request.param("refresh"));
         return channel -> requestBuilder.execute(
-                new RestBuilderListener<PutRoleMappingResponse>(channel) {
+                new RestBuilderListener<>(channel) {
                     @Override
                     public RestResponse buildResponse(PutRoleMappingResponse response, XContentBuilder builder) throws Exception {
                         return new BytesRestResponse(RestStatus.OK, builder.startObject().field("role_mapping", response).endObject());
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlAuthenticateAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlAuthenticateAction.java
index b73fb70c3fa9d..a61aaf650f99d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlAuthenticateAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/saml/RestSamlAuthenticateAction.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.xpack.core.security.action.saml.SamlAuthenticateRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.saml.SamlAuthenticateResponse;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 
 import java.io.IOException;
 import java.util.Base64;
@@ -80,8 +79,9 @@ public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient c
             logger.trace("SAML Authenticate: [{}...] [{}]", Strings.cleanTruncate(input.content, 128), input.ids);
             return channel -> {
                 final byte[] bytes = decodeBase64(input.content);
-                final SamlAuthenticateRequestBuilder requestBuilder = new SecurityClient(client).prepareSamlAuthenticate(bytes, input.ids);
-                requestBuilder.execute(new RestBuilderListener<SamlAuthenticateResponse>(channel) {
+                final SamlAuthenticateRequestBuilder requestBuilder =
+                    new SamlAuthenticateRequestBuilder(client).saml(bytes).validRequestIds(input.ids);
+                requestBuilder.execute(new RestBuilderListener<>(channel) {
                     @Override
                     public RestResponse buildResponse(SamlAuthenticateResponse response, XContentBuilder builder) throws Exception {
                         builder.startObject()
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java
index 53341ce0b7283..4a37df0f6fa77 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestChangePasswordAction.java
@@ -19,9 +19,9 @@
 import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.SecurityContext;
+import org.elasticsearch.xpack.core.security.action.user.ChangePasswordRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.user.ChangePasswordResponse;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.core.security.rest.RestRequestFilter;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
@@ -75,17 +75,16 @@ public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient c
         }
 
         final String refresh = request.param("refresh");
-        return channel ->
-                new SecurityClient(client)
-                    .prepareChangePassword(username, request.requiredContent(), request.getXContentType(), passwordHasher)
-                        .setRefreshPolicy(refresh)
-                        .execute(new RestBuilderListener<ChangePasswordResponse>(channel) {
-                            @Override
-                            public RestResponse buildResponse(ChangePasswordResponse changePasswordResponse,
-                                                              XContentBuilder builder) throws Exception {
-                                return new BytesRestResponse(RestStatus.OK, builder.startObject().endObject());
-                            }
-                        });
+        return channel -> new ChangePasswordRequestBuilder(client)
+            .username(username)
+            .source(request.requiredContent(), request.getXContentType(), passwordHasher)
+            .setRefreshPolicy(refresh)
+            .execute(new RestBuilderListener<>(channel) {
+                @Override
+                public RestResponse buildResponse(ChangePasswordResponse response, XContentBuilder builder) throws Exception {
+                    return new BytesRestResponse(RestStatus.OK, builder.startObject().endObject());
+                }
+            });
     }
 
     private static final Set<String> FILTERED_FIELDS = Collections.singleton("password");
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestDeleteUserAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestDeleteUserAction.java
index 3c810ca230830..bc0eb728c6bf6 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestDeleteUserAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestDeleteUserAction.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.core.security.action.user.DeleteUserRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.user.DeleteUserResponse;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
 import java.io.IOException;
@@ -49,16 +49,17 @@ public String getName() {
     public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
         final String username = request.param("username");
         final String refresh = request.param("refresh");
-        return channel -> new SecurityClient(client).prepareDeleteUser(username)
-                .setRefreshPolicy(refresh)
-                .execute(new RestBuilderListener<DeleteUserResponse>(channel) {
-                    @Override
-                    public RestResponse buildResponse(DeleteUserResponse response, XContentBuilder builder) throws Exception {
-                        return new BytesRestResponse(response.found() ? RestStatus.OK : RestStatus.NOT_FOUND,
-                                builder.startObject()
-                                        .field("found", response.found())
-                                        .endObject());
-                    }
-                });
+        return channel -> new DeleteUserRequestBuilder(client)
+            .username(username)
+            .setRefreshPolicy(refresh)
+            .execute(new RestBuilderListener<>(channel) {
+                @Override
+                public RestResponse buildResponse(DeleteUserResponse response, XContentBuilder builder) throws Exception {
+                    return new BytesRestResponse(response.found() ? RestStatus.OK : RestStatus.NOT_FOUND,
+                            builder.startObject()
+                                    .field("found", response.found())
+                                    .endObject());
+                }
+            });
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesAction.java
index b073349d842a3..1f32c866cb3c8 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesAction.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
@@ -66,7 +65,7 @@ public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient c
             return restChannel -> { throw new ElasticsearchSecurityException("there is no authenticated user"); };
         }
         final String username = user.principal();
-        final GetUserPrivilegesRequestBuilder requestBuilder = new SecurityClient(client).prepareGetUserPrivileges(username);
+        final GetUserPrivilegesRequestBuilder requestBuilder = new GetUserPrivilegesRequestBuilder(client).username(username);
         return channel -> requestBuilder.execute(new RestListener(channel));
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java
index 2751022d9f007..eef11d478ca65 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUsersAction.java
@@ -18,8 +18,8 @@
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.core.security.action.user.GetUsersRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.user.GetUsersResponse;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
@@ -54,7 +54,7 @@ public String getName() {
     public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
         String[] usernames = request.paramAsStringArray("username", Strings.EMPTY_ARRAY);
 
-        return channel -> new SecurityClient(client).prepareGetUsers(usernames).execute(new RestBuilderListener<GetUsersResponse>(channel) {
+        return channel -> new GetUsersRequestBuilder(client).usernames(usernames).execute(new RestBuilderListener<>(channel) {
             @Override
             public RestResponse buildResponse(GetUsersResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject();
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestHasPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestHasPrivilegesAction.java
index d26d8db206936..975f25a6d2055 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestHasPrivilegesAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestHasPrivilegesAction.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
@@ -75,8 +74,8 @@ public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient c
             return restChannel -> { throw new ElasticsearchSecurityException("there is no authenticated user"); };
         }
         final Tuple<XContentType, BytesReference> content = request.contentOrSourceParam();
-        HasPrivilegesRequestBuilder requestBuilder = new SecurityClient(client).prepareHasPrivileges(username, content.v2(), content.v1());
-        return channel -> requestBuilder.execute(new RestBuilderListener<HasPrivilegesResponse>(channel) {
+        HasPrivilegesRequestBuilder requestBuilder = new HasPrivilegesRequestBuilder(client).source(username, content.v2(), content.v1());
+        return channel -> requestBuilder.execute(new RestBuilderListener<>(channel) {
             @Override
             public RestResponse buildResponse(HasPrivilegesResponse response, XContentBuilder builder) throws Exception {
                 response.toXContent(builder, ToXContent.EMPTY_PARAMS);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java
index b56daba0bfa4a..39cd5f35fa54b 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestPutUserAction.java
@@ -22,7 +22,6 @@
 import org.elasticsearch.xpack.core.security.action.user.PutUserRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.user.PutUserResponse;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.core.security.rest.RestRequestFilter;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
@@ -60,11 +59,11 @@ public String getName() {
 
     @Override
     public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException {
-        PutUserRequestBuilder requestBuilder = new SecurityClient(client)
-            .preparePutUser(request.param("username"), request.requiredContent(), request.getXContentType(), passwordHasher)
-                .setRefreshPolicy(request.param("refresh"));
+        PutUserRequestBuilder requestBuilder = new PutUserRequestBuilder(client)
+            .source(request.param("username"), request.requiredContent(), request.getXContentType(), passwordHasher)
+            .setRefreshPolicy(request.param("refresh"));
 
-        return channel -> requestBuilder.execute(new RestBuilderListener<PutUserResponse>(channel) {
+        return channel -> requestBuilder.execute(new RestBuilderListener<>(channel) {
             @Override
             public RestResponse buildResponse(PutUserResponse putUserResponse, XContentBuilder builder) throws Exception {
                 putUserResponse.toXContent(builder, request);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestSetEnabledAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestSetEnabledAction.java
index a78b2c85aa62c..b13f0b328d993 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestSetEnabledAction.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestSetEnabledAction.java
@@ -17,8 +17,8 @@
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.xpack.core.security.action.user.SetEnabledRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.user.SetEnabledResponse;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.rest.action.SecurityBaseRestHandler;
 
 import java.io.IOException;
@@ -61,12 +61,14 @@ public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient c
         final boolean enabled = request.path().endsWith("_enable");
         assert enabled || request.path().endsWith("_disable");
         final String username = request.param("username");
-        return channel -> new SecurityClient(client).prepareSetEnabled(username, enabled)
-                .execute(new RestBuilderListener<SetEnabledResponse>(channel) {
-                    @Override
-                    public RestResponse buildResponse(SetEnabledResponse setEnabledResponse, XContentBuilder builder) throws Exception {
-                        return new BytesRestResponse(RestStatus.OK, builder.startObject().endObject());
-                    }
-                });
+        return channel -> new SetEnabledRequestBuilder(client)
+            .username(username)
+            .enabled(enabled)
+            .execute(new RestBuilderListener<>(channel) {
+                @Override
+                public RestResponse buildResponse(SetEnabledResponse setEnabledResponse, XContentBuilder builder) throws Exception {
+                    return new BytesRestResponse(RestStatus.OK, builder.startObject().endObject());
+                }
+            });
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java
index b62cb44ac028c..769465c3080a7 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java
@@ -45,8 +45,10 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.gateway.GatewayService;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.indices.IndexClosedException;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
 import org.elasticsearch.xpack.core.template.TemplateUtils;
@@ -168,9 +170,11 @@ public ElasticsearchException getUnavailableReason() {
             throw new IllegalStateException("caller must make sure to use a frozen state and check indexAvailable");
         }
 
-        if (localState.indexExists()) {
+        if (localState.indexState == IndexMetaData.State.CLOSE) {
+            return new IndexClosedException(new Index(localState.concreteIndexName, ClusterState.UNKNOWN_UUID));
+        } else if (localState.indexExists()) {
             return new UnavailableShardsException(null,
-                    "at least one primary shard for the index [" + localState.concreteIndexName + "] is unavailable");
+                "at least one primary shard for the index [" + localState.concreteIndexName + "] is unavailable");
         } else {
             return new IndexNotFoundException(localState.concreteIndexName);
         }
@@ -201,11 +205,24 @@ public void clusterChanged(ClusterChangedEvent event) {
         final boolean indexAvailable = checkIndexAvailable(event.state());
         final boolean mappingIsUpToDate = indexMetaData == null || checkIndexMappingUpToDate(event.state());
         final Version mappingVersion = oldestIndexMappingVersion(event.state());
-        final ClusterHealthStatus indexStatus = indexMetaData == null ? null :
-            new ClusterIndexHealth(indexMetaData, event.state().getRoutingTable().index(indexMetaData.getIndex())).getStatus();
         final String concreteIndexName = indexMetaData == null ? internalIndexName : indexMetaData.getIndex().getName();
+        final ClusterHealthStatus indexHealth;
+        final IndexMetaData.State indexState;
+        if (indexMetaData == null) {
+            // Index does not exist
+            indexState = null;
+            indexHealth = null;
+        } else if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
+            indexState = IndexMetaData.State.CLOSE;
+            indexHealth = null;
+            logger.warn("Index [{}] is closed. This is likely to prevent security from functioning correctly", concreteIndexName);
+        } else {
+            indexState = IndexMetaData.State.OPEN;
+            final IndexRoutingTable routingTable = event.state().getRoutingTable().index(indexMetaData.getIndex());
+            indexHealth = new ClusterIndexHealth(indexMetaData, routingTable).getStatus();
+        }
         final State newState = new State(creationTime, isIndexUpToDate, indexAvailable, mappingIsUpToDate, mappingVersion,
-                concreteIndexName, indexStatus);
+                concreteIndexName, indexHealth, indexState);
         this.indexState = newState;
 
         if (newState.equals(previousState) == false) {
@@ -216,23 +233,21 @@ public void clusterChanged(ClusterChangedEvent event) {
     }
 
     private boolean checkIndexAvailable(ClusterState state) {
-        final IndexRoutingTable routingTable = getIndexRoutingTable(state);
-        if (routingTable != null && routingTable.allPrimaryShardsActive()) {
-            return true;
-        }
-        logger.debug("Index [{}] is not yet active", aliasName);
-        return false;
-    }
-
-    /**
-     * Returns the routing-table for this index, or <code>null</code> if the index does not exist.
-     */
-    private IndexRoutingTable getIndexRoutingTable(ClusterState clusterState) {
-        IndexMetaData metaData = resolveConcreteIndex(aliasName, clusterState.metaData());
+        IndexMetaData metaData = resolveConcreteIndex(aliasName, state.metaData());
         if (metaData == null) {
-            return null;
+            logger.debug("Index [{}] is not available - no metadata", aliasName);
+            return false;
+        }
+        if (metaData.getState() == IndexMetaData.State.CLOSE) {
+            logger.warn("Index [{}] is closed", aliasName);
+            return false;
+        }
+        final IndexRoutingTable routingTable = state.routingTable().index(metaData.getIndex());
+        if (routingTable == null || routingTable.allPrimaryShardsActive() == false) {
+            logger.debug("Index [{}] is not yet active", aliasName);
+            return false;
         } else {
-            return clusterState.routingTable().index(metaData.getIndex());
+            return true;
         }
     }
 
@@ -397,15 +412,15 @@ public void onFailure(Exception e) {
      * Return true if the state moves from an unhealthy ("RED") index state to a healthy ("non-RED") state.
      */
     public static boolean isMoveFromRedToNonRed(State previousState, State currentState) {
-        return (previousState.indexStatus == null || previousState.indexStatus == ClusterHealthStatus.RED)
-                && currentState.indexStatus != null && currentState.indexStatus != ClusterHealthStatus.RED;
+        return (previousState.indexHealth == null || previousState.indexHealth == ClusterHealthStatus.RED)
+                && currentState.indexHealth != null && currentState.indexHealth != ClusterHealthStatus.RED;
     }
 
     /**
      * Return true if the state moves from the index existing to the index not existing.
      */
     public static boolean isIndexDeleted(State previousState, State currentState) {
-        return previousState.indexStatus != null && currentState.indexStatus == null;
+        return previousState.indexHealth != null && currentState.indexHealth == null;
     }
 
     private static byte[] readTemplateAsBytes(String templateName) {
@@ -435,24 +450,27 @@ private static Tuple<String, Settings> parseMappingAndSettingsFromTemplateBytes(
      * State of the security index.
      */
     public static class State {
-        public static final State UNRECOVERED_STATE = new State(null, false, false, false, null, null, null);
+        public static final State UNRECOVERED_STATE = new State(null, false, false, false, null, null, null, null);
         public final Instant creationTime;
         public final boolean isIndexUpToDate;
         public final boolean indexAvailable;
         public final boolean mappingUpToDate;
         public final Version mappingVersion;
         public final String concreteIndexName;
-        public final ClusterHealthStatus indexStatus;
+        public final ClusterHealthStatus indexHealth;
+        public final IndexMetaData.State indexState;
 
         public State(Instant creationTime, boolean isIndexUpToDate, boolean indexAvailable,
-                     boolean mappingUpToDate, Version mappingVersion, String concreteIndexName, ClusterHealthStatus indexStatus) {
+                     boolean mappingUpToDate, Version mappingVersion, String concreteIndexName, ClusterHealthStatus indexHealth,
+                     IndexMetaData.State indexState) {
             this.creationTime = creationTime;
             this.isIndexUpToDate = isIndexUpToDate;
             this.indexAvailable = indexAvailable;
             this.mappingUpToDate = mappingUpToDate;
             this.mappingVersion = mappingVersion;
             this.concreteIndexName = concreteIndexName;
-            this.indexStatus = indexStatus;
+            this.indexHealth = indexHealth;
+            this.indexState = indexState;
         }
 
         @Override
@@ -466,7 +484,8 @@ public boolean equals(Object o) {
                 mappingUpToDate == state.mappingUpToDate &&
                 Objects.equals(mappingVersion, state.mappingVersion) &&
                 Objects.equals(concreteIndexName, state.concreteIndexName) &&
-                indexStatus == state.indexStatus;
+                indexHealth == state.indexHealth &&
+                indexState == state.indexState;
         }
 
         public boolean indexExists() {
@@ -476,7 +495,7 @@ public boolean indexExists() {
         @Override
         public int hashCode() {
             return Objects.hash(creationTime, isIndexUpToDate, indexAvailable, mappingUpToDate, mappingVersion, concreteIndexName,
-                indexStatus);
+                indexHealth);
         }
     }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityStatusChangeListener.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityStatusChangeListener.java
new file mode 100644
index 0000000000000..ddc41561afabd
--- /dev/null
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityStatusChangeListener.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.support;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.license.LicenseStateListener;
+import org.elasticsearch.license.XPackLicenseState;
+
+import java.util.Objects;
+
+/**
+ * A listener for license state changes that provides log messages when a license change
+ * causes security to switch between enable and disabled (or vice versa).
+ */
+public class SecurityStatusChangeListener implements LicenseStateListener {
+
+    private final Logger logger;
+    private final XPackLicenseState licenseState;
+    private Boolean securityEnabled;
+
+    public SecurityStatusChangeListener(XPackLicenseState licenseState) {
+        this.logger = LogManager.getLogger(getClass());
+        this.licenseState = licenseState;
+        this.securityEnabled = null;
+    }
+
+    /**
+     * This listener will not be registered if security has been explicitly disabled, so we only need to account for dynamic changes due
+     * to changes in the applied license.
+     */
+    @Override
+    public synchronized void licenseStateChanged() {
+        final boolean newState = licenseState.isSecurityAvailable() && licenseState.isSecurityDisabledByLicenseDefaults() == false;
+        // old state might be null (undefined) so do Object comparison
+        if (Objects.equals(newState, securityEnabled) == false) {
+            logger.info("Active license is now [{}]; Security is {}", licenseState.getOperationMode(), newState ? "enabled" : "disabled");
+            this.securityEnabled = newState;
+        }
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
index d05e1572a1776..dfe3105af923d 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedConsumer;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -45,24 +44,13 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.Executor;
-import java.util.function.Function;
 
 import static org.elasticsearch.xpack.core.security.SecurityField.setting;
 
 public class SecurityServerTransportInterceptor implements TransportInterceptor {
 
-    private static final Function<String, Setting<String>> TRANSPORT_TYPE_SETTING_TEMPLATE = key -> new Setting<>(key, "node", v -> {
-        if (v.equals("node") || v.equals("client")) {
-            return v;
-        }
-        throw new IllegalArgumentException("type must be one of [client, node]");
-    }, Setting.Property.NodeScope);
-    private static final String TRANSPORT_TYPE_SETTING_KEY = "xpack.security.type";
     private static final Logger logger = LogManager.getLogger(SecurityServerTransportInterceptor.class);
 
-    public static final Setting.AffixSetting<String> TRANSPORT_TYPE_PROFILE_SETTING = Setting.affixKeySetting("transport.profiles.",
-            TRANSPORT_TYPE_SETTING_KEY, TRANSPORT_TYPE_SETTING_TEMPLATE);
-
     private final AuthenticationService authcService;
     private final AuthorizationService authzService;
     private final SSLService sslService;
@@ -71,7 +59,6 @@ public class SecurityServerTransportInterceptor implements TransportInterceptor
     private final ThreadPool threadPool;
     private final Settings settings;
     private final SecurityContext securityContext;
-    private final boolean reservedRealmEnabled;
 
     private volatile boolean isStateNotRecovered = true;
 
@@ -92,7 +79,6 @@ public SecurityServerTransportInterceptor(Settings settings,
         this.sslService = sslService;
         this.securityContext = securityContext;
         this.profileFilters = initializeProfileFilters(destructiveOperations);
-        this.reservedRealmEnabled = XPackSettings.RESERVED_REALM_ENABLED_SETTING.get(settings);
         clusterService.addListener(e -> isStateNotRecovered = e.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK));
     }
 
@@ -102,50 +88,55 @@ public AsyncSender interceptSender(AsyncSender sender) {
             @Override
             public <T extends TransportResponse> void sendRequest(Transport.Connection connection, String action, TransportRequest request,
                                                                   TransportRequestOptions options, TransportResponseHandler<T> handler) {
-                // make a local copy of isStateNotRecovered as this is a volatile variable and it
-                // is used multiple times in the method. The copy to a local variable allows us to
-                // guarantee we use the same value wherever we would check the value for the state
-                // being recovered
-                final boolean stateNotRecovered = isStateNotRecovered;
-                final boolean sendWithAuth = licenseState.isAuthAllowed() || stateNotRecovered;
-                if (sendWithAuth) {
-                    // the transport in core normally does this check, BUT since we are serializing to a string header we need to do it
-                    // ourselves otherwise we wind up using a version newer than what we can actually send
-                    final Version minVersion = Version.min(connection.getVersion(), Version.CURRENT);
+                final boolean requireAuth = shouldRequireExistingAuthentication();
+                // the transport in core normally does this check, BUT since we are serializing to a string header we need to do it
+                // ourselves otherwise we wind up using a version newer than what we can actually send
+                final Version minVersion = Version.min(connection.getVersion(), Version.CURRENT);
 
-                    // Sometimes a system action gets executed like a internal create index request or update mappings request
-                    // which means that the user is copied over to system actions so we need to change the user
-                    if (AuthorizationUtils.shouldReplaceUserWithSystem(threadPool.getThreadContext(), action)) {
-                        securityContext.executeAsUser(SystemUser.INSTANCE, (original) -> sendWithUser(connection, action, request, options,
-                                new ContextRestoreResponseHandler<>(threadPool.getThreadContext().wrapRestorable(original)
-                                        , handler), sender, stateNotRecovered), minVersion);
-                    } else if (AuthorizationUtils.shouldSetUserBasedOnActionOrigin(threadPool.getThreadContext())) {
-                        AuthorizationUtils.switchUserBasedOnActionOriginAndExecute(threadPool.getThreadContext(), securityContext,
-                                (original) -> sendWithUser(connection, action, request, options,
-                                        new ContextRestoreResponseHandler<>(threadPool.getThreadContext().wrapRestorable(original)
-                                                , handler), sender, stateNotRecovered));
-                    } else if (securityContext.getAuthentication() != null &&
-                            securityContext.getAuthentication().getVersion().equals(minVersion) == false) {
-                        // re-write the authentication since we want the authentication version to match the version of the connection
-                        securityContext.executeAfterRewritingAuthentication(original -> sendWithUser(connection, action, request, options,
-                            new ContextRestoreResponseHandler<>(threadPool.getThreadContext().wrapRestorable(original), handler), sender,
-                            stateNotRecovered), minVersion);
-                    } else {
-                        sendWithUser(connection, action, request, options, handler, sender, stateNotRecovered);
-                    }
+                // Sometimes a system action gets executed like a internal create index request or update mappings request
+                // which means that the user is copied over to system actions so we need to change the user
+                if (AuthorizationUtils.shouldReplaceUserWithSystem(threadPool.getThreadContext(), action)) {
+                    securityContext.executeAsUser(SystemUser.INSTANCE, (original) -> sendWithUser(connection, action, request, options,
+                            new ContextRestoreResponseHandler<>(threadPool.getThreadContext().wrapRestorable(original)
+                                    , handler), sender, requireAuth), minVersion);
+                } else if (AuthorizationUtils.shouldSetUserBasedOnActionOrigin(threadPool.getThreadContext())) {
+                    AuthorizationUtils.switchUserBasedOnActionOriginAndExecute(threadPool.getThreadContext(), securityContext,
+                            (original) -> sendWithUser(connection, action, request, options,
+                                    new ContextRestoreResponseHandler<>(threadPool.getThreadContext().wrapRestorable(original)
+                                            , handler), sender, requireAuth));
+                } else if (securityContext.getAuthentication() != null &&
+                        securityContext.getAuthentication().getVersion().equals(minVersion) == false) {
+                    // re-write the authentication since we want the authentication version to match the version of the connection
+                    securityContext.executeAfterRewritingAuthentication(original -> sendWithUser(connection, action, request, options,
+                        new ContextRestoreResponseHandler<>(threadPool.getThreadContext().wrapRestorable(original), handler), sender,
+                        requireAuth), minVersion);
                 } else {
-                    sender.sendRequest(connection, action, request, options, handler);
+                    sendWithUser(connection, action, request, options, handler, sender, requireAuth);
                 }
             }
         };
     }
 
+    /**
+     * Based on the current cluster state &amp; license, should we require that all outgoing actions have an authentication header
+     * of some sort?
+     */
+    private boolean shouldRequireExistingAuthentication() {
+        // If the license state is MISSING, then auth is not allowed.
+        // However this makes it difficult to installing a valid license, because that might implicitly turn on security.
+        // When security is enabled on the master node it will then reject any actions that do not have authentication headers
+        // but there may be in-flight internal actions (that will not have authentication headers) such as "cluster/shard/started"
+        // which we don't want to reject.
+        // So, we always send authentication headers for actions that have an implied user (system-user or explicit-origin)
+        // and then for other (user originated) actions we enforce that there is an authentication header that we can send, iff the
+        // current license allows authentication.
+        return licenseState.isAuthAllowed() && isStateNotRecovered == false;
+    }
+
     private <T extends TransportResponse> void sendWithUser(Transport.Connection connection, String action, TransportRequest request,
                                                             TransportRequestOptions options, TransportResponseHandler<T> handler,
-                                                            AsyncSender sender, final boolean stateNotRecovered) {
-        // There cannot be a request outgoing from this node that is not associated with a user
-        // unless we do not know the actual license of the cluster
-        if (securityContext.getAuthentication() == null && stateNotRecovered == false) {
+                                                            AsyncSender sender, final boolean requireAuthentication) {
+        if (securityContext.getAuthentication() == null && requireAuthentication) {
             // we use an assertion here to ensure we catch this in our testing infrastructure, but leave the ISE for cases we do not catch
             // in tests and may be hit by a user
             assertNoAuthentication(action);
@@ -182,21 +173,8 @@ private Map<String, ServerTransportFilter> initializeProfileFilters(DestructiveO
         for (Map.Entry<String, SSLConfiguration> entry : profileConfigurations.entrySet()) {
             final SSLConfiguration profileConfiguration = entry.getValue();
             final boolean extractClientCert = transportSSLEnabled && sslService.isSSLClientAuthEnabled(profileConfiguration);
-            final String type = TRANSPORT_TYPE_PROFILE_SETTING.getConcreteSettingForNamespace(entry.getKey()).get(settings);
-            switch (type) {
-                case "client":
-                    profileFilters.put(entry.getKey(), new ServerTransportFilter.ClientProfile(authcService, authzService,
-                            threadPool.getThreadContext(), extractClientCert, destructiveOperations, reservedRealmEnabled,
-                            securityContext, licenseState));
-                    break;
-                case "node":
-                    profileFilters.put(entry.getKey(), new ServerTransportFilter.NodeProfile(authcService, authzService,
-                            threadPool.getThreadContext(), extractClientCert, destructiveOperations, reservedRealmEnabled,
-                            securityContext, licenseState));
-                    break;
-                default:
-                   throw new IllegalStateException("unknown profile type: " + type);
-            }
+            profileFilters.put(entry.getKey(), new ServerTransportFilter(authcService, authzService, threadPool.getThreadContext(),
+                extractClientCert, destructiveOperations, securityContext, licenseState));
         }
 
         return Collections.unmodifiableMap(profileFilters);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java
index 2d1f63f5cc15c..75c6990dc87a3 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/ServerTransportFilter.java
@@ -32,137 +32,90 @@
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 
-import java.io.IOException;
-
-import static org.elasticsearch.xpack.core.security.support.Exceptions.authenticationError;
-
 /**
- * This interface allows to intercept messages as they come in and execute logic
- * This is used in x-pack security to execute the authentication/authorization on incoming
- * messages.
- * Note that this filter only applies for nodes, but not for clients.
+ * The server transport filter that should be used in nodes as it ensures that an incoming
+ * request is properly authenticated and authorized
  */
-public interface ServerTransportFilter {
+final class ServerTransportFilter {
+
+    private static final Logger logger = LogManager.getLogger(ServerTransportFilter.class);
+
+    private final AuthenticationService authcService;
+    private final AuthorizationService authzService;
+    private final SecurityActionMapper actionMapper = new SecurityActionMapper();
+    private final ThreadContext threadContext;
+    private final boolean extractClientCert;
+    private final DestructiveOperations destructiveOperations;
+    private final SecurityContext securityContext;
+    private final XPackLicenseState licenseState;
+
+    ServerTransportFilter(AuthenticationService authcService, AuthorizationService authzService,
+                ThreadContext threadContext, boolean extractClientCert, DestructiveOperations destructiveOperations,
+                SecurityContext securityContext, XPackLicenseState licenseState) {
+        this.authcService = authcService;
+        this.authzService = authzService;
+        this.threadContext = threadContext;
+        this.extractClientCert = extractClientCert;
+        this.destructiveOperations = destructiveOperations;
+        this.securityContext = securityContext;
+        this.licenseState = licenseState;
+    }
 
     /**
      * Called just after the given request was received by the transport. Any exception
      * thrown by this method will stop the request from being handled and the error will
      * be sent back to the sender.
      */
-    void inbound(String action, TransportRequest request, TransportChannel transportChannel, ActionListener<Void> listener)
-            throws IOException;
-
-    /**
-     * The server transport filter that should be used in nodes as it ensures that an incoming
-     * request is properly authenticated and authorized
-     */
-    class NodeProfile implements ServerTransportFilter {
-        private static final Logger logger = LogManager.getLogger(NodeProfile.class);
-
-        private final AuthenticationService authcService;
-        private final AuthorizationService authzService;
-        private final SecurityActionMapper actionMapper = new SecurityActionMapper();
-        private final ThreadContext threadContext;
-        private final boolean extractClientCert;
-        private final DestructiveOperations destructiveOperations;
-        private final boolean reservedRealmEnabled;
-        private final SecurityContext securityContext;
-        private final XPackLicenseState licenseState;
-
-        NodeProfile(AuthenticationService authcService, AuthorizationService authzService,
-                    ThreadContext threadContext, boolean extractClientCert, DestructiveOperations destructiveOperations,
-                    boolean reservedRealmEnabled, SecurityContext securityContext, XPackLicenseState licenseState) {
-            this.authcService = authcService;
-            this.authzService = authzService;
-            this.threadContext = threadContext;
-            this.extractClientCert = extractClientCert;
-            this.destructiveOperations = destructiveOperations;
-            this.reservedRealmEnabled = reservedRealmEnabled;
-            this.securityContext = securityContext;
-            this.licenseState = licenseState;
-        }
-
-        @Override
-        public void inbound(String action, TransportRequest request, TransportChannel transportChannel, ActionListener<Void> listener)
-                throws IOException {
-            if (CloseIndexAction.NAME.equals(action) || OpenIndexAction.NAME.equals(action) || DeleteIndexAction.NAME.equals(action)) {
-                IndicesRequest indicesRequest = (IndicesRequest) request;
-                try {
-                    destructiveOperations.failDestructive(indicesRequest.indices());
-                } catch(IllegalArgumentException e) {
-                    listener.onFailure(e);
-                    return;
-                }
-            }
-            /*
-             here we don't have a fallback user, as all incoming request are
-             expected to have a user attached (either in headers or in context)
-             We can make this assumption because in nodes we make sure all outgoing
-             requests from all the nodes are attached with a user (either a serialize
-             user an authentication token
-             */
-            String securityAction = actionMapper.action(action, request);
-
-            TransportChannel unwrappedChannel = transportChannel;
-            if (unwrappedChannel instanceof TaskTransportChannel) {
-                unwrappedChannel = ((TaskTransportChannel) unwrappedChannel).getChannel();
+    void inbound(String action, TransportRequest request, TransportChannel transportChannel,ActionListener<Void> listener) {
+        if (CloseIndexAction.NAME.equals(action) || OpenIndexAction.NAME.equals(action) || DeleteIndexAction.NAME.equals(action)) {
+            IndicesRequest indicesRequest = (IndicesRequest) request;
+            try {
+                destructiveOperations.failDestructive(indicesRequest.indices());
+            } catch(IllegalArgumentException e) {
+                listener.onFailure(e);
+                return;
             }
+        }
+        /*
+         here we don't have a fallback user, as all incoming request are
+         expected to have a user attached (either in headers or in context)
+         We can make this assumption because in nodes we make sure all outgoing
+         requests from all the nodes are attached with a user (either a serialize
+         user an authentication token
+         */
+        String securityAction = actionMapper.action(action, request);
+
+        TransportChannel unwrappedChannel = transportChannel;
+        if (unwrappedChannel instanceof TaskTransportChannel) {
+            unwrappedChannel = ((TaskTransportChannel) unwrappedChannel).getChannel();
+        }
 
-            if (extractClientCert && (unwrappedChannel instanceof TcpTransportChannel)) {
-                TcpChannel tcpChannel = ((TcpTransportChannel) unwrappedChannel).getChannel();
-                if (tcpChannel instanceof Netty4TcpChannel || tcpChannel instanceof NioTcpChannel) {
-                    if (tcpChannel.isOpen()) {
-                        SSLEngineUtils.extractClientCertificates(logger, threadContext, tcpChannel);
-                    }
+        if (extractClientCert && (unwrappedChannel instanceof TcpTransportChannel)) {
+            TcpChannel tcpChannel = ((TcpTransportChannel) unwrappedChannel).getChannel();
+            if (tcpChannel instanceof Netty4TcpChannel || tcpChannel instanceof NioTcpChannel) {
+                if (tcpChannel.isOpen()) {
+                    SSLEngineUtils.extractClientCertificates(logger, threadContext, tcpChannel);
                 }
             }
+        }
 
-            final Version version = transportChannel.getVersion();
-            authcService.authenticate(securityAction, request, (User)null, ActionListener.wrap((authentication) -> {
-                if (authentication != null) {
-                    if (securityAction.equals(TransportService.HANDSHAKE_ACTION_NAME) &&
-                        SystemUser.is(authentication.getUser()) == false) {
-                        securityContext.executeAsUser(SystemUser.INSTANCE, (ctx) -> {
-                            final Authentication replaced = Authentication.getAuthentication(threadContext);
-                            authzService.authorize(replaced, securityAction, request, listener);
-                        }, version);
-                    } else {
-                        authzService.authorize(authentication, securityAction, request, listener);
-                    }
-                } else if (licenseState.isAuthAllowed() == false) {
-                    listener.onResponse(null);
+        final Version version = transportChannel.getVersion();
+        authcService.authenticate(securityAction, request, (User)null, ActionListener.wrap((authentication) -> {
+            if (authentication != null) {
+                if (securityAction.equals(TransportService.HANDSHAKE_ACTION_NAME) &&
+                    SystemUser.is(authentication.getUser()) == false) {
+                    securityContext.executeAsUser(SystemUser.INSTANCE, (ctx) -> {
+                        final Authentication replaced = Authentication.getAuthentication(threadContext);
+                        authzService.authorize(replaced, securityAction, request, listener);
+                    }, version);
                 } else {
-                    listener.onFailure(new IllegalStateException("no authentication present but auth is allowed"));
+                    authzService.authorize(authentication, securityAction, request, listener);
                 }
-            }, listener::onFailure));
-        }
-    }
-
-    /**
-     * A server transport filter rejects internal calls, which should be used on connections
-     * where only clients connect to. This ensures that no client can send any internal actions
-     * or shard level actions. As it extends the NodeProfile the authentication/authorization is
-     * done as well
-     */
-    class ClientProfile extends NodeProfile {
-
-        ClientProfile(AuthenticationService authcService, AuthorizationService authzService,
-                             ThreadContext threadContext, boolean extractClientCert, DestructiveOperations destructiveOperations,
-                             boolean reservedRealmEnabled, SecurityContext securityContext, XPackLicenseState licenseState) {
-            super(authcService, authzService, threadContext, extractClientCert, destructiveOperations, reservedRealmEnabled,
-                securityContext, licenseState);
-        }
-
-        @Override
-        public void inbound(String action, TransportRequest request, TransportChannel transportChannel, ActionListener<Void> listener)
-                throws IOException {
-            // TODO is ']' sufficient to mark as shard action?
-            final boolean isInternalOrShardAction = action.startsWith("internal:") || action.endsWith("]");
-            if (isInternalOrShardAction && TransportService.HANDSHAKE_ACTION_NAME.equals(action) == false) {
-                throw authenticationError("executing internal/shard actions is considered malicious and forbidden");
+            } else if (licenseState.isAuthAllowed() == false) {
+                listener.onResponse(null);
+            } else {
+                listener.onFailure(new IllegalStateException("no authentication present but auth is allowed"));
             }
-            super.inbound(action, request, transportChannel, listener);
-        }
+        }, listener::onFailure));
     }
-
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/NioIPFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/NioIPFilter.java
index afb13ceff2edd..12f6b67d67242 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/NioIPFilter.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/NioIPFilter.java
@@ -5,28 +5,49 @@
  */
 package org.elasticsearch.xpack.security.transport.nio;
 
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.nio.NioSocketChannel;
+import org.elasticsearch.nio.DelegatingHandler;
+import org.elasticsearch.nio.InboundChannelBuffer;
+import org.elasticsearch.nio.NioChannelHandler;
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 
-import java.util.function.Predicate;
+import java.io.IOException;
+import java.net.InetSocketAddress;
 
-public final class NioIPFilter implements Predicate<NioSocketChannel> {
+public final class NioIPFilter extends DelegatingHandler {
 
+    private final InetSocketAddress remoteAddress;
     private final IPFilter filter;
     private final String profile;
+    private boolean denied = false;
 
-    NioIPFilter(@Nullable IPFilter filter, String profile) {
+    NioIPFilter(NioChannelHandler delegate, InetSocketAddress remoteAddress, IPFilter filter, String profile) {
+        super(delegate);
+        this.remoteAddress = remoteAddress;
         this.filter = filter;
         this.profile = profile;
     }
 
     @Override
-    public boolean test(NioSocketChannel nioChannel) {
-        if (filter != null) {
-            return filter.accept(profile, nioChannel.getRemoteAddress());
+    public void channelRegistered() {
+        if (filter.accept(profile, remoteAddress)) {
+            super.channelRegistered();
         } else {
-            return true;
+            denied = true;
         }
     }
+
+    @Override
+    public int consumeReads(InboundChannelBuffer channelBuffer) throws IOException {
+        if (denied) {
+            // Do not consume any reads if channel is disallowed
+            return 0;
+        } else {
+            return super.consumeReads(channelBuffer);
+        }
+    }
+
+    @Override
+    public boolean closeNow() {
+        return denied;
+    }
 }
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java
index 5f7037d57e3f1..8947447ef58d6 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java
@@ -9,9 +9,9 @@
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.nio.FlushOperation;
 import org.elasticsearch.nio.InboundChannelBuffer;
+import org.elasticsearch.nio.NioChannelHandler;
 import org.elasticsearch.nio.NioSelector;
 import org.elasticsearch.nio.NioSocketChannel;
-import org.elasticsearch.nio.ReadWriteHandler;
 import org.elasticsearch.nio.SocketChannelContext;
 import org.elasticsearch.nio.WriteOperation;
 
@@ -23,12 +23,11 @@
 import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
-import java.util.function.Predicate;
 
 /**
  * Provides a TLS/SSL read/write layer over a channel. This context will use a {@link SSLDriver} to handshake
  * with the peer channel. Once the handshake is complete, any data from the peer channel will be decrypted
- * before being passed to the {@link ReadWriteHandler}. Outbound data will be encrypted before being flushed
+ * before being passed to the {@link NioChannelHandler}. Outbound data will be encrypted before being flushed
  * to the channel.
  */
 public final class SSLChannelContext extends SocketChannelContext {
@@ -43,15 +42,14 @@ public final class SSLChannelContext extends SocketChannelContext {
     private Runnable closeTimeoutCanceller = DEFAULT_TIMEOUT_CANCELLER;
 
     SSLChannelContext(NioSocketChannel channel, NioSelector selector, Consumer<Exception> exceptionHandler, SSLDriver sslDriver,
-                      ReadWriteHandler readWriteHandler, InboundChannelBuffer applicationBuffer) {
+                      NioChannelHandler readWriteHandler, InboundChannelBuffer applicationBuffer) {
         this(channel, selector, exceptionHandler, sslDriver, readWriteHandler, InboundChannelBuffer.allocatingInstance(),
-            applicationBuffer, ALWAYS_ALLOW_CHANNEL);
+            applicationBuffer);
     }
 
     SSLChannelContext(NioSocketChannel channel, NioSelector selector, Consumer<Exception> exceptionHandler, SSLDriver sslDriver,
-                      ReadWriteHandler readWriteHandler, InboundChannelBuffer networkReadBuffer, InboundChannelBuffer channelBuffer,
-                      Predicate<NioSocketChannel> allowChannelPredicate) {
-        super(channel, selector, exceptionHandler, readWriteHandler, channelBuffer, allowChannelPredicate);
+                      NioChannelHandler readWriteHandler, InboundChannelBuffer networkReadBuffer, InboundChannelBuffer channelBuffer) {
+        super(channel, selector, exceptionHandler, readWriteHandler, channelBuffer);
         this.sslDriver = sslDriver;
         this.networkReadBuffer = networkReadBuffer;
     }
@@ -182,12 +180,7 @@ public boolean selectorShouldClose() {
     public void closeChannel() {
         if (isClosing.compareAndSet(false, true)) {
             WriteOperation writeOperation = new CloseNotifyOperation(this);
-            NioSelector selector = getSelector();
-            if (selector.isOnCurrentThread() == false) {
-                selector.queueWrite(writeOperation);
-                return;
-            }
-            selector.writeToChannel(writeOperation);
+            getSelector().queueWrite(writeOperation);
         }
     }
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransport.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransport.java
index ddf465f81d90e..bf476e5b73462 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransport.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioHttpServerTransport.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.nio.BytesChannelContext;
 import org.elasticsearch.nio.ChannelFactory;
 import org.elasticsearch.nio.InboundChannelBuffer;
+import org.elasticsearch.nio.NioChannelHandler;
 import org.elasticsearch.nio.NioSelector;
 import org.elasticsearch.nio.NioSocketChannel;
 import org.elasticsearch.nio.ServerChannelContext;
@@ -44,7 +45,6 @@ public class SecurityNioHttpServerTransport extends NioHttpServerTransport {
 
     private final SecurityHttpExceptionHandler securityExceptionHandler;
     private final IPFilter ipFilter;
-    private final NioIPFilter nioIpFilter;
     private final SSLService sslService;
     private final SSLConfiguration sslConfiguration;
     private final boolean sslEnabled;
@@ -56,7 +56,6 @@ public SecurityNioHttpServerTransport(Settings settings, NetworkService networkS
         super(settings, networkService, bigArrays, pageCacheRecycler, threadPool, xContentRegistry, dispatcher, nioGroupFactory);
         this.securityExceptionHandler = new SecurityHttpExceptionHandler(logger, lifecycle, (c, e) -> super.onException(c, e));
         this.ipFilter = ipFilter;
-        this.nioIpFilter = new NioIPFilter(ipFilter, IPFilter.HTTP_PROFILE_NAME);
         this.sslEnabled = HTTP_SSL_ENABLED.get(settings);
         this.sslService = sslService;
         if (sslEnabled) {
@@ -91,6 +90,13 @@ public NioHttpChannel createChannel(NioSelector selector, SocketChannel channel)
             NioHttpChannel httpChannel = new NioHttpChannel(channel);
             HttpReadWriteHandler httpHandler = new HttpReadWriteHandler(httpChannel,SecurityNioHttpServerTransport.this,
                 handlingSettings, corsConfig, selector.getTaskScheduler(), threadPool::relativeTimeInNanos);
+            final NioChannelHandler handler;
+            if (ipFilter != null) {
+                handler = new NioIPFilter(httpHandler, httpChannel.getRemoteAddress(), ipFilter, IPFilter.HTTP_PROFILE_NAME);
+            } else {
+                handler = httpHandler;
+            }
+
             InboundChannelBuffer networkBuffer = new InboundChannelBuffer(pageAllocator);
             Consumer<Exception> exceptionHandler = (e) -> securityExceptionHandler.accept(httpChannel, e);
 
@@ -107,10 +113,10 @@ public NioHttpChannel createChannel(NioSelector selector, SocketChannel channel)
                 }
                 SSLDriver sslDriver = new SSLDriver(sslEngine, pageAllocator, false);
                 InboundChannelBuffer applicationBuffer = new InboundChannelBuffer(pageAllocator);
-                context = new SSLChannelContext(httpChannel, selector, exceptionHandler, sslDriver, httpHandler, networkBuffer,
-                    applicationBuffer, nioIpFilter);
+                context = new SSLChannelContext(httpChannel, selector, exceptionHandler, sslDriver, handler, networkBuffer,
+                    applicationBuffer);
             } else {
-                context = new BytesChannelContext(httpChannel, selector, exceptionHandler, httpHandler, networkBuffer, nioIpFilter);
+                context = new BytesChannelContext(httpChannel, selector, exceptionHandler, handler, networkBuffer);
             }
             httpChannel.setContext(context);
 
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioTransport.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioTransport.java
index c2722cb3a0fb6..8b1a40aa266f0 100644
--- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioTransport.java
+++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SecurityNioTransport.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.nio.BytesChannelContext;
 import org.elasticsearch.nio.ChannelFactory;
 import org.elasticsearch.nio.InboundChannelBuffer;
+import org.elasticsearch.nio.NioChannelHandler;
 import org.elasticsearch.nio.NioSelector;
 import org.elasticsearch.nio.NioSocketChannel;
 import org.elasticsearch.nio.ServerChannelContext;
@@ -65,19 +66,19 @@ public class SecurityNioTransport extends NioTransport {
     private static final Logger logger = LogManager.getLogger(SecurityNioTransport.class);
 
     private final SecurityTransportExceptionHandler exceptionHandler;
-    private final IPFilter authenticator;
+    private final IPFilter ipFilter;
     private final SSLService sslService;
     private final Map<String, SSLConfiguration> profileConfiguration;
     private final boolean sslEnabled;
 
     public SecurityNioTransport(Settings settings, Version version, ThreadPool threadPool, NetworkService networkService,
                                 PageCacheRecycler pageCacheRecycler, NamedWriteableRegistry namedWriteableRegistry,
-                                CircuitBreakerService circuitBreakerService, @Nullable final IPFilter authenticator,
+                                CircuitBreakerService circuitBreakerService, @Nullable final IPFilter ipFilter,
                                 SSLService sslService, NioGroupFactory groupFactory) {
         super(settings, version, threadPool, networkService, pageCacheRecycler, namedWriteableRegistry, circuitBreakerService,
             groupFactory);
         this.exceptionHandler = new SecurityTransportExceptionHandler(logger, lifecycle, (c, e) -> super.onException(c, e));
-        this.authenticator = authenticator;
+        this.ipFilter = ipFilter;
         this.sslService = sslService;
         this.sslEnabled = XPackSettings.TRANSPORT_SSL_ENABLED.get(settings);
         if (sslEnabled) {
@@ -92,8 +93,8 @@ public SecurityNioTransport(Settings settings, Version version, ThreadPool threa
     @Override
     protected void doStart() {
         super.doStart();
-        if (authenticator != null) {
-            authenticator.setBoundTransportAddress(boundAddress(), profileBoundAddresses());
+        if (ipFilter != null) {
+            ipFilter.setBoundTransportAddress(boundAddress(), profileBoundAddresses());
         }
     }
 
@@ -132,7 +133,6 @@ private class SecurityTcpChannelFactory extends TcpChannelFactory {
 
         private final String profileName;
         private final boolean isClient;
-        private final NioIPFilter ipFilter;
 
         private SecurityTcpChannelFactory(ProfileSettings profileSettings, boolean isClient) {
             this(new RawChannelFactory(profileSettings.tcpNoDelay,
@@ -146,13 +146,18 @@ private SecurityTcpChannelFactory(RawChannelFactory rawChannelFactory, String pr
             super(rawChannelFactory);
             this.profileName = profileName;
             this.isClient = isClient;
-            this.ipFilter = new NioIPFilter(authenticator, profileName);
         }
 
         @Override
         public NioTcpChannel createChannel(NioSelector selector, SocketChannel channel) throws IOException {
             NioTcpChannel nioChannel = new NioTcpChannel(isClient == false, profileName, channel);
             TcpReadWriteHandler readWriteHandler = new TcpReadWriteHandler(nioChannel, SecurityNioTransport.this);
+            final NioChannelHandler handler;
+            if (ipFilter != null) {
+                handler = new NioIPFilter(readWriteHandler, nioChannel.getRemoteAddress(), ipFilter, profileName);
+            } else {
+                handler = readWriteHandler;
+            }
             InboundChannelBuffer networkBuffer = new InboundChannelBuffer(pageAllocator);
             Consumer<Exception> exceptionHandler = (e) -> onException(nioChannel, e);
 
@@ -160,10 +165,10 @@ public NioTcpChannel createChannel(NioSelector selector, SocketChannel channel)
             if (sslEnabled) {
                 SSLDriver sslDriver = new SSLDriver(createSSLEngine(channel), pageAllocator, isClient);
                 InboundChannelBuffer applicationBuffer = new InboundChannelBuffer(pageAllocator);
-                context = new SSLChannelContext(nioChannel, selector, exceptionHandler, sslDriver, readWriteHandler, networkBuffer,
-                    applicationBuffer, ipFilter);
+                context = new SSLChannelContext(nioChannel, selector, exceptionHandler, sslDriver, handler, networkBuffer,
+                    applicationBuffer);
             } else {
-                context = new BytesChannelContext(nioChannel, selector, exceptionHandler, readWriteHandler, networkBuffer, ipFilter);
+                context = new BytesChannelContext(nioChannel, selector, exceptionHandler, handler, networkBuffer);
             }
             nioChannel.setContext(context);
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java
index 145213094d1d8..557fd5813bf2e 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java
@@ -16,12 +16,12 @@
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.test.SecuritySettingsSourceField;
+import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheAction;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequest;
 import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheResponse;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.Realm;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authc.Realms;
 import org.junit.BeforeClass;
@@ -133,11 +133,9 @@ public void executeRequest() throws Exception {
         public abstract void executeRequest() throws Exception;
 
         static void executeTransportRequest(ClearRealmCacheRequest request) throws Exception {
-            SecurityClient securityClient = securityClient(client());
-
             final CountDownLatch latch = new CountDownLatch(1);
             final AtomicReference<Throwable> error = new AtomicReference<>();
-            securityClient.clearRealmCache(request, new ActionListener<ClearRealmCacheResponse>() {
+            client().execute(ClearRealmCacheAction.INSTANCE, request, new ActionListener<>() {
                 @Override
                 public void onResponse(ClearRealmCacheResponse response) {
                     assertThat(response.getNodes().size(), equalTo(internalCluster().getNodeNames().length));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java
index 2a9f2017c50f3..5eeab715ec148 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java
@@ -6,29 +6,33 @@
 package org.elasticsearch.integration;
 
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.security.DeleteRoleRequest;
+import org.elasticsearch.client.security.DeleteRoleResponse;
+import org.elasticsearch.client.security.GetRolesRequest;
+import org.elasticsearch.client.security.GetRolesResponse;
+import org.elasticsearch.client.security.PutRoleRequest;
+import org.elasticsearch.client.security.PutRoleResponse;
+import org.elasticsearch.client.security.RefreshPolicy;
+import org.elasticsearch.client.security.user.privileges.IndicesPrivileges;
+import org.elasticsearch.client.security.user.privileges.Role;
 import org.elasticsearch.test.NativeRealmIntegTestCase;
-import org.elasticsearch.xpack.core.security.action.role.DeleteRoleResponse;
-import org.elasticsearch.xpack.core.security.action.role.GetRolesResponse;
-import org.elasticsearch.xpack.core.security.action.role.PutRoleResponse;
 import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
 import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
-import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
+import static org.elasticsearch.test.SecuritySettingsSource.SECURITY_REQUEST_OPTIONS;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
-
 /**
  * Test for the clear roles API
  */
@@ -45,14 +49,17 @@ public static void init() throws Exception {
     }
 
     @Before
-    public void setupForTests() {
-        SecurityClient c = securityClient();
+    public void setupForTests() throws IOException {
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
         // create roles
         for (String role : roles) {
-            c.preparePutRole(role)
-                    .cluster("none")
-                    .addIndices(new String[] { "*" }, new String[] { "ALL" }, null, null, null, randomBoolean())
-                    .get();
+            restClient.security().putRole(new PutRoleRequest(
+                Role.builder().name(role)
+                    .clusterPrivileges("none")
+                    .indicesPrivileges(
+                        IndicesPrivileges.builder().indices("*").privileges("ALL").allowRestrictedIndices(randomBoolean()).build())
+                    .build(), RefreshPolicy.IMMEDIATE),
+                SECURITY_REQUEST_OPTIONS);
             logger.debug("--> created role [{}]", role);
         }
 
@@ -74,50 +81,50 @@ protected boolean addMockHttpTransport() {
     }
 
     public void testModifyingViaApiClearsCache() throws Exception {
-        Client client = client();
-        SecurityClient securityClient = securityClient(client);
-
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
         int modifiedRolesCount = randomIntBetween(1, roles.length);
         List<String> toModify = randomSubsetOf(modifiedRolesCount, roles);
         logger.debug("--> modifying roles {} to have run_as", toModify);
         for (String role : toModify) {
-            PutRoleResponse response = securityClient.preparePutRole(role)
-                    .cluster("none")
-                    .addIndices(new String[] { "*" }, new String[] { "ALL" }, null, null, null, randomBoolean())
-                    .runAs(role)
-                    .setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE)
-                    .get();
+            PutRoleResponse response = restClient.security().putRole(new PutRoleRequest(Role.builder().name(role)
+                    .clusterPrivileges("none")
+                    .indicesPrivileges(
+                        IndicesPrivileges.builder().indices("*").privileges("ALL").allowRestrictedIndices(randomBoolean()).build())
+                    .runAsPrivilege(role)
+                    .build(), randomBoolean() ? RefreshPolicy.IMMEDIATE : RefreshPolicy.NONE), SECURITY_REQUEST_OPTIONS);
             assertThat(response.isCreated(), is(false));
             logger.debug("--> updated role [{}] with run_as", role);
         }
 
-        assertRolesAreCorrect(securityClient, toModify);
+        assertRolesAreCorrect(restClient, toModify);
     }
 
     public void testDeletingViaApiClearsCache() throws Exception {
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
         final int rolesToDelete = randomIntBetween(1, roles.length - 1);
         List<String> toDelete = randomSubsetOf(rolesToDelete, roles);
         for (String role : toDelete) {
-            DeleteRoleResponse response = securityClient().prepareDeleteRole(role).get();
-            assertTrue(response.found());
+            DeleteRoleResponse response = restClient.security()
+                .deleteRole(new DeleteRoleRequest(role, RefreshPolicy.IMMEDIATE), SECURITY_REQUEST_OPTIONS);
+            assertTrue(response.isFound());
         }
 
-        GetRolesResponse roleResponse = securityClient().prepareGetRoles().names(roles).get();
-        assertTrue(roleResponse.hasRoles());
-        assertThat(roleResponse.roles().length, is(roles.length - rolesToDelete));
+        GetRolesResponse roleResponse = restClient.security().getRoles(new GetRolesRequest(roles), SECURITY_REQUEST_OPTIONS);
+        assertFalse(roleResponse.getRoles().isEmpty());
+        assertThat(roleResponse.getRoles().size(), is(roles.length - rolesToDelete));
     }
 
-    private void assertRolesAreCorrect(SecurityClient securityClient, List<String> toModify) {
+    private void assertRolesAreCorrect(RestHighLevelClient restClient, List<String> toModify) throws IOException {
         for (String role : roles) {
             logger.debug("--> getting role [{}]", role);
-            GetRolesResponse roleResponse = securityClient.prepareGetRoles().names(role).get();
-            assertThat(roleResponse.hasRoles(), is(true));
-            final String[] runAs = roleResponse.roles()[0].getRunAs();
+            GetRolesResponse roleResponse = restClient.security().getRoles(new GetRolesRequest(role), SECURITY_REQUEST_OPTIONS);
+            assertThat(roleResponse.getRoles().isEmpty(), is(false));
+            final Set<String> runAs = roleResponse.getRoles().get(0).getRunAsPrivilege();
             if (toModify.contains(role)) {
-                assertThat("role [" + role + "] should be modified and have run as", runAs == null || runAs.length == 0, is(false));
-                assertThat(Arrays.asList(runAs).contains(role), is(true));
+                assertThat("role [" + role + "] should be modified and have run as", runAs == null || runAs.size() == 0, is(false));
+                assertThat(runAs.contains(role), is(true));
             } else {
-                assertThat("role [" + role + "] should be cached and not have run as set but does!", runAs == null || runAs.length == 0,
+                assertThat("role [" + role + "] should be cached and not have run as set but does!", runAs == null || runAs.size() == 0,
                         is(true));
             }
         }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
index dc8f93a0cdcf7..59a6c4b31fe15 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java
@@ -345,8 +345,8 @@ public void testMSearch() throws Exception {
         MultiSearchResponse response = client()
                 .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
                 .prepareMultiSearch()
-                .add(client().prepareSearch("test1").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
-                .add(client().prepareSearch("test2").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery()))
                 .get();
         assertFalse(response.getResponses()[0].isFailure());
         assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
@@ -363,8 +363,8 @@ public void testMSearch() throws Exception {
         response = client()
                 .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
                 .prepareMultiSearch()
-                .add(client().prepareSearch("test1").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
-                .add(client().prepareSearch("test2").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery()))
                 .get();
         assertFalse(response.getResponses()[0].isFailure());
         assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
@@ -381,9 +381,9 @@ public void testMSearch() throws Exception {
         response = client()
                 .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
                 .prepareMultiSearch()
-                .add(client().prepareSearch("test1").setTypes("type1").addSort(SortBuilders.fieldSort("id").sortMode(SortMode.MIN))
+                .add(client().prepareSearch("test1").addSort(SortBuilders.fieldSort("id").sortMode(SortMode.MIN))
                         .setQuery(QueryBuilders.matchAllQuery()))
-                .add(client().prepareSearch("test2").setTypes("type1").addSort(SortBuilders.fieldSort("id").sortMode(SortMode.MIN))
+                .add(client().prepareSearch("test2").addSort(SortBuilders.fieldSort("id").sortMode(SortMode.MIN))
                         .setQuery(QueryBuilders.matchAllQuery()))
                 .get();
         assertFalse(response.getResponses()[0].isFailure());
@@ -424,47 +424,47 @@ public void testTVApi() throws Exception {
         boolean realtime = randomBoolean();
         TermVectorsResponse response = client()
                 .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
-                .prepareTermVectors("test", "type1", "1")
+                .prepareTermVectors("test", "1")
                 .setRealtime(realtime)
                 .get();
         assertThat(response.isExists(), is(true));
         assertThat(response.getId(), is("1"));
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
-                .prepareTermVectors("test", "type1", "2")
+                .prepareTermVectors("test", "2")
                 .setRealtime(realtime)
                 .get();
         assertThat(response.isExists(), is(true));
         assertThat(response.getId(), is("2"));
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
-                .prepareTermVectors("test", "type1", "1")
+                .prepareTermVectors("test", "1")
                 .setRealtime(realtime)
                 .get();
         assertThat(response.isExists(), is(true));
         assertThat(response.getId(), is("1"));
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
-                .prepareTermVectors("test", "type1", "2")
+                .prepareTermVectors("test", "2")
                 .setRealtime(realtime)
                 .get();
         assertThat(response.isExists(), is(true));
         assertThat(response.getId(), is("2"));
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
-                .prepareTermVectors("test", "type1", "1")
+                .prepareTermVectors("test", "1")
                 .setRealtime(realtime)
                 .get();
         assertThat(response.isExists(), is(false));
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
-                .prepareTermVectors("test", "type1", "2")
+                .prepareTermVectors("test", "2")
                 .setRealtime(realtime)
                 .get();
         assertThat(response.isExists(), is(false));
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
-                .prepareTermVectors("test", "type1", "3")
+                .prepareTermVectors("test", "3")
                 .setRealtime(realtime)
                 .get();
         assertThat(response.isExists(), is(false));
@@ -490,7 +490,7 @@ public void testMTVApi() throws Exception {
         MultiTermVectorsResponse response = client()
                 .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
                 .prepareMultiTermVectors()
-                .add(new TermVectorsRequest("test", "type1", "1").realtime(realtime))
+                .add(new TermVectorsRequest("test", "1").realtime(realtime))
                 .get();
         assertThat(response.getResponses().length, equalTo(1));
         assertThat(response.getResponses()[0].getResponse().isExists(), is(true));
@@ -498,7 +498,7 @@ public void testMTVApi() throws Exception {
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
                 .prepareMultiTermVectors()
-                .add(new TermVectorsRequest("test", "type1", "2").realtime(realtime))
+                .add(new TermVectorsRequest("test", "2").realtime(realtime))
                 .get();
         assertThat(response.getResponses().length, equalTo(1));
         assertThat(response.getResponses()[0].getResponse().isExists(), is(true));
@@ -506,8 +506,7 @@ public void testMTVApi() throws Exception {
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
                 .prepareMultiTermVectors()
-                .add(new TermVectorsRequest("test", "type1", "1").realtime(realtime))
-                .add(new TermVectorsRequest("test", "type1", "2").realtime(realtime))
+                .add(new TermVectorsRequest("test", "1").realtime(realtime)).add(new TermVectorsRequest("test", "2").realtime(realtime))
                 .get();
         assertThat(response.getResponses().length, equalTo(2));
         assertThat(response.getResponses()[0].getResponse().isExists(), is(true));
@@ -517,21 +516,21 @@ public void testMTVApi() throws Exception {
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
                 .prepareMultiTermVectors()
-                .add(new TermVectorsRequest("test", "type1", "1").realtime(realtime))
+                .add(new TermVectorsRequest("test", "1").realtime(realtime))
                 .get();
         assertThat(response.getResponses().length, equalTo(1));
         assertThat(response.getResponses()[0].getResponse().isExists(), is(false));
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
                 .prepareMultiTermVectors()
-                .add(new TermVectorsRequest("test", "type1", "2").realtime(realtime))
+                .add(new TermVectorsRequest("test", "2").realtime(realtime))
                 .get();
         assertThat(response.getResponses().length, equalTo(1));
         assertThat(response.getResponses()[0].getResponse().isExists(), is(false));
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
                 .prepareMultiTermVectors()
-                .add(new TermVectorsRequest("test", "type1", "3").realtime(realtime))
+                .add(new TermVectorsRequest("test", "3").realtime(realtime))
                 .get();
         assertThat(response.getResponses().length, equalTo(1));
         assertThat(response.getResponses()[0].getResponse().isExists(), is(false));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
index c5ff8242b6bec..6b87b5d41faf2 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java
@@ -539,8 +539,8 @@ public void testMSearchApi() throws Exception {
         MultiSearchResponse response = client()
                 .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
                 .prepareMultiSearch()
-                .add(client().prepareSearch("test1").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
-                .add(client().prepareSearch("test2").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery()))
                 .get();
         assertFalse(response.getResponses()[0].isFailure());
         assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
@@ -554,8 +554,8 @@ public void testMSearchApi() throws Exception {
         response = client()
                 .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
                 .prepareMultiSearch()
-                .add(client().prepareSearch("test1").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
-                .add(client().prepareSearch("test2").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery()))
                 .get();
         assertFalse(response.getResponses()[0].isFailure());
         assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
@@ -569,8 +569,8 @@ public void testMSearchApi() throws Exception {
         response = client()
                 .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
                 .prepareMultiSearch()
-                .add(client().prepareSearch("test1").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
-                .add(client().prepareSearch("test2").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery()))
                 .get();
         assertFalse(response.getResponses()[0].isFailure());
         assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
@@ -586,8 +586,8 @@ public void testMSearchApi() throws Exception {
         response = client()
                 .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)))
                 .prepareMultiSearch()
-                .add(client().prepareSearch("test1").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
-                .add(client().prepareSearch("test2").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery()))
                 .get();
         assertFalse(response.getResponses()[0].isFailure());
         assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
@@ -599,8 +599,8 @@ public void testMSearchApi() throws Exception {
         response = client()
                 .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user5", USERS_PASSWD)))
                 .prepareMultiSearch()
-                .add(client().prepareSearch("test1").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
-                .add(client().prepareSearch("test2").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery()))
                 .get();
         assertFalse(response.getResponses()[0].isFailure());
         assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
@@ -618,8 +618,8 @@ public void testMSearchApi() throws Exception {
         response = client()
                 .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user6", USERS_PASSWD)))
                 .prepareMultiSearch()
-                .add(client().prepareSearch("test1").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
-                .add(client().prepareSearch("test2").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery()))
                 .get();
         assertFalse(response.getResponses()[0].isFailure());
         assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
@@ -637,8 +637,8 @@ public void testMSearchApi() throws Exception {
         response = client()
                 .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user7", USERS_PASSWD)))
                 .prepareMultiSearch()
-                .add(client().prepareSearch("test1").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
-                .add(client().prepareSearch("test2").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery()))
                 .get();
         assertFalse(response.getResponses()[0].isFailure());
         assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
@@ -656,8 +656,8 @@ public void testMSearchApi() throws Exception {
         response = client()
                 .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user8", USERS_PASSWD)))
                 .prepareMultiSearch()
-                .add(client().prepareSearch("test1").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
-                .add(client().prepareSearch("test2").setTypes("type1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test1").setQuery(QueryBuilders.matchAllQuery()))
+                .add(client().prepareSearch("test2").setQuery(QueryBuilders.matchAllQuery()))
                 .get();
         assertFalse(response.getResponses()[0].isFailure());
         assertThat(response.getResponses()[0].getResponse().getHits().getTotalHits().value, is(1L));
@@ -1164,7 +1164,7 @@ public void testTVApi() throws Exception {
         boolean realtime = randomBoolean();
         TermVectorsResponse response = client()
                 .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
-                .prepareTermVectors("test", "type1", "1")
+                .prepareTermVectors("test", "1")
                 .setRealtime(realtime)
                 .get();
         assertThat(response.isExists(), is(true));
@@ -1172,7 +1172,7 @@ public void testTVApi() throws Exception {
         assertThat(response.getFields().terms("field1").size(), equalTo(1L));
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
-                .prepareTermVectors("test", "type1", "1")
+                .prepareTermVectors("test", "1")
                 .setRealtime(realtime)
                 .get();
         assertThat(response.isExists(), is(true));
@@ -1180,7 +1180,7 @@ public void testTVApi() throws Exception {
         assertThat(response.getFields().terms("field2").size(), equalTo(1L));
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
-                .prepareTermVectors("test", "type1", "1")
+                .prepareTermVectors("test", "1")
                 .setRealtime(realtime)
                 .get();
         assertThat(response.isExists(), is(true));
@@ -1189,14 +1189,14 @@ public void testTVApi() throws Exception {
         assertThat(response.getFields().terms("field2").size(), equalTo(1L));
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)))
-                .prepareTermVectors("test", "type1", "1")
+                .prepareTermVectors("test", "1")
                 .setRealtime(realtime)
                 .get();
         assertThat(response.isExists(), is(true));
         assertThat(response.getFields().size(), equalTo(0));
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user5", USERS_PASSWD)))
-                .prepareTermVectors("test", "type1", "1")
+                .prepareTermVectors("test", "1")
                 .setRealtime(realtime)
                 .get();
         assertThat(response.isExists(), is(true));
@@ -1206,7 +1206,7 @@ public void testTVApi() throws Exception {
         assertThat(response.getFields().terms("field3").size(), equalTo(1L));
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user6", USERS_PASSWD)))
-                .prepareTermVectors("test", "type1", "1")
+                .prepareTermVectors("test", "1")
                 .setRealtime(realtime)
                 .get();
         assertThat(response.isExists(), is(true));
@@ -1216,7 +1216,7 @@ public void testTVApi() throws Exception {
         assertThat(response.getFields().terms("field3").size(), equalTo(1L));
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user7", USERS_PASSWD)))
-                .prepareTermVectors("test", "type1", "1")
+                .prepareTermVectors("test", "1")
                 .setRealtime(realtime)
                 .get();
         assertThat(response.isExists(), is(true));
@@ -1226,7 +1226,7 @@ public void testTVApi() throws Exception {
         assertThat(response.getFields().terms("field3").size(), equalTo(1L));
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user8", USERS_PASSWD)))
-                .prepareTermVectors("test", "type1", "1")
+                .prepareTermVectors("test", "1")
                 .setRealtime(realtime)
                 .get();
         assertThat(response.isExists(), is(true));
@@ -1249,7 +1249,7 @@ public void testMTVApi() throws Exception {
         MultiTermVectorsResponse response = client()
                 .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
                 .prepareMultiTermVectors()
-                .add(new TermVectorsRequest("test", "type1", "1").realtime(realtime))
+                .add(new TermVectorsRequest("test", "1").realtime(realtime))
                 .get();
         assertThat(response.getResponses().length, equalTo(1));
         assertThat(response.getResponses()[0].getResponse().isExists(), is(true));
@@ -1258,7 +1258,7 @@ public void testMTVApi() throws Exception {
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
                 .prepareMultiTermVectors()
-                .add(new TermVectorsRequest("test", "type1", "1").realtime(realtime))
+                .add(new TermVectorsRequest("test", "1").realtime(realtime))
                 .get();
         assertThat(response.getResponses().length, equalTo(1));
         assertThat(response.getResponses()[0].getResponse().isExists(), is(true));
@@ -1267,7 +1267,7 @@ public void testMTVApi() throws Exception {
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
                 .prepareMultiTermVectors()
-                .add(new TermVectorsRequest("test", "type1", "1").realtime(realtime))
+                .add(new TermVectorsRequest("test", "1").realtime(realtime))
                 .get();
         assertThat(response.getResponses().length, equalTo(1));
         assertThat(response.getResponses()[0].getResponse().isExists(), is(true));
@@ -1277,7 +1277,7 @@ public void testMTVApi() throws Exception {
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)))
                 .prepareMultiTermVectors()
-                .add(new TermVectorsRequest("test", "type1", "1").realtime(realtime))
+                .add(new TermVectorsRequest("test", "1").realtime(realtime))
                 .get();
         assertThat(response.getResponses().length, equalTo(1));
         assertThat(response.getResponses()[0].getResponse().isExists(), is(true));
@@ -1285,7 +1285,7 @@ public void testMTVApi() throws Exception {
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user5", USERS_PASSWD)))
                 .prepareMultiTermVectors()
-                .add(new TermVectorsRequest("test", "type1", "1").realtime(realtime))
+                .add(new TermVectorsRequest("test", "1").realtime(realtime))
                 .get();
         assertThat(response.getResponses().length, equalTo(1));
         assertThat(response.getResponses()[0].getResponse().isExists(), is(true));
@@ -1296,7 +1296,7 @@ public void testMTVApi() throws Exception {
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user6", USERS_PASSWD)))
                 .prepareMultiTermVectors()
-                .add(new TermVectorsRequest("test", "type1", "1").realtime(realtime))
+                .add(new TermVectorsRequest("test", "1").realtime(realtime))
                 .get();
         assertThat(response.getResponses().length, equalTo(1));
         assertThat(response.getResponses()[0].getResponse().isExists(), is(true));
@@ -1307,7 +1307,7 @@ public void testMTVApi() throws Exception {
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user7", USERS_PASSWD)))
                 .prepareMultiTermVectors()
-                .add(new TermVectorsRequest("test", "type1", "1").realtime(realtime))
+                .add(new TermVectorsRequest("test", "1").realtime(realtime))
                 .get();
         assertThat(response.getResponses().length, equalTo(1));
         assertThat(response.getResponses()[0].getResponse().isExists(), is(true));
@@ -1318,7 +1318,7 @@ public void testMTVApi() throws Exception {
 
         response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user8", USERS_PASSWD)))
                 .prepareMultiTermVectors()
-                .add(new TermVectorsRequest("test", "type1", "1").realtime(realtime))
+                .add(new TermVectorsRequest("test", "1").realtime(realtime))
                 .get();
         assertThat(response.getResponses().length, equalTo(1));
         assertThat(response.getResponses()[0].getResponse().isExists(), is(true));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java
index 2035a8b6c19dd..71cdd6c03967e 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java
@@ -20,25 +20,29 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.security.PutUserRequest;
+import org.elasticsearch.client.security.RefreshPolicy;
+import org.elasticsearch.client.security.user.User;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
-import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.junit.After;
 import org.junit.Before;
 
 import java.util.Collections;
+import java.util.List;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER;
-import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.elasticsearch.test.SecuritySettingsSource.SECURITY_REQUEST_OPTIONS;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.hamcrest.Matchers.is;
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER;
+import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.is;
 
 public class MultipleIndicesPermissionsTests extends SecurityIntegTestCase {
 
@@ -47,7 +51,9 @@ public class MultipleIndicesPermissionsTests extends SecurityIntegTestCase {
     @Before
     public void waitForSecurityIndexWritable() throws Exception {
         // adds a dummy user to the native realm to force .security index creation
-        securityClient().preparePutUser("dummy_user", "password".toCharArray(), Hasher.BCRYPT, "missing_role").get();
+        new TestRestHighLevelClient().security().putUser(
+            PutUserRequest.withPassword(new User("dummy_user", List.of("missing_role")), "password".toCharArray(), true,
+                RefreshPolicy.IMMEDIATE), SECURITY_REQUEST_OPTIONS);
         assertSecurityIndexActive();
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/SecurityCachePermissionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/SecurityCachePermissionTests.java
index 8a570aac28663..c5169f9344c52 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/SecurityCachePermissionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/SecurityCachePermissionTests.java
@@ -52,7 +52,7 @@ public void loadData() {
     }
 
     public void testThatTermsFilterQueryDoesntLeakData() {
-        SearchResponse response = client().prepareSearch("data").setTypes("a").setQuery(QueryBuilders.constantScoreQuery(
+        SearchResponse response = client().prepareSearch("data").setQuery(QueryBuilders.constantScoreQuery(
                 QueryBuilders.termsLookupQuery("token", new TermsLookup("tokens", "tokens", "1", "tokens"))))
                 .execute().actionGet();
         assertThat(response.isTimedOut(), is(false));
@@ -62,7 +62,8 @@ public void testThatTermsFilterQueryDoesntLeakData() {
         try {
             response = client().filterWithHeader(singletonMap("Authorization", basicAuthHeaderValue(READ_ONE_IDX_USER,
                     SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)))
-                    .prepareSearch("data").setTypes("a").setQuery(QueryBuilders.constantScoreQuery(
+                    .prepareSearch("data")
+                    .setQuery(QueryBuilders.constantScoreQuery(
                     QueryBuilders.termsLookupQuery("token", new TermsLookup("tokens", "tokens", "1", "tokens"))))
                     .execute().actionGet();
             fail("search phase exception should have been thrown! response was:\n" + response.toString());
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/SecurityClearScrollTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/SecurityClearScrollTests.java
index 4c189e3e7f3da..2bc7a43937d36 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/SecurityClearScrollTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/SecurityClearScrollTests.java
@@ -14,8 +14,8 @@
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.xpack.core.security.SecurityField;
 import org.elasticsearch.test.SecurityIntegTestCase;
+import org.elasticsearch.xpack.core.security.SecurityField;
 import org.junit.After;
 import org.junit.Before;
 
@@ -25,9 +25,9 @@
 import java.util.Map;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
 
@@ -75,7 +75,7 @@ public void indexRandomDocuments() {
         MultiSearchRequestBuilder multiSearchRequestBuilder = client().prepareMultiSearch();
         int count = randomIntBetween(5, 15);
         for (int i = 0; i < count; i++) {
-            multiSearchRequestBuilder.add(client().prepareSearch("index").setTypes("type").setScroll("10m").setSize(1));
+            multiSearchRequestBuilder.add(client().prepareSearch("index").setScroll("10m").setSize(1));
         }
         MultiSearchResponse multiSearchResponse = multiSearchRequestBuilder.get();
         scrollIds = getScrollIds(multiSearchResponse);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java
index c115aac11d732..2e6add5c4d943 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java
@@ -31,7 +31,6 @@
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.test.SecuritySettingsSourceField;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.transport.Netty4Plugin;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
@@ -56,8 +55,6 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
-@TestLogging("org.elasticsearch.cluster.service:TRACE,org.elasticsearch.action.search:TRACE," +
-    "org.elasticsearch.search:TRACE")
 public class LicensingTests extends SecurityIntegTestCase {
     private static final String ROLES =
             SecuritySettingsSource.TEST_ROLE + ":\n" +
@@ -131,6 +128,7 @@ public void cleanupSecurityIndex() {
         deleteSecurityIndex();
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/42215")
     public void testEnableDisableBehaviour() throws Exception {
         IndexResponse indexResponse = index("test", "type", jsonBuilder()
                 .startObject()
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java
index 78d95ecbca0b8..312886b7dc450 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java
@@ -8,10 +8,11 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.security.ClearRealmCacheRequest;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.core.security.user.APMSystemUser;
 import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
@@ -24,9 +25,12 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 
+import static org.elasticsearch.test.SecuritySettingsSource.SECURITY_REQUEST_OPTIONS;
+
 /**
  * Test case with method to handle the starting and stopping the stores for native users and roles
  */
@@ -46,8 +50,9 @@ public void stopESNativeStores() throws Exception {
 
         if (getCurrentClusterScope() == Scope.SUITE) {
             // Clear the realm cache for all realms since we use a SUITE scoped cluster
-            SecurityClient client = securityClient(client());
-            client.prepareClearRealmCache().get();
+            RestHighLevelClient restClient = new TestRestHighLevelClient();
+            restClient.security()
+                .clearRealmCache(new ClearRealmCacheRequest(Collections.emptyList(), Collections.emptyList()), SECURITY_REQUEST_OPTIONS);
         }
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java
index a3b75b188a278..0db10d3f5ee57 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
@@ -24,11 +25,9 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -36,11 +35,8 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.xpack.core.XPackClient;
-import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.LocalStateSecurity;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.junit.AfterClass;
@@ -50,7 +46,6 @@
 import org.junit.Rule;
 import org.junit.rules.ExternalResource;
 
-import java.net.InetSocketAddress;
 import java.nio.file.Path;
 import java.util.Collection;
 import java.util.Collections;
@@ -399,32 +394,10 @@ protected Function<Client, Client> getClientWrapper() {
         // we need to wrap node clients because we do not specify a user for nodes and all requests will use the system
         // user. This is ok for internal n2n stuff but the test framework does other things like wiping indices, repositories, etc
         // that the system user cannot do. so we wrap the node client with a user that can do these things since the client() calls
-        // are randomized to return both node clients and transport clients
-        // transport clients do not need to be wrapped since we specify the xpack.security.user setting that sets the default user to be
-        // used for the transport client. If we did not set a default user then the transport client would not even be allowed
-        // to connect
+        // return a node client
         return client -> (client instanceof NodeClient) ? client.filterWithHeader(headers) : client;
     }
 
-    protected SecurityClient securityClient() {
-        return securityClient(client());
-    }
-
-    public static SecurityClient securityClient(Client client) {
-        return randomBoolean() ? new XPackClient(client).security() : new SecurityClient(client);
-    }
-
-    protected String getHttpURL() {
-        final NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().get();
-        final List<NodeInfo> nodes = nodeInfos.getNodes();
-        assertTrue("there is at least one node", nodes.size() > 0);
-        NodeInfo ni = randomFrom(nodes);
-        boolean useSSL = XPackSettings.HTTP_SSL_ENABLED.get(ni.getSettings());
-        TransportAddress publishAddress = ni.getHttp().address().publishAddress();
-        InetSocketAddress address = publishAddress.address();
-        return (useSSL ? "https://" : "http://") + NetworkAddress.format(address.getAddress()) + ":" + address.getPort();
-    }
-
     public void assertSecurityIndexActive() throws Exception {
         assertSecurityIndexActive(cluster());
     }
@@ -480,4 +453,10 @@ protected boolean isTransportSSLEnabled() {
     protected static Hasher getFastStoredHashAlgoForTests() {
         return Hasher.resolve(randomFrom("pbkdf2", "pbkdf2_1000", "bcrypt", "bcrypt9"));
     }
+
+    protected class TestRestHighLevelClient extends RestHighLevelClient {
+        public TestRestHighLevelClient() {
+            super(getRestClient(), client -> {}, List.of());
+        }
+    }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java
index 09e1c00cb543e..973eadcc3c846 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java
@@ -7,18 +7,17 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.reindex.ReindexPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.transport.Netty4Plugin;
-import org.elasticsearch.xpack.core.XPackClientPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.SecurityField;
 import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings;
@@ -30,11 +29,13 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.Collection;
 import java.util.List;
 import java.util.function.Consumer;
@@ -43,7 +44,7 @@
 import static org.apache.lucene.util.LuceneTestCase.createTempFile;
 import static org.elasticsearch.test.ESTestCase.inFipsJvm;
 import static org.elasticsearch.test.ESTestCase.randomFrom;
-import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD;
 import static org.elasticsearch.xpack.security.test.SecurityTestUtils.writeFile;
 
 /**
@@ -56,9 +57,13 @@ public class SecuritySettingsSource extends NodeConfigurationSource {
     public static final String TEST_USER_NAME = "test_user";
     public static final String TEST_PASSWORD_HASHED =
         new String(Hasher.resolve(randomFrom("pbkdf2", "pbkdf2_1000", "bcrypt9", "bcrypt8", "bcrypt")).
-            hash(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray())));
+            hash(new SecureString(TEST_PASSWORD.toCharArray())));
     public static final String TEST_ROLE = "user";
     public static final String TEST_SUPERUSER = "test_superuser";
+    public static final RequestOptions SECURITY_REQUEST_OPTIONS = RequestOptions.DEFAULT.toBuilder()
+        .addHeader("Authorization",
+            "Basic " + Base64.getEncoder().encodeToString((TEST_USER_NAME + ":" + TEST_PASSWORD).getBytes(StandardCharsets.UTF_8)))
+        .build();
 
     public static final String DEFAULT_TRANSPORT_CLIENT_ROLE = "transport_client";
     public static final String DEFAULT_TRANSPORT_CLIENT_USER_NAME = "test_trans_client_user";
@@ -148,22 +153,6 @@ public Path nodeConfigPath(int nodeOrdinal) {
         return nodePath(nodeOrdinal).resolve("config");
     }
 
-    @Override
-    public Settings transportClientSettings() {
-        Settings.Builder builder = Settings.builder();
-        addClientSSLSettings(builder, "xpack.security.transport.");
-        addDefaultSecurityTransportType(builder, Settings.EMPTY);
-
-        if (randomBoolean()) {
-            builder.put(SecurityField.USER_SETTING.getKey(),
-                    transportClientUsername() + ":" + new String(transportClientPassword().getChars()));
-        } else {
-            builder.put(ThreadContext.PREFIX + ".Authorization", basicAuthHeaderValue(transportClientUsername(),
-                    transportClientPassword()));
-        }
-        return builder.build();
-    }
-
     protected void addDefaultSecurityTransportType(Settings.Builder builder, Settings settings) {
         if (NetworkModule.TRANSPORT_TYPE_SETTING.exists(settings) == false) {
             builder.put(NetworkModule.TRANSPORT_TYPE_SETTING.getKey(), SecurityField.NAME4);
@@ -177,11 +166,6 @@ public Collection<Class<? extends Plugin>> nodePlugins() {
             InternalSettingsPlugin.class);
     }
 
-    @Override
-    public Collection<Class<? extends Plugin>> transportClientPlugins() {
-        return Arrays.asList(XPackClientPlugin.class, Netty4Plugin.class, ReindexPlugin.class, CommonAnalysisPlugin.class);
-    }
-
     protected String configUsers() {
         return CONFIG_STANDARD_USER;
     }
@@ -199,17 +183,9 @@ protected String nodeClientUsername() {
     }
 
     protected SecureString nodeClientPassword() {
-        return new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray());
-    }
-
-    protected String transportClientUsername() {
-        return DEFAULT_TRANSPORT_CLIENT_USER_NAME;
+        return new SecureString(TEST_PASSWORD.toCharArray());
     }
-
-    protected SecureString transportClientPassword() {
-        return new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray());
-    }
-
+    
     public static void addSSLSettingsForNodePEMFiles(Settings.Builder builder, String prefix, boolean hostnameVerificationEnabled) {
         addSSLSettingsForPEMFiles(builder, prefix,
             "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem", "testnode",
@@ -220,7 +196,7 @@ public static void addSSLSettingsForNodePEMFiles(Settings.Builder builder, Strin
                 "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/openldap.crt",
                 "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt",
                 "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_ec.crt"),
-            hostnameVerificationEnabled, false);
+            hostnameVerificationEnabled);
     }
 
     private void addNodeSSLSettings(Settings.Builder builder) {
@@ -231,7 +207,7 @@ private void addNodeSSLSettings(Settings.Builder builder) {
             } else {
                 addSSLSettingsForStore(builder, "xpack.security.transport.",
                     "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks", "testnode",
-                    hostnameVerificationEnabled, false);
+                    hostnameVerificationEnabled);
             }
         } else if (randomBoolean()) {
             builder.put(XPackSettings.TRANSPORT_SSL_ENABLED.getKey(), false);
@@ -247,10 +223,10 @@ public void addClientSSLSettings(Settings.Builder builder, String prefix) {
                 Arrays.asList("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt",
                     "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_ec.crt",
                     "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.crt"),
-                hostnameVerificationEnabled, true);
+                hostnameVerificationEnabled);
         } else {
             addSSLSettingsForStore(builder, prefix, "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.jks",
-                "testclient", hostnameVerificationEnabled, true);
+                "testclient", hostnameVerificationEnabled);
         }
     }
 
@@ -261,33 +237,22 @@ public void addClientSSLSettings(Settings.Builder builder, String prefix) {
      * @param password the password
      */
     public static void addSSLSettingsForStore(Settings.Builder builder, String resourcePathToStore, String password, String prefix) {
-        addSSLSettingsForStore(builder, prefix, resourcePathToStore, password, true, true);
+        addSSLSettingsForStore(builder, prefix, resourcePathToStore, password, true);
     }
 
     private static void addSSLSettingsForStore(Settings.Builder builder, String prefix, String resourcePathToStore, String password,
-                                               boolean hostnameVerificationEnabled, boolean transportClient) {
+                                               boolean hostnameVerificationEnabled) {
         Path store = resolveResourcePath(resourcePathToStore);
         builder.put(prefix + "ssl.verification_mode", hostnameVerificationEnabled ? "full" : "certificate");
         builder.put(prefix + "ssl.keystore.path", store);
-        if (transportClient) {
-            // continue using insecure settings for clients until we figure out what to do there...
-            builder.put(prefix + "ssl.keystore.password", password);
-        } else {
-            final String finalPrefix = prefix;
-            addSecureSettings(builder, secureSettings ->
-                secureSettings.setString(finalPrefix + "ssl.keystore.secure_password", password));
-        }
+        final String finalPrefix = prefix;
+        addSecureSettings(builder, secureSettings ->
+            secureSettings.setString(finalPrefix + "ssl.keystore.secure_password", password));
 
         if (randomBoolean()) {
             builder.put(prefix + "ssl.truststore.path", store);
-            if (transportClient) {
-                // continue using insecure settings for clients until we figure out what to do there...
-                builder.put(prefix + "ssl.truststore.password", password);
-            } else {
-                final String finalPrefix = prefix;
-                addSecureSettings(builder, secureSettings ->
-                    secureSettings.setString(finalPrefix + "ssl.truststore.secure_password", password));
-            }
+            addSecureSettings(builder, secureSettings ->
+                secureSettings.setString(finalPrefix + "ssl.truststore.secure_password", password));
         }
     }
 
@@ -304,7 +269,7 @@ private static void addSSLSettingsForStore(Settings.Builder builder, String pref
      */
     public static void addSSLSettingsForPEMFiles(Settings.Builder builder, String keyPath, String password,
                                                  String certificatePath, List<String> trustedCertificates) {
-        addSSLSettingsForPEMFiles(builder, "", keyPath, password, certificatePath, trustedCertificates, true, true);
+        addSSLSettingsForPEMFiles(builder, "", keyPath, password, certificatePath, trustedCertificates, true);
     }
 
     /**
@@ -321,26 +286,21 @@ public static void addSSLSettingsForPEMFiles(Settings.Builder builder, String ke
      */
     public static void addSSLSettingsForPEMFiles(Settings.Builder builder, String keyPath, String password,
                                                  String certificatePath, String prefix, List<String> trustedCertificates) {
-        addSSLSettingsForPEMFiles(builder, prefix, keyPath, password, certificatePath, trustedCertificates, true, true);
+        addSSLSettingsForPEMFiles(builder, prefix, keyPath, password, certificatePath, trustedCertificates, true);
     }
 
     private static void addSSLSettingsForPEMFiles(Settings.Builder builder, String prefix, String keyPath, String password,
                                                   String certificatePath, List<String> trustedCertificates,
-                                                  boolean hostnameVerificationEnabled, boolean transportClient) {
+                                                  boolean hostnameVerificationEnabled) {
         if (prefix.equals("")) {
             prefix = "xpack.security.transport.";
         }
         builder.put(prefix + "ssl.verification_mode", hostnameVerificationEnabled ? "full" : "certificate");
         builder.put(prefix + "ssl.key", resolveResourcePath(keyPath))
                 .put(prefix + "ssl.certificate", resolveResourcePath(certificatePath));
-        if (transportClient) {
-            // continue using insecure settings for clients until we figure out what to do there...
-            builder.put(prefix + "ssl.key_passphrase", password);
-        } else {
-            final String finalPrefix = prefix;
-            addSecureSettings(builder, secureSettings ->
-                secureSettings.setString(finalPrefix + "ssl.secure_key_passphrase", password));
-        }
+        final String finalPrefix = prefix;
+        addSecureSettings(builder, secureSettings ->
+            secureSettings.setString(finalPrefix + "ssl.secure_key_passphrase", password));
 
         if (trustedCertificates.isEmpty() == false) {
             builder.put(prefix + "ssl.certificate_authorities",
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySingleNodeTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySingleNodeTestCase.java
index 3c1834c84fea4..7471d96789cf4 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySingleNodeTestCase.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySingleNodeTestCase.java
@@ -179,12 +179,6 @@ protected Settings nodeSettings() {
         return builder.build();
     }
 
-    protected Settings transportClientSettings() {
-        return Settings.builder()
-                .put(customSecuritySettingsSource.transportClientSettings())
-                .build();
-    }
-
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return customSecuritySettingsSource.nodePlugins();
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilderTests.java
index 2ece398d3d19f..98bae94a33a72 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilderTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/core/security/action/privilege/PutPrivilegesRequestBuilderTests.java
@@ -23,7 +23,7 @@
 public class PutPrivilegesRequestBuilderTests extends ESTestCase {
 
     public void testBuildRequestWithMultipleElements() throws Exception {
-        final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null, PutPrivilegesAction.INSTANCE);
+        final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null);
         builder.source(new BytesArray("{ "
                 + "\"foo\":{"
                 + "  \"read\":{ \"application\":\"foo\", \"name\":\"read\", \"actions\":[ \"data:/read/*\", \"admin:/read/*\" ] },"
@@ -53,7 +53,7 @@ private ApplicationPrivilegeDescriptor descriptor(String app, String name, Strin
     }
 
     public void testPrivilegeNameValidationOfMultipleElement() throws Exception {
-        final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null, PutPrivilegesAction.INSTANCE);
+        final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null);
         final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
                 builder.source(new BytesArray("{ \"foo\":{"
                         + "\"write\":{ \"application\":\"foo\", \"name\":\"read\", \"actions\":[\"data:/read/*\",\"admin:/read/*\"] },"
@@ -65,7 +65,7 @@ public void testPrivilegeNameValidationOfMultipleElement() throws Exception {
     }
 
     public void testApplicationNameValidationOfMultipleElement() throws Exception {
-        final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null, PutPrivilegesAction.INSTANCE);
+        final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null);
         final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
                 builder.source(new BytesArray("{ \"bar\":{"
                         + "\"read\":{ \"application\":\"foo\", \"name\":\"read\", \"actions\":[ \"data:/read/*\", \"admin:/read/*\" ] },"
@@ -78,7 +78,7 @@ public void testApplicationNameValidationOfMultipleElement() throws Exception {
     }
 
     public void testInferApplicationNameAndPrivilegeName() throws Exception {
-        final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null, PutPrivilegesAction.INSTANCE);
+        final PutPrivilegesRequestBuilder builder = new PutPrivilegesRequestBuilder(null);
         builder.source(new BytesArray("{ \"foo\":{"
             + "\"read\":{ \"actions\":[ \"data:/read/*\", \"admin:/read/*\" ] },"
             + "\"write\":{ \"actions\":[ \"data:/write/*\", \"admin:/*\" ] },"
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalStateSecurity.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalStateSecurity.java
index 686c4ae12f1fc..c81c418bb4a36 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalStateSecurity.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/LocalStateSecurity.java
@@ -5,17 +5,61 @@
  */
 package org.elasticsearch.xpack.security;
 
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackInfoRequest;
+import org.elasticsearch.protocol.xpack.XPackInfoResponse;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+import org.elasticsearch.xpack.core.action.TransportXPackInfoAction;
+import org.elasticsearch.xpack.core.action.TransportXPackUsageAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageResponse;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.monitoring.Monitoring;
 
 import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
 
 public class LocalStateSecurity extends LocalStateCompositeXPackPlugin {
 
+    public static class SecurityTransportXPackUsageAction extends TransportXPackUsageAction {
+        @Inject
+        public SecurityTransportXPackUsageAction(ThreadPool threadPool, TransportService transportService,
+                                                   ClusterService clusterService, ActionFilters actionFilters,
+                                                   IndexNameExpressionResolver indexNameExpressionResolver, NodeClient client) {
+            super(threadPool, transportService, clusterService, actionFilters, indexNameExpressionResolver, client);
+        }
+        @Override
+        protected List<XPackUsageFeatureAction> usageActions() {
+            return Collections.singletonList(XPackUsageFeatureAction.SECURITY);
+        }
+    }
+
+    public static class SecurityTransportXPackInfoAction extends TransportXPackInfoAction {
+        @Inject
+        public SecurityTransportXPackInfoAction(TransportService transportService, ActionFilters actionFilters,
+                                                 LicenseService licenseService, NodeClient client) {
+            super(transportService, actionFilters, licenseService, client);
+        }
+
+        @Override
+        protected List<XPackInfoFeatureAction> infoActions() {
+            return Collections.singletonList(XPackInfoFeatureAction.SECURITY);
+        }
+    }
+
     public LocalStateSecurity(final Settings settings, final Path configPath) throws Exception {
         super(settings, configPath);
         LocalStateSecurity thisVar = this;
@@ -43,4 +87,14 @@ protected XPackLicenseState getLicenseState() {
             protected XPackLicenseState getLicenseState() { return thisVar.getLicenseState(); }
         });
     }
+
+    @Override
+    protected Class<? extends TransportAction<XPackUsageRequest, XPackUsageResponse>> getUsageAction() {
+        return SecurityTransportXPackUsageAction.class;
+    }
+
+    @Override
+    protected Class<? extends TransportAction<XPackInfoRequest, XPackInfoResponse>> getInfoAction() {
+        return SecurityTransportXPackInfoAction.class;
+    }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/PkiRealmBootstrapCheckTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/PkiRealmBootstrapCheckTests.java
index 2c62ce71d45a9..ec3e0a5132dfd 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/PkiRealmBootstrapCheckTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/PkiRealmBootstrapCheckTests.java
@@ -58,7 +58,7 @@ public void testBootstrapCheckWithPkiRealm() throws Exception {
         env = TestEnvironment.newEnvironment(settings);
         assertTrue(runCheck(settings, env).isFailure());
 
-        // set transport client auth
+        // set transport auth
         settings = Settings.builder().put(settings)
                 .put("xpack.security.transport.client_authentication", randomFrom("required", "optional"))
                 .build();
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityFeatureSetTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java
similarity index 88%
rename from x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityFeatureSetTests.java
rename to x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java
index 9bfa0b9a886f3..27b3c1ef7cc3f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityFeatureSetTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.security;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -15,9 +16,11 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.security.SecurityFeatureSetUsage;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
@@ -44,7 +47,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class SecurityFeatureSetTests extends ESTestCase {
+public class SecurityInfoTransportActionTests extends ESTestCase {
 
     private Settings settings;
     private XPackLicenseState licenseState;
@@ -52,6 +55,7 @@ public class SecurityFeatureSetTests extends ESTestCase {
     private IPFilter ipFilter;
     private CompositeRolesStore rolesStore;
     private NativeRoleMappingStore roleMappingStore;
+    private SecurityUsageServices securityServices;
 
     @Before
     public void init() throws Exception {
@@ -61,11 +65,12 @@ public void init() throws Exception {
         ipFilter = mock(IPFilter.class);
         rolesStore = mock(CompositeRolesStore.class);
         roleMappingStore = mock(NativeRoleMappingStore.class);
+        securityServices = new SecurityUsageServices(realms, rolesStore, roleMappingStore, ipFilter);
     }
 
     public void testAvailable() {
-        SecurityFeatureSet featureSet = new SecurityFeatureSet(settings, licenseState, realms,
-                rolesStore, roleMappingStore, ipFilter);
+        SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), settings, licenseState);
         when(licenseState.isSecurityAvailable()).thenReturn(true);
         assertThat(featureSet.available(), is(true));
 
@@ -74,13 +79,13 @@ public void testAvailable() {
     }
 
     public void testEnabled() {
-        SecurityFeatureSet featureSet = new SecurityFeatureSet(settings, licenseState, realms,
-                rolesStore, roleMappingStore, ipFilter);
+        SecurityInfoTransportAction featureSet = new SecurityInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), settings, licenseState);
         assertThat(featureSet.enabled(), is(true));
 
         when(licenseState.isSecurityDisabledByLicenseDefaults()).thenReturn(true);
-        featureSet = new SecurityFeatureSet(settings, licenseState, realms,
-                rolesStore, roleMappingStore, ipFilter);
+        featureSet = new SecurityInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), settings, licenseState);
         assertThat(featureSet.enabled(), is(false));
     }
 
@@ -147,11 +152,10 @@ public void testUsage() throws Exception {
             settings.put(AnonymousUser.ROLES_SETTING.getKey(), "foo");
         }
 
-        SecurityFeatureSet featureSet = new SecurityFeatureSet(settings.build(), licenseState,
-                realms, rolesStore, roleMappingStore, ipFilter);
-        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
-        featureSet.usage(future);
-        XPackFeatureSet.Usage securityUsage = future.get();
+        var usageAction = newUsageAction(settings.build());
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, null, future);
+        SecurityFeatureSetUsage securityUsage = (SecurityFeatureSetUsage) future.get().getUsage();
         BytesStreamOutput out = new BytesStreamOutput();
         securityUsage.writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new SecurityFeatureSetUsage(out.bytes().streamInput());
@@ -248,11 +252,10 @@ public void testUsageOnTrialLicenseWithSecurityDisabledByDefault() throws Except
 
         configureRealmsUsage(Collections.emptyMap());
 
-        SecurityFeatureSet featureSet = new SecurityFeatureSet(settings.build(), licenseState,
-                realms, rolesStore, roleMappingStore, ipFilter);
-        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
-        featureSet.usage(future);
-        XPackFeatureSet.Usage securityUsage = future.get();
+        var usageAction = newUsageAction(settings.build());
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, null, future);
+        SecurityFeatureSetUsage securityUsage = (SecurityFeatureSetUsage) future.get().getUsage();
         BytesStreamOutput out = new BytesStreamOutput();
         securityUsage.writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new SecurityFeatureSetUsage(out.bytes().streamInput());
@@ -322,4 +325,10 @@ private void configureRoleMappingStoreUsage(boolean roleMappingStoreEnabled) {
             return Void.TYPE;
         }).when(roleMappingStore).usageStats(any(ActionListener.class));
     }
+
+    private SecurityUsageTransportAction newUsageAction(Settings settings) {
+        return new SecurityUsageTransportAction(mock(TransportService.class),null,
+            null, mock(ActionFilters.class),null,
+            settings, licenseState, securityServices);
+    }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
index 768bc38813c0b..4d8dca8e095a8 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java
@@ -65,7 +65,6 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING;
-import static org.elasticsearch.discovery.DiscoveryModule.ZEN2_DISCOVERY_TYPE;
 import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT;
 import static org.hamcrest.Matchers.containsString;
@@ -243,61 +242,6 @@ public void testJoinValidatorOnDisabledSecurity() throws Exception {
         assertNull(joinValidator);
     }
 
-    public void testTLSJoinValidator() throws Exception {
-        createComponents(Settings.EMPTY);
-        BiConsumer<DiscoveryNode, ClusterState> joinValidator = security.getJoinValidator();
-        assertNotNull(joinValidator);
-        DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT);
-        joinValidator.accept(node, ClusterState.builder(ClusterName.DEFAULT).build());
-        int numIters = randomIntBetween(1, 10);
-        for (int i = 0; i < numIters; i++) {
-            boolean tlsOn = randomBoolean();
-            boolean securityExplicitlyEnabled = randomBoolean();
-            String discoveryType = randomFrom("single-node", ZEN2_DISCOVERY_TYPE, randomAlphaOfLength(4));
-
-            final Settings settings;
-            if (securityExplicitlyEnabled) {
-                settings = Settings.builder().put("xpack.security.enabled", true).build();
-            } else {
-                settings = Settings.EMPTY;
-            }
-            Security.ValidateTLSOnJoin validator = new Security.ValidateTLSOnJoin(tlsOn, discoveryType, settings);
-            MetaData.Builder builder = MetaData.builder();
-            License.OperationMode licenseMode = randomFrom(License.OperationMode.values());
-            License license = TestUtils.generateSignedLicense(licenseMode.description(), TimeValue.timeValueHours(24));
-            TestUtils.putLicense(builder, license);
-            ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(builder.build()).build();
-
-            final boolean expectFailure;
-            switch (licenseMode) {
-                case PLATINUM:
-                case GOLD:
-                case STANDARD:
-                    expectFailure = tlsOn == false && "single-node".equals(discoveryType) == false;
-                    break;
-                case BASIC:
-                    expectFailure = tlsOn == false && "single-node".equals(discoveryType) == false && securityExplicitlyEnabled;
-                    break;
-                case MISSING:
-                case TRIAL:
-                    expectFailure = false;
-                    break;
-                default:
-                    throw new AssertionError("unknown operation mode [" + license.operationMode() + "]");
-            }
-            logger.info("Test TLS join; Lic:{} TLS:{} Disco:{} Settings:{}  ; Expect Failure: {}",
-                licenseMode, tlsOn, discoveryType, settings.toDelimitedString(','), expectFailure);
-            if (expectFailure) {
-                IllegalStateException ise = expectThrows(IllegalStateException.class, () -> validator.accept(node, state));
-                assertEquals("Transport TLS ([xpack.security.transport.ssl.enabled]) is required for license type ["
-                    + license.operationMode().description() + "] when security is enabled", ise.getMessage());
-            } else {
-                validator.accept(node, state);
-            }
-            validator.accept(node, ClusterState.builder(ClusterName.DEFAULT).metaData(MetaData.builder().build()).build());
-        }
-    }
-
     public void testJoinValidatorForFIPSLicense() throws Exception {
         DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(),
             VersionUtils.randomVersionBetween(random(), null, Version.CURRENT));
@@ -318,29 +262,6 @@ public void testJoinValidatorForFIPSLicense() throws Exception {
         }
     }
 
-    public void testIndexJoinValidator_Old_And_Rolling() throws Exception {
-        createComponents(Settings.EMPTY);
-        BiConsumer<DiscoveryNode, ClusterState> joinValidator = security.getJoinValidator();
-        assertNotNull(joinValidator);
-        Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(),
-            VersionUtils.getPreviousVersion(Version.V_7_0_0));
-        DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT);
-        IndexMetaData indexMetaData = IndexMetaData.builder(SECURITY_MAIN_ALIAS)
-            .settings(settings(version)
-                .put(INDEX_FORMAT_SETTING.getKey(), INTERNAL_MAIN_INDEX_FORMAT - 1))
-            .numberOfShards(1).numberOfReplicas(0)
-            .build();
-        DiscoveryNode existingOtherNode = new DiscoveryNode("bar", buildNewFakeTransportAddress(), version);
-        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(existingOtherNode).build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
-            .nodes(discoveryNodes)
-            .metaData(MetaData.builder().put(indexMetaData, true).build()).build();
-        IllegalStateException e = expectThrows(IllegalStateException.class,
-            () -> joinValidator.accept(node, clusterState));
-        assertThat(e.getMessage(), equalTo("Security index is not on the current version [6] - " +
-            "The Upgrade API must be run for 7.x nodes to join the cluster"));
-    }
-
     public void testIndexJoinValidator_FullyCurrentCluster() throws Exception {
         createComponents(Settings.EMPTY);
         BiConsumer<DiscoveryNode, ClusterState> joinValidator = security.getJoinValidator();
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/SecurityActionMapperTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/SecurityActionMapperTests.java
index 6efb293f7b201..ef063c93961e4 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/SecurityActionMapperTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/SecurityActionMapperTests.java
@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.security.action;
 
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
-import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.action.search.ClearScrollAction;
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.test.ESTestCase;
@@ -73,11 +72,11 @@ public void testClearScrollAll() {
 
     public void testIndicesAnalyze() {
         SecurityActionMapper securityActionMapper = new SecurityActionMapper();
-        AnalyzeRequest analyzeRequest;
+        AnalyzeAction.Request analyzeRequest;
         if (randomBoolean()) {
-            analyzeRequest = new AnalyzeRequest(randomAlphaOfLength(randomIntBetween(1, 30))).text("text");
+            analyzeRequest = new AnalyzeAction.Request(randomAlphaOfLength(randomIntBetween(1, 30))).text("text");
         } else {
-            analyzeRequest = new AnalyzeRequest(null).text("text");
+            analyzeRequest = new AnalyzeAction.Request(null).text("text");
             analyzeRequest.index(randomAlphaOfLength(randomIntBetween(1, 30)));
         }
         assertThat(securityActionMapper.action(AnalyzeAction.NAME, analyzeRequest), equalTo(AnalyzeAction.NAME));
@@ -85,7 +84,7 @@ public void testIndicesAnalyze() {
 
     public void testClusterAnalyze() {
         SecurityActionMapper securityActionMapper = new SecurityActionMapper();
-        AnalyzeRequest analyzeRequest = new AnalyzeRequest(null).text("text");
+        AnalyzeAction.Request analyzeRequest = new AnalyzeAction.Request(null).text("text");
         assertThat(securityActionMapper.action(AnalyzeAction.NAME, analyzeRequest),
                 equalTo(SecurityActionMapper.CLUSTER_PERMISSION_ANALYZE));
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutActionTests.java
index 0ab3c96167c2c..ba6da8e31eefb 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutActionTests.java
@@ -193,7 +193,7 @@ public void setup() throws Exception {
     public void testLogoutInvalidatesTokens() throws Exception {
         final String subject = randomAlphaOfLength(8);
         final JWT signedIdToken = generateIdToken(subject, randomAlphaOfLength(8), randomAlphaOfLength(8));
-        final User user = new User("oidc-user", new String[]{"superuser"}, null, null, null, true);
+        final User user = new User("oidc-user", new String[]{"superuser"}, null, null, Map.of(), true);
         final Authentication.RealmRef realmRef = new Authentication.RealmRef(oidcRealm.name(), OpenIdConnectRealmSettings.TYPE, "node01");
         final Map<String, Object> tokenMetadata = new HashMap<>();
         tokenMetadata.put("id_token_hint", signedIdToken.serialize());
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/PutRoleMappingRequestTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/PutRoleMappingRequestTests.java
index 3079a17b77424..bb12a9872169c 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/PutRoleMappingRequestTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/PutRoleMappingRequestTests.java
@@ -5,18 +5,17 @@
  */
 package org.elasticsearch.xpack.security.action.rolemapping;
 
-import java.util.Collections;
-
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingAction;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequestBuilder;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.RoleMapperExpression;
 import org.junit.Before;
 import org.mockito.Mockito;
 
+import java.util.Collections;
+
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -27,7 +26,7 @@ public class PutRoleMappingRequestTests extends ESTestCase {
     @Before
     public void setupBuilder() {
         final ElasticsearchClient client = Mockito.mock(ElasticsearchClient.class);
-        builder = new PutRoleMappingRequestBuilder(client, PutRoleMappingAction.INSTANCE);
+        builder = new PutRoleMappingRequestBuilder(client);
     }
 
     public void testValidateMissingName() throws Exception {
@@ -70,4 +69,4 @@ private void assertValidationFailure(PutRoleMappingRequest request, String expec
         assertThat(ve.getMessage(), containsString(expectedMessage));
     }
 
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java
index 6a9c487bf2013..c5ed4365effcd 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java
@@ -7,7 +7,7 @@
 
 import org.apache.lucene.search.TotalHits;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -143,7 +143,7 @@ public void setup() throws Exception {
         final Client client = new NoOpClient(threadPool) {
             @Override
             protected <Request extends ActionRequest, Response extends ActionResponse>
-            void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+            void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
                 if (IndexAction.NAME.equals(action.name())) {
                     assertThat(request, instanceOf(IndexRequest.class));
                     IndexRequest indexRequest = (IndexRequest) request;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java
index c2050c95fc354..b32ceee1a87d9 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenActionTests.java
@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.security.action.token;
 
+import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.get.GetAction;
 import org.elasticsearch.action.get.GetRequestBuilder;
@@ -31,6 +32,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.node.Node;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -41,20 +43,25 @@
 import org.elasticsearch.xpack.core.security.action.token.CreateTokenRequest;
 import org.elasticsearch.xpack.core.security.action.token.CreateTokenResponse;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
 import org.elasticsearch.xpack.security.authc.TokenService;
+import org.elasticsearch.xpack.security.authc.kerberos.KerberosAuthenticationToken;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.junit.After;
 import org.junit.Before;
 
+import java.nio.charset.StandardCharsets;
 import java.time.Clock;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 
+import static org.hamcrest.Matchers.is;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Matchers.eq;
@@ -128,15 +135,31 @@ public void setupClient() {
         }).when(securityIndex).prepareIndexIfNeededThenExecute(any(Consumer.class), any(Runnable.class));
 
         doAnswer(invocationOnMock -> {
-            UsernamePasswordToken token = (UsernamePasswordToken) invocationOnMock.getArguments()[2];
-            User user = new User(token.principal());
+            AuthenticationToken authToken = (AuthenticationToken) invocationOnMock.getArguments()[2];
+            ActionListener<Authentication> authListener = (ActionListener<Authentication>) invocationOnMock.getArguments()[3];
+            User user = null;
+            if (authToken instanceof UsernamePasswordToken) {
+                UsernamePasswordToken token = (UsernamePasswordToken) invocationOnMock.getArguments()[2];
+                user = new User(token.principal());
+            } else if (authToken instanceof KerberosAuthenticationToken) {
+                KerberosAuthenticationToken token = (KerberosAuthenticationToken) invocationOnMock.getArguments()[2];
+                if (token.credentials() instanceof byte[]
+                        && new String((byte[]) token.credentials(), StandardCharsets.UTF_8).equals("fail")) {
+                    String errorMessage = "failed to authenticate user, gss context negotiation not complete";
+                    ElasticsearchSecurityException ese = new ElasticsearchSecurityException(errorMessage, RestStatus.UNAUTHORIZED);
+                    ese.addHeader(KerberosAuthenticationToken.WWW_AUTHENTICATE, "Negotiate FAIL");
+                    authListener.onFailure(ese);
+                    return Void.TYPE;
+                }
+                user = new User(token.principal());
+                threadPool.getThreadContext().addResponseHeader(KerberosAuthenticationToken.WWW_AUTHENTICATE, "Negotiate SUCCESS");
+            }
             Authentication authentication = new Authentication(user, new Authentication.RealmRef("fake", "mock", "n1"), null);
             authentication.writeToContext(threadPool.getThreadContext());
-            ActionListener<Authentication> authListener = (ActionListener<Authentication>) invocationOnMock.getArguments()[3];
             authListener.onResponse(authentication);
             return Void.TYPE;
         }).when(authenticationService).authenticate(eq(CreateTokenAction.NAME), any(CreateTokenRequest.class),
-            any(UsernamePasswordToken.class), any(ActionListener.class));
+            any(AuthenticationToken.class), any(ActionListener.class));
 
         this.clusterService = ClusterServiceUtils.createClusterService(threadPool);
 
@@ -202,4 +225,41 @@ public void testPasswordGrantTypeCreatesWithRefreshToken() throws Exception {
         assertNotNull(sourceMap.get("access_token"));
         assertNotNull(sourceMap.get("refresh_token"));
     }
+
+    public void testKerberosGrantTypeCreatesWithRefreshToken() throws Exception {
+        final TokenService tokenService = new TokenService(SETTINGS, Clock.systemUTC(), client, license,
+                securityIndex, securityIndex, clusterService);
+        Authentication authentication = new Authentication(new User("joe"), new Authentication.RealmRef("realm", "type", "node"), null);
+        authentication.writeToContext(threadPool.getThreadContext());
+
+        final TransportCreateTokenAction action = new TransportCreateTokenAction(threadPool,
+            mock(TransportService.class), new ActionFilters(Collections.emptySet()), tokenService,
+            authenticationService);
+        final CreateTokenRequest createTokenRequest = new CreateTokenRequest();
+        createTokenRequest.setGrantType("_kerberos");
+        String failOrSuccess = randomBoolean() ? "fail" : "success";
+        String kerbCredentialsBase64 = Base64.getEncoder().encodeToString(failOrSuccess.getBytes(StandardCharsets.UTF_8));
+        createTokenRequest.setKerberosTicket(new SecureString(kerbCredentialsBase64.toCharArray()));
+
+        PlainActionFuture<CreateTokenResponse> tokenResponseFuture = new PlainActionFuture<>();
+        action.doExecute(null, createTokenRequest, tokenResponseFuture);
+        if (failOrSuccess.equals("fail")) {
+            ElasticsearchSecurityException ese = expectThrows(ElasticsearchSecurityException.class, () -> tokenResponseFuture.actionGet());
+            assertNotNull(ese.getHeader(KerberosAuthenticationToken.WWW_AUTHENTICATE));
+            assertThat(ese.getHeader(KerberosAuthenticationToken.WWW_AUTHENTICATE).size(), is(1));
+            assertThat(ese.getHeader(KerberosAuthenticationToken.WWW_AUTHENTICATE).get(0), is("Negotiate FAIL"));
+        } else {
+            CreateTokenResponse createTokenResponse = tokenResponseFuture.get();
+            assertNotNull(createTokenResponse.getRefreshToken());
+            assertNotNull(createTokenResponse.getTokenString());
+            assertNotNull(createTokenResponse.getKerberosAuthenticationResponseToken());
+            assertThat(createTokenResponse.getKerberosAuthenticationResponseToken(), is("SUCCESS"));
+
+            assertNotNull(idxReqReference.get());
+            Map<String, Object> sourceMap = idxReqReference.get().sourceAsMap();
+            assertNotNull(sourceMap);
+            assertNotNull(sourceMap.get("access_token"));
+            assertNotNull(sourceMap.get("refresh_token"));
+        }
+    }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java
index 0aaf9db7f90d3..8ec1f9650185c 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java
@@ -613,7 +613,7 @@ public void testAccessDenied() throws Exception {
         indicesRequest(message, checkedFields, checkedArrayFields);
         opaqueId(threadContext, checkedFields);
         forwardedFor(threadContext, checkedFields);
-        
+
         assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap());
 
         // test disabled
@@ -1075,7 +1075,9 @@ private void assertMsg(Logger logger, Map<String, String> checkFields, Map<Strin
                 logLine = logEntryFieldPattern.matcher(logLine).replaceFirst("");
             }
         }
-        logLine = logLine.replaceFirst("\"@timestamp\":\"[^\"]*\"", "").replaceAll("[{},]", "");
+        logLine = logLine.replaceFirst("\"" + LoggingAuditTrail.LOG_TYPE + "\":\"audit\", ", "")
+                         .replaceFirst("\"" + LoggingAuditTrail.TIMESTAMP + "\":\"[^\"]*\"", "")
+                         .replaceAll("[{},]", "");
         // check no extra fields
         assertThat("Log event has extra unexpected content: " + logLine, Strings.hasText(logLine), is(false));
     }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java
index c8cea45037942..5c6c04b6ad491 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java
@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.security.authc;
 
 import com.google.common.collect.Sets;
-
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
@@ -26,14 +25,16 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.action.ApiKey;
+import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
+import org.elasticsearch.xpack.core.security.action.GetApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.GetApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.GetApiKeyResponse;
+import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyAction;
 import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyResponse;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 import org.junit.After;
 import org.junit.Before;
@@ -100,8 +101,7 @@ public void testCreateApiKey() {
         Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
             UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER,
                 SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
-        final CreateApiKeyResponse response = securityClient.prepareCreateApiKey()
+        final CreateApiKeyResponse response = new CreateApiKeyRequestBuilder(client)
             .setName("test key")
             .setExpiration(TimeValue.timeValueHours(TimeUnit.DAYS.toHours(7L)))
             .setRoleDescriptors(Collections.singletonList(descriptor))
@@ -115,7 +115,7 @@ public void testCreateApiKey() {
         assertThat(daysBetween, is(7L));
 
         // create simple api key
-        final CreateApiKeyResponse simple = securityClient.prepareCreateApiKey().setName("simple").get();
+        final CreateApiKeyResponse simple = new CreateApiKeyRequestBuilder(client).setName("simple").get();
         assertEquals("simple", simple.getName());
         assertNotNull(simple.getId());
         assertNotNull(simple.getKey());
@@ -151,8 +151,7 @@ public void testCreateApiKeyFailsWhenApiKeyWithSameNameAlreadyExists() throws In
             final RoleDescriptor descriptor = new RoleDescriptor("role", new String[] { "monitor" }, null, null);
             Client client = client().filterWithHeader(Collections.singletonMap("Authorization", UsernamePasswordToken
                     .basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-            SecurityClient securityClient = new SecurityClient(client);
-            final CreateApiKeyResponse response = securityClient.prepareCreateApiKey().setName(keyName).setExpiration(null)
+            final CreateApiKeyResponse response = new CreateApiKeyRequestBuilder(client).setName(keyName).setExpiration(null)
                     .setRoleDescriptors(Collections.singletonList(descriptor)).get();
             assertNotNull(response.getId());
             assertNotNull(response.getKey());
@@ -163,8 +162,7 @@ public void testCreateApiKeyFailsWhenApiKeyWithSameNameAlreadyExists() throws In
         Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
             UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER,
                 SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
-        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> securityClient.prepareCreateApiKey()
+        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> new CreateApiKeyRequestBuilder(client)
             .setName(keyName)
             .setExpiration(TimeValue.timeValueHours(TimeUnit.DAYS.toHours(7L)))
             .setRoleDescriptors(Collections.singletonList(descriptor))
@@ -173,12 +171,12 @@ public void testCreateApiKeyFailsWhenApiKeyWithSameNameAlreadyExists() throws In
 
         // Now invalidate the API key
         PlainActionFuture<InvalidateApiKeyResponse> listener = new PlainActionFuture<>();
-        securityClient.invalidateApiKey(InvalidateApiKeyRequest.usingApiKeyName(keyName), listener);
+        client.execute(InvalidateApiKeyAction.INSTANCE, InvalidateApiKeyRequest.usingApiKeyName(keyName), listener);
         InvalidateApiKeyResponse invalidateResponse = listener.get();
         verifyInvalidateResponse(1, responses, invalidateResponse);
 
         // try to create API key with same name, should succeed now
-        CreateApiKeyResponse createResponse = securityClient.prepareCreateApiKey().setName(keyName)
+        CreateApiKeyResponse createResponse = new CreateApiKeyRequestBuilder(client).setName(keyName)
                 .setExpiration(TimeValue.timeValueHours(TimeUnit.DAYS.toHours(7L)))
                 .setRoleDescriptors(Collections.singletonList(descriptor)).get();
         assertNotNull(createResponse.getId());
@@ -190,9 +188,8 @@ public void testInvalidateApiKeysForRealm() throws InterruptedException, Executi
         List<CreateApiKeyResponse> responses = createApiKeys(noOfApiKeys, null);
         Client client = client().filterWithHeader(Collections.singletonMap("Authorization", UsernamePasswordToken
                 .basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
         PlainActionFuture<InvalidateApiKeyResponse> listener = new PlainActionFuture<>();
-        securityClient.invalidateApiKey(InvalidateApiKeyRequest.usingRealmName("file"), listener);
+        client.execute(InvalidateApiKeyAction.INSTANCE, InvalidateApiKeyRequest.usingRealmName("file"), listener);
         InvalidateApiKeyResponse invalidateResponse = listener.get();
         verifyInvalidateResponse(noOfApiKeys, responses, invalidateResponse);
     }
@@ -202,9 +199,9 @@ public void testInvalidateApiKeysForUser() throws Exception {
         List<CreateApiKeyResponse> responses = createApiKeys(noOfApiKeys, null);
         Client client = client().filterWithHeader(Collections.singletonMap("Authorization", UsernamePasswordToken
                 .basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
         PlainActionFuture<InvalidateApiKeyResponse> listener = new PlainActionFuture<>();
-        securityClient.invalidateApiKey(InvalidateApiKeyRequest.usingUserName(SecuritySettingsSource.TEST_SUPERUSER), listener);
+        client.execute(InvalidateApiKeyAction.INSTANCE,
+            InvalidateApiKeyRequest.usingUserName(SecuritySettingsSource.TEST_SUPERUSER), listener);
         InvalidateApiKeyResponse invalidateResponse = listener.get();
         verifyInvalidateResponse(noOfApiKeys, responses, invalidateResponse);
     }
@@ -213,10 +210,9 @@ public void testInvalidateApiKeysForRealmAndUser() throws InterruptedException,
         List<CreateApiKeyResponse> responses = createApiKeys(1, null);
         Client client = client().filterWithHeader(Collections.singletonMap("Authorization", UsernamePasswordToken
                 .basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
         PlainActionFuture<InvalidateApiKeyResponse> listener = new PlainActionFuture<>();
-        securityClient.invalidateApiKey(InvalidateApiKeyRequest.usingRealmAndUserName("file", SecuritySettingsSource.TEST_SUPERUSER),
-                listener);
+        client.execute(InvalidateApiKeyAction.INSTANCE,
+            InvalidateApiKeyRequest.usingRealmAndUserName("file", SecuritySettingsSource.TEST_SUPERUSER), listener);
         InvalidateApiKeyResponse invalidateResponse = listener.get();
         verifyInvalidateResponse(1, responses, invalidateResponse);
     }
@@ -225,9 +221,8 @@ public void testInvalidateApiKeysForApiKeyId() throws InterruptedException, Exec
         List<CreateApiKeyResponse> responses = createApiKeys(1, null);
         Client client = client().filterWithHeader(Collections.singletonMap("Authorization", UsernamePasswordToken
                 .basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
         PlainActionFuture<InvalidateApiKeyResponse> listener = new PlainActionFuture<>();
-        securityClient.invalidateApiKey(InvalidateApiKeyRequest.usingApiKeyId(responses.get(0).getId()), listener);
+        client.execute(InvalidateApiKeyAction.INSTANCE, InvalidateApiKeyRequest.usingApiKeyId(responses.get(0).getId()), listener);
         InvalidateApiKeyResponse invalidateResponse = listener.get();
         verifyInvalidateResponse(1, responses, invalidateResponse);
     }
@@ -236,9 +231,8 @@ public void testInvalidateApiKeysForApiKeyName() throws InterruptedException, Ex
         List<CreateApiKeyResponse> responses = createApiKeys(1, null);
         Client client = client().filterWithHeader(Collections.singletonMap("Authorization", UsernamePasswordToken
                 .basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
         PlainActionFuture<InvalidateApiKeyResponse> listener = new PlainActionFuture<>();
-        securityClient.invalidateApiKey(InvalidateApiKeyRequest.usingApiKeyName(responses.get(0).getName()), listener);
+        client.execute(InvalidateApiKeyAction.INSTANCE, InvalidateApiKeyRequest.usingApiKeyName(responses.get(0).getName()), listener);
         InvalidateApiKeyResponse invalidateResponse = listener.get();
         verifyInvalidateResponse(1, responses, invalidateResponse);
     }
@@ -259,27 +253,24 @@ public void testInvalidatedApiKeysDeletedByRemover() throws Exception {
 
         List<CreateApiKeyResponse> createdApiKeys = createApiKeys(2, null);
 
-        SecurityClient securityClient = new SecurityClient(client);
-
         PlainActionFuture<InvalidateApiKeyResponse> listener = new PlainActionFuture<>();
-        securityClient.invalidateApiKey(InvalidateApiKeyRequest.usingApiKeyId(createdApiKeys.get(0).getId()), listener);
+        client.execute(InvalidateApiKeyAction.INSTANCE, InvalidateApiKeyRequest.usingApiKeyId(createdApiKeys.get(0).getId()), listener);
         InvalidateApiKeyResponse invalidateResponse = listener.get();
         assertThat(invalidateResponse.getInvalidatedApiKeys().size(), equalTo(1));
         assertThat(invalidateResponse.getPreviouslyInvalidatedApiKeys().size(), equalTo(0));
         assertThat(invalidateResponse.getErrors().size(), equalTo(0));
 
         PlainActionFuture<GetApiKeyResponse> getApiKeyResponseListener = new PlainActionFuture<>();
-        securityClient.getApiKey(GetApiKeyRequest.usingRealmName("file"), getApiKeyResponseListener);
+        client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingRealmName("file"), getApiKeyResponseListener);
         assertThat(getApiKeyResponseListener.get().getApiKeyInfos().length, is(2));
 
         client = waitForExpiredApiKeysRemoverTriggerReadyAndGetClient().filterWithHeader(
                 Collections.singletonMap("Authorization", UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER,
                         SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        securityClient = new SecurityClient(client);
 
         // invalidate API key to trigger remover
         listener = new PlainActionFuture<>();
-        securityClient.invalidateApiKey(InvalidateApiKeyRequest.usingApiKeyId(createdApiKeys.get(1).getId()), listener);
+        client.execute(InvalidateApiKeyAction.INSTANCE, InvalidateApiKeyRequest.usingApiKeyId(createdApiKeys.get(1).getId()), listener);
         assertThat(listener.get().getInvalidatedApiKeys().size(), is(1));
 
         awaitApiKeysRemoverCompletion();
@@ -288,7 +279,7 @@ public void testInvalidatedApiKeysDeletedByRemover() throws Exception {
 
         // Verify that 1st invalidated API key is deleted whereas the next one is not
         getApiKeyResponseListener = new PlainActionFuture<>();
-        securityClient.getApiKey(GetApiKeyRequest.usingRealmName("file"), getApiKeyResponseListener);
+        client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingRealmName("file"), getApiKeyResponseListener);
         assertThat(getApiKeyResponseListener.get().getApiKeyInfos().length, is(1));
         ApiKey apiKey = getApiKeyResponseListener.get().getApiKeyInfos()[0];
         assertThat(apiKey.getId(), is(createdApiKeys.get(1).getId()));
@@ -325,10 +316,8 @@ public void testExpiredApiKeysBehaviorWhenKeysExpired1WeekBeforeAnd1DayBefore()
         List<CreateApiKeyResponse> createdApiKeys = createApiKeys(noOfKeys, null);
         Instant created = Instant.now();
 
-        SecurityClient securityClient = new SecurityClient(client);
-
         PlainActionFuture<GetApiKeyResponse> getApiKeyResponseListener = new PlainActionFuture<>();
-        securityClient.getApiKey(GetApiKeyRequest.usingRealmName("file"), getApiKeyResponseListener);
+        client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingRealmName("file"), getApiKeyResponseListener);
         assertThat(getApiKeyResponseListener.get().getApiKeyInfos().length, is(noOfKeys));
 
         // Expire the 1st key such that it cannot be deleted by the remover
@@ -354,7 +343,7 @@ public void testExpiredApiKeysBehaviorWhenKeysExpired1WeekBeforeAnd1DayBefore()
 
         // Invalidate to trigger the remover
         PlainActionFuture<InvalidateApiKeyResponse> listener = new PlainActionFuture<>();
-        securityClient.invalidateApiKey(InvalidateApiKeyRequest.usingApiKeyId(createdApiKeys.get(2).getId()), listener);
+        client.execute(InvalidateApiKeyAction.INSTANCE, InvalidateApiKeyRequest.usingApiKeyId(createdApiKeys.get(2).getId()), listener);
         assertThat(listener.get().getInvalidatedApiKeys().size(), is(1));
 
         awaitApiKeysRemoverCompletion();
@@ -363,7 +352,7 @@ public void testExpiredApiKeysBehaviorWhenKeysExpired1WeekBeforeAnd1DayBefore()
 
         // Verify get API keys does not return expired and deleted key
         getApiKeyResponseListener = new PlainActionFuture<>();
-        securityClient.getApiKey(GetApiKeyRequest.usingRealmName("file"), getApiKeyResponseListener);
+        client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingRealmName("file"), getApiKeyResponseListener);
         assertThat(getApiKeyResponseListener.get().getApiKeyInfos().length, is(3));
 
         Set<String> expectedKeyIds = Sets.newHashSet(createdApiKeys.get(0).getId(), createdApiKeys.get(2).getId(),
@@ -400,17 +389,16 @@ public void testActiveApiKeysWithNoExpirationNeverGetDeletedByRemover() throws E
 
         Client client = client().filterWithHeader(Collections.singletonMap("Authorization", UsernamePasswordToken
                 .basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
         PlainActionFuture<InvalidateApiKeyResponse> listener = new PlainActionFuture<>();
         // trigger expired keys remover
-        securityClient.invalidateApiKey(InvalidateApiKeyRequest.usingApiKeyId(responses.get(1).getId()), listener);
+        client.execute(InvalidateApiKeyAction.INSTANCE, InvalidateApiKeyRequest.usingApiKeyId(responses.get(1).getId()), listener);
         InvalidateApiKeyResponse invalidateResponse = listener.get();
         assertThat(invalidateResponse.getInvalidatedApiKeys().size(), equalTo(1));
         assertThat(invalidateResponse.getPreviouslyInvalidatedApiKeys().size(), equalTo(0));
         assertThat(invalidateResponse.getErrors().size(), equalTo(0));
 
         PlainActionFuture<GetApiKeyResponse> getApiKeyResponseListener = new PlainActionFuture<>();
-        securityClient.getApiKey(GetApiKeyRequest.usingRealmName("file"), getApiKeyResponseListener);
+        client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingRealmName("file"), getApiKeyResponseListener);
         GetApiKeyResponse response = getApiKeyResponseListener.get();
         verifyGetResponse(2, responses, response, Collections.singleton(responses.get(0).getId()),
                 Collections.singletonList(responses.get(1).getId()));
@@ -421,13 +409,12 @@ public void testGetApiKeysForRealm() throws InterruptedException, ExecutionExcep
         List<CreateApiKeyResponse> responses = createApiKeys(noOfApiKeys, null);
         Client client = client().filterWithHeader(Collections.singletonMap("Authorization", UsernamePasswordToken
                 .basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
         boolean invalidate= randomBoolean();
         List<String> invalidatedApiKeyIds = null;
         Set<String> expectedValidKeyIds = null;
         if (invalidate) {
             PlainActionFuture<InvalidateApiKeyResponse> listener = new PlainActionFuture<>();
-            securityClient.invalidateApiKey(InvalidateApiKeyRequest.usingApiKeyId(responses.get(0).getId()), listener);
+            client.execute(InvalidateApiKeyAction.INSTANCE, InvalidateApiKeyRequest.usingApiKeyId(responses.get(0).getId()), listener);
             InvalidateApiKeyResponse invalidateResponse = listener.get();
             invalidatedApiKeyIds = invalidateResponse.getInvalidatedApiKeys();
             expectedValidKeyIds = responses.stream().filter(o -> !o.getId().equals(responses.get(0).getId())).map(o -> o.getId())
@@ -438,7 +425,7 @@ public void testGetApiKeysForRealm() throws InterruptedException, ExecutionExcep
         }
 
         PlainActionFuture<GetApiKeyResponse> listener = new PlainActionFuture<>();
-        securityClient.getApiKey(GetApiKeyRequest.usingRealmName("file"), listener);
+        client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingRealmName("file"), listener);
         GetApiKeyResponse response = listener.get();
         verifyGetResponse(noOfApiKeys, responses, response,
                 expectedValidKeyIds,
@@ -450,9 +437,8 @@ public void testGetApiKeysForUser() throws Exception {
         List<CreateApiKeyResponse> responses = createApiKeys(noOfApiKeys, null);
         Client client = client().filterWithHeader(Collections.singletonMap("Authorization", UsernamePasswordToken
                 .basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
         PlainActionFuture<GetApiKeyResponse> listener = new PlainActionFuture<>();
-        securityClient.getApiKey(GetApiKeyRequest.usingUserName(SecuritySettingsSource.TEST_SUPERUSER), listener);
+        client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingUserName(SecuritySettingsSource.TEST_SUPERUSER), listener);
         GetApiKeyResponse response = listener.get();
         verifyGetResponse(noOfApiKeys, responses, response, responses.stream().map(o -> o.getId()).collect(Collectors.toSet()), null);
     }
@@ -461,9 +447,8 @@ public void testGetApiKeysForRealmAndUser() throws InterruptedException, Executi
         List<CreateApiKeyResponse> responses = createApiKeys(1, null);
         Client client = client().filterWithHeader(Collections.singletonMap("Authorization", UsernamePasswordToken
                 .basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
         PlainActionFuture<GetApiKeyResponse> listener = new PlainActionFuture<>();
-        securityClient.getApiKey(GetApiKeyRequest.usingRealmAndUserName("file", SecuritySettingsSource.TEST_SUPERUSER),
+        client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingRealmAndUserName("file", SecuritySettingsSource.TEST_SUPERUSER),
                 listener);
         GetApiKeyResponse response = listener.get();
         verifyGetResponse(1, responses, response, Collections.singleton(responses.get(0).getId()), null);
@@ -473,9 +458,8 @@ public void testGetApiKeysForApiKeyId() throws InterruptedException, ExecutionEx
         List<CreateApiKeyResponse> responses = createApiKeys(1, null);
         Client client = client().filterWithHeader(Collections.singletonMap("Authorization", UsernamePasswordToken
                 .basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
         PlainActionFuture<GetApiKeyResponse> listener = new PlainActionFuture<>();
-        securityClient.getApiKey(GetApiKeyRequest.usingApiKeyId(responses.get(0).getId()), listener);
+        client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyId(responses.get(0).getId()), listener);
         GetApiKeyResponse response = listener.get();
         verifyGetResponse(1, responses, response, Collections.singleton(responses.get(0).getId()), null);
     }
@@ -484,9 +468,8 @@ public void testGetApiKeysForApiKeyName() throws InterruptedException, Execution
         List<CreateApiKeyResponse> responses = createApiKeys(1, null);
         Client client = client().filterWithHeader(Collections.singletonMap("Authorization", UsernamePasswordToken
                 .basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
         PlainActionFuture<GetApiKeyResponse> listener = new PlainActionFuture<>();
-        securityClient.getApiKey(GetApiKeyRequest.usingApiKeyName(responses.get(0).getName()), listener);
+        client.execute(GetApiKeyAction.INSTANCE, GetApiKeyRequest.usingApiKeyName(responses.get(0).getName()), listener);
         GetApiKeyResponse response = listener.get();
         verifyGetResponse(1, responses, response, Collections.singleton(responses.get(0).getId()), null);
     }
@@ -524,8 +507,7 @@ private List<CreateApiKeyResponse> createApiKeys(int noOfApiKeys, TimeValue expi
             final RoleDescriptor descriptor = new RoleDescriptor("role", new String[] { "monitor" }, null, null);
             Client client = client().filterWithHeader(Collections.singletonMap("Authorization", UsernamePasswordToken
                     .basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER, SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-            SecurityClient securityClient = new SecurityClient(client);
-            final CreateApiKeyResponse response = securityClient.prepareCreateApiKey()
+            final CreateApiKeyResponse response = new CreateApiKeyRequestBuilder(client)
                     .setName("test-key-" + randomAlphaOfLengthBetween(5, 9) + i).setExpiration(expiration)
                     .setRoleDescriptors(Collections.singletonList(descriptor)).get();
             assertNotNull(response.getId());
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
index d8a7d9447946b..75035ffcc806d 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.action.update.UpdateRequestBuilder;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.UUIDs;
@@ -364,7 +365,7 @@ public void testCacheClearOnSecurityIndexChange() {
 
         // green to yellow or yellow to green
         previousState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW));
-        currentState = dummyState(previousState.indexStatus == ClusterHealthStatus.GREEN ?
+        currentState = dummyState(previousState.indexHealth == ClusterHealthStatus.GREEN ?
             ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN);
         service.onSecurityIndexStateChange(previousState, currentState);
         assertEquals(expectedInvalidation, service.getNumInvalidation());
@@ -538,7 +539,7 @@ public void testAuthenticateTransportFallback() throws Exception {
 
     public void testAuthenticateTransportDisabledUser() throws Exception {
         final String reqId = AuditUtil.getOrGenerateRequestId(threadContext);
-        User user = new User("username", new String[] { "r1", "r2" }, null, null, null, false);
+        User user = new User("username", new String[] { "r1", "r2" }, null, null, Map.of(), false);
         User fallback = randomBoolean() ? SystemUser.INSTANCE : null;
         when(firstRealm.token(threadContext)).thenReturn(token);
         when(firstRealm.supports(token)).thenReturn(true);
@@ -552,7 +553,7 @@ public void testAuthenticateTransportDisabledUser() throws Exception {
     }
 
     public void testAuthenticateRestDisabledUser() throws Exception {
-        User user = new User("username", new String[] { "r1", "r2" }, null, null, null, false);
+        User user = new User("username", new String[] { "r1", "r2" }, null, null, Map.of(), false);
         when(firstRealm.token(threadContext)).thenReturn(token);
         when(firstRealm.supports(token)).thenReturn(true);
         mockAuthenticate(firstRealm, token, user);
@@ -938,7 +939,7 @@ public void testRunAsLookupSameRealm() throws Exception {
         when(secondRealm.token(threadContext)).thenReturn(token);
         when(secondRealm.supports(token)).thenReturn(true);
         final User user = new User("lookup user", new String[]{"user"}, "lookup user", "lookup@foo.foo",
-                Collections.singletonMap("foo", "bar"), true);
+                Map.of("foo", "bar"), true);
         mockAuthenticate(secondRealm, token, user);
         mockRealmLookupReturnsNull(firstRealm, "run_as");
         doAnswer((i) -> {
@@ -1070,7 +1071,7 @@ public void testAuthenticateTransportDisabledRunAsUser() throws Exception {
         mockRealmLookupReturnsNull(firstRealm, "run_as");
         doAnswer((i) -> {
             ActionListener<User> listener = (ActionListener<User>) i.getArguments()[1];
-            listener.onResponse(new User("looked up user", new String[]{"some role"}, null, null, null, false));
+            listener.onResponse(new User("looked up user", new String[]{"some role"}, null, null, Map.of(), false));
             return null;
         }).when(secondRealm).lookupUser(eq("run_as"), any(ActionListener.class));
         User fallback = randomBoolean() ? SystemUser.INSTANCE : null;
@@ -1092,7 +1093,7 @@ public void testAuthenticateRestDisabledRunAsUser() throws Exception {
         doAnswer((i) -> {
             @SuppressWarnings("unchecked")
             ActionListener<User> listener = (ActionListener<User>) i.getArguments()[1];
-            listener.onResponse(new User("looked up user", new String[]{"some role"}, null, null, null, false));
+            listener.onResponse(new User("looked up user", new String[]{"some role"}, null, null, Map.of(), false));
             return null;
         }).when(secondRealm).lookupUser(eq("run_as"), any(ActionListener.class));
 
@@ -1402,6 +1403,7 @@ private void setCompletedToTrue(AtomicBoolean completed) {
     }
 
     private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) {
-        return new SecurityIndexManager.State(Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus);
+        return new SecurityIndexManager.State(
+            Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetaData.State.OPEN);
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java
index 2ce089f385896..cec5e13b2e111 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/RunAsIntegTests.java
@@ -20,7 +20,7 @@
 public class RunAsIntegTests extends SecurityIntegTestCase {
 
     private static final String RUN_AS_USER = "run_as_user";
-    private static final String TRANSPORT_CLIENT_USER = "transport_user";
+    private static final String CLIENT_USER = "transport_user";
     private static final String ROLES =
             "run_as_role:\n" +
             "  run_as: [ '" + SecuritySettingsSource.TEST_USER_NAME + "', 'idontexist' ]\n";
@@ -47,14 +47,14 @@ public String configRoles() {
     public String configUsers() {
         return super.configUsers()
                 + RUN_AS_USER + ":" + SecuritySettingsSource.TEST_PASSWORD_HASHED + "\n"
-                + TRANSPORT_CLIENT_USER + ":" + SecuritySettingsSource.TEST_PASSWORD_HASHED + "\n";
+                + CLIENT_USER + ":" + SecuritySettingsSource.TEST_PASSWORD_HASHED + "\n";
     }
 
     @Override
     public String configUsersRoles() {
         String roles = super.configUsersRoles()
                 + "run_as_role:" + RUN_AS_USER + "\n"
-                + "transport_client:" + TRANSPORT_CLIENT_USER;
+                + "transport_client:" + CLIENT_USER;
         if (runAsHasSuperUserRole) {
             roles = roles + "\n"
                     + "superuser:" + RUN_AS_USER;
@@ -68,12 +68,12 @@ protected boolean transportSSLEnabled() {
     }
 
     public void testUserImpersonationUsingHttp() throws Exception {
-        // use the transport client user and try to run as
+        // use the http user and try to run as
         try {
             Request request = new Request("GET", "/_nodes");
             RequestOptions.Builder options = request.getOptions().toBuilder();
             options.addHeader("Authorization",
-                    UsernamePasswordToken.basicAuthHeaderValue(TRANSPORT_CLIENT_USER, TEST_PASSWORD_SECURE_STRING));
+                    UsernamePasswordToken.basicAuthHeaderValue(CLIENT_USER, TEST_PASSWORD_SECURE_STRING));
             options.addHeader(AuthenticationServiceField.RUN_AS_USER_HEADER, SecuritySettingsSource.TEST_USER_NAME);
             request.setOptions(options);
             getRestClient().performRequest(request);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java
index 7bac18cfcfb67..59bd9f04fedf4 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java
@@ -6,16 +6,23 @@
 package org.elasticsearch.xpack.security.authc;
 
 import org.apache.directory.api.util.Strings;
-import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.security.AuthenticateResponse;
+import org.elasticsearch.client.security.CreateTokenRequest;
+import org.elasticsearch.client.security.CreateTokenResponse;
+import org.elasticsearch.client.security.InvalidateTokenRequest;
+import org.elasticsearch.client.security.InvalidateTokenResponse;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
-import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -24,22 +31,14 @@
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.test.SecuritySettingsSourceField;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.security.action.token.CreateTokenRequest;
-import org.elasticsearch.xpack.core.security.action.token.CreateTokenResponse;
-import org.elasticsearch.xpack.core.security.action.token.InvalidateTokenRequest;
-import org.elasticsearch.xpack.core.security.action.token.InvalidateTokenResponse;
-import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction;
-import org.elasticsearch.xpack.core.security.action.user.AuthenticateRequest;
-import org.elasticsearch.xpack.core.security.action.user.AuthenticateResponse;
 import org.elasticsearch.xpack.core.security.authc.TokenMetaData;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
 import org.junit.After;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.time.Clock;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
@@ -53,11 +52,11 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
+import static org.elasticsearch.test.SecuritySettingsSource.SECURITY_REQUEST_OPTIONS;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
-@TestLogging("org.elasticsearch.xpack.security.authz.store.FileRolesStore:DEBUG")
 public class TokenAuthIntegTests extends SecurityIntegTestCase {
 
     @Override
@@ -77,48 +76,45 @@ protected int maxNumberOfNodes() {
         return defaultMaxNumberOfNodes() + 1;
     }
 
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false; // need real http
+    }
+
     public void testTokenServiceBootstrapOnNodeJoin() throws Exception {
-        final Client client = client();
-        SecurityClient securityClient = new SecurityClient(client);
-        CreateTokenResponse response = securityClient.prepareCreateToken()
-                .setGrantType("password")
-                .setUsername(SecuritySettingsSource.TEST_USER_NAME)
-                .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))
-                .get();
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
+        CreateTokenResponse response = restClient.security().createToken(CreateTokenRequest.passwordGrant(
+            SecuritySettingsSource.TEST_USER_NAME, SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()), SECURITY_REQUEST_OPTIONS);
         for (TokenService tokenService : internalCluster().getInstances(TokenService.class)) {
             PlainActionFuture<UserToken> userTokenFuture = new PlainActionFuture<>();
-            tokenService.decodeToken(response.getTokenString(), userTokenFuture);
+            tokenService.decodeToken(response.getAccessToken(), userTokenFuture);
             assertNotNull(userTokenFuture.actionGet());
         }
         // start a new node and see if it can decrypt the token
         String nodeName = internalCluster().startNode();
         for (TokenService tokenService : internalCluster().getInstances(TokenService.class)) {
             PlainActionFuture<UserToken> userTokenFuture = new PlainActionFuture<>();
-            tokenService.decodeToken(response.getTokenString(), userTokenFuture);
+            tokenService.decodeToken(response.getAccessToken(), userTokenFuture);
             assertNotNull(userTokenFuture.actionGet());
         }
 
         TokenService tokenService = internalCluster().getInstance(TokenService.class, nodeName);
         PlainActionFuture<UserToken> userTokenFuture = new PlainActionFuture<>();
-        tokenService.decodeToken(response.getTokenString(), userTokenFuture);
+        tokenService.decodeToken(response.getAccessToken(), userTokenFuture);
         assertNotNull(userTokenFuture.actionGet());
     }
 
 
     public void testTokenServiceCanRotateKeys() throws Exception {
-        final Client client = client();
-        SecurityClient securityClient = new SecurityClient(client);
-        CreateTokenResponse response = securityClient.prepareCreateToken()
-                .setGrantType("password")
-                .setUsername(SecuritySettingsSource.TEST_USER_NAME)
-                .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))
-                .get();
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
+        CreateTokenResponse response = restClient.security().createToken(CreateTokenRequest.passwordGrant(
+            SecuritySettingsSource.TEST_USER_NAME, SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()), SECURITY_REQUEST_OPTIONS);
         String masterName = internalCluster().getMasterName();
         TokenService masterTokenService = internalCluster().getInstance(TokenService.class, masterName);
         String activeKeyHash = masterTokenService.getActiveKeyHash();
         for (TokenService tokenService : internalCluster().getInstances(TokenService.class)) {
             PlainActionFuture<UserToken> userTokenFuture = new PlainActionFuture<>();
-            tokenService.decodeToken(response.getTokenString(), userTokenFuture);
+            tokenService.decodeToken(response.getAccessToken(), userTokenFuture);
             assertNotNull(userTokenFuture.actionGet());
             assertEquals(activeKeyHash, tokenService.getActiveKeyHash());
         }
@@ -131,41 +127,31 @@ public void testTokenServiceCanRotateKeys() throws Exception {
 
         for (TokenService tokenService : internalCluster().getInstances(TokenService.class)) {
             PlainActionFuture<UserToken> userTokenFuture = new PlainActionFuture<>();
-            tokenService.decodeToken(response.getTokenString(), userTokenFuture);
+            tokenService.decodeToken(response.getAccessToken(), userTokenFuture);
             assertNotNull(userTokenFuture.actionGet());
             assertNotEquals(activeKeyHash, tokenService.getActiveKeyHash());
         }
     }
 
-    @TestLogging("org.elasticsearch.xpack.security.authc:DEBUG")
     public void testExpiredTokensDeletedAfterExpiration() throws Exception {
-        final Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
-                UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER,
-                        SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
-        CreateTokenResponse response = securityClient.prepareCreateToken()
-                .setGrantType("password")
-                .setUsername(SecuritySettingsSource.TEST_USER_NAME)
-                .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))
-                .get();
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
+        CreateTokenResponse response = restClient.security().createToken(CreateTokenRequest.passwordGrant(
+            SecuritySettingsSource.TEST_USER_NAME, SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()), SECURITY_REQUEST_OPTIONS);
 
         Instant created = Instant.now();
 
-        InvalidateTokenResponse invalidateResponse = securityClient
-                .prepareInvalidateToken(response.getTokenString())
-                .setType(InvalidateTokenRequest.Type.ACCESS_TOKEN)
-                .get();
-        assertThat(invalidateResponse.getResult().getInvalidatedTokens().size(), equalTo(1));
-        assertThat(invalidateResponse.getResult().getPreviouslyInvalidatedTokens().size(), equalTo(0));
-        assertThat(invalidateResponse.getResult().getErrors().size(), equalTo(0));
+        InvalidateTokenResponse invalidateResponse = restClient.security().invalidateToken(
+            new InvalidateTokenRequest(response.getAccessToken(), null, null, null), SECURITY_REQUEST_OPTIONS);
+        assertThat(invalidateResponse.getInvalidatedTokens(), equalTo(1));
+        assertThat(invalidateResponse.getPreviouslyInvalidatedTokens(), equalTo(0));
+        assertThat(invalidateResponse.getErrors().size(), equalTo(0));
         AtomicReference<String> docId = new AtomicReference<>();
         assertBusy(() -> {
-            SearchResponse searchResponse = client.prepareSearch(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS)
-                    .setSource(SearchSourceBuilder.searchSource()
-                        .query(QueryBuilders.termQuery("doc_type", "token")))
-                    .setSize(1)
-                    .setTerminateAfter(1)
-                    .get();
+            SearchResponse searchResponse = restClient.search(new SearchRequest(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS)
+                    .source(SearchSourceBuilder.searchSource()
+                        .size(1)
+                        .terminateAfter(1)
+                        .query(QueryBuilders.termQuery("doc_type", "token"))), SECURITY_REQUEST_OPTIONS);
             assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
             docId.set(searchResponse.getHits().getAt(0).getId());
         });
@@ -173,198 +159,153 @@ public void testExpiredTokensDeletedAfterExpiration() throws Exception {
         // hack doc to modify the creation time to the day before
         Instant yesterday = created.minus(36L, ChronoUnit.HOURS);
         assertTrue(Instant.now().isAfter(yesterday));
-        client.prepareUpdate(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, SINGLE_MAPPING_NAME, docId.get())
-            .setDoc("creation_time", yesterday.toEpochMilli())
-            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
-            .get();
+        restClient.update(new UpdateRequest(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, docId.get())
+            .doc("creation_time", yesterday.toEpochMilli())
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE), SECURITY_REQUEST_OPTIONS);
 
         AtomicBoolean deleteTriggered = new AtomicBoolean(false);
         assertBusy(() -> {
             if (deleteTriggered.compareAndSet(false, true)) {
                 // invalidate a invalid token... doesn't matter that it is bad... we just want this action to trigger the deletion
-                try {
-                    securityClient.prepareInvalidateToken("fooobar")
-                            .setType(randomFrom(InvalidateTokenRequest.Type.values()))
-                            .execute()
-                            .actionGet();
-                } catch (ElasticsearchSecurityException e) {
-                    assertEquals("token malformed", e.getMessage());
-                }
+                ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () ->
+                    restClient.security().invalidateToken(new InvalidateTokenRequest("fooobar", null, null, null),
+                        SECURITY_REQUEST_OPTIONS));
+                assertThat(e.getMessage(), containsString("token malformed"));
             }
-            client.admin().indices().prepareRefresh(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS).get();
-            SearchResponse searchResponse = client.prepareSearch(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS)
-                    .setSource(SearchSourceBuilder.searchSource()
-                        .query(QueryBuilders.termQuery("doc_type", "token")))
-                    .setTerminateAfter(1)
-                    .get();
+            restClient.indices().refresh(new RefreshRequest(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS), SECURITY_REQUEST_OPTIONS);
+            SearchResponse searchResponse = restClient.search(new SearchRequest(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS)
+                    .source(SearchSourceBuilder.searchSource()
+                        .query(QueryBuilders.termQuery("doc_type", "token")).terminateAfter(1)), SECURITY_REQUEST_OPTIONS);
             assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L));
         }, 30, TimeUnit.SECONDS);
     }
 
-    public void testInvalidateAllTokensForUser() throws Exception{
+    public void testInvalidateAllTokensForUser() throws Exception {
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
         final int numOfRequests = randomIntBetween(5, 10);
         for (int i = 0; i < numOfRequests; i++) {
-            securityClient().prepareCreateToken()
-                .setGrantType("password")
-                .setUsername(SecuritySettingsSource.TEST_USER_NAME)
-                .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))
-                .get();
+            restClient.security().createToken(CreateTokenRequest.passwordGrant(SecuritySettingsSource.TEST_USER_NAME,
+                SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()), SECURITY_REQUEST_OPTIONS);
         }
-        Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
-            UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER,
-                SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClientSuperuser = new SecurityClient(client);
-        InvalidateTokenResponse invalidateResponse = securityClientSuperuser
-            .prepareInvalidateToken()
-            .setUserName(SecuritySettingsSource.TEST_USER_NAME)
-            .get();
-        assertThat(invalidateResponse.getResult().getInvalidatedTokens().size(), equalTo(2 * (numOfRequests)));
-        assertThat(invalidateResponse.getResult().getPreviouslyInvalidatedTokens().size(), equalTo(0));
-        assertThat(invalidateResponse.getResult().getErrors().size(), equalTo(0));
+        InvalidateTokenResponse invalidateResponse = restClient.security().invalidateToken(
+            new InvalidateTokenRequest(null, null, null, SecuritySettingsSource.TEST_USER_NAME),
+            RequestOptions.DEFAULT.toBuilder().addHeader("Authorization",
+                UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER,
+                SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)).build());
+        assertThat(invalidateResponse.getInvalidatedTokens(), equalTo(2 * (numOfRequests)));
+        assertThat(invalidateResponse.getPreviouslyInvalidatedTokens(), equalTo(0));
+        assertThat(invalidateResponse.getErrors().size(), equalTo(0));
     }
 
-    public void testInvalidateAllTokensForRealm() throws Exception{
+    public void testInvalidateAllTokensForRealm() throws Exception {
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
         final int numOfRequests = randomIntBetween(5, 10);
         for (int i = 0; i < numOfRequests; i++) {
-            securityClient().prepareCreateToken()
-                .setGrantType("password")
-                .setUsername(SecuritySettingsSource.TEST_USER_NAME)
-                .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))
-                .get();
+            restClient.security().createToken(CreateTokenRequest.passwordGrant(SecuritySettingsSource.TEST_USER_NAME,
+                SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()), SECURITY_REQUEST_OPTIONS);
         }
-        Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
-            UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER,
-                SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClientSuperuser = new SecurityClient(client);
-        InvalidateTokenResponse invalidateResponse = securityClientSuperuser
-            .prepareInvalidateToken()
-            .setRealmName("file")
-            .get();
-        assertThat(invalidateResponse.getResult().getInvalidatedTokens().size(), equalTo(2 * (numOfRequests)));
-        assertThat(invalidateResponse.getResult().getPreviouslyInvalidatedTokens().size(), equalTo(0));
-        assertThat(invalidateResponse.getResult().getErrors().size(), equalTo(0));
+        InvalidateTokenResponse invalidateResponse = restClient.security().invalidateToken(
+            new InvalidateTokenRequest(null, null, "file", null),
+            RequestOptions.DEFAULT.toBuilder().addHeader("Authorization",
+                UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER,
+                    SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)).build());
+        assertThat(invalidateResponse.getInvalidatedTokens(), equalTo(2 * (numOfRequests)));
+        assertThat(invalidateResponse.getPreviouslyInvalidatedTokens(), equalTo(0));
+        assertThat(invalidateResponse.getErrors().size(), equalTo(0));
     }
 
-    public void testInvalidateAllTokensForRealmThatHasNone() {
+    public void testInvalidateAllTokensForRealmThatHasNone() throws IOException {
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
         final int numOfRequests = randomIntBetween(2, 4);
         for (int i = 0; i < numOfRequests; i++) {
-            securityClient().prepareCreateToken()
-                .setGrantType("password")
-                .setUsername(SecuritySettingsSource.TEST_USER_NAME)
-                .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))
-                .get();
+            restClient.security().createToken(CreateTokenRequest.passwordGrant(SecuritySettingsSource.TEST_USER_NAME,
+                SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()), SECURITY_REQUEST_OPTIONS);
         }
-        Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
-            UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER,
-                SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClientSuperuser = new SecurityClient(client);
-        InvalidateTokenResponse invalidateResponse = securityClientSuperuser
-            .prepareInvalidateToken()
-            .setRealmName("saml")
-            .get();
-        assertThat(invalidateResponse.getResult().getInvalidatedTokens().size(), equalTo(0));
-        assertThat(invalidateResponse.getResult().getPreviouslyInvalidatedTokens().size(), equalTo(0));
-        assertThat(invalidateResponse.getResult().getErrors().size(), equalTo(0));
+        InvalidateTokenResponse invalidateResponse = restClient.security().invalidateToken(
+            new InvalidateTokenRequest(null, null, "saml", null),
+            RequestOptions.DEFAULT.toBuilder().addHeader("Authorization",
+                UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER,
+                    SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)).build());
+        assertThat(invalidateResponse.getInvalidatedTokens(), equalTo(0));
+        assertThat(invalidateResponse.getPreviouslyInvalidatedTokens(), equalTo(0));
+        assertThat(invalidateResponse.getErrors().size(), equalTo(0));
     }
 
-    public void testExpireMultipleTimes() {
-        CreateTokenResponse response = securityClient().prepareCreateToken()
-                .setGrantType("password")
-                .setUsername(SecuritySettingsSource.TEST_USER_NAME)
-                .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))
-                .get();
-
-        InvalidateTokenResponse invalidateResponse = securityClient()
-                .prepareInvalidateToken(response.getTokenString())
-                .setType(InvalidateTokenRequest.Type.ACCESS_TOKEN)
-                .get();
-        assertThat(invalidateResponse.getResult().getInvalidatedTokens().size(), equalTo(1));
-        assertThat(invalidateResponse.getResult().getPreviouslyInvalidatedTokens().size(), equalTo(0));
-        assertThat(invalidateResponse.getResult().getErrors().size(), equalTo(0));
-        InvalidateTokenResponse invalidateAgainResponse = securityClient()
-            .prepareInvalidateToken(response.getTokenString())
-            .setType(InvalidateTokenRequest.Type.ACCESS_TOKEN)
-            .get();
-        assertThat(invalidateAgainResponse.getResult().getInvalidatedTokens().size(), equalTo(0));
-        assertThat(invalidateAgainResponse.getResult().getPreviouslyInvalidatedTokens().size(), equalTo(1));
-        assertThat(invalidateAgainResponse.getResult().getErrors().size(), equalTo(0));
+    public void testExpireMultipleTimes() throws IOException {
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
+        CreateTokenResponse response = restClient.security().createToken(CreateTokenRequest.passwordGrant(
+            SecuritySettingsSource.TEST_USER_NAME, SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()), SECURITY_REQUEST_OPTIONS);
+
+        InvalidateTokenResponse invalidateResponse = restClient.security()
+            .invalidateToken(new InvalidateTokenRequest(response.getAccessToken(), null, null, null), SECURITY_REQUEST_OPTIONS);
+        assertThat(invalidateResponse.getInvalidatedTokens(), equalTo(1));
+        assertThat(invalidateResponse.getPreviouslyInvalidatedTokens(), equalTo(0));
+        assertThat(invalidateResponse.getErrors().size(), equalTo(0));
+        InvalidateTokenResponse invalidateAgainResponse = restClient.security()
+            .invalidateToken(new InvalidateTokenRequest(response.getAccessToken(), null, null, null), SECURITY_REQUEST_OPTIONS);
+        assertThat(invalidateAgainResponse.getInvalidatedTokens(), equalTo(0));
+        assertThat(invalidateAgainResponse.getPreviouslyInvalidatedTokens(), equalTo(1));
+        assertThat(invalidateAgainResponse.getErrors().size(), equalTo(0));
     }
 
-    public void testRefreshingToken() {
-        Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
-                UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                        SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
-        CreateTokenResponse createTokenResponse = securityClient.prepareCreateToken()
-                .setGrantType("password")
-                .setUsername(SecuritySettingsSource.TEST_USER_NAME)
-                .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))
-                .get();
-        assertNotNull(createTokenResponse.getRefreshToken());
+    public void testRefreshingToken() throws IOException {
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
+        CreateTokenResponse response = restClient.security().createToken(CreateTokenRequest.passwordGrant(
+            SecuritySettingsSource.TEST_USER_NAME, SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()), SECURITY_REQUEST_OPTIONS);
+        assertNotNull(response.getRefreshToken());
         // get cluster health with token
         assertNoTimeout(client()
-                .filterWithHeader(Collections.singletonMap("Authorization", "Bearer " + createTokenResponse.getTokenString()))
+                .filterWithHeader(Collections.singletonMap("Authorization", "Bearer " + response.getAccessToken()))
                 .admin().cluster().prepareHealth().get());
 
-        CreateTokenResponse refreshResponse = securityClient.prepareRefreshToken(createTokenResponse.getRefreshToken()).get();
+        CreateTokenResponse refreshResponse = restClient.security()
+            .createToken(CreateTokenRequest.refreshTokenGrant(response.getRefreshToken()), SECURITY_REQUEST_OPTIONS);
         assertNotNull(refreshResponse.getRefreshToken());
-        assertNotEquals(refreshResponse.getRefreshToken(), createTokenResponse.getRefreshToken());
-        assertNotEquals(refreshResponse.getTokenString(), createTokenResponse.getTokenString());
+        assertNotEquals(refreshResponse.getRefreshToken(), response.getRefreshToken());
+        assertNotEquals(refreshResponse.getAccessToken(), response.getAccessToken());
 
-        assertNoTimeout(client().filterWithHeader(Collections.singletonMap("Authorization", "Bearer " + refreshResponse.getTokenString()))
+        assertNoTimeout(client().filterWithHeader(Collections.singletonMap("Authorization", "Bearer " + refreshResponse.getAccessToken()))
                 .admin().cluster().prepareHealth().get());
     }
 
-    public void testRefreshingInvalidatedToken() {
-        Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
-                UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                        SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
-        CreateTokenResponse createTokenResponse = securityClient.prepareCreateToken()
-                .setGrantType("password")
-                .setUsername(SecuritySettingsSource.TEST_USER_NAME)
-                .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))
-                .get();
+    public void testRefreshingInvalidatedToken() throws IOException {
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
+        CreateTokenResponse createTokenResponse = restClient.security().createToken(CreateTokenRequest.passwordGrant(
+            SecuritySettingsSource.TEST_USER_NAME, SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()), SECURITY_REQUEST_OPTIONS);
         assertNotNull(createTokenResponse.getRefreshToken());
-        InvalidateTokenResponse invalidateResponse = securityClient
-                .prepareInvalidateToken(createTokenResponse.getRefreshToken())
-                .setType(InvalidateTokenRequest.Type.REFRESH_TOKEN)
-                .get();
-        assertThat(invalidateResponse.getResult().getInvalidatedTokens().size(), equalTo(1));
-        assertThat(invalidateResponse.getResult().getPreviouslyInvalidatedTokens().size(), equalTo(0));
-        assertThat(invalidateResponse.getResult().getErrors().size(), equalTo(0));
-
-        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
-                () -> securityClient.prepareRefreshToken(createTokenResponse.getRefreshToken()).get());
-        assertEquals("invalid_grant", e.getMessage());
+        InvalidateTokenResponse invalidateResponse = restClient.security()
+            .invalidateToken(new InvalidateTokenRequest(null, createTokenResponse.getRefreshToken(), null, null), SECURITY_REQUEST_OPTIONS);
+        assertThat(invalidateResponse.getInvalidatedTokens(), equalTo(1));
+        assertThat(invalidateResponse.getPreviouslyInvalidatedTokens(), equalTo(0));
+        assertThat(invalidateResponse.getErrors().size(), equalTo(0));
+
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
+                () -> restClient.security().createToken(
+                    CreateTokenRequest.refreshTokenGrant(createTokenResponse.getRefreshToken()), SECURITY_REQUEST_OPTIONS));
+        assertThat(e.getCause().getMessage(), containsString("invalid_grant"));
         assertEquals(RestStatus.BAD_REQUEST, e.status());
-        assertEquals("token has been invalidated", e.getHeader("error_description").get(0));
+        assertThat(e.getCause().getMessage(), containsString("token has been invalidated"));
     }
 
     public void testRefreshingMultipleTimesFails() throws Exception {
-        Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
-                UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                        SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
-        CreateTokenResponse createTokenResponse = securityClient.prepareCreateToken()
-                .setGrantType("password")
-                .setUsername(SecuritySettingsSource.TEST_USER_NAME)
-                .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))
-                .get();
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
+        CreateTokenResponse createTokenResponse = restClient.security().createToken(CreateTokenRequest.passwordGrant(
+            SecuritySettingsSource.TEST_USER_NAME, SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()), SECURITY_REQUEST_OPTIONS);
         assertNotNull(createTokenResponse.getRefreshToken());
-        CreateTokenResponse refreshResponse = securityClient.prepareRefreshToken(createTokenResponse.getRefreshToken()).get();
+
+        CreateTokenResponse refreshResponse = restClient.security()
+            .createToken(CreateTokenRequest.refreshTokenGrant(createTokenResponse.getRefreshToken()), SECURITY_REQUEST_OPTIONS);
         assertNotNull(refreshResponse);
         // We now have two documents, the original(now refreshed) token doc and the new one with the new access doc
         AtomicReference<String> docId = new AtomicReference<>();
         assertBusy(() -> {
-            SearchResponse searchResponse = client.prepareSearch(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS)
-                .setSource(SearchSourceBuilder.searchSource()
+            SearchResponse searchResponse = restClient.search(new SearchRequest(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS)
+                .source(SearchSourceBuilder.searchSource()
                     .query(QueryBuilders.boolQuery()
                         .must(QueryBuilders.termQuery("doc_type", TokenService.TOKEN_DOC_TYPE))
-                        .must(QueryBuilders.termQuery("refresh_token.refreshed", "true"))))
-                .setSize(1)
-                .setTerminateAfter(1)
-                .get();
+                        .must(QueryBuilders.termQuery("refresh_token.refreshed", "true")))
+                    .size(1)
+                    .terminateAfter(1)), SECURITY_REQUEST_OPTIONS);
             assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
             docId.set(searchResponse.getHits().getAt(0).getId());
         });
@@ -373,34 +314,27 @@ public void testRefreshingMultipleTimesFails() throws Exception {
         Instant refreshed = Instant.now();
         Instant aWhileAgo = refreshed.minus(50L, ChronoUnit.SECONDS);
         assertTrue(Instant.now().isAfter(aWhileAgo));
-        UpdateResponse updateResponse = client.prepareUpdate(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, SINGLE_MAPPING_NAME, docId.get())
-            .setDoc("refresh_token", Collections.singletonMap("refresh_time", aWhileAgo.toEpochMilli()))
+        UpdateResponse updateResponse = restClient.update(new UpdateRequest(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, docId.get())
+            .doc("refresh_token", Collections.singletonMap("refresh_time", aWhileAgo.toEpochMilli()))
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
-            .setFetchSource("refresh_token", Strings.EMPTY_STRING)
-            .get();
+            .fetchSource("refresh_token", Strings.EMPTY_STRING), SECURITY_REQUEST_OPTIONS);
         assertNotNull(updateResponse);
         Map<String, Object> refreshTokenMap = (Map<String, Object>) updateResponse.getGetResult().sourceAsMap().get("refresh_token");
         assertTrue(
             Instant.ofEpochMilli((long) refreshTokenMap.get("refresh_time")).isBefore(Instant.now().minus(30L, ChronoUnit.SECONDS)));
-        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
-                () -> securityClient.prepareRefreshToken(createTokenResponse.getRefreshToken()).get());
-        assertEquals("invalid_grant", e.getMessage());
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> restClient.security()
+            .createToken(CreateTokenRequest.refreshTokenGrant(createTokenResponse.getRefreshToken()), SECURITY_REQUEST_OPTIONS));
+        assertThat(e.getCause().getMessage(), containsString("invalid_grant"));
         assertEquals(RestStatus.BAD_REQUEST, e.status());
-        assertEquals("token has already been refreshed more than 30 seconds in the past", e.getHeader("error_description").get(0));
+        assertThat(e.getCause().getMessage(), containsString("token has already been refreshed more than 30 seconds in the past"));
     }
 
     public void testRefreshingMultipleTimesWithinWindowSucceeds() throws Exception {
         final Clock clock = Clock.systemUTC();
-        Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
-            UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
         final List<String> tokens = Collections.synchronizedList(new ArrayList<>());
-        CreateTokenResponse createTokenResponse = securityClient.prepareCreateToken()
-            .setGrantType("password")
-            .setUsername(SecuritySettingsSource.TEST_USER_NAME)
-            .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))
-            .get();
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
+        CreateTokenResponse createTokenResponse = restClient.security().createToken(CreateTokenRequest.passwordGrant(
+            SecuritySettingsSource.TEST_USER_NAME, SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()), SECURITY_REQUEST_OPTIONS);
         assertNotNull(createTokenResponse.getRefreshToken());
         final int numberOfProcessors = Runtime.getRuntime().availableProcessors();
         final int numberOfThreads = scaledRandomIntBetween((numberOfProcessors + 1) / 2, numberOfProcessors * 3);
@@ -411,13 +345,6 @@ public void testRefreshingMultipleTimesWithinWindowSucceeds() throws Exception {
         final Instant t1 = clock.instant();
         for (int i = 0; i < numberOfThreads; i++) {
             threads.add(new Thread(() -> {
-                // Each thread gets its own client so that more than one nodes will be hit
-                Client threadClient = client().filterWithHeader(Collections.singletonMap("Authorization",
-                    UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                        SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-                SecurityClient threadSecurityClient = new SecurityClient(threadClient);
-                CreateTokenRequest refreshRequest =
-                    threadSecurityClient.prepareRefreshToken(createTokenResponse.getRefreshToken()).request();
                 readyLatch.countDown();
                 try {
                     readyLatch.await();
@@ -426,21 +353,25 @@ public void testRefreshingMultipleTimesWithinWindowSucceeds() throws Exception {
                     completedLatch.countDown();
                     return;
                 }
-                threadSecurityClient.refreshToken(refreshRequest, ActionListener.wrap(result -> {
+
+                try {
+                    // safe to use same rest client across threads since it round robins between nodes
+                    CreateTokenResponse result = restClient.security()
+                        .createToken(CreateTokenRequest.refreshTokenGrant(createTokenResponse.getRefreshToken()), SECURITY_REQUEST_OPTIONS);
                     final Instant t2 = clock.instant();
-                    if (t1.plusSeconds(30L).isBefore(t2)){
+                    if (t1.plusSeconds(30L).isBefore(t2)) {
                         logger.warn("Tokens [{}], [{}] were received more than 30 seconds after the request, not checking them",
-                            result.getTokenString(), result.getRefreshToken());
+                            result.getAccessToken(), result.getRefreshToken());
                     } else {
-                        tokens.add(result.getTokenString() + result.getRefreshToken());
+                        tokens.add(result.getAccessToken() + result.getRefreshToken());
                     }
-                    logger.info("received access token [{}] and refresh token [{}]", result.getTokenString(), result.getRefreshToken());
+                    logger.info("received access token [{}] and refresh token [{}]", result.getAccessToken(), result.getRefreshToken());
                     completedLatch.countDown();
-                }, e -> {
+                } catch (IOException e) {
                     failed.set(true);
                     completedLatch.countDown();
                     logger.error("caught exception", e);
-                }));
+                }
             }));
         }
         for (Thread thread : threads) {
@@ -457,102 +388,74 @@ public void testRefreshingMultipleTimesWithinWindowSucceeds() throws Exception {
         assertThat(tokens.stream().distinct().collect(Collectors.toList()).size(), equalTo(1));
     }
 
-    public void testRefreshAsDifferentUser() {
-        Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
-                UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
-                        SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
-        CreateTokenResponse createTokenResponse = securityClient.prepareCreateToken()
-                .setGrantType("password")
-                .setUsername(SecuritySettingsSource.TEST_USER_NAME)
-                .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))
-                .get();
+    public void testRefreshAsDifferentUser() throws IOException {
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
+        CreateTokenResponse createTokenResponse = restClient.security().createToken(CreateTokenRequest.passwordGrant(
+            SecuritySettingsSource.TEST_USER_NAME, SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()), SECURITY_REQUEST_OPTIONS);
         assertNotNull(createTokenResponse.getRefreshToken());
 
-        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
-                () -> new SecurityClient(client()
-                        .filterWithHeader(Collections.singletonMap("Authorization",
-                                UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER,
-                                        SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING))))
-                        .prepareRefreshToken(createTokenResponse.getRefreshToken()).get());
-        assertEquals("invalid_grant", e.getMessage());
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
+                () -> restClient.security().createToken(CreateTokenRequest.refreshTokenGrant(createTokenResponse.getRefreshToken()),
+                    RequestOptions.DEFAULT.toBuilder().addHeader("Authorization",
+                        UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER,
+                        SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)).build()));
+        assertThat(e.getCause().getMessage(), containsString("invalid_grant"));
         assertEquals(RestStatus.BAD_REQUEST, e.status());
-        assertEquals("tokens must be refreshed by the creating client", e.getHeader("error_description").get(0));
+        assertThat(e.getCause().getMessage(), containsString("tokens must be refreshed by the creating client"));
     }
 
-    public void testCreateThenRefreshAsDifferentUser() {
-        Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
-                UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER,
-                        SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
-        CreateTokenResponse createTokenResponse = securityClient.prepareCreateToken()
-                .setGrantType("password")
-                .setUsername(SecuritySettingsSource.TEST_USER_NAME)
-                .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))
-                .get();
+    public void testCreateThenRefreshAsDifferentUser() throws IOException {
+        final RequestOptions superuserOptions = RequestOptions.DEFAULT.toBuilder().addHeader("Authorization",
+            UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER,
+                SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)).build();
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
+        CreateTokenResponse createTokenResponse = restClient.security().createToken(CreateTokenRequest.passwordGrant(
+            SecuritySettingsSource.TEST_USER_NAME, SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()), superuserOptions);
         assertNotNull(createTokenResponse.getRefreshToken());
 
-        CreateTokenResponse refreshResponse = securityClient.prepareRefreshToken(createTokenResponse.getRefreshToken()).get();
-        assertNotEquals(refreshResponse.getTokenString(), createTokenResponse.getTokenString());
+        CreateTokenResponse refreshResponse = restClient.security()
+            .createToken(CreateTokenRequest.refreshTokenGrant(createTokenResponse.getRefreshToken()), superuserOptions);
+        assertNotEquals(refreshResponse.getAccessToken(), createTokenResponse.getAccessToken());
         assertNotEquals(refreshResponse.getRefreshToken(), createTokenResponse.getRefreshToken());
 
-        PlainActionFuture<AuthenticateResponse> authFuture = new PlainActionFuture<>();
-        AuthenticateRequest request = new AuthenticateRequest();
-        request.username(SecuritySettingsSource.TEST_SUPERUSER);
-        client.execute(AuthenticateAction.INSTANCE, request, authFuture);
-        AuthenticateResponse response = authFuture.actionGet();
-        assertEquals(SecuritySettingsSource.TEST_SUPERUSER, response.authentication().getUser().principal());
-
-        authFuture = new PlainActionFuture<>();
-        request = new AuthenticateRequest();
-        request.username(SecuritySettingsSource.TEST_USER_NAME);
-        client.filterWithHeader(Collections.singletonMap("Authorization", "Bearer " + createTokenResponse.getTokenString()))
-                .execute(AuthenticateAction.INSTANCE, request, authFuture);
-        response = authFuture.actionGet();
-        assertEquals(SecuritySettingsSource.TEST_USER_NAME, response.authentication().getUser().principal());
-
-        authFuture = new PlainActionFuture<>();
-        request = new AuthenticateRequest();
-        request.username(SecuritySettingsSource.TEST_USER_NAME);
-        client.filterWithHeader(Collections.singletonMap("Authorization", "Bearer " + refreshResponse.getTokenString()))
-                .execute(AuthenticateAction.INSTANCE, request, authFuture);
-        response = authFuture.actionGet();
-        assertEquals(SecuritySettingsSource.TEST_USER_NAME, response.authentication().getUser().principal());
+        AuthenticateResponse response = restClient.security().authenticate(superuserOptions);;
+        assertEquals(SecuritySettingsSource.TEST_SUPERUSER, response.getUser().getUsername());
+
+        RequestOptions tokenAuthOptions =
+            RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", "Bearer " + createTokenResponse.getAccessToken()).build();
+        response = restClient.security().authenticate(tokenAuthOptions);
+        assertEquals(SecuritySettingsSource.TEST_USER_NAME, response.getUser().getUsername());
+
+        RequestOptions refreshedTokenAuthOptions =
+            RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", "Bearer " + refreshResponse.getAccessToken()).build();
+        response = restClient.security().authenticate(refreshedTokenAuthOptions);
+        assertEquals(SecuritySettingsSource.TEST_USER_NAME, response.getUser().getUsername());
     }
 
     public void testClientCredentialsGrant() throws Exception {
-        Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
+        final RequestOptions superuserOptions = RequestOptions.DEFAULT.toBuilder().addHeader("Authorization",
             UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_SUPERUSER,
-                SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
-        SecurityClient securityClient = new SecurityClient(client);
-        CreateTokenResponse createTokenResponse = securityClient.prepareCreateToken()
-            .setGrantType("client_credentials")
-            .get();
+                SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)).build();
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
+        CreateTokenResponse createTokenResponse =
+            restClient.security().createToken(CreateTokenRequest.clientCredentialsGrant(), superuserOptions);
         assertNull(createTokenResponse.getRefreshToken());
 
-        AuthenticateRequest request = new AuthenticateRequest();
-        request.username(SecuritySettingsSource.TEST_SUPERUSER);
-        PlainActionFuture<AuthenticateResponse> authFuture = new PlainActionFuture<>();
-        client.filterWithHeader(Collections.singletonMap("Authorization", "Bearer " + createTokenResponse.getTokenString()))
-            .execute(AuthenticateAction.INSTANCE, request, authFuture);
-        AuthenticateResponse response = authFuture.get();
-        assertEquals(SecuritySettingsSource.TEST_SUPERUSER, response.authentication().getUser().principal());
+        RequestOptions tokenAuthOptions =
+            RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", "Bearer " + createTokenResponse.getAccessToken()).build();
+        AuthenticateResponse response = restClient.security().authenticate(tokenAuthOptions);
+        assertEquals(SecuritySettingsSource.TEST_SUPERUSER, response.getUser().getUsername());
 
         // invalidate
-        PlainActionFuture<InvalidateTokenResponse> invalidateResponseFuture = new PlainActionFuture<>();
-        InvalidateTokenRequest invalidateTokenRequest =
-            new InvalidateTokenRequest(createTokenResponse.getTokenString(), InvalidateTokenRequest.Type.ACCESS_TOKEN.getValue());
-        securityClient.invalidateToken(invalidateTokenRequest, invalidateResponseFuture);
-        assertThat(invalidateResponseFuture.get().getResult().getInvalidatedTokens().size(), equalTo(1));
-        assertThat(invalidateResponseFuture.get().getResult().getPreviouslyInvalidatedTokens().size(), equalTo(0));
-        assertThat(invalidateResponseFuture.get().getResult().getErrors().size(), equalTo(0));
-
-        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> {
-            PlainActionFuture<AuthenticateResponse> responseFuture = new PlainActionFuture<>();
-            client.filterWithHeader(Collections.singletonMap("Authorization", "Bearer " + createTokenResponse.getTokenString()))
-                .execute(AuthenticateAction.INSTANCE, request, responseFuture);
-            responseFuture.actionGet();
-        });
+        InvalidateTokenResponse invalidateTokenResponse = restClient.security()
+            .invalidateToken(InvalidateTokenRequest.accessToken(createTokenResponse.getAccessToken()), superuserOptions);
+        assertThat(invalidateTokenResponse.getInvalidatedTokens(), equalTo(1));
+        assertThat(invalidateTokenResponse.getPreviouslyInvalidatedTokens(), equalTo(0));
+        assertThat(invalidateTokenResponse.getErrors().size(), equalTo(0));
+
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
+            () -> restClient.security().authenticate(tokenAuthOptions));
+        assertEquals(RestStatus.UNAUTHORIZED, e.status());
     }
 
     @Before
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java
index 49796333098ff..2a4cfb4882966 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
@@ -53,14 +54,18 @@
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.core.watcher.watch.ClockMock;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
-import org.junit.After;
 import org.elasticsearch.xpack.security.test.SecurityMocks;
 import org.hamcrest.Matchers;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
+import javax.crypto.SecretKey;
+
 import java.io.IOException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.security.GeneralSecurityException;
 import java.time.Clock;
 import java.time.Instant;
@@ -70,8 +75,6 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import javax.crypto.SecretKey;
-
 import static java.time.Clock.systemUTC;
 import static org.elasticsearch.repositories.ESBlobStoreTestCase.randomBytes;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
@@ -530,9 +533,8 @@ public void testTokenExpiry() throws Exception {
         }
 
         try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
-            // move to expiry
-            clock.fastForwardSeconds(Math.toIntExact(defaultExpiration.getSeconds()) - fastForwardAmount);
-            clock.rewind(TimeValue.timeValueNanos(clock.instant().getNano())); // trim off nanoseconds since don't store them in the index
+            // move to expiry, stripping nanoseconds, as we don't store them in the security-tokens index
+            clock.setTime(userToken.getExpirationTime().truncatedTo(ChronoUnit.MILLIS).atZone(clock.getZone()));
             PlainActionFuture<UserToken> future = new PlainActionFuture<>();
             tokenService.getAndValidateToken(requestContext, future);
             assertAuthentication(authentication, future.get().getAuthentication());
@@ -722,6 +724,11 @@ public void testCannotValidateTokenIfLicenseDoesNotAllowTokens() throws Exceptio
         assertThat(authToken, Matchers.nullValue());
     }
 
+    public void testHashedTokenIsUrlSafe() {
+        final String hashedId = TokenService.hashTokenString(UUIDs.randomBase64UUID());
+        assertEquals(hashedId, URLEncoder.encode(hashedId, StandardCharsets.UTF_8));
+    }
+
     private TokenService createTokenService(Settings settings, Clock clock) throws GeneralSecurityException {
         return new TokenService(settings, clock, client, licenseState, securityMainIndex, securityTokensIndex, clusterService);
     }
@@ -807,7 +814,7 @@ private DiscoveryNode addAnotherDataNodeWithVersion(ClusterService clusterServic
         final ClusterState currentState = clusterService.state();
         final DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(currentState.getNodes());
         final DiscoveryNode anotherDataNode = new DiscoveryNode("another_data_node#" + version, buildNewFakeTransportAddress(),
-                Collections.emptyMap(), Collections.singleton(DiscoveryNode.Role.DATA), version);
+                Collections.emptyMap(), Collections.singleton(DiscoveryNodeRole.DATA_ROLE), version);
         discoBuilder.add(anotherDataNode);
         final ClusterState.Builder newStateBuilder = ClusterState.builder(currentState);
         newStateBuilder.nodes(discoBuilder);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java
index 7021873a02878..36c9b79538272 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java
@@ -30,20 +30,29 @@
 import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder;
 import org.elasticsearch.xpack.core.action.XPackUsageResponse;
 import org.elasticsearch.xpack.core.security.SecurityFeatureSetUsage;
+import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRequestBuilder;
+import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheRequestBuilder;
+import org.elasticsearch.xpack.core.security.action.role.DeleteRoleRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.role.DeleteRoleResponse;
+import org.elasticsearch.xpack.core.security.action.role.GetRolesRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.role.GetRolesResponse;
+import org.elasticsearch.xpack.core.security.action.role.PutRoleRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.role.PutRoleResponse;
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction;
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateRequest;
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateResponse;
+import org.elasticsearch.xpack.core.security.action.user.ChangePasswordRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.user.ChangePasswordResponse;
+import org.elasticsearch.xpack.core.security.action.user.DeleteUserRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.user.DeleteUserResponse;
+import org.elasticsearch.xpack.core.security.action.user.GetUsersRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.user.GetUsersResponse;
+import org.elasticsearch.xpack.core.security.action.user.PutUserRequestBuilder;
+import org.elasticsearch.xpack.core.security.action.user.SetEnabledRequestBuilder;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
@@ -66,8 +75,8 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
-import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
 import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7;
+import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
@@ -108,11 +117,11 @@ public void setupAnonymousRoleIfNecessary() throws Exception {
         if (anonymousEnabled) {
             if (roleExists) {
                 logger.info("anonymous is enabled. creating [native_anonymous] role");
-                PutRoleResponse response = securityClient()
-                        .preparePutRole("native_anonymous")
-                        .cluster("ALL")
-                        .addIndices(new String[]{"*"}, new String[]{"ALL"}, null, null, null, randomBoolean())
-                        .get();
+                PutRoleResponse response = new PutRoleRequestBuilder(client())
+                    .name("native_anonymous")
+                    .cluster("ALL")
+                    .addIndices(new String[]{"*"}, new String[]{"ALL"}, null, null, null, randomBoolean())
+                    .get();
                 assertTrue(response.isCreated());
             } else {
                 logger.info("anonymous is enabled, but configured with a missing role");
@@ -121,42 +130,39 @@ public void setupAnonymousRoleIfNecessary() throws Exception {
     }
 
     public void testDeletingNonexistingUserAndRole() throws Exception {
-        SecurityClient c = securityClient();
         // first create the index so it exists
-        c.preparePutUser("joe", "s3kirt".toCharArray(), hasher, "role1", "user").get();
-        DeleteUserResponse resp = c.prepareDeleteUser("missing").get();
+        preparePutUser("joe", "s3krit", hasher, "role1", "user").get();
+        DeleteUserResponse resp = new DeleteUserRequestBuilder(client()).username("missing").get();
         assertFalse("user shouldn't be found", resp.found());
-        DeleteRoleResponse resp2 = c.prepareDeleteRole("role").get();
+        DeleteRoleResponse resp2 = new DeleteRoleRequestBuilder(client()).name("role").get();
         assertFalse("role shouldn't be found", resp2.found());
     }
 
     public void testGettingUserThatDoesntExist() throws Exception {
-        SecurityClient c = securityClient();
-        GetUsersResponse resp = c.prepareGetUsers("joe").get();
+        GetUsersResponse resp = new GetUsersRequestBuilder(client()).usernames("joe").get();
         assertFalse("user should not exist", resp.hasUsers());
-        GetRolesResponse resp2 = c.prepareGetRoles().names("role").get();
+        GetRolesResponse resp2 = new GetRolesRequestBuilder(client()).names("role").get();
         assertFalse("role should not exist", resp2.hasRoles());
     }
 
     public void testAddAndGetUser() throws Exception {
-        SecurityClient c = securityClient();
-        final List<User> existingUsers = Arrays.asList(c.prepareGetUsers().get().users());
+        final List<User> existingUsers = Arrays.asList(new GetUsersRequestBuilder(client()).get().users());
         final int existing = existingUsers.size();
         logger.error("--> creating user");
-        c.preparePutUser("joe", "s3kirt".toCharArray(), hasher, "role1", "user").get();
+        preparePutUser("joe", "s3kirt", hasher, "role1", "user").get();
         logger.error("--> waiting for .security index");
         ensureGreen(SECURITY_MAIN_ALIAS);
         logger.info("--> retrieving user");
-        GetUsersResponse resp = c.prepareGetUsers("joe").get();
+        GetUsersResponse resp = new GetUsersRequestBuilder(client()).usernames("joe").get();
         assertTrue("user should exist", resp.hasUsers());
         User joe = resp.users()[0];
         assertEquals("joe", joe.principal());
         assertArrayEquals(joe.roles(), new String[]{"role1", "user"});
 
         logger.info("--> adding two more users");
-        c.preparePutUser("joe2", "s3kirt2".toCharArray(), hasher, "role2", "user").get();
-        c.preparePutUser("joe3", "s3kirt3".toCharArray(), hasher, "role3", "user").get();
-        GetUsersResponse allUsersResp = c.prepareGetUsers().get();
+        preparePutUser("joe2", "s3kirt2", hasher, "role2", "user").get();
+        preparePutUser("joe3", "s3kirt3", hasher, "role3", "user").get();
+        GetUsersResponse allUsersResp = new GetUsersRequestBuilder(client()).get();
         assertTrue("users should exist", allUsersResp.hasUsers());
         assertEquals("should be " + (3 + existing) + " users total", 3 + existing, allUsersResp.users().length);
         List<String> names = new ArrayList<>(3);
@@ -168,7 +174,7 @@ public void testAddAndGetUser() throws Exception {
         CollectionUtil.timSort(names);
         assertArrayEquals(new String[] { "joe", "joe2", "joe3" }, names.toArray(Strings.EMPTY_ARRAY));
 
-        GetUsersResponse someUsersResp = c.prepareGetUsers("joe", "joe3").get();
+        GetUsersResponse someUsersResp = new GetUsersRequestBuilder(client()).usernames("joe", "joe3").get();
         assertTrue("users should exist", someUsersResp.hasUsers());
         assertEquals("should be 2 users returned", 2, someUsersResp.users().length);
         names = new ArrayList<>(2);
@@ -179,20 +185,19 @@ public void testAddAndGetUser() throws Exception {
         assertArrayEquals(new String[]{"joe", "joe3"}, names.toArray(Strings.EMPTY_ARRAY));
 
         logger.info("--> deleting user");
-        DeleteUserResponse delResp = c.prepareDeleteUser("joe").get();
+        DeleteUserResponse delResp = new DeleteUserRequestBuilder(client()).username("joe").get();
         assertTrue(delResp.found());
         logger.info("--> retrieving user");
-        resp = c.prepareGetUsers("joe").get();
+        resp = new GetUsersRequestBuilder(client()).usernames("joe").get();
         assertFalse("user should not exist after being deleted", resp.hasUsers());
     }
 
     public void testAddAndGetRole() throws Exception {
-        SecurityClient c = securityClient();
-        final List<RoleDescriptor> existingRoles = Arrays.asList(c.prepareGetRoles().get().roles());
+        final List<RoleDescriptor> existingRoles = Arrays.asList(new GetRolesRequestBuilder(client()).get().roles());
         final int existing = existingRoles.size();
         final Map<String, Object> metadata = Collections.singletonMap("key", randomAlphaOfLengthBetween(1, 10));
         logger.error("--> creating role");
-        c.preparePutRole("test_role")
+        preparePutRole("test_role")
                 .cluster("all", "none")
                 .runAs("root", "nobody")
                 .addIndices(new String[]{"index"}, new String[]{"read"}, new String[]{"body", "title"}, null,
@@ -202,20 +207,20 @@ public void testAddAndGetRole() throws Exception {
         logger.error("--> waiting for .security index");
         ensureGreen(SECURITY_MAIN_ALIAS);
         logger.info("--> retrieving role");
-        GetRolesResponse resp = c.prepareGetRoles().names("test_role").get();
+        GetRolesResponse resp = new GetRolesRequestBuilder(client()).names("test_role").get();
         assertTrue("role should exist", resp.hasRoles());
         RoleDescriptor testRole = resp.roles()[0];
         assertNotNull(testRole);
         assertThat(testRole.getMetadata().size(), is(1));
         assertThat(testRole.getMetadata().get("key"), is(metadata.get("key")));
 
-        c.preparePutRole("test_role2")
+        preparePutRole("test_role2")
                 .cluster("all", "none")
                 .runAs("root", "nobody")
                 .addIndices(new String[]{"index"}, new String[]{"read"}, new String[]{"body", "title"}, null,
                         new BytesArray("{\"query\": {\"match_all\": {}}}"), randomBoolean())
                 .get();
-        c.preparePutRole("test_role3")
+        preparePutRole("test_role3")
                 .cluster("all", "none")
                 .runAs("root", "nobody")
                 .addIndices(new String[]{"index"}, new String[]{"read"}, new String[]{"body", "title"}, null,
@@ -223,37 +228,36 @@ public void testAddAndGetRole() throws Exception {
                 .get();
 
         logger.info("--> retrieving all roles");
-        GetRolesResponse allRolesResp = c.prepareGetRoles().get();
+        GetRolesResponse allRolesResp = new GetRolesRequestBuilder(client()).get();
         assertTrue("roles should exist", allRolesResp.hasRoles());
         assertEquals("should be " + (3 + existing) + " roles total", 3 + existing, allRolesResp.roles().length);
 
         logger.info("--> retrieving test_role and test_role3");
-        GetRolesResponse someRolesResp = c.prepareGetRoles().names("test_role", "test_role3").get();
+        GetRolesResponse someRolesResp = new GetRolesRequestBuilder(client()).names("test_role", "test_role3").get();
         assertTrue("roles should exist", someRolesResp.hasRoles());
         assertEquals("should be 2 roles total", 2, someRolesResp.roles().length);
 
         logger.info("--> deleting role");
-        DeleteRoleResponse delResp = c.prepareDeleteRole("test_role").get();
+        DeleteRoleResponse delResp = new DeleteRoleRequestBuilder(client()).name("test_role").get();
         assertTrue(delResp.found());
         logger.info("--> retrieving role");
-        GetRolesResponse resp2 = c.prepareGetRoles().names("test_role").get();
+        GetRolesResponse resp2 = new GetRolesRequestBuilder(client()).names("test_role").get();
         assertFalse("role should not exist after being deleted", resp2.hasRoles());
     }
 
     public void testAddUserAndRoleThenAuth() throws Exception {
-        SecurityClient c = securityClient();
         logger.error("--> creating role");
-        c.preparePutRole("test_role")
+        preparePutRole("test_role")
                 .cluster("all")
                 .addIndices(new String[] { "*" }, new String[] { "read" }, new String[]{"body", "title"}, null,
                         new BytesArray("{\"match_all\": {}}"), randomBoolean())
                 .get();
         logger.error("--> creating user");
-        c.preparePutUser("joe", "s3krit".toCharArray(), hasher, "test_role").get();
+        preparePutUser("joe", "s3krit", hasher, "test_role").get();
         logger.error("--> waiting for .security index");
         ensureGreen(SECURITY_MAIN_ALIAS);
         logger.info("--> retrieving user");
-        GetUsersResponse resp = c.prepareGetUsers("joe").get();
+        GetUsersResponse resp = new GetUsersRequestBuilder(client()).usernames("joe").get();
         assertTrue("user should exist", resp.hasUsers());
 
         createIndex("idx");
@@ -261,20 +265,19 @@ public void testAddUserAndRoleThenAuth() throws Exception {
         // Index a document with the default test user
         client().prepareIndex("idx", "doc", "1").setSource("body", "foo").setRefreshPolicy(IMMEDIATE).get();
 
-        String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray()));
+        String token = basicAuthHeaderValue("joe", new SecureString("s3krit"));
         SearchResponse searchResp = client().filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("idx").get();
 
         assertEquals(1L, searchResp.getHits().getTotalHits().value);
     }
 
     public void testUpdatingUserAndAuthentication() throws Exception {
-        SecurityClient c = securityClient();
         logger.error("--> creating user");
-        c.preparePutUser("joe", "s3krit".toCharArray(), hasher, SecuritySettingsSource.TEST_ROLE).get();
+        preparePutUser("joe", "s3krit", hasher, SecuritySettingsSource.TEST_ROLE).get();
         logger.error("--> waiting for .security index");
         ensureGreen(SECURITY_MAIN_ALIAS);
         logger.info("--> retrieving user");
-        GetUsersResponse resp = c.prepareGetUsers("joe").get();
+        GetUsersResponse resp = new GetUsersRequestBuilder(client()).usernames("joe").get();
         assertTrue("user should exist", resp.hasUsers());
         assertThat(resp.users()[0].roles(), arrayContaining(SecuritySettingsSource.TEST_ROLE));
 
@@ -282,12 +285,12 @@ public void testUpdatingUserAndAuthentication() throws Exception {
         ensureGreen("idx");
         // Index a document with the default test user
         client().prepareIndex("idx", "doc", "1").setSource("body", "foo").setRefreshPolicy(IMMEDIATE).get();
-        String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray()));
+        String token = basicAuthHeaderValue("joe", new SecureString("s3krit"));
         SearchResponse searchResp = client().filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("idx").get();
 
         assertEquals(1L, searchResp.getHits().getTotalHits().value);
 
-        c.preparePutUser("joe", "s3krit2".toCharArray(), hasher, SecuritySettingsSource.TEST_ROLE).get();
+        preparePutUser("joe", "s3krit2", hasher, SecuritySettingsSource.TEST_ROLE).get();
 
         try {
             client().filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("idx").get();
@@ -297,20 +300,19 @@ public void testUpdatingUserAndAuthentication() throws Exception {
             assertThat(e.status(), is(RestStatus.UNAUTHORIZED));
         }
 
-        token = basicAuthHeaderValue("joe", new SecureString("s3krit2".toCharArray()));
+        token = basicAuthHeaderValue("joe", new SecureString("s3krit2"));
         searchResp = client().filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("idx").get();
         assertEquals(1L, searchResp.getHits().getTotalHits().value);
     }
 
     public void testCreateDeleteAuthenticate() {
-        SecurityClient c = securityClient();
         logger.error("--> creating user");
-        c.preparePutUser("joe", "s3krit".toCharArray(), hasher,
+        preparePutUser("joe", "s3krit", hasher,
             SecuritySettingsSource.TEST_ROLE).get();
         logger.error("--> waiting for .security index");
         ensureGreen(SECURITY_MAIN_ALIAS);
         logger.info("--> retrieving user");
-        GetUsersResponse resp = c.prepareGetUsers("joe").get();
+        GetUsersResponse resp = new GetUsersRequestBuilder(client()).usernames("joe").get();
         assertTrue("user should exist", resp.hasUsers());
         assertThat(resp.users()[0].roles(), arrayContaining(SecuritySettingsSource.TEST_ROLE));
 
@@ -318,12 +320,12 @@ public void testCreateDeleteAuthenticate() {
         ensureGreen("idx");
         // Index a document with the default test user
         client().prepareIndex("idx", "doc", "1").setSource("body", "foo").setRefreshPolicy(IMMEDIATE).get();
-        String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray()));
+        String token = basicAuthHeaderValue("joe", new SecureString("s3krit"));
         SearchResponse searchResp = client().filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("idx").get();
 
         assertEquals(1L, searchResp.getHits().getTotalHits().value);
 
-        DeleteUserResponse response = c.prepareDeleteUser("joe").get();
+        DeleteUserResponse response = new DeleteUserRequestBuilder(client()).username("joe").get();
         assertThat(response.found(), is(true));
         try {
             client().filterWithHeader(Collections.singletonMap("Authorization", token)).prepareSearch("idx").get();
@@ -336,24 +338,23 @@ public void testCreateDeleteAuthenticate() {
 
     public void testCreateAndUpdateRole() {
         final boolean authenticate = randomBoolean();
-        SecurityClient c = securityClient();
         logger.error("--> creating role");
-        c.preparePutRole("test_role")
+        preparePutRole("test_role")
                 .cluster("all")
                 .addIndices(new String[]{"*"}, new String[]{"read"}, new String[]{"body", "title"}, null,
                         new BytesArray("{\"match_all\": {}}"), randomBoolean())
                 .get();
         logger.error("--> creating user");
-        c.preparePutUser("joe", "s3krit".toCharArray(), hasher, "test_role").get();
+        preparePutUser("joe", "s3krit", hasher, "test_role").get();
         logger.error("--> waiting for .security index");
         ensureGreen(SECURITY_MAIN_ALIAS);
 
         if (authenticate) {
-            final String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray()));
+            final String token = basicAuthHeaderValue("joe", new SecureString("s3krit"));
             ClusterHealthResponse response = client().filterWithHeader(Collections.singletonMap("Authorization", token)).admin().cluster()
                     .prepareHealth().get();
             assertFalse(response.isTimedOut());
-            c.preparePutRole("test_role")
+            preparePutRole("test_role")
                     .cluster("none")
                     .addIndices(new String[]{"*"}, new String[]{"read"}, new String[]{"body", "title"}, null,
                             new BytesArray("{\"match_all\": {}}"), randomBoolean())
@@ -368,17 +369,17 @@ public void testCreateAndUpdateRole() {
             }
         } else {
             final TransportRequest request = mock(TransportRequest.class);
-            GetRolesResponse getRolesResponse = c.prepareGetRoles().names("test_role").get();
+            GetRolesResponse getRolesResponse = new GetRolesRequestBuilder(client()).names("test_role").get();
             assertTrue("test_role does not exist!", getRolesResponse.hasRoles());
             assertTrue("any cluster permission should be authorized",
                     Role.builder(getRolesResponse.roles()[0], null).build().cluster().check("cluster:admin/foo", request));
 
-            c.preparePutRole("test_role")
+            preparePutRole("test_role")
                     .cluster("none")
                     .addIndices(new String[]{"*"}, new String[]{"read"}, new String[]{"body", "title"}, null,
                             new BytesArray("{\"match_all\": {}}"), randomBoolean())
                     .get();
-            getRolesResponse = c.prepareGetRoles().names("test_role").get();
+            getRolesResponse = new GetRolesRequestBuilder(client()).names("test_role").get();
             assertTrue("test_role does not exist!", getRolesResponse.hasRoles());
 
             assertFalse("no cluster permission should be authorized",
@@ -388,12 +389,12 @@ public void testCreateAndUpdateRole() {
 
     public void testSnapshotDeleteRestore() {
         logger.error("--> creating role");
-        securityClient().preparePutRole("test_role")
-                .cluster("all")
-                .addIndices(new String[]{"*"}, new String[]{"create_index"}, null, null, null, true)
-                .get();
+        preparePutRole("test_role")
+            .cluster("all")
+            .addIndices(new String[]{"*"}, new String[]{"create_index"}, null, null, null, true)
+            .get();
         logger.error("--> creating user");
-        securityClient().preparePutUser("joe", "s3krit".toCharArray(), hasher, "test_role", "snapshot_user").get();
+        preparePutUser("joe", "s3krit", hasher, "test_role", "snapshot_user").get();
         logger.error("--> waiting for .security index");
         ensureGreen(SECURITY_MAIN_ALIAS);
         logger.info("-->  creating repository");
@@ -403,7 +404,7 @@ public void testSnapshotDeleteRestore() {
                             .put("location", randomRepoPath())
                             .put("compress", randomBoolean())
                             .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
-        final String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray()));
+        final String token = basicAuthHeaderValue("joe", new SecureString("s3krit"));
         // joe can snapshot all indices, including '.security'
         SnapshotInfo snapshotInfo = client().filterWithHeader(Collections.singletonMap("Authorization", token)).admin().cluster()
                 .prepareCreateSnapshot("test-repo", "test-snap-1")
@@ -415,15 +416,15 @@ public void testSnapshotDeleteRestore() {
         assertThat(snapshotInfo.indices(), contains(INTERNAL_SECURITY_MAIN_INDEX_7));
         deleteSecurityIndex();
         // the realm cache should clear itself but we don't wish to race it
-        securityClient().prepareClearRealmCache().get();
+        new ClearRolesCacheRequestBuilder(client()).get();
         // authn fails
         final ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> client()
                 .filterWithHeader(Collections.singletonMap("Authorization", token)).admin().indices().prepareCreate("idx").get());
         assertThat(e.status(), is(RestStatus.UNAUTHORIZED));
         // users and roles are missing
-        GetUsersResponse getUsersResponse = securityClient().prepareGetUsers("joe").get();
+        GetUsersResponse getUsersResponse = new GetUsersRequestBuilder(client()).usernames("joe").get();
         assertThat(getUsersResponse.users().length, is(0));
-        GetRolesResponse getRolesResponse = securityClient().prepareGetRoles("test_role").get();
+        GetRolesResponse getRolesResponse = new GetRolesRequestBuilder(client()).names("test_role").get();
         assertThat(getRolesResponse.roles().length, is(0));
         // restore
         RestoreSnapshotResponse response = client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap-1")
@@ -431,12 +432,12 @@ public void testSnapshotDeleteRestore() {
         assertThat(response.status(), equalTo(RestStatus.OK));
         assertThat(response.getRestoreInfo().indices(), contains(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7));
         // the realm cache should clear itself but we don't wish to race it
-        securityClient().prepareClearRealmCache().get();
+        new ClearRealmCacheRequestBuilder(client()).get();
         // users and roles are retrievable
-        getUsersResponse = securityClient().prepareGetUsers("joe").get();
+        getUsersResponse = new GetUsersRequestBuilder(client()).usernames("joe").get();
         assertThat(getUsersResponse.users().length, is(1));
         assertThat(Arrays.asList(getUsersResponse.users()[0].roles()), contains("test_role", "snapshot_user"));
-        getRolesResponse = securityClient().prepareGetRoles("test_role").get();
+        getRolesResponse = new GetRolesRequestBuilder(client()).names("test_role").get();
         assertThat(getRolesResponse.roles().length, is(1));
         assertThat(Arrays.asList(getRolesResponse.roles()[0].getClusterPrivileges()), contains("all"));
         assertThat(getRolesResponse.roles()[0].getIndicesPrivileges().length, is(1));
@@ -450,22 +451,21 @@ public void testSnapshotDeleteRestore() {
     }
 
     public void testAuthenticateWithDeletedRole() {
-        SecurityClient c = securityClient();
         logger.error("--> creating role");
-        c.preparePutRole("test_role")
+        preparePutRole("test_role")
                 .cluster("all")
                 .addIndices(new String[]{"*"}, new String[]{"read"}, new String[]{"body", "title"}, null,
                         new BytesArray("{\"match_all\": {}}"), randomBoolean())
                 .get();
-        c.preparePutUser("joe", "s3krit".toCharArray(), hasher, "test_role").get();
+        preparePutUser("joe", "s3krit", hasher, "test_role").get();
         logger.error("--> waiting for .security index");
         ensureGreen(SECURITY_MAIN_ALIAS);
 
-        final String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray()));
+        final String token = basicAuthHeaderValue("joe", new SecureString("s3krit"));
         ClusterHealthResponse response = client().filterWithHeader(Collections.singletonMap("Authorization", token)).admin().cluster()
                 .prepareHealth().get();
         assertFalse(response.isTimedOut());
-        c.prepareDeleteRole("test_role").get();
+        new DeleteRoleRequestBuilder(client()).name("test_role").get();
         if (anonymousEnabled && roleExists) {
             assertNoTimeout(
                     client().filterWithHeader(Collections.singletonMap("Authorization", token)).admin().cluster().prepareHealth().get());
@@ -477,40 +477,38 @@ public void testAuthenticateWithDeletedRole() {
     }
 
     public void testPutUserWithoutPassword() {
-        SecurityClient client = securityClient();
         // create some roles
-        client.preparePutRole("admin_role")
+        preparePutRole("admin_role")
                 .cluster("all")
                 .addIndices(new String[]{"*"}, new String[]{"all"}, null, null, null, randomBoolean())
                 .get();
-        client.preparePutRole("read_role")
+        preparePutRole("read_role")
                 .cluster("none")
                 .addIndices(new String[]{"*"}, new String[]{"read"}, null, null, null, randomBoolean())
                 .get();
 
-        assertThat(client.prepareGetUsers("joes").get().hasUsers(), is(false));
+        assertThat(new GetUsersRequestBuilder(client()).usernames("joes").get().hasUsers(), is(false));
         // check that putting a user without a password fails if the user doesn't exist
         try {
-            client.preparePutUser("joe", null, hasher, "admin_role").get();
+            preparePutUser("joe", null, hasher, "admin_role").get();
             fail("cannot create a user without a password");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("password must be specified"));
         }
 
-        assertThat(client.prepareGetUsers("joes").get().hasUsers(), is(false));
+        assertThat(new GetUsersRequestBuilder(client()).usernames("joes").get().hasUsers(), is(false));
 
         // create joe with a password and verify the user works
-        client.preparePutUser("joe", SecuritySettingsSourceField.TEST_PASSWORD.toCharArray(),
-            hasher, "admin_role").get();
-        assertThat(client.prepareGetUsers("joe").get().hasUsers(), is(true));
+        preparePutUser("joe", SecuritySettingsSourceField.TEST_PASSWORD, hasher, "admin_role").get();
+        assertThat(new GetUsersRequestBuilder(client()).usernames("joe").get().hasUsers(), is(true));
         final String token = basicAuthHeaderValue("joe", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING);
         ClusterHealthResponse response = client().filterWithHeader(Collections.singletonMap("Authorization", token)).admin().cluster()
                 .prepareHealth().get();
         assertFalse(response.isTimedOut());
 
         // modify joe without sending the password
-        client.preparePutUser("joe", null, hasher, "read_role").fullName("Joe Smith").get();
-        GetUsersResponse getUsersResponse = client.prepareGetUsers("joe").get();
+        preparePutUser("joe", null, hasher, "read_role").fullName("Joe Smith").get();
+        GetUsersResponse getUsersResponse = new GetUsersRequestBuilder(client()).usernames("joe").get();
         assertThat(getUsersResponse.hasUsers(), is(true));
         assertThat(getUsersResponse.users().length, is(1));
         User joe = getUsersResponse.users()[0];
@@ -530,9 +528,9 @@ public void testPutUserWithoutPassword() {
 
         // update the user with password and admin role again
         String secondPassword = SecuritySettingsSourceField.TEST_PASSWORD + "2";
-        client.preparePutUser("joe", secondPassword.toCharArray(), hasher, "admin_role").
+        preparePutUser("joe", secondPassword, hasher, "admin_role").
             fullName("Joe Smith").get();
-        getUsersResponse = client.prepareGetUsers("joe").get();
+        getUsersResponse = new GetUsersRequestBuilder(client()).usernames("joe").get();
         assertThat(getUsersResponse.hasUsers(), is(true));
         assertThat(getUsersResponse.users().length, is(1));
         joe = getUsersResponse.users()[0];
@@ -557,10 +555,8 @@ public void testPutUserWithoutPassword() {
     }
 
     public void testCannotCreateUserWithShortPassword() throws Exception {
-        SecurityClient client = securityClient();
         try {
-            client.preparePutUser("joe", randomAlphaOfLengthBetween(0, 5).toCharArray(), hasher,
-                "admin_role").get();
+            preparePutUser("joe", randomAlphaOfLengthBetween(0, 5), hasher, "admin_role").get();
             fail("cannot create a user without a password < 6 characters");
         } catch (IllegalArgumentException v) {
             assertThat(v.getMessage().contains("password"), is(true));
@@ -568,10 +564,8 @@ public void testCannotCreateUserWithShortPassword() throws Exception {
     }
 
     public void testCannotCreateUserWithInvalidCharactersInName() throws Exception {
-        SecurityClient client = securityClient();
         IllegalArgumentException v = expectThrows(IllegalArgumentException.class,
-            () -> client.preparePutUser("fóóbár", "my-am@zing-password".toCharArray(), hasher,
-                "admin_role").get()
+            () -> preparePutUser("fóóbár", "my-am@zing-password", hasher, "admin_role").get()
         );
         assertThat(v.getMessage(), containsString("names must be"));
     }
@@ -579,15 +573,14 @@ public void testCannotCreateUserWithInvalidCharactersInName() throws Exception {
     public void testUsersAndRolesDoNotInterfereWithIndicesStats() throws Exception {
         client().prepareIndex("foo", "bar").setSource("ignore", "me").get();
 
-        SecurityClient client = securityClient();
         if (randomBoolean()) {
-            client.preparePutUser("joe", "s3krit".toCharArray(), hasher,
+            preparePutUser("joe", "s3krit", hasher,
                 SecuritySettingsSource.TEST_ROLE).get();
         } else {
-            client.preparePutRole("read_role")
-                    .cluster("none")
-                    .addIndices(new String[]{"*"}, new String[]{"read"}, null, null, null, randomBoolean())
-                    .get();
+            preparePutRole("read_role")
+                .cluster("none")
+                .addIndices(new String[]{"*"}, new String[]{"read"}, null, null, null, randomBoolean())
+                .get();
         }
 
         IndicesStatsResponse response = client().admin().indices().prepareStats("foo", SECURITY_MAIN_ALIAS).get();
@@ -601,43 +594,41 @@ public void testUsersAndRolesDoNotInterfereWithIndicesStats() throws Exception {
     public void testOperationsOnReservedUsers() throws Exception {
         final String username = randomFrom(ElasticUser.NAME, KibanaUser.NAME);
         IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
-                () -> securityClient().preparePutUser(username, randomBoolean() ? SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()
-                    : null, hasher, "admin").get());
+                () -> preparePutUser(username, randomBoolean() ? SecuritySettingsSourceField.TEST_PASSWORD : null, hasher, "admin").get());
         assertThat(exception.getMessage(), containsString("user [" + username + "] is reserved"));
 
         exception = expectThrows(IllegalArgumentException.class,
-                () -> securityClient().prepareDeleteUser(username).get());
+                () -> new DeleteUserRequestBuilder(client()).username(username).get());
         assertThat(exception.getMessage(), containsString("user [" + username + "] is reserved"));
 
         exception = expectThrows(IllegalArgumentException.class,
-                () -> securityClient().prepareDeleteUser(AnonymousUser.DEFAULT_ANONYMOUS_USERNAME).get());
+                () -> new DeleteUserRequestBuilder(client()).username(AnonymousUser.DEFAULT_ANONYMOUS_USERNAME).get());
         assertThat(exception.getMessage(), containsString("user [" + AnonymousUser.DEFAULT_ANONYMOUS_USERNAME + "] is anonymous"));
 
         exception = expectThrows(IllegalArgumentException.class,
-            () -> securityClient().prepareChangePassword(AnonymousUser.DEFAULT_ANONYMOUS_USERNAME, "foobar".toCharArray(),
-                hasher).get());
+            () -> new ChangePasswordRequestBuilder(client()).username(AnonymousUser.DEFAULT_ANONYMOUS_USERNAME)
+                .password("foobar".toCharArray(), hasher).get());
         assertThat(exception.getMessage(), containsString("user [" + AnonymousUser.DEFAULT_ANONYMOUS_USERNAME + "] is anonymous"));
 
         exception = expectThrows(IllegalArgumentException.class,
-            () -> securityClient().preparePutUser(AnonymousUser.DEFAULT_ANONYMOUS_USERNAME, "foobar".toCharArray(),
-                hasher).get());
+            () -> preparePutUser(AnonymousUser.DEFAULT_ANONYMOUS_USERNAME, "foobar", hasher).get());
         assertThat(exception.getMessage(), containsString("user [" + AnonymousUser.DEFAULT_ANONYMOUS_USERNAME + "] is anonymous"));
 
         exception = expectThrows(IllegalArgumentException.class,
-            () -> securityClient().preparePutUser(SystemUser.NAME, "foobar".toCharArray(), hasher).get());
+            () -> preparePutUser(SystemUser.NAME, "foobar", hasher).get());
         assertThat(exception.getMessage(), containsString("user [" + SystemUser.NAME + "] is internal"));
 
         exception = expectThrows(IllegalArgumentException.class,
-            () -> securityClient().prepareChangePassword(SystemUser.NAME, "foobar".toCharArray(),
-                hasher).get());
+            () -> new ChangePasswordRequestBuilder(client()).username(SystemUser.NAME)
+                .password("foobar".toCharArray(), hasher).get());
         assertThat(exception.getMessage(), containsString("user [" + SystemUser.NAME + "] is internal"));
 
         exception = expectThrows(IllegalArgumentException.class,
-                () -> securityClient().prepareDeleteUser(SystemUser.NAME).get());
+                () -> new DeleteUserRequestBuilder(client()).username(SystemUser.NAME).get());
         assertThat(exception.getMessage(), containsString("user [" + SystemUser.NAME + "] is internal"));
 
         // get should work
-        GetUsersResponse response = securityClient().prepareGetUsers(username).get();
+        GetUsersResponse response = new GetUsersRequestBuilder(client()).usernames(username).get();
         assertThat(response.hasUsers(), is(true));
         assertThat(response.users()[0].principal(), is(username));
 
@@ -656,30 +647,30 @@ public void testOperationsOnReservedUsers() throws Exception {
     public void testOperationsOnReservedRoles() throws Exception {
         final String name = randomFrom(ReservedRolesStore.names());
         IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
-                () -> securityClient().preparePutRole(name).cluster("monitor").get());
+                () -> preparePutRole(name).cluster("monitor").get());
         assertThat(exception.getMessage(), containsString("role [" + name + "] is reserved"));
 
         exception = expectThrows(IllegalArgumentException.class,
-                () -> securityClient().prepareDeleteRole(name).get());
+                () -> new DeleteRoleRequestBuilder(client()).name(name).get());
         assertThat(exception.getMessage(), containsString("role [" + name + "] is reserved"));
 
         // get role is allowed
-        GetRolesResponse response = securityClient().prepareGetRoles(name).get();
+        GetRolesResponse response = new GetRolesRequestBuilder(client()).names(name).get();
         assertThat(response.hasRoles(), is(true));
         assertThat(response.roles()[0].getName(), is(name));
     }
 
     public void testCreateAndChangePassword() throws Exception {
-        securityClient().preparePutUser("joe", "s3krit".toCharArray(), hasher,
+        preparePutUser("joe", "s3krit", hasher,
             SecuritySettingsSource.TEST_ROLE).get();
-        final String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray()));
+        final String token = basicAuthHeaderValue("joe", new SecureString("s3krit"));
         ClusterHealthResponse response = client().filterWithHeader(Collections.singletonMap("Authorization", token))
                 .admin().cluster().prepareHealth().get();
         assertThat(response.isTimedOut(), is(false));
 
-        ChangePasswordResponse passwordResponse = securityClient(
-                client().filterWithHeader(Collections.singletonMap("Authorization", token)))
-            .prepareChangePassword("joe", SecuritySettingsSourceField.TEST_PASSWORD.toCharArray(), hasher).get();
+        ChangePasswordResponse passwordResponse =
+            new ChangePasswordRequestBuilder(client().filterWithHeader(Collections.singletonMap("Authorization", token)))
+                .username("joe").password(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray(), hasher).get();
         assertThat(passwordResponse, notNullValue());
 
 
@@ -709,8 +700,7 @@ public void testRolesUsageStats() throws Exception {
 
         final boolean fls = randomBoolean();
         final boolean dls = randomBoolean();
-        SecurityClient client = new SecurityClient(client());
-        PutRoleResponse putRoleResponse = client.preparePutRole("admin_role")
+        PutRoleResponse putRoleResponse = new PutRoleRequestBuilder(client()).name("admin_role")
                 .cluster("all")
                 .addIndices(new String[]{"*"}, new String[]{"all"}, null, null, null, randomBoolean())
                 .get();
@@ -728,7 +718,7 @@ public void testRolesUsageStats() throws Exception {
                 grantedFields = null;
                 deniedFields = fields;
             }
-            roleResponse = client.preparePutRole("admin_role_fls")
+            roleResponse = new PutRoleRequestBuilder(client()).name("admin_role_fls")
                     .cluster("all")
                     .addIndices(new String[]{"*"}, new String[]{"all"}, grantedFields, deniedFields, null, randomBoolean())
                     .get();
@@ -737,7 +727,7 @@ public void testRolesUsageStats() throws Exception {
         }
 
         if (dls) {
-            PutRoleResponse roleResponse = client.preparePutRole("admin_role_dls")
+            PutRoleResponse roleResponse = new PutRoleRequestBuilder(client()).name("admin_role_dls")
                     .cluster("all")
                     .addIndices(new String[]{"*"}, new String[]{"all"}, null, null, new BytesArray("{\"match_all\": {}}"), randomBoolean())
                     .get();
@@ -745,7 +735,7 @@ public void testRolesUsageStats() throws Exception {
             roles++;
         }
 
-        client.prepareClearRolesCache().get();
+        new ClearRolesCacheRequestBuilder(client()).get();
 
         future = new PlainActionFuture<>();
         rolesStore.usageStats(future);
@@ -757,10 +747,8 @@ public void testRolesUsageStats() throws Exception {
 
     public void testRealmUsageStats() {
         final int numNativeUsers = scaledRandomIntBetween(1, 32);
-        SecurityClient securityClient = new SecurityClient(client());
         for (int i = 0; i < numNativeUsers; i++) {
-            securityClient.preparePutUser("joe" + i, "s3krit".toCharArray(), hasher,
-                "superuser").get();
+            preparePutUser("joe" + i, "s3krit", hasher, "superuser").get();
         }
 
         XPackUsageResponse response = new XPackUsageRequestBuilder(client()).get();
@@ -779,55 +767,53 @@ public void testRealmUsageStats() {
     }
 
     public void testSetEnabled() throws Exception {
-
-        securityClient().preparePutUser("joe", "s3krit".toCharArray(), hasher,
+        preparePutUser("joe", "s3krit", hasher,
             SecuritySettingsSource.TEST_ROLE).get();
-        final String token = basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray()));
+        final String token = basicAuthHeaderValue("joe", new SecureString("s3krit"));
         ClusterHealthResponse response = client().filterWithHeader(Collections.singletonMap("Authorization", token))
                 .admin().cluster().prepareHealth().get();
         assertThat(response.isTimedOut(), is(false));
 
-        securityClient(client()).prepareSetEnabled("joe", false).get();
+        new SetEnabledRequestBuilder(client()).username("joe").enabled(false).get();
 
         ElasticsearchSecurityException expected = expectThrows(ElasticsearchSecurityException.class,
                 () -> client().filterWithHeader(Collections.singletonMap("Authorization", token)).admin().cluster().prepareHealth().get());
         assertThat(expected.status(), is(RestStatus.UNAUTHORIZED));
 
-        securityClient(client()).prepareSetEnabled("joe", true).get();
+        new SetEnabledRequestBuilder(client()).username("joe").enabled(true).get();
 
         response = client().filterWithHeader(Collections.singletonMap("Authorization", token)).admin().cluster().prepareHealth().get();
         assertThat(response.isTimedOut(), is(false));
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> securityClient(client()).prepareSetEnabled("not_a_real_user", false).get());
+                () -> new SetEnabledRequestBuilder(client()).username("not_a_real_user").enabled(false).get());
         assertThat(e.getMessage(), containsString("only existing users can be disabled"));
     }
 
     public void testNegativeLookupsThenCreateRole() throws Exception {
-        SecurityClient securityClient = new SecurityClient(client());
-        securityClient.preparePutUser("joe", "s3krit".toCharArray(), hasher, "unknown_role").get();
+        preparePutUser("joe", "s3krit", hasher, "unknown_role").get();
 
         final int negativeLookups = scaledRandomIntBetween(1, 10);
         for (int i = 0; i < negativeLookups; i++) {
             if (anonymousEnabled && roleExists) {
                 ClusterHealthResponse response = client()
                         .filterWithHeader(Collections.singletonMap("Authorization",
-                                basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray()))))
+                                basicAuthHeaderValue("joe", new SecureString("s3krit"))))
                         .admin().cluster().prepareHealth().get();
                 assertNoTimeout(response);
             } else {
                 ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, () -> client()
                         .filterWithHeader(Collections.singletonMap("Authorization",
-                                basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray()))))
+                                basicAuthHeaderValue("joe", new SecureString("s3krit"))))
                         .admin().cluster().prepareHealth().get());
                 assertThat(e.status(), is(RestStatus.FORBIDDEN));
             }
         }
 
-        securityClient.preparePutRole("unknown_role").cluster("all").get();
+        preparePutRole("unknown_role").cluster("all").get();
         ClusterHealthResponse response = client()
                 .filterWithHeader(Collections.singletonMap("Authorization",
-                        basicAuthHeaderValue("joe", new SecureString("s3krit".toCharArray()))))
+                        basicAuthHeaderValue("joe", new SecureString("s3krit"))))
                 .admin().cluster().prepareHealth().get();
         assertNoTimeout(response);
     }
@@ -841,10 +827,9 @@ public void testNegativeLookupsThenCreateRole() throws Exception {
      * the loader returned a null value, while the other caller(s) would get a null value unexpectedly
      */
     public void testConcurrentRunAs() throws Exception {
-        securityClient().preparePutUser("joe", "s3krit".toCharArray(), hasher, SecuritySettingsSource
-            .TEST_ROLE).get();
-        securityClient().preparePutUser("executor", "s3krit".toCharArray(), hasher, "superuser").get();
-        final String token = basicAuthHeaderValue("executor", new SecureString("s3krit".toCharArray()));
+        preparePutUser("joe", "s3krit", hasher, SecuritySettingsSource.TEST_ROLE).get();
+        preparePutUser("executor", "s3krit", hasher, "superuser").get();
+        final String token = basicAuthHeaderValue("executor", new SecureString("s3krit"));
         final Client client = client().filterWithHeader(Map.of(
                 "Authorization", token,
                 "es-security-runas-user", "joe"));
@@ -862,6 +847,7 @@ public void testConcurrentRunAs() throws Exception {
                         assertNoTimeout(response);
                     }
                 } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
                 }
             }));
         }
@@ -874,4 +860,15 @@ public void testConcurrentRunAs() throws Exception {
             thread.join();
         }
     }
+
+    private PutUserRequestBuilder preparePutUser(String username, String password, Hasher hasher, String... roles) {
+        return new PutUserRequestBuilder(client())
+            .username(username)
+            .password(password == null ? null : new SecureString(password.toCharArray()), hasher)
+            .roles(roles);
+    }
+
+    private PutRoleRequestBuilder preparePutRole(String name) {
+        return new PutRoleRequestBuilder(client()).name(name);
+    }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java
index 28625f20627e1..a8682c4e21d84 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.security.authc.esnative;
 
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.env.TestEnvironment;
@@ -27,7 +28,8 @@ public class NativeRealmTests extends ESTestCase {
         RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7);
 
     private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) {
-        return new SecurityIndexManager.State(Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus);
+        return new SecurityIndexManager.State(
+            Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus, IndexMetaData.State.OPEN);
     }
 
     public void testCacheClearOnIndexHealthChange() {
@@ -72,7 +74,7 @@ void clearCache() {
 
         // green to yellow or yellow to green
         previousState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW));
-        currentState = dummyState(previousState.indexStatus == ClusterHealthStatus.GREEN ?
+        currentState = dummyState(previousState.indexHealth == ClusterHealthStatus.GREEN ?
             ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN);
         nativeRealm.onSecurityIndexStateChange(previousState, currentState);
         assertEquals(expectedInvalidation, numInvalidation.get());
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java
index 53eb3fc0bdbc4..db591f98b6b90 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java
@@ -5,7 +5,7 @@
  */
 package org.elasticsearch.xpack.security.authc.esnative;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -77,7 +77,7 @@ public void setupMocks() {
 
             @Override
             protected <Request extends ActionRequest, Response extends ActionResponse>
-            void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+            void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
                 requests.add(new Tuple<>(request, listener));
             }
         };
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmIntegTests.java
index 59612d6227a71..137ba2279d96e 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmIntegTests.java
@@ -7,12 +7,15 @@
 
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.security.ChangePasswordRequest;
+import org.elasticsearch.client.security.DisableUserRequest;
+import org.elasticsearch.client.security.EnableUserRequest;
+import org.elasticsearch.client.security.RefreshPolicy;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.NativeRealmIntegTestCase;
-import org.elasticsearch.xpack.core.security.action.user.ChangePasswordResponse;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.core.security.user.APMSystemUser;
 import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
 import org.elasticsearch.xpack.core.security.user.ElasticUser;
@@ -21,14 +24,15 @@
 import org.elasticsearch.xpack.core.security.user.RemoteMonitoringUser;
 import org.junit.BeforeClass;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 
 import static java.util.Collections.singletonMap;
+import static org.elasticsearch.test.SecuritySettingsSource.SECURITY_REQUEST_OPTIONS;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
 
 /**
  * Integration tests for the built in realm
@@ -51,6 +55,11 @@ public Settings nodeSettings(int nodeOrdinal) {
         return settings;
     }
 
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false;
+    }
+
     public void testAuthenticate() {
         final List<String> usernames = Arrays.asList(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME,
             BeatsSystemUser.NAME, APMSystemUser.NAME, RemoteMonitoringUser.NAME);
@@ -70,12 +79,12 @@ public void testAuthenticate() {
      * Enabling a user forces a doc to be written to the security index, and "user doc with empty password" has a special case code in
      * the reserved realm.
      */
-    public void testAuthenticateAfterEnablingUser() {
-        final SecurityClient c = securityClient();
+    public void testAuthenticateAfterEnablingUser() throws IOException {
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
         final List<String> usernames = Arrays.asList(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME,
             BeatsSystemUser.NAME, APMSystemUser.NAME, RemoteMonitoringUser.NAME);
         for (String username : usernames) {
-            c.prepareSetEnabled(username, true).get();
+            restClient.security().enableUser(new EnableUserRequest(username, RefreshPolicy.getDefault()), SECURITY_REQUEST_OPTIONS);
             ClusterHealthResponse response = client()
                     .filterWithHeader(singletonMap("Authorization", basicAuthHeaderValue(username, getReservedPassword())))
                     .admin()
@@ -87,7 +96,7 @@ public void testAuthenticateAfterEnablingUser() {
         }
     }
 
-    public void testChangingPassword() {
+    public void testChangingPassword() throws IOException {
         String username = randomFrom(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME,
             BeatsSystemUser.NAME, APMSystemUser.NAME, RemoteMonitoringUser.NAME);
         final char[] newPassword = "supersecretvalue".toCharArray();
@@ -102,10 +111,11 @@ public void testChangingPassword() {
             assertThat(response.getClusterName(), is(cluster().getClusterName()));
         }
 
-        ChangePasswordResponse response = securityClient()
-            .prepareChangePassword(username, Arrays.copyOf(newPassword, newPassword.length), hasher)
-                .get();
-        assertThat(response, notNullValue());
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
+        final boolean changed = restClient.security()
+            .changePassword(new ChangePasswordRequest(username, Arrays.copyOf(newPassword, newPassword.length), RefreshPolicy.IMMEDIATE),
+                SECURITY_REQUEST_OPTIONS);
+        assertTrue(changed);
 
         ElasticsearchSecurityException elasticsearchSecurityException = expectThrows(ElasticsearchSecurityException.class, () -> client()
                     .filterWithHeader(singletonMap("Authorization", basicAuthHeaderValue(username, getReservedPassword())))
@@ -125,6 +135,7 @@ public void testChangingPassword() {
     }
 
     public void testDisablingUser() throws Exception {
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
         // validate the user works
         ClusterHealthResponse response = client()
                 .filterWithHeader(singletonMap("Authorization", basicAuthHeaderValue(ElasticUser.NAME, getReservedPassword())))
@@ -135,7 +146,10 @@ public void testDisablingUser() throws Exception {
         assertThat(response.getClusterName(), is(cluster().getClusterName()));
 
         // disable user
-        securityClient().prepareSetEnabled(ElasticUser.NAME, false).get();
+        final boolean disabled =
+            restClient.security().disableUser(new DisableUserRequest(ElasticUser.NAME, RefreshPolicy.getDefault()),
+                SECURITY_REQUEST_OPTIONS);
+        assertTrue(disabled);
         ElasticsearchSecurityException elasticsearchSecurityException = expectThrows(ElasticsearchSecurityException.class, () -> client()
                 .filterWithHeader(singletonMap("Authorization", basicAuthHeaderValue(ElasticUser.NAME, getReservedPassword())))
                 .admin()
@@ -145,7 +159,9 @@ public void testDisablingUser() throws Exception {
         assertThat(elasticsearchSecurityException.getMessage(), containsString("authenticate"));
 
         //enable
-        securityClient().prepareSetEnabled(ElasticUser.NAME, true).get();
+        final boolean enabled =
+            restClient.security().enableUser(new EnableUserRequest(ElasticUser.NAME, RefreshPolicy.getDefault()), SECURITY_REQUEST_OPTIONS);
+        assertTrue(enabled);
         response = client()
                 .filterWithHeader(singletonMap("Authorization", basicAuthHeaderValue(ElasticUser.NAME, getReservedPassword())))
                 .admin()
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordToolTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordToolTests.java
index 71685c444ec1c..12080077ddfab 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordToolTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordToolTests.java
@@ -236,8 +236,8 @@ public void testErrorMessagesWhenXPackIsAvailableWithCorrectLicenseAndIsEnabledB
         URL xpackSecurityPluginQueryURL = queryXPackSecurityFeatureConfigURL(url);
 
         Set<FeatureSet> featureSets = new HashSet<>();
-        featureSets.add(new FeatureSet("logstash", null, true, true, null));
-        featureSets.add(new FeatureSet("security", null, true, true, null));
+        featureSets.add(new FeatureSet("logstash", true, true));
+        featureSets.add(new FeatureSet("security", true, true));
         FeatureSetsInfo featureInfos = new FeatureSetsInfo(featureSets);
         XPackInfoResponse xpackInfo = new XPackInfoResponse(null, null, featureInfos);
         String securityPluginQueryResponseBody = null;
@@ -267,8 +267,8 @@ public void testErrorMessagesWhenXPackPluginIsAvailableButNoSecurityLicense() th
                 any(CheckedFunction.class))).thenReturn(httpResponse);
 
         Set<FeatureSet> featureSets = new HashSet<>();
-        featureSets.add(new FeatureSet("logstash", null, true, true, null));
-        featureSets.add(new FeatureSet("security", null, false, false, null));
+        featureSets.add(new FeatureSet("logstash", true, true));
+        featureSets.add(new FeatureSet("security", false, false));
         FeatureSetsInfo featureInfos = new FeatureSetsInfo(featureSets);
         XPackInfoResponse xpackInfo = new XPackInfoResponse(null, null, featureInfos);
         String securityPluginQueryResponseBody = null;
@@ -298,8 +298,8 @@ public void testErrorMessagesWhenXPackPluginIsAvailableWithValidLicenseButDisabl
                 any(CheckedFunction.class))).thenReturn(httpResponse);
 
         Set<FeatureSet> featureSets = new HashSet<>();
-        featureSets.add(new FeatureSet("logstash", null, true, true, null));
-        featureSets.add(new FeatureSet("security", null, true, false, null));
+        featureSets.add(new FeatureSet("logstash", true, true));
+        featureSets.add(new FeatureSet("security", true, false));
         FeatureSetsInfo featureInfos = new FeatureSetsInfo(featureSets);
         XPackInfoResponse xpackInfo = new XPackInfoResponse(null, null, featureInfos);
         String securityPluginQueryResponseBody = null;
@@ -416,7 +416,7 @@ public void testInteractivePasswordsFatFingers() throws Exception {
             while (failCount-- > 0) {
                 String password1 = randomAlphaOfLength(randomIntBetween(3, 10));
                 terminal.addSecretInput(password1);
-                Validation.Error err = Validation.Users.validatePassword(password1.toCharArray());
+                Validation.Error err = Validation.Users.validatePassword(new SecureString(password1.toCharArray()));
                 if (err == null) {
                     // passes strength validation, fail by mismatch
                     terminal.addSecretInput(password1 + "typo");
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryLoadBalancingTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryLoadBalancingTests.java
index 2614a1dc3d923..ff080be728db9 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryLoadBalancingTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/SessionFactoryLoadBalancingTests.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.mocksocket.MockServerSocket;
 import org.elasticsearch.mocksocket.MockSocket;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.common.socket.SocketAccess;
@@ -53,7 +52,6 @@
 /**
  * Tests that the server sets properly load balance connections without throwing exceptions
  */
-@TestLogging("org.elasticsearch.xpack.security.authc.ldap.support:DEBUG")
 public class SessionFactoryLoadBalancingTests extends LdapTestCase {
 
     private ThreadPool threadPool;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticatorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticatorTests.java
index 43b58b8d4b521..5c31ad850be96 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticatorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticatorTests.java
@@ -88,6 +88,7 @@ public class OpenIdConnectAuthenticatorTests extends OpenIdConnectTestCase {
     private Settings globalSettings;
     private Environment env;
     private ThreadContext threadContext;
+    private int callsToReloadJwk;
 
     @Before
     public void setup() {
@@ -95,6 +96,7 @@ public void setup() {
             .put("xpack.security.authc.realms.oidc.oidc-realm.ssl.verification_mode", "certificate").build();
         env = TestEnvironment.newEnvironment(globalSettings);
         threadContext = new ThreadContext(globalSettings);
+        callsToReloadJwk = 0;
     }
 
     @After
@@ -278,6 +280,7 @@ public void testClockSkewIsHonored() throws Exception {
         authenticator.authenticate(token, future);
         JWTClaimsSet claimsSet = future.actionGet();
         assertThat(claimsSet.getSubject(), equalTo(subject));
+        assertThat(callsToReloadJwk, equalTo(0));
     }
 
     public void testImplicitFlowFailsWithExpiredToken() throws Exception {
@@ -317,6 +320,11 @@ public void testImplicitFlowFailsWithExpiredToken() throws Exception {
         assertThat(e.getMessage(), containsString("Failed to parse or validate the ID Token"));
         assertThat(e.getCause(), instanceOf(BadJWTException.class));
         assertThat(e.getCause().getMessage(), containsString("Expired JWT"));
+        if (jwk.getAlgorithm().getName().startsWith("HS")) {
+            assertThat(callsToReloadJwk, equalTo(0));
+        } else {
+            assertThat(callsToReloadJwk, equalTo(1));
+        }
     }
 
     public void testImplicitFlowFailsNotYetIssuedToken() throws Exception {
@@ -356,6 +364,11 @@ public void testImplicitFlowFailsNotYetIssuedToken() throws Exception {
         assertThat(e.getMessage(), containsString("Failed to parse or validate the ID Token"));
         assertThat(e.getCause(), instanceOf(BadJWTException.class));
         assertThat(e.getCause().getMessage(), containsString("JWT issue time ahead of current time"));
+        if (jwk.getAlgorithm().getName().startsWith("HS")) {
+            assertThat(callsToReloadJwk, equalTo(0));
+        } else {
+            assertThat(callsToReloadJwk, equalTo(1));
+        }
     }
 
     public void testImplicitFlowFailsInvalidIssuer() throws Exception {
@@ -394,6 +407,11 @@ public void testImplicitFlowFailsInvalidIssuer() throws Exception {
         assertThat(e.getMessage(), containsString("Failed to parse or validate the ID Token"));
         assertThat(e.getCause(), instanceOf(BadJWTException.class));
         assertThat(e.getCause().getMessage(), containsString("Unexpected JWT issuer"));
+        if (jwk.getAlgorithm().getName().startsWith("HS")) {
+            assertThat(callsToReloadJwk, equalTo(0));
+        } else {
+            assertThat(callsToReloadJwk, equalTo(1));
+        }
     }
 
     public void testImplicitFlowFailsInvalidAudience() throws Exception {
@@ -432,6 +450,11 @@ public void testImplicitFlowFailsInvalidAudience() throws Exception {
         assertThat(e.getMessage(), containsString("Failed to parse or validate the ID Token"));
         assertThat(e.getCause(), instanceOf(BadJWTException.class));
         assertThat(e.getCause().getMessage(), containsString("Unexpected JWT audience"));
+        if (jwk.getAlgorithm().getName().startsWith("HS")) {
+            assertThat(callsToReloadJwk, equalTo(0));
+        } else {
+            assertThat(callsToReloadJwk, equalTo(1));
+        }
     }
 
     public void testAuthenticateImplicitFlowFailsWithForgedRsaIdToken() throws Exception {
@@ -456,6 +479,7 @@ public void testAuthenticateImplicitFlowFailsWithForgedRsaIdToken() throws Excep
         assertThat(e.getMessage(), containsString("Failed to parse or validate the ID Token"));
         assertThat(e.getCause(), instanceOf(BadJWSException.class));
         assertThat(e.getCause().getMessage(), containsString("Signed JWT rejected: Invalid signature"));
+        assertThat(callsToReloadJwk, equalTo(1));
     }
 
     public void testAuthenticateImplicitFlowFailsWithForgedEcsdsaIdToken() throws Exception {
@@ -480,6 +504,7 @@ public void testAuthenticateImplicitFlowFailsWithForgedEcsdsaIdToken() throws Ex
         assertThat(e.getMessage(), containsString("Failed to parse or validate the ID Token"));
         assertThat(e.getCause(), instanceOf(BadJWSException.class));
         assertThat(e.getCause().getMessage(), containsString("Signed JWT rejected: Invalid signature"));
+        assertThat(callsToReloadJwk, equalTo(1));
     }
 
     public void testAuthenticateImplicitFlowFailsWithForgedHmacIdToken() throws Exception {
@@ -503,6 +528,7 @@ public void testAuthenticateImplicitFlowFailsWithForgedHmacIdToken() throws Exce
         assertThat(e.getMessage(), containsString("Failed to parse or validate the ID Token"));
         assertThat(e.getCause(), instanceOf(BadJWSException.class));
         assertThat(e.getCause().getMessage(), containsString("Signed JWT rejected: Invalid signature"));
+        assertThat(callsToReloadJwk, equalTo(0));
     }
 
     public void testAuthenticateImplicitFlowFailsWithForgedAccessToken() throws Exception {
@@ -532,6 +558,7 @@ public void testAuthenticateImplicitFlowFailsWithForgedAccessToken() throws Exce
         assertThat(e.getMessage(), containsString("Failed to verify access token"));
         assertThat(e.getCause(), instanceOf(InvalidHashException.class));
         assertThat(e.getCause().getMessage(), containsString("Access token hash (at_hash) mismatch"));
+        assertThat(callsToReloadJwk, equalTo(0));
     }
 
     public void testImplicitFlowFailsWithNoneAlgorithm() throws Exception {
@@ -569,6 +596,7 @@ public void testImplicitFlowFailsWithNoneAlgorithm() throws Exception {
         assertThat(e.getMessage(), containsString("Failed to parse or validate the ID Token"));
         assertThat(e.getCause(), instanceOf(BadJOSEException.class));
         assertThat(e.getCause().getMessage(), containsString("Another algorithm expected, or no matching key(s) found"));
+        assertThat(callsToReloadJwk, equalTo(0));
     }
 
     /**
@@ -599,6 +627,7 @@ public void testImplicitFlowFailsWithAlgorithmMixupAttack() throws Exception {
         assertThat(e.getMessage(), containsString("Failed to parse or validate the ID Token"));
         assertThat(e.getCause(), instanceOf(BadJOSEException.class));
         assertThat(e.getCause().getMessage(), containsString("Another algorithm expected, or no matching key(s) found"));
+        assertThat(callsToReloadJwk, equalTo(1));
     }
 
     public void testImplicitFlowFailsWithUnsignedJwt() throws Exception {
@@ -635,6 +664,11 @@ public void testImplicitFlowFailsWithUnsignedJwt() throws Exception {
         assertThat(e.getMessage(), containsString("Failed to parse or validate the ID Token"));
         assertThat(e.getCause(), instanceOf(BadJWTException.class));
         assertThat(e.getCause().getMessage(), containsString("Signed ID token expected"));
+        if (jwk.getAlgorithm().getName().startsWith("HS")) {
+            assertThat(callsToReloadJwk, equalTo(0));
+        } else {
+            assertThat(callsToReloadJwk, equalTo(1));
+        }
     }
 
     public void testJsonObjectMerging() throws Exception {
@@ -832,6 +866,7 @@ private OpenIdConnectAuthenticator.ReloadableJWKSource mockSource(JWK jwk) {
         Mockito.doAnswer(invocation -> {
             @SuppressWarnings("unchecked")
             ActionListener<Void> listener = (ActionListener<Void>) invocation.getArguments()[0];
+            callsToReloadJwk += 1;
             listener.onResponse(null);
             return null;
         }).when(jwkSource).triggerReload(any(ActionListener.class));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiAuthenticationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiAuthenticationTests.java
index c3698f32b6e32..5b2ab36426363 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiAuthenticationTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiAuthenticationTests.java
@@ -70,8 +70,7 @@ protected boolean transportSSLEnabled() {
 
     @Override
     protected boolean enableWarningsCheck() {
-        // the transport client uses deprecated SSL settings since we do not know what to do about
-        // secure settings for the transport client
+        // TODO: consider setting this back to true now that the transport client is gone
         return false;
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommandTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommandTests.java
index 367921ad7635a..320eb6e14a6c1 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommandTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlMetadataCommandTests.java
@@ -688,7 +688,7 @@ private String getAliasName(final Tuple<java.security.cert.X509Certificate, Priv
     private boolean validateSignature(Signature signature) {
         try {
             Certificate[] certificates = CertParsingUtils.
-                    readCertificates(Collections.singletonList(getDataPath("saml.crt").toString()), null);
+                    readCertificates(Collections.singletonList(getDataPath("saml.crt").toString()), newEnvironment());
             PrivateKey key = PemUtils.readPrivateKey(getDataPath("saml.key"),
                     ""::toCharArray);
             Credential verificationCredential = new BasicX509Credential((java.security.cert.X509Certificate) certificates[0], key);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlRealmTests.java
index c5b5e734b6505..52961b52e3815 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlRealmTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlRealmTests.java
@@ -731,7 +731,8 @@ private static List<String> getProtocols() {
             return List.of("TLSv1.2");
         } else {
             JavaVersion full =
-                AccessController.doPrivileged((PrivilegedAction<JavaVersion>) () -> JavaVersion.parse(System.getProperty("java.version")));
+                AccessController.doPrivileged(
+                        (PrivilegedAction<JavaVersion>) () -> JavaVersion.parse(System.getProperty("java.specification.version")));
             if (full.compareTo(JavaVersion.parse("12.0.1")) < 0) {
                 return List.of("TLSv1.2");
             }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ExpressionRoleMappingTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ExpressionRoleMappingTests.java
index e93b42a6d3a04..6f78435a47c14 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ExpressionRoleMappingTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ExpressionRoleMappingTests.java
@@ -43,6 +43,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -76,7 +77,7 @@ public void testValidExpressionWithFixedRoleNames() throws Exception {
 
         final UserRoleMapper.UserData user1a = new UserRoleMapper.UserData(
             "john.smith", "cn=john.smith,ou=sales,dc=example,dc=com",
-            Collections.emptyList(), Collections.singletonMap("active", true), realm
+            List.of(), Map.of("active", true), realm
         );
         final UserRoleMapper.UserData user1b = new UserRoleMapper.UserData(
             user1a.getUsername(), user1a.getDn().toUpperCase(Locale.US), user1a.getGroups(), user1a.getMetadata(), user1a.getRealm()
@@ -89,16 +90,16 @@ public void testValidExpressionWithFixedRoleNames() throws Exception {
         );
         final UserRoleMapper.UserData user2 = new UserRoleMapper.UserData(
             "jamie.perez", "cn=jamie.perez,ou=sales,dc=example,dc=com",
-            Collections.emptyList(), Collections.singletonMap("active", false), realm
+            List.of(), Map.of("active", false), realm
         );
 
         final UserRoleMapper.UserData user3 = new UserRoleMapper.UserData(
             "simone.ng", "cn=simone.ng,ou=finance,dc=example,dc=com",
-            Collections.emptyList(), Collections.singletonMap("active", true), realm
+            List.of(), Map.of("active", true), realm
         );
 
         final UserRoleMapper.UserData user4 = new UserRoleMapper.UserData(
-                "peter.null", null, Collections.emptyList(), Collections.singletonMap("active", true), realm
+                "peter.null", null, List.of(), Map.of("active", true), realm
         );
 
         assertThat(mapping.getExpression().match(user1a.asModel()), equalTo(true));
@@ -124,21 +125,17 @@ public void testValidExpressionWithFixedRoleNames() throws Exception {
             assertThat(mapping.getExpression(), instanceOf(AnyExpression.class));
 
         final UserRoleMapper.UserData userTony = new UserRoleMapper.UserData(
-                "tony.stark", null, Collections.singletonList("Audi R8 owners"), Collections.singletonMap("boss", true), realm
+                "tony.stark", null, List.of("Audi R8 owners"), Map.of("boss", true), realm
         );
         final UserRoleMapper.UserData userPepper = new UserRoleMapper.UserData(
-                "pepper.potts", null, Arrays.asList("marvel", "cn=admins,dc=stark-enterprises,dc=com"), null, realm
+                "pepper.potts", null, List.of("marvel", "cn=admins,dc=stark-enterprises,dc=com"), Map.of(), realm
         );
         final UserRoleMapper.UserData userMax = new UserRoleMapper.UserData(
-                "max.rockatansky", null, Collections.singletonList("bronze"), Collections.singletonMap("mad", true), realm
-        );
-        final UserRoleMapper.UserData userFinn = new UserRoleMapper.UserData(
-                "finn.hackleberry", null, Arrays.asList("hacker", null), null, realm
+                "max.rockatansky", null, List.of("bronze"), Map.of("mad", true), realm
         );
         assertThat(mapping.getExpression().match(userTony.asModel()), equalTo(true));
         assertThat(mapping.getExpression().match(userPepper.asModel()), equalTo(true));
         assertThat(mapping.getExpression().match(userMax.asModel()), equalTo(false));
-        assertThat(mapping.getExpression().match(userFinn.asModel()), equalTo(false));
     }
 
     public void testParseValidJsonWithTemplatedRoleNames() throws Exception {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
index 3cca6cc4fd380..787d4069c1e1e 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -42,6 +43,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
@@ -104,10 +106,10 @@ protected void loadMappings(ActionListener<List<ExpressionRoleMapping>> listener
         final PlainActionFuture<Set<String>> future = new PlainActionFuture<>();
         final UserRoleMapper.UserData user = new UserRoleMapper.UserData("sasquatch",
                 randomiseDn("cn=walter.langowski,ou=people,ou=dept_h,o=forces,dc=gc,dc=ca"),
-                Arrays.asList(
+                List.of(
                         randomiseDn("cn=alphaflight,ou=groups,ou=dept_h,o=forces,dc=gc,dc=ca"),
                         randomiseDn("cn=mutants,ou=groups,ou=dept_h,o=forces,dc=gc,dc=ca")
-                ), Collections.singletonMap("extra_group", "flight"), realm);
+                ), Map.of("extra_group", "flight"), realm);
 
         logger.info("UserData is [{}]", user);
         store.resolveRoles(user, future);
@@ -138,7 +140,12 @@ private String randomiseDn(String dn) {
     }
 
     private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) {
-        return new SecurityIndexManager.State(Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus);
+        return indexState(true, indexStatus);
+    }
+
+    private SecurityIndexManager.State indexState(boolean isUpToDate, ClusterHealthStatus healthStatus) {
+        return new SecurityIndexManager.State(
+            Instant.now(), isUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetaData.State.OPEN);
     }
 
     public void testCacheClearOnIndexHealthChange() {
@@ -172,7 +179,7 @@ public void testCacheClearOnIndexHealthChange() {
 
         // green to yellow or yellow to green
         previousState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW));
-        currentState = dummyState(previousState.indexStatus == ClusterHealthStatus.GREEN ?
+        currentState = dummyState(previousState.indexHealth == ClusterHealthStatus.GREEN ?
             ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN);
         store.onSecurityIndexStateChange(previousState, currentState);
         assertEquals(expectedInvalidation, numInvalidation.get());
@@ -182,14 +189,10 @@ public void testCacheClearOnIndexOutOfDateChange() {
         final AtomicInteger numInvalidation = new AtomicInteger(0);
         final NativeRoleMappingStore store = buildRoleMappingStoreForInvalidationTesting(numInvalidation, true);
 
-        store.onSecurityIndexStateChange(
-            new SecurityIndexManager.State(Instant.now(), false, true, true, null, concreteSecurityIndexName, null),
-            new SecurityIndexManager.State(Instant.now(), true, true, true, null, concreteSecurityIndexName, null));
+        store.onSecurityIndexStateChange(indexState(false, null), indexState(true, null));
         assertEquals(1, numInvalidation.get());
 
-        store.onSecurityIndexStateChange(
-            new SecurityIndexManager.State(Instant.now(), true, true, true, null, concreteSecurityIndexName, null),
-            new SecurityIndexManager.State(Instant.now(), false, true, true, null, concreteSecurityIndexName, null));
+        store.onSecurityIndexStateChange(indexState(true, null), indexState(false, null));
         assertEquals(2, numInvalidation.get());
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
index b370c8e2b6b0f..7f637bdb190c5 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
@@ -22,8 +22,6 @@
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexAction;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryAction;
@@ -155,9 +153,9 @@
 import static org.elasticsearch.test.SecurityTestsUtils.assertAuthenticationException;
 import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationException;
 import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationExceptionRunAs;
-import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME;
 import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7;
 import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
+import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME;
 import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
@@ -450,7 +448,6 @@ public void testRemoteIndicesOnlyWorkWithApplicableRequestTypes() throws IOExcep
     public void testUnknownRoleCausesDenial() throws IOException {
         Tuple<String, TransportRequest> tuple = randomFrom(asList(
             new Tuple<>(SearchAction.NAME, new SearchRequest()),
-            new Tuple<>(IndicesExistsAction.NAME, new IndicesExistsRequest()),
             new Tuple<>(SqlQueryAction.NAME, new SqlQueryRequest())));
         String action = tuple.v1();
         TransportRequest request = tuple.v2();
@@ -484,7 +481,6 @@ public void testThatRoleWithNoIndicesIsDenied() throws IOException {
         @SuppressWarnings("unchecked")
         Tuple<String, TransportRequest> tuple = randomFrom(
             new Tuple<>(SearchAction.NAME, new SearchRequest()),
-            new Tuple<>(IndicesExistsAction.NAME, new IndicesExistsRequest()),
             new Tuple<>(SqlQueryAction.NAME, new SqlQueryRequest()));
         String action = tuple.v1();
         TransportRequest request = tuple.v2();
@@ -882,7 +878,7 @@ public void testGrantAllRestrictedUserCannotExecuteOperationAgainstSecurityIndic
                 new Tuple<>(BulkAction.NAME + "[s]", new IndexRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7))));
         requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7))));
         requests.add(new Tuple<>(TermVectorsAction.NAME,
-                new TermVectorsRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "type", "id")));
+                new TermVectorsRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "id")));
         requests.add(new Tuple<>(GetAction.NAME, new GetRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "id")));
         requests.add(new Tuple<>(IndicesAliasesAction.NAME, new IndicesAliasesRequest()
                 .addAliasAction(AliasActions.add().alias("security_alias").index(INTERNAL_SECURITY_MAIN_INDEX_7))));
@@ -1007,11 +1003,11 @@ public void testSuperusersCanExecuteOperationAgainstSecurityIndex() throws IOExc
                 createBulkShardRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), IndexRequest::new)));
         requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7))));
         requests.add(new Tuple<>(TermVectorsAction.NAME,
-                new TermVectorsRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "type", "id")));
+                new TermVectorsRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "id")));
         requests.add(
                 new Tuple<>(GetAction.NAME, new GetRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "type", "id")));
         requests.add(new Tuple<>(TermVectorsAction.NAME,
-                new TermVectorsRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "type", "id")));
+                new TermVectorsRequest(randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7), "id")));
         requests.add(new Tuple<>(IndicesAliasesAction.NAME, new IndicesAliasesRequest()
                 .addAliasAction(AliasActions.add().alias("security_alias").index(INTERNAL_SECURITY_MAIN_INDEX_7))));
         requests.add(new Tuple<>(ClusterHealthAction.NAME,
@@ -1244,7 +1240,7 @@ private static Tuple<String, TransportRequest> randomCompositeRequest() {
             case 1:
                 return Tuple.tuple(MultiSearchAction.NAME, new MultiSearchRequest().add(new SearchRequest()));
             case 2:
-                return Tuple.tuple(MultiTermVectorsAction.NAME, new MultiTermVectorsRequest().add("index", "type", "id"));
+                return Tuple.tuple(MultiTermVectorsAction.NAME, new MultiTermVectorsRequest().add("index", "id"));
             case 3:
                 return Tuple.tuple(BulkAction.NAME, new BulkRequest().add(new DeleteRequest("index", "type", "id")));
             case 4:
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
index 665b70a8881b2..c2db3ba7352f5 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
@@ -17,8 +17,6 @@
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
@@ -1194,28 +1192,6 @@ public void testResolveAdminAction() {
         }
     }
 
-    public void testIndicesExists() {
-        //verify that the ignore_unavailable and allow_no_indices get replaced like es core does, to make sure that
-        //indices exists api never throws exception due to missing indices, but only returns false instead.
-        {
-            IndicesExistsRequest request = new IndicesExistsRequest();
-            assertNoIndices(request, resolveIndices(request,
-                    buildAuthorizedIndices(userNoIndices, IndicesExistsAction.NAME)));
-        }
-
-        {
-            IndicesExistsRequest request = new IndicesExistsRequest("does_not_exist");
-
-            assertNoIndices(request, resolveIndices(request,
-                    buildAuthorizedIndices(user, IndicesExistsAction.NAME)));
-        }
-        {
-            IndicesExistsRequest request = new IndicesExistsRequest("does_not_exist_*");
-            assertNoIndices(request, resolveIndices(request,
-                    buildAuthorizedIndices(user, IndicesExistsAction.NAME)));
-        }
-    }
-
     public void testXPackSecurityUserHasAccessToSecurityIndex() {
         SearchRequest request = new SearchRequest();
         {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java
index b50ff864392a1..18bfe191e196e 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/ReadActionsTests.java
@@ -317,22 +317,6 @@ public void testMultiSearchWildcard() {
         }
     }
 
-    public void testIndicesExists() {
-        createIndicesWithRandomAliases("test1", "test2", "test3");
-
-        assertEquals(true, client().admin().indices().prepareExists("*").get().isExists());
-
-        assertEquals(true, client().admin().indices().prepareExists("_all").get().isExists());
-
-        assertEquals(true, client().admin().indices().prepareExists("test1", "test2").get().isExists());
-
-        assertEquals(true, client().admin().indices().prepareExists("test*").get().isExists());
-
-        assertEquals(false, client().admin().indices().prepareExists("does_not_exist").get().isExists());
-
-        assertEquals(false, client().admin().indices().prepareExists("does_not_exist*").get().isExists());
-    }
-
     public void testGet() {
         createIndicesWithRandomAliases("test1", "index1");
 
@@ -374,23 +358,23 @@ public void testMultiGet() {
 
     public void testTermVectors() {
         createIndicesWithRandomAliases("test1", "index1");
-        client().prepareTermVectors("test1", "type", "id").get();
+        client().prepareTermVectors("test1", "id").get();
 
-        assertThrowsAuthorizationExceptionDefaultUsers(client().prepareTermVectors("index1", "type", "id")::get, TermVectorsAction.NAME);
+        assertThrowsAuthorizationExceptionDefaultUsers(client().prepareTermVectors("index1", "id")::get, TermVectorsAction.NAME);
 
-        assertThrowsAuthorizationExceptionDefaultUsers(client().prepareTermVectors("missing", "type", "id")::get, TermVectorsAction.NAME);
+        assertThrowsAuthorizationExceptionDefaultUsers(client().prepareTermVectors("missing", "id")::get, TermVectorsAction.NAME);
 
-        expectThrows(IndexNotFoundException.class, () -> client().prepareTermVectors("test5", "type", "id").get());
+        expectThrows(IndexNotFoundException.class, () -> client().prepareTermVectors("test5", "id").get());
     }
 
     public void testMultiTermVectors() {
         createIndicesWithRandomAliases("test1", "test2", "test3", "index1");
         MultiTermVectorsResponse response = client().prepareMultiTermVectors()
-                .add("test1", "type", "id")
-                .add("index1", "type", "id")
-                .add("test3", "type", "id")
-                .add("missing", "type", "id")
-                .add("test5", "type", "id").get();
+                .add("test1", "id")
+                .add("index1", "id")
+                .add("test3", "id")
+                .add("missing", "id")
+                .add("test5", "id").get();
         assertEquals(5, response.getResponses().length);
         assertFalse(response.getResponses()[0].isFailed());
         assertEquals("test1", response.getResponses()[0].getResponse().getIndex());
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java
index 3290aba27e37f..2b604c8257779 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java
@@ -14,6 +14,8 @@
 import org.elasticsearch.search.SearchContextMissingException;
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSourceField;
+import org.elasticsearch.xpack.core.security.action.role.PutRoleRequestBuilder;
+import org.elasticsearch.xpack.core.security.action.user.PutUserRequestBuilder;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.junit.After;
 
@@ -28,11 +30,12 @@ public class SecurityScrollTests extends SecurityIntegTestCase {
 
     public void testScrollIsPerUser() throws Exception {
         assertSecurityIndexActive();
-        securityClient().preparePutRole("scrollable")
+        new PutRoleRequestBuilder(client()).name("scrollable")
                 .addIndices(new String[] { randomAlphaOfLengthBetween(4, 12) }, new String[] { "read" }, null, null, null, randomBoolean())
                 .get();
-        securityClient().preparePutUser("other", SecuritySettingsSourceField.TEST_PASSWORD.toCharArray(), getFastStoredHashAlgoForTests(),
-            "scrollable")
+        new PutUserRequestBuilder(client()).username("other")
+            .password(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING, getFastStoredHashAlgoForTests())
+            .roles("scrollable")
             .get();
 
         final int numDocs = randomIntBetween(4, 16);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java
index 66ea23b518c29..1994e5b702061 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SnapshotUserRoleIntegTests.java
@@ -6,31 +6,41 @@
 
 package org.elasticsearch.xpack.security.authz;
 
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.security.DeleteRoleRequest;
+import org.elasticsearch.client.security.PutRoleRequest;
+import org.elasticsearch.client.security.PutUserRequest;
+import org.elasticsearch.client.security.PutUserResponse;
+import org.elasticsearch.client.security.RefreshPolicy;
+import org.elasticsearch.client.security.user.User;
+import org.elasticsearch.client.security.user.privileges.Role;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.snapshots.SnapshotState;
 import org.elasticsearch.test.NativeRealmIntegTestCase;
-import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 import java.util.Locale;
 
-import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7;
-import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
-
+import static org.elasticsearch.test.SecuritySettingsSource.SECURITY_REQUEST_OPTIONS;
+import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationException;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.containsString;
+import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7;
+import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
 import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationException;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
 
 public class SnapshotUserRoleIntegTests extends NativeRealmIntegTestCase {
 
@@ -38,7 +48,7 @@ public class SnapshotUserRoleIntegTests extends NativeRealmIntegTestCase {
     private String ordinaryIndex;
 
     @Before
-    public void setupClusterBeforeSnapshot() {
+    public void setupClusterBeforeSnapshot() throws IOException {
         logger.info("-->  creating repository");
         assertAcked(client().admin().cluster().preparePutRepository("repo")
                 .setType("fs")
@@ -55,7 +65,10 @@ public void setupClusterBeforeSnapshot() {
         final char[] password = new char[] {'p', 'a', 's', 's', 'w', 'o', 'r', 'd'};
         final String snapshotUserToken = basicAuthHeaderValue(user, new SecureString(password));
         client = client().filterWithHeader(Collections.singletonMap("Authorization", snapshotUserToken));
-        securityClient().preparePutUser(user, password, Hasher.BCRYPT, "snapshot_user").get();
+        PutUserResponse response = new TestRestHighLevelClient().security().putUser(
+            PutUserRequest.withPassword(new User(user, List.of("snapshot_user")), password, true, RefreshPolicy.IMMEDIATE),
+            SECURITY_REQUEST_OPTIONS);
+        assertTrue(response.isCreated());
         ensureGreen(INTERNAL_SECURITY_MAIN_INDEX_7);
     }
 
@@ -75,17 +88,21 @@ public void testSnapshotUserRoleCanSnapshotAndSeeAllIndices() {
         assertThat(snapshotResponse.getSnapshotInfo().indices(), containsInAnyOrder(INTERNAL_SECURITY_MAIN_INDEX_7, ordinaryIndex));
         // view snapshots for repo
         final GetSnapshotsResponse getSnapshotResponse = client.admin().cluster().prepareGetSnapshots("repo").get();
-        assertThat(getSnapshotResponse.getSnapshots().size(), is(1));
-        assertThat(getSnapshotResponse.getSnapshots().get(0).snapshotId().getName(), is("snap"));
-        assertThat(getSnapshotResponse.getSnapshots().get(0).indices(), containsInAnyOrder(INTERNAL_SECURITY_MAIN_INDEX_7, ordinaryIndex));
+        assertThat(getSnapshotResponse.getSnapshots("repo").size(), is(1));
+        assertThat(getSnapshotResponse.getSnapshots("repo").get(0).snapshotId().getName(), is("snap"));
+        assertThat(getSnapshotResponse.getSnapshots("repo").get(0).indices(), containsInAnyOrder(INTERNAL_SECURITY_MAIN_INDEX_7,
+                ordinaryIndex));
     }
 
     public void testSnapshotUserRoleIsReserved() {
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> securityClient().preparePutRole("snapshot_user").get());
+        final RestHighLevelClient restClient = new TestRestHighLevelClient();
+        ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class,
+                () -> restClient.security().putRole(
+                    new PutRoleRequest(Role.builder().name("snapshot_user").build(), RefreshPolicy.IMMEDIATE), SECURITY_REQUEST_OPTIONS));
         assertThat(e.getMessage(), containsString("role [snapshot_user] is reserved and cannot be modified"));
-        e = expectThrows(IllegalArgumentException.class,
-                () -> securityClient().prepareDeleteRole("snapshot_user").get());
+        e = expectThrows(ElasticsearchStatusException.class,
+                () -> restClient.security().deleteRole(
+                    new DeleteRoleRequest("snapshot_user", RefreshPolicy.IMMEDIATE), SECURITY_REQUEST_OPTIONS));
         assertThat(e.getMessage(), containsString("role [snapshot_user] is reserved and cannot be deleted"));
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/WriteActionsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/WriteActionsTests.java
index 1f40f1c480f6b..e095eea09d152 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/WriteActionsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/WriteActionsTests.java
@@ -23,6 +23,9 @@
 import static org.elasticsearch.test.SecurityTestsUtils.assertAuthorizationExceptionDefaultUsers;
 import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationExceptionDefaultUsers;
 import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.core.IsInstanceOf.instanceOf;
 
 public class WriteActionsTests extends SecurityIntegTestCase {
@@ -53,6 +56,7 @@ public void testIndex() {
         //the missing index gets automatically created (user has permissions for that), but indexing fails due to missing authorization
         assertThrowsAuthorizationExceptionDefaultUsers(client().prepareIndex("missing", "type", "id").setSource("field", "value")::get,
                 BulkAction.NAME + "[s]");
+        ensureGreen();
     }
 
     public void testDelete() {
@@ -63,6 +67,7 @@ public void testDelete() {
         assertThrowsAuthorizationExceptionDefaultUsers(client().prepareDelete("index1", "type", "id")::get, BulkAction.NAME + "[s]");
 
         expectThrows(IndexNotFoundException.class, () -> client().prepareDelete("test4", "type", "id").get());
+        ensureGreen();
     }
 
     public void testUpdate() {
@@ -79,6 +84,7 @@ public void testUpdate() {
 
         assertThrowsAuthorizationExceptionDefaultUsers(client().prepareUpdate("missing", "type", "id")
                         .setDoc(Requests.INDEX_CONTENT_TYPE, "field2", "value2")::get, UpdateAction.NAME);
+        ensureGreen();
     }
 
     public void testBulk() {
@@ -98,67 +104,81 @@ public void testBulk() {
                 .add(new UpdateRequest("test4", "type", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value"))
                 .add(new UpdateRequest("missing", "type", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value")).get();
         assertTrue(bulkResponse.hasFailures());
-        assertEquals(13, bulkResponse.getItems().length);
-        assertFalse(bulkResponse.getItems()[0].isFailed());
-        assertEquals(DocWriteRequest.OpType.INDEX, bulkResponse.getItems()[0].getOpType());
-        assertEquals("test1", bulkResponse.getItems()[0].getIndex());
-        assertTrue(bulkResponse.getItems()[1].isFailed());
-        assertEquals(DocWriteRequest.OpType.INDEX, bulkResponse.getItems()[1].getOpType());
-        assertEquals("index1", bulkResponse.getItems()[1].getFailure().getIndex());
+        assertThat(bulkResponse.getItems().length, equalTo(13));
+        assertThat(bulkResponse.getItems()[0].getFailure(), nullValue());
+        assertThat(bulkResponse.getItems()[0].isFailed(), equalTo(false));
+        assertThat(bulkResponse.getItems()[0].getOpType(), equalTo(DocWriteRequest.OpType.INDEX));
+        assertThat(bulkResponse.getItems()[0].getIndex(), equalTo("test1"));
+        assertThat(bulkResponse.getItems()[1].getFailure(), notNullValue());
+        assertThat(bulkResponse.getItems()[1].isFailed(), equalTo(true));
+        assertThat(bulkResponse.getItems()[1].getOpType(), equalTo(DocWriteRequest.OpType.INDEX));
+        assertThat(bulkResponse.getItems()[1].getFailure().getIndex(), equalTo("index1"));
         assertAuthorizationExceptionDefaultUsers(bulkResponse.getItems()[1].getFailure().getCause(), BulkAction.NAME + "[s]");
         assertThat(bulkResponse.getItems()[1].getFailure().getCause().getMessage(),
                 containsString("[indices:data/write/bulk[s]] is unauthorized"));
-        assertFalse(bulkResponse.getItems()[2].isFailed());
-        assertEquals(DocWriteRequest.OpType.INDEX, bulkResponse.getItems()[2].getOpType());
-        assertEquals("test4", bulkResponse.getItems()[2].getResponse().getIndex());
-        assertTrue(bulkResponse.getItems()[3].isFailed());
-        assertEquals(DocWriteRequest.OpType.INDEX, bulkResponse.getItems()[3].getOpType());
+        assertThat(bulkResponse.getItems()[2].getFailure(), nullValue());
+        assertThat(bulkResponse.getItems()[2].isFailed(), equalTo(false));
+        assertThat(bulkResponse.getItems()[2].getOpType(), equalTo(DocWriteRequest.OpType.INDEX));
+        assertThat(bulkResponse.getItems()[2].getResponse().getIndex(), equalTo("test4"));
+        assertThat(bulkResponse.getItems()[3].getFailure(), notNullValue());
+        assertThat(bulkResponse.getItems()[3].isFailed(), equalTo(true));
+        assertThat(bulkResponse.getItems()[3].getOpType(), equalTo(DocWriteRequest.OpType.INDEX));
         //the missing index gets automatically created (user has permissions for that), but indexing fails due to missing authorization
-        assertEquals("missing", bulkResponse.getItems()[3].getFailure().getIndex());
+        assertThat(bulkResponse.getItems()[3].getFailure().getIndex(), equalTo("missing"));
         assertThat(bulkResponse.getItems()[3].getFailure().getCause(), instanceOf(ElasticsearchSecurityException.class));
         assertAuthorizationExceptionDefaultUsers(bulkResponse.getItems()[3].getFailure().getCause(), BulkAction.NAME + "[s]");
         assertThat(bulkResponse.getItems()[3].getFailure().getCause().getMessage(),
                 containsString("[indices:data/write/bulk[s]] is unauthorized"));
-        assertFalse(bulkResponse.getItems()[4].isFailed());
-        assertEquals(DocWriteRequest.OpType.DELETE, bulkResponse.getItems()[4].getOpType());
-        assertEquals("test1", bulkResponse.getItems()[4].getIndex());
-        assertTrue(bulkResponse.getItems()[5].isFailed());
-        assertEquals(DocWriteRequest.OpType.DELETE, bulkResponse.getItems()[5].getOpType());
-        assertEquals("index1", bulkResponse.getItems()[5].getFailure().getIndex());
+        assertThat(bulkResponse.getItems()[4].getFailure(), nullValue());
+        assertThat(bulkResponse.getItems()[4].isFailed(), equalTo(false));
+        assertThat(bulkResponse.getItems()[4].getOpType(), equalTo(DocWriteRequest.OpType.DELETE));
+        assertThat(bulkResponse.getItems()[4].getIndex(), equalTo("test1"));
+        assertThat(bulkResponse.getItems()[5].getFailure(), notNullValue());
+        assertThat(bulkResponse.getItems()[5].isFailed(), equalTo(true));
+        assertThat(bulkResponse.getItems()[5].getOpType(), equalTo(DocWriteRequest.OpType.DELETE));
+        assertThat(bulkResponse.getItems()[5].getFailure().getIndex(), equalTo("index1"));
         assertAuthorizationExceptionDefaultUsers(bulkResponse.getItems()[5].getFailure().getCause(), BulkAction.NAME + "[s]");
         assertThat(bulkResponse.getItems()[5].getFailure().getCause().getMessage(),
                 containsString("[indices:data/write/bulk[s]] is unauthorized"));
-        assertFalse(bulkResponse.getItems()[6].isFailed());
-        assertEquals(DocWriteRequest.OpType.DELETE, bulkResponse.getItems()[6].getOpType());
-        assertEquals("test4", bulkResponse.getItems()[6].getIndex());
-        assertTrue(bulkResponse.getItems()[7].isFailed());
-        assertEquals(DocWriteRequest.OpType.DELETE, bulkResponse.getItems()[7].getOpType());
-        assertEquals("missing", bulkResponse.getItems()[7].getFailure().getIndex());
+        assertThat(bulkResponse.getItems()[6].getFailure(), nullValue());
+        assertThat(bulkResponse.getItems()[6].isFailed(), equalTo(false));
+        assertThat(bulkResponse.getItems()[6].getOpType(), equalTo(DocWriteRequest.OpType.DELETE));
+        assertThat(bulkResponse.getItems()[6].getIndex(), equalTo("test4"));
+        assertThat(bulkResponse.getItems()[7].getFailure(), notNullValue());
+        assertThat(bulkResponse.getItems()[7].isFailed(), equalTo(true));
+        assertThat(bulkResponse.getItems()[7].getOpType(), equalTo(DocWriteRequest.OpType.DELETE));
+        assertThat(bulkResponse.getItems()[7].getFailure().getIndex(), equalTo("missing"));
         assertAuthorizationExceptionDefaultUsers(bulkResponse.getItems()[7].getFailure().getCause(), BulkAction.NAME + "[s]");
         assertThat(bulkResponse.getItems()[7].getFailure().getCause().getMessage(),
                 containsString("[indices:data/write/bulk[s]] is unauthorized"));
-        assertFalse(bulkResponse.getItems()[8].isFailed());
-        assertEquals(DocWriteRequest.OpType.INDEX, bulkResponse.getItems()[8].getOpType());
-        assertEquals("test1", bulkResponse.getItems()[8].getIndex());
-        assertFalse(bulkResponse.getItems()[9].isFailed());
-        assertEquals(DocWriteRequest.OpType.UPDATE, bulkResponse.getItems()[9].getOpType());
-        assertEquals("test1", bulkResponse.getItems()[9].getIndex());
-        assertTrue(bulkResponse.getItems()[10].isFailed());
-        assertEquals(DocWriteRequest.OpType.UPDATE, bulkResponse.getItems()[10].getOpType());
-        assertEquals("index1", bulkResponse.getItems()[10].getFailure().getIndex());
+        assertThat(bulkResponse.getItems()[8].getFailure(), nullValue());
+        assertThat(bulkResponse.getItems()[8].isFailed(), equalTo(false));
+        assertThat(bulkResponse.getItems()[8].getOpType(), equalTo(DocWriteRequest.OpType.INDEX));
+        assertThat(bulkResponse.getItems()[8].getIndex(), equalTo("test1"));
+        assertThat(bulkResponse.getItems()[9].getFailure(), nullValue());
+        assertThat(bulkResponse.getItems()[9].isFailed(), equalTo(false));
+        assertThat(bulkResponse.getItems()[9].getOpType(), equalTo(DocWriteRequest.OpType.UPDATE));
+        assertThat(bulkResponse.getItems()[9].getIndex(), equalTo("test1"));
+        assertThat(bulkResponse.getItems()[10].getFailure(), notNullValue());
+        assertThat(bulkResponse.getItems()[10].isFailed(), equalTo(true));
+        assertThat(bulkResponse.getItems()[10].getOpType(), equalTo(DocWriteRequest.OpType.UPDATE));
+        assertThat(bulkResponse.getItems()[10].getFailure().getIndex(), equalTo("index1"));
         assertAuthorizationExceptionDefaultUsers(bulkResponse.getItems()[10].getFailure().getCause(), BulkAction.NAME + "[s]");
         assertThat(bulkResponse.getItems()[10].getFailure().getCause().getMessage(),
                 containsString("[indices:data/write/bulk[s]] is unauthorized"));
-        assertTrue(bulkResponse.getItems()[11].isFailed());
-        assertEquals(DocWriteRequest.OpType.UPDATE, bulkResponse.getItems()[11].getOpType());
-        assertEquals("test4", bulkResponse.getItems()[11].getIndex());
+        assertThat(bulkResponse.getItems()[11].getFailure(), notNullValue());
+        assertThat(bulkResponse.getItems()[11].isFailed(), equalTo(true));
+        assertThat(bulkResponse.getItems()[11].getOpType(), equalTo(DocWriteRequest.OpType.UPDATE));
+        assertThat(bulkResponse.getItems()[11].getIndex(), equalTo("test4"));
         assertThat(bulkResponse.getItems()[11].getFailure().getCause(), instanceOf(DocumentMissingException.class));
-        assertTrue(bulkResponse.getItems()[12].isFailed());
-        assertEquals(DocWriteRequest.OpType.UPDATE, bulkResponse.getItems()[12].getOpType());
-        assertEquals("missing", bulkResponse.getItems()[12].getFailure().getIndex());
+        assertThat(bulkResponse.getItems()[12].getFailure(), notNullValue());
+        assertThat(bulkResponse.getItems()[12].isFailed(), equalTo(true));
+        assertThat(bulkResponse.getItems()[12].getOpType(), equalTo(DocWriteRequest.OpType.UPDATE));
+        assertThat(bulkResponse.getItems()[12].getFailure().getIndex(), equalTo("missing"));
         assertThat(bulkResponse.getItems()[12].getFailure().getCause(), instanceOf(ElasticsearchSecurityException.class));
         assertAuthorizationExceptionDefaultUsers(bulkResponse.getItems()[12].getFailure().getCause(), BulkAction.NAME + "[s]");
         assertThat(bulkResponse.getItems()[12].getFailure().getCause().getMessage(),
                 containsString("[indices:data/write/bulk[s]] is unauthorized"));
+        ensureGreen();
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
index a39545f3a9b3a..57b172f47f0cd 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
@@ -149,7 +149,7 @@ public void testRolesWhenDlsFlsUnlicensed() throws IOException {
         final AtomicReference<Collection<RoleDescriptor>> effectiveRoleDescriptors = new AtomicReference<Collection<RoleDescriptor>>();
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(Settings.EMPTY, fileRolesStore, nativeRolesStore,
                 reservedRolesStore, mock(NativePrivilegeStore.class), Collections.emptyList(),
-                new ThreadContext(Settings.EMPTY), licenseState, cache, mock(ApiKeyService.class),
+                new ThreadContext(Settings.EMPTY), licenseState, cache, mock(ApiKeyService.class), null,
                 rds -> effectiveRoleDescriptors.set(rds));
 
         PlainActionFuture<Role> roleFuture = new PlainActionFuture<>();
@@ -224,7 +224,7 @@ public void testRolesWhenDlsFlsLicensed() throws IOException {
         final AtomicReference<Collection<RoleDescriptor>> effectiveRoleDescriptors = new AtomicReference<Collection<RoleDescriptor>>();
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(Settings.EMPTY, fileRolesStore, nativeRolesStore,
                 reservedRolesStore, mock(NativePrivilegeStore.class), Collections.emptyList(),
-                new ThreadContext(Settings.EMPTY), licenseState, cache, mock(ApiKeyService.class),
+                new ThreadContext(Settings.EMPTY), licenseState, cache, mock(ApiKeyService.class), null,
                 rds -> effectiveRoleDescriptors.set(rds));
 
         PlainActionFuture<Role> roleFuture = new PlainActionFuture<>();
@@ -276,7 +276,7 @@ public void testNegativeLookupsAreCached() {
         final CompositeRolesStore compositeRolesStore =
                 new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                         nativePrivilegeStore, Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                        new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class),
+                        new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class), null,
                         rds -> effectiveRoleDescriptors.set(rds));
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
 
@@ -337,7 +337,7 @@ public void testNegativeLookupsCacheDisabled() {
         final AtomicReference<Collection<RoleDescriptor>> effectiveRoleDescriptors = new AtomicReference<Collection<RoleDescriptor>>();
         final CompositeRolesStore compositeRolesStore = new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore,
             reservedRolesStore, mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(settings),
-            new XPackLicenseState(settings), cache, mock(ApiKeyService.class), rds -> effectiveRoleDescriptors.set(rds));
+            new XPackLicenseState(settings), cache, mock(ApiKeyService.class), null, rds -> effectiveRoleDescriptors.set(rds));
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
 
         final String roleName = randomAlphaOfLengthBetween(1, 10);
@@ -374,7 +374,7 @@ public void testNegativeLookupsAreNotCachedWithFailures() {
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class),
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class), null,
                 rds -> effectiveRoleDescriptors.set(rds));
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
 
@@ -460,7 +460,7 @@ public void testCustomRolesProviders() {
                 new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                                 mock(NativePrivilegeStore.class), Arrays.asList(inMemoryProvider1, inMemoryProvider2),
                                 new ThreadContext(SECURITY_ENABLED_SETTINGS), new XPackLicenseState(SECURITY_ENABLED_SETTINGS),
-                                cache, mock(ApiKeyService.class), rds -> effectiveRoleDescriptors.set(rds));
+                                cache, mock(ApiKeyService.class), null, rds -> effectiveRoleDescriptors.set(rds));
 
         final Set<String> roleNames = Sets.newHashSet("roleA", "roleB", "unknown");
         PlainActionFuture<Role> future = new PlainActionFuture<>();
@@ -674,7 +674,7 @@ public void testCustomRolesProviderFailures() throws Exception {
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Arrays.asList(inMemoryProvider1, failingProvider),
                 new ThreadContext(SECURITY_ENABLED_SETTINGS), new XPackLicenseState(SECURITY_ENABLED_SETTINGS),
-                cache, mock(ApiKeyService.class), rds -> effectiveRoleDescriptors.set(rds));
+                cache, mock(ApiKeyService.class), null, rds -> effectiveRoleDescriptors.set(rds));
 
         final Set<String> roleNames = Sets.newHashSet("roleA", "roleB", "unknown");
         PlainActionFuture<Role> future = new PlainActionFuture<>();
@@ -720,7 +720,7 @@ public void testCustomRolesProvidersLicensing() {
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(
             Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore, mock(NativePrivilegeStore.class),
             Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState, cache,
-            mock(ApiKeyService.class), rds -> effectiveRoleDescriptors.set(rds));
+            mock(ApiKeyService.class), null, rds -> effectiveRoleDescriptors.set(rds));
 
         Set<String> roleNames = Sets.newHashSet("roleA");
         PlainActionFuture<Role> future = new PlainActionFuture<>();
@@ -735,7 +735,7 @@ Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore, mock(Nativ
         compositeRolesStore = new CompositeRolesStore(
             Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore, mock(NativePrivilegeStore.class),
             Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState, cache,
-            mock(ApiKeyService.class), rds -> effectiveRoleDescriptors.set(rds));
+            mock(ApiKeyService.class), null, rds -> effectiveRoleDescriptors.set(rds));
         // these licenses allow custom role providers
         xPackLicenseState.update(randomFrom(OperationMode.PLATINUM, OperationMode.TRIAL), true, null);
         roleNames = Sets.newHashSet("roleA");
@@ -752,7 +752,7 @@ Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore, mock(Nativ
         compositeRolesStore = new CompositeRolesStore(
             Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore, mock(NativePrivilegeStore.class),
             Arrays.asList(inMemoryProvider), new ThreadContext(Settings.EMPTY), xPackLicenseState, cache,
-            mock(ApiKeyService.class), rds -> effectiveRoleDescriptors.set(rds));
+            mock(ApiKeyService.class), null, rds -> effectiveRoleDescriptors.set(rds));
         xPackLicenseState.update(randomFrom(OperationMode.PLATINUM, OperationMode.TRIAL), false, null);
         roleNames = Sets.newHashSet("roleA");
         future = new PlainActionFuture<>();
@@ -763,7 +763,12 @@ Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore, mock(Nativ
     }
 
     private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) {
-        return new SecurityIndexManager.State(Instant.now(), true, true, true, null, concreteSecurityIndexName, indexStatus);
+        return dummyIndexState(true, indexStatus);
+    }
+
+    public SecurityIndexManager.State dummyIndexState(boolean isIndexUpToDate, ClusterHealthStatus healthStatus) {
+        return new SecurityIndexManager.State(
+            Instant.now(), isIndexUpToDate, true, true, null, concreteSecurityIndexName, healthStatus, IndexMetaData.State.OPEN);
     }
 
     public void testCacheClearOnIndexHealthChange() {
@@ -778,7 +783,7 @@ public void testCacheClearOnIndexHealthChange() {
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(
                 Settings.EMPTY, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(Settings.EMPTY),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class), rds -> {}) {
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class), null, rds -> {}) {
             @Override
             public void invalidateAll() {
                 numInvalidation.incrementAndGet();
@@ -812,7 +817,7 @@ public void invalidateAll() {
 
         // green to yellow or yellow to green
         previousState = dummyState(randomFrom(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW));
-        currentState = dummyState(previousState.indexStatus == ClusterHealthStatus.GREEN ?
+        currentState = dummyState(previousState.indexHealth == ClusterHealthStatus.GREEN ?
                                   ClusterHealthStatus.YELLOW : ClusterHealthStatus.GREEN);
         compositeRolesStore.onSecurityIndexStateChange(previousState, currentState);
         assertEquals(expectedInvalidation, numInvalidation.get());
@@ -830,21 +835,17 @@ public void testCacheClearOnIndexOutOfDateChange() {
         CompositeRolesStore compositeRolesStore = new CompositeRolesStore(SECURITY_ENABLED_SETTINGS,
                 fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class), rds -> {}) {
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class), null, rds -> {}) {
             @Override
             public void invalidateAll() {
                 numInvalidation.incrementAndGet();
             }
         };
 
-        compositeRolesStore.onSecurityIndexStateChange(
-            new SecurityIndexManager.State(Instant.now(), false, true, true, null, concreteSecurityIndexName, null),
-            new SecurityIndexManager.State(Instant.now(), true, true, true, null, concreteSecurityIndexName, null));
+        compositeRolesStore.onSecurityIndexStateChange(dummyIndexState(false, null), dummyIndexState(true, null));
         assertEquals(1, numInvalidation.get());
 
-        compositeRolesStore.onSecurityIndexStateChange(
-            new SecurityIndexManager.State(Instant.now(), true, true, true, null, concreteSecurityIndexName, null),
-            new SecurityIndexManager.State(Instant.now(), false, true, true, null, concreteSecurityIndexName, null));
+        compositeRolesStore.onSecurityIndexStateChange(dummyIndexState(true, null), dummyIndexState(false, null));
         assertEquals(2, numInvalidation.get());
     }
 
@@ -864,7 +865,7 @@ public void testDefaultRoleUserWithoutRoles() {
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class), rds -> {});
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class), null, rds -> {});
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
 
         PlainActionFuture<Role> rolesFuture = new PlainActionFuture<>();
@@ -903,7 +904,7 @@ public void testAnonymousUserEnabledRoleAdded() {
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(settings),
-                new XPackLicenseState(settings), cache, mock(ApiKeyService.class), rds -> {});
+                new XPackLicenseState(settings), cache, mock(ApiKeyService.class), null, rds -> {});
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
 
         PlainActionFuture<Role> rolesFuture = new PlainActionFuture<>();
@@ -931,7 +932,7 @@ public void testDoesNotUseRolesStoreForXPackUser() {
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class),
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class), null,
                 rds -> effectiveRoleDescriptors.set(rds));
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
 
@@ -961,7 +962,7 @@ public void testGetRolesForSystemUserThrowsException() {
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 mock(NativePrivilegeStore.class), Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class),
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, mock(ApiKeyService.class), null,
                 rds -> effectiveRoleDescriptors.set(rds));
         verify(fileRolesStore).addListener(any(Consumer.class)); // adds a listener in ctor
         IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
@@ -996,7 +997,7 @@ public void testApiKeyAuthUsesApiKeyService() throws IOException {
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 nativePrivStore, Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, apiKeyService,
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, apiKeyService, null,
                 rds -> effectiveRoleDescriptors.set(rds));
         AuditUtil.getOrGenerateRequestId(threadContext);
         final Authentication authentication = new Authentication(new User("test api key user", "superuser"),
@@ -1041,7 +1042,7 @@ public void testApiKeyAuthUsesApiKeyServiceWithScopedRole() throws IOException {
         final CompositeRolesStore compositeRolesStore =
             new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, fileRolesStore, nativeRolesStore, reservedRolesStore,
                 nativePrivStore, Collections.emptyList(), new ThreadContext(SECURITY_ENABLED_SETTINGS),
-                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, apiKeyService,
+                new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, apiKeyService, null,
                 rds -> effectiveRoleDescriptors.set(rds));
         AuditUtil.getOrGenerateRequestId(threadContext);
         final Authentication authentication = new Authentication(new User("test api key user", "api_key"),
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java
index d50663b9d7cab..486b561a5cbe0 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java
@@ -6,7 +6,7 @@
 package org.elasticsearch.xpack.security.authz.store;
 
 import org.apache.lucene.search.TotalHits;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -36,7 +36,6 @@
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheRequest;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
 import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
@@ -72,7 +71,6 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-@TestLogging("org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore:TRACE")
 public class NativePrivilegeStoreTests extends ESTestCase {
 
     private NativePrivilegeStore store;
@@ -87,7 +85,7 @@ public void setup() {
         client = new NoOpClient(getTestName()) {
             @Override
             protected <Request extends ActionRequest, Response extends ActionResponse>
-            void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+            void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
                 NativePrivilegeStoreTests.this.requests.add(request);
                 NativePrivilegeStoreTests.this.listener.set(listener);
             }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java
index 26c59a1ef5470..77d1ba667d1b9 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor.Property;
 
-import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
@@ -26,7 +25,7 @@ public class SetSecurityUserProcessorTests extends ESTestCase {
 
     public void testProcessor() throws Exception {
         User user = new User("_username", new String[]{"role1", "role2"}, "firstname lastname", "_email",
-                Collections.singletonMap("key", "value"), true);
+                Map.of("key", "value"), true);
         Authentication.RealmRef realmRef = new Authentication.RealmRef("_name", "_type", "_node_name");
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         threadContext.putTransient(AuthenticationField.AUTHENTICATION_KEY, new Authentication(user, realmRef, null));
@@ -101,7 +100,7 @@ public void testRolesProperties() throws Exception {
 
     public void testFullNameProperties() throws Exception {
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        User user = new User(null, null, "_full_name", null, null, true);
+        User user = new User(null, null, "_full_name", null, Map.of(), true);
         Authentication.RealmRef realmRef = new Authentication.RealmRef("_name", "_type", "_node_name");
         threadContext.putTransient(AuthenticationField.AUTHENTICATION_KEY, new Authentication(user, realmRef, null));
 
@@ -117,7 +116,7 @@ public void testFullNameProperties() throws Exception {
 
     public void testEmailProperties() throws Exception {
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        User user = new User(null, null, null, "_email", null, true);
+        User user = new User(null, null, null, "_email", Map.of(), true);
         Authentication.RealmRef realmRef = new Authentication.RealmRef("_name", "_type", "_node_name");
         threadContext.putTransient(AuthenticationField.AUTHENTICATION_KEY, new Authentication(user, realmRef, null));
 
@@ -133,7 +132,7 @@ public void testEmailProperties() throws Exception {
 
     public void testMetadataProperties() throws Exception {
         ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        User user = new User(null, null, null, null, Collections.singletonMap("key", "value"), true);
+        User user = new User(null, null, null, null, Map.of("key", "value"), true);
         Authentication.RealmRef realmRef = new Authentication.RealmRef("_name", "_type", "_node_name");
         threadContext.putTransient(AuthenticationField.AUTHENTICATION_KEY, new Authentication(user, realmRef, null));
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyActionTests.java
index 394c9747b6daf..3f9b743c8e788 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyActionTests.java
@@ -8,7 +8,7 @@
 
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -30,7 +30,6 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
-import org.elasticsearch.xpack.security.rest.action.apikey.RestCreateApiKeyAction;
 
 import java.time.Duration;
 import java.time.Instant;
@@ -87,8 +86,8 @@ public void sendResponse(RestResponse restResponse) {
 
         try (NodeClient client = new NodeClient(Settings.EMPTY, threadPool) {
             @Override
-            public <Request extends ActionRequest, Response extends ActionResponse> void doExecute(Action<Response> action, Request request,
-                    ActionListener<Response> listener) {
+            public <Request extends ActionRequest, Response extends ActionResponse>
+            void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
                 CreateApiKeyRequest createApiKeyRequest = (CreateApiKeyRequest) request;
                 @SuppressWarnings("unchecked")
                 RestToXContentListener<CreateApiKeyResponse> actionListener = (RestToXContentListener<CreateApiKeyResponse>) listener;
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java
index 9788bc1a5b22f..c706a251dda35 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java
@@ -8,7 +8,7 @@
 
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
@@ -31,7 +31,6 @@
 import org.elasticsearch.xpack.core.security.action.ApiKey;
 import org.elasticsearch.xpack.core.security.action.GetApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.GetApiKeyResponse;
-import org.elasticsearch.xpack.security.rest.action.apikey.RestGetApiKeyAction;
 
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
@@ -92,8 +91,8 @@ public void sendResponse(RestResponse restResponse) {
         try (NodeClient client = new NodeClient(Settings.EMPTY, threadPool) {
             @SuppressWarnings("unchecked")
             @Override
-            public <Request extends ActionRequest, Response extends ActionResponse> void doExecute(Action<Response> action, Request request,
-                                                                                                   ActionListener<Response> listener) {
+            public <Request extends ActionRequest, Response extends ActionResponse>
+            void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
                 GetApiKeyRequest getApiKeyRequest = (GetApiKeyRequest) request;
                 ActionRequestValidationException validationException = getApiKeyRequest.validate();
                 if (validationException != null) {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyActionTests.java
index e73f4e3c210d4..21e65c485fb2b 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestInvalidateApiKeyActionTests.java
@@ -8,7 +8,7 @@
 
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
@@ -29,7 +29,6 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyResponse;
-import org.elasticsearch.xpack.security.rest.action.apikey.RestInvalidateApiKeyAction;
 
 import java.util.Collections;
 
@@ -85,8 +84,8 @@ public void sendResponse(RestResponse restResponse) {
 
         try (NodeClient client = new NodeClient(Settings.EMPTY, threadPool) {
             @Override
-            public <Request extends ActionRequest, Response extends ActionResponse> void doExecute(Action<Response> action, Request request,
-                                                                                                   ActionListener<Response> listener) {
+            public <Request extends ActionRequest, Response extends ActionResponse>
+            void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
                 InvalidateApiKeyRequest invalidateApiKeyRequest = (InvalidateApiKeyRequest) request;
                 ActionRequestValidationException validationException = invalidateApiKeyRequest.validate();
                 if (validationException != null) {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestGetTokenActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestGetTokenActionTests.java
index 9e99d0cf1563c..a3e875fec2c02 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestGetTokenActionTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/oauth2/RestGetTokenActionTests.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.security.rest.action.oauth2;
 
 import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
@@ -23,8 +24,10 @@
 import org.elasticsearch.xpack.core.security.action.token.CreateTokenRequest;
 import org.elasticsearch.xpack.core.security.action.token.CreateTokenResponse;
 import org.elasticsearch.xpack.core.security.support.NoOpLogger;
+import org.elasticsearch.xpack.security.authc.kerberos.KerberosAuthenticationToken;
 import org.elasticsearch.xpack.security.rest.action.oauth2.RestGetTokenAction.CreateTokenResponseActionListener;
 
+import java.util.Locale;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.hasEntry;
@@ -43,7 +46,7 @@ public void sendResponse(RestResponse restResponse) {
         };
         CreateTokenResponseActionListener listener = new CreateTokenResponseActionListener(restChannel, restRequest, NoOpLogger.INSTANCE);
 
-        ActionRequestValidationException ve = new CreateTokenRequest(null, null, null, null, null).validate();
+        ActionRequestValidationException ve = new CreateTokenRequest(null, null, null, null, null, null).validate();
         listener.onFailure(ve);
         RestResponse response = responseSetOnce.get();
         assertNotNull(response);
@@ -67,7 +70,8 @@ public void sendResponse(RestResponse restResponse) {
         };
         CreateTokenResponseActionListener listener = new CreateTokenResponseActionListener(restChannel, restRequest, NoOpLogger.INSTANCE);
         CreateTokenResponse createTokenResponse =
-                new CreateTokenResponse(randomAlphaOfLengthBetween(1, 256), TimeValue.timeValueHours(1L), null, randomAlphaOfLength(4));
+                new CreateTokenResponse(randomAlphaOfLengthBetween(1, 256), TimeValue.timeValueHours(1L), null, randomAlphaOfLength(4),
+                        randomAlphaOfLength(5));
         listener.onResponse(createTokenResponse);
 
         RestResponse response = responseSetOnce.get();
@@ -80,7 +84,39 @@ public void sendResponse(RestResponse restResponse) {
         assertThat(map, hasEntry("access_token", createTokenResponse.getTokenString()));
         assertThat(map, hasEntry("expires_in", Math.toIntExact(createTokenResponse.getExpiresIn().seconds())));
         assertThat(map, hasEntry("refresh_token", createTokenResponse.getRefreshToken()));
-        assertEquals(4, map.size());
+        assertThat(map, hasEntry("kerberos_authentication_response_token", createTokenResponse.getKerberosAuthenticationResponseToken()));
+        assertEquals(5, map.size());
+    }
+
+    public void testSendResponseKerberosError() {
+        FakeRestRequest restRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).build();
+        final SetOnce<RestResponse> responseSetOnce = new SetOnce<>();
+        RestChannel restChannel = new AbstractRestChannel(restRequest, randomBoolean()) {
+            @Override
+            public void sendResponse(RestResponse restResponse) {
+                responseSetOnce.set(restResponse);
+            }
+        };
+        CreateTokenResponseActionListener listener = new CreateTokenResponseActionListener(restChannel, restRequest, NoOpLogger.INSTANCE);
+        String errorMessage = "failed to authenticate user, gss context negotiation not complete";
+        ElasticsearchSecurityException ese = new ElasticsearchSecurityException(errorMessage, RestStatus.UNAUTHORIZED);
+        boolean addBase64EncodedToken = randomBoolean();
+        ese.addHeader(KerberosAuthenticationToken.WWW_AUTHENTICATE, "Negotiate" + ((addBase64EncodedToken) ? " FAIL" : ""));
+        listener.onFailure(ese);
+
+        RestResponse response = responseSetOnce.get();
+        assertNotNull(response);
+
+        Map<String, Object> map = XContentHelper.convertToMap(response.content(), false,
+                XContentType.fromMediaType(response.contentType())).v2();
+        assertThat(map, hasEntry("error", RestGetTokenAction.TokenRequestError._UNAUTHORIZED.name().toLowerCase(Locale.ROOT)));
+        if (addBase64EncodedToken) {
+            assertThat(map, hasEntry("error_description", "FAIL"));
+        } else {
+            assertThat(map, hasEntry("error_description", null));
+        }
+        assertEquals(2, map.size());
+        assertEquals(RestStatus.BAD_REQUEST, response.status());
     }
 
     public void testParser() throws Exception {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java
index 55cd659509bfe..93d40c9981dff 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerIntegTests.java
@@ -6,10 +6,10 @@
 package org.elasticsearch.xpack.security.support;
 
 import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.test.SecurityIntegTestCase;
-import org.elasticsearch.xpack.core.security.action.user.PutUserRequest;
+import org.elasticsearch.xpack.core.security.action.user.PutUserRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.user.PutUserResponse;
 import org.hamcrest.Matchers;
 import org.junit.After;
@@ -44,21 +44,19 @@ public void onFailure(Exception e) {
 
                 @Override
                 protected void doRun() throws Exception {
-                    final List<PutUserRequest> requests = new ArrayList<>(numRequests);
+                    final List<PutUserRequestBuilder> requests = new ArrayList<>(numRequests);
+                    final SecureString password = new SecureString("password".toCharArray());
                     for (int i = 0; i < numRequests; i++) {
-                        requests.add(securityClient()
-                                .preparePutUser("user" + userNumber.getAndIncrement(), "password".toCharArray(),
-                                    getFastStoredHashAlgoForTests(),
-                                    randomAlphaOfLengthBetween(1, 16))
-                                .request());
+                        requests.add(new PutUserRequestBuilder(client())
+                            .username("user" + userNumber.getAndIncrement())
+                            .password(password, getFastStoredHashAlgoForTests())
+                            .roles(randomAlphaOfLengthBetween(1, 16)));
                     }
 
                     barrier.await(10L, TimeUnit.SECONDS);
 
-                    for (PutUserRequest request : requests) {
-                        PlainActionFuture<PutUserResponse> responsePlainActionFuture = new PlainActionFuture<>();
-                        securityClient().putUser(request, responsePlainActionFuture);
-                        futures.add(responsePlainActionFuture);
+                    for (PutUserRequestBuilder request : requests) {
+                        futures.add(request.execute());
                     }
                 }
             }, "create_users_thread" + i);
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java
index 157e0ffb82013..3df7c95ee56ef 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java
@@ -7,7 +7,7 @@
 
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
@@ -74,7 +74,7 @@ public class SecurityIndexManagerTests extends ESTestCase {
     private static final ClusterState EMPTY_CLUSTER_STATE = new ClusterState.Builder(CLUSTER_NAME).build();
     private static final String TEMPLATE_NAME = "SecurityIndexManagerTests-template";
     private SecurityIndexManager manager;
-    private Map<Action<?>, Map<ActionRequest, ActionListener<?>>> actions;
+    private Map<ActionType<?>, Map<ActionRequest, ActionListener<?>>> actions;
 
     @Before
     public void setUpManager() {
@@ -90,7 +90,7 @@ public void setUpManager() {
         final Client client = new FilterClient(mockClient) {
             @Override
             protected <Request extends ActionRequest, Response extends ActionResponse>
-            void doExecute(Action<Response> action, Request request, ActionListener<Response> listener) {
+            void doExecute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
                 final Map<ActionRequest, ActionListener<?>> map = actions.getOrDefault(action, new HashMap<>());
                 map.put(request, listener);
                 actions.put(action, map);
@@ -155,8 +155,8 @@ public void testIndexHealthChangeListeners() throws Exception {
         manager.clusterChanged(event(clusterStateBuilder));
 
         assertTrue(listenerCalled.get());
-        assertNull(previousState.get().indexStatus);
-        assertEquals(ClusterHealthStatus.GREEN, currentState.get().indexStatus);
+        assertNull(previousState.get().indexHealth);
+        assertEquals(ClusterHealthStatus.GREEN, currentState.get().indexHealth);
 
         // reset and call with no change to the index
         listenerCalled.set(false);
@@ -191,8 +191,8 @@ public void testIndexHealthChangeListeners() throws Exception {
         event = new ClusterChangedEvent("different index health", clusterStateBuilder.build(), previousClusterState);
         manager.clusterChanged(event);
         assertTrue(listenerCalled.get());
-        assertEquals(ClusterHealthStatus.GREEN, previousState.get().indexStatus);
-        assertEquals(ClusterHealthStatus.RED, currentState.get().indexStatus);
+        assertEquals(ClusterHealthStatus.GREEN, previousState.get().indexHealth);
+        assertEquals(ClusterHealthStatus.RED, currentState.get().indexHealth);
 
         // swap prev and current
         listenerCalled.set(false);
@@ -201,8 +201,8 @@ public void testIndexHealthChangeListeners() throws Exception {
         event = new ClusterChangedEvent("different index health swapped", previousClusterState, clusterStateBuilder.build());
         manager.clusterChanged(event);
         assertTrue(listenerCalled.get());
-        assertEquals(ClusterHealthStatus.RED, previousState.get().indexStatus);
-        assertEquals(ClusterHealthStatus.GREEN, currentState.get().indexStatus);
+        assertEquals(ClusterHealthStatus.RED, previousState.get().indexHealth);
+        assertEquals(ClusterHealthStatus.GREEN, currentState.get().indexHealth);
     }
 
     public void testWriteBeforeStateNotRecovered() throws Exception {
@@ -247,7 +247,7 @@ public void testWriteBeforeStateNotRecovered() throws Exception {
         assertThat(prepareRunnableCalled.get(), is(true));
     }
 
-    public void testListeneredNotCalledBeforeStateNotRecovered() throws Exception {
+    public void testListenerNotCalledBeforeStateNotRecovered() throws Exception {
         final AtomicBoolean listenerCalled = new AtomicBoolean(false);
         manager.addIndexStateListener((prev, current) -> {
             listenerCalled.set(true);
@@ -307,6 +307,31 @@ public void testIndexOutOfDateListeners() throws Exception {
         assertTrue(manager.isIndexUpToDate());
     }
 
+    public void testProcessClosedIndexState() throws Exception {
+        // Index initially exists
+        final ClusterState.Builder indexAvailable = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7,
+            RestrictedIndicesNames.SECURITY_MAIN_ALIAS, TEMPLATE_NAME, IndexMetaData.State.OPEN);
+        markShardsAvailable(indexAvailable);
+
+        manager.clusterChanged(event(indexAvailable));
+        assertThat(manager.indexExists(), is(true));
+        assertThat(manager.isAvailable(), is(true));
+
+        // Now close it
+        final ClusterState.Builder indexClosed = createClusterState(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7,
+            RestrictedIndicesNames.SECURITY_MAIN_ALIAS, TEMPLATE_NAME, IndexMetaData.State.CLOSE);
+        if (randomBoolean()) {
+            // In old/mixed cluster versions closed indices have no routing table
+            indexClosed.routingTable(RoutingTable.EMPTY_ROUTING_TABLE);
+        } else {
+            markShardsAvailable(indexClosed);
+        }
+
+        manager.clusterChanged(event(indexClosed));
+        assertThat(manager.indexExists(), is(true));
+        assertThat(manager.isAvailable(), is(false));
+    }
+
     private void assertInitialState() {
         assertThat(manager.indexExists(), Matchers.equalTo(false));
         assertThat(manager.isAvailable(), Matchers.equalTo(false));
@@ -322,18 +347,23 @@ private void assertIndexUpToDateButNotAvailable() {
     }
 
     public static ClusterState.Builder createClusterState(String indexName, String aliasName, String templateName) throws IOException {
-        return createClusterState(indexName, aliasName, templateName, templateName, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT);
+        return createClusterState(indexName, aliasName, templateName, IndexMetaData.State.OPEN);
+    }
+
+    public static ClusterState.Builder createClusterState(String indexName, String aliasName, String templateName,
+                                                          IndexMetaData.State state) throws IOException {
+        return createClusterState(indexName, aliasName, templateName, templateName, SecurityIndexManager.INTERNAL_MAIN_INDEX_FORMAT, state);
     }
 
     public static ClusterState.Builder createClusterState(String indexName, String aliasName, String templateName, int format)
             throws IOException {
-        return createClusterState(indexName, aliasName, templateName, templateName, format);
+        return createClusterState(indexName, aliasName, templateName, templateName, format, IndexMetaData.State.OPEN);
     }
 
     private static ClusterState.Builder createClusterState(String indexName, String aliasName, String templateName, String buildMappingFrom,
-            int format) throws IOException {
+                                                           int format, IndexMetaData.State state) throws IOException {
         IndexTemplateMetaData.Builder templateBuilder = getIndexTemplateMetaData(templateName);
-        IndexMetaData.Builder indexMeta = getIndexMetadata(indexName, aliasName, buildMappingFrom, format);
+        IndexMetaData.Builder indexMeta = getIndexMetadata(indexName, aliasName, buildMappingFrom, format, state);
 
         MetaData.Builder metaDataBuilder = new MetaData.Builder();
         metaDataBuilder.put(templateBuilder);
@@ -354,7 +384,8 @@ private static ClusterState state() {
                 .build();
     }
 
-    private static IndexMetaData.Builder getIndexMetadata(String indexName, String aliasName, String templateName, int format)
+    private static IndexMetaData.Builder getIndexMetadata(String indexName, String aliasName, String templateName, int format,
+                                                          IndexMetaData.State state)
             throws IOException {
         IndexMetaData.Builder indexMetaData = IndexMetaData.builder(indexName);
         indexMetaData.settings(Settings.builder()
@@ -364,6 +395,7 @@ private static IndexMetaData.Builder getIndexMetadata(String indexName, String a
                 .put(IndexMetaData.INDEX_FORMAT_SETTING.getKey(), format)
                 .build());
         indexMetaData.putAlias(AliasMetaData.builder(aliasName).build());
+        indexMetaData.state(state);
         final Map<String, String> mappings = getTemplateMappings(templateName);
         for (Map.Entry<String, String> entry : mappings.entrySet()) {
             indexMetaData.putMapping(entry.getKey(), entry.getValue());
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityStatusChangeListenerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityStatusChangeListenerTests.java
new file mode 100644
index 0000000000000..da18d5dc902d4
--- /dev/null
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityStatusChangeListenerTests.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.support;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.license.License;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.MockLogAppender;
+import org.junit.After;
+import org.junit.Before;
+import org.mockito.Mockito;
+
+import static org.mockito.Mockito.when;
+
+public class SecurityStatusChangeListenerTests extends ESTestCase {
+
+    private XPackLicenseState licenseState;
+    private SecurityStatusChangeListener listener;
+    private MockLogAppender logAppender;
+    private Logger listenerLogger;
+
+    @Before
+    public void setup() throws IllegalAccessException {
+        licenseState = Mockito.mock(XPackLicenseState.class);
+        when(licenseState.isSecurityAvailable()).thenReturn(true);
+
+        listener = new SecurityStatusChangeListener(licenseState);
+
+        logAppender = new MockLogAppender();
+        logAppender.start();
+        listenerLogger = LogManager.getLogger(listener.getClass());
+        Loggers.addAppender(listenerLogger, logAppender);
+    }
+
+    @After
+    public void cleanup() {
+        Loggers.removeAppender(listenerLogger, logAppender);
+        logAppender.stop();
+    }
+
+    public void testSecurityEnabledToDisabled() {
+        when(licenseState.isSecurityDisabledByLicenseDefaults()).thenReturn(false);
+
+        when(licenseState.getOperationMode()).thenReturn(License.OperationMode.GOLD);
+        logAppender.addExpectation(new MockLogAppender.SeenEventExpectation(
+            "initial change",
+            listener.getClass().getName(),
+            Level.INFO,
+            "Active license is now [GOLD]; Security is enabled"
+        ));
+        listener.licenseStateChanged();
+
+        when(licenseState.getOperationMode()).thenReturn(License.OperationMode.PLATINUM);
+        logAppender.addExpectation(new MockLogAppender.UnseenEventExpectation(
+            "no-op change",
+            listener.getClass().getName(),
+            Level.INFO,
+            "Active license is now [PLATINUM]; Security is enabled"
+        ));
+
+        when(licenseState.isSecurityDisabledByLicenseDefaults()).thenReturn(true);
+        when(licenseState.getOperationMode()).thenReturn(License.OperationMode.BASIC);
+        logAppender.addExpectation(new MockLogAppender.SeenEventExpectation(
+            "change to basic",
+            listener.getClass().getName(),
+            Level.INFO,
+            "Active license is now [BASIC]; Security is disabled"
+        ));
+        listener.licenseStateChanged();
+
+        logAppender.assertAllExpectationsMatched();
+    }
+
+    public void testSecurityDisabledToEnabled() {
+        when(licenseState.isSecurityDisabledByLicenseDefaults()).thenReturn(true);
+
+        when(licenseState.getOperationMode()).thenReturn(License.OperationMode.TRIAL);
+        logAppender.addExpectation(new MockLogAppender.SeenEventExpectation(
+            "initial change",
+            listener.getClass().getName(),
+            Level.INFO,
+            "Active license is now [TRIAL]; Security is disabled"
+        ));
+        listener.licenseStateChanged();
+
+        when(licenseState.getOperationMode()).thenReturn(License.OperationMode.BASIC);
+        logAppender.addExpectation(new MockLogAppender.UnseenEventExpectation(
+            "no-op change",
+            listener.getClass().getName(),
+            Level.INFO,
+            "Active license is now [BASIC]; Security is disabled"
+        ));
+
+        when(licenseState.isSecurityDisabledByLicenseDefaults()).thenReturn(false);
+        when(licenseState.getOperationMode()).thenReturn(License.OperationMode.PLATINUM);
+        logAppender.addExpectation(new MockLogAppender.SeenEventExpectation(
+            "change to platinum",
+            listener.getClass().getName(),
+            Level.INFO,
+            "Active license is now [PLATINUM]; Security is enabled"
+        ));
+        listener.licenseStateChanged();
+
+        logAppender.assertAllExpectationsMatched();
+    }
+
+}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/AbstractSimpleSecurityTransportTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/AbstractSimpleSecurityTransportTestCase.java
index 1b7a7f8d870f2..80802db88f942 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/AbstractSimpleSecurityTransportTestCase.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/AbstractSimpleSecurityTransportTestCase.java
@@ -8,6 +8,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
 import org.elasticsearch.common.settings.MockSecureSettings;
@@ -45,6 +46,7 @@
 import javax.net.ssl.SSLServerSocket;
 import javax.net.ssl.SSLServerSocketFactory;
 import javax.net.ssl.SSLSocket;
+
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.net.InetAddress;
@@ -53,7 +55,6 @@
 import java.net.UnknownHostException;
 import java.nio.file.Path;
 import java.util.Collections;
-import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Locale;
@@ -259,7 +260,7 @@ public boolean matches(SNIServerName sniServerName) {
                 HashMap<String, String> attributes = new HashMap<>();
                 attributes.put("server_name", sniIp);
                 DiscoveryNode node = new DiscoveryNode("server_node_id", new TransportAddress(serverAddress), attributes,
-                    EnumSet.allOf(DiscoveryNode.Role.class), Version.CURRENT);
+                    DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT);
 
                 new Thread(() -> {
                     try {
@@ -306,7 +307,7 @@ public void testInvalidSNIServerName() throws Exception {
                 HashMap<String, String> attributes = new HashMap<>();
                 attributes.put("server_name", sniIp);
                 DiscoveryNode node = new DiscoveryNode("server_node_id", new TransportAddress(serverAddress), attributes,
-                    EnumSet.allOf(DiscoveryNode.Role.class), Version.CURRENT);
+                    DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT);
 
                 ConnectTransportException connectException = expectThrows(ConnectTransportException.class,
                     () -> serviceC.connectToNode(node, TestProfiles.LIGHT_PROFILE));
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java
index 1b85049da235b..89f33f809befb 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.security.transport;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.main.MainAction;
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlocks;
@@ -45,6 +46,8 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
@@ -52,7 +55,6 @@
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
-import static org.mockito.Mockito.verifyZeroInteractions;
 import static org.mockito.Mockito.when;
 
 public class SecurityServerTransportInterceptorTests extends ESTestCase {
@@ -82,7 +84,7 @@ public void stopThreadPool() throws Exception {
         terminate(threadPool);
     }
 
-    public void testSendAsyncUnlicensed() {
+    public void testSendAsyncUserActionWhenUnlicensed() {
         SecurityServerTransportInterceptor interceptor = new SecurityServerTransportInterceptor(settings, threadPool,
                 mock(AuthenticationService.class), mock(AuthorizationService.class), xPackLicenseState, mock(SSLService.class),
                 securityContext, new DestructiveOperations(Settings.EMPTY, new ClusterSettings(Settings.EMPTY,
@@ -90,6 +92,7 @@ securityContext, new DestructiveOperations(Settings.EMPTY, new ClusterSettings(S
         ClusterServiceUtils.setState(clusterService, clusterService.state()); // force state update to trigger listener
         when(xPackLicenseState.isAuthAllowed()).thenReturn(false);
         AtomicBoolean calledWrappedSender = new AtomicBoolean(false);
+        AtomicReference<User> sendingUser = new AtomicReference<>();
         AsyncSender sender = interceptor.interceptSender(new AsyncSender() {
             @Override
             public <T extends TransportResponse> void sendRequest(Transport.Connection connection, String action, TransportRequest request,
@@ -97,13 +100,45 @@ public <T extends TransportResponse> void sendRequest(Transport.Connection conne
                 if (calledWrappedSender.compareAndSet(false, true) == false) {
                     fail("sender called more than once!");
                 }
+                sendingUser.set(securityContext.getUser());
             }
         });
-        sender.sendRequest(null, null, null, null, null);
+        Connection connection = mock(Connection.class);
+        when(connection.getVersion()).thenReturn(Version.CURRENT);
+        sender.sendRequest(connection, MainAction.NAME, null, null, null);
         assertTrue(calledWrappedSender.get());
+        assertThat(sendingUser.get(), nullValue());
         verify(xPackLicenseState).isAuthAllowed();
         verifyNoMoreInteractions(xPackLicenseState);
-        verifyZeroInteractions(securityContext);
+    }
+
+    public void testSendAsyncInternalActionWhenUnlicensed() {
+        SecurityServerTransportInterceptor interceptor = new SecurityServerTransportInterceptor(settings, threadPool,
+                mock(AuthenticationService.class), mock(AuthorizationService.class), xPackLicenseState, mock(SSLService.class),
+                securityContext, new DestructiveOperations(Settings.EMPTY, new ClusterSettings(Settings.EMPTY,
+                Collections.singleton(DestructiveOperations.REQUIRES_NAME_SETTING))), clusterService);
+        ClusterServiceUtils.setState(clusterService, clusterService.state()); // force state update to trigger listener
+        when(xPackLicenseState.isAuthAllowed()).thenReturn(false);
+        AtomicBoolean calledWrappedSender = new AtomicBoolean(false);
+        AtomicReference<User> sendingUser = new AtomicReference<>();
+        AsyncSender sender = interceptor.interceptSender(new AsyncSender() {
+            @Override
+            public <T extends TransportResponse> void sendRequest(Transport.Connection connection, String action, TransportRequest request,
+                                                                  TransportRequestOptions options, TransportResponseHandler<T> handler) {
+                if (calledWrappedSender.compareAndSet(false, true) == false) {
+                    fail("sender called more than once!");
+                }
+                sendingUser.set(securityContext.getUser());
+            }
+        });
+        Connection connection = mock(Connection.class);
+        when(connection.getVersion()).thenReturn(Version.CURRENT);
+        sender.sendRequest(connection, "internal:foo", null, null, null);
+        assertTrue(calledWrappedSender.get());
+        assertThat(sendingUser.get(), is(SystemUser.INSTANCE));
+        verify(xPackLicenseState).isAuthAllowed();
+        verify(securityContext).executeAsUser(any(User.class), any(Consumer.class), eq(Version.CURRENT));
+        verifyNoMoreInteractions(xPackLicenseState);
     }
 
     public void testSendAsyncWithStateNotRecovered() {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java
deleted file mode 100644
index 8bf5bf1d0af77..0000000000000
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.security.transport;
-
-import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.network.NetworkAddress;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.node.MockNode;
-import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeValidationException;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.MockHttpTransport;
-import org.elasticsearch.test.SecurityIntegTestCase;
-import org.elasticsearch.test.SecuritySettingsSourceField;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.ConnectionProfile;
-import org.elasticsearch.transport.Transport;
-import org.elasticsearch.transport.TransportException;
-import org.elasticsearch.transport.TransportRequestOptions;
-import org.elasticsearch.transport.TransportResponse;
-import org.elasticsearch.transport.TransportResponseHandler;
-import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.security.SecurityField;
-import org.elasticsearch.xpack.security.LocalStateSecurity;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-
-import static org.elasticsearch.discovery.SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING;
-import static org.elasticsearch.test.SecuritySettingsSource.addSSLSettingsForNodePEMFiles;
-import static org.elasticsearch.test.SecuritySettingsSource.addSSLSettingsForPEMFiles;
-import static org.elasticsearch.xpack.security.test.SecurityTestUtils.writeFile;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.instanceOf;
-
-public class ServerTransportFilterIntegrationTests extends SecurityIntegTestCase {
-    private static int randomClientPort;
-
-    @BeforeClass
-    public static void getRandomPort() {
-        randomClientPort = randomIntBetween(49000, 65500); // ephemeral port
-    }
-
-    @Override
-    public boolean transportSSLEnabled() {
-        return true;
-    }
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        Settings.Builder settingsBuilder = Settings.builder().put(super.nodeSettings(nodeOrdinal));
-        String randomClientPortRange = randomClientPort + "-" + (randomClientPort+100);
-        addSSLSettingsForNodePEMFiles(settingsBuilder, "transport.profiles.client.xpack.security.", true);
-        Path certPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt");
-        settingsBuilder.putList("transport.profiles.client.xpack.security.ssl.certificate_authorities",
-            Collections.singletonList(certPath.toString())) // settings for client truststore
-            .put("transport.profiles.client.xpack.security.type", "client")
-            .put("transport.profiles.client.port", randomClientPortRange)
-            // make sure this is "localhost", no matter if ipv4 or ipv6, but be consistent
-            .put("transport.profiles.client.bind_host", "localhost")
-            .put("xpack.security.audit.enabled", false)
-            .put(XPackSettings.WATCHER_ENABLED.getKey(), false);
-        if (randomBoolean()) {
-            settingsBuilder.put("transport.profiles.default.xpack.security.type", "node"); // this is default lets set it randomly
-        }
-
-        return settingsBuilder.build();
-    }
-
-    public void testThatConnectionToServerTypeConnectionWorks() throws IOException, NodeValidationException {
-        Path home = createTempDir();
-        Path xpackConf = home.resolve("config");
-        Files.createDirectories(xpackConf);
-
-        Transport transport = internalCluster().getMasterNodeInstance(Transport.class);
-        TransportAddress transportAddress = transport.boundAddress().publishAddress();
-        String unicastHost = NetworkAddress.format(transportAddress.address());
-
-        // test that starting up a node works
-        Settings.Builder nodeSettings = Settings.builder()
-            .put("node.name", "my-test-node")
-            .put("network.host", "localhost")
-            .put("cluster.name", internalCluster().getClusterName())
-            .put(DISCOVERY_SEED_HOSTS_SETTING.getKey(), unicastHost)
-            .put("xpack.security.enabled", true)
-            .put("xpack.security.audit.enabled", false)
-            .put("xpack.security.transport.ssl.enabled", true)
-            .put(XPackSettings.WATCHER_ENABLED.getKey(), false)
-            .put("path.home", home)
-            .put(Node.NODE_MASTER_SETTING.getKey(), false);
-        Collection<Class<? extends Plugin>> mockPlugins = Arrays.asList(LocalStateSecurity.class, MockHttpTransport.TestPlugin.class);
-        addSSLSettingsForPEMFiles(
-            nodeSettings,
-            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem",
-            "testnode",
-            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt",
-            List.of("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt",
-                "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_ec.crt"));
-        try (Node node = new MockNode(nodeSettings.build(), mockPlugins)) {
-            node.start();
-            ensureStableCluster(cluster().size() + 1);
-        }
-    }
-
-    public void testThatConnectionToClientTypeConnectionIsRejected() throws IOException, NodeValidationException, InterruptedException {
-        Path home = createTempDir();
-        Path xpackConf = home.resolve("config");
-        Files.createDirectories(xpackConf);
-        writeFile(xpackConf, "users", configUsers());
-        writeFile(xpackConf, "users_roles", configUsersRoles());
-        writeFile(xpackConf, "roles.yml", configRoles());
-
-        Transport transport = internalCluster().getMasterNodeInstance(Transport.class);
-        TransportAddress transportAddress = transport.profileBoundAddresses().get("client").publishAddress();
-        String unicastHost = NetworkAddress.format(transportAddress.address());
-
-        // test that starting up a node works
-        Settings.Builder nodeSettings = Settings.builder()
-            .put("xpack.security.authc.realms.file.file.order", 0)
-            .put("node.name", "my-test-node")
-            .put(SecurityField.USER_SETTING.getKey(), "test_user:" + SecuritySettingsSourceField.TEST_PASSWORD)
-            .put("cluster.name", internalCluster().getClusterName())
-            .put(DISCOVERY_SEED_HOSTS_SETTING.getKey(), unicastHost)
-            .put("xpack.security.enabled", true)
-            .put("xpack.security.audit.enabled", false)
-            .put("xpack.security.transport.ssl.enabled", true)
-            .put(XPackSettings.WATCHER_ENABLED.getKey(), false)
-            .put("discovery.initial_state_timeout", "0s")
-            .put("path.home", home)
-            .put(Node.NODE_MASTER_SETTING.getKey(), false);
-        Collection<Class<? extends Plugin>> mockPlugins = Arrays.asList(LocalStateSecurity.class, MockHttpTransport.TestPlugin.class);
-        addSSLSettingsForPEMFiles(
-            nodeSettings,
-            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem",
-            "testnode",
-            "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt",
-            List.of("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt",
-                "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_ec.crt"));
-        try (Node node = new MockNode(nodeSettings.build(), mockPlugins)) {
-            node.start();
-            TransportService instance = node.injector().getInstance(TransportService.class);
-            try (Transport.Connection connection = instance.openConnection(new DiscoveryNode("theNode", transportAddress, Version.CURRENT),
-                    ConnectionProfile.buildSingleChannelProfile(TransportRequestOptions.Type.REG))) {
-                // handshake should be ok
-                final DiscoveryNode handshake = instance.handshake(connection, 10000);
-                assertEquals(transport.boundAddress().publishAddress(), handshake.getAddress());
-                CountDownLatch latch = new CountDownLatch(1);
-                instance.sendRequest(connection, NodeMappingRefreshAction.ACTION_NAME,
-                        new NodeMappingRefreshAction.NodeMappingRefreshRequest("foo", "bar", "baz"),
-                        TransportRequestOptions.EMPTY,
-                        new TransportResponseHandler<TransportResponse>() {
-                    @Override
-                    public TransportResponse read(StreamInput in) {
-                        try {
-                            fail("never get that far");
-                        } finally {
-                            latch.countDown();
-                        }
-                        return null;
-                    }
-
-                    @Override
-                    public void handleResponse(TransportResponse response) {
-                        try {
-                            fail("never get that far");
-                        } finally {
-                            latch.countDown();
-                        }
-                    }
-
-                    @Override
-                    public void handleException(TransportException exp) {
-                        try {
-                            assertThat(exp.getCause(), instanceOf(ElasticsearchSecurityException.class));
-                            assertThat(exp.getCause().getMessage(),
-                                    equalTo("executing internal/shard actions is considered malicious and forbidden"));
-                        } finally {
-                            latch.countDown();
-                        }
-                    }
-
-                    @Override
-                    public String executor() {
-                        return ThreadPool.Names.SAME;
-                    }
-                });
-                latch.await();
-            }
-        }
-    }
-
-}
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java
index cce9c7ecdd0b9..d8628fe31d853 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterTests.java
@@ -33,7 +33,6 @@
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.junit.Before;
 
-import java.io.IOException;
 import java.util.Collections;
 
 import static org.elasticsearch.xpack.core.security.support.Exceptions.authenticationError;
@@ -83,7 +82,7 @@ public void testInbound() throws Exception {
             callback.onResponse(authentication);
             return Void.TYPE;
         }).when(authcService).authenticate(eq("_action"), eq(request), eq((User)null), any(ActionListener.class));
-        ServerTransportFilter filter = getClientOrNodeFilter();
+        ServerTransportFilter filter = getNodeFilter();
         PlainActionFuture<Void> future = new PlainActionFuture<>();
         filter.inbound("_action", request, channel, future);
         //future.get(); // don't block it's not called really just mocked
@@ -104,7 +103,7 @@ public void testInboundDestructiveOperations() throws Exception {
             callback.onResponse(authentication);
             return Void.TYPE;
         }).when(authcService).authenticate(eq(action), eq(request), eq((User)null), any(ActionListener.class));
-        ServerTransportFilter filter = getClientOrNodeFilter();
+        ServerTransportFilter filter = getNodeFilter();
         PlainActionFuture listener = mock(PlainActionFuture.class);
         filter.inbound(action, request, channel, listener);
         if (failDestructiveOperations) {
@@ -124,7 +123,7 @@ public void testInboundAuthenticationException() throws Exception {
             callback.onFailure(authE);
             return Void.TYPE;
         }).when(authcService).authenticate(eq("_action"), eq(request), eq((User)null), any(ActionListener.class));
-        ServerTransportFilter filter = getClientOrNodeFilter();
+        ServerTransportFilter filter = getNodeFilter();
         try {
             PlainActionFuture<Void> future = new PlainActionFuture<>();
             filter.inbound("_action", request, channel, future);
@@ -137,7 +136,7 @@ public void testInboundAuthenticationException() throws Exception {
     }
 
     public void testInboundAuthorizationException() throws Exception {
-        ServerTransportFilter filter = getClientOrNodeFilter();
+        ServerTransportFilter filter = getNodeFilter();
         TransportRequest request = mock(TransportRequest.class);
         Authentication authentication = mock(Authentication.class);
         doAnswer((i) -> {
@@ -158,23 +157,10 @@ public void testInboundAuthorizationException() throws Exception {
         assertThat(e.getMessage(), equalTo("authz failed"));
     }
 
-    public void testClientProfileRejectsNodeActions() throws Exception {
-        TransportRequest request = mock(TransportRequest.class);
-        ServerTransportFilter filter = getClientFilter(true);
-        ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
-                () -> filter.inbound("internal:foo/bar", request, channel, new PlainActionFuture<>()));
-        assertEquals("executing internal/shard actions is considered malicious and forbidden", e.getMessage());
-        e = expectThrows(ElasticsearchSecurityException.class,
-                () -> filter.inbound("indices:action" + randomFrom("[s]", "[p]", "[r]", "[n]", "[s][p]", "[s][r]", "[f]"),
-                        request, channel, new PlainActionFuture<>()));
-        assertEquals("executing internal/shard actions is considered malicious and forbidden", e.getMessage());
-        verifyZeroInteractions(authcService);
-    }
-
-    public void testNodeProfileAllowsNodeActions() throws Exception {
+    public void testAllowsNodeActions() throws Exception {
         final String internalAction = "internal:foo/bar";
         final String nodeOrShardAction = "indices:action" + randomFrom("[s]", "[p]", "[r]", "[n]", "[s][p]", "[s][r]", "[f]");
-        ServerTransportFilter filter = getNodeFilter(true);
+        ServerTransportFilter filter = getNodeFilter();
         TransportRequest request = mock(TransportRequest.class);
         Authentication authentication = new Authentication(new User("test", "superuser"), new RealmRef("test", "test", "node1"), null);
         doAnswer((i) -> {
@@ -200,21 +186,10 @@ public void testNodeProfileAllowsNodeActions() throws Exception {
         verifyNoMoreInteractions(authcService, authzService);
     }
 
-    private ServerTransportFilter getClientOrNodeFilter() throws IOException {
-        return randomBoolean() ? getNodeFilter(true) : getClientFilter(true);
-    }
-
-    private ServerTransportFilter.ClientProfile getClientFilter(boolean reservedRealmEnabled) throws IOException {
-        Settings settings = Settings.builder().put("path.home", createTempDir()).build();
-        ThreadContext threadContext = new ThreadContext(settings);
-        return new ServerTransportFilter.ClientProfile(authcService, authzService, threadContext, false, destructiveOperations,
-                reservedRealmEnabled, new SecurityContext(settings, threadContext), new XPackLicenseState(settings));
-    }
-
-    private ServerTransportFilter.NodeProfile getNodeFilter(boolean reservedRealmEnabled) throws IOException {
+    private ServerTransportFilter getNodeFilter() {
         Settings settings = Settings.builder().put("path.home", createTempDir()).build();
         ThreadContext threadContext = new ThreadContext(settings);
-        return new ServerTransportFilter.NodeProfile(authcService, authzService, threadContext, false, destructiveOperations,
-                reservedRealmEnabled, new SecurityContext(settings, threadContext), new XPackLicenseState(settings));
+        return new ServerTransportFilter(authcService, authzService, threadContext, false, destructiveOperations,
+                new SecurityContext(settings, threadContext), new XPackLicenseState(settings));
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringIntegrationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringIntegrationTests.java
index 2284a920eac11..843b0403ab24f 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringIntegrationTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/filter/IpFilteringIntegrationTests.java
@@ -24,7 +24,7 @@
 
 import static org.hamcrest.Matchers.is;
 
-// no client nodes, no transport clients, as they all get rejected on network connections
+// no client nodes as they all get rejected on network connections
 @ClusterScope(scope = Scope.SUITE, numDataNodes = 0, numClientNodes = 0)
 public class IpFilteringIntegrationTests extends SecurityIntegTestCase {
     private static int randomClientPort;
@@ -70,7 +70,7 @@ public void testThatIpFilteringIsAppliedForProfile() throws Exception {
     @SuppressForbidden(reason = "Allow opening socket for test")
     private void trySocketConnection(Socket socket, InetSocketAddress address) throws IOException {
         logger.info("connecting to {}", address);
-        SocketAccess.doPrivileged(() -> socket.connect(address, 500));
+        SocketAccess.doPrivileged(() -> socket.connect(address, 5000));
 
         assertThat(socket.isConnected(), is(true));
         try (OutputStream os = socket.getOutputStream()) {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/NioIPFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/NioIPFilterTests.java
index 3df00018af420..e7612c0c0d7fa 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/NioIPFilterTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/NioIPFilterTests.java
@@ -13,7 +13,7 @@
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.nio.NioSocketChannel;
+import org.elasticsearch.nio.NioChannelHandler;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
@@ -26,13 +26,15 @@
 import java.util.Collections;
 import java.util.HashSet;
 
-import static org.hamcrest.Matchers.is;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 public class NioIPFilterTests extends ESTestCase {
 
-    private NioIPFilter nioIPFilter;
+    private IPFilter ipFilter;
+    private String profile;
 
     @Before
     public void init() throws Exception {
@@ -59,7 +61,7 @@ public void init() throws Exception {
         XPackLicenseState licenseState = mock(XPackLicenseState.class);
         when(licenseState.isIpFilteringAllowed()).thenReturn(true);
         AuditTrailService auditTrailService = new AuditTrailService(Collections.emptyList(), licenseState);
-        IPFilter ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
+        ipFilter = new IPFilter(settings, auditTrailService, clusterSettings, licenseState);
         ipFilter.setBoundTransportAddress(transport.boundAddress(), transport.profileBoundAddresses());
         if (isHttpEnabled) {
             HttpServerTransport httpTransport = mock(HttpServerTransport.class);
@@ -70,21 +72,27 @@ public void init() throws Exception {
         }
 
         if (isHttpEnabled) {
-            nioIPFilter = new NioIPFilter(ipFilter, IPFilter.HTTP_PROFILE_NAME);
+            profile = IPFilter.HTTP_PROFILE_NAME;
         } else {
-            nioIPFilter = new NioIPFilter(ipFilter, "default");
+            profile = "default";
         }
     }
 
-    public void testThatFilteringWorksByIp() throws Exception {
+    public void testThatFilterCanPass() throws Exception {
         InetSocketAddress localhostAddr = new InetSocketAddress(InetAddresses.forString("127.0.0.1"), 12345);
-        NioSocketChannel channel1 = mock(NioSocketChannel.class);
-        when(channel1.getRemoteAddress()).thenReturn(localhostAddr);
-        assertThat(nioIPFilter.test(channel1), is(true));
+        NioChannelHandler delegate = mock(NioChannelHandler.class);
+        NioIPFilter nioIPFilter = new NioIPFilter(delegate, localhostAddr, ipFilter, profile);
+        nioIPFilter.channelRegistered();
+        verify(delegate).channelRegistered();
+        assertFalse(nioIPFilter.closeNow());
+    }
 
-        InetSocketAddress remoteAddr = new InetSocketAddress(InetAddresses.forString("10.0.0.8"), 12345);
-        NioSocketChannel channel2 = mock(NioSocketChannel.class);
-        when(channel2.getRemoteAddress()).thenReturn(remoteAddr);
-        assertThat(nioIPFilter.test(channel2), is(false));
+    public void testThatFilterCanFail() throws Exception {
+        InetSocketAddress localhostAddr = new InetSocketAddress(InetAddresses.forString("10.0.0.8"), 12345);
+        NioChannelHandler delegate = mock(NioChannelHandler.class);
+        NioIPFilter nioIPFilter = new NioIPFilter(delegate, localhostAddr, ipFilter, profile);
+        nioIPFilter.channelRegistered();
+        verify(delegate, times(0)).channelRegistered();
+        assertTrue(nioIPFilter.closeNow());
     }
 }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContextTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContextTests.java
index ad91f05868226..7efff1c0e26b9 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContextTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContextTests.java
@@ -311,7 +311,7 @@ public void testCloseTimeout() {
         context.closeChannel();
 
         ArgumentCaptor<WriteOperation> captor = ArgumentCaptor.forClass(WriteOperation.class);
-        verify(selector).writeToChannel(captor.capture());
+        verify(selector).queueWrite(captor.capture());
 
         ArgumentCaptor<Runnable> taskCaptor = ArgumentCaptor.forClass(Runnable.class);
         Runnable cancellable = mock(Runnable.class);
@@ -333,7 +333,7 @@ public void testCloseTimeoutIsCancelledOnClose() throws IOException {
             context = new SSLChannelContext(channel, selector, exceptionHandler, sslDriver, readWriteHandler, channelBuffer);
             context.closeChannel();
             ArgumentCaptor<WriteOperation> captor = ArgumentCaptor.forClass(WriteOperation.class);
-            verify(selector).writeToChannel(captor.capture());
+            verify(selector).queueWrite(captor.capture());
             ArgumentCaptor<Runnable> taskCaptor = ArgumentCaptor.forClass(Runnable.class);
             Runnable cancellable = mock(Runnable.class);
             when(nioTimer.scheduleAtRelativeTime(taskCaptor.capture(), anyLong())).thenReturn(cancellable);
@@ -360,7 +360,7 @@ public void testInitiateCloseFromSameThreadSchedulesCloseNotify() throws SSLExce
         context.closeChannel();
 
         ArgumentCaptor<WriteOperation> captor = ArgumentCaptor.forClass(WriteOperation.class);
-        verify(selector).writeToChannel(captor.capture());
+        verify(selector).queueWrite(captor.capture());
 
         context.queueWriteOperation(captor.getValue());
         verify(sslDriver).initiateClose();
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/EllipticCurveSSLTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/EllipticCurveSSLTests.java
index 24e6a5bca009b..04169c675953a 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/EllipticCurveSSLTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/EllipticCurveSSLTests.java
@@ -63,7 +63,7 @@ public void testConnection() throws Exception {
         final Path keyPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/prime256v1-key.pem");
         final Path certPath = getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/prime256v1-cert.pem");
         PrivateKey privateKey = PemUtils.readPrivateKey(keyPath, () -> null);
-        Certificate[] certs = CertParsingUtils.readCertificates(Collections.singletonList(certPath.toString()), null);
+        Certificate[] certs = CertParsingUtils.readCertificates(Collections.singletonList(certPath.toString()), newEnvironment());
         X509ExtendedKeyManager x509ExtendedKeyManager = CertParsingUtils.keyManager(certs, privateKey, new char[0]);
         SSLContext sslContext = SSLContext.getInstance("TLS");
         sslContext.init(new X509ExtendedKeyManager[] { x509ExtendedKeyManager },
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java
index 8488fe2a5e638..01f8c70fc7b36 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/ssl/SslIntegrationTests.java
@@ -68,7 +68,7 @@ public void testThatConnectionToHTTPWorks() throws Exception {
             "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.crt",
             "xpack.security.http.",
             Arrays.asList("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt"));
-        SSLService service = new SSLService(builder.build(), null);
+        SSLService service = new SSLService(builder.build(), newEnvironment());
 
         CredentialsProvider provider = new BasicCredentialsProvider();
         provider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(nodeClientUsername(),
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java
index 88d27d4171a19..7d2bcf1232519 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLClientAuthTests.java
@@ -163,7 +163,8 @@ private byte[] toByteArray(InputStream is) throws IOException {
      */
     private static List<String> getProtocols() {
         JavaVersion full =
-            AccessController.doPrivileged((PrivilegedAction<JavaVersion>) () -> JavaVersion.parse(System.getProperty("java.version")));
+            AccessController.doPrivileged(
+                    (PrivilegedAction<JavaVersion>) () -> JavaVersion.parse(System.getProperty("java.specification.version")));
         if (full.compareTo(JavaVersion.parse("11.0.3")) < 0) {
             return List.of("TLSv1.2");
         }
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java
index e2df24643aec3..c4b75f1e73d38 100644
--- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java
+++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/ssl/SSLTrustRestrictionsTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.SecurityIntegTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
@@ -50,7 +49,6 @@
  * @see RestrictedTrustManager
  */
 @ESIntegTestCase.ClusterScope(numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false)
-@TestLogging("org.elasticsearch.xpack.ssl.RestrictedTrustManager:DEBUG")
 public class SSLTrustRestrictionsTests extends SecurityIntegTestCase {
 
     private static final TimeValue MAX_WAIT_RELOAD = TimeValue.timeValueSeconds(1);
@@ -68,7 +66,6 @@ public class SSLTrustRestrictionsTests extends SecurityIntegTestCase {
     protected int maxNumberOfNodes() {
         // We are trying to test the SSL configuration for which clients/nodes may join a cluster
         // We prefer the cluster to only have 1 node, so that the SSL checking doesn't happen until the test methods run
-        // (That's not _quite_ true, because the base setup code checks the cluster using transport client, but it's the best we can do)
         return 1;
     }
 
diff --git a/x-pack/plugin/sql/build.gradle b/x-pack/plugin/sql/build.gradle
index 1d13df3b2c32e..dac38447c78fe 100644
--- a/x-pack/plugin/sql/build.gradle
+++ b/x-pack/plugin/sql/build.gradle
@@ -38,16 +38,15 @@ task internalClusterTest(type: Test) {
 check.dependsOn internalClusterTest
 
 dependencies {
-    // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
     compileOnly project(path: xpackModule('core'), configuration: 'default')
     compileOnly(project(':modules:lang-painless')) {
         // exclude ASM to not affect featureAware task on Java 10+
         exclude group: "org.ow2.asm"
     }
     compile project('sql-action')
-    compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
+    compile project(':modules:aggs-matrix-stats')
     compile "org.antlr:antlr4-runtime:4.5.3"
-    testCompile "org.elasticsearch.test:framework:${version}"
+    testCompile project(':test:framework')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     if (isEclipse) {
       testCompile project(path: xpackModule('core-tests'), configuration: 'testArtifacts')
diff --git a/x-pack/plugin/sql/jdbc/build.gradle b/x-pack/plugin/sql/jdbc/build.gradle
index 37e0baf00aa71..292b3ea890bf8 100644
--- a/x-pack/plugin/sql/jdbc/build.gradle
+++ b/x-pack/plugin/sql/jdbc/build.gradle
@@ -18,15 +18,15 @@ dependencies {
     compile (xpackProject('plugin:sql:sql-proto')) {
         transitive = false
     }
-    compile (project(':libs:x-content')) {
+    compile (project(':libs:elasticsearch-x-content')) {
         transitive = false
     }
     compile (project(':libs:elasticsearch-geo')) {
         transitive = false
     }
-    compile project(':libs:core')
+    compile project(':libs:elasticsearch-core')
     runtime "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-    testCompile "org.elasticsearch.test:framework:${version}"
+    testCompile project(":test:framework")
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     if (isEclipse) {
       testCompile project(path: xpackModule('core-tests'), configuration: 'testArtifacts')
diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java
index 7e21f2206b1e9..0a1c0826695bd 100644
--- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java
+++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.sql.jdbc;
 
+import org.elasticsearch.geo.utils.GeographyValidator;
 import org.elasticsearch.geo.utils.WellKnownText;
 import org.elasticsearch.xpack.sql.proto.StringUtils;
 
@@ -54,6 +55,8 @@
  */
 final class TypeConverter {
 
+    private static WellKnownText WKT = new WellKnownText(true, new GeographyValidator(true));
+
     private TypeConverter() {}
 
     /**
@@ -246,7 +249,7 @@ static Object convert(Object v, EsType columnType, String typeString) throws SQL
             case GEO_POINT:
             case GEO_SHAPE:
                 try {
-                    return WellKnownText.fromWKT(v.toString());
+                    return WKT.fromWKT(v.toString());
                 } catch (IOException | ParseException ex) {
                     throw new SQLException("Cannot parse geo_shape", ex);
                 }
diff --git a/x-pack/plugin/sql/qa/build.gradle b/x-pack/plugin/sql/qa/build.gradle
index f2a6acd61a058..89f04562fffb2 100644
--- a/x-pack/plugin/sql/qa/build.gradle
+++ b/x-pack/plugin/sql/qa/build.gradle
@@ -1,12 +1,10 @@
-import org.elasticsearch.gradle.test.RunTask
-
 description = 'Integration tests for SQL'
 apply plugin: 'elasticsearch.build'
 archivesBaseName = 'qa-sql'
 group = "org.elasticsearch.x-pack.qa.sql"
 
 dependencies {
-  compile "org.elasticsearch.test:framework:${version}"
+  compile project(":test:framework")
 
   // JDBC testing dependencies
   compile project(path: xpackModule('sql:jdbc'), configuration: 'nodeps')
@@ -59,7 +57,7 @@ subprojects {
     testCompile(xpackProject('plugin:sql:qa')) {
       transitive = false
     }
-    testCompile "org.elasticsearch.test:framework:${version}"
+    testCompile project(":test:framework")
 
     // JDBC testing dependencies
     testRuntime "net.sourceforge.csvjdbc:csvjdbc:${csvjdbcVersion}"
@@ -96,20 +94,20 @@ subprojects {
 
   if (project.name != 'security') {
     // The security project just configures its subprojects
+    apply plugin: 'elasticsearch.testclusters'
     apply plugin: 'elasticsearch.rest-test'
 
-    integTestCluster {
+    testClusters.integTest {
+      distribution = 'DEFAULT'
       setting 'xpack.monitoring.enabled', 'false'
       setting 'xpack.ml.enabled', 'false'
       setting 'xpack.watcher.enabled', 'false'
-      setting 'script.max_compilations_rate', '1000/1m'
     }
 
-    task runqa(type: RunTask) {
-      setting 'xpack.monitoring.enabled', 'false'
-      setting 'xpack.ml.enabled', 'false'
-      setting 'xpack.watcher.enabled', 'false'
-      setting 'script.max_compilations_rate', '1000/1m'
+    task runqa {
+      doFirst {
+        println "Run with `-Dtestclusters.inspect.failure=true integTest` to leave the cluster running after failure"
+      }
     }
   }
 }
diff --git a/x-pack/plugin/sql/qa/multi-node/build.gradle b/x-pack/plugin/sql/qa/multi-node/build.gradle
index ce7616ad1d1ae..7949f36083a07 100644
--- a/x-pack/plugin/sql/qa/multi-node/build.gradle
+++ b/x-pack/plugin/sql/qa/multi-node/build.gradle
@@ -6,8 +6,8 @@ description = 'Run a subset of SQL tests against multiple nodes'
  * feel should need to be tested against more than one node.
  */
 
-integTestCluster {
-  numNodes = 2
+testClusters.integTest{
+  numberOfNodes = 2
   setting 'xpack.security.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
 }
diff --git a/x-pack/plugin/sql/qa/security/build.gradle b/x-pack/plugin/sql/qa/security/build.gradle
index a0e6e82ed4d67..2cf410ed3d908 100644
--- a/x-pack/plugin/sql/qa/security/build.gradle
+++ b/x-pack/plugin/sql/qa/security/build.gradle
@@ -1,5 +1,5 @@
 dependencies {
-  testCompile "org.elasticsearch.plugin:x-pack-core:${version}"
+  testCompile project(':x-pack:plugin:core')
 }
 
 Project mainProject = project
@@ -26,41 +26,26 @@ subprojects {
   }
 
   dependencies {
-    testCompile "org.elasticsearch.plugin:x-pack-core:${version}"
+    testCompile project(":x-pack:plugin:core")
   }
 
-  integTestCluster {
+  testClusters.integTest {
+    distribution = "DEFAULT"
     // Setup auditing so we can use it in some tests
     setting 'xpack.security.audit.enabled', 'true'
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.license.self_generated.type', 'trial'
     // Setup roles used by tests
-    extraConfigFile 'roles.yml', '../roles.yml'
+    extraConfigFile 'roles.yml', mainProject.file('roles.yml')
     /* Setup the one admin user that we run the tests as.
      * Tests use "run as" to get different users. */
-    setupCommand 'setupUser#test_admin',
-                'bin/elasticsearch-users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser'
-    // Subprojects override the wait condition to work properly with security
+    user username: "test_admin", password: "x-pack-test-password"
   }
 
-  integTestRunner {
-    def today = new Date().format('yyyy-MM-dd')
+  integTest.runner {
     nonInputProperties.systemProperty 'tests.audit.logfile',
-        "${ -> integTest.nodes[0].homeDir}/logs/${ -> integTest.nodes[0].clusterName }_audit.json"
+        "${ -> testClusters.integTest.singleNode().getAuditLog()}"
     nonInputProperties.systemProperty 'tests.audit.yesterday.logfile',
-        "${ -> integTest.nodes[0].homeDir}/logs/${ -> integTest.nodes[0].clusterName }_audit-${today}.json"
-  }
-
-  runqa {
-    // Setup auditing so we can use it in some tests
-    setting 'xpack.security.audit.enabled', 'true'
-    setting 'xpack.security.enabled', 'true'
-    setting 'xpack.license.self_generated.type', 'trial'
-    // Setup roles used by tests
-    extraConfigFile 'roles.yml', '../roles.yml'
-    /* Setup the one admin user that we run the tests as.
-     * Tests use "run as" to get different users. */
-    setupCommand 'setupUser#test_admin',
-                'bin/elasticsearch-users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser'
+        "${ -> testClusters.integTest.singleNode().getAuditLog().getParentFile()}/integTest_audit-${new Date().format('yyyy-MM-dd')}.json"
   }
 }
diff --git a/x-pack/plugin/sql/qa/security/with-ssl/build.gradle b/x-pack/plugin/sql/qa/security/with-ssl/build.gradle
index 19459bade97a8..d73a181bef10f 100644
--- a/x-pack/plugin/sql/qa/security/with-ssl/build.gradle
+++ b/x-pack/plugin/sql/qa/security/with-ssl/build.gradle
@@ -11,7 +11,7 @@ import java.security.KeyStore
 import java.security.SecureRandom
 
 // Tell the tests we're running with ssl enabled
-integTestRunner {
+integTest.runner {
   systemProperty 'tests.ssl.enabled', 'true'
 }
 
@@ -143,78 +143,32 @@ forbiddenPatterns {
 sourceSets.test.resources.srcDir(keystoreDir)
 processTestResources.dependsOn(importNodeCertificateInClientKeyStore, importClientCertificateInNodeKeyStore)
 
-integTestCluster.dependsOn(importClientCertificateInNodeKeyStore)
+integTest.runner {
+  dependsOn(importClientCertificateInNodeKeyStore)
+  onlyIf {
+    // Do not attempt to form a cluster in a FIPS JVM, as doing so with a JKS keystore will fail.
+    // TODO Revisit this when SQL CLI client can handle key/certificate instead of only Keystores.
+    // https://github.com/elastic/elasticsearch/issues/32306
+    project.inFipsJvm == false
+  }
+}
 
-integTestCluster {
+testClusters.integTest {
   // The setup that we actually want
+  setting 'xpack.license.self_generated.type', 'trial'
   setting 'xpack.security.http.ssl.enabled', 'true'
   setting 'xpack.security.transport.ssl.enabled', 'true'
 
   // ceremony to set up ssl
   setting 'xpack.security.transport.ssl.keystore.path', 'test-node.jks'
-  keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass'
   setting 'xpack.security.http.ssl.keystore.path', 'test-node.jks'
-  keystoreSetting 'xpack.security.http.ssl.keystore.secure_password', 'keypass'
+  keystore 'xpack.security.transport.ssl.keystore.secure_password', 'keypass'
+  keystore 'xpack.security.http.ssl.keystore.secure_password', 'keypass'
 
-  setting 'xpack.license.self_generated.type', 'trial'
 
   // copy keystores into config/
   extraConfigFile nodeKeystore.name, nodeKeystore
   extraConfigFile clientKeyStore.name, clientKeyStore
-
-  // Override the wait condition to work properly with security and SSL
-  waitCondition = { NodeInfo node, AntBuilder ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    KeyStore keyStore = KeyStore.getInstance("JKS");
-    keyStore.load(clientKeyStore.newInputStream(), 'keypass'.toCharArray());
-    KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
-    kmf.init(keyStore, 'keypass'.toCharArray());
-    TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
-    tmf.init(keyStore);
-    SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
-    sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), new SecureRandom());
-    for (int i = 0; i < 10; i++) {
-      // we use custom wait logic here for HTTPS
-      HttpsURLConnection httpURLConnection = null;
-      try {
-        httpURLConnection = (HttpsURLConnection) new URL("https://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}&wait_for_status=yellow").openConnection();
-        httpURLConnection.setSSLSocketFactory(sslContext.getSocketFactory());
-        httpURLConnection.setRequestProperty("Authorization", "Basic " +
-                Base64.getEncoder().encodeToString("test_admin:x-pack-test-password".getBytes(StandardCharsets.UTF_8)));
-        httpURLConnection.setRequestMethod("GET");
-        httpURLConnection.connect();
-        if (httpURLConnection.getResponseCode() == 200) {
-          tmpFile.withWriter StandardCharsets.UTF_8.name(), {
-            it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name()))
-          }
-        }
-      } catch (IOException e) {
-        if (i == 9) {
-          logger.error("final attempt of calling cluster health failed", e)
-        } else {
-          logger.debug("failed to call cluster health", e)
-        }
-      } finally {
-        if (httpURLConnection != null) {
-          httpURLConnection.disconnect();
-        }
-      }
-
-      // did not start, so wait a bit before trying again
-      Thread.sleep(500L);
-    }
-
-    return tmpFile.exists()
-  }
-}
-
-// Do not attempt to form a cluster in a FIPS JVM, as doing so with a JKS keystore will fail.
-// TODO Revisit this when SQL CLI client can handle key/certificate instead of only Keystores.
-// https://github.com/elastic/elasticsearch/issues/32306
-tasks.matching { it.name in ["integTestCluster#init", "integTestCluster#start", "integTestCluster#wait", "integTestRunner"] }.all { 
-  onlyIf { 
-    project.inFipsJvm == false 
-  } 
 }
 
 
diff --git a/x-pack/plugin/sql/qa/security/without-ssl/build.gradle b/x-pack/plugin/sql/qa/security/without-ssl/build.gradle
index ac748527da371..691fc1a631f16 100644
--- a/x-pack/plugin/sql/qa/security/without-ssl/build.gradle
+++ b/x-pack/plugin/sql/qa/security/without-ssl/build.gradle
@@ -1,17 +1,7 @@
-integTestRunner {
+integTest.runner {
   systemProperty 'tests.ssl.enabled', 'false'
 }
 
-integTestCluster {
+testClusters.integTest {
   setting 'xpack.license.self_generated.type', 'trial'
-  waitCondition = { node, ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-            dest: tmpFile.toString(),
-            username: 'test_admin',
-            password: 'x-pack-test-password',
-            ignoreerrors: true,
-            retries: 10)
-    return tmpFile.exists()
-  }
 }
diff --git a/x-pack/plugin/sql/qa/single-node/build.gradle b/x-pack/plugin/sql/qa/single-node/build.gradle
index 3a8b0ffde0ac5..137ffed68ad52 100644
--- a/x-pack/plugin/sql/qa/single-node/build.gradle
+++ b/x-pack/plugin/sql/qa/single-node/build.gradle
@@ -1,9 +1,4 @@
-integTestCluster {
-  setting 'xpack.security.enabled', 'false'
-  setting 'xpack.license.self_generated.type', 'trial'
-}
-
-runqa {
+testClusters.integTest {
   setting 'xpack.security.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
 }
diff --git a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java
index 428bc1c21ef8f..685a36f483ba6 100644
--- a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java
+++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java
@@ -32,7 +32,7 @@ public JdbcCsvSpecIT(String fileName, String groupName, String testName, Integer
     @Override
     protected int fetchSize() {
         // using a smaller fetchSize for nested documents' tests to uncover bugs
-        // similar with https://github.com/elastic/elasticsearch/issues/35176 quicker
+        // similar to https://github.com/elastic/elasticsearch/issues/35176 quicker
         return fileName.startsWith("nested") && randomBoolean() ? randomIntBetween(1,5) : super.fetchSize();
     }
 }
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/ErrorsTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/ErrorsTestCase.java
index 5e3b034d75708..c5ae7f63ad06d 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/ErrorsTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/ErrorsTestCase.java
@@ -21,4 +21,5 @@ public interface ErrorsTestCase {
     void testSelectGroupByScore() throws Exception;
     void testSelectScoreSubField() throws Exception;
     void testSelectScoreInScalar() throws Exception;
+    void testHardLimitForSortOnAggregate() throws Exception;
 }
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/CliIntegrationTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/CliIntegrationTestCase.java
index e7a73cd12d524..cf221bbc14012 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/CliIntegrationTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/CliIntegrationTestCase.java
@@ -56,8 +56,8 @@ protected SecurityConfig securityConfig() {
         return null;
     }
 
-    protected void index(String index, CheckedConsumer<XContentBuilder, IOException> body) throws IOException {
-        Request request = new Request("PUT", "/" + index + "/_doc/1");
+    protected void index(String index, int docId, CheckedConsumer<XContentBuilder, IOException> body) throws IOException {
+        Request request = new Request("PUT", "/" + index + "/_doc/" + docId);
         request.addParameter("refresh", "true");
         XContentBuilder builder = JsonXContent.contentBuilder().startObject();
         body.accept(builder);
@@ -66,6 +66,10 @@ protected void index(String index, CheckedConsumer<XContentBuilder, IOException>
         client().performRequest(request);
     }
 
+    protected void index(String index, CheckedConsumer<XContentBuilder, IOException> body) throws IOException {
+        index(index, 1, body);
+    }
+
     public String command(String command) throws IOException {
         return cli.command(command);
     }
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ErrorsTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ErrorsTestCase.java
index ca251a31844c0..bd295d090290e 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ErrorsTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ErrorsTestCase.java
@@ -97,8 +97,15 @@ public void testSelectScoreInScalar() throws Exception {
         assertEquals("line 1:12: [SCORE()] cannot be an argument to a function" + END, readLine());
     }
 
+    @Override
+    public void testHardLimitForSortOnAggregate() throws Exception {
+        index("test", body -> body.field("a", 1).field("b", 2));
+        String commandResult = command("SELECT max(a) max FROM test GROUP BY b ORDER BY max LIMIT 12000");
+        assertEquals(START + "Bad request [[3;33;22mThe maximum LIMIT for aggregate sorting is [10000], received [12000]" + END,
+            commandResult);
+    }
+
     public static void assertFoundOneProblem(String commandResult) {
         assertEquals(START + "Bad request [[3;33;22mFound 1 problem(s)", commandResult);
     }
-
 }
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/FetchSizeTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/FetchSizeTestCase.java
index 84f74bcbac137..02de2dff4f7d6 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/FetchSizeTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/FetchSizeTestCase.java
@@ -49,4 +49,32 @@ public void testInvalidFetchSize() throws IOException {
         assertEquals(ErrorsTestCase.START + "Invalid fetch size [[3;33;22m" + Long.MAX_VALUE + ErrorsTestCase.END,
             command("fetch size = " + Long.MAX_VALUE));
     }
+
+    // Test for issue: https://github.com/elastic/elasticsearch/issues/42851
+    // Even though fetch size and limit are smaller than the noRows, all buckets
+    // should be processed to achieve the global ordering of the aggregate function.
+    public void testOrderingOnAggregate() throws IOException {
+        Request request = new Request("PUT", "/test/_bulk");
+        request.addParameter("refresh", "true");
+        StringBuilder bulk = new StringBuilder();
+        for (int i = 1; i <= 100; i++) {
+            bulk.append("{\"index\":{}}\n");
+            bulk.append("{\"a\":").append(i).append(", \"b\" : ").append(i).append("}\n");
+        }
+        request.setJsonEntity(bulk.toString());
+        client().performRequest(request);
+
+        assertEquals("[?1l>[?1000l[?2004lfetch size set to [90m4[0m", command("fetch size = 4"));
+        assertEquals("[?1l>[?1000l[?2004lfetch separator set to \"[90m -- fetch sep -- [0m\"",
+            command("fetch separator = \" -- fetch sep -- \""));
+        assertThat(command("SELECT max(b) FROM test GROUP BY a ORDER BY max(b) DESC LIMIT 20"), containsString("max(b)"));
+        assertThat(readLine(), containsString("----------"));
+        for (int i = 100; i > 80; i--) {
+            if (i < 100 && i % 4 == 0) {
+                assertThat(readLine(), containsString(" -- fetch sep -- "));
+            }
+            assertThat(readLine(), containsString(Integer.toString(i)));
+        }
+        assertEquals("", readLine());
+    }
 }
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java
deleted file mode 100644
index d5a633e5ea388..0000000000000
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.sql.qa.jdbc;
-
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.test.junit.annotations.TestLogging;
-import org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.CsvTestCase;
-
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.List;
-
-import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.csvConnection;
-import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.executeCsvQuery;
-import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.specParser;
-
-@TestLogging("org.elasticsearch.xpack.sql:TRACE")
-public abstract class DebugCsvSpec extends SpecBaseIntegrationTestCase {
-    private final CsvTestCase testCase;
-
-    @ParametersFactory(shuffle = false, argumentFormatting = SqlSpecTestCase.PARAM_FORMATTING)
-    public static List<Object[]> readScriptSpec() throws Exception {
-        Parser parser = specParser();
-        return readScriptSpec("/debug.csv-spec", parser);
-    }
-
-    public DebugCsvSpec(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {
-        super(fileName, groupName, testName, lineNumber);
-        this.testCase = testCase;
-    }
-
-    @Override
-    protected void assertResults(ResultSet expected, ResultSet elastic) throws SQLException {
-        Logger log = logEsResultSet() ? logger : null;
-
-        //
-        // uncomment this to printout the result set and create new CSV tests
-        //
-        JdbcTestUtils.logResultSetMetadata(elastic, log);
-        JdbcTestUtils.logResultSetData(elastic, log);
-        //JdbcAssert.assertResultSets(expected, elastic, log);
-    }
-
-    @Override
-    protected boolean logEsResultSet() {
-        return true;
-    }
-
-    @Override
-    protected final void doTest() throws Throwable {
-        try (Connection csv = csvConnection(testCase); Connection es = esJdbc()) {
-            // pass the testName as table for debugging purposes (in case the underlying reader is missing)
-            ResultSet expected = executeCsvQuery(csv, testName);
-            ResultSet elasticResults = executeJdbcQuery(es, testCase.query);
-            assertResults(expected, elasticResults);
-        }
-    }
-}
\ No newline at end of file
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java
index b51d66ace2e26..580fd9a28648b 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugSqlSpec.java
@@ -7,11 +7,8 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.elasticsearch.test.junit.annotations.TestLogging;
-
 import java.util.List;
 
-@TestLogging(JdbcTestUtils.SQL_TRACE)
 public abstract class DebugSqlSpec extends SqlSpecTestCase {
     @ParametersFactory(shuffle = false, argumentFormatting = PARAM_FORMATTING)
     public static List<Object[]> readScriptSpec() throws Exception {
@@ -27,4 +24,4 @@ public DebugSqlSpec(String fileName, String groupName, String testName, Integer
     protected boolean logEsResultSet() {
         return true;
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ErrorsTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ErrorsTestCase.java
index be3ba3d096ae2..d4f1cb5b30561 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ErrorsTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ErrorsTestCase.java
@@ -116,4 +116,14 @@ public void testSelectScoreInScalar() throws Exception {
             assertThat(e.getMessage(), startsWith("Found 1 problem(s)\nline 1:12: [SCORE()] cannot be an argument to a function"));
         }
     }
+
+    @Override
+    public void testHardLimitForSortOnAggregate() throws Exception {
+        index("test", body -> body.field("a", 1).field("b", 2));
+        try (Connection c = esJdbc()) {
+            SQLException e = expectThrows(SQLException.class, () ->
+                c.prepareStatement("SELECT max(a) max FROM test GROUP BY b ORDER BY max LIMIT 12000").executeQuery());
+            assertEquals("The maximum LIMIT for aggregate sorting is [10000], received [12000]", e.getMessage());
+        }
+    }
 }
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java
index 76894fc5a53d5..256d7cb612cf0 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java
@@ -10,6 +10,7 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.geo.geometry.Geometry;
 import org.elasticsearch.geo.geometry.Point;
+import org.elasticsearch.geo.utils.GeographyValidator;
 import org.elasticsearch.geo.utils.WellKnownText;
 import org.elasticsearch.xpack.sql.jdbc.EsType;
 import org.elasticsearch.xpack.sql.proto.StringUtils;
@@ -51,6 +52,8 @@ public class JdbcAssert {
 
     private static final IntObjectHashMap<EsType> SQL_TO_TYPE = new IntObjectHashMap<>();
 
+    private static final WellKnownText WKT = new WellKnownText(true, new GeographyValidator(true));
+
     static {
         for (EsType type : EsType.values()) {
             SQL_TO_TYPE.putIfAbsent(type.getVendorTypeNumber().intValue(), type);
@@ -270,7 +273,7 @@ else if (type == Types.DOUBLE) {
                         if (actualObject instanceof Geometry) {
                             // We need to convert the expected object to libs/geo Geometry for comparision
                             try {
-                                expectedObject = WellKnownText.fromWKT(expectedObject.toString());
+                                expectedObject = WKT.fromWKT(expectedObject.toString());
                             } catch (IOException | ParseException ex) {
                                 fail(ex.getMessage());
                             }
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlSpecTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlSpecTestCase.java
index ef01dc1fca11e..cfbec77a3e636 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlSpecTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlSpecTestCase.java
@@ -65,6 +65,13 @@ public SqlSpecTestCase(String fileName, String groupName, String testName, Integ
         this.query = query;
     }
 
+    @Override
+    protected int fetchSize() {
+        // using a smaller fetchSize for nested documents' tests to uncover bugs
+        // similar to https://github.com/elastic/elasticsearch/issues/42581
+        return randomIntBetween(1, 20);
+    }
+
     @Override
     protected final void doTest() throws Throwable {
         // we skip the tests in case of these locales because ES-SQL is Locale-insensitive for now
diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
index c88f31bb2fd71..7c7288d6a3539 100644
--- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
+++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java
@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.sql.qa.rest;
 
 import com.fasterxml.jackson.core.io.JsonStringEncoder;
-
 import org.apache.http.HttpEntity;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
@@ -314,7 +313,14 @@ public void testSelectScoreInScalar() throws Exception {
         expectBadRequest(() -> runSql(randomMode(), "SELECT SIN(SCORE()) FROM test"),
             containsString("line 1:12: [SCORE()] cannot be an argument to a function"));
     }
-    
+
+    @Override
+    public void testHardLimitForSortOnAggregate() throws Exception {
+        index("{\"a\": 1, \"b\": 2}");
+        expectBadRequest(() -> runSql(randomMode(), "SELECT max(a) max FROM test GROUP BY b ORDER BY max LIMIT 12000"),
+            containsString("The maximum LIMIT for aggregate sorting is [10000], received [12000]"));
+    }
+
     public void testUseColumnarForUnsupportedFormats() throws Exception {
         String format = randomFrom("txt", "csv", "tsv");
         index("{\"foo\":1}");
@@ -615,46 +621,7 @@ public void testBasicQueryText() throws IOException {
     }
 
     public void testNextPageText() throws IOException {
-        int size = 20;
-        String[] docs = new String[size];
-        for (int i = 0; i < size; i++) {
-            docs[i] = "{\"text\":\"text" + i + "\", \"number\":" + i + "}\n";
-        }
-        index(docs);
-
-        String request = "{\"query\":\"SELECT text, number, number + 5 AS sum FROM test ORDER BY number\", \"fetch_size\":2}";
-
-        String cursor = null;
-        for (int i = 0; i < 20; i += 2) {
-            Tuple<String, String> response;
-            if (i == 0) {
-                response = runSqlAsText(StringUtils.EMPTY, new StringEntity(request, ContentType.APPLICATION_JSON), "text/plain");
-            } else {
-                response = runSqlAsText(StringUtils.EMPTY, new StringEntity("{\"cursor\":\"" + cursor + "\"}",
-                        ContentType.APPLICATION_JSON), "text/plain");
-            }
-
-            StringBuilder expected = new StringBuilder();
-            if (i == 0) {
-                expected.append("     text      |    number     |      sum      \n");
-                expected.append("---------------+---------------+---------------\n");
-            }
-            expected.append(String.format(Locale.ROOT, "%-15s|%-15d|%-15d\n", "text" + i, i, i + 5));
-            expected.append(String.format(Locale.ROOT, "%-15s|%-15d|%-15d\n", "text" + (i + 1), i + 1, i + 6));
-            cursor = response.v2();
-            assertEquals(expected.toString(), response.v1());
-            assertNotNull(cursor);
-        }
-        Map<String, Object> expected = new HashMap<>();
-        expected.put("rows", emptyList());
-        assertResponse(expected, runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"}", ContentType.APPLICATION_JSON),
-                StringUtils.EMPTY));
-
-        Map<String, Object> response = runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"}", ContentType.APPLICATION_JSON),
-                "/close");
-        assertEquals(true, response.get("succeeded"));
-
-        assertEquals(0, getNumberOfSearchContexts("test"));
+        executeQueryWithNextPage("text/plain", "     text      |    number     |      sum      \n", "%-15s|%-15d|%-15d\n");
     }
 
     // CSV/TSV tests
@@ -696,6 +663,10 @@ public void testQueryWithoutHeaderInCSV() throws IOException {
         Tuple<String, String> response = runSqlAsText(query, "text/csv; header=absent");
         assertEquals(expected, response.v1());
     }
+    
+    public void testNextPageCSV() throws IOException {
+        executeQueryWithNextPage("text/csv; header=present", "text,number,sum\r\n", "%s,%d,%d\r\n");
+    }
 
     public void testQueryInTSV() throws IOException {
         index("{\"name\":" + toJson("first") + ", \"number\" : 1 }",
@@ -714,6 +685,55 @@ public void testQueryInTSV() throws IOException {
         response = runSqlAsTextFormat(query, "tsv");
         assertEquals(expected, response.v1());
     }
+    
+    public void testNextPageTSV() throws IOException {
+        executeQueryWithNextPage("text/tab-separated-values", "text\tnumber\tsum\n", "%s\t%d\t%d\n");
+    }
+    
+    private void executeQueryWithNextPage(String format, String expectedHeader, String expectedLineFormat) throws IOException {
+        int size = 20;
+        String[] docs = new String[size];
+        for (int i = 0; i < size; i++) {
+            docs[i] = "{\"text\":\"text" + i + "\", \"number\":" + i + "}\n";
+        }
+        index(docs);
+
+        String request = "{\"query\":\"SELECT text, number, number + 5 AS sum FROM test ORDER BY number\", \"fetch_size\":2}";
+
+        String cursor = null;
+        for (int i = 0; i < 20; i += 2) {
+            Tuple<String, String> response;
+            if (i == 0) {
+                response = runSqlAsText(StringUtils.EMPTY, new StringEntity(request, ContentType.APPLICATION_JSON), format);
+            } else {
+                response = runSqlAsText(StringUtils.EMPTY, new StringEntity("{\"cursor\":\"" + cursor + "\"}",
+                        ContentType.APPLICATION_JSON), format);
+            }
+
+            StringBuilder expected = new StringBuilder();
+            if (i == 0) {
+                expected.append(expectedHeader);
+                if (format == "text/plain") {
+                    expected.append("---------------+---------------+---------------\n");
+                }
+            }
+            expected.append(String.format(Locale.ROOT, expectedLineFormat, "text" + i, i, i + 5));
+            expected.append(String.format(Locale.ROOT, expectedLineFormat, "text" + (i + 1), i + 1, i + 6));
+            cursor = response.v2();
+            assertEquals(expected.toString(), response.v1());
+            assertNotNull(cursor);
+        }
+        Map<String, Object> expected = new HashMap<>();
+        expected.put("rows", emptyList());
+        assertResponse(expected, runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"}", ContentType.APPLICATION_JSON),
+                StringUtils.EMPTY));
+
+        Map<String, Object> response = runSql(new StringEntity("{\"cursor\":\"" + cursor + "\"}", ContentType.APPLICATION_JSON),
+                "/close");
+        assertEquals(true, response.get("succeeded"));
+
+        assertEquals(0, getNumberOfSearchContexts("test"));
+    }
 
     private Tuple<String, String> runSqlAsText(String sql, String accept) throws IOException {
         return runSqlAsText(StringUtils.EMPTY, new StringEntity("{\"query\":\"" + sql + "\"}", ContentType.APPLICATION_JSON), accept);
diff --git a/x-pack/plugin/sql/qa/src/main/resources/agg-ordering.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/agg-ordering.sql-spec
index 79d58c48e4469..9a193d76b3166 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/agg-ordering.sql-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/agg-ordering.sql-spec
@@ -99,13 +99,13 @@ aggNotSpecifiedWithHavingOnLargeGroupBy
 SELECT MAX(salary) AS max FROM test_emp GROUP BY emp_no HAVING AVG(salary) > 1000 ORDER BY MIN(salary);
 
 aggWithTieBreakerDescAsc
-SELECT emp_no, MIN(languages) AS min FROM test_emp GROUP BY emp_no ORDER BY MIN(languages) DESC NULLS FIRST, emp_no ASC;
+SELECT emp_no, MIN(languages) AS min FROM test_emp GROUP BY emp_no ORDER BY MIN(languages) DESC NULLS FIRST, emp_no ASC LIMIT 50;
 
 aggWithTieBreakerDescDesc
-SELECT emp_no, MIN(languages) AS min FROM test_emp GROUP BY emp_no ORDER BY MIN(languages) DESC NULLS FIRST, emp_no DESC;
+SELECT emp_no, MIN(languages) AS min FROM test_emp GROUP BY emp_no ORDER BY MIN(languages) DESC NULLS FIRST, emp_no DESC LIMIT 50;
 
 aggWithTieBreakerAscDesc
-SELECT emp_no, MIN(languages) AS min FROM test_emp GROUP BY emp_no ORDER BY MAX(languages) ASC NULLS FIRST, emp_no DESC;
+SELECT emp_no, MIN(languages) AS min FROM test_emp GROUP BY emp_no ORDER BY MAX(languages) ASC NULLS FIRST, emp_no DESC LIMIT 50;
 
 aggWithMixOfOrdinals
 SELECT gender AS g, MAX(salary) AS m FROM test_emp GROUP BY gender ORDER BY 2 DESC LIMIT 3;
diff --git a/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec
index 936c7eef88191..3cb2c3f45d4aa 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec
@@ -643,9 +643,9 @@ M              |57
 ;
 
 groupByAndAggExpression
-// tag::groupByAndAggExpression
 schema::g:s|salary:i
-SELECT gender AS g, ROUND( (MIN(salary) / 100) ) AS salary FROM emp GROUP BY gender;
+// tag::groupByAndAggExpression
+SELECT gender AS g, ROUND((MIN(salary) / 100)) AS salary FROM emp GROUP BY gender;
 
        g       |    salary     
 ---------------+---------------
@@ -1001,12 +1001,13 @@ Frank Herbert  |God Emperor of Dune|7.0029488
 
 optionalParamsForMatch
 // tag::optionalParamsForMatch
-SELECT author, name, SCORE() FROM library WHERE MATCH(name, 'to the star', 'operator=or;cutoff_frequency=0.2');
+SELECT author, name, SCORE() FROM library WHERE MATCH(name, 'to the star', 'operator=OR;fuzziness=AUTO:1,5;minimum_should_match=1')
+ORDER BY SCORE() DESC LIMIT 2;
 
      author      |                name                |    SCORE()
 -----------------+------------------------------------+---------------
-Peter F. Hamilton|Pandora's Star                      |3.0997515
 Douglas Adams    |The Hitchhiker's Guide to the Galaxy|3.1756816
+Peter F. Hamilton|Pandora's Star                      |3.0997515
 
 // end::optionalParamsForMatch
 ;
diff --git a/x-pack/plugin/sql/qa/src/main/resources/fulltext.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/fulltext.csv-spec
index cb410080e77bd..6379f6bf26f1a 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/fulltext.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/fulltext.csv-spec
@@ -92,14 +92,14 @@ SELECT emp_no, first_name, gender, last_name FROM test_emp WHERE MATCH(first_nam
 ;
 
 matchQueryWithOptions
-SELECT emp_no, first_name, gender, last_name FROM test_emp WHERE MATCH(first_name, 'Erez', 'lenient=true;cutoff_frequency=2;fuzzy_rewrite=scoring_boolean;minimum_should_match=1;operator=AND;max_expansions=30;prefix_length=1;analyzer=english;auto_generate_synonyms_phrase_query=true;fuzzy_transpositions=true');
+SELECT emp_no, first_name, gender, last_name FROM test_emp WHERE MATCH(first_name, 'Erez', 'lenient=true;fuzzy_rewrite=scoring_boolean;minimum_should_match=1;operator=AND;max_expansions=30;prefix_length=1;analyzer=english;auto_generate_synonyms_phrase_query=true;fuzzy_transpositions=true');
 
    emp_no:i    | first_name:s  |   gender:s    |  last_name:s
 10076          |Erez           |F              |Ritzmann
 ;
 
 matchQueryWithOptionsInMultipleCommaSeparatedStrings
-SELECT emp_no, first_name, gender, last_name FROM test_emp WHERE MATCH(first_name, 'Erez', 'lenient=true;cutoff_frequency=2','fuzzy_rewrite=scoring_boolean;minimum_should_match=1','operator=AND', 'max_expansions=30;prefix_length=1;analyzer=english;auto_generate_synonyms_phrase_query=true;fuzzy_transpositions=true');
+SELECT emp_no, first_name, gender, last_name FROM test_emp WHERE MATCH(first_name, 'Erez', 'lenient=true','fuzzy_rewrite=scoring_boolean;minimum_should_match=1','operator=AND', 'max_expansions=30;prefix_length=1;analyzer=english;auto_generate_synonyms_phrase_query=true;fuzzy_transpositions=true');
 
    emp_no:i    | first_name:s  |   gender:s    |  last_name:s
 10076          |Erez           |F              |Ritzmann
@@ -113,14 +113,14 @@ SELECT emp_no, first_name, gender, last_name FROM test_emp WHERE MATCH('first_na
 ;
 
 multiMatchQueryAllOptions
-SELECT emp_no, first_name, gender, last_name FROM test_emp WHERE MATCH('first_name,last_name', 'Morton', 'slop=1;lenient=true;cutoff_frequency=2;tie_breaker=0.1;fuzzy_rewrite=scoring_boolean;minimum_should_match=1;operator=AND;max_expansions=30;prefix_length=1;analyzer=english;type=best_fields;auto_generate_synonyms_phrase_query=true;fuzzy_transpositions=true');
+SELECT emp_no, first_name, gender, last_name FROM test_emp WHERE MATCH('first_name,last_name', 'Morton', 'slop=1;lenient=true;tie_breaker=0.1;fuzzy_rewrite=scoring_boolean;minimum_should_match=1;operator=AND;max_expansions=30;prefix_length=1;analyzer=english;type=best_fields;auto_generate_synonyms_phrase_query=true;fuzzy_transpositions=true');
 
    emp_no:i    | first_name:s  |   gender:s    |  last_name:s
 10095          |Hilari         |M              |Morton
 ;
 
 multiMatchQueryWithInMultipleCommaSeparatedStrings
-SELECT emp_no, first_name, gender, last_name FROM test_emp WHERE MATCH('first_name,last_name', 'Morton', 'slop=1;lenient=true', 'cutoff_frequency=2','tie_breaker=0.1;fuzzy_rewrite=scoring_boolean','minimum_should_match=1;operator=AND;max_expansions=30;prefix_length=1;analyzer=english;type=best_fields;auto_generate_synonyms_phrase_query=true;fuzzy_transpositions=true');
+SELECT emp_no, first_name, gender, last_name FROM test_emp WHERE MATCH('first_name,last_name', 'Morton', 'slop=1;lenient=true', 'tie_breaker=0.1;fuzzy_rewrite=scoring_boolean','minimum_should_match=1;operator=AND;max_expansions=30;prefix_length=1;analyzer=english;type=best_fields;auto_generate_synonyms_phrase_query=true;fuzzy_transpositions=true');
 
    emp_no:i    | first_name:s  |   gender:s    |  last_name:s
 10095          |Hilari         |M              |Morton
diff --git a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys-geo.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys-geo.csv-spec
index c9380fae2809e..6d165c33433df 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys-geo.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys-geo.csv-spec
@@ -6,10 +6,10 @@ geoSysColumns
 SYS COLUMNS TABLE LIKE 'geo';
 
                TABLE_CAT:s                       |  TABLE_SCHEM:s|  TABLE_NAME:s | COLUMN_NAME:s |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i|BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i| NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
-x-pack_plugin_sql_qa_single-node_integTestCluster|null           |geo            |city           |12             |KEYWORD        |32766          |2147483647     |null            |null            |1          |null           |null           |12             |0                 |2147483647         |1                 |YES          |null           |null          |null         |null               |NO                |NO
-x-pack_plugin_sql_qa_single-node_integTestCluster|null           |geo            |location       |114            |GEO_POINT      |58             |16             |null            |null            |1          |null           |null           |114            |0                 |null               |2                 |YES          |null           |null          |null         |null               |NO                |NO
-x-pack_plugin_sql_qa_single-node_integTestCluster|null           |geo            |location_no_dv |114            |GEO_POINT      |58             |16             |null            |null            |1          |null           |null           |114            |0                 |null               |3                 |YES          |null           |null          |null         |null               |NO                |NO
-x-pack_plugin_sql_qa_single-node_integTestCluster|null           |geo            |region         |12             |KEYWORD        |32766          |2147483647     |null            |null            |1          |null           |null           |12             |0                 |2147483647         |4                 |YES          |null           |null          |null         |null               |NO                |NO
-x-pack_plugin_sql_qa_single-node_integTestCluster|null           |geo            |region_point   |12             |KEYWORD        |32766          |2147483647     |null            |null            |1          |null           |null           |12             |0                 |2147483647         |5                 |YES          |null           |null          |null         |null               |NO                |NO
-x-pack_plugin_sql_qa_single-node_integTestCluster|null           |geo            |shape          |114            |GEO_SHAPE      |2147483647     |2147483647     |null            |null            |1          |null           |null           |114            |0                 |null               |6                 |YES          |null           |null          |null         |null               |NO                |NO
+integTest|null           |geo            |city           |12             |KEYWORD        |32766          |2147483647     |null            |null            |1          |null           |null           |12             |0                 |2147483647         |1                 |YES          |null           |null          |null         |null               |NO                |NO
+integTest|null           |geo            |location       |114            |GEO_POINT      |58             |16             |null            |null            |1          |null           |null           |114            |0                 |null               |2                 |YES          |null           |null          |null         |null               |NO                |NO
+integTest|null           |geo            |location_no_dv |114            |GEO_POINT      |58             |16             |null            |null            |1          |null           |null           |114            |0                 |null               |3                 |YES          |null           |null          |null         |null               |NO                |NO
+integTest|null           |geo            |region         |12             |KEYWORD        |32766          |2147483647     |null            |null            |1          |null           |null           |12             |0                 |2147483647         |4                 |YES          |null           |null          |null         |null               |NO                |NO
+integTest|null           |geo            |region_point   |12             |KEYWORD        |32766          |2147483647     |null            |null            |1          |null           |null           |12             |0                 |2147483647         |5                 |YES          |null           |null          |null         |null               |NO                |NO
+integTest|null           |geo            |shape          |114            |GEO_SHAPE      |2147483647     |2147483647     |null            |null            |1          |null           |null           |114            |0                 |null               |6                 |YES          |null           |null          |null         |null               |NO                |NO
 ;
\ No newline at end of file
diff --git a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec
index 1b13841c47273..6656c695ffe7b 100644
--- a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec
+++ b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec
@@ -7,16 +7,16 @@ SYS COLUMNS TABLE LIKE 'test\_emp' ESCAPE '\';
 
    TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |    COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
 ---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp      |birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp      |emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp      |first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |4               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp      |first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |5               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp      |gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp      |hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |7               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp      |languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp      |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp      |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp      |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp      |birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp      |emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp      |first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |4               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp      |first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |5               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp      |gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp      |hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp      |languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp      |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp      |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp      |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |11              |YES            |null           |null           |null           |null            |NO              |NO                
 ;
 
 sysColumnsWithTableLikeNoEscape
@@ -27,39 +27,39 @@ SYS COLUMNS TABLE LIKE 'test_emp';
 
    TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |    COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
 ---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp            |birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp            |emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp            |extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp            |extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp            |extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp            |first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp            |first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp            |gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp            |hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp            |languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp            |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp            |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp            |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp            |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
 ;
 
 sysColumnsWithCatalogAndLike
-SYS COLUMNS CATALOG 'x-pack_plugin_sql_qa_single-node_integTestCluster' TABLE LIKE 'test\_emp\_copy' ESCAPE '\';
+SYS COLUMNS CATALOG 'integTest' TABLE LIKE 'test\_emp\_copy' ESCAPE '\';
 
    TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |    COLUMN_NAME:s  |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
 ---------------+---------------+---------------+-------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy|birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy|emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy|extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy|extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy|extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy|first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy|first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy|gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy|hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy|languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy|last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy|last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy|salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy|salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
 ;
 
 sysColumnsOnAliasWithTableLike
@@ -67,19 +67,19 @@ SYS COLUMNS TABLE LIKE 'test\_alias' ESCAPE '\';
 
    TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |    COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
 ---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------
-x-pack_plugin_sql_qa_single-node_integTestCluster   |null           |test_alias    |birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster   |null           |test_alias    |emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster   |null           |test_alias    |extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster   |null           |test_alias    |extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster   |null           |test_alias    |extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster   |null           |test_alias    |first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster   |null           |test_alias    |first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster   |null           |test_alias    |gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster   |null           |test_alias    |hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster   |null           |test_alias    |languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster   |null           |test_alias    |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster   |null           |test_alias    |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster   |null           |test_alias    |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest   |null           |test_alias    |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
 ;
 
 sysColumnsAllTables
@@ -87,35 +87,35 @@ SYS COLUMNS TABLE LIKE '%';
 
    TABLE_CAT:s |  TABLE_SCHEM:s|  TABLE_NAME:s |    COLUMN_NAME:s   |   DATA_TYPE:i |   TYPE_NAME:s |  COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i|    REMARKS:s  |  COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s
 ---------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |logs           |@timestamp        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |logs           |bytes_in          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |2               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |logs           |bytes_out         |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |logs           |client_ip         |12             |IP             |0              |39             |null           |null           |1              |null           |null           |12             |0               |null             |4               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |logs           |client_port       |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |5               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |logs           |dest_ip           |12             |IP             |0              |39             |null           |null           |1              |null           |null           |12             |0               |null             |6               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |logs           |id                |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |7               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |logs           |status            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |8               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp       |birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp       |emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp       |first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |4               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp       |first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |5               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp       |gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp       |hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |7               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp       |languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp       |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp       |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp       |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |11              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy  |birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy  |emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy  |extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy  |extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy  |extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy  |first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy  |first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy  |gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy  |hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy  |languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy  |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy  |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
-x-pack_plugin_sql_qa_single-node_integTestCluster  |null           |test_emp_copy  |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |logs           |@timestamp        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |logs           |bytes_in          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |2               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |logs           |bytes_out         |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |logs           |client_ip         |12             |IP             |0              |39             |null           |null           |1              |null           |null           |12             |0               |null             |4               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |logs           |client_port       |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |5               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |logs           |dest_ip           |12             |IP             |0              |39             |null           |null           |1              |null           |null           |12             |0               |null             |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |logs           |id                |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |logs           |status            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp       |birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp       |emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp       |first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |4               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp       |first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |5               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp       |gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp       |hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp       |languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp       |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp       |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp       |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy  |birth_date        |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |1               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy  |emp_no            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |3               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy  |extra.info.gender |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |6               |YES            |null           |null           |null           |null            |NO              |NO
+integTest  |null           |test_emp_copy  |extra_gender      |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |7               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy  |extra_no          |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |8               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy  |first_name        |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |9               |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy  |first_name.keyword|12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |10              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy  |gender            |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |11              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy  |hire_date         |93             |DATETIME       |29             |8              |null           |null           |1              |null           |null           |9              |3               |null             |12              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy  |languages         |-6             |BYTE           |5              |1              |null           |10             |1              |null           |null           |-6             |0               |null             |13              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy  |last_name         |12             |TEXT           |2147483647     |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |14              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy  |last_name.keyword |12             |KEYWORD        |32766          |2147483647     |null           |null           |1              |null           |null           |12             |0               |2147483647       |15              |YES            |null           |null           |null           |null            |NO              |NO                
+integTest  |null           |test_emp_copy  |salary            |4              |INTEGER        |11             |4              |null           |10             |1              |null           |null           |4              |0               |null             |16              |YES            |null           |null           |null           |null            |NO              |NO                
 ;
\ No newline at end of file
diff --git a/x-pack/plugin/sql/sql-action/build.gradle b/x-pack/plugin/sql/sql-action/build.gradle
index 86a028186f441..defbf3dac852e 100644
--- a/x-pack/plugin/sql/sql-action/build.gradle
+++ b/x-pack/plugin/sql/sql-action/build.gradle
@@ -13,10 +13,10 @@ dependencies {
     compile (project(':server')) {
         transitive = false
     }
-    compile (project(':libs:core')) {
+    compile (project(':libs:elasticsearch-core')) {
         transitive = false
     }
-    compile (project(':libs:x-content')) {
+    compile (project(':libs:elasticsearch-x-content')) {
         transitive = false
     }
     compile xpackProject('plugin:sql:sql-proto')
@@ -26,7 +26,7 @@ dependencies {
     runtime "org.apache.logging.log4j:log4j-api:${versions.log4j}"
     runtime "org.apache.logging.log4j:log4j-core:${versions.log4j}"
 
-    testCompile "org.elasticsearch.test:framework:${version}"
+    testCompile project(":test:framework")
 }
 
 forbiddenApisMain {
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlClearCursorAction.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlClearCursorAction.java
index ee824fc04e9f5..37cc028bf993b 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlClearCursorAction.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlClearCursorAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.sql.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class SqlClearCursorAction extends Action<SqlClearCursorResponse> {
+public class SqlClearCursorAction extends StreamableResponseActionType<SqlClearCursorResponse> {
 
     public static final SqlClearCursorAction INSTANCE = new SqlClearCursorAction();
     public static final String NAME = "indices:data/read/sql/close_cursor";
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryAction.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryAction.java
index f25eef31d3dc3..660d0ff19661b 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryAction.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryAction.java
@@ -5,9 +5,9 @@
  */
 package org.elasticsearch.xpack.sql.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class SqlQueryAction extends Action<SqlQueryResponse> {
+public class SqlQueryAction extends StreamableResponseActionType<SqlQueryResponse> {
 
     public static final SqlQueryAction INSTANCE = new SqlQueryAction();
     public static final String NAME = "indices:data/read/sql";
diff --git a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateAction.java b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateAction.java
index 2431ecc1edf8c..b858c7642d163 100644
--- a/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateAction.java
+++ b/x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlTranslateAction.java
@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.sql.action;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
 /**
  * Sql action for translating SQL queries into ES requests
  */
-public class SqlTranslateAction extends Action<SqlTranslateResponse> {
+public class SqlTranslateAction extends StreamableResponseActionType<SqlTranslateResponse> {
 
     public static final SqlTranslateAction INSTANCE = new SqlTranslateAction();
     public static final String NAME = "indices:data/read/sql/translate";
diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java
index 992f55e5a3b00..5a5ec18dc9475 100644
--- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java
+++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java
@@ -41,13 +41,13 @@ public void setup() {
 
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedWriteableRegistry(searchModule.getNamedWriteables());
     }
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequestTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequestTests.java
index a22fba26a13d2..cd992dab530af 100644
--- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequestTests.java
+++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateRequestTests.java
@@ -51,13 +51,13 @@ private TimeValue randomTV() {
 
     @Override
     protected NamedWriteableRegistry getNamedWriteableRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedWriteableRegistry(searchModule.getNamedWriteables());
     }
 
     @Override
     protected NamedXContentRegistry xContentRegistry() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
diff --git a/x-pack/plugin/sql/sql-cli/build.gradle b/x-pack/plugin/sql/sql-cli/build.gradle
index 6f57ea279c5ab..bbd87c055d708 100644
--- a/x-pack/plugin/sql/sql-cli/build.gradle
+++ b/x-pack/plugin/sql/sql-cli/build.gradle
@@ -25,10 +25,10 @@ dependencies {
     
     compile xpackProject('plugin:sql:sql-client')
     compile xpackProject('plugin:sql:sql-action')    
-    compile "org.elasticsearch:elasticsearch-cli:${version}"
+    compile project(":libs:elasticsearch-cli")
 
     runtime "org.elasticsearch:jna:${versions.jna}"
-    testCompile "org.elasticsearch.test:framework:${version}"
+    testCompile project(":test:framework")
 }
 
 dependencyLicenses {
@@ -51,9 +51,9 @@ dependencyLicenses {
  * can be easily shipped around and used.
  */
 jar {
+    dependsOn configurations.runtimeClasspath
     from({
-        configurations.compile.collect { it.isDirectory() ? it : zipTree(it) }
-        configurations.runtime.collect { it.isDirectory() ? it : zipTree(it) }
+        configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) }
     }) {
         // We don't need the META-INF from the things we bundle. For now.
         exclude 'META-INF/*'
diff --git a/x-pack/plugin/sql/sql-client/build.gradle b/x-pack/plugin/sql/sql-client/build.gradle
index 613ca73a4dbad..cc6f097880e38 100644
--- a/x-pack/plugin/sql/sql-client/build.gradle
+++ b/x-pack/plugin/sql/sql-client/build.gradle
@@ -10,7 +10,7 @@ description = 'Code shared between jdbc and cli'
 dependencies {
     compile xpackProject('plugin:sql:sql-proto')
     compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-    testCompile "org.elasticsearch.test:framework:${version}"
+    testCompile project(":test:framework")
 }
 
 dependencyLicenses {
diff --git a/x-pack/plugin/sql/sql-proto/build.gradle b/x-pack/plugin/sql/sql-proto/build.gradle
index b1c055a0dfcb8..af890d82968e3 100644
--- a/x-pack/plugin/sql/sql-proto/build.gradle
+++ b/x-pack/plugin/sql/sql-proto/build.gradle
@@ -8,15 +8,15 @@ description = 'Request and response objects shared by the cli, jdbc ' +
         'and the Elasticsearch plugin'
 
 dependencies {
-    compile (project(':libs:core')) {
+    compile (project(':libs:elasticsearch-core')) {
         transitive = false
     }
-    compile (project(':libs:x-content')) {
+    compile (project(':libs:elasticsearch-x-content')) {
         transitive = false
     }
     runtime "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
 
-    testCompile "org.elasticsearch.test:framework:${version}"
+    testCompile project(":test:framework")
 }
 
 forbiddenApisMain {
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlFeatureSet.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlFeatureSet.java
deleted file mode 100644
index ae478b1d4a59c..0000000000000
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlFeatureSet.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-package org.elasticsearch.xpack.sql;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.XPackField;
-import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.sql.SqlFeatureSetUsage;
-import org.elasticsearch.xpack.core.watcher.common.stats.Counters;
-import org.elasticsearch.xpack.sql.plugin.SqlStatsAction;
-import org.elasticsearch.xpack.sql.plugin.SqlStatsRequest;
-import org.elasticsearch.xpack.sql.plugin.SqlStatsResponse;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
-public class SqlFeatureSet implements XPackFeatureSet {
-
-    private final boolean enabled;
-    private final XPackLicenseState licenseState;
-    private final Client client;
-    
-    @Inject
-    public SqlFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState, Client client) {
-        this.enabled = XPackSettings.SQL_ENABLED.get(settings);
-        this.licenseState = licenseState;
-        this.client = client;
-    }
-    
-    @Override
-    public String name() {
-        return XPackField.SQL;
-    }
-
-    @Override
-    public String description() {
-        return "SQL access to Elasticsearch";
-    }
-
-    @Override
-    public boolean available() {
-        return licenseState != null && licenseState.isSqlAllowed();
-    }
-
-    @Override
-    public boolean enabled() {
-        return enabled;
-    }
-
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return null;
-    }
-
-    @Override
-    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
-        if (enabled) {
-            SqlStatsRequest request = new SqlStatsRequest();
-            request.includeStats(true);
-            client.execute(SqlStatsAction.INSTANCE, request, ActionListener.wrap(r -> {
-                List<Counters> countersPerNode = r.getNodes()
-                        .stream()
-                        .map(SqlStatsResponse.NodeStatsResponse::getStats)
-                        .filter(Objects::nonNull)
-                        .collect(Collectors.toList());
-                Counters mergedCounters = Counters.merge(countersPerNode);
-                listener.onResponse(new SqlFeatureSetUsage(available(), enabled(), mergedCounters.toNestedMap()));
-            }, listener::onFailure));
-        } else {
-            listener.onResponse(new SqlFeatureSetUsage(available(), enabled(), Collections.emptyMap()));
-        }
-    }
-
-}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlInfoTransportAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlInfoTransportAction.java
new file mode 100644
index 0000000000000..45a076f251273
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlInfoTransportAction.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
+
+public class SqlInfoTransportAction extends XPackInfoFeatureTransportAction {
+
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+    
+    @Inject
+    public SqlInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                  Settings settings, XPackLicenseState licenseState) {
+        super(XPackInfoFeatureAction.SQL.name(), transportService, actionFilters);
+        this.enabled = XPackSettings.SQL_ENABLED.get(settings);
+        this.licenseState = licenseState;
+    }
+    
+    @Override
+    public String name() {
+        return XPackField.SQL;
+    }
+
+    @Override
+    public boolean available() {
+        return licenseState.isSqlAllowed();
+    }
+
+    @Override
+    public boolean enabled() {
+        return enabled;
+    }
+
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlUsageTransportAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlUsageTransportAction.java
new file mode 100644
index 0000000000000..3114f263eee29
--- /dev/null
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/SqlUsageTransportAction.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.sql.SqlFeatureSetUsage;
+import org.elasticsearch.xpack.core.watcher.common.stats.Counters;
+import org.elasticsearch.xpack.sql.plugin.SqlStatsAction;
+import org.elasticsearch.xpack.sql.plugin.SqlStatsRequest;
+import org.elasticsearch.xpack.sql.plugin.SqlStatsResponse;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+public class SqlUsageTransportAction extends XPackUsageFeatureTransportAction {
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+    private final Client client;
+
+    @Inject
+    public SqlUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                   ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                   Settings settings, XPackLicenseState licenseState, Client client) {
+        super(XPackUsageFeatureAction.SQL.name(), transportService, clusterService, threadPool, actionFilters,
+            indexNameExpressionResolver);
+        this.enabled = XPackSettings.SQL_ENABLED.get(settings);
+        this.licenseState = licenseState;
+        this.client = client;
+    }
+
+    @Override
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
+        boolean available = licenseState.isSqlAllowed();
+        if (enabled) {
+            SqlStatsRequest sqlRequest = new SqlStatsRequest();
+            sqlRequest.includeStats(true);
+            sqlRequest.setParentTask(clusterService.localNode().getId(), task.getId());
+            client.execute(SqlStatsAction.INSTANCE, sqlRequest, ActionListener.wrap(r -> {
+                List<Counters> countersPerNode = r.getNodes()
+                    .stream()
+                    .map(SqlStatsResponse.NodeStatsResponse::getStats)
+                    .filter(Objects::nonNull)
+                    .collect(Collectors.toList());
+                Counters mergedCounters = Counters.merge(countersPerNode);
+                SqlFeatureSetUsage usage = new SqlFeatureSetUsage(available, enabled, mergedCounters.toNestedMap());
+                listener.onResponse(new XPackUsageFeatureResponse(usage));
+            }, listener::onFailure));
+        } else {
+            SqlFeatureSetUsage usage = new SqlFeatureSetUsage(available, enabled, Collections.emptyMap());
+            listener.onResponse(new XPackUsageFeatureResponse(usage));
+        }
+    }
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursor.java
index 34fab72ca1385..26c1a690a4000 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursor.java
@@ -132,7 +132,7 @@ public void nextPage(Configuration cfg, Client client, NamedWriteableRegistry re
 
         SearchRequest search = Querier.prepareRequest(client, query, cfg.pageTimeout(), includeFrozen, indices);
 
-        client.search(search, new ActionListener<SearchResponse>() {
+        client.search(search, new ActionListener<>() {
             @Override
             public void onResponse(SearchResponse r) {
                 try {
@@ -143,9 +143,9 @@ public void onResponse(SearchResponse r) {
                         return;
                     }
 
-                    updateCompositeAfterKey(r, query);
-                    CompositeAggsRowSet rowSet = new CompositeAggsRowSet(extractors, mask, r, limit, serializeQuery(query), includeFrozen,
-                            indices);
+                    boolean hasAfterKey = updateCompositeAfterKey(r, query);
+                    CompositeAggsRowSet rowSet = new CompositeAggsRowSet(extractors, mask, r, limit,
+                            hasAfterKey ? serializeQuery(query) : null, includeFrozen, indices);
                     listener.onResponse(rowSet);
                 } catch (Exception ex) {
                     listener.onFailure(ex);
@@ -178,7 +178,7 @@ static CompositeAggregation getComposite(SearchResponse response) {
         throw new SqlIllegalArgumentException("Unrecognized root group found; {}", agg.getClass());
     }
 
-    static void updateCompositeAfterKey(SearchResponse r, SearchSourceBuilder next) {
+    static boolean updateCompositeAfterKey(SearchResponse r, SearchSourceBuilder next) {
         CompositeAggregation composite = getComposite(r);
 
         if (composite == null) {
@@ -187,22 +187,25 @@ static void updateCompositeAfterKey(SearchResponse r, SearchSourceBuilder next)
 
         Map<String, Object> afterKey = composite.afterKey();
         // a null after-key means done
-        if (afterKey != null) {
-            AggregationBuilder aggBuilder = next.aggregations().getAggregatorFactories().iterator().next();
-            // update after-key with the new value
-            if (aggBuilder instanceof CompositeAggregationBuilder) {
-                CompositeAggregationBuilder comp = (CompositeAggregationBuilder) aggBuilder;
-                comp.aggregateAfter(afterKey);
-            } else {
-                throw new SqlIllegalArgumentException("Invalid client request; expected a group-by but instead got {}", aggBuilder);
-            }
+        if (afterKey == null) {
+            return false;
+        }
+
+        AggregationBuilder aggBuilder = next.aggregations().getAggregatorFactories().iterator().next();
+        // update after-key with the new value
+        if (aggBuilder instanceof CompositeAggregationBuilder) {
+            CompositeAggregationBuilder comp = (CompositeAggregationBuilder) aggBuilder;
+            comp.aggregateAfter(afterKey);
+            return true;
+        } else {
+            throw new SqlIllegalArgumentException("Invalid client request; expected a group-by but instead got {}", aggBuilder);
         }
     }
 
     /**
      * Deserializes the search source from a byte array.
      */
-    static SearchSourceBuilder deserializeQuery(NamedWriteableRegistry registry, byte[] source) throws IOException {
+    private static SearchSourceBuilder deserializeQuery(NamedWriteableRegistry registry, byte[] source) throws IOException {
         try (NamedWriteableAwareStreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(source), registry)) {
             return new SearchSourceBuilder(in);
         }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggsRowSet.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggsRowSet.java
index f93e00eac5ac9..88b93359d0f77 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggsRowSet.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggsRowSet.java
@@ -28,8 +28,8 @@ class CompositeAggsRowSet extends ResultRowSet<BucketExtractor> {
     private final int size;
     private int row = 0;
 
-    CompositeAggsRowSet(List<BucketExtractor> exts, BitSet mask, SearchResponse response, int limit, byte[] next,
-            boolean includeFrozen, String... indices) {
+    CompositeAggsRowSet(List<BucketExtractor> exts, BitSet mask, SearchResponse response,
+                        int limit, byte[] next, boolean includeFrozen, String... indices) {
         super(exts, mask);
 
         CompositeAggregation composite = CompositeAggregationCursor.getComposite(response);
@@ -40,18 +40,22 @@ class CompositeAggsRowSet extends ResultRowSet<BucketExtractor> {
         }
 
         // page size
-        size = limit < 0 ? buckets.size() : Math.min(buckets.size(), limit);
+        size = limit == -1 ? buckets.size() : Math.min(buckets.size(), limit);
 
         if (next == null) {
             cursor = Cursor.EMPTY;
         } else {
-            // compute remaining limit
-            int remainingLimit = limit - size;
+            // Compute remaining limit
+
+            // If the limit is -1 then we have a local sorting (sort on aggregate function) that requires all the buckets
+            // to be processed so we stop only when all data is exhausted.
+            int remainingLimit = (limit == -1) ? limit : ((limit - size) >= 0 ? (limit - size) : 0);
+
             // if the computed limit is zero, or the size is zero it means either there's nothing left or the limit has been reached
             // note that a composite agg might be valid but return zero groups (since these can be filtered with HAVING/bucket selector)
             // however the Querier takes care of that and keeps making requests until either the query is invalid or at least one response
-            // is returned
-            if (next == null || size == 0 || remainingLimit == 0) {
+            // is returned.
+            if (size == 0 || remainingLimit == 0) {
                 cursor = Cursor.EMPTY;
             } else {
                 cursor = new CompositeAggregationCursor(next, exts, mask, remainingLimit, includeFrozen, indices);
@@ -92,4 +96,4 @@ public int size() {
     public Cursor nextPageCursor() {
         return cursor;
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java
index 17e5a79fa4a71..63aa239e00893 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
 import org.elasticsearch.search.aggregations.bucket.filter.Filters;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -117,7 +118,6 @@ public void query(List<Attribute> output, QueryContainer query, String index, Ac
         listener = sortingColumns.isEmpty() ? listener : new LocalAggregationSorterListener(listener, sortingColumns, query.limit());
 
         ActionListener<SearchResponse> l = null;
-
         if (query.isAggsOnly()) {
             if (query.aggs().useImplicitGroupBy()) {
                 l = new ImplicitGroupActionListener(listener, client, cfg, output, query, search);
@@ -134,7 +134,7 @@ public void query(List<Attribute> output, QueryContainer query, String index, Ac
 
     public static SearchRequest prepareRequest(Client client, SearchSourceBuilder source, TimeValue timeout, boolean includeFrozen,
             String... indices) {
-        SearchRequest search = client.prepareSearch(indices)
+        return client.prepareSearch(indices)
                 // always track total hits accurately
                 .setTrackTotalHits(true)
                 .setAllowPartialSearchResults(false)
@@ -143,7 +143,6 @@ public static SearchRequest prepareRequest(Client client, SearchSourceBuilder so
                 .setIndicesOptions(
                         includeFrozen ? IndexResolver.FIELD_CAPS_FROZEN_INDICES_OPTIONS : IndexResolver.FIELD_CAPS_INDICES_OPTIONS)
                 .request();
-            return search;
     }
 
     /**
@@ -158,11 +157,14 @@ class LocalAggregationSorterListener implements ActionListener<SchemaRowSet> {
         private final ActionListener<SchemaRowSet> listener;
 
         // keep the top N entries.
-        private final PriorityQueue<Tuple<List<?>, Integer>> data;
+        private final AggSortingQueue data;
         private final AtomicInteger counter = new AtomicInteger();
         private volatile Schema schema;
 
-        private static final int MAXIMUM_SIZE = 512;
+        /**
+         * Match the default value for {@link MultiBucketConsumerService#MAX_BUCKET_SETTING}
+         */
+        private static final int MAXIMUM_SIZE = 10_000;
         private final boolean noLimit;
 
         LocalAggregationSorterListener(ActionListener<SchemaRowSet> listener, List<Tuple<Integer, Comparator>> sortingColumns, int limit) {
@@ -174,53 +176,13 @@ class LocalAggregationSorterListener implements ActionListener<SchemaRowSet> {
             } else {
                 noLimit = false;
                 if (limit > MAXIMUM_SIZE) {
-                    throw new PlanningException("The maximum LIMIT for aggregate sorting is [{}], received [{}]", limit, MAXIMUM_SIZE);
+                    throw new PlanningException("The maximum LIMIT for aggregate sorting is [{}], received [{}]", MAXIMUM_SIZE, limit);
                 } else {
                     size = limit;
                 }
             }
 
-            this.data = new PriorityQueue<>(size) {
-
-                // compare row based on the received attribute sort
-                // if a sort item is not in the list, it is assumed the sorting happened in ES
-                // and the results are left as is (by using the row ordering), otherwise it is sorted based on the given criteria.
-                //
-                // Take for example ORDER BY a, x, b, y
-                // a, b - are sorted in ES
-                // x, y - need to be sorted client-side
-                // sorting on x kicks in, only if the values for a are equal.
-
-                // thanks to @jpountz for the row ordering idea as a way to preserve ordering
-                @SuppressWarnings("unchecked")
-                @Override
-                protected boolean lessThan(Tuple<List<?>, Integer> l, Tuple<List<?>, Integer> r) {
-                    for (Tuple<Integer, Comparator> tuple : sortingColumns) {
-                        int i = tuple.v1().intValue();
-                        Comparator comparator = tuple.v2();
-
-                        Object vl = l.v1().get(i);
-                        Object vr = r.v1().get(i);
-                        if (comparator != null) {
-                            int result = comparator.compare(vl, vr);
-                            // if things are equals, move to the next comparator
-                            if (result != 0) {
-                                return result < 0;
-                            }
-                        }
-                        // no comparator means the existing order needs to be preserved
-                        else {
-                            // check the values - if they are equal move to the next comparator
-                            // otherwise return the row order
-                            if (Objects.equals(vl, vr) == false) {
-                                return l.v2().compareTo(r.v2()) < 0;
-                            }
-                        }
-                    }
-                    // everything is equal, fall-back to the row order
-                    return l.v2().compareTo(r.v2()) < 0;
-                }
-            };
+            this.data = new AggSortingQueue(size, sortingColumns);
         }
 
         @Override
@@ -231,9 +193,8 @@ public void onResponse(SchemaRowSet schemaRowSet) {
 
         private void doResponse(RowSet rowSet) {
             // 1. consume all pages received
-            if (consumeRowSet(rowSet) == false) {
-                return;
-            }
+            consumeRowSet(rowSet);
+
             Cursor cursor = rowSet.nextPageCursor();
             // 1a. trigger a next call if there's still data
             if (cursor != Cursor.EMPTY) {
@@ -248,31 +209,21 @@ private void doResponse(RowSet rowSet) {
             sendResponse();
         }
 
-        private boolean consumeRowSet(RowSet rowSet) {
-            // use a synchronized block for visibility purposes (there's no concurrency)
+        private void consumeRowSet(RowSet rowSet) {
             ResultRowSet<?> rrs = (ResultRowSet<?>) rowSet;
-            synchronized (data) {
-                for (boolean hasRows = rrs.hasCurrentRow(); hasRows; hasRows = rrs.advanceRow()) {
-                    List<Object> row = new ArrayList<>(rrs.columnCount());
-                    rrs.forEachResultColumn(row::add);
-                    // if the queue overflows and no limit was specified, bail out
-                    if (data.insertWithOverflow(new Tuple<>(row, counter.getAndIncrement())) != null && noLimit) {
-                        onFailure(new SqlIllegalArgumentException(
-                                "The default limit [{}] for aggregate sorting has been reached; please specify a LIMIT"));
-                        return false;
-                    }
+            for (boolean hasRows = rrs.hasCurrentRow(); hasRows; hasRows = rrs.advanceRow()) {
+                List<Object> row = new ArrayList<>(rrs.columnCount());
+                rrs.forEachResultColumn(row::add);
+                // if the queue overflows and no limit was specified, throw an error
+                if (data.insertWithOverflow(new Tuple<>(row, counter.getAndIncrement())) != null && noLimit) {
+                    onFailure(new SqlIllegalArgumentException(
+                        "The default limit [{}] for aggregate sorting has been reached; please specify a LIMIT", MAXIMUM_SIZE));
                 }
             }
-            return true;
         }
 
         private void sendResponse() {
-            List<List<?>> list = new ArrayList<>(data.size());
-            Tuple<List<?>, Integer> pop = null;
-            while ((pop = data.pop()) != null) {
-                list.add(pop.v1());
-            }
-            listener.onResponse(new PagingListRowSet(schema, list, schema.size(), cfg.pageSize()));
+            listener.onResponse(new PagingListRowSet(schema, data.asList(), schema.size(), cfg.pageSize()));
         }
 
         @Override
@@ -373,7 +324,7 @@ static class CompositeActionListener extends BaseAggActionListener {
         @Override
         protected void handleResponse(SearchResponse response, ActionListener<SchemaRowSet> listener) {
             // there are some results
-            if (response.getAggregations().asList().size() > 0) {
+            if (response.getAggregations().asList().isEmpty() == false) {
 
                 // retry
                 if (CompositeAggregationCursor.shouldRetryDueToEmptyPage(response)) {
@@ -383,7 +334,7 @@ protected void handleResponse(SearchResponse response, ActionListener<SchemaRowS
                 }
 
                 CompositeAggregationCursor.updateCompositeAfterKey(response, request.source());
-                byte[] nextSearch = null;
+                byte[] nextSearch;
                 try {
                     nextSearch = CompositeAggregationCursor.serializeQuery(request.source());
                 } catch (Exception ex) {
@@ -392,7 +343,8 @@ protected void handleResponse(SearchResponse response, ActionListener<SchemaRowS
                 }
 
                 listener.onResponse(
-                        new SchemaCompositeAggsRowSet(schema, initBucketExtractors(response), mask, response, query.limit(),
+                        new SchemaCompositeAggsRowSet(schema, initBucketExtractors(response), mask, response,
+                                query.sortingColumns().isEmpty() ? query.limit() : -1,
                                 nextSearch,
                                 query.shouldIncludeFrozen(),
                                 request.indices()));
@@ -626,4 +578,63 @@ public final void onFailure(Exception ex) {
             listener.onFailure(ex);
         }
     }
+
+    @SuppressWarnings("rawtypes")
+    static class AggSortingQueue extends PriorityQueue<Tuple<List<?>, Integer>> {
+
+        private List<Tuple<Integer, Comparator>> sortingColumns;
+
+        AggSortingQueue(int maxSize, List<Tuple<Integer, Comparator>> sortingColumns) {
+            super(maxSize);
+            this.sortingColumns = sortingColumns;
+        }
+
+        // compare row based on the received attribute sort
+        // if a sort item is not in the list, it is assumed the sorting happened in ES
+        // and the results are left as is (by using the row ordering), otherwise it is sorted based on the given criteria.
+        //
+        // Take for example ORDER BY a, x, b, y
+        // a, b - are sorted in ES
+        // x, y - need to be sorted client-side
+        // sorting on x kicks in, only if the values for a are equal.
+
+        // thanks to @jpountz for the row ordering idea as a way to preserve ordering
+        @SuppressWarnings("unchecked")
+        @Override
+        protected boolean lessThan(Tuple<List<?>, Integer> l, Tuple<List<?>, Integer> r) {
+            for (Tuple<Integer, Comparator> tuple : sortingColumns) {
+                int i = tuple.v1().intValue();
+                Comparator comparator = tuple.v2();
+
+                Object vl = l.v1().get(i);
+                Object vr = r.v1().get(i);
+                if (comparator != null) {
+                    int result = comparator.compare(vl, vr);
+                    // if things are equals, move to the next comparator
+                    if (result != 0) {
+                        return result > 0;
+                    }
+                }
+                // no comparator means the existing order needs to be preserved
+                else {
+                    // check the values - if they are equal move to the next comparator
+                    // otherwise return the row order
+                    if (Objects.equals(vl, vr) == false) {
+                        return l.v2().compareTo(r.v2()) > 0;
+                    }
+                }
+            }
+            // everything is equal, fall-back to the row order
+            return l.v2().compareTo(r.v2()) > 0;
+        }
+
+        List<List<?>> asList() {
+            List<List<?>> list = new ArrayList<>(super.size());
+            Tuple<List<?>, Integer> pop;
+            while ((pop = pop()) != null) {
+                list.add(0, pop.v1());
+            }
+            return list;
+        }
+    }
 }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SchemaCompositeAggsRowSet.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SchemaCompositeAggsRowSet.java
index 64177aeab6872..3ec4ff6b11427 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SchemaCompositeAggsRowSet.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SchemaCompositeAggsRowSet.java
@@ -34,4 +34,4 @@ class SchemaCompositeAggsRowSet extends CompositeAggsRowSet implements SchemaRow
     public Schema schema() {
         return schema;
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java
index 8d9e59617aa4d..4e343c1e54f5a 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java
@@ -80,9 +80,13 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB
             if (source.size() == -1) {
                 source.size(sz);
             }
-            // limit the composite aggs only for non-local sorting
-            if (aggBuilder instanceof CompositeAggregationBuilder && container.sortingColumns().isEmpty()) {
-                ((CompositeAggregationBuilder) aggBuilder).size(sz);
+            if (aggBuilder instanceof CompositeAggregationBuilder) {
+                // limit the composite aggs only for non-local sorting
+                if (container.sortingColumns().isEmpty()) {
+                    ((CompositeAggregationBuilder) aggBuilder).size(sz);
+                } else {
+                    ((CompositeAggregationBuilder) aggBuilder).size(size);
+                }
             }
         }
 
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java
index 13294fbca221b..5446ff2493e40 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java
@@ -165,7 +165,11 @@ private Object unwrapMultiValue(Object values) {
                 return DateUtils.asDateTime(Long.parseLong(values.toString()), zoneId);
             }
         }
-        if (values instanceof Long || values instanceof Double || values instanceof String || values instanceof Boolean) {
+        // The Jackson json parser can generate for numerics - Integers, Longs, BigIntegers (if Long is not enough)
+        // and BigDecimal (if Double is not enough)
+        if (values instanceof Number
+                || values instanceof String
+                || values instanceof Boolean) {
             return values;
         }
         throw new SqlIllegalArgumentException("Type {} (returned by [{}]) is not supported", values.getClass().getSimpleName(), fieldName);
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/geo/GeoShape.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/geo/GeoShape.java
index 74b5c9646b853..1c3d1e7c9358b 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/geo/GeoShape.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/geo/GeoShape.java
@@ -29,6 +29,8 @@
 import org.elasticsearch.geo.geometry.Point;
 import org.elasticsearch.geo.geometry.Polygon;
 import org.elasticsearch.geo.geometry.Rectangle;
+import org.elasticsearch.geo.utils.GeographyValidator;
+import org.elasticsearch.geo.utils.GeometryValidator;
 import org.elasticsearch.geo.utils.WellKnownText;
 import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
 
@@ -49,6 +51,12 @@ public class GeoShape implements ToXContentFragment, NamedWriteable {
 
     private final Geometry shape;
 
+    private static final GeometryValidator validator = new GeographyValidator(true);
+
+    private static final GeometryParser GEOMETRY_PARSER = new GeometryParser(true, true, true);
+
+    private static final WellKnownText WKT_PARSER = new WellKnownText(true, validator);
+
     public GeoShape(double lon, double lat) {
         shape = new Point(lat, lon);
     }
@@ -72,17 +80,17 @@ public GeoShape(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(WellKnownText.toWKT(shape));
+        out.writeString(WKT_PARSER.toWKT(shape));
     }
 
     @Override
     public String toString() {
-        return WellKnownText.toWKT(shape);
+        return WKT_PARSER.toWKT(shape);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        return builder.value(WellKnownText.toWKT(shape));
+        return builder.value(WKT_PARSER.toWKT(shape));
     }
 
     public Geometry toGeometry() {
@@ -216,7 +224,7 @@ private static Geometry parse(Object value) throws IOException, ParseException {
             parser.nextToken(); // start object
             parser.nextToken(); // field name
             parser.nextToken(); // field value
-            return GeometryParser.parse(parser, true, true, true);
+            return GEOMETRY_PARSER.parse(parser);
         }
     }
 }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/package-info.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/package-info.java
index 9528df0332df3..2a55be6e8163f 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/package-info.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/package-info.java
@@ -13,14 +13,14 @@
  * if possible, to one (at the moment) query DSL. Of course, this means
  * not all SQL queries are supported.<br>
  *
- * <h3>Premise</h3>
+ * <h2>Premise</h2>
  * Since Elasticsearch is not a database nor does it supports arbitrary
  * {@code JOIN}s (a cornerstone of SQL), SQL module is built from the
  * ground-up with Elasticsearch in mind first and SQL second. In fact,
  * even the grammar introduces Elasticsearch specific components that
  * have no concept in ANSI SQL.
  *
- * <h3>Architecture</h3>
+ * <h2>Architecture</h2>
  * SQL module is roughly based on the Volcano project (by Graefe
  * {@code &} co)
  * <a href="http://ieeexplore.ieee.org/document/344061">[1]</a>
@@ -53,7 +53,7 @@
  * (to different degrees) by the majority of SQL engines out there such
  * as Apache Calcite, Apache Impala, Apache Spark and Facebook Presto.
  *
- * <h3>Concepts</h3>
+ * <h2>Concepts</h2>
  *
  * The building operation of the SQL engine is defined by an action,
  * namely a rule (defined in {@link org.elasticsearch.xpack.sql.rule rule}
@@ -112,8 +112,8 @@
  *  <li>The Elasticsearch query gets executed</li>
  * </ol>
  *
- * <h4>Digression - Visitors, pattern matching, {@code instanceof} and
- * Java 10/11/12</h4>
+ * <h3>Digression - Visitors, pattern matching, {@code instanceof} and
+ * Java 10/11/12</h3>
  *
  * To implement the above concepts, several choices have been made in the
  * engine (which are not common in the rest of the XPack code base). In
@@ -146,7 +146,7 @@
  * {@link org.elasticsearch.xpack.sql.tree.Node#transformDown(java.util.function.Function, Class)
  * pre-order transformation}).
  *
- * <h3>Components</h3>
+ * <h2>Components</h2>
  *
  * The SQL engine is made up of the following components:
  * <dl>
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPlugin.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPlugin.java
index c80b399d447e6..28c482adfccf2 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPlugin.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPlugin.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -32,14 +31,16 @@
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.sql.SqlFeatureSet;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.sql.SqlInfoTransportAction;
+import org.elasticsearch.xpack.sql.SqlUsageTransportAction;
 import org.elasticsearch.xpack.sql.action.SqlClearCursorAction;
 import org.elasticsearch.xpack.sql.action.SqlQueryAction;
 import org.elasticsearch.xpack.sql.action.SqlTranslateAction;
 import org.elasticsearch.xpack.sql.analysis.index.IndexResolver;
 import org.elasticsearch.xpack.sql.execution.PlanExecutor;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
@@ -103,13 +104,6 @@ Collection<Object> createComponents(Client client, String clusterName, NamedWrit
         return Arrays.asList(sqlLicenseChecker, indexResolver, new PlanExecutor(client, indexResolver, namedWriteableRegistry));
     }
 
-    @Override
-    public Collection<Module> createGuiceModules() {
-        List<Module> modules = new ArrayList<>();
-        modules.add(b -> XPackPlugin.bindFeatureSet(b, SqlFeatureSet.class));
-        return modules;
-    }
-
     @Override
     public List<RestHandler> getRestHandlers(Settings settings, RestController restController,
                                              ClusterSettings clusterSettings, IndexScopedSettings indexScopedSettings,
@@ -128,13 +122,17 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
 
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        var usageAction = new ActionHandler<>(XPackUsageFeatureAction.SQL, SqlUsageTransportAction.class);
+        var infoAction = new ActionHandler<>(XPackInfoFeatureAction.SQL, SqlInfoTransportAction.class);
         if (false == enabled) {
-            return emptyList();
+            return Arrays.asList(usageAction, infoAction);
         }
 
         return Arrays.asList(new ActionHandler<>(SqlQueryAction.INSTANCE, TransportSqlQueryAction.class),
                 new ActionHandler<>(SqlTranslateAction.INSTANCE, TransportSqlTranslateAction.class),
                 new ActionHandler<>(SqlClearCursorAction.INSTANCE, TransportSqlClearCursorAction.class),
-                new ActionHandler<>(SqlStatsAction.INSTANCE, TransportSqlStatsAction.class));
+                new ActionHandler<>(SqlStatsAction.INSTANCE, TransportSqlStatsAction.class),
+                usageAction,
+                infoAction);
     }
 }
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlStatsAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlStatsAction.java
index 5f29c743325cc..a955b442bf970 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlStatsAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlStatsAction.java
@@ -6,9 +6,9 @@
 
 package org.elasticsearch.xpack.sql.plugin;
 
-import org.elasticsearch.action.Action;
+import org.elasticsearch.action.StreamableResponseActionType;
 
-public class SqlStatsAction extends Action<SqlStatsResponse> {
+public class SqlStatsAction extends StreamableResponseActionType<SqlStatsResponse> {
 
     public static final SqlStatsAction INSTANCE = new SqlStatsAction();
     public static final String NAME = "cluster:monitor/xpack/sql/stats/dist";
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlStatsRequest.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlStatsRequest.java
index 36867cd1a55ca..ed0570bf5c16d 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlStatsRequest.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlStatsRequest.java
@@ -53,8 +53,7 @@ static class NodeStatsRequest extends BaseNodeRequest {
         
         NodeStatsRequest() {}
 
-        NodeStatsRequest(SqlStatsRequest request, String nodeId) {
-            super(nodeId);
+        NodeStatsRequest(SqlStatsRequest request) {
             includeStats = request.includeStats();
         }
         
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java
index 62963a99b2a98..f4e3e006e70f9 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java
@@ -224,7 +224,7 @@ String format(Cursor cursor, RestRequest request, SqlQueryResponse response) {
 
         boolean header = hasHeader(request);
 
-        if (header) {
+        if (header && (cursor == null || cursor == Cursor.EMPTY)) {
             row(sb, response.columns(), ColumnInfo::name);
         }
 
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlStatsAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlStatsAction.java
index 815e45175e95e..acd223d03fa05 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlStatsAction.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlStatsAction.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.sql.execution.PlanExecutor;
@@ -41,8 +42,8 @@ protected SqlStatsResponse newResponse(SqlStatsRequest request, List<SqlStatsRes
     }
 
     @Override
-    protected SqlStatsRequest.NodeStatsRequest newNodeRequest(String nodeId, SqlStatsRequest request) {
-        return new SqlStatsRequest.NodeStatsRequest(request, nodeId);
+    protected SqlStatsRequest.NodeStatsRequest newNodeRequest(SqlStatsRequest request) {
+        return new SqlStatsRequest.NodeStatsRequest(request);
     }
 
     @Override
@@ -51,7 +52,7 @@ protected SqlStatsResponse.NodeStatsResponse newNodeResponse() {
     }
 
     @Override
-    protected SqlStatsResponse.NodeStatsResponse nodeOperation(SqlStatsRequest.NodeStatsRequest request) {
+    protected SqlStatsResponse.NodeStatsResponse nodeOperation(SqlStatsRequest.NodeStatsRequest request, Task task) {
         SqlStatsResponse.NodeStatsResponse statsResponse = new SqlStatsResponse.NodeStatsResponse(clusterService.localNode());
         statsResponse.setStats(planExecutor.metrics().stats());
         
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/query/MatchQuery.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/query/MatchQuery.java
index 58ca09da929f9..3fca7630bd5a6 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/query/MatchQuery.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/query/MatchQuery.java
@@ -32,7 +32,6 @@ public class MatchQuery extends LeafQuery {
         BUILDER_APPLIERS = Map.ofEntries(
                 entry("analyzer", MatchQueryBuilder::analyzer),
                 entry("auto_generate_synonyms_phrase_query", (qb, s) -> qb.autoGenerateSynonymsPhraseQuery(Booleans.parseBoolean(s))),
-                entry("cutoff_frequency", (qb, s) -> qb.cutoffFrequency(Float.valueOf(s))),
                 entry("fuzziness", (qb, s) -> qb.fuzziness(Fuzziness.fromString(s))),
                 entry("fuzzy_transpositions", (qb, s) -> qb.fuzzyTranspositions(Booleans.parseBoolean(s))),
                 entry("fuzzy_rewrite", MatchQueryBuilder::fuzzyRewrite),
diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/query/MultiMatchQuery.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/query/MultiMatchQuery.java
index f51f8275a898c..ab6190ad6eee1 100644
--- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/query/MultiMatchQuery.java
+++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/query/MultiMatchQuery.java
@@ -32,7 +32,6 @@ public class MultiMatchQuery extends LeafQuery {
                 // appliers.put("zero_terms_query", (qb, s) -> qb.zeroTermsQuery(s));
                 entry("analyzer", MultiMatchQueryBuilder::analyzer),
                 entry("auto_generate_synonyms_phrase_query", (qb, s) -> qb.autoGenerateSynonymsPhraseQuery(Booleans.parseBoolean(s))),
-                entry("cutoff_frequency", (qb, s) -> qb.cutoffFrequency(Float.valueOf(s))),
                 entry("fuzziness", (qb, s) -> qb.fuzziness(Fuzziness.fromString(s))),
                 entry("fuzzy_rewrite", MultiMatchQueryBuilder::fuzzyRewrite),
                 entry("fuzzy_transpositions", (qb, s) -> qb.fuzzyTranspositions(Booleans.parseBoolean(s))),
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlFeatureSetTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java
similarity index 77%
rename from x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlFeatureSetTests.java
rename to x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java
index 39998885a5649..fb8f63a35aaca 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlFeatureSetTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/SqlInfoTransportActionTests.java
@@ -7,16 +7,21 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.sql.SqlFeatureSetUsage;
 import org.elasticsearch.xpack.core.watcher.common.stats.Counters;
 import org.elasticsearch.xpack.sql.plugin.SqlStatsAction;
@@ -35,7 +40,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class SqlFeatureSetTests extends ESTestCase {
+public class SqlInfoTransportActionTests extends ESTestCase {
 
     private XPackLicenseState licenseState;
     private Client client;
@@ -51,7 +56,8 @@ public void init() throws Exception {
     }
 
     public void testAvailable() {
-        SqlFeatureSet featureSet = new SqlFeatureSet(Settings.EMPTY, licenseState, client);
+        SqlInfoTransportAction featureSet = new SqlInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
         boolean available = randomBoolean();
         when(licenseState.isSqlAllowed()).thenReturn(available);
         assertThat(featureSet.available(), is(available));
@@ -67,7 +73,8 @@ public void testEnabled() {
         } else {
             settings.put("xpack.sql.enabled", enabled);
         }
-        SqlFeatureSet featureSet = new SqlFeatureSet(settings.build(), licenseState, client);
+        SqlInfoTransportAction featureSet = new SqlInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState);
         assertThat(featureSet.enabled(), is(enabled));
     }
 
@@ -97,10 +104,16 @@ public void testUsageStats() throws Exception {
             listener.onResponse(new SqlStatsResponse(new ClusterName("whatever"), nodes, Collections.emptyList()));
             return null;
         }).when(client).execute(eq(SqlStatsAction.INSTANCE), any(), any());
+        ClusterService clusterService = mock(ClusterService.class);
+        final DiscoveryNode mockNode = mock(DiscoveryNode.class);
+        when(mockNode.getId()).thenReturn("mocknode");
+        when(clusterService.localNode()).thenReturn(mockNode);
 
-        PlainActionFuture<SqlFeatureSet.Usage> future = new PlainActionFuture<>();
-        new SqlFeatureSet(Settings.EMPTY, licenseState, client).usage(future);
-        SqlFeatureSetUsage sqlUsage = (SqlFeatureSetUsage) future.get();
+        var usageAction = new SqlUsageTransportAction(mock(TransportService.class), clusterService, null,
+            mock(ActionFilters.class), null, Settings.EMPTY, licenseState, client);
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(mock(Task.class), null, null, future);
+        SqlFeatureSetUsage sqlUsage = (SqlFeatureSetUsage) future.get().getUsage();
         
         long fooBarBaz = ObjectPath.eval("foo.bar.baz", sqlUsage.stats());
         long fooFoo = ObjectPath.eval("foo.foo", sqlUsage.stats());
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursorTests.java
index 811a8ff4256f2..0baf470e4b364 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursorTests.java
@@ -46,7 +46,8 @@ protected CompositeAggregationCursor mutateInstance(CompositeAggregationCursor i
         return new CompositeAggregationCursor(instance.next(), instance.extractors(),
                 randomValueOtherThan(instance.mask(), () -> randomBitSet(instance.extractors().size())),
                 randomValueOtherThan(instance.limit(), () -> randomIntBetween(1, 512)),
-                randomBoolean(), instance.indices());
+                !instance.includeFrozen(),
+                instance.indices());
     }
 
     @Override
@@ -81,4 +82,4 @@ static BitSet randomBitSet(int size) {
         }
         return mask;
     }
-}
\ No newline at end of file
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/QuerierTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/QuerierTests.java
new file mode 100644
index 0000000000000..a6caad899dd89
--- /dev/null
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/QuerierTests.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.execution.search;
+
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.sql.execution.search.Querier.AggSortingQueue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+public class QuerierTests extends ESTestCase {
+
+    @SuppressWarnings("rawtypes")
+    public void testAggSortingAscending() {
+        Tuple<Integer, Comparator> tuple = new Tuple<>(0, Comparator.naturalOrder());
+        Querier.AggSortingQueue queue = new AggSortingQueue(10, Collections.singletonList(tuple));
+        for (int i = 50; i >= 0; i--) {
+            queue.insertWithOverflow(new Tuple<>(Collections.singletonList(i), i));
+        }
+        List<List<?>> results = queue.asList();
+
+        assertEquals(10, results.size());
+        for (int i = 0; i < 10; i ++) {
+            assertEquals(i, results.get(i).get(0));
+        }
+    }
+
+    @SuppressWarnings("rawtypes")
+    public void testAggSortingDescending() {
+        Tuple<Integer, Comparator> tuple = new Tuple<>(0, Comparator.reverseOrder());
+        Querier.AggSortingQueue queue = new AggSortingQueue(10, Collections.singletonList(tuple));
+        for (int i = 0; i <= 50; i++) {
+            queue.insertWithOverflow(new Tuple<>(Collections.singletonList(i), i));
+        }
+        List<List<?>> results = queue.asList();
+
+        assertEquals(10, results.size());
+        for (int i = 0; i < 10; i ++) {
+            assertEquals(50 - i, results.get(i).get(0));
+        }
+    }
+
+    @SuppressWarnings("rawtypes")
+    public void testAggSorting_TwoFields() {
+        List<Tuple<Integer, Comparator>> tuples = new ArrayList<>(2);
+        tuples.add(new Tuple<>(0, Comparator.reverseOrder()));
+        tuples.add(new Tuple<>(1, Comparator.naturalOrder()));
+        Querier.AggSortingQueue queue = new AggSortingQueue(10, tuples);
+
+        for (int i = 1; i <= 100; i++) {
+            queue.insertWithOverflow(new Tuple<>(Arrays.asList(i % 50 + 1, i), i));
+        }
+        List<List<?>> results = queue.asList();
+
+        assertEquals(10, results.size());
+        for (int i = 0; i < 10; i++) {
+            assertEquals(50 - (i / 2), results.get(i).get(0));
+            assertEquals(49 - (i / 2) + ((i % 2) * 50), results.get(i).get(1));
+        }
+    }
+
+    @SuppressWarnings("rawtypes")
+    public void testAggSorting_Randomized() {
+        // Initialize comparators for fields (columns)
+        int noColumns = randomIntBetween(3, 10);
+        List<Tuple<Integer, Comparator>> tuples = new ArrayList<>(noColumns);
+        boolean[] ordering = new boolean[noColumns];
+        for (int j = 0; j < noColumns; j++) {
+            boolean order = randomBoolean();
+            ordering[j] = order;
+            tuples.add(new Tuple<>(j, order ? Comparator.naturalOrder() : Comparator.reverseOrder()));
+        }
+
+        // Insert random no of documents (rows) with random 0/1 values for each field
+        int noDocs = randomIntBetween(10, 50);
+        int queueSize = randomIntBetween(4, noDocs / 2);
+        List<List<Integer>> expected = new ArrayList<>(noDocs);
+        Querier.AggSortingQueue queue = new AggSortingQueue(queueSize, tuples);
+        for (int i = 0; i < noDocs; i++) {
+            List<Integer> values = new ArrayList<>(noColumns);
+            for (int j = 0; j < noColumns; j++) {
+                values.add(randomBoolean() ? 1 : 0);
+            }
+            queue.insertWithOverflow(new Tuple<>(values, i));
+            expected.add(values);
+        }
+
+        List<List<?>> results = queue.asList();
+        assertEquals(queueSize, results.size());
+        expected.sort((o1, o2) -> {
+            for (int j = 0; j < noColumns; j++) {
+                if (o1.get(j) < o2.get(j)) {
+                    return ordering[j] ? -1 : 1;
+                } else if (o1.get(j) > o2.get(j)) {
+                    return ordering[j] ? 1 : -1;
+                }
+            }
+            return 0;
+        });
+        assertEquals(expected.subList(0, queueSize), results);
+    }
+}
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java
index 50a3b185dba86..d4ea80485f499 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java
@@ -19,6 +19,8 @@
 import org.elasticsearch.xpack.sql.util.DateUtils;
 
 import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
 import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -127,7 +129,7 @@ public void testGetDottedValueWithSource() throws Exception {
             BytesReference sourceRef = BytesReference.bytes(source);
             hit.sourceRef(sourceRef);
             Object extract = extractor.extract(hit);
-            assertEquals(hasSource ? value : null, extract);
+            assertFieldHitEquals(hasSource ? value : null, extract);
         }
     }
 
@@ -180,7 +182,7 @@ public void testGetSource() throws IOException {
             source.endObject();
             BytesReference sourceRef = BytesReference.bytes(source);
             hit.sourceRef(sourceRef);
-            assertEquals(value, extractor.extract(hit));
+            assertFieldHitEquals(value, extractor.extract(hit));
         }
     }
 
@@ -226,7 +228,7 @@ public void testSingleValueArrayInSource() throws IOException {
         source.endObject();
         BytesReference sourceRef = BytesReference.bytes(source);
         hit.sourceRef(sourceRef);
-        assertEquals(value, fe.extract(hit));
+        assertFieldHitEquals(value, fe.extract(hit));
     }
 
     public void testExtractSourcePath() {
@@ -580,6 +582,9 @@ private Object randomValue() {
                 () -> randomAlphaOfLength(10),
                 ESTestCase::randomLong,
                 ESTestCase::randomDouble,
+                ESTestCase::randomInt,
+                () -> BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE),
+                () -> new BigDecimal("20012312345621343256123456254.20012312345621343256123456254"),
                 () -> null));
         return value.get();
     }
@@ -588,9 +593,22 @@ private Object randomNonNullValue() {
         Supplier<Object> value = randomFrom(Arrays.asList(
                 () -> randomAlphaOfLength(10),
                 ESTestCase::randomLong,
-                ESTestCase::randomDouble));
+                ESTestCase::randomDouble,
+                ESTestCase::randomInt,
+                () -> BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE),
+                () -> new BigDecimal("20012312345621343256123456254.20012312345621343256123456254")));
         return value.get();
     }
+    
+    private void assertFieldHitEquals(Object expected, Object actual) {
+        if (expected instanceof BigDecimal) {
+            // parsing will, by default, build a Double even if the initial value is BigDecimal
+            // Elasticsearch does this the same when returning the results
+            assertEquals(((BigDecimal) expected).doubleValue(), actual);
+        } else {
+            assertEquals(expected, actual);
+        }
+    }
 
     private Object randomPoint(double lat, double lon) {
         Supplier<Object> value = randomFrom(Arrays.asList(
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TestMultiValueAggregation.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TestMultiValueAggregation.java
index cfa429e91e83d..7a24a5515b527 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TestMultiValueAggregation.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TestMultiValueAggregation.java
@@ -50,14 +50,4 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         throw new UnsupportedOperationException();
     }
-
-    @Override
-    protected int doHashCode() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    protected boolean doEquals(Object obj) {
-        throw new UnsupportedOperationException();
-    }
 }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TestSingleValueAggregation.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TestSingleValueAggregation.java
index 2c00f5e845f98..08f4edda3cbf6 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TestSingleValueAggregation.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/TestSingleValueAggregation.java
@@ -54,13 +54,4 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
         throw new UnsupportedOperationException();
     }
 
-    @Override
-    protected int doHashCode() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    protected boolean doEquals(Object obj) {
-        throw new UnsupportedOperationException();
-    }
 }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/geo/StWkttosqlProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/geo/StWkttosqlProcessorTests.java
index fc7b33ae905d7..818897dce343e 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/geo/StWkttosqlProcessorTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/geo/StWkttosqlProcessorTests.java
@@ -39,4 +39,13 @@ public void testTypeCheck() {
         siae = expectThrows(SqlIllegalArgumentException.class, () -> procPoint.process("point (10 10"));
         assertEquals("Cannot parse [point (10 10] as a geo_shape value", siae.getMessage());
     }
+
+    public void testCoerce() {
+        StWkttosqlProcessor proc = new StWkttosqlProcessor();
+        assertNull(proc.process(null));
+        Object result = proc.process("POLYGON ((3 1 5, 4 2 4, 5 3 3))");
+        assertThat(result, instanceOf(GeoShape.class));
+        GeoShape geoShape = (GeoShape) result;
+        assertEquals("polygon ((3.0 1.0 5.0, 4.0 2.0 4.0, 5.0 3.0 3.0, 3.0 1.0 5.0))", geoShape.toString());
+    }
 }
diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlPluginTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlPluginTests.java
index be8ac7b9968dc..43102609831b3 100644
--- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlPluginTests.java
+++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlPluginTests.java
@@ -20,6 +20,7 @@
 import java.util.Collections;
 
 import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.hasSize;
 import static org.mockito.Mockito.mock;
 
 public class SqlPluginTests extends ESTestCase {
@@ -29,7 +30,7 @@ public void testSqlDisabled() {
         SqlPlugin plugin = new SqlPlugin(settings);
         assertThat(plugin.createComponents(mock(Client.class), "cluster", new NamedWriteableRegistry(Cursors.getNamedWriteables())),
             empty());
-        assertThat(plugin.getActions(), empty());
+        assertThat(plugin.getActions(), hasSize(2)); // usage action
         assertThat(plugin.getRestHandlers(Settings.EMPTY, mock(RestController.class),
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, new SettingsFilter(Collections.emptyList()),
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.delete_auto_follow_pattern.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.delete_auto_follow_pattern.json
index 735387b173b73..dd0be5a379d30 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.delete_auto_follow_pattern.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.delete_auto_follow_pattern.json
@@ -1,9 +1,9 @@
 {
   "ccr.delete_auto_follow_pattern": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-delete-auto-follow-pattern.html",
+    "stability" : "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_ccr/auto_follow/{name}",
       "paths": [ "/_ccr/auto_follow/{name}" ],
       "parts": {
         "name": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow.json
index 588dd60261252..ca7c7e9984ee4 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow.json
@@ -1,9 +1,9 @@
 {
   "ccr.follow": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-follow.html",
+    "stability" : "stable",
     "methods": [ "PUT" ],
     "url": {
-      "path": "/{index}/_ccr/follow",
       "paths": [ "/{index}/_ccr/follow" ],
       "parts": {
         "index": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow_info.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow_info.json
index 87fd387edc13a..e2dcd8e83080c 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow_info.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow_info.json
@@ -1,9 +1,9 @@
 {
   "ccr.follow_info": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-get-follow-info.html",
+    "stability" : "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/{index}/_ccr/info",
       "paths": [ "/{index}/_ccr/info" ],
       "parts": {
         "index": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow_stats.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow_stats.json
index de4bb66056bb3..6901a39baf7b3 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow_stats.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.follow_stats.json
@@ -1,9 +1,9 @@
 {
   "ccr.follow_stats": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-get-follow-stats.html",
+    "stability" : "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/{index}/_ccr/stats",
       "paths": [ "/{index}/_ccr/stats" ],
       "parts": {
         "index": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.forget_follower.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.forget_follower.json
index 92d38e5e999e8..843a5760e2ca8 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.forget_follower.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.forget_follower.json
@@ -1,9 +1,9 @@
 {
   "ccr.forget_follower": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current",
+    "stability" : "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/{index}/_ccr/forget_follower",
       "paths": [ "/{index}/_ccr/forget_follower" ],
       "parts": {
         "index": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.get_auto_follow_pattern.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.get_auto_follow_pattern.json
index 6aea678fc004b..2511e68cb8edd 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.get_auto_follow_pattern.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.get_auto_follow_pattern.json
@@ -1,9 +1,9 @@
 {
   "ccr.get_auto_follow_pattern": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-get-auto-follow-pattern.html",
+    "stability" : "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_ccr/auto_follow/{name}",
       "paths": [ "/_ccr/auto_follow", "/_ccr/auto_follow/{name}" ],
       "parts": {
         "name": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.pause_follow.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.pause_follow.json
index e128dab917ca4..ff30da5d9ac22 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.pause_follow.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.pause_follow.json
@@ -1,9 +1,9 @@
 {
   "ccr.pause_follow": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-post-pause-follow.html",
+    "stability" : "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/{index}/_ccr/pause_follow",
       "paths": [ "/{index}/_ccr/pause_follow" ],
       "parts": {
         "index": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.put_auto_follow_pattern.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.put_auto_follow_pattern.json
index 128b262ec7a50..91a26268ce568 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.put_auto_follow_pattern.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.put_auto_follow_pattern.json
@@ -1,9 +1,9 @@
 {
   "ccr.put_auto_follow_pattern": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-auto-follow-pattern.html",
+    "stability" : "stable",
     "methods": [ "PUT" ],
     "url": {
-      "path": "/_ccr/auto_follow/{name}",
       "paths": [ "/_ccr/auto_follow/{name}" ],
       "parts": {
         "name": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.resume_follow.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.resume_follow.json
index 61e3b8580fc2c..3c4481a7d7ee0 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.resume_follow.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.resume_follow.json
@@ -1,9 +1,9 @@
 {
   "ccr.resume_follow": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-post-resume-follow.html",
+    "stability" : "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/{index}/_ccr/resume_follow",
       "paths": [ "/{index}/_ccr/resume_follow" ],
       "parts": {
         "index": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.stats.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.stats.json
index ad9ab49bcbe46..ab96a395efc2d 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.stats.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.stats.json
@@ -1,9 +1,9 @@
 {
   "ccr.stats": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-get-stats.html",
+    "stability" : "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_ccr/stats",
       "paths": [ "/_ccr/stats" ],
       "parts": {},
       "body": null
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.unfollow.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.unfollow.json
index 41be574421fc6..0c68018319b09 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.unfollow.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ccr.unfollow.json
@@ -1,9 +1,9 @@
 {
   "ccr.unfollow": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current",
+    "stability" : "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/{index}/_ccr/unfollow",
       "paths": [ "/{index}/_ccr/unfollow" ],
       "parts": {
         "index": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.delete_data_frame_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.delete_data_frame_transform.json
index 14478f72c9afe..30da72491df11 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.delete_data_frame_transform.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.delete_data_frame_transform.json
@@ -1,9 +1,9 @@
 {
   "data_frame.delete_data_frame_transform": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/delete-data-frame-transform.html",
+    "stability": "beta",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_data_frame/transforms/{transform_id}",
       "paths": [
         "/_data_frame/transforms/{transform_id}"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.get_data_frame_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.get_data_frame_transform.json
index 5bf6dd25fb546..fc89a524081c8 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.get_data_frame_transform.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.get_data_frame_transform.json
@@ -1,9 +1,9 @@
 {
   "data_frame.get_data_frame_transform": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/get-data-frame-transform.html",
+    "stability": "beta",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_data_frame/transforms/{transform_id}",
       "paths": [ "/_data_frame/transforms/{transform_id}", "/_data_frame/transforms"],
       "parts": {
         "transform_id": {
@@ -22,6 +22,11 @@
           "type": "int",
           "required": false,
           "description": "specifies a max number of transforms to get, defaults to 100"
+        },
+        "allow_no_match": {
+          "type": "boolean",
+          "required": false,
+          "description": "Whether to ignore if a wildcard expression matches no data frame transforms. (This includes `_all` string or when no data frame transforms have been specified)"
         }
       }
     },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.get_data_frame_transform_stats.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.get_data_frame_transform_stats.json
index b100c7e0a2471..e04cb4d0b0637 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.get_data_frame_transform_stats.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.get_data_frame_transform_stats.json
@@ -1,9 +1,9 @@
 {
   "data_frame.get_data_frame_transform_stats": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/get-data-frame-transform-stats.html",
+    "stability": "beta",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_data_frame/transforms/{transform_id}/_stats",
       "paths": [ "/_data_frame/transforms/{transform_id}/_stats" ],
       "parts": {
         "transform_id": {
@@ -22,6 +22,11 @@
           "type": "number",
           "required": false,
           "description": "specifies a max number of transform stats to get, defaults to 100"
+        },
+        "allow_no_match": {
+          "type": "boolean",
+          "required": false,
+          "description": "Whether to ignore if a wildcard expression matches no data frame transforms. (This includes `_all` string or when no data frame transforms have been specified)"
         }
       }
     },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.preview_data_frame_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.preview_data_frame_transform.json
index c8cf76e90d340..bbfcf61640c43 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.preview_data_frame_transform.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.preview_data_frame_transform.json
@@ -1,9 +1,9 @@
 {
   "data_frame.preview_data_frame_transform": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/preview-data-frame-transform.html",
+    "stability": "beta",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_data_frame/transforms/_preview",
       "paths": [ "/_data_frame/transforms/_preview" ]
     },
     "body": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.put_data_frame_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.put_data_frame_transform.json
index cef6de3a2b6e4..919682676da3c 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.put_data_frame_transform.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.put_data_frame_transform.json
@@ -1,9 +1,9 @@
 {
   "data_frame.put_data_frame_transform": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/put-data-frame-transform.html",
+    "stability": "beta",
     "methods": [ "PUT" ],
     "url": {
-      "path": "/_data_frame/transforms/{transform_id}",
       "paths": [ "/_data_frame/transforms/{transform_id}" ],
       "parts": {
         "transform_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.start_data_frame_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.start_data_frame_transform.json
index f2dd8e529f47d..0a1368514f1f8 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.start_data_frame_transform.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.start_data_frame_transform.json
@@ -1,9 +1,9 @@
 {
   "data_frame.start_data_frame_transform": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/start-data-frame-transform.html",
+    "stability": "beta",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_data_frame/transforms/{transform_id}/_start",
       "paths": [ "/_data_frame/transforms/{transform_id}/_start" ],
       "parts": {
         "transform_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.stop_data_frame_transform.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.stop_data_frame_transform.json
index f4f548b9aa0eb..0eb4452bc7091 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.stop_data_frame_transform.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame.stop_data_frame_transform.json
@@ -1,9 +1,9 @@
 {
   "data_frame.stop_data_frame_transform": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/stop-data-frame-transform.html",
+    "stability": "beta",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_data_frame/transforms/{transform_id}/_stop",
       "paths": [ "/_data_frame/transforms/{transform_id}/_stop" ],
       "parts": {
         "transform_id": {
@@ -22,6 +22,11 @@
           "type": "time",
           "required": false,
           "description": "Controls the time to wait until the transform has stopped. Default to 30 seconds"
+        },
+        "allow_no_match": {
+          "type": "boolean",
+          "required": false,
+          "description": "Whether to ignore if a wildcard expression matches no data frame transforms. (This includes `_all` string or when no data frame transforms have been specified)"
         }
       }
     },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/graph.explore.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/graph.explore.json
index b01c51bc6c381..4c82edadca339 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/graph.explore.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/graph.explore.json
@@ -1,9 +1,9 @@
 {
   "graph.explore": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/graph-explore-api.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/{index}/_graph/explore",
       "paths": ["/{index}/_graph/explore"],
       "deprecated_paths" : [
         {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.delete_lifecycle.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.delete_lifecycle.json
index cee1660fb1f84..1abbc0a1b6ba0 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.delete_lifecycle.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.delete_lifecycle.json
@@ -1,9 +1,9 @@
 {
   "ilm.delete_lifecycle": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-delete-lifecycle.html",
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_ilm/policy/{policy}",
       "paths": ["/_ilm/policy/{policy}"],
       "parts": {
         "policy": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.explain_lifecycle.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.explain_lifecycle.json
index 2441e64264809..8c48b22a5eba5 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.explain_lifecycle.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.explain_lifecycle.json
@@ -1,9 +1,9 @@
 {
   "ilm.explain_lifecycle": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-explain-lifecycle.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/{index}/_ilm/explain",
       "paths": ["/{index}/_ilm/explain"],
       "parts": {
         "index": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.get_lifecycle.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.get_lifecycle.json
index 3526ff40aee46..174e99059a7e2 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.get_lifecycle.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.get_lifecycle.json
@@ -1,9 +1,9 @@
 {
   "ilm.get_lifecycle": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-get-lifecycle.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_ilm/policy/{policy}",
       "paths": ["/_ilm/policy/{policy}", "/_ilm/policy"],
       "parts": {
         "policy": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.get_status.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.get_status.json
index 70aec541a115b..864a010c89e36 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.get_status.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.get_status.json
@@ -1,9 +1,9 @@
 {
   "ilm.get_status": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-get-status.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_ilm/status",
       "paths": ["/_ilm/status"],
       "parts": {},
       "params": {}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.move_to_step.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.move_to_step.json
index e3adf2d30e681..bb8d6ec54a08f 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.move_to_step.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.move_to_step.json
@@ -1,10 +1,10 @@
 {
   "ilm.move_to_step": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-move-to-step.html",
+    "stability": "stable",
     "description": "Triggers execution of a specific step in the lifecycle policy. Since this action is designed to only be run in emergency situations, clients should not implement this API",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ilm/move/{index}",
       "paths": ["/_ilm/move/{index}"],
       "parts": {
         "index": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.put_lifecycle.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.put_lifecycle.json
index 4d93becbcc9a2..7e97c46b69440 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.put_lifecycle.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.put_lifecycle.json
@@ -1,9 +1,9 @@
 {
   "ilm.put_lifecycle": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-put-lifecycle.html",
+    "stability": "stable",
     "methods": [ "PUT" ],
     "url": {
-      "path": "/_ilm/policy/{policy}",
       "paths": ["/_ilm/policy/{policy}"],
       "parts": {
         "policy": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.remove_policy.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.remove_policy.json
index d9903ff8dc40d..82c05d5560e67 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.remove_policy.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.remove_policy.json
@@ -1,9 +1,9 @@
 {
   "ilm.remove_policy": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-remove-policy.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/{index}/_ilm/remove",
       "paths": ["/{index}/_ilm/remove"],
       "parts": {
         "index": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.retry.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.retry.json
index 41edd6a83c2eb..cf9b8a30cf14b 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.retry.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.retry.json
@@ -1,9 +1,9 @@
 {
   "ilm.retry": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-retry-policy.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/{index}/_ilm/retry",
       "paths": ["/{index}/_ilm/retry"],
       "parts": {
         "index": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.start.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.start.json
index d20b3187b7e33..c13f860b9e748 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.start.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.start.json
@@ -1,9 +1,9 @@
 {
   "ilm.start": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-start.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ilm/start",
       "paths": ["/_ilm/start"],
       "parts": {},
       "params": {}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.stop.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.stop.json
index eaf75a885b8aa..bee334e9f0b42 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.stop.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ilm.stop.json
@@ -1,9 +1,9 @@
 {
   "ilm.stop": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-stop.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ilm/stop",
       "paths": ["/_ilm/stop"],
       "parts": {},
       "params": {}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/indices.freeze.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/indices.freeze.json
index a602f341e1524..0f3d4d70239e0 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/indices.freeze.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/indices.freeze.json
@@ -1,9 +1,9 @@
 {
   "indices.freeze": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/{index}/_freeze",
       "paths": [
         "/{index}/_freeze"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/indices.reload_search_analyzers.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/indices.reload_search_analyzers.json
new file mode 100644
index 0000000000000..bd79dbf4718f5
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/indices.reload_search_analyzers.json
@@ -0,0 +1,33 @@
+{
+  "indices.reload_search_analyzers": {
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-reload-analyzers.html",
+    "stability": "experimental",
+    "methods": ["GET", "POST"],
+    "url": {
+      "paths": ["/{index}/_reload_search_analyzers"],
+      "parts": {
+        "index": {
+         "type": "list",
+         "description" : "A comma-separated list of index names to reload analyzers for"
+        }
+     },
+     "params": {
+        "ignore_unavailable": {
+            "type" : "boolean",
+            "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
+        },
+        "allow_no_indices": {
+            "type" : "boolean",
+            "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
+        },
+        "expand_wildcards": {
+            "type" : "enum",
+            "options" : ["open","closed","none","all"],
+            "default" : "open",
+            "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
+        }
+     }
+    }, 
+    "body": null
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/indices.unfreeze.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/indices.unfreeze.json
index b10e869a95758..f8fb93a4401bb 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/indices.unfreeze.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/indices.unfreeze.json
@@ -1,9 +1,9 @@
 {
   "indices.unfreeze": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/frozen.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/{index}/_unfreeze",
       "paths": [ "/{index}/_unfreeze" ],
       "parts": {
         "index": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/license.delete.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/license.delete.json
index a85552e557597..f5a6c28c8a323 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/license.delete.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/license.delete.json
@@ -1,9 +1,9 @@
 {
   "license.delete": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-license.html",
+    "stability": "stable",
     "methods": ["DELETE"],
     "url": {
-      "path": "/_license",
       "paths": ["/_license"],
       "parts" : {}
     },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/license.get.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/license.get.json
index 07a570935a6b7..84d506783d695 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/license.get.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/license.get.json
@@ -1,9 +1,9 @@
 {
   "license.get": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/get-license.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_license",
       "paths": ["/_license"],
       "parts" : {
       },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/license.get_basic_status.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/license.get_basic_status.json
index cfb5608ac4063..dcf8e01837fed 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/license.get_basic_status.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/license.get_basic_status.json
@@ -1,9 +1,9 @@
 {
   "license.get_basic_status": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/get-basic-status.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_license/basic_status",
       "paths": ["/_license/basic_status"],
       "parts" : {
       },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/license.get_trial_status.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/license.get_trial_status.json
index daeb4913ad9eb..d8d3489d0cf54 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/license.get_trial_status.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/license.get_trial_status.json
@@ -1,9 +1,9 @@
 {
   "license.get_trial_status": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/get-trial-status.html",
+    "stability": "stable",
     "methods": ["GET"],
     "url": {
-      "path": "/_license/trial_status",
       "paths": ["/_license/trial_status"],
       "parts" : {
       },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/license.post.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/license.post.json
index 1bd78a1fb6af5..39cf43b03dc44 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/license.post.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/license.post.json
@@ -1,9 +1,9 @@
 {
   "license.post": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/update-license.html",
+    "stability": "stable",
     "methods": ["PUT", "POST"],
     "url": {
-      "path": "/_license",
       "paths": ["/_license"],
       "parts" : {
       },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/license.post_start_basic.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/license.post_start_basic.json
index 91da4de9ff0a2..b0da3724eac60 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/license.post_start_basic.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/license.post_start_basic.json
@@ -1,9 +1,9 @@
 {
   "license.post_start_basic": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/start-basic.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/_license/start_basic",
       "paths": ["/_license/start_basic"],
       "parts" : {
       },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/license.post_start_trial.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/license.post_start_trial.json
index 55a73df4946b5..89dbf30e7a3af 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/license.post_start_trial.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/license.post_start_trial.json
@@ -1,9 +1,9 @@
 {
   "license.post_start_trial": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/start-trial.html",
+    "stability": "stable",
     "methods": ["POST"],
     "url": {
-      "path": "/_license/start_trial",
       "paths": ["/_license/start_trial"],
       "parts" : {
       },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/migration.deprecations.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/migration.deprecations.json
index 7c7ad54f095a6..ffba21b5f5ae6 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/migration.deprecations.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/migration.deprecations.json
@@ -1,9 +1,9 @@
 {
   "migration.deprecations": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/migration-api-deprecation.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/{index}/_migration/deprecations",
       "paths": ["/_migration/deprecations", "/{index}/_migration/deprecations"],
       "parts": {
         "index": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.close_job.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.close_job.json
index ab32b89f9c264..f8aef2814a335 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.close_job.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.close_job.json
@@ -1,9 +1,9 @@
 {
   "ml.close_job": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-close-job.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/_close",
       "paths": [ "/_ml/anomaly_detectors/{job_id}/_close" ],
       "parts": {
         "job_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_calendar.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_calendar.json
index 0eb3a61cdf887..5c414f56a19a3 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_calendar.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_calendar.json
@@ -1,8 +1,8 @@
 {
   "ml.delete_calendar": {
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_ml/calendars/{calendar_id}",
       "paths": [ "/_ml/calendars/{calendar_id}" ],
       "parts": {
         "calendar_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_calendar_event.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_calendar_event.json
index 91b041f6c5141..38e49841c01d1 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_calendar_event.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_calendar_event.json
@@ -1,8 +1,8 @@
 {
   "ml.delete_calendar_event": {
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_ml/calendars/{calendar_id}/events/{event_id}",
       "paths": [ "/_ml/calendars/{calendar_id}/events/{event_id}" ],
       "parts": {
         "calendar_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_calendar_job.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_calendar_job.json
index 196eb78e9936d..8a78a850da413 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_calendar_job.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_calendar_job.json
@@ -1,8 +1,8 @@
 {
   "ml.delete_calendar_job": {
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_ml/calendars/{calendar_id}/jobs/{job_id}",
       "paths": [ "/_ml/calendars/{calendar_id}/jobs/{job_id}" ],
       "parts": {
         "calendar_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_data_frame_analytics.json
new file mode 100644
index 0000000000000..cf4d0ed4ec7f5
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_data_frame_analytics.json
@@ -0,0 +1,18 @@
+{
+  "ml.delete_data_frame_analytics": {
+    "stability": "experimental",
+    "methods": [ "DELETE" ],
+    "url": {
+      "path": "/_ml/data_frame/analytics/{id}",
+      "paths": [ "/_ml/data_frame/analytics/{id}" ],
+      "parts": {
+        "id": {
+          "type" : "string",
+          "required" : true,
+          "description" : "The ID of the data frame analytics to delete"
+        }
+      }
+    },
+    "body": null
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_datafeed.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_datafeed.json
index 039d6ee3273c3..9208c099a7324 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_datafeed.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_datafeed.json
@@ -1,9 +1,9 @@
 {
   "ml.delete_datafeed": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-datafeed.html",
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_ml/datafeeds/{datafeed_id}",
       "paths": [ "/_ml/datafeeds/{datafeed_id}" ],
       "parts": {
         "datafeed_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_expired_data.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_expired_data.json
index 2bb04bd286016..838d986ba4142 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_expired_data.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_expired_data.json
@@ -1,8 +1,8 @@
 {
   "ml.delete_expired_data": {
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_ml/_delete_expired_data",
       "paths": [ "/_ml/_delete_expired_data" ],
       "parts": {}
     },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_filter.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_filter.json
index c4bd51988b030..15a3dac18a98a 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_filter.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_filter.json
@@ -1,8 +1,8 @@
 {
   "ml.delete_filter": {
     "methods": [ "DELETE" ],
+    "stability": "stable",
     "url": {
-      "path": "/_ml/filters/{filter_id}",
       "paths": [ "/_ml/filters/{filter_id}" ],
       "parts": {
         "filter_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_forecast.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_forecast.json
index ef6ad3406e6c0..d048858c75dd6 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_forecast.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_forecast.json
@@ -1,9 +1,9 @@
 {
   "ml.delete_forecast": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-forecast.html",
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/_forecast/{forecast_id}",
       "paths": [
         "/_ml/anomaly_detectors/{job_id}/_forecast",
         "/_ml/anomaly_detectors/{job_id}/_forecast/{forecast_id}"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_job.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_job.json
index 550505a263525..2d1220324b91e 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_job.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_job.json
@@ -1,9 +1,9 @@
 {
   "ml.delete_job": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-job.html",
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}",
       "paths": [ "/_ml/anomaly_detectors/{job_id}" ],
       "parts": {
         "job_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_model_snapshot.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_model_snapshot.json
index d61c0ec46c5e5..5c4d2bfbcb8aa 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_model_snapshot.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_model_snapshot.json
@@ -1,9 +1,9 @@
 {
   "ml.delete_model_snapshot": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-snapshot.html",
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/model_snapshots/{snapshot_id}",
       "paths": [ "/_ml/anomaly_detectors/{job_id}/model_snapshots/{snapshot_id}" ],
       "parts": {
         "job_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.evaluate_data_frame.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.evaluate_data_frame.json
new file mode 100644
index 0000000000000..fb6e5ca5156cf
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.evaluate_data_frame.json
@@ -0,0 +1,15 @@
+{
+  "ml.evaluate_data_frame": {
+    "stability": "experimental",
+    "methods": [ "POST" ],
+    "url": {
+      "path": "/_ml/data_frame/_evaluate",
+      "paths": [ "/_ml/data_frame/_evaluate" ],
+      "parts": {}
+    },
+    "body": {
+      "description" : "The evaluation definition",
+      "required" : true
+    }
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.find_file_structure.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.find_file_structure.json
index 4e5550ae824a9..afab9575f7251 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.find_file_structure.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.find_file_structure.json
@@ -1,9 +1,9 @@
 {
   "ml.find_file_structure": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-find-file-structure.html",
+    "stability" : "experimental",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/find_file_structure",
       "paths": [ "/_ml/find_file_structure" ],
       "params": {
         "lines_to_sample": {
@@ -11,6 +11,11 @@
           "description": "How many lines of the file should be included in the analysis",
           "default": 1000
         },
+        "line_merge_size_limit": {
+          "type": "int",
+          "description": "Maximum number of characters permitted in a single message when lines are merged to create messages.",
+          "default": 10000
+        },
         "timeout": {
           "type": "time",
           "description": "Timeout after which the analysis will be aborted",
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.flush_job.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.flush_job.json
index 22a59928fc2bf..8cc83b7beb234 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.flush_job.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.flush_job.json
@@ -1,11 +1,9 @@
 {
   "ml.flush_job": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-flush-job.html",
-    "methods": [
-      "POST"
-    ],
+    "stability": "stable",
+    "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/_flush",
       "paths": [
         "/_ml/anomaly_detectors/{job_id}/_flush"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.forecast.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.forecast.json
index 050b80be4c9bd..18b808372e91d 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.forecast.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.forecast.json
@@ -1,8 +1,8 @@
 {
   "ml.forecast": {
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/_forecast",
       "paths": [ "/_ml/anomaly_detectors/{job_id}/_forecast" ],
       "parts": {
         "job_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_buckets.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_buckets.json
index a861eb1f7d59f..953d1c40c7eb5 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_buckets.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_buckets.json
@@ -1,9 +1,9 @@
 {
   "ml.get_buckets": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-bucket.html",
+    "stability": "stable",
     "methods": [ "GET", "POST" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/results/buckets/{timestamp}",
       "paths": [
         "/_ml/anomaly_detectors/{job_id}/results/buckets/{timestamp}",
         "/_ml/anomaly_detectors/{job_id}/results/buckets"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_calendar_events.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_calendar_events.json
index 2252db62618fb..5bc126db3dcf6 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_calendar_events.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_calendar_events.json
@@ -1,8 +1,8 @@
 {
   "ml.get_calendar_events": {
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_ml/calendars/{calendar_id}/events",
       "paths": [
         "/_ml/calendars/{calendar_id}/events"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_calendars.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_calendars.json
index de2e192d7e072..de8cd034a4065 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_calendars.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_calendars.json
@@ -1,8 +1,8 @@
 {
   "ml.get_calendars": {
+    "stability": "stable",
     "methods": [ "GET", "POST" ],
     "url": {
-      "path": "/_ml/calendars/{calendar_id}",
       "paths": [
         "/_ml/calendars",
         "/_ml/calendars/{calendar_id}"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_categories.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_categories.json
index a096ba07986cc..74d0b1dd97120 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_categories.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_categories.json
@@ -1,9 +1,9 @@
 {
   "ml.get_categories": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-category.html",
+    "stability": "stable",
     "methods": [ "GET", "POST" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/results/categories/{category_id}",
       "paths": [
         "/_ml/anomaly_detectors/{job_id}/results/categories/{category_id}",
         "/_ml/anomaly_detectors/{job_id}/results/categories/"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json
new file mode 100644
index 0000000000000..dfb8de1310d04
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json
@@ -0,0 +1,38 @@
+{
+  "ml.get_data_frame_analytics": {
+    "stability": "experimental",
+    "methods": [ "GET"],
+    "url": {
+      "path": "/_ml/data_frame/analytics/{id}",
+      "paths": [
+        "/_ml/data_frame/analytics/{id}",
+        "/_ml/data_frame/analytics"
+      ],
+      "parts": {
+        "id": {
+          "type": "string",
+          "description": "The ID of the data frame analytics to fetch"
+        }
+      },
+      "params": {
+        "allow_no_match": {
+          "type": "boolean",
+          "required": false,
+          "description": "Whether to ignore if a wildcard expression matches no data frame analytics. (This includes `_all` string or when no data frame analytics have been specified)",
+          "default": true
+        },
+        "from": {
+          "type": "int",
+          "description": "skips a number of analytics",
+          "default": 0
+        },
+        "size": {
+          "type": "int",
+          "description": "specifies a max number of analytics to get",
+          "default": 100
+        }
+      }
+    },
+    "body": null
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics_stats.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics_stats.json
new file mode 100644
index 0000000000000..4ae7b754403c5
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics_stats.json
@@ -0,0 +1,38 @@
+{
+  "ml.get_data_frame_analytics_stats": {
+    "stability": "experimental",
+    "methods": [ "GET"],
+    "url": {
+      "path": "/_ml/data_frame/analytics/{id}/_stats",
+      "paths": [
+        "/_ml/data_frame/analytics/_stats",
+        "/_ml/data_frame/analytics/{id}/_stats"
+      ],
+      "parts": {
+        "id": {
+          "type": "string",
+          "description": "The ID of the data frame analytics stats to fetch"
+        }
+      },
+      "params": {
+        "allow_no_match": {
+          "type": "boolean",
+          "required": false,
+          "description": "Whether to ignore if a wildcard expression matches no data frame analytics. (This includes `_all` string or when no data frame analytics have been specified)",
+          "default": true
+        },
+        "from": {
+          "type": "int",
+          "description": "skips a number of analytics",
+          "default": 0
+        },
+        "size": {
+          "type": "int",
+          "description": "specifies a max number of analytics to get",
+          "default": 100
+        }
+      }
+    },
+    "body": null
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeed_stats.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeed_stats.json
index 155701dbee5fa..d3ec06b88518c 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeed_stats.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeed_stats.json
@@ -1,9 +1,9 @@
 {
   "ml.get_datafeed_stats": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-datafeed-stats.html",
+    "stability": "stable",
     "methods": [ "GET"],
     "url": {
-      "path": "/_ml/datafeeds/{datafeed_id}/_stats",
       "paths": [
         "/_ml/datafeeds/{datafeed_id}/_stats",
         "/_ml/datafeeds/_stats"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json
index c911079cd2217..befcd94d93059 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json
@@ -1,9 +1,9 @@
 {
   "ml.get_datafeeds": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-datafeed.html",
+    "stability": "stable",
     "methods": [ "GET"],
     "url": {
-      "path": "/_ml/datafeeds/{datafeed_id}",
       "paths": [
         "/_ml/datafeeds/{datafeed_id}",
         "/_ml/datafeeds"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_filters.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_filters.json
index c544b6dd37a34..f788776275cb3 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_filters.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_filters.json
@@ -1,8 +1,8 @@
 {
   "ml.get_filters": {
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_ml/filters/{filter_id}",
       "paths": [ "/_ml/filters", "/_ml/filters/{filter_id}" ],
       "parts": {
         "filter_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_influencers.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_influencers.json
index adb8206dbf03c..987551fa52a61 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_influencers.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_influencers.json
@@ -1,9 +1,9 @@
 {
   "ml.get_influencers": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-influencer.html",
+    "stability": "stable",
     "methods": [ "GET", "POST" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/results/influencers",
       "paths": [ "/_ml/anomaly_detectors/{job_id}/results/influencers" ],
       "parts": {
         "job_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_job_stats.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_job_stats.json
index 66b7c836df513..8a29b68eca29f 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_job_stats.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_job_stats.json
@@ -1,9 +1,9 @@
 {
   "ml.get_job_stats": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-job-stats.html",
+    "stability": "stable",
     "methods": [ "GET"],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/_stats",
       "paths": [
         "/_ml/anomaly_detectors/_stats",
         "/_ml/anomaly_detectors/{job_id}/_stats"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json
index 6777b9ffb7747..c57d977898ef7 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json
@@ -1,9 +1,9 @@
 {
   "ml.get_jobs": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-job.html",
+    "stability": "stable",
     "methods": [ "GET"],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}",
       "paths": [
         "/_ml/anomaly_detectors/{job_id}",
         "/_ml/anomaly_detectors"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_model_snapshots.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_model_snapshots.json
index 04a6eba021494..42dca88ea2107 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_model_snapshots.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_model_snapshots.json
@@ -1,9 +1,9 @@
 {
   "ml.get_model_snapshots": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-snapshot.html",
+    "stability": "stable",
     "methods": [ "GET", "POST" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/model_snapshots/{snapshot_id}",
       "paths": [
         "/_ml/anomaly_detectors/{job_id}/model_snapshots/{snapshot_id}",
         "/_ml/anomaly_detectors/{job_id}/model_snapshots"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_overall_buckets.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_overall_buckets.json
index 679aa67d3ccd6..c0d9f5202535a 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_overall_buckets.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_overall_buckets.json
@@ -1,9 +1,9 @@
 {
   "ml.get_overall_buckets": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-overall-buckets.html",
+    "stability": "stable",
     "methods": [ "GET", "POST" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/results/overall_buckets",
       "paths": [
         "/_ml/anomaly_detectors/{job_id}/results/overall_buckets"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_records.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_records.json
index 4831629b90c6f..51316c63a8848 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_records.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_records.json
@@ -1,9 +1,9 @@
 {
   "ml.get_records": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-record.html",
+    "stability": "stable",
     "methods": ["GET", "POST"],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/results/records",
       "paths": [
         "/_ml/anomaly_detectors/{job_id}/results/records"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.info.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.info.json
index 5afbaffd1fcd6..2ee848a7f4323 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.info.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.info.json
@@ -1,8 +1,8 @@
 {
   "ml.info": {
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_ml/info",
       "paths": [ "/_ml/info" ],
       "parts": {},
       "body": null
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.open_job.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.open_job.json
index 331f71132b927..1bfcdeaefc338 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.open_job.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.open_job.json
@@ -1,9 +1,9 @@
 {
   "ml.open_job": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-open-job.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/_open",
       "paths": [ "/_ml/anomaly_detectors/{job_id}/_open" ],
       "parts": {
         "job_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.post_calendar_events.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.post_calendar_events.json
index 6775317ec6aec..e3a857fca6b3e 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.post_calendar_events.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.post_calendar_events.json
@@ -1,8 +1,8 @@
 {
   "ml.post_calendar_events": {
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/calendars/{calendar_id}/events",
       "paths": [ "/_ml/calendars/{calendar_id}/events" ],
       "parts": {
         "calendar_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.post_data.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.post_data.json
index 9acf4883e9344..dfc00fc67a055 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.post_data.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.post_data.json
@@ -1,9 +1,9 @@
 {
   "ml.post_data": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-post-data.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/_data",
       "paths": [ "/_ml/anomaly_detectors/{job_id}/_data" ],
       "parts": {
         "job_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.preview_datafeed.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.preview_datafeed.json
index 1d48948fb7c02..1afabb5c0dc9f 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.preview_datafeed.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.preview_datafeed.json
@@ -1,9 +1,9 @@
 {
   "ml.preview_datafeed": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-preview-datafeed.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_ml/datafeeds/{datafeed_id}/_preview",
       "paths": [ "/_ml/datafeeds/{datafeed_id}/_preview" ],
       "parts": {
         "datafeed_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_calendar.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_calendar.json
index 4a5473cd49699..b03a3011123ad 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_calendar.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_calendar.json
@@ -1,8 +1,8 @@
 {
   "ml.put_calendar": {
+    "stability": "stable",
     "methods": [ "PUT" ],
     "url": {
-      "path": "/_ml/calendars/{calendar_id}",
       "paths": [ "/_ml/calendars/{calendar_id}" ],
       "parts": {
         "calendar_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_calendar_job.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_calendar_job.json
index e17fdacd167f2..f69733accc8b1 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_calendar_job.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_calendar_job.json
@@ -1,8 +1,8 @@
 {
   "ml.put_calendar_job": {
+    "stability": "stable",
     "methods": [ "PUT" ],
     "url": {
-      "path": "/_ml/calendars/{calendar_id}/jobs/{job_id}",
       "paths": [ "/_ml/calendars/{calendar_id}/jobs/{job_id}" ],
       "parts": {
         "calendar_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_data_frame_analytics.json
new file mode 100644
index 0000000000000..5cee69e3ab951
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_data_frame_analytics.json
@@ -0,0 +1,21 @@
+{
+  "ml.put_data_frame_analytics": {
+    "stability": "experimental",
+    "methods": [ "PUT" ],
+    "url": {
+      "path": "/_ml/data_frame/analytics/{id}",
+      "paths": [ "/_ml/data_frame/analytics/{id}" ],
+      "parts": {
+        "id": {
+          "type": "string",
+          "required": true,
+          "description": "The ID of the data frame analytics to create"
+        }
+      }
+    },
+    "body": {
+      "description" : "The data frame analytics configuration",
+      "required" : true
+    }
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_datafeed.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_datafeed.json
index 8545d0727d7d7..8523584d604f3 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_datafeed.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_datafeed.json
@@ -1,9 +1,9 @@
 {
   "ml.put_datafeed": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-put-datafeed.html",
+    "stability": "stable",
     "methods": [ "PUT" ],
     "url": {
-      "path": "/_ml/datafeeds/{datafeed_id}",
       "paths": [ "/_ml/datafeeds/{datafeed_id}" ],
       "parts": {
         "datafeed_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_filter.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_filter.json
index 3ef67502b6464..0f5e1849eef33 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_filter.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_filter.json
@@ -1,8 +1,8 @@
 {
   "ml.put_filter": {
+    "stability": "stable",
     "methods": [ "PUT" ],
     "url": {
-      "path": "/_ml/filters/{filter_id}",
       "paths": [ "/_ml/filters/{filter_id}" ],
       "parts": {
         "filter_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_job.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_job.json
index 01bb28290acc6..c0296e577c116 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_job.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_job.json
@@ -1,9 +1,9 @@
 {
   "ml.put_job": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-put-job.html",
+    "stability": "stable",
     "methods": [ "PUT" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}",
       "paths": [ "/_ml/anomaly_detectors/{job_id}" ],
       "parts": {
         "job_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.revert_model_snapshot.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.revert_model_snapshot.json
index 3c71279c23bf0..d9ca8902d276a 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.revert_model_snapshot.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.revert_model_snapshot.json
@@ -1,9 +1,9 @@
 {
   "ml.revert_model_snapshot": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-revert-snapshot.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/model_snapshots/{snapshot_id}/_revert",
       "paths": [ "/_ml/anomaly_detectors/{job_id}/model_snapshots/{snapshot_id}/_revert" ],
       "parts": {
         "job_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.set_upgrade_mode.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.set_upgrade_mode.json
index 5406d2d06eceb..dc0f167798b61 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.set_upgrade_mode.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.set_upgrade_mode.json
@@ -1,9 +1,9 @@
 {
   "ml.set_upgrade_mode": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-set-upgrade-mode.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/set_upgrade_mode",
       "paths": [ "/_ml/set_upgrade_mode" ],
       "params": {
         "enabled": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.start_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.start_data_frame_analytics.json
new file mode 100644
index 0000000000000..3436623c61b55
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.start_data_frame_analytics.json
@@ -0,0 +1,27 @@
+{
+  "ml.start_data_frame_analytics": {
+    "stability": "experimental",
+    "methods": [ "POST" ],
+    "url": {
+      "path": "/_ml/data_frame/analytics/{id}/_start",
+      "paths": [ "/_ml/data_frame/analytics/{id}/_start" ],
+      "parts": {
+        "id": {
+          "type": "string",
+          "required": true,
+          "description": "The ID of the data frame analytics to start"
+        }
+      },
+      "params": {
+        "timeout": {
+          "type": "time",
+          "required": false,
+          "description": "Controls the time to wait until the task has started. Defaults to 20 seconds"
+        }
+      }
+    },
+    "body": {
+      "description": "The start data frame analytics parameters"
+    }
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.start_datafeed.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.start_datafeed.json
index ce3fd9cdf4b80..545f43085e36f 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.start_datafeed.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.start_datafeed.json
@@ -1,9 +1,9 @@
 {
   "ml.start_datafeed": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-start-datafeed.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/datafeeds/{datafeed_id}/_start",
       "paths": [ "/_ml/datafeeds/{datafeed_id}/_start" ],
       "parts": {
         "datafeed_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.stop_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.stop_data_frame_analytics.json
new file mode 100644
index 0000000000000..ee1a9c85be424
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.stop_data_frame_analytics.json
@@ -0,0 +1,37 @@
+{
+  "ml.stop_data_frame_analytics": {
+    "stability": "experimental",
+    "methods": [ "POST" ],
+    "url": {
+      "path": "/_ml/data_frame/analytics/{id}/_stop",
+      "paths": [ "/_ml/data_frame/analytics/{id}/_stop" ],
+      "parts": {
+        "id": {
+          "type": "string",
+          "required": true,
+          "description": "The ID of the data frame analytics to stop"
+        }
+      },
+      "params": {
+        "allow_no_match": {
+          "type": "boolean",
+          "required": false,
+          "description": "Whether to ignore if a wildcard expression matches no data frame analytics. (This includes `_all` string or when no data frame analytics have been specified)"
+        },
+        "force": {
+          "type": "boolean",
+          "required": false,
+          "description": "True if the data frame analytics should be forcefully stopped"
+        },
+        "timeout": {
+          "type": "time",
+          "required": false,
+          "description": "Controls the time to wait until the task has stopped. Defaults to 20 seconds"
+        }
+      }
+    },
+    "body": {
+      "description": "The stop data frame analytics parameters"
+    }
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.stop_datafeed.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.stop_datafeed.json
index 96d28abc15118..5774c08be588d 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.stop_datafeed.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.stop_datafeed.json
@@ -1,11 +1,9 @@
 {
   "ml.stop_datafeed": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-stop-datafeed.html",
-    "methods": [
-      "POST"
-    ],
+    "stability": "stable",
+    "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/datafeeds/{datafeed_id}/_stop",
       "paths": [
         "/_ml/datafeeds/{datafeed_id}/_stop"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_datafeed.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_datafeed.json
index aebeade84e20e..2585a7096ddd0 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_datafeed.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_datafeed.json
@@ -1,9 +1,9 @@
 {
   "ml.update_datafeed": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-update-datafeed.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/datafeeds/{datafeed_id}/_update",
       "paths": [ "/_ml/datafeeds/{datafeed_id}/_update" ],
       "parts": {
         "datafeed_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_filter.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_filter.json
index d55fb80755a31..3576b1f924cd8 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_filter.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_filter.json
@@ -1,8 +1,8 @@
 {
   "ml.update_filter": {
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/filters/{filter_id}/_update",
       "paths": [ "/_ml/filters/{filter_id}/_update" ],
       "parts": {
         "filter_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_job.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_job.json
index f2a9a29a416b4..a153547a5a84e 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_job.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_job.json
@@ -1,9 +1,9 @@
 {
   "ml.update_job": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-update-job.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/_update",
       "paths": [ "/_ml/anomaly_detectors/{job_id}/_update" ],
       "parts": {
         "job_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_model_snapshot.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_model_snapshot.json
index ba57e18b0aad6..8486aee217b3f 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_model_snapshot.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_model_snapshot.json
@@ -1,9 +1,9 @@
 {
   "ml.update_model_snapshot": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-update-snapshot.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/{job_id}/model_snapshots/{snapshot_id}/_update",
       "paths": [ "/_ml/anomaly_detectors/{job_id}/model_snapshots/{snapshot_id}/_update" ],
       "parts": {
         "job_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.validate.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.validate.json
index 140057db9fcfd..9a2ff350c908d 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.validate.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.validate.json
@@ -1,8 +1,8 @@
 {
   "ml.validate": {
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/_validate",
       "paths": [ "/_ml/anomaly_detectors/_validate" ],
       "params": {}
     },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.validate_detector.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.validate_detector.json
index 33e91b6832b9f..c28817558e5cd 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.validate_detector.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.validate_detector.json
@@ -1,8 +1,8 @@
 {
   "ml.validate_detector": {
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_ml/anomaly_detectors/_validate/detector",
       "paths": [ "/_ml/anomaly_detectors/_validate/detector" ],
       "params": {}
     },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/monitoring.bulk.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/monitoring.bulk.json
index 78a6017cba2e6..f84cf6e7bb4f9 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/monitoring.bulk.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/monitoring.bulk.json
@@ -1,9 +1,9 @@
 {
   "monitoring.bulk": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/es-monitoring.html",
+    "stability" : "experimental",
     "methods": ["POST", "PUT"],
     "url": {
-      "path": "/_monitoring/bulk",
       "paths": ["/_monitoring/bulk"],
       "deprecated_paths" : [
         {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.delete_job.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.delete_job.json
index e03cd2ae977b9..ce083b7a0b1f2 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.delete_job.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.delete_job.json
@@ -1,9 +1,9 @@
 {
   "rollup.delete_job": {
     "documentation": "",
+    "stability" : "experimental",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_rollup/job/{id}",
       "paths": [ "/_rollup/job/{id}" ],
       "parts": {
         "id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.get_jobs.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.get_jobs.json
index aa5d56e590910..9c72834a0239d 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.get_jobs.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.get_jobs.json
@@ -1,9 +1,9 @@
 {
   "rollup.get_jobs": {
     "documentation": "",
+    "stability" : "experimental",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_rollup/job/{id}",
       "paths": [ "/_rollup/job/{id}", "/_rollup/job/" ],
       "parts": {
         "id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.get_rollup_caps.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.get_rollup_caps.json
index 0fd8aa3168222..0bf94ca336617 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.get_rollup_caps.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.get_rollup_caps.json
@@ -1,9 +1,9 @@
 {
   "rollup.get_rollup_caps": {
     "documentation": "",
+    "stability" : "experimental",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_rollup/data/{id}",
       "paths": [ "/_rollup/data/{id}", "/_rollup/data/" ],
       "parts": {
         "id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.get_rollup_index_caps.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.get_rollup_index_caps.json
index c446f29e7591b..b385cd4341cf5 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.get_rollup_index_caps.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.get_rollup_index_caps.json
@@ -1,9 +1,9 @@
 {
   "rollup.get_rollup_index_caps": {
     "documentation": "",
+    "stability" : "experimental",
     "methods": [ "GET" ],
     "url": {
-      "path": "/{index}/_rollup/data",
       "paths": [ "/{index}/_rollup/data" ],
       "parts": {
         "index": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.put_job.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.put_job.json
index ca33affd7d8a6..7a792617951f8 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.put_job.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.put_job.json
@@ -1,9 +1,9 @@
 {
   "rollup.put_job": {
     "documentation": "",
+    "stability" : "experimental",
     "methods": [ "PUT" ],
     "url": {
-      "path": "/_rollup/job/{id}",
       "paths": [ "/_rollup/job/{id}" ],
       "parts": {
         "id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.rollup_search.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.rollup_search.json
index 0eeb7b29256cf..3de6cbb08de7b 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.rollup_search.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.rollup_search.json
@@ -1,9 +1,9 @@
 {
   "rollup.rollup_search": {
     "documentation": "",
+    "stability" : "experimental",
     "methods": [ "GET", "POST" ],
     "url": {
-      "path": "/{index}/_rollup_search",
       "paths": [ "{index}/_rollup_search"],
       "deprecated_paths" : [
         {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.start_job.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.start_job.json
index 8ee505b195b22..6d77b6c223095 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.start_job.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.start_job.json
@@ -1,9 +1,9 @@
 {
   "rollup.start_job": {
     "documentation": "",
+    "stability" : "experimental",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_rollup/job/{id}/_start",
       "paths": [ "/_rollup/job/{id}/_start" ],
       "parts": {
         "id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.stop_job.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.stop_job.json
index 152b72945800d..57ed0d7778e56 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.stop_job.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/rollup.stop_job.json
@@ -1,9 +1,9 @@
 {
   "rollup.stop_job": {
     "documentation": "",
+    "stability" : "experimental",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_rollup/job/{id}/_stop",
       "paths": [ "/_rollup/job/{id}/_stop" ],
       "parts": {
         "id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.authenticate.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.authenticate.json
index 394ad0d8dbe45..196f27d3e4480 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.authenticate.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.authenticate.json
@@ -1,9 +1,9 @@
 {
   "security.authenticate": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-authenticate.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_security/_authenticate",
       "paths": [
         "/_security/_authenticate"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.change_password.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.change_password.json
index 9e94102fc23e9..073277db9781c 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.change_password.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.change_password.json
@@ -1,9 +1,9 @@
 {
   "security.change_password": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-change-password.html",
+    "stability": "stable",
     "methods": [ "PUT", "POST" ],
     "url": {
-      "path": "/_security/user/{username}/_password",
       "paths": [
         "/_security/user/{username}/_password",
         "/_security/user/_password"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.clear_cached_realms.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.clear_cached_realms.json
index 9e986d615a257..2e6e5198f7b2f 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.clear_cached_realms.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.clear_cached_realms.json
@@ -1,9 +1,9 @@
 {
   "security.clear_cached_realms": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-clear-cache.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_security/realm/{realms}/_clear_cache",
       "paths": [
         "/_security/realm/{realms}/_clear_cache"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.clear_cached_roles.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.clear_cached_roles.json
index 1c39ea9ade349..f446f55324049 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.clear_cached_roles.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.clear_cached_roles.json
@@ -1,9 +1,9 @@
 {
   "security.clear_cached_roles": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-clear-role-cache.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_security/role/{name}/_clear_cache",
       "paths": [
         "/_security/role/{name}/_clear_cache"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.create_api_key.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.create_api_key.json
index 30b5b30a52b61..271232f726a06 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.create_api_key.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.create_api_key.json
@@ -1,9 +1,9 @@
 {
   "security.create_api_key": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html",
+    "stability": "stable",
     "methods": [ "PUT", "POST" ],
     "url": {
-      "path": "/_security/api_key",
       "paths": [ "/_security/api_key" ],
       "parts": {},
       "params": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.delete_privileges.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.delete_privileges.json
index 13231572ab79a..eecdd8a524248 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.delete_privileges.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.delete_privileges.json
@@ -1,9 +1,9 @@
 {
   "security.delete_privileges": {
     "documentation": "TODO",
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_security/privilege/{application}/{name}",
       "paths": [
         "/_security/privilege/{application}/{name}"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.delete_role.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.delete_role.json
index 8e4a17988f536..d36a2b7b4ce50 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.delete_role.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.delete_role.json
@@ -1,9 +1,9 @@
 {
   "security.delete_role": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-delete-role.html",
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_security/role/{name}",
       "paths": [
         "/_security/role/{name}"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.delete_role_mapping.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.delete_role_mapping.json
index 5aff743b45bdd..3c8a43ee3efe3 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.delete_role_mapping.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.delete_role_mapping.json
@@ -1,9 +1,9 @@
 {
   "security.delete_role_mapping": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-delete-role-mapping.html",
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_security/role_mapping/{name}",
       "paths": [
         "/_security/role_mapping/{name}"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.delete_user.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.delete_user.json
index 175d9925f00a6..e6697b43f837d 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.delete_user.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.delete_user.json
@@ -1,9 +1,9 @@
 {
   "security.delete_user": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-delete-user.html",
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_security/user/{username}",
       "paths": [
         "/_security/user/{username}"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.disable_user.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.disable_user.json
index 3aa50467fb546..d3530fa8e0142 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.disable_user.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.disable_user.json
@@ -1,9 +1,9 @@
 {
   "security.disable_user": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-disable-user.html",
+    "stability": "stable",
     "methods": [ "PUT", "POST" ],
     "url": {
-      "path": "/_security/user/{username}/_disable",
       "paths": [
         "/_security/user/{username}/_disable"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.enable_user.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.enable_user.json
index 950a2b54939cd..1f002662245ec 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.enable_user.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.enable_user.json
@@ -1,9 +1,9 @@
 {
   "security.enable_user": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-enable-user.html",
+    "stability": "stable",
     "methods": [ "PUT", "POST" ],
     "url": {
-      "path": "/_security/user/{username}/_enable",
       "paths": [
         "/_security/user/{username}/_enable"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_api_key.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_api_key.json
index f534d2a220e84..9f3eca4fae84b 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_api_key.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_api_key.json
@@ -1,9 +1,9 @@
 {
   "security.get_api_key": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-api-key.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_security/api_key",
       "paths": [ "/_security/api_key" ],
       "parts": {},
       "params": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_builtin_privileges.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_builtin_privileges.json
new file mode 100644
index 0000000000000..3312639a83c04
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_builtin_privileges.json
@@ -0,0 +1,15 @@
+{
+  "security.get_builtin_privileges": {
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-builtin-privileges.html",
+    "stability": "stable",
+    "methods": [ "GET" ],
+    "url": {
+      "paths": [
+        "/_security/privilege/_builtin"
+      ],
+      "parts": {},
+      "params": {}
+    },
+    "body": null
+  }
+}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_privileges.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_privileges.json
index 2cf10a872d519..13367024651fc 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_privileges.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_privileges.json
@@ -1,9 +1,9 @@
 {
   "security.get_privileges": {
-    "documentation": "TODO",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-privileges.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_security/privilege/{application}/{name}",
       "paths": [
         "/_security/privilege",
         "/_security/privilege/{application}",
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_role.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_role.json
index 412127ce157e4..73e0584ba99b3 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_role.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_role.json
@@ -1,9 +1,9 @@
 {
   "security.get_role": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-role.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_security/role/{name}",
       "paths": [
         "/_security/role/{name}",
         "/_security/role"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_role_mapping.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_role_mapping.json
index 169ef451b3cb8..da39ec0eaea80 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_role_mapping.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_role_mapping.json
@@ -1,9 +1,9 @@
 {
   "security.get_role_mapping": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-role-mapping.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_security/role_mapping/{name}",
       "paths": [
         "/_security/role_mapping/{name}",
         "/_security/role_mapping"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_token.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_token.json
index 9ecc8091797b8..3204de64dc8b0 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_token.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_token.json
@@ -1,9 +1,9 @@
 {
   "security.get_token": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-token.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_security/oauth2/token",
       "paths": [
         "/_security/oauth2/token"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_user.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_user.json
index ed6e7069dca31..65ef3c745b02e 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_user.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_user.json
@@ -1,9 +1,9 @@
 {
   "security.get_user": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-user.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_security/user/{username}",
       "paths": [
         "/_security/user/{username}",
         "/_security/user"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_user_privileges.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_user_privileges.json
index b60298f89e017..9d998513511ad 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_user_privileges.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_user_privileges.json
@@ -1,9 +1,9 @@
 {
   "security.get_user_privileges": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-privileges.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_security/user/_privileges",
       "paths": [
         "/_security/user/_privileges"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.has_privileges.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.has_privileges.json
index c388f766c9829..8b719f67f2b76 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.has_privileges.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.has_privileges.json
@@ -1,9 +1,9 @@
 {
   "security.has_privileges": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-has-privileges.html",
+    "stability": "stable",
     "methods": [ "GET", "POST" ],
     "url": {
-      "path": "/_security/user/_has_privileges",
       "paths": [
         "/_security/user/_has_privileges",
         "/_security/user/{user}/_has_privileges"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.invalidate_api_key.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.invalidate_api_key.json
index fa0639704aa9a..0e6fdd819ceb9 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.invalidate_api_key.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.invalidate_api_key.json
@@ -1,9 +1,9 @@
 {
   "security.invalidate_api_key": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-invalidate-api-key.html",
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_security/api_key",
       "paths": [ "/_security/api_key" ],
       "parts": {}
     },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.invalidate_token.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.invalidate_token.json
index 7938825e5b12e..a184314de6657 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.invalidate_token.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.invalidate_token.json
@@ -1,9 +1,9 @@
 {
   "security.invalidate_token": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-invalidate-token.html",
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_security/oauth2/token",
       "paths": [
         "/_security/oauth2/token"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.put_privileges.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.put_privileges.json
index 20b8d8418ab1f..4f25d91b616a5 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.put_privileges.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.put_privileges.json
@@ -1,9 +1,9 @@
 {
   "security.put_privileges": {
     "documentation": "TODO",
+    "stability": "stable",
     "methods": [ "PUT", "POST" ],
     "url": {
-      "path": "/_security/privilege/",
       "paths": [
         "/_security/privilege/"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.put_role.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.put_role.json
index 8539d198c70e7..6be12ec41fe4f 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.put_role.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.put_role.json
@@ -1,9 +1,9 @@
 {
   "security.put_role": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-put-role.html",
+    "stability": "stable",
     "methods": [ "PUT", "POST" ],
     "url": {
-      "path": "/_security/role/{name}",
       "paths": [
         "/_security/role/{name}"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.put_role_mapping.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.put_role_mapping.json
index d65cf8f835833..3ae8897f4d78d 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.put_role_mapping.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.put_role_mapping.json
@@ -1,9 +1,9 @@
 {
   "security.put_role_mapping": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-put-role-mapping.html",
+    "stability": "stable",
     "methods": [ "PUT", "POST" ],
     "url": {
-      "path": "/_security/role_mapping/{name}",
       "paths": [
         "/_security/role_mapping/{name}"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.put_user.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.put_user.json
index a1a77c5bb4e9f..d71cdb476abc7 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/security.put_user.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/security.put_user.json
@@ -1,9 +1,9 @@
 {
   "security.put_user": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-put-user.html",
+    "stability": "stable",
     "methods": [ "PUT", "POST" ],
     "url": {
-      "path": "/_security/user/{username}",
       "paths": [
         "/_security/user/{username}"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/sql.clear_cursor.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/sql.clear_cursor.json
index ec84f9543bfe0..cca53053513b5 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/sql.clear_cursor.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/sql.clear_cursor.json
@@ -1,9 +1,9 @@
 {
   "sql.clear_cursor": {
     "documentation": "Clear SQL cursor",
+    "stability" : "stable",
     "methods": [ "POST"],
     "url": {
-      "path": "/_sql/close",
       "paths": [ "/_sql/close" ],
       "parts": {}
     },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/sql.query.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/sql.query.json
index c12a876e8cd32..7f92622e1182f 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/sql.query.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/sql.query.json
@@ -1,9 +1,9 @@
 {
     "sql.query": {
       "documentation": "Execute SQL",
+      "stability" : "stable",
       "methods": [ "POST", "GET" ],
       "url": {
-        "path": "/_sql",
         "paths": [ "/_sql" ],
         "parts": {},
         "params": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/sql.translate.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/sql.translate.json
index 2200a61be66b2..5258d5b87641b 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/sql.translate.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/sql.translate.json
@@ -1,9 +1,9 @@
 {
     "sql.translate": {
       "documentation": "Translate SQL into Elasticsearch queries",
+      "stability" : "stable",
       "methods": [ "POST", "GET" ],
       "url": {
-        "path": "/_sql/translate",
         "paths": [ "/_sql/translate" ],
         "parts": {},
         "params": {}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ssl.certificates.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ssl.certificates.json
index 10715cd5380fe..f41e9b737dd9b 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ssl.certificates.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ssl.certificates.json
@@ -1,9 +1,9 @@
 {
   "ssl.certificates": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-ssl.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_ssl/certificates",
       "paths": [
         "/_ssl/certificates"
       ],
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.ack_watch.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.ack_watch.json
index 4920c986a042f..0ee2aad79c04d 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.ack_watch.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.ack_watch.json
@@ -1,9 +1,9 @@
 {
   "watcher.ack_watch": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-ack-watch.html",
+    "stability": "stable",
     "methods": [ "PUT", "POST" ],
     "url": {
-      "path": "/_watcher/watch/{watch_id}/_ack",
       "paths": [ "/_watcher/watch/{watch_id}/_ack", "/_watcher/watch/{watch_id}/_ack/{action_id}"],
       "parts": {
         "watch_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.activate_watch.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.activate_watch.json
index 49fb169dede77..25f13f5007b83 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.activate_watch.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.activate_watch.json
@@ -1,9 +1,9 @@
 {
   "watcher.activate_watch": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-activate-watch.html",
+    "stability": "stable",
     "methods": [ "PUT", "POST" ],
     "url": {
-      "path": "/_watcher/watch/{watch_id}/_activate",
       "paths": [ "/_watcher/watch/{watch_id}/_activate" ],
       "parts": {
         "watch_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.deactivate_watch.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.deactivate_watch.json
index ddc68b439395e..9325f03b8c2d8 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.deactivate_watch.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.deactivate_watch.json
@@ -1,9 +1,9 @@
 {
   "watcher.deactivate_watch": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-deactivate-watch.html",
+    "stability": "stable",
     "methods": [ "PUT", "POST" ],
     "url": {
-      "path": "/_watcher/watch/{watch_id}/_deactivate",
       "paths": [ "/_watcher/watch/{watch_id}/_deactivate" ],
       "parts": {
         "watch_id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.delete_watch.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.delete_watch.json
index cdf61ad52023f..87f4500f165c7 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.delete_watch.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.delete_watch.json
@@ -1,10 +1,9 @@
 {
   "watcher.delete_watch": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-delete-watch.html",
-
+    "stability": "stable",
     "methods": [ "DELETE" ],
     "url": {
-      "path": "/_watcher/watch/{id}",
       "paths": [ "/_watcher/watch/{id}" ],
       "parts": {
         "id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.execute_watch.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.execute_watch.json
index 6db8f3ae115f5..864427b93ae76 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.execute_watch.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.execute_watch.json
@@ -1,9 +1,9 @@
 {
   "watcher.execute_watch": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-execute-watch.html",
+    "stability": "stable",
     "methods": [ "PUT", "POST" ],
     "url": {
-      "path": "/_watcher/watch/{id}/_execute",
       "paths": [ "/_watcher/watch/{id}/_execute", "/_watcher/watch/_execute" ],
       "parts": {
         "id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.get_watch.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.get_watch.json
index 81f21b4b0c1e5..593ac31fe759f 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.get_watch.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.get_watch.json
@@ -1,9 +1,9 @@
 {
   "watcher.get_watch": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-get-watch.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_watcher/watch/{id}",
       "paths": [ "/_watcher/watch/{id}" ],
       "parts": {
         "id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.put_watch.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.put_watch.json
index 24f020a7b90b4..9e9c0e7a01ddc 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.put_watch.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.put_watch.json
@@ -1,9 +1,9 @@
 {
   "watcher.put_watch": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-put-watch.html",
+    "stability": "stable",
     "methods": [ "PUT", "POST" ],
     "url": {
-      "path": "/_watcher/watch/{id}",
       "paths": [ "/_watcher/watch/{id}" ],
       "parts": {
         "id": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.start.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.start.json
index 649b21c7db3f1..77e18e81fb3a8 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.start.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.start.json
@@ -1,9 +1,9 @@
 {
   "watcher.start": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-start.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_watcher/_start",
       "paths": [ "/_watcher/_start" ],
       "parts": {
       },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.stats.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.stats.json
index 01241848ab9f3..44c75dff017ca 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.stats.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.stats.json
@@ -1,9 +1,9 @@
 {
   "watcher.stats": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-stats.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_watcher/stats",
       "paths": [ "/_watcher/stats", "/_watcher/stats/{metric}" ],
       "parts": {
         "metric": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.stop.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.stop.json
index 4deee79436e2d..c036782947b4b 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.stop.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/watcher.stop.json
@@ -1,9 +1,9 @@
 {
   "watcher.stop": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-stop.html",
+    "stability": "stable",
     "methods": [ "POST" ],
     "url": {
-      "path": "/_watcher/_stop",
       "paths": [ "/_watcher/_stop" ],
       "parts": {
       },
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.info.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.info.json
index 9913bc3fba6df..75d5bf276e4aa 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.info.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.info.json
@@ -1,9 +1,9 @@
 {
   "xpack.info": {
     "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/info-api.html",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_xpack",
       "paths": [ "/_xpack" ],
       "parts": {},
       "params": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.usage.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.usage.json
index 8476c1f06d61b..9923093dcecf0 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.usage.json
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.usage.json
@@ -1,9 +1,9 @@
 {
   "xpack.usage": {
     "documentation": "Retrieve information about xpack features usage",
+    "stability": "stable",
     "methods": [ "GET" ],
     "url": {
-      "path": "/_xpack/usage",
       "paths": [ "/_xpack/usage" ],
       "parts": {},
       "params": {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/api_key/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/api_key/10_basic.yml
index 77c655207ba81..8ac56b605cbb5 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/api_key/10_basic.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/api_key/10_basic.yml
@@ -12,7 +12,7 @@ setup:
         name: "admin_role"
         body:  >
             {
-              "cluster": ["all"],
+              "cluster": ["manage_api_key"],
               "indices": [
                 {
                   "names": "*",
@@ -166,6 +166,8 @@ teardown:
   - set: { name: api_key_name }
 
   - do:
+      headers:
+        Authorization: "Basic YXBpX2tleV91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" # api_key_user
       security.get_api_key:
         id: "$api_key_id"
   - match: { "api_keys.0.id": "$api_key_id" }
@@ -198,6 +200,8 @@ teardown:
   - transform_and_set: { login_creds: "#base64EncodeCredentials(id,api_key)" }
 
   - do:
+      headers:
+        Authorization: "Basic YXBpX2tleV91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" # api_key_user
       security.invalidate_api_key:
         body:  >
             {
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/preview_transforms.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/preview_transforms.yml
index 5e58048b3bf0f..dbcacb063791f 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/preview_transforms.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/preview_transforms.yml
@@ -75,7 +75,7 @@ setup:
             "pivot": {
               "group_by": {
                 "airline": {"terms": {"field": "airline"}},
-                "by-hour": {"date_histogram": {"fixed_interval": "1h", "field": "time", "format": "yyyy-MM-dd HH"}}},
+                "by-hour": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
               "aggs": {
                 "avg_response": {"avg": {"field": "responsetime"}},
                 "time.max": {"max": {"field": "time"}},
@@ -84,20 +84,71 @@ setup:
             }
           }
   - match: { preview.0.airline: foo }
-  - match: { preview.0.by-hour: "2017-02-18 00" }
+  - match: { preview.0.by-hour: 1487376000000 }
   - match: { preview.0.avg_response: 1.0 }
   - match: { preview.0.time.max: "2017-02-18T00:30:00.000Z" }
   - match: { preview.0.time.min: "2017-02-18T00:00:00.000Z" }
   - match: { preview.1.airline: bar }
-  - match: { preview.1.by-hour: "2017-02-18 01" }
+  - match: { preview.1.by-hour: 1487379600000 }
   - match: { preview.1.avg_response: 42.0 }
   - match: { preview.1.time.max: "2017-02-18T01:00:00.000Z" }
   - match: { preview.1.time.min: "2017-02-18T01:00:00.000Z" }
   - match: { preview.2.airline: foo }
-  - match: { preview.2.by-hour: "2017-02-18 01" }
+  - match: { preview.2.by-hour: 1487379600000 }
   - match: { preview.2.avg_response: 42.0 }
   - match: { preview.2.time.max: "2017-02-18T01:01:00.000Z" }
   - match: { preview.2.time.min: "2017-02-18T01:01:00.000Z" }
+  - match: { mappings.properties.airline.type: "keyword" }
+  - match: { mappings.properties.by-hour.type: "date" }
+  - match: { mappings.properties.avg_response.type: "double" }
+  - match: { mappings.properties.time\.max.type: "date" }
+  - match: { mappings.properties.time\.min.type: "date" }
+
+  - do:
+      ingest.put_pipeline:
+        id: "data_frame_simple_pipeline"
+        body:  >
+          {
+            "processors": [
+             {
+               "set" : {
+                 "field" : "my_field",
+                 "value": 42
+               }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+  - do:
+      data_frame.preview_data_frame_transform:
+        body: >
+          {
+            "source": { "index": "airline-data" },
+            "dest": { "pipeline": "data_frame_simple_pipeline" },
+            "pivot": {
+              "group_by": {
+                "airline": {"terms": {"field": "airline"}},
+                "by-hour": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
+              "aggs": {
+                "avg_response": {"avg": {"field": "responsetime"}}
+              }
+            }
+          }
+  - match: { preview.0.airline: foo }
+  - match: { preview.0.by-hour: 1487376000000 }
+  - match: { preview.0.avg_response: 1.0 }
+  - match: { preview.0.my_field: 42 }
+  - match: { preview.1.airline: bar }
+  - match: { preview.1.by-hour: 1487379600000 }
+  - match: { preview.1.avg_response: 42.0 }
+  - match: { preview.1.my_field: 42 }
+  - match: { preview.2.airline: foo }
+  - match: { preview.2.by-hour: 1487379600000 }
+  - match: { preview.2.avg_response: 42.0 }
+  - match: { preview.2.my_field: 42 }
+  - match: { mappings.properties.airline.type: "keyword" }
+  - match: { mappings.properties.by-hour.type: "date" }
+  - match: { mappings.properties.avg_response.type: "double" }
 
 ---
 "Test preview transform with invalid config":
@@ -123,7 +174,81 @@ setup:
             "pivot": {
               "group_by": {
                 "airline": {"terms": {"field": "airline"}},
-                "by-hour": {"date_histogram": {"fixed_interval": "1h", "field": "time", "format": "yyyy-MM-dd HH"}}},
+                "by-hour": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
               "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
             }
           }
+---
+"Test preview returns bad request with invalid agg":
+  - do:
+      catch: bad_request
+      data_frame.preview_data_frame_transform:
+        body: >
+          {
+            "source": { "index": "airline-data" },
+            "pivot": {
+              "group_by": {
+                "time": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
+              "aggs": {
+                "avg_response": {"avg": {"field": "responsetime"}},
+                "time.min": {"min": {"field": "time"}}
+              }
+            }
+          }
+
+  - do:
+      catch: /field \[time\] cannot be both an object and a field/
+      data_frame.preview_data_frame_transform:
+        body: >
+          {
+            "source": { "index": "airline-data" },
+            "pivot": {
+              "group_by": {
+                "time": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
+              "aggs": {
+                "avg_response": {"avg": {"field": "responsetime"}},
+                "time.min": {"min": {"field": "time"}}
+              }
+            }
+          }
+  - do:
+      catch: /field \[super_metric\] cannot be both an object and a field/
+      data_frame.preview_data_frame_transform:
+        body: >
+          {
+            "source": { "index": "airline-data" },
+            "pivot": {
+              "group_by": {
+                "time": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
+              "aggs": {
+                "avg_response": {"avg": {"field": "responsetime"}},
+                "super_metric.time": {"min": {"field": "time"}},
+                "super_metric": {
+                  "scripted_metric": {
+                     "init_script": "",
+                     "map_script": "",
+                     "combine_script": "",
+                     "reduce_script": "return ['value1': 1, 'value2':2]"
+                  }
+                }
+              }
+            }
+          }
+---
+"Test preview with missing pipeline":
+  - do:
+      catch: bad_request
+      data_frame.preview_data_frame_transform:
+        body: >
+          {
+            "source": { "index": "airline-data" },
+            "dest": { "pipeline": "missing-pipeline" },
+            "pivot": {
+              "group_by": {
+                "time": {"date_histogram": {"fixed_interval": "1h", "field": "time"}}},
+              "aggs": {
+                "avg_response": {"avg": {"field": "responsetime"}},
+                "time.min": {"min": {"field": "time"}}
+              }
+            }
+          }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml
index 65945b6ab7429..bfde8128b491c 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_crud.yml
@@ -22,6 +22,12 @@ setup:
   - match: { count: 0 }
   - match: { transforms: [] }
 
+  - do:
+      catch: missing
+      data_frame.get_data_frame_transform:
+        transform_id: "*"
+        allow_no_match: false
+
 ---
 "Test get transform when it does not exist":
   - do:
@@ -74,7 +80,7 @@ setup:
         body: >
           {
             "source": { "index": "airline-data" },
-            "dest": { "index": "airline-data-by-airline-again" },
+            "dest": { "index": "airline-data-by-airline-again", "pipeline": "airline-pipeline" },
             "pivot": {
               "group_by": { "airline": {"terms": {"field": "airline"}}},
               "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
@@ -90,6 +96,8 @@ setup:
   - match: { transforms.0.source.index.0: "airline-data" }
   - match: { transforms.0.dest.index: "airline-data-by-airline" }
   - is_true: transforms.0.source.query.match_all
+  - is_true: transforms.0.create_time
+  - is_true: transforms.0.version
   - match: { transforms.0.pivot.group_by.airline.terms.field: "airline" }
   - match: { transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime" }
   - match: { transforms.0.description: "yaml test transform on airline-data" }
@@ -127,7 +135,7 @@ setup:
         transform_id: "airline-transform*"
         from: 0
         size: 1
-  - match: { count: 1 }
+  - match: { count: 2 }
   - match: { transforms.0.id: "airline-transform" }
 
   - do:
@@ -135,7 +143,7 @@ setup:
         transform_id: "airline-transform*"
         from: 1
         size: 1
-  - match: { count: 1 }
+  - match: { count: 2 }
   - match: { transforms.0.id: "airline-transform-dos" }
 
 ---
@@ -183,34 +191,51 @@ setup:
   - match: { transforms.0.pivot.group_by.airline.terms.field: "airline" }
   - match: { transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime" }
 ---
-"Test transform with invalid page parameter":
-  - do:
-      catch: /Param \[size\] has a max acceptable value of \[1000\]/
-      data_frame.get_data_frame_transform:
-        transform_id: "_all"
-        from: 0
-        size: 10000
----
-"Test transform where dest is included in source":
+"Test PUT continuous transform":
   - do:
-      catch: /Destination index \[airline-data-by-airline\] is included in source expression \[airline-data/
       data_frame.put_data_frame_transform:
-        transform_id: "airline-transform"
+        transform_id: "airline-transform-continuous"
         body: >
           {
             "source": {
-              "index": ["airline-data*"]
+              "index": "airline-data"
             },
-            "dest": { "index": "airline-data-by-airline" },
+            "dest": { "index": "airline-data-by-airline-continuous" },
             "pivot": {
               "group_by": { "airline": {"terms": {"field": "airline"}}},
               "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            },
+            "sync": {
+              "time": {
+                "field": "time",
+                "delay": "90m"
+              }
             }
           }
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "airline-transform-continuous"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "airline-transform-continuous" }
+  - match: { transforms.0.source.index.0: "airline-data" }
+  - match: { transforms.0.dest.index: "airline-data-by-airline-continuous" }
+  - match: { transforms.0.pivot.group_by.airline.terms.field: "airline" }
+  - match: { transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime" }
+  - match: { transforms.0.sync.time.field: "time" }
+  - match: { transforms.0.sync.time.delay: "90m" }
 ---
-"Test transform where dest is a simple index pattern":
+"Test transform with invalid page parameter":
   - do:
-      catch: /Destination index .* should refer to a single index/
+      catch: /Param \[size\] has a max acceptable value of \[1000\]/
+      data_frame.get_data_frame_transform:
+        transform_id: "_all"
+        from: 0
+        size: 10000
+---
+"Test transform where dest is included in source":
+  - do:
+      catch: /Destination index \[airline-data-by-airline\] is included in source expression \[airline-data/
       data_frame.put_data_frame_transform:
         transform_id: "airline-transform"
         body: >
@@ -218,7 +243,7 @@ setup:
             "source": {
               "index": ["airline-data*"]
             },
-            "dest": { "index": "destination*" },
+            "dest": { "index": "airline-data-by-airline" },
             "pivot": {
               "group_by": { "airline": {"terms": {"field": "airline"}}},
               "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
@@ -364,3 +389,105 @@ setup:
               "aggs": {"airline.responsetime": {"avg": {"field": "responsetime"}}}
             }
           }
+---
+"Test invalid data frame id":
+  - do:
+      catch: /can contain lowercase alphanumeric \(a-z and 0-9\), hyphens or underscores; must start and end with alphanumeric/
+      data_frame.put_data_frame_transform:
+        transform_id: "!@#$%^&*(duplicate-field-transform"
+        body: >
+          {
+            "source": {
+              "index": "source-index"
+            },
+            "dest": { "index": "dest-index" },
+            "pivot": {
+              "group_by": { "airline.id": {"terms": {"field": "airline"}}},
+              "aggs": {"airline.response": {"avg": {"field": "responsetime"}}}
+            }
+          }
+  - do:
+      catch: /The id cannot contain more than 64 character/
+      data_frame.put_data_frame_transform:
+        transform_id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+        body: >
+          {
+            "source": {
+              "index": "source-index"
+            },
+            "dest": { "index": "dest-index" },
+            "pivot": {
+              "group_by": { "airline.id": {"terms": {"field": "airline"}}},
+              "aggs": {"airline.response": {"avg": {"field": "responsetime"}}}
+            }
+          }
+---
+"Test invalid destination index name":
+  - do:
+      catch: /dest\.index \[DeStInAtIoN\] must be lowercase/
+      data_frame.put_data_frame_transform:
+        transform_id: "airline-transform"
+        body: >
+          {
+            "source": {
+              "index": "airline-data"
+            },
+            "dest": { "index": "DeStInAtIoN" },
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            }
+          }
+  - do:
+      catch: /Invalid index name \[destination#dest\], must not contain \'#\'/
+      data_frame.put_data_frame_transform:
+        transform_id: "airline-transform"
+        body: >
+          {
+            "source": {
+              "index": "airline-data"
+            },
+            "dest": { "index": "destination#dest" },
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            }
+          }
+
+---
+"Test put valid config with create_time in the body":
+
+  - do:
+      catch: /Found \[create_time\], not allowed for strict parsing/
+      data_frame.put_data_frame_transform:
+        transform_id: "airline-transform-with-create-time"
+        body: >
+          {
+            "source": { "index": "airline-data" },
+            "dest": { "index": "airline-data-by-airline" },
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            },
+            "description": "yaml test transform on airline-data",
+            "create_time": 123456789
+          }
+
+---
+"Test put valid config with version in the body":
+
+  - do:
+      catch: /Found \[version\], not allowed for strict parsing/
+      data_frame.put_data_frame_transform:
+        transform_id: "airline-transform-with-version"
+        body: >
+          {
+            "source": { "index": "airline-data" },
+            "dest": { "index": "airline-data-by-airline" },
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            },
+            "description": "yaml test transform on airline-data",
+            "version": "7.3.0"
+          }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_start_stop.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_start_stop.yml
index a475c3ceadca6..e4ff3c813ce0f 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_start_stop.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_start_stop.yml
@@ -42,7 +42,7 @@ teardown:
   - do:
       data_frame.start_data_frame_transform:
         transform_id: "airline-transform-start-stop"
-  - match: { started: true }
+  - match: { acknowledged: true }
 
 ---
 "Test start missing transform":
@@ -56,7 +56,7 @@ teardown:
   - do:
       data_frame.start_data_frame_transform:
         transform_id: "airline-transform-start-stop"
-  - match: { started: true }
+  - match: { acknowledged: true }
 
   - do:
       catch: /Unable to start data frame transform \[airline-transform-start-stop\] as it is in state \[STARTED\]/
@@ -68,7 +68,7 @@ teardown:
   - do:
       data_frame.start_data_frame_transform:
         transform_id: "airline-transform-start-stop"
-  - match: { started: true }
+  - match: { acknowledged: true }
   - do:
       indices.get_mapping:
         index: airline-data-by-airline-start-stop
@@ -83,7 +83,7 @@ teardown:
   - do:
       data_frame.start_data_frame_transform:
         transform_id: "airline-transform-start-stop"
-  - match: { started: true }
+  - match: { acknowledged: true }
   - do:
       indices.get_mapping:
         index: airline-data-by-airline-start-stop
@@ -93,7 +93,7 @@ teardown:
   - do:
       data_frame.start_data_frame_transform:
         transform_id: "airline-transform-start-stop"
-  - match: { started: true }
+  - match: { acknowledged: true }
 
   - do:
       data_frame.get_data_frame_transform_stats:
@@ -107,20 +107,20 @@ teardown:
       data_frame.stop_data_frame_transform:
         transform_id: "airline-transform-start-stop"
         wait_for_completion: true
-  - match: { stopped: true }
+  - match: { acknowledged: true }
 
   - do:
       data_frame.get_data_frame_transform_stats:
         transform_id: "airline-transform-start-stop"
   - match: { count: 1 }
   - match: { transforms.0.id: "airline-transform-start-stop" }
-#  - match: { transforms.0.state.indexer_state: "stopped" }
-#  - match: { transforms.0.state.task_state: "stopped" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
 
   - do:
       data_frame.start_data_frame_transform:
         transform_id: "airline-transform-start-stop"
-  - match: { started: true }
+  - match: { acknowledged: true }
 
   - do:
       data_frame.get_data_frame_transform_stats:
@@ -129,7 +129,70 @@ teardown:
   - match: { transforms.0.id: "airline-transform-start-stop" }
   - match: { transforms.0.state.indexer_state: "/started|indexing/" }
   - match: { transforms.0.state.task_state: "started" }
+---
+"Test start/stop/start continuous transform":
+  - do:
+      data_frame.put_data_frame_transform:
+        transform_id: "airline-transform-start-stop-continuous"
+        body: >
+          {
+            "source": { "index": "airline-data" },
+            "dest": { "index": "airline-data-by-airline-start-stop-cont" },
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            }
+          }
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "airline-transform-start-stop-continuous"
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "airline-transform-start-stop-continuous"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "airline-transform-start-stop-continuous" }
+  - match: { transforms.0.state.indexer_state: "/started|indexing/" }
+  - match: { transforms.0.state.task_state: "started" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "airline-transform-start-stop-continuous"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "airline-transform-start-stop-continuous"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "airline-transform-start-stop-continuous" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "airline-transform-start-stop-continuous"
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "airline-transform-start-stop-continuous"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "airline-transform-start-stop-continuous" }
+  - match: { transforms.0.state.indexer_state: "/started|indexing/" }
+  - match: { transforms.0.state.task_state: "started" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "airline-transform-start-stop-continuous"
+        wait_for_completion: true
+  - match: { acknowledged: true }
 
+  - do:
+      data_frame.delete_data_frame_transform:
+        transform_id: "airline-transform-start-stop-continuous"
 ---
 "Test stop missing transform":
   - do:
@@ -137,12 +200,25 @@ teardown:
       data_frame.stop_data_frame_transform:
         transform_id: "missing-transform"
 
+---
+"Test stop missing transform by expression":
+  - do:
+      data_frame.stop_data_frame_transform:
+        allow_no_match: true
+        transform_id: "missing-transform*"
+
+  - do:
+      catch: missing
+      data_frame.stop_data_frame_transform:
+        allow_no_match: false
+        transform_id: "missing-transform*"
+
 ---
 "Test stop already stopped transform":
   - do:
       data_frame.stop_data_frame_transform:
         transform_id: "airline-transform-start-stop"
-  - match: { stopped: true }
+  - match: { acknowledged: true }
 
 ---
 "Test start/stop only starts/stops specified transform":
@@ -161,7 +237,7 @@ teardown:
   - do:
       data_frame.start_data_frame_transform:
         transform_id: "airline-transform-start-stop"
-  - match: { started: true }
+  - match: { acknowledged: true }
 
   - do:
       data_frame.get_data_frame_transform_stats:
@@ -182,12 +258,14 @@ teardown:
   - do:
       data_frame.start_data_frame_transform:
         transform_id: "airline-transform-start-later"
-  - match: { started: true }
+  - match: { acknowledged: true }
 
   - do:
       data_frame.stop_data_frame_transform:
         transform_id: "airline-transform-start-stop"
-  - match: { stopped: true }
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
 
   - do:
       data_frame.get_data_frame_transform_stats:
@@ -201,8 +279,51 @@ teardown:
       data_frame.stop_data_frame_transform:
         transform_id: "airline-transform-start-later"
         wait_for_completion: true
-  - match: { stopped: true }
+  - match: { acknowledged: true }
 
   - do:
       data_frame.delete_data_frame_transform:
         transform_id: "airline-transform-start-later"
+
+---
+"Test stop all":
+  - do:
+      data_frame.put_data_frame_transform:
+        transform_id: "airline-transform-stop-all"
+        body: >
+          {
+            "source": { "index": "airline-data" },
+            "dest": { "index": "airline-data-start-later" },
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            }
+          }
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "airline-transform-stop-all"
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "airline-transform-start-stop"
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "_all"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "*"
+  - match: { count: 2 }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
+  - match: { transforms.1.state.indexer_state: "stopped" }
+  - match: { transforms.1.state.task_state: "stopped" }
+
+  - do:
+      data_frame.delete_data_frame_transform:
+        transform_id: "airline-transform-stop-all"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_stats.yml
index 79aa14cb6f628..a6066aceb969b 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_stats.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_frame/transforms_stats.yml
@@ -47,13 +47,13 @@ teardown:
         transform_id: "airline-transform-stats"
   - match: { count: 1 }
   - match: { transforms.0.id: "airline-transform-stats" }
-  - match: { transforms.0.state.indexer_state: "/started|indexing/" }
-  - match: { transforms.0.state.task_state: "started" }
+  - match: { transforms.0.state.indexer_state: "/started|indexing|stopped/" }
+  - match: { transforms.0.state.task_state: "/started|stopped/" }
   - lte: { transforms.0.state.checkpoint: 1 }
   - lte: { transforms.0.stats.pages_processed: 1 }
   - match: { transforms.0.stats.documents_processed: 0 }
   - match: { transforms.0.stats.documents_indexed: 0 }
-  - match: { transforms.0.stats.trigger_count: 1 }
+  - lte: { transforms.0.stats.trigger_count: 1 }
   - match: { transforms.0.stats.index_time_in_ms: 0 }
   - match: { transforms.0.stats.index_total: 0 }
   - match: { transforms.0.stats.index_failures: 0 }
@@ -123,7 +123,7 @@ teardown:
         transform_id: "_all"
         from: 0
         size: 1
-  - match: { count: 1 }
+  - match: { count: 3 }
   - match: { transforms.0.id: "airline-transform-stats" }
 
   - do:
@@ -131,7 +131,7 @@ teardown:
         transform_id: "_all"
         from: 1
         size: 2
-  - match: { count: 2 }
+  - match: { count: 3 }
   - match: { transforms.0.id: "airline-transform-stats-dos" }
   - match: { transforms.1.id: "airline-transform-stats-the-third" }
 
@@ -163,7 +163,7 @@ teardown:
         transform_id: "*"
   - match: { count: 2 }
   - match: { transforms.0.id: "airline-transform-stats" }
-  - match: { transforms.0.state.indexer_state: "/started|indexing/" }
+  - match: { transforms.0.state.indexer_state: "/started|indexing|stopped/" }
   - match: { transforms.1.id: "airline-transform-stats-dos" }
   - match: { transforms.1.state.indexer_state: "stopped" }
 
@@ -172,7 +172,7 @@ teardown:
         transform_id: "_all"
   - match: { count: 2 }
   - match: { transforms.0.id: "airline-transform-stats" }
-  - match: { transforms.0.state.indexer_state: "/started|indexing/" }
+  - match: { transforms.0.state.indexer_state: "/started|indexing|stopped/" }
   - match: { transforms.1.id: "airline-transform-stats-dos" }
   - match: { transforms.1.state.indexer_state: "stopped" }
 
@@ -209,3 +209,56 @@ teardown:
   - match: { transforms.0.stats.search_time_in_ms: 0 }
   - match: { transforms.0.stats.search_total: 0 }
   - match: { transforms.0.stats.search_failures: 0 }
+---
+"Test get continuous transform stats":
+  - do:
+      data_frame.put_data_frame_transform:
+        transform_id: "airline-transform-stats-continuous"
+        body: >
+          {
+            "source": { "index": "airline-data" },
+            "dest": { "index": "airline-data-by-airline-stats-continuous" },
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            },
+            "sync": { "time": { "field": "time", "delay": "1m" } }
+          }
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "airline-transform-stats-continuous"
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "airline-transform-stats-continuous"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "airline-transform-stats-continuous" }
+  - match: { transforms.0.state.indexer_state: "/started|indexing|stopped/" }
+  # Since this is continuous, there is no worry of it automatically stopping
+  - match: { transforms.0.state.task_state: "started" }
+  - lte: { transforms.0.state.checkpoint: 1 }
+  # Since this is continuous, and _start does not return until it is assigned
+  # we should see a node assignment
+  - is_true: transforms.0.state.node
+  - is_true: transforms.0.state.node.id
+  - is_true: transforms.0.state.node.name
+  - is_true: transforms.0.state.node.ephemeral_id
+  - is_true: transforms.0.state.node.transport_address
+  - lte: { transforms.0.stats.pages_processed: 1 }
+  - match: { transforms.0.stats.documents_processed: 0 }
+  - match: { transforms.0.stats.documents_indexed: 0 }
+  - lte: { transforms.0.stats.trigger_count: 1 }
+  - match: { transforms.0.stats.index_time_in_ms: 0 }
+  - match: { transforms.0.stats.index_total: 0 }
+  - match: { transforms.0.stats.index_failures: 0 }
+  - gte: { transforms.0.stats.search_time_in_ms: 0 }
+  - lte: { transforms.0.stats.search_total: 1 }
+  - match: { transforms.0.stats.search_failures: 0 }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "airline-transform-stats-continuous"
+        wait_for_completion: true
+
+  - do:
+      data_frame.delete_data_frame_transform:
+        transform_id: "airline-transform-stats-continuous"
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/10_basic.yml
new file mode 100644
index 0000000000000..c750abb7895f0
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/10_basic.yml
@@ -0,0 +1,111 @@
+---
+"Test exists query on flattened object field":
+  - skip:
+      version: " - 7.99.99"
+      reason: "Flat object fields are currently only implemented in 8.0."
+
+  - do:
+      indices.create:
+        index:  flat_object_test
+        body:
+          mappings:
+            properties:
+              flat_object:
+                type: flattened
+  - do:
+      index:
+        index:  flat_object_test
+        id:     1
+        body:
+          flat_object:
+            key: some_value
+        refresh: true
+
+  - do:
+      search:
+        index: flat_object_test
+        body:
+          query:
+            exists:
+              field: flat_object
+
+  - match: { hits.total.value: 1 }
+
+  - do:
+      search:
+        index: flat_object_test
+        body:
+          query:
+            exists:
+              field: flat_object.key
+
+  - match: { hits.total.value: 1 }
+
+  - do:
+      search:
+        index: flat_object_test
+        body:
+          query:
+            exists:
+              field: flat_object.nonexistent_key
+
+  - match: { hits.total.value: 0 }
+
+---
+"Test query string query on flattened object field":
+  - skip:
+      version: " - 7.99.99"
+      reason: "Flat object fields are currently only implemented in 8.0."
+
+  - do:
+      indices.create:
+        index:  test
+        body:
+          mappings:
+            properties:
+              headers:
+                type: flattened
+
+  - do:
+      index:
+        index:  test
+        id:     1
+        body:
+          headers:
+            content-type: application/javascript
+            origin: elastic.co
+        refresh: true
+
+  - do:
+      index:
+        index:  test
+        id:     2
+        body:
+          headers:
+            content-type: text/plain
+            origin: elastic.co
+        refresh: true
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            query_string:
+              query: "headers:text\\/plain"
+
+  - match: { hits.total.value: 1 }
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._id: "2" }
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            query_string:
+              query: "application\\/javascript AND headers.origin:elastic.co"
+
+  - match: { hits.total.value: 1 }
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._id: "1" }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/10_basic.yml
index 4ba49e53308d2..4a5f713ea74a5 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/10_basic.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/10_basic.yml
@@ -77,6 +77,25 @@
 
 - match: {hits.total: 0}
 
+- do:
+    index:
+      index: ordinary
+      id:    "1"
+      body:  { "foo": "Hello: 1" }
+      refresh: wait_for
+
+- do:
+    search:
+      rest_total_hits_as_int: true
+      index: [test, ordinary]
+      ignore_throttled: false
+      body:
+        query:
+          match:
+            foo: hello
+
+- match: {hits.total: 3}
+
 ---
 "Test index options":
 - do:
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/20_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/20_stats.yml
new file mode 100644
index 0000000000000..fd6163761dcb9
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/20_stats.yml
@@ -0,0 +1,62 @@
+---
+setup:
+  - do:
+      indices.create:
+          index: test
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
+---
+"Translog stats on frozen indices":
+  - skip:
+      version: " - 7.2.99"
+      reason:  "frozen indices have translog stats starting version 7.3.0"
+
+  - do:
+      index:
+        index: test
+        id:    1
+        body:  { "foo": "bar" }
+
+  - do:
+      index:
+        index: test
+        id:    2
+        body:  { "foo": "bar" }
+
+  - do:
+      index:
+        index: test
+        id:    3
+        body:  { "foo": "bar" }
+
+  - do:
+      indices.stats:
+        metric: [ translog ]
+  - match: { indices.test.primaries.translog.operations: 3 }
+  - match: { indices.test.primaries.translog.uncommitted_operations: 3 }
+
+  # freeze index
+  - do:
+      indices.freeze:
+        index: test
+  - is_true: acknowledged
+
+  - do:
+      indices.stats:
+        metric: [ translog ]
+  - match: { indices.test.primaries.translog.operations: 3 }
+  - match: { indices.test.primaries.translog.uncommitted_operations: 0 }
+
+  # unfreeze index
+  - do:
+      indices.freeze:
+        index: test
+  - is_true: acknowledged
+
+  - do:
+      indices.stats:
+        metric: [ translog ]
+  - match: { indices.test.primaries.translog.operations: 3 }
+  - match: { indices.test.primaries.translog.uncommitted_operations: 0 }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
new file mode 100644
index 0000000000000..a171e879fa093
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml
@@ -0,0 +1,926 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: index-source
+
+---
+"Test get-all and get-all-stats given no analytics exist":
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "_all"
+  - match: { count: 0 }
+  - match: { data_frame_analytics: [] }
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "_all"
+  - match: { count: 0 }
+  - match: { data_frame_analytics: [] }
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "*"
+  - match: { count: 0 }
+  - match: { data_frame_analytics: [] }
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "*"
+  - match: { count: 0 }
+  - match: { data_frame_analytics: [] }
+
+---
+"Test put valid config with default outlier detection, query, and filter":
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "simple-outlier-detection-with-query"
+        body: >
+          {
+            "source": {
+              "index": "index-source",
+              "query": {"term" : { "user" : "Kimchy" }}
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}},
+            "analyzed_fields": [ "obj1.*", "obj2.*" ]
+          }
+  - match: { id: "simple-outlier-detection-with-query" }
+  - match: { source.index: ["index-source"] }
+  - match: { source.query: {"term" : { "user" : "Kimchy"} } }
+  - match: { dest.index: "index-dest" }
+  - match: { analysis: {"outlier_detection":{}} }
+  - match: { analyzed_fields: {"includes" : ["obj1.*", "obj2.*" ], "excludes": [] } }
+  - is_true: create_time
+  - is_true: version
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "simple-outlier-detection-with-query"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "simple-outlier-detection-with-query" }
+  - match: { data_frame_analytics.0.source.index: ["index-source"] }
+  - match: { data_frame_analytics.0.source.query: {"term" : { "user" : "Kimchy"} } }
+  - match: { data_frame_analytics.0.dest.index: "index-dest" }
+  - match: { data_frame_analytics.0.analysis: {"outlier_detection":{}} }
+  - match: { data_frame_analytics.0.analyzed_fields: {"includes" : ["obj1.*", "obj2.*" ], "excludes": [] } }
+  - is_true: data_frame_analytics.0.create_time
+  - is_true: data_frame_analytics.0.version
+
+---
+"Test put config with security headers in the body":
+  - do:
+      catch: /unknown field \[headers\], parser not found/
+      ml.put_data_frame_analytics:
+        id: "data_frame_with_header"
+        body:  >
+          {
+            "source": {
+              "index": "index-source",
+              "query": {"term" : { "user" : "Kimchy" }}
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}},
+            "headers":{ "a_security_header" : "secret" }
+          }
+
+---
+"Test put config with create_time in the body":
+
+  - do:
+      catch: /unknown field \[create_time\], parser not found/
+      ml.put_data_frame_analytics:
+        id: "data_frame_with_create_time"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}},
+            "create_time": 123456789
+          }
+
+---
+"Test put config with version in the body":
+
+  - do:
+      catch: /unknown field \[version\], parser not found/
+      ml.put_data_frame_analytics:
+        id: "data_frame_with_version"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}},
+            "version": "7.3.0"
+          }
+
+---
+"Test put valid config with default outlier detection":
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "simple-outlier-detection"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+  - match: { id: "simple-outlier-detection" }
+  - match: { source.index: ["index-source"] }
+  - match: { source.query: {"match_all" : {} } }
+  - match: { dest.index: "index-dest" }
+  - match: { analysis: {"outlier_detection":{}} }
+  - is_true: create_time
+  - is_true: version
+
+---
+"Test put valid config with custom outlier detection":
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "custom-outlier-detection"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {
+              "outlier_detection":{
+                "n_neighbors": 5,
+                "method": "lof",
+                "feature_influence_threshold": 0.0
+              }
+            }
+          }
+  - match: { id: "custom-outlier-detection" }
+  - match: { source.index: ["index-source"] }
+  - match: { source.query: {"match_all" : {} } }
+  - match: { dest.index: "index-dest" }
+  - match: { analysis.outlier_detection.n_neighbors: 5 }
+  - match: { analysis.outlier_detection.method: "lof" }
+  - match: { analysis.outlier_detection.feature_influence_threshold: 0.0 }
+  - is_true: create_time
+  - is_true: version
+
+---
+"Test put config with inconsistent body/param ids":
+
+  - do:
+      catch: /Inconsistent id; 'body_id' specified in the body differs from 'url_id' specified as a URL argument/
+      ml.put_data_frame_analytics:
+        id: "url_id"
+        body: >
+          {
+            "id": "body_id",
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test put config with invalid id":
+
+  - do:
+      catch: /Invalid id*/
+      ml.put_data_frame_analytics:
+        id: "this id contains spaces"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test put config with invalid dest index name":
+
+  - do:
+      catch: /Invalid index name \[<script>Foo\]*/
+      ml.put_data_frame_analytics:
+        id: "config-with-invalid-dest-index"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "<script>Foo"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test put config with pattern dest index name":
+
+  - do:
+      catch: /Invalid index name \[foo\*\]/
+      ml.put_data_frame_analytics:
+        id: "config-with-invalid-dest-index"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "foo*"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test put config with missing concrete source index":
+
+  - do:
+      catch: /No index matches source index \[missing]/
+      ml.put_data_frame_analytics:
+        id: "config-with-missing-concrete-source-index"
+        body: >
+          {
+            "source": {
+              "index": "missing"
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test put config with missing wildcard source index":
+
+  - do:
+      catch: /No index matches source index \[missing\*\]/
+      ml.put_data_frame_analytics:
+        id: "config-with-missing-wildcard-source-index"
+        body: >
+          {
+            "source": {
+              "index": "missing*"
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test put config with dest index same as source index":
+
+  - do:
+      catch: /Destination index \[index-source\] must not be included in source index \[index-source\]/
+      ml.put_data_frame_analytics:
+        id: "config-with-same-source-dest-index"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-source"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test put config with dest index matching multiple indices":
+
+  - do:
+      indices.create:
+        index: multiple-dest-index-1
+
+  - do:
+      indices.create:
+        index: multiple-dest-index-2
+
+  - do:
+      indices.put_alias:
+        index: multiple-dest-index-1
+        name: multiple-dest-index
+
+  - do:
+      indices.put_alias:
+        index: multiple-dest-index-2
+        name: multiple-dest-index
+
+  - do:
+      catch: /Destination index \[multiple-dest-index\] should match a single index; matches \[multiple-dest-index-[12], multiple-dest-index-[12]\]/
+      ml.put_data_frame_analytics:
+        id: "config-with-dest-index-matching-multiple-indices"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "multiple-dest-index"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test put config with dest index included in source via alias":
+
+  - do:
+      indices.create:
+        index: another-source-index
+
+  - do:
+      indices.put_alias:
+        index: another-source-index
+        name: dest-alias
+
+  - do:
+      catch: /Destination index \[dest-alias\], which is an alias for \[another-source-index\], must not be included in source index \[another-source-index\]/
+      ml.put_data_frame_analytics:
+        id: "config-with-dest-index-included-in-source-via-alias"
+        body: >
+          {
+            "source": {
+              "index": "another-source-index"
+            },
+            "dest": {
+              "index": "dest-alias"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test put config with unknown top level field":
+
+  - do:
+      catch: /unknown field \[unknown_field\], parser not found/
+      ml.put_data_frame_analytics:
+        id: "unknown_field"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}},
+            "unknown_field": 42
+          }
+
+---
+"Test put config with unknown field in outlier detection analysis":
+
+  - do:
+      catch: /unknown field \[unknown_field\]/
+      ml.put_data_frame_analytics:
+        id: "unknown_field"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{"unknown_field":42}}
+          }
+
+---
+"Test put config given missing source":
+
+  - do:
+      catch: /\[source\] must not be null/
+      ml.put_data_frame_analytics:
+        id: "simple-outlier-detection"
+        body: >
+          {
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test put config given source with empty index array":
+
+  - do:
+      catch: /source\.index must specify at least one index/
+      ml.put_data_frame_analytics:
+        id: "simple-outlier-detection"
+        body: >
+          {
+            "source": {
+              "index": []
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test put config given source with empty string in index array":
+
+  - do:
+      catch: /source\.index must contain non-null and non-empty strings/
+      ml.put_data_frame_analytics:
+        id: "simple-outlier-detection"
+        body: >
+          {
+            "source": {
+              "index": [""]
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test put config given source without index":
+
+  - do:
+      catch: /Required \[index\]/
+      ml.put_data_frame_analytics:
+        id: "simple-outlier-detection"
+        body: >
+          {
+            "source": {
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test put config given missing dest":
+
+  - do:
+      catch: /\[dest\] must not be null/
+      ml.put_data_frame_analytics:
+        id: "simple-outlier-detection"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test put config given dest with empty index":
+
+  - do:
+      catch: /\[index\] must be non-empty/
+      ml.put_data_frame_analytics:
+        id: "simple-outlier-detection"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": ""
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test put config given dest without index":
+
+  - do:
+      catch: /Required \[index\]/
+      ml.put_data_frame_analytics:
+        id: "simple-outlier-detection"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test put config given missing analysis":
+
+  - do:
+      catch: /\[analysis\] must not be null/
+      ml.put_data_frame_analytics:
+        id: "simple-outlier-detection"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-dest"
+            }
+          }
+
+---
+"Test put config given empty analysis":
+
+  - do:
+      catch: /parsing_exception/
+      ml.put_data_frame_analytics:
+        id: "simple-outlier-detection"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {}
+          }
+
+---
+"Test get given multiple analytics":
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "foo-1"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-foo-1_dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "foo-2"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-foo-2_dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+  - match: { id: "foo-2" }
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "bar"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-bar_dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+  - match: { id: "bar" }
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "*"
+  - match: { count: 3 }
+  - match: { data_frame_analytics.0.id: "bar" }
+  - match: { data_frame_analytics.1.id: "foo-1" }
+  - match: { data_frame_analytics.2.id: "foo-2" }
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "foo-*"
+  - match: { count: 2 }
+  - match: { data_frame_analytics.0.id: "foo-1" }
+  - match: { data_frame_analytics.1.id: "foo-2" }
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "bar"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "bar" }
+
+  - do:
+      ml.get_data_frame_analytics:
+        from: 1
+  - match: { count: 3 }
+  - match: { data_frame_analytics.0.id: "foo-1" }
+  - match: { data_frame_analytics.1.id: "foo-2" }
+
+  - do:
+      ml.get_data_frame_analytics:
+        size: 2
+  - match: { count: 3 }
+  - match: { data_frame_analytics.0.id: "bar" }
+  - match: { data_frame_analytics.1.id: "foo-1" }
+
+  - do:
+      ml.get_data_frame_analytics:
+        from: 1
+        size: 1
+  - match: { count: 3 }
+  - match: { data_frame_analytics.0.id: "foo-1" }
+
+---
+"Test get given missing analytics":
+
+  - do:
+      catch: missing
+      ml.get_data_frame_analytics:
+        id: "missing-analytics"
+
+---
+"Test get given missing analytics and allow_no_match is false":
+
+  - do:
+      catch: missing
+      ml.get_data_frame_analytics:
+        id: "missing-analytics"
+        allow_no_match: false
+
+---
+"Test get given expression without matches and allow_no_match is false":
+
+  - do:
+      catch: missing
+      ml.get_data_frame_analytics:
+        id: "missing-analytics*"
+        allow_no_match: false
+
+---
+"Test get given expression without matches and allow_no_match is true":
+
+  - do:
+      ml.get_data_frame_analytics:
+        id: "missing-analytics*"
+        allow_no_match: true
+  - match: { count: 0 }
+  - match: { data_frame_analytics: [] }
+
+---
+"Test get stats given missing analytics":
+
+  - do:
+      catch: missing
+      ml.get_data_frame_analytics_stats:
+        id: "missing-analytics"
+
+---
+"Test get stats given missing analytics and allow_no_match is false":
+
+  - do:
+      catch: missing
+      ml.get_data_frame_analytics_stats:
+        id: "missing-analytics"
+        allow_no_match: false
+
+---
+"Test get stats given expression without matches and allow_no_match is false":
+
+  - do:
+      catch: missing
+      ml.get_data_frame_analytics_stats:
+        id: "missing-analytics*"
+        allow_no_match: false
+
+---
+"Test get stats given expression without matches and allow_no_match is true":
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "missing-analytics*"
+        allow_no_match: true
+  - match: { count: 0 }
+  - match: { data_frame_analytics: [] }
+
+---
+"Test get stats given multiple analytics":
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "foo-1"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-foo-1_dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "foo-2"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-foo-2_dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+  - match: { id: "foo-2" }
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "bar"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-bar_dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+  - match: { id: "bar" }
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "*"
+  - match: { count: 3 }
+  - match: { data_frame_analytics.0.id: "bar" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+  - match: { data_frame_analytics.1.id: "foo-1" }
+  - match: { data_frame_analytics.1.state: "stopped" }
+  - match: { data_frame_analytics.2.id: "foo-2" }
+  - match: { data_frame_analytics.2.state: "stopped" }
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "foo-*"
+  - match: { count: 2 }
+  - match: { data_frame_analytics.0.id: "foo-1" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+  - match: { data_frame_analytics.1.id: "foo-2" }
+  - match: { data_frame_analytics.1.state: "stopped" }
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        id: "bar"
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "bar" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        from: 2
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "foo-2" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        size: 2
+  - match: { count: 2 }
+  - match: { data_frame_analytics.0.id: "bar" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+  - match: { data_frame_analytics.1.id: "foo-1" }
+  - match: { data_frame_analytics.1.state: "stopped" }
+
+  - do:
+      ml.get_data_frame_analytics_stats:
+        from: 1
+        size: 1
+  - match: { count: 1 }
+  - match: { data_frame_analytics.0.id: "foo-1" }
+  - match: { data_frame_analytics.0.state: "stopped" }
+
+---
+"Test delete given stopped config":
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "foo"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+  - do:
+      ml.delete_data_frame_analytics:
+        id: "foo"
+  - match: { acknowledged: true }
+
+  - do:
+      catch: missing
+      ml.get_data_frame_analytics:
+        id: "foo"
+
+---
+"Test delete given missing config":
+
+  - do:
+      catch: missing
+      ml.delete_data_frame_analytics:
+        id: "missing_config"
+
+---
+"Test max model memory limit":
+  - skip:
+      features: headers
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      cluster.put_settings:
+        body:
+          transient:
+            xpack.ml.max_model_memory_limit: "20mb"
+  - match: {transient.xpack.ml.max_model_memory_limit: "20mb"}
+
+  # Explicit request higher than limit is an error
+  - do:
+      catch: /model_memory_limit \[8gb\] must be less than the value of the xpack.ml.max_model_memory_limit setting \[20mb\]/
+      ml.put_data_frame_analytics:
+        id: "simple-outlier-detection-with-query"
+        body: >
+          {
+            "source": {
+              "index": "index-source",
+              "query": {"term" : { "user" : "Kimchy" }}
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}},
+            "model_memory_limit": "8gb",
+            "analyzed_fields": [ "obj1.*", "obj2.*" ]
+          }
+
+  # Request using default higher than limit gets silently capped
+  - do:
+      ml.put_data_frame_analytics:
+        id: "simple-outlier-detection-with-query"
+        body: >
+          {
+            "source": {
+              "index": "index-source",
+              "query": {"term" : { "user" : "Kimchy" }}
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}},
+            "analyzed_fields": [ "obj1.*", "obj2.*" ]
+          }
+  - match: { id: "simple-outlier-detection-with-query" }
+  - match: { source.index: ["index-source"] }
+  - match: { source.query: {"term" : { "user" : "Kimchy"} } }
+  - match: { dest.index: "index-dest" }
+  - match: { analysis: {"outlier_detection":{}} }
+  - match: { analyzed_fields: {"includes" : ["obj1.*", "obj2.*" ], "excludes": [] } }
+  - match: { model_memory_limit: "20mb" }
+
+
+  - do:
+      headers:
+        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
+      cluster.put_settings:
+        body:
+          transient:
+            xpack.ml.max_model_memory_limit: null
+  - match: {transient: {}}
+
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/evaluate_data_frame.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/evaluate_data_frame.yml
new file mode 100644
index 0000000000000..ef844d61f1626
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/evaluate_data_frame.yml
@@ -0,0 +1,520 @@
+setup:
+
+  - do:
+      index:
+        index: utopia
+        body:  >
+          {
+            "is_outlier": false,
+            "is_outlier_int": 0,
+            "outlier_score": 0.0,
+            "all_true_field": true,
+            "all_false_field": false
+          }
+
+  - do:
+      index:
+        index: utopia
+        body:  >
+          {
+            "is_outlier": false,
+            "is_outlier_int": 0,
+            "outlier_score": 0.2,
+            "all_true_field": true,
+            "all_false_field": false
+          }
+
+  - do:
+      index:
+        index: utopia
+        body:  >
+          {
+            "is_outlier": false,
+            "is_outlier_int": 0,
+            "outlier_score": 0.3,
+            "all_true_field": true,
+            "all_false_field": false
+          }
+
+  - do:
+      index:
+        index: utopia
+        body:  >
+          {
+            "is_outlier": true,
+            "is_outlier_int": 1,
+            "outlier_score": 0.3,
+            "all_true_field": true,
+            "all_false_field": false
+          }
+
+  - do:
+      index:
+        index: utopia
+        body:  >
+          {
+            "is_outlier": true,
+            "is_outlier_int": 1,
+            "outlier_score": 0.4,
+            "all_true_field": true,
+            "all_false_field": false
+          }
+
+  - do:
+      index:
+        index: utopia
+        body:  >
+          {
+            "is_outlier": true,
+            "is_outlier_int": 1,
+            "outlier_score": 0.5,
+            "all_true_field": true,
+            "all_false_field": false
+          }
+
+  - do:
+      index:
+        index: utopia
+        body:  >
+          {
+            "is_outlier": true,
+            "is_outlier_int": 1,
+            "outlier_score": 0.9,
+            "all_true_field": true,
+            "all_false_field": false
+          }
+
+  - do:
+      index:
+        index: utopia
+        body:  >
+          {
+            "is_outlier": true,
+            "is_outlier_int": 1,
+            "outlier_score": 0.95,
+            "all_true_field": true,
+            "all_false_field": false
+          }
+
+  # This document misses the required fields and should be ignored
+  - do:
+      index:
+        index: utopia
+        body:  >
+          {
+            "foo": 0.24
+          }
+
+  - do:
+      indices.refresh: {}
+
+---
+"Test binary_soft_classifition auc_roc":
+  - do:
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier",
+                "predicted_probability_field": "outlier_score",
+                "metrics": {
+                  "auc_roc": {}
+                }
+              }
+            }
+          }
+  - match: { binary_soft_classification.auc_roc.score: 0.9899 }
+  - is_false: binary_soft_classification.auc_roc.curve
+
+---
+"Test binary_soft_classifition auc_roc given actual_field is int":
+  - do:
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier_int",
+                "predicted_probability_field": "outlier_score",
+                "metrics": {
+                  "auc_roc": {}
+                }
+              }
+            }
+          }
+  - match: { binary_soft_classification.auc_roc.score: 0.9899 }
+  - is_false: binary_soft_classification.auc_roc.curve
+
+---
+"Test binary_soft_classifition auc_roc include curve":
+  - do:
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier",
+                "predicted_probability_field": "outlier_score",
+                "metrics": {
+                  "auc_roc": { "include_curve": true }
+                }
+              }
+            }
+          }
+  - match: { binary_soft_classification.auc_roc.score: 0.9899 }
+  - is_true: binary_soft_classification.auc_roc.curve
+
+---
+"Test binary_soft_classifition auc_roc given actual_field is always true":
+  - do:
+      catch: /\[auc_roc\] requires at least one actual_field to have a different value than \[true\]/
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "all_true_field",
+                "predicted_probability_field": "outlier_score",
+                "metrics": {
+                  "auc_roc": {}
+                }
+              }
+            }
+          }
+
+---
+"Test binary_soft_classifition auc_roc given actual_field is always false":
+  - do:
+      catch: /\[auc_roc\] requires at least one actual_field to have the value \[true\]/
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "all_false_field",
+                "predicted_probability_field": "outlier_score",
+                "metrics": {
+                  "auc_roc": {}
+                }
+              }
+            }
+          }
+
+---
+"Test binary_soft_classifition precision":
+  - do:
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier",
+                "predicted_probability_field": "outlier_score",
+                "metrics": {
+                  "precision": { "at": [0, 0.5] }
+                }
+              }
+            }
+          }
+  - match:
+      binary_soft_classification:
+        precision:
+          '0.0': 0.625
+          '0.5': 1.0
+
+---
+"Test binary_soft_classifition recall":
+  - do:
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier",
+                "predicted_probability_field": "outlier_score",
+                "metrics": {
+                  "recall": { "at": [0, 0.4, 0.5] }
+                }
+              }
+            }
+          }
+  - match:
+      binary_soft_classification:
+        recall:
+          '0.0': 1.0
+          '0.4': 0.8
+          '0.5': 0.6
+
+---
+"Test binary_soft_classifition confusion_matrix":
+  - do:
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier",
+                "predicted_probability_field": "outlier_score",
+                "metrics": {
+                  "confusion_matrix": { "at": [0, 0.3, 0.5] }
+                }
+              }
+            }
+          }
+  - match:
+      binary_soft_classification:
+        confusion_matrix:
+          '0.0':
+            tp: 5
+            fp: 3
+            tn: 0
+            fn: 0
+          '0.3':
+            tp: 5
+            fp: 1
+            tn: 2
+            fn: 0
+          '0.5':
+            tp: 3
+            fp: 0
+            tn: 3
+            fn: 2
+
+---
+"Test binary_soft_classifition default metrics":
+  - do:
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier",
+                "predicted_probability_field": "outlier_score"
+              }
+            }
+          }
+  - is_true: binary_soft_classification.auc_roc.score
+  - is_true: binary_soft_classification.precision.0\.25
+  - is_true: binary_soft_classification.precision.0\.5
+  - is_true: binary_soft_classification.precision.0\.75
+  - is_true: binary_soft_classification.recall.0\.25
+  - is_true: binary_soft_classification.recall.0\.5
+  - is_true: binary_soft_classification.recall.0\.75
+  - is_true: binary_soft_classification.confusion_matrix.0\.25
+  - is_true: binary_soft_classification.confusion_matrix.0\.5
+  - is_true: binary_soft_classification.confusion_matrix.0\.75
+
+---
+"Test given missing index":
+  - do:
+      catch: /Required \[index\]/
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier",
+                "predicted_probability_field": "outlier_score"
+              }
+            }
+          }
+
+---
+"Test given index does not exist":
+  - do:
+      catch: missing
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "missing_index",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier",
+                "predicted_probability_field": "outlier_score"
+              }
+            }
+          }
+
+---
+"Test given missing evaluation":
+  - do:
+      catch: /Required \[evaluation\]/
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "foo"
+          }
+
+---
+"Test binary_soft_classification given evaluation with emtpy metrics":
+  - do:
+      catch: /\[binary_soft_classification\] must have one or more metrics/
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier",
+                "predicted_probability_field": "outlier_score",
+                "metrics": {
+                }
+              }
+            }
+          }
+
+---
+"Test binary_soft_classification given missing actual_field":
+  - do:
+      catch: /No documents found containing both \[missing, outlier_score\] fields/
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "missing",
+                "predicted_probability_field": "outlier_score"
+              }
+            }
+          }
+
+---
+"Test binary_soft_classification given missing predicted_probability_field":
+  - do:
+      catch: /No documents found containing both \[is_outlier, missing\] fields/
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier",
+                "predicted_probability_field": "missing"
+              }
+            }
+          }
+
+---
+"Test binary_soft_classification given precision with threshold less than zero":
+  - do:
+      catch: /\[precision.at\] values must be in \[0.0, 1.0\]/
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier",
+                "predicted_probability_field": "outlier_score",
+                "metrics": {
+                  "precision": { "at": [ 0.25, -0.1 ]}
+                }
+              }
+            }
+          }
+
+---
+"Test binary_soft_classification given recall with threshold less than zero":
+  - do:
+      catch: /\[recall.at\] values must be in \[0.0, 1.0\]/
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier",
+                "predicted_probability_field": "outlier_score",
+                "metrics": {
+                  "recall": { "at": [ 0.25, -0.1 ]}
+                }
+              }
+            }
+          }
+
+---
+"Test binary_soft_classification given confusion_matrix with threshold less than zero":
+  - do:
+      catch: /\[confusion_matrix.at\] values must be in \[0.0, 1.0\]/
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier",
+                "predicted_probability_field": "outlier_score",
+                "metrics": {
+                  "confusion_matrix": { "at": [ 0.25, -0.1 ]}
+                }
+              }
+            }
+          }
+
+---
+"Test binary_soft_classification given precision with empty thresholds":
+  - do:
+      catch: /\[precision.at\] must have at least one value/
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier",
+                "predicted_probability_field": "outlier_score",
+                "metrics": {
+                  "precision": { "at": []}
+                }
+              }
+            }
+          }
+
+---
+"Test binary_soft_classification given recall with empty thresholds":
+  - do:
+      catch: /\[recall.at\] must have at least one value/
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier",
+                "predicted_probability_field": "outlier_score",
+                "metrics": {
+                  "recall": { "at": []}
+                }
+              }
+            }
+          }
+
+---
+"Test binary_soft_classification given confusion_matrix with empty thresholds":
+  - do:
+      catch: /\[confusion_matrix.at\] must have at least one value/
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia",
+            "evaluation": {
+              "binary_soft_classification": {
+                "actual_field": "is_outlier",
+                "predicted_probability_field": "outlier_score",
+                "metrics": {
+                  "confusion_matrix": { "at": []}
+                }
+              }
+            }
+          }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml
index 1465f1cf0b4d2..d886ec1fb7ff6 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml
@@ -86,7 +86,13 @@ setup:
         from: 1
         size: 1
 
-  - match: { count:   1 }
+  - match: { count:   2 }
+
+  - match:
+      filters.0:
+        filter_id: "filter-foo2"
+        description: "This filter has a description"
+        items: ["123", "lmnop"]
 
 ---
 "Test get filters API with expression ID":
@@ -328,3 +334,18 @@ setup:
       ml.get_filters: {}
   - match: { count: 0 }
   - match: { filters: [] }
+
+---
+"Test get all filter given index exists but no mapping for filter_id":
+
+  - do:
+      indices.delete:
+        index: ".ml-meta"
+  - do:
+      indices.create:
+        index: ".ml-meta"
+
+  - do:
+      ml.get_filters: {}
+  - match: { count: 0 }
+  - match: { filters: [] }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/find_file_structure.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/find_file_structure.yml
index 7c6aff66e3df2..cf175961b1ef9 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/find_file_structure.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/find_file_structure.yml
@@ -10,6 +10,7 @@ setup:
         Content-Type: "application/json"
       ml.find_file_structure:
         lines_to_sample: 3
+        line_merge_size_limit: 1234
         timeout: 10s
         body:
           - airline: AAL
@@ -55,7 +56,8 @@ setup:
   - match: { field_stats.sourcetype.cardinality: 1 }
   - match: { field_stats.time.count: 3 }
   - match: { field_stats.time.cardinality: 3 }
-  - match: { field_stats.time.cardinality: 3 }
+  - match: { field_stats.time.earliest: "1403481600" }
+  - match: { field_stats.time.latest: "1403481800" }
   - is_false: explanation
 
 ---
@@ -115,5 +117,6 @@ setup:
   - match: { field_stats.sourcetype.cardinality: 1 }
   - match: { field_stats.time.count: 3 }
   - match: { field_stats.time.cardinality: 3 }
-  - match: { field_stats.time.cardinality: 3 }
+  - match: { field_stats.time.earliest: "1403481600" }
+  - match: { field_stats.time.latest: "1403481800" }
   - match: { explanation.0: "Using specified character encoding [UTF-8]" }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml
index b8f1c3df0ca9d..f52a2c21f773c 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_stats.yml
@@ -78,6 +78,7 @@ setup:
         body: >
           {"airline":"AAL","responsetime":"132.2046","time":"1403481600"}
           {"airline":"JZA","responsetime":"990.4628","time":"1403481600"}
+          {"airline":"JZA","responsetime":"244.1276","time":"1403485200"}
 
   - do:
       ml.flush_job:
@@ -88,16 +89,22 @@ setup:
   - do:
       ml.get_job_stats:
         job_id: job-stats-test
-  - match: { jobs.0.job_id : job-stats-test }
-  - match: { jobs.0.data_counts.processed_record_count: 2 }
-  - match: { jobs.0.data_counts.processed_field_count: 4 }
-  - match: { jobs.0.data_counts.input_field_count: 4 }
-  - match: { jobs.0.model_size_stats.model_bytes: 0 }
-  - match: { jobs.0.state: opened }
-  - is_true: jobs.0.node.name
-  - is_true: jobs.0.node.transport_address
-  - match: { jobs.0.node.attributes.ml\.max_open_jobs: "20"}
-  - is_true: jobs.0.open_time
+  - match:  { jobs.0.job_id : job-stats-test }
+  - match:  { jobs.0.data_counts.processed_record_count: 3 }
+  - match:  { jobs.0.data_counts.processed_field_count: 6 }
+  - match:  { jobs.0.data_counts.input_field_count: 6 }
+  - gte:    { jobs.0.model_size_stats.model_bytes: 0 }
+  - match:  { jobs.0.state: opened }
+  - is_true:  jobs.0.node.name
+  - is_true:  jobs.0.node.transport_address
+  - match:  { jobs.0.node.attributes.ml\.max_open_jobs: "20"}
+  - is_true:  jobs.0.open_time
+  - match:  { jobs.0.timing_stats.job_id: job-stats-test }
+  - match:  { jobs.0.timing_stats.bucket_count: 1 }  # Records are 1h apart and bucket span is 1h so 1 bucket is produced
+  - gte:    { jobs.0.timing_stats.minimum_bucket_processing_time_ms: 0.0 }
+  - gte:    { jobs.0.timing_stats.maximum_bucket_processing_time_ms: 0.0 }
+  - gte:    { jobs.0.timing_stats.average_bucket_processing_time_ms: 0.0 }
+  - gte:    { jobs.0.timing_stats.exponential_average_bucket_processing_time_ms: 0.0 }
 
 ---
 "Test get job stats for closed job":
@@ -108,6 +115,7 @@ setup:
         body: >
           {"airline":"AAL","responsetime":"132.2046","time":"1403481600"}
           {"airline":"JZA","responsetime":"990.4628","time":"1403481600"}
+          {"airline":"JZA","responsetime":"244.1276","time":"1403485200"}
 
   - do:
       ml.flush_job:
@@ -122,14 +130,20 @@ setup:
   - do:
       ml.get_job_stats:
         job_id: job-stats-test
-  - match: { jobs.0.job_id : job-stats-test }
-  - match: { jobs.0.data_counts.processed_record_count: 2 }
-  - match: { jobs.0.data_counts.processed_field_count: 4}
-  - match: { jobs.0.data_counts.input_field_count: 4 }
-  - gt: { jobs.0.model_size_stats.model_bytes: 0 }
-  - match: { jobs.0.state: closed }
+  - match:  { jobs.0.job_id : job-stats-test }
+  - match:  { jobs.0.data_counts.processed_record_count: 3 }
+  - match:  { jobs.0.data_counts.processed_field_count: 6 }
+  - match:  { jobs.0.data_counts.input_field_count: 6 }
+  - gt:     { jobs.0.model_size_stats.model_bytes: 0 }
+  - match:  { jobs.0.state: closed }
   - is_false: jobs.0.node
   - is_false: jobs.0.open_time
+  - match:  { jobs.0.timing_stats.job_id: job-stats-test }
+  - match:  { jobs.0.timing_stats.bucket_count: 1 }  # Records are 1h apart and bucket span is 1h so 1 bucket is produced
+  - gte:    { jobs.0.timing_stats.minimum_bucket_processing_time_ms: 0.0 }
+  - gte:    { jobs.0.timing_stats.maximum_bucket_processing_time_ms: 0.0 }
+  - gte:    { jobs.0.timing_stats.average_bucket_processing_time_ms: 0.0 }
+  - gte:    { jobs.0.timing_stats.exponential_average_bucket_processing_time_ms: 0.0 }
 
 ---
 "Test get job stats of datafeed job that has not received any data":
@@ -137,11 +151,17 @@ setup:
   - do:
       ml.get_job_stats:
         job_id: jobs-get-stats-datafeed-job
-  - match: { jobs.0.job_id : jobs-get-stats-datafeed-job }
-  - match: { jobs.0.data_counts.processed_record_count: 0 }
-  - match: { jobs.0.model_size_stats.model_bytes : 0 }
-  - match: { jobs.0.state: opened }
-  - is_true: jobs.0.open_time
+  - match:  { jobs.0.job_id : jobs-get-stats-datafeed-job }
+  - match:  { jobs.0.data_counts.processed_record_count: 0 }
+  - match:  { jobs.0.model_size_stats.model_bytes : 0 }
+  - match:  { jobs.0.state: opened }
+  - is_true:  jobs.0.open_time
+  - match:  { jobs.0.timing_stats.job_id: jobs-get-stats-datafeed-job }
+  - match:  { jobs.0.timing_stats.bucket_count: 0 }
+  - is_false: jobs.0.timing_stats.minimum_bucket_processing_time_ms
+  - is_false: jobs.0.timing_stats.maximum_bucket_processing_time_ms
+  - is_false: jobs.0.timing_stats.average_bucket_processing_time_ms
+  - is_false: jobs.0.timing_stats.exponential_average_bucket_processing_time_ms
 
 ---
 "Test get all job stats with _all":
@@ -311,20 +331,32 @@ setup:
 
   - do:
       ml.get_job_stats: {}
-  - match: { count: 2 }
-  - match: { jobs.0.job_id : job-stats-test }
-  - match: { jobs.0.data_counts.processed_record_count: 0 }
-  - match: { jobs.0.data_counts.processed_field_count: 0 }
-  - match: { jobs.0.data_counts.input_field_count: 0 }
-  - match: { jobs.0.model_size_stats.model_bytes: 0 }
-  - match: { jobs.0.state: closed }
+  - match:  { count: 2 }
+  - match:  { jobs.0.job_id : job-stats-test }
+  - match:  { jobs.0.data_counts.processed_record_count: 0 }
+  - match:  { jobs.0.data_counts.processed_field_count: 0 }
+  - match:  { jobs.0.data_counts.input_field_count: 0 }
+  - match:  { jobs.0.model_size_stats.model_bytes: 0 }
+  - match:  { jobs.0.state: closed }
   - is_false: jobs.0.node
   - is_false: jobs.0.open_time
-  - match: { jobs.1.job_id : jobs-get-stats-datafeed-job }
-  - match: { jobs.1.data_counts.processed_record_count: 0 }
-  - match: { jobs.1.data_counts.processed_field_count: 0 }
-  - match: { jobs.1.data_counts.input_field_count: 0 }
-  - match: { jobs.1.model_size_stats.model_bytes: 0 }
-  - match: { jobs.1.state: closed }
+  - match:  { jobs.0.timing_stats.job_id: job-stats-test }
+  - match:  { jobs.0.timing_stats.bucket_count: 0 }
+  - is_false: jobs.0.timing_stats.minimum_bucket_processing_time_ms
+  - is_false: jobs.0.timing_stats.maximum_bucket_processing_time_ms
+  - is_false: jobs.0.timing_stats.average_bucket_processing_time_ms
+  - is_false: jobs.0.timing_stats.exponential_average_bucket_processing_time_ms
+  - match:  { jobs.1.job_id : jobs-get-stats-datafeed-job }
+  - match:  { jobs.1.data_counts.processed_record_count: 0 }
+  - match:  { jobs.1.data_counts.processed_field_count: 0 }
+  - match:  { jobs.1.data_counts.input_field_count: 0 }
+  - match:  { jobs.1.model_size_stats.model_bytes: 0 }
+  - match:  { jobs.1.state: closed }
   - is_false: jobs.1.node
   - is_false: jobs.1.open_time
+  - match:  { jobs.1.timing_stats.job_id: jobs-get-stats-datafeed-job }
+  - match:  { jobs.1.timing_stats.bucket_count: 0 }
+  - is_false: jobs.1.timing_stats.minimum_bucket_processing_time_ms
+  - is_false: jobs.1.timing_stats.maximum_bucket_processing_time_ms
+  - is_false: jobs.1.timing_stats.average_bucket_processing_time_ms
+  - is_false: jobs.1.timing_stats.exponential_average_bucket_processing_time_ms
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml
new file mode 100644
index 0000000000000..6a8c6d0e6ed00
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml
@@ -0,0 +1,74 @@
+---
+"Test start given missing config":
+  - do:
+      catch: missing
+      ml.start_data_frame_analytics:
+        id: "missing_config"
+
+---
+"Test start given missing source index":
+
+  - do:
+      indices.create:
+        index: missing
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "missing_index"
+        body: >
+          {
+            "source": {
+              "index": "missing"
+            },
+            "dest": {
+              "index": "missing-dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+  - do:
+      indices.delete:
+        index: missing
+
+  - do:
+      catch: /No index matches source index \[missing]/
+      ml.start_data_frame_analytics:
+        id: "missing_index"
+
+---
+"Test start given source index has no compatible fields":
+
+  - do:
+      indices.create:
+        index: empty-index
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "foo"
+        body: >
+          {
+            "source": {
+              "index": "empty-index"
+            },
+            "dest": {
+              "index": "empty-index-dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+  - do:
+      catch: /No compatible fields could be detected in index \[empty-index\]/
+      ml.start_data_frame_analytics:
+        id: "foo"
+
+---
+"Test start with inconsistent body/param ids":
+
+  - do:
+      catch: /Inconsistent id; 'body_id' specified in the body differs from 'url_id' specified as a URL argument/
+      ml.start_data_frame_analytics:
+        id: "url_id"
+        body: >
+          {
+            "id": "body_id"
+          }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/stop_data_frame_analytics.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/stop_data_frame_analytics.yml
new file mode 100644
index 0000000000000..382fd59d35ad9
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/stop_data_frame_analytics.yml
@@ -0,0 +1,70 @@
+setup:
+  - do:
+      indices.create:
+        index: index-source
+
+  - do:
+      ml.put_data_frame_analytics:
+        id: "stop_data_frame_analytics_test_job"
+        body: >
+          {
+            "source": {
+              "index": "index-source"
+            },
+            "dest": {
+              "index": "index-dest"
+            },
+            "analysis": {"outlier_detection":{}}
+          }
+
+---
+"Test stop given missing config and allow_no_match is true":
+  - do:
+      catch: missing
+      ml.stop_data_frame_analytics:
+        id: "missing_config"
+        allow_no_match: true
+
+---
+"Test stop given missing config and allow_no_match is false":
+  - do:
+      catch: missing
+      ml.stop_data_frame_analytics:
+        id: "missing_config"
+        allow_no_match: false
+
+---
+"Test stop with expression that does not match and allow_no_match is true":
+  - do:
+      ml.stop_data_frame_analytics:
+        id: "missing-*"
+        allow_no_match: true
+  - match: { stopped: true }
+
+---
+"Test stop with expression that does not match and allow_no_match is false":
+  - do:
+      catch: missing
+      ml.stop_data_frame_analytics:
+        id: "missing-*"
+        allow_no_match: false
+
+---
+"Test stop given stopped":
+
+  - do:
+      ml.stop_data_frame_analytics:
+        id: "stop_data_frame_analytics_test_job"
+  - match: { stopped: true }
+
+---
+"Test stop with inconsistent body/param ids":
+
+  - do:
+      catch: /Inconsistent id; 'body_id' specified in the body differs from 'url_id' specified as a URL argument/
+      ml.stop_data_frame_analytics:
+        id: "url_id"
+        body: >
+          {
+            "id": "body_id"
+          }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/11_builtin.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/11_builtin.yml
new file mode 100644
index 0000000000000..66a2c801ff79f
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/11_builtin.yml
@@ -0,0 +1,19 @@
+---
+setup:
+  - skip:
+      features: headers
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
+---
+"Test get builtin privileges":
+  - do:
+      security.get_builtin_privileges: {}
+
+  # This is fragile - it needs to be updated every time we add a new cluster/index privilege
+  # I would much prefer we could just check that specific entries are in the array, but we don't have
+  # an assertion for that
+  - length: { "cluster" : 26 }
+  - length: { "index" : 16 }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/start_job.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/start_job.yml
index 371f7c7207fa3..3ff1c1bb6b4d7 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/start_job.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/start_job.yml
@@ -59,9 +59,10 @@ setup:
   - is_true: started
 
   - do:
-      catch: /Cannot start task for Rollup Job \[foo\] because state was/
       headers:
         Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
       rollup.start_job:
         id: foo
+  - is_true: started
+
 
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/14_cat_indices.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/14_cat_indices.yml
index e4562d731ae75..ba50ac04bcb1f 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/14_cat_indices.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/14_cat_indices.yml
@@ -44,6 +44,18 @@ setup:
             number_of_shards: "1"
             number_of_replicas: "0"
 
+  - do:
+      indices.create:
+        index: index3
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+
+  - do:
+      indices.close:
+        index:  index3
+
 ---
 teardown:
   - do:
@@ -102,6 +114,58 @@ teardown:
                 )
                 $/
 
+---
+"Test empty request while single authorized closed index":
+
+  - do:
+      indices.create:
+        index: index_to_monitor
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 1
+
+  - do:
+      index:
+        index:  index_to_monitor
+        id:     0
+        body:   { foo: bar }
+
+  - do:
+      index:
+        index:  index_to_monitor
+        id:     1
+        body:   { foo: bar }
+
+  - do:
+      index:
+        index:  index_to_monitor
+        id:     2
+        body:   { foo: bar }
+
+  - do:
+      indices.close:
+        index:  index_to_monitor
+
+  - do:
+      headers: { Authorization: "Basic Y2F0X3VzZXI6Y2F0X3Bhc3N3b3Jk" } # cat_user
+      cat.indices: {}
+
+  - match:
+      $body: |
+        /^(yellow  \s+
+           close   \s+
+           index_to_monitor \s+
+           ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
+           1      \s+
+           1      \s+
+                  \s+
+                  \s+
+                  \s+
+                  \s*
+         )
+         $/
+
 ---
 "Test explicit request while multiple authorized indices":
 
@@ -145,6 +209,67 @@ teardown:
       cat.indices:
         index: "this_*,index2"
 
+---
+"Test explicit request while multiple opened/closed authorized indices":
+
+  - do:
+      indices.create:
+        index: index_to_monitor
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+
+  - do:
+      indices.create:
+        index: this_index
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+
+  - do:
+      indices.create:
+        index: this_index_is_closed
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+
+  - do:
+      headers: { Authorization: "Basic Y2F0X3VzZXI6Y2F0X3Bhc3N3b3Jk" } # cat_user
+      cat.indices:
+        index: "this_index,index_to_monitor,this_index_is_closed"
+        v: false
+        h: i
+
+  - match:
+      $body: |
+        /^(this_index \s*\n index_to_monitor \s*\n this_index_is_closed \n?)
+          |(this_index \s*\n this_index_is_closed \s*\n index_to_monitor \n?)
+          |(this_index_is_closed \s*\n this_index \s*\n index_to_monitor \n?)
+          |(this_index_is_closed \s*\n index_to_monitor \s*\n this_index \n?)
+          |(index_to_monitor \s*\n this_index \s*\n this_index_is_closed \n?)
+          |(index_to_monitor \s*\n this_index_is_closed \s*\n this_index \n?)$/
+
+  - do:
+      catch: forbidden
+      headers: { Authorization: "Basic Y2F0X3VzZXI6Y2F0X3Bhc3N3b3Jk" } # cat_user
+      cat.indices:
+        index: "index1,index_to_monitor"
+
+  - do:
+      catch: forbidden
+      headers: { Authorization: "Basic Y2F0X3VzZXI6Y2F0X3Bhc3N3b3Jk" } # cat_user
+      cat.indices:
+        index: "this_*,index2"
+
+  - do:
+      catch: forbidden
+      headers: { Authorization: "Basic Y2F0X3VzZXI6Y2F0X3Bhc3N3b3Jk" } # cat_user
+      cat.indices:
+        index: "this_index_is_closed,index2"
+
 ---
 "Test wildcard request with multiple authorized indices":
 
diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/10_dense_vector_basic.yml
similarity index 95%
rename from modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/10_basic.yml
rename to x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/10_dense_vector_basic.yml
index 320d9ce1fa331..903b9dc3de3b0 100644
--- a/modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/10_basic.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/10_dense_vector_basic.yml
@@ -1,8 +1,8 @@
 setup:
   - skip:
       features: headers
-      version: " - 7.1.99"
-      reason: "dense_vector functions were introduced in 7.2.0"
+      version: " - 7.2.99"
+      reason: "dense_vector functions were added from 7.3"
 
   - do:
       indices.create:
@@ -15,6 +15,7 @@ setup:
             properties:
               my_dense_vector:
                  type: dense_vector
+                 dims: 5
   - do:
       index:
         index: test-index
diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/20_special_cases.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/20_dense_vector_special_cases.yml
similarity index 60%
rename from modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/20_special_cases.yml
rename to x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/20_dense_vector_special_cases.yml
index 0df3a8fcaadc8..22cfd5169686d 100644
--- a/modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/20_special_cases.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/20_dense_vector_special_cases.yml
@@ -1,8 +1,8 @@
 setup:
   - skip:
       features: headers
-      version: " - 7.1.99"
-      reason: "dense_vector functions were introduced in 7.2.0"
+      version: " - 7.2.99"
+      reason: "dense_vector functions were added from 7.3"
 
   - do:
       indices.create:
@@ -17,31 +17,36 @@ setup:
             properties:
               my_dense_vector:
                  type: dense_vector
+                 dims: 3
 
 
 ---
-"Vectors of different dimensions and data types":
-# document vectors of different dimensions
+"Indexing of Dense vectors should error when dims don't match defined in the mapping":
+
   - do:
+      catch: bad_request
       index:
         index: test-index
         id: 1
         body:
-          my_dense_vector: [10]
+          my_dense_vector: [10, 2]
+  - match: { error.type: "mapper_parsing_exception" }
 
+---
+"Vectors of mixed integers and floats":
   - do:
       index:
         index: test-index
-        id: 2
+        id: 1
         body:
-          my_dense_vector: [10, 10.5]
+          my_dense_vector: [10, 10, 10]
 
   - do:
       index:
         index: test-index
-        id: 3
+        id: 2
         body:
-          my_dense_vector: [10, 10.5, 100.5]
+          my_dense_vector: [10.9, 10.9, 10.9]
 
   - do:
       indices.refresh: {}
@@ -59,14 +64,13 @@ setup:
               script:
                 source: "cosineSimilarity(params.query_vector, doc['my_dense_vector'])"
                 params:
-                  query_vector: [10]
+                  query_vector: [10, 10, 10]
 
-  - match: {hits.total: 3}
+  - match: {hits.total: 2}
   - match: {hits.hits.0._id: "1"}
   - match: {hits.hits.1._id: "2"}
-  - match: {hits.hits.2._id: "3"}
 
-# query vector of type double
+# query vector of type float
   - do:
       headers:
         Content-Type: application/json
@@ -79,12 +83,52 @@ setup:
               script:
                 source: "cosineSimilarity(params.query_vector, doc['my_dense_vector'])"
                 params:
-                  query_vector: [10.0]
+                  query_vector: [10.0, 10.0, 10.0]
 
-  - match: {hits.total: 3}
+  - match: {hits.total: 2}
   - match: {hits.hits.0._id: "1"}
   - match: {hits.hits.1._id: "2"}
-  - match: {hits.hits.2._id: "3"}
+
+
+---
+"Functions with query vectors with dims different from docs vectors should error":
+  - do:
+      index:
+        index: test-index
+        id: 1
+        body:
+          my_dense_vector: [1, 2, 3]
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch: bad_request
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query:
+            script_score:
+              query: {match_all: {} }
+              script:
+                source: "cosineSimilarity(params.query_vector, doc['my_dense_vector'])"
+                params:
+                  query_vector: [1, 2, 3, 4]
+  - match: { error.root_cause.0.type: "script_exception" }
+
+  - do:
+      catch: bad_request
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query:
+            script_score:
+              query: {match_all: {} }
+              script:
+                source: "dotProduct(params.query_vector, doc['my_dense_vector'])"
+                params:
+                  query_vector: [1, 2, 3, 4]
+  - match: { error.root_cause.0.type: "script_exception" }
 
 ---
 "Distance functions for documents missing vector field should return 0":
@@ -93,7 +137,7 @@ setup:
       index: test-index
       id: 1
       body:
-        my_dense_vector: [10]
+        my_dense_vector: [10, 10, 10]
 
 - do:
     index:
@@ -117,7 +161,7 @@ setup:
             script:
               source: "cosineSimilarity(params.query_vector, doc['my_dense_vector'])"
               params:
-                query_vector: [10.0]
+                query_vector: [10.0, 10.0, 10.0]
 
 - match: {hits.total: 2}
 - match: {hits.hits.0._id: "1"}
@@ -149,5 +193,5 @@ setup:
             script:
               source: "dotProductSparse(params.query_vector, doc['my_dense_vector'])"
               params:
-                query_vector: {"2": 0.5, "10" : 111.3}
+                query_vector: {"2": 0.5, "10" : 111.3, "3": 44}
 - match: { error.root_cause.0.type: "script_exception" }
diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/30_sparse_vector_basic.yml
similarity index 96%
rename from modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/10_basic.yml
rename to x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/30_sparse_vector_basic.yml
index b1330bbe852d3..b031a0e2b618b 100644
--- a/modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/10_basic.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/30_sparse_vector_basic.yml
@@ -1,8 +1,8 @@
 setup:
   - skip:
       features: headers
-      version: " - 7.1.99"
-      reason: "sparse_vector functions were introduced in 7.2.0"
+      version: " - 7.2.99"
+      reason: "sparse_vector functions were introduced in 7.3.0"
 
   - do:
       indices.create:
diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/20_special_cases.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/40_sparse_vector_special_cases.yml
similarity index 98%
rename from modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/20_special_cases.yml
rename to x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/40_sparse_vector_special_cases.yml
index d2ae0bb1a7549..615da302cae69 100644
--- a/modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/20_special_cases.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/40_sparse_vector_special_cases.yml
@@ -1,8 +1,8 @@
 setup:
   - skip:
       features: headers
-      version: " - 7.1.99"
-      reason: "sparse_vector functions were introduced in 7.2.0"
+      version: " - 7.2.99"
+      reason: "sparse_vector functions were introduced in 7.3.0"
 
   - do:
       indices.create:
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/voting_only_node/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/voting_only_node/10_basic.yml
new file mode 100644
index 0000000000000..bba86b2ac0cc2
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/voting_only_node/10_basic.yml
@@ -0,0 +1,16 @@
+---
+"cluster stats with voting only node stats":
+
+  - skip:
+      version: " - 7.99.99"
+      reason:  "voting only nodes are added in v8.0.0"
+
+  - do:
+      cluster.stats: {}
+
+  - gte: { nodes.count.total: 1}
+  - gte: { nodes.count.master: 1}
+  - gte: { nodes.count.data: 1}
+  - gte: { nodes.count.ingest: 0}
+  - gte: { nodes.count.coordinating_only: 0}
+  - gte: { nodes.count.voting_only: 0}
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/execute_watch/80_foreach.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/execute_watch/80_foreach.yml
new file mode 100644
index 0000000000000..0c5ec6fbcbacc
--- /dev/null
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/execute_watch/80_foreach.yml
@@ -0,0 +1,111 @@
+---
+setup:
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+
+---
+"Test execute watch api with foreach action":
+  - do:
+      watcher.execute_watch:
+        body:  >
+          {
+            "watch" : {
+              "trigger": {
+                "schedule" : { "cron" : "0 0 0 1 * ? 2099" }
+              },
+              "input": {
+                "simple" : {
+                  "hits" : {
+                    "hits" : [
+                    { "key" : "first" },
+                    { "key" : "second" },
+                    { "key" : "third" }
+                    ]
+                  }
+                }
+              },
+              "actions": {
+                "log_hits" : {
+                  "foreach" : "ctx.payload.hits.hits",
+                  "logging" : {
+                    "text" : "Logging {{ctx.payload.key}}"
+                  }
+                }
+              }
+            }
+          }
+
+  - match: { watch_record.trigger_event.type: "manual" }
+  - match: { watch_record.state: "executed" }
+  - match: { watch_record.status.execution_state: "executed" }
+  - match: { watch_record.result.actions.0.foreach.0.logging.logged_text: "Logging first" }
+  - match: { watch_record.result.actions.0.foreach.1.logging.logged_text: "Logging second" }
+  - match: { watch_record.result.actions.0.foreach.2.logging.logged_text: "Logging third" }
+
+
+---
+"Test execute watch api with foreach action using an array":
+  - do:
+      watcher.execute_watch:
+        body:  >
+          {
+            "watch" : {
+              "trigger": {
+                "schedule" : { "cron" : "0 0 0 1 * ? 2099" }
+              },
+              "input": {
+                "simple" : {
+                  "values" : [1, 2, 3]
+                }
+              },
+              "actions": {
+                "log_hits" : {
+                  "foreach" : "ctx.payload.values",
+                  "logging" : {
+                    "text" : "Logging {{ctx.payload._value}}"
+                  }
+                }
+              }
+            }
+          }
+
+  - match: { watch_record.trigger_event.type: "manual" }
+  - match: { watch_record.state: "executed" }
+  - match: { watch_record.status.execution_state: "executed" }
+  - match: { watch_record.result.actions.0.foreach.0.logging.logged_text: "Logging 1" }
+  - match: { watch_record.result.actions.0.foreach.1.logging.logged_text: "Logging 2" }
+  - match: { watch_record.result.actions.0.foreach.2.logging.logged_text: "Logging 3" }
+
+---
+"Test execute watch api with foreach action using an array of arrays":
+  - do:
+      watcher.execute_watch:
+        body:  >
+          {
+            "watch" : {
+              "trigger": {
+                "schedule" : { "cron" : "0 0 0 1 * ? 2099" }
+              },
+              "input": {
+                "simple" : {
+                  "values" : [[1, 2], [2, 3], [3, 4]]
+                }
+              },
+              "actions": {
+                "log_hits" : {
+                  "foreach" : "ctx.payload.values",
+                  "logging" : {
+                    "text" : "Logging {{ctx.payload._value.1}}"
+                  }
+                }
+              }
+            }
+          }
+
+  - match: { watch_record.trigger_event.type: "manual" }
+  - match: { watch_record.state: "executed" }
+  - match: { watch_record.status.execution_state: "executed" }
+  - match: { watch_record.result.actions.0.foreach.0.logging.logged_text: "Logging 2" }
+  - match: { watch_record.result.actions.0.foreach.1.logging.logged_text: "Logging 3" }
+  - match: { watch_record.result.actions.0.foreach.2.logging.logged_text: "Logging 4" }
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/15_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/15_basic.yml
index 213d935a5c52f..1f2e5ce9625e8 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/15_basic.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/xpack/15_basic.yml
@@ -68,19 +68,15 @@
   - is_true:    features.watcher
   - is_true:    features.watcher.enabled
   - is_true:    features.watcher.available
-  - is_true:    features.watcher.description
   - is_true:    features.security
   - is_true:    features.security.enabled
   - is_true:    features.security.available
-  - is_true:    features.security.description
   - is_true:    features.graph
   - is_true:    features.graph.enabled
   - is_true:    features.graph.available
-  - is_true:    features.graph.description
   - is_true:    features.monitoring
   - is_true:    features.monitoring.enabled
   - is_true:    features.monitoring.available
-  - is_true:    features.monitoring.description
   - is_true:    tagline
 
   - do:
@@ -152,19 +148,15 @@
   - is_true:    features.watcher
   - is_true:    features.watcher.enabled
   - is_true:    features.watcher.available
-  - is_false:   features.watcher.description
   - is_true:    features.security
   - is_true:    features.security.enabled
   - is_true:    features.security.available
-  - is_false:   features.security.description
   - is_true:    features.graph
   - is_true:    features.graph.enabled
   - is_true:    features.graph.available
-  - is_false:   features.graph.description
   - is_true:    features.monitoring
   - is_true:    features.monitoring.enabled
   - is_true:    features.monitoring.available
-  - is_false:   features.monitoring.description
   - is_false:   tagline
 
 
diff --git a/x-pack/plugin/vectors/build.gradle b/x-pack/plugin/vectors/build.gradle
new file mode 100644
index 0000000000000..e780bd7f8a1a2
--- /dev/null
+++ b/x-pack/plugin/vectors/build.gradle
@@ -0,0 +1,22 @@
+evaluationDependsOn(xpackModule('core'))
+
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+    name 'vectors'
+    description 'A plugin for working with vectors'
+    classname 'org.elasticsearch.xpack.vectors.Vectors'
+    extendedPlugins = ['x-pack-core', 'lang-painless']
+}
+archivesBaseName = 'x-pack-vectors'
+
+dependencies {
+    compileOnly project(':modules:lang-painless:spi')
+    compileOnly project(path: xpackModule('core'), configuration: 'default')
+    testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+    if (isEclipse) {
+        testCompile project(path: xpackModule('core-tests'), configuration: 'testArtifacts')
+    }
+}
+
+integTest.enabled = false
diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/Vectors.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/Vectors.java
new file mode 100644
index 0000000000000..6e714984077ca
--- /dev/null
+++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/Vectors.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.vectors;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.vectors.mapper.DenseVectorFieldMapper;
+import org.elasticsearch.xpack.vectors.mapper.SparseVectorFieldMapper;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+
+public class Vectors extends Plugin implements MapperPlugin, ActionPlugin {
+
+    protected final boolean enabled;
+
+    public Vectors(Settings settings) {
+        this.enabled = XPackSettings.VECTORS_ENABLED.get(settings);
+    }
+
+    @Override
+    public List<ActionPlugin.ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        return Arrays.asList(
+            new ActionPlugin.ActionHandler<>(XPackUsageFeatureAction.VECTORS, VectorsUsageTransportAction.class),
+            new ActionPlugin.ActionHandler<>(XPackInfoFeatureAction.VECTORS, VectorsInfoTransportAction.class));
+    }
+
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        if (enabled == false) {
+            return emptyMap();
+        }
+        Map<String, Mapper.TypeParser> mappers = new LinkedHashMap<>();
+        mappers.put(DenseVectorFieldMapper.CONTENT_TYPE, new DenseVectorFieldMapper.TypeParser());
+        mappers.put(SparseVectorFieldMapper.CONTENT_TYPE, new SparseVectorFieldMapper.TypeParser());
+        return Collections.unmodifiableMap(mappers);
+    }
+}
diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/VectorsInfoTransportAction.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/VectorsInfoTransportAction.java
new file mode 100644
index 0000000000000..7974b6ec246f3
--- /dev/null
+++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/VectorsInfoTransportAction.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.vectors;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
+
+public class VectorsInfoTransportAction extends XPackInfoFeatureTransportAction {
+
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public VectorsInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                      Settings settings, XPackLicenseState licenseState) {
+        super(XPackInfoFeatureAction.VECTORS.name(), transportService, actionFilters);
+        this.enabled = XPackSettings.VECTORS_ENABLED.get(settings);
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    public String name() {
+        return XPackField.VECTORS;
+    }
+
+    @Override
+    public boolean available() {
+        return licenseState.isVectorsAllowed();
+    }
+
+    @Override
+    public boolean enabled() {
+        return enabled;
+    }
+
+}
diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/VectorsUsageTransportAction.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/VectorsUsageTransportAction.java
new file mode 100644
index 0000000000000..8c16d651f47ee
--- /dev/null
+++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/VectorsUsageTransportAction.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.vectors;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.vectors.VectorsFeatureSetUsage;
+
+public class VectorsUsageTransportAction extends XPackUsageFeatureTransportAction {
+
+    private final Settings settings;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public VectorsUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                       ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                       Settings settings, XPackLicenseState licenseState) {
+        super(XPackUsageFeatureAction.VECTORS.name(), transportService, clusterService,
+            threadPool, actionFilters, indexNameExpressionResolver);
+        this.settings = settings;
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
+        VectorsFeatureSetUsage usage =
+            new VectorsFeatureSetUsage(licenseState.isVectorsAllowed(), XPackSettings.VECTORS_ENABLED.get(settings));
+        listener.onResponse(new XPackUsageFeatureResponse(usage));
+    }
+}
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java
similarity index 67%
rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/DenseVectorFieldMapper.java
rename to x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java
index d48a457ba08cd..a7773e3e3c527 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/DenseVectorFieldMapper.java
+++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java
@@ -1,36 +1,31 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.index.mapper;
+
+package org.elasticsearch.xpack.vectors.mapper;
 
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.mapper.ArrayValueMapperParser;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.query.VectorDVIndexFieldData;
+import org.elasticsearch.xpack.vectors.query.VectorDVIndexFieldData;
 import org.elasticsearch.search.DocValueFormat;
 
 import java.io.IOException;
@@ -62,12 +57,28 @@ public static class Defaults {
     }
 
     public static class Builder extends FieldMapper.Builder<Builder, DenseVectorFieldMapper> {
+        private int dims = 0;
 
         public Builder(String name) {
             super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
             builder = this;
         }
 
+        public Builder dims(int dims) {
+            if ((dims > MAX_DIMS_COUNT) || (dims < 1)) {
+                throw new MapperParsingException("The number of dimensions for field [" + name +
+                    "] should be in the range [1, " + MAX_DIMS_COUNT + "]");
+            }
+            this.dims = dims;
+            return this;
+        }
+
+        @Override
+        protected void setupFieldType(BuilderContext context) {
+            super.setupFieldType(context);
+            fieldType().setDims(dims);
+        }
+
         @Override
         public DenseVectorFieldType fieldType() {
             return (DenseVectorFieldType) super.fieldType();
@@ -86,11 +97,17 @@ public static class TypeParser implements Mapper.TypeParser {
         @Override
         public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             DenseVectorFieldMapper.Builder builder = new DenseVectorFieldMapper.Builder(name);
-            return builder;
+            Object dimsField = node.remove("dims");
+            if (dimsField == null) {
+                throw new MapperParsingException("The [dims] property must be specified for field [" + name + "].");
+            }
+            int dims = XContentMapValues.nodeIntegerValue(dimsField);
+            return builder.dims(dims);
         }
     }
 
     public static final class DenseVectorFieldType extends MappedFieldType {
+        private int dims;
 
         public DenseVectorFieldType() {}
 
@@ -102,6 +119,14 @@ public DenseVectorFieldType clone() {
             return new DenseVectorFieldType(this);
         }
 
+        int dims() {
+            return dims;
+        }
+
+        void setDims(int dims) {
+            this.dims = dims;
+        }
+
         @Override
         public String typeName() {
             return CONTENT_TYPE;
@@ -151,28 +176,30 @@ public void parse(ParseContext context) throws IOException {
         if (context.externalValueSet()) {
             throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] can't be used in multi-fields");
         }
+        int dims = fieldType().dims(); //number of vector dimensions
 
         // encode array of floats as array of integers and store into buf
         // this code is here and not int the VectorEncoderDecoder so not to create extra arrays
-        byte[] buf = new byte[0];
+        byte[] buf = new byte[dims * INT_BYTES];
         int offset = 0;
         int dim = 0;
         for (Token token = context.parser().nextToken(); token != Token.END_ARRAY; token = context.parser().nextToken()) {
+            if (dim++ >= dims) {
+                throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] of doc [" +
+                    context.sourceToParse().id() + "] has exceeded the number of dimensions [" + dims + "] defined in mapping");
+            }
             ensureExpectedToken(Token.VALUE_NUMBER, token, context.parser()::getTokenLocation);
             float value = context.parser().floatValue(true);
-            if (buf.length < (offset + INT_BYTES)) {
-                buf = ArrayUtil.grow(buf, (offset + INT_BYTES));
-            }
             int intValue = Float.floatToIntBits(value);
-            buf[offset] =  (byte) (intValue >> 24);
-            buf[offset+1] = (byte) (intValue >> 16);
-            buf[offset+2] = (byte) (intValue >>  8);
-            buf[offset+3] = (byte) intValue;
-            offset += INT_BYTES;
-            if (dim++ >= MAX_DIMS_COUNT) {
-                throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() +
-                    "] has exceeded the maximum allowed number of dimensions of [" + MAX_DIMS_COUNT + "]");
-            }
+            buf[offset++] = (byte) (intValue >> 24);
+            buf[offset++] = (byte) (intValue >> 16);
+            buf[offset++] = (byte) (intValue >>  8);
+            buf[offset++] = (byte) intValue;
+        }
+        if (dim != dims) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] of doc [" +
+                context.sourceToParse().id() + "] has number of dimensions [" + dim +
+                "] less than defined in the mapping [" +  dims +"]");
         }
         BinaryDocValuesField field = new BinaryDocValuesField(fieldType().name(), new BytesRef(buf, 0, offset));
         if (context.doc().getByKey(fieldType().name()) != null) {
@@ -182,6 +209,12 @@ public void parse(ParseContext context) throws IOException {
         context.doc().addWithKey(fieldType().name(), field);
     }
 
+    @Override
+    protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
+        super.doXContentBody(builder, includeDefaults, params);
+        builder.field("dims", fieldType().dims());
+    }
+
     @Override
     protected void parseCreateField(ParseContext context, List<IndexableField> fields) {
         throw new AssertionError("parse is implemented directly");
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SparseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java
similarity index 89%
rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SparseVectorFieldMapper.java
rename to x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java
index 931e27bc1c19f..3c551a4ee525f 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SparseVectorFieldMapper.java
+++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java
@@ -1,23 +1,11 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.index.mapper;
+
+package org.elasticsearch.xpack.vectors.mapper;
 
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.index.IndexOptions;
@@ -29,8 +17,13 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.query.VectorDVIndexFieldData;
+import org.elasticsearch.xpack.vectors.query.VectorDVIndexFieldData;
 import org.elasticsearch.search.DocValueFormat;
 
 import java.io.IOException;
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/VectorEncoderDecoder.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/VectorEncoderDecoder.java
similarity index 84%
rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/VectorEncoderDecoder.java
rename to x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/VectorEncoderDecoder.java
index fbf9955f46621..31b94ae108e63 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/VectorEncoderDecoder.java
+++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/VectorEncoderDecoder.java
@@ -1,23 +1,11 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.index.mapper;
+
+package org.elasticsearch.xpack.vectors.mapper;
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.InPlaceMergeSorter;
@@ -174,12 +162,11 @@ public static float[] decodeDenseVector(BytesRef vectorBR) {
         float[] vector = new float[dimCount];
         int offset = vectorBR.offset;
         for (int dim = 0; dim < dimCount; dim++) {
-            int intValue = ((vectorBR.bytes[offset] & 0xFF) << 24)   |
-                ((vectorBR.bytes[offset+1] & 0xFF) << 16) |
-                ((vectorBR.bytes[offset+2] & 0xFF) <<  8) |
-                (vectorBR.bytes[offset+3] & 0xFF);
+            int intValue = ((vectorBR.bytes[offset++] & 0xFF) << 24)   |
+                ((vectorBR.bytes[offset++] & 0xFF) << 16) |
+                ((vectorBR.bytes[offset++] & 0xFF) <<  8) |
+                (vectorBR.bytes[offset++] & 0xFF);
             vector[dim] = Float.intBitsToFloat(intValue);
-            offset = offset + INT_BYTES;
         }
         return vector;
     }
diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DocValuesWhitelistExtension.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DocValuesWhitelistExtension.java
new file mode 100644
index 0000000000000..7d8d1d97da92e
--- /dev/null
+++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DocValuesWhitelistExtension.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+package org.elasticsearch.xpack.vectors.query;
+
+
+import org.elasticsearch.painless.spi.PainlessExtension;
+import org.elasticsearch.painless.spi.Whitelist;
+import org.elasticsearch.painless.spi.WhitelistLoader;
+import org.elasticsearch.script.ScoreScript;
+import org.elasticsearch.script.ScriptContext;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class DocValuesWhitelistExtension implements PainlessExtension {
+
+    private static final Whitelist WHITELIST =
+        WhitelistLoader.loadFromResourceFiles(DocValuesWhitelistExtension.class, "whitelist.txt");
+
+    @Override
+    public Map<ScriptContext<?>, List<Whitelist>> getContextWhitelists() {
+        return Collections.singletonMap(ScoreScript.CONTEXT, Collections.singletonList(WHITELIST));
+    }
+}
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/ScoreScriptUtils.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/ScoreScriptUtils.java
similarity index 87%
rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/query/ScoreScriptUtils.java
rename to x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/ScoreScriptUtils.java
index 93e80d2a653fb..fcb02cee68822 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/ScoreScriptUtils.java
+++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/ScoreScriptUtils.java
@@ -1,32 +1,20 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.index.query;
+
+package org.elasticsearch.xpack.vectors.query;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.mapper.VectorEncoderDecoder;
+import org.elasticsearch.xpack.vectors.mapper.VectorEncoderDecoder;
 
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.index.mapper.VectorEncoderDecoder.sortSparseDimsDoubleValues;
+import static org.elasticsearch.xpack.vectors.mapper.VectorEncoderDecoder.sortSparseDimsDoubleValues;
 
 public class ScoreScriptUtils {
 
@@ -42,6 +30,10 @@ public static double dotProduct(List<Number> queryVector, VectorScriptDocValues.
         BytesRef value = dvs.getEncodedValue();
         if (value == null) return 0;
         float[] docVector = VectorEncoderDecoder.decodeDenseVector(value);
+        if (queryVector.size() != docVector.length) {
+            throw new IllegalArgumentException("Can't calculate dotProduct! The number of dimensions of the query vector [" +
+                queryVector.size() + "] is different from the documents' vectors [" + docVector.length + "].");
+        }
         return intDotProduct(queryVector, docVector);
     }
 
@@ -73,6 +65,10 @@ public double cosineSimilarity(VectorScriptDocValues.DenseVectorScriptDocValues
             BytesRef value = dvs.getEncodedValue();
             if (value == null) return 0;
             float[] docVector = VectorEncoderDecoder.decodeDenseVector(value);
+            if (queryVector.size() != docVector.length) {
+                throw new IllegalArgumentException("Can't calculate cosineSimilarity! The number of dimensions of the query vector [" +
+                    queryVector.size() + "] is different from the documents' vectors [" + docVector.length + "].");
+            }
 
             // calculate docVector magnitude
             double dotProduct = 0f;
@@ -87,13 +83,10 @@ public double cosineSimilarity(VectorScriptDocValues.DenseVectorScriptDocValues
     }
 
     private static double intDotProduct(List<Number> v1, float[] v2){
-        int dims = Math.min(v1.size(), v2.length);
         double v1v2DotProduct = 0;
-        int dim = 0;
         Iterator<Number> v1Iter = v1.iterator();
-        while(dim < dims) {
+        for (int dim = 0; dim < v2.length; dim++) {
             v1v2DotProduct += v1Iter.next().doubleValue() * v2[dim];
-            dim++;
         }
         return v1v2DotProduct;
     }
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorDVAtomicFieldData.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVAtomicFieldData.java
similarity index 69%
rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorDVAtomicFieldData.java
rename to x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVAtomicFieldData.java
index 99e581ce4e514..201bf00de1ea6 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorDVAtomicFieldData.java
+++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVAtomicFieldData.java
@@ -1,23 +1,11 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.index.query;
+
+package org.elasticsearch.xpack.vectors.query;
 
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DocValues;
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorDVIndexFieldData.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVIndexFieldData.java
similarity index 72%
rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorDVIndexFieldData.java
rename to x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVIndexFieldData.java
index 9badf9f11b443..04922c4131426 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorDVIndexFieldData.java
+++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVIndexFieldData.java
@@ -1,23 +1,11 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.index.query;
+
+package org.elasticsearch.xpack.vectors.query;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.SortField;
diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorScriptDocValues.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorScriptDocValues.java
similarity index 67%
rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorScriptDocValues.java
rename to x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorScriptDocValues.java
index 603881d390718..000caf74656eb 100644
--- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorScriptDocValues.java
+++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorScriptDocValues.java
@@ -1,23 +1,11 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.index.query;
+
+package org.elasticsearch.xpack.vectors.query;
 
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.util.BytesRef;
diff --git a/x-pack/plugin/vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension
new file mode 100644
index 0000000000000..0c8f2e6cfd023
--- /dev/null
+++ b/x-pack/plugin/vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension
@@ -0,0 +1 @@
+org.elasticsearch.xpack.vectors.query.DocValuesWhitelistExtension
\ No newline at end of file
diff --git a/x-pack/plugin/vectors/src/main/resources/org/elasticsearch/xpack/vectors/query/whitelist.txt b/x-pack/plugin/vectors/src/main/resources/org/elasticsearch/xpack/vectors/query/whitelist.txt
new file mode 100644
index 0000000000000..d385744e88fd5
--- /dev/null
+++ b/x-pack/plugin/vectors/src/main/resources/org/elasticsearch/xpack/vectors/query/whitelist.txt
@@ -0,0 +1,18 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues {
+}
+class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues$DenseVectorScriptDocValues {
+}
+class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues$SparseVectorScriptDocValues {
+}
+
+static_import {
+    double cosineSimilarity(List, VectorScriptDocValues.DenseVectorScriptDocValues) bound_to org.elasticsearch.xpack.vectors.query.ScoreScriptUtils$CosineSimilarity
+    double dotProduct(List, VectorScriptDocValues.DenseVectorScriptDocValues) from_class org.elasticsearch.xpack.vectors.query.ScoreScriptUtils
+    double dotProductSparse(Map, VectorScriptDocValues.SparseVectorScriptDocValues) bound_to org.elasticsearch.xpack.vectors.query.ScoreScriptUtils$DotProductSparse
+    double cosineSimilaritySparse(Map, VectorScriptDocValues.SparseVectorScriptDocValues) bound_to org.elasticsearch.xpack.vectors.query.ScoreScriptUtils$CosineSimilaritySparse
+}
\ No newline at end of file
diff --git a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/VectorsInfoTransportActionTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/VectorsInfoTransportActionTests.java
new file mode 100644
index 0000000000000..2ffab717faf03
--- /dev/null
+++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/VectorsInfoTransportActionTests.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.vectors;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.vectors.VectorsFeatureSetUsage;
+import org.junit.Before;
+
+import static org.hamcrest.core.Is.is;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class VectorsInfoTransportActionTests extends ESTestCase {
+
+    private XPackLicenseState licenseState;
+
+    @Before
+    public void init() {
+        licenseState = mock(XPackLicenseState.class);
+    }
+
+    public void testAvailable() throws Exception {
+        VectorsInfoTransportAction featureSet = new VectorsInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
+        boolean available = randomBoolean();
+        when(licenseState.isVectorsAllowed()).thenReturn(available);
+        assertThat(featureSet.available(), is(available));
+
+        var usageAction = new VectorsUsageTransportAction(mock(TransportService.class), null, null,
+            mock(ActionFilters.class), null, Settings.EMPTY, licenseState);
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, null, future);
+        XPackFeatureSet.Usage usage = future.get().getUsage();
+        assertThat(usage.available(), is(available));
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        usage.writeTo(out);
+        XPackFeatureSet.Usage serializedUsage = new VectorsFeatureSetUsage(out.bytes().streamInput());
+        assertThat(serializedUsage.available(), is(available));
+    }
+
+    public void testEnabled() throws Exception {
+        boolean enabled = randomBoolean();
+        Settings.Builder settings = Settings.builder();
+        if (enabled) {
+            if (randomBoolean()) {
+                settings.put("xpack.vectors.enabled", enabled);
+            }
+        } else {
+            settings.put("xpack.vectors.enabled", enabled);
+        }
+        VectorsInfoTransportAction featureSet = new VectorsInfoTransportAction(
+mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState);
+        assertThat(featureSet.enabled(), is(enabled));
+
+        VectorsUsageTransportAction usageAction = new VectorsUsageTransportAction(mock(TransportService.class),
+            null, null, mock(ActionFilters.class), null, settings.build(), licenseState);
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(null, null, null, future);
+        XPackFeatureSet.Usage usage = future.get().getUsage();
+        assertThat(usage.enabled(), is(enabled));
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        usage.writeTo(out);
+        XPackFeatureSet.Usage serializedUsage = new VectorsFeatureSetUsage(out.bytes().streamInput());
+        assertThat(serializedUsage.enabled(), is(enabled));
+    }
+
+}
diff --git a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java
new file mode 100644
index 0000000000000..d1b37c73a246e
--- /dev/null
+++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+package org.elasticsearch.xpack.vectors.mapper;
+
+import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.DocumentMapperParser;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.vectors.Vectors;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class DenseVectorFieldMapperTests extends ESSingleNodeTestCase {
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(Vectors.class, XPackPlugin.class);
+    }
+
+    public void testMappingExceedDimsLimit() throws IOException {
+        IndexService indexService = createIndex("test-index");
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("_doc")
+            .startObject("properties")
+            .startObject("my-dense-vector").field("type", "dense_vector").field("dims", DenseVectorFieldMapper.MAX_DIMS_COUNT + 1)
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject());
+        MapperParsingException e = expectThrows(MapperParsingException.class, () -> parser.parse("_doc", new CompressedXContent(mapping)));
+        assertEquals(e.getMessage(), "The number of dimensions for field [my-dense-vector] should be in the range [1, 1024]");
+    }
+
+    public void testDefaults() throws Exception {
+        IndexService indexService = createIndex("test-index");
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("_doc")
+            .startObject("properties")
+            .startObject("my-dense-vector").field("type", "dense_vector").field("dims", 3)
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject());
+        DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping));
+
+        float[] validVector = {-12.1f, 100.7f, -4};
+        ParsedDocument doc1 = mapper.parse(new SourceToParse("test-index", "_doc", "1", BytesReference
+            .bytes(XContentFactory.jsonBuilder()
+                .startObject()
+                    .startArray("my-dense-vector").value(validVector[0]).value(validVector[1]).value(validVector[2]).endArray()
+                .endObject()),
+            XContentType.JSON));
+        IndexableField[] fields = doc1.rootDoc().getFields("my-dense-vector");
+        assertEquals(1, fields.length);
+        assertThat(fields[0], instanceOf(BinaryDocValuesField.class));
+        // assert that after decoding the indexed value is equal to expected
+        BytesRef vectorBR = fields[0].binaryValue();
+        float[] decodedValues = VectorEncoderDecoder.decodeDenseVector(vectorBR);
+        assertArrayEquals(
+            "Decoded dense vector values is not equal to the indexed one.",
+            validVector,
+            decodedValues,
+            0.001f
+        );
+    }
+
+    public void testDocumentsWithIncorrectDims() throws Exception {
+        IndexService indexService = createIndex("test-index");
+        int dims = 3;
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String mapping = Strings.toString(XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("_doc")
+            .startObject("properties")
+            .startObject("my-dense-vector").field("type", "dense_vector").field("dims", dims)
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject());
+        DocumentMapper mapper = parser.parse("_doc", new CompressedXContent(mapping));
+
+        // test that error is thrown when a document has number of dims more than defined in the mapping
+        float[] invalidVector = new float[dims + 1];
+        BytesReference invalidDoc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+            .array("my-dense-vector", invalidVector)
+            .endObject());
+        MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapper.parse(
+            new SourceToParse("test-index", "_doc", "1", invalidDoc, XContentType.JSON)));
+        assertThat(e.getCause().getMessage(), containsString("has exceeded the number of dimensions [3] defined in mapping"));
+
+        // test that error is thrown when a document has number of dims less than defined in the mapping
+        float[] invalidVector2 = new float[dims - 1];
+        BytesReference invalidDoc2 = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
+            .array("my-dense-vector", invalidVector2)
+            .endObject());
+        MapperParsingException e2 = expectThrows(MapperParsingException.class, () -> mapper.parse(
+            new SourceToParse("test-index", "_doc", "2", invalidDoc2, XContentType.JSON)));
+        assertThat(e2.getCause().getMessage(), containsString("has number of dimensions [2] less than defined in the mapping [3]"));
+    }
+}
diff --git a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldTypeTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldTypeTests.java
new file mode 100644
index 0000000000000..51d4d3f52bd04
--- /dev/null
+++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldTypeTests.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+package org.elasticsearch.xpack.vectors.mapper;
+
+import org.elasticsearch.index.mapper.FieldTypeTestCase;
+import org.elasticsearch.index.mapper.MappedFieldType;
+
+public class DenseVectorFieldTypeTests extends FieldTypeTestCase {
+
+    @Override
+    protected MappedFieldType createDefaultFieldType() {
+        return new DenseVectorFieldMapper.DenseVectorFieldType();
+    }
+}
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SparseVectorFieldMapperTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java
similarity index 90%
rename from modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SparseVectorFieldMapperTests.java
rename to x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java
index 754a6f1a31803..e1e110a750b93 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SparseVectorFieldMapperTests.java
+++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java
@@ -1,23 +1,11 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.index.mapper;
+
+package org.elasticsearch.xpack.vectors.mapper;
 
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.index.IndexableField;
@@ -28,8 +16,15 @@
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.DocumentMapperParser;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.vectors.Vectors;
 import org.hamcrest.Matchers;
 import org.junit.Before;
 
@@ -64,7 +59,7 @@ public void setUpMapper() throws Exception {
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(MapperExtrasPlugin.class);
+        return pluginList(Vectors.class, XPackPlugin.class);
     }
 
     public void testDefaults() throws Exception {
diff --git a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java
new file mode 100644
index 0000000000000..7f6af470d1825
--- /dev/null
+++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+package org.elasticsearch.xpack.vectors.mapper;
+
+import org.elasticsearch.index.mapper.FieldTypeTestCase;
+import org.elasticsearch.index.mapper.MappedFieldType;
+
+public class SparseVectorFieldTypeTests extends FieldTypeTestCase {
+
+    @Override
+    protected MappedFieldType createDefaultFieldType() {
+        return new SparseVectorFieldMapper.SparseVectorFieldType();
+    }
+}
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/VectorEncoderDecoderTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/VectorEncoderDecoderTests.java
similarity index 69%
rename from modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/VectorEncoderDecoderTests.java
rename to x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/VectorEncoderDecoderTests.java
index 9b8a741192c4f..939d999b0d9aa 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/VectorEncoderDecoderTests.java
+++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/VectorEncoderDecoderTests.java
@@ -1,23 +1,10 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.index.mapper;
+package org.elasticsearch.xpack.vectors.mapper;
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.test.ESTestCase;
@@ -29,14 +16,14 @@
 public class VectorEncoderDecoderTests extends ESTestCase {
 
     public void testDenseVectorEncodingDecoding() {
-        int dimCount = randomIntBetween(0, 300);
+        int dimCount = randomIntBetween(0, DenseVectorFieldMapper.MAX_DIMS_COUNT);
         float[] expectedValues = new float[dimCount];
         for (int i = 0; i < dimCount; i++) {
             expectedValues[i] = randomFloat();
         }
 
         // test that values that went through encoding and decoding are equal to their original
-        BytesRef encodedDenseVector =  mockEncodeDenseVector(expectedValues);
+        BytesRef encodedDenseVector = mockEncodeDenseVector(expectedValues);
         float[] decodedValues = VectorEncoderDecoder.decodeDenseVector(encodedDenseVector);
         assertArrayEquals(
             "Decoded dense vector values are not equal to their original.",
@@ -44,7 +31,6 @@ public void testDenseVectorEncodingDecoding() {
             decodedValues,
             0.001f
         );
-
     }
 
     public void testSparseVectorEncodingDecoding() {
@@ -83,18 +69,17 @@ public void testSparseVectorEncodingDecoding() {
     }
 
     // imitates the code in DenseVectorFieldMapper::parse
-    public static BytesRef mockEncodeDenseVector(float[] dims) {
+    public static BytesRef mockEncodeDenseVector(float[] values) {
         final short INT_BYTES = VectorEncoderDecoder.INT_BYTES;
-        byte[] buf = new byte[INT_BYTES * dims.length];
+        byte[] buf = new byte[INT_BYTES * values.length];
         int offset = 0;
         int intValue;
-        for (float value: dims) {
+        for (float value: values) {
             intValue = Float.floatToIntBits(value);
-            buf[offset] =  (byte) (intValue >> 24);
-            buf[offset+1] = (byte) (intValue >> 16);
-            buf[offset+2] = (byte) (intValue >>  8);
-            buf[offset+3] = (byte) intValue;
-            offset += INT_BYTES;
+            buf[offset++] =  (byte) (intValue >> 24);
+            buf[offset++] = (byte) (intValue >> 16);
+            buf[offset++] = (byte) (intValue >>  8);
+            buf[offset++] = (byte) intValue;
         }
         return new BytesRef(buf, 0, offset);
     }
diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/ScoreScriptUtilsTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/query/ScoreScriptUtilsTests.java
similarity index 61%
rename from modules/mapper-extras/src/test/java/org/elasticsearch/index/query/ScoreScriptUtilsTests.java
rename to x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/query/ScoreScriptUtilsTests.java
index bcdf0387c3f71..699a9b09fb537 100644
--- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/ScoreScriptUtilsTests.java
+++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/query/ScoreScriptUtilsTests.java
@@ -1,38 +1,26 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
  */
 
-package org.elasticsearch.index.query;
+package org.elasticsearch.xpack.vectors.query;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.mapper.VectorEncoderDecoder;
+import org.elasticsearch.xpack.vectors.mapper.VectorEncoderDecoder;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.index.query.ScoreScriptUtils.CosineSimilarity;
-import org.elasticsearch.index.query.ScoreScriptUtils.DotProductSparse;
-import org.elasticsearch.index.query.ScoreScriptUtils.CosineSimilaritySparse;
+import org.elasticsearch.xpack.vectors.query.ScoreScriptUtils.CosineSimilarity;
+import org.elasticsearch.xpack.vectors.query.ScoreScriptUtils.DotProductSparse;
+import org.elasticsearch.xpack.vectors.query.ScoreScriptUtils.CosineSimilaritySparse;
 
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.index.mapper.VectorEncoderDecoderTests.mockEncodeDenseVector;
-import static org.elasticsearch.index.query.ScoreScriptUtils.dotProduct;
+import static org.elasticsearch.xpack.vectors.mapper.VectorEncoderDecoderTests.mockEncodeDenseVector;
+import static org.elasticsearch.xpack.vectors.query.ScoreScriptUtils.dotProduct;
+import static org.hamcrest.Matchers.containsString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -53,6 +41,17 @@ public void testDenseVectorFunctions() {
         CosineSimilarity cosineSimilarity = new CosineSimilarity(queryVector);
         double result2 = cosineSimilarity.cosineSimilarity(dvs);
         assertEquals("cosineSimilarity result is not equal to the expected value!", 0.78, result2, 0.1);
+
+        // test dotProduct fails when queryVector has wrong number of dims
+        List<Number> invalidQueryVector = Arrays.asList(0.5, 111.3);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> dotProduct(invalidQueryVector, dvs));
+        assertThat(e.getMessage(), containsString("dimensions of the query vector [2] is different from the documents' vectors [5]"));
+
+        // test cosineSimilarity fails when queryVector has wrong number of dims
+        CosineSimilarity cosineSimilarity2 = new CosineSimilarity(invalidQueryVector);
+        e = expectThrows(IllegalArgumentException.class, () -> cosineSimilarity2.cosineSimilarity(dvs));
+        assertThat(e.getMessage(), containsString("dimensions of the query vector [2] is different from the documents' vectors [5]"));
+
     }
 
     public void testSparseVectorFunctions() {
diff --git a/x-pack/plugin/voting-only-node/build.gradle b/x-pack/plugin/voting-only-node/build.gradle
new file mode 100644
index 0000000000000..1c79c665f2d04
--- /dev/null
+++ b/x-pack/plugin/voting-only-node/build.gradle
@@ -0,0 +1,22 @@
+evaluationDependsOn(xpackModule('core'))
+
+apply plugin: 'elasticsearch.esplugin'
+esplugin {
+    name 'x-pack-voting-only-node'
+    description 'Elasticsearch Expanded Pack Plugin - Voting-only node'
+    classname 'org.elasticsearch.cluster.coordination.VotingOnlyNodePlugin'
+    extendedPlugins = ['x-pack-core']
+}
+
+dependencies {
+    compileOnly project(path: xpackModule('core'), configuration: 'default')
+    testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+    if (isEclipse) {
+        testCompile project(path: xpackModule('core-tests'), configuration: 'testArtifacts')
+    }
+}
+
+// xpack modules are installed in real clusters as the meta plugin, so
+// installing them as individual plugins for integ tests doesn't make sense,
+// so we disable integ tests
+integTest.enabled = false
\ No newline at end of file
diff --git a/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/VotingOnlyNodeFeatureSet.java b/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/VotingOnlyNodeFeatureSet.java
new file mode 100644
index 0000000000000..a412df53e6205
--- /dev/null
+++ b/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/VotingOnlyNodeFeatureSet.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.votingonly.VotingOnlyNodeFeatureSetUsage;
+
+public class VotingOnlyNodeFeatureSet implements XPackFeatureSet {
+
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public VotingOnlyNodeFeatureSet(@Nullable XPackLicenseState licenseState) {
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    public String name() {
+        return XPackField.VOTING_ONLY;
+    }
+
+    @Override
+    public boolean available() {
+        return licenseState != null && licenseState.isVotingOnlyAllowed();
+    }
+
+    @Override
+    public boolean enabled() {
+        return true;
+    }
+
+    public static class UsageTransportAction extends XPackUsageFeatureTransportAction {
+
+        private final XPackLicenseState licenseState;
+
+        @Inject
+        public UsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                    ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                    XPackLicenseState licenseState) {
+            super(XPackUsageFeatureAction.VOTING_ONLY.name(), transportService, clusterService,
+                threadPool, actionFilters, indexNameExpressionResolver);
+            this.licenseState = licenseState;
+        }
+
+        @Override
+        protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                       ActionListener<XPackUsageFeatureResponse> listener) {
+            final boolean available = licenseState.isVotingOnlyAllowed();
+            final VotingOnlyNodeFeatureSetUsage usage =
+                new VotingOnlyNodeFeatureSetUsage(available);
+            listener.onResponse(new XPackUsageFeatureResponse(usage));
+        }
+    }
+
+    public static class UsageInfoAction extends XPackInfoFeatureTransportAction {
+
+        private final XPackLicenseState licenseState;
+
+        @Inject
+        public UsageInfoAction(TransportService transportService, ActionFilters actionFilters,
+                               XPackLicenseState licenseState) {
+            super(XPackInfoFeatureAction.VOTING_ONLY.name(), transportService, actionFilters);
+            this.licenseState = licenseState;
+        }
+
+        @Override
+        protected String name() {
+            return XPackField.VOTING_ONLY;
+        }
+
+        @Override
+        protected boolean available() {
+            return licenseState.isVotingOnlyAllowed();
+        }
+
+        @Override
+        protected boolean enabled() {
+            return true;
+        }
+    }
+}
diff --git a/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/VotingOnlyNodePlugin.java b/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/VotingOnlyNodePlugin.java
new file mode 100644
index 0000000000000..202cd3f095316
--- /dev/null
+++ b/x-pack/plugin/voting-only-node/src/main/java/org/elasticsearch/cluster/coordination/VotingOnlyNodePlugin.java
@@ -0,0 +1,230 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
+import org.elasticsearch.cluster.coordination.CoordinationState.VoteCollection;
+import org.elasticsearch.cluster.coordination.VotingOnlyNodeFeatureSet.UsageTransportAction;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.discovery.DiscoveryModule;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.DiscoveryPlugin;
+import org.elasticsearch.plugins.NetworkPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportInterceptor;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportResponse;
+import org.elasticsearch.transport.TransportResponseHandler;
+import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+
+public class VotingOnlyNodePlugin extends Plugin implements DiscoveryPlugin, NetworkPlugin, ActionPlugin {
+
+    public static final Setting<Boolean> VOTING_ONLY_NODE_SETTING
+        = Setting.boolSetting("node.voting_only", false, Setting.Property.NodeScope);
+
+    private static final String VOTING_ONLY_ELECTION_STRATEGY = "supports_voting_only";
+
+    static DiscoveryNodeRole VOTING_ONLY_NODE_ROLE = new DiscoveryNodeRole("voting_only", "v") {
+        @Override
+        protected Setting<Boolean> roleSetting() {
+            return VOTING_ONLY_NODE_SETTING;
+        }
+    };
+
+    private final Settings settings;
+    private final SetOnce<ThreadPool> threadPool;
+
+    private final boolean isVotingOnlyNode;
+
+    public VotingOnlyNodePlugin(Settings settings) {
+        this.settings = settings;
+        threadPool = new SetOnce<>();
+        isVotingOnlyNode = VOTING_ONLY_NODE_SETTING.get(settings);
+    }
+
+    public static boolean isVotingOnlyNode(DiscoveryNode discoveryNode) {
+        return discoveryNode.getRoles().contains(VOTING_ONLY_NODE_ROLE);
+    }
+
+    public static boolean isFullMasterNode(DiscoveryNode discoveryNode) {
+        return discoveryNode.isMasterNode() && discoveryNode.getRoles().contains(VOTING_ONLY_NODE_ROLE) == false;
+    }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Collections.singletonList(VOTING_ONLY_NODE_SETTING);
+    }
+
+    @Override
+    public Set<DiscoveryNodeRole> getRoles() {
+        if (isVotingOnlyNode && Node.NODE_MASTER_SETTING.get(settings) == false) {
+            throw new IllegalStateException("voting-only node must be master-eligible");
+        }
+        return Collections.singleton(VOTING_ONLY_NODE_ROLE);
+    }
+
+    @Override
+    public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
+                                               ResourceWatcherService resourceWatcherService, ScriptService scriptService,
+                                               NamedXContentRegistry xContentRegistry, Environment environment,
+                                               NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) {
+        this.threadPool.set(threadPool);
+        return Collections.emptyList();
+    }
+
+    @Override
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        return Arrays.asList(new ActionHandler<>(XPackUsageFeatureAction.VOTING_ONLY, UsageTransportAction.class),
+            new ActionHandler<>(XPackInfoFeatureAction.VOTING_ONLY, VotingOnlyNodeFeatureSet.UsageInfoAction.class));
+    }
+
+    @Override
+    public Map<String, ElectionStrategy> getElectionStrategies() {
+        return Collections.singletonMap(VOTING_ONLY_ELECTION_STRATEGY, new VotingOnlyNodeElectionStrategy());
+    }
+
+    @Override
+    public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry, ThreadContext threadContext) {
+        if (isVotingOnlyNode) {
+            return Collections.singletonList(new TransportInterceptor() {
+                @Override
+                public AsyncSender interceptSender(AsyncSender sender) {
+                    return new VotingOnlyNodeAsyncSender(sender, threadPool::get);
+                }
+            });
+        } else {
+            return Collections.emptyList();
+        }
+    }
+
+    @Override
+    public Settings additionalSettings() {
+        return Settings.builder().put(DiscoveryModule.ELECTION_STRATEGY_SETTING.getKey(), VOTING_ONLY_ELECTION_STRATEGY).build();
+    }
+
+    static class VotingOnlyNodeElectionStrategy extends ElectionStrategy {
+
+        @Override
+        public boolean satisfiesAdditionalQuorumConstraints(DiscoveryNode localNode, long localCurrentTerm, long localAcceptedTerm,
+                                                            long localAcceptedVersion, VotingConfiguration lastCommittedConfiguration,
+                                                            VotingConfiguration lastAcceptedConfiguration, VoteCollection joinVotes) {
+            // if local node is voting only, have additional checks on election quorum definition
+            if (isVotingOnlyNode(localNode)) {
+                // if all votes are from voting only nodes, do not elect as master (no need to transfer state)
+                if (joinVotes.nodes().stream().filter(DiscoveryNode::isMasterNode).allMatch(VotingOnlyNodePlugin::isVotingOnlyNode)) {
+                    return false;
+                }
+                // if there's a vote from a full master node with same state (i.e. last accepted term and version match), then that node
+                // should become master instead, so we should stand down. There are two exceptional cases, however:
+                // 1) if we are in term 0. In that case, we allow electing the voting-only node to avoid poisonous situations where only
+                //    voting-only nodes are bootstrapped.
+                // 2) if there is another full master node with an older state. In that case, we ensure that
+                //    satisfiesAdditionalQuorumConstraints cannot go from true to false when adding new joinVotes in the same election.
+                //    As voting-only nodes only broadcast the state to the full master nodes, eventually all of them will have caught up
+                //    and there should not be any remaining full master nodes with older state, effectively disabling election of
+                //    voting-only nodes.
+                if (joinVotes.getJoins().stream().anyMatch(fullMasterWithSameState(localAcceptedTerm, localAcceptedVersion)) &&
+                    localAcceptedTerm > 0 &&
+                    joinVotes.getJoins().stream().noneMatch(fullMasterWithOlderState(localAcceptedTerm, localAcceptedVersion))) {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        private static Predicate<Join> fullMasterWithSameState(long localAcceptedTerm, long localAcceptedVersion) {
+            return join -> isFullMasterNode(join.getSourceNode()) &&
+                join.getLastAcceptedTerm() == localAcceptedTerm &&
+                join.getLastAcceptedVersion() == localAcceptedVersion;
+        }
+
+        private static Predicate<Join> fullMasterWithOlderState(long localAcceptedTerm, long localAcceptedVersion) {
+            return join -> isFullMasterNode(join.getSourceNode()) &&
+                (join.getLastAcceptedTerm() < localAcceptedTerm ||
+                    (join.getLastAcceptedTerm() == localAcceptedTerm && join.getLastAcceptedVersion() < localAcceptedVersion));
+        }
+    }
+
+    static class VotingOnlyNodeAsyncSender implements TransportInterceptor.AsyncSender {
+        private final TransportInterceptor.AsyncSender sender;
+        private final Supplier<ThreadPool> threadPoolSupplier;
+
+        VotingOnlyNodeAsyncSender(TransportInterceptor.AsyncSender sender, Supplier<ThreadPool> threadPoolSupplier) {
+            this.sender = sender;
+            this.threadPoolSupplier = threadPoolSupplier;
+        }
+
+        @Override
+        public <T extends TransportResponse> void sendRequest(Transport.Connection connection, String action, TransportRequest request,
+                                                              TransportRequestOptions options, TransportResponseHandler<T> handler) {
+            if (action.equals(PublicationTransportHandler.PUBLISH_STATE_ACTION_NAME)) {
+                final DiscoveryNode destinationNode = connection.getNode();
+                if (isFullMasterNode(destinationNode)) {
+                    sender.sendRequest(connection, action, request, options, new TransportResponseHandler<>() {
+                        @Override
+                        public void handleResponse(TransportResponse response) {
+                            handler.handleException(new TransportException(new ElasticsearchException(
+                                "ignoring successful publish response used purely for state transfer: " + response)));
+                        }
+
+                        @Override
+                        public void handleException(TransportException exp) {
+                            handler.handleException(exp);
+                        }
+
+                        @Override
+                        public String executor() {
+                            return handler.executor();
+                        }
+
+                        @Override
+                        public TransportResponse read(StreamInput in) throws IOException {
+                            return handler.read(in);
+                        }
+                    });
+                } else {
+                    threadPoolSupplier.get().generic().execute(() -> handler.handleException(new TransportException(
+                        new ElasticsearchException("voting-only node skipping publication to " + destinationNode))));
+                }
+            } else {
+                sender.sendRequest(connection, action, request, options, handler);
+            }
+        }
+    }
+}
diff --git a/x-pack/plugin/voting-only-node/src/test/java/org/elasticsearch/cluster/coordination/LocalStateVotingOnlyNodePlugin.java b/x-pack/plugin/voting-only-node/src/test/java/org/elasticsearch/cluster/coordination/LocalStateVotingOnlyNodePlugin.java
new file mode 100644
index 0000000000000..aaacb6b76edfe
--- /dev/null
+++ b/x-pack/plugin/voting-only-node/src/test/java/org/elasticsearch/cluster/coordination/LocalStateVotingOnlyNodePlugin.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
+
+import java.nio.file.Path;
+
+public class LocalStateVotingOnlyNodePlugin extends LocalStateCompositeXPackPlugin {
+
+    public LocalStateVotingOnlyNodePlugin(final Settings settings, final Path configPath) throws Exception {
+        super(settings, configPath);
+
+        plugins.add(new VotingOnlyNodePlugin(settings));
+    }
+}
diff --git a/x-pack/plugin/voting-only-node/src/test/java/org/elasticsearch/cluster/coordination/VotingOnlyNodeCoordinationStateTests.java b/x-pack/plugin/voting-only-node/src/test/java/org/elasticsearch/cluster/coordination/VotingOnlyNodeCoordinationStateTests.java
new file mode 100644
index 0000000000000..414ba350adf40
--- /dev/null
+++ b/x-pack/plugin/voting-only-node/src/test/java/org/elasticsearch/cluster/coordination/VotingOnlyNodeCoordinationStateTests.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+public class VotingOnlyNodeCoordinationStateTests extends ESTestCase {
+
+    public void testSafety() {
+        new CoordinationStateTestCluster(IntStream.range(0, randomIntBetween(1, 5))
+            .mapToObj(i -> new DiscoveryNode("node_" + i, buildNewFakeTransportAddress(), Map.of(),
+                randomBoolean() ? DiscoveryNodeRole.BUILT_IN_ROLES :
+                Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.INGEST_ROLE, DiscoveryNodeRole.MASTER_ROLE,
+                    VotingOnlyNodePlugin.VOTING_ONLY_NODE_ROLE), Version.CURRENT))
+            .collect(Collectors.toList()), new VotingOnlyNodePlugin.VotingOnlyNodeElectionStrategy())
+            .runRandomly();
+    }
+
+}
diff --git a/x-pack/plugin/voting-only-node/src/test/java/org/elasticsearch/cluster/coordination/VotingOnlyNodeCoordinatorTests.java b/x-pack/plugin/voting-only-node/src/test/java/org/elasticsearch/cluster/coordination/VotingOnlyNodeCoordinatorTests.java
new file mode 100644
index 0000000000000..08afd3de70f2b
--- /dev/null
+++ b/x-pack/plugin/voting-only-node/src/test/java/org/elasticsearch/cluster/coordination/VotingOnlyNodeCoordinatorTests.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportInterceptor;
+import org.junit.BeforeClass;
+
+import java.util.Collections;
+import java.util.Set;
+
+import static java.util.Collections.emptySet;
+
+public class VotingOnlyNodeCoordinatorTests extends AbstractCoordinatorTestCase {
+
+    @BeforeClass
+    public static void setPossibleRolesWithVotingOnly() {
+        DiscoveryNode.setPossibleRoles(
+            Sets.union(DiscoveryNodeRole.BUILT_IN_ROLES, Sets.newHashSet(VotingOnlyNodePlugin.VOTING_ONLY_NODE_ROLE)));
+    }
+
+    @Override
+    protected TransportInterceptor getTransportInterceptor(DiscoveryNode localNode, ThreadPool threadPool) {
+        if (VotingOnlyNodePlugin.isVotingOnlyNode(localNode)) {
+            return new TransportInterceptor() {
+                @Override
+                public AsyncSender interceptSender(AsyncSender sender) {
+                    return new VotingOnlyNodePlugin.VotingOnlyNodeAsyncSender(sender, () -> threadPool);
+                }
+            };
+        } else {
+            return super.getTransportInterceptor(localNode, threadPool);
+        }
+    }
+
+    @Override
+    protected ElectionStrategy getElectionStrategy() {
+        return new VotingOnlyNodePlugin.VotingOnlyNodeElectionStrategy();
+    }
+
+    public void testDoesNotElectVotingOnlyMasterNode() {
+        final Cluster cluster = new Cluster(randomIntBetween(1, 5), false, Settings.EMPTY);
+        cluster.runRandomly();
+        cluster.stabilise();
+
+        final Cluster.ClusterNode leader = cluster.getAnyLeader();
+        assertTrue(leader.getLocalNode().isMasterNode());
+        assertFalse(leader.getLocalNode().toString(), VotingOnlyNodePlugin.isVotingOnlyNode(leader.getLocalNode()));
+    }
+
+    @Override
+    protected DiscoveryNode createDiscoveryNode(int nodeIndex, boolean masterEligible) {
+        final TransportAddress address = buildNewFakeTransportAddress();
+        return new DiscoveryNode("", "node" + nodeIndex,
+            UUIDs.randomBase64UUID(random()), // generated deterministically for repeatable tests
+            address.address().getHostString(), address.getAddress(), address, Collections.emptyMap(),
+            masterEligible ? DiscoveryNodeRole.BUILT_IN_ROLES :
+                randomBoolean() ? emptySet() : Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.INGEST_ROLE,
+                    DiscoveryNodeRole.MASTER_ROLE, VotingOnlyNodePlugin.VOTING_ONLY_NODE_ROLE), Version.CURRENT);
+    }
+
+}
diff --git a/x-pack/plugin/voting-only-node/src/test/java/org/elasticsearch/cluster/coordination/VotingOnlyNodePluginTests.java b/x-pack/plugin/voting-only-node/src/test/java/org/elasticsearch/cluster/coordination/VotingOnlyNodePluginTests.java
new file mode 100644
index 0000000000000..c55a405059a70
--- /dev/null
+++ b/x-pack/plugin/voting-only-node/src/test/java/org/elasticsearch/cluster/coordination/VotingOnlyNodePluginTests.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.discovery.MasterNotDiscoveredException;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESIntegTestCase.Scope;
+
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.nullValue;
+
+@ESIntegTestCase.ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
+public class VotingOnlyNodePluginTests extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(LocalStateVotingOnlyNodePlugin.class);
+    }
+
+    public void testRequireVotingOnlyNodeToBeMasterEligible() {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        IllegalStateException ise = expectThrows(IllegalStateException.class, () -> internalCluster().startNode(Settings.builder()
+            .put(Node.NODE_MASTER_SETTING.getKey(), false)
+            .put(VotingOnlyNodePlugin.VOTING_ONLY_NODE_SETTING.getKey(), true)
+            .build()));
+        assertThat(ise.getMessage(), containsString("voting-only node must be master-eligible"));
+    }
+
+    public void testVotingOnlyNodeStats() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        internalCluster().startNodes(2);
+        internalCluster().startNode(Settings.builder().put(VotingOnlyNodePlugin.VOTING_ONLY_NODE_SETTING.getKey(), true));
+        assertBusy(() -> assertThat(client().admin().cluster().prepareState().get().getState().getLastCommittedConfiguration().getNodeIds(),
+            hasSize(3)));
+        assertThat(client().admin().cluster().prepareClusterStats().get().getNodesStats().getCounts().getRoles().get(
+            VotingOnlyNodePlugin.VOTING_ONLY_NODE_ROLE.roleName()).intValue(), equalTo(1));
+        assertThat(client().admin().cluster().prepareNodesStats("voting_only:true").get().getNodes(), hasSize(1));
+        assertThat(client().admin().cluster().prepareNodesStats("master:true", "voting_only:false").get().getNodes(), hasSize(2));
+    }
+
+    public void testPreferFullMasterOverVotingOnlyNodes() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        internalCluster().startNodes(2);
+        internalCluster().startNode(Settings.builder().put(VotingOnlyNodePlugin.VOTING_ONLY_NODE_SETTING.getKey(), true).build());
+        internalCluster().startDataOnlyNodes(randomInt(2));
+        assertBusy(() -> assertThat(
+            client().admin().cluster().prepareState().get().getState().getLastCommittedConfiguration().getNodeIds().size(),
+            equalTo(3)));
+        final String originalMaster = internalCluster().getMasterName();
+
+        internalCluster().stopCurrentMasterNode();
+        client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).get();
+        assertNotEquals(originalMaster, internalCluster().getMasterName());
+        assertThat(
+            VotingOnlyNodePlugin.isVotingOnlyNode(client().admin().cluster().prepareState().get().getState().nodes().getMasterNode()),
+            equalTo(false));
+    }
+
+    public void testBootstrapOnlyVotingOnlyNodes() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        internalCluster().startNodes(Settings.builder().put(VotingOnlyNodePlugin.VOTING_ONLY_NODE_SETTING.getKey(), true).build(),
+            Settings.EMPTY, Settings.EMPTY);
+        assertBusy(() -> assertThat(
+            client().admin().cluster().prepareState().get().getState().getLastCommittedConfiguration().getNodeIds().size(),
+            equalTo(3)));
+        assertThat(
+            VotingOnlyNodePlugin.isVotingOnlyNode(client().admin().cluster().prepareState().get().getState().nodes().getMasterNode()),
+            equalTo(false));
+    }
+
+    public void testVotingOnlyNodesCannotBeMasterWithoutFullMasterNodes() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        internalCluster().startNode();
+        internalCluster().startNodes(2, Settings.builder().put(VotingOnlyNodePlugin.VOTING_ONLY_NODE_SETTING.getKey(), true).build());
+        internalCluster().startDataOnlyNodes(randomInt(2));
+        assertBusy(() -> assertThat(
+            client().admin().cluster().prepareState().get().getState().getLastCommittedConfiguration().getNodeIds().size(),
+            equalTo(3)));
+        final String oldMasterId = client().admin().cluster().prepareState().get().getState().nodes().getMasterNodeId();
+
+        internalCluster().stopCurrentMasterNode();
+
+        expectThrows(MasterNotDiscoveredException.class, () ->
+            assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("100ms").execute().actionGet()
+                .getState().nodes().getMasterNodeId(), nullValue()));
+
+        // start a fresh full master node, which will be brought into the cluster as master by the voting-only nodes
+        final String newMaster = internalCluster().startNode();
+        assertEquals(newMaster, internalCluster().getMasterName());
+        final String newMasterId = client().admin().cluster().prepareState().get().getState().nodes().getMasterNodeId();
+        assertNotEquals(oldMasterId, newMasterId);
+    }
+}
diff --git a/x-pack/plugin/watcher/build.gradle b/x-pack/plugin/watcher/build.gradle
index 1585488bdb5b3..20695d805190b 100644
--- a/x-pack/plugin/watcher/build.gradle
+++ b/x-pack/plugin/watcher/build.gradle
@@ -23,9 +23,7 @@ dependencyLicenses {
 }
 
 dependencies {
-    compileOnly "org.elasticsearch:elasticsearch:${version}"
-
-    // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
+    compileOnly project(':server')
     compileOnly project(path: xpackModule('core'), configuration: 'default')
     compileOnly project(path: ':modules:transport-netty4', configuration: 'runtime')
     compileOnly project(path: ':plugins:transport-nio', configuration: 'runtime')
@@ -34,7 +32,7 @@ dependencies {
     if (isEclipse) {
       testCompile project(path: xpackModule('core-tests'), configuration: 'testArtifacts')
     }
-    testCompile "org.elasticsearch.plugin:x-pack-ilm:${version}"
+    testCompile project(xpackModule('ilm'))
 
     // watcher deps
     compile 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:r239'
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/ClockHolder.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/ClockHolder.java
new file mode 100644
index 0000000000000..f94c90e8fd0eb
--- /dev/null
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/ClockHolder.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.watcher;
+
+import java.time.Clock;
+
+/**
+ * A wrapper around {@link java.time.Clock} to provide a concrete type for Guice injection.
+ *
+ * This class is temporary until {@link java.time.Clock} can be passed to action constructors
+ * directly, or the actions can be rewritten to be unit tested with the clock overriden
+ * just for unit tests instead of via Node construction.
+ */
+public final class ClockHolder {
+    public final Clock clock;
+
+    public ClockHolder(Clock clock) {
+        this.clock = clock;
+    }
+}
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
index 4226ff4fed0d9..1b8fcc2658f79 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
@@ -21,8 +21,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.regex.Regex;
@@ -58,6 +56,8 @@
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.core.watcher.WatcherField;
 import org.elasticsearch.xpack.core.watcher.actions.ActionFactory;
@@ -408,7 +408,8 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure)
         listener = new WatcherIndexingListener(watchParser, getClock(), triggerService);
         clusterService.addListener(listener);
 
-        return Arrays.asList(registry, inputRegistry, historyStore, triggerService, triggeredWatchParser,
+        // note: clock is needed here until actions can be constructed directly instead of by guice
+        return Arrays.asList(new ClockHolder(getClock()), registry, inputRegistry, historyStore, triggerService, triggeredWatchParser,
                 watcherLifeCycleService, executionService, triggerEngineListener, watcherService, watchParser,
                 configuredTriggerEngine, triggeredWatchStore, watcherSearchTemplateService, slackService, pagerDutyService);
     }
@@ -425,20 +426,6 @@ protected Consumer<Iterable<TriggerEvent>> getTriggerEngineListener(ExecutionSer
         return new AsyncTriggerEventConsumer(executionService);
     }
 
-    @Override
-    public Collection<Module> createGuiceModules() {
-        List<Module> modules = new ArrayList<>();
-        modules.add(b -> b.bind(Clock.class).toInstance(getClock())); //currently assuming the only place clock is bound
-        modules.add(b -> {
-            XPackPlugin.bindFeatureSet(b, WatcherFeatureSet.class);
-            if (enabled == false) {
-                b.bind(WatcherService.class).toProvider(Providers.of(null));
-            }
-        });
-
-        return modules;
-    }
-
     @Override
     public List<Setting<?>> getSettings() {
         List<Setting<?>> settings = new ArrayList<>();
@@ -532,8 +519,10 @@ static int getWatcherThreadPoolSize(Settings settings) {
 
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        var usageAction = new ActionHandler<>(XPackUsageFeatureAction.WATCHER, WatcherUsageTransportAction.class);
+        var infoAction = new ActionHandler<>(XPackInfoFeatureAction.WATCHER, WatcherInfoTransportAction.class);
         if (false == enabled) {
-            return emptyList();
+            return Arrays.asList(usageAction, infoAction);
         }
         return Arrays.asList(new ActionHandler<>(PutWatchAction.INSTANCE, TransportPutWatchAction.class),
                 new ActionHandler<>(DeleteWatchAction.INSTANCE, TransportDeleteWatchAction.class),
@@ -542,7 +531,9 @@ static int getWatcherThreadPoolSize(Settings settings) {
                 new ActionHandler<>(AckWatchAction.INSTANCE, TransportAckWatchAction.class),
                 new ActionHandler<>(ActivateWatchAction.INSTANCE, TransportActivateWatchAction.class),
                 new ActionHandler<>(WatcherServiceAction.INSTANCE, TransportWatcherServiceAction.class),
-                new ActionHandler<>(ExecuteWatchAction.INSTANCE, TransportExecuteWatchAction.class));
+                new ActionHandler<>(ExecuteWatchAction.INSTANCE, TransportExecuteWatchAction.class),
+                usageAction,
+                infoAction);
     }
 
     @Override
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherFeatureSet.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherFeatureSet.java
deleted file mode 100644
index 3f02cfde7c7e7..0000000000000
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherFeatureSet.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.watcher;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.XPackField;
-import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.watcher.WatcherFeatureSetUsage;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
-import org.elasticsearch.xpack.core.watcher.common.stats.Counters;
-import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsRequest;
-import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsResponse;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
-import static org.elasticsearch.xpack.core.ClientHelper.WATCHER_ORIGIN;
-
-public class WatcherFeatureSet implements XPackFeatureSet {
-
-    private final boolean enabled;
-    private final XPackLicenseState licenseState;
-    private Client client;
-
-    @Inject
-    public WatcherFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState, Client client) {
-        this.enabled = XPackSettings.WATCHER_ENABLED.get(settings);
-        this.licenseState = licenseState;
-        this.client = client;
-    }
-
-    @Override
-    public String name() {
-        return XPackField.WATCHER;
-    }
-
-    @Override
-    public String description() {
-        return "Alerting, Notification and Automation for the Elastic Stack";
-    }
-
-    @Override
-    public boolean available() {
-        return licenseState != null && licenseState.isWatcherAllowed();
-    }
-
-    @Override
-    public boolean enabled() {
-        return enabled;
-    }
-
-    @Override
-    public Map<String, Object> nativeCodeInfo() {
-        return null;
-    }
-
-    @Override
-    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
-        if (enabled) {
-            try (ThreadContext.StoredContext ignore =
-                    client.threadPool().getThreadContext().stashWithOrigin(WATCHER_ORIGIN)) {
-                WatcherClient watcherClient = new WatcherClient(client);
-                WatcherStatsRequest request = new WatcherStatsRequest();
-                request.includeStats(true);
-                watcherClient.watcherStats(request, ActionListener.wrap(r -> {
-                    List<Counters> countersPerNode = r.getNodes()
-                            .stream()
-                            .map(WatcherStatsResponse.Node::getStats)
-                            .filter(Objects::nonNull)
-                            .collect(Collectors.toList());
-                    Counters mergedCounters = Counters.merge(countersPerNode);
-                    listener.onResponse(new WatcherFeatureSetUsage(available(), enabled(), mergedCounters.toNestedMap()));
-                }, listener::onFailure));
-            }
-        } else {
-            listener.onResponse(new WatcherFeatureSetUsage(available(), enabled(), Collections.emptyMap()));
-        }
-    }
-}
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportAction.java
new file mode 100644
index 0000000000000..63f071e7d1e50
--- /dev/null
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportAction.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.watcher;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
+
+public class WatcherInfoTransportAction extends XPackInfoFeatureTransportAction {
+
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public WatcherInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                      Settings settings, XPackLicenseState licenseState) {
+        super(XPackInfoFeatureAction.WATCHER.name(), transportService, actionFilters);
+        this.enabled = XPackSettings.WATCHER_ENABLED.get(settings);
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    public String name() {
+        return XPackField.WATCHER;
+    }
+
+    @Override
+    public boolean available() {
+        return licenseState.isWatcherAllowed();
+    }
+
+    @Override
+    public boolean enabled() {
+        return enabled;
+    }
+
+}
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherUsageTransportAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherUsageTransportAction.java
new file mode 100644
index 0000000000000..231d6a7f811a2
--- /dev/null
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherUsageTransportAction.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.watcher;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.watcher.WatcherFeatureSetUsage;
+import org.elasticsearch.xpack.core.watcher.common.stats.Counters;
+import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsAction;
+import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsRequest;
+import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsResponse;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.xpack.core.ClientHelper.WATCHER_ORIGIN;
+
+public class WatcherUsageTransportAction extends XPackUsageFeatureTransportAction {
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+    private final Client client;
+
+    @Inject
+    public WatcherUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                       ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                       Settings settings, XPackLicenseState licenseState, Client client) {
+        super(XPackUsageFeatureAction.WATCHER.name(), transportService, clusterService, threadPool, actionFilters,
+              indexNameExpressionResolver);
+        this.enabled = XPackSettings.WATCHER_ENABLED.get(settings);
+        this.licenseState = licenseState;
+        this.client = client;
+    }
+
+    @Override
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
+        if (enabled) {
+            try (ThreadContext.StoredContext ignore =
+                     client.threadPool().getThreadContext().stashWithOrigin(WATCHER_ORIGIN)) {
+                WatcherStatsRequest statsRequest = new WatcherStatsRequest();
+                statsRequest.includeStats(true);
+                statsRequest.setParentTask(clusterService.localNode().getId(), task.getId());
+                client.execute(WatcherStatsAction.INSTANCE, statsRequest, ActionListener.wrap(r -> {
+                    List<Counters> countersPerNode = r.getNodes()
+                        .stream()
+                        .map(WatcherStatsResponse.Node::getStats)
+                        .filter(Objects::nonNull)
+                        .collect(Collectors.toList());
+                    Counters mergedCounters = Counters.merge(countersPerNode);
+                    WatcherFeatureSetUsage usage =
+                        new WatcherFeatureSetUsage(licenseState.isWatcherAllowed(), true, mergedCounters.toNestedMap());
+                    listener.onResponse(new XPackUsageFeatureResponse(usage));
+                }, listener::onFailure));
+            }
+        } else {
+            WatcherFeatureSetUsage usage =
+                new WatcherFeatureSetUsage(licenseState.isWatcherAllowed(), false, Collections.emptyMap());
+            listener.onResponse(new XPackUsageFeatureResponse(usage));
+        }
+    }
+}
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/Scheme.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/Scheme.java
index 04557271c26ce..15afb8885b572 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/Scheme.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/Scheme.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.watcher.common.http;
 
 import java.util.Locale;
+import java.util.Objects;
 
 public enum Scheme {
 
@@ -29,6 +30,7 @@ public int defaultPort() {
     }
 
     public static Scheme parse(String value) {
+        Objects.requireNonNull(value, "Scheme should not be Null");
         value = value.toLowerCase(Locale.ROOT);
         switch (value) {
             case "http":
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java
index c2a079e519f0f..c6c041a6571b1 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/NotificationService.java
@@ -8,6 +8,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.SecureString;
@@ -179,12 +180,13 @@ private static SecureSettings extractSecureSettings(Settings source, List<Settin
         // get the secure settings out
         final SecureSettings sourceSecureSettings = Settings.builder().put(source, true).getSecureSettings();
         // filter and cache them...
-        final Map<String, SecureString> cache = new HashMap<>();
+        final Map<String, Tuple<SecureString, byte[]>> cache = new HashMap<>();
         if (sourceSecureSettings != null && securePluginSettings != null) {
             for (final String settingKey : sourceSecureSettings.getSettingNames()) {
                 for (final Setting<?> secureSetting : securePluginSettings) {
                     if (secureSetting.match(settingKey)) {
-                        cache.put(settingKey, sourceSecureSettings.getString(settingKey));
+                        cache.put(settingKey,
+                                new Tuple<>(sourceSecureSettings.getString(settingKey), sourceSecureSettings.getSHA256Digest(settingKey)));
                     }
                 }
             }
@@ -197,8 +199,8 @@ public boolean isLoaded() {
             }
 
             @Override
-            public SecureString getString(String setting) throws GeneralSecurityException {
-                return cache.get(setting);
+            public SecureString getString(String setting) {
+                return cache.get(setting).v1();
             }
 
             @Override
@@ -207,10 +209,15 @@ public Set<String> getSettingNames() {
             }
 
             @Override
-            public InputStream getFile(String setting) throws GeneralSecurityException {
+            public InputStream getFile(String setting) {
                 throw new IllegalStateException("A NotificationService setting cannot be File.");
             }
 
+            @Override
+            public byte[] getSHA256Digest(String setting) {
+                return cache.get(setting).v2();
+            }
+
             @Override
             public void close() throws IOException {
             }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/jira/JiraAccount.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/jira/JiraAccount.java
index b539d007eeef8..d59f7e6f645f0 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/jira/JiraAccount.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/jira/JiraAccount.java
@@ -61,6 +61,9 @@ public JiraAccount(String name, Settings settings, HttpClient httpClient) {
         String url = getSetting(name, settings, SECURE_URL_SETTING);
         try {
             URI uri = new URI(url);
+            if (uri.getScheme() == null) {
+                throw new URISyntaxException("null", "No scheme defined in url");
+            }
             Scheme protocol = Scheme.parse(uri.getScheme());
             if ((protocol == Scheme.HTTP) && (Booleans.isTrue(settings.get(ALLOW_HTTP_SETTING)) == false)) {
                 throw new SettingsException("invalid jira [" + name + "] account settings. unsecure scheme [" + protocol + "]");
@@ -68,7 +71,7 @@ public JiraAccount(String name, Settings settings, HttpClient httpClient) {
             this.url = uri;
         } catch (URISyntaxException | IllegalArgumentException e) {
             throw new SettingsException(
-                    "invalid jira [" + name + "] account settings. invalid [" + SECURE_URL_SETTING.getKey() + "] setting", e);
+                "invalid jira [" + name + "] account settings. invalid [" + SECURE_URL_SETTING.getKey() + "] setting", e);
         }
         this.user = getSetting(name, settings, SECURE_USER_SETTING);
         this.password = getSetting(name, settings, SECURE_PASSWORD_SETTING);
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/WatcherRestHandler.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/WatcherRestHandler.java
deleted file mode 100644
index 3588df514c377..0000000000000
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/WatcherRestHandler.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.watcher.rest;
-
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
-
-import java.io.IOException;
-
-public abstract class WatcherRestHandler extends BaseRestHandler {
-
-    protected static String URI_BASE = "/_xpack";
-
-    public WatcherRestHandler(Settings settings) {
-        super(settings);
-    }
-
-    @Override
-    public final RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
-        return doPrepareRequest(request, new WatcherClient(client));
-    }
-
-    protected abstract RestChannelConsumer doPrepareRequest(RestRequest request, WatcherClient client) throws IOException;
-
-}
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestAckWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestAckWatchAction.java
index 23180631902b8..027054c499894 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestAckWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestAckWatchAction.java
@@ -7,21 +7,22 @@
 package org.elasticsearch.xpack.watcher.rest.action;
 
 import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams;
+import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.WatchField;
-import org.elasticsearch.xpack.watcher.rest.WatcherRestHandler;
 
 import java.io.IOException;
 
@@ -31,7 +32,7 @@
 /**
  * The rest action to ack a watch
  */
-public class RestAckWatchAction extends WatcherRestHandler {
+public class RestAckWatchAction extends BaseRestHandler {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestAckWatchAction.class));
 
@@ -40,16 +41,16 @@ public RestAckWatchAction(Settings settings, RestController controller) {
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
             POST, "/_watcher/watch/{id}/_ack", this,
-            POST, URI_BASE + "/watcher/watch/{id}/_ack", deprecationLogger);
+            POST, "/_xpack/watcher/watch/{id}/_ack", deprecationLogger);
         controller.registerWithDeprecatedHandler(
             PUT, "/_watcher/watch/{id}/_ack", this,
-            PUT, URI_BASE + "/watcher/watch/{id}/_ack", deprecationLogger);
+            PUT, "/_xpack/watcher/watch/{id}/_ack", deprecationLogger);
         controller.registerWithDeprecatedHandler(
             POST, "/_watcher/watch/{id}/_ack/{actions}", this,
-            POST, URI_BASE + "/watcher/watch/{id}/_ack/{actions}", deprecationLogger);
+            POST, "/_xpack/watcher/watch/{id}/_ack/{actions}", deprecationLogger);
         controller.registerWithDeprecatedHandler(
             PUT, "/_watcher/watch/{id}/_ack/{actions}", this,
-            PUT, URI_BASE + "/watcher/watch/{id}/_ack/{actions}", deprecationLogger);
+            PUT, "/_xpack/watcher/watch/{id}/_ack/{actions}", deprecationLogger);
     }
 
     @Override
@@ -58,13 +59,13 @@ public String getName() {
     }
 
     @Override
-    public RestChannelConsumer doPrepareRequest(RestRequest request, WatcherClient client) throws IOException {
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         AckWatchRequest ackWatchRequest = new AckWatchRequest(request.param("id"));
         String[] actions = request.paramAsStringArray("actions", null);
         if (actions != null) {
             ackWatchRequest.setActionIds(actions);
         }
-        return channel -> client.ackWatch(ackWatchRequest, new RestBuilderListener<AckWatchResponse>(channel) {
+        return channel -> client.execute(AckWatchAction.INSTANCE, ackWatchRequest, new RestBuilderListener<AckWatchResponse>(channel) {
             @Override
             public RestResponse buildResponse(AckWatchResponse response, XContentBuilder builder) throws Exception {
                 return new BytesRestResponse(RestStatus.OK, builder.startObject()
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestActivateWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestActivateWatchAction.java
index 93ce7f1b322b0..23db7054a68c0 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestActivateWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestActivateWatchAction.java
@@ -7,21 +7,22 @@
 package org.elasticsearch.xpack.watcher.rest.action;
 
 import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams;
+import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.WatchField;
-import org.elasticsearch.xpack.watcher.rest.WatcherRestHandler;
 
 import java.io.IOException;
 
@@ -31,7 +32,7 @@
 /**
  * The rest action to de/activate a watch
  */
-public class RestActivateWatchAction extends WatcherRestHandler {
+public class RestActivateWatchAction extends BaseRestHandler {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestActivateWatchAction.class));
 
@@ -40,18 +41,18 @@ public RestActivateWatchAction(Settings settings, RestController controller) {
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
             POST, "/_watcher/watch/{id}/_activate", this,
-            POST, URI_BASE + "/watcher/watch/{id}/_activate", deprecationLogger);
+            POST, "/_xpack/watcher/watch/{id}/_activate", deprecationLogger);
         controller.registerWithDeprecatedHandler(
             PUT, "/_watcher/watch/{id}/_activate", this,
-            PUT, URI_BASE + "/watcher/watch/{id}/_activate", deprecationLogger);
+            PUT, "/_xpack/watcher/watch/{id}/_activate", deprecationLogger);
 
         final DeactivateRestHandler deactivateRestHandler = new DeactivateRestHandler(settings);
         controller.registerWithDeprecatedHandler(
             POST, "/_watcher/watch/{id}/_deactivate", deactivateRestHandler,
-            POST, URI_BASE + "/watcher/watch/{id}/_deactivate", deprecationLogger);
+            POST, "/_xpack/watcher/watch/{id}/_deactivate", deprecationLogger);
         controller.registerWithDeprecatedHandler(
             PUT, "/_watcher/watch/{id}/_deactivate", deactivateRestHandler,
-            PUT, URI_BASE + "/watcher/watch/{id}/_deactivate", deprecationLogger);
+            PUT, "/_xpack/watcher/watch/{id}/_deactivate", deprecationLogger);
     }
 
     @Override
@@ -60,20 +61,21 @@ public String getName() {
     }
 
     @Override
-    public RestChannelConsumer doPrepareRequest(RestRequest request, WatcherClient client) throws IOException {
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         String watchId = request.param("id");
         return channel ->
-                client.activateWatch(new ActivateWatchRequest(watchId, true), new RestBuilderListener<ActivateWatchResponse>(channel) {
-                    @Override
-                    public RestResponse buildResponse(ActivateWatchResponse response, XContentBuilder builder) throws Exception {
-                        return new BytesRestResponse(RestStatus.OK, builder.startObject()
-                                .field(WatchField.STATUS.getPreferredName(), response.getStatus(), WatcherParams.HIDE_SECRETS)
-                                .endObject());
-                    }
-                });
+                client.execute(ActivateWatchAction.INSTANCE, new ActivateWatchRequest(watchId, true),
+                    new RestBuilderListener<ActivateWatchResponse>(channel) {
+                        @Override
+                        public RestResponse buildResponse(ActivateWatchResponse response, XContentBuilder builder) throws Exception {
+                            return new BytesRestResponse(RestStatus.OK, builder.startObject()
+                                    .field(WatchField.STATUS.getPreferredName(), response.getStatus(), WatcherParams.HIDE_SECRETS)
+                                    .endObject());
+                        }
+                    });
     }
 
-    private static class DeactivateRestHandler extends WatcherRestHandler {
+    private static class DeactivateRestHandler extends BaseRestHandler {
 
         DeactivateRestHandler(Settings settings) {
             super(settings);
@@ -85,17 +87,18 @@ public String getName() {
         }
 
         @Override
-        public RestChannelConsumer doPrepareRequest(RestRequest request, WatcherClient client) throws IOException {
+        public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
             String watchId = request.param("id");
             return channel ->
-                    client.activateWatch(new ActivateWatchRequest(watchId, false), new RestBuilderListener<ActivateWatchResponse>(channel) {
-                        @Override
-                        public RestResponse buildResponse(ActivateWatchResponse response, XContentBuilder builder) throws Exception {
-                            return new BytesRestResponse(RestStatus.OK, builder.startObject()
-                                    .field(WatchField.STATUS.getPreferredName(), response.getStatus(), WatcherParams.HIDE_SECRETS)
-                                    .endObject());
-                        }
-                    });
+                    client.execute(ActivateWatchAction.INSTANCE, new ActivateWatchRequest(watchId, false),
+                        new RestBuilderListener<ActivateWatchResponse>(channel) {
+                            @Override
+                            public RestResponse buildResponse(ActivateWatchResponse response, XContentBuilder builder) throws Exception {
+                                return new BytesRestResponse(RestStatus.OK, builder.startObject()
+                                        .field(WatchField.STATUS.getPreferredName(), response.getStatus(), WatcherParams.HIDE_SECRETS)
+                                        .endObject());
+                            }
+                        });
         }
     }
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestDeleteWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestDeleteWatchAction.java
index 0b0fbc478df36..1594e2c2ef92b 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestDeleteWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestDeleteWatchAction.java
@@ -7,19 +7,20 @@
 package org.elasticsearch.xpack.watcher.rest.action;
 
 import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
-import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest;
-import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
-import org.elasticsearch.xpack.watcher.rest.WatcherRestHandler;
+import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchAction;
 
 import java.io.IOException;
 
@@ -27,7 +28,7 @@
 import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
-public class RestDeleteWatchAction extends WatcherRestHandler {
+public class RestDeleteWatchAction extends BaseRestHandler {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestDeleteWatchAction.class));
 
@@ -36,7 +37,7 @@ public RestDeleteWatchAction(Settings settings, RestController controller) {
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
             DELETE, "/_watcher/watch/{id}", this,
-            DELETE, URI_BASE + "/watcher/watch/{id}", deprecationLogger);
+            DELETE, "/_xpack/watcher/watch/{id}", deprecationLogger);
     }
 
     @Override
@@ -45,9 +46,9 @@ public String getName() {
     }
 
     @Override
-    protected RestChannelConsumer doPrepareRequest(final RestRequest request, WatcherClient client) throws IOException {
+    protected RestChannelConsumer prepareRequest(final RestRequest request, NodeClient client) throws IOException {
         DeleteWatchRequest deleteWatchRequest = new DeleteWatchRequest(request.param("id"));
-        return channel -> client.deleteWatch(deleteWatchRequest, new RestBuilderListener<DeleteWatchResponse>(channel) {
+        return channel -> client.execute(DeleteWatchAction.INSTANCE, deleteWatchRequest, new RestBuilderListener<>(channel) {
             @Override
             public RestResponse buildResponse(DeleteWatchResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject()
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchAction.java
index 2c85dca35c0e4..a19c795c83672 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchAction.java
@@ -8,6 +8,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.logging.DeprecationLogger;
@@ -15,6 +16,7 @@
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -22,14 +24,13 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.xpack.core.security.rest.RestRequestFilter;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.execution.ActionExecutionMode;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams;
+import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.WatchField;
-import org.elasticsearch.xpack.watcher.rest.WatcherRestHandler;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -42,7 +43,7 @@
 import static org.elasticsearch.xpack.watcher.rest.action.RestExecuteWatchAction.Field.IGNORE_CONDITION;
 import static org.elasticsearch.xpack.watcher.rest.action.RestExecuteWatchAction.Field.RECORD_EXECUTION;
 
-public class RestExecuteWatchAction extends WatcherRestHandler implements RestRequestFilter {
+public class RestExecuteWatchAction extends BaseRestHandler implements RestRequestFilter {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestExecuteWatchAction.class));
 
@@ -57,16 +58,16 @@ public RestExecuteWatchAction(Settings settings, RestController controller) {
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
             POST, "/_watcher/watch/{id}/_execute", this,
-            POST, URI_BASE + "/watcher/watch/{id}/_execute", deprecationLogger);
+            POST, "/_xpack/watcher/watch/{id}/_execute", deprecationLogger);
         controller.registerWithDeprecatedHandler(
             PUT, "/_watcher/watch/{id}/_execute", this,
-            PUT, URI_BASE + "/watcher/watch/{id}/_execute", deprecationLogger);
+            PUT, "/_xpack/watcher/watch/{id}/_execute", deprecationLogger);
         controller.registerWithDeprecatedHandler(
             POST, "/_watcher/watch/_execute", this,
-            POST, URI_BASE + "/watcher/watch/_execute", deprecationLogger);
+            POST, "/_xpack/watcher/watch/_execute", deprecationLogger);
         controller.registerWithDeprecatedHandler(
             PUT, "/_watcher/watch/_execute", this,
-            PUT, URI_BASE + "/watcher/watch/_execute", deprecationLogger);
+            PUT, "/_xpack/watcher/watch/_execute", deprecationLogger);
     }
 
     @Override
@@ -75,10 +76,10 @@ public String getName() {
     }
 
     @Override
-    protected RestChannelConsumer doPrepareRequest(final RestRequest request, WatcherClient client) throws IOException {
+    protected RestChannelConsumer prepareRequest(final RestRequest request, NodeClient client) throws IOException {
         ExecuteWatchRequest executeWatchRequest = parseRequest(request, client);
 
-        return channel -> client.executeWatch(executeWatchRequest, new RestBuilderListener<ExecuteWatchResponse>(channel) {
+        return channel -> client.execute(ExecuteWatchAction.INSTANCE, executeWatchRequest, new RestBuilderListener<>(channel) {
             @Override
             public RestResponse buildResponse(ExecuteWatchResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject();
@@ -90,9 +91,9 @@ public RestResponse buildResponse(ExecuteWatchResponse response, XContentBuilder
         });
     }
 
-    //This tightly binds the REST API to the java API
-    private ExecuteWatchRequest parseRequest(RestRequest request, WatcherClient client) throws IOException {
-        ExecuteWatchRequestBuilder builder = client.prepareExecuteWatch();
+    //This tightly binds the REST API to the java API. pkg private for testing
+    static ExecuteWatchRequest parseRequest(RestRequest request, NodeClient client) throws IOException {
+        ExecuteWatchRequestBuilder builder = new ExecuteWatchRequestBuilder(client);
         builder.setId(request.param("id"));
         builder.setDebug(WatcherParams.debug(request));
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java
index 0d9b1ee6a1991..be2d588bb0cc4 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestGetWatchAction.java
@@ -7,25 +7,26 @@
 package org.elasticsearch.xpack.watcher.rest.action;
 
 import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
+import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
-import org.elasticsearch.xpack.watcher.rest.WatcherRestHandler;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
-public class RestGetWatchAction extends WatcherRestHandler {
+public class RestGetWatchAction extends BaseRestHandler {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetWatchAction.class));
 
@@ -34,7 +35,7 @@ public RestGetWatchAction(Settings settings, RestController controller) {
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
             GET, "/_watcher/watch/{id}", this,
-            GET, URI_BASE + "/watcher/watch/{id}", deprecationLogger);
+            GET, "/_xpack/watcher/watch/{id}", deprecationLogger);
     }
 
     @Override
@@ -43,9 +44,9 @@ public String getName() {
     }
 
     @Override
-    protected RestChannelConsumer doPrepareRequest(final RestRequest request, WatcherClient client) {
+    protected RestChannelConsumer prepareRequest(final RestRequest request, NodeClient client) {
         final GetWatchRequest getWatchRequest = new GetWatchRequest(request.param("id"));
-        return channel -> client.getWatch(getWatchRequest, new RestBuilderListener<GetWatchResponse>(channel) {
+        return channel -> client.execute(GetWatchAction.INSTANCE, getWatchRequest, new RestBuilderListener<>(channel) {
             @Override
             public RestResponse buildResponse(GetWatchResponse response, XContentBuilder builder) throws Exception {
                 response.toXContent(builder, request);
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java
index edbfa3cf1fe28..213cb68b71088 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestPutWatchAction.java
@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.watcher.rest.action;
 
 import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
@@ -14,6 +15,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -21,8 +23,7 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.xpack.core.security.rest.RestRequestFilter;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
-import org.elasticsearch.xpack.watcher.rest.WatcherRestHandler;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -33,7 +34,7 @@
 import static org.elasticsearch.rest.RestStatus.CREATED;
 import static org.elasticsearch.rest.RestStatus.OK;
 
-public class RestPutWatchAction extends WatcherRestHandler implements RestRequestFilter {
+public class RestPutWatchAction extends BaseRestHandler implements RestRequestFilter {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestPutWatchAction.class));
 
@@ -42,10 +43,10 @@ public RestPutWatchAction(Settings settings, RestController controller) {
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
             POST, "/_watcher/watch/{id}", this,
-            POST, URI_BASE + "/watcher/watch/{id}", deprecationLogger);
+            POST, "/_xpack/watcher/watch/{id}", deprecationLogger);
         controller.registerWithDeprecatedHandler(
             PUT, "/_watcher/watch/{id}", this,
-            PUT, URI_BASE + "/watcher/watch/{id}", deprecationLogger);
+            PUT, "/_xpack/watcher/watch/{id}", deprecationLogger);
     }
 
     @Override
@@ -54,14 +55,14 @@ public String getName() {
     }
 
     @Override
-    protected RestChannelConsumer doPrepareRequest(final RestRequest request, WatcherClient client) throws IOException {
+    protected RestChannelConsumer prepareRequest(final RestRequest request, NodeClient client) throws IOException {
         PutWatchRequest putWatchRequest =
                 new PutWatchRequest(request.param("id"), request.content(), request.getXContentType());
         putWatchRequest.setVersion(request.paramAsLong("version", Versions.MATCH_ANY));
         putWatchRequest.setIfSeqNo(request.paramAsLong("if_seq_no", putWatchRequest.getIfSeqNo()));
         putWatchRequest.setIfPrimaryTerm(request.paramAsLong("if_primary_term", putWatchRequest.getIfPrimaryTerm()));
         putWatchRequest.setActive(request.paramAsBoolean("active", putWatchRequest.isActive()));
-        return channel -> client.putWatch(putWatchRequest, new RestBuilderListener<PutWatchResponse>(channel) {
+        return channel -> client.execute(PutWatchAction.INSTANCE, putWatchRequest, new RestBuilderListener<>(channel) {
             @Override
             public RestResponse buildResponse(PutWatchResponse response, XContentBuilder builder) throws Exception {
                 response.toXContent(builder, request);
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatchServiceAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatchServiceAction.java
index d2ec2bd423e54..bd8c6f781ebcc 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatchServiceAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatchServiceAction.java
@@ -7,18 +7,19 @@
 package org.elasticsearch.xpack.watcher.rest.action;
 
 import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
+import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceRequest;
-import org.elasticsearch.xpack.watcher.rest.WatcherRestHandler;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
-public class RestWatchServiceAction extends WatcherRestHandler {
+public class RestWatchServiceAction extends BaseRestHandler {
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestWatchServiceAction.class));
 
@@ -27,10 +28,10 @@ public RestWatchServiceAction(Settings settings, RestController controller) {
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
             POST, "/_watcher/_start", this,
-            POST, URI_BASE + "/watcher/_start", deprecationLogger);
+            POST, "/_xpack/watcher/_start", deprecationLogger);
         controller.registerWithDeprecatedHandler(
             POST, "/_watcher/_stop", new StopRestHandler(settings),
-            POST, URI_BASE + "/watcher/_stop", deprecationLogger);
+            POST, "/_xpack/watcher/_stop", deprecationLogger);
     }
 
     @Override
@@ -39,11 +40,12 @@ public String getName() {
     }
 
     @Override
-    public RestChannelConsumer doPrepareRequest(RestRequest request, WatcherClient client) {
-        return channel -> client.watcherService(new WatcherServiceRequest().start(), new RestToXContentListener<>(channel));
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
+        return channel ->
+            client.execute(WatcherServiceAction.INSTANCE, new WatcherServiceRequest().start(), new RestToXContentListener<>(channel));
     }
 
-    private static class StopRestHandler extends WatcherRestHandler {
+    private static class StopRestHandler extends BaseRestHandler {
 
         StopRestHandler(Settings settings) {
             super(settings);
@@ -55,8 +57,9 @@ public String getName() {
         }
 
         @Override
-        public RestChannelConsumer doPrepareRequest(RestRequest request, WatcherClient client) {
-            return channel -> client.watcherService(new WatcherServiceRequest().stop(), new RestToXContentListener<>(channel));
+        public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
+            return channel ->
+                client.execute(WatcherServiceAction.INSTANCE, new WatcherServiceRequest().stop(), new RestToXContentListener<>(channel));
         }
     }
 }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatcherStatsAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatcherStatsAction.java
index 6d2640d66da3d..201a56363bad3 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatcherStatsAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/rest/action/RestWatcherStatsAction.java
@@ -6,17 +6,18 @@
 
 package org.elasticsearch.xpack.watcher.rest.action;
 
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
-import org.elasticsearch.common.logging.DeprecationLogger;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
+import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsRequest;
-import org.elasticsearch.xpack.watcher.rest.WatcherRestHandler;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -24,7 +25,7 @@
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
-public class RestWatcherStatsAction extends WatcherRestHandler {
+public class RestWatcherStatsAction extends BaseRestHandler {
     private static final Logger logger = LogManager.getLogger(RestWatcherStatsAction.class);
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
 
@@ -33,10 +34,10 @@ public RestWatcherStatsAction(Settings settings, RestController controller) {
         // TODO: remove deprecated endpoint in 8.0.0
         controller.registerWithDeprecatedHandler(
             GET, "/_watcher/stats", this,
-            GET, URI_BASE + "/watcher/stats", deprecationLogger);
+            GET, "/_xpack/watcher/stats", deprecationLogger);
         controller.registerWithDeprecatedHandler(
             GET, "/_watcher/stats/{metric}", this,
-            GET, URI_BASE + "/watcher/stats/{metric}", deprecationLogger);
+            GET, "/_xpack/watcher/stats/{metric}", deprecationLogger);
     }
 
     @Override
@@ -45,7 +46,7 @@ public String getName() {
     }
 
     @Override
-    protected RestChannelConsumer doPrepareRequest(final RestRequest restRequest, WatcherClient client) throws IOException {
+    protected RestChannelConsumer prepareRequest(final RestRequest restRequest, NodeClient client) throws IOException {
         Set<String> metrics = Strings.tokenizeByCommaToSet(restRequest.param("metric", ""));
 
         WatcherStatsRequest request = new WatcherStatsRequest();
@@ -62,7 +63,7 @@ protected RestChannelConsumer doPrepareRequest(final RestRequest restRequest, Wa
         }
 
 
-        return channel -> client.watcherStats(request, new RestActions.NodesResponseRestListener<>(channel));
+        return channel -> client.execute(WatcherStatsAction.INSTANCE, request, new RestActions.NodesResponseRestListener<>(channel));
     }
 
     private static final Set<String> RESPONSE_PARAMS = Collections.singleton("emit_stacktraces");
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateService.java
index d86a63948c7ac..f3ae021eb560f 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateService.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/search/WatcherSearchTemplateService.java
@@ -57,9 +57,6 @@ public String renderTemplate(Script source, WatchExecutionContext ctx, Payload p
 
     public SearchRequest toSearchRequest(WatcherSearchTemplateRequest request) throws IOException {
         SearchRequest searchRequest = new SearchRequest(request.getIndices());
-        if (request.getTypes() != null) {
-            searchRequest.types(request.getTypes());
-        }
         searchRequest.searchType(request.getSearchType());
         searchRequest.indicesOptions(request.getIndicesOptions());
         SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchAction.java
index 36643ed18f8d8..ab26c5df6303d 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchAction.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsRequest;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.core.watcher.watch.WatchField;
+import org.elasticsearch.xpack.watcher.ClockHolder;
 import org.elasticsearch.xpack.watcher.transport.actions.WatcherTransportAction;
 import org.elasticsearch.xpack.watcher.watch.WatchParser;
 
@@ -54,10 +55,10 @@ public class TransportAckWatchAction extends WatcherTransportAction<AckWatchRequ
 
     @Inject
     public TransportAckWatchAction(TransportService transportService, ActionFilters actionFilters,
-                                   Clock clock, XPackLicenseState licenseState, WatchParser parser,
+                                   ClockHolder clockHolder, XPackLicenseState licenseState, WatchParser parser,
                                    Client client, ClusterService clusterService) {
         super(AckWatchAction.NAME, transportService, actionFilters, licenseState, AckWatchRequest::new);
-        this.clock = clock;
+        this.clock = clockHolder.clock;
         this.parser = parser;
         this.client = client;
         this.clusterService = clusterService;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/activate/TransportActivateWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/activate/TransportActivateWatchAction.java
index 0801f1e3ce4c4..7393f4b96bbac 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/activate/TransportActivateWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/activate/TransportActivateWatchAction.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.core.watcher.watch.WatchField;
 import org.elasticsearch.xpack.core.watcher.watch.WatchStatus;
+import org.elasticsearch.xpack.watcher.ClockHolder;
 import org.elasticsearch.xpack.watcher.transport.actions.WatcherTransportAction;
 import org.elasticsearch.xpack.watcher.watch.WatchParser;
 
@@ -50,9 +51,9 @@ public class TransportActivateWatchAction extends WatcherTransportAction<Activat
 
     @Inject
     public TransportActivateWatchAction(TransportService transportService, ActionFilters actionFilters,
-                                        Clock clock, XPackLicenseState licenseState, WatchParser parser, Client client) {
+                                        ClockHolder clockHolder, XPackLicenseState licenseState, WatchParser parser, Client client) {
         super(ActivateWatchAction.NAME, transportService, actionFilters, licenseState, ActivateWatchRequest::new);
-        this.clock = clock;
+        this.clock = clockHolder.clock;
         this.parser = parser;
         this.client = client;
     }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/execute/TransportExecuteWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/execute/TransportExecuteWatchAction.java
index a049267594611..db0e204b51d44 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/execute/TransportExecuteWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/execute/TransportExecuteWatchAction.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.xpack.core.watcher.trigger.TriggerEvent;
 import org.elasticsearch.xpack.core.watcher.watch.Payload;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
+import org.elasticsearch.xpack.watcher.ClockHolder;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.execution.ExecutionService;
 import org.elasticsearch.xpack.watcher.execution.ManualExecutionContext;
@@ -67,13 +68,13 @@ public class TransportExecuteWatchAction extends WatcherTransportAction<ExecuteW
 
     @Inject
     public TransportExecuteWatchAction(TransportService transportService, ThreadPool threadPool,
-                                       ActionFilters actionFilters, ExecutionService executionService, Clock clock,
+                                       ActionFilters actionFilters, ExecutionService executionService, ClockHolder clockHolder,
                                        XPackLicenseState licenseState, WatchParser watchParser, Client client,
                                        TriggerService triggerService) {
         super(ExecuteWatchAction.NAME, transportService, actionFilters, licenseState, ExecuteWatchRequest::new);
         this.threadPool = threadPool;
         this.executionService = executionService;
-        this.clock = clock;
+        this.clock = clockHolder.clock;
         this.triggerService = triggerService;
         this.watchParser = watchParser;
         this.client = client;
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/get/TransportGetWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/get/TransportGetWatchAction.java
index b8d6be360de9a..ca37b02c05469 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/get/TransportGetWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/get/TransportGetWatchAction.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
+import org.elasticsearch.xpack.watcher.ClockHolder;
 import org.elasticsearch.xpack.watcher.transport.actions.WatcherTransportAction;
 import org.elasticsearch.xpack.watcher.watch.WatchParser;
 
@@ -43,10 +44,10 @@ public class TransportGetWatchAction extends WatcherTransportAction<GetWatchRequ
 
     @Inject
     public TransportGetWatchAction(TransportService transportService, ActionFilters actionFilters,
-                                   XPackLicenseState licenseState, WatchParser parser, Clock clock, Client client) {
+                                   XPackLicenseState licenseState, WatchParser parser, ClockHolder clockHolder, Client client) {
         super(GetWatchAction.NAME, transportService, actionFilters, licenseState, GetWatchRequest::new);
         this.parser = parser;
-        this.clock = clock;
+        this.clock = clockHolder.clock;
         this.client = client;
     }
 
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchAction.java
index 0074ec9ac3fa9..fc887b4d12f6a 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchAction.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.xpack.core.watcher.support.xcontent.WatcherParams;
 import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
+import org.elasticsearch.xpack.watcher.ClockHolder;
 import org.elasticsearch.xpack.watcher.transport.actions.WatcherTransportAction;
 import org.elasticsearch.xpack.watcher.watch.WatchParser;
 
@@ -66,10 +67,10 @@ public class TransportPutWatchAction extends WatcherTransportAction<PutWatchRequ
 
     @Inject
     public TransportPutWatchAction(TransportService transportService, ThreadPool threadPool, ActionFilters actionFilters,
-                                   Clock clock, XPackLicenseState licenseState, WatchParser parser, Client client) {
+                                   ClockHolder clockHolder, XPackLicenseState licenseState, WatchParser parser, Client client) {
         super(PutWatchAction.NAME, transportService, actionFilters, licenseState, PutWatchRequest::new);
         this.threadPool = threadPool;
-        this.clock = clock;
+        this.clock = clockHolder.clock;
         this.parser = parser;
         this.client = client;
     }
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/service/TransportWatcherServiceAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/service/TransportWatcherServiceAction.java
index 047e43acf35ac..a992a9a3675c3 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/service/TransportWatcherServiceAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/service/TransportWatcherServiceAction.java
@@ -20,7 +20,9 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackPlugin;
@@ -28,6 +30,8 @@
 import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceRequest;
 
+import java.io.IOException;
+
 public class TransportWatcherServiceAction extends TransportMasterNodeAction<WatcherServiceRequest, AcknowledgedResponse> {
 
     private AckedRequest ackedRequest = new AckedRequest() {
@@ -55,13 +59,18 @@ protected String executor() {
         return ThreadPool.Names.MANAGEMENT;
     }
 
+    @Override
+    protected AcknowledgedResponse read(StreamInput in) throws IOException {
+        return new AcknowledgedResponse(in);
+    }
+
     @Override
     protected AcknowledgedResponse newResponse() {
-        return new AcknowledgedResponse();
+        throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");
     }
 
     @Override
-    protected void masterOperation(WatcherServiceRequest request, ClusterState state,
+    protected void masterOperation(Task task, WatcherServiceRequest request, ClusterState state,
                                    ActionListener<AcknowledgedResponse> listener) {
         switch (request.getCommand()) {
             case STOP:
diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/stats/TransportWatcherStatsAction.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/stats/TransportWatcherStatsAction.java
index 51251e58003e8..39de29f5eaf40 100644
--- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/stats/TransportWatcherStatsAction.java
+++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/stats/TransportWatcherStatsAction.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.watcher.WatcherMetaData;
@@ -52,8 +53,8 @@ protected WatcherStatsResponse newResponse(WatcherStatsRequest request, List<Wat
     }
 
     @Override
-    protected WatcherStatsRequest.Node newNodeRequest(String nodeId, WatcherStatsRequest request) {
-        return new WatcherStatsRequest.Node(request, nodeId);
+    protected WatcherStatsRequest.Node newNodeRequest(WatcherStatsRequest request) {
+        return new WatcherStatsRequest.Node(request);
     }
 
     @Override
@@ -62,7 +63,7 @@ protected WatcherStatsResponse.Node newNodeResponse() {
     }
 
     @Override
-    protected WatcherStatsResponse.Node nodeOperation(WatcherStatsRequest.Node request) {
+    protected WatcherStatsResponse.Node nodeOperation(WatcherStatsRequest.Node request, Task task) {
         WatcherStatsResponse.Node statsResponse = new WatcherStatsResponse.Node(clusterService.localNode());
         statsResponse.setWatcherState(lifeCycleService.getState());
         statsResponse.setThreadPoolQueueSize(executionService.executionThreadPoolQueueSize());
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherConcreteIndexTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherConcreteIndexTests.java
index 237c0a2bdf153..5eddbb07884dc 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherConcreteIndexTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherConcreteIndexTests.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
@@ -33,7 +34,7 @@ public void testCanUseAnyConcreteIndexName() throws Exception {
         ensureGreen(newWatcherIndexName);
         startWatcher();
 
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("mywatch").setSource(watchBuilder()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client(), "mywatch").setSource(watchBuilder()
             .trigger(schedule(interval("3s")))
             .input(noneInput())
             .condition(InternalAlwaysCondition.INSTANCE)
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java
index 6cea483995ac9..ef3275c3f9ab8 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingNode;
@@ -432,14 +433,14 @@ public void testOnNonDataNodes() {
 
         DiscoveryNode node1 = new DiscoveryNode("node_1", ESTestCase.buildNewFakeTransportAddress(),
                 Collections.emptyMap(), new HashSet<>(Collections.singletonList(
-                        randomFrom(DiscoveryNode.Role.INGEST, DiscoveryNode.Role.MASTER))),
+                        randomFrom(DiscoveryNodeRole.INGEST_ROLE, DiscoveryNodeRole.MASTER_ROLE))),
                 Version.CURRENT);
 
         DiscoveryNode node2 = new DiscoveryNode("node_2", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(),
-                new HashSet<>(Collections.singletonList(DiscoveryNode.Role.DATA)), Version.CURRENT);
+                new HashSet<>(Collections.singletonList(DiscoveryNodeRole.DATA_ROLE)), Version.CURRENT);
 
         DiscoveryNode node3 = new DiscoveryNode("node_3", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(),
-                new HashSet<>(Collections.singletonList(DiscoveryNode.Role.DATA)), Version.CURRENT);
+                new HashSet<>(Collections.singletonList(DiscoveryNodeRole.DATA_ROLE)), Version.CURRENT);
 
         IndexMetaData.Builder indexMetaDataBuilder = createIndexBuilder(Watch.INDEX, 1 ,0);
 
@@ -698,6 +699,6 @@ private IndexMetaData.Builder createIndexBuilder(String name, int numberOfShards
 
     private static DiscoveryNode newNode(String nodeId) {
         return new DiscoveryNode(nodeId, ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(),
-                new HashSet<>(asList(DiscoveryNode.Role.values())), Version.CURRENT);
+                DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT);
     }
 }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherFeatureSetTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java
similarity index 81%
rename from x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherFeatureSetTests.java
rename to x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java
index 46b0fc0c30e7b..e33ee16b63dff 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherFeatureSetTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherInfoTransportActionTests.java
@@ -7,23 +7,28 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
 import org.elasticsearch.xpack.core.watcher.WatcherFeatureSetUsage;
 import org.elasticsearch.xpack.core.watcher.WatcherMetaData;
 import org.elasticsearch.xpack.core.watcher.common.stats.Counters;
-import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsResponse;
@@ -44,7 +49,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class WatcherFeatureSetTests extends ESTestCase {
+public class WatcherInfoTransportActionTests extends ESTestCase {
 
     private XPackLicenseState licenseState;
     private Client client;
@@ -60,7 +65,8 @@ public void init() throws Exception {
     }
 
     public void testAvailable() {
-        WatcherFeatureSet featureSet = new WatcherFeatureSet(Settings.EMPTY, licenseState, client);
+        WatcherInfoTransportAction featureSet = new WatcherInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState);
         boolean available = randomBoolean();
         when(licenseState.isWatcherAllowed()).thenReturn(available);
         assertThat(featureSet.available(), is(available));
@@ -76,7 +82,8 @@ public void testEnabled() {
         } else {
             settings.put("xpack.watcher.enabled", enabled);
         }
-        WatcherFeatureSet featureSet = new WatcherFeatureSet(settings.build(), licenseState, client);
+        WatcherInfoTransportAction featureSet = new WatcherInfoTransportAction(
+            mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState);
         assertThat(featureSet.enabled(), is(enabled));
     }
 
@@ -106,10 +113,16 @@ public void testUsageStats() throws Exception {
                     nodes, Collections.emptyList()));
             return null;
         }).when(client).execute(eq(WatcherStatsAction.INSTANCE), any(), any());
-
-        PlainActionFuture<WatcherFeatureSet.Usage> future = new PlainActionFuture<>();
-        new WatcherFeatureSet(Settings.EMPTY, licenseState, client).usage(future);
-        WatcherFeatureSetUsage watcherUsage = (WatcherFeatureSetUsage) future.get();
+        ClusterService clusterService = mock(ClusterService.class);
+        final DiscoveryNode mockNode = mock(DiscoveryNode.class);
+        when(mockNode.getId()).thenReturn("mocknode");
+        when(clusterService.localNode()).thenReturn(mockNode);
+
+        var usageAction = new WatcherUsageTransportAction(mock(TransportService.class), clusterService, null,
+            mock(ActionFilters.class), null, Settings.EMPTY, licenseState, client);
+        PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
+        usageAction.masterOperation(mock(Task.class), null, null, future);
+        WatcherFeatureSetUsage watcherUsage = (WatcherFeatureSetUsage) future.get().getUsage();
         assertThat(watcherUsage.stats().keySet(), containsInAnyOrder("foo", "spam"));
         long fooBarBaz = ObjectPath.eval("foo.bar.baz", watcherUsage.stats());
         assertThat(fooBarBaz, is(5L));
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java
index 19e2a13487123..548583ac14b7d 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
@@ -327,13 +328,13 @@ public void testNonDataNode() {
         IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index).addShard(shardRouting);
 
         DiscoveryNode node1 = new DiscoveryNode("node_1", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(),
-            new HashSet<>(asList(randomFrom(DiscoveryNode.Role.INGEST, DiscoveryNode.Role.MASTER))), Version.CURRENT);
+            new HashSet<>(asList(randomFrom(DiscoveryNodeRole.INGEST_ROLE, DiscoveryNodeRole.MASTER_ROLE))), Version.CURRENT);
 
         DiscoveryNode node2 = new DiscoveryNode("node_2", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(),
-            new HashSet<>(asList(DiscoveryNode.Role.DATA)), Version.CURRENT);
+            new HashSet<>(asList(DiscoveryNodeRole.DATA_ROLE)), Version.CURRENT);
 
         DiscoveryNode node3 = new DiscoveryNode("node_3", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(),
-            new HashSet<>(asList(DiscoveryNode.Role.DATA)), Version.CURRENT);
+            new HashSet<>(asList(DiscoveryNodeRole.DATA_ROLE)), Version.CURRENT);
 
         IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(Watch.INDEX)
             .settings(Settings.builder()
@@ -465,7 +466,8 @@ public void testWatcherStopsOnClusterLevelBlock() {
     }
 
     public void testMasterOnlyNodeCanStart() {
-        List<DiscoveryNode.Role> roles = Collections.singletonList(randomFrom(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.INGEST));
+        List<DiscoveryNodeRole> roles =
+                Collections.singletonList(randomFrom(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.INGEST_ROLE));
         ClusterState state = ClusterState.builder(new ClusterName("my-cluster"))
             .nodes(new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1")
                 .add(new DiscoveryNode("node_1", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(),
@@ -596,6 +598,6 @@ private static DiscoveryNode newNode(String nodeName) {
 
     private static DiscoveryNode newNode(String nodeName, Version version) {
         return new DiscoveryNode(nodeName, ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(),
-            new HashSet<>(asList(DiscoveryNode.Role.values())), version);
+            DiscoveryNodeRole.BUILT_IN_ROLES, version);
     }
 }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java
index a4131889f84a0..c9e0ed8934359 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherPluginTests.java
@@ -67,8 +67,7 @@ public void testWatcherDisabledTests() throws Exception {
 
         List<ExecutorBuilder<?>> executorBuilders = watcher.getExecutorBuilders(settings);
         assertThat(executorBuilders, hasSize(0));
-        assertThat(watcher.createGuiceModules(), hasSize(2));
-        assertThat(watcher.getActions(), hasSize(0));
+        assertThat(watcher.getActions(), hasSize(2));
         assertThat(watcher.getRestHandlers(settings, null, null, null, null, null, null), hasSize(0));
 
         // ensure index module is not called, even if watches index is tried
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java
index 3d1fe78e27a00..f4ee831266b38 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
@@ -62,11 +63,9 @@
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 
-import static java.util.Arrays.asList;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 import static org.mockito.Matchers.any;
@@ -277,6 +276,6 @@ void stopExecutor() {
 
     private static DiscoveryNode newNode() {
         return new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(),
-                new HashSet<>(asList(DiscoveryNode.Role.values())), Version.CURRENT);
+                DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT);
     }
 }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherXpackUsageStatsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherXpackUsageStatsTests.java
deleted file mode 100644
index 7c07c98eb4725..0000000000000
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherXpackUsageStatsTests.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.watcher;
-
-import org.elasticsearch.xpack.core.XPackFeatureSet;
-import org.elasticsearch.xpack.core.action.XPackUsageAction;
-import org.elasticsearch.protocol.xpack.XPackUsageRequest;
-import org.elasticsearch.xpack.core.action.XPackUsageResponse;
-import org.elasticsearch.xpack.core.watcher.WatcherFeatureSetUsage;
-import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
-
-import java.util.Map;
-import java.util.Optional;
-
-import static org.elasticsearch.xpack.watcher.actions.ActionBuilders.loggingAction;
-import static org.elasticsearch.xpack.watcher.client.WatchSourceBuilders.watchBuilder;
-import static org.elasticsearch.xpack.watcher.input.InputBuilders.simpleInput;
-import static org.elasticsearch.xpack.watcher.trigger.TriggerBuilders.schedule;
-import static org.elasticsearch.xpack.watcher.trigger.schedule.Schedules.cron;
-import static org.hamcrest.Matchers.is;
-
-public class WatcherXpackUsageStatsTests extends AbstractWatcherIntegrationTestCase {
-
-    // as these tests use three data nodes, those watches will be across two of those
-    // nodes due to having two watcher shards, so that we can be sure that the count
-    // was merged
-    public void testWatcherUsageStatsTests() {
-        long watchCount = randomLongBetween(5, 20);
-        for (int i = 0; i < watchCount; i++) {
-            watcherClient().preparePutWatch("_id" + i).setSource(watchBuilder()
-                    .trigger(schedule(cron("0/5 * * * * ? 2050")))
-                    .input(simpleInput())
-                    .addAction("_id", loggingAction("whatever " + i)))
-                    .get();
-        }
-
-        XPackUsageRequest request = new XPackUsageRequest();
-        XPackUsageResponse usageResponse = client().execute(XPackUsageAction.INSTANCE, request).actionGet();
-        Optional<XPackFeatureSet.Usage> usage = usageResponse.getUsages().stream()
-                .filter(u -> u instanceof WatcherFeatureSetUsage)
-                .findFirst();
-        assertThat(usage.isPresent(), is(true));
-        WatcherFeatureSetUsage featureSetUsage = (WatcherFeatureSetUsage) usage.get();
-
-        long activeWatchCount = (long) ((Map) featureSetUsage.stats().get("count")).get("active");
-        assertThat(activeWatchCount, is(watchCount));
-    }
-
-}
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/ActionErrorIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/ActionErrorIntegrationTests.java
index 6590d6c271b26..dbf4da8a7a1a6 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/ActionErrorIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/ActionErrorIntegrationTests.java
@@ -10,7 +10,9 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
+import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.actions.index.IndexAction;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 
@@ -32,7 +34,7 @@ public void testErrorInAction() throws Exception {
         createIndex("foo");
         client().admin().indices().prepareUpdateSettings("foo").setSettings(Settings.builder().put("index.blocks.write", true)).get();
 
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id").setSource(watchBuilder()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client(), "_id").setSource(watchBuilder()
                 .trigger(schedule(interval("10m")))
 
                         // adding an action that throws an error and is associated with a 60 minute throttle period
@@ -73,7 +75,7 @@ public void testErrorInAction() throws Exception {
         });
 
         // now lets confirm that the ack status of the action is awaits_successful_execution
-        GetWatchResponse getWatchResponse = watcherClient().prepareGetWatch("_id").get();
+        GetWatchResponse getWatchResponse = new GetWatchRequestBuilder(client(), "_id").get();
         XContentSource watch = getWatchResponse.getSource();
         watch.getValue("status.actions._action.ack.awaits_successful_execution");
     }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/ActionWrapperTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/ActionWrapperTests.java
index e3a41062caee5..8931319501b6d 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/ActionWrapperTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/ActionWrapperTests.java
@@ -5,6 +5,11 @@
  */
 package org.elasticsearch.xpack.watcher.actions;
 
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.watcher.actions.Action;
 import org.elasticsearch.xpack.core.watcher.actions.ActionStatus;
@@ -13,16 +18,31 @@
 import org.elasticsearch.xpack.core.watcher.actions.ActionWrapperResult;
 import org.elasticsearch.xpack.core.watcher.actions.ExecutableAction;
 import org.elasticsearch.xpack.core.watcher.execution.WatchExecutionContext;
+import org.elasticsearch.xpack.core.watcher.execution.Wid;
+import org.elasticsearch.xpack.core.watcher.trigger.TriggerEvent;
+import org.elasticsearch.xpack.core.watcher.watch.Payload;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.core.watcher.watch.WatchStatus;
+import org.elasticsearch.xpack.watcher.actions.logging.ExecutableLoggingAction;
+import org.elasticsearch.xpack.watcher.actions.logging.LoggingAction;
+import org.elasticsearch.xpack.watcher.common.text.TextTemplate;
+import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.condition.NeverCondition;
+import org.elasticsearch.xpack.watcher.test.MockTextTemplateEngine;
+import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTriggerEvent;
 
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
@@ -34,7 +54,7 @@ public class ActionWrapperTests extends ESTestCase {
     private Watch watch = mock(Watch.class);
     @SuppressWarnings("unchecked")
     private ExecutableAction<Action> executableAction = mock(ExecutableAction.class);
-    private ActionWrapper actionWrapper = new ActionWrapper("_action", null, NeverCondition.INSTANCE, null, executableAction);
+    private ActionWrapper actionWrapper = new ActionWrapper("_action", null, NeverCondition.INSTANCE, null, executableAction, null);
 
     public void testThatUnmetActionConditionResetsAckStatus() throws Exception {
         WatchStatus watchStatus = new WatchStatus(now, Collections.singletonMap("_action", createActionStatus(State.ACKED)));
@@ -59,10 +79,135 @@ public void testOtherActionsAreNotAffectedOnActionConditionReset() throws Except
         assertThat(watch.status().actionStatus("other").ackStatus().state(), is(otherState));
     }
 
+    public void testThatMultipleResultsCanBeReturned() throws Exception {
+        final LoggingAction loggingAction = new LoggingAction(new TextTemplate("{{key}}"), null, null);
+        final ExecutableAction<LoggingAction> executableAction =
+            new ExecutableLoggingAction(loggingAction, logger, new MockTextTemplateEngine());
+        ActionWrapper wrapper = new ActionWrapper("_action", null, InternalAlwaysCondition.INSTANCE, null, executableAction,
+            "ctx.payload.my_path");
+
+        WatchExecutionContext ctx = mockExecutionContent(watch);
+        Payload.Simple payload = new Payload.Simple(Map.of("my_path",
+            List.of(
+                Map.of("key", "first"),
+                Map.of("key", "second"),
+                Map.of("key", "third")
+            )));
+        when(ctx.payload()).thenReturn(payload);
+
+        ActionWrapperResult result = wrapper.execute(ctx);
+        assertThat(result.action().status(), is(Action.Result.Status.SUCCESS));
+        // check that action toXContent contains all the results
+        try (XContentBuilder builder = jsonBuilder()) {
+            result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            final String json = Strings.toString(builder);
+            final Map<String, Object> map = XContentHelper.convertToMap(JsonXContent.jsonXContent, json, true);
+            assertThat(map, hasKey("foreach"));
+            assertThat(map.get("foreach"), instanceOf(List.class));
+            List<Map<String, Object>> actions = (List) map.get("foreach");
+            assertThat(actions, hasSize(3));
+        }
+    }
+
+    public void testThatSpecifiedPathIsNotCollection() {
+        ActionWrapper wrapper = new ActionWrapper("_action", null, InternalAlwaysCondition.INSTANCE, null, executableAction,
+            "ctx.payload.my_path");
+        WatchExecutionContext ctx = mockExecutionContent(watch);
+        Payload.Simple payload = new Payload.Simple(Map.of("my_path", "not a map"));
+        when(ctx.payload()).thenReturn(payload);
+        when(executableAction.logger()).thenReturn(logger);
+
+        ActionWrapperResult result = wrapper.execute(ctx);
+        assertThat(result.action().status(), is(Action.Result.Status.FAILURE));
+        assertThat(result.action(), instanceOf(Action.Result.FailureWithException.class));
+        Action.Result.FailureWithException failureWithException = (Action.Result.FailureWithException) result.action();
+        assertThat(failureWithException.getException().getMessage(),
+            is("specified foreach object was not a an array/collection: [ctx.payload.my_path]"));
+    }
+
+    public void testEmptyCollection() {
+        ActionWrapper wrapper = new ActionWrapper("_action", null, InternalAlwaysCondition.INSTANCE, null, executableAction,
+            "ctx.payload.my_path");
+        WatchExecutionContext ctx = mockExecutionContent(watch);
+        Payload.Simple payload = new Payload.Simple(Map.of("my_path", Collections.emptyList()));
+        when(ctx.payload()).thenReturn(payload);
+        when(executableAction.logger()).thenReturn(logger);
+
+        ActionWrapperResult result = wrapper.execute(ctx);
+        assertThat(result.action().status(), is(Action.Result.Status.FAILURE));
+        assertThat(result.action(), instanceOf(Action.Result.FailureWithException.class));
+        Action.Result.FailureWithException failureWithException = (Action.Result.FailureWithException) result.action();
+        assertThat(failureWithException.getException().getMessage(),
+            is("foreach object [ctx.payload.my_path] was an empty list, could not run any action"));
+    }
+
+    public void testPartialFailure() throws Exception {
+        ActionWrapper wrapper = new ActionWrapper("_action", null, InternalAlwaysCondition.INSTANCE, null, executableAction,
+            "ctx.payload.my_path");
+        WatchExecutionContext ctx = mockExecutionContent(watch);
+        Payload.Simple payload = new Payload.Simple(Map.of("my_path",
+            List.of(
+                Map.of("key", "first"),
+                Map.of("key", "second")
+            )));
+        when(ctx.payload()).thenReturn(payload);
+        when(executableAction.logger()).thenReturn(logger);
+
+        final Action.Result firstResult = new LoggingAction.Result.Success("log_message");;
+        final Payload firstPayload = new Payload.Simple(Map.of("key", "first"));
+        when(executableAction.execute(eq("_action"), eq(ctx), eq(firstPayload))).thenReturn(firstResult);
+
+        final Action.Result secondResult = new Action.Result.Failure("MY_TYPE", "second reason");
+        final Payload secondPayload = new Payload.Simple(Map.of("key", "second"));
+        when(executableAction.execute(eq("_action"), eq(ctx), eq(secondPayload))).thenReturn(secondResult);
+
+        ActionWrapperResult result = wrapper.execute(ctx);
+        assertThat(result.action().status(), is(Action.Result.Status.PARTIAL_FAILURE));
+    }
+
+    public void testLimitOfNumberOfActionsExecuted() throws Exception {
+        ActionWrapper wrapper = new ActionWrapper("_action", null, InternalAlwaysCondition.INSTANCE, null, executableAction,
+            "ctx.payload.my_path");
+        WatchExecutionContext ctx = mockExecutionContent(watch);
+        List<Map<String, String>> itemsPayload = new ArrayList<>();
+        for (int i = 0; i < 101; i++) {
+            final Action.Result actionResult = new LoggingAction.Result.Success("log_message " + i);;
+            final Payload singleItemPayload = new Payload.Simple(Map.of("key", String.valueOf(i)));
+            itemsPayload.add(Map.of("key", String.valueOf(i)));
+            when(executableAction.execute(eq("_action"), eq(ctx), eq(singleItemPayload))).thenReturn(actionResult);
+        }
+
+        Payload.Simple payload = new Payload.Simple(Map.of("my_path", itemsPayload));
+        when(ctx.payload()).thenReturn(payload);
+        when(executableAction.logger()).thenReturn(logger);
+
+        ActionWrapperResult result = wrapper.execute(ctx);
+        assertThat(result.action().status(), is(Action.Result.Status.SUCCESS));
+
+        // check that action toXContent contains all the results
+        try (XContentBuilder builder = jsonBuilder()) {
+            result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            final String json = Strings.toString(builder);
+            final Map<String, Object> map = XContentHelper.convertToMap(JsonXContent.jsonXContent, json, true);
+            assertThat(map, hasKey("foreach"));
+            assertThat(map.get("foreach"), instanceOf(List.class));
+            List<Map<String, Object>> actions = (List) map.get("foreach");
+            assertThat(actions, hasSize(100));
+            assertThat(map, hasKey("number_of_actions_executed"));
+            assertThat(map.get("number_of_actions_executed"), is(100));
+        }
+    }
+
     private WatchExecutionContext mockExecutionContent(Watch watch) {
         WatchExecutionContext ctx = mock(WatchExecutionContext.class);
         when(watch.id()).thenReturn("watchId");
         when(ctx.watch()).thenReturn(watch);
+        final ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
+        final Wid wid = new Wid("watchId", now);
+        when(ctx.id()).thenReturn(wid);
+        when(ctx.executionTime()).thenReturn(now);
+        final TriggerEvent triggerEvent = new ScheduleTriggerEvent("watchId", now, now);
+        when(ctx.triggerEvent()).thenReturn(triggerEvent);
         when(ctx.skipThrottling(eq("_action"))).thenReturn(true);
         return ctx;
     }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/TimeThrottleIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/TimeThrottleIntegrationTests.java
index 393932f8c8ce6..671d3ec44ed5e 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/TimeThrottleIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/TimeThrottleIntegrationTests.java
@@ -7,14 +7,14 @@
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
-import org.elasticsearch.common.xcontent.ObjectPath;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 
 import java.util.Map;
@@ -29,15 +29,11 @@
 import static org.elasticsearch.xpack.watcher.trigger.schedule.Schedules.interval;
 import static org.hamcrest.Matchers.is;
 
-@TestLogging("org.elasticsearch.xpack.watcher:DEBUG," +
-        "org.elasticsearch.xpack.watcher.WatcherLifeCycleService:DEBUG," +
-        "org.elasticsearch.xpack.watcher.trigger.ScheduleTriggerMock:TRACE," +
-        "org.elasticsearch.xpack.watcher.WatcherIndexingListener:TRACE")
 public class TimeThrottleIntegrationTests extends AbstractWatcherIntegrationTestCase {
 
     public void testTimeThrottle(){
         String id = randomAlphaOfLength(20);
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client())
                 .setId(id)
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("5s")))
@@ -63,7 +59,7 @@ public void testTimeThrottle(){
 
     public void testTimeThrottleDefaults() {
         String id = randomAlphaOfLength(30);
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client())
                 .setId(id)
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("1s")))
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailAttachmentTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailAttachmentTests.java
index f8d7171a13d8c..35f84abcb881c 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailAttachmentTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/email/EmailAttachmentTests.java
@@ -16,8 +16,8 @@
 import org.elasticsearch.test.http.MockResponse;
 import org.elasticsearch.test.http.MockWebServer;
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.common.http.Scheme;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
@@ -35,7 +35,6 @@
 import javax.mail.Multipart;
 import javax.mail.Part;
 import javax.mail.internet.MimeMessage;
-
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
@@ -150,7 +149,6 @@ public void testThatEmailAttachmentsAreSent() throws Exception {
             latch.countDown();
         });
 
-        WatcherClient watcherClient = watcherClient();
         createIndex("idx");
         // Have a sample document in the index, the watch is going to evaluate
         client().prepareIndex("idx", "type").setSource("field", "value").get();
@@ -181,7 +179,8 @@ public void testThatEmailAttachmentsAreSent() throws Exception {
                 .addAction("_email", emailAction(emailBuilder).setAuthentication(EmailServer.USERNAME, EmailServer.PASSWORD.toCharArray())
                 .setAttachments(emailAttachments));
 
-        watcherClient.preparePutWatch("_test_id")
+        new PutWatchRequestBuilder(client())
+                .setId("_test_id")
                 .setSource(watchSourceBuilder)
                 .get();
 
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/throttler/ActionThrottleTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/throttler/ActionThrottleTests.java
index 1834b70d92fcb..dd71580490011 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/throttler/ActionThrottleTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/throttler/ActionThrottleTests.java
@@ -11,13 +11,14 @@
 import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest;
 import org.elasticsearch.xpack.core.watcher.actions.Action;
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
 import org.elasticsearch.xpack.core.watcher.execution.ActionExecutionMode;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
+import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.actions.email.EmailAction;
 import org.elasticsearch.xpack.watcher.actions.index.IndexAction;
@@ -60,11 +61,11 @@ public void testSingleActionAckThrottle() throws Exception {
         Action.Builder<?> action = availableAction.action();
         watchSourceBuilder.addAction("test_id", action);
 
-        watcherClient().putWatch(new PutWatchRequest("_id", watchSourceBuilder.buildAsBytes(XContentType.JSON),
-                XContentType.JSON)).actionGet();
+        new PutWatchRequestBuilder(client())
+            .setId("_id").setSource(watchSourceBuilder.buildAsBytes(XContentType.JSON), XContentType.JSON).get();
         refresh(Watch.INDEX);
 
-        ExecuteWatchRequestBuilder executeWatchRequestBuilder = watcherClient().prepareExecuteWatch("_id")
+        ExecuteWatchRequestBuilder executeWatchRequestBuilder = new ExecuteWatchRequestBuilder(client()).setId("_id")
                 .setRecordExecution(true)
                 .setActionMode("test_id", ActionExecutionMode.SIMULATE);
 
@@ -76,10 +77,10 @@ public void testSingleActionAckThrottle() throws Exception {
 
         boolean ack = randomBoolean();
         if (ack) {
-            watcherClient().prepareAckWatch("_id").setActionIds("test_id").get();
+            new AckWatchRequestBuilder(client(), "_id").setActionIds("test_id").get();
         }
 
-        executeWatchRequestBuilder = watcherClient().prepareExecuteWatch("_id")
+        executeWatchRequestBuilder = new ExecuteWatchRequestBuilder(client()).setId("_id")
                 .setRecordExecution(true)
                 .setActionMode("test_id", ActionExecutionMode.SIMULATE);
         responseMap = executeWatchRequestBuilder.get().getRecordSource().getAsMap();
@@ -105,13 +106,13 @@ public void testRandomMultiActionAckThrottle() throws Exception {
             }
         }
 
-        watcherClient().putWatch(new PutWatchRequest("_id",
-                watchSourceBuilder.buildAsBytes(XContentType.JSON), XContentType.JSON)).actionGet();
+        new PutWatchRequestBuilder(client())
+            .setId("_id").setSource(watchSourceBuilder.buildAsBytes(XContentType.JSON), XContentType.JSON).get();
         refresh(Watch.INDEX);
         executeWatch("_id");
 
         for (String actionId : ackingActions)  {
-            watcherClient().prepareAckWatch("_id").setActionIds(actionId).get();
+            new AckWatchRequestBuilder(client(), "_id").setActionIds(actionId).get();
         }
 
         timeWarp().clock().fastForwardSeconds(15);
@@ -128,7 +129,7 @@ public void testRandomMultiActionAckThrottle() throws Exception {
     }
 
     private Map<String, Object> executeWatch(String id) {
-        return watcherClient().prepareExecuteWatch(id)
+        return new ExecuteWatchRequestBuilder(client()).setId(id)
                 .setRecordExecution(true)
                 .setActionMode("_all", ActionExecutionMode.SIMULATE).get().getRecordSource().getAsMap();
     }
@@ -143,8 +144,8 @@ public void testDifferentThrottlePeriods() throws Exception {
         watchSourceBuilder.addAction("fifteen_sec_throttle", new TimeValue(15, TimeUnit.SECONDS),
                 randomFrom(AvailableAction.values()).action());
 
-        watcherClient().putWatch(new PutWatchRequest("_id",
-                watchSourceBuilder.buildAsBytes(XContentType.JSON), XContentType.JSON)).actionGet();
+        new PutWatchRequestBuilder(client())
+            .setId("_id").setSource(watchSourceBuilder.buildAsBytes(XContentType.JSON), XContentType.JSON).get();
         refresh(Watch.INDEX);
 
         timeWarp().clock().fastForwardSeconds(1);
@@ -181,13 +182,13 @@ public void testDefaultThrottlePeriod() throws Exception {
         AvailableAction availableAction = randomFrom(AvailableAction.values());
         watchSourceBuilder.addAction("default_global_throttle", availableAction.action());
 
-        watcherClient().putWatch(new PutWatchRequest("_id",
-                watchSourceBuilder.buildAsBytes(XContentType.JSON), XContentType.JSON)).actionGet();
+        new PutWatchRequestBuilder(client())
+            .setId("_id").setSource(watchSourceBuilder.buildAsBytes(XContentType.JSON), XContentType.JSON).get();
         refresh(Watch.INDEX);
 
         timeWarp().clock().setTime(ZonedDateTime.now(ZoneOffset.UTC));
 
-        ExecuteWatchResponse executeWatchResponse = watcherClient().prepareExecuteWatch("_id")
+        ExecuteWatchResponse executeWatchResponse = new ExecuteWatchRequestBuilder(client()).setId("_id")
                 .setTriggerEvent(new ManualTriggerEvent("execute_id",
                         new ScheduleTriggerEvent(ZonedDateTime.now(ZoneOffset.UTC), ZonedDateTime.now(ZoneOffset.UTC))))
                 .setActionMode("default_global_throttle", ActionExecutionMode.SIMULATE)
@@ -199,7 +200,7 @@ public void testDefaultThrottlePeriod() throws Exception {
 
         timeWarp().clock().fastForwardSeconds(1);
 
-        executeWatchResponse = watcherClient().prepareExecuteWatch("_id")
+        executeWatchResponse = new ExecuteWatchRequestBuilder(client()).setId("_id")
                 .setTriggerEvent(new ManualTriggerEvent("execute_id",
                         new ScheduleTriggerEvent(ZonedDateTime.now(ZoneOffset.UTC), ZonedDateTime.now(ZoneOffset.UTC))))
                 .setActionMode("default_global_throttle", ActionExecutionMode.SIMULATE)
@@ -212,7 +213,7 @@ public void testDefaultThrottlePeriod() throws Exception {
 
         assertBusy(() -> {
             try {
-                ExecuteWatchResponse executeWatchResponse1 = watcherClient().prepareExecuteWatch("_id")
+                ExecuteWatchResponse executeWatchResponse1 = new ExecuteWatchRequestBuilder(client()).setId("_id")
                         .setTriggerEvent(new ManualTriggerEvent("execute_id",
                                 new ScheduleTriggerEvent(ZonedDateTime.now(ZoneOffset.UTC), ZonedDateTime.now(ZoneOffset.UTC))))
                         .setActionMode("default_global_throttle", ActionExecutionMode.SIMULATE)
@@ -234,13 +235,13 @@ public void testWatchThrottlePeriod() throws Exception {
         AvailableAction availableAction = randomFrom(AvailableAction.values());
         watchSourceBuilder.addAction("default_global_throttle", availableAction.action());
 
-        watcherClient().putWatch(new PutWatchRequest("_id",
-                watchSourceBuilder.buildAsBytes(XContentType.JSON), XContentType.JSON)).actionGet();
+        new PutWatchRequestBuilder(client())
+            .setId("_id").setSource(watchSourceBuilder.buildAsBytes(XContentType.JSON), XContentType.JSON).get();
         refresh(Watch.INDEX);
 
         timeWarp().clock().setTime(ZonedDateTime.now(ZoneOffset.UTC));
 
-        ExecuteWatchResponse executeWatchResponse = watcherClient().prepareExecuteWatch("_id")
+        ExecuteWatchResponse executeWatchResponse = new ExecuteWatchRequestBuilder(client()).setId("_id")
                 .setTriggerEvent(new ManualTriggerEvent("execute_id",
                         new ScheduleTriggerEvent(ZonedDateTime.now(ZoneOffset.UTC), ZonedDateTime.now(ZoneOffset.UTC))))
                 .setActionMode("default_global_throttle", ActionExecutionMode.SIMULATE)
@@ -251,7 +252,7 @@ public void testWatchThrottlePeriod() throws Exception {
 
         timeWarp().clock().fastForwardSeconds(1);
 
-        executeWatchResponse = watcherClient().prepareExecuteWatch("_id")
+        executeWatchResponse = new ExecuteWatchRequestBuilder(client()).setId("_id")
                 .setTriggerEvent(new ManualTriggerEvent("execute_id",
                         new ScheduleTriggerEvent(ZonedDateTime.now(ZoneOffset.UTC), ZonedDateTime.now(ZoneOffset.UTC))))
                 .setActionMode("default_global_throttle", ActionExecutionMode.SIMULATE)
@@ -265,7 +266,7 @@ public void testWatchThrottlePeriod() throws Exception {
         assertBusy(() -> {
             try {
                 //Since the default throttle period is 5 seconds but we have overridden the period in the watch this should trigger
-                ExecuteWatchResponse executeWatchResponse1 = watcherClient().prepareExecuteWatch("_id")
+                ExecuteWatchResponse executeWatchResponse1 = new ExecuteWatchRequestBuilder(client()).setId("_id")
                         .setTriggerEvent(new ManualTriggerEvent("execute_id",
                                 new ScheduleTriggerEvent(ZonedDateTime.now(ZoneOffset.UTC), ZonedDateTime.now(ZoneOffset.UTC))))
                         .setActionMode("default_global_throttle", ActionExecutionMode.SIMULATE)
@@ -296,7 +297,7 @@ public void testFailingActionDoesGetThrottled() throws Exception {
 
         TimeValue throttlePeriod = new TimeValue(60, TimeUnit.MINUTES);
 
-        watcherClient().preparePutWatch("_id").setSource(watchBuilder()
+        new PutWatchRequestBuilder(client(), "_id").setSource(watchBuilder()
                 .trigger(new ScheduleTrigger(new IntervalSchedule(
                         new IntervalSchedule.Interval(60, IntervalSchedule.Interval.Unit.MINUTES))))
                 .defaultThrottlePeriod(throttlePeriod)
@@ -306,7 +307,7 @@ public void testFailingActionDoesGetThrottled() throws Exception {
         refresh(Watch.INDEX);
 
         {
-            Map<String, Object> responseMap = watcherClient().prepareExecuteWatch("_id")
+            Map<String, Object> responseMap = new ExecuteWatchRequestBuilder(client()).setId("_id")
                     .setRecordExecution(true)
                     .get().getRecordSource().getAsMap();
 
@@ -328,7 +329,7 @@ public void testFailingActionDoesGetThrottled() throws Exception {
         }
 
         {
-            Map<String, Object> responseMap = watcherClient().prepareExecuteWatch("_id")
+            Map<String, Object> responseMap = new ExecuteWatchRequestBuilder(client()).setId("_id")
                     .setRecordExecution(true)
                     .get().getRecordSource().getAsMap();
             String state = ObjectPath.eval("state", responseMap);
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java
index 377ffa67c9a8f..e5cf3a54a4c66 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookHttpsIntegrationTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.xpack.core.ssl.TestsSSLService;
 import org.elasticsearch.xpack.core.watcher.history.WatchRecord;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.actions.ActionBuilders;
 import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
 import org.elasticsearch.xpack.watcher.common.http.HttpMethod;
@@ -82,7 +83,7 @@ public void testHttps() throws Exception {
                 .body(new TextTemplate("{key=value}"))
                 .method(HttpMethod.POST);
 
-        watcherClient().preparePutWatch("_id")
+        new PutWatchRequestBuilder(client(), "_id")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("5s")))
                         .input(simpleInput("key", "value"))
@@ -121,7 +122,7 @@ public void testHttpsAndBasicAuth() throws Exception {
                 .body(new TextTemplate("{key=value}"))
                 .method(HttpMethod.POST);
 
-        watcherClient().preparePutWatch("_id")
+        new PutWatchRequestBuilder(client(), "_id")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("5s")))
                         .input(simpleInput("key", "value"))
@@ -148,7 +149,8 @@ private static List<String> getProtocols() {
             return List.of("TLSv1.2");
         } else {
             JavaVersion full =
-                AccessController.doPrivileged((PrivilegedAction<JavaVersion>) () -> JavaVersion.parse(System.getProperty("java.version")));
+                AccessController.doPrivileged(
+                        (PrivilegedAction<JavaVersion>) () -> JavaVersion.parse(System.getProperty("java.specification.version")));
             if (full.compareTo(JavaVersion.parse("12.0.1")) < 0) {
                 return List.of("TLSv1.2");
             }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookIntegrationTests.java
index 2c961db6187fe..3395e5d879331 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookIntegrationTests.java
@@ -16,6 +16,8 @@
 import org.elasticsearch.transport.Netty4Plugin;
 import org.elasticsearch.xpack.core.watcher.history.WatchRecord;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
+import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.actions.ActionBuilders;
 import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
 import org.elasticsearch.xpack.watcher.common.http.HttpMethod;
@@ -79,7 +81,7 @@ public void testWebhook() throws Exception {
                 .auth(new BasicAuth("user", "pass".toCharArray()))
                 .method(HttpMethod.POST);
 
-        watcherClient().preparePutWatch("_id")
+        new PutWatchRequestBuilder(client(), "_id")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("5s")))
                         .input(simpleInput("key", "value"))
@@ -119,7 +121,7 @@ public void testWebhookWithBasicAuth() throws Exception {
                 .body(new TextTemplate("_body"))
                 .method(HttpMethod.POST);
 
-        watcherClient().preparePutWatch("_id")
+        new PutWatchRequestBuilder(client(), "_id")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("5s")))
                         .input(simpleInput("key", "value"))
@@ -152,7 +154,7 @@ public void testWebhookWithTimebasedIndex() throws Exception {
                 .putHeader("Content-Type", new TextTemplate("application/json"))
                 .method(HttpMethod.PUT);
 
-        watcherClient().preparePutWatch("_id")
+        new PutWatchRequestBuilder(client(), "_id")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("5s")))
                         .input(simpleInput("key", "value"))
@@ -160,7 +162,7 @@ public void testWebhookWithTimebasedIndex() throws Exception {
                         .addAction("_id", ActionBuilders.webhookAction(builder)))
                 .get();
 
-        watcherClient().prepareExecuteWatch("_id").get();
+        new ExecuteWatchRequestBuilder(client(), "_id").get();
 
         GetResponse response = client().prepareGet().setIndex("<logstash-{now/d}>").setId("1").get();
         assertExists(response);
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java
index 6e9957a404d9a..d24bba0b1ba57 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java
@@ -758,7 +758,8 @@ private static List<String> getProtocols() {
             return List.of("TLSv1.2");
         } else {
             JavaVersion full =
-                AccessController.doPrivileged((PrivilegedAction<JavaVersion>) () -> JavaVersion.parse(System.getProperty("java.version")));
+                AccessController.doPrivileged(
+                        (PrivilegedAction<JavaVersion>) () -> JavaVersion.parse(System.getProperty("java.specification.version")));
             if (full.compareTo(JavaVersion.parse("12.0.1")) < 0) {
                 return List.of("TLSv1.2");
             }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecuteWatchQueuedStatsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecuteWatchQueuedStatsTests.java
index d4686652eaeff..4e99dc9803027 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecuteWatchQueuedStatsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecuteWatchQueuedStatsTests.java
@@ -8,13 +8,17 @@
 
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.execution.ActionExecutionMode;
+import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequestBuilder;
+import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequest;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
+import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsResponse;
 import org.elasticsearch.xpack.watcher.actions.index.IndexAction;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
@@ -55,8 +59,8 @@ protected boolean timeWarped() {
      * happens yet succeeds with the production code change that accompanies this test.
      */
     public void testQueuedStats() throws ExecutionException, InterruptedException {
-        final WatcherClient client = new WatcherClient(client());
-        client.preparePutWatch("id")
+        final Client client = client();
+        new PutWatchRequestBuilder(client, "id")
                 .setActive(true)
                 .setSource(
                         new WatchSourceBuilder()
@@ -90,7 +94,7 @@ public void testQueuedStats() throws ExecutionException, InterruptedException {
                 }
                 request.setActionMode("_all", ActionExecutionMode.EXECUTE);
                 request.setRecordExecution(true);
-                futures.add(client.executeWatch(request));
+                futures.add(client.execute(ExecuteWatchAction.INSTANCE, request));
             }
         });
         executeWatchThread.start();
@@ -103,7 +107,7 @@ public void testQueuedStats() throws ExecutionException, InterruptedException {
                 fail(e.toString());
             }
             for (int i = 0; i < numberOfIterations; i++) {
-                final WatcherStatsResponse response = client.prepareWatcherStats().setIncludeQueuedWatches(true).get();
+                final WatcherStatsResponse response = new WatcherStatsRequestBuilder(client).setIncludeQueuedWatches(true).get();
                 failures.addAll(response.failures());
             }
         });
@@ -118,7 +122,7 @@ public void testQueuedStats() throws ExecutionException, InterruptedException {
 
         assertThat(failures, empty());
 
-        client.prepareDeleteWatch("id").get();
+        new DeleteWatchRequestBuilder(client, "id").get();
     }
 
 }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java
index fd06045204710..40478f45dfa8d 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
@@ -87,13 +88,11 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import static java.util.Arrays.asList;
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
@@ -156,7 +155,7 @@ public void init() throws Exception {
         parser = mock(WatchParser.class);
 
         DiscoveryNode discoveryNode = new DiscoveryNode("node_1", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(),
-                new HashSet<>(asList(DiscoveryNode.Role.values())), Version.CURRENT);
+                DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT);
         ClusterService clusterService = mock(ClusterService.class);
         when(clusterService.localNode()).thenReturn(discoveryNode);
 
@@ -228,7 +227,7 @@ public void testExecute() throws Exception {
         when(action.type()).thenReturn("MY_AWESOME_TYPE");
         when(action.execute("_action", context, payload)).thenReturn(actionResult);
 
-        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action);
+        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action, null);
 
         WatchStatus watchStatus = new WatchStatus(now, singletonMap("_action", new ActionStatus(now)));
 
@@ -314,7 +313,7 @@ public void testExecuteFailedInput() throws Exception {
         ExecutableAction action = mock(ExecutableAction.class);
         when(action.execute("_action", context, payload)).thenReturn(actionResult);
 
-        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action);
+        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action, null);
         WatchStatus watchStatus = new WatchStatus(now, singletonMap("_action", new ActionStatus(now)));
 
         when(watch.input()).thenReturn(input);
@@ -379,7 +378,7 @@ public void testExecuteFailedCondition() throws Exception {
         ExecutableAction action = mock(ExecutableAction.class);
         when(action.execute("_action", context, payload)).thenReturn(actionResult);
 
-        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action);
+        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action, null);
         WatchStatus watchStatus = new WatchStatus(now, singletonMap("_action", new ActionStatus(now)));
 
         when(watch.input()).thenReturn(input);
@@ -443,7 +442,7 @@ public void testExecuteFailedWatchTransform() throws Exception {
         ExecutableAction action = mock(ExecutableAction.class);
         when(action.execute("_action", context, payload)).thenReturn(actionResult);
 
-        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action);
+        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action, null);
         WatchStatus watchStatus = new WatchStatus(now, singletonMap("_action", new ActionStatus(now)));
 
         when(watch.input()).thenReturn(input);
@@ -521,7 +520,7 @@ public void testExecuteFailedActionTransform() throws Exception {
         when(action.logger()).thenReturn(logger);
         when(action.execute("_action", context, payload)).thenReturn(actionResult);
 
-        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action);
+        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action, null);
 
         WatchStatus watchStatus = new WatchStatus(now, singletonMap("_action", new ActionStatus(now)));
 
@@ -601,7 +600,7 @@ public void testExecuteInner() throws Exception {
         ExecutableAction action = mock(ExecutableAction.class);
         when(action.execute("_action", context, payload)).thenReturn(actionResult);
 
-        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action);
+        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action, null);
         ZonedDateTime time = clock.instant().atZone(ZoneOffset.UTC);
         WatchStatus watchStatus = new WatchStatus(time, singletonMap("_action", new ActionStatus(now)));
 
@@ -650,7 +649,7 @@ public void testExecuteInnerThrottled() throws Exception {
 
         ExecutableAction action = mock(ExecutableAction.class);
         when(action.type()).thenReturn("_type");
-        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action);
+        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action, null);
         ZonedDateTime time = clock.instant().atZone(ZoneOffset.UTC);
         WatchStatus watchStatus = new WatchStatus(time, singletonMap("_action", new ActionStatus(now)));
 
@@ -713,7 +712,7 @@ public void testExecuteInnerConditionNotMet() throws Exception {
 
         ExecutableAction action = mock(ExecutableAction.class);
         when(action.type()).thenReturn("_type");
-        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action);
+        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action, null);
         ZonedDateTime time = clock.instant().atZone(ZoneOffset.UTC);
         WatchStatus watchStatus = new WatchStatus(time, singletonMap("_action", new ActionStatus(now)));
 
@@ -770,7 +769,7 @@ public void testExecuteInnerConditionNotMetDueToException() throws Exception {
         ExecutableAction action = mock(ExecutableAction.class);
         when(action.type()).thenReturn("_type");
         when(action.logger()).thenReturn(logger);
-        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action);
+        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action, null);
         ZonedDateTime time = clock.instant().atZone(ZoneOffset.UTC);
         WatchStatus watchStatus = new WatchStatus(time, singletonMap("_action", new ActionStatus(now)));
 
@@ -818,7 +817,7 @@ public void testExecuteConditionNotMet() throws Exception {
         ExecutableCondition actionCondition = mock(ExecutableCondition.class);
         ExecutableTransform actionTransform = mock(ExecutableTransform.class);
         ExecutableAction action = mock(ExecutableAction.class);
-        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action);
+        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, actionCondition, actionTransform, action, null);
 
         ZonedDateTime time = clock.instant().atZone(ZoneOffset.UTC);
         WatchStatus watchStatus = new WatchStatus(time, singletonMap("_action", new ActionStatus(now)));
@@ -947,7 +946,7 @@ public void testThatTriggeredWatchDeletionHappensOnlyIfWatchExists() throws Exce
         when(action.type()).thenReturn("MY_AWESOME_TYPE");
         when(action.execute("_action", context, payload)).thenReturn(actionResult);
 
-        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, null, null, action);
+        ActionWrapper actionWrapper = new ActionWrapper("_action", throttler, null, null, action, null);
 
         WatchStatus watchStatus = new WatchStatus(now, singletonMap("_action", new ActionStatus(now)));
 
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryActionConditionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryActionConditionTests.java
index 15a74537f70db..01fe96c480da0 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryActionConditionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryActionConditionTests.java
@@ -13,12 +13,12 @@
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
 import org.elasticsearch.xpack.core.watcher.condition.Condition;
 import org.elasticsearch.xpack.core.watcher.condition.ExecutableCondition;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.input.Input;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.condition.CompareCondition;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.condition.NeverCondition;
@@ -46,7 +46,6 @@
 /**
  * This test makes sure per-action conditions are honored.
  */
-@TestLogging("org.elasticsearch.xpack.watcher:DEBUG,org.elasticsearch.xpack.watcher.WatcherIndexingListener:TRACE")
 public class HistoryActionConditionTests extends AbstractWatcherIntegrationTestCase {
 
     private final Input input = simpleInput("key", 15).build();
@@ -253,7 +252,7 @@ private void putAndTriggerWatch(final String id, final Input input, final Condit
             source.addAction("action" + i, actionConditions[i], loggingAction(Integer.toString(i)));
         }
 
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch(id).setSource(source).get();
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client(), id).setSource(source).get();
 
         assertThat(putWatchResponse.isCreated(), is(true));
 
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateEmailMappingsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateEmailMappingsTests.java
index af1a7ad19fcce..9bbddf9e0aee6 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateEmailMappingsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateEmailMappingsTests.java
@@ -11,9 +11,9 @@
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.notification.email.EmailTemplate;
 import org.elasticsearch.xpack.watcher.notification.email.support.EmailServer;
@@ -34,8 +34,6 @@
  * This test makes sure that the email address fields in the watch_record action result are
  * not analyzed so they can be used in aggregations
  */
-@TestLogging("org.elasticsearch.xpack.watcher:DEBUG," +
-        "org.elasticsearch.xpack.watcher.WatcherIndexingListener:TRACE")
 public class HistoryTemplateEmailMappingsTests extends AbstractWatcherIntegrationTestCase {
 
     private EmailServer server;
@@ -68,7 +66,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
     }
 
     public void testEmailFields() throws Exception {
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id").setSource(watchBuilder()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client(), "_id").setSource(watchBuilder()
                 .trigger(schedule(interval("5s")))
                 .input(simpleInput())
                 .condition(InternalAlwaysCondition.INSTANCE)
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java
index bcae850e17c22..d97415ea8ca9d 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.search.aggregations.Aggregations;
@@ -18,7 +19,8 @@
 import org.elasticsearch.test.http.MockWebServer;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
-import org.elasticsearch.common.xcontent.ObjectPath;
+import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.common.http.HttpMethod;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
@@ -66,7 +68,7 @@ public void cleanup() throws Exception {
     }
 
     public void testHttpFields() throws Exception {
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id").setSource(watchBuilder()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client(), "_id").setSource(watchBuilder()
                 .trigger(schedule(interval("5s")))
                 .input(httpInput(HttpRequestTemplate.builder("localhost", webServer.getPort()).path("/input/path")))
                 .condition(InternalAlwaysCondition.INSTANCE)
@@ -130,7 +132,7 @@ public void testExceptionMapping() {
             webServer.enqueue(new MockResponse().setBeforeReplyDelay(TimeValue.timeValueSeconds(5)));
         }
 
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch(id).setSource(watchBuilder()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client(), id).setSource(watchBuilder()
                 .trigger(schedule(interval("1h")))
                 .input(httpInput(HttpRequestTemplate.builder("localhost", webServer.getPort())
                         .path("/")
@@ -144,7 +146,7 @@ public void testExceptionMapping() {
                 .get();
 
         assertThat(putWatchResponse.isCreated(), is(true));
-        watcherClient().prepareExecuteWatch(id).setRecordExecution(true).get();
+        new ExecuteWatchRequestBuilder(client(), id).setRecordExecution(true).get();
 
         // ensure watcher history index has been written with this id
         flushAndRefresh(HistoryStoreField.INDEX_PREFIX + "*");
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateIndexActionMappingsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateIndexActionMappingsTests.java
index 50efdd335c82c..0b8205935dfe6 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateIndexActionMappingsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateIndexActionMappingsTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 
 import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
@@ -32,7 +33,7 @@ public class HistoryTemplateIndexActionMappingsTests extends AbstractWatcherInte
     public void testIndexActionFields() throws Exception {
         String index = "the-index";
 
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id").setSource(watchBuilder()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client(), "_id").setSource(watchBuilder()
                 .trigger(schedule(interval("5m")))
                 .addAction("index", indexAction(index)))
                 .get();
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateSearchInputMappingsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateSearchInputMappingsTests.java
index b512fc46d8f4d..ddfcbb52a17fa 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateSearchInputMappingsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateSearchInputMappingsTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
@@ -45,7 +46,7 @@ public void testHttpFields() throws Exception {
                 new String[]{index}, new String[]{type}, SearchType.QUERY_THEN_FETCH,
                 WatcherSearchTemplateRequest.DEFAULT_INDICES_OPTIONS, new BytesArray("{}")
         );
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id").setSource(watchBuilder()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client(), "_id").setSource(watchBuilder()
                 .trigger(schedule(interval("5s")))
                 .input(searchInput(request))
                 .condition(InternalAlwaysCondition.INSTANCE)
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java
index b51715c64e2c2..69d0ea9b0b0e6 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 
@@ -34,7 +35,7 @@
 public class HistoryTemplateTimeMappingsTests extends AbstractWatcherIntegrationTestCase {
 
     public void testTimeFields() throws Exception {
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id").setSource(watchBuilder()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client(), "_id").setSource(watchBuilder()
                 .trigger(schedule(interval("5s")))
                 .input(simpleInput())
                 .condition(InternalAlwaysCondition.INSTANCE)
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTransformMappingsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTransformMappingsTests.java
index a1b0f170b9fd0..11a379546b310 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTransformMappingsTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTransformMappingsTests.java
@@ -8,6 +8,8 @@
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 
 import java.util.List;
@@ -48,7 +50,7 @@ public void testTransformFields() throws Exception {
                                 .startObject("foo").field("what", "ever").endObject().endObject()))
                 .get();
 
-        watcherClient().preparePutWatch("_first").setSource(watchBuilder()
+        new PutWatchRequestBuilder(client(), "_first").setSource(watchBuilder()
                 .trigger(schedule(interval("5s")))
                 .input(simpleInput())
                 .transform(searchTransform(templateRequest(searchSource().query(QueryBuilders.termQuery("name", "first")), "idx")))
@@ -59,7 +61,7 @@ public void testTransformFields() throws Exception {
 
         // execute another watch which with a transform that should conflict with the previous watch. Since the
         // mapping for the transform construct is disabled, there should be no problems.
-        watcherClient().preparePutWatch("_second").setSource(watchBuilder()
+        new PutWatchRequestBuilder(client(), "_second").setSource(watchBuilder()
                 .trigger(schedule(interval("5s")))
                 .input(simpleInput())
                 .transform(searchTransform(templateRequest(searchSource().query(QueryBuilders.termQuery("name", "second")), "idx")))
@@ -68,8 +70,8 @@ public void testTransformFields() throws Exception {
                         loggingAction("indexed")))
                 .get();
 
-        watcherClient().prepareExecuteWatch("_first").setRecordExecution(true).get();
-        watcherClient().prepareExecuteWatch("_second").setRecordExecution(true).get();
+        new ExecuteWatchRequestBuilder(client(), "_first").setRecordExecution(true).get();
+        new ExecuteWatchRequestBuilder(client(), "_second").setRecordExecution(true).get();
 
         assertBusy(() -> {
             GetFieldMappingsResponse response = client().admin().indices()
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/chain/ChainIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/chain/ChainIntegrationTests.java
index 508da647a2cfa..0328fc2678b8f 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/chain/ChainIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/chain/ChainIntegrationTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.transport.Netty4Plugin;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.input.http.HttpInput;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
@@ -68,7 +69,7 @@ public void testChainedInputsAreWorking() throws Exception {
                 .add("first", simpleInput("url", "/" + index  + "/_search"))
                 .add("second", httpInputBuilder);
 
-        watcherClient().preparePutWatch("_name")
+        new PutWatchRequestBuilder(client(), "_name")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval(5, SECONDS)))
                         .input(chainedInputBuilder)
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java
index efbefdd640893..0fa05e900e518 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/NotificationServiceTests.java
@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.watcher.notification;
 
+import org.elasticsearch.common.hash.MessageDigests;
 import org.elasticsearch.common.settings.SecureSetting;
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.SecureString;
@@ -16,6 +17,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 import java.security.GeneralSecurityException;
 import java.util.Arrays;
 import java.util.Collections;
@@ -247,6 +249,11 @@ public InputStream getFile(String setting) throws GeneralSecurityException {
                 return null;
             }
 
+            @Override
+            public byte[] getSHA256Digest(String setting) throws GeneralSecurityException {
+                return MessageDigests.sha256().digest(new String(secureSettingsMap.get(setting)).getBytes(StandardCharsets.UTF_8));
+            }
+
             @Override
             public void close() throws IOException {
             }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/EmailSecretsIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/EmailSecretsIntegrationTests.java
index b5d48e5033b0e..d96f9f07ed479 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/EmailSecretsIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/EmailSecretsIntegrationTests.java
@@ -10,13 +10,15 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.xpack.core.watcher.WatcherField;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.crypto.CryptoService;
 import org.elasticsearch.xpack.core.watcher.crypto.CryptoServiceTests;
 import org.elasticsearch.xpack.core.watcher.execution.ActionExecutionMode;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
+import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.trigger.TriggerEvent;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.actions.ActionBuilders;
@@ -80,8 +82,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
     }
 
     public void testEmail() throws Exception {
-        WatcherClient watcherClient = watcherClient();
-        watcherClient.preparePutWatch("_id")
+        new PutWatchRequestBuilder(client(), "_id")
                 .setSource(watchBuilder()
                         .trigger(schedule(cron("0 0 0 1 * ? 2020")))
                         .input(simpleInput())
@@ -113,7 +114,7 @@ public void testEmail() throws Exception {
         }
 
         // verifying the password is not returned by the GET watch API
-        GetWatchResponse watchResponse = watcherClient.prepareGetWatch("_id").get();
+        GetWatchResponse watchResponse = new GetWatchRequestBuilder(client(), "_id").get();
         assertThat(watchResponse, notNullValue());
         assertThat(watchResponse.getId(), is("_id"));
         XContentSource contentSource = watchResponse.getSource();
@@ -136,7 +137,7 @@ public void testEmail() throws Exception {
         });
 
         TriggerEvent triggerEvent = new ScheduleTriggerEvent(ZonedDateTime.now(ZoneOffset.UTC), ZonedDateTime.now(ZoneOffset.UTC));
-        ExecuteWatchResponse executeResponse = watcherClient.prepareExecuteWatch("_id")
+        ExecuteWatchResponse executeResponse = new ExecuteWatchRequestBuilder(client(), "_id")
                 .setRecordExecution(false)
                 .setTriggerEvent(triggerEvent)
                 .setActionMode("_all", ActionExecutionMode.FORCE_EXECUTE)
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/jira/JiraAccountTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/jira/JiraAccountTests.java
index 997a6aa6a8dc3..1b911172dc108 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/jira/JiraAccountTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/jira/JiraAccountTests.java
@@ -80,6 +80,16 @@ public void testJiraAccountSettings() {
         assertThat(e.getMessage(), containsString("invalid jira [test] account settings. missing required [secure_password] setting"));
     }
 
+    public void testInvalidSchemeUrl() throws Exception{
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString(JiraAccount.SECURE_URL_SETTING.getKey(),"test"); //Setting test as invalid scheme url
+        secureSettings.setString(JiraAccount.SECURE_USER_SETTING.getKey(), "foo");
+        secureSettings.setString(JiraAccount.SECURE_PASSWORD_SETTING.getKey(), "password");
+        Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+        SettingsException e = expectThrows(SettingsException.class, () -> new JiraAccount("test", settings, null));
+        assertThat(e.getMessage(), containsString("invalid jira [test] account settings. invalid [secure_url] setting"));
+    }
+
     public void testUnsecureAccountUrl() throws Exception {
         final MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString(JiraAccount.SECURE_USER_SETTING.getKey(), "foo");
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchActionTests.java
index f71999a068c34..d372b2ece7578 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchActionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/rest/action/RestExecuteWatchActionTests.java
@@ -5,17 +5,14 @@
  */
 package org.elasticsearch.xpack.watcher.rest.action;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.rest.RestController;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
 import org.elasticsearch.test.rest.FakeRestRequest.Builder;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
-import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
+import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequest;
 
 import java.util.Arrays;
 import java.util.HashMap;
@@ -23,30 +20,24 @@
 
 import static org.hamcrest.core.Is.is;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 public class RestExecuteWatchActionTests extends ESTestCase {
 
-    private RestController restController = mock(RestController.class);
-    private Client client = mock(Client.class);
-    private WatcherClient watcherClient = mock(WatcherClient.class);
+    private NodeClient client = mock(NodeClient.class);
 
     public void testThatFlagsCanBeSpecifiedViaParameters() throws Exception {
         String randomId = randomAlphaOfLength(10);
         for (String recordExecution : Arrays.asList("true", "false", null)) {
             for (String ignoreCondition : Arrays.asList("true", "false", null)) {
                 for (String debugCondition : Arrays.asList("true", "false", null)) {
-                    ExecuteWatchRequestBuilder builder = new ExecuteWatchRequestBuilder(client);
-                    when(watcherClient.prepareExecuteWatch()).thenReturn(builder);
 
-                    RestExecuteWatchAction restExecuteWatchAction = new RestExecuteWatchAction(Settings.EMPTY, restController);
-                    restExecuteWatchAction.doPrepareRequest(createFakeRestRequest(randomId, recordExecution, ignoreCondition,
-                            debugCondition), watcherClient);
+                    ExecuteWatchRequest request = RestExecuteWatchAction.parseRequest(
+                        createFakeRestRequest(randomId, recordExecution, ignoreCondition, debugCondition), client);
 
-                    assertThat(builder.request().getId(), is(randomId));
-                    assertThat(builder.request().isRecordExecution(), is(Boolean.parseBoolean(recordExecution)));
-                    assertThat(builder.request().isIgnoreCondition(), is(Boolean.parseBoolean(ignoreCondition)));
-                    assertThat(builder.request().isDebug(), is(Boolean.parseBoolean(debugCondition)));
+                    assertThat(request.getId(), is(randomId));
+                    assertThat(request.isRecordExecution(), is(Boolean.parseBoolean(recordExecution)));
+                    assertThat(request.isIgnoreCondition(), is(Boolean.parseBoolean(ignoreCondition)));
+                    assertThat(request.isDebug(), is(Boolean.parseBoolean(debugCondition)));
                 }
             }
         }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java
index 58a649de48c02..d128b907a73e6 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java
@@ -321,9 +321,5 @@ private static class TestPutIndexTemplateResponse extends AcknowledgedResponse {
         TestPutIndexTemplateResponse(boolean acknowledged) {
             super(acknowledged);
         }
-
-        TestPutIndexTemplateResponse() {
-            super();
-        }
     }
 }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java
index 5b11b444db3ca..7ad5b5e98bf4a 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java
@@ -41,19 +41,20 @@
 import org.elasticsearch.test.disruption.ServiceDisruptionScheme;
 import org.elasticsearch.test.store.MockFSIndexStore;
 import org.elasticsearch.test.transport.MockTransportService;
-import org.elasticsearch.xpack.core.XPackClient;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.watcher.WatcherState;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.execution.TriggeredWatchStoreField;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
 import org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
+import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceRequestBuilder;
+import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsResponse;
 import org.elasticsearch.xpack.core.watcher.watch.ClockMock;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.indexlifecycle.IndexLifecycle;
+import org.elasticsearch.xpack.watcher.ClockHolder;
 import org.elasticsearch.xpack.watcher.notification.email.Authentication;
 import org.elasticsearch.xpack.watcher.notification.email.Email;
 import org.elasticsearch.xpack.watcher.notification.email.EmailService;
@@ -64,7 +65,6 @@
 import org.junit.After;
 import org.junit.Before;
 
-import java.time.Clock;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
@@ -170,7 +170,7 @@ protected boolean timeWarped() {
     public void _setup() throws Exception {
         if (timeWarped()) {
             timeWarp = new TimeWarp(internalCluster().getInstances(ScheduleTriggerEngineMock.class),
-                    (ClockMock)getInstanceFromMaster(Clock.class));
+                    (ClockMock)getInstanceFromMaster(ClockHolder.class).clock);
         }
 
         if (internalCluster().size() > 0) {
@@ -278,7 +278,7 @@ public void replaceWatcherIndexWithRandomlyNamedIndex(String originalIndexOrAlia
         ensureGreen(to);
 
         AtomicReference<String> originalIndex = new AtomicReference<>(originalIndexOrAlias);
-        boolean watchesIsAlias = client().admin().indices().prepareAliasesExist(originalIndexOrAlias).get().isExists();
+        boolean watchesIsAlias = client().admin().indices().prepareGetAliases(originalIndexOrAlias).get().getAliases().isEmpty() == false;
         if (watchesIsAlias) {
             GetAliasesResponse aliasesResponse = client().admin().indices().prepareGetAliases(originalIndexOrAlias).get();
             assertEquals(1, aliasesResponse.getAliases().size());
@@ -327,10 +327,6 @@ protected WatchParser watchParser() {
         return getInstanceFromMaster(WatchParser.class);
     }
 
-    protected WatcherClient watcherClient() {
-        return randomBoolean() ? new XPackClient(client()).watcher() : new WatcherClient(client());
-    }
-
     private IndexNameExpressionResolver indexNameExpressionResolver() {
         return internalCluster().getInstance(IndexNameExpressionResolver.class);
     }
@@ -475,7 +471,7 @@ private void ensureWatcherTemplatesAdded() throws Exception {
 
     protected void startWatcher() throws Exception {
         assertBusy(() -> {
-            WatcherStatsResponse watcherStatsResponse = watcherClient().prepareWatcherStats().get();
+            WatcherStatsResponse watcherStatsResponse = new WatcherStatsRequestBuilder(client()).get();
             assertThat(watcherStatsResponse.hasFailures(), is(false));
             List<Tuple<String, WatcherState>> currentStatesFromStatsRequest = watcherStatsResponse.getNodes().stream()
                     .map(response -> Tuple.tuple(response.getNode().getName(), response.getWatcherState()))
@@ -496,7 +492,7 @@ protected void startWatcher() throws Exception {
 
             boolean isAllStateStopped = states.stream().allMatch(w -> w == WatcherState.STOPPED);
             if (isAllStateStopped) {
-                assertAcked(watcherClient().prepareWatchService().start().get());
+                assertAcked(new WatcherServiceRequestBuilder(client()).start().get());
                 throw new AssertionError("all nodes are stopped, restarting");
             }
 
@@ -519,7 +515,7 @@ protected void ensureLicenseEnabled() throws Exception {
 
     protected void stopWatcher() throws Exception {
         assertBusy(() -> {
-            WatcherStatsResponse watcherStatsResponse = watcherClient().prepareWatcherStats().get();
+            WatcherStatsResponse watcherStatsResponse = new WatcherStatsRequestBuilder(client()).get();
             assertThat(watcherStatsResponse.hasFailures(), is(false));
             List<Tuple<String, WatcherState>> currentStatesFromStatsRequest = watcherStatsResponse.getNodes().stream()
                     .map(response -> Tuple.tuple(response.getNode().getName(), response.getWatcherState()))
@@ -530,7 +526,7 @@ protected void stopWatcher() throws Exception {
 
             boolean isAllStateStarted = states.stream().allMatch(w -> w == WatcherState.STARTED);
             if (isAllStateStarted) {
-                assertAcked(watcherClient().prepareWatchService().stop().get());
+                assertAcked(new WatcherServiceRequestBuilder(client()).stop().get());
                 throw new AssertionError("all nodes are started, stopping");
             }
 
@@ -623,13 +619,13 @@ public void removeAndEnsureHealthy(InternalTestCluster cluster) {
         @Override
         public synchronized void applyToNode(String node, InternalTestCluster cluster) {
             if (frozen) {
-                ((ClockMock)cluster.getInstance(Clock.class, node)).freeze();
+                ((ClockMock)cluster.getInstance(ClockHolder.class, node).clock).freeze();
             }
         }
 
         @Override
         public void removeFromNode(String node, InternalTestCluster cluster) {
-            ((ClockMock)cluster.getInstance(Clock.class, node)).unfreeze();
+            ((ClockMock)cluster.getInstance(ClockHolder.class, node).clock).unfreeze();
         }
 
         @Override
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java
index 9636d159b52b4..aa1231baa17a4 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/WatcherTestUtils.java
@@ -158,7 +158,7 @@ public static Watch createTestWatch(String watchName, Client client, HttpClient
         httpRequest.path(new TextTemplate("/foobarbaz/{{ctx.watch_id}}"));
         httpRequest.body(new TextTemplate("{{ctx.watch_id}} executed with {{ctx.payload.response.hits.total_hits}} hits"));
         actions.add(new ActionWrapper("_webhook", null, null, null, new ExecutableWebhookAction(new WebhookAction(httpRequest.build()),
-                logger, httpClient, engine)));
+                logger, httpClient, engine), null));
 
 
         EmailTemplate email = EmailTemplate.builder().from("from@test.com").to("to@test.com").build();
@@ -166,7 +166,7 @@ public static Watch createTestWatch(String watchName, Client client, HttpClient
         EmailAction action = new EmailAction(email, "testaccount", auth, Profile.STANDARD, null, null);
         ExecutableEmailAction executale = new ExecutableEmailAction(action, logger, emailService, engine,
                 new HtmlSanitizer(Settings.EMPTY), Collections.emptyMap());
-        actions.add(new ActionWrapper("_email", null, null, null, executale));
+        actions.add(new ActionWrapper("_email", null, null, null, executale, null));
 
         ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
         Map<String, ActionStatus> statuses = new HashMap<>();
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherExecutorServiceBenchmark.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherExecutorServiceBenchmark.java
index b530811e1c7e4..08c2dc38c03f4 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherExecutorServiceBenchmark.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherExecutorServiceBenchmark.java
@@ -16,7 +16,7 @@
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction;
 import org.elasticsearch.xpack.watcher.Watcher;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.condition.ScriptCondition;
@@ -56,7 +56,6 @@ public class WatcherExecutorServiceBenchmark {
             .build();
 
     private static Client client;
-    private static WatcherClient watcherClient;
     private static ScheduleTriggerEngineMock scheduler;
 
     protected static void start() throws Exception {
@@ -65,7 +64,6 @@ protected static void start() throws Exception {
         client = node.client();
         client.admin().cluster().prepareHealth("*").setWaitForGreenStatus().get();
         Thread.sleep(5000);
-        watcherClient = node.injector().getInstance(WatcherClient.class);
         scheduler = node.injector().getInstance(ScheduleTriggerEngineMock.class);
     }
 
@@ -88,7 +86,7 @@ public static void main(String[] args) throws Exception {
                                 "ctx.payload.hits.total.value > 0",
                                 emptyMap()))).buildAsBytes(XContentType.JSON), XContentType.JSON);
                 putAlertRequest.setId(name);
-                watcherClient.putWatch(putAlertRequest).actionGet();
+                client.execute(PutWatchAction.INSTANCE, putAlertRequest).actionGet();
             }
 
             int numThreads = 50;
@@ -131,7 +129,7 @@ public static void main(String[] args) throws Exception {
                         .condition(new ScriptCondition(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "1 == 1", emptyMap())))
                         .addAction("_id", indexAction("index")).buildAsBytes(XContentType.JSON), XContentType.JSON);
                 putAlertRequest.setId(name);
-                watcherClient.putWatch(putAlertRequest).actionGet();
+                client.execute(PutWatchAction.INSTANCE, putAlertRequest).actionGet();
             }
 
             int numThreads = 50;
@@ -178,7 +176,7 @@ public static void main(String[] args) throws Exception {
                                 "ctx.payload.tagline == \"You Know, for Search\"",
                                 emptyMap()))).buildAsBytes(XContentType.JSON), XContentType.JSON);
                 putAlertRequest.setId(name);
-                watcherClient.putWatch(putAlertRequest).actionGet();
+                client.execute(PutWatchAction.INSTANCE, putAlertRequest).actionGet();
             }
 
             int numThreads = 50;
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherScheduleEngineBenchmark.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherScheduleEngineBenchmark.java
index 9c7c9027e7f85..c4c810681e9b0 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherScheduleEngineBenchmark.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherScheduleEngineBenchmark.java
@@ -30,8 +30,9 @@
 import org.elasticsearch.threadpool.ThreadPoolStats;
 import org.elasticsearch.xpack.core.watcher.WatcherState;
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
+import org.elasticsearch.xpack.core.watcher.transport.actions.service.WatcherServiceRequestBuilder;
+import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.actions.ActionBuilders;
 import org.elasticsearch.xpack.watcher.actions.logging.LoggingLevel;
@@ -153,12 +154,11 @@ public static void main(String[] args) throws Exception {
                     client.admin().cluster().prepareHealth(Watch.INDEX, "test").setWaitForYellowStatus().get();
 
                     Clock clock = node.injector().getInstance(Clock.class);
-                    WatcherClient watcherClient = node.injector().getInstance(WatcherClient.class);
-                    while (!watcherClient.prepareWatcherStats().get().getNodes().stream()
+                    while (!new WatcherStatsRequestBuilder(client).get().getNodes().stream()
                             .allMatch(r -> r.getWatcherState() == WatcherState.STARTED)) {
                         Thread.sleep(100);
                     }
-                    long actualLoadedWatches = watcherClient.prepareWatcherStats().get().getWatchesCount();
+                    long actualLoadedWatches = new WatcherStatsRequestBuilder(client).get().getWatchesCount();
                     if (actualLoadedWatches != numWatches) {
                         throw new IllegalStateException("Expected [" + numWatches + "] watched to be loaded, but only [" +
                                 actualLoadedWatches + "] watches were actually loaded");
@@ -228,7 +228,7 @@ public void run() {
                     Percentiles percentiles = searchResponse.getAggregations().get("percentile_delay");
                     stats.setDelayPercentiles(percentiles);
                     stats.setAvgJvmUsed(jvmUsedHeapSpace);
-                    watcherClient.prepareWatchService().stop().get();
+                    new WatcherServiceRequestBuilder(client).stop().get();
                 }
             }
         }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java
index 2f2299d7d65e0..08854743a41ef 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BasicWatcherTests.java
@@ -16,11 +16,12 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
+import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequestBuilder;
+import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.condition.CompareCondition;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
@@ -60,18 +61,15 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
-@TestLogging("org.elasticsearch.xpack.watcher:DEBUG," +
-             "org.elasticsearch.xpack.watcher.WatcherIndexingListener:TRACE")
 public class BasicWatcherTests extends AbstractWatcherIntegrationTestCase {
 
     public void testIndexWatch() throws Exception {
-        WatcherClient watcherClient = watcherClient();
         createIndex("idx");
         // Have a sample document in the index, the watch is going to evaluate
         client().prepareIndex("idx", "type").setSource("field", "foo").get();
         refresh();
         WatcherSearchTemplateRequest request = templateRequest(searchSource().query(termQuery("field", "foo")), "idx");
-        watcherClient.preparePutWatch("_name")
+        new PutWatchRequestBuilder(client()).setId("_name")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS)))
                         .input(searchInput(request))
@@ -83,15 +81,14 @@ public void testIndexWatch() throws Exception {
         timeWarp().trigger("_name");
         assertWatchWithMinimumPerformedActionsCount("_name", 1);
 
-        GetWatchResponse getWatchResponse = watcherClient().prepareGetWatch().setId("_name").get();
+        GetWatchResponse getWatchResponse = new GetWatchRequestBuilder(client()).setId("_name").get();
         assertThat(getWatchResponse.isFound(), is(true));
         assertThat(getWatchResponse.getSource(), notNullValue());
     }
 
     public void testIndexWatchRegisterWatchBeforeTargetIndex() throws Exception {
-        WatcherClient watcherClient = watcherClient();
         WatcherSearchTemplateRequest searchRequest = templateRequest(searchSource().query(termQuery("field", "value")), "idx");
-        watcherClient.preparePutWatch("_name")
+        new PutWatchRequestBuilder(client()).setId("_name")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS)))
                         .input(searchInput(searchRequest))
@@ -113,16 +110,15 @@ public void testIndexWatchRegisterWatchBeforeTargetIndex() throws Exception {
     }
 
     public void testDeleteWatch() throws Exception {
-        WatcherClient watcherClient = watcherClient();
         WatcherSearchTemplateRequest searchRequest = templateRequest(searchSource().query(matchAllQuery()), "idx");
-        PutWatchResponse indexResponse = watcherClient.preparePutWatch("_name")
+        PutWatchResponse indexResponse = new PutWatchRequestBuilder(client()).setId("_name")
                 .setSource(watchBuilder()
                         .trigger(schedule(cron("0/1 * * * * ? 2020")))
                         .input(searchInput(searchRequest))
                         .condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L)))
                 .get();
         assertThat(indexResponse.isCreated(), is(true));
-        DeleteWatchResponse deleteWatchResponse = watcherClient.prepareDeleteWatch("_name").get();
+        DeleteWatchResponse deleteWatchResponse = new DeleteWatchRequestBuilder(client()).setId("_name").get();
         assertThat(deleteWatchResponse, notNullValue());
         assertThat(deleteWatchResponse.isFound(), is(true));
 
@@ -130,13 +126,12 @@ public void testDeleteWatch() throws Exception {
         assertHitCount(client().prepareSearch(Watch.INDEX).setSize(0).get(), 0L);
 
         // Deleting the same watch for the second time
-        deleteWatchResponse = watcherClient.prepareDeleteWatch("_name").get();
+        deleteWatchResponse = new DeleteWatchRequestBuilder(client()).setId("_name").get();
         assertThat(deleteWatchResponse, notNullValue());
         assertThat(deleteWatchResponse.isFound(), is(false));
     }
 
     public void testMalformedWatch() throws Exception {
-        WatcherClient watcherClient = watcherClient();
         createIndex("idx");
         // Have a sample document in the index, the watch is going to evaluate
         client().prepareIndex("idx", "type").setSource("field", "value").get();
@@ -152,7 +147,7 @@ public void testMalformedWatch() throws Exception {
 
         watchSource.endObject();
         try {
-            watcherClient.preparePutWatch("_name")
+            new PutWatchRequestBuilder(client()).setId("_name")
                     .setSource(BytesReference.bytes(watchSource), watchSource.contentType())
                     .get();
             fail();
@@ -169,7 +164,6 @@ public void testMalformedWatch() throws Exception {
         }
     }
 
-    @TestLogging("org.elasticsearch.xpack.watcher:DEBUG")
     public void testModifyWatches() throws Exception {
         createIndex("idx");
         WatcherSearchTemplateRequest searchRequest = templateRequest(searchSource().query(matchAllQuery()), "idx");
@@ -179,7 +173,7 @@ public void testModifyWatches() throws Exception {
                 .input(searchInput(searchRequest))
                 .addAction("_id", indexAction("idx"));
 
-        watcherClient().preparePutWatch("_name")
+        new PutWatchRequestBuilder(client()).setId("_name")
                 .setSource(source.condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L)))
                 .get();
 
@@ -187,7 +181,7 @@ public void testModifyWatches() throws Exception {
         timeWarp().trigger("_name");
         assertWatchWithMinimumPerformedActionsCount("_name", 0, false);
 
-        watcherClient().preparePutWatch("_name")
+        new PutWatchRequestBuilder(client()).setId("_name")
                 .setSource(source.condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 0L)))
                 .get();
 
@@ -196,7 +190,7 @@ public void testModifyWatches() throws Exception {
         refresh();
         assertWatchWithMinimumPerformedActionsCount("_name", 1, false);
 
-        watcherClient().preparePutWatch("_name")
+        new PutWatchRequestBuilder(client()).setId("_name")
                 .setSource(source
                         .trigger(schedule(Schedules.cron("0/1 * * * * ? 2020")))
                         .condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 0L)))
@@ -235,20 +229,19 @@ public void testConditionSearchWithIndexedTemplate() throws Exception {
     }
 
     public void testInputFiltering() throws Exception {
-        WatcherClient watcherClient = watcherClient();
         createIndex("idx");
         // Have a sample document in the index, the watch is going to evaluate
         client().prepareIndex("idx", "type").setSource(jsonBuilder().startObject().field("field", "foovalue").endObject()).get();
         refresh();
         WatcherSearchTemplateRequest request = templateRequest(searchSource().query(termQuery("field", "foovalue")), "idx");
-        watcherClient.preparePutWatch("_name1")
+        new PutWatchRequestBuilder(client()).setId("_name1")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS)))
                         .input(searchInput(request).extractKeys("hits.total.value"))
                         .condition(new CompareCondition("ctx.payload.hits.total", CompareCondition.Op.EQ, 1L)))
                 .get();
         // in this watcher the condition will fail, because max_score isn't extracted, only total:
-        watcherClient.preparePutWatch("_name2")
+        new PutWatchRequestBuilder(client()).setId("_name2")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.SECONDS)))
                         .input(searchInput(request).extractKeys("hits.total.value"))
@@ -270,7 +263,7 @@ public void testInputFiltering() throws Exception {
 
     public void testPutWatchWithNegativeSchedule() throws Exception {
         try {
-            watcherClient().preparePutWatch("_name")
+            new PutWatchRequestBuilder(client()).setId("_name")
                     .setSource(watchBuilder()
                             .trigger(schedule(interval(-5, IntervalSchedule.Interval.Unit.SECONDS)))
                             .input(simpleInput("key", "value"))
@@ -283,7 +276,7 @@ public void testPutWatchWithNegativeSchedule() throws Exception {
         }
 
         try {
-            watcherClient().preparePutWatch("_name")
+            new PutWatchRequestBuilder(client()).setId("_name")
                     .setSource(watchBuilder()
                             .trigger(schedule(hourly().minutes(-10).build()))
                             .input(simpleInput("key", "value"))
@@ -296,7 +289,7 @@ public void testPutWatchWithNegativeSchedule() throws Exception {
         }
 
         try {
-            watcherClient().preparePutWatch("_name")
+            new PutWatchRequestBuilder(client()).setId("_name")
                     .setSource(watchBuilder()
                             .trigger(schedule(daily().atRoundHour(-10).build()))
                             .input(simpleInput("key", "value"))
@@ -310,7 +303,7 @@ public void testPutWatchWithNegativeSchedule() throws Exception {
         }
 
         try {
-            watcherClient().preparePutWatch("_name")
+            new PutWatchRequestBuilder(client()).setId("_name")
                     .setSource(watchBuilder()
                             .trigger(schedule(weekly().time(WeekTimes.builder().atRoundHour(-10).build()).build()))
                             .input(simpleInput("key", "value"))
@@ -324,7 +317,7 @@ public void testPutWatchWithNegativeSchedule() throws Exception {
         }
 
         try {
-            watcherClient().preparePutWatch("_name")
+            new PutWatchRequestBuilder(client()).setId("_name")
                     .setSource(watchBuilder()
                             .trigger(schedule(monthly().time(MonthTimes.builder().atRoundHour(-10).build()).build()))
                             .input(simpleInput("key", "value"))
@@ -345,7 +338,7 @@ private void testConditionSearch(WatcherSearchTemplateRequest request) throws Ex
         String watchName = "_name";
         assertAcked(prepareCreate("events").addMapping("event", "level", "type=text"));
 
-        watcherClient().preparePutWatch(watchName)
+        new PutWatchRequestBuilder(client()).setId(watchName)
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("5s")))
                         .input(searchInput(request))
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java
index 9c32e07346b0d..2f2cae2cb41c3 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java
@@ -18,6 +18,8 @@
 import org.elasticsearch.xpack.core.watcher.execution.Wid;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
 import org.elasticsearch.xpack.core.watcher.history.WatchRecord;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
+import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsResponse;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.core.watcher.watch.WatchField;
@@ -135,7 +137,7 @@ public void testLoadMalformedWatchRecord() throws Exception {
         startWatcher();
 
         assertBusy(() -> {
-            WatcherStatsResponse response = watcherClient().prepareWatcherStats().get();
+            WatcherStatsResponse response = new WatcherStatsRequestBuilder(client()).get();
             assertThat(response.getWatchesCount(), equalTo(1L));
         });
     }
@@ -165,7 +167,7 @@ public void testLoadExistingWatchesUponStartup() throws Exception {
         startWatcher();
 
         assertBusy(() -> {
-            WatcherStatsResponse response = watcherClient().prepareWatcherStats().get();
+            WatcherStatsResponse response = new WatcherStatsRequestBuilder(client()).get();
             assertThat(response.getWatchesCount(), equalTo((long) numWatches));
         });
     }
@@ -176,7 +178,7 @@ public void testMixedTriggeredWatchLoading() throws Exception {
                 .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                 .setSource("field", "value").get();
 
-        WatcherStatsResponse response = watcherClient().prepareWatcherStats().get();
+        WatcherStatsResponse response = new WatcherStatsRequestBuilder(client()).get();
         assertThat(response.getWatchesCount(), equalTo(0L));
 
         WatcherSearchTemplateRequest request = templateRequest(searchSource().query(termQuery("field", "value")), "my-index");
@@ -185,7 +187,7 @@ public void testMixedTriggeredWatchLoading() throws Exception {
         int numWatches = 8;
         for (int i = 0; i < numWatches; i++) {
             String watchId = "_id" + i;
-            watcherClient().preparePutWatch(watchId).setSource(watchBuilder()
+            new PutWatchRequestBuilder(client()).setId(watchId).setSource(watchBuilder()
                     .trigger(schedule(cron("0/5 * * * * ? 2050")))
                     .input(searchInput(request))
                     .condition(InternalAlwaysCondition.INSTANCE)
@@ -225,12 +227,12 @@ public void testTriggeredWatchLoading() throws Exception {
                 .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
                 .setSource("field", "value").get();
 
-        WatcherStatsResponse response = watcherClient().prepareWatcherStats().get();
+        WatcherStatsResponse response = new WatcherStatsRequestBuilder(client()).get();
         assertThat(response.getWatchesCount(), equalTo(0L));
 
         String watchId = "_id";
         WatcherSearchTemplateRequest request = templateRequest(searchSource().query(termQuery("field", "value")), "my-index");
-        watcherClient().preparePutWatch(watchId).setSource(watchBuilder()
+        new PutWatchRequestBuilder(client()).setId(watchId).setSource(watchBuilder()
                 .trigger(schedule(cron("0/5 * * * * ? 2050")))
                 .input(searchInput(request))
                 .condition(InternalAlwaysCondition.INSTANCE)
@@ -266,7 +268,7 @@ private void assertSingleExecutionAndCompleteWatchHistory(final long numberOfWat
         assertBusy(() -> {
             // We need to wait until all the records are processed from the internal execution queue, only then we can assert
             // that numRecords watch records have been processed as part of starting up.
-            WatcherStatsResponse response = watcherClient().prepareWatcherStats().setIncludeCurrentWatches(true).get();
+            WatcherStatsResponse response = new WatcherStatsRequestBuilder(client()).setIncludeCurrentWatches(true).get();
             long maxSize = response.getNodes().stream().map(WatcherStatsResponse.Node::getSnapshots).mapToLong(List::size).sum();
             assertThat(maxSize, equalTo(0L));
 
@@ -288,13 +290,13 @@ private void assertSingleExecutionAndCompleteWatchHistory(final long numberOfWat
     }
 
     public void testManuallyStopped() throws Exception {
-        WatcherStatsResponse response = watcherClient().prepareWatcherStats().get();
+        WatcherStatsResponse response = new WatcherStatsRequestBuilder(client()).get();
         assertThat(response.watcherMetaData().manuallyStopped(), is(false));
         stopWatcher();
-        response = watcherClient().prepareWatcherStats().get();
+        response = new WatcherStatsRequestBuilder(client()).get();
         assertThat(response.watcherMetaData().manuallyStopped(), is(true));
         startWatcher();
-        response = watcherClient().prepareWatcherStats().get();
+        response = new WatcherStatsRequestBuilder(client()).get();
         assertThat(response.watcherMetaData().manuallyStopped(), is(false));
     }
 
@@ -302,7 +304,7 @@ public void testWatchRecordSavedTwice() throws Exception {
         // Watcher could prevent to start if a watch record tried to executed twice or more and the watch didn't exist
         // for that watch record or the execution threadpool rejected the watch record.
         // A watch record without a watch is the easiest to simulate, so that is what this test does.
-        if (client().admin().indices().prepareExists(Watch.INDEX).get().isExists() == false) {
+        if (indexExists(Watch.INDEX) == false) {
             // we rarely create an .watches alias in the base class
             assertAcked(client().admin().indices().prepareCreate(Watch.INDEX));
         }
@@ -339,7 +341,7 @@ public void testWatchRecordSavedTwice() throws Exception {
         assertBusy(() -> {
             // We need to wait until all the records are processed from the internal execution queue, only then we can assert
             // that numRecords watch records have been processed as part of starting up.
-            WatcherStatsResponse response = watcherClient().prepareWatcherStats().setIncludeCurrentWatches(true).get();
+            WatcherStatsResponse response = new WatcherStatsRequestBuilder(client()).setIncludeCurrentWatches(true).get();
             long maxSize = response.getNodes().stream().map(WatcherStatsResponse.Node::getSnapshots).mapToLong(List::size).sum();
             assertThat(maxSize, equalTo(0L));
 
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/ExecutionVarsIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/ExecutionVarsIntegrationTests.java
index 9b92294a07022..a0d704c1f0850 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/ExecutionVarsIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/ExecutionVarsIntegrationTests.java
@@ -6,13 +6,14 @@
 package org.elasticsearch.xpack.watcher.test.integration;
 
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
-import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
+import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.condition.ScriptCondition;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.elasticsearch.xpack.watcher.test.WatcherMockScriptPlugin;
@@ -108,9 +109,7 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
     }
 
     public void testVars() throws Exception {
-        WatcherClient watcherClient = watcherClient();
-
-        PutWatchResponse putWatchResponse = watcherClient.preparePutWatch(watchId).setSource(watchBuilder()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client()).setId(watchId).setSource(watchBuilder()
                 .trigger(schedule(cron("0/1 * * * * ?")))
                 .input(simpleInput("value", 5))
                 .condition(new ScriptCondition(
@@ -172,9 +171,7 @@ public void testVars() throws Exception {
     }
 
     public void testVarsManual() throws Exception {
-        WatcherClient watcherClient = watcherClient();
-
-        PutWatchResponse putWatchResponse = watcherClient.preparePutWatch(watchId).setSource(watchBuilder()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client()).setId(watchId).setSource(watchBuilder()
                 .trigger(schedule(cron("0/1 * * * * ? 2020")))
                 .input(simpleInput("value", 5))
                 .condition(new ScriptCondition(
@@ -195,8 +192,8 @@ public void testVarsManual() throws Exception {
 
         boolean debug = randomBoolean();
 
-        ExecuteWatchResponse executeWatchResponse = watcherClient
-                .prepareExecuteWatch(watchId)
+        ExecuteWatchResponse executeWatchResponse = new ExecuteWatchRequestBuilder(client())
+                .setId(watchId)
                 .setDebug(debug)
                 .get();
         assertThat(executeWatchResponse.getRecordId(), notNullValue());
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java
index d58664fb40760..7c9e3107042fc 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java
@@ -18,6 +18,9 @@
 import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder;
 import org.elasticsearch.xpack.core.watcher.input.Input;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
+import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
+import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequestBuilder;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.watch.WatchStatus;
 import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
@@ -59,10 +62,10 @@ public void testThatHistoryIsWrittenWithChainedInput() throws Exception {
                 templateRequest(searchSource().sort(SortBuilders.fieldSort("inner.date").order(SortOrder.DESC)), "foo")))
         );
 
-        PutWatchResponse response = watcherClient().preparePutWatch("test_watch").setSource(builder).get();
+        PutWatchResponse response = new PutWatchRequestBuilder(client()).setId("test_watch").setSource(builder).get();
         assertThat(response.isCreated(), is(true));
 
-        watcherClient().prepareExecuteWatch("test_watch").setRecordExecution(true).get();
+        new ExecuteWatchRequestBuilder(client()).setId("test_watch").setRecordExecution(true).get();
 
         flushAndRefresh(".watcher-history-*");
         SearchResponse searchResponse = client().prepareSearch(".watcher-history-*").get();
@@ -85,14 +88,14 @@ public void testFailedInputResultWithDotsInFieldNameGetsStored() throws Exceptio
             input = chainInput().add("chained", input);
         }
 
-        watcherClient().preparePutWatch("test_watch")
+        new PutWatchRequestBuilder(client()).setId("test_watch")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.HOURS)))
                         .input(input)
                         .addAction("_logger", loggingAction("#### randomLogging")))
                 .get();
 
-        watcherClient().prepareExecuteWatch("test_watch").setRecordExecution(true).get();
+        new ExecuteWatchRequestBuilder(client()).setId("test_watch").setRecordExecution(true).get();
 
         refresh(".watcher-history*");
         SearchResponse searchResponse = client().prepareSearch(".watcher-history*").setSize(0).get();
@@ -126,14 +129,14 @@ public void testPayloadInputWithDotsInFieldNameWorks() throws Exception {
             input = chainInput().add("chained", input);
         }
 
-        watcherClient().preparePutWatch("test_watch")
+        new PutWatchRequestBuilder(client()).setId("test_watch")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.HOURS)))
                         .input(input)
                         .addAction("_logger", loggingAction("#### randomLogging")))
                 .get();
 
-        watcherClient().prepareExecuteWatch("test_watch").setRecordExecution(true).get();
+        new ExecuteWatchRequestBuilder(client()).setId("test_watch").setRecordExecution(true).get();
 
         refresh(".watcher-history*");
         SearchResponse searchResponse = client().prepareSearch(".watcher-history*").setSize(0).get();
@@ -157,16 +160,16 @@ public void testPayloadInputWithDotsInFieldNameWorks() throws Exception {
     }
 
     public void testThatHistoryContainsStatus() throws Exception {
-        watcherClient().preparePutWatch("test_watch")
+        new PutWatchRequestBuilder(client()).setId("test_watch")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval(5, IntervalSchedule.Interval.Unit.HOURS)))
                         .input(simpleInput("foo", "bar"))
                         .addAction("_logger", loggingAction("#### randomLogging")))
                 .get();
 
-        watcherClient().prepareExecuteWatch("test_watch").setRecordExecution(true).get();
+        new ExecuteWatchRequestBuilder(client()).setId("test_watch").setRecordExecution(true).get();
 
-        WatchStatus status = watcherClient().prepareGetWatch("test_watch").get().getStatus();
+        WatchStatus status = new GetWatchRequestBuilder(client()).setId("test_watch").get().getStatus();
 
         refresh(".watcher-history*");
         SearchResponse searchResponse = client().prepareSearch(".watcher-history*").setSize(1).get();
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java
index fe548abe1f536..271821783e3a5 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/HttpSecretsIntegrationTests.java
@@ -13,13 +13,15 @@
 import org.elasticsearch.test.http.MockResponse;
 import org.elasticsearch.test.http.MockWebServer;
 import org.elasticsearch.xpack.core.watcher.WatcherField;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.crypto.CryptoService;
 import org.elasticsearch.xpack.core.watcher.crypto.CryptoServiceTests;
 import org.elasticsearch.xpack.core.watcher.execution.ActionExecutionMode;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
+import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.trigger.TriggerEvent;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.common.http.BasicAuth;
@@ -88,8 +90,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
     }
 
     public void testHttpInput() throws Exception {
-        WatcherClient watcherClient = watcherClient();
-        watcherClient.preparePutWatch("_id")
+        new PutWatchRequestBuilder(client()).setId("_id")
                 .setSource(watchBuilder()
                         .trigger(schedule(cron("0 0 0 1 * ? 2020")))
                         .input(httpInput(HttpRequestTemplate.builder(webServer.getHostName(), webServer.getPort())
@@ -119,7 +120,7 @@ public void testHttpInput() throws Exception {
         }
 
         // verifying the password is not returned by the GET watch API
-        GetWatchResponse watchResponse = watcherClient.prepareGetWatch("_id").get();
+        GetWatchResponse watchResponse = new GetWatchRequestBuilder(client()).setId("_id").get();
         assertThat(watchResponse, notNullValue());
         assertThat(watchResponse.getId(), is("_id"));
         XContentSource contentSource = watchResponse.getSource();
@@ -142,7 +143,7 @@ public void testHttpInput() throws Exception {
                 BytesReference.bytes(jsonBuilder().startObject().field("key", "value").endObject()).utf8ToString()));
 
         TriggerEvent triggerEvent = new ScheduleTriggerEvent(ZonedDateTime.now(ZoneOffset.UTC), ZonedDateTime.now(ZoneOffset.UTC));
-        ExecuteWatchResponse executeResponse = watcherClient.prepareExecuteWatch("_id")
+        ExecuteWatchResponse executeResponse = new ExecuteWatchRequestBuilder(client()).setId("_id")
                 .setRecordExecution(false)
                 .setTriggerEvent(triggerEvent)
                 .setActionMode("_all", ActionExecutionMode.FORCE_EXECUTE)
@@ -167,8 +168,7 @@ public void testHttpInput() throws Exception {
     }
 
     public void testWebhookAction() throws Exception {
-        WatcherClient watcherClient = watcherClient();
-        watcherClient.preparePutWatch("_id")
+        new PutWatchRequestBuilder(client()).setId("_id")
                 .setSource(watchBuilder()
                         .trigger(schedule(cron("0 0 0 1 * ? 2020")))
                         .input(simpleInput())
@@ -199,7 +199,7 @@ public void testWebhookAction() throws Exception {
         }
 
         // verifying the password is not returned by the GET watch API
-        GetWatchResponse watchResponse = watcherClient.prepareGetWatch("_id").get();
+        GetWatchResponse watchResponse = new GetWatchRequestBuilder(client()).setId("_id").get();
         assertThat(watchResponse, notNullValue());
         assertThat(watchResponse.getId(), is("_id"));
         XContentSource contentSource = watchResponse.getSource();
@@ -222,7 +222,7 @@ public void testWebhookAction() throws Exception {
                 BytesReference.bytes(jsonBuilder().startObject().field("key", "value").endObject()).utf8ToString()));
 
         TriggerEvent triggerEvent = new ScheduleTriggerEvent(ZonedDateTime.now(ZoneOffset.UTC), ZonedDateTime.now(ZoneOffset.UTC));
-        ExecuteWatchResponse executeResponse = watcherClient.prepareExecuteWatch("_id")
+        ExecuteWatchResponse executeResponse = new ExecuteWatchRequestBuilder(client()).setId("_id")
                 .setRecordExecution(false)
                 .setActionMode("_all", ActionExecutionMode.FORCE_EXECUTE)
                 .setTriggerEvent(triggerEvent)
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/RejectedExecutionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/RejectedExecutionTests.java
index 9492e50048d4e..600dd2320e40e 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/RejectedExecutionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/RejectedExecutionTests.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.xpack.core.XPackSettings;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.condition.CompareCondition;
 import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
@@ -34,12 +34,11 @@ protected boolean timeWarped() {
     }
 
     public void testHistoryOnRejection() throws Exception {
-        WatcherClient watcherClient = watcherClient();
         createIndex("idx");
         client().prepareIndex("idx", "_doc").setSource("field", "a").get();
         refresh();
         WatcherSearchTemplateRequest request = templateRequest(searchSource().query(termQuery("field", "a")), "idx");
-        watcherClient.preparePutWatch(randomAlphaOfLength(5))
+        new PutWatchRequestBuilder(client()).setId(randomAlphaOfLength(5))
             .setSource(watchBuilder()
                 .trigger(schedule(interval(1, IntervalSchedule.Interval.Unit.SECONDS)))
                 .input(searchInput(request))
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java
index 8714d89415195..e45b1a7b91ee1 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java
@@ -188,7 +188,7 @@ public void testThatEmptyRequestBodyWorks() throws Exception {
     }
 
     private WatcherSearchTemplateService watcherSearchTemplateService() {
-        SearchModule module = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        SearchModule module = new SearchModule(Settings.EMPTY, Collections.emptyList());
         return new WatcherSearchTemplateService(scriptService, new NamedXContentRegistry(module.getNamedXContents()));
     }
 }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SingleNodeTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SingleNodeTests.java
index 318d1f1a8b1a1..04d43a7aa8b0f 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SingleNodeTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SingleNodeTests.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule;
@@ -50,7 +51,7 @@ public void testThatLoadingWithNonExistingIndexWorks() throws Exception {
 
         String watchId = randomAlphaOfLength(20);
         // now we start with an empty set up, store a watch and expected it to be executed
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch(watchId)
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client()).setId(watchId)
             .setSource(watchBuilder()
                 .trigger(schedule(interval(1, IntervalSchedule.Interval.Unit.SECONDS)))
                 .input(simpleInput())
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java
index a0ef5e97d8534..d31e083f63f51 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchAckTests.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.watcher.test.integration;
 
-
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.get.GetRequest;
@@ -15,15 +14,16 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.watcher.actions.ActionStatus;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
 import org.elasticsearch.xpack.core.watcher.history.WatchRecord;
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchResponse;
-import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
+import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
+import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.condition.CompareCondition;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
@@ -47,7 +47,6 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsEqual.equalTo;
 
-@TestLogging("org.elasticsearch.xpack.watcher:DEBUG")
 public class WatchAckTests extends AbstractWatcherIntegrationTestCase {
 
     private String id = randomAlphaOfLength(10);
@@ -62,7 +61,7 @@ public void indexTestDocument() {
     }
 
     public void testAckSingleAction() throws Exception {
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client())
                 .setId("_id")
                 .setSource(watchBuilder()
                         .trigger(schedule(cron("0/5 * * * * ? *")))
@@ -75,10 +74,10 @@ public void testAckSingleAction() throws Exception {
                 .get();
 
         assertThat(putWatchResponse.isCreated(), is(true));
-        assertThat(watcherClient().prepareWatcherStats().get().getWatchesCount(), is(1L));
+        assertThat(new WatcherStatsRequestBuilder(client()).get().getWatchesCount(), is(1L));
 
         timeWarp().trigger("_id", 4, TimeValue.timeValueSeconds(5));
-        AckWatchResponse ackResponse = watcherClient().prepareAckWatch("_id").setActionIds("_a1").get();
+        AckWatchResponse ackResponse = new AckWatchRequestBuilder(client(), "_id").setActionIds("_a1").get();
         assertThat(ackResponse.getStatus().actionStatus("_a1").ackStatus().state(), is(ActionStatus.AckStatus.State.ACKED));
         assertThat(ackResponse.getStatus().actionStatus("_a2").ackStatus().state(), is(ActionStatus.AckStatus.State.ACKABLE));
 
@@ -107,7 +106,7 @@ public void testAckSingleAction() throws Exception {
 
         timeWarp().trigger("_id", 4, TimeValue.timeValueSeconds(5));
 
-        GetWatchResponse getWatchResponse = watcherClient().prepareGetWatch("_id").get();
+        GetWatchResponse getWatchResponse = new GetWatchRequestBuilder(client()).setId("_id").get();
         assertThat(getWatchResponse.isFound(), is(true));
 
         Watch parsedWatch = watchParser().parse(getWatchResponse.getId(), true, getWatchResponse.getSource().getBytes(),
@@ -123,7 +122,7 @@ public void testAckSingleAction() throws Exception {
     }
 
     public void testAckAllActions() throws Exception {
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client())
                 .setId("_id")
                 .setSource(watchBuilder()
                         .trigger(schedule(cron("0/5 * * * * ? *")))
@@ -136,11 +135,11 @@ public void testAckAllActions() throws Exception {
                 .get();
 
         assertThat(putWatchResponse.isCreated(), is(true));
-        assertThat(watcherClient().prepareWatcherStats().get().getWatchesCount(), is(1L));
+        assertThat(new WatcherStatsRequestBuilder(client()).get().getWatchesCount(), is(1L));
 
         timeWarp().trigger("_id", 4, TimeValue.timeValueSeconds(5));
 
-        AckWatchRequestBuilder ackWatchRequestBuilder = watcherClient().prepareAckWatch("_id");
+        AckWatchRequestBuilder ackWatchRequestBuilder = new AckWatchRequestBuilder(client(), "_id");
         if (randomBoolean()) {
             ackWatchRequestBuilder.setActionIds("_all");
         } else if (randomBoolean()) {
@@ -176,7 +175,7 @@ public void testAckAllActions() throws Exception {
 
         timeWarp().trigger("_id", 4, TimeValue.timeValueSeconds(5));
 
-        GetWatchResponse getWatchResponse = watcherClient().prepareGetWatch("_id").get();
+        GetWatchResponse getWatchResponse = new GetWatchRequestBuilder(client()).setId("_id").get();
         assertThat(getWatchResponse.isFound(), is(true));
 
         Watch parsedWatch = watchParser().parse(getWatchResponse.getId(), true,
@@ -192,7 +191,7 @@ public void testAckAllActions() throws Exception {
     }
 
     public void testAckWithRestart() throws Exception {
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client())
                 .setId("_name")
                 .setSource(watchBuilder()
                         .trigger(schedule(cron("0/5 * * * * ? *")))
@@ -202,12 +201,12 @@ public void testAckWithRestart() throws Exception {
                         .addAction("_id", indexAction("actions")))
                 .get();
         assertThat(putWatchResponse.isCreated(), is(true));
-        assertThat(watcherClient().prepareWatcherStats().get().getWatchesCount(), is(1L));
+        assertThat(new WatcherStatsRequestBuilder(client()).get().getWatchesCount(), is(1L));
 
         timeWarp().trigger("_name", 4, TimeValue.timeValueSeconds(5));
         restartWatcherRandomly();
 
-        AckWatchResponse ackResponse = watcherClient().prepareAckWatch("_name").get();
+        AckWatchResponse ackResponse = new AckWatchRequestBuilder(client(), "_name").get();
         assertThat(ackResponse.getStatus().actionStatus("_id").ackStatus().state(), is(ActionStatus.AckStatus.State.ACKED));
 
         refresh("actions");
@@ -217,7 +216,7 @@ public void testAckWithRestart() throws Exception {
 
         restartWatcherRandomly();
 
-        GetWatchResponse watchResponse = watcherClient().getWatch(new GetWatchRequest("_name")).actionGet();
+        GetWatchResponse watchResponse = new GetWatchRequestBuilder(client()).setId("_name").get();
         assertThat(watchResponse.getStatus().actionStatus("_id").ackStatus().state(), Matchers.equalTo(ActionStatus.AckStatus.State.ACKED));
 
         refresh();
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchMetadataTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchMetadataTests.java
index 1e2c1ddbc64f1..a192c9315cd46 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchMetadataTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/WatchMetadataTests.java
@@ -10,7 +10,9 @@
 import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.xpack.core.watcher.execution.ActionExecutionMode;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
+import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.trigger.TriggerEvent;
 import org.elasticsearch.xpack.watcher.actions.logging.LoggingAction;
 import org.elasticsearch.xpack.watcher.actions.logging.LoggingLevel;
@@ -48,7 +50,7 @@ public void testWatchMetadata() throws Exception {
         metaList.add("test");
 
         metadata.put("baz", metaList);
-        watcherClient().preparePutWatch("_name")
+        new PutWatchRequestBuilder(client()).setId("_name")
                 .setSource(watchBuilder()
                         .trigger(schedule(cron("0/5 * * * * ? *")))
                         .input(noneInput())
@@ -74,7 +76,7 @@ public void testWatchMetadataAvailableAtExecution() throws Exception {
                 .setLevel(LoggingLevel.DEBUG)
                 .setCategory("test");
 
-        watcherClient().preparePutWatch("_name")
+        new PutWatchRequestBuilder(client()).setId("_name")
                 .setSource(watchBuilder()
                         .trigger(schedule(cron("0 0 0 1 1 ? 2050")))
                         .input(noneInput())
@@ -85,7 +87,7 @@ public void testWatchMetadataAvailableAtExecution() throws Exception {
                 .get();
 
         TriggerEvent triggerEvent = new ScheduleTriggerEvent(ZonedDateTime.now(ZoneOffset.UTC), ZonedDateTime.now(ZoneOffset.UTC));
-        ExecuteWatchResponse executeWatchResponse = watcherClient().prepareExecuteWatch("_name")
+        ExecuteWatchResponse executeWatchResponse = new ExecuteWatchRequestBuilder(client()).setId("_name")
                 .setTriggerEvent(triggerEvent).setActionMode("_all", ActionExecutionMode.SIMULATE).get();
         Map<String, Object> result = executeWatchResponse.getRecordSource().getAsMap();
         logger.info("result=\n{}", result);
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java
index 3684597c3529d..4c3f4ff26524e 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transform/TransformIntegrationTests.java
@@ -15,7 +15,8 @@
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
-import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateRequest;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.elasticsearch.xpack.watcher.test.WatcherMockScriptPlugin;
@@ -116,7 +117,7 @@ public void testScriptTransform() throws Exception {
         }
 
         // put a watch that has watch level transform:
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id1")
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client(), "_id1")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("5s")))
                         .input(simpleInput(MapBuilder.<String, Object>newMapBuilder().put("key1", 10).put("key2", 10)))
@@ -125,7 +126,7 @@ public void testScriptTransform() throws Exception {
                 .get();
         assertThat(putWatchResponse.isCreated(), is(true));
         // put a watch that has a action level transform:
-        putWatchResponse = watcherClient().preparePutWatch("_id2")
+        putWatchResponse = new PutWatchRequestBuilder(client(), "_id2")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("5s")))
                         .input(simpleInput(MapBuilder.<String, Object>newMapBuilder().put("key1", 10).put("key2", 10)))
@@ -154,7 +155,6 @@ public void testScriptTransform() throws Exception {
         assertThat(response.getHits().getAt(0).getSourceAsMap().get("key3").toString(), equalTo("20"));
     }
 
-    @TestLogging("org.elasticsearch.xpack.watcher:DEBUG")
     public void testSearchTransform() throws Exception {
         createIndex("my-condition-index", "my-payload-index");
         ensureGreen("my-condition-index", "my-payload-index");
@@ -165,7 +165,7 @@ public void testSearchTransform() throws Exception {
         WatcherSearchTemplateRequest inputRequest = templateRequest(searchSource().query(matchAllQuery()), "my-condition-index");
         WatcherSearchTemplateRequest transformRequest = templateRequest(searchSource().query(matchAllQuery()), "my-payload-index");
 
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id1")
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client(), "_id1")
                 .setSource(watchBuilder()
                                 .trigger(schedule(interval("5s")))
                                 .input(searchInput(inputRequest))
@@ -173,7 +173,7 @@ public void testSearchTransform() throws Exception {
                                 .addAction("_id", indexAction("output1", "result"))
                 ).get();
         assertThat(putWatchResponse.isCreated(), is(true));
-        putWatchResponse = watcherClient().preparePutWatch("_id2")
+        putWatchResponse = new PutWatchRequestBuilder(client(), "_id2")
                 .setSource(watchBuilder()
                                 .trigger(schedule(interval("5s")))
                                 .input(searchInput(inputRequest))
@@ -205,7 +205,7 @@ public void testChainTransform() throws Exception {
         Script script2 = mockScript("['key4' : ctx.payload.key3 + 10]");
 
         // put a watch that has watch level transform:
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch("_id1")
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client(), "_id1")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("5s")))
                         .input(simpleInput(MapBuilder.<String, Object>newMapBuilder().put("key1", 10).put("key2", 10)))
@@ -214,7 +214,7 @@ public void testChainTransform() throws Exception {
                 .get();
         assertThat(putWatchResponse.isCreated(), is(true));
         // put a watch that has a action level transform:
-        putWatchResponse = watcherClient().preparePutWatch("_id2")
+        putWatchResponse = new PutWatchRequestBuilder(client(), "_id2")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("5s")))
                         .input(simpleInput(MapBuilder.<String, Object>newMapBuilder().put("key1", 10).put("key2", 10)))
@@ -245,7 +245,7 @@ public void testChainTransform() throws Exception {
     }
 
     private void executeWatch(String watchId) {
-        watcherClient().prepareExecuteWatch(watchId)
+        new ExecuteWatchRequestBuilder(client(), watchId)
             .setRecordExecution(true)
             .get();
     }
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/activate/ActivateWatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/activate/ActivateWatchTests.java
index f89d0eee7c941..eef257ce46b25 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/activate/ActivateWatchTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/activate/ActivateWatchTests.java
@@ -17,11 +17,14 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.test.junit.annotations.TestLogging;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.execution.ExecutionState;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
+import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.activate.ActivateWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
+import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsResponse;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 
@@ -40,7 +43,8 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
-@TestLogging("org.elasticsearch.xpack.watcher:DEBUG,org.elasticsearch.xpack.watcher.WatcherIndexingListener:TRACE")
+@TestLogging("org.elasticsearch.xpack.watcher:DEBUG,org.elasticsearch.xpack.core.watcher:DEBUG," +
+    "org.elasticsearch.xpack.watcher.WatcherIndexingListener:TRACE")
 public class ActivateWatchTests extends AbstractWatcherIntegrationTestCase {
 
     @Override
@@ -48,9 +52,8 @@ protected boolean timeWarped() {
         return false;
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30699")
     public void testDeactivateAndActivate() throws Exception {
-        PutWatchResponse putWatchResponse = watcherClient().preparePutWatch()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client())
                 .setId("_id")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("1s")))
@@ -61,7 +64,7 @@ public void testDeactivateAndActivate() throws Exception {
 
         assertThat(putWatchResponse.isCreated(), is(true));
 
-        GetWatchResponse getWatchResponse = watcherClient().prepareGetWatch("_id").get();
+        GetWatchResponse getWatchResponse = new GetWatchRequestBuilder(client(), "_id").get();
         assertThat(getWatchResponse, notNullValue());
         assertThat(getWatchResponse.getStatus().state().isActive(), is(true));
 
@@ -69,17 +72,17 @@ public void testDeactivateAndActivate() throws Exception {
         assertWatchWithMinimumActionsCount("_id", ExecutionState.EXECUTED, 1);
 
         // we now know the watch is executing... lets deactivate it
-        ActivateWatchResponse activateWatchResponse = watcherClient().prepareActivateWatch("_id", false).get();
+        ActivateWatchResponse activateWatchResponse = new ActivateWatchRequestBuilder(client(), "_id", false).get();
         assertThat(activateWatchResponse, notNullValue());
         assertThat(activateWatchResponse.getStatus().state().isActive(), is(false));
 
-        getWatchResponse = watcherClient().prepareGetWatch("_id").get();
+        getWatchResponse = new GetWatchRequestBuilder(client(), "_id").get();
         assertThat(getWatchResponse, notNullValue());
         assertThat(getWatchResponse.getStatus().state().isActive(), is(false));
 
         // wait until no watch is executing
         assertBusy(() -> {
-            WatcherStatsResponse statsResponse = watcherClient().prepareWatcherStats().setIncludeCurrentWatches(true).get();
+            WatcherStatsResponse statsResponse = new WatcherStatsRequestBuilder(client()).setIncludeCurrentWatches(true).get();
             int sum = statsResponse.getNodes().stream().map(WatcherStatsResponse.Node::getSnapshots).mapToInt(List::size).sum();
             assertThat(sum, is(0));
         });
@@ -88,36 +91,30 @@ public void testDeactivateAndActivate() throws Exception {
         refresh();
         long count1 = docCount(".watcher-history*", matchAllQuery());
 
-        logger.info("Sleeping for 5 seconds, watch history count [{}]", count1);
-        Thread.sleep(5000);
-
         refresh();
-        long count2 = docCount(".watcher-history*", matchAllQuery());
-
-        assertThat(count2, is(count1));
+        //ensure no new watch history
+        awaitBusy(() -> count1 != docCount(".watcher-history*", matchAllQuery()), 5, TimeUnit.SECONDS);
 
         // lets activate it again
         logger.info("Activating watch again");
 
-        activateWatchResponse = watcherClient().prepareActivateWatch("_id", true).get();
+        activateWatchResponse = new ActivateWatchRequestBuilder(client(), "_id", true).get();
         assertThat(activateWatchResponse, notNullValue());
         assertThat(activateWatchResponse.getStatus().state().isActive(), is(true));
 
-        getWatchResponse = watcherClient().prepareGetWatch("_id").get();
+        getWatchResponse = new GetWatchRequestBuilder(client(), "_id").get();
         assertThat(getWatchResponse, notNullValue());
         assertThat(getWatchResponse.getStatus().state().isActive(), is(true));
 
-        logger.info("Sleeping for another five seconds, ensuring that watch is executed");
-        Thread.sleep(5000);
         refresh();
-        long count3 = docCount(".watcher-history*", matchAllQuery());
-        assertThat(count3, greaterThan(count1));
+        assertBusy(() -> {
+            long count2 = docCount(".watcher-history*", matchAllQuery());
+            assertThat(count2, greaterThan(count1));
+        });
     }
 
     public void testLoadWatchWithoutAState() throws Exception {
-        WatcherClient watcherClient = watcherClient();
-
-        PutWatchResponse putWatchResponse = watcherClient.preparePutWatch()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client())
                 .setId("_id")
                 .setSource(watchBuilder()
                         .trigger(schedule(cron("0 0 0 1 1 ? 2050"))) // some time in 2050
@@ -128,7 +125,7 @@ public void testLoadWatchWithoutAState() throws Exception {
 
         assertThat(putWatchResponse.isCreated(), is(true));
 
-        GetWatchResponse getWatchResponse = watcherClient.prepareGetWatch("_id").get();
+        GetWatchResponse getWatchResponse = new GetWatchRequestBuilder(client(), "_id").get();
         assertThat(getWatchResponse, notNullValue());
         assertThat(getWatchResponse.getStatus().state().isActive(), is(true));
 
@@ -161,7 +158,7 @@ public void testLoadWatchWithoutAState() throws Exception {
         stopWatcher();
         startWatcher();
 
-        getWatchResponse = watcherClient.prepareGetWatch("_id").get();
+        getWatchResponse = new GetWatchRequestBuilder(client(), "_id").get();
         assertThat(getWatchResponse, notNullValue());
         assertThat(getWatchResponse.getStatus().state(), notNullValue());
         assertThat(getWatchResponse.getStatus().state().isActive(), is(true));
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java
index b0c3f6359b52b..d5ed8fb45a157 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/delete/DeleteWatchTests.java
@@ -8,14 +8,18 @@
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ObjectPath;
+import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.test.http.MockResponse;
 import org.elasticsearch.test.http.MockWebServer;
 import org.elasticsearch.xpack.core.watcher.history.HistoryStoreField;
-import org.elasticsearch.common.xcontent.ObjectPath;
-import org.elasticsearch.protocol.xpack.watcher.DeleteWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchRequestBuilder;
+import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.common.http.HttpRequestTemplate;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 
@@ -51,7 +55,7 @@ public void testWatchDeletionDuringExecutionWorks() throws Exception {
             server.start();
             HttpRequestTemplate template = HttpRequestTemplate.builder(server.getHostName(), server.getPort()).path("/").build();
 
-            PutWatchResponse responseFuture = watcherClient().preparePutWatch("_name").setSource(watchBuilder()
+            PutWatchResponse responseFuture = new PutWatchRequestBuilder(client(), "_name").setSource(watchBuilder()
                     .trigger(schedule(interval("6h")))
                     .input(httpInput(template))
                     .addAction("_action1", loggingAction("anything")))
@@ -59,17 +63,17 @@ public void testWatchDeletionDuringExecutionWorks() throws Exception {
             assertThat(responseFuture.isCreated(), is(true));
 
             ActionFuture<ExecuteWatchResponse> executeWatchFuture =
-                    watcherClient().prepareExecuteWatch("_name").setRecordExecution(true).execute();
+                    new ExecuteWatchRequestBuilder(client(), "_name").setRecordExecution(true).execute();
 
             // without this sleep the delete operation might overtake the watch execution
             sleep(1000);
-            DeleteWatchResponse deleteWatchResponse = watcherClient().prepareDeleteWatch("_name").get();
+            DeleteWatchResponse deleteWatchResponse = new DeleteWatchRequestBuilder(client(), "_name").get();
             assertThat(deleteWatchResponse.isFound(), is(true));
 
             executeWatchFuture.get();
 
             // the watch is gone, no leftovers
-            GetWatchResponse getWatchResponse = watcherClient().prepareGetWatch("_name").get();
+            GetWatchResponse getWatchResponse = new GetWatchRequestBuilder(client(), "_name").get();
             assertThat(getWatchResponse.isFound(), is(false));
 
             // the watch history shows a successful execution, even though the watch was deleted
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/execute/ExecuteWatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/execute/ExecuteWatchTests.java
index c743922a4d715..a849036d9a3c6 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/execute/ExecuteWatchTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/execute/ExecuteWatchTests.java
@@ -8,13 +8,14 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
 import org.elasticsearch.xpack.core.watcher.actions.ActionStatus;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.execution.ActionExecutionMode;
 import org.elasticsearch.xpack.core.watcher.execution.Wid;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.execute.ExecuteWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.watch.WatchStatus;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
@@ -31,9 +32,7 @@
 public class ExecuteWatchTests extends AbstractWatcherIntegrationTestCase {
 
     public void testExecuteAllDefaults() throws Exception {
-        WatcherClient watcherClient = watcherClient();
-
-        PutWatchResponse putWatchResponse = watcherClient.preparePutWatch()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client())
                 .setId("_id")
                 .setSource(watchBuilder()
                         .trigger(schedule(cron("0/5 * * * * ? 2099")))
@@ -44,7 +43,7 @@ public void testExecuteAllDefaults() throws Exception {
 
         assertThat(putWatchResponse.isCreated(), is(true));
 
-        ExecuteWatchResponse response = watcherClient.prepareExecuteWatch("_id").get();
+        ExecuteWatchResponse response = new ExecuteWatchRequestBuilder(client(), "_id").get();
         assertThat(response, notNullValue());
         assertThat(response.getRecordId(), notNullValue());
         Wid wid = new Wid(response.getRecordId());
@@ -73,9 +72,7 @@ public void testExecuteAllDefaults() throws Exception {
     }
 
     public void testExecuteActionMode() throws Exception {
-        final WatcherClient watcherClient = watcherClient();
-
-        PutWatchResponse putWatchResponse = watcherClient.preparePutWatch()
+        PutWatchResponse putWatchResponse = new PutWatchRequestBuilder(client())
                 .setId("_id")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval("1s"))) // run every second so we can ack it
@@ -117,7 +114,7 @@ public void testExecuteActionMode() throws Exception {
                     new String[] { "foo", "_all" },
                     null
             );
-            AckWatchRequestBuilder ackWatchRequestBuilder = watcherClient.prepareAckWatch("_id");
+            AckWatchRequestBuilder ackWatchRequestBuilder = new AckWatchRequestBuilder(client(), "_id");
             if (actionIds != null) {
                 ackWatchRequestBuilder.setActionIds(actionIds);
             }
@@ -130,7 +127,7 @@ public void testExecuteActionMode() throws Exception {
             assertThat(actionStatus.ackStatus().state(), is(ActionStatus.AckStatus.State.ACKED));
         }
 
-        ExecuteWatchResponse response = watcherClient.prepareExecuteWatch("_id")
+        ExecuteWatchResponse response = new ExecuteWatchRequestBuilder(client(), "_id")
                 .setActionMode(randomBoolean() ? "log" : "_all", mode)
                 .get();
         assertThat(response, notNullValue());
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/get/GetWatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/get/GetWatchTests.java
index 226879b5e3047..72a2ffc5d62a0 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/get/GetWatchTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/action/get/GetWatchTests.java
@@ -8,10 +8,10 @@
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.protocol.xpack.watcher.PutWatchResponse;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
-import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
+import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.watcher.condition.InternalAlwaysCondition;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
@@ -30,12 +30,10 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-// added due to https://github.com/elastic/x-pack-elasticsearch/issues/3854
-@TestLogging("org.elasticsearch.action.search:DEBUG")
 public class GetWatchTests extends AbstractWatcherIntegrationTestCase {
 
     public void testGet() throws Exception {
-        PutWatchResponse putResponse = watcherClient().preparePutWatch("_name").setSource(watchBuilder()
+        PutWatchResponse putResponse = new PutWatchRequestBuilder(client(), "_name").setSource(watchBuilder()
                 .trigger(schedule(interval("5m")))
                 .input(simpleInput())
                 .condition(InternalAlwaysCondition.INSTANCE)
@@ -45,7 +43,7 @@ public void testGet() throws Exception {
         assertThat(putResponse, notNullValue());
         assertThat(putResponse.isCreated(), is(true));
 
-        GetWatchResponse getResponse = watcherClient().getWatch(new GetWatchRequest("_name")).get();
+        GetWatchResponse getResponse = new GetWatchRequestBuilder(client(), "_name").get();
         assertThat(getResponse, notNullValue());
         assertThat(getResponse.isFound(), is(true));
         assertThat(getResponse.getId(), is("_name"));
@@ -73,7 +71,7 @@ public void testGetNotFound() throws Exception {
             } catch (IndexNotFoundException e) {}
         }
 
-        GetWatchResponse getResponse = watcherClient().getWatch(new GetWatchRequest("_name")).get();
+        GetWatchResponse getResponse = new GetWatchRequestBuilder(client(), "_name").get();
         assertThat(getResponse, notNullValue());
         assertThat(getResponse.getId(), is("_name"));
         assertThat(getResponse.isFound(), is(false));
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchActionTests.java
index 0f7d64527fe26..bfaed67f6de23 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchActionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/ack/TransportAckWatchActionTests.java
@@ -32,6 +32,7 @@
 import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.stats.WatcherStatsResponse;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
+import org.elasticsearch.xpack.watcher.ClockHolder;
 import org.elasticsearch.xpack.watcher.watch.WatchParser;
 import org.junit.Before;
 
@@ -63,7 +64,8 @@ public void setupAction() {
         client = mock(Client.class);
         when(client.threadPool()).thenReturn(threadPool);
         action = new TransportAckWatchAction(transportService, new ActionFilters(Collections.emptySet()),
-            Clock.systemUTC(), new XPackLicenseState(Settings.EMPTY), watchParser, client, createClusterService(threadPool));
+            new ClockHolder(Clock.systemUTC()), new XPackLicenseState(Settings.EMPTY),
+            watchParser, client, createClusterService(threadPool));
     }
 
     public void testWatchNotFound() {
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchActionTests.java
index 411bee5e2f9d9..0ee82d86c76a9 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchActionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/put/TransportPutWatchActionTests.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.watcher.watch.ClockMock;
 import org.elasticsearch.xpack.core.watcher.watch.Watch;
+import org.elasticsearch.xpack.watcher.ClockHolder;
 import org.elasticsearch.xpack.watcher.test.WatchExecutionContextMockBuilder;
 import org.elasticsearch.xpack.watcher.watch.WatchParser;
 import org.junit.Before;
@@ -75,7 +76,7 @@ public void setupAction() throws Exception {
         }).when(client).execute(any(), any(), any());
 
         action = new TransportPutWatchAction(transportService, threadPool, new ActionFilters(Collections.emptySet()),
-            new ClockMock(), new XPackLicenseState(Settings.EMPTY), parser, client);
+            new ClockHolder(new ClockMock()), new XPackLicenseState(Settings.EMPTY), parser, client);
     }
 
     public void testHeadersAreFilteredWhenPuttingWatches() throws Exception {
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/stats/TransportWatcherStatsActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/stats/TransportWatcherStatsActionTests.java
index 9a7c9f58f3d43..49307e73b25c2 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/stats/TransportWatcherStatsActionTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/transport/actions/stats/TransportWatcherStatsActionTests.java
@@ -85,8 +85,8 @@ public void setupTransportAction() {
     public void testWatcherStats() throws Exception {
         WatcherStatsRequest request = new WatcherStatsRequest();
         request.includeStats(true);
-        WatcherStatsResponse.Node nodeResponse1 = action.nodeOperation(new WatcherStatsRequest.Node(request, "nodeId"));
-        WatcherStatsResponse.Node nodeResponse2 = action.nodeOperation(new WatcherStatsRequest.Node(request, "nodeId2"));
+        WatcherStatsResponse.Node nodeResponse1 = action.nodeOperation(new WatcherStatsRequest.Node(request), null);
+        WatcherStatsResponse.Node nodeResponse2 = action.nodeOperation(new WatcherStatsRequest.Node(request), null);
 
         WatcherStatsResponse response = action.newResponse(request,
             Arrays.asList(nodeResponse1, nodeResponse2), Collections.emptyList());
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchStatusIntegrationTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchStatusIntegrationTests.java
index 74068538511bb..d8b9202aad5d6 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchStatusIntegrationTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchStatusIntegrationTests.java
@@ -7,9 +7,10 @@
 
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.xpack.core.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource;
+import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequestBuilder;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse;
+import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder;
 import org.elasticsearch.xpack.watcher.condition.NeverCondition;
 import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase;
 import org.elasticsearch.xpack.watcher.test.WatcherTestUtils;
@@ -33,8 +34,7 @@
 public class WatchStatusIntegrationTests extends AbstractWatcherIntegrationTestCase {
 
     public void testThatStatusGetsUpdated() {
-        WatcherClient watcherClient = watcherClient();
-        watcherClient.preparePutWatch("_name")
+        new PutWatchRequestBuilder(client(), "_name")
                 .setSource(watchBuilder()
                         .trigger(schedule(interval(5, SECONDS)))
                         .input(simpleInput())
@@ -43,7 +43,7 @@ public void testThatStatusGetsUpdated() {
                 .get();
         timeWarp().trigger("_name");
 
-        GetWatchResponse getWatchResponse = watcherClient.prepareGetWatch().setId("_name").get();
+        GetWatchResponse getWatchResponse = new GetWatchRequestBuilder(client(), "_name").get();
         assertThat(getWatchResponse.isFound(), is(true));
         assertThat(getWatchResponse.getSource(), notNullValue());
         assertThat(getWatchResponse.getStatus().lastChecked(), is(notNullValue()));
diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java
index 84144b2866b32..feadeba084d3e 100644
--- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java
+++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/watch/WatchTests.java
@@ -441,7 +441,7 @@ public void testParseWatchWithoutTriggerDoesNotWork() throws Exception {
     private WatchParser createWatchparser() throws Exception {
         LoggingAction loggingAction = new LoggingAction(new TextTemplate("foo"), null, null);
         List<ActionWrapper> actions = Collections.singletonList(new ActionWrapper("_logging_", randomThrottler(), null, null,
-                new ExecutableLoggingAction(loggingAction, logger, new MockTextTemplateEngine())));
+                new ExecutableLoggingAction(loggingAction, logger, new MockTextTemplateEngine()), null));
 
         ScheduleRegistry scheduleRegistry = registry(new IntervalSchedule(new IntervalSchedule.Interval(1,
                 IntervalSchedule.Interval.Unit.SECONDS)));
@@ -585,7 +585,7 @@ private List<ActionWrapper> randomActions() {
                     randomFrom(DataAttachment.JSON, DataAttachment.YAML), EmailAttachments.EMPTY_ATTACHMENTS);
             list.add(new ActionWrapper("_email_" + randomAlphaOfLength(8), randomThrottler(),
                     AlwaysConditionTests.randomCondition(scriptService), randomTransform(),
-                    new ExecutableEmailAction(action, logger, emailService, templateEngine, htmlSanitizer, Collections.emptyMap())));
+                    new ExecutableEmailAction(action, logger, emailService, templateEngine, htmlSanitizer, Collections.emptyMap()), null));
         }
         if (randomBoolean()) {
             ZoneOffset timeZone = randomBoolean() ? ZoneOffset.UTC : null;
@@ -596,7 +596,7 @@ private List<ActionWrapper> randomActions() {
             list.add(new ActionWrapper("_index_" + randomAlphaOfLength(8), randomThrottler(),
                     AlwaysConditionTests.randomCondition(scriptService),  randomTransform(),
                     new ExecutableIndexAction(action, logger, client, TimeValue.timeValueSeconds(30),
-                            TimeValue.timeValueSeconds(30))));
+                            TimeValue.timeValueSeconds(30)), null));
         }
         if (randomBoolean()) {
             HttpRequestTemplate httpRequest = HttpRequestTemplate.builder("test.host", randomIntBetween(8000, 9000))
@@ -606,7 +606,7 @@ private List<ActionWrapper> randomActions() {
             WebhookAction action = new WebhookAction(httpRequest);
             list.add(new ActionWrapper("_webhook_" + randomAlphaOfLength(8), randomThrottler(),
                     AlwaysConditionTests.randomCondition(scriptService), randomTransform(),
-                    new ExecutableWebhookAction(action, logger, httpClient, templateEngine)));
+                    new ExecutableWebhookAction(action, logger, httpClient, templateEngine), null));
         }
         return list;
     }
diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle
index 72b62f94b5542..c9c444ee9f115 100644
--- a/x-pack/qa/core-rest-tests-with-security/build.gradle
+++ b/x-pack/qa/core-rest-tests-with-security/build.gradle
@@ -1,3 +1,4 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -7,39 +8,27 @@ dependencies {
 
 integTest {
   includePackaged = true
-}
-
-integTestRunner {
-  systemProperty 'tests.rest.blacklist',
-    [
-     'index/10_with_id/Index with ID',
-     'indices.get_alias/10_basic/Get alias against closed indices'
-    ].join(',')
+  runner {
+    systemProperty 'tests.rest.blacklist',
+            [
+                    'index/10_with_id/Index with ID',
+                    'indices.get_alias/10_basic/Get alias against closed indices'
+            ].join(',')
 
-  systemProperty 'tests.rest.cluster.username', System.getProperty('tests.rest.cluster.username', 'test_user')
-  systemProperty 'tests.rest.cluster.password', System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')
+    systemProperty 'tests.rest.cluster.username', System.getProperty('tests.rest.cluster.username', 'test_user')
+    systemProperty 'tests.rest.cluster.password', System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')
+  }
 }
 
-integTestCluster {
+testClusters.integTest {
+  distribution = "DEFAULT"
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.watcher.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
-  setupCommand 'setupDummyUser',
-               'bin/elasticsearch-users',
-               'useradd', System.getProperty('tests.rest.cluster.username', 'test_user'),
-               '-p', System.getProperty('tests.rest.cluster.password', 'x-pack-test-password'),
-               '-r', 'superuser'
-  waitCondition = { node, ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-            dest: tmpFile.toString(),
-            username: 'test_user',
-            password: 'x-pack-test-password',
-            ignoreerrors: true,
-            retries: 10)
-    return tmpFile.exists()
-  }
+
+  user username: System.getProperty('tests.rest.cluster.username', 'test_user'),
+          password:  System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')
 }
diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
index 3ac0b20f95d0f..aa0788bd2b426 100644
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java
@@ -277,7 +277,6 @@ private String loadWatch(String watch) throws IOException {
         return StreamsUtils.copyToStringFromClasspath("/org/elasticsearch/xpack/restart/" + watch);
     }
 
-    @SuppressWarnings("unchecked")
     private void assertOldTemplatesAreDeleted() throws IOException {
         Map<String, Object> templates = entityAsMap(client().performRequest(new Request("GET", "/_template")));
         assertThat(templates.keySet(), not(hasItems(is("watches"), startsWith("watch-history"), is("triggered_watches"))));
diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java
index 3c849811f5620..14f5d66a0fff2 100644
--- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java
+++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlMigrationFullClusterRestartIT.java
@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.restart;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
@@ -30,12 +29,10 @@
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
 import java.util.Base64;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.isEmptyOrNullString;
@@ -128,10 +125,6 @@ private void oldClusterTests() throws IOException {
     }
 
     private void upgradedClusterTests() throws Exception {
-        // wait for the closed and open jobs and datafeed to be migrated
-        waitForMigration(Arrays.asList(OLD_CLUSTER_CLOSED_JOB_ID, OLD_CLUSTER_OPEN_JOB_ID),
-                Arrays.asList(OLD_CLUSTER_STOPPED_DATAFEED_ID, OLD_CLUSTER_STARTED_DATAFEED_ID));
-
         waitForJobToBeAssigned(OLD_CLUSTER_OPEN_JOB_ID);
         waitForDatafeedToBeAssigned(OLD_CLUSTER_STARTED_DATAFEED_ID);
         // The persistent task params for the job & datafeed left open
@@ -181,40 +174,6 @@ private void waitForDatafeedToBeAssigned(String datafeedId) throws Exception {
         }, 30, TimeUnit.SECONDS);
     }
 
-    @SuppressWarnings("unchecked")
-    private void waitForMigration(List<String> expectedMigratedJobs, List<String> expectedMigratedDatafeeds) throws Exception {
-
-        // After v6.6.0 jobs are created in the index so no migration will take place
-        if (getOldClusterVersion().onOrAfter(Version.V_6_6_0)) {
-            return;
-        }
-
-        assertBusy(() -> {
-            // wait for the eligible configs to be moved from the clusterstate
-            Request getClusterState = new Request("GET", "/_cluster/state/metadata");
-            Response response = client().performRequest(getClusterState);
-            Map<String, Object> responseMap = entityAsMap(response);
-
-            List<Map<String, Object>> jobs =
-                    (List<Map<String, Object>>) XContentMapValues.extractValue("metadata.ml.jobs", responseMap);
-
-            if (jobs != null) {
-                for (String jobId : expectedMigratedJobs) {
-                    assertJobNotPresent(jobId, jobs);
-                }
-            }
-
-            List<Map<String, Object>> datafeeds =
-                    (List<Map<String, Object>>) XContentMapValues.extractValue("metadata.ml.datafeeds", responseMap);
-
-            if (datafeeds != null) {
-                for (String datafeedId : expectedMigratedDatafeeds) {
-                    assertDatafeedNotPresent(datafeedId, datafeeds);
-                }
-            }
-        }, 30, TimeUnit.SECONDS);
-    }
-
     @SuppressWarnings("unchecked")
     private void checkTaskParamsAreUpdated(String jobId, String datafeedId) throws Exception {
         Request getClusterState = new Request("GET", "/_cluster/state/metadata");
@@ -240,18 +199,6 @@ else if (id.equals(MlTasks.datafeedTaskId(datafeedId))) {
         }
     }
 
-    private void assertDatafeedNotPresent(String datafeedId, List<Map<String, Object>> datafeeds) {
-        Optional<Object> config = datafeeds.stream().map(map -> map.get("datafeed_id"))
-                .filter(id -> id.equals(datafeedId)).findFirst();
-        assertFalse(config.isPresent());
-    }
-
-    private void assertJobNotPresent(String jobId, List<Map<String, Object>> jobs) {
-        Optional<Object> config = jobs.stream().map(map -> map.get("job_id"))
-                .filter(id -> id.equals(jobId)).findFirst();
-        assertFalse(config.isPresent());
-    }
-
     private void addAggregations(DatafeedConfig.Builder dfBuilder) {
         TermsAggregationBuilder airline = AggregationBuilders.terms("airline");
         MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time").subAggregation(airline);
diff --git a/x-pack/qa/kerberos-tests/build.gradle b/x-pack/qa/kerberos-tests/build.gradle
index e4c261b4c5d57..3095bb6364ffa 100644
--- a/x-pack/qa/kerberos-tests/build.gradle
+++ b/x-pack/qa/kerberos-tests/build.gradle
@@ -2,17 +2,15 @@ import java.nio.file.Path
 import java.nio.file.Paths
 import java.nio.file.Files
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 
 testFixtures.useFixture ":test:fixtures:krb5kdc-fixture"
 
-// https://github.com/elastic/elasticsearch/issues/40624
-integTest.enabled = false
-
 dependencies {
-    testCompile "org.elasticsearch.plugin:x-pack-core:${version}"
+    testCompile project(':x-pack:plugin:core')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     if (isEclipse) {
       testCompile project(path: xpackModule('core-tests'), configuration: 'testArtifacts')
@@ -20,7 +18,8 @@ dependencies {
     testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
 }
 
-integTestCluster {
+testClusters.integTest {
+    distribution = 'DEFAULT'
     // force localhost IPv4 otherwise it is a chicken and egg problem where we need the keytab for the hostname when starting the cluster
     // but do not know the exact address that is first in the http ports file
     setting 'http.host', '127.0.0.1'
@@ -29,51 +28,40 @@ integTestCluster {
     setting 'xpack.security.authc.realms.file.file1.order', '0'
     setting 'xpack.ml.enabled', 'false'
     setting 'xpack.security.audit.enabled', 'true'
+    setting 'xpack.security.authc.token.enabled', 'true'
     // Kerberos realm
     setting 'xpack.security.authc.realms.kerberos.kerberos.order', '1'
     setting 'xpack.security.authc.realms.kerberos.kerberos.keytab.path', 'es.keytab'
     setting 'xpack.security.authc.realms.kerberos.kerberos.krb.debug', 'true'
     setting 'xpack.security.authc.realms.kerberos.kerberos.remove_realm_name', 'false'
 
-    jvmArgs += " -Djava.security.krb5.conf=${project(':test:fixtures:krb5kdc-fixture').ext.krb5Conf("peppa")}"
-    jvmArgs += " -Dsun.security.krb5.debug=true"
 
-    extraConfigFile("es.keytab", project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("peppa", "HTTP_localhost.keytab"))
+    systemProperty "java.security.krb5.conf", { project(':test:fixtures:krb5kdc-fixture').ext.krb5Conf("peppa").toString() }
+    systemProperty "sun.security.krb5.debug", "true"
 
-    setupCommand 'setupTestAdmin',
-            'bin/elasticsearch-users', 'useradd', "test_admin", '-p', 'x-pack-test-password', '-r', "superuser"
+    extraConfigFile "es.keytab", project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("peppa", "HTTP_localhost.keytab")
 
-    waitCondition = { node, ant ->
-        File tmpFile = new File(node.cwd, 'wait.success')
-        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-        dest: tmpFile.toString(),
-        username: 'test_admin',
-        password: 'x-pack-test-password',
-        ignoreerrors: true,
-        retries: 10)
-        return tmpFile.exists()
-    }
+    user username: "test_admin", password: "x-pack-test-password"
+    user username: "test_kibana_user", password: "x-pack-test-password", role: "kibana_system"
+}
 
+task copyKeytabToGeneratedResources(type: Copy) {
+    from project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("peppa", "peppa.keytab")
+    into "$buildDir/generated-resources/keytabs"
+    dependsOn project(':test:fixtures:krb5kdc-fixture').postProcessFixture
 }
 
 String realm = "BUILD.ELASTIC.CO"
-integTestRunner {
+integTest.runner {
     Path peppaKeytab = Paths.get("${project.buildDir}", "generated-resources", "keytabs", "peppa.keytab")
-    systemProperty 'test.userkt', "peppa@${realm}"
-    systemProperty 'test.userkt.keytab', "${peppaKeytab}"
-    systemProperty 'test.userpwd', "george@${realm}"
+    nonInputProperties.systemProperty 'test.userkt', "peppa@${realm}"
+    nonInputProperties.systemProperty 'test.userkt.keytab', "${peppaKeytab}"
+    nonInputProperties.systemProperty 'test.userpwd', "george@${realm}"
     systemProperty 'test.userpwd.password', "dino"
     systemProperty 'tests.security.manager', 'true'
     jvmArgs([
             "-Djava.security.krb5.conf=${project(':test:fixtures:krb5kdc-fixture').ext.krb5Conf("peppa")}",
             "-Dsun.security.krb5.debug=true"
     ])
+    classpath += copyKeytabToGeneratedResources.outputs.files
 }
-
-def generatedResources = "$buildDir/generated-resources/keytabs"
-task copyKeytabToGeneratedResources(type: Copy) {
-    from project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("peppa", "peppa.keytab")
-    into generatedResources
-    dependsOn project(':test:fixtures:krb5kdc-fixture').postProcessFixture
-}
-project.sourceSets.test.output.dir(generatedResources, builtBy:copyKeytabToGeneratedResources)
diff --git a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
index 0281f38933b03..8363705d00cc1 100644
--- a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
+++ b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java
@@ -21,9 +21,9 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.rest.ESRestTestCase;
+import org.ietf.jgss.GSSException;
 import org.junit.Before;
 
-import javax.security.auth.login.LoginContext;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
@@ -34,11 +34,16 @@
 import java.util.List;
 import java.util.Map;
 
+import javax.security.auth.login.LoginContext;
+
 import static org.elasticsearch.common.xcontent.XContentHelper.convertToMap;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
 /**
  * Integration test to demonstrate authentication against a real MIT Kerberos
@@ -102,6 +107,37 @@ public void testLoginByUsernamePassword() throws IOException, PrivilegedActionEx
         executeRequestAndVerifyResponse(userPrincipalName, callbackHandler);
     }
 
+    public void testGetOauth2TokenInExchangeForKerberosTickets() throws PrivilegedActionException, GSSException, IOException {
+        final String userPrincipalName = System.getProperty(TEST_USER_WITH_PWD_KEY);
+        final String password = System.getProperty(TEST_USER_WITH_PWD_PASSWD_KEY);
+        final boolean enabledDebugLogs = Boolean.parseBoolean(System.getProperty(ENABLE_KERBEROS_DEBUG_LOGS_KEY));
+        final SpnegoHttpClientConfigCallbackHandler callbackHandler = new SpnegoHttpClientConfigCallbackHandler(userPrincipalName,
+                new SecureString(password.toCharArray()), enabledDebugLogs);
+        final String host = getClusterHosts().get(0).getHostName();
+        final String kerberosTicket = callbackHandler.getBase64EncodedTokenForSpnegoHeader(host);
+
+        final Request request = new Request("POST", "/_security/oauth2/token");
+        String json = "{" +
+                      "  \"grant_type\" : \"_kerberos\", " +
+                      "  \"kerberos_ticket\" : \"" + kerberosTicket  + "\"" +
+                      "}";
+        request.setJsonEntity(json);
+
+        try (RestClient client = buildClientForUser("test_kibana_user")) {
+            final Response response = client.performRequest(request);
+            assertOK(response);
+            final Map<String, Object> map = parseResponseAsMap(response.getEntity());
+            assertThat(map.get("access_token"), notNullValue());
+            assertThat(map.get("type"), is("Bearer"));
+            assertThat(map.get("refresh_token"), notNullValue());
+            final Object base64OutToken = map.get("kerberos_authentication_response_token");
+            assertThat(base64OutToken, notNullValue());
+            final String outToken = callbackHandler.handleResponse((String) base64OutToken);
+            assertThat(outToken, is(nullValue()));
+            assertThat(callbackHandler.isEstablished(), is(true));
+        }
+    }
+
     @Override
     @SuppressForbidden(reason = "SPNEGO relies on hostnames and we need to ensure host isn't a IP address")
     protected HttpHost buildHttpHost(String host, int port) {
@@ -137,6 +173,13 @@ private Map<String, Object> parseResponseAsMap(final HttpEntity entity) throws I
         return convertToMap(XContentType.JSON.xContent(), entity.getContent(), false);
     }
 
+    private RestClient buildClientForUser(String user) throws IOException {
+        final String token = basicAuthHeaderValue(user, new SecureString("x-pack-test-password".toCharArray()));
+        Settings settings = Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+        final HttpHost[] hosts = getClusterHosts().toArray(new HttpHost[getClusterHosts().size()]);
+        return buildClient(settings, hosts);
+    }
+
     private RestClient buildRestClientForKerberos(final SpnegoHttpClientConfigCallbackHandler callbackHandler) throws IOException {
         final Settings settings = restAdminSettings();
         final HttpHost[] hosts = getClusterHosts().toArray(new HttpHost[getClusterHosts().size()]);
diff --git a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoHttpClientConfigCallbackHandler.java b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoHttpClientConfigCallbackHandler.java
index e5768d8f2e944..4694340047467 100644
--- a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoHttpClientConfigCallbackHandler.java
+++ b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SpnegoHttpClientConfigCallbackHandler.java
@@ -19,6 +19,7 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback;
 import org.elasticsearch.common.settings.SecureString;
+import org.ietf.jgss.GSSContext;
 import org.ietf.jgss.GSSCredential;
 import org.ietf.jgss.GSSException;
 import org.ietf.jgss.GSSManager;
@@ -28,11 +29,14 @@
 import java.io.IOException;
 import java.security.AccessControlContext;
 import java.security.AccessController;
+import java.security.Principal;
 import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 
 import javax.security.auth.Subject;
 import javax.security.auth.callback.Callback;
@@ -43,6 +47,7 @@
 import javax.security.auth.login.AppConfigurationEntry;
 import javax.security.auth.login.Configuration;
 import javax.security.auth.login.LoginContext;
+import javax.security.auth.login.LoginException;
 
 /**
  * This class implements {@link HttpClientConfigCallback} which allows for
@@ -72,6 +77,7 @@ private static Oid getSpnegoOid() {
     private final String keytabPath;
     private final boolean enableDebugLogs;
     private LoginContext loginContext;
+    private GSSContext gssContext;
 
     /**
      * Constructs {@link SpnegoHttpClientConfigCallbackHandler} with given
@@ -314,4 +320,74 @@ public AppConfigurationEntry[] getAppConfigurationEntry(final String name) {
 
         abstract void addOptions(Map<String, String> options);
     }
+
+    /**
+     * Initiates GSS context establishment and returns the
+     * base64 encoded token to be sent to server.
+     *
+     * @return Base64 encoded token
+     * @throws PrivilegedActionException when privileged action threw exception
+     * @throws GSSException when GSS context creation fails
+     */
+    String getBase64EncodedTokenForSpnegoHeader(final String serviceHost) throws PrivilegedActionException, GSSException {
+        final GSSManager gssManager = GSSManager.getInstance();
+        final GSSName gssServicePrincipalName = AccessController
+                .doPrivileged((PrivilegedExceptionAction<GSSName>) () -> gssManager.createName("HTTP/" + serviceHost, null));
+        final GSSName gssUserPrincipalName = gssManager.createName(userPrincipalName, GSSName.NT_USER_NAME);
+        loginContext = AccessController
+                .doPrivileged((PrivilegedExceptionAction<LoginContext>) () -> loginUsingPassword(userPrincipalName, password));
+        final GSSCredential userCreds = doAsWrapper(loginContext.getSubject(), (PrivilegedExceptionAction<GSSCredential>) () -> gssManager
+                .createCredential(gssUserPrincipalName, GSSCredential.DEFAULT_LIFETIME, SPNEGO_OID, GSSCredential.INITIATE_ONLY));
+        gssContext = gssManager.createContext(gssServicePrincipalName, SPNEGO_OID, userCreds, GSSCredential.DEFAULT_LIFETIME);
+        gssContext.requestMutualAuth(true);
+
+        final byte[] outToken = doAsWrapper(loginContext.getSubject(),
+                (PrivilegedExceptionAction<byte[]>) () -> gssContext.initSecContext(new byte[0], 0, 0));
+        return Base64.getEncoder().encodeToString(outToken);
+    }
+
+    private LoginContext loginUsingPassword(final String principal, final SecureString password) throws LoginException {
+        final Set<Principal> principals = Collections.singleton(new KerberosPrincipal(principal));
+
+        final Subject subject = new Subject(false, principals, Collections.emptySet(), Collections.emptySet());
+
+        final Configuration conf = new PasswordJaasConf(principal, enableDebugLogs);
+        final CallbackHandler callback = new KrbCallbackHandler(principal, password);
+        final LoginContext loginContext = new LoginContext(CRED_CONF_NAME, subject, callback, conf);
+        loginContext.login();
+        return loginContext;
+    }
+
+    /**
+     * Handles server response and returns new token if any to be sent to server.
+     *
+     * @param base64Token inToken received from server passed to initSecContext for
+     *            gss negotiation
+     * @return Base64 encoded token to be sent to server. May return {@code null} if
+     *         nothing to be sent.
+     * @throws PrivilegedActionException when privileged action threw exception
+     */
+    String handleResponse(final String base64Token) throws PrivilegedActionException {
+        if (gssContext.isEstablished()) {
+            throw new IllegalStateException("GSS Context has already been established");
+        }
+        final byte[] token = Base64.getDecoder().decode(base64Token);
+        final byte[] outToken = doAsWrapper(loginContext.getSubject(),
+                (PrivilegedExceptionAction<byte[]>) () -> gssContext.initSecContext(token, 0, token.length));
+        if (outToken == null || outToken.length == 0) {
+            return null;
+        }
+        return Base64.getEncoder().encodeToString(outToken);
+    }
+
+    /**
+     * @return {@code true} If the gss security context was established
+     */
+    boolean isEstablished() {
+        return gssContext.isEstablished();
+    }
+
+    static <T> T doAsWrapper(final Subject subject, final PrivilegedExceptionAction<T> action) throws PrivilegedActionException {
+        return AccessController.doPrivileged((PrivilegedExceptionAction<T>) () -> Subject.doAs(subject, action));
+    }
 }
diff --git a/x-pack/qa/multi-cluster-search-security/build.gradle b/x-pack/qa/multi-cluster-search-security/build.gradle
index 8406345575dfd..4efcbdae8836b 100644
--- a/x-pack/qa/multi-cluster-search-security/build.gradle
+++ b/x-pack/qa/multi-cluster-search-security/build.gradle
@@ -1,76 +1,59 @@
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
   testCompile project(':x-pack:qa')
 }
 
-task remoteClusterTest(type: RestIntegTestTask) {
+task 'remote-cluster'(type: RestIntegTestTask) {
   mustRunAfter(precommit)
+  runner {
+    systemProperty 'tests.rest.suite', 'remote_cluster'
+  }
 }
 
-remoteClusterTestCluster {
-  numNodes = 2
-  clusterName = 'remote-cluster'
-  setting 'cluster.remote.connect', false
+testClusters.'remote-cluster' {
+  distribution = "DEFAULT"
+  numberOfNodes = 2
+  setting 'cluster.remote.connect', "false"
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.watcher.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
-  setupCommand 'setupDummyUser',
-          'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
-  waitCondition = { node, ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-            dest: tmpFile.toString(),
-            username: 'test_user',
-            password: 'x-pack-test-password',
-            ignoreerrors: true,
-            retries: 10)
-    return tmpFile.exists()
-  }
-}
 
-remoteClusterTestRunner {
-  systemProperty 'tests.rest.suite', 'remote_cluster'
+  user username: "test_user", password: "x-pack-test-password"
 }
 
-task mixedClusterTest(type: RestIntegTestTask) {}
+task 'mixed-cluster'(type: RestIntegTestTask) {
+  dependsOn 'remote-cluster'
+  runner {
+    useCluster testClusters.'remote-cluster'
+    systemProperty 'tests.rest.suite', 'multi_cluster'
+  }
+}
 
-mixedClusterTestCluster {
-  dependsOn remoteClusterTestRunner
+testClusters.'mixed-cluster' {
+  distribution = "DEFAULT"
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.watcher.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
-  setupCommand 'setupDummyUser',
-          'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
-  waitCondition = { node, ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-            dest: tmpFile.toString(),
-            username: 'test_user',
-            password: 'x-pack-test-password',
-            ignoreerrors: true,
-            retries: 10)
-    return tmpFile.exists()
+  setting 'cluster.remote.my_remote_cluster.seeds', {
+    testClusters.'remote-cluster'.getAllTransportPortURI().collect { "\"$it\"" }.toString()
   }
-  setting 'cluster.remote.my_remote_cluster.seeds', "\"${-> remoteClusterTest.nodes.get(0).transportUri()}\""
-  setting 'cluster.remote.connections_per_cluster', 1
-  setting 'cluster.remote.connect', true
-}
+  setting 'cluster.remote.connections_per_cluster', "1"
+  setting 'cluster.remote.connect', "true"
 
-mixedClusterTestRunner {
-  systemProperty 'tests.rest.suite', 'multi_cluster'
-  finalizedBy 'remoteClusterTestCluster#node0.stop','remoteClusterTestCluster#node1.stop'
+  user username: "test_user", password: "x-pack-test-password"
 }
 
 task integTest {
-  dependsOn = [mixedClusterTest]
+  dependsOn 'mixed-cluster'
 }
 
 test.enabled = false // no unit tests for multi-cluster-search, only the rest integration test
diff --git a/x-pack/qa/multi-node/build.gradle b/x-pack/qa/multi-node/build.gradle
index 28de4a4c114a9..5bf248482aa7f 100644
--- a/x-pack/qa/multi-node/build.gradle
+++ b/x-pack/qa/multi-node/build.gradle
@@ -1,3 +1,4 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -5,26 +6,16 @@ dependencies {
   testCompile project(':x-pack:qa')
 }
 
-integTestCluster {
-  numNodes = 2
-  clusterName = 'multi-node'
+testClusters.integTest {
+  distribution = 'DEFAULT'
+  numberOfNodes = 2
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.watcher.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
-  extraConfigFile 'roles.yml', 'roles.yml'
-  setupCommand 'setup-test-user', 'bin/elasticsearch-users', 'useradd', 'test-user', '-p', 'x-pack-test-password', '-r', 'test'
-  setupCommand 'setup-super-user', 'bin/elasticsearch-users', 'useradd', 'super-user', '-p', 'x-pack-super-password', '-r', 'superuser'
-  waitCondition = { node, ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-            dest: tmpFile.toString(),
-            username: 'super-user',
-            password: 'x-pack-super-password',
-            ignoreerrors: true,
-            retries: 10)
-    return tmpFile.exists()
-  }
+  extraConfigFile 'roles.yml', file('roles.yml')
+  user username: "test-user", password: "x-pack-test-password", role: "test"
+  user username: "super-user", password: "x-pack-super-password"
 }
diff --git a/x-pack/qa/oidc-op-tests/build.gradle b/x-pack/qa/oidc-op-tests/build.gradle
index 34325bc69b624..32a77d2b98cb3 100644
--- a/x-pack/qa/oidc-op-tests/build.gradle
+++ b/x-pack/qa/oidc-op-tests/build.gradle
@@ -1,11 +1,11 @@
 Project idpFixtureProject = xpackProject("test:idp-fixture")
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 
 dependencies {
-    // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
     testCompile project(path: xpackModule('core'), configuration: 'default')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
     if (isEclipse) {
@@ -25,8 +25,12 @@ task setupPorts {
     }
 }
 
-integTestCluster {
-    dependsOn setupPorts
+integTest.runner {
+  dependsOn setupPorts
+}
+
+testClusters.integTest {
+    distribution = 'DEFAULT'
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.security.http.ssl.enabled', 'false'
@@ -36,13 +40,13 @@ integTestCluster {
     // OpenID Connect Realm 1 configured for authorization grant flow
     setting 'xpack.security.authc.realms.oidc.c2id.order', '2'
     setting 'xpack.security.authc.realms.oidc.c2id.op.issuer', 'http://localhost:8080'
-    setting 'xpack.security.authc.realms.oidc.c2id.op.authorization_endpoint', "http://127.0.0.1:${-> ephemeralPort}/c2id-login"
-    setting 'xpack.security.authc.realms.oidc.c2id.op.token_endpoint', "http://127.0.0.1:${-> ephemeralPort}/c2id/token"
-    setting 'xpack.security.authc.realms.oidc.c2id.op.userinfo_endpoint', "http://127.0.0.1:${-> ephemeralPort}/c2id/userinfo"
+    setting 'xpack.security.authc.realms.oidc.c2id.op.authorization_endpoint', { "http://127.0.0.1:${ephemeralPort}/c2id-login" }
+    setting 'xpack.security.authc.realms.oidc.c2id.op.token_endpoint', { "http://127.0.0.1:${ephemeralPort}/c2id/token" }
+    setting 'xpack.security.authc.realms.oidc.c2id.op.userinfo_endpoint', { "http://127.0.0.1:${ephemeralPort}/c2id/userinfo" }
     setting 'xpack.security.authc.realms.oidc.c2id.op.jwkset_path', 'op-jwks.json'
     setting 'xpack.security.authc.realms.oidc.c2id.rp.redirect_uri', 'https://my.fantastic.rp/cb'
-    setting 'xpack.security.authc.realms.oidc.c2id.rp.client_id', 'elasticsearch-rp'
-    keystoreSetting 'xpack.security.authc.realms.oidc.c2id.rp.client_secret', 'b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2'
+    setting 'xpack.security.authc.realms.oidc.c2id.rp.client_id', 'https://my.elasticsearch.org/rp'
+    keystore 'xpack.security.authc.realms.oidc.c2id.rp.client_secret', 'b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2'
     setting 'xpack.security.authc.realms.oidc.c2id.rp.response_type', 'code'
     setting 'xpack.security.authc.realms.oidc.c2id.claims.principal', 'sub'
     setting 'xpack.security.authc.realms.oidc.c2id.claims.name', 'name'
@@ -51,13 +55,13 @@ integTestCluster {
     // OpenID Connect Realm 2 configured for implicit flow
     setting 'xpack.security.authc.realms.oidc.c2id-implicit.order', '3'
     setting 'xpack.security.authc.realms.oidc.c2id-implicit.op.issuer', 'http://localhost:8080'
-    setting 'xpack.security.authc.realms.oidc.c2id-implicit.op.authorization_endpoint', "http://127.0.0.1:${-> ephemeralPort}/c2id-login"
-    setting 'xpack.security.authc.realms.oidc.c2id-implicit.op.token_endpoint', "http://127.0.0.1:${-> ephemeralPort}/c2id/token"
-    setting 'xpack.security.authc.realms.oidc.c2id-implicit.op.userinfo_endpoint', "http://127.0.0.1:${-> ephemeralPort}/c2id/userinfo"
+    setting 'xpack.security.authc.realms.oidc.c2id-implicit.op.authorization_endpoint', { "http://127.0.0.1:${ephemeralPort}/c2id-login" }
+    setting 'xpack.security.authc.realms.oidc.c2id-implicit.op.token_endpoint', { "http://127.0.0.1:${ephemeralPort}/c2id/token" }
+    setting 'xpack.security.authc.realms.oidc.c2id-implicit.op.userinfo_endpoint', { "http://127.0.0.1:${ephemeralPort}/c2id/userinfo" }
     setting 'xpack.security.authc.realms.oidc.c2id-implicit.op.jwkset_path', 'op-jwks.json'
     setting 'xpack.security.authc.realms.oidc.c2id-implicit.rp.redirect_uri', 'https://my.fantastic.rp/cb'
     setting 'xpack.security.authc.realms.oidc.c2id-implicit.rp.client_id', 'elasticsearch-rp'
-    keystoreSetting 'xpack.security.authc.realms.oidc.c2id-implicit.rp.client_secret', 'b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2'
+    keystore 'xpack.security.authc.realms.oidc.c2id-implicit.rp.client_secret', 'b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2'
     setting 'xpack.security.authc.realms.oidc.c2id-implicit.rp.response_type', 'id_token token'
     setting 'xpack.security.authc.realms.oidc.c2id-implicit.claims.principal', 'sub'
     setting 'xpack.security.authc.realms.oidc.c2id-implicit.claims.name', 'name'
@@ -67,19 +71,7 @@ integTestCluster {
 
     extraConfigFile 'op-jwks.json', idpFixtureProject.file("oidc/op-jwks.json")
 
-    setupCommand 'setupTestAdmin',
-            'bin/elasticsearch-users', 'useradd', "test_admin", '-p', 'x-pack-test-password', '-r', "superuser"
-
-    waitCondition = { node, ant ->
-        File tmpFile = new File(node.cwd, 'wait.success')
-        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-                dest: tmpFile.toString(),
-                username: 'test_admin',
-                password: 'x-pack-test-password',
-                ignoreerrors: true,
-                retries: 10)
-        return tmpFile.exists()
-    }
+    user username: "test_admin", password: "x-pack-test-password"
 }
 
 thirdPartyAudit.enabled = false
diff --git a/x-pack/qa/oidc-op-tests/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java b/x-pack/qa/oidc-op-tests/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java
index 7835b236ed84d..3022b34c4aeec 100644
--- a/x-pack/qa/oidc-op-tests/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java
+++ b/x-pack/qa/oidc-op-tests/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java
@@ -76,23 +76,38 @@ public void setupUserAndRoles() throws IOException {
      * C2id server only supports dynamic registration, so we can't pre-seed it's config with our client data. Execute only once
      */
     @BeforeClass
-    public static void registerClient() throws Exception {
+    public static void registerClients() throws Exception {
         try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
-            HttpPost httpPost = new HttpPost(REGISTRATION_URL);
-            final BasicHttpContext context = new BasicHttpContext();
-            String json = "{" +
-                "\"grant_types\": [\"implicit\", \"authorization_code\"]," +
-                "\"response_types\": [\"code\", \"token id_token\"]," +
+            String codeClient = "{" +
+                "\"grant_types\": [\"authorization_code\"]," +
+                "\"response_types\": [\"code\"]," +
+                "\"preferred_client_id\":\"https://my.elasticsearch.org/rp\"," +
+                "\"preferred_client_secret\":\"b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2\"," +
+                "\"redirect_uris\": [\"https://my.fantastic.rp/cb\"]" +
+                "}";
+            String implicitClient = "{" +
+                "\"grant_types\": [\"implicit\"]," +
+                "\"response_types\": [\"token id_token\"]," +
                 "\"preferred_client_id\":\"elasticsearch-rp\"," +
                 "\"preferred_client_secret\":\"b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2\"," +
                 "\"redirect_uris\": [\"https://my.fantastic.rp/cb\"]" +
                 "}";
-            httpPost.setEntity(new StringEntity(json, ContentType.APPLICATION_JSON));
+            HttpPost httpPost = new HttpPost(REGISTRATION_URL);
+            final BasicHttpContext context = new BasicHttpContext();
+            httpPost.setEntity(new StringEntity(codeClient, ContentType.APPLICATION_JSON));
             httpPost.setHeader("Accept", "application/json");
             httpPost.setHeader("Content-type", "application/json");
             httpPost.setHeader("Authorization", "Bearer 811fa888f3e0fdc9e01d4201bfeee46a");
             CloseableHttpResponse response = SocketAccess.doPrivileged(() -> httpClient.execute(httpPost, context));
             assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+            httpPost.setEntity(new StringEntity(implicitClient, ContentType.APPLICATION_JSON));
+            HttpPost httpPost2 = new HttpPost(REGISTRATION_URL);
+            httpPost2.setEntity(new StringEntity(implicitClient, ContentType.APPLICATION_JSON));
+            httpPost2.setHeader("Accept", "application/json");
+            httpPost2.setHeader("Content-type", "application/json");
+            httpPost2.setHeader("Authorization", "Bearer 811fa888f3e0fdc9e01d4201bfeee46a");
+            CloseableHttpResponse response2 = SocketAccess.doPrivileged(() -> httpClient.execute(httpPost2, context));
+            assertThat(response2.getStatusLine().getStatusCode(), equalTo(200));
         }
     }
 
diff --git a/x-pack/qa/openldap-tests/build.gradle b/x-pack/qa/openldap-tests/build.gradle
index f732d8fc5b030..0f1d728e9b8aa 100644
--- a/x-pack/qa/openldap-tests/build.gradle
+++ b/x-pack/qa/openldap-tests/build.gradle
@@ -2,7 +2,6 @@ apply plugin: 'elasticsearch.standalone-test'
 apply plugin: 'elasticsearch.test.fixtures'
 
 dependencies {
-    // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
     testCompile project(path: xpackModule('core'), configuration: 'default')
     testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
diff --git a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java
index b899d28273d5a..2f732110eecbe 100644
--- a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java
+++ b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java
@@ -15,7 +15,6 @@
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.OpenLdapTests;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
@@ -43,7 +42,6 @@
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
 
-@TestLogging("org.elasticsearch.xpack.core.ssl.SSLService:TRACE")
 public class OpenLdapUserSearchSessionFactoryTests extends ESTestCase {
 
     private Settings globalSettings;
diff --git a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java
index 036cf8ad0db33..f24bcface06bb 100644
--- a/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java
+++ b/x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java
@@ -23,7 +23,6 @@
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
-@SuppressWarnings("unchecked")
 public class SearchGroupsResolverTests extends GroupsResolverTestCase {
 
     private static final String BRUCE_BANNER_DN = "uid=hulk,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com";
diff --git a/x-pack/qa/reindex-tests-with-security/build.gradle b/x-pack/qa/reindex-tests-with-security/build.gradle
index 7f878e6356b73..1cedd1f5bf6c6 100644
--- a/x-pack/qa/reindex-tests-with-security/build.gradle
+++ b/x-pack/qa/reindex-tests-with-security/build.gradle
@@ -1,10 +1,8 @@
-import org.elasticsearch.gradle.http.WaitForHttpResource
-
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
   testCompile project(path: xpackModule('core'), configuration: 'default')
   testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
@@ -23,10 +21,11 @@ forbiddenPatterns {
 
 File caFile = project.file('src/test/resources/ssl/ca.p12')
 
-integTestCluster {
+testClusters.integTest {
+  distribution = "DEFAULT"
   // Whitelist reindexing from the local node so we can test it.
-  extraConfigFile 'http.key', project.projectDir.toPath().resolve('src/test/resources/ssl/http.key')
-  extraConfigFile 'http.crt', project.projectDir.toPath().resolve('src/test/resources/ssl/http.crt')
+  extraConfigFile 'http.key', file('src/test/resources/ssl/http.key')
+  extraConfigFile 'http.crt', file('src/test/resources/ssl/http.crt')
   extraConfigFile 'ca.p12', caFile
   setting 'reindex.remote.whitelist', '127.0.0.1:*'
   setting 'xpack.ilm.enabled', 'false'
@@ -47,26 +46,13 @@ integTestCluster {
     }
   }
 
-  extraConfigFile 'roles.yml', 'roles.yml'
-  [
-    test_admin: 'superuser',
-    powerful_user: 'superuser',
-    minimal_user: 'minimal',
-    minimal_with_task_user: 'minimal_with_task',
-    readonly_user: 'readonly',
-    dest_only_user: 'dest_only',
-    can_not_see_hidden_docs_user: 'can_not_see_hidden_docs',
-    can_not_see_hidden_fields_user: 'can_not_see_hidden_fields',
-  ].each { String user, String role ->
-    setupCommand 'setupUser#' + user,
-                 'bin/elasticsearch-users', 'useradd', user, '-p', 'x-pack-test-password', '-r', role
-  }
-  waitCondition = { node, ant ->
-    WaitForHttpResource http = new WaitForHttpResource("https", node.httpUri(), numNodes)
-    http.setTrustStoreFile(caFile)
-    http.setTrustStorePassword("password")
-    http.setUsername("test_admin")
-    http.setPassword("x-pack-test-password")
-    return http.wait(5000)
-  }
+  extraConfigFile 'roles.yml', file('roles.yml')
+  user username: "test_admin", password: 'x-pack-test-password', role: "superuser"
+  user username: "powerful_user", password: 'x-pack-test-password', role: "superuser"
+  user username: "minimal_user", password: 'x-pack-test-password', role: "minimal"
+  user username: "minimal_with_task_user", password: 'x-pack-test-password', role: "minimal_with_task"
+  user username: "readonly_user", password: 'x-pack-test-password', role: "readonly"
+  user username: "dest_only_user", password: 'x-pack-test-password', role: "dest_only"
+  user username: "can_not_see_hidden_docs_user", password: 'x-pack-test-password', role: "can_not_see_hidden_docs"
+  user username: "can_not_see_hidden_fields_user", password: 'x-pack-test-password', role: "can_not_see_hidden_fields"
 }
diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle
index d75ecbd7a55ed..46e9bc10b21ed 100644
--- a/x-pack/qa/rolling-upgrade/build.gradle
+++ b/x-pack/qa/rolling-upgrade/build.gradle
@@ -11,7 +11,7 @@ test.enabled = false
 
 dependencies {
   testCompile project(':x-pack:qa')
-  testCompile ("org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}")
+  testCompile project(':client:rest-high-level')
 }
 
 Closure waitWithAuth = { NodeInfo node, AntBuilder ant ->
@@ -121,7 +121,6 @@ for (Version version : bwcVersions.wireCompatible) {
         setting 'xpack.security.transport.ssl.enabled', 'true'
         setting 'xpack.security.authc.token.enabled', 'true'
         setting 'xpack.security.authc.token.timeout', '60m'
-        setting 'logger.org.elasticsearch.xpack.security.authc.TokenService', 'trace'
         setting 'xpack.security.audit.enabled', 'true'
         rootProject.globalInfo.ready {
             if (project.inFipsJvm) {
@@ -189,7 +188,6 @@ for (Version version : bwcVersions.wireCompatible) {
             setting 'xpack.security.enabled', 'true'
             setting 'xpack.security.transport.ssl.enabled', 'true'
             setting 'xpack.security.authc.token.timeout', '60m'
-            setting 'logger.org.elasticsearch.xpack.security.authc.TokenService', 'trace'
             rootProject.globalInfo.ready {
                 if (project.inFipsJvm) {
                     setting 'xpack.security.transport.ssl.key', 'testnode.pem'
@@ -242,7 +240,9 @@ for (Version version : bwcVersions.wireCompatible) {
                 'mixed_cluster/30_ml_jobs_crud/Create a job in the mixed cluster and write some data',
                 'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed in mixed cluster',
                 'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed without aggs in mixed cluster',
-                'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed with aggs in mixed cluster'
+                'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed with aggs in mixed cluster',
+                'mixed_cluster/80_data_frame_jobs_crud/Test put batch data frame transforms on mixed cluster',
+                'mixed_cluster/80_data_frame_jobs_crud/Test put continuous data frame transform on mixed cluster'
         ].join(',')
         finalizedBy "${baseName}#oldClusterTestCluster#node1.stop"
     }
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataFrameSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataFrameSurvivesUpgradeIT.java
new file mode 100644
index 0000000000000..1aee9130ddc50
--- /dev/null
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataFrameSurvivesUpgradeIT.java
@@ -0,0 +1,279 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.upgrades;
+
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.dataframe.GetDataFrameTransformStatsResponse;
+import org.elasticsearch.client.dataframe.transforms.DataFrameTransformConfig;
+import org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStats;
+import org.elasticsearch.client.dataframe.transforms.DataFrameTransformTaskState;
+import org.elasticsearch.client.dataframe.transforms.DestConfig;
+import org.elasticsearch.client.dataframe.transforms.SourceConfig;
+import org.elasticsearch.client.dataframe.transforms.TimeSyncConfig;
+import org.elasticsearch.client.dataframe.transforms.pivot.GroupConfig;
+import org.elasticsearch.client.dataframe.transforms.pivot.PivotConfig;
+import org.elasticsearch.client.dataframe.transforms.pivot.TermsGroupSource;
+import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.xpack.test.rest.XPackRestTestConstants;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
+
+@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/43662")
+public class DataFrameSurvivesUpgradeIT extends AbstractUpgradeTestCase {
+
+    private static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version"));
+    private static final String DATAFRAME_ENDPOINT = "/_data_frame/transforms/";
+    private static final String CONTINUOUS_DATA_FRAME_ID = "continuous-data-frame-upgrade-job";
+    private static final String CONTINUOUS_DATA_FRAME_SOURCE = "data-frame-upgrade-continuous-source";
+    private static final List<String> ENTITIES = Stream.iterate(1, n -> n + 1)
+        .limit(5)
+        .map(v -> "user_" + v)
+        .collect(Collectors.toList());
+    private static final List<TimeValue> BUCKETS = Stream.iterate(1, n -> n + 1)
+        .limit(5)
+        .map(TimeValue::timeValueMinutes)
+        .collect(Collectors.toList());
+
+    @Override
+    protected Collection<String> templatesToWaitFor() {
+        return Stream.concat(XPackRestTestConstants.DATA_FRAME_TEMPLATES.stream(),
+            super.templatesToWaitFor().stream()).collect(Collectors.toSet());
+    }
+
+    protected static void waitForPendingDataFrameTasks() throws Exception {
+        waitForPendingTasks(adminClient(), taskName -> taskName.startsWith("data_frame/transforms") == false);
+    }
+
+    /**
+     * The purpose of this test is to ensure that when a job is open through a rolling upgrade we upgrade the results
+     * index mappings when it is assigned to an upgraded node even if no other ML endpoint is called after the upgrade
+     */
+    public void testDataFramesRollingUpgrade() throws Exception {
+        assumeTrue("Continuous data frames not supported until 7.3", UPGRADE_FROM_VERSION.onOrAfter(Version.V_7_3_0));
+        Request waitForYellow = new Request("GET", "/_cluster/health");
+        waitForYellow.addParameter("wait_for_nodes", "3");
+        waitForYellow.addParameter("wait_for_status", "yellow");
+        switch (CLUSTER_TYPE) {
+            case OLD:
+                createAndStartContinuousDataFrame();
+                break;
+            case MIXED:
+                client().performRequest(waitForYellow);
+                long lastCheckpoint = 1;
+                if (Booleans.parseBoolean(System.getProperty("tests.first_round")) == false) {
+                    lastCheckpoint = 2;
+                }
+                verifyContinuousDataFrameHandlesData(lastCheckpoint);
+                break;
+            case UPGRADED:
+                client().performRequest(waitForYellow);
+                verifyContinuousDataFrameHandlesData(3);
+                cleanUpTransforms();
+                break;
+            default:
+                throw new UnsupportedOperationException("Unknown cluster type [" + CLUSTER_TYPE + "]");
+        }
+    }
+
+    private void cleanUpTransforms() throws Exception {
+        stopTransform(CONTINUOUS_DATA_FRAME_ID);
+        deleteTransform(CONTINUOUS_DATA_FRAME_ID);
+        waitForPendingDataFrameTasks();
+    }
+
+    private void createAndStartContinuousDataFrame() throws Exception {
+        createIndex(CONTINUOUS_DATA_FRAME_SOURCE);
+        long totalDocsWritten = 0;
+        for (TimeValue bucket : BUCKETS) {
+            int docs = randomIntBetween(1, 25);
+            putData(CONTINUOUS_DATA_FRAME_SOURCE, docs, bucket, ENTITIES);
+            totalDocsWritten += docs * ENTITIES.size();
+        }
+
+        DataFrameTransformConfig config = DataFrameTransformConfig.builder()
+            .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(30)))
+            .setPivotConfig(PivotConfig.builder()
+                .setAggregations(new AggregatorFactories.Builder().addAggregator(AggregationBuilders.avg("stars").field("stars")))
+                .setGroups(GroupConfig.builder().groupBy("user_id", TermsGroupSource.builder().setField("user_id").build()).build())
+                .build())
+            .setDest(DestConfig.builder().setIndex(CONTINUOUS_DATA_FRAME_ID + "_idx").build())
+            .setSource(SourceConfig.builder().setIndex(CONTINUOUS_DATA_FRAME_SOURCE).build())
+            .setId(CONTINUOUS_DATA_FRAME_ID)
+            .build();
+        putTransform(CONTINUOUS_DATA_FRAME_ID, config);
+
+        startTransform(CONTINUOUS_DATA_FRAME_ID);
+        waitUntilAfterCheckpoint(CONTINUOUS_DATA_FRAME_ID, 0L);
+
+        DataFrameTransformStateAndStats stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID);
+
+        assertThat(stateAndStats.getTransformStats().getOutputDocuments(), equalTo((long)ENTITIES.size()));
+        assertThat(stateAndStats.getTransformStats().getNumDocuments(), equalTo(totalDocsWritten));
+        assertThat(stateAndStats.getTransformState().getTaskState(), equalTo(DataFrameTransformTaskState.STARTED));
+    }
+
+    private void verifyContinuousDataFrameHandlesData(long expectedLastCheckpoint) throws Exception {
+
+        // A continuous data frame should automatically become started when it gets assigned to a node
+        // if it was assigned to the node that was removed from the cluster
+        assertBusy(() -> {
+            DataFrameTransformStateAndStats stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID);
+            assertThat(stateAndStats.getTransformState().getTaskState(), equalTo(DataFrameTransformTaskState.STARTED));
+        },
+        120,
+        TimeUnit.SECONDS);
+
+        DataFrameTransformStateAndStats previousStateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID);
+
+        // Add a new user and write data to it
+        // This is so we can have more reliable data counts, as writing to existing entities requires
+        // rescanning the past data
+        List<String> entities = new ArrayList<>(1);
+        entities.add("user_" + ENTITIES.size() + expectedLastCheckpoint);
+        int docs = 5;
+        // Index the data very recently in the past so that the transform sync delay can catch up to reading it in our spin
+        // wait later.
+        putData(CONTINUOUS_DATA_FRAME_SOURCE, docs, TimeValue.timeValueSeconds(1), entities);
+
+        waitUntilAfterCheckpoint(CONTINUOUS_DATA_FRAME_ID, expectedLastCheckpoint);
+
+        assertBusy(() -> assertThat(
+            getTransformStats(CONTINUOUS_DATA_FRAME_ID).getTransformStats().getNumDocuments(),
+            greaterThanOrEqualTo(docs + previousStateAndStats.getTransformStats().getNumDocuments())),
+            120,
+            TimeUnit.SECONDS);
+        DataFrameTransformStateAndStats stateAndStats = getTransformStats(CONTINUOUS_DATA_FRAME_ID);
+
+        assertThat(stateAndStats.getTransformState().getTaskState(),
+            equalTo(DataFrameTransformTaskState.STARTED));
+        assertThat(stateAndStats.getTransformStats().getOutputDocuments(),
+            greaterThan(previousStateAndStats.getTransformStats().getOutputDocuments()));
+        assertThat(stateAndStats.getTransformStats().getNumDocuments(),
+            greaterThanOrEqualTo(docs + previousStateAndStats.getTransformStats().getNumDocuments()));
+    }
+
+    private void putTransform(String id, DataFrameTransformConfig config) throws IOException {
+        final Request createDataframeTransformRequest = new Request("PUT", DATAFRAME_ENDPOINT + id);
+        createDataframeTransformRequest.setJsonEntity(Strings.toString(config));
+        Response response = client().performRequest(createDataframeTransformRequest);
+        assertEquals(200, response.getStatusLine().getStatusCode());
+    }
+
+    private void deleteTransform(String id) throws IOException {
+        Response response = client().performRequest(new Request("DELETE", DATAFRAME_ENDPOINT + id));
+        assertEquals(200, response.getStatusLine().getStatusCode());
+    }
+
+    private void startTransform(String id) throws IOException  {
+        final Request startDataframeTransformRequest = new Request("POST", DATAFRAME_ENDPOINT + id + "/_start");
+        Response response = client().performRequest(startDataframeTransformRequest);
+        assertEquals(200, response.getStatusLine().getStatusCode());
+    }
+
+    private void stopTransform(String id) throws IOException  {
+        final Request stopDataframeTransformRequest = new Request("POST",
+            DATAFRAME_ENDPOINT + id + "/_stop?wait_for_completion=true");
+        Response response = client().performRequest(stopDataframeTransformRequest);
+        assertEquals(200, response.getStatusLine().getStatusCode());
+    }
+
+    private DataFrameTransformStateAndStats getTransformStats(String id) throws IOException {
+        final Request getStats = new Request("GET", DATAFRAME_ENDPOINT + id + "/_stats");
+        Response response = client().performRequest(getStats);
+        assertEquals(200, response.getStatusLine().getStatusCode());
+        XContentType xContentType = XContentType.fromMediaTypeOrFormat(response.getEntity().getContentType().getValue());
+        try (XContentParser parser = xContentType.xContent().createParser(
+            NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+            response.getEntity().getContent())) {
+            GetDataFrameTransformStatsResponse resp = GetDataFrameTransformStatsResponse.fromXContent(parser);
+            assertThat(resp.getTransformsStateAndStats(), hasSize(1));
+            return resp.getTransformsStateAndStats().get(0);
+        }
+    }
+
+    private void waitUntilAfterCheckpoint(String id, long currentCheckpoint) throws Exception {
+        assertBusy(() -> assertThat(getTransformStats(id).getTransformState().getCheckpoint(), greaterThan(currentCheckpoint)),
+            60, TimeUnit.SECONDS);
+    }
+
+    private void createIndex(String indexName) throws IOException {
+        // create mapping
+        try (XContentBuilder builder = jsonBuilder()) {
+            builder.startObject();
+            {
+                builder.startObject("mappings")
+                    .startObject("properties")
+                    .startObject("timestamp")
+                    .field("type", "date")
+                    .endObject()
+                    .startObject("user_id")
+                    .field("type", "keyword")
+                    .endObject()
+                    .startObject("stars")
+                    .field("type", "integer")
+                    .endObject()
+                    .endObject()
+                    .endObject();
+            }
+            builder.endObject();
+            final StringEntity entity = new StringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON);
+            Request req = new Request("PUT", indexName);
+            req.setEntity(entity);
+            client().performRequest(req);
+        }
+    }
+
+    private void putData(String indexName, int numDocs, TimeValue fromTime, List<String> entityIds) throws IOException {
+        long timeStamp = Instant.now().toEpochMilli() - fromTime.getMillis();
+
+        // create index
+        final StringBuilder bulk = new StringBuilder();
+        for (int i = 0; i < numDocs; i++) {
+            for (String entity : entityIds) {
+                bulk.append("{\"index\":{\"_index\":\"" + indexName + "\"}}\n")
+                    .append("{\"user_id\":\"")
+                    .append(entity)
+                    .append("\",\"stars\":")
+                    .append(randomLongBetween(0, 5))
+                    .append(",\"timestamp\":")
+                    .append(timeStamp)
+                    .append("}\n");
+            }
+        }
+        bulk.append("\r\n");
+        final Request bulkRequest = new Request("POST", "/_bulk");
+        bulkRequest.addParameter("refresh", "true");
+        bulkRequest.setJsonEntity(bulk.toString());
+        entityAsMap(client().performRequest(bulkRequest));
+    }
+}
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml
index 7ac23d3cc1554..812f4d2683170 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml
@@ -35,6 +35,57 @@
         job_id: old-cluster-job
   - match: { count: 1 }
 
+---
+"Test get old cluster job's timing stats":
+  - do:
+      ml.get_job_stats:
+        job_id: old-cluster-job-with-ts
+  - match: { jobs.0.job_id: old-cluster-job-with-ts }
+  - match: { jobs.0.state: "closed" }
+  - match: { jobs.0.timing_stats.job_id: old-cluster-job-with-ts }
+  - match: { jobs.0.timing_stats.bucket_count: 1 }
+  - gte:   { jobs.0.timing_stats.minimum_bucket_processing_time_ms: 0.0 }
+  - gte:   { jobs.0.timing_stats.maximum_bucket_processing_time_ms: 0.0 }
+  - gte:   { jobs.0.timing_stats.average_bucket_processing_time_ms: 0.0 }
+  - gte:   { jobs.0.timing_stats.exponential_average_bucket_processing_time_ms: 0.0 }
+
+---
+"Test get old cluster categorization job":
+  - do:
+      ml.get_jobs:
+        job_id: old-cluster-categorization-job
+  - match: { count: 1 }
+
+  - do:
+      ml.get_job_stats:
+        job_id: old-cluster-categorization-job
+  - match: { jobs.0.state: "closed" }
+  - match: { jobs.0.data_counts.processed_record_count: 2 }
+  - is_true: jobs.0.model_size_stats
+  - is_false: node
+
+  - do:
+      ml.open_job:
+        job_id: old-cluster-categorization-job
+
+  - do:
+      ml.get_job_stats:
+        job_id: old-cluster-categorization-job
+  - match: { jobs.0.state: "opened" }
+  - match: { jobs.0.data_counts.processed_record_count: 2 }
+  - is_true: jobs.0.model_size_stats
+  - is_true: jobs.0.node
+  - is_true: jobs.0.open_time
+
+  - do:
+      ml.close_job:
+        job_id: old-cluster-categorization-job
+
+  - do:
+      ml.get_categories:
+        job_id: old-cluster-categorization-job
+  - match: { count: 1 }
+
 ---
 "Create a job in the mixed cluster and write some data":
   - do:
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_data_frame_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_data_frame_jobs_crud.yml
new file mode 100644
index 0000000000000..9454423e98953
--- /dev/null
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_data_frame_jobs_crud.yml
@@ -0,0 +1,292 @@
+---
+"Test put batch data frame transforms on mixed cluster":
+  - do:
+      cluster.health:
+        index: "dataframe-transform-airline-data"
+        wait_for_status: green
+        timeout: 70s
+
+  - do:
+      data_frame.put_data_frame_transform:
+        transform_id: "mixed-simple-transform"
+        body: >
+          {
+            "source": { "index": "dataframe-transform-airline-data" },
+            "dest": { "index": "mixed-simple-transform-idx" },
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "mixed-simple-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-simple-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-transform" }
+  - match: { transforms.0.state.task_state: "/started|stopped/" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "mixed-simple-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-simple-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-transform" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
+
+  - do:
+      data_frame.put_data_frame_transform:
+        transform_id: "mixed-complex-transform"
+        body: >
+          {
+            "source": {
+              "index": "dataframe-transform-airline-data",
+              "query": {
+                "bool": {
+                  "filter": {"term": {"airline": "ElasticAir"}}
+                }
+              }
+            },
+            "dest": {
+              "index": "mixed-complex-transform-idx",
+              "pipeline": "data_frame_simple_pipeline"
+            },
+            "pivot": {
+              "group_by": {
+                "airline": {"terms": {"field": "airline"}},
+                "day": {"date_histogram": {"field": "timestamp", "calendar_interval": "1d"}},
+                "every_50": {"histogram": {"field": "responsetime", "interval": 50}}
+              },
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "mixed-complex-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-complex-transform" }
+  - is_true: transforms.0.version
+  - is_true: transforms.0.create_time
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "mixed-complex-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-complex-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-complex-transform" }
+  - match: { transforms.0.state.task_state: "/started|stopped/" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "mixed-complex-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-complex-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-complex-transform" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
+
+---
+"Test put continuous data frame transform on mixed cluster":
+  - do:
+      cluster.health:
+        index: "dataframe-transform-airline-data-cont"
+        wait_for_status: green
+        timeout: 70s
+
+  - do:
+      data_frame.put_data_frame_transform:
+        transform_id: "mixed-simple-continuous-transform"
+        body: >
+          {
+            "source": { "index": "dataframe-transform-airline-data-cont" },
+            "dest": { "index": "mixed-simple-continuous-transform-idx" },
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            },
+            "sync": {
+              "time": {
+                "field": "time",
+                "delay": "90m"
+              }
+            }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-continuous-transform" }
+  - match: { transforms.0.sync.time.field: "time" }
+  - match: { transforms.0.sync.time.delay: "90m" }
+  - is_true: transforms.0.version
+  - is_true: transforms.0.create_time
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-continuous-transform" }
+  - match: { transforms.0.state.task_state: "started" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "mixed-simple-continuous-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-continuous-transform" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
+
+---
+"Test GET, start, and stop old cluster batch transforms":
+  - do:
+      cluster.health:
+        index: "dataframe-transform-airline-data"
+        wait_for_status: green
+        timeout: 70s
+
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "old-simple-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-transform" }
+  - match: { transforms.0.source.index.0: "dataframe-transform-airline-data" }
+  - match: { transforms.0.dest.index: "old-simple-transform-idx" }
+  - match: { transforms.0.pivot.group_by.airline.terms.field: "airline" }
+  - match: { transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime" }
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "old-simple-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-transform" }
+  - match: { transforms.0.state.task_state: "/started|stopped/" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "old-simple-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-transform" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
+
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "old-complex-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-complex-transform" }
+  - match: { transforms.0.source.index.0: "dataframe-transform-airline-data" }
+  - match: { transforms.0.dest.index: "old-complex-transform-idx" }
+  - match: { transforms.0.dest.pipeline: "data_frame_simple_pipeline" }
+  - match: { transforms.0.pivot.group_by.airline.terms.field: "airline" }
+  - match: { transforms.0.pivot.group_by.day.date_histogram.field: "timestamp" }
+  - match: { transforms.0.pivot.group_by.every_50.histogram.field: "responsetime" }
+  - match: { transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime" }
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "old-complex-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-complex-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-complex-transform" }
+  - match: { transforms.0.state.task_state: "/started|stopped/" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "old-complex-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-complex-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-complex-transform" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
+
+---
+"Test GET, stop, start, old continuous transforms":
+  - do:
+      cluster.health:
+        index: "dataframe-transform-airline-data-cont"
+        wait_for_status: green
+        timeout: 70s
+
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.sync.time.field: "time" }
+  - match: { transforms.0.sync.time.delay: "90m" }
+  - is_true: transforms.0.version
+  - is_true: transforms.0.create_time
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.state.task_state: "started" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml
index bca44b5f16a38..e42b971e550ad 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml
@@ -56,6 +56,65 @@
        index: [".ml-state", ".ml-anomalies-shared"]
        wait_for_status: green
 
+---
+"Put categorization job on the old cluster and post some data":
+
+  - do:
+      ml.put_job:
+        job_id: old-cluster-categorization-job
+        body:  >
+          {
+            "description":"Cluster upgrade with categorization",
+            "analysis_config" : {
+                "bucket_span": "60s",
+                "detectors" :[{"function":"count","by_field_name":"mlcategory"}],
+                "categorization_field_name": "message",
+                "categorization_analyzer" : {
+                    "char_filter" : ["html_strip"],
+                    "tokenizer" : "classic",
+                    "filter" : ["stop"]
+                }
+            },
+            "analysis_limits" : {
+                "model_memory_limit": "50mb"
+            },
+            "data_description" : {
+                "time_field":"time",
+                "time_format":"epoch"
+            }
+          }
+  - match: { job_id: old-cluster-categorization-job }
+
+  - do:
+      ml.open_job:
+        job_id: old-cluster-categorization-job
+
+  - do:
+      ml.post_data:
+        job_id: old-cluster-categorization-job
+        body:
+          - message: "this is the first message to be categorized"
+            time: 1403481600
+          - message: "this is the second message to be categorized"
+            time: 1403481700
+  - match: { processed_record_count: 2 }
+
+  - do:
+      ml.close_job:
+        job_id: old-cluster-categorization-job
+
+  - do:
+      ml.get_categories:
+        job_id: old-cluster-categorization-job
+  - match: { count: 1 }
+
+# Wait for indices to be fully allocated before
+# killing the node
+  - do:
+     cluster.health:
+       index: [".ml-state", ".ml-anomalies-shared"]
+       wait_for_status: green
+
 ---
 "Put job on the old cluster with the default model memory limit and post some data":
   - do:
@@ -110,6 +169,64 @@
        index: [".ml-state", ".ml-anomalies-shared"]
        wait_for_status: green
 
+---
+"Put job with timing stats checking on the old cluster and post some data":
+
+  - do:
+      ml.put_job:
+        job_id: old-cluster-job-with-ts
+        body:  >
+          {
+            "description":"Cluster upgrade with timing stats checking",
+            "analysis_config" : {
+                "bucket_span": "60s",
+                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
+            },
+            "analysis_limits" : {
+                "model_memory_limit": "50mb"
+            },
+            "data_description" : {
+                "format":"xcontent",
+                "time_field":"time",
+                "time_format":"epoch"
+            }
+          }
+  - match: { job_id: old-cluster-job-with-ts }
+
+  - do:
+      ml.open_job:
+        job_id: old-cluster-job-with-ts
+
+  - do:
+      ml.post_data:
+        job_id: old-cluster-job-with-ts
+        body:
+          - airline: AAL
+            responsetime: 132.2046
+            sourcetype: post-data-job
+            time: 1403481600
+          - airline: JZA
+            responsetime: 990.4628
+            sourcetype: post-data-job
+            time: 1403481700
+  - match: { processed_record_count: 2 }
+
+  - do:
+      ml.close_job:
+        job_id: old-cluster-job-with-ts
+
+  - do:
+      ml.get_buckets:
+        job_id: old-cluster-job-with-ts
+  - match: { count: 1 }
+
+  # Wait for indices to be fully allocated before
+  # killing the node
+  - do:
+      cluster.health:
+        index: [".ml-state", ".ml-anomalies-shared"]
+        wait_for_status: green
+
 ---
 "Put job with empty strings in the configuration":
   - do:
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/80_data_frame_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/80_data_frame_jobs_crud.yml
new file mode 100644
index 0000000000000..7b666c2caa35f
--- /dev/null
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/80_data_frame_jobs_crud.yml
@@ -0,0 +1,218 @@
+---
+"Test put batch data frame transforms on old cluster":
+  - do:
+      indices.create:
+        index: dataframe-transform-airline-data
+        body:
+          mappings:
+            properties:
+              time:
+                type: date
+              airline:
+                type: keyword
+              responsetime:
+                type: float
+              event_rate:
+                type: integer
+  - do:
+      cluster.health:
+        index: "dataframe-transform-airline-data"
+        wait_for_status: green
+        timeout: 70s
+
+  - do:
+      ingest.put_pipeline:
+        id: "data_frame_simple_pipeline"
+        body:  >
+          {
+            "processors": [
+             {
+               "set" : {
+                 "field" : "my_field",
+                 "value": 42
+               }
+              }
+            ]
+          }
+  - do:
+      data_frame.put_data_frame_transform:
+        transform_id: "old-simple-transform"
+        body: >
+          {
+            "source": { "index": "dataframe-transform-airline-data" },
+            "dest": { "index": "old-simple-transform-idx" },
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "old-simple-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-transform" }
+  - is_true: transforms.0.version
+  - is_true: transforms.0.create_time
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "old-simple-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-transform" }
+  - match: { transforms.0.state.task_state: "/started|stopped/" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "old-simple-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-transform" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
+
+  - do:
+      data_frame.put_data_frame_transform:
+        transform_id: "old-complex-transform"
+        body: >
+          {
+            "source": {
+              "index": "dataframe-transform-airline-data",
+              "query": {
+                "bool": {
+                  "filter": {"term": {"airline": "ElasticAir"}}
+                }
+              }
+            },
+            "dest": {
+              "index": "old-complex-transform-idx",
+              "pipeline": "data_frame_simple_pipeline"
+            },
+            "pivot": {
+              "group_by": {
+                "airline": {"terms": {"field": "airline"}},
+                "day": {"date_histogram": {"field": "timestamp", "calendar_interval": "1d"}},
+                "every_50": {"histogram": {"field": "responsetime", "interval": 50}}
+              },
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "old-complex-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-complex-transform" }
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "old-complex-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-complex-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-complex-transform" }
+  - match: { transforms.0.state.task_state: "/started|stopped/" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "old-complex-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-complex-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-complex-transform" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
+
+---
+"Test put continuous data frame transform on old cluster":
+  - do:
+      indices.create:
+        index: dataframe-transform-airline-data-cont
+        body:
+          mappings:
+            properties:
+              time:
+                type: date
+              airline:
+                type: keyword
+              responsetime:
+                type: float
+              event_rate:
+                type: integer
+  - do:
+      cluster.health:
+        index: "dataframe-transform-airline-data-cont"
+        wait_for_status: green
+        timeout: 70s
+
+  - do:
+      data_frame.put_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+        body: >
+          {
+            "source": { "index": "dataframe-transform-airline-data-cont" },
+            "dest": { "index": "old-simple-continuous-transform-idx" },
+            "pivot": {
+              "group_by": { "airline": {"terms": {"field": "airline"}}},
+              "aggs": {"avg_response": {"avg": {"field": "responsetime"}}}
+            },
+            "sync": {
+              "time": {
+                "field": "time",
+                "delay": "90m"
+              }
+            }
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.sync.time.field: "time" }
+  - match: { transforms.0.sync.time.delay: "90m" }
+  - is_true: transforms.0.version
+  - is_true: transforms.0.create_time
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.state.task_state: "started" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml
index 5d8775dfb2c11..b558583bd066e 100644
--- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml
@@ -21,6 +21,19 @@ setup:
   - is_true: jobs.0.node
   - is_true: jobs.0.open_time
 
+  - do:
+      ml.open_job:
+        job_id: old-cluster-categorization-job
+
+  - do:
+      ml.get_job_stats:
+        job_id: old-cluster-categorization-job
+  - match: { jobs.0.state: "opened" }
+  - match: { jobs.0.data_counts.processed_record_count: 2 }
+  - is_true: jobs.0.model_size_stats
+  - is_true: jobs.0.node
+  - is_true: jobs.0.open_time
+
   - do:
       ml.open_job:
         job_id: mixed-cluster-job
@@ -38,6 +51,10 @@ setup:
       ml.close_job:
         job_id: old-cluster-job
 
+  - do:
+      ml.close_job:
+        job_id: old-cluster-categorization-job
+
   - do:
       ml.close_job:
         job_id: mixed-cluster-job
@@ -47,6 +64,11 @@ setup:
         job_id: old-cluster-job
   - match: { count: 1 }
 
+  - do:
+      ml.get_categories:
+        job_id: old-cluster-categorization-job
+  - match: { count: 1 }
+
   - do:
       ml.get_buckets:
         job_id: mixed-cluster-job
@@ -62,6 +84,16 @@ setup:
       ml.get_jobs:
         job_id: old-cluster-job
 
+  - do:
+      ml.delete_job:
+        job_id: old-cluster-categorization-job
+  - match: { acknowledged: true }
+
+  - do:
+      catch: missing
+      ml.get_jobs:
+        job_id: old-cluster-categorization-job
+
   - do:
       ml.delete_job:
         job_id: mixed-cluster-job
@@ -77,6 +109,30 @@ setup:
         name: ".ml-state-write"
   - is_true: ''
 
+---
+"Test get old cluster job's timing stats":
+  - do:
+      ml.get_job_stats:
+        job_id: old-cluster-job-with-ts
+  - match: { jobs.0.job_id: old-cluster-job-with-ts }
+  - match: { jobs.0.state: "closed" }
+  - match: { jobs.0.timing_stats.job_id: old-cluster-job-with-ts }
+  - match: { jobs.0.timing_stats.bucket_count: 1 }
+  - gte:   { jobs.0.timing_stats.minimum_bucket_processing_time_ms: 0.0 }
+  - gte:   { jobs.0.timing_stats.maximum_bucket_processing_time_ms: 0.0 }
+  - gte:   { jobs.0.timing_stats.average_bucket_processing_time_ms: 0.0 }
+  - gte:   { jobs.0.timing_stats.exponential_average_bucket_processing_time_ms: 0.0 }
+
+  - do:
+      ml.delete_job:
+        job_id: old-cluster-job-with-ts
+  - match: { acknowledged: true }
+
+  - do:
+      catch: missing
+      ml.get_jobs:
+        job_id: old-cluster-job-with-ts
+
 ---
 "Test job with pre 6.4 rules":
 
diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_data_frame_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_data_frame_jobs_crud.yml
new file mode 100644
index 0000000000000..ea63950d7fc64
--- /dev/null
+++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_data_frame_jobs_crud.yml
@@ -0,0 +1,259 @@
+setup:
+  - do:
+      cluster.health:
+        wait_for_status: green
+        wait_for_nodes: 3
+        # wait for long enough that we give delayed unassigned shards to stop being delayed
+        timeout: 70s
+---
+"Get start, stop, and delete old and mixed cluster batch data frame transforms":
+  # Simple and complex OLD transforms
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "old-simple-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-transform" }
+  - match: { transforms.0.source.index.0: "dataframe-transform-airline-data" }
+  - match: { transforms.0.dest.index: "old-simple-transform-idx" }
+  - match: { transforms.0.pivot.group_by.airline.terms.field: "airline" }
+  - match: { transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime" }
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "old-simple-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-transform" }
+  - match: { transforms.0.state.task_state: "/started|stopped/" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "old-simple-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-transform" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "old-complex-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-complex-transform" }
+  - match: { transforms.0.source.index.0: "dataframe-transform-airline-data" }
+  - match: { transforms.0.dest.index: "old-complex-transform-idx" }
+  - match: { transforms.0.dest.pipeline: "data_frame_simple_pipeline" }
+  - match: { transforms.0.pivot.group_by.airline.terms.field: "airline" }
+  - match: { transforms.0.pivot.group_by.day.date_histogram.field: "timestamp" }
+  - match: { transforms.0.pivot.group_by.every_50.histogram.field: "responsetime" }
+  - match: { transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime" }
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "old-complex-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-complex-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-complex-transform" }
+  - match: { transforms.0.state.task_state: "/started|stopped/" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "old-complex-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-complex-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-complex-transform" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
+
+  # Simple and complex Mixed cluster transforms
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "mixed-simple-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-transform" }
+  - match: { transforms.0.source.index.0: "dataframe-transform-airline-data" }
+  - match: { transforms.0.dest.index: "mixed-simple-transform-idx" }
+  - match: { transforms.0.pivot.group_by.airline.terms.field: "airline" }
+  - match: { transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime" }
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "mixed-simple-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-simple-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-transform" }
+  - match: { transforms.0.state.task_state: "/started|stopped/" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "mixed-simple-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-simple-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-transform" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
+
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "mixed-complex-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-complex-transform" }
+  - match: { transforms.0.source.index.0: "dataframe-transform-airline-data" }
+  - match: { transforms.0.dest.index: "mixed-complex-transform-idx" }
+  - match: { transforms.0.dest.pipeline: "data_frame_simple_pipeline" }
+  - match: { transforms.0.pivot.group_by.airline.terms.field: "airline" }
+  - match: { transforms.0.pivot.group_by.day.date_histogram.field: "timestamp" }
+  - match: { transforms.0.pivot.group_by.every_50.histogram.field: "responsetime" }
+  - match: { transforms.0.pivot.aggregations.avg_response.avg.field: "responsetime" }
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "mixed-complex-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-complex-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-complex-transform" }
+  - match: { transforms.0.state.task_state: "/started|stopped/" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "mixed-complex-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-complex-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-complex-transform" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
+
+# Delete all old and mixed transforms
+  - do:
+      data_frame.delete_data_frame_transform:
+        transform_id: "old-simple-transform"
+
+  - do:
+      data_frame.delete_data_frame_transform:
+        transform_id: "mixed-simple-transform"
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-transform,mixed-simple-transform"
+  - match: { count: 0 }
+
+---
+"Test GET, stop, delete, old and mixed continuous transforms":
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.sync.time.field: "time" }
+  - match: { transforms.0.sync.time.delay: "90m" }
+  - is_true: transforms.0.version
+  - is_true: transforms.0.create_time
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.state.task_state: "started" }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.state.task_state: "started" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "old-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "old-simple-continuous-transform" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
+
+  - do:
+      data_frame.delete_data_frame_transform:
+        transform_id: "old-simple-continuous-transform"
+
+  - do:
+      data_frame.get_data_frame_transform:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-continuous-transform" }
+  - match: { transforms.0.sync.time.field: "time" }
+  - match: { transforms.0.sync.time.delay: "90m" }
+  - is_true: transforms.0.version
+  - is_true: transforms.0.create_time
+
+  - do:
+      data_frame.start_data_frame_transform:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { acknowledged: true }
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-continuous-transform" }
+  - match: { transforms.0.state.task_state: "started" }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-continuous-transform" }
+  - match: { transforms.0.state.task_state: "started" }
+
+  - do:
+      data_frame.stop_data_frame_transform:
+        transform_id: "mixed-simple-continuous-transform"
+        wait_for_completion: true
+  - match: { acknowledged: true }
+
+  - do:
+      data_frame.get_data_frame_transform_stats:
+        transform_id: "mixed-simple-continuous-transform"
+  - match: { count: 1 }
+  - match: { transforms.0.id: "mixed-simple-continuous-transform" }
+  - match: { transforms.0.state.indexer_state: "stopped" }
+  - match: { transforms.0.state.task_state: "stopped" }
+
+  - do:
+      data_frame.delete_data_frame_transform:
+        transform_id: "mixed-simple-continuous-transform"
diff --git a/x-pack/qa/saml-idp-tests/build.gradle b/x-pack/qa/saml-idp-tests/build.gradle
index 25d9d9037b719..9e8c15cb94150 100644
--- a/x-pack/qa/saml-idp-tests/build.gradle
+++ b/x-pack/qa/saml-idp-tests/build.gradle
@@ -1,6 +1,6 @@
 Project idpFixtureProject = xpackProject("test:idp-fixture")
-evaluationDependsOn(idpFixtureProject.path)
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
@@ -21,10 +21,12 @@ project.sourceSets.test.output.dir(outputDir, builtBy: copyIdpFiles)
 
 task setupPorts {
     dependsOn copyIdpFiles, idpFixtureProject.postProcessFixture
+    // Don't attempt to get ephemeral ports when Docker is not available
+    onlyIf { idpFixtureProject.postProcessFixture.enabled }
     doLast {
         String portString = idpFixtureProject.postProcessFixture.ext."test.fixtures.shibboleth-idp.tcp.4443"
         int ephemeralPort = Integer.valueOf(portString)
-        File idpMetaFile = file(outputDir + '/idp-metadata.xml')
+        File idpMetaFile = file("$outputDir/idp-metadata.xml")
         List<String> lines = idpMetaFile.readLines("UTF-8")
         StringBuilder content = new StringBuilder()
         for (String line : lines) {
@@ -35,56 +37,44 @@ task setupPorts {
         idpMetaFile.write(content.toString(), "UTF-8")
     }
 }
-// Don't attempt to get ephemeral ports when Docker is not available
-setupPorts.onlyIf { idpFixtureProject.postProcessFixture.enabled }
 
-integTestCluster.dependsOn setupPorts
 
-integTestCluster {
-  setting 'xpack.license.self_generated.type', 'trial'
-  setting 'xpack.security.enabled', 'true'
-  setting 'xpack.security.http.ssl.enabled', 'false'
-  setting 'xpack.security.authc.token.enabled', 'true'
-  setting 'xpack.security.authc.realms.file.file.order', '0'
-  // SAML realm 1 (no authorization_realms)
-  setting 'xpack.security.authc.realms.saml.shibboleth.order', '1'
-  setting 'xpack.security.authc.realms.saml.shibboleth.idp.entity_id', 'https://test.shibboleth.elastic.local/'
-  setting 'xpack.security.authc.realms.saml.shibboleth.idp.metadata.path', 'idp-metadata.xml'
-  setting 'xpack.security.authc.realms.saml.shibboleth.sp.entity_id', 'http://mock1.http.elastic.local/'
-  // The port in the ACS URL is fake - the test will bind the mock webserver
-  // to a random port and then whenever it needs to connect to a URL on the
-  // mock webserver it will replace 54321 with the real port
-  setting 'xpack.security.authc.realms.saml.shibboleth.sp.acs', 'http://localhost:54321/saml/acs1'
-  setting 'xpack.security.authc.realms.saml.shibboleth.attributes.principal', 'uid'
-  setting 'xpack.security.authc.realms.saml.shibboleth.attributes.name', 'urn:oid:2.5.4.3'
-  // SAML realm 2 (uses authorization_realms)
-  setting 'xpack.security.authc.realms.saml.shibboleth_native.order', '2'
-  setting 'xpack.security.authc.realms.saml.shibboleth_native.idp.entity_id', 'https://test.shibboleth.elastic.local/'
-  setting 'xpack.security.authc.realms.saml.shibboleth_native.idp.metadata.path', 'idp-metadata.xml'
-  setting 'xpack.security.authc.realms.saml.shibboleth_native.sp.entity_id', 'http://mock2.http.elastic.local/'
-  setting 'xpack.security.authc.realms.saml.shibboleth_native.sp.acs', 'http://localhost:54321/saml/acs2'
-  setting 'xpack.security.authc.realms.saml.shibboleth_native.attributes.principal', 'uid'
-  setting 'xpack.security.authc.realms.saml.shibboleth_native.authorization_realms', 'native'
-  setting 'xpack.security.authc.realms.native.native.order', '3'
+integTest.runner.dependsOn setupPorts
 
-  setting 'xpack.ml.enabled', 'false'
-  setting 'logger.org.elasticsearch.xpack.security', 'TRACE'
+testClusters.integTest {
+    distribution = "DEFAULT"
+    setting 'xpack.license.self_generated.type', 'trial'
+    setting 'xpack.security.enabled', 'true'
+    setting 'xpack.security.http.ssl.enabled', 'false'
+    setting 'xpack.security.authc.token.enabled', 'true'
+    setting 'xpack.security.authc.realms.file.file.order', '0'
+    // SAML realm 1 (no authorization_realms)
+    setting 'xpack.security.authc.realms.saml.shibboleth.order', '1'
+    setting 'xpack.security.authc.realms.saml.shibboleth.idp.entity_id', 'https://test.shibboleth.elastic.local/'
+    setting 'xpack.security.authc.realms.saml.shibboleth.idp.metadata.path', 'idp-metadata.xml'
+    setting 'xpack.security.authc.realms.saml.shibboleth.sp.entity_id', 'http://mock1.http.elastic.local/'
+    // The port in the ACS URL is fake - the test will bind the mock webserver
+    // to a random port and then whenever it needs to connect to a URL on the
+    // mock webserver it will replace 54321 with the real port
+    setting 'xpack.security.authc.realms.saml.shibboleth.sp.acs', 'http://localhost:54321/saml/acs1'
+    setting 'xpack.security.authc.realms.saml.shibboleth.attributes.principal', 'uid'
+    setting 'xpack.security.authc.realms.saml.shibboleth.attributes.name', 'urn:oid:2.5.4.3'
+    // SAML realm 2 (uses authorization_realms)
+    setting 'xpack.security.authc.realms.saml.shibboleth_native.order', '2'
+    setting 'xpack.security.authc.realms.saml.shibboleth_native.idp.entity_id', 'https://test.shibboleth.elastic.local/'
+    setting 'xpack.security.authc.realms.saml.shibboleth_native.idp.metadata.path', 'idp-metadata.xml'
+    setting 'xpack.security.authc.realms.saml.shibboleth_native.sp.entity_id', 'http://mock2.http.elastic.local/'
+    setting 'xpack.security.authc.realms.saml.shibboleth_native.sp.acs', 'http://localhost:54321/saml/acs2'
+    setting 'xpack.security.authc.realms.saml.shibboleth_native.attributes.principal', 'uid'
+    setting 'xpack.security.authc.realms.saml.shibboleth_native.authorization_realms', 'native'
+    setting 'xpack.security.authc.realms.native.native.order', '3'
 
-  extraConfigFile 'idp-metadata.xml', file(outputDir + "/idp-metadata.xml")
+    setting 'xpack.ml.enabled', 'false'
+    setting 'logger.org.elasticsearch.xpack.security', 'TRACE'
 
-  setupCommand 'setupTestAdmin',
-            'bin/elasticsearch-users', 'useradd', "test_admin", '-p', 'x-pack-test-password', '-r', "superuser"
+    extraConfigFile 'idp-metadata.xml', file(outputDir + "/idp-metadata.xml")
 
-  waitCondition = { node, ant ->
-        File tmpFile = new File(node.cwd, 'wait.success')
-        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-                dest: tmpFile.toString(),
-                username: 'test_admin',
-                password: 'x-pack-test-password',
-                ignoreerrors: true,
-                retries: 10)
-        return tmpFile.exists()
-    }
+    user username: "test_admin", password: 'x-pack-test-password'
 }
 
 forbiddenPatterns {
diff --git a/x-pack/qa/security-example-spi-extension/build.gradle b/x-pack/qa/security-example-spi-extension/build.gradle
index 4790df3609c35..311cf6c67d7f4 100644
--- a/x-pack/qa/security-example-spi-extension/build.gradle
+++ b/x-pack/qa/security-example-spi-extension/build.gradle
@@ -1,3 +1,4 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.esplugin'
 
 esplugin {
@@ -8,17 +9,21 @@ esplugin {
 }
 
 dependencies {
-  compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
-  testCompile "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}"
+  compileOnly project(':x-pack:plugin:core')
+  testCompile project(':client:rest-high-level')
 }
 
 
-integTestRunner {
-    systemProperty 'tests.security.manager', 'false'
+integTest.runner {
+    dependsOn buildZip
 }
 
-integTestCluster {
-  dependsOn buildZip
+testClusters.integTest {
+  // This is important, so that all the modules are available too.
+  // There are index templates that use token filters that are in analysis-module and
+  // processors are being used that are in ingest-common module.
+  distribution = 'default'
+
   setting 'xpack.security.authc.realms.custom.custom.order', '0'
   setting 'xpack.security.authc.realms.custom.custom.filtered_setting', 'should be filtered'
   setting 'xpack.security.authc.realms.file.esusers.order', '1'
@@ -29,22 +34,5 @@ integTestCluster {
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
 
-  // This is important, so that all the modules are available too.
-  // There are index templates that use token filters that are in analysis-module and
-  // processors are being used that are in ingest-common module.
-  distribution = 'default'
-
-  setupCommand 'setupDummyUser',
-               'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
-  waitCondition = { node, ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-            dest: tmpFile.toString(),
-            username: 'test_user',
-            password: 'x-pack-test-password',
-            ignoreerrors: true,
-            retries: 10)
-    return tmpFile.exists()
-  }
+  user  username: "test_user", password: "x-pack-test-password"
 }
-check.dependsOn integTest
diff --git a/x-pack/qa/security-setup-password-tests/build.gradle b/x-pack/qa/security-setup-password-tests/build.gradle
index 604fffbe32f76..ee77aab06bb99 100644
--- a/x-pack/qa/security-setup-password-tests/build.gradle
+++ b/x-pack/qa/security-setup-password-tests/build.gradle
@@ -1,8 +1,8 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-    // "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
     testCompile project(path: xpackModule('core'), configuration: 'default')
     testCompile project(path: xpackModule('security'), configuration: 'runtime')
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
@@ -11,24 +11,15 @@ dependencies {
     }
 }
 
-integTestRunner {
+integTest.runner {
+    nonInputProperties.systemProperty 'tests.config.dir', "${-> testClusters.integTest.singleNode().getConfigDir()}"
     systemProperty 'tests.security.manager', 'false'
-    // TODO add  tests.config.dir = {cluster.singleNode().getConfigDir()} when converting to testclusters
 }
 
-integTestCluster {
-    setupCommand 'setupTestAdmin',
-            'bin/elasticsearch-users', 'useradd', "test_admin", '-p', 'x-pack-test-password', '-r', "superuser"
+testClusters.integTest {
+    distribution = "DEFAULT"
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.license.self_generated.type', 'trial'
-    waitCondition = { node, ant ->
-        File tmpFile = new File(node.cwd, 'wait.success')
-        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-                dest: tmpFile.toString(),
-                username: 'test_admin',
-                password: 'x-pack-test-password',
-                ignoreerrors: true,
-                retries: 10)
-        return tmpFile.exists()
-    }
+
+    user username: "test_admin", password: "x-pack-test-password"
 }
diff --git a/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle b/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle
index 18bf5656f1991..9bb5d5899db6a 100644
--- a/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle
+++ b/x-pack/qa/smoke-test-monitoring-with-watcher/build.gradle
@@ -1,3 +1,4 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -5,21 +6,12 @@ dependencies {
   testCompile project(':x-pack:qa')
 }
 
-integTestCluster {
+testClusters.integTest {
+  distribution = "DEFAULT"
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'true'
   setting 'xpack.watcher.enabled', 'true'
   setting 'xpack.security.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
-
-  // exporter settings are configured dynamically in our tests
-  // configure a local exporter, the HTTP exporter is configured via dynamic settings change
-  //setting 'xpack.monitoring.exporters.my_local.type', 'local'
-  //setting 'xpack.monitoring.exporters.my_local.index.name.time_format', 'YYYY'
-  //setting 'xpack.monitoring.exporters.my_http.type', 'http'
-  //setting 'xpack.monitoring.exporters.my_http.host', 'http'
-  //setting 'xpack.monitoring.exporters.my_http.index.name.time_format', 'YYYY-MM'
-  // one of the exporters should configure cluster alerts
-  // setting 'xpack.monitoring.exporters.my_http.cluster_alerts.management.enabled', 'true'
 }
diff --git a/x-pack/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java b/x-pack/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java
index 40ed71df842f3..780428d815dc8 100644
--- a/x-pack/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java
+++ b/x-pack/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java
@@ -9,7 +9,6 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ObjectPath;
 import org.junit.After;
@@ -19,7 +18,6 @@
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.is;
 
-@TestLogging("org.elasticsearch.client:TRACE,tracer:TRACE")
 @AwaitsFix(bugUrl = "flaky tests")
 public class MonitoringWithWatcherRestIT extends ESRestTestCase {
 
diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle
index d4fe2129363c5..78c0356c5c2a4 100644
--- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle
+++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle
@@ -2,11 +2,12 @@ import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.test.NodeInfo
 import org.elasticsearch.gradle.http.WaitForHttpResource
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile "org.elasticsearch.plugin:x-pack-core:${version}"
+  testCompile project(':x-pack:plugin:core')
   testCompile project(':client:rest-high-level')
 }
 
@@ -38,16 +39,11 @@ task copyKeyCerts(type: Copy) {
 sourceSets.test.resources.srcDir(keystoreDir)
 processTestResources.dependsOn(copyKeyCerts)
 
-integTestCluster.dependsOn(copyKeyCerts)
+integTest.runner.dependsOn(copyKeyCerts)
 
-ext.pluginsCount = 0
-project(':plugins').getChildProjects().each { pluginName, pluginProject ->
-  // need to get a non-decorated project object, so must re-lookup the project by path
-  integTestCluster.plugin(pluginProject.path)
-  pluginsCount += 1
-}
-
-integTestCluster {
+def pluginsCount = 0
+testClusters.integTest {
+  distribution = "DEFAULT"
   setting 'xpack.monitoring.collection.interval', '1s'
   setting 'xpack.monitoring.exporters._http.type', 'http'
   setting 'xpack.monitoring.exporters._http.enabled', 'false'
@@ -62,7 +58,7 @@ integTestCluster {
   setting 'xpack.security.http.ssl.key', 'testnode.pem'
   setting 'xpack.security.http.ssl.certificate', 'testnode.crt'
   setting 'xpack.security.http.ssl.certificate_authorities', 'testnode.crt'
-  keystoreSetting 'xpack.security.http.ssl.secure_key_passphrase', 'testnode'
+  keystore 'xpack.security.http.ssl.secure_key_passphrase', 'testnode'
 
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
@@ -74,18 +70,13 @@ integTestCluster {
   extraConfigFile clientKey.name, clientKey
   extraConfigFile clientCert.name, clientCert
 
-  setupCommand 'setupTestUser',
-               'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
-  setupCommand 'setupMonitoringUser',
-               'bin/elasticsearch-users', 'useradd', 'monitoring_agent', '-p', 'x-pack-test-password', '-r', 'remote_monitoring_agent'
+  user username: "test_user", password: "x-pack-test-password"
+  user username: "monitoring_agent", password: "x-pack-test-password", role: "remote_monitoring_agent"
 
-  waitCondition = { NodeInfo node, AntBuilder ant ->
-    WaitForHttpResource http = new WaitForHttpResource("https", node.httpUri(), numNodes)
-    http.setTrustStoreFile(clientKeyStore)
-    http.setTrustStorePassword("testclient")
-    http.setUsername("test_user")
-    http.setPassword("x-pack-test-password")
-    return http.wait(5000)
+  project(':plugins').getChildProjects().each { pluginName, pluginProject ->
+    plugin file(pluginProject.tasks.bundlePlugin.archiveFile)
+    tasks.integTest.dependsOn pluginProject.tasks.bundlePlugin
+    pluginsCount += 1
   }
 }
 
diff --git a/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java b/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java
index 5662990580f0f..bb9ce6854560e 100644
--- a/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java
+++ b/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java
@@ -63,7 +63,7 @@
  *
  * It sets up a cluster with Monitoring and Security configured with SSL. Once started,
  * an HTTP exporter is activated and it exports data locally over HTTPS/SSL. The test
- * then uses a transport client to check that the data have been correctly received and
+ * then uses a rest client to check that the data have been correctly received and
  * indexed in the cluster.
  */
 public class SmokeTestMonitoringWithSecurityIT extends ESRestTestCase {
diff --git a/x-pack/qa/smoke-test-plugins/build.gradle b/x-pack/qa/smoke-test-plugins/build.gradle
index 5aa3adbdf09f5..86fb4ee0f07d6 100644
--- a/x-pack/qa/smoke-test-plugins/build.gradle
+++ b/x-pack/qa/smoke-test-plugins/build.gradle
@@ -1,5 +1,6 @@
 import org.elasticsearch.gradle.MavenFilteringHack
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -7,27 +8,16 @@ dependencies {
   testCompile project(':x-pack:qa')
 }
 
-ext.pluginsCount = 0
-project(':plugins').getChildProjects().each { pluginName, pluginProject ->
-  // need to get a non-decorated project object, so must re-lookup the project by path
-  integTestCluster.plugin(pluginProject.path)
-  pluginsCount += 1
-}
-
-integTestCluster {
+int pluginsCount = 0
+testClusters.integTest {
+  distribution = "DEFAULT"
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.license.self_generated.type', 'trial'
-  setupCommand 'setupDummyUser',
-               'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
-  waitCondition = { node, ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-            dest: tmpFile.toString(),
-            username: 'test_user',
-            password: 'x-pack-test-password',
-            ignoreerrors: true,
-            retries: 10)
-    return tmpFile.exists()
+  user username: "test_user", password: "x-pack-test-password"
+  project(':plugins').getChildProjects().each { pluginName, pluginProject ->
+    plugin file(pluginProject.tasks.bundlePlugin.archiveFile)
+    tasks.integTest.dependsOn pluginProject.tasks.bundlePlugin
+    pluginsCount += 1
   }
 }
 
diff --git a/x-pack/qa/smoke-test-security-with-mustache/build.gradle b/x-pack/qa/smoke-test-security-with-mustache/build.gradle
index bf2e6c325730e..3edf8d22cbe1d 100644
--- a/x-pack/qa/smoke-test-security-with-mustache/build.gradle
+++ b/x-pack/qa/smoke-test-security-with-mustache/build.gradle
@@ -1,3 +1,4 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -5,21 +6,11 @@ dependencies {
   testCompile project(':x-pack:qa')
 }
 
-integTestCluster {
+testClusters.integTest {
+  distribution = "DEFAULT"
   setting 'xpack.watcher.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.license.self_generated.type', 'trial'
-  setupCommand 'setupDummyUser',
-          'bin/elasticsearch-users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser'
-  waitCondition = { node, ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-            dest: tmpFile.toString(),
-            username: 'test_admin',
-            password: 'x-pack-test-password',
-            ignoreerrors: true,
-            retries: 10)
-    return tmpFile.exists()
-  }
+  user username: "test_admin", password: "x-pack-test-password"
 }
diff --git a/x-pack/qa/smoke-test-watcher-with-security/build.gradle b/x-pack/qa/smoke-test-watcher-with-security/build.gradle
index fc66785d47957..d24c52b8d9005 100644
--- a/x-pack/qa/smoke-test-watcher-with-security/build.gradle
+++ b/x-pack/qa/smoke-test-watcher-with-security/build.gradle
@@ -1,3 +1,4 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -12,8 +13,9 @@ task copyWatcherRestTests(type: Copy) {
   include 'rest-api-spec/test/watcher/**'
 }
 
-integTestCluster {
-  dependsOn copyWatcherRestTests
+integTest.runner.dependsOn copyWatcherRestTests
+testClusters.integTest {
+  distribution = "DEFAULT"
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
@@ -22,27 +24,13 @@ integTestCluster {
   setting 'xpack.notification.email.account._email.smtp.host', 'host.domain'
   setting 'xpack.notification.email.account._email.smtp.port', '587'
   setting 'xpack.notification.email.account._email.smtp.user', '_user'
-  keystoreSetting 'xpack.notification.email.account._email.smtp.secure_password', '_passwd'
+  keystore  'xpack.notification.email.account._email.smtp.secure_password', '_passwd'
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'logger.org.elasticsearch.xpack.watcher', 'debug'
   setting 'logger.org.elasticsearch.xpack.core.watcher', 'debug'
-  extraConfigFile 'roles.yml', 'roles.yml'
-  setupCommand 'setupTestAdminUser',
-               'bin/elasticsearch-users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser'
-  setupCommand 'setupXpackUserForTests',
-               'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'watcher_manager'
-  setupCommand 'setupWatcherManagerUser',
-               'bin/elasticsearch-users', 'useradd', 'watcher_manager', '-p', 'x-pack-test-password', '-r', 'watcher_manager'
-  setupCommand 'setupPowerlessUser',
-               'bin/elasticsearch-users', 'useradd', 'powerless_user', '-p', 'x-pack-test-password', '-r', 'crappy_role'
-  waitCondition = { node, ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-            dest: tmpFile.toString(),
-            username: 'test_admin',
-            password: 'x-pack-test-password',
-            ignoreerrors: true,
-            retries: 10)
-    return tmpFile.exists()
-  }
+  extraConfigFile 'roles.yml', file('roles.yml')
+  user username: "test_admin", password: "x-pack-test-password"
+  user username: "x_pack_rest_user", password: "x-pack-test-password", role: "watcher_manager"
+  user username: "watcher_manager", password: "x-pack-test-password", role: "watcher_manager"
+  user username: "powerless_user", password: "x-pack-test-password", role: "crappy_role"
 }
diff --git a/x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java b/x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java
index e184ef19596af..20fd53b72e416 100644
--- a/x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java
+++ b/x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java
@@ -8,6 +8,7 @@
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
@@ -164,7 +165,7 @@ public void testSearchInputWithInsufficientPrivileges() throws Exception {
         String indexName = "index_not_allowed_to_read";
         try (XContentBuilder builder = jsonBuilder()) {
             builder.startObject();
-            builder.startObject("trigger").startObject("schedule").field("interval", "1s").endObject().endObject();
+            builder.startObject("trigger").startObject("schedule").field("interval", "4s").endObject().endObject();
             builder.startObject("input").startObject("search").startObject("request")
                     .startArray("indices").value(indexName).endArray()
                     .startObject("body").startObject("query").startObject("match_all").endObject().endObject().endObject()
@@ -180,8 +181,10 @@ public void testSearchInputWithInsufficientPrivileges() throws Exception {
 
         // check history, after watch has fired
         ObjectPath objectPath = getWatchHistoryEntry(watchId);
-        String state = objectPath.evaluate("hits.hits.0._source.state");
-        assertThat(state, is("execution_not_needed"));
+        assertBusy(() -> {
+            String state = objectPath.evaluate("hits.hits.0._source.state");
+            assertThat(state, is("execution_not_needed"));
+        });
         boolean conditionMet = objectPath.evaluate("hits.hits.0._source.result.condition.met");
         assertThat(conditionMet, is(false));
     }
@@ -328,6 +331,10 @@ private ObjectPath getWatchHistoryEntry(String watchId, String state) throws Exc
                 String watchid = objectPath.evaluate("hits.hits.0._source.watch_id");
                 assertThat(watchid, is(watchId));
                 objectPathReference.set(objectPath);
+            } catch (ResponseException e) {
+                final String err = "Failed to perform search of watcher history";
+                logger.info(err, e);
+                fail(err);
             }
         });
         return objectPathReference.get();
diff --git a/x-pack/qa/smoke-test-watcher/build.gradle b/x-pack/qa/smoke-test-watcher/build.gradle
index 9194c46daed01..d4ccba66f4966 100644
--- a/x-pack/qa/smoke-test-watcher/build.gradle
+++ b/x-pack/qa/smoke-test-watcher/build.gradle
@@ -1,3 +1,4 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
@@ -5,15 +6,12 @@ dependencies {
   testCompile project(':x-pack:qa')
 }
 
-integTestCluster {
+testClusters.integTest {
+  distribution = "DEFAULT"
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.security.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG'
-}
-
-integTestRunner {
-  include "**/*Tests.class"
 }
\ No newline at end of file
diff --git a/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java b/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java
index ec5015345fcb2..60866661617f8 100644
--- a/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java
+++ b/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java
@@ -216,8 +216,8 @@ private ObjectPath getWatchHistoryEntry(String watchId) throws Exception {
                 assertThat("watch_id for hit 0 in watcher history", foundWatchId, is(watchId));
                 objectPathReference.set(objectPath);
             } catch (ResponseException e) {
-                final String err = "Failed to perform search of watcher history - " + e;
-                logger.info(err);
+                final String err = "Failed to perform search of watcher history";
+                logger.info(err, e);
                 fail(err);
             }
         });
diff --git a/x-pack/qa/smoke-test-watcher/src/test/resources/rest-api-spec/test/mustache/50_webhook_url_escaping.yml b/x-pack/qa/smoke-test-watcher/src/test/resources/rest-api-spec/test/mustache/50_webhook_url_escaping.yml
index e11809a79baa5..bff41eceab3f7 100644
--- a/x-pack/qa/smoke-test-watcher/src/test/resources/rest-api-spec/test/mustache/50_webhook_url_escaping.yml
+++ b/x-pack/qa/smoke-test-watcher/src/test/resources/rest-api-spec/test/mustache/50_webhook_url_escaping.yml
@@ -1,8 +1,8 @@
 ---
 "Test url escaping with url mustache function":
   - skip:
-        version: "all"
-        reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/41172"
+      version: "all"
+      reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/41172"
   - do:
       cluster.health:
           wait_for_status: yellow
@@ -82,7 +82,6 @@
   - do:
       count:
           index: <date-index-{now/d}>
-          type: log
 
   - match: {count : 2}
 
diff --git a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java
index bfdf051a29235..633fec7a5b63d 100644
--- a/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java
+++ b/x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java
@@ -10,7 +10,7 @@
 public final class XPackRestTestConstants {
 
     // Watcher constants:
-    public static final String INDEX_TEMPLATE_VERSION = "9";
+    public static final String INDEX_TEMPLATE_VERSION = "10";
     public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION;
     public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches";
     public static final String WATCHES_TEMPLATE_NAME = ".watches";
@@ -30,6 +30,12 @@ public final class XPackRestTestConstants {
     public static final List<String> ML_POST_V660_TEMPLATES =
         List.of(AUDITOR_NOTIFICATIONS_INDEX, ML_META_INDEX_NAME, STATE_INDEX_PREFIX, RESULTS_INDEX_PREFIX, CONFIG_INDEX);
 
+    // Data Frame constants:
+    public static final String DATA_FRAME_INTERNAL_INDEX = ".data-frame-internal-1";
+    public static final String DATA_FRAME_NOTIFICATIONS_INDEX = ".data-frame-notifications-1";
+
+    public static final List<String> DATA_FRAME_TEMPLATES = List.of(DATA_FRAME_INTERNAL_INDEX, DATA_FRAME_NOTIFICATIONS_INDEX);
+
     private XPackRestTestConstants() {
     }
 }
diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java
index 86d411053afe9..94c23b77d3ab0 100644
--- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java
+++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java
@@ -26,7 +26,6 @@
 import org.elasticsearch.xpack.core.security.authc.ldap.ActiveDirectorySessionFactorySettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.LdapRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -177,12 +176,11 @@ public void setupRoleMappings() throws Exception {
         if (content.isEmpty()) {
             return;
         }
-        SecurityClient securityClient = securityClient();
         Map<String, ActionFuture<PutRoleMappingResponse>> futures = new LinkedHashMap<>(content.size());
         for (int i = 0; i < content.size(); i++) {
             final String name = "external_" + i;
-            final PutRoleMappingRequestBuilder builder = securityClient.preparePutRoleMapping(
-                    name, new BytesArray(content.get(i)), XContentType.JSON);
+            final PutRoleMappingRequestBuilder builder = new PutRoleMappingRequestBuilder(client())
+                .source(name, new BytesArray(content.get(i)), XContentType.JSON);
             futures.put(name, builder.execute());
         }
         for (String mappingName : futures.keySet()) {
diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryGroupsResolverTests.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryGroupsResolverTests.java
index 1a4fd0242dbaa..7fbbd217ae90b 100644
--- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryGroupsResolverTests.java
+++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryGroupsResolverTests.java
@@ -35,7 +35,6 @@ public void setReferralFollowing() {
     }
 
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/35738")
-    @SuppressWarnings("unchecked")
     public void testResolveSubTree() throws Exception {
         Settings settings = Settings.builder()
                 .put("xpack.security.authc.realms.active_directory.ad.group_search.scope", LdapSearchScope.SUB_TREE)
diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java
index 73e1df5dd08bd..3dc432b482bd6 100644
--- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java
+++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactoryTests.java
@@ -66,7 +66,6 @@ public boolean enableWarningsCheck() {
         return false;
     }
 
-    @SuppressWarnings("unchecked")
     public void testAdAuth() throws Exception {
         RealmConfig config = configureRealm("ad-test", buildAdSettings(AD_LDAP_URL, AD_DOMAIN, false));
         try (ActiveDirectorySessionFactory sessionFactory = getActiveDirectorySessionFactory(config, sslService, threadPool)) {
@@ -101,7 +100,6 @@ private RealmConfig configureRealm(String name, Settings settings) {
         return new RealmConfig(identifier, mergedSettings, env, new ThreadContext(globalSettings));
     }
 
-    @SuppressWarnings("unchecked")
     public void testNetbiosAuth() throws Exception {
         final String adUrl = randomFrom(AD_LDAP_URL, AD_LDAP_GC_URL);
         RealmConfig config = configureRealm("ad-test", buildAdSettings(adUrl, AD_DOMAIN, false));
@@ -139,7 +137,6 @@ public void testAdAuthAvengers() throws Exception {
         }
     }
 
-    @SuppressWarnings("unchecked")
     public void testAuthenticate() throws Exception {
         Settings settings = buildAdSettings(REALM_ID, AD_LDAP_URL, AD_DOMAIN, "CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com",
                 LdapSearchScope.ONE_LEVEL, false);
@@ -163,7 +160,6 @@ public void testAuthenticate() throws Exception {
         }
     }
 
-    @SuppressWarnings("unchecked")
     public void testAuthenticateBaseUserSearch() throws Exception {
         Settings settings = buildAdSettings(REALM_ID, AD_LDAP_URL, AD_DOMAIN,
                 "CN=Bruce Banner, CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com", LdapSearchScope.BASE, false);
@@ -208,7 +204,6 @@ public void testAuthenticateBaseGroupSearch() throws Exception {
         }
     }
 
-    @SuppressWarnings("unchecked")
     public void testAuthenticateWithUserPrincipalName() throws Exception {
         Settings settings = buildAdSettings(REALM_ID, AD_LDAP_URL, AD_DOMAIN, "CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com",
                 LdapSearchScope.ONE_LEVEL, false);
@@ -229,7 +224,6 @@ public void testAuthenticateWithUserPrincipalName() throws Exception {
         }
     }
 
-    @SuppressWarnings("unchecked")
     public void testAuthenticateWithSAMAccountName() throws Exception {
         Settings settings = buildAdSettings(REALM_ID, AD_LDAP_URL, AD_DOMAIN, "CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com",
                 LdapSearchScope.ONE_LEVEL, false);
@@ -251,7 +245,6 @@ public void testAuthenticateWithSAMAccountName() throws Exception {
         }
     }
 
-    @SuppressWarnings("unchecked")
     public void testCustomUserFilter() throws Exception {
         Settings settings = Settings.builder()
                 .put(buildAdSettings(REALM_ID, AD_LDAP_URL, AD_DOMAIN, "CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com",
@@ -275,7 +268,6 @@ public void testCustomUserFilter() throws Exception {
     }
 
 
-    @SuppressWarnings("unchecked")
     public void testStandardLdapConnection() throws Exception {
         String groupSearchBase = "DC=ad,DC=test,DC=elasticsearch,DC=com";
         String userTemplate = "CN={0},CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com";
@@ -341,7 +333,6 @@ public void testHandlingLdapReferralErrors() throws Exception {
         }
     }
 
-    @SuppressWarnings("unchecked")
     public void testStandardLdapWithAttributeGroups() throws Exception {
         String userTemplate = "CN={0},CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com";
         Settings settings = LdapTestCase.buildLdapSettings(new String[]{AD_LDAP_URL}, userTemplate, false);
diff --git a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/UserAttributeGroupsResolverTests.java b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/UserAttributeGroupsResolverTests.java
index 24f0ecace67b9..38adbbe019048 100644
--- a/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/UserAttributeGroupsResolverTests.java
+++ b/x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/UserAttributeGroupsResolverTests.java
@@ -29,7 +29,6 @@ public class UserAttributeGroupsResolverTests extends GroupsResolverTestCase {
     public static final String BRUCE_BANNER_DN = "cn=Bruce Banner,CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com";
     private static final RealmConfig.RealmIdentifier REALM_ID = new RealmConfig.RealmIdentifier("ldap", "realm1");
 
-    @SuppressWarnings("unchecked")
     public void testResolve() throws Exception {
         //falling back on the 'memberOf' attribute
         UserAttributeGroupsResolver resolver = new UserAttributeGroupsResolver(config(REALM_ID, Settings.EMPTY));
@@ -42,7 +41,6 @@ public void testResolve() throws Exception {
                 containsString("Philanthropists")));
     }
 
-    @SuppressWarnings("unchecked")
     public void testResolveFromPreloadedAttributes() throws Exception {
         SearchRequest preSearch = new SearchRequest(BRUCE_BANNER_DN, SearchScope.BASE, LdapUtils.OBJECT_CLASS_PRESENCE_FILTER, "memberOf");
         final Collection<Attribute> attributes = ldapConnection.searchForEntry(preSearch).getAttributes();
@@ -57,7 +55,6 @@ public void testResolveFromPreloadedAttributes() throws Exception {
                 containsString("Philanthropists")));
     }
 
-    @SuppressWarnings("unchecked")
     public void testResolveCustomGroupAttribute() throws Exception {
         Settings settings = Settings.builder()
                 .put("user_group_attribute", "seeAlso")
diff --git a/x-pack/qa/third-party/jira/build.gradle b/x-pack/qa/third-party/jira/build.gradle
index 43667300a3383..c503981a1a8c2 100644
--- a/x-pack/qa/third-party/jira/build.gradle
+++ b/x-pack/qa/third-party/jira/build.gradle
@@ -3,31 +3,27 @@ import groovy.json.JsonSlurper
 import javax.net.ssl.HttpsURLConnection
 import java.nio.charset.StandardCharsets
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile "org.elasticsearch.plugin:x-pack-core:${version}"
+  testCompile project(':x-pack:plugin:core')
   testCompile project(path: xpackModule('watcher'), configuration: 'runtime')
 }
 
-ext {
-  jiraUrl      = System.getenv('jira_url')
-  jiraUser     = System.getenv('jira_user')
-  jiraPassword = System.getenv('jira_password')
-  jiraProject  = System.getenv('jira_project')
-}
 
-integTestCluster {
+String jiraUrl = System.getenv('jira_url')
+String jiraUser = System.getenv('jira_user')
+String jiraPassword = System.getenv('jira_password')
+String jiraProject = System.getenv('jira_project')
+
+testClusters.integTest {
   setting 'xpack.security.enabled', 'false'
   setting 'xpack.monitoring.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG'
-  keystoreSetting 'xpack.notification.jira.account.test.secure_url', jiraUrl
-  keystoreSetting 'xpack.notification.jira.account.test.secure_user', jiraUser
-  keystoreSetting 'xpack.notification.jira.account.test.secure_password', jiraPassword
-  setting 'xpack.notification.jira.account.test.issue_defaults.project.key', jiraProject
   setting 'xpack.notification.jira.account.test.issue_defaults.issuetype.name', 'Bug'
   setting 'xpack.notification.jira.account.test.issue_defaults.labels.0', 'integration-tests'
 }
@@ -49,7 +45,13 @@ if (!jiraUrl && !jiraUser && !jiraPassword && !jiraProject) {
     integTest.enabled = false
     testingConventions.enabled = false
 } else {
-    integTestRunner.finalizedBy cleanJira
+    testClusters.integTest {
+        setting 'xpack.notification.jira.account.test.issue_defaults.project.key', jiraProject
+        keystore 'xpack.notification.jira.account.test.secure_url', jiraUrl
+        keystore 'xpack.notification.jira.account.test.secure_user', jiraUser
+        keystore 'xpack.notification.jira.account.test.secure_password', jiraPassword
+    }
+    integTest.runner.finalizedBy cleanJira
 }
 
 /** List all issues associated to a given Jira project **/
diff --git a/x-pack/qa/third-party/pagerduty/build.gradle b/x-pack/qa/third-party/pagerduty/build.gradle
index 9013d8c281538..ebee73f9eaaed 100644
--- a/x-pack/qa/third-party/pagerduty/build.gradle
+++ b/x-pack/qa/third-party/pagerduty/build.gradle
@@ -1,23 +1,24 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile "org.elasticsearch.plugin:x-pack-core:${version}"
+  testCompile project(':x-pack:plugin:core')
   testCompile project(path: xpackModule('watcher'), configuration: 'runtime')
 }
 
 String pagerDutyServiceKey = System.getenv('pagerduty_service_api_key')
 
-integTestCluster {
-  setting 'xpack.security.enabled', 'false'
-  setting 'xpack.monitoring.enabled', 'false'
-  setting 'xpack.ml.enabled', 'false'
-  setting 'xpack.license.self_generated.type', 'trial'
-  setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG'
-  keystoreSetting 'xpack.notification.pagerduty.account.test_account.secure_service_api_key', pagerDutyServiceKey
-}
-
 if (!pagerDutyServiceKey) {
   integTest.enabled = false
   testingConventions.enabled = false
+} else {
+  testClusters.integTest {
+    setting 'xpack.security.enabled', 'false'
+    setting 'xpack.monitoring.enabled', 'false'
+    setting 'xpack.ml.enabled', 'false'
+    setting 'xpack.license.self_generated.type', 'trial'
+    setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG'
+    keystore 'xpack.notification.pagerduty.account.test_account.secure_service_api_key', pagerDutyServiceKey
+  }
 }
diff --git a/x-pack/qa/third-party/slack/build.gradle b/x-pack/qa/third-party/slack/build.gradle
index 9fdfaeb826667..de0550be81943 100644
--- a/x-pack/qa/third-party/slack/build.gradle
+++ b/x-pack/qa/third-party/slack/build.gradle
@@ -1,23 +1,24 @@
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-  testCompile "org.elasticsearch.plugin:x-pack-core:${version}"
+  testCompile project(':x-pack:plugin:core')
   testCompile project(path: xpackModule('watcher'), configuration: 'runtime')
 }
 
 String slackUrl = System.getenv('slack_url')
 
-integTestCluster {
-  setting 'xpack.security.enabled', 'false'
-  setting 'xpack.monitoring.enabled', 'false'
-  setting 'xpack.ml.enabled', 'false'
-  setting 'xpack.license.self_generated.type', 'trial'
-  setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG'
-  keystoreSetting 'xpack.notification.slack.account.test_account.secure_url', slackUrl
-}
-
 if (!slackUrl) {
   integTest.enabled = false
   testingConventions.enabled = false
+} else {
+  testClusters.integTest {
+    setting 'xpack.security.enabled', 'false'
+    setting 'xpack.monitoring.enabled', 'false'
+    setting 'xpack.ml.enabled', 'false'
+    setting 'xpack.license.self_generated.type', 'trial'
+    setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG'
+    keystore  'xpack.notification.slack.account.test_account.secure_url', slackUrl
+  }
 }
diff --git a/x-pack/test/feature-aware/build.gradle b/x-pack/test/feature-aware/build.gradle
index e3c7ae96063d0..9d7f1504418d2 100644
--- a/x-pack/test/feature-aware/build.gradle
+++ b/x-pack/test/feature-aware/build.gradle
@@ -2,9 +2,9 @@ apply plugin: 'elasticsearch.build'
 
 dependencies {
   compile 'org.ow2.asm:asm:7.1'
-  compile "org.elasticsearch:elasticsearch:${version}"
-  compile "org.elasticsearch.plugin:x-pack-core:${version}"
-  testCompile "org.elasticsearch.test:framework:${version}"
+  compile project(':server')
+  compile project(':x-pack:plugin:core')
+  testCompile project(':test:framework')
 }
 
 forbiddenApisMain.enabled = true

From ca5cfa0e7b7adf9029afe4c170ab799da2146184 Mon Sep 17 00:00:00 2001
From: Bukhtawar Khan <bukhtawa@amazon.com>
Date: Mon, 8 Jul 2019 22:22:46 +0530
Subject: [PATCH 4/9] Resolving merge conflicts

---
 .../allocation/DiskThresholdMonitor.java      |  82 ++++++++--
 .../allocation/DiskThresholdMonitorTests.java | 149 +++++++++++++++---
 .../allocation/decider/MockDiskUsagesIT.java  |   3 +
 3 files changed, 194 insertions(+), 40 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
index 7177cf8bef4be..0fa94ca4e16a0 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
@@ -33,19 +33,23 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.RerouteService;
 import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.LongSupplier;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 /**
  * Listens for a node to go over the high watermark and kicks off an empty
@@ -64,6 +68,7 @@ public class DiskThresholdMonitor {
     private final RerouteService rerouteService;
     private final AtomicLong lastRunTimeMillis = new AtomicLong(Long.MIN_VALUE);
     private final AtomicBoolean checkInProgress = new AtomicBoolean();
+    private final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
 
     public DiskThresholdMonitor(Settings settings, Supplier<ClusterState> clusterStateSupplier, ClusterSettings clusterSettings,
                                 Client client, LongSupplier currentTimeMillisSupplier, RerouteService rerouteService) {
@@ -135,14 +140,23 @@ public void onNewInfo(ClusterInfo info) {
         }
         final ClusterState state = clusterStateSupplier.get();
         final Set<String> indicesToMarkReadOnly = new HashSet<>();
+        RoutingNodes routingNodes = state.getRoutingNodes();
+        Set<String> indicesToMarkIneligibleForAutoRelease = new HashSet<>();
+        //Ensure we release indices on nodes that have a usage response from node stats
+        markNodesMissingUsageIneligibleForRelease(routingNodes, usages, indicesToMarkIneligibleForAutoRelease);
 
         for (final ObjectObjectCursor<String, DiskUsage> entry : usages) {
             final String node = entry.key;
             final DiskUsage usage = entry.value;
             warnAboutDiskIfNeeded(usage);
+            RoutingNode routingNode = routingNodes.node(node);
+            // Only unblock index if all nodes that contain shards of it are below the high disk watermark
+            if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes()
+                || usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) {
+                markIneligiblityForAutoRelease(routingNode, indicesToMarkIneligibleForAutoRelease);
+            }
             if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdFloodStage().getBytes() ||
                 usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdFloodStage()) {
-                final RoutingNode routingNode = state.getRoutingNodes().node(node);
                 if (routingNode != null) { // this might happen if we haven't got the full cluster-state yet?!
                     for (ShardRouting routing : routingNode) {
                         indicesToMarkReadOnly.add(routing.index().getName());
@@ -181,7 +195,7 @@ public void onNewInfo(ClusterInfo info) {
             }
         }
 
-        final ActionListener<Void> listener = new GroupedActionListener<>(ActionListener.wrap(this::checkFinished), 2);
+        final ActionListener<Void> listener = new GroupedActionListener<>(ActionListener.wrap(this::checkFinished), 3);
 
         if (reroute) {
             logger.info("rerouting shards: [{}]", explanation);
@@ -196,30 +210,70 @@ public void onNewInfo(ClusterInfo info) {
         } else {
             listener.onResponse(null);
         }
+        // Get set of indices that are eligible to be automatically unblocked
+        // Only collect indices that are currently blocked
+        final String[] indices = state.routingTable().indicesRouting().keys().toArray(String.class);
+        Set<String> indicesToAutoRelease = Arrays.stream(indices)
+            .filter(index -> indicesToMarkIneligibleForAutoRelease.contains(index) == false)
+            .filter(index -> state.getBlocks().hasIndexBlock(index, IndexMetaData.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK))
+            .collect(Collectors.toCollection(HashSet::new));
+
+        if (indicesToAutoRelease.isEmpty() == false) {
+            if (diskThresholdSettings.isAutoReleaseIndexEnabled()) {
+                logger.info("Releasing read-only allow delete block on indices: [{}]", indicesToAutoRelease);
+                updateIndicesReadOnly(indicesToAutoRelease, listener, false);
+            } else {
+                deprecationLogger.deprecated("[{}] will be removed in 8.0.0", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY);
+                listener.onResponse(null);
+            }
+        } else {
+            listener.onResponse(null);
+        }
 
         indicesToMarkReadOnly.removeIf(index -> state.getBlocks().indexBlocked(ClusterBlockLevel.WRITE, index));
         if (indicesToMarkReadOnly.isEmpty() == false) {
-            markIndicesReadOnly(indicesToMarkReadOnly, ActionListener.wrap(r -> {
-                setLastRunTimeMillis();
-                listener.onResponse(r);
-            }, e -> {
-                logger.debug("marking indices readonly failed", e);
-                setLastRunTimeMillis();
-                listener.onFailure(e);
-            }));
+            updateIndicesReadOnly(indicesToMarkReadOnly, listener, true);
         } else {
             listener.onResponse(null);
         }
     }
 
+    private void markNodesMissingUsageIneligibleForRelease(RoutingNodes routingNodes, ImmutableOpenMap<String, DiskUsage> usages,
+                                                           Set<String> indicesToMarkIneligibleForAutoRelease) {
+        for (RoutingNode routingNode : routingNodes) {
+            if (usages.containsKey(routingNode.nodeId()) == false) {
+                markIneligiblityForAutoRelease(routingNode, indicesToMarkIneligibleForAutoRelease);
+            }
+        }
+
+    }
+
+    private void markIneligiblityForAutoRelease(RoutingNode routingNode, Set<String> indicesToMarkIneligibleForAutoRelease) {
+        if (routingNode != null) {
+            for (ShardRouting routing : routingNode) {
+                String indexName = routing.index().getName();
+                indicesToMarkIneligibleForAutoRelease.add(indexName);
+            }
+        }
+    }
+
     private void setLastRunTimeMillis() {
         lastRunTimeMillis.getAndUpdate(l -> Math.max(l, currentTimeMillisSupplier.getAsLong()));
     }
 
-    protected void markIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionListener<Void> listener) {
+    protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener<Void> listener, boolean readOnly) {
         // set read-only block but don't block on the response
-        client.admin().indices().prepareUpdateSettings(indicesToMarkReadOnly.toArray(Strings.EMPTY_ARRAY))
-            .setSettings(Settings.builder().put(IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE, true).build())
-            .execute(ActionListener.map(listener, r -> null));
+        String value = readOnly ? Boolean.TRUE.toString() : null;
+        ActionListener<Void> wrappedListener = ActionListener.wrap(r -> {
+            setLastRunTimeMillis();
+            listener.onResponse(r);
+        }, e -> {
+            logger.debug("marking indices read-only [{}] failed", readOnly,  e);
+            setLastRunTimeMillis();
+            listener.onFailure(e);
+        });
+        client.admin().indices().prepareUpdateSettings(indicesToUpdate.toArray(Strings.EMPTY_ARRAY)).
+            setSettings(Settings.builder().put(IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE, value).build()).
+            execute(ActionListener.map(wrappedListener, r -> null));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
index e46c899dbec91..9235e8ccaba75 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
@@ -48,30 +48,8 @@
 public class DiskThresholdMonitorTests extends ESAllocationTestCase {
 
     public void testMarkFloodStageIndicesReadOnly() {
-        AllocationService allocation = createAllocationService(Settings.builder()
-            .put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
         Settings settings = Settings.EMPTY;
-        MetaData metaData = MetaData.builder()
-            .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)
-                .put("index.routing.allocation.require._id", "node2")).numberOfShards(1).numberOfReplicas(0))
-            .put(IndexMetaData.builder("test_1").settings(settings(Version.CURRENT)
-                .put("index.routing.allocation.require._id", "node1")).numberOfShards(1).numberOfReplicas(0))
-            .put(IndexMetaData.builder("test_2").settings(settings(Version.CURRENT)
-                .put("index.routing.allocation.require._id", "node1")).numberOfShards(1).numberOfReplicas(0))
-            .build();
-        RoutingTable routingTable = RoutingTable.builder()
-            .addAsNew(metaData.index("test"))
-            .addAsNew(metaData.index("test_1"))
-            .addAsNew(metaData.index("test_2"))
-            .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
-            .metaData(metaData).routingTable(routingTable).build();
-        logger.info("adding two nodes and performing rerouting");
-        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))
-            .add(newNode("node2"))).build();
-        clusterState = allocation.reroute(clusterState, "reroute");
-        logger.info("start primary shard");
-        clusterState = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+        ClusterState clusterState = bootstrapCluster();
         ClusterState finalState = clusterState;
         AtomicBoolean reroute = new AtomicBoolean(false);
         AtomicReference<Set<String>> indices = new AtomicReference<>();
@@ -82,7 +60,7 @@ public void testMarkFloodStageIndicesReadOnly() {
             listener.onResponse(null);
         }) {
             @Override
-            protected void markIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionListener<Void> listener) {
+            protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionListener<Void> listener, boolean readOnly) {
                 assertTrue(indices.compareAndSet(null, indicesToMarkReadOnly));
                 listener.onResponse(null);
             }
@@ -122,7 +100,7 @@ protected void markIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionList
             listener.onResponse(null);
         }) {
             @Override
-            protected void markIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionListener<Void> listener) {
+            protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionListener<Void> listener, boolean readOnly) {
                 assertTrue(indices.compareAndSet(null, indicesToMarkReadOnly));
                 listener.onResponse(null);
             }
@@ -149,7 +127,7 @@ public void testDoesNotSubmitRerouteTaskTooFrequently() {
             assertTrue(listenerReference.compareAndSet(null, listener));
         }) {
             @Override
-            protected void markIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionListener<Void> listener) {
+            protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionListener<Void> listener, boolean readOnly) {
                 throw new AssertionError("unexpected");
             }
         };
@@ -219,4 +197,123 @@ protected void markIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionList
         monitor.onNewInfo(new ClusterInfo(allDisksOk, null, null, null));
         assertNull(listenerReference.get());
     }
+
+    public void testAutoReleaseIndices() {
+        Settings settings = Settings.EMPTY;
+        ClusterState clusterState = bootstrapCluster();
+
+        AtomicReference<Set<String>> indicesToMarkReadOnly = new AtomicReference<>();
+        AtomicReference<Set<String>> indicesToRelease = new AtomicReference<>();
+        // Change cluster state so that "test" index is blocked (read only)
+        IndexMetaData indexMetaData = IndexMetaData.builder(clusterState.metaData().index("test")).settings(Settings.builder()
+            .put(clusterState.metaData()
+                .index("test").getSettings())
+            .put(IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), true)).build();
+
+        final ClusterState anotherFinalClusterState = ClusterState.builder(clusterState).metaData(MetaData.builder(clusterState.metaData())
+            .put(clusterState.metaData().index("test_1"), false)
+            .put(clusterState.metaData().index("test_2"), false)
+            .put(indexMetaData, true).build())
+            .blocks(ClusterBlocks.builder().addBlocks(indexMetaData).build()).build();
+
+        assertTrue(anotherFinalClusterState.blocks().indexBlocked(ClusterBlockLevel.WRITE, "test"));
+
+        AtomicLong currentTime = new AtomicLong();
+        DiskThresholdMonitor monitor = new DiskThresholdMonitor(settings, () -> anotherFinalClusterState,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, currentTime::get, (reason, listener) -> {
+            assertNotNull(listener);
+            listener.onResponse(null);
+        }){
+            @Override
+            protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener<Void> listener, boolean readOnly) {
+                if (readOnly) {
+                    assertTrue(indicesToMarkReadOnly.compareAndSet(null, indicesToUpdate));
+                } else {
+                    assertTrue(indicesToRelease.compareAndSet(null, indicesToUpdate));
+                }
+                listener.onResponse(null);
+            }
+        };
+
+        // When free disk on node2 goes above threshold (10% high watermark in this test case), index on node2 marked for
+        // auto release
+        ImmutableOpenMap.Builder<String, DiskUsage> builder = ImmutableOpenMap.builder();
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 4));
+        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 11));
+        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
+        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
+        assertEquals(new HashSet<>(Arrays.asList("test")), indicesToRelease.get());
+
+        // When free disk on node1 and node2 goes below threshold (5% flood watermark in this test case).
+        // index with read only block isn't marked again to be read only and no indices to be auto-released
+        indicesToMarkReadOnly.set(null);
+        indicesToRelease.set(null);
+        builder = ImmutableOpenMap.builder();
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 4));
+        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 4));
+        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
+        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
+        assertNull(indicesToRelease.get());
+
+        // When free disk on node1 and node2 goes above threshold (10% high watermark in this test case).
+        // index with read only block to be auto-released
+        indicesToMarkReadOnly.set(null);
+        indicesToRelease.set(null);
+        builder = ImmutableOpenMap.builder();
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 14));
+        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 14));
+        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
+        assertNull(indicesToMarkReadOnly.get());
+        assertEquals(new HashSet<>(Arrays.asList("test")), indicesToRelease.get());
+
+
+        //Remove index block settings from cluster state
+        indexMetaData = IndexMetaData.builder(clusterState.metaData().index("test")).settings(Settings.builder()
+            .put(clusterState.metaData()
+                .index("test").getSettings())
+            .put(IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), false)).build();
+
+        ClusterState clusterStateWithoutBlocks = ClusterState.builder(clusterState).metaData(MetaData.builder(clusterState.metaData())
+            .put(clusterState.metaData().index("test_1"), false)
+            .put(clusterState.metaData().index("test_2"), false)
+            .put(indexMetaData, true).build())
+            .blocks(ClusterBlocks.builder().addBlocks(indexMetaData).build()).build();
+
+        assertFalse(clusterStateWithoutBlocks.blocks().indexBlocked(ClusterBlockLevel.WRITE, "test"));
+        indicesToMarkReadOnly.set(null);
+        indicesToRelease.set(null);
+        builder = ImmutableOpenMap.builder();
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 4));
+        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 4));
+        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
+        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
+        assertNull(indicesToRelease.get());
+    }
+
+    private ClusterState bootstrapCluster() {
+        AllocationService allocation = createAllocationService(Settings.builder()
+            .put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
+        Settings settings = Settings.EMPTY;
+        MetaData metaData = MetaData.builder()
+            .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)
+                .put("index.routing.allocation.require._id", "node2")).numberOfShards(1).numberOfReplicas(0))
+            .put(IndexMetaData.builder("test_1").settings(settings(Version.CURRENT)
+                .put("index.routing.allocation.require._id", "node1")).numberOfShards(1).numberOfReplicas(0))
+            .put(IndexMetaData.builder("test_2").settings(settings(Version.CURRENT)
+                .put("index.routing.allocation.require._id", "node1")).numberOfShards(1).numberOfReplicas(0))
+            .build();
+        RoutingTable routingTable = RoutingTable.builder()
+            .addAsNew(metaData.index("test"))
+            .addAsNew(metaData.index("test_1"))
+            .addAsNew(metaData.index("test_2"))
+            .build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .metaData(metaData).routingTable(routingTable).build();
+        logger.info("adding two nodes and performing rerouting");
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))
+            .add(newNode("node2"))).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
+        logger.info("start primary shard");
+        return allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
index 3d9e77e48b2b9..2a0113312fdfd 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.cluster.routing.allocation.decider;
 
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterState;
@@ -37,6 +39,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 

From 041e7a8b514810337ba4c18e2007a5853ef07036 Mon Sep 17 00:00:00 2001
From: Bukhtawar Khan <bukhtawa@amazon.com>
Date: Wed, 10 Jul 2019 14:48:03 +0530
Subject: [PATCH 5/9] Addressed minor comments

---
 .../allocation/DiskThresholdMonitor.java      | 32 +++++++++++--------
 .../allocation/DiskThresholdSettings.java     |  2 +-
 .../allocation/DiskThresholdMonitorTests.java |  5 +--
 3 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
index 0fa94ca4e16a0..1a6e5992b5dd0 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
@@ -23,6 +23,7 @@
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.GroupedActionListener;
 import org.elasticsearch.client.Client;
@@ -42,7 +43,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 
-import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -50,6 +50,7 @@
 import java.util.function.LongSupplier;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
 /**
  * Listens for a node to go over the high watermark and kicks off an empty
@@ -77,6 +78,9 @@ public DiskThresholdMonitor(Settings settings, Supplier<ClusterState> clusterSta
         this.rerouteService = rerouteService;
         this.diskThresholdSettings = new DiskThresholdSettings(settings, clusterSettings);
         this.client = client;
+        if(diskThresholdSettings.isAutoReleaseIndexEnabled() == false) {
+            deprecationLogger.deprecated("[{}] will be removed in version {}", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, Version.V_7_4_0.major + 1);
+        }
     }
 
     /**
@@ -142,7 +146,6 @@ public void onNewInfo(ClusterInfo info) {
         final Set<String> indicesToMarkReadOnly = new HashSet<>();
         RoutingNodes routingNodes = state.getRoutingNodes();
         Set<String> indicesToMarkIneligibleForAutoRelease = new HashSet<>();
-        //Ensure we release indices on nodes that have a usage response from node stats
         markNodesMissingUsageIneligibleForRelease(routingNodes, usages, indicesToMarkIneligibleForAutoRelease);
 
         for (final ObjectObjectCursor<String, DiskUsage> entry : usages) {
@@ -210,20 +213,20 @@ public void onNewInfo(ClusterInfo info) {
         } else {
             listener.onResponse(null);
         }
-        // Get set of indices that are eligible to be automatically unblocked
-        // Only collect indices that are currently blocked
-        final String[] indices = state.routingTable().indicesRouting().keys().toArray(String.class);
-        Set<String> indicesToAutoRelease = Arrays.stream(indices)
+        Set<String> indicesToAutoRelease = StreamSupport.stream(state.routingTable().indicesRouting()
+            .spliterator(), false)
+            .map(c -> c.key)
             .filter(index -> indicesToMarkIneligibleForAutoRelease.contains(index) == false)
             .filter(index -> state.getBlocks().hasIndexBlock(index, IndexMetaData.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK))
-            .collect(Collectors.toCollection(HashSet::new));
+            .collect(Collectors.toSet());
 
         if (indicesToAutoRelease.isEmpty() == false) {
             if (diskThresholdSettings.isAutoReleaseIndexEnabled()) {
-                logger.info("Releasing read-only allow delete block on indices: [{}]", indicesToAutoRelease);
+                logger.info("releasing read-only allow delete block on indices: [{}]", indicesToAutoRelease);
                 updateIndicesReadOnly(indicesToAutoRelease, listener, false);
             } else {
-                deprecationLogger.deprecated("[{}] will be removed in 8.0.0", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY);
+                deprecationLogger.deprecated("[{}] will be removed in version {}", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, Version.V_7_4_0.major + 1);
+                logger.debug("[{}] disabled, not releasing read-only allow delete block on indices: [{}]", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, indicesToAutoRelease);
                 listener.onResponse(null);
             }
         } else {
@@ -263,17 +266,18 @@ private void setLastRunTimeMillis() {
 
     protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener<Void> listener, boolean readOnly) {
         // set read-only block but don't block on the response
-        String value = readOnly ? Boolean.TRUE.toString() : null;
         ActionListener<Void> wrappedListener = ActionListener.wrap(r -> {
             setLastRunTimeMillis();
             listener.onResponse(r);
         }, e -> {
-            logger.debug("marking indices read-only [{}] failed", readOnly,  e);
+            logger.debug("marking indices read-only [{}] failed", readOnly, e);
             setLastRunTimeMillis();
             listener.onFailure(e);
         });
-        client.admin().indices().prepareUpdateSettings(indicesToUpdate.toArray(Strings.EMPTY_ARRAY)).
-            setSettings(Settings.builder().put(IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE, value).build()).
-            execute(ActionListener.map(wrappedListener, r -> null));
+        Settings readOnlySettings = readOnly ? Settings.builder().put(IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE, Boolean.TRUE.toString()).build() :
+            Settings.builder().putNull(IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE).build();
+        client.admin().indices().prepareUpdateSettings(indicesToUpdate.toArray(Strings.EMPTY_ARRAY))
+            .setSettings(readOnlySettings)
+            .execute(ActionListener.map(wrappedListener, r -> null));
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java
index ebd4b90996d8e..2477763a3138c 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdSettings.java
@@ -72,7 +72,7 @@ public class DiskThresholdSettings {
     private volatile TimeValue rerouteInterval;
     private volatile Double freeDiskThresholdFloodStage;
     private volatile ByteSizeValue freeBytesThresholdFloodStage;
-    private static boolean autoReleaseIndexEnabled;
+    private static final boolean autoReleaseIndexEnabled;
     public static final String AUTO_RELEASE_INDEX_ENABLED_KEY = "es.disk.auto_release_flood_stage_block";
 
     static {
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
index 9235e8ccaba75..db98a4638eead 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
@@ -50,11 +50,10 @@ public class DiskThresholdMonitorTests extends ESAllocationTestCase {
     public void testMarkFloodStageIndicesReadOnly() {
         Settings settings = Settings.EMPTY;
         ClusterState clusterState = bootstrapCluster();
-        ClusterState finalState = clusterState;
         AtomicBoolean reroute = new AtomicBoolean(false);
         AtomicReference<Set<String>> indices = new AtomicReference<>();
         AtomicLong currentTime = new AtomicLong();
-        DiskThresholdMonitor monitor = new DiskThresholdMonitor(settings, () -> finalState,
+        DiskThresholdMonitor monitor = new DiskThresholdMonitor(settings, () -> clusterState,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, currentTime::get, (reason, listener) -> {
             assertTrue(reroute.compareAndSet(false, true));
             listener.onResponse(null);
@@ -62,6 +61,7 @@ public void testMarkFloodStageIndicesReadOnly() {
             @Override
             protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionListener<Void> listener, boolean readOnly) {
                 assertTrue(indices.compareAndSet(null, indicesToMarkReadOnly));
+                assertTrue(readOnly);
                 listener.onResponse(null);
             }
         };
@@ -102,6 +102,7 @@ protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionLi
             @Override
             protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionListener<Void> listener, boolean readOnly) {
                 assertTrue(indices.compareAndSet(null, indicesToMarkReadOnly));
+                assertTrue(readOnly);
                 listener.onResponse(null);
             }
         };

From 6d4228af1482e0d78891594398c6f09ac7fb7aa0 Mon Sep 17 00:00:00 2001
From: Bukhtawar Khan <bukhtawa@amazon.com>
Date: Wed, 10 Jul 2019 23:08:07 +0530
Subject: [PATCH 6/9] Simplified if-block

---
 .../allocation/DiskThresholdMonitor.java      | 21 +++++++++----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
index 1a6e5992b5dd0..a7ef23fd8f22b 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
@@ -78,7 +78,7 @@ public DiskThresholdMonitor(Settings settings, Supplier<ClusterState> clusterSta
         this.rerouteService = rerouteService;
         this.diskThresholdSettings = new DiskThresholdSettings(settings, clusterSettings);
         this.client = client;
-        if(diskThresholdSettings.isAutoReleaseIndexEnabled() == false) {
+        if (diskThresholdSettings.isAutoReleaseIndexEnabled() == false) {
             deprecationLogger.deprecated("[{}] will be removed in version {}", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, Version.V_7_4_0.major + 1);
         }
     }
@@ -145,21 +145,18 @@ public void onNewInfo(ClusterInfo info) {
         final ClusterState state = clusterStateSupplier.get();
         final Set<String> indicesToMarkReadOnly = new HashSet<>();
         RoutingNodes routingNodes = state.getRoutingNodes();
-        Set<String> indicesToMarkIneligibleForAutoRelease = new HashSet<>();
-        markNodesMissingUsageIneligibleForRelease(routingNodes, usages, indicesToMarkIneligibleForAutoRelease);
+        Set<String> indicesNotToAutoRelease = new HashSet<>();
+        markNodesMissingUsageIneligibleForRelease(routingNodes, usages, indicesNotToAutoRelease);
 
         for (final ObjectObjectCursor<String, DiskUsage> entry : usages) {
             final String node = entry.key;
             final DiskUsage usage = entry.value;
             warnAboutDiskIfNeeded(usage);
             RoutingNode routingNode = routingNodes.node(node);
-            // Only unblock index if all nodes that contain shards of it are below the high disk watermark
-            if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes()
-                || usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) {
-                markIneligiblityForAutoRelease(routingNode, indicesToMarkIneligibleForAutoRelease);
-            }
             if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdFloodStage().getBytes() ||
                 usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdFloodStage()) {
+                // Only unblock index if all nodes that contain shards of it are below the high disk watermark
+                markIneligiblityForAutoRelease(routingNode, indicesNotToAutoRelease);
                 if (routingNode != null) { // this might happen if we haven't got the full cluster-state yet?!
                     for (ShardRouting routing : routingNode) {
                         indicesToMarkReadOnly.add(routing.index().getName());
@@ -167,6 +164,8 @@ public void onNewInfo(ClusterInfo info) {
                 }
             } else if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes() ||
                 usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) {
+                // Only unblock index if all nodes that contain shards of it are below the high disk watermark
+                markIneligiblityForAutoRelease(routingNode, indicesNotToAutoRelease);
                 if (lastRunTimeMillis.get() < currentTimeMillis - diskThresholdSettings.getRerouteInterval().millis()) {
                     reroute = true;
                     explanation = "high disk watermark exceeded on one or more nodes";
@@ -216,17 +215,17 @@ public void onNewInfo(ClusterInfo info) {
         Set<String> indicesToAutoRelease = StreamSupport.stream(state.routingTable().indicesRouting()
             .spliterator(), false)
             .map(c -> c.key)
-            .filter(index -> indicesToMarkIneligibleForAutoRelease.contains(index) == false)
+            .filter(index -> indicesNotToAutoRelease.contains(index) == false)
             .filter(index -> state.getBlocks().hasIndexBlock(index, IndexMetaData.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK))
             .collect(Collectors.toSet());
 
         if (indicesToAutoRelease.isEmpty() == false) {
             if (diskThresholdSettings.isAutoReleaseIndexEnabled()) {
-                logger.info("releasing read-only allow delete block on indices: [{}]", indicesToAutoRelease);
+                logger.info("releasing read-only-allow-delete block on indices: [{}]", indicesToAutoRelease);
                 updateIndicesReadOnly(indicesToAutoRelease, listener, false);
             } else {
                 deprecationLogger.deprecated("[{}] will be removed in version {}", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, Version.V_7_4_0.major + 1);
-                logger.debug("[{}] disabled, not releasing read-only allow delete block on indices: [{}]", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, indicesToAutoRelease);
+                logger.debug("[{}] disabled, not releasing read-only-allow-delete block on indices: [{}]", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, indicesToAutoRelease);
                 listener.onResponse(null);
             }
         } else {

From a6334c8bae57eaed14a28f6102e32de84c906b72 Mon Sep 17 00:00:00 2001
From: Bukhtawar Khan <bukhtawa@amazon.com>
Date: Thu, 18 Jul 2019 21:36:57 +0530
Subject: [PATCH 7/9] Incorporating test case comments

---
 .../allocation/DiskThresholdMonitor.java      |   3 +-
 .../allocation/DiskThresholdMonitorTests.java | 181 ++++++++++++++++--
 .../allocation/decider/MockDiskUsagesIT.java  |  28 ++-
 3 files changed, 174 insertions(+), 38 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
index 8f2c3bf2bb410..93cfc31330903 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
@@ -154,9 +154,9 @@ public void onNewInfo(ClusterInfo info) {
             final DiskUsage usage = entry.value;
             warnAboutDiskIfNeeded(usage);
             RoutingNode routingNode = routingNodes.node(node);
+            // Only unblock index if all nodes that contain shards of it are below the high disk watermark
             if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdFloodStage().getBytes() ||
                 usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdFloodStage()) {
-                // Only unblock index if all nodes that contain shards of it are below the high disk watermark
                 markIneligiblityForAutoRelease(routingNode, indicesNotToAutoRelease);
                 if (routingNode != null) { // this might happen if we haven't got the full cluster-state yet?!
                     for (ShardRouting routing : routingNode) {
@@ -165,7 +165,6 @@ public void onNewInfo(ClusterInfo info) {
                 }
             } else if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes() ||
                 usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) {
-                // Only unblock index if all nodes that contain shards of it are below the high disk watermark
                 markIneligiblityForAutoRelease(routingNode, indicesNotToAutoRelease);
                 if (lastRunTimeMillis.get() < currentTimeMillis - diskThresholdSettings.getRerouteInterval().millis()) {
                     reroute = true;
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
index 5a9f9e8e27615..89c36a54cbf8e 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
@@ -31,6 +31,7 @@
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -49,12 +50,11 @@
 public class DiskThresholdMonitorTests extends ESAllocationTestCase {
 
     public void testMarkFloodStageIndicesReadOnly() {
-        Settings settings = Settings.EMPTY;
         ClusterState clusterState = bootstrapCluster();
         AtomicBoolean reroute = new AtomicBoolean(false);
         AtomicReference<Set<String>> indices = new AtomicReference<>();
         AtomicLong currentTime = new AtomicLong();
-        DiskThresholdMonitor monitor = new DiskThresholdMonitor(settings, () -> clusterState,
+        DiskThresholdMonitor monitor = new DiskThresholdMonitor(Settings.EMPTY, () -> clusterState,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, currentTime::get,
             (reason, priority, listener) -> {
                 assertTrue(reroute.compareAndSet(false, true));
@@ -98,7 +98,7 @@ protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionLi
             .blocks(ClusterBlocks.builder().addBlocks(indexMetaData).build()).build();
         assertTrue(anotherFinalClusterState.blocks().indexBlocked(ClusterBlockLevel.WRITE, "test_2"));
 
-        monitor = new DiskThresholdMonitor(settings, () -> anotherFinalClusterState,
+        monitor = new DiskThresholdMonitor(Settings.EMPTY, () -> anotherFinalClusterState,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, currentTime::get,
             (reason, priority, listener) -> {
                 assertTrue(reroute.compareAndSet(false, true));
@@ -207,10 +207,8 @@ protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionLi
         assertNull(listenerReference.get());
     }
 
-    public void testAutoReleaseIndices() {
-        Settings settings = Settings.EMPTY;
+    public void testAutoReleaseSingleShardIndices() {
         ClusterState clusterState = bootstrapCluster();
-
         AtomicReference<Set<String>> indicesToMarkReadOnly = new AtomicReference<>();
         AtomicReference<Set<String>> indicesToRelease = new AtomicReference<>();
         // Change cluster state so that "test" index is blocked (read only)
@@ -228,9 +226,10 @@ public void testAutoReleaseIndices() {
         assertTrue(anotherFinalClusterState.blocks().indexBlocked(ClusterBlockLevel.WRITE, "test"));
 
         AtomicLong currentTime = new AtomicLong();
-        DiskThresholdMonitor monitor = new DiskThresholdMonitor(settings, () -> anotherFinalClusterState,
-            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, currentTime::get, (reason, listener) -> {
+        DiskThresholdMonitor monitor = new DiskThresholdMonitor(Settings.EMPTY, () -> anotherFinalClusterState,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, currentTime::get, (reason, priority, listener) -> {
             assertNotNull(listener);
+            assertThat(priority, equalTo(Priority.HIGH));
             listener.onResponse(null);
         }){
             @Override
@@ -247,34 +246,42 @@ protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener
         // When free disk on node2 goes above threshold (10% high watermark in this test case), index on node2 marked for
         // auto release
         ImmutableOpenMap.Builder<String, DiskUsage> builder = ImmutableOpenMap.builder();
-        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 4));
-        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 11));
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
+        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(10, 100)));
         monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
         assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
         assertEquals(new HashSet<>(Arrays.asList("test")), indicesToRelease.get());
 
-        // When free disk on node1 and node2 goes below threshold (5% flood watermark in this test case).
+        // When free disk on node1 goes below(5% flood watermark in this test case) and node2 stays below 10% high watermark threshold,
         // index with read only block isn't marked again to be read only and no indices to be auto-released
         indicesToMarkReadOnly.set(null);
         indicesToRelease.set(null);
         builder = ImmutableOpenMap.builder();
-        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 4));
-        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 4));
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
+        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(0, 9)));
         monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
         assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
         assertNull(indicesToRelease.get());
 
-        // When free disk on node1 and node2 goes above threshold (10% high watermark in this test case).
+        // When free disk on node1 goes above 5% flood watermark and node2 goes above 10% high watermark,
         // index with read only block to be auto-released
         indicesToMarkReadOnly.set(null);
         indicesToRelease.set(null);
         builder = ImmutableOpenMap.builder();
-        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 14));
-        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 14));
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(5, 100)));
+        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(10, 100)));
         monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
         assertNull(indicesToMarkReadOnly.get());
         assertEquals(new HashSet<>(Arrays.asList("test")), indicesToRelease.get());
 
+        //When no usage information is present for node2, we don't release the block
+        indicesToMarkReadOnly.set(null);
+        indicesToRelease.set(null);
+        builder = ImmutableOpenMap.builder();
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
+        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
+        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
+        assertNull(indicesToRelease.get());
 
         //Remove index block settings from cluster state
         indexMetaData = IndexMetaData.builder(clusterState.metaData().index("test")).settings(Settings.builder()
@@ -288,12 +295,145 @@ protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener
             .put(indexMetaData, true).build())
             .blocks(ClusterBlocks.builder().addBlocks(indexMetaData).build()).build();
 
-        assertFalse(clusterStateWithoutBlocks.blocks().indexBlocked(ClusterBlockLevel.WRITE, "test"));
+         assertFalse(clusterStateWithoutBlocks.blocks().indexBlocked(ClusterBlockLevel.WRITE, "test"));
+
+         monitor = new DiskThresholdMonitor(Settings.EMPTY, () -> clusterStateWithoutBlocks,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, currentTime::get, (reason, priority, listener) -> {
+            assertNotNull(listener);
+            assertThat(priority, equalTo(Priority.HIGH));
+            listener.onResponse(null);
+        }){
+            @Override
+            protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener<Void> listener, boolean readOnly) {
+                if (readOnly) {
+                    assertTrue(indicesToMarkReadOnly.compareAndSet(null, indicesToUpdate));
+                } else {
+                    assertTrue(indicesToRelease.compareAndSet(null, indicesToUpdate));
+                }
+                listener.onResponse(null);
+            }
+        };
+        indicesToMarkReadOnly.set(null);
+        indicesToRelease.set(null);
+        builder = ImmutableOpenMap.builder();
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
+        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(0, 4)));
+        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
+        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2", "test")), indicesToMarkReadOnly.get());
+        assertNull(indicesToRelease.get());
+    }
+
+    public void testAutoReleaseMultiShardIndices(){
+        AtomicReference<Set<String>> indicesToMarkReadOnly = new AtomicReference<>();
+        AtomicReference<Set<String>> indicesToRelease = new AtomicReference<>();
+        AllocationService allocation = createAllocationService(Settings.builder()
+            .put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
+        MetaData metaData = MetaData.builder()
+            .put(IndexMetaData.builder("test_1").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1))
+            .put(IndexMetaData.builder("test_2").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1))
+            .build();
+        RoutingTable routingTable = RoutingTable.builder()
+            .addAsNew(metaData.index("test_1"))
+            .addAsNew(metaData.index("test_2"))
+            .build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .metaData(metaData).routingTable(routingTable).build();
+        logger.info("adding two nodes and performing rerouting");
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))
+            .add(newNode("node2"))).build();
+        clusterState = allocation.reroute(clusterState, "reroute");
+        logger.info("start primary shard");
+        clusterState = startInitializingShardsAndReroute(allocation, clusterState);
+        assertThat(clusterState.getRoutingTable().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(4));
+        logger.info("start replica shards");
+        clusterState = startInitializingShardsAndReroute(allocation, clusterState);
+        assertThat(clusterState.getRoutingTable().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(8));
+
+        // Change cluster state so that "test_2" index is blocked (read only)
+        IndexMetaData indexMetaData = IndexMetaData.builder(clusterState.metaData().index("test_2")).settings(Settings.builder()
+            .put(clusterState.metaData()
+                .index("test_2").getSettings())
+            .put(IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), true)).build();
+
+        ClusterState clusterStateWithBlocks = ClusterState.builder(clusterState).metaData(MetaData.builder(clusterState.metaData())
+            .put(clusterState.metaData().index("test_1"), false)
+            .put(clusterState.metaData().index("test_2"), false)
+            .put(indexMetaData, true).build())
+            .blocks(ClusterBlocks.builder().addBlocks(indexMetaData).build()).build();
+
+        assertTrue(clusterStateWithBlocks.blocks().indexBlocked(ClusterBlockLevel.WRITE, "test_2"));
+        AtomicLong currentTime = new AtomicLong();
+        DiskThresholdMonitor monitor = new DiskThresholdMonitor(Settings.EMPTY, () -> clusterStateWithBlocks,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, currentTime::get, (reason, priority, listener) -> {
+            assertNotNull(listener);
+            assertThat(priority, equalTo(Priority.HIGH));
+            listener.onResponse(null);
+        }){
+            @Override
+            protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener<Void> listener, boolean readOnly) {
+                if (readOnly) {
+                    assertTrue(indicesToMarkReadOnly.compareAndSet(null, indicesToUpdate));
+                } else {
+                    assertTrue(indicesToRelease.compareAndSet(null, indicesToUpdate));
+                }
+                listener.onResponse(null);
+            }
+        };
+
+        // When free disk on any of node1 or node2 goes below 5% flood watermark, then apply index block on indices not having the block
+        indicesToMarkReadOnly.set(null);
+        indicesToRelease.set(null);
+        ImmutableOpenMap.Builder<String, DiskUsage> builder = ImmutableOpenMap.builder();
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(10, 100)));
+        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(0, 4)));
+        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
+        assertEquals(new HashSet<>(Arrays.asList("test_1")), indicesToMarkReadOnly.get());
+        assertNull(indicesToRelease.get());
+
+        // When free disk on node1 and node2 goes above 10% high watermark, then only release index block
+        indicesToMarkReadOnly.set(null);
+        indicesToRelease.set(null);
+        builder = ImmutableOpenMap.builder();
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(10, 100)));
+        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(10, 100)));
+        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
+        assertNull(indicesToMarkReadOnly.get());
+        assertEquals(new HashSet<>(Arrays.asList("test_2")), indicesToRelease.get());
+
+        //Remove index block settings from cluster state
+        indexMetaData = IndexMetaData.builder(clusterState.metaData().index("test_1")).settings(Settings.builder()
+            .put(clusterState.metaData()
+                .index("test_1").getSettings())
+            .put(IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), false)).build();
+
+        ClusterState clusterStateWithoutBlocks = ClusterState.builder(clusterState).metaData(MetaData.builder(clusterState.metaData())
+            .put(clusterState.metaData().index("test_1"), false)
+            .put(clusterState.metaData().index("test_2"), false)
+            .put(indexMetaData, true).build())
+            .blocks(ClusterBlocks.builder().addBlocks(indexMetaData).build()).build();
+
+        assertFalse(clusterStateWithoutBlocks.blocks().indexBlocked(ClusterBlockLevel.WRITE, "test_1"));
+        monitor = new DiskThresholdMonitor(Settings.EMPTY, () -> clusterStateWithoutBlocks,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, currentTime::get, (reason, priority, listener) -> {
+            assertNotNull(listener);
+            assertThat(priority, equalTo(Priority.HIGH));
+            listener.onResponse(null);
+        }){
+            @Override
+            protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener<Void> listener, boolean readOnly) {
+                if (readOnly) {
+                    assertTrue(indicesToMarkReadOnly.compareAndSet(null, indicesToUpdate));
+                } else {
+                    assertTrue(indicesToRelease.compareAndSet(null, indicesToUpdate));
+                }
+                listener.onResponse(null);
+            }
+        };
         indicesToMarkReadOnly.set(null);
         indicesToRelease.set(null);
         builder = ImmutableOpenMap.builder();
-        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 4));
-        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 4));
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
+        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(0, 4)));
         monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
         assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
         assertNull(indicesToRelease.get());
@@ -302,7 +442,6 @@ protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener
     private ClusterState bootstrapCluster() {
         AllocationService allocation = createAllocationService(Settings.builder()
             .put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
-        Settings settings = Settings.EMPTY;
         MetaData metaData = MetaData.builder()
             .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)
                 .put("index.routing.allocation.require._id", "node2")).numberOfShards(1).numberOfReplicas(0))
@@ -323,6 +462,6 @@ private ClusterState bootstrapCluster() {
             .add(newNode("node2"))).build();
         clusterState = allocation.reroute(clusterState, "reroute");
         logger.info("start primary shard");
-        return allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+        return allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING));
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
index 2a0113312fdfd..5731b04be4c43 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
@@ -21,6 +21,7 @@
 
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterState;
@@ -32,6 +33,7 @@
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -167,11 +169,10 @@ public void testAutomaticReleaseOfIndexBlock() throws Exception {
                 DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(),
                 watermarkBytes ? "5b" : "95%")
             .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "150ms")).get();
-        // Create an index with 10 shards so we can check allocation for it
+        // Create an index with 6 shards so we can check allocation for it
         prepareCreate("test").setSettings(Settings.builder()
             .put("number_of_shards", 6)
-            .put("number_of_replicas", 0)
-            .put("index.routing.allocation.exclude._name", "")).get();
+            .put("number_of_replicas", 0)).get();
         ensureGreen("test");
 
         // Block until the "fake" cluster info is retrieved at least once
@@ -191,8 +192,8 @@ public void testAutomaticReleaseOfIndexBlock() throws Exception {
                 node.nodeId(), resp.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
         }
 
-        client().prepareIndex().setIndex("test").setType("doc").setId("1").setSource("foo", "bar").get();
-        refresh();
+        client().prepareIndex("test", "doc", "1").setSource("{\"foo\": \"bar\"}", XContentType.JSON)
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
         assertSearchHits(client().prepareSearch().get(), "1");
 
         // Block all nodes so that re-balancing does not occur (BalancedShardsAllocator)
@@ -200,35 +201,32 @@ public void testAutomaticReleaseOfIndexBlock() throws Exception {
         cis.setN2Usage(realNodeNames.get(1), new DiskUsage(nodes.get(1), "n2", "_na_", 100, 3));
         cis.setN3Usage(realNodeNames.get(2), new DiskUsage(nodes.get(2), "n3", "_na_", 100, 3));
 
-        refresh();
         // Wait until index "test" is blocked
         assertBusy(() -> {
-            assertBlocked(client().prepareIndex().setIndex("test").setType("doc").setId("2").setSource("foo", "bar"),
+            assertBlocked(client().prepareIndex().setIndex("test").setType("doc").setId("1").setSource("foo", "bar"),
                 IndexMetaData.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK);
         });
 
         // Cannot add further documents
-        assertBlocked(client().prepareIndex().setIndex("test").setType("doc").setId("3").setSource("foo", "bar"),
+        assertBlocked(client().prepareIndex().setIndex("test").setType("doc").setId("2").setSource("foo", "bar"),
             IndexMetaData.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK);
-        refresh();
-        assertSearchHits(client().prepareSearch().get(), "1", "2");
+        assertSearchHits(client().prepareSearch().get(), "1");
 
         // Update the disk usages so all nodes are back under the high and flood watermarks
         cis.setN1Usage(realNodeNames.get(0), new DiskUsage(nodes.get(0), "n1", "_na_", 100, 11));
         cis.setN2Usage(realNodeNames.get(1), new DiskUsage(nodes.get(1), "n2", "_na_", 100, 11));
         cis.setN3Usage(realNodeNames.get(2), new DiskUsage(nodes.get(2), "n3", "_na_", 100, 11));
 
-        refresh();
         // Attempt to create a new document until DiskUsageMonitor unblocks the index
         assertBusy(() -> {
             try {
-                client().prepareIndex().setIndex("test").setType("doc").setId("3").setSource("foo", "bar").get();
+                client().prepareIndex("test", "doc", "3").setSource("{\"foo\": \"bar\"}", XContentType.JSON)
+                    .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
             } catch (ClusterBlockException e) {
-                fail();
+                throw new AssertionError("retrying", e);
             }
         });
-        refresh();
-        assertSearchHits(client().prepareSearch().get(), "1", "2", "3");
+        assertSearchHits(client().prepareSearch().get(), "1", "3");
     }
 
 }

From 2444c6d44ae1748468c94b226171f325bf010a8a Mon Sep 17 00:00:00 2001
From: Bukhtawar Khan <bukhtawa@amazon.com>
Date: Tue, 23 Jul 2019 23:31:39 +0530
Subject: [PATCH 8/9] Yet another commit

---
 .../allocation/DiskThresholdMonitor.java      |  28 +-
 .../allocation/DiskThresholdMonitorTests.java | 287 ++++++------------
 2 files changed, 109 insertions(+), 206 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
index 93cfc31330903..c997567f01c7f 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
@@ -157,15 +157,21 @@ public void onNewInfo(ClusterInfo info) {
             // Only unblock index if all nodes that contain shards of it are below the high disk watermark
             if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdFloodStage().getBytes() ||
                 usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdFloodStage()) {
-                markIneligiblityForAutoRelease(routingNode, indicesNotToAutoRelease);
                 if (routingNode != null) { // this might happen if we haven't got the full cluster-state yet?!
                     for (ShardRouting routing : routingNode) {
-                        indicesToMarkReadOnly.add(routing.index().getName());
+                        String indexName = routing.index().getName();
+                        indicesToMarkReadOnly.add(indexName);
+                        indicesNotToAutoRelease.add(indexName);
                     }
                 }
             } else if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes() ||
                 usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) {
-                markIneligiblityForAutoRelease(routingNode, indicesNotToAutoRelease);
+                if (routingNode != null) {
+                    for (ShardRouting routing : routingNode) {
+                        String indexName = routing.index().getName();
+                        indicesNotToAutoRelease.add(indexName);
+                    }
+                }
                 if (lastRunTimeMillis.get() < currentTimeMillis - diskThresholdSettings.getRerouteInterval().millis()) {
                     reroute = true;
                     explanation = "high disk watermark exceeded on one or more nodes";
@@ -244,21 +250,17 @@ private void markNodesMissingUsageIneligibleForRelease(RoutingNodes routingNodes
                                                            Set<String> indicesToMarkIneligibleForAutoRelease) {
         for (RoutingNode routingNode : routingNodes) {
             if (usages.containsKey(routingNode.nodeId()) == false) {
-                markIneligiblityForAutoRelease(routingNode, indicesToMarkIneligibleForAutoRelease);
+                if (routingNode != null) {
+                    for (ShardRouting routing : routingNode) {
+                        String indexName = routing.index().getName();
+                        indicesToMarkIneligibleForAutoRelease.add(indexName);
+                    }
+                }
             }
         }
 
     }
 
-    private void markIneligiblityForAutoRelease(RoutingNode routingNode, Set<String> indicesToMarkIneligibleForAutoRelease) {
-        if (routingNode != null) {
-            for (ShardRouting routing : routingNode) {
-                String indexName = routing.index().getName();
-                indicesToMarkIneligibleForAutoRelease.add(indexName);
-            }
-        }
-    }
-
     private void setLastRunTimeMillis() {
         lastRunTimeMillis.getAndUpdate(l -> Math.max(l, currentTimeMillisSupplier.getAsLong()));
     }
diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
index 89c36a54cbf8e..64406888bc239 100644
--- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java
@@ -45,12 +45,31 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 
 public class DiskThresholdMonitorTests extends ESAllocationTestCase {
 
     public void testMarkFloodStageIndicesReadOnly() {
-        ClusterState clusterState = bootstrapCluster();
+        AllocationService allocation = createAllocationService(Settings.builder()
+            .put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
+        MetaData metaData = MetaData.builder()
+            .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)
+                .put("index.routing.allocation.require._id", "node2")).numberOfShards(1).numberOfReplicas(0))
+            .put(IndexMetaData.builder("test_1").settings(settings(Version.CURRENT)
+                .put("index.routing.allocation.require._id", "node1")).numberOfShards(1).numberOfReplicas(0))
+            .put(IndexMetaData.builder("test_2").settings(settings(Version.CURRENT)
+                .put("index.routing.allocation.require._id", "node1")).numberOfShards(1).numberOfReplicas(0))
+            .build();
+        RoutingTable routingTable = RoutingTable.builder()
+            .addAsNew(metaData.index("test"))
+            .addAsNew(metaData.index("test_1"))
+            .addAsNew(metaData.index("test_2"))
+            .build();
+        final ClusterState clusterState = applyStartedShardsUntilNoChange(
+            ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+                .metaData(metaData).routingTable(routingTable)
+                .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build(), allocation);
         AtomicBoolean reroute = new AtomicBoolean(false);
         AtomicReference<Set<String>> indices = new AtomicReference<>();
         AtomicLong currentTime = new AtomicLong();
@@ -143,8 +162,8 @@ protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionLi
 
         final ImmutableOpenMap.Builder<String, DiskUsage> allDisksOkBuilder;
         allDisksOkBuilder = ImmutableOpenMap.builder();
-        allDisksOkBuilder.put("node1", new DiskUsage("node1","node1", "/foo/bar", 100, 50));
-        allDisksOkBuilder.put("node2", new DiskUsage("node2","node2", "/foo/bar", 100, 50));
+        allDisksOkBuilder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, 50));
+        allDisksOkBuilder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, 50));
         final ImmutableOpenMap<String, DiskUsage> allDisksOk = allDisksOkBuilder.build();
 
         final ImmutableOpenMap.Builder<String, DiskUsage> oneDiskAboveWatermarkBuilder = ImmutableOpenMap.builder();
@@ -207,102 +226,32 @@ protected void updateIndicesReadOnly(Set<String> indicesToMarkReadOnly, ActionLi
         assertNull(listenerReference.get());
     }
 
-    public void testAutoReleaseSingleShardIndices() {
-        ClusterState clusterState = bootstrapCluster();
+    public void testAutoReleaseIndices() {
         AtomicReference<Set<String>> indicesToMarkReadOnly = new AtomicReference<>();
         AtomicReference<Set<String>> indicesToRelease = new AtomicReference<>();
-        // Change cluster state so that "test" index is blocked (read only)
-        IndexMetaData indexMetaData = IndexMetaData.builder(clusterState.metaData().index("test")).settings(Settings.builder()
-            .put(clusterState.metaData()
-                .index("test").getSettings())
-            .put(IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), true)).build();
-
-        final ClusterState anotherFinalClusterState = ClusterState.builder(clusterState).metaData(MetaData.builder(clusterState.metaData())
-            .put(clusterState.metaData().index("test_1"), false)
-            .put(clusterState.metaData().index("test_2"), false)
-            .put(indexMetaData, true).build())
-            .blocks(ClusterBlocks.builder().addBlocks(indexMetaData).build()).build();
-
-        assertTrue(anotherFinalClusterState.blocks().indexBlocked(ClusterBlockLevel.WRITE, "test"));
+        AllocationService allocation = createAllocationService(Settings.builder()
+            .put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
+        MetaData metaData = MetaData.builder()
+            .put(IndexMetaData.builder("test_1").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1))
+            .put(IndexMetaData.builder("test_2").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1))
+            .build();
+        RoutingTable routingTable = RoutingTable.builder()
+            .addAsNew(metaData.index("test_1"))
+            .addAsNew(metaData.index("test_2"))
+            .build();
+        final ClusterState clusterState = applyStartedShardsUntilNoChange(
+            ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+                .metaData(metaData).routingTable(routingTable)
+                .nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build(), allocation);
+        assertThat(clusterState.getRoutingTable().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(8));
 
-        AtomicLong currentTime = new AtomicLong();
-        DiskThresholdMonitor monitor = new DiskThresholdMonitor(Settings.EMPTY, () -> anotherFinalClusterState,
-            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, currentTime::get, (reason, priority, listener) -> {
-            assertNotNull(listener);
-            assertThat(priority, equalTo(Priority.HIGH));
-            listener.onResponse(null);
-        }){
-            @Override
-            protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener<Void> listener, boolean readOnly) {
-                if (readOnly) {
-                    assertTrue(indicesToMarkReadOnly.compareAndSet(null, indicesToUpdate));
-                } else {
-                    assertTrue(indicesToRelease.compareAndSet(null, indicesToUpdate));
-                }
+        DiskThresholdMonitor monitor = new DiskThresholdMonitor(Settings.EMPTY, () -> clusterState,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, () -> 0L,
+            (reason, priority, listener) -> {
+                assertNotNull(listener);
+                assertThat(priority, equalTo(Priority.HIGH));
                 listener.onResponse(null);
-            }
-        };
-
-        // When free disk on node2 goes above threshold (10% high watermark in this test case), index on node2 marked for
-        // auto release
-        ImmutableOpenMap.Builder<String, DiskUsage> builder = ImmutableOpenMap.builder();
-        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
-        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(10, 100)));
-        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
-        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
-        assertEquals(new HashSet<>(Arrays.asList("test")), indicesToRelease.get());
-
-        // When free disk on node1 goes below(5% flood watermark in this test case) and node2 stays below 10% high watermark threshold,
-        // index with read only block isn't marked again to be read only and no indices to be auto-released
-        indicesToMarkReadOnly.set(null);
-        indicesToRelease.set(null);
-        builder = ImmutableOpenMap.builder();
-        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
-        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(0, 9)));
-        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
-        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
-        assertNull(indicesToRelease.get());
-
-        // When free disk on node1 goes above 5% flood watermark and node2 goes above 10% high watermark,
-        // index with read only block to be auto-released
-        indicesToMarkReadOnly.set(null);
-        indicesToRelease.set(null);
-        builder = ImmutableOpenMap.builder();
-        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(5, 100)));
-        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(10, 100)));
-        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
-        assertNull(indicesToMarkReadOnly.get());
-        assertEquals(new HashSet<>(Arrays.asList("test")), indicesToRelease.get());
-
-        //When no usage information is present for node2, we don't release the block
-        indicesToMarkReadOnly.set(null);
-        indicesToRelease.set(null);
-        builder = ImmutableOpenMap.builder();
-        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
-        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
-        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
-        assertNull(indicesToRelease.get());
-
-        //Remove index block settings from cluster state
-        indexMetaData = IndexMetaData.builder(clusterState.metaData().index("test")).settings(Settings.builder()
-            .put(clusterState.metaData()
-                .index("test").getSettings())
-            .put(IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), false)).build();
-
-        ClusterState clusterStateWithoutBlocks = ClusterState.builder(clusterState).metaData(MetaData.builder(clusterState.metaData())
-            .put(clusterState.metaData().index("test_1"), false)
-            .put(clusterState.metaData().index("test_2"), false)
-            .put(indexMetaData, true).build())
-            .blocks(ClusterBlocks.builder().addBlocks(indexMetaData).build()).build();
-
-         assertFalse(clusterStateWithoutBlocks.blocks().indexBlocked(ClusterBlockLevel.WRITE, "test"));
-
-         monitor = new DiskThresholdMonitor(Settings.EMPTY, () -> clusterStateWithoutBlocks,
-            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, currentTime::get, (reason, priority, listener) -> {
-            assertNotNull(listener);
-            assertThat(priority, equalTo(Priority.HIGH));
-            listener.onResponse(null);
-        }){
+            }) {
             @Override
             protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener<Void> listener, boolean readOnly) {
                 if (readOnly) {
@@ -315,39 +264,12 @@ protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener
         };
         indicesToMarkReadOnly.set(null);
         indicesToRelease.set(null);
-        builder = ImmutableOpenMap.builder();
+        ImmutableOpenMap.Builder<String, DiskUsage> builder = ImmutableOpenMap.builder();
         builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
         builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(0, 4)));
         monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
-        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2", "test")), indicesToMarkReadOnly.get());
+        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
         assertNull(indicesToRelease.get());
-    }
-
-    public void testAutoReleaseMultiShardIndices(){
-        AtomicReference<Set<String>> indicesToMarkReadOnly = new AtomicReference<>();
-        AtomicReference<Set<String>> indicesToRelease = new AtomicReference<>();
-        AllocationService allocation = createAllocationService(Settings.builder()
-            .put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
-        MetaData metaData = MetaData.builder()
-            .put(IndexMetaData.builder("test_1").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1))
-            .put(IndexMetaData.builder("test_2").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1))
-            .build();
-        RoutingTable routingTable = RoutingTable.builder()
-            .addAsNew(metaData.index("test_1"))
-            .addAsNew(metaData.index("test_2"))
-            .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
-            .metaData(metaData).routingTable(routingTable).build();
-        logger.info("adding two nodes and performing rerouting");
-        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))
-            .add(newNode("node2"))).build();
-        clusterState = allocation.reroute(clusterState, "reroute");
-        logger.info("start primary shard");
-        clusterState = startInitializingShardsAndReroute(allocation, clusterState);
-        assertThat(clusterState.getRoutingTable().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(4));
-        logger.info("start replica shards");
-        clusterState = startInitializingShardsAndReroute(allocation, clusterState);
-        assertThat(clusterState.getRoutingTable().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(8));
 
         // Change cluster state so that "test_2" index is blocked (read only)
         IndexMetaData indexMetaData = IndexMetaData.builder(clusterState.metaData().index("test_2")).settings(Settings.builder()
@@ -356,19 +278,17 @@ public void testAutoReleaseMultiShardIndices(){
             .put(IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), true)).build();
 
         ClusterState clusterStateWithBlocks = ClusterState.builder(clusterState).metaData(MetaData.builder(clusterState.metaData())
-            .put(clusterState.metaData().index("test_1"), false)
-            .put(clusterState.metaData().index("test_2"), false)
             .put(indexMetaData, true).build())
             .blocks(ClusterBlocks.builder().addBlocks(indexMetaData).build()).build();
 
         assertTrue(clusterStateWithBlocks.blocks().indexBlocked(ClusterBlockLevel.WRITE, "test_2"));
-        AtomicLong currentTime = new AtomicLong();
-        DiskThresholdMonitor monitor = new DiskThresholdMonitor(Settings.EMPTY, () -> clusterStateWithBlocks,
-            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, currentTime::get, (reason, priority, listener) -> {
-            assertNotNull(listener);
-            assertThat(priority, equalTo(Priority.HIGH));
-            listener.onResponse(null);
-        }){
+        monitor = new DiskThresholdMonitor(Settings.EMPTY, () -> clusterStateWithBlocks,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, () -> 0L,
+            (reason, priority, listener) -> {
+                assertNotNull(listener);
+                assertThat(priority, equalTo(Priority.HIGH));
+                listener.onResponse(null);
+            }) {
             @Override
             protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener<Void> listener, boolean readOnly) {
                 if (readOnly) {
@@ -379,15 +299,14 @@ protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener
                 listener.onResponse(null);
             }
         };
-
         // When free disk on any of node1 or node2 goes below 5% flood watermark, then apply index block on indices not having the block
         indicesToMarkReadOnly.set(null);
         indicesToRelease.set(null);
-        ImmutableOpenMap.Builder<String, DiskUsage> builder = ImmutableOpenMap.builder();
-        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(10, 100)));
+        builder = ImmutableOpenMap.builder();
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 100)));
         builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(0, 4)));
         monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
-        assertEquals(new HashSet<>(Arrays.asList("test_1")), indicesToMarkReadOnly.get());
+        assertThat(indicesToMarkReadOnly.get(), contains("test_1"));
         assertNull(indicesToRelease.get());
 
         // When free disk on node1 and node2 goes above 10% high watermark, then only release index block
@@ -398,70 +317,52 @@ protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener
         builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(10, 100)));
         monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
         assertNull(indicesToMarkReadOnly.get());
-        assertEquals(new HashSet<>(Arrays.asList("test_2")), indicesToRelease.get());
+        assertThat(indicesToRelease.get(), contains("test_2"));
 
-        //Remove index block settings from cluster state
-        indexMetaData = IndexMetaData.builder(clusterState.metaData().index("test_1")).settings(Settings.builder()
-            .put(clusterState.metaData()
-                .index("test_1").getSettings())
-            .put(IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.getKey(), false)).build();
+        // When no usage information is present for node2, we don't release the block
+        indicesToMarkReadOnly.set(null);
+        indicesToRelease.set(null);
+        builder = ImmutableOpenMap.builder();
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
+        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
+        assertThat(indicesToMarkReadOnly.get(), contains("test_1"));
+        assertNull(indicesToRelease.get());
 
-        ClusterState clusterStateWithoutBlocks = ClusterState.builder(clusterState).metaData(MetaData.builder(clusterState.metaData())
-            .put(clusterState.metaData().index("test_1"), false)
-            .put(clusterState.metaData().index("test_2"), false)
-            .put(indexMetaData, true).build())
-            .blocks(ClusterBlocks.builder().addBlocks(indexMetaData).build()).build();
+        // When disk usage on one node is between the high and flood-stage watermarks, nothing changes
+        indicesToMarkReadOnly.set(null);
+        indicesToRelease.set(null);
+        builder = ImmutableOpenMap.builder();
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(5, 9)));
+        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(5, 100)));
+        if (randomBoolean()) {
+            builder.put("node3", new DiskUsage("node3", "node3", "/foo/bar", 100, between(0, 100)));
+        }
+        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
+        assertNull(indicesToMarkReadOnly.get());
+        assertNull(indicesToRelease.get());
 
-        assertFalse(clusterStateWithoutBlocks.blocks().indexBlocked(ClusterBlockLevel.WRITE, "test_1"));
-        monitor = new DiskThresholdMonitor(Settings.EMPTY, () -> clusterStateWithoutBlocks,
-            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, currentTime::get, (reason, priority, listener) -> {
-            assertNotNull(listener);
-            assertThat(priority, equalTo(Priority.HIGH));
-            listener.onResponse(null);
-        }){
-            @Override
-            protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener<Void> listener, boolean readOnly) {
-                if (readOnly) {
-                    assertTrue(indicesToMarkReadOnly.compareAndSet(null, indicesToUpdate));
-                } else {
-                    assertTrue(indicesToRelease.compareAndSet(null, indicesToUpdate));
-                }
-                listener.onResponse(null);
-            }
-        };
+        // When disk usage on one node is missing and the other is below the high watermark, nothing changes
         indicesToMarkReadOnly.set(null);
         indicesToRelease.set(null);
         builder = ImmutableOpenMap.builder();
-        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
-        builder.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(0, 4)));
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(5, 100)));
+        if (randomBoolean()) {
+            builder.put("node3", new DiskUsage("node3", "node3", "/foo/bar", 100, between(0, 100)));
+        }
         monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
-        assertEquals(new HashSet<>(Arrays.asList("test_1", "test_2")), indicesToMarkReadOnly.get());
+        assertNull(indicesToMarkReadOnly.get());
         assertNull(indicesToRelease.get());
-    }
 
-    private ClusterState bootstrapCluster() {
-        AllocationService allocation = createAllocationService(Settings.builder()
-            .put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
-        MetaData metaData = MetaData.builder()
-            .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)
-                .put("index.routing.allocation.require._id", "node2")).numberOfShards(1).numberOfReplicas(0))
-            .put(IndexMetaData.builder("test_1").settings(settings(Version.CURRENT)
-                .put("index.routing.allocation.require._id", "node1")).numberOfShards(1).numberOfReplicas(0))
-            .put(IndexMetaData.builder("test_2").settings(settings(Version.CURRENT)
-                .put("index.routing.allocation.require._id", "node1")).numberOfShards(1).numberOfReplicas(0))
-            .build();
-        RoutingTable routingTable = RoutingTable.builder()
-            .addAsNew(metaData.index("test"))
-            .addAsNew(metaData.index("test_1"))
-            .addAsNew(metaData.index("test_2"))
-            .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
-            .metaData(metaData).routingTable(routingTable).build();
-        logger.info("adding two nodes and performing rerouting");
-        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))
-            .add(newNode("node2"))).build();
-        clusterState = allocation.reroute(clusterState, "reroute");
-        logger.info("start primary shard");
-        return allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING));
+        // When disk usage on one node is missing and the other is above the flood-stage watermark, affected indices are blocked
+        indicesToMarkReadOnly.set(null);
+        indicesToRelease.set(null);
+        builder = ImmutableOpenMap.builder();
+        builder.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
+        if (randomBoolean()) {
+            builder.put("node3", new DiskUsage("node3", "node3", "/foo/bar", 100, between(0, 100)));
+        }
+        monitor.onNewInfo(new ClusterInfo(builder.build(), null, null, null));
+        assertThat(indicesToMarkReadOnly.get(), contains("test_1"));
+        assertNull(indicesToRelease.get());
     }
 }

From bd6a990701ddc779d91874f883f8ecb7d8d28d52 Mon Sep 17 00:00:00 2001
From: Bukhtawar Khan <bukhtawa@amazon.com>
Date: Wed, 7 Aug 2019 13:15:37 +0530
Subject: [PATCH 9/9] Fixes checkstyles

---
 .../routing/allocation/DiskThresholdMonitor.java  | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
index c997567f01c7f..9071d23ce526d 100644
--- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
+++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java
@@ -23,6 +23,7 @@
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.GroupedActionListener;
@@ -80,7 +81,8 @@ public DiskThresholdMonitor(Settings settings, Supplier<ClusterState> clusterSta
         this.diskThresholdSettings = new DiskThresholdSettings(settings, clusterSettings);
         this.client = client;
         if (diskThresholdSettings.isAutoReleaseIndexEnabled() == false) {
-            deprecationLogger.deprecated("[{}] will be removed in version {}", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, Version.V_7_4_0.major + 1);
+            deprecationLogger.deprecated("[{}] will be removed in version {}",
+                DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, Version.V_7_4_0.major + 1);
         }
     }
 
@@ -230,8 +232,10 @@ public void onNewInfo(ClusterInfo info) {
                 logger.info("releasing read-only-allow-delete block on indices: [{}]", indicesToAutoRelease);
                 updateIndicesReadOnly(indicesToAutoRelease, listener, false);
             } else {
-                deprecationLogger.deprecated("[{}] will be removed in version {}", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, Version.V_7_4_0.major + 1);
-                logger.debug("[{}] disabled, not releasing read-only-allow-delete block on indices: [{}]", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, indicesToAutoRelease);
+                deprecationLogger.deprecated("[{}] will be removed in version {}",
+                    DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, Version.V_7_4_0.major + 1);
+                logger.debug("[{}] disabled, not releasing read-only-allow-delete block on indices: [{}]",
+                    DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, indicesToAutoRelease);
                 listener.onResponse(null);
             }
         } else {
@@ -271,11 +275,12 @@ protected void updateIndicesReadOnly(Set<String> indicesToUpdate, ActionListener
             setLastRunTimeMillis();
             listener.onResponse(r);
         }, e -> {
-            logger.debug("marking indices read-only [{}] failed", readOnly, e);
+            logger.debug(new ParameterizedMessage("setting indices [{}] read-only failed", readOnly), e);
             setLastRunTimeMillis();
             listener.onFailure(e);
         });
-        Settings readOnlySettings = readOnly ? Settings.builder().put(IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE, Boolean.TRUE.toString()).build() :
+        Settings readOnlySettings = readOnly ? Settings.builder()
+            .put(IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE, Boolean.TRUE.toString()).build() :
             Settings.builder().putNull(IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE).build();
         client.admin().indices().prepareUpdateSettings(indicesToUpdate.toArray(Strings.EMPTY_ARRAY))
             .setSettings(readOnlySettings)